github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-06-28 16:41:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

sync code

Browse Source
This commit is contained in:
Ned Wright 2024-11-28 10:41:59 +00:00
parent 6255e1f30d
commit 1c1f0b7e29
59 changed files with 842 additions and 707 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
components/attachment-intakers/nginx_attachment/nginx_attachment.cc
View File

@ -1135,7 +1135,11 @@ private:
"webUserResponse"
);
bool remove_event_id_param =
getProfileAgentSettingWithDefault<string>("false", "nginxAttachment.removeRedirectEventId") == "true";
string uuid;
string redirectUrl;
if (i_transaction_table->hasState<NginxAttachmentOpaque>()) {
NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
uuid = opaque.getSessionUUID();
@ -1145,7 +1149,12 @@ private:
if (web_trigger_conf.getDetailsLevel() == "Redirect") {
web_response_data.response_data.redirect_data.redirect_location_size =
web_trigger_conf.getRedirectURL().size();
web_response_data.response_data.redirect_data.add_event_id = web_trigger_conf.getAddEventId() ? 1 : 0;
bool add_event = web_trigger_conf.getAddEventId();
if (add_event && !remove_event_id_param) {
web_response_data.response_data.redirect_data.redirect_location_size +=
strlen("?event_id=") + uuid.size();
}
web_response_data.response_data.redirect_data.add_event_id = add_event ? 1 : 0;
web_response_data.web_repsonse_type = static_cast<uint8_t>(ngx_web_response_type_e::REDIRECT_WEB_RESPONSE);
} else {
web_response_data.response_data.custom_response_data.title_size =
@ -1159,8 +1168,13 @@ private:
verdict_data_sizes.push_back(sizeof(ngx_http_cp_web_response_data_t));
if (web_trigger_conf.getDetailsLevel() == "Redirect") {
verdict_data.push_back(reinterpret_cast<const char *>(web_trigger_conf.getRedirectURL().data()));
verdict_data_sizes.push_back(web_trigger_conf.getRedirectURL().size());
redirectUrl = web_trigger_conf.getRedirectURL();
if (!remove_event_id_param && web_trigger_conf.getAddEventId()) {
redirectUrl += "?event-id=" + uuid;
}
verdict_data.push_back(reinterpret_cast<const char *>(redirectUrl.data()));
verdict_data_sizes.push_back(redirectUrl.size());
} else {
verdict_data.push_back(reinterpret_cast<const char *>(web_trigger_conf.getResponseTitle().data()));
verdict_data_sizes.push_back(web_trigger_conf.getResponseTitle().size());

30
components/attachment-intakers/nginx_attachment/user_identifiers_config.cc
View File

@ -282,7 +282,7 @@ isIpTrusted(const string &value, const vector<CIDRSData> &cidr_values)
}
Maybe<string>
UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const
UsersAllIdentifiersConfig::parseXForwardedFor(const string &str, ExtractType type) const
{
vector<string> header_values = split(str);
@ -291,12 +291,23 @@ UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const
vector<string> xff_values = getHeaderValuesFromConfig("x-forwarded-for");
vector<CIDRSData> cidr_values(xff_values.begin(), xff_values.end());
for (const string &value : header_values) {
if (!IPAddr::createIPAddr(value).ok()) {
dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << value;
for (auto it = header_values.rbegin(); it != header_values.rend() - 1; ++it) {
if (!IPAddr::createIPAddr(*it).ok()) {
dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << *it;
return genError("Invalid IP address");
}
if (!isIpTrusted(value, cidr_values)) return genError("Untrusted Ip found");
if (type == ExtractType::PROXYIP) continue;
if (!isIpTrusted(*it, cidr_values)) {
dbgDebug(D_NGINX_ATTACHMENT_PARSER) << "Found untrusted IP in the xff header IPs list: " << *it;
return *it;
}
}
if (!IPAddr::createIPAddr(header_values[0]).ok()) {
dbgWarning(D_NGINX_ATTACHMENT_PARSER)
<< "Invalid IP address found in the xff header IPs list: "
<< header_values[0];
return genError("Invalid IP address");
}
return header_values[0];
@ -312,7 +323,7 @@ UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, Ext
return;
}
NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
auto value = parseXForwardedFor(header.getValue());
auto value = parseXForwardedFor(header.getValue(), type);
if (!value.ok()) {
dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Could not extract source identifier from X-Forwarded-For header";
return;
@ -321,12 +332,13 @@ UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, Ext
if (type == ExtractType::SOURCEIDENTIFIER) {
opaque.setSourceIdentifier(header.getKey(), value.unpack());
dbgDebug(D_NGINX_ATTACHMENT_PARSER)
<< "Added source identifir to XFF "
<< "Added source identifier from XFF header"
<< value.unpack();
opaque.setSavedData(HttpTransactionData::xff_vals_ctx, header.getValue());
opaque.setSavedData(HttpTransactionData::source_identifier, value.unpack());
dbgTrace(D_NGINX_ATTACHMENT_PARSER)
<< "XFF found, set ctx with value from header: "
<< static_cast<string>(header.getValue());
<< "XFF found, set ctx with value from header: "
<< static_cast<string>(header.getValue());
} else {
opaque.setSavedData(HttpTransactionData::proxy_ip_ctx, value.unpack());
}

2
components/include/reverse_proxy_defaults.h
View File

@ -28,7 +28,7 @@ static const std::string default_nginx_config_file = "/etc/cp/conf/rpmanager/ngi
static const std::string default_prepare_nginx_config_file = "/etc/cp/conf/rpmanager/nginx_prepare.conf";
static const std::string default_global_conf_template = "/etc/cp/conf/rpmanager/nginx-conf-template";
static const std::string default_nginx_config_include_file =
"/etc/cp/conf/rpmanager/servers/nginx_conf_include.conf";
"/etc/cp/conf/rpmanager/servers/00_nginx_conf_include.conf";
static const std::string default_global_conf_include_template =
"/etc/cp/conf/rpmanager/nginx-conf-include-template";
static const std::string default_global_conf_include_template_no_responses =

2
components/include/user_identifiers_config.h
View File

@ -58,7 +58,7 @@ private:
const std::string::const_iterator &end,
const std::string &key) const;
Buffer extractKeyValueFromCookie(const std::string &cookie_value, const std::string &key) const;
Maybe<std::string> parseXForwardedFor(const std::string &str) const;
Maybe<std::string> parseXForwardedFor(const std::string &str, ExtractType type) const;
std::vector<UsersIdentifiersConfig> user_identifiers;
};

4
components/include/waap.h
View File

@ -33,7 +33,6 @@ class I_WaapAssetStatesManager;
class I_Messaging;
class I_AgentDetails;
class I_Encryptor;
class I_WaapModelResultLogger;
const std::string WAAP_APPLICATION_NAME = "waap application";
@ -51,8 +50,7 @@ class WaapComponent
Singleton::Consume<I_AgentDetails>,
Singleton::Consume<I_Messaging>,
Singleton::Consume<I_Encryptor>,
Singleton::Consume<I_Environment>,
Singleton::Consume<I_WaapModelResultLogger>
Singleton::Consume<I_Environment>
{
public:
WaapComponent();

2
components/security_apps/orchestration/details_resolver/details_resolver.cc
View File

@ -350,7 +350,7 @@ DetailsResolver::Impl::readCloudMetadata()
}
if (!cloud_metadata.ok()) {
dbgWarning(D_ORCHESTRATOR) << cloud_metadata.getErr();
dbgDebug(D_ORCHESTRATOR) << cloud_metadata.getErr();
return genError("Failed to fetch cloud metadata");
}

45
components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h
View File

@ -18,6 +18,8 @@
#include <regex>
#include <boost/regex.hpp>
#include <boost/algorithm/string.hpp>
#include <cereal/external/rapidjson/document.h>
#include <cereal/external/rapidjson/filereadstream.h>
#if defined(gaia)
@ -100,6 +102,14 @@ checkIsInstallHorizonTelemetrySucceeded(const string &command_output)
return command_output;
}
Maybe<string>
getOtlpAgentGaiaOsRole(const string &command_output)
{
if (command_output == "" ) return string("-1");
return command_output;
}
Maybe<string>
getQUID(const string &command_output)
{
@ -111,6 +121,13 @@ getQUID(const string &command_output)
return command_output;
}
Maybe<string>
getIsAiopsRunning(const string &command_output)
{
if (command_output == "" ) return string("false");
return command_output;
}
Maybe<string>
checkHasSDWan(const string &command_output)
@ -186,6 +203,24 @@ getMgmtObjAttr(shared_ptr<istream> file_stream, const string &attr)
return genError("Object attribute was not found. Attr: " + attr);
}
Maybe<string>
getAttrFromCpsdwanGetDataJson(const string &attr)
{
static const std::string get_data_json_path = "/tmp/cpsdwan_getdata_orch.json";
std::ifstream ifs(get_data_json_path);
if (ifs.is_open()) {
rapidjson::IStreamWrapper isw(ifs);
rapidjson::Document document;
document.ParseStream(isw);
if (!document.HasParseError() && document.HasMember(attr.c_str()) && document[attr.c_str()].IsString()) {
return string(document[attr.c_str()].GetString());
}
}
return genError("Attribute " + attr + " was not found in " + get_data_json_path);
}
Maybe<string>
getMgmtObjUid(const string &command_output)
{
@ -193,6 +228,11 @@ getMgmtObjUid(const string &command_output)
return command_output;
}
Maybe<string> obj_uuid = getAttrFromCpsdwanGetDataJson("uuid");
if (obj_uuid.ok()) {
return obj_uuid.unpack();
}
static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C";
auto file_stream = std::make_shared<std::ifstream>(obj_path);
if (!file_stream->is_open()) {
@ -311,6 +351,11 @@ getSmbObjectName(const string &command_output)
return genError("Object name was not found");
}
Maybe<string> obj_name = getAttrFromCpsdwanGetDataJson("name");
if (obj_name.ok()) {
return obj_name.unpack();
}
static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C";
auto ifs = std::make_shared<std::ifstream>(obj_path);
if (!ifs->is_open()) {

15
components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
View File

@ -73,6 +73,15 @@ SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''",
getQUID)
SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "[ -d /opt/CPOtlpAgent/custom_scripts ] "
"&& ENV_NO_FORMAT=1 /opt/CPOtlpAgent/custom_scripts/agent_role.sh",
getOtlpAgentGaiaOsRole)
SHELL_CMD_HANDLER(
"IS_AIOPS_RUNNING",
"FS_PATH=<FILESYSTEM-PREFIX>; "
"PID=$(ps auxf | grep -v grep | grep -E ${FS_PATH}.*cp-nano-horizon-telemetry | awk -F' ' '{printf $2}'); "
"[ -z \"{PID}\" ] && echo 'false' || echo 'true'",
getIsAiopsRunning)
SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
SHELL_CMD_HANDLER(
"canUpdateSDWanData",
@ -180,8 +189,7 @@ SHELL_CMD_HANDLER(
)
SHELL_CMD_HANDLER(
"managements",
"sed -n '/:masters (/,$p' $FWDIR/database/myself_objects.C |"
" sed -e ':a' -e 'N' -e '$!ba' -e 's/\\n//g' -e 's/\t//g' -e 's/ //g' | sed 's/))):.*/)))):/'",
"echo 1",
extractManagements
)
#endif //gaia
@ -237,8 +245,7 @@ SHELL_CMD_HANDLER(
SHELL_CMD_HANDLER(
"managements",
"sed -n '/:masters (/,$p' /tmp/local.cfg |"
" sed -e ':a' -e 'N' -e '$!ba' -e 's/\\n//g' -e 's/\t//g' -e 's/ //g' | sed 's/))):.*/)))):/'",
"echo 1",
extractManagements
)
#endif//smb

4
components/security_apps/orchestration/hybrid_mode_telemetry.cc
View File

@ -34,7 +34,9 @@ HybridModeMetric::upon(const HybridModeMetricEvent &)
{
auto shell_cmd = Singleton::Consume<I_ShellCmd>::by<OrchestrationComp>();
auto maybe_cmd_output = shell_cmd->getExecOutput(
getFilesystemPathConfig() + "/watchdog/cp-nano-watchdog --restart_count"
getFilesystemPathConfig() + "/watchdog/cp-nano-watchdog --restart_count",
1000,
false
);
// get wd process restart count

6
components/security_apps/orchestration/include/declarative_policy_utils.h
View File

@ -79,8 +79,8 @@ public:
) override;
std::string getUpdate(CheckUpdateRequest &request) override;
bool shouldApplyPolicy() override;
void turnOffApplyPolicyFlag() override;
void turnOnApplyPolicyFlag() override;
void turnOffApplyLocalPolicyFlag() override;
void turnOnApplyLocalPolicyFlag() override;
std::string getCurrPolicy() override { return curr_policy; }
@ -94,7 +94,7 @@ private:
std::string curr_version;
std::string curr_policy;
std::string curr_checksum;
bool should_apply_policy;
bool should_apply_local_policy;
};
#endif // __DECLARATIVE_POLICY_UTILS_H__

4
components/security_apps/orchestration/include/i_declarative_policy.h
View File

@ -22,8 +22,8 @@ public:
virtual std::string getCurrPolicy() = 0;
virtual void turnOffApplyPolicyFlag() = 0;
virtual void turnOnApplyPolicyFlag() = 0;
virtual void turnOffApplyLocalPolicyFlag() = 0;
virtual void turnOnApplyLocalPolicyFlag() = 0;
protected:
virtual ~I_DeclarativePolicy() {}

2
components/security_apps/orchestration/orchestration_comp.cc
View File

@ -2033,7 +2033,7 @@ private:
}
auto policy_mgmt_mode = getSettingWithDefault<string>("management", "profileManagedMode");
if (getOrchestrationMode() == OrchestrationMode::HYBRID || policy_mgmt_mode == "declarative") {
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOnApplyPolicyFlag();
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOnApplyLocalPolicyFlag();
}
auto policy_version = i_service_controller->getPolicyVersion();

6
components/security_apps/orchestration/service_controller/service_controller.cc
View File

@ -793,7 +793,7 @@ ServiceController::Impl::updateServiceConfiguration(
<< "Policy file was not updated. Sending reload command regarding settings and data";
auto signal_services = sendSignalForServices(nano_services_to_update, "");
if (!signal_services.ok()) return signal_services.passErr();
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyPolicyFlag();
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyLocalPolicyFlag();
return Maybe<void>();
}
@ -940,7 +940,7 @@ ServiceController::Impl::updateServiceConfiguration(
if (new_policy_path.compare(config_file_path) == 0) {
dbgDebug(D_SERVICE_CONTROLLER) << "Enforcing the default policy file";
policy_version = version_value;
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyPolicyFlag();
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyLocalPolicyFlag();
return Maybe<void>();
}
@ -959,7 +959,7 @@ ServiceController::Impl::updateServiceConfiguration(
}
if (!was_policy_updated && !send_signal_for_services_err.empty()) return genError(send_signal_for_services_err);
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyPolicyFlag();
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyLocalPolicyFlag();
return Maybe<void>();
}

21
components/security_apps/orchestration/update_communication/declarative_policy_utils.cc
View File

@ -17,7 +17,7 @@ void
DeclarativePolicyUtils::init()
{
local_policy_path = getFilesystemPathConfig() + "/conf/local_policy.yaml";
should_apply_policy = true;
should_apply_local_policy = true;
Singleton::Consume<I_RestApi>::by<DeclarativePolicyUtils>()->addRestCall<ApplyPolicyRest>(
RestAction::SET, "apply-policy"
);
@ -40,7 +40,7 @@ DeclarativePolicyUtils::upon(const ApplyPolicyEvent &event)
{
dbgTrace(D_ORCHESTRATOR) << "Apply policy event";
local_policy_path = event.getPolicyPath();
should_apply_policy = true;
should_apply_local_policy = true;
}
// LCOV_EXCL_STOP
@ -48,19 +48,24 @@ bool
DeclarativePolicyUtils::shouldApplyPolicy()
{
auto env_type = Singleton::Consume<I_EnvDetails>::by<DeclarativePolicyUtils>()->getEnvType();
return env_type == EnvType::K8S ? true : should_apply_policy;
if (env_type == EnvType::K8S) {
I_OrchestrationTools *orch_tools = Singleton::Consume<I_OrchestrationTools>::by<DeclarativePolicyUtils>();
auto maybe_new_version = orch_tools->readFile("/etc/cp/conf/k8s-policy-check.trigger");
return maybe_new_version != curr_version;
}
return should_apply_local_policy;
}
void
DeclarativePolicyUtils::turnOffApplyPolicyFlag()
DeclarativePolicyUtils::turnOffApplyLocalPolicyFlag()
{
should_apply_policy = false;
should_apply_local_policy = false;
}
void
DeclarativePolicyUtils::turnOnApplyPolicyFlag()
DeclarativePolicyUtils::turnOnApplyLocalPolicyFlag()
{
should_apply_policy = true;
should_apply_local_policy = true;
}
Maybe<string>
@ -211,6 +216,6 @@ DeclarativePolicyUtils::periodicPolicyLoad()
if (*new_checksum == curr_checksum) return;
should_apply_policy = true;
should_apply_local_policy = true;
curr_checksum = *new_checksum;
}

2
components/security_apps/orchestration/update_communication/fog_communication.cc
View File

@ -74,7 +74,7 @@ FogCommunication::getUpdate(CheckUpdateRequest &request)
<< " to: "
<< policy_mgmt_mode;
profile_mode = policy_mgmt_mode;
i_declarative_policy->turnOnApplyPolicyFlag();
i_declarative_policy->turnOnApplyLocalPolicyFlag();
}
if (i_declarative_policy->shouldApplyPolicy()) {

9
components/security_apps/rate_limit/rate_limit.cc
View File

@ -293,8 +293,13 @@ public:
return ACCEPT;
}
burst = rule.getRateLimit();
limit = calcRuleLimit(rule);
auto replicas = getenv("REPLICA_COUNT") ? std::stoi(getenv("REPLICA_COUNT")) : 1;
if (replicas == 0) {
dbgWarning(D_RATE_LIMIT) << "REPLICA_COUNT environment variable is set to 0, setting REPLICA_COUNT to 1";
replicas = 1;
}
burst = static_cast<float>(rule.getRateLimit()) / replicas;
limit = static_cast<float>(calcRuleLimit(rule)) / replicas;
dbgTrace(D_RATE_LIMIT)
<< "found rate limit rule with: "

37
components/security_apps/waap/include/i_waap_model_result_logger.h
View File

@ -1,37 +0,0 @@
// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
class IWaf2Transaction;
struct Waf2ScanResult;
namespace Waap {
namespace Scores {
struct ModelLoggingSettings;
}
}
class I_WaapModelResultLogger {
public:
virtual ~I_WaapModelResultLogger() {}
virtual void
logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
std::string modelName,
std::string otherModelName,
double newScore,
double baseScore) = 0;
};

3
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@ -87,9 +87,10 @@ add_library(waap_clib
ParserPairs.cc
Waf2Util2.cc
ParserPDF.cc
ParserKnownBenignSkipper.cc
ParserScreenedJson.cc
ParserBinaryFile.cc
RegexComparator.cc
WaapModelResultLogger.cc
)
add_definitions("-Wno-unused-function")

71
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@ -28,6 +28,8 @@
#include "ParserDelimiter.h"
#include "ParserPDF.h"
#include "ParserBinaryFile.h"
#include "ParserKnownBenignSkipper.h"
#include "ParserScreenedJson.h"
#include "WaapAssetState.h"
#include "Waf2Regex.h"
#include "Waf2Util.h"
@ -359,6 +361,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth,
base64BinaryFileType
@ -410,6 +413,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth,
base64BinaryFileType
@ -461,6 +465,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth,
base64ParamFound,
@ -835,6 +840,7 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth,
bool base64ParamFound,
@ -854,6 +860,7 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth,
b64FileType
@ -918,6 +925,7 @@ bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth
) {
@ -959,6 +967,7 @@ DeepParser::createInternalParser(
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth,
Waap::Util::BinaryFileType b64FileType
@ -978,7 +987,19 @@ DeepParser::createInternalParser(
<< "\n\tflags: "
<< flags
<< "\n\tparser_depth: "
<< parser_depth;
<< parser_depth
<< "\n\tisBodyPayload: "
<< isBodyPayload
<< "\n\tisRefererPayload: "
<< isRefererPayload
<< "\n\tisRefererParamPayload: "
<< isRefererParamPayload
<< "\n\tisUrlPayload: "
<< isUrlPayload
<< "\n\tisUrlParamPayload: "
<< isUrlParamPayload
<< "\n\tisCookiePayload: "
<< isCookiePayload;
bool isPipesType = false, isSemicolonType = false, isAsteriskType = false, isCommaType = false,
isAmperType = false;
bool isKeyValDelimited = false;
@ -1045,6 +1066,53 @@ DeepParser::createInternalParser(
}
}
if (Waap::Util::isScreenedJson(cur_val)) {
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse screened JSON";
m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserScreenedJson>>(*this, parser_depth + 1));
offset = 0;
return offset;
}
dbgTrace(D_WAAP_DEEP_PARSER)
<< "Offset = "
<< offset
<< " depth = "
<< m_depth
<< " isBodyPayload = "
<< isBodyPayload;
//Detect sensor_data format in body and just use dedicated filter for it
if (m_depth == 1
&& isBodyPayload
&& Waap::Util::detectKnownSource(cur_val) == Waap::Util::SOURCE_TYPE_SENSOR_DATA) {
m_parsersDeque.push_back(
std::make_shared<BufferedParser<ParserKnownBenignSkipper>>(
*this,
parser_depth + 1,
Waap::Util::SOURCE_TYPE_SENSOR_DATA
)
);
offset = 0;
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse data_sensor data - skipping it";
return offset;
}
// Detect cookie parameter sensorsdata2015jssdkcross
// and causes false positives due to malformed JSON. Make preprocessing to parse it correctly
if (m_depth == 2
&& isCookiePayload) {
offset = Waap::Util::definePrefixedJson(cur_val);
if (offset >= 0) {
m_parsersDeque.push_back(
std::make_shared<BufferedParser<ParserJson>>(
*this,
parser_depth + 1,
m_pTransaction
)
);
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse JSON data";
return offset;
}
}
// Detect wbxml (binary XML) data type
if (m_depth == 1 && isBodyPayload && !valueStats.isUTF16 && m_pWaapAssetState->isWBXMLSampleType(cur_val)) {
m_is_wbxml = true;
@ -1374,6 +1442,7 @@ DeepParser::createInternalParser(
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth
);

3
components/security_apps/waap/waap_clib/DeepParser.h
View File

@ -129,6 +129,7 @@ private:
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth,
Waap::Util::BinaryFileType b64FileType
@ -144,6 +145,7 @@ private:
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth
);
@ -160,6 +162,7 @@ private:
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth,
bool base64ParamFound,

96
components/security_apps/waap/waap_clib/WaapAssetState.cc
View File

@ -783,6 +783,55 @@ WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const
#endif
}
// std::string nicePrint() - is a function used to create std::string that will represent all data that is
// collected inside Waf2ScanResult object. This function is used for debugging purposes. it should make deep-dive
// into the object easier.
std::string
WaapAssetState::nicePrint(Waf2ScanResult &res) const
{
std::string result = "Waf2ScanResult:\n";
result += "keyword_matches:\n";
for (const auto &keyword : res.keyword_matches) {
result += keyword + "\n";
}
result += "regex_matches:\n";
for (const auto &regex : res.regex_matches) {
result += regex + "\n";
}
result += "filtered_keywords:\n";
for (const auto &filtered : res.filtered_keywords) {
result += filtered + "\n";
}
result += "found_patterns:\n";
for (const auto &pattern : res.found_patterns) {
result += pattern.first + ":\n";
for (const auto &value : pattern.second) {
result += value + "\n";
}
}
result += "unescaped_line: " + res.unescaped_line + "\n";
result += "param_name: " + res.param_name + "\n";
result += "location: " + res.location + "\n";
result += "score: " + std::to_string(res.score) + "\n";
result += "scoreNoFilter: " + std::to_string(res.scoreNoFilter) + "\n";
result += "scoreArray:\n";
for (const auto &score : res.scoreArray) {
result += std::to_string(score) + "\n";
}
result += "keywordCombinations:\n";
for (const auto &combination : res.keywordCombinations) {
result += combination + "\n";
}
result += "attack_types:\n";
for (const auto &attack : res.attack_types) {
result += attack + "\n";
}
result += "m_isAttackInParam: " + std::to_string(res.m_isAttackInParam) + "\n";
return result;
}
bool
checkBinaryData(const std::string &line, bool binaryDataFound)
{
@ -1033,7 +1082,7 @@ WaapAssetState::apply(
// Scan unescaped_line with aho-corasick once, and reuse it in multiple calls to checkRegex below
// This is done to improve performance of regex matching.
SampleValue unescapedLineSample(res.unescaped_line, m_Signatures->m_regexPreconditions);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "after doing second set of checkRegex calls..." << nicePrint(res);
checkRegex(
unescapedLineSample,
m_Signatures->specific_acuracy_keywords_regex,
@ -1111,7 +1160,7 @@ WaapAssetState::apply(
}
bool os_cmd_ev = Waap::Util::find_in_map_of_stringlists_keys("os_cmd_ev", res.found_patterns);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "before evasion checking " << nicePrint(res);
if (os_cmd_ev) {
dbgTrace(D_WAAP_EVASIONS) << "os command evasion found";
@ -1295,6 +1344,47 @@ WaapAssetState::apply(
}
}
bool path_traversal_ev = Waap::Util::find_in_map_of_stringlists_keys("path_traversal", res.found_patterns);
dbgTrace(D_WAAP_EVASIONS)
<< "path_traversal_ev = " << path_traversal_ev
<< " sample = " << res.unescaped_line
<< " res.unescaped_line.find(2f) = " << res.unescaped_line.find("2f");
if ((path_traversal_ev) && (res.unescaped_line.find("2f") != std::string::npos)) {
// Possible path traversal evasion .2f. detected: - clean up and scan with regexes again.
dbgTrace(D_WAAP_EVASIONS) << "comment evasion .2f. found" << res.unescaped_line
<< "Status beroe evasion checking " << nicePrint(res);
std::string unescaped = line;
replaceAll(unescaped, "2f", "/");
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
}
if (kwCount == res.keyword_matches.size()) {
// Remove the evasion keyword if no real evasion found
keywordsToRemove.push_back("path_traversal");
path_traversal_ev = false;
}
else if (!binaryDataFound) {
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
newWordsCount);
// Take minimal words count because empirically it means evasion was probably succesfully decoded
wordsCount = std::min(wordsCount, newWordsCount);
}
dbgTrace(D_WAAP_EVASIONS) << "status after evasion checking " << nicePrint(res);
}
bool quoutes_space_evasion = Waap::Util::find_in_map_of_stringlists_keys(
"quotes_space_ev_fast_reg",
res.found_patterns
@ -1726,7 +1816,7 @@ WaapAssetState::apply(
wordsCount = std::min(wordsCount, newWordsCount);
}
}
dbgTrace(D_WAAP_SAMPLE_SCAN) << "after evasions..." << nicePrint(res);
// Remove evasion keywords that should not be reported because there's no real evasion found
if (!keywordsToRemove.empty()) {
dbgTrace(D_WAAP_SAMPLE_SCAN)

1
components/security_apps/waap/waap_clib/WaapAssetState.h
View File

@ -49,6 +49,7 @@ private: //ugly but needed for build
Waap::Util::map_of_stringlists_t & found_patterns, bool longTextFound, bool binaryDataFound) const;
void filterKeywordsDueToLongText(Waf2ScanResult &res) const;
std::string nicePrint(Waf2ScanResult &res) const;
public:
// Load and compile signatures from file

250
components/security_apps/waap/waap_clib/WaapModelResultLogger.cc
View File

@ -1,250 +0,0 @@
// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "WaapModelResultLogger.h"
#include "Waf2Engine.h"
#include "i_time_get.h"
#include "i_messaging.h"
#include "i_instance_awareness.h"
#include "http_manager.h"
#include "LogGenWrapper.h"
#include "rest.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_MODEL_LOGGER);
using namespace std;
static const unsigned int MAX_FILES_PER_WINDOW = 5;
static const unsigned int MAX_LOGS_PER_WINDOW = 1800;
static constexpr std::chrono::minutes RATE_LIMIT_WINDOW_MINUTES = std::chrono::minutes(30);
class WaapModelReport : public RestGetFile
{
public:
WaapModelReport(const vector<WaapModelResult> &_data) : data(_data) {}
private:
C2S_PARAM(vector<WaapModelResult>, data);
};
class WaapModelResultLogger::Impl
:
Singleton::Provide<I_WaapModelResultLogger>::From<WaapModelResultLogger>
{
public:
Impl(size_t maxLogs) : max_logs(maxLogs), sent_files_count(0), sent_logs_count(0),
last_sent_s3(std::chrono::minutes::zero()),
last_kusto_log_window(std::chrono::minutes::zero()) {}
virtual ~Impl();
void
logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
string modelName,
string otherModelName,
double score,
double otherScore) override;
private:
void logToStream(WaapModelResult &result, chrono::minutes now);
void logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now);
bool shouldSendLogsToS3(chrono::minutes now);
void sendLogsToS3();
size_t max_logs;
unsigned int sent_files_count;
unsigned int sent_logs_count;
std::chrono::minutes last_sent_s3;
std::chrono::minutes last_kusto_log_window;
std::map<std::string, vector<WaapModelResult>> logs;
};
WaapModelResultLogger::WaapModelResultLogger(size_t maxLogs) : pimpl(make_unique<WaapModelResultLogger::Impl>(maxLogs))
{
}
WaapModelResultLogger::~WaapModelResultLogger()
{
}
void
WaapModelResultLogger::logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
std::string modelName,
std::string otherModelName,
double score,
double otherScore
)
{
pimpl->logModelResult(settings, transaction, res, modelName, otherModelName, score, otherScore);
}
void
WaapModelResultLogger::Impl::logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
string modelName,
string otherModelName,
double score,
double otherScore)
{
if (transaction == NULL) return;
if (!Singleton::exists<I_Messaging>()) {
dbgError(D_WAAP_MODEL_LOGGER) << "Messaging service is not available, will not log";
return;
}
double score_diff = score - otherScore;
if (settings.logLevel == Waap::Scores::ModelLogLevel::DIFF &&
! ((score_diff > 0 && score >= 1.5f && otherScore < 4.0f) ||
(score_diff < 0 && score < 4.0f && otherScore >= 1.5f))) {
return;
}
auto current_time = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
auto now = chrono::duration_cast<chrono::minutes>(current_time);
WaapModelResult result = WaapModelResult(
*transaction,
res,
modelName,
otherModelName,
score,
otherScore,
now.count()
);
if (settings.logToStream) logToStream(result, now);
if (settings.logToS3) logToS3(result, transaction, now);
}
void WaapModelResultLogger::Impl::logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now)
{
auto asset_state = transaction->getAssetState();
string asset_id = (asset_state != nullptr) ? asset_state->m_assetId : "";
auto asset_logs = logs.find(asset_id);
if (asset_logs == logs.end()) {
logs.emplace(asset_id, vector<WaapModelResult>());
}
logs.at(asset_id).push_back(result);
if (shouldSendLogsToS3(now)) {
sendLogsToS3();
}
}
void WaapModelResultLogger::Impl::logToStream(WaapModelResult &result, chrono::minutes now)
{
if (now - last_kusto_log_window > RATE_LIMIT_WINDOW_MINUTES) {
last_kusto_log_window = now;
sent_logs_count = 0;
}
else if (sent_logs_count > MAX_LOGS_PER_WINDOW) {
return;
}
sent_logs_count++;
dbgTrace(D_WAAP_MODEL_LOGGER) << "Logging WAAP model telemetry";
auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
LogGenWrapper logGenWrapper(
maybeLogTriggerConf,
"WAAP Model Telemetry",
ReportIS::Audience::SECURITY,
LogTriggerConf::SecurityType::ThreatPrevention,
ReportIS::Severity::CRITICAL,
ReportIS::Priority::HIGH,
false);
LogGen& waap_log = logGenWrapper.getLogGen();
waap_log.addMarkerSuffix(result.location);
waap_log << LogField("httpuripath", result.uri);
waap_log << LogField("matchedlocation", result.location);
waap_log << LogField("matchedparameter", result.param);
waap_log << LogField("matchedindicators", Waap::Util::vecToString(result.keywords), LogFieldOption::XORANDB64);
waap_log << LogField("matchedsample", result.sample, LogFieldOption::XORANDB64);
waap_log << LogField("waapkeywordsscore", (int)(result.otherScore * 100));
waap_log << LogField("waapfinalscore", (int)(result.score * 100));
waap_log << LogField("indicatorssource", result.modelName);
waap_log << LogField("indicatorsversion", result.otherModelName);
}
bool WaapModelResultLogger::Impl::shouldSendLogsToS3(chrono::minutes now)
{
if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) return true;
for (const auto &asset_logs : logs) {
if (asset_logs.second.size() >= max_logs) return true;
}
return false;
}
void WaapModelResultLogger::Impl::sendLogsToS3()
{
dbgFlow(D_WAAP_MODEL_LOGGER) << "Sending logs to fog";
I_Messaging *msg = Singleton::Consume<I_Messaging>::by<WaapComponent>();
for (auto &asset_logs : logs) {
if (asset_logs.second.empty()) {
continue;
}
if (sent_files_count >= MAX_FILES_PER_WINDOW) {
dbgInfo(D_WAAP_MODEL_LOGGER) << "Reached max files per window, will wait for next window";
asset_logs.second.clear();
continue;
}
I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
string tenant_id = agentDetails->getTenantId();
string agent_id = agentDetails->getAgentId();
string asset_id = asset_logs.first;
if (Singleton::exists<I_InstanceAwareness>()) {
I_InstanceAwareness* instance = Singleton::Consume<I_InstanceAwareness>::by<WaapComponent>();
Maybe<string> uniqueId = instance->getUniqueID();
if (uniqueId.ok())
{
agent_id += "/" + uniqueId.unpack();
}
}
string uri = "/storage/waap/" +
tenant_id + "/" + asset_id + "/waap_model_results/window_" +
to_string(last_sent_s3.count()) + "-" + to_string(sent_files_count) +
"/" + agent_id + "/data.data";
WaapModelReport report = WaapModelReport(asset_logs.second);
dbgInfo(D_WAAP_MODEL_LOGGER) << "Sending logs for asset " << asset_logs.first <<
", length " << asset_logs.second.size() <<
", uri " << uri;
msg->sendAsyncMessage(
HTTPMethod::PUT,
uri,
report,
MessageCategory::LOG
);
asset_logs.second.clear();
}
auto current_time = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
auto now = chrono::duration_cast<chrono::minutes>(current_time);
if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) {
last_sent_s3 = now;
sent_files_count = 0;
} else {
sent_files_count++;
}
}
WaapModelResultLogger::Impl::~Impl()
{}

109
components/security_apps/waap/waap_clib/WaapModelResultLogger.h
View File

@ -1,109 +0,0 @@
// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <memory>
#include <string>
#include <map>
#include <chrono>
#include <vector>
#include <ostream>
#include "cereal/archives/json.hpp"
#include "i_waap_model_result_logger.h"
#include "DeepAnalyzer.h"
#include "i_transaction.h"
#include "ScanResult.h"
#include "WaapAssetState.h"
#include "WaapScores.h"
class WaapModelResultLogger
:
Singleton::Provide<I_WaapModelResultLogger>
{
public:
WaapModelResultLogger(size_t maxLogs = MAX_WAAP_MODEL_LOGS);
virtual ~WaapModelResultLogger();
virtual void logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
std::string modelName,
std::string otherModelName,
double score,
double otherScore
);
class Impl;
protected:
std::unique_ptr<Impl> pimpl;
static const size_t MAX_WAAP_MODEL_LOGS = 20000;
};
class WaapModelResult
{
public:
WaapModelResult(
IWaf2Transaction &transaction,
Waf2ScanResult &res,
const std::string &modelName,
const std::string &otherModelName,
double score,
double otherScore,
uint64_t time
) : uri(transaction.getUri()), location(res.location), param(res.param_name),
modelName(modelName), otherModelName(otherModelName),
score(score), otherScore(otherScore), keywords(res.keywordsAfterFilter),
sample(res.unescaped_line.substr(0, 100)), id(transaction.getIndex()), time(time)
{
}
template<class Archive>
void serialize(Archive &ar) const
{
ar(cereal::make_nvp("uri", uri));
ar(cereal::make_nvp("location", location));
ar(cereal::make_nvp("param", param));
ar(cereal::make_nvp("modelName", modelName));
ar(cereal::make_nvp("otherModelName", otherModelName));
ar(cereal::make_nvp("score", score));
ar(cereal::make_nvp("otherScore", otherScore));
ar(cereal::make_nvp("keywords", keywords));
ar(cereal::make_nvp("sample", sample));
ar(cereal::make_nvp("id", id));
ar(cereal::make_nvp("time", time));
}
std::string toString() const
{
std::stringstream message_stream;
{
cereal::JSONOutputArchive ar(message_stream);
serialize(ar);
}
return message_stream.str();
}
std::string uri;
std::string location;
std::string param;
std::string modelName;
std::string otherModelName;
double score;
double otherScore;
std::vector<std::string> keywords;
std::string sample;
uint64_t id;
uint64_t time;
};

5
components/security_apps/waap/waap_clib/WaapOverride.h
View File

@ -48,8 +48,9 @@ public:
m_tag = to_lower_copy(m_tag);
if (m_tag != "sourceip" && m_tag != "sourceidentifier" && m_tag != "url" && m_tag != "hostname" &&
m_tag != "keyword" && m_tag != "paramname" && m_tag != "paramvalue" && m_tag != "paramlocation" &&
m_tag != "responsebody" && m_tag != "headername" && m_tag != "headervalue" && m_tag != "method") {
m_tag != "keyword" && m_tag != "indicator" && m_tag != "paramname" && m_tag != "paramvalue" &&
m_tag != "paramlocation" && m_tag != "responsebody" && m_tag != "headername" &&
m_tag != "headervalue" && m_tag != "method") {
m_isValid = false;
dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag;
}

2
components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
View File

@ -105,7 +105,7 @@ bool WaapOverrideFunctor::operator()(
}
return false;
}
else if (tagLower == "keyword") {
else if (tagLower == "keyword" || tagLower == "indicator") {
for (const auto &rx : rxes) {
for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
if (REGX_MATCH(keywordStr)) {

12
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@ -15,7 +15,6 @@
#include "WaapScores.h"
#include "Waf2Engine.h"
#include "i_transaction.h"
#include "WaapModelResultLogger.h"
#include <string>
#include "debug.h"
#include "reputation_features_events.h"
@ -111,10 +110,6 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
for (auto keyword : newKeywords) {
res.keywordsAfterFilter.push_back(keyword);
}
res.scoreArray.clear();
res.coefArray.clear();
res.keywordCombinations.clear();
double res_score = getScoreFromPool(res, newKeywords, poolName);
std::string other_pool_name = Waap::Scores::getOtherScorePoolName();
@ -123,14 +118,10 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
if (applyLearning && poolName != other_pool_name &&
modelLoggingSettings.logLevel != Waap::Scores::ModelLogLevel::OFF) {
double other_score = getScoreFromPool(res, newKeywords, other_pool_name);
dbgDebug(D_WAAP_SCANNER) << "Comparing score from pool " << poolName << ": " << res_score
<< ", vs. pool " << other_pool_name << ": " << other_score
<< ", score difference: " << res_score - other_score
<< ", sample: " << res.unescaped_line;
Singleton::Consume<I_WaapModelResultLogger>::by<WaapComponent>()->logModelResult(
modelLoggingSettings, m_transaction, res, poolName, other_pool_name, res_score, other_score
);
res.other_model_score = other_score;
} else {
res.other_model_score = res_score;
@ -142,6 +133,9 @@ double Waap::Scanner::getScoreFromPool(
Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
)
{
res.scoreArray.clear();
res.coefArray.clear();
res.keywordCombinations.clear();
KeywordsStats stats = m_transaction->getAssetState()->scoreBuilder.getSnapshotStats(poolName);
if (!newKeywords.empty()) {

11
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@ -1358,7 +1358,7 @@ Waf2Transaction::isHtmlType(const char* data, int data_len){
dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false";
return false;
}
std::string body(data);
std::string body(data, data_len);
if(!m_pWaapAssetState->getSignatures()->html_regex.hasMatch(body))
{
dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false";
@ -1661,6 +1661,9 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
waapLog << LogField("sourcePort", m_remote_port);
waapLog << LogField("httpHostName", m_hostStr);
waapLog << LogField("httpMethod", m_methodStr);
if (!m_siteConfig->get_AssetId().empty()) waapLog << LogField("assetId", m_siteConfig->get_AssetId());
if (!m_siteConfig->get_AssetName().empty()) waapLog << LogField("assetName", m_siteConfig->get_AssetName());
const auto& autonomousSecurityDecision = std::dynamic_pointer_cast<AutonomousSecurityDecision>(
m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
bool send_extended_log = shouldSendExtendedLog(triggerLog);
@ -2343,6 +2346,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
exceptions_dict["sourceIdentifier"].insert(m_source_identifier);
exceptions_dict["url"].insert(getUriStr());
exceptions_dict["hostName"].insert(m_hostStr);
exceptions_dict["method"].insert(m_methodStr);
for (auto &keyword : res.keyword_matches) {
exceptions_dict["indicator"].insert(keyword);
@ -2355,8 +2359,9 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
auto behaviors = exceptions.unpack().getBehavior(exceptions_dict,
getAssetState()->m_filtersMngr->getMatchedOverrideKeywords());
for (const auto &behavior : behaviors) {
dbgTrace(D_WAAP_OVERRIDE) << "got behavior: " << behavior.getId();
if (!res.filtered_keywords.empty() || res.score > 0) {
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " with filtered indicators";
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "' with filtered indicators";
std::string overrideId = behavior.getId();
if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){
m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
@ -2375,7 +2380,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
}
if (behavior == action_ignore)
{
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " should ignore.";
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "': should ignore.";
std::string overrideId = behavior.getId();
if (!overrideId.empty()) {
m_matchedOverrideIds.insert(overrideId);

1
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@ -41,7 +41,6 @@
#include "i_waap_telemetry.h"
#include "i_deepAnalyzer.h"
#include "i_time_get.h"
#include "i_waap_model_result_logger.h"
#include "table_opaque.h"
#include "WaapResponseInspectReasons.h"
#include "WaapResponseInjectReasons.h"

23
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@ -35,6 +35,7 @@
#include "user_identifiers_config.h"
#include "Waf2Regex.h"
#include "ParserBinaryFile.h"
#include "ParserKnownBenignSkipper.h"
using boost::algorithm::to_lower_copy;
using namespace std;
@ -1218,21 +1219,21 @@ static const SingleRegex csp_report_policy_re(
"csp_report_policy"
);
static const SingleRegex base64_key_value_detector_re(
"^[^<>{};,&\\?|=\\s]+={1}\\s*.+",
err,
"base64_key_value");
"^[^<>{};,&\\?|=\\s]+={1}\\s*.+",
err,
"base64_key_value");
static const SingleRegex json_key_value_detector_re(
"\\A[^<>{};,&\\?|=\\s]+=[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]",
err,
"json_key_value");
err,
"json_key_value");
static const SingleRegex base64_key_detector_re(
"^[^<>{};,&\\?|=\\s]+={1}",
err,
"base64_key");
"^[^<>{};,&\\?|=\\s]+={1}",
err,
"base64_key");
static const SingleRegex base64_prefix_detector_re(
"data:\\S*;base64,\\S+|base64,\\S+",
err,
"base64_prefix");
"data:\\S*;base64,\\S+|base64,\\S+",
err,
"base64_prefix");
// looks for combination <param>={<some text>*:<some text>*}
//used to allow parsing param=JSON to reduce false positives

10
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@ -894,6 +894,16 @@ namespace Util {
bool isValidJson(const std::string &input);
enum KnownSourceType {
SOURCE_TYPE_UNKNOWN = 0,
SOURCE_TYPE_SENSOR_DATA = 1
};
KnownSourceType detectKnownSource(const std::string &input);
bool isScreenedJson(const std::string &input);
int definePrefixedJson(const std::string &input);
bool detectJSONasParameter(const std::string &s,
std::string &key,
std::string &value);

49
components/security_apps/waap/waap_clib/Waf2Util2.cc
View File

@ -1,5 +1,7 @@
#include "Waf2Util.h"
#include "Waf2Regex.h"
#include <string>
#include "debug.h"
namespace Waap {
namespace Util {
@ -628,5 +630,52 @@ isValidJson(const std::string &input)
return false;
}
KnownSourceType
detectKnownSource(const std::string &input)
{
static bool err = false;
static const SingleRegex known_source_sensor_data_re(
"^\\{\\\"sensor_data\\\":\\\"",
err,
"known_source_sensor_data"
);
if (known_source_sensor_data_re.hasMatch(input)) {
return SOURCE_TYPE_SENSOR_DATA;
}
return SOURCE_TYPE_UNKNOWN;
}
int
definePrefixedJson(const std::string &input)
{
static const size_t MAX_JSON_PREFIX_LEN = 32;
static const size_t MIN_PARAMETER_LEN = 4;
if (input.size() < MIN_PARAMETER_LEN) {
return -1;
}
for (size_t i = 0; i < std::min(input.size(), MAX_JSON_PREFIX_LEN) - 2 ; ++i) {
if (input[i] == '-' && input[i+1] == '{') return i + 1;
}
return -1;
}
bool
isScreenedJson(const std::string &input)
{
static bool err = false;
static const SingleRegex screened_json_re(
R"(^"{\s*\\"\w+\\"\s*:\s*\\"["\w])",
err,
"screened_json"
);
if (screened_json_re.hasMatch(input)) {
return true;
}
return false;
}
} // namespace Util
} // namespace Waap

13
components/security_apps/waap/waap_component_impl.cc
View File

@ -50,8 +50,7 @@ WaapComponent::Impl::Impl() :
drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP),
waapStateTable(NULL),
transactionsCount(0),
deepAnalyzer(),
waapModelResultLogger()
deepAnalyzer()
{
}
@ -536,9 +535,15 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event)
verdict = drop_response.getVerdict();
}
bool sould_inject_response = waf2Transaction.shouldInjectResponse();
// in Chunked transfer encoding the last chunk is always empty - and we leave it empty
bool should_stay_empty_chunk = event.isLastChunk() && dataBufLen == 0;
dbgTrace(D_WAAP)
<< (sould_inject_response ? "should Inject Response" : "should not Inject Response")
<< (should_stay_empty_chunk ? " empty last chunk will stay empty" : "");
if (verdict == pending_response.getVerdict() &&
waf2Transaction.shouldInjectResponse() &&
!event.isLastChunk()
sould_inject_response &&
!should_stay_empty_chunk
) {
// Inject if needed. Note that this is only reasonable to do if there was no DROP decision above

2
components/security_apps/waap/waap_component_impl.h
View File

@ -19,7 +19,6 @@
#include "table_opaque.h"
#include "i_transaction.h"
#include "waap_clib/DeepAnalyzer.h"
#include "waap_clib/WaapModelResultLogger.h"
#include "waap_clib/WaapAssetState.h"
#include "waap_clib/WaapAssetStatesManager.h"
#include "reputation_features_agg.h"
@ -81,7 +80,6 @@ private:
uint64_t transactionsCount;
// instance of singleton classes
DeepAnalyzer deepAnalyzer;
WaapModelResultLogger waapModelResultLogger;
WaapAssetStatesManager waapAssetStatesManager;
std::unordered_set<std::string> m_seen_assets_id;
};

14
components/utils/generic_rulebase/match_query.cc
View File

@ -111,7 +111,7 @@ MatchQuery::load(cereal::JSONInputArchive &archive_in)
is_specific_label = false;
}
}
is_ignore_keyword = (key == "indicator");
is_ignore_keyword = (key == "indicator" || key == "keyword");
if (condition_type != Conditions::Exist) {
archive_in(cereal::make_nvp("value", value));
@ -244,9 +244,10 @@ MatchQuery::getAllKeys() const
bool
MatchQuery::matchAttributes(
const unordered_map<string, set<string>> &key_value_pairs,
set<string> &matched_override_keywords ) const
set<string> &matched_override_keywords) const
{
dbgTrace(D_RULEBASE_CONFIG) << "Start matching attributes";
if (type == MatchType::Condition) {
auto key_value_pair = key_value_pairs.find(key);
if (key_value_pair == key_value_pairs.end()) {
@ -257,9 +258,11 @@ MatchQuery::matchAttributes(
} else if (type == MatchType::Operator && operator_type == Operators::And) {
for (const MatchQuery &inner_match: items) {
if (!inner_match.matchAttributes(key_value_pairs, matched_override_keywords)) {
dbgTrace(D_RULEBASE_CONFIG) << "Failed to match attributes for AND operator";
return false;
}
}
dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched all inner matches for AND operator";
return true;
} else if (type == MatchType::Operator && operator_type == Operators::Or) {
// With 'or' condition, evaluate matched override keywords first and add the ones that were fully matched
@ -272,6 +275,7 @@ MatchQuery::matchAttributes(
res = true;
}
}
dbgTrace(D_RULEBASE_CONFIG) << "Match result for OR operator is: " << res;
return res;
} else {
dbgWarning(D_RULEBASE_CONFIG) << "Unsupported match query type";
@ -285,6 +289,7 @@ MatchQuery::getMatch( const unordered_map<string, set<string>> &key_value_pairs)
MatchQuery::MatchResult matches;
matches.matched_keywords = make_shared<set<string>>();
matches.is_match = matchAttributes(key_value_pairs, *matches.matched_keywords);
dbgTrace(D_RULEBASE_CONFIG) << "Match result: " << matches.is_match;
return matches;
}
@ -306,10 +311,13 @@ MatchQuery::matchAttributes(
if (isKeyTypeIp()) {
match = matchAttributesIp(values);
dbgTrace(D_RULEBASE_CONFIG) << "Match result for IP address: " << match;
} else if (isRegEx()) {
match = matchAttributesRegEx(values, matched_override_keywords);
dbgTrace(D_RULEBASE_CONFIG) << "Match result for regex: " << match;
} else {
match = matchAttributesString(values);
dbgTrace(D_RULEBASE_CONFIG) << "Match result for string: " << match;
}
return negate ? !match : match;
@ -324,6 +332,8 @@ MatchQuery::matchAttributesRegEx(
boost::cmatch value_matcher;
for (const boost::regex &val_regex : regex_values) {
for (const string &requested_match_value : values) {
dbgTrace(D_RULEBASE_CONFIG) << "Matching value: '" << requested_match_value
<< "' with regex: '" << val_regex << "'";
if (NGEN::Regex::regexMatch(
__FILE__,
__LINE__,

5
components/utils/generic_rulebase/parameters_config.cc
View File

@ -119,10 +119,13 @@ ParameterException::getBehavior(
for (const MatchBehaviorPair &match_behavior_pair: match_queries) {
MatchQuery::MatchResult match_res = match_behavior_pair.match.getMatch(key_value_pairs);
if (match_res.is_match) {
dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception from a list of matches.";
dbgTrace(D_RULEBASE_CONFIG)
<< "Successfully matched an exception from a list of matches, behavior: "
<< match_behavior_pair.behavior.getId();
// When matching indicators with action=ignore, we expect no behavior override.
// Instead, a matched keywords list should be returned which will be later removed from score calculation
if (match_res.matched_keywords->size() > 0 && match_behavior_pair.behavior == action_ignore) {
dbgTrace(D_RULEBASE_CONFIG) << "Got action ignore";
matched_override_keywords.insert(match_res.matched_keywords->begin(),
match_res.matched_keywords->end());
} else {

2
core/CMakeLists.txt
View File

@ -37,7 +37,7 @@ target_link_libraries(
ngen_core
-Wl,-whole-archive
"table;debug_is;shell_cmd;metric;tenant_manager;messaging;encryptor;time_proxy;singleton;mainloop;environment;logging;report;rest"
"config;intelligence_is_v2;event_is;memory_consumption;connkey"
"compression_utils;-lz;config;intelligence_is_v2;event_is;memory_consumption;connkey"
"instance_awareness;socket_is;agent_details;agent_details_reporter;buffers;cpu;agent_core_utilities"
"report_messaging"
-Wl,-no-whole-archive

2
core/agent_details_reporter/agent_details_reporter.cc
View File

@ -117,7 +117,7 @@ private:
return;
}
dbgInfo(D_AGENT_DETAILS)
dbgDebug(D_AGENT_DETAILS)
<< "Successfully handled attributes persistence. Operation: "
<< operation
<< ", Path "

13
core/include/services_sdk/interfaces/messaging/http_response.h
View File

@ -31,15 +31,26 @@ public:
HTTPResponse() = default;
// LCOV_EXCL_STOP
HTTPResponse(HTTPStatusCode _status_code, const std::string &_body) : status_code(_status_code), body(_body) {}
HTTPResponse(
HTTPStatusCode _status_code,
const std::string &_body,
std::unordered_map<std::string, std::string> _headers = std::unordered_map<std::string, std::string>()
)
:
status_code(_status_code),
body(_body),
headers(_headers)
{}
HTTPStatusCode getHTTPStatusCode() const;
const std::string & getBody() const;
std::string toString() const;
Maybe<std::string> getHeaderVal(const std::string &header_key);
private:
HTTPStatusCode status_code;
std::string body;
std::unordered_map<std::string, std::string> headers;
};
#endif // __HTTP_RESPONSE_H__

1
core/include/services_sdk/interfaces/messaging/messaging_enums.h
View File

@ -63,6 +63,7 @@ enum class HTTPStatusCode
HTTP_PROXY_AUTHENTICATION_REQUIRED = 407,
HTTP_REQUEST_TIME_OUT = 408,
HTTP_PAYLOAD_TOO_LARGE = 413,
HTTP_TOO_MANY_REQUESTS = 429,
// 5xx - Server error responses.
HTTP_INTERNAL_SERVER_ERROR = 500,
HTTP_NOT_IMPLEMENTED = 501,

29
core/include/services_sdk/interfaces/messaging/messaging_metadata.h
View File

@ -8,6 +8,7 @@
#include "config.h"
#include "singleton.h"
#include "i_agent_details.h"
#include "i_time_get.h"
class MessageProxySettings
{
@ -54,7 +55,7 @@ private:
uint16_t proxy_port = 0;
};
class MessageMetadata
class MessageMetadata : Singleton::Consume<I_TimeGet>
{
public:
inline MessageMetadata();
@ -227,6 +228,26 @@ public:
return sni_host_name;
}
void
setRateLimitBlock(uint block_time)
{
is_rate_limit_block = true;
auto timer = Singleton::Consume<I_TimeGet>::by<MessageMetadata>();
auto current_timeout = timer->getMonotonicTime() + std::chrono::seconds(block_time);
rate_limit_block_time = current_timeout.count();
}
bool
isRateLimitBlock() const
{
if (is_rate_limit_block) {
auto timer = Singleton::Consume<I_TimeGet>::by<MessageMetadata>();
uint current_time = timer->getMonotonicTime().count();
if (current_time < rate_limit_block_time) return true;
}
return false;
}
template <class Archive>
void
serialize(Archive &ar)
@ -243,7 +264,9 @@ public:
cereal::make_nvp("is_to_fog", is_to_fog),
cereal::make_nvp("ca_path", ca_path),
cereal::make_nvp("client_cert_path", client_cert_path),
cereal::make_nvp("client_key_path", client_key_path)
cereal::make_nvp("client_key_path", client_key_path),
cereal::make_nvp("is_rate_limit_block", is_rate_limit_block),
cereal::make_nvp("rate_limit_block_time", rate_limit_block_time)
);
}
@ -262,6 +285,8 @@ private:
std::string external_certificate = "";
bool should_buffer = false;
bool is_to_fog = false;
bool is_rate_limit_block = false;
uint rate_limit_block_time = 0;
};
#endif // __MESSAGING_METADATA_H__

2
core/include/services_sdk/resources/debug_flags.h
View File

@ -107,6 +107,8 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_WAAP_PARSER_PAIRS, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_PDF, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_BINARY_FILE, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_SCREENED_JSON, D_WAAP_PARSER)
DEFINE_FLAG(D_IPS, D_COMPONENT)
DEFINE_FLAG(D_FILE_UPLOAD, D_COMPONENT)

2
core/include/services_sdk/resources/generic_metric.h
View File

@ -24,6 +24,7 @@
#include "i_mainloop.h"
#include "i_time_get.h"
#include "i_agent_details.h"
#include "i_encryptor.h"
#include "i_instance_awareness.h"
#include "i_environment.h"
#include "i_messaging.h"
@ -57,6 +58,7 @@ class GenericMetric
Singleton::Consume<I_Environment>,
Singleton::Consume<I_Messaging>,
Singleton::Consume<I_RestApi>,
Singleton::Consume<I_Encryptor>,
public Listener<AllMetricEvent>
{
public:

77
core/include/services_sdk/resources/metric/metric_calc.h
View File

@ -23,6 +23,8 @@
#include "report/report.h"
#include "customized_cereal_map.h"
#include "compression_utils.h"
#include "i_encryptor.h"
class GenericMetric;
@ -56,6 +58,11 @@ public:
value(_value)
{
timestamp = Singleton::Consume<I_TimeGet>::by<GenericMetric>()->getWalltimeStr();
// convert timestamp to RFC 3339 format
std::size_t pos = timestamp.find('.');
if (pos != std::string::npos) {
timestamp = timestamp.substr(0, pos) + "Z";
}
asset_id = Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId();
}
@ -118,7 +125,73 @@ public:
}
// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
std::string
Maybe<std::string>
toString() const
{
std::stringstream ss;
{
cereal::JSONOutputArchive ar(ss);
serialize(ar);
}
auto res = compressAndEncodeData(ss.str());
if (!res.ok()) {
return genError("Failed to compress and encode the data");
}
return res.unpack();
}
// LCOV_EXCL_STOP
private:
Maybe<std::string>
compressAndEncodeData(const std::string &unhandled_data) const
{
std::string data_holder = unhandled_data;
auto compression_stream = initCompressionStream();
CompressionResult compression_response = compressData(
compression_stream,
CompressionType::GZIP,
data_holder.size(),
reinterpret_cast<const unsigned char *>(data_holder.c_str()),
true
);
finiCompressionStream(compression_stream);
if (!compression_response.ok) {
// send log to Kibana
return genError("Failed to compress(gzip) data");
}
std::string compressed_data =
std::string((const char *)compression_response.output, compression_response.num_output_bytes);
auto encryptor = Singleton::Consume<I_Encryptor>::by<GenericMetric>();
Maybe<std::string> handled_data = encryptor->base64Encode(compressed_data);
if (compression_response.output) free(compression_response.output);
compression_response.output = nullptr;
compression_response.num_output_bytes = 0;
return handled_data;
}
std::vector<AiopsMetricData> metrics;
};
class CompressAndEncodeAIOPSMetrics
{
public:
CompressAndEncodeAIOPSMetrics(const AiopsMetricList &_aiops_metrics) : aiops_metrics(_aiops_metrics) {}
void
serialize(cereal::JSONOutputArchive &ar) const
{
auto metric_str = aiops_metrics.toString();
if (!metric_str.ok()) {
return;
}
ar(cereal::make_nvp("records", metric_str.unpack()));
}
// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
Maybe<std::string>
toString() const
{
std::stringstream ss;
@ -131,7 +204,7 @@ public:
// LCOV_EXCL_STOP
private:
std::vector<AiopsMetricData> metrics;
AiopsMetricList aiops_metrics;
};
class MetricCalc

2
core/include/services_sdk/resources/rest.h
View File

@ -33,6 +33,8 @@
#include "maybe_res.h"
#include "rest/schema_printer.h"
static const std::string BULK_ARRAY_NAME = "bulkArray";
/// @class JsonError
/// @brief Class representing JSON parsing errors.
///

39
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@ -93,10 +93,13 @@ public:
res << "\"name\": \"" << details->getAgentId() << "\", ";
auto rest = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
res << "\"url\": \"http://127.0.0.1:" << rest->getListeningPort() <<"/set-new-invalidation\", ";
res << "\"capabilities\": { \"getBulkCallback\": " << "true" << " }, ";
res << "\"dataMap\": [";
res << stream.str();
res << " ] }";
dbgTrace(D_INTELLIGENCE) << res.str();
return res;
}
@ -148,13 +151,19 @@ public:
void
performCallBacks(const Invalidation &invalidation, const string &registration_id) const override
{
dbgDebug(D_INTELLIGENCE) << "Looking for callbacks for invalidation " << invalidation.genObject();
dbgTrace(D_INTELLIGENCE)
<< "Looking for callbacks for invalidation "
<< invalidation.genObject()
<< " registration id: "
<< registration_id;
if (registration_id != "") {
auto invalidation_cb = registration_id_to_cb.find(registration_id);
if (invalidation_cb != registration_id_to_cb.end()) return invalidation_cb->second(invalidation);
}
dbgDebug(D_INTELLIGENCE) << "Have not found callback per registration id";
for (auto &registed_invalidation : callbacks) {
dbgTrace(D_INTELLIGENCE) << "Checking against: " << registed_invalidation.second.first.genObject();
dbgDebug(D_INTELLIGENCE) << "Checking against: " << registed_invalidation.second.first.genObject();
performCallBacksImpl(invalidation, registed_invalidation.second);
}
}
@ -169,6 +178,7 @@ private:
auto &registereed_invalidation = invalidation_and_cb.first;
auto &cb = invalidation_and_cb.second;
if (!registereed_invalidation.matches(actual_invalidation)) return;
dbgTrace(D_INTELLIGENCE) << "Found a matching invalidation registration, should callback";
cb(actual_invalidation);
}
@ -177,7 +187,7 @@ private:
uint running_id = 0;
};
class ReceiveInvalidation : public ServerRest
class SingleReceivedInvalidation : public ServerRest
{
public:
void
@ -248,6 +258,29 @@ private:
C2S_OPTIONAL_PARAM(string, invalidationType);
};
class ReceiveInvalidation : public ServerRest
{
public:
void
doCall() override
{
dbgTrace(D_INTELLIGENCE)
<< (bulkArray.isActive() ?
"BULK invalidations, receiving invalidations in bulks"
: "error in format, expected bulk invalidations, not single");
for (SingleReceivedInvalidation &r : bulkArray.get()) {
r.doCall();
}
return;
}
private:
C2S_LABEL_PARAM(vector<SingleReceivedInvalidation>, bulkArray, BULK_ARRAY_NAME);
};
class PagingController
{
public:

51
core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc
View File

@ -486,6 +486,8 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation)
EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]"));
EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"attr3\": \"3\" } ]"));
EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true }"));
}
TEST_F(IntelligenceInvalidation, register_for_multiple_assets_invalidation)
@ -529,6 +531,7 @@ TEST_F(IntelligenceInvalidation, register_for_multiple_assets_invalidation)
));
EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0);
EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true }"));
EXPECT_THAT(
body,
@ -606,7 +609,7 @@ TEST_F(IntelligenceInvalidation, invalidation_callback)
.setObjectType(Intelligence::ObjectType::ASSET);
stringstream json;
json << invalidation2.genObject();
json << "[" << invalidation2.genObject() << "]";
mock_invalidation->performRestCall(json);
EXPECT_EQ(recieved_invalidations.size(), 1u);
@ -650,7 +653,7 @@ TEST_F(IntelligenceInvalidation, delete_invalidation_callback)
.setObjectType(Intelligence::ObjectType::ASSET);
stringstream json;
json << invalidation2.genObject();
json << "[" << invalidation2.genObject() << "]";
mock_invalidation->performRestCall(json);
EXPECT_EQ(recieved_invalidations.size(), 0u);
@ -694,7 +697,7 @@ TEST_F(IntelligenceInvalidation, invalidation_short_handling)
.setObjectType(Intelligence::ObjectType::ASSET);
stringstream json;
json << invalidation2.genObject();
json << "[" << invalidation2.genObject() << "]";
mock_invalidation->performRestCall(json);
EXPECT_EQ(recieved_invalidations.size(), 0u);
@ -789,7 +792,7 @@ TEST_F(IntelligenceInvalidation, invalidation_flow_with_multiple_assets)
.setObjectType(Intelligence::ObjectType::ASSET);
stringstream json1;
json1 << not_matching_invalidation.genObject();
json1 << "[" << not_matching_invalidation.genObject() << "]";
mock_invalidation->performRestCall(json1);
EXPECT_EQ(recieved_invalidations.size(), 0u);
@ -805,7 +808,7 @@ TEST_F(IntelligenceInvalidation, invalidation_flow_with_multiple_assets)
.setObjectType(Intelligence::ObjectType::ASSET);
stringstream json2;
json2 << matching_invalidation.genObject();
json2 << "[" << matching_invalidation.genObject() << "]";
mock_invalidation->performRestCall(json2);
EXPECT_EQ(recieved_invalidations.size(), 1u);
@ -865,7 +868,7 @@ TEST_F(IntelligenceInvalidation, invalidation_cb_match_2_registred_assets)
auto stop_listening_2 = make_scope_exit([&] { invalidation_2_to_register.stopListening(i_intelligence); });
stringstream json;
json << matching_invalidation.genObject();
json << "[" << matching_invalidation.genObject() << "]";
mock_invalidation->performRestCall(json);
EXPECT_EQ(recieved_invalidations.size(), 2u);
@ -927,9 +930,43 @@ TEST_F(IntelligenceInvalidation, invalidation_cb_match_by_registration_id)
string modifiedJsonString = matching_invalidation.genObject().substr(2);
stringstream json;
json << "{ \"invalidationRegistrationId\": \""<< *registration_id << "\", " << modifiedJsonString;
json << "[{ \"invalidationRegistrationId\": \""<< *registration_id << "\", " << modifiedJsonString << "]";
cout << json.str() << endl;
mock_invalidation->performRestCall(json);
EXPECT_EQ(recieved_invalidations.size(), 1u);
}
TEST_F(IntelligenceInvalidation, bulk_invalidation_callback)
{
stringstream configuration;
configuration << "{";
configuration << " \"agentSettings\":[";
configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}";
configuration << " ],";
configuration << " \"intelligence\":{";
configuration << " \"local intelligence server ip\":\"127.0.0.1\",";
configuration << " \"local intelligence server primary port\":9090";
configuration << " }";
configuration << "}";
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
EXPECT_CALL(
messaging_mock,
sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _)
).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")));
auto invalidation = Invalidation("agent_y");
EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0);
auto invalidation2 = Invalidation("agent2");
EXPECT_NE(i_intelligence->registerInvalidation(invalidation2, callback), 0);
auto invalidation3 = Invalidation("agent3");
EXPECT_NE(i_intelligence->registerInvalidation(invalidation3, callback), 0);
dbgTrace(D_INTELLIGENCE) << "2 callbacks: ";
stringstream json3;
json3 << "[{\"class\":\"agent3\"},{\"class\":\"agent2\"}]";
mock_invalidation->performRestCall(json3);
EXPECT_EQ(recieved_invalidations.size(), 2u);
}

2
core/messaging/include/response_parser.h
View File

@ -25,7 +25,7 @@ private:
bool isLegalChunkedResponse(const std::string &res);
Maybe<HTTPStatusCode> status_code = genError("Not received");
Maybe<std::map<std::string, std::string>> headers = genError("Not received");
Maybe<std::unordered_map<std::string, std::string>> headers = genError("Not received");
std::string body;
std::string raw_response;
bool error = false;

18
core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc
View File

@ -276,11 +276,18 @@ MessagingBufferComponent::Impl::sendMessage()
}
if (res == HTTPStatusCode::HTTP_SUSPEND) {
dbgDebug(D_MESSAGING) << "Suspended connection - sleeping for a while";
dbgDebug(D_MESSAGING) << "Message is Suspended - sleeping for a while";
mainloop->yield(chrono::seconds(1));
return true;
}
if (res == HTTPStatusCode::HTTP_TOO_MANY_REQUESTS) {
dbgDebug(D_MESSAGING) << "Suspended message due to rate limit block - sleeping for a while";
mainloop->yield(chrono::seconds(1));
popMessage();
return true;
}
++curr_no_retries;
if (curr_no_retries >= getProfileAgentSettingWithDefault<uint>(10, "eventBuffer.maxNumOfSendigRetries")) {
dbgWarning(D_MESSAGING) << "Reached maximum number of retries - poping message";
@ -295,6 +302,12 @@ MessagingBufferComponent::Impl::sendMessage(const BufferedMessage &message) cons
{
MessageMetadata message_metadata = message.getMessageMetadata();
message_metadata.setShouldBufferMessage(false);
if (message_metadata.isRateLimitBlock()) {
mainloop->yield(chrono::seconds(1));
return HTTPStatusCode::HTTP_SUSPEND;
}
auto res = messaging->sendSyncMessage(
message.getMethod(),
message.getURI(),
@ -305,6 +318,9 @@ MessagingBufferComponent::Impl::sendMessage(const BufferedMessage &message) cons
if (res.ok()) return HTTPStatusCode::HTTP_OK;
if (res.getErr().getHTTPStatusCode() == HTTPStatusCode::HTTP_SUSPEND) return HTTPStatusCode::HTTP_SUSPEND;
if (res.getErr().getHTTPStatusCode() == HTTPStatusCode::HTTP_TOO_MANY_REQUESTS) {
return HTTPStatusCode::HTTP_TOO_MANY_REQUESTS;
}
return HTTPStatusCode::HTTP_UNKNOWN;
}

18
core/messaging/messaging_comp/http_response.cc
View File

@ -36,6 +36,7 @@ static const map<HTTPStatusCode, string> status_code_to_string = {
{ HTTPStatusCode::HTTP_PROXY_AUTHENTICATION_REQUIRED, "407 - HTTP_PROXY_AUTHENTICATION_REQUIRED" },
{ HTTPStatusCode::HTTP_REQUEST_TIME_OUT, "408 - HTTP_REQUEST_TIME_OUT" },
{ HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE, "413 - HTTP_PAYLOAD_TOO_LARGE" },
{ HTTPStatusCode::HTTP_TOO_MANY_REQUESTS, "429 - HTTP_TOO_MANY_REQUESTS" },
{ HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR, "500 - HTTP_INTERNAL_SERVER_ERROR" },
{ HTTPStatusCode::HTTP_NOT_IMPLEMENTED, "501 - HTTP_NOT_IMPLEMENTED" },
{ HTTPStatusCode::HTTP_BAD_GATEWAY, "502 - HTTP_BAD_GATEWAY" },
@ -63,6 +64,7 @@ static const map<int, HTTPStatusCode> num_to_status_code = {
{ 407, HTTPStatusCode::HTTP_PROXY_AUTHENTICATION_REQUIRED },
{ 408, HTTPStatusCode::HTTP_REQUEST_TIME_OUT },
{ 413, HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE },
{ 429, HTTPStatusCode::HTTP_TOO_MANY_REQUESTS },
{ 500, HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR },
{ 501, HTTPStatusCode::HTTP_NOT_IMPLEMENTED },
{ 502, HTTPStatusCode::HTTP_BAD_GATEWAY },
@ -100,6 +102,16 @@ HTTPResponse::toString() const
return "[Status-code]: " + code->second + ", [Body]: " + (body.empty() ? "{}" : body);
}
Maybe<string>
HTTPResponse::getHeaderVal(const string &header_key)
{
auto header = headers.find(header_key);
if (header == headers.end()) {
return genError("Header \'" + header_key + "\' not found.");
}
return header->second;
}
Maybe<HTTPResponse>
HTTPResponseParser::parseData(const string &data, bool is_connect)
{
@ -116,7 +128,7 @@ HTTPResponseParser::parseData(const string &data, bool is_connect)
if (!handleBody(is_connect)) return genError("Response not ready!");
return HTTPResponse(status_code.unpack(), body);
return HTTPResponse(status_code.unpack(), body, headers.unpack());
}
static string
@ -133,7 +145,7 @@ bool
HTTPResponseParser::handleHeaders()
{
stringstream ss(raw_response);
map<string, string> header_map;
unordered_map<string, string> header_map;
while (true) {
string header;
@ -171,7 +183,7 @@ HTTPResponseParser::getHeaderVal(const string &header_key)
auto headers_map = headers.unpack();
auto header = headers_map.find(header_key);
if (header == headers_map.end()) {
return genError("Header\'" + header_key + "\' not found.");
return genError("Header \'" + header_key + "\' not found.");
}
return header->second;
}

24
core/messaging/messaging_comp/messaging_comp.cc
View File

@ -148,6 +148,21 @@ MessagingComp::sendMessage(
auto response = i_conn->sendRequest(conn, *req);
if (!response.ok()) return response.passErr();
auto response_data = response.unpack();
if (response_data.getHTTPStatusCode() == HTTPStatusCode::HTTP_TOO_MANY_REQUESTS) {
dbgDebug(D_MESSAGING) << "Too many requests. Suspend the message";
auto rate_limit_metadata = message_metadata;
uint retry_after_sec = 60;
auto retry_after_header = response_data.getHeaderVal("retry-after");
if (retry_after_header.ok()) {
retry_after_sec = stoi(*retry_after_header);
}
rate_limit_metadata.setShouldBufferMessage(true);
rate_limit_metadata.setRateLimitBlock(retry_after_sec);
return suspendMessage(body, method, uri, category, rate_limit_metadata);
}
if (is_to_fog && method == HTTPMethod::GET) fog_get_requests_cache.emplaceEntry(uri, *response);
return response;
}
@ -355,6 +370,15 @@ MessagingComp::suspendMessage(
const MessageMetadata &message_metadata
) const
{
if (message_metadata.isRateLimitBlock()) {
dbgInfo(D_MESSAGING) << "Rate limit block is active, message is suspended, message is buffered.";
i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false);
return genError<HTTPResponse>(
HTTPStatusCode::HTTP_TOO_MANY_REQUESTS,
"The connection is suspended due to rate limit block, message is buffered."
);
}
if (message_metadata.shouldBufferMessage()) {
dbgWarning(D_MESSAGING) << "Buffering message due to connection suspended";
i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false);

27
core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc
View File

@ -276,3 +276,30 @@ TEST_F(TestMessagingComp, testSetFogConnection)
EXPECT_CALL(mock_messaging_connection, establishConnection(metadata, category)).WillOnce(Return(conn));
EXPECT_TRUE(messaging_comp.setFogConnection(category));
}
TEST_F(TestMessagingComp, testRateLimitBlock)
{
setAgentDetails();
string body = "test body";
HTTPMethod method = HTTPMethod::POST;
string uri = "/test-uri";
MessageCategory category = MessageCategory::GENERIC;
MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC);
Flags<MessageConnectionConfig> conn_flags;
conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags, false, true);
Connection conn(conn_key, conn_metadata);
EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC))
.WillOnce(Return(conn));
unordered_map<string, string> res_headers = {{"Retry-After", "10"}};
HTTPResponse res(HTTPStatusCode::HTTP_TOO_MANY_REQUESTS, "response!!", res_headers);
EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res));
auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, conn_metadata);
ASSERT_FALSE(sending_res.ok());
HTTPResponse http_res = sending_res.getErr();
EXPECT_EQ(http_res.getBody(), "The connection is suspended due to rate limit block, message is buffered.");
EXPECT_EQ(http_res.getHTTPStatusCode(), HTTPStatusCode::HTTP_TOO_MANY_REQUESTS);
}

30
core/metric/generic_metric.cc
View File

@ -32,7 +32,10 @@ MetricMetadata::Units operator"" _unit(const char *str, size_t) { return MetricM
MetricMetadata::Description operator"" _desc(const char *str, size_t) { return MetricMetadata::Description{str}; }
// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
static ostream & operator<<(ostream &os, const AiopsMetricList &metrics) { return os << metrics.toString(); }
static ostream & operator<<(ostream &os, const CompressAndEncodeAIOPSMetrics &metrics)
{
return os << metrics.toString();
}
// LCOV_EXCL_STOP
vector<AiopsMetricData>
@ -44,7 +47,7 @@ MetricCalc::getAiopsMetrics() const
string name = getMetricDotName() != "" ? getMetricDotName() : getMetricName();
string units = getMetircUnits();
string description = getMetircDescription();
string type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter";
string type = getMetricType() == MetricType::GAUGE ? "Gauge" : "Counter";
return { AiopsMetricData(name, type, units, description, getBasicLabels(), value) };
}
@ -320,17 +323,17 @@ class PrometheusRest : public ClientRest
public:
Metric(const string &n, const string &t, const string &d, const string &l, const string &v)
:
name(n),
type(t),
description(d),
metric_name(n),
metric_type(t),
metric_description(d),
labels(l),
value(v)
{}
private:
C2S_PARAM(string, name);
C2S_PARAM(string, type);
C2S_PARAM(string, description);
C2S_PARAM(string, metric_name);
C2S_PARAM(string, metric_type);
C2S_PARAM(string, metric_description);
C2S_PARAM(string, labels);
C2S_PARAM(string, value);
};
@ -356,6 +359,7 @@ void
GenericMetric::generatePrometheus()
{
if (!getProfileAgentSettingWithDefault(false, "prometheus")) return;
dbgTrace(D_METRICS) << "Generate prometheus metric";
vector<PrometheusData> all_metrics;
for (auto &calc : calcs) {
@ -371,7 +375,7 @@ GenericMetric::generatePrometheus()
new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
Singleton::Consume<I_Messaging>::by<GenericMetric>()->sendSyncMessage(
HTTPMethod::POST,
"/set-prometheus-data",
"/add-metrics",
rest,
MessageCategory::GENERIC,
new_config_req_md
@ -382,6 +386,7 @@ void
GenericMetric::generateAiopsLog()
{
if (!getConfigurationWithDefault<bool>(true, "metric", "aiopsMetricSendEnable")) return;
dbgTrace(D_METRICS) << "Generate AIOPS metric";
AiopsMetricList aiops_metrics;
@ -397,18 +402,17 @@ GenericMetric::generateAiopsLog()
Level::LOG,
LogLevel::INFO,
audience,
team,
ReportIS::AudienceTeam::HORIZON_TELEMETRY,
Severity::INFO,
Priority::LOW,
report_interval,
LogField("agentId", Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId()),
tags,
Tags::INFORMATIONAL,
issuing_engine
ReportIS::IssuingEngine::HORIZON_TELEMETRY_METRICS
);
metric_to_fog << LogField("eventObject", aiops_metrics);
metric_to_fog << LogField("eventObject", CompressAndEncodeAIOPSMetrics(aiops_metrics));
LogRest metric_client_rest(metric_to_fog);
sendLog(metric_client_rest);
}

284
core/metric/metric_ut/metric_ut.cc
View File

@ -9,6 +9,7 @@
#include "mock/mock_time_get.h"
#include "mock/mock_rest_api.h"
#include "agent_details.h"
#include "mock/mock_encryptor.h"
#include "mock/mock_messaging.h"
#include "mock/mock_instance_awareness.h"
#include "config.h"
@ -219,6 +220,7 @@ public:
::Environment env;
ConfigComponent conf;
AgentDetails agent_details;
StrictMock<MockEncryptor> mock_encryptor;
NiceMock<MockMessaging> messaging_mock;
stringstream debug_output;
I_MainLoop::Routine routine;
@ -556,7 +558,7 @@ TEST_F(MetricTest, printPromeathus)
cpu_event.notify();
string message_body;
EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _))
EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/add-metrics", _, _, _))
.WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
routine();
@ -564,44 +566,44 @@ TEST_F(MetricTest, printPromeathus)
"{\n"
" \"metrics\": [\n"
" {\n"
" \"name\": \"cpuMax\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"metric_name\": \"cpuMax\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"name\": \"cpuMin\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"metric_name\": \"cpuMin\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"name\": \"cpuAvg\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"metric_name\": \"cpuAvg\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"name\": \"cpuCurrent\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"metric_name\": \"cpuCurrent\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"name\": \"cpuCounter\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"metric_name\": \"cpuCounter\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"name\": \"cpuTotalCounter\",\n"
" \"type\": \"counter\",\n"
" \"description\": \"\",\n"
" \"metric_name\": \"cpuTotalCounter\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"1\"\n"
" }\n"
@ -636,7 +638,7 @@ TEST_F(MetricTest, printPromeathusMultiMap)
HttpTransaction("/index.html", "POST", 40).notify();
string message_body;
EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _))
EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/add-metrics", _, _, _))
.WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
routine();
@ -644,25 +646,25 @@ TEST_F(MetricTest, printPromeathusMultiMap)
"{\n"
" \"metrics\": [\n"
" {\n"
" \"name\": \"request.total\",\n"
" \"type\": \"counter\",\n"
" \"description\": \"\",\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"name\": \"request.total\",\n"
" \"type\": \"counter\",\n"
" \"description\": \"\",\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"name\": \"request.total\",\n"
" \"type\": \"counter\",\n"
" \"description\": \"\",\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n"
" \"value\": \"1\"\n"
@ -1396,6 +1398,8 @@ TEST_F(MetricTest, testManyValuesOutOfOrder)
TEST_F(MetricTest, basicAIOPSMetricTest)
{
EXPECT_CALL(timer, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087")));
EXPECT_CALL(mock_encryptor, base64Encode(_)).WillRepeatedly(Return("compress and encode metric payload"));
CPUMetric cpu_mt;
cpu_mt.init(
"CPU usage",
@ -1452,14 +1456,14 @@ TEST_F(MetricTest, basicAIOPSMetricTest)
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"Agent Core\",\n"
" \"eventAudienceTeam\": \"\",\n"
" \"eventFrequency\": 5,\n"
" \"eventTags\": [\n"
" \"Informational\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"issuingEngine\": \"Agent Core\",\n"
" \"issuingEngine\": \"horizonTelemetryMetrics\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
@ -1469,92 +1473,7 @@ TEST_F(MetricTest, basicAIOPSMetricTest)
" },\n"
" \"eventData\": {\n"
" \"eventObject\": {\n"
" \"Metrics\": [\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpu.max\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"percrnt\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 89.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuMin\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 89.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuAvg\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 89.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuCurrent\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 89.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuCounter\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 1.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuTotalCounter\",\n"
" \"MetricType\": \"counter\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 1.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" }\n"
" ]\n"
" \"records\": \"compress and encode metric payload\"\n"
" }\n"
" }\n"
" }\n"
@ -1595,8 +1514,68 @@ TEST_F(MetricTest, testAIOPSMapMetric)
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
EXPECT_CALL(mock_encryptor, base64Encode(_)).WillRepeatedly(Return("compress and encode metric payload"));
routine();
// aiops data example
// " \"Metrics\": [\n"
// " {\n"
// " \"Timestamp\": \"2016-11-13T17:31:24Z\",\n"
// " \"MetricName\": \"/index.html\",\n"
// " \"MetricType\": \"Gauge\",\n"
// " \"MetricUnit\": \"\",\n"
// " \"MetricDescription\": \"\",\n"
// " \"MetricValue\": 0.0,\n"
// " \"ResourceAttributes\": {},\n"
// " \"MetricAttributes\": {\n"
// " \"key1\": \"value1\",\n"
// " \"key2\": \"value2\"\n"
// " },\n"
// " \"AssetID\": \"Unknown\"\n"
// " },\n"
// " {\n"
// " \"Timestamp\": \"2016-11-13T17:31:24Z\",\n"
// " \"MetricName\": \"/index2.html\",\n"
// " \"MetricType\": \"Gauge\",\n"
// " \"MetricUnit\": \"\",\n"
// " \"MetricDescription\": \"\",\n"
// " \"MetricValue\": 0.0,\n"
// " \"ResourceAttributes\": {},\n"
// " \"MetricAttributes\": {\n"
// " \"key1\": \"value1\",\n"
// " \"key2\": \"value2\"\n"
// " },\n"
// " \"AssetID\": \"Unknown\"\n"
// " },\n"
// " {\n"
// " \"Timestamp\": \"2016-11-13T17:31:24Z\",\n"
// " \"MetricName\": \"/index.html\",\n"
// " \"MetricType\": \"Counter\",\n"
// " \"MetricUnit\": \"\",\n"
// " \"MetricDescription\": \"\",\n"
// " \"MetricValue\": 0.0,\n"
// " \"ResourceAttributes\": {},\n"
// " \"MetricAttributes\": {},\n"
// " \"AssetID\": \"Unknown\"\n"
// " },\n"
// " {\n"
// " \"Timestamp\": \"2016-11-13T17:31:24Z\",\n"
// " \"MetricName\": \"/index2.html\",\n"
// " \"MetricType\": \"Counter\",\n"
// " \"MetricUnit\": \"\",\n"
// " \"MetricDescription\": \"\",\n"
// " \"MetricValue\": 0.0,\n"
// " \"ResourceAttributes\": {},\n"
// " \"MetricAttributes\": {},\n"
// " \"AssetID\": \"Unknown\"\n"
// " }\n"
// " ]\n"
// " }\n"
// " }\n"
// " }\n"
// "}";
string expected_message =
"{\n"
" \"log\": {\n"
@ -1608,14 +1587,14 @@ TEST_F(MetricTest, testAIOPSMapMetric)
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"Agent Core\",\n"
" \"eventAudienceTeam\": \"\",\n"
" \"eventFrequency\": 5,\n"
" \"eventTags\": [\n"
" \"Informational\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"issuingEngine\": \"Agent Core\",\n"
" \"issuingEngine\": \"horizonTelemetryMetrics\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
@ -1625,72 +1604,7 @@ TEST_F(MetricTest, testAIOPSMapMetric)
" },\n"
" \"eventData\": {\n"
" \"eventObject\": {\n"
" \"Metrics\": [\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"/index.html\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 25.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {\n"
" \"url\": \"/index.html\"\n"
" },\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"/index2.html\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 20.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {\n"
" \"url\": \"/index2.html\"\n"
" },\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"/index.html\",\n"
" \"MetricType\": \"counter\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 2.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {\n"
" \"url\": \"/index.html\"\n"
" },\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"/index2.html\",\n"
" \"MetricType\": \"counter\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 1.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {\n"
" \"url\": \"/index2.html\"\n"
" },\n"
" \"AssetID\": \"Unknown\"\n"
" }\n"
" ]\n"
" \"records\": \"compress and encode metric payload\"\n"
" }\n"
" }\n"
" }\n"

12
core/rest/rest.cc
View File

@ -33,6 +33,18 @@ ServerRest::performRestCall(istream &in)
{
try {
try {
int firstChar = in.peek();
if (firstChar != EOF) {
// array as root is not supported in JSONInputArchive format
// but this struct is in bulk invalidations requirements
// we will change the format here
if (firstChar == '[') {
std::string content((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
std::string modifiedContent = "{\"" + BULK_ARRAY_NAME + "\": " + content + "}";
std::istringstream* modifiedStream = new std::istringstream(modifiedContent);
in.rdbuf(modifiedStream->rdbuf());
}
}
cereal::JSONInputArchive in_ar(in);
load(in_ar);
} catch (cereal::Exception &e) {

5
nodes/orchestration/package/open-appsec-cloud-mgmt
View File

@ -58,6 +58,7 @@ send_notification_to_the_fog()
"$var_fog/api/v1/agents/events/bulk" -H "X-Trace-Id:${correlation_id}" \
--header "Authorization: Bearer ${ra_token}" --header "user-agent: Infinity Next (a7030abf93a4c13)" \
--header "Content-Type: application/json" \
--header "x-rate-limit-product-type: openappsec" \
--data "{\"logs\": [{\"log\": {\"eventTime\": \
\"$DATE\",\"eventName\": \"Agent started onboarding process to cloud management\",\"eventSeverity\": \
\"Info\",\"eventPriority\": \"Urgent\",\"eventLogLevel\": \"info\",\"eventType\": \"Event Driven\",
@ -73,7 +74,9 @@ send_notification_to_the_fog()
upload_res=$(curl -s -o /dev/null -s -w "%{http_code}\n" \
--request POST "$var_fog/api/v1/agents/events/bulk" -H "X-Trace-Id:${correlation_id}" \
--header "Authorization: Bearer ${ra_token}" --header "user-agent: Infinity Next (a7030abf93a4c13)" \
--header "Content-Type: application/json" --data "{\"logs\": \
--header "Content-Type: application/json" \
--header "x-rate-limit-product-type: openappsec" \
--data "{\"logs\": \
[{\"log\": {\"eventTime\": \"$DATE\",\"eventName\": \"Agent started onboarding process to cloud management\",
\"eventSeverity\": \"Info\",\"eventPriority\": \"Urgent\",\"eventLogLevel\": \"info\",\"eventType\": \"Event Driven\",
\"eventLevel\": \"Log\",\"eventAudience\": \"Internal\",\"eventAudienceTeam\": \"Agent Core\",\"eventFrequency\": 0,

8
nodes/orchestration/package/open-appsec-cloud-mgmt-k8s
View File

@ -157,7 +157,9 @@ send_notification_to_the_fog()
DATE=$(date "+%FT%T.000")
upload_res=$(curl -o /dev/null -s -w "%{http_code}\n" --request POST "$var_fog/api/v1/agents/events/bulk" \
-H "X-Trace-Id:${correlation_id}" --header "Authorization: Bearer ${ra_token}" \
--header "user-agent: Infinity Next (a7030abf93a4c13)" --header "Content-Type: application/json" \
--header "user-agent: Infinity Next (a7030abf93a4c13)" \
--header "Content-Type: application/json" \
--header "x-rate-limit-product-type: openappsec" \
--data "{\"logs\": [{\"log\": {\"eventTime\": \"$DATE\",\"eventName\": \
\"Agent started onboarding process to cloud management\",\"eventSeverity\": \"Info\",\"eventPriority\": \
\"Urgent\",\"eventLogLevel\": \"info\",\"eventType\": \"Event Driven\",\"eventLevel\": \"Log\",\"eventAudience\": \
@ -171,7 +173,9 @@ send_notification_to_the_fog()
sleep 5
upload_res=$(curl -o /dev/null -s -w "%{http_code}\n" --request POST "$var_fog/api/v1/agents/events/bulk" \
-H "X-Trace-Id:${correlation_id}" --header "Authorization: Bearer ${ra_token}" \
--header "user-agent: Infinity Next (a7030abf93a4c13)" --header "Content-Type: application/json" \
--header "user-agent: Infinity Next (a7030abf93a4c13)" \
--header "Content-Type: application/json" \
--header "x-rate-limit-product-type: openappsec" \
--data "{\"logs\": [{\"log\": {\"eventTime\": \"$DATE\",\"eventName\": \
\"Agent started onboarding process to cloud management\",\"eventSeverity\": \"Info\",\"eventPriority\": \
\"Urgent\",\"eventLogLevel\": \"info\",\"eventType\": \"Event Driven\",\"eventLevel\": \"Log\",\

2
nodes/orchestration/package/orchestration_package.sh
View File

@ -924,6 +924,8 @@ install_orchestration()
cp_exec "ln -sf /storage/nano_agent${USR_LIB_PATH}/cpnano ${USR_LIB_PATH}/cpnano"
cp_exec "mkdir -p /storage/nano_agent/${FILESYSTEM_PATH}"
cp_exec "ln -sf /storage/nano_agent/${FILESYSTEM_PATH} ${FILESYSTEM_PATH}"
cp_copy smb_egg/nano-egg-internal /opt/fw1/bin/nano-egg-internal
chmod +x /opt/fw1/bin/nano-egg-internal
fi
${INSTALL_COMMAND} lib/*.so* ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/
${INSTALL_COMMAND} lib/boost/*.so* ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/

2
unit_test.cmake
View File

@ -2,7 +2,7 @@ enable_testing()
function(add_unit_test ut_name ut_sources use_libs)
add_executable(${ut_name} ${ut_sources})
target_link_libraries(${ut_name} -Wl,--start-group ${use_libs} debug_is report cptest pthread packet singleton environment metric event_is buffers rest config ${GTEST_BOTH_LIBRARIES} gmock boost_regex pthread dl -Wl,--end-group)
target_link_libraries(${ut_name} -Wl,--start-group ${use_libs} debug_is report cptest pthread packet singleton environment metric event_is buffers rest config compression_utils z ${GTEST_BOTH_LIBRARIES} gmock boost_regex pthread dl -Wl,--end-group)
add_test(NAME ${ut_name}
COMMAND ${ut_name}