github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-06-28 16:41:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #200 from openappsec/Oct_14_2024-Dev

Browse Source 
Oct 14 2024 dev
This commit is contained in:
Daniel-Eisenberg 2024-11-05 12:12:10 +02:00 committed by GitHub
commit 027ddfea21
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
96 changed files with 3012 additions and 448 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
components/include/env_details.h
View File

@ -29,12 +29,15 @@ public:
virtual EnvType getEnvType() override;
virtual std::string getToken() override;
virtual std::string getNameSpace() override;
private:
std::string retrieveToken();
std::string retrieveNamespace();
std::string readFileContent(const std::string &file_path);
std::string token;
std::string agent_namespace;
EnvType env_type;
};

2
components/include/generic_rulebase/match_query.h
View File

@ -91,7 +91,7 @@ private:
bool matchAttributesString(const std::set<std::string> &values) const;
bool matchAttributesIp(const std::set<std::string> &values) const;
bool isRegEx() const;
bool isIP() const;
void sortAndMergeIpRangesValues();
MatchType type;
Operators operator_type;

3
components/include/ip_utilities.h
View File

@ -28,8 +28,9 @@
// LCOV_EXCL_START Reason: temporary until we add relevant UT until 07/10
bool operator<(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr);
bool operator==(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr);
bool operator<=(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr);
bool operator<(const IPRange &range1, const IPRange &range2);
// LCOV_EXCL_STOP
Maybe<std::pair<std::string, int>> extractAddressAndMaskSize(const std::string &cidr);

4
components/include/waap.h
View File

@ -33,6 +33,7 @@ class I_WaapAssetStatesManager;
class I_Messaging;
class I_AgentDetails;
class I_Encryptor;
class I_WaapModelResultLogger;
const std::string WAAP_APPLICATION_NAME = "waap application";
@ -50,7 +51,8 @@ class WaapComponent
Singleton::Consume<I_AgentDetails>,
Singleton::Consume<I_Messaging>,
Singleton::Consume<I_Encryptor>,
Singleton::Consume<I_Environment>
Singleton::Consume<I_Environment>,
Singleton::Consume<I_WaapModelResultLogger>
{
public:
WaapComponent();

6
components/security_apps/local_policy_mgmt_gen/access_control_practice.cc
View File

@ -228,7 +228,11 @@ AccessControlPracticeSpec::load(cereal::JSONInputArchive &archive_in)
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec";
parseAppsecJSONKey<string>("name", practice_name, archive_in);
parseAppsecJSONKey<string>("practiceMode", mode, archive_in);
parseAppsecJSONKey<string>("practiceMode", mode, archive_in, "inherited");
if (valid_modes.count(mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Access control practice mode invalid: " << mode;
throw PolicyGenException("AppSec Access control practice mode invalid: " + mode);
}
parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
parseMandatoryAppsecJSONKey<AccessControlRateLimit>("rateLimit", rate_limit, archive_in);
}

95
components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc
View File

@ -438,19 +438,30 @@ WebAppSection::WebAppSection(
csrf_protection_mode("Disabled"),
open_redirect_mode("Disabled"),
error_disclosure_mode("Disabled"),
schema_validation_mode("Disabled"),
schema_validation_enforce_level("fullSchema"),
practice_advanced_config(parsed_appsec_spec),
anti_bots(parsed_appsec_spec.getAntiBot()),
trusted_sources({ parsed_trusted_sources })
{
auto mitigation_sevirity = parsed_appsec_spec.getWebAttacks().getMinimumConfidence();
if (key_to_mitigation_severity.find(mitigation_sevirity) == key_to_mitigation_severity.end()) {
dbgWarning(D_LOCAL_POLICY)
<< "web attack mitigation severity invalid: "
<< mitigation_sevirity;
throw PolicyGenException("web attack mitigation severity invalid: " + mitigation_sevirity);
} else {
web_attack_mitigation_severity = key_to_mitigation_severity.at(mitigation_sevirity);
}
web_attack_mitigation = web_attack_mitigation_mode != "Disabled";
web_attack_mitigation_severity =
web_attack_mitigation_mode != "Prevent" ? "Transparent" :
parsed_appsec_spec.getWebAttacks().getMinimumConfidence();
web_attack_mitigation_severity;
web_attack_mitigation_action =
web_attack_mitigation_mode != "Prevent" ? "Transparent" :
web_attack_mitigation_severity == "critical" ? "low" :
web_attack_mitigation_severity == "high" ? "balanced" :
web_attack_mitigation_severity == "medium" ? "high" :
web_attack_mitigation_severity == "Critical" ? "Low" :
web_attack_mitigation_severity == "High" ? "Balanced" :
web_attack_mitigation_severity == "Medium" ? "High" :
"Error";
triggers.push_back(TriggersInWaapSection(parsed_log_trigger));
@ -479,6 +490,9 @@ WebAppSection::WebAppSection(
const string &_web_attack_mitigation_severity,
const string &_web_attack_mitigation_mode,
const string &_bot_protection,
const string &_schema_validation_mode,
const string &_schema_validation_enforce_level,
const vector<string> &_schema_validation_oas,
const PracticeAdvancedConfig &_practice_advanced_config,
const AppsecPracticeAntiBotSection &_anti_bots,
const LogTriggerSection &parsed_log_trigger,
@ -493,19 +507,29 @@ WebAppSection::WebAppSection(
practice_id(_practice_id),
practice_name(_practice_name),
context(_context),
web_attack_mitigation_severity(_web_attack_mitigation_severity),
web_attack_mitigation_mode(_web_attack_mitigation_mode),
bot_protection(_bot_protection),
schema_validation_mode(_schema_validation_mode),
schema_validation_enforce_level(_schema_validation_enforce_level),
schema_validation_oas(_schema_validation_oas),
practice_advanced_config(_practice_advanced_config),
anti_bots(_anti_bots),
trusted_sources({ parsed_trusted_sources })
{
if (key_to_mitigation_severity.find(_web_attack_mitigation_severity) == key_to_mitigation_severity.end()) {
dbgWarning(D_LOCAL_POLICY)
<< "web attack mitigation severity invalid: "
<< _web_attack_mitigation_severity;
throw PolicyGenException("web attack mitigation severity invalid: " + _web_attack_mitigation_severity);
} else {
web_attack_mitigation_severity = key_to_mitigation_severity.at(_web_attack_mitigation_severity);
}
web_attack_mitigation = web_attack_mitigation_mode != "Disabled";
web_attack_mitigation_action =
web_attack_mitigation_mode != "Prevent" ? "Transparent" :
web_attack_mitigation_severity == "critical" ? "low" :
web_attack_mitigation_severity == "high" ? "balanced" :
web_attack_mitigation_severity == "medium" ? "high" :
web_attack_mitigation_severity == "Critical" ? "Low" :
web_attack_mitigation_severity == "High" ? "Balanced" :
web_attack_mitigation_severity == "Medium" ? "High" :
"Error";
csrf_protection_mode = protections.getCsrfProtectionMode(_web_attack_mitigation_mode);
@ -516,6 +540,7 @@ WebAppSection::WebAppSection(
for (const SourcesIdentifiers &source_ident : parsed_trusted_sources.getSourcesIdentifiers()) {
overrides.push_back(AppSecOverride(source_ident));
}
}
// LCOV_EXCL_STOP
@ -523,35 +548,35 @@ WebAppSection::WebAppSection(
void
WebAppSection::save(cereal::JSONOutputArchive &out_ar) const
{
string disabled_str = "Disabled";
vector<string> empty_list;
out_ar(
cereal::make_nvp("context", context),
cereal::make_nvp("webAttackMitigation", web_attack_mitigation),
cereal::make_nvp("webAttackMitigationSeverity", web_attack_mitigation_severity),
cereal::make_nvp("webAttackMitigationAction", web_attack_mitigation_action),
cereal::make_nvp("webAttackMitigationMode", web_attack_mitigation_mode),
cereal::make_nvp("practiceAdvancedConfig", practice_advanced_config),
cereal::make_nvp("csrfProtection", csrf_protection_mode),
cereal::make_nvp("openRedirect", open_redirect_mode),
cereal::make_nvp("errorDisclosure", error_disclosure_mode),
cereal::make_nvp("practiceId", practice_id),
cereal::make_nvp("practiceName", practice_name),
cereal::make_nvp("assetId", asset_id),
cereal::make_nvp("assetName", asset_name),
cereal::make_nvp("ruleId", rule_id),
cereal::make_nvp("ruleName", rule_name),
cereal::make_nvp("schemaValidation", false),
cereal::make_nvp("schemaValidation_v2", disabled_str),
cereal::make_nvp("oas", empty_list),
cereal::make_nvp("triggers", triggers),
cereal::make_nvp("applicationUrls", application_urls),
cereal::make_nvp("overrides", overrides),
cereal::make_nvp("trustedSources", trusted_sources),
cereal::make_nvp("waapParameters", empty_list),
cereal::make_nvp("botProtection", false),
cereal::make_nvp("antiBot", anti_bots),
cereal::make_nvp("botProtection_v2", bot_protection != "" ? bot_protection : string("Detect"))
cereal::make_nvp("context", context),
cereal::make_nvp("webAttackMitigation", web_attack_mitigation),
cereal::make_nvp("webAttackMitigationSeverity", web_attack_mitigation_severity),
cereal::make_nvp("webAttackMitigationAction", web_attack_mitigation_action),
cereal::make_nvp("webAttackMitigationMode", web_attack_mitigation_mode),
cereal::make_nvp("practiceAdvancedConfig", practice_advanced_config),
cereal::make_nvp("csrfProtection", csrf_protection_mode),
cereal::make_nvp("openRedirect", open_redirect_mode),
cereal::make_nvp("errorDisclosure", error_disclosure_mode),
cereal::make_nvp("practiceId", practice_id),
cereal::make_nvp("practiceName", practice_name),
cereal::make_nvp("assetId", asset_id),
cereal::make_nvp("assetName", asset_name),
cereal::make_nvp("ruleId", rule_id),
cereal::make_nvp("ruleName", rule_name),
cereal::make_nvp("schemaValidation", schema_validation_mode == "Prevent"),
cereal::make_nvp("schemaValidation_v2", schema_validation_mode),
cereal::make_nvp("oas", schema_validation_oas),
cereal::make_nvp("schemaValidationEnforceLevel", schema_validation_enforce_level),
cereal::make_nvp("triggers", triggers),
cereal::make_nvp("applicationUrls", application_urls),
cereal::make_nvp("overrides", overrides),
cereal::make_nvp("trustedSources", trusted_sources),
cereal::make_nvp("waapParameters", empty_list),
cereal::make_nvp("botProtection", false),
cereal::make_nvp("antiBot", anti_bots),
cereal::make_nvp("botProtection_v2", bot_protection != "" ? bot_protection : string("Detect"))
);
}

50
components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h
View File

@ -291,6 +291,9 @@ public:
const std::string &_web_attack_mitigation_severity,
const std::string &_web_attack_mitigation_mode,
const std::string &_bot_protection,
const std::string &schema_validation_mode,
const std::string &schema_validation_enforce_level,
const std::vector<std::string> &schema_validation_oas,
const PracticeAdvancedConfig &_practice_advanced_config,
const AppsecPracticeAntiBotSection &_anti_bots,
const LogTriggerSection &parsed_log_trigger,
@ -302,27 +305,30 @@ public:
bool operator< (const WebAppSection &other) const;
private:
std::string application_urls;
std::string asset_id;
std::string asset_name;
std::string rule_id;
std::string rule_name;
std::string practice_id;
std::string practice_name;
std::string context;
std::string web_attack_mitigation_action;
std::string web_attack_mitigation_severity;
std::string web_attack_mitigation_mode;
std::string csrf_protection_mode;
std::string open_redirect_mode;
std::string error_disclosure_mode;
std::string bot_protection;
bool web_attack_mitigation;
std::vector<TriggersInWaapSection> triggers;
PracticeAdvancedConfig practice_advanced_config;
AppsecPracticeAntiBotSection anti_bots;
std::vector<AppSecTrustedSources> trusted_sources;
std::vector<AppSecOverride> overrides;
bool web_attack_mitigation;
std::string application_urls;
std::string asset_id;
std::string asset_name;
std::string rule_id;
std::string rule_name;
std::string practice_id;
std::string practice_name;
std::string context;
std::string web_attack_mitigation_action;
std::string web_attack_mitigation_severity;
std::string web_attack_mitigation_mode;
std::string csrf_protection_mode;
std::string open_redirect_mode;
std::string error_disclosure_mode;
std::string bot_protection;
std::string schema_validation_mode;
std::string schema_validation_enforce_level;
std::vector<std::string> schema_validation_oas;
PracticeAdvancedConfig practice_advanced_config;
AppsecPracticeAntiBotSection anti_bots;
std::vector<AppSecOverride> overrides;
std::vector<AppSecTrustedSources> trusted_sources;
std::vector<TriggersInWaapSection> triggers;
};
class WebAPISection
@ -410,7 +416,7 @@ class ParsedRule
{
public:
ParsedRule() {}
ParsedRule(const std::string &_host) : host(_host) {}
ParsedRule(const std::string &_host, const std::string &_mode) : host(_host), mode(_mode) {}
void load(cereal::JSONInputArchive &archive_in);
const std::vector<std::string> & getExceptions() const;

5
components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h
View File

@ -24,6 +24,7 @@
#include "maybe_res.h"
#include "i_orchestration_tools.h"
#include "i_shell_cmd.h"
#include "i_encryptor.h"
#include "i_messaging.h"
#include "i_env_details.h"
#include "i_agent_details.h"
@ -40,6 +41,7 @@ class K8sPolicyUtils
Singleton::Consume<I_Messaging>,
Singleton::Consume<I_ShellCmd>,
Singleton::Consume<I_EnvDetails>,
Singleton::Consume<I_Encryptor>,
Singleton::Consume<I_AgentDetails>
{
public:
@ -80,6 +82,8 @@ private:
void createSnortFile(std::vector<NewAppSecPracticeSpec> &practices) const;
void createSchemaValidationOas(std::vector<NewAppSecPracticeSpec> &practices) const;
template<class T>
std::vector<T> extractV1Beta2ElementsFromCluster(
const std::string &crd_plural,
@ -112,6 +116,7 @@ private:
I_Messaging* messaging = nullptr;
EnvType env_type;
std::string token;
std::string agent_ns;
};
#endif // __K8S_POLICY_UTILS_H__

17
components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h
View File

@ -49,6 +49,13 @@ static const std::unordered_map<std::string, TriggerType> string_to_trigger_type
{ "WebUserResponse", TriggerType::WebUserResponse }
};
static const std::unordered_map<std::string, std::string> key_to_mitigation_severity = {
{ "high", "High"},
{ "medium", "Medium"},
{ "critical", "Critical"},
{ "Transparent", "Transparent"}
};
static const std::unordered_map<std::string, std::string> key_to_practices_val = {
{ "prevent-learn", "Prevent"},
{ "detect-learn", "Learn"},
@ -57,6 +64,14 @@ static const std::unordered_map<std::string, std::string> key_to_practices_val =
{ "inactive", "Inactive"}
};
static const std::unordered_map<std::string, std::string> key_to_practices_mode_val = {
{ "prevent-learn", "Prevent"},
{ "detect-learn", "Detect"},
{ "prevent", "Prevent"},
{ "detect", "Detect"},
{ "inactive", "Disabled"}
};
static const std::unordered_map<std::string, std::string> key_to_practices_val2 = {
{ "prevent-learn", "Prevent"},
{ "detect-learn", "Learn"},
@ -66,6 +81,8 @@ static const std::unordered_map<std::string, std::string> key_to_practices_val2
};
static const std::string default_appsec_url = "http://*:*";
static const std::string default_appsec_name = "Any";
class PolicyGenException : public std::exception
{

2
components/security_apps/local_policy_mgmt_gen/include/new_appsec_policy_crd_parser.h
View File

@ -31,7 +31,7 @@ class NewParsedRule
{
public:
NewParsedRule() {}
NewParsedRule(const std::string &_host) : host(_host) {}
NewParsedRule(const std::string &_host, const std::string &_mode) : host(_host), mode(_mode) {}
void load(cereal::JSONInputArchive &archive_in);

50
components/security_apps/local_policy_mgmt_gen/include/new_practice.h
View File

@ -23,6 +23,8 @@
#include "config.h"
#include "debug.h"
#include "local_policy_common.h"
#include "i_orchestration_tools.h"
#include "i_encryptor.h"
bool isModeInherited(const std::string &mode);
@ -88,6 +90,8 @@ public:
void save(cereal::JSONOutputArchive &out_ar) const;
bool operator<(const IpsProtectionsSection &other) const;
private:
std::string context;
std::string name;
@ -105,7 +109,7 @@ public:
// LCOV_EXCL_START Reason: no test exist
IPSSection() {};
IPSSection(const std::vector<IpsProtectionsSection> &_ips) : ips(_ips) {};
IPSSection(const std::vector<IpsProtectionsSection> &_ips);
// LCOV_EXCL_STOP
void save(cereal::JSONOutputArchive &out_ar) const;
@ -138,6 +142,12 @@ public:
const std::string & getMode(const std::string &default_mode = "inactive") const;
private:
const std::string & getRulesMode(
const std::string &mode,
const std::string &default_mode = "inactive"
) const;
std::string override_mode;
std::string max_performance_impact;
std::string min_severity_level;
@ -487,15 +497,16 @@ private:
SnortSection snort;
};
class NewSnortSignaturesAndOpenSchemaAPI
class NewSnortSignatures
{
public:
NewSnortSignaturesAndOpenSchemaAPI() : is_temporary(false) {};
NewSnortSignatures() : is_temporary(false) {};
void load(cereal::JSONInputArchive &archive_in);
void addFile(const std::string &file_name);
const std::string & getOverrideMode(const std::string &default_mode = "inactive") const;
const std::string & getEnforceLevel() const;
const std::vector<std::string> & getConfigMap() const;
const std::vector<std::string> & getFiles() const;
bool isTemporary() const;
@ -503,17 +514,40 @@ public:
private:
std::string override_mode;
std::string enforcement_level;
std::vector<std::string> config_map;
std::vector<std::string> files;
bool is_temporary;
};
class NewOpenApiSchema : Singleton::Consume<I_OrchestrationTools>, Singleton::Consume<I_Encryptor>
{
public:
NewOpenApiSchema() {};
void load(cereal::JSONInputArchive &archive_in);
void addOas(const std::string &file);
const std::string & getOverrideMode(const std::string &default_mode = "inactive") const;
const std::string & getEnforceLevel() const;
const std::vector<std::string> & getConfigMap() const;
const std::vector<std::string> & getFiles() const;
const std::vector<std::string> & getOas() const;
private:
std::string override_mode;
std::string enforcement_level;
std::vector<std::string> config_map;
std::vector<std::string> files;
std::vector<std::string> oas;
};
class NewAppSecPracticeAntiBot
{
public:
const std::vector<std::string> & getIjectedUris() const;
const std::vector<std::string> & getValidatedUris() const;
const std::string & getMode() const;
const std::string & getMode(const std::string &default_mode = "inactive") const;
void load(cereal::JSONInputArchive &archive_in);
void save(cereal::JSONOutputArchive &out_ar) const;
@ -569,8 +603,8 @@ class NewAppSecPracticeSpec
public:
void load(cereal::JSONInputArchive &archive_in);
NewSnortSignaturesAndOpenSchemaAPI & getSnortSignatures();
const NewSnortSignaturesAndOpenSchemaAPI & getOpenSchemaValidation() const;
NewSnortSignatures & getSnortSignatures();
NewOpenApiSchema & getOpenSchemaValidation();
const NewAppSecPracticeWebAttacks & getWebAttacks() const;
const NewAppSecPracticeAntiBot & getAntiBot() const;
const NewIntrusionPrevention & getIntrusionPrevention() const;
@ -583,8 +617,8 @@ public:
private:
NewFileSecurity file_security;
NewIntrusionPrevention intrusion_prevention;
NewSnortSignaturesAndOpenSchemaAPI openapi_schema_validation;
NewSnortSignaturesAndOpenSchemaAPI snort_signatures;
NewOpenApiSchema openapi_schema_validation;
NewSnortSignatures snort_signatures;
NewAppSecPracticeWebAttacks web_attacks;
NewAppSecPracticeAntiBot anti_bot;
std::string appsec_class_name;

2
components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h
View File

@ -123,6 +123,7 @@ public:
);
const std::string & getIdentifier() const;
const std::string & getIdentifierValue() const;
void save(cereal::JSONOutputArchive &out_ar) const;
@ -145,6 +146,7 @@ public:
);
const std::string & getIdentifier() const;
const std::string & getIdentifierValue() const;
void save(cereal::JSONOutputArchive &out_ar) const;

87
components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc
View File

@ -35,6 +35,14 @@ convertAnnotationKeysTostring(const AnnotationKeys &key)
}
}
string
getAppSecScopeType()
{
auto env_res = getenv("CRDS_SCOPE");
if (env_res != nullptr) return env_res;
return "cluster";
}
void
K8sPolicyUtils::init()
{
@ -42,6 +50,7 @@ K8sPolicyUtils::init()
env_type = env_details->getEnvType();
if (env_type == EnvType::K8S) {
token = env_details->getToken();
agent_ns = getAppSecScopeType() == "namespaced" ? env_details->getNameSpace() + "/" : "";
messaging = Singleton::Consume<I_Messaging>::by<K8sPolicyUtils>();
}
}
@ -140,10 +149,12 @@ extractElementsFromNewRule(
const NewParsedRule &rule,
map<AnnotationTypes, unordered_set<string>> &policy_elements_names)
{
policy_elements_names[AnnotationTypes::EXCEPTION].insert(
rule.getExceptions().begin(),
rule.getExceptions().end()
);
if (rule.getExceptions().size() > 0) {
policy_elements_names[AnnotationTypes::EXCEPTION].insert(
rule.getExceptions().begin(),
rule.getExceptions().end()
);
}
policy_elements_names[AnnotationTypes::THREAT_PREVENTION_PRACTICE].insert(
rule.getPractices().begin(),
rule.getPractices().end()
@ -152,14 +163,24 @@ extractElementsFromNewRule(
rule.getAccessControlPractices().begin(),
rule.getAccessControlPractices().end()
);
policy_elements_names[AnnotationTypes::TRIGGER].insert(
rule.getLogTriggers().begin(),
rule.getLogTriggers().end()
);
policy_elements_names[AnnotationTypes::WEB_USER_RES].insert(rule.getCustomResponse());
policy_elements_names[AnnotationTypes::SOURCE_IDENTIFIERS].insert(rule.getSourceIdentifiers());
policy_elements_names[AnnotationTypes::TRUSTED_SOURCES].insert(rule.getTrustedSources());
policy_elements_names[AnnotationTypes::UPGRADE_SETTINGS].insert(rule.getUpgradeSettings());
if (rule.getLogTriggers().size() > 0) {
policy_elements_names[AnnotationTypes::TRIGGER].insert(
rule.getLogTriggers().begin(),
rule.getLogTriggers().end()
);
}
if (rule.getCustomResponse() != "" ) {
policy_elements_names[AnnotationTypes::WEB_USER_RES].insert(rule.getCustomResponse());
}
if (rule.getSourceIdentifiers() != "" ) {
policy_elements_names[AnnotationTypes::SOURCE_IDENTIFIERS].insert(rule.getSourceIdentifiers());
}
if (rule.getTrustedSources() != "" ) {
policy_elements_names[AnnotationTypes::TRUSTED_SOURCES].insert(rule.getTrustedSources());
}
if (rule.getUpgradeSettings() != "" ) {
policy_elements_names[AnnotationTypes::UPGRADE_SETTINGS].insert(rule.getUpgradeSettings());
}
}
map<AnnotationTypes, unordered_set<string>>
@ -259,9 +280,11 @@ K8sPolicyUtils::extractV1Beta2ElementsFromCluster(
dbgTrace(D_LOCAL_POLICY) << "Retrieve AppSec elements. type: " << crd_plural;
vector<T> elements;
for (const string &element_name : elements_names) {
string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : "";
string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : "";
dbgTrace(D_LOCAL_POLICY) << "AppSec element name: " << element_name;
auto maybe_appsec_element = getObjectFromCluster<AppsecSpecParser<T>>(
"/apis/openappsec.io/v1beta2/" + crd_plural + "/" + element_name
"/apis/openappsec.io/v1beta2/" + ns + agent_ns + crd_plural + ns_suffix + "/" + element_name
);
if (!maybe_appsec_element.ok()) {
@ -362,8 +385,9 @@ K8sPolicyUtils::createSnortFile(vector<NewAppSecPracticeSpec> &practices) const
practice.getSnortSignatures().setTemporary(true);
for (const string &config_map : practice.getSnortSignatures().getConfigMap())
{
string ns = agent_ns == "" ? "default/" : agent_ns;
auto maybe_configmap = getObjectFromCluster<ConfigMaps>(
"/api/v1/namespaces/default/configmaps/" + config_map
"/api/v1/namespaces/" + ns + "configmaps/" + config_map
);
if (!maybe_configmap.ok()) {
dbgWarning(D_LOCAL_POLICY) << "Failed to get configMaps from the cluster.";
@ -381,6 +405,28 @@ K8sPolicyUtils::createSnortFile(vector<NewAppSecPracticeSpec> &practices) const
}
}
void
K8sPolicyUtils::createSchemaValidationOas(vector<NewAppSecPracticeSpec> &practices) const
{
for (NewAppSecPracticeSpec &practice : practices) {
vector<string> res;
for (const string &config_map : practice.getOpenSchemaValidation().getConfigMap())
{
string ns = agent_ns == "" ? "default/" : agent_ns;
auto maybe_configmap = getObjectFromCluster<ConfigMaps>(
"/api/v1/namespaces/" + ns + "configmaps/" + config_map
);
if (!maybe_configmap.ok()) {
dbgWarning(D_LOCAL_POLICY) << "Failed to get configMaps from the cluster.";
continue;
}
string file_content = maybe_configmap.unpack().getFileContent();
string res = Singleton::Consume<I_Encryptor>::by<K8sPolicyUtils>()->base64Encode(file_content);
practice.getOpenSchemaValidation().addOas(res);
}
}
}
Maybe<V1beta2AppsecLinuxPolicy>
K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
const AppsecSpecParser<NewAppsecPolicySpec> &appsec_policy_spec,
@ -396,6 +442,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
}
if (default_rule.getMode().empty() && !ingress_mode.empty()) {
dbgTrace(D_LOCAL_POLICY) << "setting the policy default rule mode to the ingress mode: " << ingress_mode;
default_rule.setMode(ingress_mode);
}
@ -411,6 +458,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
);
createSnortFile(threat_prevention_practices);
createSchemaValidationOas(threat_prevention_practices);
vector<AccessControlPracticeSpec> access_control_practices =
extractV1Beta2ElementsFromCluster<AccessControlPracticeSpec>(
@ -493,9 +541,12 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i
maybe_appsec_policy_spec.ok() ? "There is no v1beta1 policy" : maybe_appsec_policy_spec.getErr();
dbgWarning(D_LOCAL_POLICY
) << "Failed to retrieve Appsec policy with crds version: v1beta1, Trying version: v1beta2";
string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : "";
string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : "";
auto maybe_v1beta2_appsec_policy_spec = getObjectFromCluster<AppsecSpecParser<NewAppsecPolicySpec>>(
"/apis/openappsec.io/v1beta2/policies/" + policy_name
"/apis/openappsec.io/v1beta2/" + ns + agent_ns + "policies" + ns_suffix + "/" + policy_name
);
if (!maybe_v1beta2_appsec_policy_spec.ok()) {
dbgWarning(D_LOCAL_POLICY)
<< "Failed to retrieve AppSec policy. Error: " << maybe_v1beta2_appsec_policy_spec.getErr();
@ -535,10 +586,11 @@ K8sPolicyUtils::createPolicy(
if (policies.find(annotations_values[AnnotationKeys::PolicyKey]) == policies.end()) {
policies[annotations_values[AnnotationKeys::PolicyKey]] = appsec_policy;
}
auto default_mode = appsec_policy.getAppsecPolicySpec().getDefaultRule().getMode();
if (item.getSpec().doesDefaultBackendExist()) {
dbgTrace(D_LOCAL_POLICY)
<< "Inserting Any host rule to the specific asset set";
K ingress_rule = K("*");
K ingress_rule = K("*", default_mode);
policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule);
}
@ -556,12 +608,11 @@ K8sPolicyUtils::createPolicy(
<< "' uri: '"
<< uri.getPath()
<< "'";
K ingress_rule = K(host);
K ingress_rule = K(host, default_mode);
policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule);
}
}
}
}
std::tuple<map<string, AppsecLinuxPolicy>, map<string, V1beta2AppsecLinuxPolicy>>

259
components/security_apps/local_policy_mgmt_gen/new_practice.cc
View File

@ -22,6 +22,7 @@ static const set<string> performance_impacts = {"low", "medium", "high"};
static const set<string> severity_levels = {"low", "medium", "high", "critical"};
static const set<string> size_unit = {"bytes", "KB", "MB", "GB"};
static const set<string> confidences_actions = {"prevent", "detect", "inactive", "as-top-level", "inherited"};
static const set<string> valied_enforcement_level = {"fullSchema", "endpointOnly"};
static const set<string> valid_modes = {
"prevent",
"detect",
@ -32,38 +33,38 @@ static const set<string> valid_modes = {
"inherited"
};
static const set<string> valid_confidences = {"medium", "high", "critical"};
static const std::unordered_map<std::string, std::string> key_to_performance_impact_val = {
static const unordered_map<string, string> key_to_performance_impact_val = {
{ "low", "Low or lower"},
{ "medium", "Medium or lower"},
{ "high", "High or lower"}
};
static const std::unordered_map<std::string, std::string> key_to_severity_level_val = {
static const unordered_map<string, string> key_to_severity_level_val = {
{ "low", "Low or above"},
{ "medium", "Medium or above"},
{ "high", "High or above"},
{ "critical", "Critical"}
};
static const std::unordered_map<std::string, std::string> key_to_mode_val = {
static const unordered_map<string, string> key_to_mode_val = {
{ "prevent-learn", "Prevent"},
{ "detect-learn", "Detect"},
{ "prevent", "Prevent"},
{ "detect", "Detect"},
{ "inactive", "Inactive"}
};
static const std::unordered_map<std::string, std::string> anti_bot_key_to_mode_val = {
static const unordered_map<string, string> anti_bot_key_to_mode_val = {
{ "prevent-learn", "Prevent"},
{ "detect-learn", "Detect"},
{ "prevent", "Prevent"},
{ "detect", "Detect"},
{ "inactive", "Disabled"}
};
static const std::unordered_map<std::string, uint64_t> unit_to_int = {
static const unordered_map<string, uint64_t> unit_to_int = {
{ "bytes", 1},
{ "KB", 1024},
{ "MB", 1048576},
{ "GB", 1073741824}
};
static const std::string TRANSPARENT_MODE = "Transparent";
static const string TRANSPARENT_MODE = "Transparent";
bool
isModeInherited(const string &mode)
@ -71,11 +72,11 @@ isModeInherited(const string &mode)
return mode == "as-top-level" || mode == "inherited";
}
const std::string &
const string &
getModeWithDefault(
const std::string &mode,
const std::string &default_mode,
const std::unordered_map<std::string, std::string> &key_to_val)
const string &mode,
const string &default_mode,
const unordered_map<string, string> &key_to_val)
{
if (isModeInherited(mode) && (key_to_val.find(default_mode) != key_to_val.end())) {
dbgError(D_LOCAL_POLICY) << "Setting to top-level mode: " << default_mode;
@ -88,36 +89,35 @@ getModeWithDefault(
return key_to_val.at(mode);
}
const std::vector<std::string> &
const vector<string> &
NewAppSecPracticeAntiBot::getIjectedUris() const
{
return injected_uris;
}
const std::vector<std::string> &
const vector<string> &
NewAppSecPracticeAntiBot::getValidatedUris() const
{
return validated_uris;
}
const std::string &
NewAppSecPracticeAntiBot::getMode() const
const string &
NewAppSecPracticeAntiBot::getMode(const string &default_mode) const
{
return override_mode;
return getModeWithDefault(override_mode, default_mode, anti_bot_key_to_mode_val);
}
void
NewAppSecPracticeAntiBot::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Web Bots";
string mode;
parseAppsecJSONKey<vector<string>>("injectedUris", injected_uris, archive_in);
parseAppsecJSONKey<vector<string>>("validatedUris", validated_uris, archive_in);
parseMandatoryAppsecJSONKey<string>("overrideMode", mode, archive_in, "inactive");
if (valid_modes.count(mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << mode;
parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
if (valid_modes.count(override_mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << override_mode;
throw PolicyGenException("AppSec Web Bots override mode invalid: " + override_mode);
}
override_mode = anti_bot_key_to_mode_val.at(mode);
}
void
@ -242,14 +242,14 @@ NewAppSecPracticeWebAttacks::getProtections() const
}
SnortProtectionsSection::SnortProtectionsSection(
const std::string &_context,
const std::string &_asset_name,
const std::string &_asset_id,
const std::string &_practice_name,
const std::string &_practice_id,
const std::string &_source_identifier,
const std::string &_mode,
const std::vector<std::string> &_files)
const string &_context,
const string &_asset_name,
const string &_asset_id,
const string &_practice_name,
const string &_practice_id,
const string &_source_identifier,
const string &_mode,
const vector<string> &_files)
:
context(_context),
asset_name(_asset_name),
@ -278,10 +278,10 @@ SnortProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
}
DetectionRules::DetectionRules(
const std::string &_type,
const std::string &_SSM,
const std::string &_keywords,
const std::vector<std::string> &_context)
const string &_type,
const string &_SSM,
const string &_keywords,
const vector<string> &_context)
:
type(_type),
SSM(_SSM),
@ -314,14 +314,14 @@ DetectionRules::save(cereal::JSONOutputArchive &out_ar) const
ProtectionMetadata::ProtectionMetadata(
bool _silent,
const std::string &_protection_name,
const std::string &_severity,
const std::string &_confidence_level,
const std::string &_performance_impact,
const std::string &_last_update,
const std::string &_maintrain_id,
const std::vector<std::string> &_tags,
const std::vector<std::string> &_cve_list)
const string &_protection_name,
const string &_severity,
const string &_confidence_level,
const string &_performance_impact,
const string &_last_update,
const string &_maintrain_id,
const vector<string> &_tags,
const vector<string> &_cve_list)
:
silent(_silent),
protection_name(_protection_name),
@ -394,9 +394,9 @@ ProtectionsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
}
ProtectionsSection::ProtectionsSection(
const std::vector<ProtectionsProtectionsSection> &_protections,
const std::string &_name,
const std::string &_modification_time)
const vector<ProtectionsProtectionsSection> &_protections,
const string &_name,
const string &_modification_time)
:
protections(_protections),
name(_name),
@ -460,12 +460,16 @@ SnortSectionWrapper::save(cereal::JSONOutputArchive &out_ar) const
}
void
NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in)
NewSnortSignatures::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Snort Signatures practice";
parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
parseAppsecJSONKey<vector<string>>("configmap", config_map, archive_in);
parseAppsecJSONKey<vector<string>>("files", files, archive_in);
if (valid_modes.count(override_mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode;
throw PolicyGenException("AppSec Snort Signatures override mode invalid: " + override_mode);
}
is_temporary = false;
if (valid_modes.count(override_mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode;
@ -474,42 +478,107 @@ NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in)
}
void
NewSnortSignaturesAndOpenSchemaAPI::addFile(const string &file_name)
NewSnortSignatures::addFile(const string &file_name)
{
files.push_back(file_name);
}
const string &
NewSnortSignaturesAndOpenSchemaAPI::getOverrideMode(const string &default_mode) const
NewSnortSignatures::getOverrideMode(const string &default_mode) const
{
const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val);
const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_mode_val);
return res;
}
const vector<string> &
NewSnortSignaturesAndOpenSchemaAPI::getFiles() const
NewSnortSignatures::getFiles() const
{
return files;
}
const vector<string> &
NewSnortSignaturesAndOpenSchemaAPI::getConfigMap() const
NewSnortSignatures::getConfigMap() const
{
return config_map;
}
bool
NewSnortSignaturesAndOpenSchemaAPI::isTemporary() const
NewSnortSignatures::isTemporary() const
{
return is_temporary;
}
void
NewSnortSignaturesAndOpenSchemaAPI::setTemporary(bool val)
NewSnortSignatures::setTemporary(bool val)
{
is_temporary = val;
}
void
NewOpenApiSchema::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Schema Validation practice";
parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
parseAppsecJSONKey<vector<string>>("configmap", config_map, archive_in);
parseAppsecJSONKey<vector<string>>("files", files, archive_in);
parseAppsecJSONKey<string>("enforcementLevel", enforcement_level, archive_in, "fullSchema");
if (valied_enforcement_level.count(enforcement_level) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Schema Validation enforcement level invalid: " << enforcement_level;
throw PolicyGenException("AppSec Schema Validation enforcement level invalid: " + enforcement_level);
}
if (valid_modes.count(override_mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Schema Validation override mode invalid: " << override_mode;
throw PolicyGenException("AppSec Schema Validation override mode invalid: " + override_mode);
}
for (const string &file : files)
{
auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<NewOpenApiSchema>();
auto file_content = i_orchestration_tools->readFile(file);
if (!file_content.ok()) {
dbgWarning(D_LOCAL_POLICY) << "Couldn't open the schema validation file";
continue;
}
oas.push_back(Singleton::Consume<I_Encryptor>::by<NewOpenApiSchema>()->base64Encode(file_content.unpack()));
}
}
void
NewOpenApiSchema::addOas(const string &file)
{
oas.push_back(file);
}
const string &
NewOpenApiSchema::getOverrideMode(const string &default_mode) const
{
const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val2);
return res;
}
const string &
NewOpenApiSchema::getEnforceLevel() const
{
return enforcement_level;
}
const vector<string> &
NewOpenApiSchema::getFiles() const
{
return files;
}
const vector<string> &
NewOpenApiSchema::getConfigMap() const
{
return config_map;
}
const vector<string> &
NewOpenApiSchema::getOas() const
{
return oas;
}
void
IpsProtectionsRulesSection::save(cereal::JSONOutputArchive &out_ar) const
{
@ -548,7 +617,7 @@ IpsProtectionsSection::IpsProtectionsSection(
{
}
std::string &
string &
IpsProtectionsSection::getMode()
{
return mode;
@ -570,6 +639,20 @@ IpsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
);
}
bool
IpsProtectionsSection::operator<(const IpsProtectionsSection &other) const
{
// for sorting from the most specific to the least specific rule
if (name == default_appsec_name) return false;
if (other.name == default_appsec_name) return true;
return name.size() > other.name.size();
}
IPSSection::IPSSection(const vector<IpsProtectionsSection> &_ips) : ips(_ips)
{
sort(ips.begin(), ips.end());
}
void
IPSSection::save(cereal::JSONOutputArchive &out_ar) const
{
@ -648,7 +731,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
vector<IpsProtectionsRulesSection> ips_rules;
IpsProtectionsRulesSection high_rule(
min_cve_Year,
getModeWithDefault(high_confidence_event_action, default_mode, key_to_practices_val),
getRulesMode(high_confidence_event_action, default_mode),
string("High"),
max_performance_impact,
string(""),
@ -658,7 +741,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
IpsProtectionsRulesSection med_rule(
min_cve_Year,
getModeWithDefault(medium_confidence_event_action, default_mode, key_to_practices_val),
getRulesMode(medium_confidence_event_action, default_mode),
string("Medium"),
max_performance_impact,
string(""),
@ -668,7 +751,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
IpsProtectionsRulesSection low_rule(
min_cve_Year,
getModeWithDefault(low_confidence_event_action, default_mode, key_to_practices_val),
getRulesMode(low_confidence_event_action, default_mode),
string("Low"),
max_performance_impact,
string(""),
@ -679,33 +762,45 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
return ips_rules;
}
const std::string &
NewIntrusionPrevention::getMode(const std::string &default_mode) const
const string &
NewIntrusionPrevention::getMode(const string &default_mode) const
{
const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val);
const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_mode_val);
return res;
}
const string &
NewIntrusionPrevention::getRulesMode(const string &mode, const string &default_mode) const
{
if (isModeInherited(mode)) return default_mode;
if (key_to_practices_mode_val.find(mode) == key_to_practices_mode_val.end()) {
dbgError(D_LOCAL_POLICY) << "Given mode: " << mode << " or top-level: " << default_mode << " is invalid.";
return key_to_practices_mode_val.at("inactive");
}
return key_to_practices_mode_val.at(mode);
}
FileSecurityProtectionsSection::FileSecurityProtectionsSection(
uint64_t _file_size_limit,
uint64_t _archive_file_size_limit,
bool _allow_files_without_name,
bool _required_file_size_limit,
bool _required_archive_extraction,
const std::string &_context,
const std::string &_name,
const std::string &_asset_id,
const std::string &_practice_name,
const std::string &_practice_id,
const std::string &_action,
const std::string &_files_without_name_action,
const std::string &_high_confidence_action,
const std::string &_medium_confidence_action,
const std::string &_low_confidence_action,
const std::string &_severity_level,
const std::string &_file_size_limit_action,
const std::string &_multi_level_archive_action,
const std::string &_unopened_archive_action)
const string &_context,
const string &_name,
const string &_asset_id,
const string &_practice_name,
const string &_practice_id,
const string &_action,
const string &_files_without_name_action,
const string &_high_confidence_action,
const string &_medium_confidence_action,
const string &_low_confidence_action,
const string &_severity_level,
const string &_file_size_limit_action,
const string &_multi_level_archive_action,
const string &_unopened_archive_action)
:
file_size_limit(_file_size_limit),
archive_file_size_limit(_archive_file_size_limit),
@ -831,13 +926,13 @@ NewFileSecurityArchiveInspection::getrequiredArchiveExtraction() const
return extract_archive_files;
}
const std::string &
const string &
NewFileSecurityArchiveInspection::getMultiLevelArchiveAction() const
{
return archived_files_within_archived_files;
}
const std::string &
const string &
NewFileSecurityArchiveInspection::getUnopenedArchiveAction() const
{
return archived_files_where_content_extraction_failed;
@ -886,7 +981,7 @@ NewFileSecurityLargeFileInspection::getFileSizeLimit() const
return (file_size_limit * unit_to_int.at(file_size_limit_unit));
}
const std::string &
const string &
NewFileSecurityLargeFileInspection::getFileSizeLimitAction() const
{
return files_exceeding_size_limit_action;
@ -1007,7 +1102,7 @@ void
NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec";
parseAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>(
parseAppsecJSONKey<NewOpenApiSchema>(
"schemaValidation",
openapi_schema_validation,
archive_in
@ -1015,11 +1110,15 @@ NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in)
parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
parseMandatoryAppsecJSONKey<NewFileSecurity>("fileSecurity", file_security, archive_in);
parseMandatoryAppsecJSONKey<NewIntrusionPrevention>("intrusionPrevention", intrusion_prevention, archive_in);
parseMandatoryAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>("snortSignatures", snort_signatures, archive_in);
parseMandatoryAppsecJSONKey<NewSnortSignatures>("snortSignatures", snort_signatures, archive_in);
parseMandatoryAppsecJSONKey<NewAppSecPracticeWebAttacks>("webAttacks", web_attacks, archive_in);
parseAppsecJSONKey<NewAppSecPracticeAntiBot>("antiBot", anti_bot, archive_in);
parseAppsecJSONKey<string>("name", practice_name, archive_in);
parseAppsecJSONKey<string>("practiceMode", mode, archive_in, "inherited");
if (valid_modes.count(mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Threat prevention practice mode invalid: " << mode;
throw PolicyGenException("AppSec Threat prevention practice mode invalid: " + mode);
}
}
void
@ -1028,13 +1127,13 @@ NewAppSecPracticeSpec::setName(const string &_name)
practice_name = _name;
}
const NewSnortSignaturesAndOpenSchemaAPI &
NewAppSecPracticeSpec::getOpenSchemaValidation() const
NewOpenApiSchema &
NewAppSecPracticeSpec::getOpenSchemaValidation()
{
return openapi_schema_validation;
}
NewSnortSignaturesAndOpenSchemaAPI &
NewSnortSignatures &
NewAppSecPracticeSpec::getSnortSignatures()
{
return snort_signatures;

47
components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc
View File

@ -23,6 +23,14 @@ using namespace std;
USE_DEBUG_FLAG(D_NGINX_POLICY);
USE_DEBUG_FLAG(D_LOCAL_POLICY);
static const std::unordered_map<std::string, std::string> key_to_source_identefier_val = {
{ "sourceip", "Source IP"},
{ "cookie", "Cookie:"},
{ "headerkey", "Header:"},
{ "JWTKey", ""},
{ "x-forwarded-for", "X-Forwarded-For"}
};
void
SecurityAppsWrapper::save(cereal::JSONOutputArchive &out_ar) const
{
@ -1038,7 +1046,7 @@ PolicyMakerUtils::createIpsSections(
practice_name,
practice_id,
source_identifier,
override_mode,
"Inactive",
apssec_practice.getIntrusionPrevention().createIpsRules(override_mode)
);
@ -1048,8 +1056,7 @@ PolicyMakerUtils::createIpsSections(
void
PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_temporary)
{
auto path = getFilesystemPathConfig() + "/conf/snort/" + file_name;
string in_file = is_temporary ? path + ".rule" : path;
auto path = is_temporary ? getFilesystemPathConfig() + "/conf/snort/" + file_name + ".rule" : file_name;
if (snort_protections.find(path) != snort_protections.end()) {
dbgTrace(D_LOCAL_POLICY) << "Snort protections section for file " << file_name << " already exists";
@ -1060,7 +1067,9 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_
<< (is_temporary ? " temporary" : "") << " file " << path;
auto snort_script_path = getFilesystemPathConfig() + "/scripts/snort_to_ips_local.py";
auto cmd = "python3 " + snort_script_path + " " + in_file + " " + path + ".out " + path + ".err";
auto tmp_out = "/tmp/" + file_name + ".out";
auto tmp_err = "/tmp/" + file_name + ".err";
auto cmd = "python3 " + snort_script_path + " " + path + " " + tmp_out + " " + tmp_err;
auto res = Singleton::Consume<I_ShellCmd>::by<LocalPolicyMgmtGenerator>()->getExecOutput(cmd);
@ -1069,16 +1078,16 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_
return;
}
Maybe<ProtectionsSectionWrapper> maybe_protections = openFileAsJson<ProtectionsSectionWrapper>(path + ".out");
Maybe<ProtectionsSectionWrapper> maybe_protections = openFileAsJson<ProtectionsSectionWrapper>(tmp_out);
if (!maybe_protections.ok()){
dbgWarning(D_LOCAL_POLICY) << maybe_protections.getErr();
return;
}
auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<LocalPolicyMgmtGenerator>();
if (is_temporary) i_orchestration_tools->removeFile(in_file);
i_orchestration_tools->removeFile(path + ".out");
i_orchestration_tools->removeFile(path + ".err");
if (is_temporary) i_orchestration_tools->removeFile(path);
i_orchestration_tools->removeFile(tmp_out);
i_orchestration_tools->removeFile(tmp_err);
snort_protections[path] = ProtectionsSection(
maybe_protections.unpack().getProtections(),
@ -1208,7 +1217,8 @@ void
PolicyMakerUtils::createWebAppSection(
const V1beta2AppsecLinuxPolicy &policy,
const RulesConfigRulebase& rule_config,
const string &practice_id, const string &full_url,
const string &practice_id,
const string &full_url,
const string &default_mode,
map<AnnotationTypes, string> &rule_annotations)
{
@ -1225,6 +1235,7 @@ PolicyMakerUtils::createWebAppSection(
apssec_practice.getWebAttacks().getMaxObjectDepth(),
apssec_practice.getWebAttacks().getMaxUrlSizeBytes()
);
WebAppSection web_app = WebAppSection(
full_url == "Any" ? default_appsec_url : full_url,
rule_config.getAssetId(),
@ -1236,7 +1247,10 @@ PolicyMakerUtils::createWebAppSection(
rule_config.getContext(),
apssec_practice.getWebAttacks().getMinimumConfidence(practice_mode),
apssec_practice.getWebAttacks().getMode(practice_mode),
apssec_practice.getAntiBot().getMode(),
apssec_practice.getAntiBot().getMode(practice_mode),
apssec_practice.getOpenSchemaValidation().getOverrideMode(practice_mode),
apssec_practice.getOpenSchemaValidation().getEnforceLevel(),
apssec_practice.getOpenSchemaValidation().getOas(),
practice_advance_config,
apssec_practice.getAntiBot(),
log_triggers[rule_annotations[AnnotationTypes::TRIGGER]],
@ -1290,7 +1304,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
);
rules_config[rule_config.getAssetName()] = rule_config;
string current_identifier;
string current_identifier, current_identifier_value;
if (!rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS].empty()) {
UsersIdentifiersRulebase user_identifiers = createUserIdentifiers<V1beta2AppsecLinuxPolicy>(
rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS],
@ -1299,6 +1313,15 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
);
users_identifiers[rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS]] = user_identifiers;
current_identifier = user_identifiers.getIdentifier();
current_identifier_value = user_identifiers.getIdentifierValue();
}
string ips_identifier, ips_identifier_value;
if(key_to_source_identefier_val.find(current_identifier) != key_to_source_identefier_val.end()) {
ips_identifier = key_to_source_identefier_val.at(current_identifier);
}
if (current_identifier == "cookie" || current_identifier == "headerkey") {
ips_identifier_value = current_identifier_value;
}
createIpsSections(
@ -1306,7 +1329,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
rule_config.getAssetName(),
practice_id,
rule_annotations[AnnotationTypes::PRACTICE],
current_identifier,
ips_identifier + ips_identifier_value,
rule_config.getContext(),
policy,
rule_annotations,

16
components/security_apps/local_policy_mgmt_gen/rules_config_section.cc
View File

@ -17,6 +17,8 @@ using namespace std;
USE_DEBUG_FLAG(D_LOCAL_POLICY);
static const string empty_string="";
AssetUrlParser
AssetUrlParser::parse(const string &uri)
{
@ -242,6 +244,13 @@ UsersIdentifier::getIdentifier() const
{
return source_identifier;
}
const string &
UsersIdentifier::getIdentifierValue() const
{
if (identifier_values.empty()) return empty_string;
return identifier_values[0];
}
// LCOV_EXCL_STOP
void
@ -272,6 +281,13 @@ UsersIdentifiersRulebase::getIdentifier() const
if (source_identifiers.empty()) return source_identifier;
return source_identifiers[0].getIdentifier();
}
const string &
UsersIdentifiersRulebase::getIdentifierValue() const
{
if (source_identifiers.empty()) return empty_string;
return source_identifiers[0].getIdentifierValue();
}
// LCOV_EXCL_STOP
void

7
components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
View File

@ -66,7 +66,12 @@ SHELL_CMD_HANDLER("QUID", "FS_PATH=<FILESYSTEM-PREFIX>;"
"/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');",
getQUID)
SHELL_CMD_HANDLER("SMO_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message || echo ''",
"&& python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message[0].SMO_QUID || echo ''",
getQUID)
SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''",
getQUID)
SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
SHELL_CMD_HANDLER(

13
components/security_apps/orchestration/env_details/env_details.cc
View File

@ -28,6 +28,7 @@ EnvDetails::EnvDetails() : env_type(EnvType::LINUX)
auto tools = Singleton::Consume<I_OrchestrationTools>::from<OrchestrationTools>();
if (tools->doesFileExist("/.dockerenv")) env_type = EnvType::DOCKER;
token = retrieveToken();
agent_namespace = retrieveNamespace();
if (!token.empty()) {
auto env_res = getenv("deployment_type");
env_type = env_res != nullptr && env_res == string("non_crd_k8s") ? EnvType::NON_CRD_K8S : EnvType::K8S;
@ -46,12 +47,24 @@ EnvDetails::getToken()
return token;
}
string
EnvDetails::getNameSpace()
{
return agent_namespace;
}
string
EnvDetails::retrieveToken()
{
return readFileContent(k8s_service_account + "/token");
}
string
EnvDetails::retrieveNamespace()
{
return readFileContent(k8s_service_account + "/namespace");
}
string
EnvDetails::readFileContent(const string &file_path)
{

9
components/security_apps/orchestration/orchestration_comp.cc
View File

@ -1485,11 +1485,10 @@ private:
}
void
setUpgradeTime()
setDelayedUpgradeTime()
{
if (getConfigurationFlag("service_startup") != "true") return;
if (i_service_controller->getServiceToPortMap().empty()) return;
if (!i_agent_details->isOpenAppsecAgent() && i_service_controller->getServiceToPortMap().empty()) return;
try {
string upgrade_delay_interval_str = getAttribute("no-setting", "UPGRADE_DELAY_INTERVAL_MIN");
int upgrade_delay_interval = upgrade_delay_interval_str != "" ? stoi(upgrade_delay_interval_str) : 30;
@ -1506,6 +1505,7 @@ private:
void
run()
{
loadExistingPolicy();
sleep_interval = policy.getErrorSleepInterval();
Maybe<void> registration_status(genError("Not running yet."));
while (!(registration_status = registerToTheFog()).ok()) {
@ -1530,7 +1530,6 @@ private:
<< " seconds";
Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->yield(seconds(sleep_interval));
}
loadExistingPolicy();
failure_count = 0;
Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->yield(chrono::seconds(1));
@ -1587,7 +1586,7 @@ private:
).notify();
}
setUpgradeTime();
setDelayedUpgradeTime();
while (true) {
Singleton::Consume<I_Environment>::by<OrchestrationComp>()->startNewTrace(false);
if (shouldReportAgentDetailsMetadata()) {

2
components/security_apps/waap/include/i_transaction.h
View File

@ -70,6 +70,7 @@ public:
virtual const std::string getParam() const = 0;
virtual const std::vector<std::string> getKeywordMatches() const = 0;
virtual const std::vector<std::string> getKeywordsCombinations() const = 0;
virtual const std::vector<std::string> getKeywordsAfterFilter() const = 0;
virtual const std::string getContentTypeStr() const = 0;
virtual Waap::Util::ContentType getContentType() const = 0;
virtual const std::string getKeywordMatchesStr() const = 0;
@ -84,6 +85,7 @@ public:
virtual const std::string getUriStr() const = 0;
virtual const std::string& getSourceIdentifier() const = 0;
virtual double getScore() const = 0;
virtual double getOtherModelScore() const = 0;
virtual const std::vector<double> getScoreArray() const = 0;
virtual Waap::CSRF::State& getCsrfState() = 0;
virtual ngx_http_cp_verdict_e getUserLimitVerdict() = 0;

37
components/security_apps/waap/include/i_waap_model_result_logger.h Normal file
View File

@ -0,0 +1,37 @@
// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
class IWaf2Transaction;
struct Waf2ScanResult;
namespace Waap {
namespace Scores {
struct ModelLoggingSettings;
}
}
class I_WaapModelResultLogger {
public:
virtual ~I_WaapModelResultLogger() {}
virtual void
logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
std::string modelName,
std::string otherModelName,
double newScore,
double baseScore) = 0;
};

2
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@ -88,6 +88,8 @@ add_library(waap_clib
Waf2Util2.cc
ParserPDF.cc
ParserBinaryFile.cc
RegexComparator.cc
WaapModelResultLogger.cc
)
add_definitions("-Wno-unused-function")

96
components/security_apps/waap/waap_clib/CidrMatch.cc
View File

@ -25,6 +25,51 @@
#include "log_generator.h"
#include <stdexcept>
static in_addr applyMaskV4(const in_addr& addr, uint8_t prefixLength) {
in_addr maskedAddr;
if (prefixLength == 0) {
maskedAddr.s_addr = 0;
} else {
uint32_t mask = htonl(~((1 << (32 - prefixLength)) - 1)); // Create mask
maskedAddr.s_addr = addr.s_addr & mask; // Apply mask
}
return maskedAddr;
}
// Function to apply a network mask to an IPv6 address
static in6_addr applyMaskV6(const in6_addr& addr, uint8_t prefixLength) {
in6_addr maskedAddr = addr;
int fullBytes = prefixLength / 8;
int remainingBits = prefixLength % 8;
// Mask full bytes
for (int i = fullBytes; i < 16; ++i) {
maskedAddr.s6_addr[i] = 0;
}
// Mask remaining bits
if (remainingBits > 0) {
uint8_t mask = ~((1 << (8 - remainingBits)) - 1);
maskedAddr.s6_addr[fullBytes] &= mask;
}
return maskedAddr;
}
// Helper function to convert an IPv4 address to string
static std::string ipv4ToString(const in_addr& ipv4) {
char str[INET_ADDRSTRLEN];
inet_ntop(AF_INET, &ipv4, str, INET_ADDRSTRLEN);
return std::string(str);
}
// Helper function to convert an IPv6 address to string
static std::string ipv6ToString(const in6_addr& ipv6) {
char str[INET6_ADDRSTRLEN];
inet_ntop(AF_INET6, &ipv6, str, INET6_ADDRSTRLEN);
return std::string(str);
}
USE_DEBUG_FLAG(D_WAAP);
namespace Waap {
namespace Util {
@ -38,6 +83,15 @@ bool CIDRData::operator==(const CIDRData &other) const {
isIPV6 == other.isIPV6;
}
bool CIDRData::operator<(const CIDRData &other) const {
if (isIPV6) {
if (!other.isIPV6) return false;
return memcmp(ipCIDRV6.s6_addr, other.ipCIDRV6.s6_addr, sizeof(ipCIDRV6.s6_addr)) < 0;
}
if (other.isIPV6) return true;
return ntohl(ipCIDRV4.s_addr) < ntohl(other.ipCIDRV4.s_addr);
}
bool cidr4_match(const in_addr &addr, const in_addr &net, uint8_t bits) {
if (bits == 0) {
// C99 6.5.7 (3): u32 << 32 is undefined behaviour
@ -114,9 +168,11 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr)
memset(&cidr.ipCIDRV6, 0, sizeof(struct in6_addr));
if (inet_pton(AF_INET, strPrefix.c_str(), &cidr.ipCIDRV4) == 1 && bits <= 32) {
cidr.ipCIDRV4 = applyMaskV4(cidr.ipCIDRV4, bits);
cidr.isIPV6 = false;
}
else if (inet_pton(AF_INET6, strPrefix.c_str(), &cidr.ipCIDRV6) == 1 && bits <= 128) {
cidr.ipCIDRV6 = applyMaskV6(cidr.ipCIDRV6, bits);
cidr.isIPV6 = true;
}
else
@ -128,6 +184,7 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr)
return true;
}
bool cidrMatch(const std::string& sourceip, const std::string& targetCidr) {
CIDRData cidrData;
@ -139,6 +196,7 @@ bool cidrMatch(const std::string& sourceip, const std::string& targetCidr) {
return cidrMatch(sourceip, cidrData);
}
bool cidrMatch(const std::string & sourceip, const CIDRData & cidr){
struct in_addr source_inaddr;
struct in6_addr source_inaddr6;
@ -155,5 +213,43 @@ bool cidrMatch(const std::string & sourceip, const CIDRData & cidr){
dbgDebug(D_WAAP) << "Source IP address does not match any of the CIDR definitions.";
return false;
}
bool doesFirstCidrContainSecond(const CIDRData &first, const CIDRData &second) {
if (first.isIPV6 != second.isIPV6) return false; // IPv4 and IPv6 cannot overlap
if (first.networkBits >= second.networkBits) return false;
if (!first.isIPV6) {
// IPv4 containment check
in_addr smallerNetwork = applyMaskV4(second.ipCIDRV4, first.networkBits);
return (first.ipCIDRV4.s_addr == smallerNetwork.s_addr);
}
// IPv6 containment check
in6_addr smallerNetwork = applyMaskV6(second.ipCIDRV6, first.networkBits);
for (int i = 0; i < 16; ++i) {
if (first.ipCIDRV6.s6_addr[i] != smallerNetwork.s6_addr[i]) {
return false;
}
}
return true;
}
std::string cidrsToString(const std::vector<CIDRData>& cidrs) {
std::stringstream ss;
bool is_first = true;
ss << "[";
for (const auto& cidr : cidrs) {
if (!is_first) ss << ", ";
if (cidr.isIPV6) {
ss << ipv6ToString(cidr.ipCIDRV6) << "/" << static_cast<int>(cidr.networkBits);
} else {
ss << ipv4ToString(cidr.ipCIDRV4) << "/" << static_cast<int>(cidr.networkBits);
}
is_first = false;
}
ss << "]";
return ss.str();
}
}
}

4
components/security_apps/waap/waap_clib/CidrMatch.h
View File

@ -18,6 +18,7 @@
#include <sys/socket.h>
#include <arpa/inet.h>
#include <errno.h>
#include <vector>
namespace Waap {
namespace Util {
@ -29,11 +30,14 @@ struct CIDRData {
uint8_t networkBits;
bool isIPV6;
bool operator==(const CIDRData &other) const;
bool operator<(const CIDRData &other) const;
};
bool isCIDR(const std::string& strCIDR, CIDRData& cidr);
bool cidrMatch(const std::string& sourceip, const CIDRData& cidr);
bool cidrMatch(const std::string &sourceip, const std::string &target);
bool doesFirstCidrContainSecond(const CIDRData &first, const CIDRData &second);
std::string cidrsToString(const std::vector<CIDRData>& cidrs);
}
}

7
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@ -1308,7 +1308,7 @@ DeepParser::createInternalParser(
*this,
parser_depth + 1,
'&',
valueStats.isUrlEncoded)
valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val))
);
} else if (!Waap::Util::testUrlBareUtf8Evasion(cur_val)) {
dbgTrace(D_WAAP_DEEP_PARSER) << "!Waap::Util::testUrlBareUtf8Evasion(cur_val)";
@ -1323,7 +1323,7 @@ DeepParser::createInternalParser(
*this,
parser_depth + 1,
'&',
valueStats.isUrlEncoded)
valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val))
);
offset = 0;
return offset;
@ -1545,5 +1545,6 @@ DeepParser::isPDFDetected(const std::string &cur_val) const
static const std::string PDF_header("%PDF-");
if (cur_val.size() < 10)
return false;
return cur_val.substr(0, cur_val.size() > 64 ? 64 : cur_val.size()).find(PDF_header) != std::string::npos;
return cur_val.substr(0, cur_val.size() > MAX_PDF_HEADER_LOOKUP ? MAX_PDF_HEADER_LOOKUP : cur_val.size())
.find(PDF_header) != std::string::npos;
}

3
components/security_apps/waap/waap_clib/ParserBinaryFile.cc
View File

@ -150,7 +150,8 @@ ParserBinaryFile::push(const char *buf, size_t len)
}
} else {
dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "parsing binary. Searching for tail: " << tail;
c = strstr(buf + len - tail.size(), tail.c_str());
size_t tail_lookup_offset = (len > MAX_TAIL_LOOKUP) ? len - MAX_TAIL_LOOKUP : 0;
c = strstr(buf + tail_lookup_offset, tail.c_str());
dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "search result: c=" << c;
if (c) {
m_state = s_end;

2
components/security_apps/waap/waap_clib/ParserBinaryFile.h
View File

@ -20,7 +20,7 @@
#define MIN_HEADER_LOOKUP 16
#define MAX_HEADER_LOOKUP 64
#define MAX_TAIL_LOOKUP 5
#define MAX_TAIL_LOOKUP 20
class ParserBinaryFile : public ParserBase {
public:

26
components/security_apps/waap/waap_clib/ParserPDF.cc
View File

@ -38,9 +38,10 @@ size_t
ParserPDF::push(const char *buf, size_t len)
{
dbgTrace(D_WAAP_PARSER_PDF)
<< "buf="
<< buf
<< "len="
<< "buf='"
<< std::string(buf, std::min((size_t)200, len))
<< (len > 200 ? "..." : "")
<< "' len="
<< len;
const char *c;
@ -65,13 +66,18 @@ ParserPDF::push(const char *buf, size_t len)
m_state = s_body;
CP_FALL_THROUGH;
case s_body:
c = strstr(buf + len - MAX_TAIL_LOOKUP, PDF_TAIL);
dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): c=" << c;
if (c) {
m_state = s_end;
CP_FALL_THROUGH;
} else {
break;
{
size_t tail_lookup_offset = (len > MAX_PDF_TAIL_LOOKUP) ? len - MAX_PDF_TAIL_LOOKUP : 0;
c = strstr(buf + tail_lookup_offset, PDF_TAIL);
dbgTrace(D_WAAP_PARSER_PDF)
<< "string to search: " << std::string(buf + tail_lookup_offset)
<< " c=" << c;
if (c) {
m_state = s_end;
CP_FALL_THROUGH;
} else {
break;
}
}
case s_end:
if (m_receiver.onKey("PDF", 3) != 0) {

4
components/security_apps/waap/waap_clib/ParserPDF.h
View File

@ -17,8 +17,8 @@
#include "ParserBase.h"
#include <string.h>
#define MAX_HEADER_LOOKUP 64
#define MAX_TAIL_LOOKUP 5
#define MAX_PDF_HEADER_LOOKUP 64
#define MAX_PDF_TAIL_LOOKUP 20
class ParserPDF : public ParserBase {
public:

36
components/security_apps/waap/waap_clib/RegexComparator.cc Normal file
View File

@ -0,0 +1,36 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "RegexComparator.h"
#include <sstream>
namespace Waap {
namespace Util {
std::string regexSetToString(const std::set<std::shared_ptr<boost::regex>, RegexComparator> &regexSet) {
std::stringstream ss;
ss << "[";
bool first = true;
for (const auto &regexPtr : regexSet) {
if (!first) ss << ", ";
if (regexPtr) {
first = false;
ss << regexPtr->str();
}
}
ss << "]";
return ss.str();
}
}
}

32
components/security_apps/waap/waap_clib/RegexComparator.h Normal file
View File

@ -0,0 +1,32 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <set>
#include <boost/regex.hpp>
namespace Waap {
namespace Util {
// Custom comparator for std::shared_ptr<boost::regex>
struct RegexComparator {
bool operator()(const std::shared_ptr<boost::regex>& lhs, const std::shared_ptr<boost::regex>& rhs) const {
// Compare the actual regex patterns by string representation
return lhs->str() < rhs->str();
}
};
std::string regexSetToString(const std::set<std::shared_ptr<boost::regex>, RegexComparator> &regexSet);
}
}

4
components/security_apps/waap/waap_clib/ScanResult.cc
View File

@ -23,9 +23,11 @@ unescaped_line(),
param_name(),
location(),
score(0.0f),
other_model_score(0.0f),
scoreNoFilter(0.0f),
scoreArray(),
keywordCombinations(),
keywordsAfterFilter(),
attack_types(),
m_isAttackInParam(false)
{
@ -41,9 +43,11 @@ void Waf2ScanResult::clear()
param_name.clear();
location.clear();
score = 0;
other_model_score = 0;
scoreNoFilter = 0;
scoreArray.clear();
keywordCombinations.clear();
keywordsAfterFilter.clear();
attack_types.clear();
}

3
components/security_apps/waap/waap_clib/ScanResult.h
View File

@ -29,9 +29,12 @@ struct Waf2ScanResult {
std::string param_name;
std::string location;
double score;
double other_model_score;
double scoreNoFilter;
std::vector<double> scoreArray;
std::vector<double> coefArray;
std::vector<std::string> keywordCombinations;
std::vector<std::string> keywordsAfterFilter;
std::set<std::string> attack_types;
bool m_isAttackInParam;
void clear(); // clear Waf2ScanResult

75
components/security_apps/waap/waap_clib/ScoreBuilder.cc
View File

@ -95,6 +95,10 @@ void KeywordsScorePool::mergeScores(const KeywordsScorePool& baseScores)
m_keywordsDataMap[it->first] = it->second;
}
}
m_stats.linModelIntercept = baseScores.m_stats.linModelIntercept;
m_stats.linModelNNZCoef = baseScores.m_stats.linModelNNZCoef;
m_stats.isLinModel = baseScores.m_stats.isLinModel;
}
@ -118,7 +122,6 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& base
m_pWaapAssetState(pWaapAssetState)
{
restore();
// merge
mergeScores(baseScores);
}
@ -352,16 +355,42 @@ void ScoreBuilder::snap()
const std::string &poolName = pool.first;
const KeywordsScorePool& keywordScorePool = pool.second;
m_snapshotKwScoreMap[poolName];
m_snapshotKwCoefMap[poolName];
if (keywordScorePool.m_keywordsDataMap.empty()) {
m_snapshotStatsMap[poolName] = KeywordsStats();
} else {
m_snapshotStatsMap[poolName] = keywordScorePool.m_stats;
}
for (const auto &kwData : keywordScorePool.m_keywordsDataMap)
{
const std::string &kwName = kwData.first;
double kwScore = kwData.second.score;
m_snapshotKwScoreMap[poolName][kwName] = kwScore;
if (keywordScorePool.m_stats.isLinModel) {
m_snapshotKwScoreMap[poolName][kwName] = kwData.second.new_score;
} else {
m_snapshotKwScoreMap[poolName][kwName] = kwData.second.score;
}
m_snapshotKwCoefMap[poolName][kwName] = kwData.second.coef;
}
}
}
KeywordsStats
ScoreBuilder::getSnapshotStats(const std::string &poolName) const
{
std::map<std::string, KeywordsStats>::const_iterator poolIt = m_snapshotStatsMap.find(poolName);
if (poolIt == m_snapshotStatsMap.end()) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting stats from base pool";
poolIt = m_snapshotStatsMap.find(KEYWORDS_SCORE_POOL_BASE);
if (poolIt == m_snapshotStatsMap.end()) {
dbgWarning(D_WAAP_SCORE_BUILDER) <<
"base pool does not exist! This is probably a bug. Returning empty stats";
return KeywordsStats();
}
}
return poolIt->second;
}
double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double defaultScore,
const std::string &poolName) const
{
@ -369,12 +398,11 @@ double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double
if (poolIt == m_snapshotKwScoreMap.end()) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting score from base pool";
poolIt = m_snapshotKwScoreMap.find(KEYWORDS_SCORE_POOL_BASE);
}
if (poolIt == m_snapshotKwScoreMap.end()) {
dbgDebug(D_WAAP_SCORE_BUILDER) <<
"base pool does not exist! This is probably a bug. Returning default score " << defaultScore;
return defaultScore;
if (poolIt == m_snapshotKwScoreMap.end()) {
dbgDebug(D_WAAP_SCORE_BUILDER) <<
"base pool does not exist! This is probably a bug. Returning default score " << defaultScore;
return defaultScore;
}
}
const KeywordScoreMap &kwScoreMap = poolIt->second;
@ -391,6 +419,35 @@ double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double
return kwScoreFound->second;
}
double
ScoreBuilder::getSnapshotKeywordCoef(const std::string &keyword, double defaultCoef, const std::string &poolName) const
{
std::map<std::string, KeywordScoreMap>::const_iterator poolIt = m_snapshotKwCoefMap.find(poolName);
if (poolIt == m_snapshotKwCoefMap.end()) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting coef from base pool";
poolIt = m_snapshotKwCoefMap.find(KEYWORDS_SCORE_POOL_BASE);
}
if (poolIt == m_snapshotKwCoefMap.end()) {
dbgWarning(D_WAAP_SCORE_BUILDER) <<
"base pool does not exist! This is probably a bug. Returning default coef " << defaultCoef;
return defaultCoef;
}
const KeywordScoreMap &kwCoefMap = poolIt->second;
KeywordScoreMap::const_iterator kwCoefFound = kwCoefMap.find(keyword);
if (kwCoefFound == kwCoefMap.end()) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "keywordCoef:'" << keyword << "': " << defaultCoef <<
" (default, keyword not found in pool '" << poolName << "')";
return defaultCoef;
}
dbgTrace(D_WAAP_SCORE_BUILDER) << "keywordCoef:'" << keyword << "': " << kwCoefFound->second << " (pool '" <<
poolName << "')";
return kwCoefFound->second;
}
keywords_set ScoreBuilder::getIpItemKeywordsSet(std::string ip)
{
return m_fpStore.ipItems[ip];

40
components/security_apps/waap/waap_clib/ScoreBuilder.h
View File

@ -26,6 +26,8 @@
#include <cereal/types/string.hpp>
#include "WaapDefines.h"
USE_DEBUG_FLAG(D_WAAP_SCORE_BUILDER);
struct ScoreBuilderData {
std::string m_sourceIdentifier;
std::string m_userAgent;
@ -52,11 +54,15 @@ enum KeywordType {
};
struct KeywordData {
KeywordData() : truePositiveCtr(0), falsePositiveCtr(0), score(0.0), type(KEYWORD_TYPE_UNKNOWN) {}
KeywordData() :
truePositiveCtr(0), falsePositiveCtr(0), score(0.0), coef(0.0), new_score(0.0), type(KEYWORD_TYPE_UNKNOWN)
{}
unsigned int truePositiveCtr;
unsigned int falsePositiveCtr;
double score;
double coef;
double new_score;
KeywordType type;
template <class Archive>
@ -65,20 +71,42 @@ struct KeywordData {
cereal::make_nvp("true_positives", truePositiveCtr),
cereal::make_nvp("score", score),
cereal::make_nvp("type", type));
try {
ar(cereal::make_nvp("coef", coef),
cereal::make_nvp("new_score", new_score));
} catch (const cereal::Exception &e) {
ar.setNextName(nullptr);
coef = 0;
new_score = 0;
}
}
};
struct KeywordsStats {
KeywordsStats() : truePositiveCtr(0), falsePositiveCtr(0) {}
KeywordsStats() : truePositiveCtr(0), falsePositiveCtr(0), linModelIntercept(0), linModelNNZCoef(0), isLinModel(0)
{}
template <class Archive>
void serialize(Archive& ar) {
ar(cereal::make_nvp("false_positives", falsePositiveCtr),
cereal::make_nvp("true_positives", truePositiveCtr));
cereal::make_nvp("true_positives", truePositiveCtr));
try {
ar(cereal::make_nvp("intercept", linModelIntercept),
cereal::make_nvp("log_nnz_coef", linModelNNZCoef));
isLinModel = true;
} catch (const cereal::Exception &e) {
ar.setNextName(nullptr);
linModelIntercept = 0;
linModelNNZCoef = 0;
isLinModel = false;
}
}
unsigned int truePositiveCtr;
unsigned int falsePositiveCtr;
double linModelIntercept;
double linModelNNZCoef;
bool isLinModel;
};
typedef std::unordered_set<std::string> keywords_set;
@ -106,6 +134,7 @@ struct KeywordsScorePool {
KeywordsScorePool();
// LCOV_EXCL_START Reason: no test exist
template <typename _A>
KeywordsScorePool(_A &iarchive)
{
@ -120,6 +149,7 @@ struct KeywordsScorePool {
m_keywordsDataMap[key] = item.second;
}
}
// LCOV_EXCL_STOP
template <class Archive>
void serialize(Archive& ar) {
@ -148,6 +178,8 @@ public:
void snap();
double getSnapshotKeywordScore(const std::string &keyword, double defaultScore, const std::string &poolName) const;
double getSnapshotKeywordCoef(const std::string &keyword, double defaultCoef, const std::string &poolName) const;
KeywordsStats getSnapshotStats(const std::string &poolName) const;
keywords_set getIpItemKeywordsSet(std::string ip);
keywords_set getUaItemKeywordsSet(std::string userAgent);
@ -200,6 +232,8 @@ protected:
SerializedData m_serializedData;
std::map<std::string, KeywordsScorePool> &m_keywordsScorePools; // live data continuously updated during traffic
std::map<std::string, KeywordScoreMap> m_snapshotKwScoreMap; // the snapshot is updated only by a call to snap()
std::map<std::string, KeywordScoreMap> m_snapshotKwCoefMap; // the snapshot is updated only by a call to snap()
std::map<std::string, KeywordsStats> m_snapshotStatsMap; // the snapshot is updated only by a call to snap()
std::list<std::string> m_falsePositivesSetsIntersection;
I_WaapAssetState* m_pWaapAssetState;
};

3
components/security_apps/waap/waap_clib/Serializator.cc
View File

@ -195,6 +195,9 @@ void SerializeToFileBase::saveData()
} else {
ss.str(string((const char *)res.output, res.num_output_bytes));
}
if (res.output) free(res.output);
res.output = nullptr;
res.num_output_bytes = 0;
filestream << ss.str();

250
components/security_apps/waap/waap_clib/WaapModelResultLogger.cc Normal file
View File

@ -0,0 +1,250 @@
// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "WaapModelResultLogger.h"
#include "Waf2Engine.h"
#include "i_time_get.h"
#include "i_messaging.h"
#include "i_instance_awareness.h"
#include "http_manager.h"
#include "LogGenWrapper.h"
#include "rest.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_MODEL_LOGGER);
using namespace std;
static const unsigned int MAX_FILES_PER_WINDOW = 5;
static const unsigned int MAX_LOGS_PER_WINDOW = 1800;
static constexpr std::chrono::minutes RATE_LIMIT_WINDOW_MINUTES = std::chrono::minutes(30);
class WaapModelReport : public RestGetFile
{
public:
WaapModelReport(const vector<WaapModelResult> &_data) : data(_data) {}
private:
C2S_PARAM(vector<WaapModelResult>, data);
};
class WaapModelResultLogger::Impl
:
Singleton::Provide<I_WaapModelResultLogger>::From<WaapModelResultLogger>
{
public:
Impl(size_t maxLogs) : max_logs(maxLogs), sent_files_count(0), sent_logs_count(0),
last_sent_s3(std::chrono::minutes::zero()),
last_kusto_log_window(std::chrono::minutes::zero()) {}
virtual ~Impl();
void
logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
string modelName,
string otherModelName,
double score,
double otherScore) override;
private:
void logToStream(WaapModelResult &result, chrono::minutes now);
void logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now);
bool shouldSendLogsToS3(chrono::minutes now);
void sendLogsToS3();
size_t max_logs;
unsigned int sent_files_count;
unsigned int sent_logs_count;
std::chrono::minutes last_sent_s3;
std::chrono::minutes last_kusto_log_window;
std::map<std::string, vector<WaapModelResult>> logs;
};
WaapModelResultLogger::WaapModelResultLogger(size_t maxLogs) : pimpl(make_unique<WaapModelResultLogger::Impl>(maxLogs))
{
}
WaapModelResultLogger::~WaapModelResultLogger()
{
}
void
WaapModelResultLogger::logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
std::string modelName,
std::string otherModelName,
double score,
double otherScore
)
{
pimpl->logModelResult(settings, transaction, res, modelName, otherModelName, score, otherScore);
}
void
WaapModelResultLogger::Impl::logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
string modelName,
string otherModelName,
double score,
double otherScore)
{
if (transaction == NULL) return;
if (!Singleton::exists<I_Messaging>()) {
dbgError(D_WAAP_MODEL_LOGGER) << "Messaging service is not available, will not log";
return;
}
double score_diff = score - otherScore;
if (settings.logLevel == Waap::Scores::ModelLogLevel::DIFF &&
! ((score_diff > 0 && score >= 1.5f && otherScore < 4.0f) ||
(score_diff < 0 && score < 4.0f && otherScore >= 1.5f))) {
return;
}
auto current_time = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
auto now = chrono::duration_cast<chrono::minutes>(current_time);
WaapModelResult result = WaapModelResult(
*transaction,
res,
modelName,
otherModelName,
score,
otherScore,
now.count()
);
if (settings.logToStream) logToStream(result, now);
if (settings.logToS3) logToS3(result, transaction, now);
}
void WaapModelResultLogger::Impl::logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now)
{
auto asset_state = transaction->getAssetState();
string asset_id = (asset_state != nullptr) ? asset_state->m_assetId : "";
auto asset_logs = logs.find(asset_id);
if (asset_logs == logs.end()) {
logs.emplace(asset_id, vector<WaapModelResult>());
}
logs.at(asset_id).push_back(result);
if (shouldSendLogsToS3(now)) {
sendLogsToS3();
}
}
void WaapModelResultLogger::Impl::logToStream(WaapModelResult &result, chrono::minutes now)
{
if (now - last_kusto_log_window > RATE_LIMIT_WINDOW_MINUTES) {
last_kusto_log_window = now;
sent_logs_count = 0;
}
else if (sent_logs_count > MAX_LOGS_PER_WINDOW) {
return;
}
sent_logs_count++;
dbgTrace(D_WAAP_MODEL_LOGGER) << "Logging WAAP model telemetry";
auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
LogGenWrapper logGenWrapper(
maybeLogTriggerConf,
"WAAP Model Telemetry",
ReportIS::Audience::SECURITY,
LogTriggerConf::SecurityType::ThreatPrevention,
ReportIS::Severity::CRITICAL,
ReportIS::Priority::HIGH,
false);
LogGen& waap_log = logGenWrapper.getLogGen();
waap_log.addMarkerSuffix(result.location);
waap_log << LogField("httpuripath", result.uri);
waap_log << LogField("matchedlocation", result.location);
waap_log << LogField("matchedparameter", result.param);
waap_log << LogField("matchedindicators", Waap::Util::vecToString(result.keywords), LogFieldOption::XORANDB64);
waap_log << LogField("matchedsample", result.sample, LogFieldOption::XORANDB64);
waap_log << LogField("waapkeywordsscore", (int)(result.otherScore * 100));
waap_log << LogField("waapfinalscore", (int)(result.score * 100));
waap_log << LogField("indicatorssource", result.modelName);
waap_log << LogField("indicatorsversion", result.otherModelName);
}
bool WaapModelResultLogger::Impl::shouldSendLogsToS3(chrono::minutes now)
{
if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) return true;
for (const auto &asset_logs : logs) {
if (asset_logs.second.size() >= max_logs) return true;
}
return false;
}
void WaapModelResultLogger::Impl::sendLogsToS3()
{
dbgFlow(D_WAAP_MODEL_LOGGER) << "Sending logs to fog";
I_Messaging *msg = Singleton::Consume<I_Messaging>::by<WaapComponent>();
for (auto &asset_logs : logs) {
if (asset_logs.second.empty()) {
continue;
}
if (sent_files_count >= MAX_FILES_PER_WINDOW) {
dbgInfo(D_WAAP_MODEL_LOGGER) << "Reached max files per window, will wait for next window";
asset_logs.second.clear();
continue;
}
I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
string tenant_id = agentDetails->getTenantId();
string agent_id = agentDetails->getAgentId();
string asset_id = asset_logs.first;
if (Singleton::exists<I_InstanceAwareness>()) {
I_InstanceAwareness* instance = Singleton::Consume<I_InstanceAwareness>::by<WaapComponent>();
Maybe<string> uniqueId = instance->getUniqueID();
if (uniqueId.ok())
{
agent_id += "/" + uniqueId.unpack();
}
}
string uri = "/storage/waap/" +
tenant_id + "/" + asset_id + "/waap_model_results/window_" +
to_string(last_sent_s3.count()) + "-" + to_string(sent_files_count) +
"/" + agent_id + "/data.data";
WaapModelReport report = WaapModelReport(asset_logs.second);
dbgInfo(D_WAAP_MODEL_LOGGER) << "Sending logs for asset " << asset_logs.first <<
", length " << asset_logs.second.size() <<
", uri " << uri;
msg->sendAsyncMessage(
HTTPMethod::PUT,
uri,
report,
MessageCategory::LOG
);
asset_logs.second.clear();
}
auto current_time = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
auto now = chrono::duration_cast<chrono::minutes>(current_time);
if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) {
last_sent_s3 = now;
sent_files_count = 0;
} else {
sent_files_count++;
}
}
WaapModelResultLogger::Impl::~Impl()
{}

109
components/security_apps/waap/waap_clib/WaapModelResultLogger.h Normal file
View File

@ -0,0 +1,109 @@
// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <memory>
#include <string>
#include <map>
#include <chrono>
#include <vector>
#include <ostream>
#include "cereal/archives/json.hpp"
#include "i_waap_model_result_logger.h"
#include "DeepAnalyzer.h"
#include "i_transaction.h"
#include "ScanResult.h"
#include "WaapAssetState.h"
#include "WaapScores.h"
class WaapModelResultLogger
:
Singleton::Provide<I_WaapModelResultLogger>
{
public:
WaapModelResultLogger(size_t maxLogs = MAX_WAAP_MODEL_LOGS);
virtual ~WaapModelResultLogger();
virtual void logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
std::string modelName,
std::string otherModelName,
double score,
double otherScore
);
class Impl;
protected:
std::unique_ptr<Impl> pimpl;
static const size_t MAX_WAAP_MODEL_LOGS = 20000;
};
class WaapModelResult
{
public:
WaapModelResult(
IWaf2Transaction &transaction,
Waf2ScanResult &res,
const std::string &modelName,
const std::string &otherModelName,
double score,
double otherScore,
uint64_t time
) : uri(transaction.getUri()), location(res.location), param(res.param_name),
modelName(modelName), otherModelName(otherModelName),
score(score), otherScore(otherScore), keywords(res.keywordsAfterFilter),
sample(res.unescaped_line.substr(0, 100)), id(transaction.getIndex()), time(time)
{
}
template<class Archive>
void serialize(Archive &ar) const
{
ar(cereal::make_nvp("uri", uri));
ar(cereal::make_nvp("location", location));
ar(cereal::make_nvp("param", param));
ar(cereal::make_nvp("modelName", modelName));
ar(cereal::make_nvp("otherModelName", otherModelName));
ar(cereal::make_nvp("score", score));
ar(cereal::make_nvp("otherScore", otherScore));
ar(cereal::make_nvp("keywords", keywords));
ar(cereal::make_nvp("sample", sample));
ar(cereal::make_nvp("id", id));
ar(cereal::make_nvp("time", time));
}
std::string toString() const
{
std::stringstream message_stream;
{
cereal::JSONOutputArchive ar(message_stream);
serialize(ar);
}
return message_stream.str();
}
std::string uri;
std::string location;
std::string param;
std::string modelName;
std::string otherModelName;
double score;
double otherScore;
std::vector<std::string> keywords;
std::string sample;
uint64_t id;
uint64_t time;
};

5
components/security_apps/waap/waap_clib/WaapOverride.cc
View File

@ -12,7 +12,6 @@
// limitations under the License.
#include "WaapOverride.h"
#include "Waf2Util.h"
USE_DEBUG_FLAG(D_WAAP);
@ -25,8 +24,8 @@ bool Match::operator==(const Match &other) const
(m_operand1 == other.m_operand1) &&
(m_operand2 == other.m_operand2) &&
(m_tag == other.m_tag) &&
Waap::Util::compareObjects(m_valueRegex, other.m_valueRegex) &&
m_cidr == other.m_cidr &&
(m_valuesRegex == other.m_valuesRegex) &&
m_ip_addr_values == other.m_ip_addr_values &&
m_isCidr == other.m_isCidr;
}

105
components/security_apps/waap/waap_clib/WaapOverride.h
View File

@ -21,6 +21,7 @@
#include <memory>
#include "debug.h"
#include "CidrMatch.h"
#include "RegexComparator.h"
USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
@ -52,23 +53,52 @@ public:
m_isValid = false;
dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag;
}
// The name "value" here is misleading. The real meaning is "regex pattern string"
ar(cereal::make_nvp("value", m_value));
try {
ar(cereal::make_nvp("values", m_values));
dbgDebug(D_WAAP_OVERRIDE) << "Values list is missing, using single value instead.";
} catch (const cereal::Exception &e) {
// The name "value" here is misleading. The real meaning is "regex pattern string"
ar(cereal::make_nvp("value", m_value));
m_values.insert(m_value);
}
if (m_tag == "sourceip" || m_tag == "sourceidentifier") {
m_isCidr = Waap::Util::isCIDR(m_value, m_cidr);
m_isCidr = true;
m_ip_addr_values.resize(m_values.size());
int val_idx = 0;
for (const auto &cur_val : m_values) {
if (!Waap::Util::isCIDR(cur_val, m_ip_addr_values[val_idx])) {
dbgDebug(D_WAAP_OVERRIDE) << "Invalid value in list of IP addresses: " << cur_val;
m_isValid = false;
break;
}
val_idx++;
}
sortAndMergeCIDRs();
dbgTrace(D_WAAP_OVERRIDE) << "CIDR list: " << cidrsToString(m_ip_addr_values);
}
m_isOverrideResponse = (m_tag == "responsebody" || m_tag == "responseBody");
if (!m_isCidr) {
// regex build may throw boost::regex_error
m_valueRegex = nullptr;
try {
m_valueRegex = std::make_shared<boost::regex>(m_value);
}
catch (const boost::regex_error &err) {
dbgDebug(D_WAAP_OVERRIDE) << "Waap::Override::Match(): Failed to compile regex pattern '" <<
m_value << "' on position " << err.position() << ". Reason: '" << err.what() << "'";
for (const auto &cur_val : m_values) {
try {
m_valuesRegex.emplace(std::make_shared<boost::regex>(cur_val));
}
catch (const boost::regex_error &err) {
dbgDebug(D_WAAP_OVERRIDE)
<< "Waap::Override::Match(): Failed to compile regex pattern '"
<< cur_val
<< "' on position "
<< err.position()
<< ". Reason: '"
<< err.what()
<< "'";
m_isValid = false;
m_valuesRegex.clear();
break;
}
}
}
}
@ -95,13 +125,21 @@ public:
template<typename TestFunctor>
bool match(TestFunctor testFunctor) const {
if (m_op == "basic" && m_isCidr) {
bool result = testFunctor(m_tag, m_cidr);
dbgTrace(D_WAAP_OVERRIDE) << "Override matching CIDR: " << m_value << " result: " << result;
bool result = testFunctor(m_tag, m_ip_addr_values);
dbgTrace(D_WAAP_OVERRIDE)
<< "Override matching CIDR list: "
<< cidrsToString(m_ip_addr_values)
<< " result: "
<< result;
return result;
}
else if (m_op == "basic" && m_valueRegex) {
bool result = testFunctor(m_tag, *m_valueRegex);
dbgTrace(D_WAAP_OVERRIDE) << "Override matching regex: " << m_value << " result: " << result;
else if (m_op == "basic" && !m_valuesRegex.empty()) {
bool result = testFunctor(m_tag, m_valuesRegex);
dbgTrace(D_WAAP_OVERRIDE)
<< "Override matching regex list: "
<< regexSetToString(m_valuesRegex)
<< " result: "
<< result;
return result;
}
if (m_op == "and") {
@ -125,24 +163,39 @@ public:
return false;
}
bool isOverrideResponse() const {
return m_isOverrideResponse;
}
bool isOverrideResponse() const { return m_isOverrideResponse; }
bool isValidMatch() const{
return m_isValid;
}
bool isValidMatch() const { return m_isValid; }
private:
void sortAndMergeCIDRs() {
if (m_ip_addr_values.empty()) return;
std::sort(m_ip_addr_values.begin(), m_ip_addr_values.end());
size_t mergedIndex = 0;
for (size_t i = 1; i < m_ip_addr_values.size(); ++i) {
Waap::Util::CIDRData &current = m_ip_addr_values[mergedIndex];
Waap::Util::CIDRData &next = m_ip_addr_values[i];
if (!doesFirstCidrContainSecond(current, next)) {
++mergedIndex;
if (i != mergedIndex) m_ip_addr_values[mergedIndex] = next;
}
}
m_ip_addr_values.resize(mergedIndex + 1);
}
std::string m_op;
std::shared_ptr<Match> m_operand1;
std::shared_ptr<Match> m_operand2;
std::string m_tag;
std::string m_value;
std::shared_ptr<boost::regex> m_valueRegex;
Waap::Util::CIDRData m_cidr;
bool m_isCidr;
bool m_isOverrideResponse;
std::set<std::string> m_values;
std::vector<Waap::Util::CIDRData> m_ip_addr_values;
std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> m_valuesRegex;
bool m_isCidr;
bool m_isOverrideResponse;
bool m_isValid;
};

165
components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
View File

@ -17,100 +17,150 @@
#include "WaapOverrideFunctor.h"
#include "Waf2Engine.h"
#include "CidrMatch.h"
#include "RegexComparator.h"
#include "agent_core_utilities.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
#define REGX_MATCH(FIELD_FETCH) \
NGEN::Regex::regexMatch(__FILE__, __LINE__, FIELD_FETCH.c_str(), what, *rx)
#define W2T_REGX_MATCH(FIELD_GETTER) \
REGX_MATCH(waf2Transaction.FIELD_GETTER())
WaapOverrideFunctor::WaapOverrideFunctor(Waf2Transaction& waf2Transaction) :waf2Transaction(waf2Transaction)
{
}
bool WaapOverrideFunctor::operator()(const std::string& tag, const Waap::Util::CIDRData& value) {
bool WaapOverrideFunctor::operator()(const std::string& tag, const std::vector<Waap::Util::CIDRData> &values) {
std::string sourceIp;
if (tag == "sourceip") {
dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr() << " CIDR: " <<
value.cidrString;
std::string sourceIp = waf2Transaction.getRemoteAddr();
// match sourceIp against the cidr
return Waap::Util::cidrMatch(sourceIp, value);
dbgDebug(D_WAAP_OVERRIDE)
<< "Remote IP Address : "
<< waf2Transaction.getRemoteAddr();
sourceIp = waf2Transaction.getRemoteAddr();
} else if (tag == "sourceidentifier") {
dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr();
sourceIp = waf2Transaction.getSourceIdentifier();
} else {
dbgWarning(D_WAAP_OVERRIDE) << "Unsupported tag: " << tag;
return false;
}
else if (tag == "sourceidentifier") {
dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr() << " CIDR: " <<
value.cidrString;
std::string sourceIp = waf2Transaction.getSourceIdentifier();
// match source against the cidr
return Waap::Util::cidrMatch(sourceIp, value);
Waap::Util::CIDRData source_cidr;
if (!Waap::Util::isCIDR(sourceIp, source_cidr)) {
dbgWarning(D_WAAP_OVERRIDE) << "Failed to create subnet from: " << sourceIp;
return false;
}
int left = 0;
int right = values.size() - 1;
while (left <= right) {
int mid = left + (right - left) / 2;
if (Waap::Util::cidrMatch(sourceIp, values[mid])) return true;
if (values[mid] < source_cidr) {
left = mid + 1;
} else {
right = mid - 1;
}
}
return false;
}
bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& rx)
bool WaapOverrideFunctor::operator()(
const std::string &tag,
const std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> &rxes)
{
boost::cmatch what;
std::string tagLower = tag;
std::transform(tagLower.begin(), tagLower.end(), tagLower.begin(), ::tolower);
try {
if (tagLower == "method") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getMethod().c_str(), what, rx);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getMethod)) return true;
}
return false;
}
else if (tagLower == "url") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUriStr().c_str(), what, rx);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getUriStr)) return true;
}
return false;
}
else if (tagLower == "hostname") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getHost().c_str(), what, rx);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getHost)) return true;
}
return false;
}
else if (tagLower == "sourceidentifier") {
return NGEN::Regex::regexMatch(
__FILE__,
__LINE__,
waf2Transaction.getSourceIdentifier().c_str(),
what,
rx
);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getSourceIdentifier)) return true;
}
return false;
}
else if (tagLower == "keyword") {
for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordStr.c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
if (REGX_MATCH(keywordStr)) {
return true;
}
}
}
return false;
}
else if (tagLower == "paramname") {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getName().c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (REGX_MATCH(keywordInfo.getName())) {
return true;
}
}
}
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getParamKey().c_str(), what, rx)) {
return true;
}
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getParam().c_str(), what, rx)) {
return true;
if (W2T_REGX_MATCH(getParamKey)) return true;
if (W2T_REGX_MATCH(getParam)) return true;
}
return false;
}
else if (tagLower == "paramvalue") {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getValue().c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (REGX_MATCH(keywordInfo.getValue())) {
return true;
}
}
if (W2T_REGX_MATCH(getSample)) return true;
}
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getSample().c_str(), what, rx)) {
return true;
}
return false;
}
else if (tagLower == "paramlocation") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getLocation().c_str(), what, rx);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getLocation)) return true;
}
return false;
}
else if (tagLower == "responsebody") {
waf2Transaction.getResponseInspectReasons().setApplyOverride(true);
if (!waf2Transaction.getResponseBody().empty()) {
boost::smatch matcher;
return NGEN::Regex::regexSearch(__FILE__, __LINE__,
waf2Transaction.getResponseBody().c_str(), matcher, rx);
for (const auto &rx : rxes) {
boost::smatch matcher;
if (NGEN::Regex::regexSearch(
__FILE__,
__LINE__,
waf2Transaction.getResponseBody().c_str(),
matcher,
*rx
)) {
return true;
}
}
return false;
} else {
return false;
}
@ -119,11 +169,13 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
dbgDebug(D_WAAP_OVERRIDE) << "Header name override scan is not required";
return false;
}
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.first;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if(NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.first;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if (REGX_MATCH(value)) {
return true;
}
}
}
return false;
@ -132,11 +184,13 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
dbgDebug(D_WAAP_OVERRIDE) << "Header value override scan is not required";
return false;
}
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.second;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.second;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if (REGX_MATCH(value)) {
return true;
}
}
}
return false;
@ -146,7 +200,6 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
dbgDebug(D_WAAP_OVERRIDE) << "RegEx match for tag " << tag << " failed due to: " << e.what();
return false;
}
// Unknown tag: should not occur
dbgDebug(D_WAAP) << "Invalid override tag: " << tag;
return false;

12
components/security_apps/waap/waap_clib/WaapOverrideFunctor.h
View File

@ -16,6 +16,7 @@
namespace Waap {
namespace Util {
struct CIDRData; // forward decleration
struct RegexComparator;
}
}
@ -24,12 +25,15 @@ class Waf2Transaction;
// Functor used to match Override rules against request data
class WaapOverrideFunctor {
public:
WaapOverrideFunctor(Waf2Transaction& waf2Transaction);
WaapOverrideFunctor(Waf2Transaction &waf2Transaction);
bool operator()(const std::string& tag, const Waap::Util::CIDRData& value);
bool operator()(const std::string &tag, const std::vector<Waap::Util::CIDRData> &values);
bool operator()(const std::string& tag, const boost::regex& rx);
bool operator()(
const std::string &tag,
const std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> &rxes
);
private:
Waf2Transaction& waf2Transaction;
Waf2Transaction &waf2Transaction;
};

49
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@ -13,7 +13,9 @@
#include "WaapScanner.h"
#include "WaapScores.h"
#include "Waf2Engine.h"
#include "i_transaction.h"
#include "WaapModelResultLogger.h"
#include <string>
#include "debug.h"
#include "reputation_features_events.h"
@ -105,20 +107,59 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
}
std::sort(newKeywords.begin(), newKeywords.end());
res.keywordsAfterFilter.clear();
for (auto keyword : newKeywords) {
res.keywordsAfterFilter.push_back(keyword);
}
res.scoreArray.clear();
res.coefArray.clear();
res.keywordCombinations.clear();
double res_score = getScoreFromPool(res, newKeywords, poolName);
std::string other_pool_name = Waap::Scores::getOtherScorePoolName();
Waap::Scores::ModelLoggingSettings modelLoggingSettings = Waap::Scores::getModelLoggingSettings();
if (applyLearning && poolName != other_pool_name &&
modelLoggingSettings.logLevel != Waap::Scores::ModelLogLevel::OFF) {
double other_score = getScoreFromPool(res, newKeywords, other_pool_name);
dbgDebug(D_WAAP_SCANNER) << "Comparing score from pool " << poolName << ": " << res_score
<< ", vs. pool " << other_pool_name << ": " << other_score
<< ", score difference: " << res_score - other_score
<< ", sample: " << res.unescaped_line;
Singleton::Consume<I_WaapModelResultLogger>::by<WaapComponent>()->logModelResult(
modelLoggingSettings, m_transaction, res, poolName, other_pool_name, res_score, other_score
);
res.other_model_score = other_score;
} else {
res.other_model_score = res_score;
}
return res_score;
}
double Waap::Scanner::getScoreFromPool(
Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
)
{
KeywordsStats stats = m_transaction->getAssetState()->scoreBuilder.getSnapshotStats(poolName);
if (!newKeywords.empty()) {
// Collect scores of individual keywords
Waap::Scores::calcIndividualKeywords(m_transaction->getAssetState()->scoreBuilder, poolName, newKeywords,
res.scoreArray);
res.scoreArray, res.coefArray);
// Collect keyword combinations and their scores. Append scores to scoresArray,
// and also populate m_scanResultKeywordCombinations list
Waap::Scores::calcCombinations(m_transaction->getAssetState()->scoreBuilder, poolName, newKeywords,
res.scoreArray, res.keywordCombinations);
res.scoreArray, res.coefArray, res.keywordCombinations);
}
if (stats.isLinModel) {
return Waap::Scores::calcLogisticRegressionScore(
res.coefArray, stats.linModelIntercept, stats.linModelNNZCoef
);
}
// use base_scores calculation
return Waap::Scores::calcArrayScore(res.scoreArray);
}
@ -127,7 +168,7 @@ bool Waap::Scanner::isKeyCspReport(const std::string &key, Waf2ScanResult &res,
{
if (res.score < 8.0f && res.location == "body" && dp.getActualParser(0) == "jsonParser") {
if (key == "csp-report.blocked-uri" || key == "csp-report.script-sample" ||
(key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) {
(key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) {
dbgTrace(D_WAAP_SCANNER) << "CSP report detected, ignoring.";
return true;
}

3
components/security_apps/waap/waap_clib/WaapScanner.h
View File

@ -44,6 +44,9 @@ namespace Waap {
static const std::string xmlEntityAttributeId;
private:
double getScoreData(Waf2ScanResult& res, const std::string &poolName, bool applyLearning = true);
double getScoreFromPool(
Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
);
bool shouldIgnoreOverride(const Waf2ScanResult &res);
bool isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp);

79
components/security_apps/waap/waap_clib/WaapScores.cc
View File

@ -14,6 +14,8 @@
#include "WaapScores.h"
#include <vector>
#include <string>
#include <cmath>
#include "ScoreBuilder.h"
#include "WaapDefines.h"
#include "debug.h"
@ -24,11 +26,44 @@ namespace Waap {
namespace Scores {
std::string getScorePoolNameByLocation(const std::string &location) {
std::string poolName = KEYWORDS_SCORE_POOL_BASE;
if (location == "header") {
poolName = KEYWORDS_SCORE_POOL_HEADERS;
auto maybePoolName = getProfileAgentSetting<std::string>("agent.waap.scorePoolName");
std::string res = KEYWORDS_SCORE_POOL_BASE;
if (maybePoolName.ok()) {
res = maybePoolName.unpack();
}
return poolName;
else if (location == "header") {
res = KEYWORDS_SCORE_POOL_HEADERS;
}
return res;
}
std::string getOtherScorePoolName()
{
auto maybePoolName = getProfileAgentSetting<std::string>("agent.waap.otherScorePoolName");
if (maybePoolName.ok()) {
return maybePoolName.unpack();
}
return KEYWORDS_SCORE_POOL_BASE;
}
ModelLoggingSettings getModelLoggingSettings()
{
ModelLoggingSettings settings = {.logLevel = ModelLogLevel::DIFF,
.logToS3 = false,
.logToStream = true};
auto maybeLogS3 = getProfileAgentSetting<bool>("agent.waap.modelLogToS3");
if (maybeLogS3.ok()) {
settings.logToS3 = maybeLogS3.unpack();
}
auto maybeLogKusto = getProfileAgentSetting<bool>("agent.waap.modelLogToStream");
if (maybeLogKusto.ok()) {
settings.logToStream = maybeLogKusto.unpack();
}
auto maybeLogLevel = getProfileAgentSetting<uint>("agent.waap.modelLogLevel");
if (maybeLogLevel.ok() && (settings.logToS3 || settings.logToStream)) {
settings.logLevel = static_cast<ModelLogLevel>(maybeLogLevel.unpack());
}
return settings;
}
void
@ -37,9 +72,16 @@ addKeywordScore(
const std::string &poolName,
std::string keyword,
double defaultScore,
std::vector<double>& scoresArray)
double defaultCoef,
std::vector<double>& scoresArray,
std::vector<double>& coefArray)
{
scoresArray.push_back(scoreBuilder.getSnapshotKeywordScore(keyword, defaultScore, poolName));
double score = scoreBuilder.getSnapshotKeywordScore(keyword, defaultScore, poolName);
double coef = scoreBuilder.getSnapshotKeywordCoef(keyword, defaultCoef, poolName);
dbgDebug(D_WAAP_SCORE_BUILDER) << "Adding score: " << score << " coef: " << coef
<< " keyword: '" << keyword << "' pool: " << poolName;
scoresArray.push_back(score);
coefArray.push_back(coef);
}
// Calculate score of individual keywords
@ -48,13 +90,14 @@ calcIndividualKeywords(
const ScoreBuilder& scoreBuilder,
const std::string &poolName,
const std::vector<std::string>& keyword_matches,
std::vector<double>& scoresArray)
std::vector<double>& scoresArray,
std::vector<double>& coefArray)
{
std::vector<std::string> keywords = keyword_matches; // deep copy!! (PERFORMANCE WARNING!)
std::sort(keywords.begin(), keywords.end());
for (auto pKeyword = keywords.begin(); pKeyword != keywords.end(); ++pKeyword) {
addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, scoresArray);
addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, 0.3f, scoresArray, coefArray);
}
}
@ -65,10 +108,12 @@ calcCombinations(
const std::string &poolName,
const std::vector<std::string>& keyword_matches,
std::vector<double>& scoresArray,
std::vector<double>& coefArray,
std::vector<std::string>& keyword_combinations)
{
keyword_combinations.clear();
static const double max_combi_score = 1.0f;
double default_coef = 0.8f;
for (size_t i = 0; i < keyword_matches.size(); ++i) {
std::vector<std::string> combinations;
@ -93,7 +138,7 @@ calcCombinations(
}
// set default combination score to be the sum of its keywords, bounded by 1
default_score = std::min(default_score, max_combi_score);
addKeywordScore(scoreBuilder, poolName, combination, default_score, scoresArray);
addKeywordScore(scoreBuilder, poolName, combination, default_score, default_coef, scoresArray, coefArray);
keyword_combinations.push_back(combination);
}
}
@ -105,7 +150,6 @@ calcArrayScore(std::vector<double>& scoreArray)
// Calculate cumulative score from array of individual scores
double score = 1.0f;
for (auto pScore = scoreArray.begin(); pScore != scoreArray.end(); ++pScore) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "scoreArr[]=" << *pScore;
double left = 10.0f - score;
double divisor = (*pScore / 3.0f + 10.0f); // note: divisor can't be empty because
// *pScore is always positive and there's a +10 offset
@ -115,5 +159,20 @@ calcArrayScore(std::vector<double>& scoreArray)
return score;
}
double
calcLogisticRegressionScore(std::vector<double> &coefArray, double intercept, double nnzCoef)
{
// Sparse logistic regression model, with boolean feature values
// Instead of performing a dot product of features*coefficients, we sum the coefficients of the non-zero features
// An additional feature was added for the log of the number of non-zero features, as a regularization term
double log_odds = intercept + nnzCoef * log(static_cast<double>(coefArray.size()) + 1);
for (double &pCoef : coefArray) {
log_odds += pCoef;
}
// Apply the expit function to the log-odds to obtain the probability,
// and multiply by 10 to obtain a 'score' in the range [0, 10]
return 1.0f / (1.0f + exp(-log_odds)) * 10.0f;
}
}
}

23
components/security_apps/waap/waap_clib/WaapScores.h
View File

@ -20,7 +20,21 @@
namespace Waap {
namespace Scores {
enum class ModelLogLevel {
OFF = 0,
DIFF = 1,
ALL = 2
};
struct ModelLoggingSettings {
ModelLogLevel logLevel;
bool logToS3;
bool logToStream;
};
std::string getScorePoolNameByLocation(const std::string &location);
std::string getOtherScorePoolName();
ModelLoggingSettings getModelLoggingSettings();
void
addKeywordScore(
@ -28,7 +42,9 @@ addKeywordScore(
const std::string &poolName,
std::string keyword,
double defaultScore,
std::vector<double>& scoresArray);
double defaultCoef,
std::vector<double>& scoresArray,
std::vector<double>& coefArray);
// Calculate score of individual keywords
void
@ -36,7 +52,8 @@ calcIndividualKeywords(
const ScoreBuilder& scoreBuilder,
const std::string &poolName,
const std::vector<std::string>& keyword_matches,
std::vector<double>& scoresArray);
std::vector<double>& scoresArray,
std::vector<double>& coefArray);
// Calculate keyword combinations and their scores
void
@ -45,9 +62,11 @@ calcCombinations(
const std::string &poolName,
const std::vector<std::string>& keyword_matches,
std::vector<double>& scoresArray,
std::vector<double>& coefArray,
std::vector<std::string>& keyword_combinations);
double calcArrayScore(std::vector<double>& scoreArray);
double calcLogisticRegressionScore(std::vector<double> &coefArray, double intercept, double nnzCoef=0.0);
}
}

2
components/security_apps/waap/waap_clib/WaapTrigger.h
View File

@ -125,6 +125,7 @@ struct Log {
struct Trigger {
template <typename _A>
void serialize(_A &ar) {
ar(cereal::make_nvp("id", triggerId));
ar(cereal::make_nvp("$triggerType", triggerType));
triggerType = to_lower_copy(triggerType);
@ -137,6 +138,7 @@ struct Trigger {
Trigger();
bool operator==(const Trigger &other) const;
std::string triggerId;
std::string triggerType;
std::shared_ptr<Log> log;
};

28
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@ -819,8 +819,15 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
std::string tag = scanStage + "_param";
m_deepParser.m_key.push(tag.data(), tag.size());
size_t buff_len = uriEnd - p;
dbgTrace(D_WAAP) << "% will be encoded?'" << checkUrlEncoded(p, buff_len) << "'";
ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, checkUrlEncoded(p, buff_len));
bool should_decode = checkUrlEncoded(p, buff_len);
if (should_decode) {
std::string url_as_string(p, buff_len);
should_decode = should_decode && (Waap::Util::testUrlBadUtf8Evasion(url_as_string) != true);
}
dbgTrace(D_WAAP) << "should_decode % = " << should_decode;
ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, should_decode);
up.push(p, buff_len);
up.finish();
m_deepParser.m_key.pop(tag.c_str());
@ -1300,6 +1307,15 @@ void Waf2Transaction::set_ignoreScore(bool ignoreScore) {
m_ignoreScore = ignoreScore;
}
bool Waf2Transaction::get_ignoreScore() const
{
auto maybe_override_ignore_score = getProfileAgentSetting<std::string>("agent.waap.alwaysReportScore");
if (maybe_override_ignore_score.ok()) {
return maybe_override_ignore_score.unpack() == "true";
}
return m_ignoreScore;
}
void
Waf2Transaction::decide(
bool& bForceBlock,
@ -1889,6 +1905,14 @@ Waf2Transaction::sendLog()
return;
}
std::set<std::string> triggers_set;
for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) {
triggers_set.insert(trigger.triggerId);
dbgTrace(D_WAAP) << "Add waap log trigger id to triggers set:" << trigger.triggerId;
}
ScopedContext ctx;
ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
switch (decision_type)
{

9
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@ -20,6 +20,8 @@
#include "DeepParser.h"
#include "WaapAssetState.h"
#include "PatternMatcher.h"
#include "generic_rulebase/rulebase_config.h"
#include "generic_rulebase/evaluators/trigger_eval.h"
#include "Waf2Util.h"
#include "WaapConfigApplication.h"
#include "WaapConfigApi.h"
@ -39,6 +41,7 @@
#include "i_waap_telemetry.h"
#include "i_deepAnalyzer.h"
#include "i_time_get.h"
#include "i_waap_model_result_logger.h"
#include "table_opaque.h"
#include "WaapResponseInspectReasons.h"
#include "WaapResponseInjectReasons.h"
@ -91,6 +94,7 @@ public:
const std::vector<std::string> getFilteredKeywords() const;
const std::map<std::string, std::vector<std::string>> getFilteredVerbose() const;
virtual const std::vector<std::string> getKeywordsCombinations() const;
virtual const std::vector<std::string> getKeywordsAfterFilter() const;
const std::vector<DeepParser::KeywordInfo>& getKeywordInfo() const;
const std::vector<std::pair<std::string, std::string> >& getKvPairs() const;
const std::string getKeywordMatchesStr() const;
@ -99,6 +103,9 @@ public:
const std::string getLastScanSample() const;
virtual const std::string& getLastScanParamName() const;
double getScore() const;
// LCOV_EXCL_START Reason: model testing
double getOtherModelScore() const;
// LCOV_EXCL_STOP
const std::vector<double> getScoreArray() const;
Waap::CSRF::State& getCsrfState();
const std::set<std::string> getFoundPatterns() const;
@ -161,7 +168,7 @@ public:
// decision functions
void set_ignoreScore(bool ignoreScore);
bool get_ignoreScore() const { return m_ignoreScore; }
bool get_ignoreScore() const;
void decide(
bool& bForceBlock,
bool& bForceException,

19
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@ -189,6 +189,15 @@ const std::vector<std::string> Waf2Transaction::getKeywordsCombinations() const
}
return std::vector<std::string>();
}
const std::vector<std::string> Waf2Transaction::getKeywordsAfterFilter() const
{
if (m_scanResult)
{
return m_scanResult->keywordsAfterFilter;
}
return std::vector<std::string>();
}
const std::vector<DeepParser::KeywordInfo>& Waf2Transaction::getKeywordInfo() const
{
return m_deepParser.m_keywordInfo;
@ -230,6 +239,15 @@ double Waf2Transaction::getScore() const
}
return 0;
}
// LCOV_EXCL_START Reason: model testing
double Waf2Transaction::getOtherModelScore() const
{
if (m_scanResult) {
return m_scanResult->other_model_score;
}
return 0;
}
// LCOV_EXCL_STOP
const std::vector<double> Waf2Transaction::getScoreArray() const
{
if (m_scanResult) {
@ -397,6 +415,7 @@ void Waf2Transaction::sendAutonomousSecurityLog(
waap_log << LogField("waapUriFalsePositiveScore", (int)(
autonomousSecurityDecision->getFpMitigationScore() * 100));
waap_log << LogField("waapKeywordsScore", (int)(getScore() * 100));
waap_log << LogField("reservedNgenA", (int)(getOtherModelScore() * 100));
waap_log << LogField("waapFinalScore", (int)(autonomousSecurityDecision->getFinalScore() * 100));
waap_log << LogField("waapCalculatedThreatLevel", autonomousSecurityDecision->getThreatLevel());
}

39
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@ -508,6 +508,8 @@ const char* g_htmlTags[] = {
};
static const string b64_prefix("base64,");
// Empirically calculated entropy threshold for base64 encoded data, value above it is considered as base64 encoded
static const double base64_entropy_threshold = 4.01;
const size_t g_htmlTagsCount = sizeof(g_htmlTags) / sizeof(g_htmlTags[0]);
@ -966,7 +968,8 @@ base64_decode_status decodeBase64Chunk(
string::const_iterator it,
string::const_iterator end,
string& decoded,
bool clear_on_error)
bool clear_on_error,
bool called_with_prefix)
{
decoded.clear();
uint32_t acc = 0;
@ -974,6 +977,7 @@ base64_decode_status decodeBase64Chunk(
int terminatorCharsSeen = 0; // whether '=' character was seen, and how many of them.
uint32_t nonPrintableCharsCount = 0;
uint32_t spacer_count = 0;
uint32_t length = end - it;
dbgTrace(D_WAAP) << "decodeBase64Chunk: value='" << value << "' match='" << string(it, end) << "'";
string::const_iterator begin = it;
@ -986,6 +990,8 @@ base64_decode_status decodeBase64Chunk(
return B64_DECODE_INVALID;
}
std::unordered_map<char, double> frequency;
while (it != end) {
unsigned char c = *it;
@ -1008,6 +1014,7 @@ base64_decode_status decodeBase64Chunk(
// allow for more terminator characters
it++;
frequency[c]++;
continue;
}
@ -1032,6 +1039,7 @@ base64_decode_status decodeBase64Chunk(
// Start tracking terminator characters
terminatorCharsSeen++;
it++;
frequency[c]++;
continue;
}
else {
@ -1062,6 +1070,7 @@ base64_decode_status decodeBase64Chunk(
}
it++;
frequency[c]++;
}
// end of encoded sequence decoded.
@ -1078,6 +1087,27 @@ base64_decode_status decodeBase64Chunk(
<< "; decoded='"
<< decoded << "'";
// Check if entropy is correlates with b64 threshold (initially > 4.5)
if (!called_with_prefix) {
double entropy = 0;
double p = 0;
for (const auto& pair : frequency) {
p = pair.second / length;
entropy -= p * std::log2(p);
}
dbgTrace(D_WAAP_BASE64) << " ===b64Test===: base entropy = " << entropy << "length = " << length;
// Add short payload factor
if (length < 16)
entropy = entropy * 16 / length;
// Enforce tailoring '=' characters
entropy+=terminatorCharsSeen;
dbgTrace(D_WAAP_BASE64) << " ===b64Test===: corrected entropy = " << entropy << "length = " << length;
if (entropy <= base64_entropy_threshold) {
return B64_DECODE_INVALID;
}
}
// Return success only if decoded.size>=5 and there are less than 10% of non-printable
// characters in output.
if (decoded.size() >= 5) {
@ -1323,10 +1353,11 @@ processDecodedChunk(
string::const_iterator start,
string::const_iterator end,
string &value,
BinaryFileType &binaryFileType
BinaryFileType &binaryFileType,
bool called_with_prefix = false
)
{
base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false);
base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false, called_with_prefix);
dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: after decode. retVal=" << retVal
<< " value.size()=" << value.size();
if (retVal != B64_DECODE_INVALID && !value.empty()) {
@ -1349,7 +1380,7 @@ bool isBase64PrefixProcessingOK (
if (detectBase64Chunk(s, start, end)) {
dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk detected";
if ((start != s.end()) && (end == s.end())) {
retVal = processDecodedChunk(s, start, end, value, binaryFileType);
retVal = processDecodedChunk(s, start, end, value, binaryFileType, true);
}
} else if (start != s.end()) {
dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk not detected."

3
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@ -871,7 +871,8 @@ decodeBase64Chunk(
std::string::const_iterator it,
std::string::const_iterator end,
std::string &decoded,
bool clear_on_error = true);
bool clear_on_error = true,
bool called_with_prefix = false);
bool
b64DecodeChunk(

3
components/security_apps/waap/waap_component_impl.cc
View File

@ -50,7 +50,8 @@ WaapComponent::Impl::Impl() :
drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP),
waapStateTable(NULL),
transactionsCount(0),
deepAnalyzer()
deepAnalyzer(),
waapModelResultLogger()
{
}

2
components/security_apps/waap/waap_component_impl.h
View File

@ -19,6 +19,7 @@
#include "table_opaque.h"
#include "i_transaction.h"
#include "waap_clib/DeepAnalyzer.h"
#include "waap_clib/WaapModelResultLogger.h"
#include "waap_clib/WaapAssetState.h"
#include "waap_clib/WaapAssetStatesManager.h"
#include "reputation_features_agg.h"
@ -80,6 +81,7 @@ private:
uint64_t transactionsCount;
// instance of singleton classes
DeepAnalyzer deepAnalyzer;
WaapModelResultLogger waapModelResultLogger;
WaapAssetStatesManager waapAssetStatesManager;
std::unordered_set<std::string> m_seen_assets_id;
};

42
components/utils/generic_rulebase/match_query.cc
View File

@ -157,6 +157,9 @@ MatchQuery::load(cereal::JSONInputArchive &archive_in)
dbgDebug(D_RULEBASE_CONFIG) << "Failed to compile regex. Error: " << e.what();
}
}
if (isKeyTypeIp()) {
sortAndMergeIpRangesValues();
}
first_value = *(value.begin());
}
break;
@ -301,7 +304,7 @@ MatchQuery::matchAttributes(
bool negate = type == MatchQuery::Conditions::NotEquals || type == MatchQuery::Conditions::NotIn;
bool match = false;
if (isIP()) {
if (isKeyTypeIp()) {
match = matchAttributesIp(values);
} else if (isRegEx()) {
match = matchAttributesRegEx(values, matched_override_keywords);
@ -352,10 +355,18 @@ MatchQuery::matchAttributesString(const set<string> &values) const
bool
MatchQuery::matchAttributesIp(const set<string> &values) const
{
for (const IPRange &rule_ip_range : ip_addr_value) {
for (const string &requested_value : values) {
IpAddress ip_addr = IPUtilities::createIpFromString(requested_value);
if (IPUtilities::isIpAddrInRange(rule_ip_range, ip_addr)) return true;
for (const string &requested_value : values) {
int left = 0;
int right = ip_addr_value.size() - 1;
IpAddress ip_addr = IPUtilities::createIpFromString(requested_value);
while (left <= right) {
int mid = left + (right - left) / 2;
if (IPUtilities::isIpAddrInRange(ip_addr_value[mid], ip_addr)) return true;
if (ip_addr_value[mid].start < ip_addr) {
left = mid + 1;
} else {
right = mid - 1;
}
}
}
return false;
@ -367,8 +378,23 @@ MatchQuery::isRegEx() const
return key != "protectionName";
}
bool
MatchQuery::isIP() const
void
MatchQuery::sortAndMergeIpRangesValues()
{
return key == "sourceIP" || key == "destinationIP";
if (ip_addr_value.empty()) return;
sort(ip_addr_value.begin(), ip_addr_value.end());
size_t mergedIndex = 0;
for (size_t i = 1; i < ip_addr_value.size(); ++i) {
if (ip_addr_value[i].start <= ip_addr_value[mergedIndex].end) {
if (ip_addr_value[mergedIndex].end <= ip_addr_value[i].end) {
ip_addr_value[mergedIndex].end = ip_addr_value[i].end;
}
} else {
++mergedIndex;
ip_addr_value[mergedIndex] = ip_addr_value[i];
}
}
ip_addr_value.resize(mergedIndex + 1);
}

18
components/utils/ip_utilities/ip_utilities.cc
View File

@ -22,7 +22,10 @@ bool
operator<(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr)
{
if (this_ip_addr.ip_type < other_ip_addr.ip_type) return true;
if (this_ip_addr.ip_type == IP_VERSION_4) return this_ip_addr.addr4_t.s_addr < other_ip_addr.addr4_t.s_addr;
if (this_ip_addr.ip_type > other_ip_addr.ip_type) return false;
if (this_ip_addr.ip_type == IP_VERSION_4) {
return ntohl(this_ip_addr.addr4_t.s_addr) < ntohl(other_ip_addr.addr4_t.s_addr);
}
return memcmp(&this_ip_addr.addr6_t, &other_ip_addr.addr6_t, sizeof(struct in6_addr)) < 0;
}
@ -33,6 +36,19 @@ operator==(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr)
if (this_ip_addr.ip_type == IP_VERSION_4) return this_ip_addr.addr4_t.s_addr == other_ip_addr.addr4_t.s_addr;
return memcmp(&this_ip_addr.addr6_t, &other_ip_addr.addr6_t, sizeof(struct in6_addr)) == 0;
}
bool
operator<=(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr)
{
if (this_ip_addr < other_ip_addr || this_ip_addr == other_ip_addr) return true;
return false;
}
bool
operator<(const IPRange &range1, const IPRange &range2)
{
return range1.start < range2.start || (range1.start == range2.start && range1.end < range2.end);
}
// LCOV_EXCL_STOP
Maybe<pair<string, int>>

24
components/utils/keywords/keywords_rule.cc
View File

@ -5,6 +5,8 @@
using namespace std;
USE_DEBUG_FLAG(D_KEYWORD);
static const string whitespaces = " \t";
static string
@ -14,6 +16,13 @@ getSubStrNoPadding(const string &str, uint start, uint end)
auto r_end = str.find_last_not_of(whitespaces, end-1);
if (r_end==string::npos || r_start==string::npos || r_start>r_end) {
dbgWarning(D_KEYWORD)
<< "Can't extract substring from '"
<< str
<< "', padded start: "
<< r_start
<< ", padded end: "
<< r_end;
throw KeywordError("Found an empty section in the '"+ str + "'");
}
@ -45,13 +54,24 @@ split(const string &str, const string &delim, uint start = 0)
}
default:
if (!in_string && delim.find(str[index])!=string::npos) {
res.push_back(getSubStrNoPadding(str, part_start, index));
if (part_start == index) {
dbgTrace(D_KEYWORD) << "Encountered consecutive delimiter in: " << str;
} else {
res.push_back(getSubStrNoPadding(str, part_start, index));
}
part_start = index+1;
}
}
}
if (escape||in_string) throw KeywordError("Split has ended in the middle of the parsing");
if (escape||in_string) {
dbgWarning(D_KEYWORD)
<< "Failed to split "
<< str
<< ". Split ended in "
<< (escape ? "escape" : "middle of string");
throw KeywordError("Split has ended in the middle of the parsing");
}
if (str.find_first_not_of(whitespaces, part_start)!=string::npos) {
res.push_back(getSubStrNoPadding(str, part_start, str.size()));

7
components/utils/keywords/keywords_ut/keywords_ut.cc
View File

@ -55,6 +55,13 @@ TEST_F(KeywordsRuleTest, data_basic_test) {
EXPECT_FALSE(ruleRun("data: \"75\", part HTTP_RESPONSE_BODY;"));
}
TEST_F(KeywordsRuleTest, consecutive_delimiter_test) {
appendBuffer("HTTP_RESPONSE_BODY", "123456789");
EXPECT_TRUE(ruleRun("data: \"234\" , part HTTP_RESPONSE_BODY;"));
EXPECT_FALSE(ruleRun("data: \"75\", part HTTP_RESPONSE_BODY;"));
}
TEST_F(KeywordsRuleTest, data_relative_test) {
appendBuffer("HTTP_RESPONSE_BODY", "1234567890");

6
core/agent_details/agent_details.cc
View File

@ -304,6 +304,12 @@ AgentDetails::getOrchestrationMode() const
return orchestration_mode;
}
bool
AgentDetails::isOpenAppsecAgent() const
{
return (orchestration_mode == OrchestrationMode::HYBRID) || (tenant_id.rfind("org_", 0) == 0);
}
string
AgentDetails::getAccessToken() const
{

11
core/config/config.cc
View File

@ -315,22 +315,13 @@ void
ConfigComponent::Impl::preload()
{
I_Environment *environment = Singleton::Consume<I_Environment>::by<ConfigComponent>();
auto executable = environment->get<string>("Executable Name");
auto executable = environment->get<string>("Base Executable Name");
if (!executable.ok() || *executable == "") {
dbgWarning(D_CONFIG)
<< "Could not load nano service's settings since \"Executable Name\" in not found in the environment";
return;
}
executable_name = *executable;
auto file_path_end = executable_name.find_last_of("/");
if (file_path_end != string::npos) {
executable_name = executable_name.substr(file_path_end + 1);
}
auto file_sufix_start = executable_name.find_first_of(".");
if (file_sufix_start != string::npos) {
executable_name = executable_name.substr(0, file_sufix_start);
}
config_file_paths.insert(executable_name + "-conf.json");
config_file_paths.insert(executable_name + "-debug-conf.json");

18
core/debug_is/debug.cc
View File

@ -766,22 +766,8 @@ Debug::findDebugFilePrefix(const string &file_name)
string
Debug::getExecutableName()
{
auto executable = env->get<string>("Executable Name");
if (!executable.ok() || *executable == "") {
return "";
}
string executable_name = *executable;
auto file_path_end = executable_name.find_last_of("/");
if (file_path_end != string::npos) {
executable_name = executable_name.substr(file_path_end + 1);
}
auto file_sufix_start = executable_name.find_first_of(".");
if (file_sufix_start != string::npos) {
executable_name = executable_name.substr(0, file_sufix_start);
}
return executable_name;
auto executable = env->get<string>("Base Executable Name");
return executable.ok() ? *executable : "";
}
void

2
core/debug_is/debug_is_ut/debug_ut.cc
View File

@ -836,7 +836,7 @@ TEST_F(DebugConfigTest, testSetConfig)
EXPECT_CALL(mock_rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true));
env.preload();
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "debug-ut");
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "debug-ut");
env.init();
Debug::init();

13
core/environment/environment.cc
View File

@ -265,18 +265,9 @@ Environment::Impl::getCurrentHeadersMap()
tracing_headers["X-Span-Id"] = span_id;
}
auto exec_name = get<string>("Executable Name");
auto exec_name = get<string>("Base Executable Name");
if (exec_name.ok() && *exec_name != "") {
string executable_name = *exec_name;
auto file_path_end = executable_name.find_last_of("/");
if (file_path_end != string::npos) {
executable_name = executable_name.substr(file_path_end + 1);
}
auto file_sufix_start = executable_name.find_first_of(".");
if (file_sufix_start != string::npos) {
executable_name = executable_name.substr(0, file_sufix_start);
}
tracing_headers["X-Calling-Service"] = executable_name;
tracing_headers["X-Calling-Service"] = *exec_name;
}
return tracing_headers;

2
core/environment/environment_ut/tracing_ut.cc
View File

@ -346,7 +346,7 @@ TEST_F(TracingCompRoutinesTest, 2SpansDifFlow)
{
I_MainLoop::Routine routine = [&] () {
string service_name = "test-service-name";
i_env->registerValue("Executable Name", service_name);
i_env->registerValue("Base Executable Name", service_name);
i_env->startNewTrace(true, "a687b388-1108-4083-9852-07c33b1074e9");
trace_id = i_env->getCurrentTrace();

1
core/include/services_sdk/interfaces/i_agent_details.h
View File

@ -46,6 +46,7 @@ public:
virtual OrchestrationMode getOrchestrationMode() const = 0;
virtual std::string getAccessToken() const = 0;
virtual void loadAccessToken() = 0;
virtual bool isOpenAppsecAgent() const = 0;
// OpenSSL
virtual void setOpenSSLDir(const std::string &openssl_dir) = 0;

1
core/include/services_sdk/interfaces/i_env_details.h
View File

@ -24,6 +24,7 @@ class I_EnvDetails
public:
virtual EnvType getEnvType() = 0;
virtual std::string getToken() = 0;
virtual std::string getNameSpace() = 0;
protected:
virtual ~I_EnvDetails() {}

13
core/include/services_sdk/interfaces/messaging/messaging_metadata.h
View File

@ -215,6 +215,18 @@ public:
return is_to_fog;
}
void
setSniHostName(const std::string &_host_name)
{
sni_host_name = _host_name;
}
Maybe<std::string>
getSniHostName() const
{
return sni_host_name;
}
template <class Archive>
void
serialize(Archive &ar)
@ -237,6 +249,7 @@ public:
private:
std::string host_name = "";
Maybe<std::string> sni_host_name = genError("SNI host name not set");
std::string ca_path = "";
std::string client_cert_path = "";
std::string client_key_path = "";

1
core/include/services_sdk/interfaces/mock/mock_agent_details.h
View File

@ -28,6 +28,7 @@ public:
MOCK_CONST_METHOD0(getAgentId, std::string());
MOCK_METHOD0(loadAccessToken, void());
MOCK_CONST_METHOD0(getAccessToken, std::string());
MOCK_CONST_METHOD0(isOpenAppsecAgent, bool());
// OpenSSL
MOCK_METHOD1(setOpenSSLDir, void(const std::string&));

1
core/include/services_sdk/resources/agent_details.h
View File

@ -73,6 +73,7 @@ public:
Maybe<std::string> getOpenSSLDir() const;
std::string getClusterId() const;
OrchestrationMode getOrchestrationMode() const;
bool isOpenAppsecAgent() const;
std::string getAccessToken() const;
void loadAccessToken();

8
core/include/services_sdk/resources/component_is/components_list_impl.h
View File

@ -165,6 +165,14 @@ public:
}
registerGlobalValue<std::string>("Executable Name", arg_vec.front());
auto file_path_end = arg_vec.front().find_last_of("/");
auto executable_name = arg_vec.front().substr(file_path_end + 1);
auto file_sufix_start = executable_name.find_first_of(".");
if (file_sufix_start != std::string::npos) {
executable_name = executable_name.substr(0, file_sufix_start);
}
registerGlobalValue<std::string>("Base Executable Name", executable_name);
}
void

2
core/include/services_sdk/resources/debug_flags.h
View File

@ -76,6 +76,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_WAAP_SCORE_BUILDER, D_WAAP)
DEFINE_FLAG(D_WAAP_ULIMITS, D_WAAP)
DEFINE_FLAG(D_WAAP_SCANNER, D_WAAP)
DEFINE_FLAG(D_WAAP_MODEL_LOGGER, D_WAAP)
DEFINE_FLAG(D_WAAP_DEEP_PARSER, D_WAAP)
DEFINE_FLAG(D_WAAP_BASE64, D_WAAP)
DEFINE_FLAG(D_WAAP_JSON, D_WAAP)
@ -190,6 +191,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_L7_ACCESS_CONTROL, D_COMPONENT)
DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT)
DEFINE_FLAG(D_HORIZON_TELEMETRY, D_COMPONENT)
DEFINE_FLAG(D_PROMETHEUS, D_COMPONENT)
DEFINE_FLAG(D_FLOW, D_ALL)
DEFINE_FLAG(D_DROP, D_FLOW)

4
core/include/services_sdk/resources/generic_metric.h
View File

@ -24,6 +24,7 @@
#include "i_mainloop.h"
#include "i_time_get.h"
#include "i_agent_details.h"
#include "i_instance_awareness.h"
#include "i_environment.h"
#include "i_messaging.h"
#include "i_rest_api.h"
@ -52,6 +53,7 @@ class GenericMetric
Singleton::Consume<I_MainLoop>,
Singleton::Consume<I_TimeGet>,
Singleton::Consume<I_AgentDetails>,
Singleton::Consume<I_InstanceAwareness>,
Singleton::Consume<I_Environment>,
Singleton::Consume<I_Messaging>,
Singleton::Consume<I_RestApi>,
@ -111,7 +113,9 @@ private:
void handleMetricStreamSending();
void generateLog();
void generatePrometheus();
void generateDebug();
void generateAiopsLog();
I_MainLoop *i_mainloop;
I_TimeGet *i_time;

2
core/include/services_sdk/resources/log_generator.h
View File

@ -102,6 +102,8 @@ public:
std::string getLogInsteadOfSending();
void addMarkerSuffix(const std::string &suffix);
private:
std::chrono::microseconds getCurrentTime() const;
void loadBaseLogFields();

6
core/include/services_sdk/resources/metric/average.h
View File

@ -54,6 +54,12 @@ public:
return (count > 0) ? double(sum)/count : 0;
}
float
getValue() const override
{
return static_cast<float>(getAverage());
}
void
save(cereal::JSONOutputArchive &ar) const override
{

6
core/include/services_sdk/resources/metric/counter.h
View File

@ -44,6 +44,12 @@ public:
return counter;
}
float
getValue() const override
{
return static_cast<float>(counter);
}
void
save(cereal::JSONOutputArchive &ar) const override
{

6
core/include/services_sdk/resources/metric/last_reported_value.h
View File

@ -44,6 +44,12 @@ public:
return last_reported;
}
float
getValue() const override
{
return static_cast<float>(last_reported);
}
void
save(cereal::JSONOutputArchive &ar) const override
{

6
core/include/services_sdk/resources/metric/max.h
View File

@ -56,6 +56,12 @@ public:
return max;
}
float
getValue() const override
{
return static_cast<float>(max);
}
void
save(cereal::JSONOutputArchive &ar) const override
{

115
core/include/services_sdk/resources/metric/metric_calc.h
View File

@ -18,21 +18,129 @@
#error metric/metric_calc.h should not be included directly
#endif // __GENERIC_METRIC_H_
#include <cmath>
#include <cereal/archives/json.hpp>
#include "report/report.h"
#include "customized_cereal_map.h"
class GenericMetric;
enum class MetricType { GAUGE, COUNTER };
struct PrometheusData
{
std::string name;
std::string type;
std::string desc;
std::string label;
std::string value;
};
class AiopsMetricData
{
public:
AiopsMetricData(
const std::string &_name,
const std::string &_type,
const std::string &_units,
const std::string &_description,
std::map<std::string, std::string> _resource_attributes,
float _value)
:
name(_name),
type(_type),
units(_units),
description(_description),
resource_attributes(_resource_attributes),
value(_value)
{
timestamp = Singleton::Consume<I_TimeGet>::by<GenericMetric>()->getWalltimeStr();
asset_id = Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId();
}
void
serialize(cereal::JSONOutputArchive &ar) const
{
ar(cereal::make_nvp("Timestamp", timestamp));
ar(cereal::make_nvp("MetricName", name));
ar(cereal::make_nvp("MetricType", type));
ar(cereal::make_nvp("MetricUnit", units));
ar(cereal::make_nvp("MetricDescription", description));
ar(cereal::make_nvp("MetricValue", value));
ar(cereal::make_nvp("ResourceAttributes", resource_attributes));
ar(cereal::make_nvp("MetricAttributes", metric_attributes));
ar(cereal::make_nvp("AssetID", asset_id));
}
std::string
toString() const
{
std::stringstream ss;
{
cereal::JSONOutputArchive ar(ss);
serialize(ar);
}
return ss.str();
}
void
addMetricAttribute(const std::string &label, const std::string &value)
{
metric_attributes[label] = value;
}
private:
std::string timestamp = "";
std::string asset_id = "";
std::string name;
std::string type;
std::string units;
std::string description;
std::map<std::string, std::string> resource_attributes;
std::map<std::string, std::string> metric_attributes;
float value = 0;
};
class AiopsMetricList
{
public:
void
addMetrics(const std::vector<AiopsMetricData> &_metrics)
{
metrics.insert(metrics.end(), _metrics.begin(), _metrics.end());
}
void
serialize(cereal::JSONOutputArchive &ar) const
{
ar(cereal::make_nvp("Metrics", metrics));
}
// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
std::string
toString() const
{
std::stringstream ss;
{
cereal::JSONOutputArchive ar(ss);
serialize(ar);
}
return ss.str();
}
// LCOV_EXCL_STOP
private:
std::vector<AiopsMetricData> metrics;
};
class MetricCalc
{
public:
template<typename ... Args>
MetricCalc(GenericMetric *metric, const std::string &calc_title, const Args & ... args)
{
setMetadata("BaseName", calc_title);
setMetricName(calc_title);
addMetric(metric);
parseMetadata(args ...);
}
@ -47,7 +155,11 @@ public:
std::string getMetircDescription() const { return getMetadata("Description"); }
std::string getMetadata(const std::string &metadata) const;
virtual MetricType getMetricType() const { return MetricType::GAUGE; }
virtual std::vector<PrometheusData> getPrometheusMetrics() const;
virtual float getValue() const = 0;
virtual std::vector<AiopsMetricData> getAiopsMetrics() const;
void setMetricName(const std::string &name) { setMetadata("BaseName", name); }
void setMetricDotName(const std::string &name) { setMetadata("DotName", name); }
void setMetircUnits(const std::string &units) { setMetadata("Units", units); }
void setMetircDescription(const std::string &description) { setMetadata("Description", description); }
@ -55,6 +167,7 @@ public:
protected:
void addMetric(GenericMetric *metric);
std::map<std::string, std::string> getBasicLabels() const;
template <typename Metadata, typename ... OtherMetadata>
void

69
core/include/services_sdk/resources/metric/metric_map.h
View File

@ -54,6 +54,37 @@ class MetricMap : public MetricCalc
return first->second.getMetricType();
}
std::vector<PrometheusData>
getPrometheusMetrics(const std::string &label, const std::string &name) const
{
std::vector<PrometheusData> res;
for (auto &metric : inner_map) {
auto sub_res = metric.second.getPrometheusMetrics();
for (auto &sub_metric : sub_res) {
sub_metric.label += "," + label + "=\"" + metric.first + "\"";
sub_metric.name = name;
}
res.insert(res.end(), sub_res.begin(), sub_res.end());
}
return res;
}
std::vector<AiopsMetricData>
getAiopsMetrics(const std::string &label) const
{
std::vector<AiopsMetricData> aiops_metrics;
for (auto &metric : inner_map) {
auto metric_data = metric.second.getAiopsMetrics();
for (auto &sub_metric : metric_data) {
sub_metric.addMetricAttribute(label, metric.first);
}
aiops_metrics.insert(aiops_metrics.end(), metric_data.begin(), metric_data.end());
}
return aiops_metrics;
}
typename std::map<std::string, Metric>::const_iterator begin() const { return inner_map.begin(); }
typename std::map<std::string, Metric>::const_iterator end() const { return inner_map.end(); }
@ -63,9 +94,17 @@ class MetricMap : public MetricCalc
public:
template <typename ... Args>
MetricMap(GenericMetric *metric, const std::string &title, const Args & ... args)
MetricMap(
const Metric &sub_metric,
GenericMetric *metric,
const std::string &l,
const std::string &title,
const Args & ... args
)
:
MetricCalc(metric, title, args ...)
MetricCalc(metric, title, args ...),
base_metric(sub_metric),
label(l)
{
}
@ -75,6 +114,14 @@ public:
if (getMetricType() == MetricType::GAUGE) metric_map.clear();
}
// LCOV_EXCL_START Reason: Covered by printPromeathusMultiMap unit-test, but misdetected by the coverage
float
getValue() const override
{
return std::nanf("");
}
// LCOV_EXCL_STOP
void
save(cereal::JSONOutputArchive &ar) const override
{
@ -89,7 +136,9 @@ public:
{
std::stringstream string_key;
string_key << key;
auto metric = metric_map.emplace(string_key.str(), Metric(nullptr, string_key.str())).first;
auto new_metric = base_metric;
new_metric.setMetricName(string_key.str());
auto metric = metric_map.emplace(string_key.str(), std::move(new_metric)).first;
metric->second.report(new_values...);
}
@ -105,8 +154,22 @@ public:
return field;
}
std::vector<PrometheusData>
getPrometheusMetrics() const override
{
return metric_map.getPrometheusMetrics(label, getMetricName());
}
std::vector<AiopsMetricData>
getAiopsMetrics() const
{
return metric_map.getAiopsMetrics(label);
}
private:
InnerMap metric_map;
Metric base_metric;
std::string label;
};
} // namespace MetricCalculations

6
core/include/services_sdk/resources/metric/min.h
View File

@ -56,6 +56,12 @@ public:
return min;
}
float
getValue() const override
{
return static_cast<float>(min);
}
void
save(cereal::JSONOutputArchive &ar) const override
{

6
core/include/services_sdk/resources/metric/no_reset_counter.h
View File

@ -41,6 +41,12 @@ public:
return counter;
}
float
getValue() const override
{
return static_cast<float>(counter);
}
void
save(cereal::JSONOutputArchive &ar) const override
{

6
core/include/services_sdk/resources/metric/top_values.h
View File

@ -59,6 +59,12 @@ public:
values.clear();
}
float
getValue() const override
{
return std::nanf("");
}
std::vector<T>
getTopValues() const
{

1
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@ -485,6 +485,7 @@ private:
MessageMetadata req_md(server, *port);
req_md.insertHeaders(getHTTPHeaders());
req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
return sendIntelligenceRequestImpl(rest_req, req_md);
}

9
core/logging/log_generator.cc
View File

@ -123,3 +123,12 @@ LogGen::loadBaseLogFields()
log.getMarkers() = env->getAllStrings(EnvKeyAttr::LogSection::MARKER);
}
// Adding a suffix to log markers will allow for creating a unique log that won't be reduced
void
LogGen::addMarkerSuffix(const string &suffix)
{
for (auto &marker : log.getMarkers()) {
marker.second += suffix;
}
}

11
core/messaging/connection/connection.cc
View File

@ -96,6 +96,8 @@ public:
client_key_path = metadata.getClientKeyPath();
is_dual_auth = true;
}
sni_hostname = metadata.getSniHostName();
}
void
@ -329,6 +331,12 @@ private:
if (!SSL_set1_host(ssl_socket, host)) {
return genError("Failed to set host name verification. Host: " + string(host));
}
if (sni_hostname.ok()) {
host = sni_hostname->c_str();
}
dbgDebug(D_CONNECTION) << "Setting TLS host name extension. Host: " << host;
if (!SSL_set_tlsext_host_name(ssl_socket, host)) {
return genError("Failed to set TLS host name extension. Host: " + string(host));
}
@ -656,7 +664,7 @@ private:
printOut(const string &data)
{
string type = getConfigurationWithDefault<string>("chopped", "message", "Data printout type");
uint length = getConfigurationWithDefault<uint>(10, "message", "Data printout length");
uint length = getConfigurationWithDefault<uint>(50, "message", "Data printout length");
if (type == "full") return data;
if (type == "size") return to_string(data.size()) + " bytes";
if (type == "none") return "";
@ -689,6 +697,7 @@ private:
bool lock = false;
bool should_close_connection = false;
bool is_dual_auth = false;
Maybe<string> sni_hostname = genError<string>("Uninitialized");
};
Connection::Connection(const MessageConnectionKey &key, const MessageMetadata &metadata)

5
core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc
View File

@ -112,9 +112,8 @@ MessagingBufferComponent::Impl::init()
auto sub_path = getProfileAgentSettingWithDefault<string>("nano_agent/event_buffer/", "eventBuffer.baseFolder");
buffer_root_path = getLogFilesPathConfig() + "/" + sub_path;
string full_executable_name =
Singleton::Consume<I_Environment>::by<Messaging>()->get<string>("Executable Name").unpack();
string executable_name = full_executable_name.substr(full_executable_name.find_last_of("/") + 1);
string executable_name =
Singleton::Consume<I_Environment>::by<Messaging>()->get<string>("Base Executable Name").unpack();
removeLegacyBuffer(buffer_root_path, executable_name);
mkdir(buffer_root_path.c_str(), 0644);

2
core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc
View File

@ -26,7 +26,7 @@ public:
TestMessagingBuffer()
{
env.preload();
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "tmp_test_file");
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "tmp_test_file");
config.preload();
config.init();

170
core/metric/generic_metric.cc
View File

@ -17,6 +17,7 @@
#include "debug.h"
#include "report/log_rest.h"
#include "config.h"
#include "report_messaging.h"
#include <fstream>
#include <sstream>
@ -30,6 +31,24 @@ MetricMetadata::DotName operator"" _dot(const char *str, size_t) { return Metric
MetricMetadata::Units operator"" _unit(const char *str, size_t) { return MetricMetadata::Units{str}; }
MetricMetadata::Description operator"" _desc(const char *str, size_t) { return MetricMetadata::Description{str}; }
// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
static ostream & operator<<(ostream &os, const AiopsMetricList &metrics) { return os << metrics.toString(); }
// LCOV_EXCL_STOP
vector<AiopsMetricData>
MetricCalc::getAiopsMetrics() const
{
float value = getValue();
if (isnan(value)) return {};
string name = getMetricDotName() != "" ? getMetricDotName() : getMetricName();
string units = getMetircUnits();
string description = getMetircDescription();
string type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter";
return { AiopsMetricData(name, type, units, description, getBasicLabels(), value) };
}
string
MetricCalc::getMetadata(const string &key) const
{
@ -54,6 +73,54 @@ MetricCalc::addMetric(GenericMetric *metric)
if (metric != nullptr) metric->addCalc(this);
}
vector<PrometheusData>
MetricCalc::getPrometheusMetrics() const
{
float value = getValue();
if (isnan(value)) return {};
PrometheusData res;
res.name = getMetricDotName() != "" ? getMetricDotName() : getMetricName();
res.type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter";
res.desc = getMetircDescription();
stringstream labels;
const auto &label_pairs = getBasicLabels();
bool first = true;
for (auto &pair : label_pairs) {
if (!first) labels << ',';
labels << pair.first << "=\"" << pair.second << '"';
first = false;
}
res.label = labels.str();
stringstream value_str;
value_str << value;
res.value = value_str.str();
return {res};
}
map<string, string>
MetricCalc::getBasicLabels() const
{
map<string, string> res;
auto i_instance = Singleton::Consume<I_InstanceAwareness>::by<GenericMetric>();
auto id = i_instance->getUniqueID();
if (id.ok()) res["id"] = *id;
auto details = Singleton::Consume<I_AgentDetails>::by<GenericMetric>();
res["agent"] = details->getAgentId();
auto env = Singleton::Consume<I_Environment>::by<GenericMetric>();
auto executable = env->get<string>("Base Executable Name");
if (executable.ok()) res["process"] = *executable;
return res;
}
static const string metric_file = "/tmp/metrics_output.txt";
class GenericMetric::MetricsRest : public ServerRest
@ -121,7 +188,9 @@ void
GenericMetric::handleMetricStreamSending()
{
if (active_streams.isSet(Stream::DEBUG)) generateDebug();
if (active_streams.isSet(Stream::PROMETHEUS)) generatePrometheus();
if (active_streams.isSet(Stream::FOG)) generateLog();
if (active_streams.isSet(Stream::AIOPS)) generateAiopsLog();
if (reset) resetMetrics();
}
@ -244,6 +313,106 @@ GenericMetric::generateLog()
sendLog(metric_client_rest);
}
class PrometheusRest : public ClientRest
{
class Metric : public ClientRest
{
public:
Metric(const string &n, const string &t, const string &d, const string &l, const string &v)
:
name(n),
type(t),
description(d),
labels(l),
value(v)
{}
private:
C2S_PARAM(string, name);
C2S_PARAM(string, type);
C2S_PARAM(string, description);
C2S_PARAM(string, labels);
C2S_PARAM(string, value);
};
public:
PrometheusRest() : metrics(vector<Metric>()) {}
void
addMetric(const vector<PrometheusData> &vec)
{
auto &metric_vec = metrics.get();
metric_vec.reserve(vec.size());
for (auto &metric : vec) {
metric_vec.emplace_back(metric.name, metric.type, metric.desc, "{" + metric.label + "}", metric.value);
}
}
private:
C2S_PARAM(vector<Metric>, metrics);
};
void
GenericMetric::generatePrometheus()
{
if (!getProfileAgentSettingWithDefault(false, "prometheus")) return;
vector<PrometheusData> all_metrics;
for (auto &calc : calcs) {
const auto &cal_metrics = calc->getPrometheusMetrics();
all_metrics.insert(all_metrics.end(), cal_metrics.begin(), cal_metrics.end());
}
PrometheusRest rest;
rest.addMetric(all_metrics);
MessageMetadata new_config_req_md("127.0.0.1", 7465);
new_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
Singleton::Consume<I_Messaging>::by<GenericMetric>()->sendSyncMessage(
HTTPMethod::POST,
"/set-prometheus-data",
rest,
MessageCategory::GENERIC,
new_config_req_md
);
}
void
GenericMetric::generateAiopsLog()
{
if (!getConfigurationWithDefault<bool>(true, "metric", "aiopsMetricSendEnable")) return;
AiopsMetricList aiops_metrics;
for (auto &calc : calcs) {
aiops_metrics.addMetrics(calc->getAiopsMetrics());
}
set<ReportIS::Tags> tags;
Report metric_to_fog(
"AIOPS Metric Data",
Singleton::Consume<I_TimeGet>::by<GenericMetric>()->getWalltime(),
Type::PERIODIC,
Level::LOG,
LogLevel::INFO,
audience,
team,
Severity::INFO,
Priority::LOW,
report_interval,
LogField("agentId", Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId()),
tags,
Tags::INFORMATIONAL,
issuing_engine
);
metric_to_fog << LogField("eventObject", aiops_metrics);
LogRest metric_client_rest(metric_to_fog);
sendLog(metric_client_rest);
}
void
GenericMetric::generateDebug()
{
@ -270,6 +439,7 @@ GenericMetric::preload()
{
registerExpectedConfiguration<bool>("metric", "fogMetricSendEnable");
registerExpectedConfiguration<bool>("metric", "debugMetricSendEnable");
registerExpectedConfiguration<bool>("metric", "aiopsMetricSendEnable");
registerExpectedConfiguration<bool>("metric", "fogMetricUri");
registerExpectedConfiguration<string>("metric", "metricsOutputTmpFile");
}

539
core/metric/metric_ut/metric_ut.cc
View File

@ -10,6 +10,7 @@
#include "mock/mock_rest_api.h"
#include "agent_details.h"
#include "mock/mock_messaging.h"
#include "mock/mock_instance_awareness.h"
#include "config.h"
#include "config_component.h"
@ -67,6 +68,14 @@ public:
top_usage.report(event.getCPU());
}
void
setAiopsMetric()
{
turnOnStream(Stream::AIOPS);
max.setMetricDotName("cpu.max");
max.setMetircUnits("percrnt");
}
Max<double> max{this, "cpuMax"};
Min<double> min{this, "cpuMin"};
Average<double> avg{this, "cpuAvg"};
@ -121,13 +130,15 @@ public:
class HttpTransaction : public Event<HttpTransaction>
{
public:
HttpTransaction(const string &_url, uint _bytes) : url(_url), bytes(_bytes) {}
HttpTransaction(const string &_url, const string &m, uint _bytes) : url(_url), method(m), bytes(_bytes) {}
const string & getUrl() const { return url;}
const string & getUrl() const { return url; }
const string & getMethod() const { return method; }
uint getBytes() const { return bytes; }
private:
string url;
string method;
uint bytes;
};
@ -144,9 +155,33 @@ public:
total.report(event.getUrl(), 1);
}
void
setAiopsMetric()
{
turnOnStream(Stream::AIOPS);
}
private:
MetricMap<string, Average<double>> avg{this, "PerUrlAvg"};
MetricMap<string, NoResetCounter> total{this, "TotalRequests"};
MetricMap<string, Average<double>> avg{Average<double>{nullptr, ""}, this, "url", "PerUrlAvg"};
MetricMap<string, NoResetCounter> total{NoResetCounter{nullptr, ""}, this, "url", "TotalRequests"};
};
class UrlMetric2 : public GenericMetric, public Listener<HttpTransaction>
{
public:
void
upon(const HttpTransaction &event) override
{
total.report(event.getUrl(), event.getMethod(), 1);
}
private:
MetricMap<string, MetricMap<string, NoResetCounter>> total{
MetricMap<string, NoResetCounter>{NoResetCounter{nullptr, ""}, nullptr, "method", ""},
this,
"url",
"request.total"
};
};
class MetricTest : public Test
@ -155,6 +190,8 @@ public:
MetricTest()
{
EXPECT_CALL(rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true));
ON_CALL(instance, getUniqueID()).WillByDefault(Return(string("87")));
ON_CALL(instance, getFamilyID()).WillByDefault(Return(string("")));
env.init();
Debug::setNewDefaultStdout(&debug_output);
Debug::setUnitTestFlag(D_METRICS, Debug::DebugLevel::TRACE);
@ -177,6 +214,7 @@ public:
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> timer;
NiceMock<MockInstanceAwareness> instance;
StrictMock<MockRestApi> rest;
::Environment env;
ConfigComponent conf;
@ -255,7 +293,9 @@ TEST_F(MetricTest, basicMetricTest)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 89,\n"
@ -326,7 +366,9 @@ TEST_F(MetricTest, basicMetricTest)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 90,\n"
@ -399,7 +441,9 @@ TEST_F(MetricTest, basicMetricTest)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 100,\n"
@ -485,6 +529,150 @@ TEST_F(MetricTest, printMetricsTest)
GenericMetric::fini();
}
TEST_F(MetricTest, printPromeathus)
{
conf.preload();
stringstream configuration;
configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
EXPECT_TRUE(Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration));
CPUMetric cpu_mt;
cpu_mt.init(
"CPU usage",
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
false
);
cpu_mt.turnOffStream(GenericMetric::Stream::FOG);
cpu_mt.turnOffStream(GenericMetric::Stream::DEBUG);
cpu_mt.turnOnStream(GenericMetric::Stream::PROMETHEUS);
cpu_mt.registerListener();
CPUEvent cpu_event;
cpu_event.setProcessCPU(89);
cpu_event.notify();
string message_body;
EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _))
.WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
routine();
string res =
"{\n"
" \"metrics\": [\n"
" {\n"
" \"name\": \"cpuMax\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"name\": \"cpuMin\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"name\": \"cpuAvg\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"name\": \"cpuCurrent\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"name\": \"cpuCounter\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"name\": \"cpuTotalCounter\",\n"
" \"type\": \"counter\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"1\"\n"
" }\n"
" ]\n"
"}";
EXPECT_EQ(message_body, res);
}
TEST_F(MetricTest, printPromeathusMultiMap)
{
conf.preload();
stringstream configuration;
configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
EXPECT_TRUE(Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration));
UrlMetric2 metric;
metric.init(
"Bytes per URL",
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
true
);
metric.turnOnStream(GenericMetric::Stream::PROMETHEUS);
metric.registerListener();
HttpTransaction("/index.html", "GET", 10).notify();
HttpTransaction("/index2.html", "GET", 20).notify();
HttpTransaction("/index.html", "POST", 40).notify();
string message_body;
EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _))
.WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
routine();
string res =
"{\n"
" \"metrics\": [\n"
" {\n"
" \"name\": \"request.total\",\n"
" \"type\": \"counter\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"name\": \"request.total\",\n"
" \"type\": \"counter\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"name\": \"request.total\",\n"
" \"type\": \"counter\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n"
" \"value\": \"1\"\n"
" }\n"
" ]\n"
"}";
EXPECT_EQ(message_body, res);
}
TEST_F(MetricTest, metricTestWithReset)
{
CPUMetric cpu_mt;
@ -554,7 +742,9 @@ TEST_F(MetricTest, metricTestWithReset)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 89,\n"
@ -624,7 +814,9 @@ TEST_F(MetricTest, metricTestWithReset)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 90,\n"
@ -694,7 +886,9 @@ TEST_F(MetricTest, metricTestWithReset)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 100,\n"
@ -796,7 +990,9 @@ TEST_F(MetricTest, generateReportWithReset)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 89,\n"
@ -888,7 +1084,9 @@ TEST_F(MetricTest, generateReportWithReset)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 90,\n"
@ -959,7 +1157,9 @@ TEST_F(MetricTest, generateReportWithReset)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"My named nano service\"\n"
" \"serviceName\": \"My named nano service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 100,\n"
@ -1015,6 +1215,7 @@ TEST_F(MetricTest, allMetricTest)
seconds(5),
false
);
msg_size_mt.registerListener();
EXPECT_EQ(msg_size_mt.getMetricName(), "Message size");
@ -1074,9 +1275,9 @@ TEST_F(MetricTest, testMapMetric)
);
url_mt.registerListener();
HttpTransaction("/index.html", 10).notify();
HttpTransaction("/index2.html", 20).notify();
HttpTransaction("/index.html", 40).notify();
HttpTransaction("/index.html", "GET", 10).notify();
HttpTransaction("/index2.html", "GET", 20).notify();
HttpTransaction("/index.html", "POST", 40).notify();
string message_body;
@ -1191,3 +1392,309 @@ TEST_F(MetricTest, testManyValuesOutOfOrder)
EXPECT_THAT(AllMetricEvent().query(), ElementsAre(HasSubstr(cpu_str)));
}
TEST_F(MetricTest, basicAIOPSMetricTest)
{
EXPECT_CALL(timer, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087")));
CPUMetric cpu_mt;
cpu_mt.init(
"CPU usage",
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
false
);
cpu_mt.setAiopsMetric();
cpu_mt.registerListener();
EXPECT_EQ(cpu_mt.getMetricName(), "CPU usage");
EXPECT_EQ(cpu_mt.getReportInterval().count(), 5);
routine();
CPUEvent cpu_event;
cpu_event.setProcessCPU(89);
cpu_event.notify();
string metric_str =
"{\n"
" \"Metric\": \"CPU usage\",\n"
" \"Reporting interval\": 5,\n"
" \"cpuMax\": 89.0,\n"
" \"cpuMin\": 89.0,\n"
" \"cpuAvg\": 89.0,\n"
" \"cpuCurrent\": 89.0,\n"
" \"cpuCounter\": 1,\n"
" \"cpuTotalCounter\": 1,\n"
" \"cpuTops\": [\n"
" 89.0\n"
" ]\n"
"}";
string message_body;
EXPECT_CALL(messaging_mock, sendAsyncMessage(
_,
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
string expected_message =
"{\n"
" \"log\": {\n"
" \"eventTime\": \"2016-11-13T17:31:24.087\",\n"
" \"eventName\": \"AIOPS Metric Data\",\n"
" \"eventSeverity\": \"Info\",\n"
" \"eventPriority\": \"Low\",\n"
" \"eventType\": \"Periodic\",\n"
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"Agent Core\",\n"
" \"eventFrequency\": 5,\n"
" \"eventTags\": [\n"
" \"Informational\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"issuingEngine\": \"Agent Core\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"eventObject\": {\n"
" \"Metrics\": [\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpu.max\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"percrnt\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 89.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuMin\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 89.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuAvg\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 89.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuCurrent\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 89.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuCounter\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 1.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuTotalCounter\",\n"
" \"MetricType\": \"counter\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 1.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" }\n"
" ]\n"
" }\n"
" }\n"
" }\n"
"}";
routine();
EXPECT_THAT(debug_output.str(), HasSubstr(metric_str));
EXPECT_EQ(message_body, expected_message);
debug_output.str("");
}
TEST_F(MetricTest, testAIOPSMapMetric)
{
EXPECT_CALL(timer, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087")));
UrlMetric url_mt;
url_mt.init(
"Bytes per URL",
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
true
);
url_mt.registerListener();
url_mt.setAiopsMetric();
HttpTransaction("/index.html", "GET", 10).notify();
HttpTransaction("/index2.html", "GET", 20).notify();
HttpTransaction("/index.html", "POST", 40).notify();
string message_body;
EXPECT_CALL(messaging_mock, sendAsyncMessage(
_,
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
routine();
string expected_message =
"{\n"
" \"log\": {\n"
" \"eventTime\": \"2016-11-13T17:31:24.087\",\n"
" \"eventName\": \"AIOPS Metric Data\",\n"
" \"eventSeverity\": \"Info\",\n"
" \"eventPriority\": \"Low\",\n"
" \"eventType\": \"Periodic\",\n"
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"Agent Core\",\n"
" \"eventFrequency\": 5,\n"
" \"eventTags\": [\n"
" \"Informational\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"issuingEngine\": \"Agent Core\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"eventObject\": {\n"
" \"Metrics\": [\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"/index.html\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 25.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {\n"
" \"url\": \"/index.html\"\n"
" },\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"/index2.html\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 20.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {\n"
" \"url\": \"/index2.html\"\n"
" },\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"/index.html\",\n"
" \"MetricType\": \"counter\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 2.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {\n"
" \"url\": \"/index.html\"\n"
" },\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"/index2.html\",\n"
" \"MetricType\": \"counter\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 1.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {\n"
" \"url\": \"/index2.html\"\n"
" },\n"
" \"AssetID\": \"Unknown\"\n"
" }\n"
" ]\n"
" }\n"
" }\n"
" }\n"
"}";
EXPECT_EQ(message_body, expected_message);
}

15
core/rest/rest_conn.cc
View File

@ -23,11 +23,12 @@ using namespace std;
USE_DEBUG_FLAG(D_API);
RestConn::RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke)
RestConn::RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke, bool is_external)
:
fd(_fd),
mainloop(_mainloop),
invoke(_invoke)
fd(_fd),
mainloop(_mainloop),
invoke(_invoke),
is_external_ip(is_external)
{}
RestConn::~RestConn()
@ -101,6 +102,12 @@ RestConn::parseConn() const
return sendResponse("200 OK", invoke->invokeGet(identifier), false);
}
if (is_external_ip) {
dbgWarning(D_API) << "External IP tried to POST";
sendResponse("500 Internal Server Error", "", false);
stop();
}
stringstream body;
body.str(readSize(len));

3
core/rest/rest_conn.h
View File

@ -21,7 +21,7 @@
class RestConn
{
public:
RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke);
RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke, bool is_external = false);
~RestConn();
void parseConn() const;
@ -35,6 +35,7 @@ private:
int fd;
I_MainLoop *mainloop;
const I_RestInvoke *invoke;
bool is_external_ip = false;
};
#endif // __REST_CONN_H__

109
core/rest/rest_server.cc
View File

@ -47,6 +47,7 @@ public:
void startNewConnection() const;
bool bindRestServerSocket(struct sockaddr_in &addr, vector<uint16_t> port_range);
bool bindRestServerSocket(struct sockaddr_in6 &addr, vector<uint16_t> port_range);
bool addRestCall(RestAction oper, const string &uri, unique_ptr<RestInit> &&init) override;
bool addGetCall(const string &uri, const function<string()> &cb) override;
uint16_t getListeningPort() const override { return listening_port; }
@ -73,10 +74,36 @@ private:
bool
RestServer::Impl::bindRestServerSocket(struct sockaddr_in &addr, vector<uint16_t> port_range)
{
dbgFlow(D_API) << "Binding IPv4 socket";
for (uint16_t port : port_range) {
addr.sin_port = htons(port);
if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) == 0) return true;
if (errno == EADDRINUSE) {
dbgDebug(D_API) << "Port " << port << " is already in use";
} else {
dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno);
}
}
return false;
}
bool
RestServer::Impl::bindRestServerSocket(struct sockaddr_in6 &addr, vector<uint16_t> port_range)
{
dbgFlow(D_API) << "Binding IPv6 socket";
for (uint16_t port : port_range) {
addr.sin6_port = htons(port);
if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in6)) == 0) return true;
if (errno == EADDRINUSE) {
dbgDebug(D_API) << "Port " << port << " is already in use";
} else {
dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno);
}
}
return false;
@ -119,21 +146,55 @@ RestServer::Impl::init()
mainloop = Singleton::Consume<I_MainLoop>::by<RestServer>();
auto init_connection = [this] () {
fd = socket(AF_INET, SOCK_STREAM, 0);
auto allow_external_conn = "Nano service API Allow Get From External IP";
auto conf_value = getConfiguration<bool>("connection", allow_external_conn);
bool accept_get_from_external_ip = false;
if (conf_value.ok()) {
accept_get_from_external_ip = *conf_value;
} else {
auto env_value = Singleton::Consume<I_Environment>::by<RestServer>()->get<bool>(allow_external_conn);
if (env_value.ok()) {
accept_get_from_external_ip = *env_value;
}
}
if (accept_get_from_external_ip) {
fd = socket(AF_INET6, SOCK_STREAM, 0);
} else {
fd = socket(AF_INET, SOCK_STREAM, 0);
}
dbgAssert(fd >= 0) << alert << "Failed to open a socket";
int socket_enable = 1;
if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)) < 0) {
dbgWarning(D_API) << "Could not set the socket options";
}
struct sockaddr_in addr;
bzero(&addr, sizeof(addr));
if (accept_get_from_external_ip) {
int option = 0;
if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &option, sizeof(option)) < 0) {
dbgWarning(D_API) << "Could not set the IPV6_V6ONLY option";
}
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
struct sockaddr_in6 addr6;
bzero(&addr6, sizeof(addr6));
addr6.sin6_family = AF_INET6;
addr6.sin6_addr = in6addr_any;
while (!bindRestServerSocket(addr, port_range)) {
mainloop->yield(bind_retry_interval_msec);
while (!bindRestServerSocket(addr6, port_range)) {
mainloop->yield(bind_retry_interval_msec);
}
listening_port = ntohs(addr6.sin6_port);
} else {
struct sockaddr_in addr;
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
while (!bindRestServerSocket(addr, port_range)) {
mainloop->yield(bind_retry_interval_msec);
}
listening_port = ntohs(addr.sin_port);
}
listen(fd, listen_limit);
@ -146,9 +207,12 @@ RestServer::Impl::init()
"REST server listener",
is_primary.ok() && *is_primary
);
listening_port = ntohs(addr.sin_port);
dbgInfo(D_API) << "REST server started: " << listening_port;
dbgInfo(D_API)
<< "REST server started: "
<< listening_port
<< ". Accepting: "
<< (accept_get_from_external_ip ? "external" : "loopback")
<< " connections";
Singleton::Consume<I_Environment>::by<RestServer>()->registerValue<int>("Listening Port", listening_port);
};
@ -172,14 +236,30 @@ void
RestServer::Impl::startNewConnection() const
{
dbgFlow(D_API) << "Starting a new connection";
int new_socket = accept(fd, nullptr, nullptr);
struct sockaddr_storage addr;
socklen_t addr_len = sizeof(addr);
int new_socket = accept(fd, (struct sockaddr *)&addr, &addr_len);
if (new_socket < 0) {
dbgWarning(D_API) << "Failed to accept a new socket";
dbgWarning(D_API) << "Failed to accept a new socket: " << strerror(errno);
return;
}
dbgDebug(D_API) << "Starting a new socket: " << new_socket;
RestConn conn(new_socket, mainloop, this);
dbgDebug(D_API) << "Starting a new socket: " << new_socket;
bool is_external = false;
if (addr.ss_family == AF_INET6) {
struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *) &addr;
if (IN6_IS_ADDR_V4MAPPED(&addr_in6->sin6_addr)) {
struct in_addr ipv4_addr;
memcpy(&ipv4_addr, &addr_in6->sin6_addr.s6_addr[12], sizeof(ipv4_addr));
is_external = ipv4_addr.s_addr != htonl(INADDR_LOOPBACK);
} else {
is_external = memcmp(&addr_in6->sin6_addr, &in6addr_loopback, sizeof(in6addr_loopback)) != 0;
}
} else {
struct sockaddr_in *addr_in = (struct sockaddr_in *)&addr;
is_external = addr_in->sin_addr.s_addr != htonl(INADDR_LOOPBACK);
}
RestConn conn(new_socket, mainloop, this, is_external);
mainloop->addFileRoutine(
I_MainLoop::RoutineType::Offline,
new_socket,
@ -283,4 +363,5 @@ RestServer::preload()
registerExpectedConfiguration<uint>("connection", "Nano service API Port Alternative");
registerExpectedConfiguration<uint>("connection", "Nano service API Port Range start");
registerExpectedConfiguration<uint>("connection", "Nano service API Port Range end");
registerExpectedConfiguration<bool>("connection", "Nano service API Allow Get From External IP");
}

224
core/rest/rest_ut/rest_config_ut.cc
View File

@ -14,10 +14,49 @@
#include "agent_details.h"
#include "mock/mock_messaging.h"
#include "tenant_manager.h"
#include <netdb.h>
#include <arpa/inet.h>
using namespace std;
using namespace testing;
static const string config_json_allow_external =
"{\n"
" \"connection\": {\n"
" \"Nano service API Port Primary\": [\n"
" {\n"
" \"value\": 9777\n"
" }\n"
" ],\n"
" \"Nano service API Port Alternative\": [\n"
" {\n"
" \"value\": 9778\n"
" }\n"
" ],\n"
" \"Nano service API Allow Get From External IP\": [\n"
" {\n"
" \"value\": true\n"
" }\n"
" ]\n"
" }\n"
"}\n";
static const string config_json =
"{\n"
" \"connection\": {\n"
" \"Nano service API Port Primary\": [\n"
" {\n"
" \"value\": 9777\n"
" }\n"
" ],\n"
" \"Nano service API Port Alternative\": [\n"
" {\n"
" \"value\": 9778\n"
" }\n"
" ]\n"
" }\n"
"}\n";
USE_DEBUG_FLAG(D_API);
USE_DEBUG_FLAG(D_MAINLOOP);
@ -31,22 +70,6 @@ public:
time_proxy.init();
mainloop_comp.init();
string config_json =
"{\n"
" \"connection\": {\n"
" \"Nano service API Port Primary\": [\n"
" {\n"
" \"value\": 9777\n"
" }\n"
" ],\n"
" \"Nano service API Port Alternative\": [\n"
" {\n"
" \"value\": 9778\n"
" }\n"
" ]\n"
" }\n"
"}\n";
istringstream ss(config_json);
Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss);
@ -58,6 +81,9 @@ public:
~RestConfigTest()
{
Debug::setNewDefaultStdout(&cout);
auto mainloop = Singleton::Consume<I_MainLoop>::from(mainloop_comp);
mainloop->stopAll();
rest_server.fini();
time_proxy.fini();
mainloop_comp.fini();
}
@ -133,7 +159,7 @@ int TestServer::g_num = 0;
TEST_F(RestConfigTest, basic_flow)
{
env.preload();
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "tmp_test_file");
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "tmp_test_file");
config.preload();
config.init();
@ -165,11 +191,15 @@ TEST_F(RestConfigTest, basic_flow)
auto mainloop = Singleton::Consume<I_MainLoop>::from(mainloop_comp);
I_MainLoop::Routine stop_routine = [&] () {
EXPECT_EQ(connect(file_descriptor1, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0);
EXPECT_EQ(connect(file_descriptor1, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0)
<< "file_descriptor1 Error: "
<< strerror(errno);
string msg1 = "GET /stuff HTTP/1.1\r\n\r\n";
EXPECT_EQ(write(file_descriptor1, msg1.data(), msg1.size()), static_cast<int>(msg1.size()));
EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0);
EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0)
<< "file_descriptor2 Error: "
<< strerror(errno);
string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}";
EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast<int>(msg2.size()));
@ -204,3 +234,159 @@ TEST_F(RestConfigTest, basic_flow)
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 6\r\n\r\nblabla"
);
}
string
getLocalIPAddress() {
char hostname[1024];
hostname[1024 - 1] = '\0';
// Get the hostname
if (gethostname(hostname, sizeof(hostname)) == -1) {
return "";
}
struct addrinfo hints, *info, *p;
int gai_result;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_INET; // Use AF_INET for IPv4
hints.ai_socktype = SOCK_STREAM;
// Get the address info
if ((gai_result = getaddrinfo(hostname, nullptr, &hints, &info)) != 0) {
return "";
}
std::string ip_address;
for (p = info; p != nullptr; p = p->ai_next) {
void *addr;
char ipstr[INET_ADDRSTRLEN];
struct sockaddr_in *ipv4 = (struct sockaddr_in *)p->ai_addr;
addr = &(ipv4->sin_addr);
// Convert the IP to a string and print it
inet_ntop(p->ai_family, addr, ipstr, sizeof(ipstr));
if (std::string(ipstr) != "127.0.0.1") {
ip_address = ipstr;
break;
}
}
freeaddrinfo(info); // Free the linked list
return ip_address;
}
TEST_F(RestConfigTest, not_loopback_flow)
{
env.preload();
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "tmp_test_file");
istringstream ss(config_json_allow_external);
Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss);
config.preload();
config.init();
rest_server.init();
time_proxy.init();
mainloop_comp.init();
auto i_rest = Singleton::Consume<I_RestApi>::from(rest_server);
ASSERT_TRUE(i_rest->addRestCall<TestServer>(RestAction::ADD, "test"));
ASSERT_TRUE(i_rest->addGetCall("stuff", [] () { return string("blabla"); }));
int file_descriptor1 = socket(AF_INET, SOCK_STREAM, 0);
EXPECT_NE(file_descriptor1, -1);
int file_descriptor2 = socket(AF_INET, SOCK_STREAM, 0);
EXPECT_NE(file_descriptor2, -1);
auto primary_port = getConfiguration<uint>("connection", "Nano service API Port Primary");
auto second_port = getConfiguration<uint>("connection", "Nano service API Port Alternative");
auto local_ip = getLocalIPAddress();
struct sockaddr_in sa_primary;
sa_primary.sin_family = AF_INET;
sa_primary.sin_port = htons(primary_port.unpack());
sa_primary.sin_addr.s_addr = inet_addr(local_ip.c_str());
struct sockaddr_in sa_second;
sa_second.sin_family = AF_INET;
sa_second.sin_port = htons(second_port.unpack());
sa_second.sin_addr.s_addr = inet_addr(local_ip.c_str());
int socket_enable = 1;
EXPECT_EQ(setsockopt(file_descriptor1, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0);
EXPECT_EQ(setsockopt(file_descriptor2, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0);
EXPECT_CALL(messaging, sendSyncMessage(_, _, _, _, _))
.WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")));
Debug::setNewDefaultStdout(&cout);
auto mainloop = Singleton::Consume<I_MainLoop>::from(mainloop_comp);
Debug::setNewDefaultStdout(&cout);
I_MainLoop::Routine stop_routine = [&] () {
int socket_client_2 = -1;
auto socket_client_1 = connect(file_descriptor1, (struct sockaddr*)&sa_primary, sizeof(struct sockaddr));
dbgDebug(D_API) << "socket_client_1: " << socket_client_1;
if (socket_client_1 == -1) {
dbgDebug(D_API) << "Error: " << strerror(errno);
socket_client_2 = connect(file_descriptor1, (struct sockaddr*)&sa_second, sizeof(struct sockaddr));
dbgDebug(D_API) << "socket_client_2: " << socket_client_2;
if (socket_client_2 == -1) {
dbgDebug(D_API) << "Error: " << strerror(errno) << endl;
} else {
EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa_second, sizeof(struct sockaddr)), 0);
string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}";
EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast<int>(msg2.size()));
}
} else {
EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa_primary, sizeof(struct sockaddr)), 0);
string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}";
EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast<int>(msg2.size()));
}
EXPECT_TRUE(socket_client_1 != -1 || socket_client_2 != -1);
string msg1 = "GET /stuff HTTP/1.1\r\n\r\n";
EXPECT_EQ(write(file_descriptor1, msg1.data(), msg1.size()), static_cast<int>(msg1.size()));
mainloop->yield(true);
struct pollfd s_poll;
s_poll.fd = file_descriptor1;
s_poll.events = POLLIN;
s_poll.revents = 0;
while(poll(&s_poll, 1, 0) <= 0) {
mainloop->yield(true);
}
struct pollfd s_poll2;
s_poll2.fd = file_descriptor2;
s_poll2.events = POLLIN;
s_poll2.revents = 0;
while(poll(&s_poll2, 1, 0) <= 0) {
mainloop->yield(true);
}
mainloop->stopAll();
};
mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::RealTime,
stop_routine,
"RestConfigTest-alternative_port_used stop routine",
true
);
mainloop->run();
char respose[1000];
EXPECT_EQ(read(file_descriptor1, respose, 1000), 76);
EXPECT_EQ(
string(respose, 76),
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 6\r\n\r\nblabla"
);
EXPECT_EQ(read(file_descriptor2, respose, 1000), 89);
EXPECT_EQ(
string(respose, 89),
"HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\nContent-Length: 0\r\n\r\n"
);
}

2
core/rest/rest_ut/rest_schema_ut.cc
View File

@ -437,7 +437,7 @@ TEST(RestSchema, server_schema)
env.preload();
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "tmp_test_file");
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "tmp_test_file");
config.preload();
config.init();

2
core/version/version_ut/CMakeLists.txt
View File

@ -1,5 +1,5 @@
add_unit_test(
version_ut
"version_ut.cc"
"version;rest;singleton;environment;event_is;metric;"
"version;rest;singleton;environment;event_is;metric;messaging"
)

8
nodes/orchestration/package/open-appsec-ctl.sh
View File

@ -1452,12 +1452,12 @@ upload_ai() # Initials - uai
if [ "$AI_VERBOSE" = "true" ]; then
echo "Uploading file $file"
fi
if [ -z "${is_gaia}" -o "$is_smb_release" = "1" ]; then
uai_curl_output=$(${curl_cmd} -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1)
if [ -n "${is_gaia}" -o "$is_smb_release" = "1" ]; then
uai_curl_output=$(${curl_cmd} -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1 | tail -1)
elif [ "${remove_curl_ld_path}" = "true" ]; then
uai_curl_output=$(LD_LIBRARY_PATH="" ${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1)
uai_curl_output=$(LD_LIBRARY_PATH="" ${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1 | tail -1)
else
uai_curl_output=$(${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1)
uai_curl_output=$(${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1 | tail -1)
fi
if [ "$AI_UPLOAD_TOO_LARGE_FLAG" = "false" ] && [ "$uai_curl_output" = "413" ]; then
AI_UPLOAD_TOO_LARGE_FLAG=true