github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-06-28 16:41:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #117 from openappsec/Mar_21_2024-Dev

Browse Source 
Mar 21 2024 dev
This commit is contained in:
WrightNed 2024-03-26 13:53:49 +02:00 committed by GitHub
commit 262b2e59ff
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
102 changed files with 3853 additions and 454 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build_system/docker/entry.sh
View File

@ -67,7 +67,7 @@ fi
/nano-service-installers/$CACHE_INSTALLATION_SCRIPT --install
/nano-service-installers/$HTTP_TRANSACTION_HANDLER_SERVICE --install
if [ ! -z $CROWDSEC_ENABLED ]; then
if [ "$CROWDSEC_ENABLED" == "true" ]; then
/nano-service-installers/$INTELLIGENCE_INSTALLATION_SCRIPT --install
/nano-service-installers/$CROWDSEC_INSTALLATION_SCRIPT --install
fi

1
components/CMakeLists.txt
View File

@ -1,6 +1,5 @@
add_subdirectory(report_messaging)
add_subdirectory(http_manager)
add_subdirectory(generic_rulebase)
add_subdirectory(signal_handler)
add_subdirectory(gradual_deployment)
add_subdirectory(packet)

3
components/attachment-intakers/nginx_attachment/nginx_attachment.cc
View File

@ -1136,8 +1136,7 @@ private:
NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
uuid = opaque.getSessionUUID();
}
web_response_data.uuid_size =
string("Incident Id: ").length() + uuid.size();
web_response_data.uuid_size = uuid.size();
if (web_trigger_conf.getDetailsLevel() == "Redirect") {
web_response_data.response_data.redirect_data.redirect_location_size =

4
components/include/i_generic_rulebase.h
View File

@ -17,6 +17,7 @@
#include <vector>
#include "generic_rulebase/parameters_config.h"
#include "generic_rulebase/triggers_config.h"
#include "generic_rulebase/zone.h"
#include "config.h"
@ -26,6 +27,9 @@ public:
virtual Maybe<Zone, Config::Errors> getLocalZone() const = 0;
virtual Maybe<Zone, Config::Errors> getOtherZone() const = 0;
virtual LogTriggerConf getLogTriggerConf(const std::string &trigger_Id) const = 0;
virtual ParameterException getParameterException(const std::string &parameter_Id) const = 0;
using ParameterKeyValues = std::unordered_map<std::string, std::set<std::string>>;
virtual std::set<ParameterBehavior> getBehavior(const ParameterKeyValues &key_value_pairs) const = 0;

2
components/include/i_waap_telemetry.h
View File

@ -25,6 +25,7 @@ struct DecisionTelemetryData
std::string source;
TrafficMethod method;
int responseCode;
uint64_t elapsedTime;
std::set<std::string> attackTypes;
DecisionTelemetryData() :
@ -36,6 +37,7 @@ struct DecisionTelemetryData
source(),
method(POST),
responseCode(0),
elapsedTime(0),
attackTypes()
{
}

3
components/include/report_messaging.h
View File

@ -141,9 +141,12 @@ public:
ReportMessaging & operator<<(const LogField &field);
void setForceBuffering(bool _force_buffering);
private:
Report report;
bool is_async_message;
bool force_buffering = false;
MessageCategory message_type_tag;
};

1
components/include/telemetry.h
View File

@ -91,6 +91,7 @@ private:
MetricCalculations::Counter response_2xx{this, "reservedNgenG"};
MetricCalculations::Counter response_4xx{this, "reservedNgenH"};
MetricCalculations::Counter response_5xx{this, "reservedNgenI"};
MetricCalculations::Average<uint64_t> average_latency{this, "reservedNgenJ"};
};
class WaapAttackTypesMetrics : public WaapTelemetryBase

10
components/report_messaging/report_messaging.cc
View File

@ -33,7 +33,9 @@ ReportMessaging::~ReportMessaging()
HTTPMethod::POST,
url,
log_rest,
message_type_tag
message_type_tag,
MessageMetadata(),
force_buffering
);
} catch (...) {}
}
@ -44,3 +46,9 @@ ReportMessaging::operator<<(const LogField &field)
report << field;
return *this;
}
void
ReportMessaging::setForceBuffering(bool _force_buffering)
{
force_buffering = _force_buffering;
}

47
components/report_messaging/report_messaging_ut/report_messaging_ut.cc
View File

@ -99,12 +99,55 @@ TEST_F(ReportMessagingTest, title_only)
" }\n"
"}",
_,
_,
_
)
).Times(1);
ReportMessaging("test", ReportIS::AudienceTeam::AGENT_CORE, 1, true, ReportIS::Tags::ACCESS_CONTROL);
}
TEST_F(ReportMessagingTest, with_buffering)
{
EXPECT_CALL(
mock_messaging,
sendAsyncMessage(
_,
_,
"{\n"
" \"log\": {\n"
" \"eventTime\": \"Best Time ever\",\n"
" \"eventName\": \"test\",\n"
" \"eventSeverity\": \"Info\",\n"
" \"eventPriority\": \"Low\",\n"
" \"eventType\": \"Event Driven\",\n"
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"Agent Core\",\n"
" \"eventFrequency\": 0,\n"
" \"eventTags\": [\n"
" \"Access Control\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" },\n"
" \"eventData\": {\n"
" \"eventObject\": 1\n"
" }\n"
" }\n"
"}",
_,
_,
true
)
).Times(1);
ReportMessaging report("test", ReportIS::AudienceTeam::AGENT_CORE, 1, true, ReportIS::Tags::ACCESS_CONTROL);
report.setForceBuffering(true);
}
TEST_F(ReportMessagingTest, with_dynamic_fields)
{
EXPECT_CALL(
@ -140,6 +183,7 @@ TEST_F(ReportMessagingTest, with_dynamic_fields)
" }\n"
"}",
_,
_,
_
)
).Times(1);
@ -189,6 +233,7 @@ TEST_F(ReportMessagingTest, custom_event_object)
" }\n"
"}",
_,
_,
_
)
).Times(1);
@ -243,6 +288,7 @@ TEST_F(ReportMessagingTest, custom_priority)
" }\n"
"}",
_,
_,
_
)
).Times(1);
@ -309,6 +355,7 @@ TEST_F(ReportMessagingTest, with_env_details)
" }\n"
"}",
_,
_,
_
)
).Times(1);

1
components/security_apps/layer_7_access_control/layer_7_access_control.cc
View File

@ -37,6 +37,7 @@ public:
if (!ipv4_addresses.empty()) ipv4_address = ipv4_addresses.front();
} catch (const cereal::Exception &e) {
dbgWarning(D_L7_ACCESS_CONTROL) << "Failed to load IP reputation data JSON. Error: " << e.what();
ar.setNextName(nullptr);
}
}

36
components/security_apps/local_policy_mgmt_gen/access_control_practice.cc
View File

@ -12,20 +12,34 @@
// limitations under the License.
#include "access_control_practice.h"
#include "new_practice.h"
using namespace std;
USE_DEBUG_FLAG(D_LOCAL_POLICY);
// LCOV_EXCL_START Reason: no test exist
static const map<string, string> valid_modes_to_key = {
static const set<string> valid_modes = {
"prevent",
"detect",
"inactive",
"prevent-learn",
"detect-learn",
"as-top-level",
"inherited"
};
static const unordered_map<string, string> valid_modes_to_key = {
{"prevent", "Active"},
{"prevent-learn", "Active"},
{"detect", "Detect"},
{"detect-learn", "Detect"},
{"inactive", "Inactive"}
};
static const set<string> valid_units = {"minute", "second"};
static const std::unordered_map<std::string, std::string> key_to_units_val = {
static const unordered_map<std::string, std::string> key_to_units_val = {
{ "second", "Second"},
{ "minute", "Minute"}
};
@ -177,13 +191,10 @@ void
AccessControlRateLimit::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading Access control rate limit";
string in_mode;
parseAppsecJSONKey<string>("overrideMode", in_mode, archive_in, "detect");
if (valid_modes_to_key.find(in_mode) == valid_modes_to_key.end()) {
dbgWarning(D_LOCAL_POLICY) << "AppSec access control rate limit override mode invalid: " << in_mode;
throw PolicyGenException("AppSec access control rate limit override mode invalid: " + in_mode);
} else {
mode = valid_modes_to_key.at(in_mode);
parseMandatoryAppsecJSONKey<string>("overrideMode", mode, archive_in, "inactive");
if (valid_modes.find(mode) == valid_modes.end()) {
dbgWarning(D_LOCAL_POLICY) << "AppSec access control rate limit override mode invalid: " << mode;
throw PolicyGenException("AppSec access control rate limit override mode invalid: " + mode);
}
parseAppsecJSONKey<std::vector<AccessControlRateLimiteRules>>("rules", rules, archive_in);
}
@ -205,9 +216,10 @@ AccessControlRateLimit::getRules() const
}
const string &
AccessControlRateLimit::getMode() const
AccessControlRateLimit::getMode(const std::string &default_mode) const
{
return mode;
const string &res = getModeWithDefault(mode, default_mode, valid_modes_to_key);
return res;
}
void
@ -227,7 +239,7 @@ AccessControlPracticeSpec::setName(const string &_name)
}
const AccessControlRateLimit &
AccessControlPracticeSpec::geRateLimit() const
AccessControlPracticeSpec::getRateLimit() const
{
return rate_limit;
}

21
components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc
View File

@ -133,7 +133,7 @@ AppSecPracticeWebAttacks::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec";
parseAppsecJSONKey<AppSecWebAttackProtections>("protections", protections, archive_in);
parseAppsecJSONKey<string>("override-mode", mode, archive_in, "Unset");
parseAppsecJSONKey<string>("override-mode", mode, archive_in, "as-top-level");
if (valid_modes.count(mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec practice override mode invalid: " << mode;
}
@ -187,7 +187,7 @@ AppSecPracticeWebAttacks::getMinimumConfidence() const
const string &
AppSecPracticeWebAttacks::getMode(const string &default_mode) const
{
if (mode == "Unset" || (key_to_practices_val2.find(mode) == key_to_practices_val2.end())) {
if (isModeInherited(mode) || (key_to_practices_val2.find(mode) == key_to_practices_val2.end())) {
dbgError(D_LOCAL_POLICY) << "Couldn't find a value for key: " << mode << ". Returning " << default_mode;
return default_mode;
}
@ -429,6 +429,9 @@ WebAppSection::WebAppSection(
context(_context),
web_attack_mitigation_severity(parsed_appsec_spec.getWebAttacks().getMinimumConfidence()),
web_attack_mitigation_mode(parsed_appsec_spec.getWebAttacks().getMode(default_mode)),
csrf_protection_mode("Disabled"),
open_redirect_mode("Disabled"),
error_disclosure_mode("Disabled"),
practice_advanced_config(parsed_appsec_spec),
anti_bots(parsed_appsec_spec.getAntiBot()),
trusted_sources({ parsed_trusted_sources })
@ -451,6 +454,7 @@ WebAppSection::WebAppSection(
}
}
// Used for V1Beta2
WebAppSection::WebAppSection(
const string &_application_urls,
const string &_asset_id,
@ -465,7 +469,8 @@ WebAppSection::WebAppSection(
const PracticeAdvancedConfig &_practice_advanced_config,
const AppsecPracticeAntiBotSection &_anti_bots,
const LogTriggerSection &parsed_log_trigger,
const AppSecTrustedSources &parsed_trusted_sources)
const AppSecTrustedSources &parsed_trusted_sources,
const NewAppSecWebAttackProtections &protections)
:
application_urls(_application_urls),
asset_id(_asset_id),
@ -489,6 +494,10 @@ WebAppSection::WebAppSection(
web_attack_mitigation_severity == "medium" ? "high" :
"Error";
csrf_protection_mode = protections.getCsrfProtectionMode(_web_attack_mitigation_mode);
open_redirect_mode = protections.getOpenRedirectMode(_web_attack_mitigation_mode);
error_disclosure_mode = protections.getErrorDisclosureMode(_web_attack_mitigation_mode);
triggers.push_back(TriggersInWaapSection(parsed_log_trigger));
for (const SourcesIdentifiers &source_ident : parsed_trusted_sources.getSourcesIdentifiers()) {
overrides.push_back(AppSecOverride(source_ident));
@ -510,9 +519,9 @@ WebAppSection::save(cereal::JSONOutputArchive &out_ar) const
cereal::make_nvp("webAttackMitigationAction", web_attack_mitigation_action),
cereal::make_nvp("webAttackMitigationMode", web_attack_mitigation_mode),
cereal::make_nvp("practiceAdvancedConfig", practice_advanced_config),
cereal::make_nvp("csrfProtection", disabled_str),
cereal::make_nvp("openRedirect", disabled_str),
cereal::make_nvp("errorDisclosure", disabled_str),
cereal::make_nvp("csrfProtection", csrf_protection_mode),
cereal::make_nvp("openRedirect", open_redirect_mode),
cereal::make_nvp("errorDisclosure", error_disclosure_mode),
cereal::make_nvp("practiceId", practice_id),
cereal::make_nvp("practiceName", practice_name),
cereal::make_nvp("assetId", asset_id),

4
components/security_apps/local_policy_mgmt_gen/include/access_control_practice.h
View File

@ -165,7 +165,7 @@ public:
void load(cereal::JSONInputArchive &archive_in);
const std::vector<AccessControlRateLimiteRules> & getRules() const;
const std::string & getMode() const;
const std::string & getMode(const std::string &default_mode = "inactive") const;
std::vector<RateLimitRulesSection> createRateLimitRulesSection(const RateLimitRulesTriggerSection &trigger) const;
private:
@ -178,7 +178,7 @@ class AccessControlPracticeSpec
public:
void load(cereal::JSONInputArchive &archive_in);
const AccessControlRateLimit & geRateLimit() const;
const AccessControlRateLimit &getRateLimit() const;
const std::string & getAppSecClassName() const;
const std::string & getName() const;
void setName(const std::string &_name);

7
components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h
View File

@ -278,6 +278,7 @@ public:
const std::vector<InnerException> &parsed_exceptions
);
// used for V1beta2
WebAppSection(
const std::string &_application_urls,
const std::string &_asset_id,
@ -292,7 +293,8 @@ public:
const PracticeAdvancedConfig &_practice_advanced_config,
const AppsecPracticeAntiBotSection &_anti_bots,
const LogTriggerSection &parsed_log_trigger,
const AppSecTrustedSources &parsed_trusted_sources);
const AppSecTrustedSources &parsed_trusted_sources,
const NewAppSecWebAttackProtections &protections);
void save(cereal::JSONOutputArchive &out_ar) const;
@ -310,6 +312,9 @@ private:
std::string web_attack_mitigation_action;
std::string web_attack_mitigation_severity;
std::string web_attack_mitigation_mode;
std::string csrf_protection_mode;
std::string open_redirect_mode;
std::string error_disclosure_mode;
bool web_attack_mitigation;
std::vector<TriggersInWaapSection> triggers;
PracticeAdvancedConfig practice_advanced_config;

3
components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h
View File

@ -97,8 +97,7 @@ parseAppsecJSONKey(
value = default_value;
if (!mandatory) {
dbgDebug(D_LOCAL_POLICY)
<< "Could not parse the required key. Key: \""<< key_name
<< "\", Error: " << e.what();
<< "Could not parse a non-mandatory key: \""<< key_name << "\", Error: " << e.what();
} else {
throw PolicyGenException(
"Could not parse a mandatory key: \"" + key_name + "\", Error: " + std::string(e.what())

29
components/security_apps/local_policy_mgmt_gen/include/new_practice.h
View File

@ -24,6 +24,14 @@
#include "debug.h"
#include "local_policy_common.h"
bool isModeInherited(const std::string &mode);
const std::string &getModeWithDefault(
const std::string &mode,
const std::string &default_mode,
const std::unordered_map<std::string, std::string> &key_to_val
);
class IpsProtectionsRulesSection
{
public:
@ -126,8 +134,8 @@ class NewIntrusionPrevention
public:
void load(cereal::JSONInputArchive &archive_in);
std::vector<IpsProtectionsRulesSection> createIpsRules() const;
const std::string & getMode() const;
std::vector<IpsProtectionsRulesSection> createIpsRules(const std::string &default_mode) const;
const std::string & getMode(const std::string &default_mode = "inactive") const;
private:
std::string override_mode;
@ -273,7 +281,8 @@ public:
const std::string &asset_name,
const std::string &asset_id,
const std::string &practice_name,
const std::string &practice_id
const std::string &practice_id,
const std::string &default_mode
) const;
private:
@ -486,7 +495,7 @@ public:
void load(cereal::JSONInputArchive &archive_in);
void addFile(const std::string &file_name);
const std::string & getOverrideMode() const;
const std::string & getOverrideMode(const std::string &default_mode = "inactive") const;
const std::vector<std::string> & getConfigMap() const;
const std::vector<std::string> & getFiles() const;
bool isTemporary() const;
@ -530,10 +539,10 @@ class NewAppSecWebAttackProtections
public:
void load(cereal::JSONInputArchive &archive_in);
const std::string getCsrfProtectionMode() const;
const std::string & getErrorDisclosureMode() const;
const std::string & getCsrfProtectionMode(const std::string &default_mode = "inactive") const;
const std::string & getErrorDisclosureMode(const std::string &default_mode = "inactive") const;
const std::string & getOpenRedirectMode(const std::string &default_mode = "inactive") const;
bool getNonValidHttpMethods() const;
const std::string getOpenRedirectMode() const;
private:
std::string csrf_protection;
@ -551,9 +560,9 @@ public:
int getMaxHeaderSizeBytes() const;
int getMaxObjectDepth() const;
int getMaxUrlSizeBytes() const;
const std::string & getMinimumConfidence() const;
const NewAppSecWebAttackProtections & getprotections() const;
const std::string & getMode(const std::string &default_mode = "Inactive") const;
const std::string & getMinimumConfidence(const std::string &default_mode = "inactive") const;
const NewAppSecWebAttackProtections & getProtections() const;
const std::string & getMode(const std::string &default_mode = "inactive") const;
private:
int max_body_size_kb;

10
components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h
View File

@ -158,7 +158,8 @@ private:
const std::string &source_identifier,
const std::string & context,
const V1beta2AppsecLinuxPolicy &policy,
std::map<AnnotationTypes, std::string> &rule_annotations
std::map<AnnotationTypes, std::string> &rule_annotations,
const std::string &default_mode
);
void createSnortProtecionsSection(const std::string &file_name, bool is_temporary);
@ -172,7 +173,8 @@ private:
const std::string &practice_id,
const std::string &source_identifier,
const V1beta2AppsecLinuxPolicy &policy,
std::map<AnnotationTypes, std::string> &rule_annotations
std::map<AnnotationTypes, std::string> &rule_annotations,
const std::string &default_mode
);
void
@ -183,7 +185,8 @@ private:
const std::string &practice_name,
const std::string & context,
const V1beta2AppsecLinuxPolicy &policy,
std::map<AnnotationTypes, std::string> &rule_annotations
std::map<AnnotationTypes, std::string> &rule_annotations,
const std::string &default_mode
);
void
@ -192,6 +195,7 @@ private:
const std::string &url,
const std::string &uri,
const std::string &trigger_id,
const std::string &default_mode,
const V1beta2AppsecLinuxPolicy &policy,
std::map<AnnotationTypes, std::string> &rule_annotations
);

10
components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc
View File

@ -414,7 +414,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
vector<AccessControlPracticeSpec> access_control_practices =
extractV1Beta2ElementsFromCluster<AccessControlPracticeSpec>(
"accesscontrolpractice",
"accesscontrolpractices",
policy_elements_names[AnnotationTypes::ACCESS_CONTROL_PRACTICE]
);
@ -489,6 +489,8 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i
!doesVersionExist(maybe_appsec_policy_spec.unpack().getMetaData().getAnnotations(), "v1beta1")
) {
try {
std::string v1beta1_error =
maybe_appsec_policy_spec.ok() ? "There is no v1beta1 policy" : maybe_appsec_policy_spec.getErr();
dbgWarning(D_LOCAL_POLICY
) << "Failed to retrieve Appsec policy with crds version: v1beta1, Trying version: v1beta2";
auto maybe_v1beta2_appsec_policy_spec = getObjectFromCluster<AppsecSpecParser<NewAppsecPolicySpec>>(
@ -498,7 +500,7 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i
dbgWarning(D_LOCAL_POLICY)
<< "Failed to retrieve AppSec policy. Error: " << maybe_v1beta2_appsec_policy_spec.getErr();
return std::make_tuple(
genError("Failed to retrieve AppSec v1beta1 policy. Error: " + maybe_appsec_policy_spec.getErr()),
genError("Failed to retrieve AppSec v1beta1 policy. Error: " + v1beta1_error),
genError(
"Failed to retrieve AppSec v1beta2 policy. Error: " + maybe_v1beta2_appsec_policy_spec.getErr()
)
@ -584,7 +586,9 @@ K8sPolicyUtils::createAppsecPoliciesFromIngresses()
);
if (!std::get<0>(maybe_appsec_policy).ok() && !std::get<1>(maybe_appsec_policy).ok()) {
dbgWarning(D_LOCAL_POLICY)
<< "Failed to create appsec policy. Error: "
<< "Failed to create appsec policy. v1beta1 Error: "
<< std::get<0>(maybe_appsec_policy).getErr()
<< ". v1beta2 Error: "
<< std::get<1>(maybe_appsec_policy).getErr();
continue;
}

2
components/security_apps/local_policy_mgmt_gen/new_appsec_linux_policy.cc
View File

@ -99,7 +99,7 @@ V1beta2AppsecLinuxPolicy::serialize(cereal::JSONInputArchive &archive_in)
archive_in
);
parseAppsecJSONKey<vector<NewAppsecLogTrigger>>("logTriggers", log_triggers, archive_in);
parseAppsecJSONKey<vector<NewAppSecCustomResponse>>("customResponse", custom_responses, archive_in);
parseAppsecJSONKey<vector<NewAppSecCustomResponse>>("customResponses", custom_responses, archive_in);
parseAppsecJSONKey<vector<NewAppsecException>>("exceptions", exceptions, archive_in);
parseAppsecJSONKey<vector<NewTrustedSourcesSpec>>("trustedSources", trusted_sources, archive_in);
parseAppsecJSONKey<vector<NewSourcesIdentifiers>>("sourcesIdentifiers", sources_identifiers, archive_in);

2
components/security_apps/local_policy_mgmt_gen/new_exceptions.cc
View File

@ -44,7 +44,7 @@ void
NewAppsecException::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading New AppSec exception";
parseAppsecJSONKey<string>("name", name, archive_in, "exception");
parseAppsecJSONKey<string>("name", name, archive_in);
parseMandatoryAppsecJSONKey<string>("action", action, archive_in, "accept");
parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
if (valid_actions.count(action) == 0) {

246
components/security_apps/local_policy_mgmt_gen/new_practice.cc
View File

@ -21,8 +21,16 @@ USE_DEBUG_FLAG(D_LOCAL_POLICY);
static const set<string> performance_impacts = {"low", "medium", "high"};
static const set<string> severity_levels = {"low", "medium", "high", "critical"};
static const set<string> size_unit = {"bytes", "KB", "MB", "GB"};
static const set<string> confidences_actions = {"prevent", "detect", "inactive"};
static const set<string> valid_modes = {"prevent", "detect", "inactive", "prevent-learn", "detect-learn"};
static const set<string> confidences_actions = {"prevent", "detect", "inactive", "as-top-level", "inherited"};
static const set<string> valid_modes = {
"prevent",
"detect",
"inactive",
"prevent-learn",
"detect-learn",
"as-top-level",
"inherited"
};
static const set<string> valid_confidences = {"medium", "high", "critical"};
static const std::unordered_map<std::string, std::string> key_to_performance_impact_val = {
{ "low", "Low or lower"},
@ -48,6 +56,30 @@ static const std::unordered_map<std::string, uint64_t> unit_to_int = {
{ "MB", 1048576},
{ "GB", 1073741824}
};
static const std::string TRANSPARENT_MODE = "Transparent";
bool
isModeInherited(const string &mode)
{
return mode == "as-top-level" || mode == "inherited";
}
const std::string &
getModeWithDefault(
const std::string &mode,
const std::string &default_mode,
const std::unordered_map<std::string, std::string> &key_to_val)
{
if (isModeInherited(mode) && (key_to_val.find(default_mode) != key_to_val.end())) {
dbgError(D_LOCAL_POLICY) << "Setting to top-level mode: " << default_mode;
return key_to_val.at(default_mode);
}
else if (key_to_val.find(mode) == key_to_val.end()) {
dbgError(D_LOCAL_POLICY) << "Given mode: " << mode << " or top-level: " << default_mode << " is invalid.";
return key_to_val.at("inactive");
}
return key_to_val.at(mode);
}
void
NewAppSecWebBotsURI::load(cereal::JSONInputArchive &archive_in)
@ -84,7 +116,7 @@ NewAppSecPracticeAntiBot::load(cereal::JSONInputArchive &archive_in)
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Web Bots";
parseAppsecJSONKey<vector<NewAppSecWebBotsURI>>("injectedUris", injected_uris, archive_in);
parseAppsecJSONKey<vector<NewAppSecWebBotsURI>>("validatedUris", validated_uris, archive_in);
parseAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "Inactive");
parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
if (valid_modes.count(override_mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << override_mode;
}
@ -110,26 +142,33 @@ NewAppSecWebAttackProtections::load(cereal::JSONInputArchive &archive_in)
parseAppsecJSONKey<string>("csrfProtection", csrf_protection, archive_in, "inactive");
parseAppsecJSONKey<string>("errorDisclosure", error_disclosure, archive_in, "inactive");
parseAppsecJSONKey<string>("openRedirect", open_redirect, archive_in, "inactive");
if (valid_modes.count(csrf_protection) == 0 ||
valid_modes.count(error_disclosure) == 0 ||
valid_modes.count(open_redirect) == 0) {
string error_msg = "AppSec Attack Protections mode invalid. csrf_protection: " + csrf_protection +
" error_disclosure: " + error_disclosure + " open_redirect: " + open_redirect;
dbgWarning(D_LOCAL_POLICY) << error_msg;
throw PolicyGenException(error_msg);
}
parseAppsecJSONKey<bool>("nonValidHttpMethods", non_valid_http_methods, archive_in, false);
}
const string
NewAppSecWebAttackProtections::getCsrfProtectionMode() const
const string &
NewAppSecWebAttackProtections::getCsrfProtectionMode(const string &default_mode) const
{
if (key_to_practices_val.find(csrf_protection) == key_to_practices_val.end()) {
dbgError(D_LOCAL_POLICY)
<< "Failed to find a value for "
<< csrf_protection
<< ". Setting CSRF protection to Inactive";
return "Inactive";
}
return key_to_practices_val.at(csrf_protection);
return getModeWithDefault(csrf_protection, default_mode, key_to_practices_val2);
}
const string &
NewAppSecWebAttackProtections::getErrorDisclosureMode() const
NewAppSecWebAttackProtections::getErrorDisclosureMode(const string &default_mode) const
{
return error_disclosure;
return getModeWithDefault(error_disclosure, default_mode, key_to_practices_val2);
}
const string &
NewAppSecWebAttackProtections::getOpenRedirectMode(const string &default_mode) const
{
return getModeWithDefault(open_redirect, default_mode, key_to_practices_val2);
}
bool
@ -138,40 +177,24 @@ NewAppSecWebAttackProtections::getNonValidHttpMethods() const
return non_valid_http_methods;
}
const string
NewAppSecWebAttackProtections::getOpenRedirectMode() const
{
if (key_to_practices_val.find(open_redirect) == key_to_practices_val.end()) {
dbgError(D_LOCAL_POLICY)
<< "Failed to find a value for "
<< open_redirect
<< ". Setting Open Redirect mode to Inactive";
return "Inactive";
}
return key_to_practices_val.at(open_redirect);
}
void
NewAppSecPracticeWebAttacks::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice web attacks spec";
parseAppsecJSONKey<NewAppSecWebAttackProtections>("protections", protections, archive_in);
parseAppsecJSONKey<string>("overrideMode", mode, archive_in, "Unset");
parseMandatoryAppsecJSONKey<string>("overrideMode", mode, archive_in, "inactive");
if (valid_modes.count(mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec practice override mode invalid: " << mode;
}
if (getMode() == "Prevent") {
parseMandatoryAppsecJSONKey<string>("minimumConfidence", minimum_confidence, archive_in, "critical");
if (valid_confidences.count(minimum_confidence) == 0) {
dbgWarning(D_LOCAL_POLICY)
<< "AppSec practice override minimum confidence invalid: "
<< minimum_confidence;
throw PolicyGenException("AppSec practice override minimum confidence invalid: " + minimum_confidence);
}
} else {
minimum_confidence = "Transparent";
parseAppsecJSONKey<string>("minimumConfidence", minimum_confidence, archive_in, "critical");
if (valid_confidences.count(minimum_confidence) == 0) {
dbgWarning(D_LOCAL_POLICY)
<< "AppSec practice override minimum confidence invalid: "
<< minimum_confidence;
throw PolicyGenException("AppSec practice override minimum confidence invalid: " + minimum_confidence);
}
parseAppsecJSONKey<int>("maxBodySizeKb", max_body_size_kb, archive_in, 1000000);
parseAppsecJSONKey<int>("maxHeaderSizeBytes", max_header_size_bytes, archive_in, 102400);
parseAppsecJSONKey<int>("maxObjectDepth", max_object_depth, archive_in, 40);
@ -203,19 +226,25 @@ NewAppSecPracticeWebAttacks::getMaxUrlSizeBytes() const
}
const string &
NewAppSecPracticeWebAttacks::getMinimumConfidence() const
NewAppSecPracticeWebAttacks::getMinimumConfidence(const string &default_mode) const
{
if (getMode(default_mode) != "Prevent") {
return TRANSPARENT_MODE;
}
return minimum_confidence;
}
const string &
NewAppSecPracticeWebAttacks::getMode(const string &default_mode) const
{
if (mode == "Unset" || (key_to_practices_val2.find(mode) == key_to_practices_val2.end())) {
dbgError(D_LOCAL_POLICY) << "Couldn't find a value for key: " << mode << ". Returning " << default_mode;
return default_mode;
}
return key_to_practices_val2.at(mode);
const string &res = getModeWithDefault(mode, default_mode, key_to_practices_val);
return res;
}
const NewAppSecWebAttackProtections &
NewAppSecPracticeWebAttacks::getProtections() const
{
return protections;
}
SnortProtectionsSection::SnortProtectionsSection(
@ -244,7 +273,7 @@ SnortProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
{
out_ar(
cereal::make_nvp("context", context),
cereal::make_nvp("mode", key_to_mode_val.at(mode)),
cereal::make_nvp("mode", mode),
cereal::make_nvp("files", files),
cereal::make_nvp("assetName", asset_name),
cereal::make_nvp("assetId", asset_id),
@ -440,8 +469,8 @@ void
NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Snort Signatures practice";
parseAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
parseMandatoryAppsecJSONKey<vector<string>>("configmap", config_map, archive_in);
parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
parseAppsecJSONKey<vector<string>>("configmap", config_map, archive_in);
parseAppsecJSONKey<vector<string>>("files", files, archive_in);
is_temporary = false;
if (valid_modes.count(override_mode) == 0) {
@ -457,9 +486,10 @@ NewSnortSignaturesAndOpenSchemaAPI::addFile(const string &file_name)
}
const string &
NewSnortSignaturesAndOpenSchemaAPI::getOverrideMode() const
NewSnortSignaturesAndOpenSchemaAPI::getOverrideMode(const string &default_mode) const
{
return override_mode;
const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val);
return res;
}
const vector<string> &
@ -491,7 +521,7 @@ IpsProtectionsRulesSection::save(cereal::JSONOutputArchive &out_ar) const
{
vector<string> protections;
out_ar(
cereal::make_nvp("action", key_to_mode_val.at(action)),
cereal::make_nvp("action", action),
cereal::make_nvp("confidenceLevel", confidence_level),
cereal::make_nvp("clientProtections", true),
cereal::make_nvp("serverProtections", true),
@ -541,7 +571,7 @@ IpsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
cereal::make_nvp("practiceName", practice_name),
cereal::make_nvp("practiceId", practice_id),
cereal::make_nvp("sourceIdentifier", source_identifier),
cereal::make_nvp("defaultAction", key_to_mode_val.at(mode)),
cereal::make_nvp("defaultAction", mode),
cereal::make_nvp("rules", rules)
);
}
@ -566,7 +596,7 @@ void
NewIntrusionPrevention::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Intrusion Prevention practice";
parseAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
if (valid_modes.count(override_mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Intrusion Prevention override mode invalid: " << override_mode;
throw PolicyGenException("AppSec Intrusion Prevention override mode invalid: " + override_mode);
@ -580,13 +610,13 @@ NewIntrusionPrevention::load(cereal::JSONInputArchive &archive_in)
"AppSec Intrusion Prevention max performance impact invalid: " + max_performance_impact
);
}
parseAppsecJSONKey<string>("minSeverityLevel", min_severity_level, archive_in, "low");
parseAppsecJSONKey<string>("minSeverityLevel", min_severity_level, archive_in, "medium");
if (severity_levels.count(min_severity_level) == 0) {
dbgWarning(D_LOCAL_POLICY)
<< "AppSec Intrusion Prevention min severity level invalid: "
<< min_severity_level;
}
parseAppsecJSONKey<string>("highConfidenceEventAction", high_confidence_event_action, archive_in, "prevent");
parseAppsecJSONKey<string>("highConfidenceEventAction", high_confidence_event_action, archive_in, "inherited");
if (confidences_actions.count(high_confidence_event_action) == 0) {
dbgWarning(D_LOCAL_POLICY)
<< "AppSec Intrusion Prevention high confidence event invalid: "
@ -595,7 +625,9 @@ NewIntrusionPrevention::load(cereal::JSONInputArchive &archive_in)
"AppSec Intrusion Prevention high confidence event invalid: " + high_confidence_event_action
);
}
parseAppsecJSONKey<string>("mediumConfidenceEventAction", medium_confidence_event_action, archive_in, "prevent");
parseAppsecJSONKey<string>(
"mediumConfidenceEventAction", medium_confidence_event_action, archive_in, "inherited"
);
if (confidences_actions.count(medium_confidence_event_action) == 0) {
dbgWarning(D_LOCAL_POLICY)
<< "AppSec Intrusion Prevention medium confidence event invalid: "
@ -613,16 +645,16 @@ NewIntrusionPrevention::load(cereal::JSONInputArchive &archive_in)
"AppSec Intrusion Prevention low confidence event action invalid: " + low_confidence_event_action
);
}
parseAppsecJSONKey<int>("minCveYear", min_cve_Year, archive_in);
parseAppsecJSONKey<int>("minCveYear", min_cve_Year, archive_in, 2016);
}
vector<IpsProtectionsRulesSection>
NewIntrusionPrevention::createIpsRules() const
NewIntrusionPrevention::createIpsRules(const string &default_mode) const
{
vector<IpsProtectionsRulesSection> ips_rules;
IpsProtectionsRulesSection high_rule(
min_cve_Year,
high_confidence_event_action,
getModeWithDefault(high_confidence_event_action, default_mode, key_to_practices_val),
string("High"),
max_performance_impact,
string(""),
@ -632,7 +664,7 @@ NewIntrusionPrevention::createIpsRules() const
IpsProtectionsRulesSection med_rule(
min_cve_Year,
medium_confidence_event_action,
getModeWithDefault(medium_confidence_event_action, default_mode, key_to_practices_val),
string("Medium"),
max_performance_impact,
string(""),
@ -642,7 +674,7 @@ NewIntrusionPrevention::createIpsRules() const
IpsProtectionsRulesSection low_rule(
min_cve_Year,
low_confidence_event_action,
getModeWithDefault(low_confidence_event_action, default_mode, key_to_practices_val),
string("Low"),
max_performance_impact,
string(""),
@ -654,9 +686,10 @@ NewIntrusionPrevention::createIpsRules() const
}
const std::string &
NewIntrusionPrevention::getMode() const
NewIntrusionPrevention::getMode(const std::string &default_mode) const
{
return override_mode;
const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val);
return res;
}
FileSecurityProtectionsSection::FileSecurityProtectionsSection(
@ -711,20 +744,20 @@ FileSecurityProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
cereal::make_nvp("assetId", asset_id),
cereal::make_nvp("practiceName", practice_name),
cereal::make_nvp("practiceId", practice_id),
cereal::make_nvp("action", key_to_mode_val.at(action)),
cereal::make_nvp("filesWithoutNameAction", key_to_mode_val.at(files_without_name_action)),
cereal::make_nvp("action", action),
cereal::make_nvp("filesWithoutNameAction", files_without_name_action),
cereal::make_nvp("allowFilesWithoutName", allow_files_without_name),
cereal::make_nvp("highConfidence", key_to_mode_val.at(high_confidence_action)),
cereal::make_nvp("mediumConfidence", key_to_mode_val.at(medium_confidence_action)),
cereal::make_nvp("lowConfidence", key_to_mode_val.at(low_confidence_action)),
cereal::make_nvp("highConfidence", high_confidence_action),
cereal::make_nvp("mediumConfidence", medium_confidence_action),
cereal::make_nvp("lowConfidence", low_confidence_action),
cereal::make_nvp("severityLevel", key_to_severity_level_val.at(severity_level)),
cereal::make_nvp("fileSizeLimitAction", key_to_mode_val.at(file_size_limit_action)),
cereal::make_nvp("fileSizeLimitAction", file_size_limit_action),
cereal::make_nvp("fileSizeLimit", file_size_limit),
cereal::make_nvp("requiredFileSizeLimit", required_file_size_limit),
cereal::make_nvp("requiredArchiveExtraction", required_archive_extraction),
cereal::make_nvp("archiveFileSizeLimit", archive_file_size_limit),
cereal::make_nvp("MultiLevelArchiveAction", key_to_mode_val.at(multi_level_archive_action)),
cereal::make_nvp("UnopenedArchiveAction", key_to_mode_val.at(unopened_archive_action))
cereal::make_nvp("MultiLevelArchiveAction", multi_level_archive_action),
cereal::make_nvp("UnopenedArchiveAction", unopened_archive_action)
);
}
@ -748,7 +781,7 @@ void
NewFileSecurityArchiveInspection::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec File Security Archive Inspection practice";
parseAppsecJSONKey<bool>("extractArchiveFiles", extract_archive_files, archive_in, true);
parseAppsecJSONKey<bool>("extractArchiveFiles", extract_archive_files, archive_in, false);
parseAppsecJSONKey<uint64_t>("scanMaxFileSize", scan_max_file_size, archive_in, 10);
parseAppsecJSONKey<string>("scanMaxFileSizeUnit", scan_max_file_size_unit, archive_in, "MB");
if (size_unit.count(scan_max_file_size_unit) == 0) {
@ -763,7 +796,7 @@ NewFileSecurityArchiveInspection::load(cereal::JSONInputArchive &archive_in)
"archivedFilesWithinArchivedFiles",
archived_files_within_archived_files,
archive_in,
"prevent");
"inherited");
if (confidences_actions.count(archived_files_within_archived_files) == 0) {
dbgWarning(D_LOCAL_POLICY)
<< "AppSec File Security Archive Inspection archived files within archived files invalid: "
@ -777,7 +810,7 @@ NewFileSecurityArchiveInspection::load(cereal::JSONInputArchive &archive_in)
"archivedFilesWhereContentExtractionFailed",
archived_files_where_content_extraction_failed,
archive_in,
"prevent");
"inherited");
if (confidences_actions.count(archived_files_where_content_extraction_failed) == 0) {
dbgWarning(D_LOCAL_POLICY)
<< "AppSec File Security Archive Inspection archived files within archived file invalid: "
@ -834,7 +867,7 @@ NewFileSecurityLargeFileInspection::load(cereal::JSONInputArchive &archive_in)
"filesExceedingSizeLimitAction",
files_exceeding_size_limit_action,
archive_in,
"prevent");
"inherited");
if (confidences_actions.count(files_exceeding_size_limit_action) == 0) {
dbgWarning(D_LOCAL_POLICY)
<< "AppSec File Security Archive Inspection archived files within archived files invalid: "
@ -869,18 +902,18 @@ void
NewFileSecurity::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec File Security practice";
parseAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
if (valid_modes.count(override_mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec File Security override mode invalid: " << override_mode;
throw PolicyGenException("AppSec File Security override mode invalid: " + override_mode);
}
parseMandatoryAppsecJSONKey<string>("minSeverityLevel", min_severity_level, archive_in, "low");
parseAppsecJSONKey<string>("minSeverityLevel", min_severity_level, archive_in, "medium");
if (severity_levels.count(min_severity_level) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec File Security min severity level invalid: " << min_severity_level;
min_severity_level = "low";
}
parseMandatoryAppsecJSONKey<string>(
"highConfidenceEventAction", high_confidence_event_action, archive_in, "inactive"
parseAppsecJSONKey<string>(
"highConfidenceEventAction", high_confidence_event_action, archive_in, "inherited"
);
if (confidences_actions.count(high_confidence_event_action) == 0) {
dbgWarning(D_LOCAL_POLICY)
@ -888,8 +921,8 @@ NewFileSecurity::load(cereal::JSONInputArchive &archive_in)
<< high_confidence_event_action;
high_confidence_event_action = "inactive";
}
parseMandatoryAppsecJSONKey<string>(
"mediumConfidenceEventAction", medium_confidence_event_action, archive_in, "inactive"
parseAppsecJSONKey<string>(
"mediumConfidenceEventAction", medium_confidence_event_action, archive_in, "inherited"
);
if (confidences_actions.count(medium_confidence_event_action) == 0) {
dbgWarning(D_LOCAL_POLICY)
@ -897,8 +930,8 @@ NewFileSecurity::load(cereal::JSONInputArchive &archive_in)
<< medium_confidence_event_action;
medium_confidence_event_action = "inactive";
}
parseMandatoryAppsecJSONKey<string>(
"lowConfidenceEventAction", low_confidence_event_action, archive_in, "inactive"
parseAppsecJSONKey<string>(
"lowConfidenceEventAction", low_confidence_event_action, archive_in, "detect"
);
if (confidences_actions.count(low_confidence_event_action) == 0) {
dbgWarning(D_LOCAL_POLICY)
@ -906,7 +939,7 @@ NewFileSecurity::load(cereal::JSONInputArchive &archive_in)
<< low_confidence_event_action;
low_confidence_event_action = "inactive";
}
parseMandatoryAppsecJSONKey<string>("unnamedFilesAction", unnamed_files_action, archive_in, "inactive");
parseAppsecJSONKey<string>("unnamedFilesAction", unnamed_files_action, archive_in, "inherited");
if (confidences_actions.count(unnamed_files_action) == 0) {
dbgWarning(D_LOCAL_POLICY)
<< "AppSec File Security low unnamed files action invalid: "
@ -914,10 +947,8 @@ NewFileSecurity::load(cereal::JSONInputArchive &archive_in)
unnamed_files_action = "inactive";
}
parseAppsecJSONKey<bool>("threatEmulationEnabled", threat_emulation_enabled, archive_in);
parseMandatoryAppsecJSONKey<NewFileSecurityArchiveInspection>("archiveInspection", archive_inspection, archive_in);
parseMandatoryAppsecJSONKey<NewFileSecurityLargeFileInspection>(
"largeFileInspection", large_file_inspection, archive_in
);
parseAppsecJSONKey<NewFileSecurityArchiveInspection>("archiveInspection", archive_inspection, archive_in);
parseAppsecJSONKey<NewFileSecurityLargeFileInspection>("largeFileInspection", large_file_inspection, archive_in);
}
const string &
@ -944,28 +975,37 @@ NewFileSecurity::createFileSecurityProtectionsSection(
const string &asset_name,
const string &asset_id,
const string &practice_name,
const string &practice_id) const
const string &practice_id,
const string &default_mode) const
{
string practice_action = (isModeInherited(override_mode) ? default_mode : override_mode);
const string &unnamed_files_action_val =
getModeWithDefault(unnamed_files_action, practice_action, key_to_mode_val);
const string &large_file_action_val = getModeWithDefault(
getLargeFileInspection().getFileSizeLimitAction(),
practice_action,
key_to_mode_val
);
return FileSecurityProtectionsSection(
getLargeFileInspection().getFileSizeLimit(),
getArchiveInspection().getArchiveFileSizeLimit(),
unnamed_files_action == "prevent" ? true : false,
getLargeFileInspection().getFileSizeLimitAction() == "prevent" ? true : false,
unnamed_files_action_val == "Prevent" ? true : false,
large_file_action_val == "Prevent" ? true : false,
getArchiveInspection().getrequiredArchiveExtraction(),
context,
asset_name,
asset_id,
practice_name,
practice_id,
override_mode,
unnamed_files_action,
high_confidence_event_action,
medium_confidence_event_action,
low_confidence_event_action,
getModeWithDefault(override_mode, practice_action, key_to_mode_val),
unnamed_files_action_val,
getModeWithDefault(high_confidence_event_action, practice_action, key_to_mode_val),
getModeWithDefault(medium_confidence_event_action, practice_action, key_to_mode_val),
getModeWithDefault(low_confidence_event_action, practice_action, key_to_mode_val),
min_severity_level,
getLargeFileInspection().getFileSizeLimitAction(),
getArchiveInspection().getMultiLevelArchiveAction(),
getArchiveInspection().getUnopenedArchiveAction()
large_file_action_val,
getModeWithDefault(getArchiveInspection().getMultiLevelArchiveAction(), practice_action, key_to_mode_val),
getModeWithDefault(getArchiveInspection().getUnopenedArchiveAction(), practice_action, key_to_mode_val)
);
}
@ -974,14 +1014,14 @@ NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec";
parseAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>(
"openapi-schema-validation",
"schemaValidation",
openapi_schema_validation,
archive_in
);
parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
parseAppsecJSONKey<NewFileSecurity>("fileSecurity", file_security, archive_in);
parseAppsecJSONKey<NewIntrusionPrevention>("intrusionPrevention", intrusion_prevention, archive_in);
parseAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>("snortSignatures", snort_signatures, archive_in);
parseMandatoryAppsecJSONKey<NewFileSecurity>("fileSecurity", file_security, archive_in);
parseMandatoryAppsecJSONKey<NewIntrusionPrevention>("intrusionPrevention", intrusion_prevention, archive_in);
parseMandatoryAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>("snortSignatures", snort_signatures, archive_in);
parseMandatoryAppsecJSONKey<NewAppSecPracticeWebAttacks>("webAttacks", web_attacks, archive_in);
parseAppsecJSONKey<NewAppSecPracticeAntiBot>("antiBot", anti_bot, archive_in);
parseAppsecJSONKey<string>("name", practice_name, archive_in);

53
components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc
View File

@ -996,13 +996,15 @@ PolicyMakerUtils::createIpsSections(
const string &source_identifier,
const string & context,
const V1beta2AppsecLinuxPolicy &policy,
map<AnnotationTypes, string> &rule_annotations)
map<AnnotationTypes, string> &rule_annotations,
const string &default_mode)
{
auto apssec_practice = getAppsecPracticeSpec<V1beta2AppsecLinuxPolicy, NewAppSecPracticeSpec>(
rule_annotations[AnnotationTypes::PRACTICE],
policy);
if (apssec_practice.getIntrusionPrevention().getMode().empty()) return;
const string &override_mode = apssec_practice.getIntrusionPrevention().getMode(default_mode);
if (override_mode == "Inactive" || override_mode == "Disabled") return;
IpsProtectionsSection ips_section = IpsProtectionsSection(
context,
@ -1011,8 +1013,8 @@ PolicyMakerUtils::createIpsSections(
practice_name,
practice_id,
source_identifier,
apssec_practice.getIntrusionPrevention().getMode(),
apssec_practice.getIntrusionPrevention().createIpsRules()
override_mode,
apssec_practice.getIntrusionPrevention().createIpsRules(override_mode)
);
ips[asset_name] = ips_section;
@ -1068,13 +1070,16 @@ PolicyMakerUtils::createSnortSections(
const string &practice_id,
const string &source_identifier,
const V1beta2AppsecLinuxPolicy &policy,
map<AnnotationTypes, string> &rule_annotations)
map<AnnotationTypes, string> &rule_annotations,
const string &default_mode)
{
auto apssec_practice = getAppsecPracticeSpec<V1beta2AppsecLinuxPolicy, NewAppSecPracticeSpec>(
rule_annotations[AnnotationTypes::PRACTICE],
policy);
if (apssec_practice.getSnortSignatures().getOverrideMode() == "inactive" ||
const string &override_mode = apssec_practice.getSnortSignatures().getOverrideMode(default_mode);
if (override_mode == "Inactive" ||
override_mode == "Disabled" ||
apssec_practice.getSnortSignatures().getFiles().size() == 0) {
return;
}
@ -1094,7 +1099,7 @@ PolicyMakerUtils::createSnortSections(
practice_name,
practice_id,
source_identifier,
apssec_practice.getSnortSignatures().getOverrideMode(),
override_mode,
apssec_practice.getSnortSignatures().getFiles()
);
@ -1109,7 +1114,8 @@ PolicyMakerUtils::createFileSecuritySections(
const string &practice_name,
const string &context,
const V1beta2AppsecLinuxPolicy &policy,
map<AnnotationTypes, string> &rule_annotations)
map<AnnotationTypes, string> &rule_annotations,
const string &default_mode)
{
auto apssec_practice = getAppsecPracticeSpec<V1beta2AppsecLinuxPolicy, NewAppSecPracticeSpec>(
rule_annotations[AnnotationTypes::PRACTICE],
@ -1122,7 +1128,8 @@ PolicyMakerUtils::createFileSecuritySections(
asset_name,
asset_id,
practice_name,
practice_id
practice_id,
default_mode
);
file_security[asset_name] = file_security_section;
@ -1134,6 +1141,7 @@ PolicyMakerUtils::createRateLimitSection(
const string &url,
const string &uri,
const string &trigger_id,
const std::string &default_mode,
const V1beta2AppsecLinuxPolicy &policy,
map<AnnotationTypes, string> &rule_annotations)
{
@ -1157,13 +1165,13 @@ PolicyMakerUtils::createRateLimitSection(
trigger = RateLimitRulesTriggerSection(trigger_id, trigger_name, "Trigger");
}
auto rules = access_control_practice.geRateLimit().createRateLimitRulesSection(trigger);
auto rules = access_control_practice.getRateLimit().createRateLimitRulesSection(trigger);
rate_limit[rule_annotations[AnnotationTypes::ACCESS_CONTROL_PRACTICE]] = RateLimitSection(
asset_name,
url,
uri,
access_control_practice.geRateLimit().getMode(),
access_control_practice.getRateLimit().getMode(default_mode),
practice_id,
rule_annotations[AnnotationTypes::ACCESS_CONTROL_PRACTICE],
rules
@ -1198,12 +1206,13 @@ PolicyMakerUtils::createWebAppSection(
practice_id,
rule_annotations[AnnotationTypes::PRACTICE],
rule_config.getContext(),
apssec_practice.getWebAttacks().getMinimumConfidence(),
apssec_practice.getWebAttacks().getMinimumConfidence(default_mode),
apssec_practice.getWebAttacks().getMode(default_mode),
practice_advance_config,
apssec_practice.getAntiBot(),
log_triggers[rule_annotations[AnnotationTypes::TRIGGER]],
trusted_sources[rule_annotations[AnnotationTypes::TRUSTED_SOURCES]]
trusted_sources[rule_annotations[AnnotationTypes::TRUSTED_SOURCES]],
apssec_practice.getWebAttacks().getProtections()
);
web_apps[rule_config.getAssetName()] = web_app;
}
@ -1271,7 +1280,8 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
current_identifier,
rule_config.getContext(),
policy,
rule_annotations
rule_annotations,
default_mode
);
createSnortSections(
@ -1282,7 +1292,8 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
practice_id,
current_identifier,
policy,
rule_annotations
rule_annotations,
default_mode
);
createFileSecuritySections(
@ -1292,11 +1303,18 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
rule_annotations[AnnotationTypes::PRACTICE],
"assetId(" + rule_config.getAssetId() + ")",
policy,
rule_annotations
rule_annotations,
default_mode
);
if (!web_apps.count(rule_config.getAssetName())) {
createWebAppSection(policy, rule_config, practice_id, asset_name, default_mode, rule_annotations);
createWebAppSection(
policy,
rule_config,
practice_id,
asset_name,
default_mode,
rule_annotations);
}
}
@ -1568,6 +1586,7 @@ PolicyMakerUtils::createPolicyElementsByRule<V1beta2AppsecLinuxPolicy, NewParsed
std::get<0>(splited_host_name),
std::get<2>(splited_host_name),
log_triggers[rule_annotations[AnnotationTypes::TRIGGER]].getTriggerId(),
rule.getMode(),
policy,
rule_annotations
);

12
components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h
View File

@ -216,6 +216,18 @@ getFecApplicable(const string &command_output)
return genError("Could not determine if fec applicable");
}
Maybe<string>
getSMCBasedMgmtId(const string &command_output)
{
return getAttr(command_output, "Mgmt object UUID was not found");
}
Maybe<string>
getSMCBasedMgmtName(const string &command_output)
{
return getAttr(command_output, "Mgmt object Name was not found");
}
Maybe<string>
getSmbObjectName(const string &command_output)
{

44
components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
View File

@ -33,7 +33,10 @@
SHELL_PRE_CMD("read sdwan data",
"(cpsdwan get_data > /tmp/cpsdwan_getdata_orch.json~) "
"&& (mv /tmp/cpsdwan_getdata_orch.json~ /tmp/cpsdwan_getdata_orch.json)")
#endif
#endif //gaia || smb
#if defined(smb)
SHELL_PRE_CMD("gunzip local.cfg", "gunzip -c $FWDIR/state/local/FW1/local.cfg.gz > /tmp/local.cfg")
#endif //smb
#endif
#ifdef SHELL_CMD_HANDLER
@ -115,6 +118,22 @@ SHELL_CMD_HANDLER(
"cat $FWDIR/database/myself_objects.C | awk -F '[:()]' '/:VPN_1/ {print $3}' | head -n 1",
getGWIPSecVPNBlade
)
SHELL_CMD_HANDLER(
"SMCBasedMgmtId",
"domain_uuid=$(jq -r .domain_uuid /tmp/cpsdwan_getdata_orch.json);"
"[ \"$domain_uuid\" != \"null\" ] && echo \"$domain_uuid\" ||"
"cat $FWDIR/database/myself_objects.C "
"| awk -F'[{}]' '/:masters/ { found=1; next } found && /:Uid/ { uid=tolower($2); print uid; exit }'",
getSMCBasedMgmtId
)
SHELL_CMD_HANDLER(
"SMCBasedMgmtName",
"domain_name=$(jq -r .domain_name /tmp/cpsdwan_getdata_orch.json);"
"[ \"$domain_name\" != \"null\" ] && echo \"$domain_name\" ||"
"cat $FWDIR/database/myself_objects.C "
"| awk -F '[:()]' '/:masters/ {found=1; next} found && /:Name/ {print $3; exit}'",
getSMCBasedMgmtName
)
#endif //gaia
#if defined(smb)
@ -148,6 +167,23 @@ SHELL_CMD_HANDLER(
"cat $FWDIR/conf/active_blades.txt | grep -o 'IPS [01]' | cut -d ' ' -f2",
getSmbGWIPSecVPNBlade
)
SHELL_CMD_HANDLER(
"SMCBasedMgmtId",
"domain_uuid=$(jq -r .domain_uuid /tmp/cpsdwan_getdata_orch.json);"
"[ \"$domain_uuid\" != \"null\" ] && echo \"$domain_uuid\" ||"
"cat /tmp/local.cfg "
"| awk -F'[{}]' '/:masters/ { found=1; next } found && /:Uid/ { uid=tolower($2); print uid; exit }'",
getSMCBasedMgmtId
)
SHELL_CMD_HANDLER(
"SMCBasedMgmtName",
"domain_name=$(jq -r .domain_name /tmp/cpsdwan_getdata_orch.json);"
"[ \"$domain_name\" != \"null\" ] && echo \"$domain_name\" ||"
"cat /tmp/local.cfg "
"| awk -F '[:()]' '/:masters/ {found=1; next} found && /:Name/ {print $3; exit}'",
getSMCBasedMgmtName
)
#endif//smb
SHELL_CMD_OUTPUT("kernel_version", "uname -r")
@ -190,3 +226,9 @@ FILE_CONTENT_HANDLER("os_release", "/etc/os-release", getOsRelease)
FILE_CONTENT_HANDLER("AppSecModelVersion", "/etc/cp/conf/waap/waap.data", getWaapModelVersion)
#endif // FILE_CONTENT_HANDLER
#ifdef SHELL_POST_CMD
#if defined(smb)
SHELL_POST_CMD("remove local.cfg", "rm -rf /tmp/local.cfg")
#endif //smb
#endif

18
components/security_apps/orchestration/details_resolver/details_resolving_handler.cc
View File

@ -64,6 +64,12 @@ private:
#undef FILE_CONTENT_HANDLER
};
#define SHELL_POST_CMD(NAME, COMMAND) {NAME, COMMAND},
map<string, string> shell_post_commands = {
#include "details_resolver_impl.h"
};
#undef SHELL_POST_CMD
map<string, string>
DetailsResolvingHanlder::Impl::getResolvedDetails() const
{
@ -114,6 +120,18 @@ DetailsResolvingHanlder::Impl::getResolvedDetails() const
in_file->close();
}
for (auto &shell_post_command : shell_post_commands) {
const string &name = shell_post_command.first;
const string &command = shell_post_command.second;
Maybe<int> command_ret = shell->getExecReturnCode(command, timeout);
if (!command_ret.ok()) {
dbgWarning(D_AGENT_DETAILS) << "Failed to run post-command " << name;
} else if (*command_ret) {
dbgWarning(D_AGENT_DETAILS) << "Post-command " << name << " failed (rc: " << *command_ret << ")";
}
}
I_AgentDetailsReporter *reporter = Singleton::Consume<I_AgentDetailsReporter>::by<DetailsResolvingHanlder>();
reporter->addAttr(resolved_details, true);

3
components/security_apps/orchestration/orchestration_comp.cc
View File

@ -1012,10 +1012,11 @@ private:
HybridModeMetricEvent().notify();
if (!response.ok()) {
dbgWarning(D_ORCHESTRATOR) << "Failed to get the update. Error: " << response.getErr();
orch_status->setFieldStatus(
OrchestrationStatusFieldType::LAST_UPDATE,
OrchestrationStatusResult::FAILED,
response.getErr()
"Warning: Agent/Gateway failed during the update process. Contact Check Point support."
);
return genError(response.getErr());

1
components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc
View File

@ -90,6 +90,7 @@ public:
"/api/v1/agents/events",
_,
MessageCategory::LOG,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));

5
components/security_apps/orchestration/update_communication/fog_authenticator.cc
View File

@ -473,10 +473,11 @@ FogAuthenticator::authenticateAgent()
auto orc_status = Singleton::Consume<I_OrchestrationStatus>::by<FogAuthenticator>();
credentials = getCredentials();
if (!credentials.ok()) {
dbgWarning(D_ORCHESTRATOR) << "Failed to authenticate the agent: " << credentials.getErr();
orc_status->setFieldStatus(
OrchestrationStatusFieldType::REGISTRATION,
OrchestrationStatusResult::FAILED,
credentials.getErr()
"Warning: Agent/Gateway failed the authentication. Contact Check Point support."
);
return genError(credentials.getErr());
}
@ -516,7 +517,7 @@ FogAuthenticator::authenticateAgent()
orc_status->setFieldStatus(
OrchestrationStatusFieldType::REGISTRATION,
OrchestrationStatusResult::FAILED,
access_token.getErr()
"Warning: Agent/Gateway failed to receive access token. Contact Check Point support."
);
}
int next_session_req = max(

10
components/security_apps/waap/include/i_serialize.h
View File

@ -154,6 +154,7 @@ protected:
if (agentDetails->getOrchestrationMode() == OrchestrationMode::HYBRID) {
MessageMetadata req_md(getSharedStorageHost(), 80);
req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId());
req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
auto req_status = messaging->sendSyncMessage(
method,
uri,
@ -161,6 +162,10 @@ protected:
MessageCategory::GENERIC,
req_md
);
if (!req_status.ok()) {
dbgWarning(D_WAAP) << "failed to send request to uri: " << uri
<< ", error: " << req_status.getErr().toString();
}
return req_status.ok();
}
auto req_status = messaging->sendSyncMessage(
@ -169,6 +174,10 @@ protected:
obj,
MessageCategory::GENERIC
);
if (!req_status.ok()) {
dbgWarning(D_WAAP) << "failed to send request to uri: " << uri
<< ", error: " << req_status.getErr().toString();
}
return req_status.ok();
}
@ -204,6 +213,7 @@ protected:
if (agentDetails->getOrchestrationMode() == OrchestrationMode::HYBRID) {
MessageMetadata req_md(getSharedStorageHost(), 80);
req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId());
req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
return messaging->sendSyncMessageWithoutResponse(
method,
uri,

3
components/security_apps/waap/include/i_transaction.h
View File

@ -20,6 +20,7 @@
#include "../waap_clib/WaapOpenRedirect.h"
#include "../waap_clib/FpMitigation.h"
#include "../waap_clib/DeepParser.h"
#include "../waap_clib/OASchemaUpdaterConfConstant.h"
#include "http_inspection_events.h"
enum HeaderType {
@ -29,6 +30,7 @@ enum HeaderType {
COOKIE_HEADER,
REFERER_HEADER,
CONTENT_TYPE_HEADER,
AUTHORIZATION_HEADER,
CLEAN_HEADER,
OTHER_KNOWN_HEADERS
};
@ -135,6 +137,7 @@ public:
virtual void add_response_body_chunk(const char* data, int data_len) = 0;
virtual void end_response_body() = 0;
virtual void end_response() = 0;
virtual const std::string& getResponseBody() = 0;
virtual void collectFoundPatterns() = 0;
virtual ReportIS::Severity computeEventSeverityFromDecision() const = 0;

1
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@ -86,6 +86,7 @@ add_library(waap_clib
ParserPercentEncode.cc
ParserPairs.cc
Waf2Util2.cc
ParserPDF.cc
)
add_definitions("-Wno-unused-function")

22
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@ -26,6 +26,7 @@
#include "ParserPercentEncode.h"
#include "ParserPairs.h"
#include "ParserDelimiter.h"
#include "ParserPDF.h"
#include "WaapAssetState.h"
#include "Waf2Regex.h"
#include "Waf2Util.h"
@ -1146,9 +1147,15 @@ DeepParser::createInternalParser(
));
offset = 0;
} else if (isTopData && (isBinaryType || m_pWaapAssetState->isBinarySampleType(cur_val))) {
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a binary file";
m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserBinary>>(*this, parser_depth + 1));
offset = 0;
if (isPDFDetected(cur_val)) {
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a PDF file";
m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserPDF>>(*this, parser_depth + 1));
offset = 0;
} else {
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a binary file";
m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserBinary>>(*this, parser_depth + 1));
offset = 0;
}
}
}
if (offset < 0) {
@ -1473,3 +1480,12 @@ DeepParser::shouldEnforceDepthLimit(const std::shared_ptr<ParserBase> &parser) c
}
return false;
}
bool
DeepParser::isPDFDetected(const std::string &cur_val) const
{
static const std::string PDF_header("%PDF-");
if (cur_val.size() < 10)
return false;
return cur_val.substr(0, cur_val.size() > 64 ? 64 : cur_val.size()).find(PDF_header) != std::string::npos;
}

1
components/security_apps/waap/waap_clib/DeepParser.h
View File

@ -173,6 +173,7 @@ private:
bool shouldEnforceDepthLimit(const std::shared_ptr<ParserBase>& parser) const;
void setLocalMaxObjectDepth(size_t depth) { m_localMaxObjectDepth = depth; }
void setGlobalMaxObjectDepthReached() { m_globalMaxObjectDepthReached = true; }
bool isPDFDetected(const std::string &cur_val) const;
bool m_deepParserFlag;
std::stack<std::tuple<size_t, size_t, std::string>> m_splitTypesStack; // depth, splitIndex, splitType
std::deque<std::shared_ptr<ParserBase>> m_parsersDeque;

13
components/security_apps/waap/waap_clib/OASchemaUpdaterConfConstant.h Normal file
View File

@ -0,0 +1,13 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

111
components/security_apps/waap/waap_clib/ParserPDF.cc Normal file
View File

@ -0,0 +1,111 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ParserPDF.h"
#include "Waf2Util.h"
#include "debug.h"
#include <string.h>
USE_DEBUG_FLAG(D_WAAP_PARSER_PDF);
USE_DEBUG_FLAG(D_WAAP);
const std::string ParserPDF::m_parserName = "ParserPDF";
const char* PDF_TAIL = "%%EOF";
ParserPDF::ParserPDF(
IParserStreamReceiver &receiver,
size_t parser_depth
) :
m_receiver(receiver),
m_state(s_start),
m_parser_depth(parser_depth)
{}
ParserPDF::~ParserPDF()
{}
size_t
ParserPDF::push(const char *buf, size_t len)
{
dbgTrace(D_WAAP_PARSER_PDF)
<< "buf="
<< buf
<< "len="
<< len;
const char *c;
if (m_state == s_error) {
return 0;
}
if (len == 0)
{
dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): end of stream. m_state=" << m_state;
if (m_state == s_end) {
m_receiver.onKvDone();
} else {
m_state = s_error;
}
return 0;
}
switch (m_state) {
case s_start:
m_state = s_body;
CP_FALL_THROUGH;
case s_body:
c = strstr(buf + len - MAX_TAIL_LOOKUP, PDF_TAIL);
dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): c=" << c;
if (c) {
m_state = s_end;
CP_FALL_THROUGH;
} else {
break;
}
case s_end:
if (m_receiver.onKey("PDF", 3) != 0) {
m_state = s_error;
return 0;
}
if (m_receiver.onValue("", 0) != 0) {
m_state = s_error;
return 0;
}
break;
case s_error:
break;
default:
dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): unknown state: " << m_state;
m_state = s_error;
return 0;
}
return len;
}
void ParserPDF::finish()
{
push(NULL, 0);
}
const std::string& ParserPDF::name() const
{
return m_parserName;
}
bool ParserPDF::error() const
{
return m_state == s_error;
}

47
components/security_apps/waap/waap_clib/ParserPDF.h Normal file
View File

@ -0,0 +1,47 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __PARSER_PDF_H__
#define __PARSER_PDF_H__
#include "ParserBase.h"
#include <string.h>
#define MAX_HEADER_LOOKUP 64
#define MAX_TAIL_LOOKUP 5
class ParserPDF : public ParserBase {
public:
ParserPDF(IParserStreamReceiver &receiver, size_t parser_depth);
virtual ~ParserPDF();
virtual size_t push(const char *buf, size_t len);
virtual void finish();
virtual const std::string &name() const;
virtual bool error() const;
virtual size_t depth() { return 1; }
private:
enum state {
s_start,
s_body,
s_end,
s_error
};
IParserStreamReceiver &m_receiver;
enum state m_state;
static const std::string m_parserName;
size_t m_parser_depth;
};
#endif // __PARSER_PDF_H__

1
components/security_apps/waap/waap_clib/Serializator.cc
View File

@ -706,6 +706,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
MessageMetadata req_md(getLearningHost(), 80);
req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId());
req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
bool ok = messaging->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
"/api/sync",

1
components/security_apps/waap/waap_clib/Telemetry.cc
View File

@ -156,6 +156,7 @@ void
WaapTrafficTelemetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data)
{
initMetrics();
average_latency.report(data.elapsedTime);
switch (data.method)
{
case POST:

49
components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
View File

@ -12,6 +12,7 @@
// limitations under the License.
#include <string>
#include <cctype>
#include <boost/regex.hpp>
#include "WaapOverrideFunctor.h"
#include "Waf2Engine.h"
@ -47,14 +48,16 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const Waap::Util::C
bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& rx)
{
boost::cmatch what;
std::string tagLower = tag;
std::transform(tagLower.begin(), tagLower.end(), tagLower.begin(), ::tolower);
try {
if (tag == "url") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUri().c_str(), what, rx);
if (tagLower == "url") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUriStr().c_str(), what, rx);
}
else if (tag == "hostname") {
else if (tagLower == "hostname") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getHost().c_str(), what, rx);
}
else if (tag == "sourceidentifier") {
else if (tagLower == "sourceidentifier") {
return NGEN::Regex::regexMatch(
__FILE__,
__LINE__,
@ -63,7 +66,7 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
rx
);
}
else if (tag == "keyword") {
else if (tagLower == "keyword") {
for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordStr.c_str(), what, rx)) {
return true;
@ -71,7 +74,7 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
}
return false;
}
else if (tag == "paramname" || tag == "paramName") {
else if (tagLower == "paramname") {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getName().c_str(), what, rx)) {
return true;
@ -85,7 +88,7 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
}
return false;
}
else if (tag == "paramvalue" || tag == "paramValue") {
else if (tagLower == "paramvalue") {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getValue().c_str(), what, rx)) {
return true;
@ -96,10 +99,10 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
}
return false;
}
else if (tag == "paramlocation" || tag == "paramLocation") {
else if (tagLower == "paramlocation") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getLocation().c_str(), what, rx);
}
else if (tag == "responsebody" || tag == "responseBody") {
else if (tagLower == "responsebody") {
waf2Transaction.getResponseInspectReasons().setApplyOverride(true);
if (!waf2Transaction.getResponseBody().empty()) {
boost::smatch matcher;
@ -108,6 +111,32 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
} else {
return false;
}
} else if (tagLower == "headername") {
if (!waf2Transaction.checkIsHeaderOverrideScanRequired()) {
dbgDebug(D_WAAP_OVERRIDE) << "Header name override scan is not required";
return false;
}
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.first;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if(NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) {
return true;
}
}
return false;
} else if (tagLower == "headervalue") {
if (!waf2Transaction.checkIsHeaderOverrideScanRequired()) {
dbgDebug(D_WAAP_OVERRIDE) << "Header value override scan is not required";
return false;
}
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.second;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) {
return true;
}
}
return false;
}
}
catch (std::runtime_error & e) {
@ -116,6 +145,6 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
}
// Unknown tag: should not occur
dbgWarning(D_WAAP) << "Invalid override tag:" << tag;
dbgWarning(D_WAAP) << "Invalid override tag: " << tag;
return false;
}

16
components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc
View File

@ -26,7 +26,8 @@ namespace Waap {
errorLimiter(false),
rateLimiting(false),
collectResponseForLog(false),
applyOverride(false)
applyOverride(false),
triggerReport(false)
{
}
@ -39,11 +40,12 @@ namespace Waap {
" RateLimiting=" << rateLimiting <<
" ErrorLimiter=" << errorLimiter <<
" collectResponseForLog=" << collectResponseForLog <<
" applyOverride=" << applyOverride;
" applyOverride=" << applyOverride <<
" triggerReport=" << triggerReport;
return
openRedirect || errorDisclosure || rateLimiting || errorLimiter ||
collectResponseForLog || applyOverride;
collectResponseForLog || applyOverride || triggerReport;
}
void
@ -91,6 +93,14 @@ namespace Waap {
applyOverride = flag;
}
void
ResponseInspectReasons::setTriggerReport(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(setTriggerReport) " << triggerReport << " to " <<
flag;
triggerReport = flag;
}
bool
ResponseInspectReasons::getApplyOverride(void)
{

2
components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h
View File

@ -25,6 +25,7 @@ public:
void setErrorLimiter(bool flag);
void setCollectResponseForLog(bool flag);
void setApplyOverride(bool flag);
void setTriggerReport(bool flag);
bool getApplyOverride(void);
private:
@ -34,6 +35,7 @@ private:
bool rateLimiting;
bool collectResponseForLog;
bool applyOverride;
bool triggerReport;
};
}

183
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@ -56,8 +56,10 @@ USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_WAAP_ULIMITS);
USE_DEBUG_FLAG(D_WAAP_BOT_PROTECTION);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
USE_DEBUG_FLAG(D_WAAP_HEADERS);
using namespace ReportIS;
using namespace std;
#define MAX_REQUEST_BODY_SIZE (2*1024)
#define MAX_RESPONSE_BODY_SIZE (2*1024)
@ -89,6 +91,9 @@ void Waf2Transaction::start_response(int response_status, int http_version)
dbgTrace(D_WAAP) << "[transaction:" << this << "] start_response(response_status=" << response_status
<< "," << " http_version=" << http_version << ")";
m_responseStatus = response_status;
if (m_triggerReport) {
m_responseInspectReasons.setTriggerReport(false);
}
if(m_responseStatus == 404)
{
@ -233,6 +238,12 @@ void Waf2Transaction::end_response_body()
dbgTrace(D_WAAP) << "[transaction:" << this << "] end_response_body";
}
const std::string &
Waf2Transaction::getResponseBody()
{
return m_response_body;
}
void Waf2Transaction::scanErrDisclosureBuffer()
{
if (m_responseStatus >= 400 && m_responseStatus <= 599) {
@ -302,19 +313,25 @@ Waf2Transaction::Waf2Transaction() :
m_deepParser(m_pWaapAssetState, m_scanner, this),
m_deepParserReceiver(m_deepParser),
m_scanResult(NULL),
m_response_body(),
m_request_body_bytes_received(0),
m_response_body_bytes_received(0),
m_processedUri(false),
m_processedHeaders(false),
m_isHeaderOverrideScanRequired(false),
m_isScanningRequired(false),
m_responseStatus(0),
m_responseInspectReasons(),
m_responseInjectReasons(),
m_index(-1),
m_triggerLog(),
m_triggerReport(false),
is_schema_validation(false),
m_waf2TransactionFlags()
{}
{
I_TimeGet *timeGet = Singleton::Consume<I_TimeGet>::by<Waf2Transaction>();
m_entry_time = chrono::duration_cast<chrono::milliseconds>(timeGet->getMonotonicTime());
}
Waf2Transaction::Waf2Transaction(std::shared_ptr<WaapAssetState> pWaapAssetState) :
TableOpaqueSerialize<Waf2Transaction>(this),
@ -334,19 +351,25 @@ Waf2Transaction::Waf2Transaction(std::shared_ptr<WaapAssetState> pWaapAssetState
m_deepParser(m_pWaapAssetState, m_scanner, this),
m_deepParserReceiver(m_deepParser),
m_scanResult(NULL),
m_response_body(),
m_request_body_bytes_received(0),
m_response_body_bytes_received(0),
m_processedUri(false),
m_processedHeaders(false),
m_isHeaderOverrideScanRequired(false),
m_isScanningRequired(false),
m_responseStatus(0),
m_responseInspectReasons(),
m_responseInjectReasons(),
m_index(-1),
m_triggerLog(),
m_triggerReport(false),
is_schema_validation(false),
m_waf2TransactionFlags()
{}
{
I_TimeGet *timeGet = Singleton::Consume<I_TimeGet>::by<Waf2Transaction>();
m_entry_time = chrono::duration_cast<chrono::milliseconds>(timeGet->getMonotonicTime());
}
Waf2Transaction::~Waf2Transaction() {
dbgTrace(D_WAAP) << "Waf2Transaction::~Waf2Transaction: deleting m_requestBodyParser";
@ -362,6 +385,8 @@ HeaderType Waf2Transaction::detectHeaderType(const char* name, int name_len) {
static const char content_type[] = "content-Type";
static const char cookie[] = "cookie";
static const char referer[] = "referer";
static const char authorization[] = "authorization";
dbgTrace(D_WAAP_HEADERS) << "Detecting header type for header >>>" << std::string(name, name_len) << "<<<";
if (memcaseinsensitivecmp(name, name_len, host, sizeof(host) - 1)) {
return HeaderType::HOST_HEADER;
@ -378,6 +403,9 @@ HeaderType Waf2Transaction::detectHeaderType(const char* name, int name_len) {
if (memcaseinsensitivecmp(name, name_len, referer, sizeof(referer) - 1)) {
return HeaderType::REFERER_HEADER;
}
if (memcaseinsensitivecmp(name, name_len, authorization, sizeof(authorization) - 1)) {
return HeaderType::AUTHORIZATION_HEADER;
}
return UNKNOWN_HEADER;
}
@ -422,37 +450,100 @@ HeaderType Waf2Transaction::checkCleanHeader(const char* name, int name_len, con
return CLEAN_HEADER;
}
}
static const std::string authorization("authorization");
if (memcaseinsensitivecmp(name, name_len, authorization.data(), authorization.size())) {
dbgTrace(D_WAAP) << "[transaction:" << this << "] special header '" << std::string(name, name_len) <<
"' - detect base64 to determine cleanliness ...";
std::string result;
int decodedCount = 0;
int deletedCount = 0;
std::string v(value, value_len);
boost::algorithm::to_lower(v);
const std::string negotiate("negotiate ");
if (boost::algorithm::starts_with(v, negotiate)) {
v = v.substr(negotiate.size(), v.size() - negotiate.size());
// Detect potential base64 match after the "Negotiate " prefix
Waap::Util::b64Decode(v, b64DecodeChunk, decodedCount, deletedCount, result);
if (result.empty() && (deletedCount + decodedCount == 1)) {
// Decoded 1 base64 chunk and nothing left behind it
dbgTrace(D_WAAP) << "[transaction:" << this << "] special header '" <<
std::string(name, name_len) << " is clean";
return CLEAN_HEADER;
}
}
}
}
return UNKNOWN_HEADER;
}
void Waf2Transaction::parseAuthorization(const char* value, int value_len, const std::string &header_name)
{
#ifdef NO_HEADERS_SCAN
return;
#endif
static const std::string authorization("authorization");
dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this
<< "] special header '" << header_name;
std::string result;
std::string v(value, value_len);
int decodedCount;
int deletedCount;
static const char *bearer = "bearer ";
static const std::string bearer_str(bearer);
if (memcaseinsensitivecmp(value, sizeof(bearer) - 1, bearer, sizeof(bearer) - 1)) {
dbgTrace(D_WAAP_HEADERS) << "Checking JWT chunk for alg = none" << v;
// parse buffer v till first dot,
// convert this chunk from base 64 to json string and get value for "alg" key
// if alg == "none" then this is JWT with no signature and we should block it
// if alg != "none" then this is JWT with signature and we should not block it
char* p = (char*)v.c_str() + bearer_str.size();
char* p1 = strchr(p, '.');
if (p1 != NULL) {
*p1 = '\0';
std::string header(p);
alignBase64Chunk(header);
Waap::Util::b64Decode(header, b64DecodeChunk, decodedCount, deletedCount, result);
if (!result.empty()) {
dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] special header '" << header_name
<< " base64 encoded for JWT alg testing";
// parse JSON string and get value for "alg" key and check its value to be equal to "none"
// if alg == "none" then this is JWT with no signature and we should block it
boost::algorithm::to_lower(result);
static bool err = false;
static const SingleRegex invalid_jwt_alg_re(
"\"alg\"\\s*:\\s*\"none\"",
err,
"invalid_jwt_alg_"
);
if (invalid_jwt_alg_re.hasMatch(result)) {
dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] special header '" << header_name
<< " bad JWT algotitm detected, blocking";
Waf2ScanResult new_res = Waf2ScanResult();
new_res.location = "header";
new_res.param_name = header_name;
new_res.unescaped_line = header_name + ": " + result;
new_res.score = 9.0;
new_res.keyword_matches.push_back("alg:none");
new_res.scoreArray.push_back(9.0);
new_res.attack_types.insert("JWT");
if (m_scanResult != NULL) {
dbgTrace(D_WAAP_HEADERS) << "LastScanResult = " << m_scanResult;
m_scanResult->mergeFrom(new_res);
} else {
dbgTrace(D_WAAP_HEADERS) << "LastScanResult is NULL";
m_scanResult = new Waf2ScanResult(new_res);
}
return;
}
dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] special header '" << header_name
<< " clean JWT algotitm detected";
return;
} else {
parseGenericHeaderValue(header_name, value, value_len);
}
}
return;
}
static const char *negotiate = "negotiate ";
static const std::string negotiate_str(negotiate);
if (memcaseinsensitivecmp(value, sizeof(negotiate) - 1, negotiate, sizeof(negotiate) - 1)) {
v = v.substr(negotiate_str.size(), v.size() - negotiate_str.size());
// Detect potential base64 match after the "Negotiate " prefix
alignBase64Chunk(v);
Waap::Util::b64Decode(v, b64DecodeChunk, decodedCount, deletedCount, result);
if (result.empty() && (deletedCount + decodedCount == 1)) {
// Decoded 1 base64 chunk and nothing left behind it
dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] special header '" << header_name
<< " is clean - clean base64 for negotiate";
return;
} else {
parseGenericHeaderValue(header_name, value, value_len);
}
}
}
// Methods below are callbacks that are called during HTTP transaction processing by the front-end server/proxy
void Waf2Transaction::start() {
dbgTrace(D_WAAP) << "[Waf2Transaction::start():" << this << "] start";
@ -744,14 +835,14 @@ void Waf2Transaction::parseContentType(const char* value, int value_len)
ctp.push(value, value_len);
ctp.finish();
dbgTrace(D_WAAP) << "[transaction:" << this << "] ctp detected content type: '" <<
dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] ctp detected content type: '" <<
ctp.contentTypeDetected.c_str() << "'";
// The above fills m_contentTypeDetected
m_contentType = Waap::Util::detectContentType(ctp.contentTypeDetected.c_str());
// extract boundary string required for parsing multipart-form-data stream
if (m_contentType == Waap::Util::CONTENT_TYPE_MULTIPART_FORM) {
dbgTrace(D_WAAP) << "content_type detected: " << Waap::Util::getContentTypeStr(m_contentType) <<
dbgTrace(D_WAAP_HEADERS) << "content_type detected: " << Waap::Util::getContentTypeStr(m_contentType) <<
"; boundary='" << ctp.boundaryFound.c_str() << "'";
m_deepParser.setMultipartBoundary(ctp.boundaryFound);
}
@ -773,7 +864,7 @@ void Waf2Transaction::parseCookie(const char* value, int value_len)
#endif
if (value_len > 0) {
dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the cookie value";
dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] scanning the cookie value";
m_deepParser.m_key.push("cookie", 6);
ParserUrlEncode cookieValueParser(m_deepParserReceiver, 0, ';');
cookieValueParser.push(value, value_len);
@ -788,7 +879,7 @@ void Waf2Transaction::parseReferer(const char* value, int value_len)
#ifdef NO_HEADERS_SCAN
return;
#endif
dbgTrace(D_WAAP) << "Parsed Referer. Referer URI: " << m_uriReferer;
dbgTrace(D_WAAP_HEADERS) << "Parsed Referer. Referer URI: " << m_uriReferer;
std::string referer(value, value_len);
std::vector<RegexMatch> regex_matches;
@ -815,7 +906,7 @@ void Waf2Transaction::parseUnknownHeaderName(const char* name, int name_len)
// Apply signatures on all other, header names, unless they are considered "good" ones to skip scanning them.
if (name_len &&
!m_pWaapAssetState->getSignatures()->good_header_name_re.hasMatch(std::string(name, name_len))) {
dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the header name";
dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] scanning the header name";
m_deepParser.m_key.push("header", 6);
ParserRaw headerNameParser(m_deepParserReceiver, 0, std::string(name, name_len));
headerNameParser.push(name, name_len);
@ -834,7 +925,7 @@ void Waf2Transaction::parseGenericHeaderValue(const std::string &headerName, con
return;
}
dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the header value";
dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] scanning the header value";
m_deepParser.m_key.push("header", 6);
ParserRaw headerValueParser(m_deepParserReceiver, 0, headerName);
headerValueParser.push(value, value_len);
@ -844,11 +935,11 @@ void Waf2Transaction::parseGenericHeaderValue(const std::string &headerName, con
};
// Scan relevant headers to detect attacks inside them
void Waf2Transaction::scanSpecificHeder(const char* name, int name_len, const char* value, int value_len)
void Waf2Transaction::scanSpecificHeader(const char* name, int name_len, const char* value, int value_len)
{
HeaderType header_t = detectHeaderType(name, name_len);
std::string headerName = std::string(name, name_len);
dbgTrace(D_WAAP_HEADERS) << "Processing the header " << headerName;
switch (header_t)
{
case HeaderType::COOKIE_HEADER:
@ -857,6 +948,9 @@ void Waf2Transaction::scanSpecificHeder(const char* name, int name_len, const ch
case HeaderType::REFERER_HEADER:
parseReferer(value, value_len);
break;
case HeaderType::AUTHORIZATION_HEADER:
parseAuthorization(value, value_len, headerName);
break;
case HeaderType::UNKNOWN_HEADER: {
HeaderType headerType = checkCleanHeader(name, name_len, value, value_len);
if(headerType == HeaderType::CLEAN_HEADER) {
@ -954,7 +1048,7 @@ void Waf2Transaction::scanHeaders()
// Scan relevant headers for attacks
for (auto it = hdrs_map.begin(); it != hdrs_map.end(); ++it)
{
scanSpecificHeder(it->first.c_str(), it->first.size(),
scanSpecificHeader(it->first.c_str(), it->first.size(),
it->second.c_str(), it->second.size());
}
}
@ -1171,6 +1265,12 @@ void Waf2Transaction::end_request() {
// Enable response headers processing if response scanning is enabled in policy
auto errorDisclosurePolicy = m_siteConfig ? m_siteConfig->get_ErrorDisclosurePolicy() : NULL;
m_responseInspectReasons.setErrorDisclosure(errorDisclosurePolicy && errorDisclosurePolicy->enable);
auto triggerPolicy = m_siteConfig ? m_siteConfig->get_TriggerPolicy() : NULL;
if (isTriggerReportExists(triggerPolicy)) {
m_responseInspectReasons.setTriggerReport(true);
dbgTrace(D_WAAP) << "setTriggerReport(true)";
}
}
void Waf2Transaction::extractEnvSourceIdentifier()
@ -1393,6 +1493,7 @@ Waf2Transaction::decideAfterHeaders()
return false;
}
m_isHeaderOverrideScanRequired = true;
m_overrideState = getOverrideState(sitePolicy);
// Select scores pool by location (but use forced pool when forced)
@ -1648,6 +1749,12 @@ Waf2Transaction::sendLog()
const auto& autonomousSecurityDecision = std::dynamic_pointer_cast<AutonomousSecurityDecision>(
m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
I_TimeGet *timeGet = Singleton::Consume<I_TimeGet>::by<Waf2Transaction>();
auto finish = timeGet->getMonotonicTime();
auto diff = chrono::duration_cast<chrono::milliseconds>(finish) - m_entry_time;
telemetryData.elapsedTime = diff.count();
dbgTrace(D_WAAP) << "latency updated, time: " << diff.count() << "ms";
if (m_methodStr == "POST") {
telemetryData.method = POST;
} else if (m_methodStr == "GET") {

17
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@ -38,6 +38,7 @@
#include "i_transaction.h"
#include "i_waap_telemetry.h"
#include "i_deepAnalyzer.h"
#include "i_time_get.h"
#include "table_opaque.h"
#include "WaapResponseInspectReasons.h"
#include "WaapResponseInjectReasons.h"
@ -45,6 +46,7 @@
#include "WaapOpenRedirectPolicy.h"
#include "WaapScanner.h"
#include "singleton.h"
#include "OASchemaUpdaterConfConstant.h"
struct DecisionTelemetryData;
class Waf2Transaction;
@ -60,6 +62,7 @@ class Waf2Transaction :
public Singleton::Consume<I_Table>,
private boost::noncopyable,
Singleton::Consume<I_AgentDetails>,
Singleton::Consume<I_TimeGet>,
Singleton::Consume<I_Environment>
{
public:
@ -76,6 +79,7 @@ public:
void set_host(const char *host);
// getters
uint64_t getElapsedTime() const;
const std::string& getRemoteAddr() const;
virtual const std::string getUri() const;
const std::string getUriStr() const;
@ -143,6 +147,7 @@ public:
void start_response_body();
void add_response_body_chunk(const char* data, int data_len);
void end_response_body();
const std::string & getResponseBody();
void end_response();
void extractEnvSourceIdentifier();
void finish();
@ -212,19 +217,21 @@ public:
}
// LCOV_EXCL_STOP
bool reportScanResult(const Waf2ScanResult &res);
bool shouldIgnoreOverride(const Waf2ScanResult &res);
Waap::OpenRedirect::State &getOpenRedirectState() { return m_openRedirectState; }
IWaapConfig* getSiteConfig() { return m_siteConfig; }
void addNote(const std::string &note) { m_notes.push_back(note); }
const std::string &getResponseBody(void) const { return m_response_body; }
Waap::ResponseInspectReasons &getResponseInspectReasons(void) { return m_responseInspectReasons; }
// LCOV_EXCL_START Reason: This function is tested in system tests
bool checkIsHeaderOverrideScanRequired();
// LCOV_EXCL_STOP
private:
int finalizeDecision(IWaapConfig *sitePolicy, bool shouldBlock);
const std::shared_ptr<Waap::Trigger::Log> getTriggerLog(const std::shared_ptr<Waap::Trigger::Policy>&
triggerPolicy) const;
bool isTriggerReportExists(const std::shared_ptr<Waap::Trigger::Policy> &triggerPolicy);
void sendAutonomousSecurityLog(
const std::shared_ptr<Waap::Trigger::Log>& triggerLog,
bool shouldBlock,
@ -261,15 +268,17 @@ private:
void parseContentType(const char* value, int value_len);
void parseCookie(const char* value, int value_len);
void parseReferer(const char* value, int value_len);
void parseAuthorization(const char* value, int value_len, const std::string &header_name);
void parseUnknownHeaderName(const char* name, int name_len);
void parseGenericHeaderValue(const std::string &headerName, const char* value, int value_len);
void scanSpecificHeder(const char* name, int name_len, const char* value, int value_len);
void scanSpecificHeader(const char* name, int name_len, const char* value, int value_len);
void detectSpecificHeader(const char* name, int name_len, const char* value, int value_len);
void detectHeaders();
void scanHeaders();
void clearRequestParserState();
void scanErrDisclosureBuffer();
std::chrono::milliseconds m_entry_time;
std::shared_ptr<WaapAssetState> m_pWaapAssetState;
bool m_ignoreScore; // override the scoring filter and (effectively) take the last suspicious parameter,
// instead of the one with highest score that is > SCORE_THRESHOLD
@ -334,6 +343,7 @@ private:
bool m_processedUri;
bool m_processedHeaders;
bool m_isHeaderOverrideScanRequired;
bool m_isScanningRequired;
int m_responseStatus;
Waap::ResponseInspectReasons m_responseInspectReasons;
@ -345,6 +355,7 @@ private:
// Cached pointer to const triggerLog (hence mutable)
mutable std::shared_ptr<Waap::Trigger::Log> m_triggerLog;
bool m_triggerReport;
bool is_schema_validation = false;
Waf2TransactionFlags m_waf2TransactionFlags;
};

23
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@ -551,6 +551,11 @@ const std::set<std::string> Waf2Transaction::getFoundPatterns() const
return m_found_patterns;
}
bool Waf2Transaction::checkIsHeaderOverrideScanRequired()
{
return m_isHeaderOverrideScanRequired;
}
Waap::Override::State Waf2Transaction::getOverrideState(IWaapConfig* sitePolicy)
{
Waap::Override::State overrideState;
@ -566,6 +571,7 @@ Waap::Override::State Waf2Transaction::getOverrideState(IWaapConfig* sitePolicy)
if (overridePolicy) { // later we will run response overrides
overrideStateResponse.applyOverride(*overridePolicy, WaapOverrideFunctor(*this), m_matchedOverrideIds, false);
}
m_isHeaderOverrideScanRequired = false;
return overrideStateResponse;
}
@ -592,6 +598,23 @@ const std::shared_ptr<Waap::Trigger::Log> Waf2Transaction::getTriggerLog(const s
return m_triggerLog;
}
bool Waf2Transaction::isTriggerReportExists(const std::shared_ptr<
Waap::Trigger::Policy> &triggerPolicy)
{
if (!triggerPolicy) {
return false;
}
if (m_triggerReport) {
return m_triggerReport;
}
for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) {
if (trigger.triggerType == "report") {
return m_triggerReport = true;
}
}
return m_triggerReport;
}
ReportIS::Severity Waf2Transaction::computeEventSeverityFromDecision() const
{
DecisionType type = m_waapDecision.getHighestPriorityDecisionToLog();

7
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@ -732,6 +732,13 @@ inline void replaceAll(std::string& str, const std::string& from, const std::str
start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx'
}
}
inline void alignBase64Chunk (std::string &chunk)
{
size_t len = chunk.length() % 4;
if (len >= 2) {
chunk.append(4-len, '=');
}
}
// Count items in v that are not in ignored_set
inline size_t countNotInSet(const std::vector<std::string> &v, const std::set<std::string> &ignored_set) {

15
components/security_apps/waap/waap_component_impl.cc
View File

@ -170,11 +170,15 @@ WaapComponent::Impl::respond(const NewHttpTransactionEvent &event)
waf2Transaction.start();
char sourceIpStr[INET_ADDRSTRLEN];
inet_ntop(AF_INET, &(event.getSourceIP()), sourceIpStr, INET_ADDRSTRLEN);
char listeningIpStr[INET_ADDRSTRLEN];
inet_ntop(AF_INET, &(event.getListeningIP()), listeningIpStr, INET_ADDRSTRLEN);
char sourceIpStr[INET6_ADDRSTRLEN];
char listeningIpStr[INET6_ADDRSTRLEN];
if (event.getSourceIP().getType() == IPType::V4) {
inet_ntop(AF_INET, &(event.getSourceIP()), sourceIpStr, INET6_ADDRSTRLEN);
inet_ntop(AF_INET, &(event.getListeningIP()), listeningIpStr, INET6_ADDRSTRLEN);
} else {
inet_ntop(AF_INET6, &(event.getSourceIP()), sourceIpStr, INET6_ADDRSTRLEN);
inet_ntop(AF_INET6, &(event.getListeningIP()), listeningIpStr, INET6_ADDRSTRLEN);
}
// Set envelope data
waf2Transaction.set_transaction_remote(sourceIpStr, event.getSourcePort());
@ -515,7 +519,6 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event)
dbgTrace(D_WAAP) << "HttpBodyResponse";
// Push the response data chunk to the waf2 engine
const char *dataBuf = (const char*)event.getData().data();
size_t dataBufLen = event.getData().size();

1
components/utils/CMakeLists.txt
View File

@ -1,3 +1,4 @@
add_subdirectory(generic_rulebase)
add_subdirectory(geo_location)
add_subdirectory(http_transaction_data)
add_subdirectory(ip_utilities)

0
components/generic_rulebase/CMakeLists.txt → components/utils/generic_rulebase/CMakeLists.txt
View File

137
components/utils/generic_rulebase/assets_config.cc Normal file
View File

@ -0,0 +1,137 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/assets_config.h"
#include <string>
#include <algorithm>
#include <unordered_map>
#include "generic_rulebase/generic_rulebase_utils.h"
#include "config.h"
#include "debug.h"
#include "ip_utilities.h"
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
using namespace std;
void
RuleAsset::load(cereal::JSONInputArchive &archive_in)
{
archive_in(cereal::make_nvp("assetId", asset_id));
archive_in(cereal::make_nvp("assetName", asset_name));
archive_in(cereal::make_nvp("assetUrls", asset_urls));
dbgWarning(D_RULEBASE_CONFIG) << "Adding asset with UID: " << asset_id;
}
void
RuleAsset::AssetUrl::load(cereal::JSONInputArchive &archive_in)
{
archive_in(cereal::make_nvp("protocol", protocol));
transform(protocol.begin(), protocol.end(), protocol.begin(), [](unsigned char c) { return tolower(c); });
archive_in(cereal::make_nvp("ip", ip));
archive_in(cereal::make_nvp("port", port));
int value;
if (protocol == "*") {
is_any_proto = true;
} else {
is_any_proto = false;
try {
value = 0;
if(protocol == "udp") value = IPPROTO_UDP;
if(protocol == "tcp") value = IPPROTO_TCP;
if(protocol == "dccp") value = IPPROTO_DCCP;
if(protocol == "sctp") value = IPPROTO_SCTP;
if(protocol == "icmp") value = IPPROTO_ICMP;
if(protocol == "icmpv6") value = IPPROTO_ICMP;
if (value > static_cast<int>(UINT8_MAX) || value < 0) {
dbgWarning(D_RULEBASE_CONFIG)
<< "provided value is not a legal IP protocol number. Value: "
<< protocol;
} else {
parsed_proto = value;
}
} catch (...) {
dbgWarning(D_RULEBASE_CONFIG) << "provided value is not a legal IP protocol. Value: " << protocol;
}
}
if (port == "*") {
is_any_port = true;
} else {
is_any_port = false;
try {
value = stoi(port);
if (value > static_cast<int>(UINT16_MAX) || value < 0) {
dbgWarning(D_RULEBASE_CONFIG) << "provided value is not a legal port number. Value: " << port;
} else {
parsed_port = value;
}
} catch (...) {
dbgWarning(D_RULEBASE_CONFIG) << "provided value is not a legal port. Value: " << port;
}
}
if (ip == "*") {
is_any_ip = true;
} else {
is_any_ip = false;
auto ip_addr = IPAddr::createIPAddr(ip);
if (!ip_addr.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Could not create IP address. Error: " << ip_addr.getErr();
} else {
parsed_ip = ConvertToIpAddress(ip_addr.unpackMove());
}
}
}
IpAddress
RuleAsset::AssetUrl::ConvertToIpAddress(const IPAddr &addr)
{
IpAddress address;
switch (addr.getType()) {
case IPType::UNINITIALIZED: {
address.addr4_t = {0};
address.ip_type = IP_VERSION_ANY;
break;
}
case IPType::V4: {
address.addr4_t = addr.getIPv4();
address.ip_type = IP_VERSION_4;
break;
}
case IPType::V6: {
address.addr6_t = addr.getIPv6();
address.ip_type = IP_VERSION_6;
break;
}
default:
address.addr4_t = {0};
address.ip_type = IP_VERSION_ANY;
dbgWarning(D_RULEBASE_CONFIG) << "Unsupported IP type: " << static_cast<int>(addr.getType());
}
return address;
}
const Assets Assets::empty_assets_config = Assets();
void
Assets::preload()
{
registerExpectedSetting<Assets>("rulebase", "usedAssets");
}

0
components/generic_rulebase/evaluators/CMakeLists.txt → components/utils/generic_rulebase/evaluators/CMakeLists.txt
View File

52
components/utils/generic_rulebase/evaluators/asset_eval.cc Normal file
View File

@ -0,0 +1,52 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/evaluators/asset_eval.h"
#include <vector>
#include <string>
#include "generic_rulebase/assets_config.h"
#include "config.h"
#include "debug.h"
using namespace std;
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
string AssetMatcher::ctx_key = "asset_id";
AssetMatcher::AssetMatcher(const vector<string> &params)
{
if (params.size() != 1) reportWrongNumberOfParams(AssetMatcher::getName(), params.size(), 1, 1);
asset_id = params[0];
}
Maybe<bool, Context::Error>
AssetMatcher::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<AssetMatcher>();
auto bc_asset_id_ctx = env->get<GenericConfigId>(AssetMatcher::ctx_key);
if (bc_asset_id_ctx.ok()) {
dbgTrace(D_RULEBASE_CONFIG)
<< "Asset ID: "
<< asset_id
<< "; Current set assetId context: "
<< *bc_asset_id_ctx;
} else {
dbgTrace(D_RULEBASE_CONFIG) << "Asset ID: " << asset_id << ". Empty context";
}
return bc_asset_id_ctx.ok() && *bc_asset_id_ctx == asset_id;
}

299
components/utils/generic_rulebase/evaluators/connection_eval.cc Normal file
View File

@ -0,0 +1,299 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/evaluators/connection_eval.h"
#include <vector>
#include <string>
#include "generic_rulebase/rulebase_config.h"
#include "config.h"
#include "debug.h"
#include "ip_utilities.h"
using namespace std;
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
string IpAddressMatcher::ctx_key = "ipAddress";
string SourceIpMatcher::ctx_key = "sourceIP";
string DestinationIpMatcher::ctx_key = "destinationIP";
string SourcePortMatcher::ctx_key = "sourcePort";
string ListeningPortMatcher::ctx_key = "listeningPort";
string IpProtocolMatcher::ctx_key = "ipProtocol";
string UrlMatcher::ctx_key = "url";
Maybe<IPAddr>
getIpAddrFromEnviroment(I_Environment *env, Context::MetaDataType enum_data_type, const string &str_data_type)
{
auto ip_str = env->get<string>(enum_data_type);
if (!ip_str.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to get " << str_data_type << " from the enviroment.";
return genError("Failed to get " + str_data_type + " from the enviroment.");
}
return IPAddr::createIPAddr(ip_str.unpack());
}
bool
checkIfIpInRangesVec(const vector<CustomRange<IPAddr>> &values, const IPAddr &ip_to_check)
{
if (values.size() == 0) {
dbgTrace(D_RULEBASE_CONFIG) << "Ip addersses vector empty. Match is true.";
return true;
}
for (const CustomRange<IPAddr> &range : values) {
if (range.contains(ip_to_check)) {
dbgTrace(D_RULEBASE_CONFIG) << "Ip adderss matched: " << ip_to_check;
return true;
}
}
dbgTrace(D_RULEBASE_CONFIG) << "Ip adderss not match: " << ip_to_check;
return false;
}
IpAddressMatcher::IpAddressMatcher(const vector<string> &params)
{
for (const string &param : params) {
Maybe<CustomRange<IPAddr>> ip_range = CustomRange<IPAddr>::createRange(param);
if (!ip_range.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to create ip. Error: " + ip_range.getErr();
continue;
}
values.push_back(ip_range.unpack());
}
}
Maybe<bool, Context::Error>
IpAddressMatcher::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<IpAddressMatcher>();
Maybe<IPAddr> subject_ip = getIpAddrFromEnviroment(
env,
Context::MetaDataType::SubjectIpAddr,
"subject ip address"
);
if (subject_ip.ok() && checkIfIpInRangesVec(values, subject_ip.unpack())) return true;
Maybe<IPAddr> other_ip = getIpAddrFromEnviroment(
env,
Context::MetaDataType::OtherIpAddr,
"other ip address"
);
if (other_ip.ok() && checkIfIpInRangesVec(values, other_ip.unpack())) return true;
if (!subject_ip.ok() && !other_ip.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Error in getting subject ip and other ip from the enviroment";
return false;
}
dbgTrace(D_RULEBASE_CONFIG) << "Ip adderss didn't match";
return false;
}
SourceIpMatcher::SourceIpMatcher(const vector<string> &params)
{
for (const string &param : params) {
Maybe<CustomRange<IPAddr>> ip_range = CustomRange<IPAddr>::createRange(param);
if (!ip_range.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to create source ip. Error: " + ip_range.getErr();
continue;
}
values.push_back(ip_range.unpack());
}
}
Maybe<bool, Context::Error>
SourceIpMatcher::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<SourceIpMatcher>();
auto direction_maybe = env->get<string>(Context::MetaDataType::Direction);
if (!direction_maybe.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to get direction from the enviroment.";
return false;
}
string direction = direction_maybe.unpack();
if (direction == "incoming") {
Maybe<IPAddr> other_ip = getIpAddrFromEnviroment(
env,
Context::MetaDataType::OtherIpAddr,
"other ip address"
);
return other_ip.ok() && checkIfIpInRangesVec(values, other_ip.unpack());
} else if (direction == "outgoing") {
Maybe<IPAddr> subject_ip = getIpAddrFromEnviroment(
env,
Context::MetaDataType::SubjectIpAddr,
"subject ip address"
);
return subject_ip.ok() && checkIfIpInRangesVec(values, subject_ip.unpack());
}
dbgTrace(D_RULEBASE_CONFIG) << "Source ip adderss didn't match";
return false;
}
DestinationIpMatcher::DestinationIpMatcher(const vector<string> &params)
{
for (const string &param : params) {
Maybe<CustomRange<IPAddr>> ip_range = CustomRange<IPAddr>::createRange(param);
if (!ip_range.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to create destination ip. Error: " + ip_range.getErr();
continue;
}
values.push_back(ip_range.unpack());
}
}
Maybe<bool, Context::Error>
DestinationIpMatcher::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<DestinationIpMatcher>();
auto direction_maybe = env->get<string>(Context::MetaDataType::Direction);
if (!direction_maybe.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to get direction.";
return false;
}
string direction = direction_maybe.unpack();
if (direction == "outgoing") {
Maybe<IPAddr> other_ip = getIpAddrFromEnviroment(
env,
Context::MetaDataType::OtherIpAddr,
"other ip address"
);
return other_ip.ok() && checkIfIpInRangesVec(values, other_ip.unpack());
} else if (direction == "incoming") {
Maybe<IPAddr> subject_ip = getIpAddrFromEnviroment(
env,
Context::MetaDataType::SubjectIpAddr,
"subject ip address"
);
return subject_ip.ok() && checkIfIpInRangesVec(values, subject_ip.unpack());
}
dbgTrace(D_RULEBASE_CONFIG) << "Destination ip adderss didn't match";
return false;
}
SourcePortMatcher::SourcePortMatcher(const vector<string> &params)
{
for (const string &param : params) {
Maybe<CustomRange<PortNumber>> port_range = CustomRange<PortNumber>::createRange(param);
if (!port_range.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to create source port.";
continue;
}
values.push_back(port_range.unpack());
}
}
Maybe<bool, Context::Error>
SourcePortMatcher::evalVariable() const
{
dbgTrace(D_RULEBASE_CONFIG) << "Source is not a match";
return false;
}
ListeningPortMatcher::ListeningPortMatcher(const vector<string> &params)
{
for (const string &param : params) {
Maybe<CustomRange<PortNumber>> port_range = CustomRange<PortNumber>::createRange(param);
if (!port_range.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to create listening port range.";
continue;
}
values.push_back(port_range.unpack());
}
}
Maybe<bool, Context::Error>
ListeningPortMatcher::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<ListeningPortMatcher>();
auto port_str = env->get<string>(Context::MetaDataType::Port);
if (!port_str.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to get port from the enviroment.";
return false;
}
PortNumber port;
if (ConnKeyUtil::fromString(port_str.unpack(), port)) {
if (values.size() == 0) return true;
for (const CustomRange<PortNumber> &port_range : values) {
if (port_range.contains(port)) {
dbgTrace(D_RULEBASE_CONFIG) << "Listening port is a match. Value: " << port_str.unpack();
return true;
}
}
}
dbgTrace(D_RULEBASE_CONFIG) << "Listening port is not a match. Value: " << port_str.unpack();
return false;
}
IpProtocolMatcher::IpProtocolMatcher(const vector<string> &params)
{
for (const string &param : params) {
Maybe<CustomRange<IPProto>> proto_range = CustomRange<IPProto>::createRange(param);
if (!proto_range.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to create ip protocol.";
continue;
}
values.push_back(proto_range.unpack());
}
}
Maybe<bool, Context::Error>
IpProtocolMatcher::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<IpProtocolMatcher>();
auto proto_str = env->get<string>(Context::MetaDataType::Protocol);
if (!proto_str.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to get ip protocol from the enviroment.";
return false;
}
IPProto protocol;
if (ConnKeyUtil::fromString(proto_str.unpack(), protocol)) {
if (values.size() == 0) return true;
for (const CustomRange<IPProto> &proto_range : values) {
if (proto_range.contains(protocol)) {
dbgTrace(D_RULEBASE_CONFIG) << "Ip protocol is a match. Value: " << proto_str.unpack();
return true;
}
}
}
dbgTrace(D_RULEBASE_CONFIG) << "Source port is not a match. Value: " << proto_str.unpack();
return false;
}
UrlMatcher::UrlMatcher(const vector<string> &params) : values(params) {}
Maybe<bool, Context::Error>
UrlMatcher::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<UrlMatcher>();
auto curr_url_ctx = env->get<string>(Context::MetaDataType::Url);
if (!curr_url_ctx.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to get URL from the enviroment.";
return false;
}
if (values.size() == 0) {
dbgTrace(D_RULEBASE_CONFIG) << "Matched URL on \"any\". Url: " << *curr_url_ctx;
return true;
}
for (const string &url : values) {
if (*curr_url_ctx == url) {
dbgTrace(D_RULEBASE_CONFIG) << "Matched URL. Value: " << *curr_url_ctx;
return true;
}
}
dbgTrace(D_RULEBASE_CONFIG) << "URL is not a match. Value: " << *curr_url_ctx;
return false;
}

168
components/utils/generic_rulebase/evaluators/http_transaction_data_eval.cc Normal file
View File

@ -0,0 +1,168 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/evaluators/http_transaction_data_eval.h"
#include <boost/lexical_cast.hpp>
#include <algorithm>
#include "http_transaction_data.h"
#include "environment/evaluator_templates.h"
#include "i_environment.h"
#include "singleton.h"
#include "debug.h"
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
using namespace std;
using namespace EnvironmentHelper;
EqualHost::EqualHost(const vector<string> &params)
{
if (params.size() != 1) reportWrongNumberOfParams("EqualHost", params.size(), 1, 1);
host = params[0];
}
Maybe<bool, Context::Error>
EqualHost::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<EqualHost>();
auto host_ctx = env->get<string>(HttpTransactionData::host_name_ctx);
if (!host_ctx.ok())
{
return false;
}
std::string lower_host_ctx = host_ctx.unpack();
std::transform(lower_host_ctx.begin(), lower_host_ctx.end(), lower_host_ctx.begin(), ::tolower);
std::string lower_host = host;
std::transform(lower_host.begin(), lower_host.end(), lower_host.begin(), ::tolower);
if (lower_host_ctx == lower_host) return true;
size_t pos = lower_host_ctx.find_last_of(':');
if (pos == string::npos) return false;
lower_host_ctx = string(lower_host_ctx.data(), pos);
return lower_host_ctx == lower_host;
}
WildcardHost::WildcardHost(const vector<string> &params)
{
if (params.size() != 1) reportWrongNumberOfParams("WildcardHost", params.size(), 1, 1);
host = params[0];
}
Maybe<bool, Context::Error>
WildcardHost::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<WildcardHost>();
auto host_ctx = env->get<string>(HttpTransactionData::host_name_ctx);
if (!host_ctx.ok())
{
return false;
}
string lower_host_ctx = host_ctx.unpack();
transform(lower_host_ctx.begin(), lower_host_ctx.end(), lower_host_ctx.begin(), ::tolower);
dbgTrace(D_RULEBASE_CONFIG) << "found host in current context: " << lower_host_ctx;
size_t pos = lower_host_ctx.find_first_of(".");
if (pos == string::npos) {
return false;
}
lower_host_ctx = "*" + lower_host_ctx.substr(pos, lower_host_ctx.length());
string lower_host = host;
transform(lower_host.begin(), lower_host.end(), lower_host.begin(), ::tolower);
dbgTrace(D_RULEBASE_CONFIG)
<< "trying to match host context with its corresponding wildcard address: "
<< lower_host_ctx
<< ". Matcher host: "
<< lower_host;
if (lower_host_ctx == lower_host) return true;
pos = lower_host_ctx.find_last_of(':');
if (pos == string::npos) return false;
lower_host_ctx = string(lower_host_ctx.data(), pos);
return lower_host_ctx == lower_host;
}
EqualListeningIP::EqualListeningIP(const vector<string> &params)
{
if (params.size() != 1) reportWrongNumberOfParams("EqualListeningIP", params.size(), 1, 1);
auto maybe_ip = IPAddr::createIPAddr(params[0]);
if (!maybe_ip.ok()) reportWrongParamType(getName(), params[0], "Not a valid IP Address");
listening_ip = maybe_ip.unpack();
}
Maybe<bool, Context::Error>
EqualListeningIP::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<EqualListeningIP>();
auto listening_ip_ctx = env->get<IPAddr>(HttpTransactionData::listening_ip_ctx);
return listening_ip_ctx.ok() && listening_ip_ctx.unpack() == listening_ip;
}
EqualListeningPort::EqualListeningPort(const vector<string> &params)
{
if (params.size() != 1) reportWrongNumberOfParams("EqualListeningPort", params.size(), 1, 1);
try {
listening_port = boost::lexical_cast<PortNumber>(params[0]);
} catch (boost::bad_lexical_cast const&) {
reportWrongParamType(getName(), params[0], "Not a valid port number");
}
}
Maybe<bool, Context::Error>
EqualListeningPort::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<EqualListeningPort>();
auto port_ctx = env->get<PortNumber>(HttpTransactionData::listening_port_ctx);
return port_ctx.ok() && port_ctx.unpack() == listening_port;
}
BeginWithUri::BeginWithUri(const vector<string> &params)
{
if (params.size() != 1) reportWrongNumberOfParams("BeginWithUri", params.size(), 1, 1);
uri_prefix = params[0];
}
Maybe<bool, Context::Error>
BeginWithUri::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<BeginWithUri>();
auto uri_ctx = env->get<string>(HttpTransactionData::uri_ctx);
if (!uri_ctx.ok())
{
return false;
}
std::string lower_uri_ctx = uri_ctx.unpack();
std::transform(lower_uri_ctx.begin(), lower_uri_ctx.end(), lower_uri_ctx.begin(), ::tolower);
std::string lower_uri_prefix = uri_prefix;
std::transform(lower_uri_prefix.begin(), lower_uri_prefix.end(), lower_uri_prefix.begin(), ::tolower);
return lower_uri_ctx.find(lower_uri_prefix) == 0;
}

38
components/utils/generic_rulebase/evaluators/parameter_eval.cc Normal file
View File

@ -0,0 +1,38 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/evaluators/parameter_eval.h"
#include <vector>
#include <string>
#include "generic_rulebase/rulebase_config.h"
#include "config.h"
#include "debug.h"
using namespace std;
string ParameterMatcher::ctx_key = "parameters";
ParameterMatcher::ParameterMatcher(const vector<string> &params)
{
if (params.size() != 1) reportWrongNumberOfParams(ParameterMatcher::getName(), params.size(), 1, 1);
parameter_id = params[0];
}
Maybe<bool, Context::Error>
ParameterMatcher::evalVariable() const
{
auto rule = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
return rule.ok() && rule.unpack().isParameterActive(parameter_id);
}

50
components/utils/generic_rulebase/evaluators/practice_eval.cc Normal file
View File

@ -0,0 +1,50 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/evaluators/practice_eval.h"
#include <vector>
#include <string>
#include "generic_rulebase/rulebase_config.h"
#include "config.h"
#include "debug.h"
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
using namespace std;
string PracticeMatcher::ctx_key = "practices";
PracticeMatcher::PracticeMatcher(const vector<string> &params)
{
if (params.size() != 1) reportWrongNumberOfParams(PracticeMatcher::getName(), params.size(), 1, 1);
practice_id = params[0];
}
Maybe<bool, Context::Error>
PracticeMatcher::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<PracticeMatcher>();
auto bc_practice_id_ctx = env->get<set<GenericConfigId>>(PracticeMatcher::ctx_key);
dbgTrace(D_RULEBASE_CONFIG)
<< "Trying to match practice. ID: "
<< practice_id << ", Current set IDs: "
<< makeSeparatedStr(bc_practice_id_ctx.ok() ? *bc_practice_id_ctx : set<GenericConfigId>(), ", ");
if (bc_practice_id_ctx.ok()) {
return bc_practice_id_ctx.unpack().count(practice_id) > 0;
}
auto rule = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
return rule.ok() && rule.unpack().isPracticeActive(practice_id);
}

136
components/utils/generic_rulebase/evaluators/query_eval.cc Normal file
View File

@ -0,0 +1,136 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/evaluators/query_eval.h"
#include <vector>
#include <string>
#include <map>
#include "generic_rulebase/rulebase_config.h"
#include "generic_rulebase/zones_config.h"
#include "i_environment.h"
#include "singleton.h"
#include "config.h"
#include "debug.h"
#include "enum_range.h"
using namespace std;
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
QueryMatcher::QueryMatcher(const vector<string> &params)
{
if (params.size() < 1) reportWrongNumberOfParams(QueryMatcher::getName(), params.size(), 1);
key = params.front();
if (key == "any") {
is_any = true;
} else {
values.reserve(params.size() - 1);
for (uint i = 1; i < params.size() ; i++) {
if (params[i] == "any") {
values.clear();
break;
}
values.insert(params[i]);
}
}
}
const string
QueryMatcher::contextKeyToString(Context::MetaDataType type)
{
if (type == Context::MetaDataType::SubjectIpAddr || type == Context::MetaDataType::OtherIpAddr) return "ip";
return Context::convertToString(type);
}
class QueryMatchSerializer
{
public:
static const string req_attr_ctx_key;
template <typename Archive>
void
serialize(Archive &ar)
{
I_Environment *env = Singleton::Consume<I_Environment>::by<QueryMatcher>();
auto req_attr = env->get<string>(req_attr_ctx_key);
if (!req_attr.ok()) return;
try {
ar(cereal::make_nvp(*req_attr, value));
dbgDebug(D_RULEBASE_CONFIG)
<< "Found value for requested attribute. Tag: "
<< *req_attr
<< ", Value: "
<< value;
} catch (exception &e) {
dbgDebug(D_RULEBASE_CONFIG) << "Could not find values for requested attribute. Tag: " << *req_attr;
ar.finishNode();
}
}
template <typename Values>
bool
matchValues(const Values &requested_vals) const
{
return value != "" && (requested_vals.empty() || requested_vals.count(value) > 0);
}
private:
string value;
};
const string QueryMatchSerializer::req_attr_ctx_key = "requested attribute key";
Maybe<bool, Context::Error>
QueryMatcher::evalVariable() const
{
if (is_any) return true;
I_Environment *env = Singleton::Consume<I_Environment>::by<QueryMatcher>();
auto local_asset_ctx = env->get<bool>("is local asset");
bool is_remote_asset = local_asset_ctx.ok() && !(*local_asset_ctx);
QueryRequest request;
for (Context::MetaDataType name : makeRange<Context::MetaDataType>()) {
auto val = env->get<string>(name);
if (val.ok()) {
if ((name == Context::MetaDataType::SubjectIpAddr && is_remote_asset) ||
(name == Context::MetaDataType::OtherIpAddr && !is_remote_asset)) {
continue;
}
request.addCondition(Condition::EQUALS, contextKeyToString(name), *val);
}
}
if (request.empty()) return false;
request.setRequestedAttr(key);
ScopedContext req_attr_key;
req_attr_key.registerValue<string>(QueryMatchSerializer::req_attr_ctx_key, key);
I_Intelligence_IS_V2 *intelligence = Singleton::Consume<I_Intelligence_IS_V2>::by<Zone>();
auto query_res = intelligence->queryIntelligence<QueryMatchSerializer>(request);
if (!query_res.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to perform intelligence query. Error: " << query_res.getErr();
return false;
}
for (const AssetReply<QueryMatchSerializer> &asset : query_res.unpack()) {
if (asset.matchValues<unordered_set<string>>(values)) return true;
}
return false;
}

57
components/utils/generic_rulebase/evaluators/trigger_eval.cc Normal file
View File

@ -0,0 +1,57 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/evaluators/trigger_eval.h"
#include <vector>
#include <string>
#include "generic_rulebase/rulebase_config.h"
#include "config.h"
#include "debug.h"
using namespace std;
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
string TriggerMatcher::ctx_key = "triggers";
TriggerMatcher::TriggerMatcher(const vector<string> &params)
{
if (params.size() != 1) reportWrongNumberOfParams(TriggerMatcher::getName(), params.size(), 1, 1);
trigger_id = params[0];
}
Maybe<bool, Context::Error>
TriggerMatcher::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<TriggerMatcher>();
auto ac_bc_trigger_id_ctx = env->get<set<GenericConfigId>>("ac_trigger_id");
dbgTrace(D_RULEBASE_CONFIG)
<< "Trying to match trigger for access control rule. ID: "
<< trigger_id << ", Current set IDs: "
<< makeSeparatedStr(ac_bc_trigger_id_ctx.ok() ? *ac_bc_trigger_id_ctx : set<GenericConfigId>(), ", ");
if (ac_bc_trigger_id_ctx.ok()) {
return ac_bc_trigger_id_ctx.unpack().count(trigger_id) > 0;
}
auto bc_trigger_id_ctx = env->get<set<GenericConfigId>>(TriggerMatcher::ctx_key);
dbgTrace(D_RULEBASE_CONFIG)
<< "Trying to match trigger. ID: "
<< trigger_id << ", Current set IDs: "
<< makeSeparatedStr(bc_trigger_id_ctx.ok() ? *bc_trigger_id_ctx : set<GenericConfigId>(), ", ");
if (bc_trigger_id_ctx.ok() && bc_trigger_id_ctx.unpack().count(trigger_id) > 0 ) return true;
auto rule = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
return rule.ok() && rule.unpack().isTriggerActive(trigger_id);
}

44
components/utils/generic_rulebase/evaluators/zone_eval.cc Normal file
View File

@ -0,0 +1,44 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/evaluators/zone_eval.h"
#include <vector>
#include <string>
#include "generic_rulebase/zone.h"
#include "generic_rulebase/rulebase_config.h"
#include "config.h"
using namespace std;
string ZoneMatcher::ctx_key = "zone_id";
ZoneMatcher::ZoneMatcher(const vector<string> &params)
{
if (params.size() != 1) reportWrongNumberOfParams(ZoneMatcher::getName(), params.size(), 1, 1);
zone_id = params[0];
}
Maybe<bool, Context::Error>
ZoneMatcher::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<ZoneMatcher>();
auto bc_zone_id_ctx = env->get<GenericConfigId>(ZoneMatcher::ctx_key);
if (bc_zone_id_ctx.ok() && *bc_zone_id_ctx == zone_id) return true;
if (!getProfileAgentSettingWithDefault<bool>(false, "rulebase.enableQueryBasedMatch")) return false;
auto zone = getConfiguration<Zone>("rulebase", "zones");
return zone.ok() && zone.unpack().getId() == zone_id;
}

149
components/utils/generic_rulebase/generic_rulebase.cc Normal file
View File

@ -0,0 +1,149 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/generic_rulebase.h"
#include <unordered_set>
#include "generic_rulebase/evaluators/trigger_eval.h"
#include "generic_rulebase/evaluators/practice_eval.h"
#include "generic_rulebase/evaluators/parameter_eval.h"
#include "generic_rulebase/evaluators/zone_eval.h"
#include "generic_rulebase/evaluators/asset_eval.h"
#include "generic_rulebase/evaluators/query_eval.h"
#include "generic_rulebase/evaluators/connection_eval.h"
#include "generic_rulebase/evaluators/http_transaction_data_eval.h"
#include "generic_rulebase/zone.h"
#include "generic_rulebase/triggers_config.h"
#include "generic_rulebase/parameters_config.h"
#include "singleton.h"
#include "common.h"
#include "debug.h"
#include "cache.h"
#include "config.h"
#include "i_environment.h"
using namespace std;
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
class GenericRulebase::Impl : Singleton::Provide<I_GenericRulebase>::From<GenericRulebase>
{
public:
void init() {}
void fini() {}
void preload();
Maybe<Zone, Config::Errors> getLocalZone() const override { return getZoneConfig(true); }
Maybe<Zone, Config::Errors> getOtherZone() const override { return getZoneConfig(false); }
LogTriggerConf getLogTriggerConf(const string &trigger_Id) const override;
ParameterException getParameterException(const string &parameter_Id) const override;
set<ParameterBehavior> getBehavior(const ParameterKeyValues &key_value_pairs) const override;
private:
Maybe<Zone, Config::Errors>
getZoneConfig(bool is_local_zone) const
{
ScopedContext asset_location_ctx;
asset_location_ctx.registerValue<bool>("is local asset", is_local_zone);
return getConfiguration<Zone>("rulebase", "zones");
}
};
void
GenericRulebase::Impl::preload()
{
addMatcher<TriggerMatcher>();
addMatcher<PracticeMatcher>();
addMatcher<ParameterMatcher>();
addMatcher<ZoneMatcher>();
addMatcher<AssetMatcher>();
addMatcher<QueryMatcher>();
addMatcher<IpAddressMatcher>();
addMatcher<SourceIpMatcher>();
addMatcher<DestinationIpMatcher>();
addMatcher<SourcePortMatcher>();
addMatcher<ListeningPortMatcher>();
addMatcher<IpProtocolMatcher>();
addMatcher<UrlMatcher>();
addMatcher<EqualHost>();
addMatcher<WildcardHost>();
addMatcher<EqualListeningIP>();
addMatcher<EqualListeningPort>();
addMatcher<BeginWithUri>();
BasicRuleConfig::preload();
LogTriggerConf::preload();
ParameterException::preload();
registerExpectedConfiguration<Zone>("rulebase", "zones");
registerExpectedConfigFile("zones", Config::ConfigFileType::Policy);
registerExpectedConfigFile("triggers", Config::ConfigFileType::Policy);
registerExpectedConfigFile("rules", Config::ConfigFileType::Policy);
registerExpectedConfigFile("parameters", Config::ConfigFileType::Policy);
registerExpectedConfigFile("exceptions", Config::ConfigFileType::Policy);
}
LogTriggerConf
GenericRulebase::Impl::getLogTriggerConf(const string& trigger_Id) const
{
ScopedContext ctx;
set<string> triggers = {trigger_Id};
ctx.registerValue<set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers);
return getConfigurationWithDefault(LogTriggerConf(), "rulebase", "log");
}
ParameterException
GenericRulebase::Impl::getParameterException(const string& parameter_Id) const
{
ScopedContext ctx;
set<string> exceptions = {parameter_Id};
ctx.registerValue<set<GenericConfigId>>(ParameterMatcher::ctx_key, exceptions);
return getConfigurationWithDefault(ParameterException(), "rulebase", "exception");
}
set<ParameterBehavior>
GenericRulebase::Impl::getBehavior(const ParameterKeyValues &key_value_pairs) const
{
auto &exceptions = getConfiguration<ParameterException>("rulebase", "exception");
if (!exceptions.ok()) {
dbgTrace(D_RULEBASE_CONFIG) << "Could not find any exception with the current rule's context";
return {};
}
return (*exceptions).getBehavior(key_value_pairs);
}
GenericRulebase::GenericRulebase() : Component("GenericRulebase"), pimpl(make_unique<Impl>()) {}
GenericRulebase::~GenericRulebase() {}
void
GenericRulebase::init()
{
pimpl->init();
}
void
GenericRulebase::fini()
{
pimpl->fini();
}
void
GenericRulebase::preload()
{
pimpl->preload();
}

109
components/utils/generic_rulebase/generic_rulebase_context.cc Normal file
View File

@ -0,0 +1,109 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/generic_rulebase_context.h"
#include <vector>
#include "context.h"
#include "config.h"
#include "generic_rulebase/evaluators/trigger_eval.h"
#include "generic_rulebase/evaluators/parameter_eval.h"
#include "generic_rulebase/evaluators/practice_eval.h"
#include "generic_rulebase/evaluators/zone_eval.h"
#include "generic_rulebase/evaluators/asset_eval.h"
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
using namespace std;
template<typename Configs>
set<GenericConfigId>
extractIds(const vector<Configs> &configurations)
{
set<GenericConfigId> ids;
for (const Configs &conf : configurations) {
ids.insert(conf.getId());
}
return ids;
}
void
GenericRulebaseContext::activate(const BasicRuleConfig &rule)
{
switch(registration_state) {
case RuleRegistrationState::UNINITIALIZED: {
registration_state = RuleRegistrationState::REGISTERED;
ctx.registerValue<set<GenericConfigId>>(
TriggerMatcher::ctx_key,
extractIds<RuleTrigger>(rule.getTriggers())
);
ctx.registerValue<set<GenericConfigId>>(
PracticeMatcher::ctx_key,
extractIds<RulePractice>(rule.getPractices())
);
dbgTrace(D_RULEBASE_CONFIG)
<< "Activating current practices. Current practice IDs: "
<< makeSeparatedStr(extractIds<RulePractice>(rule.getPractices()), ", ");
ctx.registerValue<set<GenericConfigId>>(
ParameterMatcher::ctx_key,
extractIds<RuleParameter>(rule.getParameters())
);
ctx.registerValue<GenericConfigId>(
ZoneMatcher::ctx_key,
rule.getZoneId()
);
ctx.registerValue<GenericConfigId>(
AssetMatcher::ctx_key,
rule.getAssetId()
);
ctx.activate();
break;
}
case RuleRegistrationState::REGISTERED: {
dbgTrace(D_RULEBASE_CONFIG) << "Activating registered rule values";
ctx.activate();
break;
}
case RuleRegistrationState::UNREGISTERED: {
dbgTrace(D_RULEBASE_CONFIG) << "Failed to register rule values";
}
}
}
void
GenericRulebaseContext::activate()
{
switch(registration_state) {
case RuleRegistrationState::UNINITIALIZED: {
auto maybe_rule = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
if (!maybe_rule.ok()) {
registration_state = RuleRegistrationState::UNREGISTERED;
return;
}
dbgTrace(D_RULEBASE_CONFIG) << "Registering new rule values";
activate(maybe_rule.unpack());
registration_state = RuleRegistrationState::REGISTERED;
break;
}
case RuleRegistrationState::REGISTERED: {
dbgTrace(D_RULEBASE_CONFIG) << "Activating registered rule values";
ctx.activate();
break;
}
case RuleRegistrationState::UNREGISTERED: {
dbgTrace(D_RULEBASE_CONFIG) << "Failed to register rule values";
}
}
}

347
components/utils/generic_rulebase/match_query.cc Normal file
View File

@ -0,0 +1,347 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/match_query.h"
#include "cereal/types/set.hpp"
#include "generic_rulebase/generic_rulebase_utils.h"
#include "config.h"
#include "ip_utilities.h"
#include "agent_core_utilities.h"
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
using namespace std;
static const unordered_map<string, MatchQuery::MatchType> string_to_match_type = {
{ "condition", MatchQuery::MatchType::Condition },
{ "operator", MatchQuery::MatchType::Operator }
};
static const unordered_map<string, MatchQuery::Operators> string_to_operator = {
{ "and", MatchQuery::Operators::And },
{ "or", MatchQuery::Operators::Or }
};
static const unordered_map<string, MatchQuery::Conditions> string_to_condition = {
{ "equals", MatchQuery::Conditions::Equals },
{ "not-equals", MatchQuery::Conditions::NotEquals },
{ "not equals", MatchQuery::Conditions::NotEquals },
{ "in", MatchQuery::Conditions::In },
{ "not-in", MatchQuery::Conditions::NotIn },
{ "not in", MatchQuery::Conditions::NotIn },
{ "exist", MatchQuery::Conditions::Exist }
};
static const string ip_addr_type_name = "IP address";
static const string port_type_name = "port";
static const string ip_proto_type_name = "IP protocol";
static const unordered_map<string, MatchQuery::StaticKeys> string_to_key = {
{ "sourceIP", MatchQuery::StaticKeys::SrcIpAddress },
{ "sourceIpAddr", MatchQuery::StaticKeys::SrcIpAddress },
{ "destinationIP", MatchQuery::StaticKeys::DstIpAddress },
{ "destinationIpAddr", MatchQuery::StaticKeys::DstIpAddress },
{ "ipAddress", MatchQuery::StaticKeys::IpAddress },
{ "sourcePort", MatchQuery::StaticKeys::SrcPort },
{ "listeningPort", MatchQuery::StaticKeys::ListeningPort },
{ "ipProtocol", MatchQuery::StaticKeys::IpProtocol },
{ "domain", MatchQuery::StaticKeys::Domain }
};
MatchQuery::MatchQuery(const string &match) : is_specific_label(false), is_ignore_keyword(false)
{
try {
stringstream ss;
ss.str(match);
cereal::JSONInputArchive archive_in(ss);
load(archive_in);
} catch (const exception &e) {
dbgWarning(D_RULEBASE_CONFIG)
<< "Unable to load match query JSON. JSON content: "
<< match
<< ", Error: "
<< e.what();
}
}
void
MatchQuery::load(cereal::JSONInputArchive &archive_in)
{
string type_as_string;
archive_in(cereal::make_nvp("type", type_as_string));
string op_as_string;
archive_in(cereal::make_nvp("op", op_as_string));
auto maybe_type = string_to_match_type.find(type_as_string);
if (maybe_type == string_to_match_type.end()) {
reportConfigurationError("Illegal Zone match query type. Provided type in configuration: " + type_as_string);
}
type = maybe_type->second;
switch (type) {
case (MatchType::Condition): {
auto maybe_condition = string_to_condition.find(op_as_string);
if (maybe_condition == string_to_condition.end()) {
reportConfigurationError(
"Illegal op provided for condition. Provided op in configuration: " +
op_as_string
);
}
condition_type = maybe_condition->second;
operator_type = Operators::None;
archive_in(cereal::make_nvp("key", key));
key_type = getKeyByName(key);
if (key_type == StaticKeys::NotStatic) {
if (key.rfind("containerLabels.", 0) == 0) {
is_specific_label = true;
} else {
is_specific_label = false;
}
}
is_ignore_keyword = (key == "indicator");
if (condition_type != Conditions::Exist) {
archive_in(cereal::make_nvp("value", value));
for(const auto &val: value) {
if (isKeyTypeIp()) {
auto ip_range = IPUtilities::createRangeFromString<IPRange, IpAddress>(val, ip_addr_type_name);
if (ip_range.ok()) {
ip_addr_value.push_back(ip_range.unpack());
} else {
dbgWarning(D_RULEBASE_CONFIG)
<< "Failed to parse IP address range. Error: "
<< ip_range.getErr();
}
} else if (isKeyTypePort()) {
auto port_range = IPUtilities::createRangeFromString<PortsRange, uint16_t>(
val,
port_type_name
);
if (port_range.ok()) {
port_value.push_back(port_range.unpack());
} else {
dbgWarning(D_RULEBASE_CONFIG)
<< "Failed to parse port range. Error: "
<< port_range.getErr();
}
} else if (isKeyTypeProtocol()) {
auto proto_range = IPUtilities::createRangeFromString<IpProtoRange, uint8_t>(
val,
ip_proto_type_name
);
if (proto_range.ok()) {
ip_proto_value.push_back(proto_range.unpack());
} else {
dbgWarning(D_RULEBASE_CONFIG)
<< "Failed to parse IP protocol range. Error: "
<< proto_range.getErr();
}
}
try {
regex_values.insert(boost::regex(val));
} catch (const exception &e) {
dbgDebug(D_RULEBASE_CONFIG) << "Failed to compile regex. Error: " << e.what();
}
}
first_value = *(value.begin());
}
break;
}
case (MatchType::Operator): {
auto maybe_operator = string_to_operator.find(op_as_string);
if (maybe_operator == string_to_operator.end()) {
reportConfigurationError(
"Illegal op provided for operator. Provided op in configuration: " +
op_as_string
);
}
operator_type = maybe_operator->second;
condition_type = Conditions::None;
archive_in(cereal::make_nvp("items", items));
break;
}
}
}
MatchQuery::StaticKeys
MatchQuery::getKeyByName(const string &key_type_name)
{
auto key = string_to_key.find(key_type_name);
if (key == string_to_key.end()) return StaticKeys::NotStatic;
return key->second;
}
bool
MatchQuery::isKeyTypeIp() const
{
return (key_type >= StaticKeys::IpAddress && key_type <= StaticKeys::DstIpAddress);
}
bool
MatchQuery::isKeyTypePort() const
{
return (key_type == StaticKeys::SrcPort || key_type == StaticKeys::ListeningPort);
}
bool
MatchQuery::isKeyTypeProtocol() const
{
return (key_type == StaticKeys::IpProtocol);
}
bool
MatchQuery::isKeyTypeDomain() const
{
return (key_type == StaticKeys::Domain);
}
bool
MatchQuery::isKeyTypeSpecificLabel() const
{
return is_specific_label;
}
bool
MatchQuery::isKeyTypeStatic() const
{
return (key_type != StaticKeys::NotStatic);
}
set<string>
MatchQuery::getAllKeys() const
{
set<string> keys;
if (type == MatchType::Condition) {
if (!key.empty()) keys.insert(key);
return keys;
}
for (const MatchQuery &inner_match: items) {
set<string> iner_keys = inner_match.getAllKeys();
keys.insert(iner_keys.begin(), iner_keys.end());
}
return keys;
}
bool
MatchQuery::matchAttributes(
const unordered_map<string, set<string>> &key_value_pairs,
set<string> &matched_override_keywords ) const
{
if (type == MatchType::Condition) {
auto key_value_pair = key_value_pairs.find(key);
if (key_value_pair == key_value_pairs.end()) {
dbgTrace(D_RULEBASE_CONFIG) << "Ignoring irrelevant key: " << key;
return false;
}
return matchAttributes(key_value_pair->second, matched_override_keywords);
} else if (type == MatchType::Operator && operator_type == Operators::And) {
for (const MatchQuery &inner_match: items) {
if (!inner_match.matchAttributes(key_value_pairs, matched_override_keywords)) {
return false;
}
}
return true;
} else if (type == MatchType::Operator && operator_type == Operators::Or) {
// With 'or' condition, evaluate matched override keywords first and add the ones that were fully matched
set<string> inner_override_keywords;
bool res = false;
for (const MatchQuery &inner_match: items) {
inner_override_keywords.clear();
if (inner_match.matchAttributes(key_value_pairs, inner_override_keywords)) {
matched_override_keywords.insert(inner_override_keywords.begin(), inner_override_keywords.end());
res = true;
}
}
return res;
} else {
dbgWarning(D_RULEBASE_CONFIG) << "Unsupported match query type";
}
return false;
}
MatchQuery::MatchResult
MatchQuery::getMatch( const unordered_map<string, set<string>> &key_value_pairs) const
{
MatchQuery::MatchResult matches;
matches.matched_keywords = make_shared<set<string>>();
matches.is_match = matchAttributes(key_value_pairs, *matches.matched_keywords);
return matches;
}
bool
MatchQuery::matchAttributes(
const unordered_map<string, set<string>> &key_value_pairs) const
{
return getMatch(key_value_pairs).is_match;
}
bool
MatchQuery::matchAttributes(
const set<string> &values,
set<string> &matched_override_keywords) const
{
auto &type = condition_type;
bool negate = type == MatchQuery::Conditions::NotEquals || type == MatchQuery::Conditions::NotIn;
bool match = isRegEx() ? matchAttributesRegEx(values, matched_override_keywords) : matchAttributesString(values);
return negate ? !match : match;
}
bool
MatchQuery::matchAttributesRegEx(
const set<string> &values,
set<string> &matched_override_keywords) const
{
bool res = false;
boost::cmatch value_matcher;
for (const boost::regex &val_regex : regex_values) {
for (const string &requested_match_value : values) {
if (NGEN::Regex::regexMatch(
__FILE__,
__LINE__,
requested_match_value.c_str(),
value_matcher,
val_regex))
{
res = true;
if (is_ignore_keyword) {
matched_override_keywords.insert(requested_match_value);
} else {
return res;
}
}
}
}
return res;
}
bool
MatchQuery::matchAttributesString(const set<string> &values) const
{
for (const string &requested_value : values) {
if (value.find(requested_value) != value.end()) return true;
}
return false;
}
bool
MatchQuery::isRegEx() const
{
return key != "protectionName";
}

157
components/utils/generic_rulebase/parameters_config.cc Normal file
View File

@ -0,0 +1,157 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/parameters_config.h"
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
using namespace std;
bool ParameterException::is_geo_location_exception_exists(false);
bool ParameterException::is_geo_location_exception_being_loaded(false);
void
ParameterOverrides::load(cereal::JSONInputArchive &archive_in)
{
parseJSONKey<vector<ParsedBehavior>>("parsedBehavior", parsed_behaviors, archive_in);
}
void
ParameterTrustedSources::load(cereal::JSONInputArchive &archive_in)
{
parseJSONKey<uint>("numOfSources", num_of_sources, archive_in);
parseJSONKey<vector<SourcesIdentifier>>("sourcesIdentifiers", sources_identidiers, archive_in);
}
void
ParameterBehavior::load(cereal::JSONInputArchive &archive_in)
{
string key_string;
string val_string;
parseJSONKey<string>("id", id, archive_in);
parseJSONKey<string>("key", key_string, archive_in);
parseJSONKey<string>("value", val_string, archive_in);
if (string_to_behavior_key.find(key_string) == string_to_behavior_key.end()) {
dbgWarning(D_RULEBASE_CONFIG) << "Unsupported behavior key: " << key_string;
return;
}
key = string_to_behavior_key.at(key_string);
if (string_to_behavior_val.find(val_string) == string_to_behavior_val.end()) {
dbgWarning(D_RULEBASE_CONFIG) << "Unsupported behavior value: " << val_string;
return;
}
value = string_to_behavior_val.at(val_string);
}
void
ParameterAntiBot::load(cereal::JSONInputArchive &archive_in)
{
parseJSONKey<vector<string>>("injected", injected, archive_in);
parseJSONKey<vector<string>>("validated", validated, archive_in);
}
void
ParameterOAS::load(cereal::JSONInputArchive &archive_in)
{
parseJSONKey<string>("value", value, archive_in);
}
void
ParameterException::MatchBehaviorPair::load(cereal::JSONInputArchive &archive_in)
{
parseJSONKey<MatchQuery>("match", match, archive_in);
parseJSONKey<ParameterBehavior>("behavior", behavior, archive_in);
}
void
ParameterException::load(cereal::JSONInputArchive &archive_in)
{
try {
archive_in(
cereal::make_nvp("match", match),
cereal::make_nvp("behavior", behavior)
);
} catch (...) {
parseJSONKey<vector<MatchBehaviorPair>>("exceptions", match_queries, archive_in);
}
function<bool(const MatchQuery &)> isGeoLocationExists =
[&](const MatchQuery &query)
{
if (query.getKey() == "countryCode" || query.getKey() == "countryName") {
is_geo_location_exception_being_loaded = true;
return true;
}
for (const MatchQuery &query_item : query.getItems()) {
if (isGeoLocationExists(query_item)) return true;
}
return false;
};
if (isGeoLocationExists(match)) return;
for (const MatchBehaviorPair &match_query : match_queries) {
if (isGeoLocationExists(match_query.match)) return;
}
}
set<ParameterBehavior>
ParameterException::getBehavior(
const unordered_map<string, set<string>> &key_value_pairs,
set<string> &matched_override_keywords) const
{
set<ParameterBehavior> matched_behaviors;
matched_override_keywords.clear();
dbgTrace(D_RULEBASE_CONFIG) << "Matching exception";
for (const MatchBehaviorPair &match_behavior_pair: match_queries) {
MatchQuery::MatchResult match_res = match_behavior_pair.match.getMatch(key_value_pairs);
if (match_res.is_match) {
dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception from a list of matches.";
// When matching indicators with action=ignore, we expect no behavior override.
// Instead, a matched keywords list should be returned which will be later removed from score calculation
if (match_res.matched_keywords->size() > 0 && match_behavior_pair.behavior == action_ignore) {
matched_override_keywords.insert(match_res.matched_keywords->begin(),
match_res.matched_keywords->end());
} else {
matched_behaviors.insert(match_behavior_pair.behavior);
}
}
}
if (match_queries.empty()) {
MatchQuery::MatchResult match_res = match.getMatch(key_value_pairs);
if (match_res.is_match) {
dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception.";
// When matching indicators with action=ignore, we expect no behavior override.
// Instead, a matched keywords list should be returned which will be later removed from score calculation
if (match_res.matched_keywords->size() > 0 && behavior == action_ignore) {
matched_override_keywords.insert(match_res.matched_keywords->begin(),
match_res.matched_keywords->end());
} else {
matched_behaviors.insert(behavior);
}
}
}
return matched_behaviors;
}
set<ParameterBehavior>
ParameterException::getBehavior(const unordered_map<string, set<string>> &key_value_pairs) const
{
set<string> keywords;
return getBehavior(key_value_pairs, keywords);
}

79
components/utils/generic_rulebase/rulebase_config.cc Normal file
View File

@ -0,0 +1,79 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/rulebase_config.h"
#include "telemetry.h"
#include "config.h"
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
using namespace std;
set<string> BasicRuleConfig::assets_ids{};
set<string> BasicRuleConfig::assets_ids_aggregation{};
void
BasicRuleConfig::load(cereal::JSONInputArchive &ar)
{
parseJSONKey<vector<RulePractice>>("practices", practices, ar);
parseJSONKey<vector<RuleTrigger>>("triggers", triggers, ar);
parseJSONKey<vector<RuleParameter>>("parameters", parameters, ar);
parseJSONKey<uint8_t>("priority", priority, ar);
parseJSONKey<string>("ruleId", rule_id, ar);
parseJSONKey<string>("ruleName", rule_name, ar);
parseJSONKey<string>("assetId", asset_id, ar);
parseJSONKey<string>("assetName", asset_name, ar);
parseJSONKey<string>("zoneId", zone_id, ar);
parseJSONKey<string>("zoneName", zone_name, ar);
assets_ids_aggregation.insert(asset_id);
}
void
BasicRuleConfig::updateCountMetric()
{
BasicRuleConfig::assets_ids = BasicRuleConfig::assets_ids_aggregation;
AssetCountEvent(AssetType::ALL, BasicRuleConfig::assets_ids.size()).notify();
}
bool
BasicRuleConfig::isPracticeActive(const string &practice_id) const
{
for (auto practice: practices) {
if (practice.getId() == practice_id) return true;
}
return false;
}
bool
BasicRuleConfig::isTriggerActive(const string &trigger_id) const
{
for (auto trigger: triggers) {
if (trigger.getId() == trigger_id) {
return true;
}
}
return false;
}
bool
BasicRuleConfig::isParameterActive(const string &parameter_id) const
{
for (auto param: parameters) {
if (param.getId() == parameter_id) {
return true;
}
}
return false;
}

243
components/utils/generic_rulebase/triggers_config.cc Normal file
View File

@ -0,0 +1,243 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <string>
#include <map>
#include "generic_rulebase/triggers_config.h"
#include "generic_rulebase/generic_rulebase_utils.h"
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
using namespace std;
WebTriggerConf::WebTriggerConf() : response_title(""), response_body(""), response_code(0) {}
WebTriggerConf::WebTriggerConf(const string &title, const string &body, uint code)
:
response_title(title),
response_body(body),
response_code(code)
{}
WebTriggerConf WebTriggerConf::default_trigger_conf = WebTriggerConf(
"Attack blocked by web application protection", // title
"Check Point's <b>Application Security</b> has detected an attack and blocked it.", // body
403
);
void
WebTriggerConf::load(cereal::JSONInputArchive &archive_in)
{
try {
parseJSONKey<string>("details level", details_level, archive_in);
if (details_level == "Redirect") {
parseJSONKey<string>("redirect URL", redirect_url, archive_in);
parseJSONKey<bool>("xEventId", add_event_id_to_header, archive_in);
parseJSONKey<bool>("eventIdInHeader", add_event_id_to_header, archive_in);
return;
}
parseJSONKey<uint>("response code", response_code, archive_in);
if (response_code < 100 || response_code > 599) {
throw cereal::Exception(
"illegal web trigger response code: " +
to_string(response_code) +
" is out of range (100-599)"
);
}
if (details_level == "Response Code") return;
parseJSONKey<string>("response body", response_body, archive_in);
parseJSONKey<string>("response title", response_title, archive_in);
} catch (const exception &e) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to parse the web trigger configuration: '" << e.what() << "'";
archive_in.setNextName(nullptr);
}
}
bool
WebTriggerConf::operator==(const WebTriggerConf &other) const
{
return
response_code == other.response_code &&
response_title == other.response_title &&
response_body == other.response_body;
}
LogTriggerConf::LogTriggerConf(string trigger_name, bool log_detect, bool log_prevent) : name(trigger_name)
{
if (log_detect) should_log_on_detect.setAll();
if (log_prevent) should_log_on_prevent.setAll();
active_streams.setFlag(ReportIS::StreamType::JSON_FOG);
active_streams.setFlag(ReportIS::StreamType::JSON_LOG_FILE);
}
ReportIS::Severity
LogTriggerConf::getSeverity(bool is_action_drop_or_prevent) const
{
return is_action_drop_or_prevent ? ReportIS::Severity::MEDIUM : ReportIS::Severity::LOW;
}
ReportIS::Priority
LogTriggerConf::getPriority(bool is_action_drop_or_prevent) const
{
return is_action_drop_or_prevent ? ReportIS::Priority::HIGH : ReportIS::Priority::MEDIUM;
}
Flags<ReportIS::StreamType>
LogTriggerConf::getStreams(SecurityType security_type, bool is_action_drop_or_prevent) const
{
if (is_action_drop_or_prevent && should_log_on_prevent.isSet(security_type)) return active_streams;
if (!is_action_drop_or_prevent && should_log_on_detect.isSet(security_type)) return active_streams;
return Flags<ReportIS::StreamType>();
}
Flags<ReportIS::Enreachments>
LogTriggerConf::getEnrechments(SecurityType security_type) const
{
Flags<ReportIS::Enreachments> enreachments;
if (log_geo_location.isSet(security_type)) enreachments.setFlag(ReportIS::Enreachments::GEOLOCATION);
if (should_format_output) enreachments.setFlag(ReportIS::Enreachments::BEAUTIFY_OUTPUT);
return enreachments;
}
template <typename EnumClass>
static void
setTriggersFlag(const string &key, cereal::JSONInputArchive &ar, EnumClass flag, Flags<EnumClass> &flags)
{
bool value = false;
parseJSONKey<bool>(key, value, ar);
if (value) flags.setFlag(flag);
}
static void
setLogConfiguration(
const ReportIS::StreamType &log_type,
const string &log_server_url = "",
const string &protocol = ""
)
{
dbgTrace(D_RULEBASE_CONFIG) << "log server url:" << log_server_url;
if (log_server_url != "" && protocol != "") {
Singleton::Consume<I_Logging>::by<LogTriggerConf>()->addStream(log_type, log_server_url, protocol);
} else {
Singleton::Consume<I_Logging>::by<LogTriggerConf>()->addStream(log_type);
}
}
static string
parseProtocolWithDefault(
const std::string &default_value,
const std::string &key_name,
cereal::JSONInputArchive &archive_in
)
{
string value;
try {
archive_in(cereal::make_nvp(key_name, value));
} catch (const cereal::Exception &e) {
return default_value;
}
return value;
}
void
LogTriggerConf::load(cereal::JSONInputArchive& archive_in)
{
try {
parseJSONKey<string>("triggerName", name, archive_in);
parseJSONKey<string>("verbosity", verbosity, archive_in);
parseJSONKey<string>("urlForSyslog", url_for_syslog, archive_in);
parseJSONKey<string>("urlForCef", url_for_cef, archive_in);
parseJSONKey<string>("syslogProtocol", syslog_protocol, archive_in);
syslog_protocol = parseProtocolWithDefault("UDP", "syslogProtocol", archive_in);
cef_protocol = parseProtocolWithDefault("UDP", "cefProtocol", archive_in);
setTriggersFlag("webBody", archive_in, WebLogFields::webBody, log_web_fields);
setTriggersFlag("webHeaders", archive_in, WebLogFields::webHeaders, log_web_fields);
setTriggersFlag("webRequests", archive_in, WebLogFields::webRequests, log_web_fields);
setTriggersFlag("webUrlPath", archive_in, WebLogFields::webUrlPath, log_web_fields);
setTriggersFlag("webUrlQuery", archive_in, WebLogFields::webUrlQuery, log_web_fields);
setTriggersFlag("logToAgent", archive_in, ReportIS::StreamType::JSON_LOG_FILE, active_streams);
setTriggersFlag("logToCloud", archive_in, ReportIS::StreamType::JSON_FOG, active_streams);
setTriggersFlag("logToK8sService", archive_in, ReportIS::StreamType::JSON_K8S_SVC, active_streams);
setTriggersFlag("logToSyslog", archive_in, ReportIS::StreamType::SYSLOG, active_streams);
setTriggersFlag("logToCef", archive_in, ReportIS::StreamType::CEF, active_streams);
setTriggersFlag("acAllow", archive_in, SecurityType::AccessControl, should_log_on_detect);
setTriggersFlag("acDrop", archive_in, SecurityType::AccessControl, should_log_on_prevent);
setTriggersFlag("tpDetect", archive_in, SecurityType::ThreatPrevention, should_log_on_detect);
setTriggersFlag("tpPrevent", archive_in, SecurityType::ThreatPrevention, should_log_on_prevent);
setTriggersFlag("complianceWarnings", archive_in, SecurityType::Compliance, should_log_on_detect);
setTriggersFlag("complianceViolations", archive_in, SecurityType::Compliance, should_log_on_prevent);
setTriggersFlag("acLogGeoLocation", archive_in, SecurityType::AccessControl, log_geo_location);
setTriggersFlag("tpLogGeoLocation", archive_in, SecurityType::ThreatPrevention, log_geo_location);
setTriggersFlag("complianceLogGeoLocation", archive_in, SecurityType::Compliance, log_geo_location);
bool extend_logging = false;
parseJSONKey<bool>("extendLogging", extend_logging, archive_in);
if (extend_logging) {
setTriggersFlag("responseCode", archive_in, WebLogFields::responseCode, log_web_fields);
setTriggersFlag("responseBody", archive_in, WebLogFields::responseBody, log_web_fields);
string severity;
static const map<string, extendLoggingSeverity> extend_logging_severity_strings = {
{"High", extendLoggingSeverity::High},
{"Critical", extendLoggingSeverity::Critical}
};
parseJSONKey<string>("extendLoggingMinSeverity", severity, archive_in);
auto extended_severity = extend_logging_severity_strings.find(severity);
if (extended_severity != extend_logging_severity_strings.end()) {
extend_logging_severity = extended_severity->second;
} else {
dbgWarning(D_RULEBASE_CONFIG)
<< "Failed to parse the extendLoggingMinSeverityfield: '"
<< severity
<< "'";
}
}
for (ReportIS::StreamType log_stream : makeRange<ReportIS::StreamType>()) {
if (!active_streams.isSet(log_stream)) continue;
switch (log_stream) {
case ReportIS::StreamType::JSON_DEBUG:
setLogConfiguration(ReportIS::StreamType::JSON_DEBUG);
break;
case ReportIS::StreamType::JSON_FOG:
setLogConfiguration(ReportIS::StreamType::JSON_FOG);
break;
case ReportIS::StreamType::JSON_LOG_FILE:
setLogConfiguration(ReportIS::StreamType::JSON_LOG_FILE);
break;
case ReportIS::StreamType::JSON_K8S_SVC:
setLogConfiguration(ReportIS::StreamType::JSON_K8S_SVC);
break;
case ReportIS::StreamType::SYSLOG:
setLogConfiguration(ReportIS::StreamType::SYSLOG, getUrlForSyslog(), syslog_protocol);
break;
case ReportIS::StreamType::CEF:
setLogConfiguration(ReportIS::StreamType::CEF, getUrlForCef(), cef_protocol);
break;
case ReportIS::StreamType::NONE: break;
case ReportIS::StreamType::COUNT: break;
}
}
parseJSONKey<bool>("formatLoggingOutput", should_format_output, archive_in);
} catch (const exception &e) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to parse the log trigger configuration: '" << e.what() << "'";
archive_in.setNextName(nullptr);
}
}

179
components/utils/generic_rulebase/zone.cc Normal file
View File

@ -0,0 +1,179 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/zone.h"
#include <set>
#include <vector>
#include <string>
using namespace std;
static const unordered_map<string, Zone::Direction> string_to_direction = {
{ "to", Zone::Direction::To },
{ "from", Zone::Direction::From },
{ "bidirectional", Zone::Direction::Bidirectional }
};
class AdjacentZone
{
public:
void
load(cereal::JSONInputArchive &archive_in)
{
string direction_as_string;
archive_in(cereal::make_nvp("direction", direction_as_string));
archive_in(cereal::make_nvp("zoneId", id));
auto maybe_direction = string_to_direction.find(direction_as_string);
if (maybe_direction == string_to_direction.end()) {
reportConfigurationError(
"Illegal direction provided for adjacency. Provided direction in configuration: " +
direction_as_string
);
}
dir = maybe_direction->second;
}
pair<Zone::Direction, GenericConfigId> getValue() const { return make_pair(dir, id); }
private:
Zone::Direction dir;
GenericConfigId id;
};
class TagsValues
{
public:
static const string req_attrs_ctx_key;
TagsValues() {}
template <typename Archive>
void
serialize(Archive &ar)
{
I_Environment *env = Singleton::Consume<I_Environment>::by<Zone>();
auto req_attrs = env->get<set<string>>(req_attrs_ctx_key);
if (!req_attrs.ok()) return;
for (const string &req_attr : *req_attrs) {
try {
string data;
ar(cereal::make_nvp(req_attr, data));
dbgDebug(D_RULEBASE_CONFIG)
<< "Found value for requested attribute. Tag: "
<< req_attr
<< ", Value: "
<< data;
tags_set[req_attr].insert(data);
} catch (const exception &e) {
dbgDebug(D_RULEBASE_CONFIG) << "Could not find values for requested attribute. Tag: " << req_attr;
ar.setNextName(nullptr);
}
}
}
bool
matchValueByKey(const string &requested_key, const unordered_set<string> &possible_values) const
{
auto values = tags_set.find(requested_key);
if (values == tags_set.end()) return false;
for (const string &val : possible_values) {
if (values->second.count(val)) return true;
}
return false;
}
void
insert(const TagsValues &other)
{
for (auto &single_tags_value : other.getData()) {
tags_set[single_tags_value.first].insert(single_tags_value.second.begin(), single_tags_value.second.end());
}
}
const unordered_map<string, set<string>> & getData() const { return tags_set; }
private:
unordered_map<string, set<string>> tags_set;
};
const string TagsValues::req_attrs_ctx_key = "requested attributes key";
void
Zone::load(cereal::JSONInputArchive &archive_in)
{
archive_in(cereal::make_nvp("id", zone_id));
archive_in(cereal::make_nvp("name", zone_name));
vector<AdjacentZone> adjacency;
try {
archive_in(cereal::make_nvp("adjacentZones", adjacency));
} catch (const cereal::Exception &) {
dbgTrace(D_RULEBASE_CONFIG)
<< "List of adjacentZones does not exist for current object. Zone id: "
<< zone_id
<< ", Zone name: "
<< zone_name;
archive_in.setNextName(nullptr);
}
for (const AdjacentZone &zone : adjacency) {
adjacent_zones.push_back(zone.getValue());
}
archive_in(cereal::make_nvp("match", match_query));
is_any =
match_query.getType() == MatchQuery::MatchType::Condition &&
match_query.getKey() == "any" &&
match_query.getValue().count("any") > 0;
set<string> keys = match_query.getAllKeys();
}
const string
contextKeyToString(Context::MetaDataType type)
{
if (type == Context::MetaDataType::SubjectIpAddr || type == Context::MetaDataType::OtherIpAddr) return "ip";
return Context::convertToString(type);
}
bool
Zone::contains(const Asset &asset)
{
QueryRequest request;
for (const auto &main_attr : asset.getAttrs()) {
request.addCondition(Condition::EQUALS, contextKeyToString(main_attr.first), main_attr.second);
}
ScopedContext req_attrs_key;
req_attrs_key.registerValue<set<string>>(TagsValues::req_attrs_ctx_key, match_query.getAllKeys());
I_Intelligence_IS_V2 *intelligence = Singleton::Consume<I_Intelligence_IS_V2>::by<Zone>();
auto query_res = intelligence->queryIntelligence<TagsValues>(request);
if (!query_res.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to perform intelligence query. Error: " << query_res.getErr();
return false;
}
for (const AssetReply<TagsValues> &asset : query_res.unpack()) {
TagsValues tag_values = asset.mergeReplyData();
if (match_query.matchAttributes(tag_values.getData())) return true;
}
return false;
}

114
components/utils/generic_rulebase/zones_config.cc Normal file
View File

@ -0,0 +1,114 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "generic_rulebase/zones_config.h"
#include <string>
#include <unordered_map>
#include "generic_rulebase/generic_rulebase_utils.h"
#include "config.h"
#include "ip_utilities.h"
#include "connkey.h"
#include "i_generic_rulebase.h"
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
using namespace std;
void
ZonesConfig::load(cereal::JSONInputArchive &archive_in)
{
dbgFlow(D_RULEBASE_CONFIG) << "Saving active zones";
set<string> used_zones;
cereal::load(archive_in, used_zones);
dbgTrace(D_RULEBASE_CONFIG) << "Loading all zones";
auto all_zones_maybe = getSetting<Zones>("rulebase", "zones");
if (!all_zones_maybe.ok()) {
dbgWarning(D_RULEBASE_CONFIG) << "Failed to load zones";
return;
}
dbgTrace(D_RULEBASE_CONFIG) << "Creating cache of all zones by ID";
map<GenericConfigId, Zone> all_zones;
for (const auto &single_zone : all_zones_maybe.unpack().zones) {
if (used_zones.count(single_zone.getId()) > 0 && single_zone.isAnyZone()) {
dbgTrace(D_RULEBASE_CONFIG) << "Found used zone of type \"Any\": saving all zones as active zones";
zones = all_zones_maybe.unpack().zones;
return;
}
dbgDebug(D_RULEBASE_CONFIG)
<< "Adding specific zone to cache. Zone ID: "
<< single_zone.getId()
<< ", name: "
<< single_zone.getName();
all_zones.emplace(single_zone.getId(), single_zone);
}
dbgTrace(D_RULEBASE_CONFIG) << "Creating list of active zones";
map<GenericConfigId, Zone> active_zones_set;
for (const auto &single_used_zone_id : used_zones) {
const auto &found_zone = all_zones[single_used_zone_id];
dbgTrace(D_RULEBASE_CONFIG)
<< "Adding zone to list of active zones. Zone ID: "
<< single_used_zone_id
<< ", zone name: "
<< found_zone.getName();
active_zones_set.emplace(found_zone.getId(), found_zone);
for (const auto &adjacent_zone : found_zone.getAdjacentZones()) {
const auto &adjacent_zone_obj = all_zones[adjacent_zone.second];
dbgTrace(D_RULEBASE_CONFIG)
<< "Adding adjacent zone to list of active zones. Zone ID: "
<< adjacent_zone_obj.getId()
<< ", zone name: "
<< adjacent_zone_obj.getName();
active_zones_set.emplace(adjacent_zone_obj.getId(), adjacent_zone_obj);
}
}
vector<GenericConfigId> implied_zones = {
"impliedAzure",
"impliedDNS",
"impliedSSH",
"impliedProxy",
"impliedFog"
};
GenericConfigId any_zone_id = "";
for (const auto &single_zone : all_zones_maybe.unpack().zones) {
if (single_zone.isAnyZone()) any_zone_id = single_zone.getId();
}
for (GenericConfigId &implied_id: implied_zones) {
if (all_zones.find(implied_id) != all_zones.end()) {
dbgDebug(D_RULEBASE_CONFIG) << "Adding implied zone to cache. Zone ID: " << implied_id;
active_zones_set.emplace(implied_id, all_zones[implied_id]);
if (any_zone_id != "" && active_zones_set.count(any_zone_id) == 0) {
active_zones_set.emplace(any_zone_id, all_zones[any_zone_id]);
}
}
}
for (const auto &single_id_zone_pair : active_zones_set) {
zones.push_back(single_id_zone_pair.second);
}
}
void
ZonesConfig::preload()
{
registerExpectedSetting<Zones>("rulebase", "zones");
registerExpectedSetting<ZonesConfig>("rulebase", "usedZones");
}

11
core/agent_details_reporter/agent_details_reporter_ut/agent_details_reporter_ut.cc
View File

@ -95,6 +95,7 @@ TEST_F(AgentReporterTest, dataReport)
" }"
"\n}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
AgentDataReport() << AgentReportField(custom_data);;
@ -112,6 +113,7 @@ TEST_F(AgentReporterTest, labeledDataReport)
" }"
"\n}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
AgentDataReport() << AgentReportFieldWithLabel("this_is_custom_label", data);
@ -131,6 +133,7 @@ TEST_F(AgentReporterTest, multiDataReport)
" }"
"\n}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@ -158,6 +161,7 @@ TEST_F(AgentReporterTest, multiDataReportWithRegistrationData)
" \"architecture\": \"aaa\"\n"
"}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@ -181,6 +185,7 @@ TEST_F(AgentReporterTest, basicAttrTest)
" \"additionalMetaData\": {}\n"
"}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@ -200,6 +205,7 @@ TEST_F(AgentReporterTest, basicAttrTest)
" }\n"
"}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@ -219,6 +225,7 @@ TEST_F(AgentReporterTest, basicAttrTest)
" \"additionalMetaData\": {}\n"
"}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@ -247,6 +254,7 @@ TEST_F(AgentReporterTest, advancedAttrTest)
" }\n"
"}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@ -272,6 +280,7 @@ TEST_F(AgentReporterTest, advancedAttrTest)
" }\n"
"}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@ -297,6 +306,7 @@ TEST_F(AgentReporterTest, RestDetailsTest)
"/agents",
rest_call_parameters.str(),
MessageCategory::GENERIC,
_,
_
)).Times(1);
@ -366,6 +376,7 @@ TEST_F(AgentReporterTest, PersistenceAttrTest)
"/agents",
expected_attributes,
MessageCategory::GENERIC,
_,
_
)).Times(1);
EXPECT_TRUE(report->sendAttributes());

3
core/debug_is/debug_is_ut/debug_ut.cc
View File

@ -824,6 +824,7 @@ TEST(DebugFogTest, fog_stream)
"/api/v1/agents/events/bulk",
_,
_,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@ -980,7 +981,7 @@ TEST(DebugFogTest, fog_stream)
.WillOnce(DoAll(InvokeMainLoopCB(), Return(0)));
string message_body_1, message_body_2;
EXPECT_CALL(messaging_mock, sendAsyncMessage(_, "/api/v1/agents/events", _, MessageCategory::DEBUG, _))
EXPECT_CALL(messaging_mock, sendAsyncMessage(_, "/api/v1/agents/events", _, MessageCategory::DEBUG, _, _))
.WillOnce(SaveArg<2>(&message_body_1))
.WillOnce(SaveArg<2>(&message_body_2));

13
core/include/general/common.h
View File

@ -27,6 +27,7 @@
#include <memory>
#include <string>
#include <vector>
#include <sstream>
#include <iomanip>
#include <sys/types.h>
@ -132,6 +133,18 @@ operator<<(ostream &os, const pair<const First, Second> &printable_pair)
return os;
}
template <typename Printable>
ostream &
operator<<(ostream &os, const vector<Printable> &obj)
{
bool first = true;
for (const auto &val : obj) {
os << (first ? "" : ", ") << val;
first = false;
}
return os;
}
} // namespace std
#endif // __COMMON_H__

6
core/include/services_sdk/interfaces/i_messaging.h
View File

@ -56,14 +56,16 @@ public:
const std::string &uri,
serializableObject &req_obj,
const MessageCategory category = MessageCategory::GENERIC,
MessageMetadata message_metadata = MessageMetadata());
MessageMetadata message_metadata = MessageMetadata(),
bool force_buffering = true);
virtual void sendAsyncMessage(
const HTTPMethod method,
const std::string &uri,
const std::string &body,
const MessageCategory category,
MessageMetadata message_metadata
const MessageMetadata &message_metadata = MessageMetadata(),
bool force_buffering = true
) = 0;
virtual Maybe<HTTPResponse, HTTPResponse> sendSyncMessage(

2
core/include/services_sdk/interfaces/intelligence_is_v2/asset_replay_impl.h
View File

@ -28,6 +28,7 @@ AssetReply<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
SerializableMultiMap<std::string, std::vector<std::string>> tmp_multimap;
ar(
cereal::make_nvp("schemaVersion", asset_schema_version),
cereal::make_nvp("assetType", asset_type),
cereal::make_nvp("assetTypeSchemaVersion", asset_type_schema_version),
cereal::make_nvp("class", asset_class),
cereal::make_nvp("category", asset_category),
@ -65,7 +66,6 @@ AssetReply<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
} catch(...) {}
ar(cereal::make_nvp("sources", sources));
ar(cereal::make_nvp("assetType", asset_type));
}
template <typename UserSerializableReplyAttr>

6
core/include/services_sdk/interfaces/messaging/interface_impl.h
View File

@ -128,7 +128,8 @@ I_Messaging::sendAsyncMessage(
const std::string &uri,
serializableObject &req_obj,
const MessageCategory category,
MessageMetadata message_metadata)
MessageMetadata message_metadata,
bool force_buffering)
{
Maybe<std::string> req_body = req_obj.genJson();
if (!req_body.ok()) {
@ -141,7 +142,8 @@ I_Messaging::sendAsyncMessage(
uri,
req_body.unpack(),
category,
message_metadata
message_metadata,
force_buffering
);
}

5
core/include/services_sdk/interfaces/mock/mock_messaging.h
View File

@ -18,14 +18,15 @@ public:
MessageMetadata
)
);
MOCK_METHOD5(
MOCK_METHOD6(
sendAsyncMessage,
void (
HTTPMethod,
const string &,
const string &,
MessageCategory,
MessageMetadata
const MessageMetadata &,
bool
)
);

2
core/include/services_sdk/resources/debug_flags.h
View File

@ -80,6 +80,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_WAAP_JSON, D_WAAP)
DEFINE_FLAG(D_WAAP_BOT_PROTECTION, D_WAAP)
DEFINE_FLAG(D_WAAP_STREAMING_PARSING, D_WAAP)
DEFINE_FLAG(D_WAAP_HEADERS, D_WAAP)
DEFINE_FLAG(D_WAAP_PARSER, D_WAAP)
DEFINE_FLAG(D_WAAP_PARSER_XML, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_HTML, D_WAAP_PARSER)
@ -96,6 +97,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_WAAP_PARSER_PHPSERIALIZE, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_PERCENT, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_PAIRS, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_PDF, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_OVERRIDE, D_WAAP)
DEFINE_FLAG(D_IPS, D_COMPONENT)

4
core/include/services_sdk/resources/generic_metric.h
View File

@ -58,7 +58,8 @@ public:
const ReportIS::IssuingEngine &_issuing_engine,
std::chrono::seconds _report_interval,
bool _reset,
ReportIS::Audience _audience = ReportIS::Audience::INTERNAL
ReportIS::Audience _audience = ReportIS::Audience::INTERNAL,
bool _force_buffering = false
);
template <typename Value>
@ -107,6 +108,7 @@ private:
std::chrono::seconds report_interval;
std::vector<MetricCalc *> calcs;
bool reset;
bool force_buffering = false;
Context ctx;
};

13
core/include/services_sdk/resources/intelligence_invalidation.h
View File

@ -29,6 +29,7 @@ namespace Intelligence
enum class ClassifierType { CLASS, CATEGORY, FAMILY, GROUP, ORDER, KIND };
enum class ObjectType { ASSET, ZONE, POLICY_PACKAGE, CONFIGURATION, SESSION, SHORTLIVED };
enum class InvalidationType { ADD, DELETE, UPDATE };
class Invalidation
{
@ -36,16 +37,20 @@ public:
Invalidation(const std::string &class_value);
Invalidation & setClassifier(ClassifierType type, const std::string &val);
Invalidation & setStringAttr(const std::string &attr, const std::string &val);
Invalidation & setStringSetAttr(const std::string &attr, const std::set<std::string> &val);
Invalidation & setStringAttr(const std::string &attr, const std::string &val, bool is_main = true);
Invalidation & setStringSetAttr(const std::string &attr, const std::set<std::string> &val, bool is_main = true);
Invalidation & setSourceId(const std::string &id);
Invalidation & setObjectType(ObjectType type);
Invalidation & setInvalidationType(InvalidationType type);
std::string getClassifier(ClassifierType type) const { return classifiers[type]; }
Maybe<std::string, void> getStringMainAttr(const std::string &attr) const;
Maybe<std::set<std::string>, void> getStringSetMainAttr(const std::string &attr) const;
Maybe<std::string, void> getStringAttr(const std::string &attr) const;
Maybe<std::set<std::string>, void> getStringSetAttr(const std::string &attr) const;
const Maybe<std::string, void> & getSourceId() const { return source_id; }
const Maybe<ObjectType, void> & getObjectType() const { return object_type; }
InvalidationType getInvalidationType() const { return invalidation_type; }
bool report(I_Intelligence_IS_V2 *interface) const;
@ -59,13 +64,17 @@ public:
bool matches(const Invalidation &other) const;
private:
bool hasMainAttr(const std::string &key, const std::string &value) const;
bool hasAttr(const std::string &key, const std::string &value) const;
EnumArray<ClassifierType, std::string, 6> classifiers;
std::map<std::string, std::string> string_main_attr;
std::map<std::string, std::set<std::string>> set_string_main_attr;
std::map<std::string, std::string> string_attr;
std::map<std::string, std::set<std::string>> set_string_attr;
Maybe<std::string, void> source_id;
Maybe<ObjectType, void> object_type;
InvalidationType invalidation_type = InvalidationType::ADD;
Maybe<uint, void> listening_id;
};

2
core/intelligence_is_v2/include/intelligence_server.h
View File

@ -37,7 +37,7 @@ private:
IntelligenceRequest request;
Flags<MessageConnectionConfig> conn_flags;
bool is_local_intelligence;
bool is_local_intelligence = false;
Maybe<std::string> server_ip = genError("No server ip set");
Maybe<unsigned int> server_port = genError("No port is set");
I_Messaging * i_message = nullptr;

20
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@ -244,7 +244,12 @@ public:
bool
sendInvalidation(const Invalidation &invalidation) const override
{
return hasLocalIntelligence() ? sendLocalInvalidation(invalidation) : sendGlobalInvalidation(invalidation);
if (hasLocalInvalidationSupport()) {
return sendLocalInvalidation(invalidation);
}
else {
return sendGlobalInvalidation(invalidation);
}
}
Maybe<uint>
@ -287,9 +292,14 @@ public:
private:
bool
hasLocalIntelligence() const
hasLocalInvalidationSupport() const
{
return getProfileAgentSettingWithDefault<bool>(false, "agent.config.useLocalIntelligence");
auto is_supported = getProfileAgentSettingWithDefault<bool>(false, "agent.config.useLocalIntelligence");
if (!is_supported) {
is_supported = getProfileAgentSettingWithDefault<bool>(false, "agent.config.supportInvalidation");
}
return is_supported;
}
bool
@ -330,6 +340,7 @@ private:
MessageMetadata invalidation_req_md(server, *port);
invalidation_req_md.insertHeaders(getHTTPHeaders());
invalidation_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
return message->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
invalidation_uri,
@ -411,6 +422,7 @@ private:
dbgTrace(D_INTELLIGENCE) << "Invalidation value: " << registration.genJson();
MessageMetadata registration_req_md(server, *port);
registration_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
return message->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
registration_uri,
@ -423,7 +435,7 @@ private:
void
sendReccurringInvalidationRegistration() const
{
if (!hasLocalIntelligence() || invalidations.empty()) return;
if (!hasLocalInvalidationSupport() || invalidations.empty()) return;
sendLocalRegistrationImpl(invalidations.getRegistration());
}

5
core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc
View File

@ -411,8 +411,9 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest)
"}\n"
);
MessageMetadata md;
EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _)
).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str)));
).WillOnce(DoAll(SaveArg<4>(&md), Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str))));
auto maybe_ans = intell->queryIntelligence<Profile>(request);
EXPECT_TRUE(maybe_ans.ok());
@ -421,6 +422,7 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest)
auto iter = vec.begin();
EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry");
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing");
EXPECT_FALSE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceComponentTestV2, fakeLocalIntelligenceTest)
@ -490,6 +492,7 @@ TEST_F(IntelligenceComponentTestV2, fakeLocalIntelligenceTest)
EXPECT_TRUE(maybe_ans.ok());
EXPECT_EQ(md.getHostName(), "127.0.0.1");
EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceComponentTestV2, multiAssetsIntelligenceTest)

110
core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc
View File

@ -25,59 +25,106 @@ TEST(InvalidationBasic, SettersAndGetters)
EXPECT_EQ(invalidation.getClassifier(ClassifierType::GROUP), "");
EXPECT_EQ(invalidation.getClassifier(ClassifierType::ORDER), "");
EXPECT_EQ(invalidation.getClassifier(ClassifierType::KIND), "");
EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::ADD);
EXPECT_FALSE(invalidation.getStringMainAttr("main_attr1").ok());
EXPECT_FALSE(invalidation.getStringSetMainAttr("main_attr2").ok());
EXPECT_FALSE(invalidation.getStringAttr("attr1").ok());
EXPECT_FALSE(invalidation.getStringSetAttr("attr2").ok());
EXPECT_FALSE(invalidation.getSourceId().ok());
EXPECT_FALSE(invalidation.getObjectType().ok());
set<string> vals = { "2", "3" };
set<string> main_vals = { "2", "3" };
set<string> vals = { "5", "6" };
invalidation
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setStringAttr("attr1", "1")
.setStringSetAttr("attr2", vals)
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
.setObjectType(Intelligence::ObjectType::ASSET)
.setInvalidationType(InvalidationType::DELETE);
EXPECT_EQ(invalidation.getClassifier(ClassifierType::CATEGORY), "bbb");
EXPECT_EQ(invalidation.getClassifier(ClassifierType::FAMILY), "ccc");
EXPECT_EQ(invalidation.getStringAttr("attr1").unpack(), "1");
EXPECT_EQ(invalidation.getStringMainAttr("main_attr1").unpack(), "1");
EXPECT_EQ(invalidation.getStringSetMainAttr("main_attr2").unpack(), main_vals);
EXPECT_EQ(invalidation.getStringAttr("attr1").unpack(), "4");
EXPECT_EQ(invalidation.getStringSetAttr("attr2").unpack(), vals);
EXPECT_EQ(invalidation.getSourceId().unpack(), "id");
EXPECT_EQ(invalidation.getObjectType().unpack(), Intelligence::ObjectType::ASSET);
EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::DELETE);
}
TEST(InvalidationBasic, Matching)
{
set<string> vals = { "2", "3" };
set<string> main_vals = { "2", "3" };
set<string> vals = { "5", "6" };
auto base_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setStringAttr("attr1", "1")
.setStringSetAttr("attr2", vals);
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false);
auto matching_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("attr1", "1")
.setStringSetAttr("attr2", vals)
.setStringAttr("attr3", "6")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setStringAttr("main_attr3", "6")
.setStringAttr("attr3", "7", false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET)
.setInvalidationType(InvalidationType::ADD);
EXPECT_TRUE(base_invalidation.matches(matching_invalidation));
auto not_matching_invalidation_type = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET)
.setInvalidationType(InvalidationType::DELETE);
EXPECT_FALSE(base_invalidation.matches(not_matching_invalidation_type));
auto missing_attr_invalidation_main = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("main_attr1", "1")
.setStringAttr("main_attr2", "2")
.setStringAttr("main_attr3", "6")
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setStringAttr("attr3", "7", false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
EXPECT_TRUE(base_invalidation.matches(matching_invalidation));
EXPECT_FALSE(base_invalidation.matches(missing_attr_invalidation_main));
auto missing_attr_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("attr1", "1")
.setStringAttr("attr2", "2")
.setStringAttr("attr3", "6")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("main_attr3", "6")
.setStringAttr("attr1", "4", false)
.setStringAttr("attr2", "2", false)
.setStringAttr("attr3", "7", false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
@ -88,9 +135,12 @@ TEST(InvalidationBasic, Matching)
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringSetAttr("attr1", vals2)
.setStringSetAttr("attr2", vals)
.setStringAttr("attr3", "6")
.setStringSetAttr("main_attr1", vals2)
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("main_attr3", "6")
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setStringAttr("attr3", "7", false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
@ -170,17 +220,20 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation)
{
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.setStringAttr("attr3", "3", false)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
string invalidation_json;
MessageMetadata md;
EXPECT_CALL(
messaging_mock,
sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _)
).WillOnce(DoAll(
SaveArg<2>(&invalidation_json),
SaveArg<4>(&md),
Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))
));
@ -192,16 +245,20 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation)
"\"category\": \"bbb\", "
"\"family\": \"ccc\", "
"\"objectType\": \"asset\", "
"\"invalidationType\": \"add\", "
"\"sourceId\": \"id\", "
"\"mainAttributes\": [ { \"attr2\": \"2\" } ]"
"\"mainAttributes\": [ { \"attr2\": \"2\" } ], "
"\"attributes\": [ { \"attr3\": \"3\" } ]"
" } ] }";
EXPECT_EQ(invalidation_json, expected_json);
EXPECT_FALSE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceInvalidation, sending_private_invalidation)
{
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.setStringAttr("attr3", "3", false)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@ -221,11 +278,13 @@ TEST_F(IntelligenceInvalidation, sending_private_invalidation)
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
string invalidation_json;
MessageMetadata md;
EXPECT_CALL(
messaging_mock,
sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _)
).WillOnce(DoAll(
SaveArg<2>(&invalidation_json),
SaveArg<4>(&md),
Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))
));
@ -237,10 +296,13 @@ TEST_F(IntelligenceInvalidation, sending_private_invalidation)
"\"category\": \"bbb\", "
"\"family\": \"ccc\", "
"\"objectType\": \"asset\", "
"\"invalidationType\": \"add\", "
"\"sourceId\": \"id\", "
"\"mainAttributes\": [ { \"attr2\": \"2\" } ]"
"\"mainAttributes\": [ { \"attr2\": \"2\" } ], "
"\"attributes\": [ { \"attr3\": \"3\" } ]"
" } ] }";
EXPECT_EQ(invalidation_json, expected_json);
EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceInvalidation, register_for_invalidation)
@ -257,19 +319,23 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation)
configuration << "}";
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
set<string> vals = { "11", "55", "22" };
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.setStringSetAttr("attr3", vals, false)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
string body;
MessageMetadata md;
EXPECT_CALL(
messaging_mock,
sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _)
).WillOnce(DoAll(
SaveArg<2>(&body),
SaveArg<4>(&md),
Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))
));
@ -278,6 +344,8 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation)
EXPECT_THAT(body, HasSubstr("\"url\": \"http://127.0.0.1:7000/set-new-invalidation\""));
EXPECT_THAT(body, HasSubstr("\"apiVersion\": \"v2\", \"communicationType\": \"sync\""));
EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]"));
EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"attr3\": [ \"11\", \"22\", \"55\" ] } ]"));
EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceInvalidation, invalidation_callback)
@ -321,7 +389,7 @@ TEST_F(IntelligenceInvalidation, invalidation_callback)
mock_invalidation->performRestCall(json);
EXPECT_EQ(recieved_invalidations.size(), 1);
EXPECT_EQ(recieved_invalidations[0].getStringSetAttr("attr2").unpack(), vals);
EXPECT_EQ(recieved_invalidations[0].getStringSetMainAttr("attr2").unpack(), vals);
}
TEST_F(IntelligenceInvalidation, delete_invalidation_callback)

5
core/intelligence_is_v2/intelligence_server.cc
View File

@ -43,7 +43,7 @@ Sender::Sender(IntelligenceRequest request) : request(request)
}
auto setting_server_ip = getSetting<string>("intelligence", "local intelligence server ip");
if (setting_server_ip.ok()) server_ip = *setting_server_ip;
if (setting_server_ip.ok() && is_local_intelligence) server_ip = *setting_server_ip;
}
Maybe<Response>
@ -80,6 +80,7 @@ Sender::sendQueryObjectToLocalServer(bool is_primary_port)
server_port = *local_port;
conn_flags.reset();
conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
auto res = sendQueryMessage();
@ -173,7 +174,7 @@ Sender::sendMessage()
MessageMetadata req_md(*server_ip, *server_port, conn_flags);
auto req_status = i_message->sendSyncMessage(
HTTPMethod::POST,
query_uri,
request.isBulk() ? queries_uri : query_uri,
request,
MessageCategory::INTELLIGENCE,
req_md

103
core/intelligence_is_v2/invalidation.cc
View File

@ -37,16 +37,16 @@ Invalidation::setClassifier(ClassifierType type, const string &val)
}
Invalidation &
Invalidation::setStringAttr(const string &attr, const string &val)
Invalidation::setStringAttr(const string &attr, const string &val, bool is_main)
{
string_main_attr[attr] = val;
is_main ? string_main_attr[attr] = val : string_attr[attr] = val;
return *this;
}
Invalidation &
Invalidation::setStringSetAttr(const string &attr, const set<string> &val)
Invalidation::setStringSetAttr(const string &attr, const set<string> &val, bool is_main)
{
set_string_main_attr[attr] = val;
is_main ? set_string_main_attr[attr] = val : set_string_attr[attr] = val;
return *this;
}
@ -64,8 +64,15 @@ Invalidation::setObjectType(ObjectType type)
return *this;
}
Invalidation &
Invalidation::setInvalidationType(InvalidationType type)
{
invalidation_type = type;
return *this;
}
Maybe<string, void>
Invalidation::getStringAttr(const string &attr) const
Invalidation::getStringMainAttr(const string &attr) const
{
auto val_ref = string_main_attr.find(attr);
if (val_ref == string_main_attr.end()) return genError<void>();
@ -73,13 +80,29 @@ Invalidation::getStringAttr(const string &attr) const
}
Maybe<set<string>, void>
Invalidation::getStringSetAttr(const string &attr) const
Invalidation::getStringSetMainAttr(const string &attr) const
{
auto val_ref = set_string_main_attr.find(attr);
if (val_ref == set_string_main_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<string, void>
Invalidation::getStringAttr(const string &attr) const
{
auto val_ref = string_attr.find(attr);
if (val_ref == string_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<set<string>, void>
Invalidation::getStringSetAttr(const string &attr) const
{
auto val_ref = set_string_attr.find(attr);
if (val_ref == set_string_attr.end()) return genError<void>();
return val_ref->second;
}
bool
Invalidation::report(I_Intelligence_IS_V2 *interface) const
{
@ -110,6 +133,12 @@ static const map<Intelligence::ObjectType, string> convertObjectType = {
{ Intelligence::ObjectType::SHORTLIVED, "shortLived" }
};
static const map<Intelligence::InvalidationType, string> convertInvalidationType = {
{ Intelligence::InvalidationType::ADD, "add" },
{ Intelligence::InvalidationType::DELETE, "delete" },
{ Intelligence::InvalidationType::UPDATE, "update" }
};
Maybe<string>
Invalidation::genJson() const
{
@ -145,6 +174,7 @@ Invalidation::genObject() const
}
if (object_type.ok()) invalidation <<", \"objectType\": \"" << convertObjectType.at(*object_type) << '"';
invalidation << ", \"invalidationType\": \"" << convertInvalidationType.at(invalidation_type) << '"';
if (source_id.ok()) invalidation <<", \"sourceId\": \"" << *source_id << '"';
if (!string_main_attr.empty() || !set_string_main_attr.empty()) {
@ -161,9 +191,35 @@ Invalidation::genObject() const
auto val = makeSeparatedStr(attr.second, ", ");
invalidation << "{ \"" << attr.first << "\": [ ";
bool internal_first = true;
for (auto &val : attr.second) {
for (auto &value : attr.second) {
if (!internal_first) invalidation << ", ";
invalidation << "\"" << val << "\"";
invalidation << "\"" << value << "\"";
internal_first = false;
}
invalidation << " ] }";
first = false;
}
invalidation << " ]";
}
if (!string_attr.empty() || !set_string_attr.empty()) {
invalidation << ", \"attributes\": [ ";
bool first = true;
for (auto &attr : string_attr) {
if (!first) invalidation << ", ";
invalidation << "{ \"" << attr.first << "\": \"" << attr.second << "\" }";
first = false;
}
for (auto &attr : set_string_attr) {
if (!first) invalidation << ", ";
auto val = makeSeparatedStr(attr.second, ", ");
invalidation << "{ \"" << attr.first << "\": [ ";
bool internal_first = true;
for (auto &value : attr.second) {
if (!internal_first) invalidation << ", ";
invalidation << "\"" << value << "\"";
internal_first = false;
}
invalidation << " ] }";
@ -208,16 +264,29 @@ Invalidation::matches(const Invalidation &other) const
if (!other.object_type.ok() || *object_type != *other.object_type) return false;
}
if (invalidation_type != other.invalidation_type) return false;
if (source_id.ok()) {
if (!other.source_id.ok() || *source_id != *other.source_id) return false;
}
for (auto &key_value : string_main_attr) {
if (!other.hasAttr(key_value.first, key_value.second)) return false;
if (!other.hasMainAttr(key_value.first, key_value.second)) return false;
}
for (auto &key_values : set_string_main_attr) {
for (auto &value : key_values.second) {
if (!other.hasMainAttr(key_values.first, value)) return false;
}
}
for (auto &key_value : string_attr) {
if (!other.hasAttr(key_value.first, key_value.second)) return false;
}
for (auto &key_values : set_string_attr) {
for (auto &value : key_values.second) {
if (!other.hasAttr(key_values.first, value)) return false;
}
@ -227,7 +296,7 @@ Invalidation::matches(const Invalidation &other) const
}
bool
Invalidation::hasAttr(const string &key, const string &value) const
Invalidation::hasMainAttr(const string &key, const string &value) const
{
auto string_elem = string_main_attr.find(key);
if (string_elem != string_main_attr.end()) return string_elem->second == value;
@ -239,3 +308,17 @@ Invalidation::hasAttr(const string &key, const string &value) const
return false;
}
bool
Invalidation::hasAttr(const string &key, const string &value) const
{
auto string_elem = string_attr.find(key);
if (string_elem != string_attr.end()) return string_elem->second == value;
auto set_string_elem = set_string_attr.find(key);
if (set_string_elem != set_string_attr.end()) {
return set_string_elem->second.find(value) != set_string_elem->second.end();
}
return false;
}

3
core/logging/k8s_svc_stream.cc
View File

@ -41,6 +41,8 @@ K8sSvcStream::sendLog(const Report &log)
MessageMetadata rest_req_md(svc_host, 80);
rest_req_md.insertHeader("X-Tenant-Id", Singleton::Consume<I_AgentDetails>::by<LoggingComp>()->getTenantId());
rest_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
bool ok = i_msg->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
K8sSvc_log_uri,
@ -69,6 +71,7 @@ K8sSvcStream::sendLog(const LogBulkRest &logs, bool persistence_only)
MessageMetadata rest_req_md(svc_host, 80);
rest_req_md.insertHeader("X-Tenant-Id", Singleton::Consume<I_AgentDetails>::by<LoggingComp>()->getTenantId());
rest_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
bool ok = i_msg->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
K8sSvc_log_uri,

6
core/logging/logging_ut/logging_ut.cc
View File

@ -171,6 +171,7 @@ public:
_,
_,
MessageCategory::LOG,
_,
_
)).WillRepeatedly(SaveArg<2>(&body));
}
@ -745,6 +746,7 @@ TEST_F(LogTest, FogBulkLogs)
_,
_,
MessageCategory::LOG,
_,
_
)).WillRepeatedly(SaveArg<2>(&local_body));
@ -1143,6 +1145,7 @@ TEST(LogTestInstanceAwareness, LogGenInstanceAwareness)
_,
_,
MessageCategory::LOG,
_,
_
)).Times(AnyNumber());
@ -1248,6 +1251,7 @@ TEST(LogTestWithoutComponent, RegisterBasicConfig)
_,
_,
MessageCategory::LOG,
_,
_
)).Times(AnyNumber());
@ -1295,6 +1299,7 @@ TEST(LogTestWithoutComponent, RegisterAdvancedConfig)
_,
_,
MessageCategory::LOG,
_,
_
)).Times(AnyNumber());
@ -1366,6 +1371,7 @@ TEST_F(LogTest, BulkModification)
_,
_,
MessageCategory::LOG,
_,
_
)).WillRepeatedly(SaveArg<2>(&local_body));

2
core/mainloop/mainloop_ut/mainloop_ut.cc
View File

@ -51,7 +51,7 @@ public:
{
EXPECT_CALL(
mock_msg,
sendAsyncMessage(_, "/api/v1/agents/events", _, _, _)
sendAsyncMessage(_, "/api/v1/agents/events", _, _, _, _)
).Times(2).WillRepeatedly(
WithArgs<2, 3>(
Invoke(

79
core/messaging/connection/connection.cc
View File

@ -38,6 +38,10 @@ USE_DEBUG_FLAG(D_CONNECTION);
static const HTTPResponse sending_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to send all data in time");
static const HTTPResponse receving_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to receive all data in time");
static const HTTPResponse parsing_error(HTTPStatusCode::HTTP_UNKNOWN, "Failed to parse the HTTP response");
static const HTTPResponse close_error(
HTTPStatusCode::HTTP_UNKNOWN,
"The previous request failed to receive a response. Closing the connection"
);
const string &
MessageConnectionKey::getHostName() const
@ -125,6 +129,12 @@ public:
return key;
}
bool
shouldCloseConnection() const
{
return should_close_connection;
}
bool
isOverProxy() const
{
@ -152,12 +162,12 @@ public:
if (establishConnection().ok()) {
dbgDebug(D_MESSAGING) << "Reestablish connection";
return true;
return false;
}
dbgWarning(D_MESSAGING) << "Reestablish connection failed";
active = genError(curr_time + chrono::seconds(300));
return false;
return true;
}
Maybe<void>
@ -200,6 +210,7 @@ public:
<< key.getPort()
<< (isOverProxy() ? ", Over proxy: " + settings.getProxyHost() + ":" + to_string(key.getPort()) : "");
active = Maybe<void, chrono::seconds>();
should_close_connection = false;
return Maybe<void>();
}
@ -443,25 +454,40 @@ private:
I_MainLoop *i_mainloop = Singleton::Consume<I_MainLoop>::by<Messaging>();
I_TimeGet *i_time = Singleton::Consume<I_TimeGet>::by<Messaging>();
auto conn_end_time = i_time->getMonotonicTime() + getConnectionTimeout();
auto bio_connect = BIO_do_connect(bio.get());
uint attempts_count = 0;
while (i_time->getMonotonicTime() < conn_end_time) {
attempts_count++;
if (BIO_do_connect(bio.get()) > 0) {
if (isUnsecure() || isOverProxy()) return Maybe<void>();
return performHandshakeAndVerifyCert(i_time, i_mainloop);
}
auto conn_end_time = i_time->getMonotonicTime() + getConnectionTimeout();
while (i_time->getMonotonicTime() < conn_end_time && bio_connect <= 0) {
if (!BIO_should_retry(bio.get())) {
auto curr_time = chrono::duration_cast<chrono::seconds>(i_time->getMonotonicTime());
active = genError(curr_time + chrono::seconds(60));
string bio_error = ERR_error_string(ERR_get_error(), nullptr);
return genError("Failed to connect (BIO won't retry!): " + bio_error);
return genError(
"Failed to connect to: " +
full_address +
", error: " +
bio_error +
". Connection suspended for 60 seconds");
}
if ((attempts_count % 10) == 0) i_mainloop->yield(true);
attempts_count++;
if (!isBioSocketReady()) {
i_mainloop->yield((attempts_count % 10) == 0);
continue;
}
bio_connect = BIO_do_connect(bio.get());
}
return genError("Failed to establish new connection to " + full_address + " after reaching timeout.");
if (bio_connect > 0) {
if (isUnsecure() || isOverProxy()) return Maybe<void>();
return performHandshakeAndVerifyCert(i_time, i_mainloop);
}
auto curr_time = chrono::duration_cast<chrono::seconds>(i_time->getMonotonicTime());
active = genError(curr_time + chrono::seconds(60));
return genError(
"Failed to establish new connection to: " +
full_address +
" after reaching timeout." +
" Connection suspended for 60 seconds");
}
Maybe<uint, HTTPResponse>
@ -544,6 +570,7 @@ private:
Maybe<HTTPResponse, HTTPResponse>
sendAndReceiveData(const string &request, bool is_connect)
{
dbgFlow(D_CONNECTION) << "Sending and receiving data";
I_MainLoop *i_mainloop = Singleton::Consume<I_MainLoop>::by<Messaging>();
while (lock) {
i_mainloop->yield(true);
@ -551,6 +578,11 @@ private:
lock = true;
auto unlock = make_scope_exit([&] () { lock = false; });
if (should_close_connection) {
dbgWarning(D_CONNECTION) << close_error.getBody();
return genError(close_error);
}
I_TimeGet *i_time = Singleton::Consume<I_TimeGet>::by<Messaging>();
auto sending_end_time = i_time->getMonotonicTime() + getConnectionTimeout();
size_t data_left_to_send = request.length();
@ -567,9 +599,15 @@ private:
HTTPResponseParser http_parser;
dbgTrace(D_CONNECTION) << "Sent the message, now waiting for response";
while (!http_parser.hasReachedError()) {
if (i_time->getMonotonicTime() > receiving_end_time) return genError(receving_timeout);
if (i_time->getMonotonicTime() > receiving_end_time) {
should_close_connection = true;
return genError(receving_timeout);
};
auto receieved = receiveData();
if (!receieved.ok()) return receieved.passErr();
if (!receieved.ok()) {
should_close_connection = true;
return receieved.passErr();
}
auto response = http_parser.parseData(*receieved, is_connect);
if (response.ok()) {
dbgTrace(D_MESSAGING) << printOut(response.unpack().toString());
@ -611,6 +649,7 @@ private:
uint failed_attempts = 0;
bool lock = false;
bool should_close_connection = false;
};
Connection::Connection(const MessageConnectionKey &key, const MessageMetadata &metadata)
@ -663,6 +702,12 @@ Connection::getConnKey() const
return pimpl->getConnKey();
}
bool
Connection::shouldCloseConnection() const
{
return pimpl->shouldCloseConnection();
}
bool
Connection::isOverProxy() const
{

6
core/messaging/connection/connection_comp.cc
View File

@ -53,6 +53,10 @@ public:
{
auto conn = persistent_connections.find(MessageConnectionKey(host_name, port, category));
if (conn == persistent_connections.end()) return genError("No persistent connection found");
if (conn->second.shouldCloseConnection()) {
persistent_connections.erase(conn);
return genError("The connection needs to reestablish");
}
return conn->second;
}
@ -93,6 +97,7 @@ private:
if (!external_certificate.empty()) conn.setExternalCertificate(external_certificate);
auto connected = conn.establishConnection();
persistent_connections.emplace(conn_key, conn);
if (!connected.ok()) {
string connection_err = "Failed to establish connection. Error: " + connected.getErr();
@ -101,7 +106,6 @@ private:
}
dbgTrace(D_CONNECTION) << "Connection establish succssesfuly";
persistent_connections.emplace(conn_key, conn);
return conn;
}

110
core/messaging/connection/connection_ut/connection_comp_ut.cc
View File

@ -23,83 +23,13 @@
#include "mock/mock_encryptor.h"
#include "rest.h"
#include "rest_server.h"
#include "dummy_socket.h"
using namespace std;
using namespace testing;
USE_DEBUG_FLAG(D_CONNECTION);
class DummySocket : Singleton::Consume<I_MainLoop>
{
public:
~DummySocket()
{
if (server_fd != -1) close(server_fd);
if (connection_fd != -1) close(connection_fd);
}
void
init()
{
server_fd = socket(AF_INET, SOCK_STREAM, 0);
dbgAssert(server_fd >= 0) << "Failed to open a socket";
int socket_enable = 1;
setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int));
struct sockaddr_in addr;
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
addr.sin_port = htons(8080);
bind(server_fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in));
listen(server_fd, 100);
}
void
acceptSocket()
{
if (connection_fd == -1) connection_fd = accept(server_fd, nullptr, nullptr);
}
string
readFromSocket()
{
acceptSocket();
string res;
char buffer[1024];
while (int bytesRead = readRaw(buffer, sizeof(buffer))) {
res += string(buffer, bytesRead);
}
return res;
}
void
writeToSocket(const string &msg)
{
acceptSocket();
EXPECT_EQ(write(connection_fd, msg.data(), msg.size()), msg.size());
}
private:
int
readRaw(char *buf, uint len)
{
struct pollfd s_poll;
s_poll.fd = connection_fd;
s_poll.events = POLLIN;
s_poll.revents = 0;
if (poll(&s_poll, 1, 0) <= 0 || (s_poll.revents & POLLIN) == 0) return 0;
return read(connection_fd, buf, len);
}
int server_fd = -1;
int connection_fd = -1;
};
static ostream &
operator<<(ostream &os, const BufferedMessage &)
{
@ -179,7 +109,6 @@ TEST_F(TestConnectionComp, testEstablishNewConnection)
conn_metadata.setExternalCertificate("external cert");
auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG);
cout << "[OREN] read: " << endl;
ASSERT_TRUE(maybe_connection.ok());
auto get_conn = maybe_connection.unpack();
EXPECT_EQ(get_conn.getConnKey().getHostName(), "127.0.0.1");
@ -228,6 +157,43 @@ TEST_F(TestConnectionComp, testSendRequest)
EXPECT_EQ(dummy_socket.readFromSocket(), expected_msg);
}
TEST_F(TestConnectionComp, testCloseConnectionBeforeResponse)
{
// Create a connection
Flags<MessageConnectionConfig> conn_flags;
conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
MessageMetadata conn_metadata("127.0.0.1", 8080, conn_flags);
// Insert the connection to the map
auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG);
ASSERT_TRUE(maybe_connection.ok());
// Get the connection from the map - Should be successful
auto maybe_get_connection = i_conn->getPersistentConnection("127.0.0.1", 8080, MessageCategory::LOG);
ASSERT_TRUE(maybe_get_connection.ok());
auto conn = maybe_get_connection.unpack();
auto req = HTTPRequest::prepareRequest(conn, HTTPMethod::POST, "/test", conn_metadata.getHeaders(), "test-body");
ASSERT_TRUE(req.ok());
// force the connection to be closed
ON_CALL(mock_mainloop, yield(false)).WillByDefault(InvokeWithoutArgs([&] () { return; }));
EXPECT_CALL(mock_timer, getMonotonicTime())
.WillRepeatedly(Invoke([] () {static int j = 0; return chrono::microseconds(++j * 1000 * 1000);}));
auto maybe_response = i_conn->sendRequest(conn, *req);
ASSERT_TRUE(!maybe_response.ok());
ASSERT_EQ(
maybe_response.getErr().toString(),
"[Status-code]: -1 - HTTP_UNKNOWN, [Body]: Failed to receive all data in time"
);
auto maybe_get_closed_connection = i_conn->getPersistentConnection("127.0.0.1", 8080, MessageCategory::LOG);
ASSERT_TRUE(!maybe_get_closed_connection.ok());
ASSERT_EQ(maybe_get_closed_connection.getErr(), "The connection needs to reestablish");
}
TEST_F(TestConnectionComp, testSendRequestReplyChunked)
{
Flags<MessageConnectionConfig> conn_flags;

1
core/messaging/include/connection.h
View File

@ -63,6 +63,7 @@ public:
bool isOverProxy() const;
bool isUnsecure() const;
bool isSuspended();
bool shouldCloseConnection() const;
Maybe<void> establishConnection();
Maybe<HTTPResponse, HTTPResponse> sendRequest(const std::string &request);

100
core/messaging/include/dummy_socket.h Normal file
View File

@ -0,0 +1,100 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __DUMMY_SOCKET_H__
#define __DUMMY_SOCKET_H__
#include <sys/socket.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <pthread.h>
#include <thread>
#include <fcntl.h>
#include <poll.h>
#include "singleton.h"
#include "i_mainloop.h"
#include "agent_core_utilities.h"
class DummySocket : Singleton::Consume<I_MainLoop>
{
public:
~DummySocket()
{
if (server_fd != -1) close(server_fd);
if (connection_fd != -1) close(connection_fd);
}
void
init()
{
server_fd = socket(AF_INET, SOCK_STREAM, 0);
dbgAssert(server_fd >= 0) << "Failed to open a socket";
int socket_enable = 1;
setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int));
struct sockaddr_in addr;
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
addr.sin_port = htons(8080);
bind(server_fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in));
listen(server_fd, 100);
}
void
acceptSocket()
{
if (connection_fd == -1) connection_fd = accept(server_fd, nullptr, nullptr);
}
std::string
readFromSocket()
{
acceptSocket();
std::string res;
char buffer[1024];
while (int bytesRead = readRaw(buffer, sizeof(buffer))) {
res += std::string(buffer, bytesRead);
}
return res;
}
void
writeToSocket(const std::string &msg)
{
acceptSocket();
EXPECT_EQ(write(connection_fd, msg.data(), msg.size()), msg.size());
}
private:
int
readRaw(char *buf, uint len)
{
struct pollfd s_poll;
s_poll.fd = connection_fd;
s_poll.events = POLLIN;
s_poll.revents = 0;
if (poll(&s_poll, 1, 0) <= 0 || (s_poll.revents & POLLIN) == 0) return 0;
return read(connection_fd, buf, len);
}
int server_fd = -1;
int connection_fd = -1;
};
#endif // __DUMMY_SOCKET_H__

3
core/messaging/include/messaging_comp.h
View File

@ -50,7 +50,8 @@ public:
const std::string &uri,
const std::string &body,
MessageCategory category,
const MessageMetadata &message_metadata
const MessageMetadata &message_metadata,
bool force_buffering = true
);
Maybe<HTTPStatusCode, HTTPResponse> downloadFile(

5
core/messaging/messaging.cc
View File

@ -55,10 +55,11 @@ public:
const std::string &uri,
const std::string &body,
const MessageCategory category,
MessageMetadata message_metadata
const MessageMetadata &message_metadata,
bool force_buffering
) override
{
return messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata);
return messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata, force_buffering);
}
Maybe<HTTPStatusCode, HTTPResponse>

10
core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc
View File

@ -153,8 +153,6 @@ MessagingBufferComponent::Impl::pushNewBufferedMessage(
{
dbgTrace(D_MESSAGING_BUFFER) << "Pushing new message to buffer";
message_metadata.setShouldBufferMessage(false);
if (!force_immediate_writing) {
dbgDebug(D_MESSAGING_BUFFER) << "Holding message temporarily in memory";
memory_messages.emplace_back(body, method, uri, category, message_metadata);
@ -296,12 +294,14 @@ MessagingBufferComponent::Impl::sendMessage()
HTTPStatusCode
MessagingBufferComponent::Impl::sendMessage(const BufferedMessage &message) const
{
MessageMetadata message_metadata = message.getMessageMetadata();
message_metadata.setShouldBufferMessage(false);
auto res = messaging->sendSyncMessage(
message.getMethod(),
message.getURI(),
message.getBody(),
message.getCategory(),
message.getMessageMetadata()
message_metadata
);
if (res.ok()) return HTTPStatusCode::HTTP_OK;
@ -316,7 +316,9 @@ MessagingBufferComponent::Impl::handleInMemoryMessages()
memory_messages.reserve(32);
for (const auto &message : messages) {
if (sendMessage(message) != HTTPStatusCode::HTTP_OK) writeToDisk(message);
if (sendMessage(message) != HTTPStatusCode::HTTP_OK) {
if (message.getMessageMetadata().shouldBufferMessage()) writeToDisk(message);
}
mainloop->yield();
}
}

22
core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc
View File

@ -233,17 +233,36 @@ TEST_F(TestMessagingBuffer, testRoutinInMemory)
string body_1 = "body1";
string body_2 = "body2";
string body_3 = "body3";
string body_4 = "body4";
string uri_1 = "uri_1";
string uri_2 = "uri_2";
string uri_3 = "uri_3";
string uri_4 = "uri_4";
MessageCategory category = MessageCategory::GENERIC;
MessageMetadata message_metadata = MessageMetadata();
MessageMetadata msg_2_message_metadata = MessageMetadata();
msg_2_message_metadata.setShouldBufferMessage(true);
HTTPMethod method = HTTPMethod::POST;
buffer_provider->pushNewBufferedMessage(body_1, method, uri_1, category, message_metadata, false);
buffer_provider->pushNewBufferedMessage(body_2, method, uri_2, category, message_metadata, false);
buffer_provider->pushNewBufferedMessage(
body_2,
method,
uri_2,
category,
msg_2_message_metadata,
false
); // should be buffered
buffer_provider->pushNewBufferedMessage(body_3, method, uri_3, category, message_metadata, false);
buffer_provider->pushNewBufferedMessage(
body_4,
method,
uri_4,
category,
message_metadata,
false
); // shouldn't be buffered
HTTPResponse res(HTTPStatusCode::HTTP_OK, "");
Maybe<HTTPResponse, HTTPResponse> err = genError(res);
@ -251,6 +270,7 @@ TEST_F(TestMessagingBuffer, testRoutinInMemory)
EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_1, body_1, _, _)).WillOnce(Return(res));
EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_2, body_2, _, _)).WillOnce(Return(err));
EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_3, body_3, _, _)).WillOnce(Return(res));
EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_4, body_4, _, _)).WillOnce(Return(err));
memory_routine();

7
core/messaging/messaging_comp/messaging_comp.cc
View File

@ -136,10 +136,13 @@ MessagingComp::sendAsyncMessage(
const string &uri,
const string &body,
MessageCategory category,
const MessageMetadata &message_metadata
const MessageMetadata &message_metadata,
bool force_buffering
)
{
i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false);
MessageMetadata new_message_metadata = message_metadata;
new_message_metadata.setShouldBufferMessage(force_buffering);
i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, new_message_metadata, false);
}
Maybe<HTTPStatusCode, HTTPResponse>

35
core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc
View File

@ -15,6 +15,7 @@
#include "mocks/mock_messaging_connection.h"
#include "rest.h"
#include "rest_server.h"
#include "dummy_socket.h"
using namespace std;
using namespace testing;
@ -48,11 +49,13 @@ class TestMessagingComp : public testing::Test
public:
TestMessagingComp()
{
Debug::setUnitTestFlag(D_MESSAGING, Debug::DebugLevel::TRACE);
EXPECT_CALL(mock_time_get, getMonotonicTime()).WillRepeatedly(Return(chrono::microseconds(0)));
ON_CALL(mock_agent_details, getFogDomain()).WillByDefault(Return(Maybe<string>(fog_addr)));
ON_CALL(mock_agent_details, getFogPort()).WillByDefault(Return(Maybe<uint16_t>(fog_port)));
messaging_comp.init();
dummy_socket.init();
}
void
@ -70,8 +73,8 @@ public:
EXPECT_CALL(mock_proxy_conf, getProxyAuthentication(_)).WillRepeatedly(Return(string("cred")));
}
const string fog_addr = "1.2.3.4";
uint16_t fog_port = 80;
const string fog_addr = "127.0.0.1";
int fog_port = 8080;
CPTestTempfile agent_details_file;
MessagingComp messaging_comp;
::Environment env;
@ -82,6 +85,7 @@ public:
NiceMock<MockTimeGet> mock_time_get;
NiceMock<MockAgentDetails> mock_agent_details;
NiceMock<MockProxyConfiguration> mock_proxy_conf;
DummySocket dummy_socket;
};
TEST_F(TestMessagingComp, testInitComp)
@ -100,16 +104,19 @@ TEST_F(TestMessagingComp, testSendSyncMessage)
HTTPMethod method = HTTPMethod::POST;
string uri = "/test-uri";
MessageCategory category = MessageCategory::GENERIC;
MessageMetadata message_metadata;
MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC);
Connection conn(conn_key, message_metadata);
Flags<MessageConnectionConfig> conn_flags;
conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags, false, true);
Connection conn(conn_key, conn_metadata);
EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC))
.WillOnce(Return(conn));
HTTPResponse res(HTTPStatusCode::HTTP_OK, "response!!");
EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res));
auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, message_metadata);
auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, conn_metadata);
ASSERT_TRUE(sending_res.ok());
HTTPResponse http_res = sending_res.unpack();
EXPECT_EQ(http_res.getBody(), "response!!");
@ -126,7 +133,7 @@ TEST_F(TestMessagingComp, testSendAsyncMessage)
MessageMetadata message_metadata;
EXPECT_CALL(mock_messaging_buffer, pushNewBufferedMessage(body, method, uri, category, _, _)).Times(1);
messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata);
messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata, true);
}
TEST_F(TestMessagingComp, testSendSyncMessageOnSuspendedConn)
@ -163,16 +170,18 @@ TEST_F(TestMessagingComp, testUploadFile)
setAgentDetails();
string uri = "/test-uri";
MessageCategory category = MessageCategory::GENERIC;
MessageMetadata message_metadata;
MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC);
Connection conn(conn_key, message_metadata);
Flags<MessageConnectionConfig> conn_flags;
conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags, false, true);
Connection conn(conn_key, conn_metadata);
EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC))
.WillOnce(Return(conn));
HTTPResponse res(HTTPStatusCode::HTTP_OK, "");
EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res));
auto upload_res = messaging_comp.uploadFile(uri, path, category, message_metadata);
auto upload_res = messaging_comp.uploadFile(uri, path, category, conn_metadata);
ASSERT_TRUE(upload_res.ok());
EXPECT_EQ(upload_res.unpack(), HTTPStatusCode::HTTP_OK);
}
@ -185,16 +194,18 @@ TEST_F(TestMessagingComp, testDownloadFile)
string uri = "/test-uri";
HTTPMethod method = HTTPMethod::GET;
MessageCategory category = MessageCategory::GENERIC;
MessageMetadata message_metadata;
MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC);
Connection conn(conn_key, message_metadata);
Flags<MessageConnectionConfig> conn_flags;
conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags, false, true);
Connection conn(conn_key, conn_metadata);
EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC))
.WillOnce(Return(conn));
HTTPResponse res(HTTPStatusCode::HTTP_OK, "");
EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res));
auto upload_res = messaging_comp.downloadFile(method, uri, "/tmp/test.txt", category, message_metadata);
auto upload_res = messaging_comp.downloadFile(method, uri, "/tmp/test.txt", category, conn_metadata);
ASSERT_TRUE(upload_res.ok());
EXPECT_EQ(upload_res.unpack(), HTTPStatusCode::HTTP_OK);
}

8
core/metric/generic_metric.cc
View File

@ -65,7 +65,8 @@ GenericMetric::init(
const ReportIS::IssuingEngine &_issuing_engine,
chrono::seconds _report_interval,
bool _reset,
Audience _audience
Audience _audience,
bool _force_buffering
)
{
i_mainloop = Singleton::Consume<I_MainLoop>::by<GenericMetric>();
@ -76,6 +77,7 @@ GenericMetric::init(
team = _team;
issuing_engine = _issuing_engine;
audience = _audience;
force_buffering = _force_buffering;
i_mainloop->addRecurringRoutine(
I_MainLoop::RoutineType::System,
@ -233,7 +235,9 @@ GenericMetric::sendLog(const LogRest &metric_client_rest) const
HTTPMethod::POST,
fog_metric_uri,
metric_client_rest,
MessageCategory::METRIC
MessageCategory::METRIC,
MessageMetadata(),
force_buffering
);
}

11
core/metric/metric_ut/metric_ut.cc
View File

@ -215,6 +215,7 @@ TEST_F(MetricTest, basicMetricTest)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@ -283,6 +284,7 @@ TEST_F(MetricTest, basicMetricTest)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@ -353,6 +355,7 @@ TEST_F(MetricTest, basicMetricTest)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@ -433,6 +436,7 @@ TEST_F(MetricTest, printMetricsTest)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).Times(AnyNumber());
@ -504,6 +508,7 @@ TEST_F(MetricTest, metricTestWithReset)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@ -571,6 +576,7 @@ TEST_F(MetricTest, metricTestWithReset)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@ -638,6 +644,7 @@ TEST_F(MetricTest, metricTestWithReset)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@ -727,6 +734,7 @@ TEST_F(MetricTest, generateReportWithReset)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@ -801,6 +809,7 @@ TEST_F(MetricTest, generateReportWithReset)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@ -869,6 +878,7 @@ TEST_F(MetricTest, generateReportWithReset)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@ -1018,6 +1028,7 @@ TEST_F(MetricTest, testMapMetric)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
routine();

4
core/rest/rest_server.cc
View File

@ -133,7 +133,7 @@ RestServer::Impl::init()
auto is_primary = Singleton::Consume<I_Environment>::by<RestServer>()->get<bool>("Is Rest primary routine");
id = mainloop->addFileRoutine(
I_MainLoop::RoutineType::Offline,
I_MainLoop::RoutineType::RealTime,
fd,
[&] () { this->startNewConnection(); },
"REST server listener",
@ -174,7 +174,7 @@ RestServer::Impl::startNewConnection() const
RestConn conn(new_socket, mainloop, this);
mainloop->addFileRoutine(
I_MainLoop::RoutineType::Offline,
I_MainLoop::RoutineType::RealTime,
new_socket,
[conn] () { conn.parseConn(); },
"REST server connection handler"

2
nodes/orchestration/package/local-default-policy.yaml
View File

@ -52,7 +52,7 @@ log-triggers:
url-path: false
url-query: false
log-destination:
cloud: true
cloud: false
stdout:
format: json

Some files were not shown because too many files have changed in this diff Show More