From fd2d9fa081db56c74f1585b1c529024c131f9273 Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Thu, 27 Apr 2023 19:05:49 +0000 Subject: [PATCH] Apr 27th Update --- README.md | 3 +- build_system/docker/Dockerfile | 1 - components/generic_rulebase/match_query.cc | 5 +- .../generic_rulebase/triggers_config.cc | 1 + components/http_manager/http_manager.cc | 36 ++ components/http_manager/http_manager_opaque.h | 3 + components/include/i_local_policy_mgmt_gen.h | 4 +- components/include/i_pm_scan.h | 9 +- components/include/i_service_controller.h | 4 + components/include/layer_7_access_control.h | 33 ++ components/include/mock/mock_http_manager.h | 19 + components/security_apps/CMakeLists.txt | 1 + .../layer_7_access_control/CMakeLists.txt | 3 + .../layer_7_access_control.cc | 348 +++++++++++++++ .../layer_7_access_control_ut/CMakeLists.txt | 7 + .../data/malicious_intelligence_response.json | 87 ++++ .../data/ok_intelligence_response.json | 6 + .../layer_7_access_control_ut.cc | 413 ++++++++++++++++++ .../details_resolver/details_resolver.cc | 13 +- .../details_resolver_impl.h | 9 + .../include/declarative_policy_utils.h | 7 +- .../include/mock/mock_orchestration_tools.h | 7 + .../include/mock/mock_service_controller.h | 4 + .../include/policy_maker_utils.h | 17 +- .../include/triggers_section.h | 12 +- .../local_policy_mgmt_gen.cc | 78 +--- .../policy_maker_utils.cc | 54 ++- .../local_policy_mgmt_gen/triggers_section.cc | 13 + .../orchestration/orchestration_comp.cc | 138 +++++- .../orchestration_multitenant_ut.cc | 23 +- .../orchestration_ut/orchestration_ut.cc | 35 +- .../service_controller/service_controller.cc | 41 +- .../service_controller_ut.cc | 36 +- .../declarative_policy_utils.cc | 9 +- .../reputation/reputation_features_agg.cc | 10 + .../waap/waap_clib/TuningDecision.cc | 14 +- .../waap/waap_clib/WaapRegexPreconditions.cc | 77 ++-- .../waap/waap_clib/WaapRegexPreconditions.h | 4 +- .../waap/waap_clib/Waf2Engine.cc | 3 + .../security_apps/waap/waap_clib/Waf2Util.cc | 2 +- components/utils/pm/pm_hook.cc | 29 +- components/utils/pm/pm_ut/pm_scan_ut.cc | 55 ++- .../agent_details_report.cc | 18 + .../agent_details_reporter.cc | 13 + core/config/config.cc | 9 +- core/core_ut/cache_ut.cc | 12 + core/include/general/debug.h | 3 + .../interfaces/i_agent_details_reporter.h | 1 + .../interfaces/i_intelligence_is_v2.h | 24 +- .../services_sdk/interfaces/i_mainloop.h | 2 + .../intelligence_types_v2.h | 2 + .../intelligence_is_v2/query_request_v2.h | 4 + .../intelligence_is_v2/query_types_v2.h | 25 +- .../interfaces/mock/mock_mainloop.h | 2 + .../interfaces/mock/mock_shell_cmd.h | 13 +- .../interfaces/mock/mock_socket_is.h | 2 + .../resources/agent_details_report.h | 5 + .../services_sdk/resources/debug_flags.h | 6 + .../resources/report/Log_modifiers.h | 82 ++++ .../resources/report/base_field.h | 50 +-- .../resources/report/report_enums.h | 1 + core/include/services_sdk/utilities/cache.h | 1 + .../utilities/caching/cache_impl.h | 8 + .../utilities/caching/cache_types.h | 1 + .../intelligence_comp_v2.cc | 3 + .../query_request_v2_ut.cc | 103 +++++ core/intelligence_is_v2/query_request_v2.cc | 40 ++ core/intelligence_is_v2/query_types_v2.cc | 31 +- core/logging/log_streams.h | 1 + core/logging/logging_ut/logging_ut.cc | 2 +- core/logging/syslog_stream.cc | 43 +- core/mainloop/mainloop.cc | 3 +- core/message/http_decoder.cc | 112 ++--- core/message/http_decoder.h | 1 - core/message/message.cc | 25 +- core/report/report.cc | 15 +- core/report/report_ut/report_ut.cc | 23 +- core/report/tag_and_enum_management.cc | 6 +- external/graphqlparser/CMakeLists.txt | 6 +- external/graphqlparser/ast/c.py | 16 +- external/graphqlparser/ast/c_impl.py | 18 +- external/graphqlparser/ast/c_visitor_impl.py | 6 +- external/graphqlparser/ast/cxx.py | 88 ++-- external/graphqlparser/ast/cxx_impl.py | 22 +- .../ast/cxx_json_visitor_header.py | 8 +- .../ast/cxx_json_visitor_impl.py | 24 +- external/graphqlparser/ast/cxx_visitor.py | 26 +- .../orchestration/package/open-appsec-ctl.sh | 103 ++++- .../package/orchestration_package.sh | 37 +- 89 files changed, 2175 insertions(+), 544 deletions(-) create mode 100644 components/include/layer_7_access_control.h create mode 100755 components/include/mock/mock_http_manager.h create mode 100644 components/security_apps/layer_7_access_control/CMakeLists.txt create mode 100644 components/security_apps/layer_7_access_control/layer_7_access_control.cc create mode 100644 components/security_apps/layer_7_access_control/layer_7_access_control_ut/CMakeLists.txt create mode 100644 components/security_apps/layer_7_access_control/layer_7_access_control_ut/data/malicious_intelligence_response.json create mode 100644 components/security_apps/layer_7_access_control/layer_7_access_control_ut/data/ok_intelligence_response.json create mode 100644 components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc create mode 100644 core/include/services_sdk/resources/report/Log_modifiers.h diff --git a/README.md b/README.md index 43770e4..ba3553d 100644 --- a/README.md +++ b/README.md @@ -90,13 +90,12 @@ Before compiling the services, you'll need to ensure the latest development vers * GTest * GMock * cURL -* Python2 An example of installing the packages on Alpine: ```bash $ apk update - $ apk add boost-dev openssl-dev pcre2-dev libxml2-dev gtest-dev curl-dev python2 + $ apk add boost-dev openssl-dev pcre2-dev libxml2-dev gtest-dev curl-dev ``` ## Compiling and packaging the agent code diff --git a/build_system/docker/Dockerfile b/build_system/docker/Dockerfile index cc56970..2152321 100644 --- a/build_system/docker/Dockerfile +++ b/build_system/docker/Dockerfile @@ -4,7 +4,6 @@ RUN apk add --no-cache -u busybox RUN apk add --no-cache -u zlib RUN apk add --no-cache bash RUN apk add --no-cache libstdc++ -RUN apk add --no-cache libexecinfo RUN apk add --no-cache boost RUN apk add --no-cache icu-libs RUN apk add --no-cache curl diff --git a/components/generic_rulebase/match_query.cc b/components/generic_rulebase/match_query.cc index 16b1582..f472ae7 100755 --- a/components/generic_rulebase/match_query.cc +++ b/components/generic_rulebase/match_query.cc @@ -245,14 +245,15 @@ MatchQuery::matchAttributes( } else if (type == MatchType::Operator && operator_type == Operators::Or) { // With 'or' condition, evaluate matched override keywords first and add the ones that were fully matched set inner_override_keywords; + bool res = false; for (const MatchQuery &inner_match: items) { inner_override_keywords.clear(); if (inner_match.matchAttributes(key_value_pairs, inner_override_keywords)) { matched_override_keywords.insert(inner_override_keywords.begin(), inner_override_keywords.end()); - return true; + res = true; } } - return false; + return res; } else { dbgWarning(D_RULEBASE_CONFIG) << "Unsupported match query type"; } diff --git a/components/generic_rulebase/triggers_config.cc b/components/generic_rulebase/triggers_config.cc index e0f43b0..01429ec 100755 --- a/components/generic_rulebase/triggers_config.cc +++ b/components/generic_rulebase/triggers_config.cc @@ -173,6 +173,7 @@ LogTriggerConf::load(cereal::JSONInputArchive& archive_in) setTriggersFlag("webUrlQuery", archive_in, WebLogFields::webUrlQuery, log_web_fields); setTriggersFlag("logToAgent", archive_in, ReportIS::StreamType::JSON_LOG_FILE, active_streams); setTriggersFlag("logToCloud", archive_in, ReportIS::StreamType::JSON_FOG, active_streams); + setTriggersFlag("logToK8sService", archive_in, ReportIS::StreamType::JSON_K8S_SVC, active_streams); setTriggersFlag("logToSyslog", archive_in, ReportIS::StreamType::SYSLOG, active_streams); setTriggersFlag("logToCef", archive_in, ReportIS::StreamType::CEF, active_streams); setTriggersFlag("acAllow", archive_in, SecurityType::AccessControl, should_log_on_detect); diff --git a/components/http_manager/http_manager.cc b/components/http_manager/http_manager.cc index 331fc90..4d544b3 100755 --- a/components/http_manager/http_manager.cc +++ b/components/http_manager/http_manager.cc @@ -90,6 +90,23 @@ public: ScopedContext ctx; ctx.registerValue(app_sec_marker_key, i_transaction_table->keyToString(), EnvKeyAttr::LogSection::MARKER); + HttpManagerOpaque &state = i_transaction_table->getState(); + string event_key = static_cast(event.getKey()); + if (event_key == getProfileAgentSettingWithDefault("", "agent.customHeaderValueLogging")) { + string event_value = static_cast(event.getValue()); + dbgTrace(D_HTTP_MANAGER) + << "Found header key and value - (" + << event_key + << ": " + << event_value + << ") that matched agent settings"; + state.setUserDefinedValue(event_value); + } + + if (state.getUserDefinedValue().ok()) { + ctx.registerValue("UserDefined", state.getUserDefinedValue().unpack(), EnvKeyAttr::LogSection::DATA); + } + auto event_responds = is_request ? HttpRequestHeaderEvent(event).performNamedQuery() : @@ -118,6 +135,9 @@ public: ScopedContext ctx; ctx.registerValue(app_sec_marker_key, i_transaction_table->keyToString(), EnvKeyAttr::LogSection::MARKER); + if (state.getUserDefinedValue().ok()) { + ctx.registerValue("UserDefined", state.getUserDefinedValue().unpack(), EnvKeyAttr::LogSection::DATA); + } FilterVerdict verdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT); if (!is_request && event.getData().size() == 0 && !event.isLastChunk()) { @@ -148,6 +168,11 @@ public: ScopedContext ctx; ctx.registerValue(app_sec_marker_key, i_transaction_table->keyToString(), EnvKeyAttr::LogSection::MARKER); + HttpManagerOpaque &state = i_transaction_table->getState(); + if (state.getUserDefinedValue().ok()) { + ctx.registerValue("UserDefined", state.getUserDefinedValue().unpack(), EnvKeyAttr::LogSection::DATA); + } + return handleEvent(ResponseCodeEvent(event).performNamedQuery()); } @@ -164,6 +189,9 @@ public: ScopedContext ctx; ctx.registerValue(app_sec_marker_key, i_transaction_table->keyToString(), EnvKeyAttr::LogSection::MARKER); + if (state.getUserDefinedValue().ok()) { + ctx.registerValue("UserDefined", state.getUserDefinedValue().unpack(), EnvKeyAttr::LogSection::DATA); + } return handleEvent(EndRequestEvent().performNamedQuery()); } @@ -181,6 +209,9 @@ public: ScopedContext ctx; ctx.registerValue(app_sec_marker_key, i_transaction_table->keyToString(), EnvKeyAttr::LogSection::MARKER); + if (state.getUserDefinedValue().ok()) { + ctx.registerValue("UserDefined", state.getUserDefinedValue().unpack(), EnvKeyAttr::LogSection::DATA); + } return handleEvent(EndTransactionEvent().performNamedQuery()); } @@ -196,6 +227,11 @@ public: ScopedContext ctx; ctx.registerValue(app_sec_marker_key, i_transaction_table->keyToString(), EnvKeyAttr::LogSection::MARKER); + HttpManagerOpaque &state = i_transaction_table->getState(); + if (state.getUserDefinedValue().ok()) { + ctx.registerValue("UserDefined", state.getUserDefinedValue().unpack(), EnvKeyAttr::LogSection::DATA); + } + return handleEvent(WaitTransactionEvent().performNamedQuery()); } diff --git a/components/http_manager/http_manager_opaque.h b/components/http_manager/http_manager_opaque.h index 5a11f62..23a50f1 100644 --- a/components/http_manager/http_manager_opaque.h +++ b/components/http_manager/http_manager_opaque.h @@ -31,6 +31,8 @@ public: ngx_http_cp_verdict_e getManagerVerdict() const { return manager_verdict; } ngx_http_cp_verdict_e getCurrVerdict() const; void saveCurrentDataToCache(const Buffer &full_data); + void setUserDefinedValue(const std::string &value) { user_defined_value = value; } + Maybe getUserDefinedValue() const { return user_defined_value; } const Buffer & getPreviousDataCache() const { return prev_data_cache; } uint getAggeregatedPayloadSize() const { return aggregated_payload_size; } void updatePayloadSize(const uint curr_payload); @@ -50,6 +52,7 @@ private: ngx_http_cp_verdict_e manager_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; Buffer prev_data_cache; uint aggregated_payload_size = 0; + Maybe user_defined_value = genError("uninitialized"); }; #endif // __HTTP_MANAGER_OPAQUE_H__ diff --git a/components/include/i_local_policy_mgmt_gen.h b/components/include/i_local_policy_mgmt_gen.h index e9f9b94..a4ae7ed 100755 --- a/components/include/i_local_policy_mgmt_gen.h +++ b/components/include/i_local_policy_mgmt_gen.h @@ -18,7 +18,9 @@ class I_LocalPolicyMgmtGen { public: virtual std::string parsePolicy(const std::string &policy_version) = 0; - virtual const std::string & getPolicyPath(void) const = 0; + virtual const std::string & getAgentPolicyPath(void) const = 0; + virtual const std::string & getLocalPolicyPath(void) const = 0; + virtual void setPolicyPath(const std::string &new_local_policy_path) = 0; protected: ~I_LocalPolicyMgmtGen() {} diff --git a/components/include/i_pm_scan.h b/components/include/i_pm_scan.h index 84f55cc..eb43265 100755 --- a/components/include/i_pm_scan.h +++ b/components/include/i_pm_scan.h @@ -27,12 +27,13 @@ class PMPattern { public: PMPattern() {} - PMPattern(const std::string &pat, bool start, bool end, uint index = 0) + PMPattern(const std::string &pat, bool start, bool end, uint index = 0, bool noRegex = false) : pattern(pat), match_start(start), match_end(end), - index(index) + index(index), + noRegex(noRegex) {} bool operator<(const PMPattern &other) const; @@ -44,18 +45,20 @@ public: size_t size() const { return pattern.size(); } bool empty() const { return pattern.empty(); } uint getIndex() const { return index; } + bool isNoRegex() const { return noRegex; } private: std::string pattern; bool match_start = false; bool match_end = false; uint index; + bool noRegex = false; }; class I_PMScan { public: - using CBFunction = std::function; + using CBFunction = std::function; virtual std::set scanBuf(const Buffer &buf) const = 0; virtual std::set> scanBufWithOffset(const Buffer &buf) const = 0; diff --git a/components/include/i_service_controller.h b/components/include/i_service_controller.h index a941180..a2f5070 100755 --- a/components/include/i_service_controller.h +++ b/components/include/i_service_controller.h @@ -47,6 +47,10 @@ public: const bool last_iteration = false ) = 0; + virtual bool doesFailedServicesExist() = 0; + + virtual void clearFailedServices() = 0; + virtual bool isServiceInstalled(const std::string &service_name) = 0; virtual void registerServiceConfig( diff --git a/components/include/layer_7_access_control.h b/components/include/layer_7_access_control.h new file mode 100644 index 0000000..f28ba9c --- /dev/null +++ b/components/include/layer_7_access_control.h @@ -0,0 +1,33 @@ +#ifndef __LAYER_7_ACCESS_CONTROL_H__ +#define __LAYER_7_ACCESS_CONTROL_H__ + +#include + +#include "singleton.h" +#include "i_mainloop.h" +#include "component.h" +#include "i_intelligence_is_v2.h" + +#include + +class Layer7AccessControl + : + public Component, + Singleton::Consume, + Singleton::Consume, + Singleton::Consume, + Singleton::Consume +{ +public: + Layer7AccessControl(); + ~Layer7AccessControl(); + + void init() override; + void fini() override; + +private: + class Impl; + std::unique_ptr pimpl; +}; + +#endif // __LAYER_7_ACCESS_CONTROL_H__ diff --git a/components/include/mock/mock_http_manager.h b/components/include/mock/mock_http_manager.h new file mode 100755 index 0000000..b941d8b --- /dev/null +++ b/components/include/mock/mock_http_manager.h @@ -0,0 +1,19 @@ +#ifndef __MOCK_HTTP_MANAGER_H__ +#define __MOCK_HTTP_MANAGER_H__ + +#include "i_http_manager.h" +#include "cptest.h" + +class MockHttpManager : public Singleton::Provide::From> +{ +public: + MOCK_METHOD1(inspect, FilterVerdict(const HttpTransactionData &)); + MOCK_METHOD2(inspect, FilterVerdict(const HttpHeader &, bool is_request)); + MOCK_METHOD2(inspect, FilterVerdict(const HttpBody &, bool is_request)); + MOCK_METHOD0(inspectEndRequest, FilterVerdict()); + MOCK_METHOD1(inspect, FilterVerdict(const ResponseCode &)); + MOCK_METHOD0(inspectEndTransaction, FilterVerdict()); + MOCK_METHOD0(inspectDelayedVerdict, FilterVerdict()); +}; + +#endif // __MOCK_HTTP_MANAGER_H__ diff --git a/components/security_apps/CMakeLists.txt b/components/security_apps/CMakeLists.txt index 8717429..68e330c 100644 --- a/components/security_apps/CMakeLists.txt +++ b/components/security_apps/CMakeLists.txt @@ -1,2 +1,3 @@ +add_subdirectory(layer_7_access_control) add_subdirectory(orchestration) add_subdirectory(waap) diff --git a/components/security_apps/layer_7_access_control/CMakeLists.txt b/components/security_apps/layer_7_access_control/CMakeLists.txt new file mode 100644 index 0000000..a3a3ee8 --- /dev/null +++ b/components/security_apps/layer_7_access_control/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(l7_access_control layer_7_access_control.cc) + +add_subdirectory(layer_7_access_control_ut) diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control.cc b/components/security_apps/layer_7_access_control/layer_7_access_control.cc new file mode 100644 index 0000000..189b3b6 --- /dev/null +++ b/components/security_apps/layer_7_access_control/layer_7_access_control.cc @@ -0,0 +1,348 @@ +#include "layer_7_access_control.h" + +#include +#include +#include + +#include "config.h" +#include "cache.h" +#include "http_inspection_events.h" +#include "http_transaction_common.h" +#include "nginx_attachment_common.h" +#include "intelligence_comp_v2.h" +#include "intelligence_is_v2/intelligence_query_v2.h" +#include "intelligence_is_v2/query_request_v2.h" +#include "log_generator.h" + +USE_DEBUG_FLAG(D_L7_ACCESS_CONTROL); + +using namespace std; +using namespace Intelligence_IS_V2; + +static const string crowdsec_enabled_value = "true"; +static const string crowdsec_asset_type = "data-cloud-ip-crowdSec"; + +class IntelligenceIpReputation +{ +public: + template + void + load(Archive &ar) + { + try { + vector ipv4_addresses; + ar(cereal::make_nvp("type", type)); + ar(cereal::make_nvp("scenario", scenario)); + ar(cereal::make_nvp("origin", origin)); + ar(cereal::make_nvp("crowdsecId", crowdsec_event_id)); + ar(cereal::make_nvp("ipv4Addresses", ipv4_addresses)); + if (!ipv4_addresses.empty()) ipv4_address = ipv4_addresses.front(); + } catch (const cereal::Exception &e) { + dbgWarning(D_L7_ACCESS_CONTROL) << "Failed to load IP reputation data JSON. Error: " << e.what(); + } + } + + Maybe + getType() const + { + if (type.empty()) return genError("Empty type"); + return LogField("externalVendorRecommendedAction", type); + } + + Maybe + getScenario() const + { + if (scenario.empty()) return genError("Empty scenario"); + return LogField("externalVendorRecommendationOriginDetails", scenario); + } + + Maybe + getOrigin() const + { + if (origin.empty()) return genError("Empty origin"); + return LogField("externalVendorRecommendationOrigin", origin); + } + + Maybe + getIpv4Address() const + { + if (ipv4_address.empty()) return genError("Empty ipv4 address"); + return LogField("externalVendorRecommendedAffectedScope", ipv4_address); + } + + Maybe + getCrowdsecEventId() const + { + if (!crowdsec_event_id) return genError("Empty ID"); + return LogField("externalVendorRecommendationId", crowdsec_event_id); + } + + bool isMalicious() const { return type == "ban"; } + + void + print(std::ostream &out) const + { + out + << "Crowdsec event ID: " + << crowdsec_event_id + << ", IPV4 address: " + << ipv4_address + << ", type: " + << type + << ", origin: " + << origin + << ", scenario: " + << scenario; + } + +private: + string type; + string scenario; + string origin; + string ipv4_address; + unsigned int crowdsec_event_id; +}; + +class Layer7AccessControl::Impl : public Listener +{ +public: + void init(); + void fini(); + + string getListenerName() const override { return "Layer-7 Access Control app"; } + + EventVerdict + respond(const HttpRequestHeaderEvent &event) override + { + dbgTrace(D_L7_ACCESS_CONTROL) << "Handling a new layer-7 access control event: " << event; + + if (!isAppEnabled()) { + dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Accept verdict as the Layer-7 Access Control app is disabled"; + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + } + + if (!event.isLastHeader()) { + dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Inspect verdict"; + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + } + + auto source_identifier = i_env->get(HttpTransactionData::source_identifier); + if (source_identifier.ok() && IPAddr::createIPAddr(source_identifier.unpack()).ok()) { + dbgTrace(D_L7_ACCESS_CONTROL) << "Found a valid source identifier value: " << source_identifier.unpack(); + return checkReputation(source_identifier.unpack()); + } + + auto orig_source_ip = i_env->get(HttpTransactionData::client_ip_ctx); + if (!orig_source_ip.ok()) { + dbgWarning(D_L7_ACCESS_CONTROL) << "Could not extract the Client IP address from context"; + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + } + + stringstream ss_client_ip; + ss_client_ip << orig_source_ip.unpack(); + return checkReputation(ss_client_ip.str()); + } + +private: + Maybe getIpReputation(const string &ip); + ngx_http_cp_verdict_e checkReputation(const string &source_ip); + void generateLog(const string &source_ip, const IntelligenceIpReputation &ip_reputation) const; + + bool isAppEnabled() const; + bool isPrevent() const; + + Maybe genLogField(const string &log_key, const string &env_key) const; + Maybe genLogIPField(const string &log_key, const string &env_key) const; + + I_Environment *i_env = nullptr; + I_Intelligence_IS_V2 *i_intelligence = nullptr; + TemporaryCache ip_reputation_cache; +}; + +bool +Layer7AccessControl::Impl::isAppEnabled() const +{ + bool enabled = getenv("CROWDSEC_ENABLED") ? string(getenv("CROWDSEC_ENABLED")) == crowdsec_enabled_value : false; + return getProfileAgentSettingWithDefault(enabled, "layer7AccessControl.crowdsec.enabled"); +} + +bool +Layer7AccessControl::Impl::isPrevent() const +{ + string security_mode_env = getenv("CROWDSEC_MODE") ? getenv("CROWDSEC_MODE") : "prevent"; + string mode = getProfileAgentSettingWithDefault(security_mode_env, "layer7AccessControl.securityMode"); + + dbgTrace(D_L7_ACCESS_CONTROL) << "Selected security mode: " << mode; + + return mode == "prevent"; +} + +Maybe +Layer7AccessControl::Impl::getIpReputation(const string &ip) +{ + dbgFlow(D_L7_ACCESS_CONTROL) << "Getting reputation of IP " << ip; + + if (ip_reputation_cache.doesKeyExists(ip)) return ip_reputation_cache.getEntry(ip); + + dbgTrace(D_L7_ACCESS_CONTROL) << "Not found in cache - about to query intelligence"; + + QueryRequest request = QueryRequest( + Condition::EQUALS, + "ipv4Addresses", + ip, + true, + AttributeKeyType::REGULAR + ); + + auto response = i_intelligence->queryIntelligence(request); + + if (!response.ok()) { + dbgWarning(D_L7_ACCESS_CONTROL) << "Failed to query intelligence about reputation of IP: " << ip; + return genError("Failed to query intelligence"); + } + + auto &unpacked_response = response.unpack(); + if (unpacked_response.empty()) { + dbgTrace(D_L7_ACCESS_CONTROL) << "Intelligence reputation response collection is empty. IP is clean."; + return IntelligenceIpReputation(); + } + + for (const auto &intelligence_reply : unpacked_response) { + if (intelligence_reply.getAssetType() == crowdsec_asset_type && !intelligence_reply.getData().empty()){ + dbgTrace(D_L7_ACCESS_CONTROL) << intelligence_reply.getData().front(); + return intelligence_reply.getData().front(); + } + } + + + return IntelligenceIpReputation(); +} + +ngx_http_cp_verdict_e +Layer7AccessControl::Impl::checkReputation(const string &source_ip) +{ + auto ip_reputation = getIpReputation(source_ip); + if (!ip_reputation.ok()) { + dbgWarning(D_L7_ACCESS_CONTROL) << "Could not query intelligence. Retruning default verdict"; + bool is_drop_by_default = getProfileAgentSettingWithDefault(false, "layer7AccessControl.dropByDefault"); + if (!(is_drop_by_default && isPrevent())) return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + generateLog(source_ip, IntelligenceIpReputation()); + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + } + + if (!ip_reputation.unpack().isMalicious()) { + dbgTrace(D_L7_ACCESS_CONTROL) << "Accepting IP: " << source_ip; + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + } + + ip_reputation_cache.emplaceEntry(source_ip, ip_reputation.unpack()); + + if (isPrevent()) { + dbgTrace(D_L7_ACCESS_CONTROL) << "Dropping IP: " << source_ip; + generateLog(source_ip, ip_reputation.unpack()); + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + } + + dbgTrace(D_L7_ACCESS_CONTROL) << "Detecting IP: " << source_ip; + generateLog(source_ip, ip_reputation.unpack()); + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; +} + + +void +Layer7AccessControl::Impl::generateLog(const string &source_ip, const IntelligenceIpReputation &ip_reputation) const +{ + dbgFlow(D_L7_ACCESS_CONTROL) << "About to generate Layer-7 Access Control log"; + + string security_action = isPrevent() ? "Prevent" : "Detect"; + + LogGen log( + "Access Control External Vendor Reputation", + ReportIS::Audience::SECURITY, + ReportIS::Severity::CRITICAL, + ReportIS::Priority::HIGH, + ReportIS::Tags::LAYER_7_ACCESS_CONTROL + ); + log + << genLogField("sourcePort", HttpTransactionData::client_port_ctx) + << genLogField("httpHostName", HttpTransactionData::host_name_ctx) + << genLogField("httpUriPath", HttpTransactionData::uri_ctx) + << genLogField("httpMethod", HttpTransactionData::method_ctx) + << genLogField("ipProtocol", HttpTransactionData::http_proto_ctx) + << genLogField("destinationPort", HttpTransactionData::listening_port_ctx) + << genLogField("proxyIP", HttpTransactionData::proxy_ip_ctx) + << genLogField("httpSourceId", HttpTransactionData::source_identifier) + << genLogField("httpUriPath", HttpTransactionData::uri_path_decoded) + << genLogField("httpUriQuery", HttpTransactionData::uri_query_decoded) + << genLogField("httpRequestHeaders", HttpTransactionData::req_headers) + << genLogIPField("destinationIP", HttpTransactionData::listening_ip_ctx) + << LogField("securityAction", security_action) + << LogField("sourceIP", source_ip) + << LogField("externalVendorName", "crowdsec") + << ip_reputation.getCrowdsecEventId() + << ip_reputation.getType() + << ip_reputation.getOrigin() + << ip_reputation.getIpv4Address() + << ip_reputation.getScenario(); +} + +Maybe +Layer7AccessControl::Impl::genLogField(const string &log_key, const string &env_key) const +{ + auto value = i_env->get(env_key); + if (value.ok()) return LogField(log_key, *value); + return value.passErr(); +} + +Maybe +Layer7AccessControl::Impl::genLogIPField(const string &log_key, const string &env_key) const +{ + auto value = i_env->get(env_key); + if (value.ok()) { + stringstream value_str; + value_str << value.unpack(); + return LogField(log_key, value_str.str()); + } + return value.passErr(); +} + +void +Layer7AccessControl::Impl::init() +{ + registerListener(); + i_env = Singleton::Consume::by(); + i_intelligence = Singleton::Consume::by(); + + chrono::minutes expiration( + getProfileAgentSettingWithDefault(60u, "layer7AccessControl.crowdsec.cacheExpiration") + ); + + ip_reputation_cache.startExpiration( + expiration, + Singleton::Consume::by(), + Singleton::Consume::by() + ); +} + +void +Layer7AccessControl::Impl::fini() +{ + unregisterListener(); + ip_reputation_cache.endExpiration(); +} + +Layer7AccessControl::Layer7AccessControl() : Component("Layer-7 Access Control"), pimpl(make_unique()) {} + +Layer7AccessControl::~Layer7AccessControl() {} + +void +Layer7AccessControl::init() +{ + pimpl->init(); +} + +void +Layer7AccessControl::fini() +{ + pimpl->fini(); +} diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/CMakeLists.txt b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/CMakeLists.txt new file mode 100644 index 0000000..7c2cdc9 --- /dev/null +++ b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/CMakeLists.txt @@ -0,0 +1,7 @@ +file(COPY data DESTINATION .) + +add_unit_test( + layer_7_access_control_ut + "layer_7_access_control_ut.cc" + "l7_access_control;logging;agent_details;table;singleton;time_proxy;metric;event_is;connkey;http_transaction_data;generic_rulebase;generic_rulebase_evaluators;ip_utilities;intelligence_is_v2" +) diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/data/malicious_intelligence_response.json b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/data/malicious_intelligence_response.json new file mode 100644 index 0000000..a36caca --- /dev/null +++ b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/data/malicious_intelligence_response.json @@ -0,0 +1,87 @@ +{ + "assetCollections": [ + { + "schemaVersion": 1, + "assetType": "not-crowdsec", + "assetTypeSchemaVersion": 1, + "permissionType": "allTenants", + "name": "050 Plus", + "objectType": "asset", + "class": "appiApplication", + "category": "cloud", + "family": "applicationsAndCategories", + "group": "appiObjects", + "order": "application", + "mainAttributes": { + "appiObjUuid": "00FA9E4440350F65E05308241DC22DA2" + }, + "sources": [ + { + "tenantId": "27278218-0e7f-4cd8-bdfe-2a5897d68fd0", + "sourceId": "434eabf4-651f-4a45-a9f7-159dc8183b78", + "assetId": "2222222222222222222222222222222", + "ttl": 86400, + "expirationTime": "2023-03-29T15:16:11.367873254Z", + "confidence": 900, + "attributes": { + "appType": "core", + "blockOnAny": false, + "categoryId": 40000060, + "categoryName": "VoIP", + "categoryUuid": "00FA9E44404E0F65E05308241DC22DA2", + "cpId": 60517839, + "dataType": false, + "type": "appfw_application" + } + } + ] + }, + { + "schemaVersion": 1, + "assetType": "data-cloud-ip-crowdSec", + "assetTypeSchemaVersion": 1, + "permissionType": "tenant", + "name": "1.2.3.4", + "objectType": "asset", + "class": "data", + "category": "cloud", + "family": "ip", + "group": "crowdSec", + "mainAttributes": { + "ipv4Addresses": [ + "1.2.3.4" + ] + }, + "sources": [ + { + "tenantId": "c6f606b1-e59b-4f94-b829-ce597bd03067", + "sourceId": "529185bd-9c91-4853-9c26-3140950ecad7", + "assetId": "YXNzZXQ7OztkYXRhOzs7Y2xvdWQ7OztpcDs7O2Nyb3dkU2VjOzs7Ozs7Ozs7eyJpcHY0QWRkcmVzc2VzIjpbIjEuMi4zLjQiXX0=", + "ttl": 86400, + "expirationTime": "2023-03-27T12:08:00.279Z", + "confidence": 500, + "attributes": { + "crowdsecId": 2253734, + "duration": "29m33.009472691s", + "ipv4Addresses": [ + "1.2.3.4" + ], + "ipv4AddressesRange": [ + { + "max": "1.2.3.4", + "min": "1.2.3.4" + } + ], + "origin": "cscli", + "scenario": "manual 'ban' from 'localhost'", + "scope": "Ip", + "type": "ban" + } + } + ] + } + ], + "status": "done", + "totalNumAssets": 1, + "cursor": "" +} diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/data/ok_intelligence_response.json b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/data/ok_intelligence_response.json new file mode 100644 index 0000000..f90f9ea --- /dev/null +++ b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/data/ok_intelligence_response.json @@ -0,0 +1,6 @@ +{ + "assetCollections": [], + "status": "done", + "totalNumAssets": 0, + "cursor": "" +} diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc new file mode 100644 index 0000000..13de087 --- /dev/null +++ b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc @@ -0,0 +1,413 @@ +#include "layer_7_access_control.h" + +#include "cptest.h" +#include "config_component.h" +#include "mock/mock_mainloop.h" +#include "mock/mock_time_get.h" +#include "mock/mock_http_manager.h" +#include "mock/mock_logging.h" +#include "mock/mock_messaging.h" +#include "intelligence_comp_v2.h" +#include "agent_details.h" + +using namespace std; +using namespace testing; + +USE_DEBUG_FLAG(D_L7_ACCESS_CONTROL); + +class Layer7AccessControlTest : public Test +{ +public: + Layer7AccessControlTest() + { + Debug::setUnitTestFlag(D_L7_ACCESS_CONTROL, Debug::DebugLevel::TRACE); + EXPECT_CALL(mock_logging, getCurrentLogId()).Times(AnyNumber()); + EXPECT_CALL(mock_time, getWalltimeStr(_)).WillRepeatedly(Return(string("2016-11-13T17:31:24.087"))); + EXPECT_CALL(mock_time, getWalltime()).WillRepeatedly(Return(chrono::seconds(0))); + EXPECT_CALL(mock_time, getMonotonicTime()).WillRepeatedly(Return(chrono::seconds(60))); + EXPECT_CALL(mock_ml, doesRoutineExist(_)).WillRepeatedly(Return(true)); + EXPECT_CALL(mock_ml, stop(_)).WillRepeatedly(Return()); + env.preload(); + env.init(); + config.preload(); + intelligence_comp.preload(); + intelligence_comp.init(); + l7_access_control.preload(); + l7_access_control.init(); + ctx.activate(); + } + + ~Layer7AccessControlTest() + { + ctx.deactivate(); + l7_access_control.fini(); + } + + string loadIntelligenceResponse(const string &file_path); + void registerTransactionData(); + void verifyReport(const Report &report, const string &source_identifier, const string &security_action); + + const EventVerdict drop_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + const EventVerdict accept_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + const EventVerdict inspect_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + Layer7AccessControl l7_access_control; + ::Environment env; + ConfigComponent config; + StrictMock mock_logging; + StrictMock mock_time; + StrictMock mock_ml; + StrictMock messaging_mock; + AgentDetails agent_details; + IntelligenceComponentV2 intelligence_comp; + Context ctx; +}; + +string prevent_settings = +"{\n" + "\"agentSettings\": [\n" + "{\n" + "\"id\": \"aac36348-5826-17d4-de11-195dd4dfca4a\"," + "\"key\": \"agent.config.useLocalIntelligence\"," + "\"value\": \"true\"" + "}," + "{" + "\"id\": \"f6c386fb-e221-59af-dbf5-b9bed680ec6b\"," + "\"key\": \"layer7AccessControl.logOnDrop\"," + "\"value\": \"true\"" + "}," + "{" + "\"id\": \"5ac38ee8-8b3c-481b-b382-f1f0735c0468\"," + "\"key\": \"layer7AccessControl.securityMode\"," + "\"value\": \"prevent\"" + "}," + "{" + "\"id\": \"54c38f89-8fe2-871e-b29a-31e088f1b1d3\"," + "\"key\": \"layer7AccessControl.crowdsec.enabled\"," + "\"value\": \"true\"" + "}" + "],\n"; + +string detect_settings = +"{\n" + "\"agentSettings\": [\n" + "{\n" + "\"id\": \"aac36348-5826-17d4-de11-195dd4dfca4a\"," + "\"key\": \"agent.config.useLocalIntelligence\"," + "\"value\": \"true\"" + "}," + "{" + "\"id\": \"f6c386fb-e221-59af-dbf5-b9bed680ec6b\"," + "\"key\": \"layer7AccessControl.logOnDrop\"," + "\"value\": \"true\"" + "}," + "{" + "\"id\": \"5ac38ee8-8b3c-481b-b382-f1f0735c0468\"," + "\"key\": \"layer7AccessControl.securityMode\"," + "\"value\": \"detect\"" + "}," + "{" + "\"id\": \"54c38f89-8fe2-871e-b29a-31e088f1b1d3\"," + "\"key\": \"layer7AccessControl.crowdsec.enabled\"," + "\"value\": \"true\"" + "}" + "],\n"; + +string disabled_settings = +"{" + "\"agentSettings\": [\n" + "{\n" + "\"id\": \"aac36348-5826-17d4-de11-195dd4dfca4a\"," + "\"key\": \"agent.config.useLocalIntelligence\"," + "\"value\": \"true\"" + "}," + "{" + "\"id\": \"f6c386fb-e221-59af-dbf5-b9bed680ec6b\"," + "\"key\": \"layer7AccessControl.logOnDrop\"," + "\"value\": \"true\"" + "}," + "{" + "\"id\": \"5ac38ee8-8b3c-481b-b382-f1f0735c0468\"," + "\"key\": \"layer7AccessControl.securityMode\"," + "\"value\": \"detect\"" + "}," + "{" + "\"id\": \"54c38f89-8fe2-871e-b29a-31e088f1b1d3\"," + "\"key\": \"layer7AccessControl.crowdsec.enabled\"," + "\"value\": \"false\"" + "}" + "],\n"; + +string policy = + "\"rulebase\": {" + "\"usersIdentifiers\": [" + "{" + "\"context\": \"Any(All(Any(EqualHost(juice-shop.checkpoint.com)),EqualListeningPort(80)))\"," + "\"identifierValues\": []," + "\"sourceIdentifier\": \"\"," + "\"sourceIdentifiers\": [" + "{" + "\"identifierValues\": []," + "\"sourceIdentifier\": \"x-forwarded-for\"" + "}" + "]" + "}" + "],\n" + "\"rulesConfig\": [" + "{" + "\"assetId\": \"00c37544-047b-91d4-e5e5-31d90070bcfd\"," + "\"assetName\": \"juice\"," + "\"context\": \"Any(All(Any(EqualHost(juice-shop.checkpoint.com)),EqualListeningPort(80)))\"," + "\"isCleanup\": false," + "\"parameters\": []," + "\"practices\": [" + "{" + "\"practiceId\": \"36be58f5-2c99-1f16-f816-bf25118d9bc1\"," + "\"practiceName\": \"WEB APPLICATION BEST PRACTICE\"," + "\"practiceType\": \"WebApplication\"" + "}" + "]," + "\"priority\": 1," + "\"ruleId\": \"00c37544-047b-91d4-e5e5-31d90070bcfd\"," + "\"ruleName\": \"juice\"," + "\"triggers\": [" + "{" + "\"triggerId\": \"86be58f5-2b65-18ee-2bd7-b4429dab245d\"," + "\"triggerName\": \"Log Trigger\"," + "\"triggerType\": \"log\"" + "}" + "]," + "\"zoneId\": \"\"," + "\"zoneName\": \"\"" + "}" + "]" + "}\n" +"}\n"; + +void +Layer7AccessControlTest::registerTransactionData() +{ + ctx.registerValue(HttpTransactionData::client_ip_ctx, IPAddr::createIPAddr("4.4.4.4").unpack()); + ctx.registerValue(HttpTransactionData::listening_ip_ctx, IPAddr::createIPAddr("5.6.7.8").unpack()); + ctx.registerValue(HttpTransactionData::http_proto_ctx, "http"); + ctx.registerValue(HttpTransactionData::method_ctx, "POST"); + ctx.registerValue(HttpTransactionData::host_name_ctx, "juice-shop.checkpoint.com"); + ctx.registerValue(HttpTransactionData::listening_port_ctx, 80); + ctx.registerValue(HttpTransactionData::client_port_ctx, 12345); + ctx.registerValue(HttpTransactionData::uri_ctx, "/"); +} + +static bool +operator==(const EventVerdict &first, const EventVerdict &second) +{ + return first.getVerdict() == second.getVerdict(); +} + +string +Layer7AccessControlTest::loadIntelligenceResponse(const string &file_path) +{ + stringstream ss; + ifstream f(cptestFnameInExeDir(file_path), ios::in); + dbgTrace(D_L7_ACCESS_CONTROL) << "Loading intelligence response from: " << file_path; + ss << f.rdbuf(); + f.close(); + return ss.str(); +} + +template +string +reportToStr(const T &obj) +{ + stringstream ss; + { + cereal::JSONOutputArchive ar(ss); + obj.serialize(ar); + } + return ss.str(); +} + +void +Layer7AccessControlTest::verifyReport( + const Report &report, + const string &source_identifier, + const string &security_action +) +{ + string log = reportToStr(report); + dbgTrace(D_L7_ACCESS_CONTROL) << "Report: " << log; + + if (!source_identifier.empty()) EXPECT_THAT(log, HasSubstr("\"httpSourceId\": \"" + source_identifier + "\"")); + EXPECT_THAT(log, HasSubstr("\"securityAction\": \"" + security_action + "\"")); + EXPECT_THAT(log, HasSubstr("\"eventName\": \"Access Control External Vendor Reputation\"")); + EXPECT_THAT(log, HasSubstr("\"httpHostName\": \"juice-shop.checkpoint.com\"")); + EXPECT_THAT(log, HasSubstr("\"httpUriPath\": \"/\"")); + EXPECT_THAT(log, HasSubstr("\"httpMethod\": \"POST\"")); + EXPECT_THAT(log, HasSubstr("\"ipProtocol\": \"http\"")); + EXPECT_THAT(log, HasSubstr("\"destinationIP\": \"5.6.7.8\"")); + EXPECT_THAT(log, HasSubstr("\"externalVendorName\": \"crowdsec\"")); + EXPECT_THAT(log, HasSubstr("\"externalVendorRecommendationId\": 2253734")); + EXPECT_THAT(log, HasSubstr("\"externalVendorRecommendedAction\": \"ban\"")); + EXPECT_THAT(log, HasSubstr("\"externalVendorRecommendationOrigin\": \"cscli\"")); + EXPECT_THAT(log, HasSubstr("\"externalVendorRecommendedAffectedScope\": \"1.2.3.4\"")); + EXPECT_THAT(log, HasSubstr("\"externalVendorRecommendationOriginDetails\": \"manual 'ban' from 'localhost'\"")); +} + +TEST_F(Layer7AccessControlTest, ReturnAcceptVerdict) +{ + stringstream ss_conf(prevent_settings + policy); + Singleton::Consume::from(config)->loadConfiguration(ss_conf); + + string intelligence_response_ok = loadIntelligenceResponse("data/ok_intelligence_response.json"); + + EXPECT_CALL( + messaging_mock, + sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) + ).WillOnce(Return(intelligence_response_ok)); + + registerTransactionData(); + ctx.registerValue(HttpTransactionData::source_identifier, "1.2.3.4"); + const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 }; + const HttpHeader header2{ Buffer("date"), Buffer("Sun, 26 Mar 2023 18:45:22 GMT"), 1 }; + const HttpHeader header3{ Buffer("x-forwarded-for"), Buffer("1.2.3.4"), 2, true}; + + EXPECT_THAT( + HttpRequestHeaderEvent(header1).performNamedQuery(), + ElementsAre(Pair("Layer-7 Access Control app", inspect_verdict)) + ); + EXPECT_THAT( + HttpRequestHeaderEvent(header2).performNamedQuery(), + ElementsAre(Pair("Layer-7 Access Control app", inspect_verdict)) + ); + EXPECT_THAT( + HttpRequestHeaderEvent(header3).performNamedQuery(), + ElementsAre(Pair("Layer-7 Access Control app", accept_verdict)) + ); +} + +TEST_F(Layer7AccessControlTest, ReturnDropVerdictOnMaliciousReputation) +{ + stringstream ss_conf(prevent_settings + policy); + Singleton::Consume::from(config)->loadConfiguration(ss_conf); + + string malicious_intelligence_response = loadIntelligenceResponse("data/malicious_intelligence_response.json"); + + EXPECT_CALL( + messaging_mock, + sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) + ).WillOnce(Return(malicious_intelligence_response)); + + registerTransactionData(); + ctx.registerValue(HttpTransactionData::source_identifier, "1.2.3.4"); + const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 }; + const HttpHeader header2{ Buffer("date"), Buffer("Sun, 26 Mar 2023 18:45:22 GMT"), 1 }; + const HttpHeader header3{ Buffer("x-forwarded-for"), Buffer("1.2.3.4"), 2, true}; + + Report report; + EXPECT_CALL(mock_logging, sendLog(_)).WillOnce(SaveArg<0>(&report)); + + EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict)); + EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(inspect_verdict)); + EXPECT_THAT(HttpRequestHeaderEvent(header3).query(), ElementsAre(drop_verdict)); + + verifyReport(report, "1.2.3.4", "Prevent"); +} + +TEST_F(Layer7AccessControlTest, ReturnDropVerdictCacheBased) +{ + stringstream ss_conf(prevent_settings + policy); + Singleton::Consume::from(config)->loadConfiguration(ss_conf); + + string malicious_intelligence_response = loadIntelligenceResponse("data/malicious_intelligence_response.json"); + + EXPECT_CALL( + messaging_mock, + sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) + ).WillOnce(Return(malicious_intelligence_response)); + + registerTransactionData(); + ctx.registerValue(HttpTransactionData::source_identifier, "1.2.3.4"); + const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 }; + const HttpHeader header2{ Buffer("date"), Buffer("Sun, 26 Mar 2023 18:45:22 GMT"), 1 }; + const HttpHeader header3{ Buffer("x-forwarded-for"), Buffer("1.2.3.4"), 2, true}; + + Report report; + EXPECT_CALL(mock_logging, sendLog(_)).Times(2).WillRepeatedly(SaveArg<0>(&report)); + + EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict)); + EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(inspect_verdict)); + EXPECT_THAT(HttpRequestHeaderEvent(header3).query(), ElementsAre(drop_verdict)); + + verifyReport(report, "1.2.3.4", "Prevent"); + + EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict)); + EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(inspect_verdict)); + EXPECT_THAT(HttpRequestHeaderEvent(header3).query(), ElementsAre(drop_verdict)); + + verifyReport(report, "1.2.3.4", "Prevent"); +} + +TEST_F(Layer7AccessControlTest, AcceptOnDetect) +{ + stringstream ss_conf(detect_settings + policy); + Singleton::Consume::from(config)->loadConfiguration(ss_conf); + + string malicious_intelligence_response = loadIntelligenceResponse("data/malicious_intelligence_response.json"); + + EXPECT_CALL( + messaging_mock, + sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) + ).WillOnce(Return(malicious_intelligence_response)); + + registerTransactionData(); + ctx.registerValue(HttpTransactionData::source_identifier, "1.2.3.4"); + const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 }; + const HttpHeader header2{ Buffer("date"), Buffer("Sun, 26 Mar 2023 18:45:22 GMT"), 1 }; + const HttpHeader header3{ Buffer("x-forwarded-for"), Buffer("1.2.3.4"), 2, true}; + + Report report; + EXPECT_CALL(mock_logging, sendLog(_)).WillOnce(SaveArg<0>(&report)); + + EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict)); + EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(inspect_verdict)); + EXPECT_THAT(HttpRequestHeaderEvent(header3).query(), ElementsAre(accept_verdict)); + + verifyReport(report, "1.2.3.4", "Detect"); +} + +TEST_F(Layer7AccessControlTest, FallbackToSourceIPAndDrop) +{ + stringstream ss_conf(prevent_settings + policy); + Singleton::Consume::from(config)->loadConfiguration(ss_conf); + + string malicious_intelligence_response = loadIntelligenceResponse("data/malicious_intelligence_response.json"); + + EXPECT_CALL( + messaging_mock, + sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) + ).WillOnce(Return(malicious_intelligence_response)); + + registerTransactionData(); + const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 }; + const HttpHeader header2{ Buffer("date"), Buffer("Sun, 26 Mar 2023 18:45:22 GMT"), 1, true }; + + Report report; + EXPECT_CALL(mock_logging, sendLog(_)).WillOnce(SaveArg<0>(&report)); + + EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict)); + EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(drop_verdict)); + + verifyReport(report, "", "Prevent"); +} + +TEST_F(Layer7AccessControlTest, AcceptOnDisabled) +{ + stringstream ss_conf(disabled_settings + policy); + Singleton::Consume::from(config)->loadConfiguration(ss_conf); + + registerTransactionData(); + ctx.registerValue(HttpTransactionData::source_identifier, "1.2.3.4"); + const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 }; + const HttpHeader header2{ Buffer("date"), Buffer("Sun, 26 Mar 2023 18:45:22 GMT"), 1 }; + const HttpHeader header3{ Buffer("x-forwarded-for"), Buffer("1.2.3.4"), 2, true}; + + EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(accept_verdict)); +} diff --git a/components/security_apps/orchestration/details_resolver/details_resolver.cc b/components/security_apps/orchestration/details_resolver/details_resolver.cc index 5b39e7b..82c12d1 100644 --- a/components/security_apps/orchestration/details_resolver/details_resolver.cc +++ b/components/security_apps/orchestration/details_resolver/details_resolver.cc @@ -201,6 +201,13 @@ DetailsResolver::Impl::isVersionEqualOrAboveR8110() return false; } +static bool +isNoResponse(const string &cmd) +{ + auto res = DetailsResolvingHanlder::getCommandOutput(cmd); + return !res.ok() || res.unpack().empty(); +} + Maybe> DetailsResolver::Impl::parseNginxMetadata() { @@ -215,9 +222,8 @@ DetailsResolver::Impl::parseNginxMetadata() output_path; dbgTrace(D_ORCHESTRATOR) << "Details resolver, srcipt exe cmd: " << srcipt_exe_cmd; - auto is_nginx_exist = DetailsResolvingHanlder::getCommandOutput("which nginx"); - if (!is_nginx_exist.ok() || is_nginx_exist.unpack().size() == 0) { - return genError("Nginx isn't installed"); + if (isNoResponse("which nginx") && isNoResponse("which kong")) { + return genError("Nginx or Kong isn't installed"); } auto script_output = DetailsResolvingHanlder::getCommandOutput(srcipt_exe_cmd); @@ -259,6 +265,7 @@ DetailsResolver::Impl::parseNginxMetadata() for(string &line : lines) { if (line.size() == 0) continue; if (line.find("RELEASE_VERSION") != string::npos) continue; + if (line.find("KONG_VERSION") != string::npos) continue; if (line.find("--with-cc=") != string::npos) continue; if (line.find("NGINX_VERSION") != string::npos) { auto eq_index = line.find("="); diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index 940b15a..50f1f69 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -15,6 +15,15 @@ #error details_resolver_handlers/details_resolver_impl.h should not be included directly. #endif // __DETAILS_RESOLVER_HANDLER_CC__ +// Retrieve artifacts by incorporating nano service names into additional metadata: +// To include a required nano service in the additional metadata sent to the manifest generator, +// add a handler in this file. The key to use is 'requiredNanoServices', and its value should be +// a string representing an array of nano service prefix names, separated by semicolons. +// For example: "httpTransactionHandler_linux;iotSnmp_gaia;" +// +// Handler example for reading the content of a configuration file: +// FILE_CONTENT_HANDLER("requiredNanoServices", "/tmp/nano_services_list", getRequiredNanoServices) + // use SHELL_CMD_HANDLER(key as string, shell command as string, ptr to Maybe handler(const string&)) // to return a string value for an attribute key based on a logic executed in a handler that receives // shell command execution output as its input diff --git a/components/security_apps/orchestration/include/declarative_policy_utils.h b/components/security_apps/orchestration/include/declarative_policy_utils.h index 1afe87a..c742124 100644 --- a/components/security_apps/orchestration/include/declarative_policy_utils.h +++ b/components/security_apps/orchestration/include/declarative_policy_utils.h @@ -25,6 +25,7 @@ #include "i_env_details.h" #include "maybe_res.h" #include "event.h" +#include "rest.h" class ApplyPolicyEvent : public Event { @@ -46,13 +47,15 @@ public: class ApplyPolicyRest : public ServerRest { public: - // LCOV_EXCL_START Reason: no test exist void doCall() override { + Singleton::Consume::by()->setPolicyPath(policy_path.get()); ApplyPolicyEvent().notify(); } - // LCOV_EXCL_STOP + + private: + C2S_PARAM(std::string, policy_path); }; void init(); diff --git a/components/security_apps/orchestration/include/mock/mock_orchestration_tools.h b/components/security_apps/orchestration/include/mock/mock_orchestration_tools.h index be317f7..20b6cfb 100755 --- a/components/security_apps/orchestration/include/mock/mock_orchestration_tools.h +++ b/components/security_apps/orchestration/include/mock/mock_orchestration_tools.h @@ -31,6 +31,13 @@ operator<<(std::ostream &os, const std::map &) return os; } +template +std::ostream & +operator<<(std::ostream &os, const Maybe> &) +{ + return os; +} + class MockOrchestrationTools : public Singleton::Provide::From> diff --git a/components/security_apps/orchestration/include/mock/mock_service_controller.h b/components/security_apps/orchestration/include/mock/mock_service_controller.h index 7a3e8d5..8d8e104 100755 --- a/components/security_apps/orchestration/include/mock/mock_service_controller.h +++ b/components/security_apps/orchestration/include/mock/mock_service_controller.h @@ -26,6 +26,10 @@ class MockServiceController : public: MOCK_METHOD0(refreshPendingServices, void()); + MOCK_METHOD0(doesFailedServicesExist, bool()); + + MOCK_METHOD0(clearFailedServices, void()); + MOCK_CONST_METHOD0(getPolicyVersion, const std::string &()); MOCK_CONST_METHOD0(getUpdatePolicyVersion, const std::string &()); diff --git a/components/security_apps/orchestration/local_policy_mgmt_gen/include/policy_maker_utils.h b/components/security_apps/orchestration/local_policy_mgmt_gen/include/policy_maker_utils.h index 9dbc856..4cae78e 100644 --- a/components/security_apps/orchestration/local_policy_mgmt_gen/include/policy_maker_utils.h +++ b/components/security_apps/orchestration/local_policy_mgmt_gen/include/policy_maker_utils.h @@ -91,6 +91,7 @@ private: SettingsWrapper settings; SecurityAppsWrapper security_apps; }; + class PolicyMakerUtils : Singleton::Consume, @@ -99,6 +100,19 @@ class PolicyMakerUtils Singleton::Consume { public: + std::string proccesSingleAppsecPolicy( + const std::string &policy_path, + const std::string &policy_version, + const std::string &local_appsec_policy_path + ); + + std::string proccesMultipleAppsecPolicies( + const std::map &appsec_policies, + const std::string &policy_version, + const std::string &local_appsec_policy_path + ); + +private: std::string getPolicyName(const std::string &policy_path); Maybe openPolicyAsJson(const std::string &policy_path); @@ -129,7 +143,8 @@ public: const std::string &policy_name ); -private: + void createAgentPolicyFromAppsecPolicy(const std::string &policy_name, const AppsecLinuxPolicy &appsec_policy); + std::map log_triggers; std::map web_user_res_triggers; std::map inner_exceptions; diff --git a/components/security_apps/orchestration/local_policy_mgmt_gen/include/triggers_section.h b/components/security_apps/orchestration/local_policy_mgmt_gen/include/triggers_section.h index e7ccea9..5a76463 100644 --- a/components/security_apps/orchestration/local_policy_mgmt_gen/include/triggers_section.h +++ b/components/security_apps/orchestration/local_policy_mgmt_gen/include/triggers_section.h @@ -22,6 +22,8 @@ #include "config.h" #include "debug.h" #include "local_policy_common.h" +#include "i_agent_details.h" +#include "i_env_details.h" class LogTriggerSection { @@ -37,6 +39,7 @@ public: bool _logToAgent, bool _logToCef, bool _logToCloud, + bool _logToK8sService, bool _logToSyslog, bool _responseBody, bool _tpDetect, @@ -68,6 +71,7 @@ private: bool logToAgent; bool logToCef; bool logToCloud; + bool logToK8sService; bool logToSyslog; bool responseBody; bool tpDetect; @@ -233,7 +237,11 @@ private: std::string format; }; -class AppsecTriggerLogDestination : public ClientRest +class AppsecTriggerLogDestination + : + public ClientRest, + Singleton::Consume, + Singleton::Consume { public: void load(cereal::JSONInputArchive &archive_in); @@ -244,6 +252,7 @@ public: bool shouldBeautifyLogs() const; bool getCloud() const; + bool isK8SNeeded() const; bool isCefNeeded() const; bool isSyslogNeeded() const; const std::string & getSyslogServerIpv4Address() const; @@ -254,6 +263,7 @@ private: const LoggingService & getCefServiceData() const; bool cloud = false; + bool k8s_service = false; bool agent_local = true; bool beautify_logs = true; LoggingService syslog_service; diff --git a/components/security_apps/orchestration/local_policy_mgmt_gen/local_policy_mgmt_gen.cc b/components/security_apps/orchestration/local_policy_mgmt_gen/local_policy_mgmt_gen.cc index aec2403..d6f36fa 100644 --- a/components/security_apps/orchestration/local_policy_mgmt_gen/local_policy_mgmt_gen.cc +++ b/components/security_apps/orchestration/local_policy_mgmt_gen/local_policy_mgmt_gen.cc @@ -50,14 +50,9 @@ using namespace std; USE_DEBUG_FLAG(D_LOCAL_POLICY); -const static string local_appsec_policy_path = "/tmp/local_appsec.policy"; -const static string open_appsec_io = "openappsec.io/"; -const static string policy_key = "policy"; -const static string syslog_key = "syslog"; -const static string mode_key = "mode"; -const static string local_mgmt_policy_path = "/conf/local_policy.yaml"; +const static string default_local_appsec_policy_path = "/tmp/local_appsec.policy"; +const static string default_local_mgmt_policy_path = "/conf/local_policy.yaml"; -// LCOV_EXCL_STOP class LocalPolicyMgmtGenerator::Impl : public Singleton::Provide::From, @@ -66,7 +61,6 @@ class LocalPolicyMgmtGenerator::Impl { public: -// LCOV_EXCL_START Reason: no test exist void init() { @@ -74,6 +68,7 @@ public: env_type = env_details->getEnvType(); if (env_type == EnvType::LINUX) { dbgInfo(D_LOCAL_POLICY) << "Initializing Linux policy generator"; + local_policy_path = getFilesystemPathConfig() + default_local_mgmt_policy_path; return; } dbgInfo(D_LOCAL_POLICY) << "Initializing K8S policy generator"; @@ -97,36 +92,10 @@ public: { dbgFlow(D_LOCAL_POLICY) << "Starting to parse policy - embedded environment"; - string policy_path = getConfigurationFlagWithDefault( - getFilesystemPathConfig() + local_mgmt_policy_path, - "local_mgmt_policy" - ); - - Maybe maybe_policy = policy_maker_utils.openPolicyAsJson(policy_path); - if (!maybe_policy.ok()){ - dbgWarning(D_LOCAL_POLICY) << maybe_policy.getErr(); - return ""; - } - AppsecLinuxPolicy policy = maybe_policy.unpack(); - string policy_name = policy_maker_utils.getPolicyName(policy_path); - dbgTrace(D_LOCAL_POLICY) << "Proccesing policy, name: " << policy_name; - - ParsedRule default_rule = policy.getAppsecPolicySpec().getDefaultRule(); - - // add default rule to policy - policy_maker_utils.createPolicyElementsByRule(default_rule, default_rule, policy, policy_name); - - vector specific_rules = policy.getAppsecPolicySpec().getSpecificRules(); - policy_maker_utils.createPolicyElements( - specific_rules, - default_rule, - policy, - policy_name - ); - PolicyWrapper policy_wrapper = policy_maker_utils.combineElementsToPolicy(policy_version); - return policy_maker_utils.dumpPolicyToFile( - policy_wrapper, - local_appsec_policy_path + return policy_maker_utils.proccesSingleAppsecPolicy( + local_policy_path, + policy_version, + default_local_appsec_policy_path ); } @@ -136,30 +105,10 @@ public: dbgFlow(D_LOCAL_POLICY) << "Starting to parse policy - K8S environment"; map appsec_policies = k8s_policy_utils.createAppsecPoliciesFromIngresses(); - - for (const auto &appsec_policy : appsec_policies) { - string policy_name = appsec_policy.first; - dbgTrace(D_LOCAL_POLICY) << "Proccesing policy, name: " << policy_name; - AppsecLinuxPolicy policy = appsec_policy.second; - - ParsedRule default_rule = policy.getAppsecPolicySpec().getDefaultRule(); - - // add default rule to policy - policy_maker_utils.createPolicyElementsByRule(default_rule, default_rule, policy, policy_name); - - vector specific_rules = policy.getAppsecPolicySpec().getSpecificRules(); - policy_maker_utils.createPolicyElements( - specific_rules, - default_rule, - policy, - policy_name - ); - } - - PolicyWrapper policy_wrapper = policy_maker_utils.combineElementsToPolicy(policy_version); - return policy_maker_utils.dumpPolicyToFile( - policy_wrapper, - local_appsec_policy_path + return policy_maker_utils.proccesMultipleAppsecPolicies( + appsec_policies, + policy_version, + default_local_appsec_policy_path ); } @@ -169,7 +118,9 @@ public: return isK8sEnv() ? parseK8sPolicy(policy_version) : parseLinuxPolicy(policy_version); } - const string & getPolicyPath(void) const override { return local_appsec_policy_path; } + const string & getAgentPolicyPath(void) const override { return default_local_appsec_policy_path; } + const string & getLocalPolicyPath(void) const override { return local_policy_path; } + void setPolicyPath(const string &new_local_policy_path) override { local_policy_path = new_local_policy_path; } private: bool @@ -182,6 +133,7 @@ private: EnvType env_type; PolicyMakerUtils policy_maker_utils; K8sPolicyUtils k8s_policy_utils; + string local_policy_path; }; diff --git a/components/security_apps/orchestration/local_policy_mgmt_gen/policy_maker_utils.cc b/components/security_apps/orchestration/local_policy_mgmt_gen/policy_maker_utils.cc index 17bbab3..0c1df95 100644 --- a/components/security_apps/orchestration/local_policy_mgmt_gen/policy_maker_utils.cc +++ b/components/security_apps/orchestration/local_policy_mgmt_gen/policy_maker_utils.cc @@ -144,7 +144,6 @@ PolicyMakerUtils::dumpPolicyToFile(const PolicyWrapper &policy, const string &po policy.save(ar); } string policy_str = ss.str(); - dbgTrace(D_NGINX_POLICY) << "policy: " << policy_str; try { ofstream policy_file(policy_path); policy_file << policy_str; @@ -365,6 +364,7 @@ createLogTriggerSection( bool webHeaders = trigger_spec.getAppsecTriggerExtendedLogging().isHttpHeaders(); bool webBody = trigger_spec.getAppsecTriggerExtendedLogging().isRequestBody(); bool logToCloud = trigger_spec.getAppsecTriggerLogDestination().getCloud(); + bool logToK8sService = trigger_spec.getAppsecTriggerLogDestination().isK8SNeeded(); bool logToAgent = trigger_spec.getAppsecTriggerLogDestination().isAgentLocal(); bool beautify_logs = trigger_spec.getAppsecTriggerLogDestination().shouldBeautifyLogs(); bool logToCef = trigger_spec.getAppsecTriggerLogDestination().isCefNeeded(); @@ -391,6 +391,7 @@ createLogTriggerSection( logToAgent, logToCef, logToCloud, + logToK8sService, logToSyslog, responseBody, tpDetect, @@ -758,3 +759,54 @@ PolicyMakerUtils::createPolicyElements( createPolicyElementsByRule(rule, default_rule, policy, policy_name); } } + +void +PolicyMakerUtils::createAgentPolicyFromAppsecPolicy(const string &policy_name, const AppsecLinuxPolicy &appsec_policy) +{ + dbgTrace(D_LOCAL_POLICY) << "Proccesing policy, name: " << policy_name; + + ParsedRule default_rule = appsec_policy.getAppsecPolicySpec().getDefaultRule(); + + // add default rule to policy + createPolicyElementsByRule(default_rule, default_rule, appsec_policy, policy_name); + + vector specific_rules = appsec_policy.getAppsecPolicySpec().getSpecificRules(); + createPolicyElements(specific_rules, default_rule, appsec_policy, policy_name); +} + +string +PolicyMakerUtils::proccesSingleAppsecPolicy( + const string &policy_path, + const string &policy_version, + const string &local_appsec_policy_path) +{ + Maybe maybe_policy = openPolicyAsJson(policy_path); + if (!maybe_policy.ok()){ + dbgWarning(D_LOCAL_POLICY) << maybe_policy.getErr(); + return ""; + } + createAgentPolicyFromAppsecPolicy(getPolicyName(policy_path), maybe_policy.unpack()); + + PolicyWrapper policy_wrapper = combineElementsToPolicy(policy_version); + return dumpPolicyToFile( + policy_wrapper, + local_appsec_policy_path + ); +} + +string +PolicyMakerUtils::proccesMultipleAppsecPolicies( + const map &appsec_policies, + const string &policy_version, + const string &local_appsec_policy_path) +{ + for (const auto &appsec_policy : appsec_policies) { + createAgentPolicyFromAppsecPolicy(appsec_policy.first, appsec_policy.second); + } + + PolicyWrapper policy_wrapper = combineElementsToPolicy(policy_version); + return dumpPolicyToFile( + policy_wrapper, + local_appsec_policy_path + ); +} diff --git a/components/security_apps/orchestration/local_policy_mgmt_gen/triggers_section.cc b/components/security_apps/orchestration/local_policy_mgmt_gen/triggers_section.cc index 8f63a2c..7c50436 100644 --- a/components/security_apps/orchestration/local_policy_mgmt_gen/triggers_section.cc +++ b/components/security_apps/orchestration/local_policy_mgmt_gen/triggers_section.cc @@ -30,6 +30,7 @@ LogTriggerSection::LogTriggerSection( bool _logToAgent, bool _logToCef, bool _logToCloud, + bool _logToK8sService, bool _logToSyslog, bool _responseBody, bool _tpDetect, @@ -52,6 +53,7 @@ LogTriggerSection::LogTriggerSection( logToAgent(_logToAgent), logToCef(_logToCef), logToCloud(_logToCloud), + logToK8sService(_logToK8sService), logToSyslog(_logToSyslog), responseBody(_responseBody), tpDetect(_tpDetect), @@ -95,6 +97,7 @@ LogTriggerSection::save(cereal::JSONOutputArchive &out_ar) const cereal::make_nvp("logToAgent", logToAgent), cereal::make_nvp("logToCef", logToCef), cereal::make_nvp("logToCloud", logToCloud), + cereal::make_nvp("logToK8sService", logToK8sService), cereal::make_nvp("logToSyslog", logToSyslog), cereal::make_nvp("responseBody", responseBody), cereal::make_nvp("responseCode", false), @@ -382,6 +385,10 @@ AppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in) dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger LogDestination"; // TBD: support "file" parseAppsecJSONKey("cloud", cloud, archive_in, false); + auto mode = Singleton::Consume::by()->getOrchestrationMode(); + auto env_type = Singleton::Consume::by()->getEnvType(); + bool k8s_service_default = (mode == OrchestrationMode::HYBRID && env_type == EnvType::K8S); + parseAppsecJSONKey("k8s-service", k8s_service, archive_in, k8s_service_default); StdoutLogging stdout_log; parseAppsecJSONKey("stdout", stdout_log, archive_in); @@ -421,6 +428,12 @@ AppsecTriggerLogDestination::getCloud() const return cloud; } +bool +AppsecTriggerLogDestination::isK8SNeeded() const +{ + return k8s_service; +} + bool AppsecTriggerLogDestination::isCefNeeded() const { diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 6adffe9..2b4cbe9 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -13,6 +13,7 @@ #include "orchestration_comp.h" +#include #include #include #include @@ -157,7 +158,7 @@ private: class OrchestrationComp::Impl { public: - explicit Impl() {} + explicit Impl() : curr_agent_data_report(false) {} void init() @@ -171,6 +172,7 @@ public: doEncrypt(); health_check_status_listener.registerListener(); + curr_agent_data_report.disableReportSending(); auto rest = Singleton::Consume::by(); rest->addRestCall(RestAction::SHOW, "orchestration-status"); rest->addRestCall(RestAction::ADD, "proxy"); @@ -197,6 +199,7 @@ public: fini() { Singleton::Consume::by()->writeStatusToFile(); + curr_agent_data_report.disableReportSending(); } private: @@ -514,6 +517,81 @@ private: return Maybe(); } + // LCOV_EXCL_START Reason: future changes will be done + bool + updateServiceConfigurationFromBackup() + { + auto policy_file_path = getConfigurationWithDefault( + filesystem_prefix + "/conf/policy.json", + "orchestration", + "Policy file path" + ); + + auto orchestration_policy_file = getPolicyConfigPath("orchestration", Config::ConfigFileType::Policy); + + auto settings_file_path = getConfigurationWithDefault( + filesystem_prefix + "/conf/settings.json", + "orchestration", + "Settings file path" + ); + + dbgInfo(D_ORCHESTRATOR) + << "Enforcing new configuration. Policy file: " + << policy_file_path + << ", Settings file: " + << settings_file_path; + + string backup_ext = getConfigurationWithDefault( + ".bk", + "orchestration", + "Backup file extension" + ); + + auto orchestration_tools = Singleton::Consume::by(); + auto service_controller = Singleton::Consume::by(); + + // Try to use the backup policy.json file and re-write the services's policies. + dbgInfo(D_ORCHESTRATOR) << "Updating services with the new policy."; + if (service_controller->updateServiceConfiguration(policy_file_path + backup_ext, settings_file_path)) { + dbgInfo(D_ORCHESTRATOR) << "Recovering the policy file from backup."; + if (!orchestration_tools->copyFile(policy_file_path + backup_ext, policy_file_path)) { + dbgWarning (D_ORCHESTRATOR) + << "Failed to recover policy file from backup. File: " + << policy_file_path + backup_ext; + return false; + } + return true; + } + dbgWarning (D_ORCHESTRATOR) << "Failed to load Orchestration policy."; + return false; + } + // LCOV_EXCL_STOP + + string + updatePolicyAndFogAddress(const OrchestrationPolicy &orchestration_policy) + { + if (!updateFogAddress(orchestration_policy.getFogAddress())) { + dbgWarning(D_ORCHESTRATOR) << "Failed to update the new Fog address."; + if (!updateFogAddress(policy.getFogAddress())) { + dbgWarning(D_ORCHESTRATOR) << "Failed to restore the old Fog address."; + } + return ""; + } + + policy = orchestration_policy; + + auto service_controller = Singleton::Consume::by(); + string new_policy_version = service_controller->getPolicyVersion(); + Singleton::Consume::by()->setPolicyVersion(new_policy_version); + auto update_communication = Singleton::Consume::by(); + auto path_policy_version = update_communication->sendPolicyVersion(new_policy_version); + if (!path_policy_version.ok()) { + dbgWarning(D_ORCHESTRATOR) << path_policy_version.getErr(); + } + + return new_policy_version; + } + Maybe handlePolicyUpdate(const OrchPolicy &new_policy, const string &settings_path, const vector &data_updates) { @@ -531,6 +609,17 @@ private: return genError("Failed to download the new policy file. Error: " + new_policy_file.getErr()); } + auto orchestration_tools = Singleton::Consume::by(); + auto conf_path = filesystem_prefix + "/conf/policy.json"; + string last_ext = getConfigurationWithDefault( + ".last", + "orchestration", + "last fog policy file extension" + ); + if (!orchestration_tools->copyFile(new_policy_file.unpack(), conf_path + last_ext)) { + dbgWarning(D_ORCHESTRATOR) << "Failed to copy a new policy file to " << conf_path + last_ext; + } + // Calculate the changes between the existing policy to the new one. auto service_controller = Singleton::Consume::by(); string old_policy_version = service_controller->getPolicyVersion(); @@ -579,27 +668,29 @@ private: return genError("Failed to load new Orchestration policy file."); } - if (!updateFogAddress(orchestration_policy.unpack().getFogAddress())) { - dbgWarning(D_ORCHESTRATOR) << "Failed to update the new Fog address."; - if (!updateFogAddress(policy.getFogAddress())) { - dbgWarning(D_ORCHESTRATOR) << "Failed to restore the old Fog address."; - } - return genError("Failed to load Orchestration new policy file, fog update failed."); - } - - policy = orchestration_policy.unpack(); - - string new_policy_version = service_controller->getPolicyVersion(); - Singleton::Consume::by()->setPolicyVersion(new_policy_version); - auto update_communication = Singleton::Consume::by(); - auto path_policy_version = update_communication->sendPolicyVersion(new_policy_version); - if (!path_policy_version.ok()) { - dbgWarning(D_ORCHESTRATOR) << path_policy_version.getErr(); + string new_policy_version = updatePolicyAndFogAddress(orchestration_policy.unpack()); + if (new_policy_version.empty()) { + return genError("Failed to load Orchestration new policy file."); } reloadConfiguration(); if (getProfileAgentSettingWithDefault(false, "agent.config.orchestration.reportAgentDetail")) { + service_controller->clearFailedServices(); reportAgentDetailsMetaData(); + if(service_controller->doesFailedServicesExist()) { + dbgWarning(D_ORCHESTRATOR) << "Failed to enforce Orchestration policy."; + updateServiceConfigurationFromBackup(); + // Reload the orchestration policy, in case of the policy updated + orchestration_policy = loadDefaultOrchestrationPolicy(); + if (!orchestration_policy.ok()) { + return genError("Failed to load new Orchestration policy file."); + } + + new_policy_version = updatePolicyAndFogAddress(orchestration_policy.unpack()); + if (new_policy_version.empty()) { + return genError("Failed to load Orchestration new policy file."); + } + } } dbgTrace(D_ORCHESTRATOR) @@ -1230,6 +1321,8 @@ private: reportAgentDetailsMetaData() { I_DetailsResolver *i_details_resolver = Singleton::Consume::by(); + i_details_resolver->getResolvedDetails(); + AgentDataReport agent_data_report; agent_data_report << AgentReportFieldWithLabel("agent_version", i_details_resolver->getAgentVersion()); @@ -1280,6 +1373,13 @@ private: agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionGER81", "true"); } #endif // gaia || smb + + if (agent_data_report == curr_agent_data_report) { + agent_data_report.disableReportSending(); + } else { + curr_agent_data_report = agent_data_report; + curr_agent_data_report.disableReportSending(); + } } void @@ -1371,6 +1471,7 @@ private: while (true) { static int failure_count = 0; Singleton::Consume::by()->startNewTrace(false); + reportAgentDetailsMetaData(); auto check_update_result = checkUpdate(); if (!check_update_result.ok()) { failure_count++; @@ -1477,7 +1578,7 @@ private: getAttribute(const string &setting, const string &env) { auto res = getSetting(setting); - if (res.ok()) return res.unpack(); + if (res.ok() && *res != "") return res.unpack(); auto env_res = getenv(env.c_str()); if (env_res != nullptr) return env_res; return ""; @@ -1705,6 +1806,7 @@ private: EnvDetails env_details; string filesystem_prefix = ""; + AgentDataReport curr_agent_data_report; }; OrchestrationComp::OrchestrationComp() diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc index bdebc35..306e6f6 100644 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc @@ -104,19 +104,17 @@ public: expectDetailsResolver() { Maybe> no_nginx(genError("No nginx")); - EXPECT_CALL(mock_details_resolver, getPlatform()).WillOnce(Return(string("linux"))); - EXPECT_CALL(mock_details_resolver, getArch()).WillOnce(Return(string("x86_64"))); - EXPECT_CALL(mock_details_resolver, isReverseProxy()).WillOnce(Return(false)); - EXPECT_CALL(mock_details_resolver, isKernelVersion3OrHigher()).WillOnce(Return(false)); - EXPECT_CALL(mock_details_resolver, isGwNotVsx()).WillOnce(Return(false)); - EXPECT_CALL(mock_details_resolver, isVersionEqualOrAboveR8110()).WillOnce(Return(false)); - EXPECT_CALL(mock_details_resolver, parseNginxMetadata()).WillOnce(Return(no_nginx)); - EXPECT_CALL(mock_details_resolver, getAgentVersion()) - .WillOnce(Return("1.1.1")) - .WillOnce(Return("1.1.1")); + EXPECT_CALL(mock_details_resolver, getPlatform()).WillRepeatedly(Return(string("linux"))); + EXPECT_CALL(mock_details_resolver, getArch()).WillRepeatedly(Return(string("x86_64"))); + EXPECT_CALL(mock_details_resolver, isReverseProxy()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, isKernelVersion3OrHigher()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, isGwNotVsx()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, isVersionEqualOrAboveR8110()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, parseNginxMetadata()).WillRepeatedly(Return(no_nginx)); + EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("1.1.1")); map resolved_mgmt_details({{"kernel_version", "4.4.0-87-generic"}}); - EXPECT_CALL(mock_details_resolver, getResolvedDetails()).WillOnce(Return(resolved_mgmt_details)); + EXPECT_CALL(mock_details_resolver, getResolvedDetails()).WillRepeatedly(Return(resolved_mgmt_details)); } void @@ -156,7 +154,6 @@ public: } ::Environment env; - OrchestrationComp orchestration_comp; AgentDetails agent_details; ConfigComponent config_comp; Config::I_Config *config; @@ -185,6 +182,7 @@ public: NiceMock mock_agent_reporter; NiceMock mock_log; + OrchestrationComp orchestration_comp; private: bool @@ -215,6 +213,7 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource) string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; + string last_policy_file_path = "/etc/cp/conf/policy.json.last"; string data_file_path = "/etc/cp/conf/data.json"; string host_address = "1.2.3.5"; diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc index 6a01003..94910af 100755 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc @@ -130,19 +130,17 @@ public: expectDetailsResolver() { Maybe> no_nginx(genError("No nginx")); - EXPECT_CALL(mock_details_resolver, getPlatform()).WillOnce(Return(string("linux"))); - EXPECT_CALL(mock_details_resolver, getArch()).WillOnce(Return(string("x86_64"))); - EXPECT_CALL(mock_details_resolver, isReverseProxy()).WillOnce(Return(false)); - EXPECT_CALL(mock_details_resolver, isKernelVersion3OrHigher()).WillOnce(Return(false)); - EXPECT_CALL(mock_details_resolver, isGwNotVsx()).WillOnce(Return(false)); - EXPECT_CALL(mock_details_resolver, isVersionEqualOrAboveR8110()).WillOnce(Return(false)); - EXPECT_CALL(mock_details_resolver, parseNginxMetadata()).WillOnce(Return(no_nginx)); - EXPECT_CALL(mock_details_resolver, getAgentVersion()) - .WillOnce(Return("1.1.1")) - .WillOnce(Return("1.1.1")); + EXPECT_CALL(mock_details_resolver, getPlatform()).WillRepeatedly(Return(string("linux"))); + EXPECT_CALL(mock_details_resolver, getArch()).WillRepeatedly(Return(string("x86_64"))); + EXPECT_CALL(mock_details_resolver, isReverseProxy()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, isKernelVersion3OrHigher()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, isGwNotVsx()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, isVersionEqualOrAboveR8110()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, parseNginxMetadata()).WillRepeatedly(Return(no_nginx)); + EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("1.1.1")); map resolved_mgmt_details({{"kernel_version", "4.4.0-87-generic"}}); - EXPECT_CALL(mock_details_resolver, getResolvedDetails()).WillOnce(Return(resolved_mgmt_details)); + EXPECT_CALL(mock_details_resolver, getResolvedDetails()).WillRepeatedly(Return(resolved_mgmt_details)); } string @@ -561,6 +559,7 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate) string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; + string last_policy_file_path = "/etc/cp/conf/policy.json.last"; string data_file_path = "/etc/cp/conf/data.json"; string host_address = "1.2.3.5"; string new_host_address = "6.2.3.5"; @@ -603,6 +602,8 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate) EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)) .WillOnce(Return(policy_response)) .WillOnce(Return(new_policy_response)); + EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_path, policy_file_path + ".last")) + .WillOnce(Return(true)); EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)).WillOnce(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); @@ -727,6 +728,7 @@ TEST_F(OrchestrationTest, startOrchestrationPoliceWithFailures) string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; + string last_policy_file_path = "/etc/cp/conf/policy.json.last"; string data_file_path = "/etc/cp/conf/data.json"; string host_address = "1.2.3.5"; @@ -853,6 +855,7 @@ TEST_F(OrchestrationTest, loadOrchestrationPolicyFromBackup) string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; + string last_policy_file_path = "/etc/cp/conf/policy.json.last"; string data_file_path = "/etc/cp/conf/data.json"; string host_address = "1.2.3.5"; @@ -987,6 +990,7 @@ TEST_F(OrchestrationTest, manifestUpdate) string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; + string last_policy_file_path = "/etc/cp/conf/policy.json.last"; string data_file_path = "/etc/cp/conf/data.json"; string host_address = "1.2.3.5"; @@ -1139,7 +1143,9 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; + string last_policy_file_path = "/etc/cp/conf/policy.json.last"; string data_file_path = "/etc/cp/conf/data.json"; + string new_policy_path = "policy path"; string manifest_checksum = "manifest"; string policy_checksum = "policy"; @@ -1166,6 +1172,8 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response)); + EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_path, policy_file_path + ".last")) + .WillOnce(Return(true)); EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)).WillOnce(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); @@ -1194,7 +1202,7 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) Package::ChecksumTypes::SHA256, policy_file ) - ).WillOnce(Return(Maybe(string("policy path")))); + ).WillOnce(Return(Maybe(string(new_policy_path)))); string manifest = ""; string policy = "111111"; string setting = ""; @@ -1284,6 +1292,7 @@ TEST_F(OrchestrationTest, failedDownloadSettings) string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; + string last_policy_file_path = "/etc/cp/conf/policy.json.last"; string data_file_path = "/etc/cp/conf/data.json"; string host_address = "1.2.3.5"; @@ -1445,6 +1454,7 @@ TEST_P(OrchestrationTest, orchestrationFirstRun) string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; + string last_policy_file_path = "/etc/cp/conf/policy.json.last"; string data_file_path = "/etc/cp/conf/data.json"; string host_address = "1.2.3.5"; @@ -1665,6 +1675,7 @@ TEST_F(OrchestrationTest, dataUpdate) string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; + string last_policy_file_path = "/etc/cp/conf/policy.json.last"; string data_file_path = "/etc/cp/conf/data.json"; string host_address = "1.2.3.5"; diff --git a/components/security_apps/orchestration/service_controller/service_controller.cc b/components/security_apps/orchestration/service_controller/service_controller.cc index 1e1e821..5c90440 100755 --- a/components/security_apps/orchestration/service_controller/service_controller.cc +++ b/components/security_apps/orchestration/service_controller/service_controller.cc @@ -296,6 +296,10 @@ public: const string &service_id ) override; + bool doesFailedServicesExist() override; + + void clearFailedServices() override; + private: void cleanUpVirtualFiles(); @@ -323,6 +327,7 @@ private: string update_policy_version; string settings_path; map services_reconf_status; + map failed_services; map services_reconf_names; map services_reconf_ids; string filesystem_prefix; @@ -387,9 +392,24 @@ ServiceController::Impl::getUpdatedReconfStatus() if (res < service_and_reconf_status.second) res = service_and_reconf_status.second; } + return res; } +// LCOV_EXCL_START Reason: future fix will be done +void +ServiceController::Impl::clearFailedServices() +{ + failed_services.clear(); +} + +bool +ServiceController::Impl::doesFailedServicesExist() +{ + return (failed_services.size() > 0); +} +// LCOV_EXCL_STOP + void ServiceController::Impl::init() { @@ -775,18 +795,11 @@ ServiceController::Impl::updateServiceConfiguration( if (new_policy_path.compare(config_file_path) == 0) { dbgDebug(D_ORCHESTRATOR) << "Enforcing the default policy file"; policy_version = version_value; - return true; } string backup_ext = getConfigurationWithDefault(".bk", "orchestration", "Backup file extension"); - // Save the new configuration file. - if (!orchestration_tools->copyFile(new_policy_path, config_file_path)) { - dbgWarning(D_ORCHESTRATOR) << "Failed to save the policy file."; - return false; - } - // Backup the current configuration file. uint max_backup_attempts = 3; bool is_backup_succeed = false; @@ -794,7 +807,7 @@ ServiceController::Impl::updateServiceConfiguration( I_MainLoop *mainloop = Singleton::Consume::by(); for (size_t i = 0; i < max_backup_attempts; i++) { - if (orchestration_tools->copyFile(new_policy_path, backup_file)) { + if (orchestration_tools->copyFile(config_file_path, backup_file)) { is_backup_succeed = true; break; } @@ -807,6 +820,12 @@ ServiceController::Impl::updateServiceConfiguration( } policy_version = version_value; + + // Save the new configuration file. + if (!orchestration_tools->copyFile(new_policy_path, config_file_path)) { + dbgWarning(D_ORCHESTRATOR) << "Failed to save the policy file."; + return false; + } } return was_policy_updated; @@ -835,7 +854,7 @@ ServiceController::Impl::sendSignalForServices( } if (reconf_status == ReconfStatus::FAILED) { - dbgDebug(D_ORCHESTRATOR) << "The reconfiguration failed for serivce " << service_id; + dbgDebug(D_ORCHESTRATOR) << "The reconfiguration failed for serivce: " << service_id; services_reconf_status.clear(); services_reconf_names.clear(); return false; @@ -972,6 +991,10 @@ ServiceController::Impl::getUpdatePolicyVersion() const void ServiceController::Impl::updateReconfStatus(int id, ReconfStatus status) { + if (status == ReconfStatus::FAILED) { + failed_services.emplace(id, status); + } + if (services_reconf_status.find(id) == services_reconf_status.end()) { dbgError(D_ORCHESTRATOR) << "Service reconfiguration monitor received illegal id :" << id; return; diff --git a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc index c5f4ab1..61af44d 100755 --- a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc +++ b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc @@ -254,7 +254,7 @@ TEST_F(ServiceControllerTest, UpdateConfiguration) EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path + backup_extension)) + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -346,7 +346,7 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration) EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path + backup_extension)) + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -465,7 +465,7 @@ TEST_F(ServiceControllerTest, writeRegisteredServicesFromFile) EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path + backup_extension)) + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -606,7 +606,7 @@ TEST_F(ServiceControllerTest, noPolicyUpdate) .WillOnce(Return(json_parser_return)); EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(l4_firewall)); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path + backup_extension)) + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_status, @@ -697,7 +697,7 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations) EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path + backup_extension)) + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -743,7 +743,7 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations) .WillOnce(Return(json_parser_return)); EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(l4_firewall)); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path + backup_extension)) + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_status, @@ -849,7 +849,7 @@ TEST_F(ServiceControllerTest, backup) mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path)).WillOnce(Return(true) ); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path + backup_extension)) + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -963,7 +963,7 @@ TEST_F(ServiceControllerTest, backupAttempts) writeFile(l4_firewall, l4_firewall_policy_path)).WillOnce(Return(true) ); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path + backup_extension)) + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(false)) .WillOnce(Return(false)) .WillOnce(Return(true)); @@ -1078,7 +1078,7 @@ TEST_F(ServiceControllerTest, MultiUpdateConfiguration) EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, writeFile(orchestration, orchestration_policy_path)).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path + backup_extension)) + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -1136,7 +1136,7 @@ TEST_F(ServiceControllerTest, emptyServices) Return(json_parser_return) ); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path + backup_extension)) + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -1355,19 +1355,17 @@ TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest) _ ) ).WillRepeatedly(Return(string("not-registered"))); - EXPECT_CALL( - mock_orchestration_tools, - copyFile(file_name, policy_file_path) - ).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) + .WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); - EXPECT_FALSE(i_service_controller->updateServiceConfiguration(file_name, "")); + EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, "")); EXPECT_THAT( capture_debug.str(), HasSubstr("Service mock access control is inactive") ); EXPECT_FALSE(i_service_controller->isServiceInstalled("family1_id2")); - EXPECT_NE(i_service_controller->getPolicyVersion(), version_value); - EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); + EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value); EXPECT_EQ(i_service_controller->getUpdatePolicyVersion(), version_value); } @@ -1567,7 +1565,7 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles) ); string new_policy_file_path = "/etc/cp/conf/tenant_" + tenant + "_profile_" + profile + "/" + "policy.json"; - EXPECT_CALL(mock_orchestration_tools, copyFile(conf_file_name, new_policy_file_path + backup_extension)) + EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_file_path, new_policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(conf_file_name, new_policy_file_path)).WillOnce(Return(true)); @@ -1664,7 +1662,7 @@ TEST_F(ServiceControllerTest, test_delayed_reconf) EXPECT_CALL(mock_orchestration_status, setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY)); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path + backup_extension)) + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_ml, yield(false)).Times(AnyNumber()); diff --git a/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc b/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc index 38ea9bf..f5b5dae 100755 --- a/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc +++ b/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc @@ -1,5 +1,5 @@ #include "declarative_policy_utils.h" -#include "rest.h" + #include "config.h" #include "log_generator.h" #include "agent_details.h" @@ -54,10 +54,7 @@ DeclarativePolicyUtils::getLocalPolicyChecksum() return orchestration_tools->readFile("/etc/cp/conf/k8s-policy-check.trigger"); } - string policy_path = getConfigurationFlagWithDefault( - getFilesystemPathConfig() + "/conf/local_policy.yaml", - "local_mgmt_policy" - ); + string policy_path = Singleton::Consume::by()->getLocalPolicyPath(); Maybe file_checksum = orchestration_tools->calculateChecksum( I_OrchestrationTools::SELECTED_CHECKSUM_TYPE, @@ -97,7 +94,7 @@ DeclarativePolicyUtils::getPolicyChecksum() I_OrchestrationTools *orchestration_tools = Singleton::Consume::by(); Maybe file_checksum = orchestration_tools->calculateChecksum( I_OrchestrationTools::SELECTED_CHECKSUM_TYPE, - Singleton::Consume::by()->getPolicyPath() + Singleton::Consume::by()->getAgentPolicyPath() ); if (!file_checksum.ok()) { diff --git a/components/security_apps/waap/reputation/reputation_features_agg.cc b/components/security_apps/waap/reputation/reputation_features_agg.cc index 747ddf4..52c5c97 100755 --- a/components/security_apps/waap/reputation/reputation_features_agg.cc +++ b/components/security_apps/waap/reputation/reputation_features_agg.cc @@ -68,6 +68,11 @@ public: void init() { + I_AgentDetails *agentDetails = Singleton::Consume::by(); + + if (agentDetails->getOrchestrationMode() != OrchestrationMode::ONLINE) { + return; + } registerListener(); I_MainLoop* i_mainLoop = Singleton::Consume::by(); I_MainLoop::Routine routine = [this]() { reportReputationFeatures(); }; @@ -77,6 +82,11 @@ public: void fini() { + I_AgentDetails *agentDetails = Singleton::Consume::by(); + + if (agentDetails->getOrchestrationMode() != OrchestrationMode::ONLINE) { + return; + } unregisterListener(); } diff --git a/components/security_apps/waap/waap_clib/TuningDecision.cc b/components/security_apps/waap/waap_clib/TuningDecision.cc index d587bf2..562b51d 100755 --- a/components/security_apps/waap/waap_clib/TuningDecision.cc +++ b/components/security_apps/waap/waap_clib/TuningDecision.cc @@ -118,15 +118,13 @@ void TuningDecision::updateDecisions() { TuningEvents tuningEvents; RemoteFilesList tuningDecisionFiles; - if (m_baseUri == "") { - I_AgentDetails *agentDetails = Singleton::Consume::by(); - if (agentDetails->getOrchestrationMode() != OrchestrationMode::ONLINE) { - m_baseUri = "/api/"; - } else { - m_baseUri = "/storage/waap/"; - } - dbgTrace(D_WAAP) << "URI prefix: " << m_baseUri; + I_AgentDetails *agentDetails = Singleton::Consume::by(); + if (agentDetails->getOrchestrationMode() != OrchestrationMode::ONLINE) { + m_baseUri = "/api/"; + } else { + m_baseUri = "/storage/waap/"; } + dbgTrace(D_WAAP) << "URI prefix: " << m_baseUri; bool isSuccessful = sendObject(tuningDecisionFiles, I_Messaging::Method::GET, m_baseUri + "?list-type=2&prefix=" + m_remotePath); diff --git a/components/security_apps/waap/waap_clib/WaapRegexPreconditions.cc b/components/security_apps/waap/waap_clib/WaapRegexPreconditions.cc index f4bca8e..f3b49f8 100755 --- a/components/security_apps/waap/waap_clib/WaapRegexPreconditions.cc +++ b/components/security_apps/waap/waap_clib/WaapRegexPreconditions.cc @@ -42,35 +42,6 @@ namespace Waap { auto preconditions = jsObj.at("preconditions").get(); - // Build full list of words to load into aho-corasick pattern matcher - dbgTrace(D_WAAP_REGEX) << "Loading regex precondition_keys into Aho-Corasick pattern matcher..."; - - auto preconditionKeys = jsObj.at("precondition_keys").get(); - std::set pmPatterns; - - for (const auto &preconditionKey : preconditionKeys) { - std::string wordStr(preconditionKey.get()); - - // Do not load the "empty" word into Aho-Corasick. It's meaningless and Aho prepare() call would fail. - if (wordStr.empty()) { - continue; - } - - WordIndex wordIndex = registerWord(wordStr); - pmPatterns.insert(PMPattern(wordStr, false, false, wordIndex)); - } - - // Initialize the aho-corasick pattern matcher with the patterns - Maybe pmHookStatus = m_pmHook.prepare(pmPatterns); - - if (!pmHookStatus.ok()) { - dbgError(D_WAAP_REGEX) << "Aho-Corasick engine failed to load!"; - error = true; - return; - } - - dbgTrace(D_WAAP_REGEX) << "Aho-Corasick engine loaded."; - // Loop over pre-conditions (rules) and load them dbgTrace(D_WAAP_REGEX) << "Loading regex preconditions..."; @@ -140,6 +111,7 @@ namespace Waap { if (flags == "_noregex") { // Add regex pattern to set of "noRegex" patterns m_noRegexPatterns.insert(regexPattern); + m_pmWordInfo[wordIndex].noRegex = true; } m_regexToWordMap[regexPattern] = wordIndex; @@ -167,6 +139,43 @@ namespace Waap { } } + // Build full list of words to load into aho-corasick pattern matcher + dbgTrace(D_WAAP_REGEX) << "Loading regex precondition_keys into Aho-Corasick pattern matcher..."; + + auto preconditionKeys = jsObj.at("precondition_keys").get(); + std::set pmPatterns; + + for (const auto &preconditionKey : preconditionKeys) { + std::string wordStr(preconditionKey.get()); + + // Do not load the "empty" word into Aho-Corasick. It's meaningless and Aho prepare() call would fail. + if (wordStr.empty()) { + continue; + } + + WordIndex wordIndex = registerWord(wordStr); + WordIndex napreWordIndex = m_pmWordInfo[wordIndex].napreWordIndex; + WordIndex napostWordIndex = m_pmWordInfo[wordIndex].napostWordIndex; + WordIndex napostNapreWordIndex = m_pmWordInfo[wordIndex].napostNapreWordIndex; + + bool noRegex = ((napreWordIndex != emptyWordIndex) && m_pmWordInfo[napreWordIndex].noRegex) || + ((napostWordIndex != emptyWordIndex) && m_pmWordInfo[napostWordIndex].noRegex) || + ((napostNapreWordIndex != emptyWordIndex) && m_pmWordInfo[napostNapreWordIndex].noRegex); + + pmPatterns.insert(PMPattern(wordStr, false, false, wordIndex, noRegex)); + } + + // Initialize the aho-corasick pattern matcher with the patterns + Maybe pmHookStatus = m_pmHook.prepare(pmPatterns); + + if (!pmHookStatus.ok()) { + dbgError(D_WAAP_REGEX) << "Aho-Corasick engine failed to load!"; + error = true; + return; + } + + dbgTrace(D_WAAP_REGEX) << "Aho-Corasick engine loaded."; + dbgTrace(D_WAAP_REGEX) << "Aho-corasick pattern matching engine initialized!"; } @@ -225,17 +234,17 @@ namespace Waap { dbgTrace(D_WAAP_REGEX) << "Rules pass #1: collect OR sets"; m_pmHook.scanBufWithOffsetLambda(buffer, [this, &wordsSet, &buffer] - (u_int endMatchOffset, const PMPattern &pmPattern) + (u_int endMatchOffset, const PMPattern &pmPattern, bool matchAll) { uint offset = endMatchOffset + 1 - pmPattern.size(); // reported offset points to last character of a match // Extract the word index from the PMPattern object (we do not need the string part of it) WordIndex wordIndex = pmPattern.getIndex(); - bool regexWordBefore = (offset != 0) && - (isRegexWordChar(buffer.data()[offset - 1])); - bool regexWordAfter = (offset + pmPattern.size() < buffer.size()) && - (isRegexWordChar(buffer.data()[offset + pmPattern.size()])); + bool regexWordBefore = !matchAll && (offset != 0) && + (isRegexWordChar(buffer.data()[offset - 1])); + bool regexWordAfter = !matchAll && (offset + pmPattern.size() < buffer.size()) && + (isRegexWordChar(buffer.data()[offset + pmPattern.size()])); processWord(wordsSet, wordIndex); diff --git a/components/security_apps/waap/waap_clib/WaapRegexPreconditions.h b/components/security_apps/waap/waap_clib/WaapRegexPreconditions.h index f3c2c69..e84f725 100755 --- a/components/security_apps/waap/waap_clib/WaapRegexPreconditions.h +++ b/components/security_apps/waap/waap_clib/WaapRegexPreconditions.h @@ -67,6 +67,7 @@ namespace Waap { WordIndex napreWordIndex; WordIndex baseWordIndex; std::string wordStr; + bool noRegex; WordInfo() : @@ -74,7 +75,8 @@ namespace Waap { napostWordIndex(emptyWordIndex), napreWordIndex(emptyWordIndex), baseWordIndex(0), - wordStr() + wordStr(), + noRegex(false) { } }; diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.cc b/components/security_apps/waap/waap_clib/Waf2Engine.cc index 7541503..9196af0 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.cc +++ b/components/security_apps/waap/waap_clib/Waf2Engine.cc @@ -2205,6 +2205,9 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) { for (auto &keyword : res.keyword_matches) { exceptions_dict["indicator"].insert(keyword); } + for (auto &it : res.found_patterns) { + exceptions_dict["indicator"].insert(it.first); + } // calling behavior and check if there is a behavior that match to this specific param name. auto behaviors = exceptions.unpack().getBehavior(exceptions_dict, diff --git a/components/security_apps/waap/waap_clib/Waf2Util.cc b/components/security_apps/waap/waap_clib/Waf2Util.cc index a20df05..d0017d5 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.cc +++ b/components/security_apps/waap/waap_clib/Waf2Util.cc @@ -1186,7 +1186,7 @@ static const SingleRegex base64_key_value_detector_re( err, "base64_key_value"); static const SingleRegex json_key_value_detector_re( - "^[^<>{};,&\\?|=\\s]+={.+:.+}\\z", + "^[^<>{};,&\\?|=\\s]+={.+(?s):.+(?s)}\\z", err, "json_key_value"); static const SingleRegex base64_key_detector_re( diff --git a/components/utils/pm/pm_hook.cc b/components/utils/pm/pm_hook.cc index 21f071a..cc0fdaa 100644 --- a/components/utils/pm/pm_hook.cc +++ b/components/utils/pm/pm_hook.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include "kiss_patterns.h" #include "kiss_thin_nfa_impl.h" @@ -132,17 +133,41 @@ PMHook::scanBufWithOffset(const Buffer &buf) const } void -PMHook::scanBufWithOffsetLambda(const Buffer &buf, function cb) const +PMHook::scanBufWithOffsetLambda(const Buffer &buf, I_PMScan::CBFunction cb) const { dbgAssert(handle != nullptr) << "Unusable Pattern Matcher"; + unordered_map match_counts; vector> pm_matches; + static const uint maxCbCount = 3; + uint totalCount = 0; + kiss_thin_nfa_exec(handle.get(), buf, pm_matches); dbgTrace(D_PM) << pm_matches.size() << " raw matches found"; for (auto &res : pm_matches) { - cb(res.second, patterns.at(res.first)); + uint patIndex = res.first; + uint cbCount = match_counts[patIndex]; + const PMPattern &pat = patterns.at(patIndex); + bool noRegex = pat.isNoRegex(); + bool isShort = (pat.size() == 1); + + // Limit the max number of callback calls per precondition, unless it's used as a regex substitute + // On the last callback call, make sure to add the pre/post-word associated preconditions + if (noRegex || cbCount < maxCbCount) { + bool matchAll = !noRegex && (cbCount == maxCbCount-1 || isShort); + + totalCount++; + cb(res.second, pat, matchAll); + + if (matchAll) + match_counts[patIndex] = maxCbCount; + else + match_counts[patIndex]++; + } } + + dbgTrace(D_PM) << totalCount << " filtered matches found"; } bool diff --git a/components/utils/pm/pm_ut/pm_scan_ut.cc b/components/utils/pm/pm_ut/pm_scan_ut.cc index ab8f38c..7be9a98 100644 --- a/components/utils/pm/pm_ut/pm_scan_ut.cc +++ b/components/utils/pm/pm_ut/pm_scan_ut.cc @@ -461,20 +461,69 @@ TEST(pm_scan, pm_offsets_test_pat_getIndex_method) TEST(pm_scan, pm_offsets_lambda_test_pat_getIndex_method) { set initPatts; + initPatts.insert(PMPattern("ABC", false, false)); // initialized with the default index 0 initPatts.insert(PMPattern("ABCD", false, false, 4)); initPatts.insert(PMPattern("CDE", false, false, 7)); + initPatts.insert(PMPattern("DCB", false, false)); + initPatts.insert(PMPattern("*", false, false)); + PMHook pm; EXPECT_TRUE(pm.prepare(initPatts).ok()); - Buffer buf("12345ABCDEF5678"); + Buffer buf("12345ABCDEF5678 * DCB * DCB * DCB * DCB"); std::set> results; - pm.scanBufWithOffsetLambda(buf, [&] (uint offset, const PMPattern &pat) { results.emplace(offset, pat); }); + pm.scanBufWithOffsetLambda(buf, [&] (uint offset, const PMPattern &pat, bool matchAll) + { results.emplace(offset, pat); (void)matchAll; } ); + // limit to 1 cb call for 1 character long matches, and 3 cb calles for longer matches std::set> expected{ {8, {"ABCD", false, false, 4}}, {7, {"ABC", false, false, 0}}, - {9, {"CDE", false, false, 7}} + {9, {"CDE", false, false, 7}}, + {20, {"DCB", false, false, 0}}, + {26, {"DCB", false, false, 0}}, + {32, {"DCB", false, false, 0}}, + {22, {"*", false, false, 0}} + }; + + EXPECT_EQ(results, expected); +} + +TEST(pm_scan, pm_offsets_lambda_test_pat_limit_noregex) +{ + set initPatts; + + initPatts.insert(PMPattern("ABC", false, false)); // initialized with the default index 0 + initPatts.insert(PMPattern("ABCD", false, false)); + initPatts.insert(PMPattern("CDE", false, false)); + initPatts.insert(PMPattern("DCB", false, false, 0, true)); + initPatts.insert(PMPattern("*", false, false, 0, true)); + + PMHook pm; + EXPECT_TRUE(pm.prepare(initPatts).ok()); + + Buffer buf("12345ABCDEF5678 * DCB * DCB * DCB * DCB"); + std::set> results; + pm.scanBufWithOffsetLambda(buf, [&] (uint offset, const PMPattern &pat, bool matchAll) + { + results.emplace(offset, pat); + EXPECT_FALSE(matchAll); + } ); + + // don't limit no. of cb when noregex is set + std::set> expected{ + {8, {"ABCD", false, false, 0}}, + {7, {"ABC", false, false, 0}}, + {9, {"CDE", false, false, 0}}, + {20, {"DCB", false, false, 0, true}}, + {26, {"DCB", false, false, 0, true}}, + {32, {"DCB", false, false, 0, true}}, + {38, {"DCB", false, false, 0, true}}, + {16, {"*", false, false, 0, true}}, + {22, {"*", false, false, 0, true}}, + {28, {"*", false, false, 0, true}}, + {34, {"*", false, false, 0, true}} }; EXPECT_EQ(results, expected); diff --git a/core/agent_details_reporter/agent_details_report.cc b/core/agent_details_reporter/agent_details_report.cc index c84455f..b1ecc9d 100644 --- a/core/agent_details_reporter/agent_details_report.cc +++ b/core/agent_details_reporter/agent_details_report.cc @@ -19,6 +19,7 @@ using namespace std; AgentDataReport::~AgentDataReport() { + if (!should_report) return; Singleton::Consume::by()->sendReport( agent_details, policy_version, @@ -35,6 +36,17 @@ AgentDataReport::operator<<(const pair &data) return *this; } +bool +AgentDataReport::operator==(const AgentDataReport &other) const +{ + return policy_version == other.policy_version && + platform == other.platform && + architecture == other.architecture && + agent_version == other.agent_version && + agent_details == other.agent_details && + attributes == other.attributes; +} + void AgentDataReport::setPolicyVersion(const string &_policy_version) { @@ -58,3 +70,9 @@ AgentDataReport::setAgentVersion(const string &_agent_version) { agent_version = _agent_version; } + +void +AgentDataReport::disableReportSending() +{ + should_report = false; +} diff --git a/core/agent_details_reporter/agent_details_reporter.cc b/core/agent_details_reporter/agent_details_reporter.cc index 38d198d..886562b 100644 --- a/core/agent_details_reporter/agent_details_reporter.cc +++ b/core/agent_details_reporter/agent_details_reporter.cc @@ -127,6 +127,7 @@ private: const string &operation; }; + map persistant_attributes; map new_attributes; map attributes; @@ -141,6 +142,12 @@ metaDataReport::operator<<(const pair &data) return *this; } +bool +metaDataReport::operator==(const metaDataReport &other) const +{ + return agent_details == other.agent_details; +} + void metaDataReport::serialize(cereal::JSONOutputArchive &out_ar) const { @@ -169,7 +176,12 @@ AgentDetailsReporter::Impl::addAttr(const string &key, const string &val, bool a } } + if (persistant_attributes[key] == val) { + dbgDebug(D_AGENT_DETAILS) << "Attribute " << key << " did not change. Value: " << val; + return true; + } new_attributes[key] = val; + persistant_attributes[key] = val; dbgDebug(D_AGENT_DETAILS) << "Successfully added new attribute"; return true; @@ -194,6 +206,7 @@ AgentDetailsReporter::Impl::deleteAttr(const string &key) dbgDebug(D_AGENT_DETAILS) << "Deleting existing attributes. Key: " << key; attributes.erase(key); new_attributes.erase(key); + persistant_attributes.erase(key); } bool diff --git a/core/config/config.cc b/core/config/config.cc index decc78a..44432f0 100644 --- a/core/config/config.cc +++ b/core/config/config.cc @@ -80,14 +80,11 @@ public: error = load_config_staus == I_Config::AsyncLoadConfigStatus::Error; if (error) { + error_message = "Reload already in progress - can't start another one"; dbgWarning(D_CONFIG) << "Configuration reload status: " << status_map.at(load_config_staus); } else { dbgDebug(D_CONFIG) << "Configuration reload status: " << status_map.at(load_config_staus); } - - if (!finished) { - error_message = "Reload already in progress - can't start another one"; - } } private: @@ -625,7 +622,7 @@ ConfigComponent::Impl::reloadConfiguration(const string &version, bool is_async, { if (is_continuous_report) { dbgWarning(D_CONFIG) << "Cannot start another continuous reload while another is running."; - return AsyncLoadConfigStatus::InProgress; + return AsyncLoadConfigStatus::Error; } if (!is_async) { @@ -643,7 +640,7 @@ ConfigComponent::Impl::reloadConfiguration(const string &version, bool is_async, "A-Synchronize reload configuraion" ); - return AsyncLoadConfigStatus::Success; + return AsyncLoadConfigStatus::InProgress; } void diff --git a/core/core_ut/cache_ut.cc b/core/core_ut/cache_ut.cc index b1effc1..9170c60 100644 --- a/core/core_ut/cache_ut.cc +++ b/core/core_ut/cache_ut.cc @@ -78,6 +78,18 @@ TEST(TempCaching, value_emplace) EXPECT_EQ(val, 9); } +TEST(TempCaching, value_get_const) +{ + TemporaryCache cache; + cache.emplaceEntry(3, 27); + + auto &const_cache = const_cast &>(cache); + + EXPECT_FALSE(const_cache.getEntry(0).ok()); + EXPECT_TRUE(const_cache.getEntry(3).ok()); + EXPECT_EQ(const_cache.getEntry(3).unpack(), 27); +} + TEST(TempCaching, get_uninitialized_value) { TemporaryCache cache; diff --git a/core/include/general/debug.h b/core/include/general/debug.h index 7c8bc1a..c62d92f 100755 --- a/core/include/general/debug.h +++ b/core/include/general/debug.h @@ -32,6 +32,9 @@ class I_Encryptor; class I_AgentDetails; class I_SignalHandler; +namespace Config { enum class Errors; } +std::ostream & operator<<(std::ostream &, const Config::Errors &); + class Debug : Singleton::Consume, diff --git a/core/include/services_sdk/interfaces/i_agent_details_reporter.h b/core/include/services_sdk/interfaces/i_agent_details_reporter.h index f6e8fcc..f85193a 100644 --- a/core/include/services_sdk/interfaces/i_agent_details_reporter.h +++ b/core/include/services_sdk/interfaces/i_agent_details_reporter.h @@ -27,6 +27,7 @@ public: metaDataReport(const metaDataReport &) = default; metaDataReport & operator<<(const std::pair &data); + bool operator==(const metaDataReport &other) const; void serialize(cereal::JSONOutputArchive &out_ar) const; private: diff --git a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h index 8ffa170..f319eea 100755 --- a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h +++ b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h @@ -15,6 +15,7 @@ #define __I_INTELLIGENCE_IS_V2_H__ #include +#include #include "maybe_res.h" #include "i_messaging.h" @@ -130,6 +131,14 @@ private: ); } + dbgTrace(D_INTELLIGENCE) + << "Sending intelligence request with IP: " + << ip + << " port: " + << server_port + << " query_uri: " + << query_uri; + return i_message->sendObject( intelligence_query, I_Messaging::Method::POST, @@ -248,10 +257,11 @@ private: "intelligence", is_primary_port ? primary_port_setting : secondary_port_setting ); + if (!server_port.ok()) return false; conn_flags.reset(); - + if (intelligence_query.getPagingStatus().ok()) { return sendPagingQueryMessage( intelligence_query, @@ -275,12 +285,22 @@ private: auto i_message = getMessaging(); Flags conn_flags; + bool crowdsec_enabled = std::getenv("CROWDSEC_ENABLED") ? + std::string(std::getenv("CROWDSEC_ENABLED")) == "true" : + false; + + crowdsec_enabled = getProfileAgentSettingWithDefault( + crowdsec_enabled, + "layer7AccessControl.crowdsec.enabled" + ); + bool use_local_intelligence = getProfileAgentSettingWithDefault( false, "agent.config.useLocalIntelligence" ); + auto server_ip = getSetting("intelligence", "local intelligence server ip"); - if (server_ip.ok() && use_local_intelligence) { + if (server_ip.ok() && (use_local_intelligence || crowdsec_enabled)) { if (sendQueryObjectToLocalServer( intelligence_query, query_uri, diff --git a/core/include/services_sdk/interfaces/i_mainloop.h b/core/include/services_sdk/interfaces/i_mainloop.h index 8e82d55..4f6006f 100644 --- a/core/include/services_sdk/interfaces/i_mainloop.h +++ b/core/include/services_sdk/interfaces/i_mainloop.h @@ -61,6 +61,8 @@ public: virtual Maybe getCurrentRoutineId() const = 0; + virtual void updateCurrentStress(bool is_busy) = 0; + virtual void run() = 0; // When a routine yields the scheduler may choose to let it continue to run (in the case the routine didn't use diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h index a078ad9..542cdf2 100755 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h @@ -58,6 +58,8 @@ enum class ResponseStatus IN_PROGRESS }; +enum class ObjectType { ASSET, ZONE, CONFIGURATION, COUNT }; + const std::string & convertConditionTypeToString(const Condition &condition_type); const std::string & convertOperationTypeToString(const Operator &operation_type); std::string createAttributeString(const std::string &key, AttributeKeyType type); diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h index e052f66..c3653d0 100755 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h @@ -63,6 +63,8 @@ public: ); void setTenantsList(const std::vector tenants); + void setCrossTenantAssetDB(bool cross_tenant_asset_db); + void setObjectType(const ObjectType &obj_type); void setAssetsLimit(uint _assets_limit); bool checkMinConfidence(uint upper_confidence_limit); @@ -83,11 +85,13 @@ public: private: uint assets_limit = default_assets_limit; bool full_response = false; + Maybe object_type = genError("uninitialized"); Maybe cursor = genError("Cursor not initialized"); SerializableQueryFilter query; SerializableAttributesMap requested_attributes; SerializableQueryTypes query_types; QueryRequest calcQueryRequestOperator(const QueryRequest &other_query, const Operator &operator_type); + Maybe convertObjectTypeToString() const; }; class BulkQueryRequest diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_types_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/query_types_v2.h index 8dd858c..ccfeea9 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/query_types_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/query_types_v2.h @@ -19,35 +19,26 @@ #include "cereal/types/tuple.hpp" #include "cereal/types/vector.hpp" #include "intelligence_types_v2.h" +#include "maybe_res.h" #include #include -class serializableTenantList -{ -public: - serializableTenantList(const std::vector &_tenants) - : - tenants(_tenants) - {} - - void serialize(cereal::JSONOutputArchive &ar) const; - -private: - std::vector tenants; -}; - class SerializableQueryTypes { public: SerializableQueryTypes() {}; void save(cereal::JSONOutputArchive &ar) const; - void setSerializableTenantList(const std::vector tenants); + void setSerializableTenantList(const std::vector &tenant_list); + void setQueryCrossTenantAssetDB(bool query_cross_tenant_asset_db); private: - std::vector tenants; - bool is_nsaas = false; + void serializeMultiTenant(cereal::JSONOutputArchive &ar) const; + void serializeCrossTenantAssetDB(cereal::JSONOutputArchive &ar) const; + + Maybe> tenants = genError("tenant list is uninitialized"); + Maybe query_cross_tenant_asset_db = genError("cross tenant asset db query is uninitialized"); }; #endif // __QUERY_TYPES_V2_H__ diff --git a/core/include/services_sdk/interfaces/mock/mock_mainloop.h b/core/include/services_sdk/interfaces/mock/mock_mainloop.h index 3143bf7..a439995 100644 --- a/core/include/services_sdk/interfaces/mock/mock_mainloop.h +++ b/core/include/services_sdk/interfaces/mock/mock_mainloop.h @@ -26,6 +26,8 @@ public: MOCK_CONST_METHOD0(getCurrentRoutineId, Maybe ()); + MOCK_METHOD1(updateCurrentStress, void (bool)); + MOCK_METHOD1(yield, void (bool)); MOCK_METHOD1(yield, void (std::chrono::microseconds)); diff --git a/core/include/services_sdk/interfaces/mock/mock_shell_cmd.h b/core/include/services_sdk/interfaces/mock/mock_shell_cmd.h index 8631ca6..29fcb34 100755 --- a/core/include/services_sdk/interfaces/mock/mock_shell_cmd.h +++ b/core/include/services_sdk/interfaces/mock/mock_shell_cmd.h @@ -7,6 +7,13 @@ #include "singleton.h" #include "cptest.h" +static std::ostream & +operator<<(std::ostream &os, const Maybe> &val) +{ + if (val.ok()) return os << "<" << (*val).first << ", " << (*val).second << ">"; + return os; +} + class MockShellCmd : public Singleton::Provide::From> { public: @@ -15,10 +22,4 @@ public: MOCK_METHOD3(getExecOutputAndCode, Maybe>(const std::string &, uint, bool)); }; -static std::ostream & -operator<<(std::ostream &os, const std::pair &val) -{ - return os << "<" << val.first << ", " << val.second << ">"; -} - #endif // __MOCK_SHELL_CMD_H__ diff --git a/core/include/services_sdk/interfaces/mock/mock_socket_is.h b/core/include/services_sdk/interfaces/mock/mock_socket_is.h index f825475..bcda2b4 100755 --- a/core/include/services_sdk/interfaces/mock/mock_socket_is.h +++ b/core/include/services_sdk/interfaces/mock/mock_socket_is.h @@ -5,6 +5,8 @@ #include "cptest.h" #include "common.h" +std::ostream & operator<<(std::ostream &os, const Maybe> &) { return os; } + class MockSocketIS : public Singleton::Provide::From> { public: diff --git a/core/include/services_sdk/resources/agent_details_report.h b/core/include/services_sdk/resources/agent_details_report.h index 983901b..8aec594 100644 --- a/core/include/services_sdk/resources/agent_details_report.h +++ b/core/include/services_sdk/resources/agent_details_report.h @@ -30,17 +30,22 @@ class AgentDataReport { public: AgentDataReport() = default; + AgentDataReport(bool disable_report_sending) { should_report = disable_report_sending; } ~AgentDataReport(); AgentDataReport & operator<<(const std::pair &data); + bool operator==(const AgentDataReport& other) const; + void setPolicyVersion(const std::string &policy_version); void setPlatform(const std::string &platform); void setArchitecture(const std::string &architecture); void setAgentVersion(const std::string &_agent_version); + void disableReportSending(); private: metaDataReport agent_details; + bool should_report = true; Maybe policy_version = genError("Not set"); Maybe platform = genError("Not set"); Maybe architecture = genError("Not set"); diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index 11357b8..446bbd4 100755 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -131,6 +131,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_SDWAN_POLICY, D_SDWAN) DEFINE_FLAG(D_SDWAN_DATA, D_SDWAN) DEFINE_FLAG(D_LOGGER_SDWAN, D_SDWAN) + DEFINE_FLAG(D_SDWAN_API, D_SDWAN) DEFINE_FLAG(D_REVERSE_PROXY, D_COMPONENT) DEFINE_FLAG(D_PLATFORM, D_REVERSE_PROXY) DEFINE_FLAG(D_NGINX_MESSAGE_READER, D_REVERSE_PROXY) @@ -143,11 +144,15 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_IOT_AUXILIARY, D_IOT_NEXT) DEFINE_FLAG(D_IOT_REPORT_STATUS, D_IOT_AUXILIARY) DEFINE_FLAG(D_IOT_COLLECT_METADATA, D_IOT_AUXILIARY) + DEFINE_FLAG(D_IOT_QUERY_INTELLIGENCE, D_IOT_AUXILIARY) + DEFINE_FLAG(D_IOT_SAVE_PERSISTENT, D_IOT_AUXILIARY) + DEFINE_FLAG(D_IOT_DOCKER, D_IOT_AUXILIARY) DEFINE_FLAG(D_IOT_ENFORCE, D_IOT_NEXT) DEFINE_FLAG(D_IOT_ENFORCE_POLICY, D_IOT_ENFORCE) DEFINE_FLAG(D_IOT_ENFORCE_ASSETS, D_IOT_ENFORCE) DEFINE_FLAG(D_IOT_DOCTOR, D_IOT_NEXT) DEFINE_FLAG(D_IOT_RISK, D_IOT_NEXT) + DEFINE_FLAG(D_IOT_QUERY_ASSETS, D_IOT_RISK) DEFINE_FLAG(D_IOT_INDICATOR_DATA, D_IOT_RISK) DEFINE_FLAG(D_IOT_INDICATORS, D_IOT_RISK) DEFINE_FLAG(D_IOT_DISCOVERY, D_IOT_NEXT) @@ -159,6 +164,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_CPVIEW_METRIC_PROVIDER, D_COMPONENT) DEFINE_FLAG(D_GEO_FILTER, D_COMPONENT) DEFINE_FLAG(D_URL_FILTERING, D_COMPONENT) + DEFINE_FLAG(D_L7_ACCESS_CONTROL, D_COMPONENT) DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT) DEFINE_FLAG(D_FLOW, D_ALL) diff --git a/core/include/services_sdk/resources/report/Log_modifiers.h b/core/include/services_sdk/resources/report/Log_modifiers.h new file mode 100644 index 0000000..122bc9d --- /dev/null +++ b/core/include/services_sdk/resources/report/Log_modifiers.h @@ -0,0 +1,82 @@ +#ifndef __LOG_MODIFIERS_H__ +#define __LOG_MODIFIERS_H__ + +#include +#include "virtual_modifiers.h" + +namespace LogModifiers +{ + +class ReplaceBackslash : public ReplaceSubContiners +{ +public: + ReplaceBackslash() { init(&src, &dst); } + +private: + std::string src = "\\"; + std::string dst = "\\\\"; +}; + +class ReplaceCR : public ReplaceSubContiners +{ +public: + ReplaceCR() { init(&src, &dst); } + +private: + std::string src = "\r"; + std::string dst = "\\r"; +}; + +class ReplaceLF : public ReplaceSubContiners +{ +public: + ReplaceLF() { init(&src, &dst); } + +private: + std::string src = "\n"; + std::string dst = "\\n"; +}; + +class ReplaceDoubleOuotes : public ReplaceSubContiners +{ +public: + ReplaceDoubleOuotes() { init(&src, &dst); } + +private: + std::string src = "\""; + std::string dst = "\\\""; +}; + +class ReplaceQuote : public ReplaceSubContiners +{ +public: + ReplaceQuote() { init(&src, &dst); } + +private: + std::string src = "'"; + std::string dst = "\\'"; +}; + +class ReplaceClosingBrace : public ReplaceSubContiners +{ +public: + ReplaceClosingBrace() { init(&src, &dst); } + +private: + std::string src = "]"; + std::string dst = "\\]"; +}; + +class ReplaceEqualSign : public ReplaceSubContiners +{ +public: + ReplaceEqualSign() { init(&src, &dst); } + +private: + std::string src = "="; + std::string dst = "\\="; +}; + +} // namesapce LogModifiers + +#endif // __LOG_MODIFIERS_H__ diff --git a/core/include/services_sdk/resources/report/base_field.h b/core/include/services_sdk/resources/report/base_field.h index bd8c88a..d517eaa 100755 --- a/core/include/services_sdk/resources/report/base_field.h +++ b/core/include/services_sdk/resources/report/base_field.h @@ -25,6 +25,8 @@ #include "debug.h" #include "flags.h" #include "config.h" +#include "virtual_container.h" +#include "Log_modifiers.h" enum class LogFieldOption { XORANDB64, COUNT }; @@ -72,8 +74,7 @@ class LogField : Singleton::Consume virtual void serialize(cereal::JSONOutputArchive &ar) const = 0; virtual void addFields(const LogField &log) = 0; - virtual std::string getSyslog() const = 0; - virtual std::string getCef() const = 0; + virtual std::string getSyslogAndCef() const = 0; template Maybe @@ -107,15 +108,17 @@ class LogField : Singleton::Consume } std::string - getSyslog() const override + getSyslogAndCef() const override { - return name + "='" + Details::getValueAsString(getValue()) + "'"; - } - - std::string - getCef() const override - { - return name + "=" + Details::getValueAsString(getValue()); + std::string value(Details::getValueAsString(getValue())); + auto modifier1 = makeVirtualContainer(value); + auto modifier2 = makeVirtualContainer(modifier1); + auto modifier3 = makeVirtualContainer(modifier2); + auto modifier4 = makeVirtualContainer(modifier3); + auto modifier5 = makeVirtualContainer(modifier4); + auto modifier6 = makeVirtualContainer(modifier5); + auto modifier7 = makeVirtualContainer(modifier6); + return name + "=\"" + std::string(modifier7.begin(), modifier7.end()) + "\""; } // LCOV_EXCL_START Reason: seems that assert prevent the LCOV from identifying that method was tested @@ -180,27 +183,14 @@ class LogField : Singleton::Consume } std::string - getSyslog() const override + getSyslogAndCef() const override { if (fields.size() == 0) return ""; std::string res; for (auto &field : fields) { if (res.size() > 0) res += " "; - res += field.getSyslog(); - } - return res; - } - - std::string - getCef() const override - { - if (fields.size() == 0) return ""; - - std::string res; - for (auto &field : fields) { - if (res.size() > 0) res += " "; - res += field.getCef(); + res += field.getSyslogAndCef(); } return res; } @@ -252,15 +242,9 @@ public: } std::string - getSyslog() const + getSyslogAndCef() const { - return field->getSyslog(); - } - - std::string - getCef() const - { - return field->getCef(); + return field->getSyslogAndCef(); } void diff --git a/core/include/services_sdk/resources/report/report_enums.h b/core/include/services_sdk/resources/report/report_enums.h index 628d49a..5ef1d81 100755 --- a/core/include/services_sdk/resources/report/report_enums.h +++ b/core/include/services_sdk/resources/report/report_enums.h @@ -61,6 +61,7 @@ enum class Tags { WEB_SERVER_KONG, DEPLOYMENT_EMBEDDED, DEPLOYMENT_K8S, + LAYER_7_ACCESS_CONTROL, COUNT }; diff --git a/core/include/services_sdk/utilities/cache.h b/core/include/services_sdk/utilities/cache.h index df10626..d9558a9 100644 --- a/core/include/services_sdk/utilities/cache.h +++ b/core/include/services_sdk/utilities/cache.h @@ -74,6 +74,7 @@ public: bool emplaceEntry(const Key &key, const Value &val); bool emplaceEntry(const Key &key, Value &&val); Value & getEntry(const Key &key); + Maybe getEntry(const Key &key) const; microseconds getEntryTimeLeft(const Key &key); }; diff --git a/core/include/services_sdk/utilities/caching/cache_impl.h b/core/include/services_sdk/utilities/caching/cache_impl.h index 83225ba..aa0d04e 100644 --- a/core/include/services_sdk/utilities/caching/cache_impl.h +++ b/core/include/services_sdk/utilities/caching/cache_impl.h @@ -232,6 +232,14 @@ TemporaryCache::getEntry(const Key &key) return entries.at(key).getValue(); } +template +Maybe +TemporaryCache::getEntry(const Key &key) const +{ + if (!BaseTemporaryCache::doesKeyExists(key)) return genError(0); + return entries.at(key).getValue(); +} + template std::chrono::microseconds TemporaryCache::getEntryTimeLeft(const Key &key) diff --git a/core/include/services_sdk/utilities/caching/cache_types.h b/core/include/services_sdk/utilities/caching/cache_types.h index b6d4780..60eafd2 100644 --- a/core/include/services_sdk/utilities/caching/cache_types.h +++ b/core/include/services_sdk/utilities/caching/cache_types.h @@ -50,6 +50,7 @@ public: void setNewTime(I_TimeGet *timer) { timer != nullptr ? time = timer->getMonotonicTime() : microseconds(0); } bool isExpired(const microseconds &expired) const { return time < expired; } Value & getValue() { return val; } + const Value & getValue() const { return val; } microseconds getTime() { return time; } private: diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc index 4f872bf..c714d17 100755 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -140,6 +140,9 @@ IntelligenceComponentV2::preload() registerExpectedConfiguration("intelligence", "offline intelligence only"); registerExpectedConfiguration("intelligence", "maximum request overall time"); registerExpectedConfiguration("intelligence", "maximum request lap time"); + registerExpectedSetting("intelligence", "local intelligence server ip"); + registerExpectedSetting("intelligence", "local intelligence server secondary port"); + registerExpectedSetting("intelligence", "local intelligence server primary port"); registerExpectedConfigFile("agent-intelligence", Config::ConfigFileType::Policy); } diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc index 36aa02c..8381482 100755 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc @@ -5,6 +5,8 @@ using namespace std; using namespace testing; +USE_DEBUG_FLAG(D_INTELLIGENCE); + TEST(QueryRequestTestV2, QueryTest) { QueryRequest request(Condition::EQUALS, "phase", "testing", true); @@ -438,3 +440,104 @@ TEST(QueryRequestTestV2, OneLinerComplexQueryTest) "}"; EXPECT_EQ(out.str(), output_json); } + +TEST(QueryRequestTestV2, CrossTenantAssetDBTest) +{ + QueryRequest request(Condition::EQUALS, "class", "risk", true); + + request.setObjectType(ObjectType::CONFIGURATION); + request.setCrossTenantAssetDB(true); + + string output_json = + "{\n" + " \"limit\": 20,\n" + " \"fullResponse\": true,\n" + " \"query\": {\n" + " \"operator\": \"equals\",\n" + " \"key\": \"mainAttributes.class\",\n" + " \"value\": \"risk\"\n" + " },\n" + " \"objectType\": \"configuration\",\n" + " \"queryTypes\": {\n" + " \"queryCrossTenantAssetDB\": true\n" + " }\n" + "}"; + + stringstream out; + { + cereal::JSONOutputArchive out_ar(out); + request.saveToJson(out_ar); + } + EXPECT_EQ(out.str(), output_json); +} + +TEST(QueryRequestTestV2, IllegalObjectTypeTest) +{ + QueryRequest request(Condition::EQUALS, "class", "risk", true); + stringstream debug_output; + Debug::setNewDefaultStdout(&debug_output); + Debug::setUnitTestFlag(D_INTELLIGENCE, Debug::DebugLevel::TRACE); + + request.setObjectType(static_cast(static_cast(ObjectType::COUNT) + 1)); + request.setCrossTenantAssetDB(true); + + string output_json = + "{\n" + " \"limit\": 20,\n" + " \"fullResponse\": true,\n" + " \"query\": {\n" + " \"operator\": \"equals\",\n" + " \"key\": \"mainAttributes.class\",\n" + " \"value\": \"risk\"\n" + " },\n" + " \"queryTypes\": {\n" + " \"queryCrossTenantAssetDB\": true\n" + " }\n" + "}"; + + stringstream out; + { + cereal::JSONOutputArchive out_ar(out); + request.saveToJson(out_ar); + } + EXPECT_EQ(out.str(), output_json); + + string debug_str = "Illegal Object Type."; + EXPECT_THAT(debug_output.str(), HasSubstr(debug_str)); + Debug::setNewDefaultStdout(&cout); +} + +TEST(QueryRequestTestV2, UninitializedObjectTypeTest) +{ + QueryRequest request(Condition::EQUALS, "class", "risk", true); + stringstream debug_output; + Debug::setNewDefaultStdout(&debug_output); + Debug::setUnitTestFlag(D_INTELLIGENCE, Debug::DebugLevel::TRACE); + + request.setCrossTenantAssetDB(true); + + string output_json = + "{\n" + " \"limit\": 20,\n" + " \"fullResponse\": true,\n" + " \"query\": {\n" + " \"operator\": \"equals\",\n" + " \"key\": \"mainAttributes.class\",\n" + " \"value\": \"risk\"\n" + " },\n" + " \"queryTypes\": {\n" + " \"queryCrossTenantAssetDB\": true\n" + " }\n" + "}"; + + stringstream out; + { + cereal::JSONOutputArchive out_ar(out); + request.saveToJson(out_ar); + } + EXPECT_EQ(out.str(), output_json); + + string debug_str = "uninitialized"; + EXPECT_THAT(debug_output.str(), HasSubstr(debug_str)); + Debug::setNewDefaultStdout(&cout); +} diff --git a/core/intelligence_is_v2/query_request_v2.cc b/core/intelligence_is_v2/query_request_v2.cc index d60d977..2113311 100755 --- a/core/intelligence_is_v2/query_request_v2.cc +++ b/core/intelligence_is_v2/query_request_v2.cc @@ -13,6 +13,7 @@ #include "intelligence_is_v2/query_request_v2.h" #include "debug.h" +#include "enum_array.h" const uint QueryRequest::default_min_confidence = 500; const uint QueryRequest::default_assets_limit = 20; @@ -22,6 +23,8 @@ using namespace Intelligence_IS_V2; USE_DEBUG_FLAG(D_INTELLIGENCE); +static const EnumArray object_type_to_string_array{ "asset", "zone", "configuration" }; + BulkQueryRequest::BulkQueryRequest(QueryRequest &_request, int _index) : request(_request), @@ -55,6 +58,17 @@ QueryRequest::QueryRequest( full_response = full_reponse; } +Maybe +QueryRequest::convertObjectTypeToString() const +{ + if (!object_type.ok()) return object_type.passErr(); + if (static_cast(*object_type) < static_cast(ObjectType::COUNT)) { + return object_type_to_string_array[*object_type]; + } + + return genError("Illegal Object Type."); +} + void QueryRequest::saveToJson(cereal::JSONOutputArchive &ar) const { @@ -64,6 +78,13 @@ QueryRequest::saveToJson(cereal::JSONOutputArchive &ar) const cereal::make_nvp("query", query) ); + auto objTypeString = convertObjectTypeToString(); + if (objTypeString.ok()) { + ar(cereal::make_nvp("objectType", *objTypeString)); + } else { + dbgTrace(D_INTELLIGENCE) << objTypeString.getErr(); + } + if (cursor.ok()) ar(cereal::make_nvp("cursor", cursor.unpack().second)); requested_attributes.save(ar); query_types.save(ar); @@ -78,6 +99,13 @@ QueryRequest::save(cereal::JSONOutputArchive &ar) const cereal::make_nvp("query", query) ); + auto objTypeString = convertObjectTypeToString(); + if (objTypeString.ok()) { + ar(cereal::make_nvp("objectType", *objTypeString)); + } else { + dbgTrace(D_INTELLIGENCE) << objTypeString.getErr(); + } + if (cursor.ok()) ar(cereal::make_nvp("cursor", cursor.unpack().second)); requested_attributes.save(ar); query_types.save(ar); @@ -129,6 +157,12 @@ QueryRequest::setTenantsList(const vector tenants) query_types.setSerializableTenantList(tenants); } +void +QueryRequest::setCrossTenantAssetDB(bool cross_tenant_asset_db) +{ + query_types.setQueryCrossTenantAssetDB(cross_tenant_asset_db); +} + void QueryRequest::setAssetsLimit(uint _assets_limit) { @@ -173,6 +207,12 @@ QueryRequest::setCursor(CursorState state, const string &value) cursor = RequestCursor(state, value); } +void +QueryRequest::setObjectType(const ObjectType &obj_type) +{ + object_type = obj_type; +} + QueryRequest QueryRequest::calcQueryRequestOperator(const QueryRequest &other_query, const Operator &operator_type) { diff --git a/core/intelligence_is_v2/query_types_v2.cc b/core/intelligence_is_v2/query_types_v2.cc index c0d34ae..374863e 100644 --- a/core/intelligence_is_v2/query_types_v2.cc +++ b/core/intelligence_is_v2/query_types_v2.cc @@ -17,22 +17,37 @@ using namespace std; using namespace Intelligence_IS_V2; void -serializableTenantList::serialize(cereal::JSONOutputArchive &ar) const +SerializableQueryTypes::serializeMultiTenant(cereal::JSONOutputArchive &ar) const { - ar(cereal::make_nvp("multiTenant", tenants)); + ar(cereal::make_nvp("multiTenant", *tenants)); +} + +void +SerializableQueryTypes::serializeCrossTenantAssetDB(cereal::JSONOutputArchive &ar) const +{ + ar(cereal::make_nvp("queryCrossTenantAssetDB", *query_cross_tenant_asset_db)); } void SerializableQueryTypes::save(cereal::JSONOutputArchive &ar) const { - if (!is_nsaas) return; - serializableTenantList serializable_tenants(tenants); - ar(cereal::make_nvp("queryTypes", serializable_tenants)); + if (!tenants.ok() && !query_cross_tenant_asset_db.ok()) return; + + ar.setNextName("queryTypes"); + ar.startNode(); + if (tenants.ok()) serializeMultiTenant(ar); + if (query_cross_tenant_asset_db.ok()) serializeCrossTenantAssetDB(ar); + ar.finishNode(); } void -SerializableQueryTypes::setSerializableTenantList(const std::vector _tenants) +SerializableQueryTypes::setSerializableTenantList(const vector &tenant_list) { - tenants = _tenants; - is_nsaas = true; + tenants = tenant_list; }; + +void +SerializableQueryTypes::setQueryCrossTenantAssetDB(bool cross_tenant_asset_db) +{ + query_cross_tenant_asset_db = cross_tenant_asset_db; +} diff --git a/core/logging/log_streams.h b/core/logging/log_streams.h index d620dc0..0948604 100755 --- a/core/logging/log_streams.h +++ b/core/logging/log_streams.h @@ -103,6 +103,7 @@ public: void sendLog(const Report &log) override; private: + void sendLog(const std::vector &data); void connect(); I_Socket *i_socket = nullptr; diff --git a/core/logging/logging_ut/logging_ut.cc b/core/logging/logging_ut/logging_ut.cc index d28637d..5b7cb11 100755 --- a/core/logging/logging_ut/logging_ut.cc +++ b/core/logging/logging_ut/logging_ut.cc @@ -1489,7 +1489,7 @@ TEST_F(LogTest, ObfuscationTest) sysog_routine(); EXPECT_EQ(capture_syslog_cef_data.size(), 2); for (const string &str : capture_syslog_cef_data) { - EXPECT_THAT(str, AnyOf(HasSubstr("String='Another string'"), HasSubstr("String=Another string"))); + EXPECT_THAT(str, AnyOf(HasSubstr("String='Another string'"), HasSubstr("String=\"Another string\""))); } } diff --git a/core/logging/syslog_stream.cc b/core/logging/syslog_stream.cc index b452cef..4bcd8e9 100755 --- a/core/logging/syslog_stream.cc +++ b/core/logging/syslog_stream.cc @@ -54,31 +54,32 @@ SyslogStream::sendLog(const Report &log) vector data(syslog_report.begin(), syslog_report.end()); mainloop->addOneTimeRoutine( I_MainLoop::RoutineType::Offline, - [this, data] () - { - if (!socket.ok()) { - connect(); - if (!socket.ok()) { - dbgWarning(D_REPORT) << "Failed to connect to the syslog server, Log will not be sent."; - return; - } - dbgTrace(D_REPORT) << "Successfully connect to the syslog server"; - } - - int tries = 1; - for (; tries <=3; tries++) { - if (i_socket->writeData(socket.unpack(), data)) { - dbgTrace(D_REPORT) << "log was sent to syslog server"; - return; - } else { - dbgWarning(D_REPORT) << "Failed to send log to syslog server"; - } - } - }, + [this, data] () { sendLog(data); }, "Logging Syslog stream messaging" ); } +void +SyslogStream::sendLog(const vector &data) +{ + for (int tries = 0; tries < 3; ++tries) { + if (!socket.ok()) { + connect(); + if (!socket.ok()) { + dbgWarning(D_REPORT) << "Failed to connect to the syslog server, Log will not be sent."; + return; + } + dbgTrace(D_REPORT) << "Successfully connect to the syslog server"; + } + + if (i_socket->writeData(socket.unpack(), data)) { + dbgTrace(D_REPORT) << "log was sent to syslog server"; + return; + } + } + dbgWarning(D_REPORT) << "Failed to send log to syslog server"; +} + void SyslogStream::connect() { diff --git a/core/mainloop/mainloop.cc b/core/mainloop/mainloop.cc index b058665..78f7ac6 100644 --- a/core/mainloop/mainloop.cc +++ b/core/mainloop/mainloop.cc @@ -73,6 +73,8 @@ public: Maybe getCurrentRoutineId() const override; + void updateCurrentStress(bool is_busy) override; + void yield(bool force) override; void yield(chrono::microseconds time) override; void stopAll() override; @@ -118,7 +120,6 @@ public: private: void reportStartupEvent(); void stop(const RoutineMap::iterator &iter); - void updateCurrentStress(bool is_busy); uint32_t getCurrentTimeSlice(uint32_t current_stress); RoutineID getNextID(); diff --git a/core/message/http_decoder.cc b/core/message/http_decoder.cc index a194e03..a744a37 100755 --- a/core/message/http_decoder.cc +++ b/core/message/http_decoder.cc @@ -129,17 +129,7 @@ HTTPDecoder::handleBody() auto maybe_transfer_encoding = unpacked_headers.getHeaderVal("transfer-encoding"); if (maybe_transfer_encoding.ok()) { auto transfer_encoding_type = maybe_transfer_encoding.unpack(); - if (transfer_encoding_type == "chunked") { - if (Singleton::exists()) { - I_EnvDetails *env_details = Singleton::Consume::by(); - EnvType env_type = env_details->getEnvType(); - if (env_type == EnvType::K8S) { - dbgDebug(D_COMMUNICATION) << "Getting Chunked Response in a k8s env"; - return getChunkedResponseK8s(); - } - } - return getChunkedResponse(); - } + if (transfer_encoding_type == "chunked") return getChunkedResponse(); } auto connection_header = unpacked_headers.getHeaderVal("connection"); @@ -157,91 +147,43 @@ HTTPDecoder::getChunkedResponse() { if(!isLegalChunkedResponse(response)) return false; - stringstream ss(response); string line; + string res = response; string chunk_body = ""; - size_t chunk_length = 0; - while (getline(ss, line) && line != "\r") { - if (chunk_body.length() == chunk_length) { - body += chunk_body; - chunk_body = ""; - try { - chunk_length = stoi(line, nullptr, 16); - } catch (const exception& err) { - dbgDebug(D_COMMUNICATION) << "Failed to convert chunk length to a number. Line: " << line; - return false; - } - } else if (chunk_body.length() > chunk_length) { - dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure."; + string CRLF = "\r\n"; + size_t chunk_size = 0; + + for (auto end_of_line = res.find(CRLF); end_of_line != string::npos; end_of_line = res.find(CRLF)) { + line = res.substr(0, end_of_line); + try { + chunk_size = stoi(line, nullptr, 16); + } catch (const exception& err) { + dbgDebug(D_COMMUNICATION) << "Failed to convert chunk length to a number. Line: " << line; return false; - } else { - if (line.back() == '\r') { - line.pop_back(); - } - if (!chunk_body.empty()) { - chunk_body += '\n'; - } - chunk_body += line; } + + if (end_of_line + 2 + chunk_size > res.length()) { + dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure - chunk-size is bigger than chunk-data"; + return false; + } + chunk_body = res.substr(end_of_line + 2, chunk_size); + res = res.substr(end_of_line + 2 + chunk_size); + + if (res.find(CRLF) != 0) { + dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure - chunk-data missing final CRLF sequence"; + return false; + } + res = res.substr(2); + body += chunk_body; } - if (chunk_length != 0) { - dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure."; + if (chunk_size != 0) { + dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure - last-chunk of the body is not sized 0"; return false; } return true; } -// LCOV_EXCL_START Reason: Will be deleted in INXT-31454 -bool -HTTPDecoder::getChunkedResponseK8s() -{ - if(!isLegalChunkedResponse(response)) return false; - - stringstream ss(response); - string line; - string chunk_body = ""; - size_t chunk_length = 0; - while (getline(ss, line)) { - if(line == "\r"){ - continue; - } - if (chunk_body.length() == chunk_length) { - body += chunk_body; - chunk_body = ""; - try { - chunk_length = stoi(line, nullptr, 16); - } catch (const exception& err) { - dbgDebug(D_COMMUNICATION) << "Failed to convert chunk length to a number. Line: " << line; - return false; - } - } else if (chunk_body.length() > chunk_length) { - dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure."; - return false; - } else { - if (line.back() == '\r') { - line.pop_back(); - } - if (!chunk_body.empty()) { - chunk_body += '\n'; - chunk_length = chunk_length + 1; - } - chunk_body += line; - } - } - - if (chunk_length != 0) { - dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure."; - if (chunk_body.length() == chunk_length) { - body += chunk_body; - return true; - } - return false; - } - return true; -} -// LCOV_EXCL_STOP - bool HTTPDecoder::isLegalChunkedResponse(const string &res) { diff --git a/core/message/http_decoder.h b/core/message/http_decoder.h index 352c4c8..23c2bd3 100755 --- a/core/message/http_decoder.h +++ b/core/message/http_decoder.h @@ -39,7 +39,6 @@ private: bool handleBody(); bool getChunkedResponse(); - bool getChunkedResponseK8s(); bool isLegalChunkedResponse(const std::string &res); I_Messaging::Method method; diff --git a/core/message/message.cc b/core/message/message.cc index a5e722c..278bb67 100755 --- a/core/message/message.cc +++ b/core/message/message.cc @@ -513,17 +513,22 @@ public: http_status_code == HTTPStatusCode::HTTP_BAD_REQUEST; }; pending_signatures.insert(req_sig); - auto res = sendMessage(get_reply, body, method, url, headers, fog_server_err, should_yield, tag); - pending_signatures.erase(req_sig); - if (res.ok()) return res; + try { + auto res = sendMessage(get_reply, body, method, url, headers, fog_server_err, should_yield, tag); + pending_signatures.erase(req_sig); + if (res.ok()) return res; - bool should_buffer_default = getProfileAgentSettingWithDefault( - true, - "eventBuffer.bufferFailedRequests" - ); - if (!getConfigurationWithDefault(should_buffer_default, "message", "Buffer Failed Requests")) { - dbgWarning(D_COMMUNICATION) << "Failed to send Request."; - return res; + bool should_buffer_default = getProfileAgentSettingWithDefault( + true, + "eventBuffer.bufferFailedRequests" + ); + if (!getConfigurationWithDefault(should_buffer_default, "message", "Buffer Failed Requests")) { + dbgWarning(D_COMMUNICATION) << "Failed to send Request."; + return res; + } + } catch (...) { + dbgWarning(D_COMMUNICATION) << "Can't send a persistent message, mainloop has been stopped"; + return genError("mainloop has been stopped"); } dbgWarning(D_COMMUNICATION) << "Failed to send Request. Buffering the request."; } diff --git a/core/report/report.cc b/core/report/report.cc index a522f44..c41e654 100644 --- a/core/report/report.cc +++ b/core/report/report.cc @@ -149,8 +149,8 @@ Report::getSyslog() const } time_stamp += "Z"; - string origin_syslog = origin.getSyslog(); - string event_data_syslog = event_data.getSyslog(); + string origin_syslog = origin.getSyslogAndCef(); + string event_data_syslog = event_data.getSyslogAndCef(); string agent_id = "cpnano-agent-" + Singleton::Consume::by()->getAgentId(); auto service_name = Singleton::Consume::by()->get("Service Name"); @@ -189,6 +189,12 @@ Report::getCef() const CefReport report; auto service_name = Singleton::Consume::by()->get("Service Name"); + auto i_time = Singleton::Consume::by(); + string time_stamp = i_time->getWalltimeStr(time); + if (time_stamp.size() > 7 && time_stamp[time_stamp.size() - 7] == '.') { + time_stamp.erase(time_stamp.size() - 3); // downscale micro-sec resollution to milli-sec + } + if (service_name.ok()) { string tmp = service_name.unpack(); tmp.erase(remove(tmp.begin(), tmp.end(), ' '), tmp.end()); @@ -205,9 +211,10 @@ Report::getCef() const report.pushMandatory(title); report.pushMandatory(TagAndEnumManagement::convertToString(priority)); - string origin_cef = origin.getCef(); - string event_data_cef = event_data.getCef(); + string origin_cef = origin.getSyslogAndCef(); + string event_data_cef = event_data.getSyslogAndCef(); + report.pushExtension("eventTime=" + time_stamp); if (!origin_cef.empty()) { report.pushExtension(origin_cef); } diff --git a/core/report/report_ut/report_ut.cc b/core/report/report_ut/report_ut.cc index 86b3dd8..683aab0 100644 --- a/core/report/report_ut/report_ut.cc +++ b/core/report/report_ut/report_ut.cc @@ -570,8 +570,8 @@ TEST_F(ReportTest, testSyslogWithoutServiceName) EXPECT_EQ( report.getSyslog(), "<133>1 0:0:0.123Z cpnano-agent-001 UnnamedNanoService - 0 - " - "title='Log Test' agent='Secret' eventTraceId='' eventSpanId='' " - "issuingEngineVersion='' serviceName='Unnamed Nano Service' serviceId='' serviceFamilyId=''" + "title='Log Test' agent=\"Secret\" eventTraceId=\"\" eventSpanId=\"\" " + "issuingEngineVersion=\"\" serviceName=\"Unnamed Nano Service\" serviceId=\"\" serviceFamilyId=\"\"" ); } @@ -604,13 +604,17 @@ TEST_F(ReportTest, testSyslog) vector> f1 = { { "a", "b"}, {"1", "2"} }; report << LogField("ArrayOfArraies", f1); + report << LogField("DataWithNewLine", "new\r\nline"); + report << LogField("DataWithQuote", "data'bla"); string result = string("<133>1 0:0:0.123Z cpnano-agent-001 AccessControlApp - 1 - " - "title='Log Test' agent='Secret'") + - " eventTraceId='' eventSpanId='' issuingEngineVersion=''" + - " serviceName='Access Control App' serviceId='' serviceFamilyId=''" + - string(" ArrayOfArraies='[ [ a, b ], [ 1, 2 ] ]'"); + "title='Log Test' agent=\"Secret\"") + + " eventTraceId=\"\" eventSpanId=\"\" issuingEngineVersion=\"\"" + + " serviceName=\"Access Control App\" serviceId=\"\" serviceFamilyId=\"\"" + + string(" ArrayOfArraies=\"[ [ a, b \\], [ 1, 2 \\] \\]\"") + + string(" DataWithNewLine=\"new\\r\\nline\"") + + string(" DataWithQuote=\"data\\'bla\""); EXPECT_EQ(report.getSyslog(), result); } @@ -643,11 +647,14 @@ TEST_F(ReportTest, testCef) ); report.addToOrigin(another_origin); + report << LogField("DataWithQuote", "data'bla"); + EXPECT_EQ( report.getCef(), "CEF:0|Check Point|AccessControlApp||Event Driven|Log Test|Low|" - "agent=Secret eventTraceId= eventSpanId= issuingEngineVersion=" - " serviceName=Access Control App serviceId= serviceFamilyId= Bond=1" + "eventTime=0:0:0.123 agent=\"Secret\" eventTraceId=\"\" eventSpanId=\"\" issuingEngineVersion=\"\"" + " serviceName=\"Access Control App\" serviceId=\"\"" + " serviceFamilyId=\"\" Bond=\"1\" DataWithQuote=\"data\\'bla\"" ); } diff --git a/core/report/tag_and_enum_management.cc b/core/report/tag_and_enum_management.cc index dcf3190..81badab 100755 --- a/core/report/tag_and_enum_management.cc +++ b/core/report/tag_and_enum_management.cc @@ -103,7 +103,8 @@ TagAndEnumManagement::convertStringToTag(const string &tag) {"NGINX Server", ReportIS::Tags::WEB_SERVER_NGINX}, {"Kong Server", ReportIS::Tags::WEB_SERVER_KONG}, {"Embedded Deployment", ReportIS::Tags::DEPLOYMENT_EMBEDDED}, - {"Kubernetes Deployment", ReportIS::Tags::DEPLOYMENT_K8S} + {"Kubernetes Deployment", ReportIS::Tags::DEPLOYMENT_K8S}, + {"Layer 7 Access Control", ReportIS::Tags::LAYER_7_ACCESS_CONTROL} }; auto report_is_tag = strings_to_tags.find(tag); @@ -300,7 +301,8 @@ EnumArray TagAndEnumManagement::tags_translation_arr { "NGINX Server", "Kong Server", "Embedded Deployment", - "Kubernetes Deployment" + "Kubernetes Deployment", + "Layer 7 Access Control" }; EnumArray TagAndEnumManagement::audience_team_translation { diff --git a/external/graphqlparser/CMakeLists.txt b/external/graphqlparser/CMakeLists.txt index 2f030cb..ee9a958 100644 --- a/external/graphqlparser/CMakeLists.txt +++ b/external/graphqlparser/CMakeLists.txt @@ -7,9 +7,9 @@ INCLUDE(version) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++11") -FIND_PACKAGE(PythonInterp 2 REQUIRED) -IF (NOT PYTHON_VERSION_MAJOR EQUAL 2) - MESSAGE(FATAL_ERROR "Python 2 is required.") +FIND_PACKAGE(PythonInterp 3 REQUIRED) +IF (NOT PYTHON_VERSION_MAJOR EQUAL 3) + MESSAGE(FATAL_ERROR "Python 3 is required.") ENDIF() FIND_PROGRAM(CTYPESGEN_FOUND ctypesgen.py) diff --git a/external/graphqlparser/ast/c.py b/external/graphqlparser/ast/c.py index 3a60ebf..0bf8492 100644 --- a/external/graphqlparser/ast/c.py +++ b/external/graphqlparser/ast/c.py @@ -60,7 +60,7 @@ class Printer(object): self._current_type = None def start_file(self): - print C_LICENSE_COMMENT + '''/** @generated */ + print(C_LICENSE_COMMENT + '''/** @generated */ #pragma once @@ -68,30 +68,30 @@ class Printer(object): extern "C" { #endif -''' +''') def end_file(self): - print ''' + print(''' #ifdef __cplusplus } #endif -''' +''') def start_type(self, name): # Forward declarations for AST nodes. st_name = struct_name(name) - print 'struct ' + st_name + ';' + print('struct ' + st_name + ';') self._current_type = name def field(self, type, name, nullable, plural): - print field_prototype(self._current_type, type, name, nullable, plural) + ';' + print(field_prototype(self._current_type, type, name, nullable, plural) + ';') def end_type(self, name): - print + print() def start_union(self, name): - print 'struct ' + struct_name(name) + ';' + print('struct ' + struct_name(name) + ';') def union_option(self, option): pass diff --git a/external/graphqlparser/ast/c_impl.py b/external/graphqlparser/ast/c_impl.py index f82299d..cb6a581 100644 --- a/external/graphqlparser/ast/c_impl.py +++ b/external/graphqlparser/ast/c_impl.py @@ -15,13 +15,13 @@ class Printer(object): self._current_type = None def start_file(self): - print C_LICENSE_COMMENT + '''/** @generated */ + print(C_LICENSE_COMMENT + '''/** @generated */ #include "GraphQLAst.h" #include "../Ast.h" using namespace facebook::graphql::ast; // NOLINT -''' +''') def end_file(self): pass @@ -30,23 +30,23 @@ using namespace facebook::graphql::ast; // NOLINT self._current_type = name def field(self, type, name, nullable, plural): - print field_prototype(self._current_type, type, name, nullable, plural) + ' {' - print ' const auto *realNode = reinterpret_cast(node);' % self._current_type + print(field_prototype(self._current_type, type, name, nullable, plural) + ' {') + print(' const auto *realNode = reinterpret_cast(node);' % self._current_type) title_name = title(name) call_get = 'realNode->get%s()' % title_name if plural: if nullable: - print ' return %s ? %s->size() : 0;' % (call_get, call_get) + print(' return %s ? %s->size() : 0;' % (call_get, call_get)) else: - print ' return %s.size();' % call_get + print(' return %s.size();' % call_get) else: if type in ['string', 'OperationKind', 'boolean']: - print ' return %s;' % call_get + print(' return %s;' % call_get) else: fmt = ' return reinterpret_cast(%s%s);' - print fmt % (struct_name(type), '' if nullable else '&', call_get) + print(fmt % (struct_name(type), '' if nullable else '&', call_get)) - print '}' + print('}') def end_type(self, name): pass diff --git a/external/graphqlparser/ast/c_visitor_impl.py b/external/graphqlparser/ast/c_visitor_impl.py index cab7062..17b9f8d 100644 --- a/external/graphqlparser/ast/c_visitor_impl.py +++ b/external/graphqlparser/ast/c_visitor_impl.py @@ -14,8 +14,8 @@ class Printer(object): self._types = [] def start_file(self): - print C_LICENSE_COMMENT + '/** @generated */' - print '#define FOR_EACH_CONCRETE_TYPE(MACRO) \\' + print(C_LICENSE_COMMENT + '/** @generated */') + print('#define FOR_EACH_CONCRETE_TYPE(MACRO) \\') def start_type(self, name): self._types.append(name) @@ -27,7 +27,7 @@ class Printer(object): pass def end_file(self): - print ' \\\n'.join('MACRO(%s, %s)' % (name, snake(name)) for name in self._types) + print(' \\\n'.join('MACRO(%s, %s)' % (name, snake(name)) for name in self._types)) def start_union(self, name): pass diff --git a/external/graphqlparser/ast/cxx.py b/external/graphqlparser/ast/cxx.py index 3706cad..b27063d 100644 --- a/external/graphqlparser/ast/cxx.py +++ b/external/graphqlparser/ast/cxx.py @@ -3,7 +3,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -import cStringIO as StringIO +from io import StringIO from casing import title from license import C_LICENSE_COMMENT @@ -16,12 +16,12 @@ class Printer(object): # HACK: Defer everything we print so that forward declarations for # all classes come first. Avoids having to do 2 passes over the # input file. - self._deferredOutput = StringIO.StringIO() + self._deferredOutput = StringIO() self._fields = [] def start_file(self): - print C_LICENSE_COMMENT + '''/** @generated */ + print(C_LICENSE_COMMENT + '''/** @generated */ #pragma once #include "AstNode.h" @@ -40,14 +40,14 @@ namespace ast { struct CDeleter { void operator()(const char *p) const { free((void *)p); } }; -''' +''') def end_file(self): - print - print self._deferredOutput.getvalue() - print '}' - print '}' - print '}' + print() + print(self._deferredOutput.getvalue()) + print('}') + print('}') + print('}') def _base_class(self, type): return self._bases.get(type, 'Node') @@ -56,8 +56,8 @@ struct CDeleter { self._type_name = name base = self._base_class(name) # non-deferred! - print 'class %s;' % name - print >> self._deferredOutput, 'class %s : public %s {' % (name, base) + print('class %s;' % name) + print('class %s : public %s {' % (name, base), file=self._deferredOutput) self._fields = [] def field(self, type, name, nullable, plural): @@ -67,18 +67,18 @@ struct CDeleter { def end_type(self, name): self._print_fields() - print >> self._deferredOutput, ' public:' + print(' public:', file=self._deferredOutput) self._print_constructor() - print >> self._deferredOutput + print('', file=self._deferredOutput) self._print_destructor_prototype() - print >> self._deferredOutput + print('', file=self._deferredOutput) self._print_noncopyable() - print >> self._deferredOutput + print('', file=self._deferredOutput) self._print_getters() - print >> self._deferredOutput, ' void accept(visitor::AstVisitor *visitor) const override;' - print >> self._deferredOutput, '};' - print >> self._deferredOutput - print >> self._deferredOutput + print(' void accept(visitor::AstVisitor *visitor) const override;', file=self._deferredOutput) + print('};', file=self._deferredOutput) + print('', file=self._deferredOutput) + print('', file=self._deferredOutput) self._type_name = None self._fields = [] @@ -95,7 +95,7 @@ struct CDeleter { storage_type = self._storage_type(type) if plural: storage_type = 'std::unique_ptr>' % storage_type - print >> self._deferredOutput, ' %s %s_;' % (storage_type, name) + print(' %s %s_;' % (storage_type, name), file=self._deferredOutput) def _ctor_singular_type(self, type): if type == 'string': @@ -109,28 +109,28 @@ struct CDeleter { return 'std::vector<%s> *' % self._storage_type(type) def _print_constructor(self): - print >> self._deferredOutput, ' explicit %s(' % self._type_name - print >> self._deferredOutput, ' const yy::location &location%s' % (',' if self._fields else '') + print(' explicit %s(' % self._type_name, file=self._deferredOutput) + print(' const yy::location &location%s' % (',' if self._fields else ''), file=self._deferredOutput) def ctor_arg(type, name, plural): if plural: ctor_type = self._ctor_plural_type(type) else: ctor_type = self._ctor_singular_type(type) return ' %s %s' % (ctor_type, name) - print >> self._deferredOutput, ',\n'.join(ctor_arg(type, name, plural) - for (type, name, nullable, plural) in self._fields) - print >> self._deferredOutput, ' )' + print(',\n'.join(ctor_arg(type, name, plural) + for (type, name, nullable, plural) in self._fields), file=self._deferredOutput) + print(' )', file=self._deferredOutput) def ctor_init(type, name, plural): # Strings are const char *, just pass. # Vectors are passed by pointer and we take ownership. # Node types are passed in by pointer and we take ownership. value = name return ' %s_(%s)' % (name, value) - print >> self._deferredOutput, ' : %s(location)%s' % (self._base_class(self._type_name), ',' if self._fields else '') - print >> self._deferredOutput, ',\n'.join(ctor_init(type, name, plural) + print(' : %s(location)%s' % (self._base_class(self._type_name), ',' if self._fields else ''), file=self._deferredOutput) + print(',\n'.join(ctor_init(type, name, plural) for (type, name, nullable, plural) - in self._fields) - print >> self._deferredOutput, ' {}' + in self._fields), file=self._deferredOutput) + print(' {}', file=self._deferredOutput) def _getter_type(self, type, nullable, plural): if plural and nullable: @@ -163,31 +163,31 @@ struct CDeleter { def _print_getters(self): for (type, name, nullable, plural) in self._fields: - print >> self._deferredOutput, ' %s get%s() const' % ( + print(' %s get%s() const' % ( self._getter_type(type, nullable, plural), - title(name)) - print >> self._deferredOutput, ' { return %s; }' % ( - self._getter_value_to_return(name + '_', type, nullable, plural)) - print >> self._deferredOutput + title(name)), file=self._deferredOutput) + print(' { return %s; }' % ( + self._getter_value_to_return(name + '_', type, nullable, plural)), file=self._deferredOutput) + print('', file=self._deferredOutput) def _print_destructor_prototype(self): - print >> self._deferredOutput, ' ~%s() {}' % self._type_name + print(' ~%s() {}' % self._type_name, file=self._deferredOutput) def _print_noncopyable(self): - print >> self._deferredOutput, ' %s(const %s&) = delete;' % ( - self._type_name, self._type_name) - print >> self._deferredOutput, ' %s& operator=(const %s&) = delete;' % ( - self._type_name, self._type_name) + print(' %s(const %s&) = delete;' % ( + self._type_name, self._type_name), file=self._deferredOutput) + print(' %s& operator=(const %s&) = delete;' % ( + self._type_name, self._type_name), file=self._deferredOutput) def start_union(self, name): self._type_name = name # non-deferred! - print 'class %s;' % name - print >> self._deferredOutput, 'class %s : public Node {' % name - print >> self._deferredOutput, ' public:' + print('class %s;' % name) + print('class %s : public Node {' % name, file=self._deferredOutput) + print(' public:', file=self._deferredOutput) self._print_constructor() - print >> self._deferredOutput, '};' - print >> self._deferredOutput + print('};', file=self._deferredOutput) + print('', file=self._deferredOutput) def union_option(self, type): assert type not in self._bases, '%s cannot appear in more than one union!' % type diff --git a/external/graphqlparser/ast/cxx_impl.py b/external/graphqlparser/ast/cxx_impl.py index ab161c6..7c36887 100644 --- a/external/graphqlparser/ast/cxx_impl.py +++ b/external/graphqlparser/ast/cxx_impl.py @@ -10,7 +10,7 @@ class Printer(object): pass def start_file(self): - print C_LICENSE_COMMENT + '''/** @generated */ + print(C_LICENSE_COMMENT + '''/** @generated */ #include "Ast.h" #include "AstVisitor.h" @@ -18,17 +18,17 @@ class Printer(object): namespace facebook { namespace graphql { namespace ast { -''' +''') def end_file(self): - print '} // namespace ast' - print '} // namespace graphql' - print '} // namespace facebook' + print('} // namespace ast') + print('} // namespace graphql') + print('} // namespace facebook') def start_type(self, name): - print '''void %s::accept(visitor::AstVisitor *visitor) const { + print('''void %s::accept(visitor::AstVisitor *visitor) const { if (visitor->visit%s(*this)) { -''' % (name, name) +''' % (name, name)) def field(self, type, name, nullable, plural): if type in ['OperationKind', 'string', 'boolean']: @@ -38,18 +38,18 @@ namespace ast { accept = '{ for (const auto &x : *%s_) { x->accept(visitor); } }' % name if nullable: accept = 'if (%s_) %s' % (name, accept) - print ' ' + accept + print(' ' + accept) else: accept = '%s_->accept(visitor);' % name if nullable: accept = 'if (%s_) { %s }' % (name, accept) - print ' ' + accept + print(' ' + accept) def end_type(self, name): - print ''' } + print(''' } visitor->endVisit%s(*this); } -''' % name +''' % name) def start_union(self, name): pass diff --git a/external/graphqlparser/ast/cxx_json_visitor_header.py b/external/graphqlparser/ast/cxx_json_visitor_header.py index fca744f..bd8a0bb 100644 --- a/external/graphqlparser/ast/cxx_json_visitor_header.py +++ b/external/graphqlparser/ast/cxx_json_visitor_header.py @@ -12,7 +12,7 @@ class Printer(object): self._anyFieldIsANode = False def start_file(self): - print C_LICENSE_COMMENT + '/** @generated */' + print(C_LICENSE_COMMENT + '/** @generated */') def end_file(self): pass @@ -23,9 +23,9 @@ class Printer(object): def end_type(self, name): titleName = title(name) if self._anyFieldIsANode: - print 'bool visit%s(const %s &node) override;' % (titleName, titleName) - print 'void endVisit%s(const %s &node) override;' % (titleName, titleName) - print + print('bool visit%s(const %s &node) override;' % (titleName, titleName)) + print('void endVisit%s(const %s &node) override;' % (titleName, titleName)) + print() def field(self, type, name, nullable, plural): if (not self._anyFieldIsANode and diff --git a/external/graphqlparser/ast/cxx_json_visitor_impl.py b/external/graphqlparser/ast/cxx_json_visitor_impl.py index 0df2bac..64c37ca 100644 --- a/external/graphqlparser/ast/cxx_json_visitor_impl.py +++ b/external/graphqlparser/ast/cxx_json_visitor_impl.py @@ -12,7 +12,7 @@ class Printer(object): self._fields = [] def start_file(self): - print C_LICENSE_COMMENT + '/** @generated */' + print(C_LICENSE_COMMENT + '/** @generated */') def end_file(self): pass @@ -30,13 +30,13 @@ class Printer(object): anyFieldIsANode = any(type not in ('string, boolean') for (type, _, _ ,_) in self._fields) if anyFieldIsANode: - print '''bool JsonVisitor::visit%s(const %s &node) { + print('''bool JsonVisitor::visit%s(const %s &node) { visitNode(); return true; } -''' % (titleName, titleName) - print '''void JsonVisitor::endVisit%(tn)s(const %(tn)s &node) { - NodeFieldPrinter fields(*this, "%(tn)s", node);''' % {'tn': titleName} +''' % (titleName, titleName)) + print('''void JsonVisitor::endVisit%(tn)s(const %(tn)s &node) { + NodeFieldPrinter fields(*this, "%(tn)s", node);''' % {'tn': titleName}) for (type, fieldName, nullable, plural) in self._fields: funcName = None @@ -48,7 +48,7 @@ class Printer(object): funcName = 'printSingularBooleanField' elif not nullable and not plural: # Special case: singular object fields don't need the value passed. - print ' fields.printSingularObjectField("%s");' % fieldName + print(' fields.printSingularObjectField("%s");' % fieldName) continue else: nullable_str = 'Nullable' if nullable else '' @@ -56,19 +56,19 @@ class Printer(object): funcName = 'print%s%sField' % (nullable_str, plural_str) assert funcName is not None - print ' fields.%s("%s", node.get%s());' % ( - funcName, fieldName, title(fieldName)) + print(' fields.%s("%s", node.get%s());' % ( + funcName, fieldName, title(fieldName))) if anyFieldIsANode: - print ''' + print(''' endVisitNode(fields.finishPrinting()); } -''' +''') else: - print ''' + print(''' printed_.back().emplace_back(fields.finishPrinting()); } -''' +''') def start_union(self, name): pass diff --git a/external/graphqlparser/ast/cxx_visitor.py b/external/graphqlparser/ast/cxx_visitor.py index f1b7029..12c5470 100644 --- a/external/graphqlparser/ast/cxx_visitor.py +++ b/external/graphqlparser/ast/cxx_visitor.py @@ -11,7 +11,7 @@ class Printer(object): pass def start_file(self): - print C_LICENSE_COMMENT + '''/** @generated */ + print(C_LICENSE_COMMENT + '''/** @generated */ #pragma once @@ -25,28 +25,28 @@ namespace visitor { class AstVisitor { public: virtual ~AstVisitor() {} -''' +''') def end_file(self): - print '};' # end AstVisitor - print - print '}' - print '}' - print '}' - print '}' + print('};') # end AstVisitor + print() + print('}') + print('}') + print('}') + print('}') def start_type(self, name): titleName = title(name) camelName = camel(titleName) - print ' virtual bool visit%s(const %s &%s) { return true; }' % ( + print(' virtual bool visit%s(const %s &%s) { return true; }' % ( titleName, titleName, - camelName) - print ' virtual void endVisit%s(const %s &%s) { }' % ( + camelName)) + print(' virtual void endVisit%s(const %s &%s) { }' % ( titleName, titleName, - camelName) - print + camelName)) + print() def end_type(self, name): pass diff --git a/nodes/orchestration/package/open-appsec-ctl.sh b/nodes/orchestration/package/open-appsec-ctl.sh index b1b8c1c..0e20fa3 100644 --- a/nodes/orchestration/package/open-appsec-ctl.sh +++ b/nodes/orchestration/package/open-appsec-ctl.sh @@ -48,6 +48,8 @@ NO_COLOR='\033[0m' pidof_cmd="pidof -x" is_alpine_release= +var_last_policy_modification_time=0 + ls -l /etc/ | grep release > /dev/null 2>&1 retval=$? @@ -120,6 +122,14 @@ load_paths() load_paths +AGENT_POLICY_PATH="${FILESYSTEM_PATH}/${cp_nano_conf_location}/policy.json" +CUSTOM_POLICY_CONF_FILE="${FILESYSTEM_PATH}/${cp_nano_conf_location}/custom_policy.cfg" +if [ -f ${CUSTOM_POLICY_CONF_FILE} ]; then + . $CUSTOM_POLICY_CONF_FILE +else + var_policy_file="${FILESYSTEM_PATH}/${cp_nano_conf_location}/local_policy.yaml" +fi + is_arm32= if [ -n "$(uname -a | grep armv7l)" ]; then pidof_cmd="pidof" @@ -144,7 +154,7 @@ lines_to_skip=$((PACKAGE_LIST_LINE_OFFSET)) } <"${FILESYSTEM_PATH}/${CP_SCRIPTS_PATH}/${CP_NANO_PACKAGE_LIST_NAME}" is_valid_var_name() # Initials - ivvn -{ +{ ivvn_var_name=$1 # Check that string $ivvn_var_name is a valid variable name # [[:alnum:]] - Alphanumeric [a-z A-Z 0-9] @@ -949,14 +959,14 @@ run_status() # Initials - rs rs_temp_old_status=$(echo "$rs_orch_status" | sed -r "${rs_line_count},${rs_line_count}d; "' 1,1d; s/^\s*//g; s/^\n//g; s/\"//g; s/\\n/\n/g; s/\,//g') else rs_temp_old_status=$(sed 's/{//g' <${FILESYSTEM_PATH}/$cp_nano_conf_location/orchestration_status.json | sed 's/}//g' | sed 's/"//g' | sed 's/,//g' | sed -r '/^\s*$/d' | sed -r 's/^ //g') - rs_policy_load_time="$(cat /etc/cp/conf/orchestration_status.json | grep "Last policy update" | sed "s|\"||g" | sed "s|,||g")" + rs_policy_load_time="$(cat ${FILESYSTEM_PATH}/conf/orchestration_status.json | grep "Last policy update" | sed "s|\"||g" | sed "s|,||g")" fi - if [ -n "$(cat /etc/cp/conf/agent_details.json | grep "hybrid_mode")" ]; then + if [ -n "$(cat ${FILESYSTEM_PATH}/conf/agent_details.json | grep "hybrid_mode")" ]; then add_policy_file=true rs_mgmt_mode_text="Local management" else - if [ -n "$(cat /etc/cp/conf/settings.json | grep "\"profileManagedMode\":\"management\"")" ]; then + if [ -n "$(cat ${FILESYSTEM_PATH}/conf/settings.json | grep "\"profileManagedMode\":\"management\"")" ]; then add_policy_file=false rs_mgmt_mode_text="Cloud management (Fully managed)" else @@ -968,9 +978,9 @@ run_status() # Initials - rs if [ "${add_policy_file}" = "true" ]; then echo "Policy files: " - echo " /etc/cp/conf/local_policy.yaml" + echo " ${var_policy_file}" else - policy=`cat /etc/cp/conf/policy.json` + policy=`cat ${AGENT_POLICY_PATH}` version="version" policy_version=${policy#*version} policy_version=`echo $policy_version | cut -d"\"" -f3` @@ -1475,7 +1485,7 @@ set_mode() rm ${FILESYSTEM_PATH}/${cp_nano_conf_location}/agent_details.json rm ${FILESYSTEM_PATH}/${cp_nano_conf_location}/orchestration_status.json - echo '{}'>${FILESYSTEM_PATH}/${cp_nano_conf_location}/policy.json + echo '{}'>${AGENT_POLICY_PATH} if [ -f ${FILESYSTEM_PATH}/data/data5.a ]; then rm ${FILESYSTEM_PATH}/data/data5.a @@ -1588,9 +1598,31 @@ stop_service() # Initials - stops record_command() # Initials - rc { - touch /var/log/nano_agent/operations.log - echo "$(tail -99 /var/log/nano_agent/operations.log)" > /var/log/nano_agent/operations.log - echo $(date "+%Y.%m.%d-%H.%M.%S") ": " $0 $@ >> /var/log/nano_agent/operations.log + touch ${LOG_FILE_PATH}/nano_agent/operations.log + echo "$(tail -99 ${LOG_FILE_PATH}/nano_agent/operations.log)" > ${LOG_FILE_PATH}/nano_agent/operations.log + echo $(date "+%Y.%m.%d-%H.%M.%S") ": " $0 $@ >> ${LOG_FILE_PATH}/nano_agent/operations.log +} + +is_apply_policy_needed() +{ + if [ "${var_policy_file}" != "${var_new_policy_file}" ]; then + var_policy_file=$var_new_policy_file + return 0 + fi + local_policy_modification_time=$(stat -c %Y ${var_policy_file}) + if [ "${local_policy_modification_time}" -eq "${last_local_policy_modification_time}" ] || [ -z ${last_local_policy_modification_time} ]; then + return 1 + fi + return 0 +} + +is_policy_file_changed() +{ + new_modification_time=$(stat -c %Y ${AGENT_POLICY_PATH}) + if [ "${new_modification_time}" -gt "${var_last_policy_modification_time}" ]; then + return 1 + fi + return 0 } run() # Initials - r @@ -1680,35 +1712,63 @@ run() # Initials - r elif [ "-vp" = "$1" ] || [ "--view-policy" = "$1" ]; then record_command $@ shift - var_policy_file=$1 - if [ -z ${var_policy_file} ]; then - var_policy_file="/etc/cp/conf/local_policy.yaml" + if [ ! -z $1 ]; then + var_policy_file=$1 fi less ${var_policy_file} elif [ "-ep" = "$1" ] || [ "--edit-policy" = "$1" ]; then record_command $@ shift - var_policy_file=$1 - if [ -z ${var_policy_file} ]; then - var_policy_file="/etc/cp/conf/local_policy.yaml" + if [ ! -z $1 ]; then + var_policy_file=$1 fi vi ${var_policy_file} elif [ "-ap" = "$1" ] || [ "--apply-policy" = "$1" ]; then record_command $@ - curl_apply_policy=$(${curl_cmd} -S -w "%{http_code}\n" -m 1 --noproxy "*" --header "Content-Type: application/json" \ - --request POST --data {} http://127.0.0.1:"$(extract_api_port 'orchestration')"/set-apply-policy 2>&1) - while [ /etc/cp/conf/local_policy.yaml -nt /etc/cp/conf/policy.json ]; do + shift + if [ ! -z $1 ]; then + if [ "-d" = "$1" ] || [ "--default-policy" = "$1" ]; then + var_new_policy_file="${FILESYSTEM_PATH}/${cp_nano_conf_location}/local_policy.yaml" + elif [ -f $1 ]; then + var_new_policy_file=$1 + else + echo "Invalid policy path: $1" + exit 1 + fi + else + var_new_policy_file="${FILESYSTEM_PATH}/${cp_nano_conf_location}/local_policy.yaml" + fi + + is_apply_policy_needed + if [ $? -eq 1 ]; then + echo "Policy didn't changed. Policy path: ${var_policy_file}" + exit 0 + fi + echo "Applying new policy. Policy path: ${var_policy_file}" + var_last_policy_modification_time=$(stat -c %Y ${AGENT_POLICY_PATH}) + curl_apply_policy=$(${curl_cmd} -S -w "%{http_code}\n" -m 1 --noproxy "*" \ + --header "Content-Type: application/json" --request POST --data '{"policy_path":"'"${var_policy_file}"'"}' \ + http://127.0.0.1:"$(extract_api_port 'orchestration')"/set-apply-policy 2>&1) + is_policy_file_changed + is_changed=$? + while [ ${is_changed} -eq 0 ]; do echo -n "." sleep 3 + is_policy_file_changed + is_changed=$? done + + var_last_policy_modification_time=$(stat -c %Y ${AGENT_POLICY_PATH}) + echo "var_policy_file=${var_policy_file}" > ${CUSTOM_POLICY_CONF_FILE} + echo "last_local_policy_modification_time=$(stat -c %Y ${var_policy_file})" >> ${CUSTOM_POLICY_CONF_FILE} echo "New policy applied." exit 1 elif [ "-lp" = "$1" ] || [ "--list-policies" = "$1" ]; then record_command $@ - echo "/etc/cp/conf/local_policy.yaml" + echo $var_policy_file elif [ "-vl" = "$1" ] || [ "--view-logs" = "$1" ]; then record_command $@ - less /var/log/nano_agent/cp-nano-http-transaction-handler.log? + less $LOG_FILE_PATH/nano_agent/cp-nano-http-transaction-handler.log? else usage fi @@ -1718,4 +1778,3 @@ load_paths run "${@}" exit 0 - diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index 9e4e7f1..980fb9d 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -179,6 +179,14 @@ verify_proxy_config() fi } +save_local_policy_config() +{ + custom_policy_conf_file=${FILESYSTEM_PATH}/${CONF_PATH}/custom_policy.cfg + var_policy_file=${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml + echo "var_policy_file=${var_policy_file}" > ${custom_policy_conf_file} + echo "last_local_policy_modification_time=$(stat -c %Y ${var_policy_file})" >> ${custom_policy_conf_file} +} + [ -f /etc/environment ] && . "/etc/environment" if [ -n "${CP_ENV_FILESYSTEM}" ] ; then FILESYSTEM_PATH=$CP_ENV_FILESYSTEM @@ -632,6 +640,10 @@ upgrade_conf_if_needed() var_orchestration_mode=${previous_mode} fi + if [ ${var_orchestration_mode} = "hybrid_mode" ]; then + save_local_policy_config + fi + cp_exec "cp -f configuration/orchestration.cfg ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" execution_flags="execution_flags=\"--orchestration-mode=${var_orchestration_mode}\"" echo $execution_flags >> ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg @@ -667,7 +679,11 @@ copy_orchestration_executable() cp_copy open-appsec-cloud-mgmt-k8s ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/open-appsec-cloud-mgmt-k8s cp_copy open-appsec-ctl.sh ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/open-appsec-ctl.sh if [ $var_hybrid_mode = true ]; then - cp_copy local-default-policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml + if [ -f /ext/appsec/local_policy.yaml ]; then + cp_exec "ln -s /ext/appsec/local_policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml" + else + cp_copy local-default-policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml + fi fi } @@ -761,6 +777,10 @@ install_orchestration() fi fi + if [ -f "$FILESYSTEM_PATH/$CONF_PATH/custom_policy.cfg" ]; then + cp_exec "rm -f $FILESYSTEM_PATH/$CONF_PATH/custom_policy.cfg" + fi + if command -v ldconfig &>/dev/null; then cp_exec "ldconfig" ${FORCE_STDOUT} fi @@ -793,6 +813,11 @@ install_orchestration() if ! [ -z "$previous_mode" ]; then var_orchestration_mode=${previous_mode} fi + + if [ ${var_orchestration_mode} = "hybrid_mode" ]; then + save_local_policy_config + fi + cp_exec "cp -f configuration/orchestration.cfg ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" execution_flags="execution_flags=\"--orchestration-mode=${var_orchestration_mode}\"" echo $execution_flags >> ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg @@ -920,6 +945,8 @@ install_orchestration() elif [ $var_hybrid_mode = true ]; then cp_print "Run Orchestration nano service in hybrid mode" ${FORCE_STDOUT} cp_copy certificate/ngen.body.crt ${FILESYSTEM_PATH}/${CERTS_PATH}/fog.pem + + save_local_policy_config else cp_copy certificate/ngen.body.crt ${FILESYSTEM_PATH}/${CERTS_PATH}/fog.pem fi @@ -1016,19 +1043,19 @@ run_pre_install_test() run_post_install_test() { if [ $var_is_alpine = false ]; then - if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_chrono.so.1.78.0 ]; then + if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_chrono.so ]; then cp_print "Error, libboost_chrono .so file is missing" ${FORCE_STDOUT} exit 1 fi - if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_context.so.1.78.0 ]; then + if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_context.so ]; then cp_print "Error, libboost_context .so file is missing" ${FORCE_STDOUT} exit 1 fi - if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_system.so.1.78.0 ]; then + if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_system.so ]; then cp_print "Error, libboost_system .so file is missing" ${FORCE_STDOUT} exit 1 fi - if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_thread.so.1.78.0 ]; then + if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_thread.so ]; then cp_print "Error, libboost_thread .so file is missing" ${FORCE_STDOUT} exit 1 fi