From 942b2ef8b4c74cc3d877c85b9ba693a6d03b9524 Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Sun, 14 Apr 2024 12:55:54 +0000 Subject: [PATCH] 2024 April 14th update --- .../nginx_attachment/nginx_attachment.cc | 3 +- components/generic_rulebase/assets_config.cc | 137 ---- .../generic_rulebase/evaluators/asset_eval.cc | 52 -- .../evaluators/connection_eval.cc | 299 -------- .../evaluators/http_transaction_data_eval.cc | 168 ----- .../evaluators/parameter_eval.cc | 38 - .../evaluators/practice_eval.cc | 50 -- .../generic_rulebase/evaluators/query_eval.cc | 136 ---- .../evaluators/trigger_eval.cc | 57 -- .../generic_rulebase/evaluators/zone_eval.cc | 44 -- .../generic_rulebase/generic_rulebase.cc | 126 ---- .../generic_rulebase_context.cc | 109 --- components/generic_rulebase/match_query.cc | 347 --------- .../generic_rulebase/parameters_config.cc | 157 ----- .../generic_rulebase/rulebase_config.cc | 79 --- .../generic_rulebase/triggers_config.cc | 243 ------- components/generic_rulebase/zone.cc | 179 ----- components/generic_rulebase/zones_config.cc | 114 --- components/include/details_resolver.h | 1 + components/include/downloader.h | 2 + components/include/i_orchestration_tools.h | 2 +- components/include/orchestration_status.h | 2 +- components/include/package_handler.h | 4 +- components/include/url_parser.h | 3 + .../ips/include/ips_basic_policy.h | 4 + .../ips/include/ips_signatures.h | 20 +- .../ips/include/snort_basic_policy.h | 2 + .../security_apps/ips/ips_basic_policy.cc | 26 +- .../security_apps/ips/ips_signatures.cc | 22 +- .../security_apps/ips/ips_ut/component_ut.cc | 41 +- .../security_apps/ips/snort_basic_policy.cc | 15 +- .../access_control_practice.cc | 7 + .../include/access_control_practice.h | 2 + .../include/new_practice.h | 2 + .../local_policy_mgmt_gen/new_practice.cc | 8 + .../policy_maker_utils.cc | 16 +- .../details_resolver/details_resolver.cc | 7 + .../details_resolver_impl.h | 2 +- .../details_resolving_handler.cc | 23 + .../details_resolving_handler.h | 1 + .../orchestration/downloader/CMakeLists.txt | 2 +- .../orchestration/downloader/downloader.cc | 5 + .../orchestration/downloader/http_client.cc | 14 +- .../orchestration/downloader/http_client.h | 1 + .../orchestration/downloader/https_client.cc | 2 +- .../downloader/https_client_helper.cc | 20 + .../orchestration/hybrid_mode_telemetry.cc | 1 + .../orchestration/include/namespace_data.h | 2 +- .../include/orchestration_policy.h | 8 +- .../modules_ut/orchestration_policy_ut.cc | 16 +- .../modules/modules_ut/url_parser_ut.cc | 9 + .../modules/orchestration_policy.cc | 15 +- .../modules/orchestration_status.cc | 2 - .../orchestration/modules/url_parser.cc | 12 + .../orchestration/orchestration_comp.cc | 657 +++++++++--------- .../orchestration_tools/namespace_data.cc | 2 +- .../orchestration_tools.cc | 102 +-- .../orchestration_tools_ut.cc | 25 +- .../orchestration_multitenant_ut.cc | 45 +- .../orchestration_ut/orchestration_ut.cc | 461 ++---------- .../package_handler/package_handler.cc | 4 + .../service_controller/service_controller.cc | 139 ++-- .../service_controller_ut.cc | 174 ++--- core/config/config.cc | 52 +- .../interfaces/messaging/messaging_enums.h | 9 + .../services_sdk/resources/debug_flags.h | 1 + .../resources/intelligence_invalidation.h | 43 +- .../include/intelligence_request.h | 2 - .../include/intelligence_server.h | 2 +- .../intelligence_comp_v2.cc | 81 ++- .../intelligence_is_v2_ut/invalidation_ut.cc | 550 +++++++++++++-- .../intelligence_request.cc | 13 - .../intelligence_is_v2/intelligence_server.cc | 69 +- core/intelligence_is_v2/invalidation.cc | 277 +++++--- core/messaging/connection/connection.cc | 70 +- .../messaging_comp/messaging_comp.cc | 2 +- core/rest/rest_server.cc | 4 +- .../package/cpnano_debug/cpnano_debug.cc | 2 + .../package/orchestration_package.sh | 133 ++-- 79 files changed, 1800 insertions(+), 3778 deletions(-) delete mode 100755 components/generic_rulebase/assets_config.cc delete mode 100755 components/generic_rulebase/evaluators/asset_eval.cc delete mode 100755 components/generic_rulebase/evaluators/connection_eval.cc delete mode 100755 components/generic_rulebase/evaluators/http_transaction_data_eval.cc delete mode 100755 components/generic_rulebase/evaluators/parameter_eval.cc delete mode 100755 components/generic_rulebase/evaluators/practice_eval.cc delete mode 100755 components/generic_rulebase/evaluators/query_eval.cc delete mode 100755 components/generic_rulebase/evaluators/trigger_eval.cc delete mode 100755 components/generic_rulebase/evaluators/zone_eval.cc delete mode 100755 components/generic_rulebase/generic_rulebase.cc delete mode 100755 components/generic_rulebase/generic_rulebase_context.cc delete mode 100755 components/generic_rulebase/match_query.cc delete mode 100755 components/generic_rulebase/parameters_config.cc delete mode 100755 components/generic_rulebase/rulebase_config.cc delete mode 100755 components/generic_rulebase/triggers_config.cc delete mode 100755 components/generic_rulebase/zone.cc delete mode 100755 components/generic_rulebase/zones_config.cc create mode 100644 components/security_apps/orchestration/downloader/https_client_helper.cc diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc index 9f52aa5..6174c65 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc @@ -1136,7 +1136,8 @@ private: NginxAttachmentOpaque &opaque = i_transaction_table->getState(); uuid = opaque.getSessionUUID(); } - web_response_data.uuid_size = uuid.size(); + web_response_data.uuid_size = + string("Incident Id: ").length() + uuid.size(); if (web_trigger_conf.getDetailsLevel() == "Redirect") { web_response_data.response_data.redirect_data.redirect_location_size = diff --git a/components/generic_rulebase/assets_config.cc b/components/generic_rulebase/assets_config.cc deleted file mode 100755 index 64e0984..0000000 --- a/components/generic_rulebase/assets_config.cc +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/assets_config.h" - -#include -#include -#include - -#include "generic_rulebase/generic_rulebase_utils.h" -#include "config.h" -#include "debug.h" -#include "ip_utilities.h" - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -using namespace std; - -void -RuleAsset::load(cereal::JSONInputArchive &archive_in) -{ - archive_in(cereal::make_nvp("assetId", asset_id)); - archive_in(cereal::make_nvp("assetName", asset_name)); - archive_in(cereal::make_nvp("assetUrls", asset_urls)); - - dbgWarning(D_RULEBASE_CONFIG) << "Adding asset with UID: " << asset_id; -} - -void -RuleAsset::AssetUrl::load(cereal::JSONInputArchive &archive_in) -{ - archive_in(cereal::make_nvp("protocol", protocol)); - transform(protocol.begin(), protocol.end(), protocol.begin(), [](unsigned char c) { return tolower(c); }); - - archive_in(cereal::make_nvp("ip", ip)); - archive_in(cereal::make_nvp("port", port)); - - int value; - if (protocol == "*") { - is_any_proto = true; - } else { - is_any_proto = false; - try { - value = 0; - if(protocol == "udp") value = IPPROTO_UDP; - if(protocol == "tcp") value = IPPROTO_TCP; - if(protocol == "dccp") value = IPPROTO_DCCP; - if(protocol == "sctp") value = IPPROTO_SCTP; - if(protocol == "icmp") value = IPPROTO_ICMP; - if(protocol == "icmpv6") value = IPPROTO_ICMP; - - if (value > static_cast(UINT8_MAX) || value < 0) { - dbgWarning(D_RULEBASE_CONFIG) - << "provided value is not a legal IP protocol number. Value: " - << protocol; - } else { - parsed_proto = value; - } - } catch (...) { - dbgWarning(D_RULEBASE_CONFIG) << "provided value is not a legal IP protocol. Value: " << protocol; - } - } - - if (port == "*") { - is_any_port = true; - } else { - is_any_port = false; - try { - value = stoi(port); - if (value > static_cast(UINT16_MAX) || value < 0) { - dbgWarning(D_RULEBASE_CONFIG) << "provided value is not a legal port number. Value: " << port; - } else { - parsed_port = value; - } - } catch (...) { - dbgWarning(D_RULEBASE_CONFIG) << "provided value is not a legal port. Value: " << port; - } - } - - if (ip == "*") { - is_any_ip = true; - } else { - is_any_ip = false; - auto ip_addr = IPAddr::createIPAddr(ip); - if (!ip_addr.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Could not create IP address. Error: " << ip_addr.getErr(); - } else { - parsed_ip = ConvertToIpAddress(ip_addr.unpackMove()); - } - } -} - -IpAddress -RuleAsset::AssetUrl::ConvertToIpAddress(const IPAddr &addr) -{ - IpAddress address; - switch (addr.getType()) { - case IPType::UNINITIALIZED: { - address.addr4_t = {0}; - address.ip_type = IP_VERSION_ANY; - break; - } - case IPType::V4: { - address.addr4_t = addr.getIPv4(); - address.ip_type = IP_VERSION_4; - break; - } - case IPType::V6: { - address.addr6_t = addr.getIPv6(); - address.ip_type = IP_VERSION_6; - break; - } - default: - address.addr4_t = {0}; - address.ip_type = IP_VERSION_ANY; - dbgWarning(D_RULEBASE_CONFIG) << "Unsupported IP type: " << static_cast(addr.getType()); - } - return address; -} - -const Assets Assets::empty_assets_config = Assets(); - -void -Assets::preload() -{ - registerExpectedSetting("rulebase", "usedAssets"); -} diff --git a/components/generic_rulebase/evaluators/asset_eval.cc b/components/generic_rulebase/evaluators/asset_eval.cc deleted file mode 100755 index ad0e35c..0000000 --- a/components/generic_rulebase/evaluators/asset_eval.cc +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/evaluators/asset_eval.h" - -#include -#include - -#include "generic_rulebase/assets_config.h" -#include "config.h" -#include "debug.h" - -using namespace std; - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -string AssetMatcher::ctx_key = "asset_id"; - -AssetMatcher::AssetMatcher(const vector ¶ms) -{ - if (params.size() != 1) reportWrongNumberOfParams(AssetMatcher::getName(), params.size(), 1, 1); - asset_id = params[0]; -} - -Maybe -AssetMatcher::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto bc_asset_id_ctx = env->get(AssetMatcher::ctx_key); - - if (bc_asset_id_ctx.ok()) { - dbgTrace(D_RULEBASE_CONFIG) - << "Asset ID: " - << asset_id - << "; Current set assetId context: " - << *bc_asset_id_ctx; - } else { - dbgTrace(D_RULEBASE_CONFIG) << "Asset ID: " << asset_id << ". Empty context"; - } - - return bc_asset_id_ctx.ok() && *bc_asset_id_ctx == asset_id; -} diff --git a/components/generic_rulebase/evaluators/connection_eval.cc b/components/generic_rulebase/evaluators/connection_eval.cc deleted file mode 100755 index 589a96d..0000000 --- a/components/generic_rulebase/evaluators/connection_eval.cc +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/evaluators/connection_eval.h" - -#include -#include - -#include "generic_rulebase/rulebase_config.h" -#include "config.h" -#include "debug.h" -#include "ip_utilities.h" - -using namespace std; -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -string IpAddressMatcher::ctx_key = "ipAddress"; -string SourceIpMatcher::ctx_key = "sourceIP"; -string DestinationIpMatcher::ctx_key = "destinationIP"; -string SourcePortMatcher::ctx_key = "sourcePort"; -string ListeningPortMatcher::ctx_key = "listeningPort"; -string IpProtocolMatcher::ctx_key = "ipProtocol"; -string UrlMatcher::ctx_key = "url"; - -Maybe -getIpAddrFromEnviroment(I_Environment *env, Context::MetaDataType enum_data_type, const string &str_data_type) -{ - auto ip_str = env->get(enum_data_type); - if (!ip_str.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to get " << str_data_type << " from the enviroment."; - return genError("Failed to get " + str_data_type + " from the enviroment."); - } - return IPAddr::createIPAddr(ip_str.unpack()); -} - -bool -checkIfIpInRangesVec(const vector> &values, const IPAddr &ip_to_check) -{ - if (values.size() == 0) { - dbgTrace(D_RULEBASE_CONFIG) << "Ip addersses vector empty. Match is true."; - return true; - } - for (const CustomRange &range : values) { - if (range.contains(ip_to_check)) { - dbgTrace(D_RULEBASE_CONFIG) << "Ip adderss matched: " << ip_to_check; - return true; - } - } - dbgTrace(D_RULEBASE_CONFIG) << "Ip adderss not match: " << ip_to_check; - return false; -} - - -IpAddressMatcher::IpAddressMatcher(const vector ¶ms) -{ - for (const string ¶m : params) { - Maybe> ip_range = CustomRange::createRange(param); - if (!ip_range.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to create ip. Error: " + ip_range.getErr(); - continue; - } - values.push_back(ip_range.unpack()); - } -} - -Maybe -IpAddressMatcher::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - Maybe subject_ip = getIpAddrFromEnviroment( - env, - Context::MetaDataType::SubjectIpAddr, - "subject ip address" - ); - if (subject_ip.ok() && checkIfIpInRangesVec(values, subject_ip.unpack())) return true; - - Maybe other_ip = getIpAddrFromEnviroment( - env, - Context::MetaDataType::OtherIpAddr, - "other ip address" - ); - if (other_ip.ok() && checkIfIpInRangesVec(values, other_ip.unpack())) return true; - if (!subject_ip.ok() && !other_ip.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Error in getting subject ip and other ip from the enviroment"; - return false; - } - dbgTrace(D_RULEBASE_CONFIG) << "Ip adderss didn't match"; - return false; -} - -SourceIpMatcher::SourceIpMatcher(const vector ¶ms) -{ - for (const string ¶m : params) { - Maybe> ip_range = CustomRange::createRange(param); - if (!ip_range.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to create source ip. Error: " + ip_range.getErr(); - continue; - } - values.push_back(ip_range.unpack()); - } -} - -Maybe -SourceIpMatcher::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto direction_maybe = env->get(Context::MetaDataType::Direction); - if (!direction_maybe.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to get direction from the enviroment."; - return false; - } - string direction = direction_maybe.unpack(); - if (direction == "incoming") { - Maybe other_ip = getIpAddrFromEnviroment( - env, - Context::MetaDataType::OtherIpAddr, - "other ip address" - ); - return other_ip.ok() && checkIfIpInRangesVec(values, other_ip.unpack()); - } else if (direction == "outgoing") { - Maybe subject_ip = getIpAddrFromEnviroment( - env, - Context::MetaDataType::SubjectIpAddr, - "subject ip address" - ); - return subject_ip.ok() && checkIfIpInRangesVec(values, subject_ip.unpack()); - } - dbgTrace(D_RULEBASE_CONFIG) << "Source ip adderss didn't match"; - return false; -} - -DestinationIpMatcher::DestinationIpMatcher(const vector ¶ms) -{ - for (const string ¶m : params) { - Maybe> ip_range = CustomRange::createRange(param); - if (!ip_range.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to create destination ip. Error: " + ip_range.getErr(); - continue; - } - values.push_back(ip_range.unpack()); - } -} - -Maybe -DestinationIpMatcher::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto direction_maybe = env->get(Context::MetaDataType::Direction); - if (!direction_maybe.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to get direction."; - return false; - } - string direction = direction_maybe.unpack(); - if (direction == "outgoing") { - Maybe other_ip = getIpAddrFromEnviroment( - env, - Context::MetaDataType::OtherIpAddr, - "other ip address" - ); - return other_ip.ok() && checkIfIpInRangesVec(values, other_ip.unpack()); - } else if (direction == "incoming") { - Maybe subject_ip = getIpAddrFromEnviroment( - env, - Context::MetaDataType::SubjectIpAddr, - "subject ip address" - ); - return subject_ip.ok() && checkIfIpInRangesVec(values, subject_ip.unpack()); - } - dbgTrace(D_RULEBASE_CONFIG) << "Destination ip adderss didn't match"; - return false; -} - -SourcePortMatcher::SourcePortMatcher(const vector ¶ms) -{ - for (const string ¶m : params) { - Maybe> port_range = CustomRange::createRange(param); - if (!port_range.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to create source port."; - continue; - } - values.push_back(port_range.unpack()); - } -} - -Maybe -SourcePortMatcher::evalVariable() const -{ - dbgTrace(D_RULEBASE_CONFIG) << "Source is not a match"; - return false; -} - - -ListeningPortMatcher::ListeningPortMatcher(const vector ¶ms) -{ - for (const string ¶m : params) { - Maybe> port_range = CustomRange::createRange(param); - if (!port_range.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to create listening port range."; - continue; - } - values.push_back(port_range.unpack()); - } -} - -Maybe -ListeningPortMatcher::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto port_str = env->get(Context::MetaDataType::Port); - if (!port_str.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to get port from the enviroment."; - return false; - } - PortNumber port; - if (ConnKeyUtil::fromString(port_str.unpack(), port)) { - if (values.size() == 0) return true; - for (const CustomRange &port_range : values) { - if (port_range.contains(port)) { - dbgTrace(D_RULEBASE_CONFIG) << "Listening port is a match. Value: " << port_str.unpack(); - return true; - } - } - } - dbgTrace(D_RULEBASE_CONFIG) << "Listening port is not a match. Value: " << port_str.unpack(); - return false; -} - -IpProtocolMatcher::IpProtocolMatcher(const vector ¶ms) -{ - for (const string ¶m : params) { - Maybe> proto_range = CustomRange::createRange(param); - if (!proto_range.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to create ip protocol."; - continue; - } - values.push_back(proto_range.unpack()); - } -} - -Maybe -IpProtocolMatcher::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto proto_str = env->get(Context::MetaDataType::Protocol); - if (!proto_str.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to get ip protocol from the enviroment."; - return false; - } - IPProto protocol; - if (ConnKeyUtil::fromString(proto_str.unpack(), protocol)) { - if (values.size() == 0) return true; - for (const CustomRange &proto_range : values) { - if (proto_range.contains(protocol)) { - dbgTrace(D_RULEBASE_CONFIG) << "Ip protocol is a match. Value: " << proto_str.unpack(); - return true; - } - } - } - dbgTrace(D_RULEBASE_CONFIG) << "Source port is not a match. Value: " << proto_str.unpack(); - return false; -} - -UrlMatcher::UrlMatcher(const vector ¶ms) : values(params) {} - -Maybe -UrlMatcher::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto curr_url_ctx = env->get(Context::MetaDataType::Url); - if (!curr_url_ctx.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to get URL from the enviroment."; - return false; - } - - if (values.size() == 0) { - dbgTrace(D_RULEBASE_CONFIG) << "Matched URL on \"any\". Url: " << *curr_url_ctx; - return true; - } - - for (const string &url : values) { - if (*curr_url_ctx == url) { - dbgTrace(D_RULEBASE_CONFIG) << "Matched URL. Value: " << *curr_url_ctx; - return true; - } - } - - dbgTrace(D_RULEBASE_CONFIG) << "URL is not a match. Value: " << *curr_url_ctx; - return false; -} diff --git a/components/generic_rulebase/evaluators/http_transaction_data_eval.cc b/components/generic_rulebase/evaluators/http_transaction_data_eval.cc deleted file mode 100755 index d6560fb..0000000 --- a/components/generic_rulebase/evaluators/http_transaction_data_eval.cc +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/evaluators/http_transaction_data_eval.h" - -#include -#include - -#include "http_transaction_data.h" -#include "environment/evaluator_templates.h" -#include "i_environment.h" -#include "singleton.h" -#include "debug.h" - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -using namespace std; -using namespace EnvironmentHelper; - -EqualHost::EqualHost(const vector ¶ms) -{ - if (params.size() != 1) reportWrongNumberOfParams("EqualHost", params.size(), 1, 1); - host = params[0]; -} - -Maybe -EqualHost::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto host_ctx = env->get(HttpTransactionData::host_name_ctx); - - if (!host_ctx.ok()) - { - return false; - } - - std::string lower_host_ctx = host_ctx.unpack(); - std::transform(lower_host_ctx.begin(), lower_host_ctx.end(), lower_host_ctx.begin(), ::tolower); - - std::string lower_host = host; - std::transform(lower_host.begin(), lower_host.end(), lower_host.begin(), ::tolower); - - - if (lower_host_ctx == lower_host) return true; - size_t pos = lower_host_ctx.find_last_of(':'); - if (pos == string::npos) return false; - lower_host_ctx = string(lower_host_ctx.data(), pos); - return lower_host_ctx == lower_host; -} - -WildcardHost::WildcardHost(const vector ¶ms) -{ - if (params.size() != 1) reportWrongNumberOfParams("WildcardHost", params.size(), 1, 1); - host = params[0]; -} - -Maybe -WildcardHost::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto host_ctx = env->get(HttpTransactionData::host_name_ctx); - - if (!host_ctx.ok()) - { - return false; - } - - string lower_host_ctx = host_ctx.unpack(); - transform(lower_host_ctx.begin(), lower_host_ctx.end(), lower_host_ctx.begin(), ::tolower); - - dbgTrace(D_RULEBASE_CONFIG) << "found host in current context: " << lower_host_ctx; - - size_t pos = lower_host_ctx.find_first_of("."); - if (pos == string::npos) { - return false; - } - - lower_host_ctx = "*" + lower_host_ctx.substr(pos, lower_host_ctx.length()); - - string lower_host = host; - transform(lower_host.begin(), lower_host.end(), lower_host.begin(), ::tolower); - - dbgTrace(D_RULEBASE_CONFIG) - << "trying to match host context with its corresponding wildcard address: " - << lower_host_ctx - << ". Matcher host: " - << lower_host; - - if (lower_host_ctx == lower_host) return true; - pos = lower_host_ctx.find_last_of(':'); - if (pos == string::npos) return false; - lower_host_ctx = string(lower_host_ctx.data(), pos); - return lower_host_ctx == lower_host; -} - -EqualListeningIP::EqualListeningIP(const vector ¶ms) -{ - if (params.size() != 1) reportWrongNumberOfParams("EqualListeningIP", params.size(), 1, 1); - - auto maybe_ip = IPAddr::createIPAddr(params[0]); - if (!maybe_ip.ok()) reportWrongParamType(getName(), params[0], "Not a valid IP Address"); - - listening_ip = maybe_ip.unpack(); -} - -Maybe -EqualListeningIP::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto listening_ip_ctx = env->get(HttpTransactionData::listening_ip_ctx); - return listening_ip_ctx.ok() && listening_ip_ctx.unpack() == listening_ip; -} - -EqualListeningPort::EqualListeningPort(const vector ¶ms) -{ - if (params.size() != 1) reportWrongNumberOfParams("EqualListeningPort", params.size(), 1, 1); - - try { - listening_port = boost::lexical_cast(params[0]); - } catch (boost::bad_lexical_cast const&) { - reportWrongParamType(getName(), params[0], "Not a valid port number"); - } -} - -Maybe -EqualListeningPort::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto port_ctx = env->get(HttpTransactionData::listening_port_ctx); - - return port_ctx.ok() && port_ctx.unpack() == listening_port; -} - -BeginWithUri::BeginWithUri(const vector ¶ms) -{ - if (params.size() != 1) reportWrongNumberOfParams("BeginWithUri", params.size(), 1, 1); - uri_prefix = params[0]; -} - -Maybe -BeginWithUri::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto uri_ctx = env->get(HttpTransactionData::uri_ctx); - - if (!uri_ctx.ok()) - { - return false; - } - - std::string lower_uri_ctx = uri_ctx.unpack(); - std::transform(lower_uri_ctx.begin(), lower_uri_ctx.end(), lower_uri_ctx.begin(), ::tolower); - - std::string lower_uri_prefix = uri_prefix; - std::transform(lower_uri_prefix.begin(), lower_uri_prefix.end(), lower_uri_prefix.begin(), ::tolower); - - return lower_uri_ctx.find(lower_uri_prefix) == 0; -} diff --git a/components/generic_rulebase/evaluators/parameter_eval.cc b/components/generic_rulebase/evaluators/parameter_eval.cc deleted file mode 100755 index 2430ba6..0000000 --- a/components/generic_rulebase/evaluators/parameter_eval.cc +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/evaluators/parameter_eval.h" - -#include -#include - -#include "generic_rulebase/rulebase_config.h" -#include "config.h" -#include "debug.h" - -using namespace std; - -string ParameterMatcher::ctx_key = "parameters"; - -ParameterMatcher::ParameterMatcher(const vector ¶ms) -{ - if (params.size() != 1) reportWrongNumberOfParams(ParameterMatcher::getName(), params.size(), 1, 1); - parameter_id = params[0]; -} - -Maybe -ParameterMatcher::evalVariable() const -{ - auto rule = getConfiguration("rulebase", "rulesConfig"); - return rule.ok() && rule.unpack().isParameterActive(parameter_id); -} diff --git a/components/generic_rulebase/evaluators/practice_eval.cc b/components/generic_rulebase/evaluators/practice_eval.cc deleted file mode 100755 index fc570e1..0000000 --- a/components/generic_rulebase/evaluators/practice_eval.cc +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/evaluators/practice_eval.h" - -#include -#include - -#include "generic_rulebase/rulebase_config.h" -#include "config.h" -#include "debug.h" - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -using namespace std; - -string PracticeMatcher::ctx_key = "practices"; - -PracticeMatcher::PracticeMatcher(const vector ¶ms) -{ - if (params.size() != 1) reportWrongNumberOfParams(PracticeMatcher::getName(), params.size(), 1, 1); - practice_id = params[0]; -} - -Maybe -PracticeMatcher::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto bc_practice_id_ctx = env->get>(PracticeMatcher::ctx_key); - dbgTrace(D_RULEBASE_CONFIG) - << "Trying to match practice. ID: " - << practice_id << ", Current set IDs: " - << makeSeparatedStr(bc_practice_id_ctx.ok() ? *bc_practice_id_ctx : set(), ", "); - if (bc_practice_id_ctx.ok()) { - return bc_practice_id_ctx.unpack().count(practice_id) > 0; - } - - auto rule = getConfiguration("rulebase", "rulesConfig"); - return rule.ok() && rule.unpack().isPracticeActive(practice_id); -} diff --git a/components/generic_rulebase/evaluators/query_eval.cc b/components/generic_rulebase/evaluators/query_eval.cc deleted file mode 100755 index 23f722f..0000000 --- a/components/generic_rulebase/evaluators/query_eval.cc +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/evaluators/query_eval.h" - -#include -#include -#include - -#include "generic_rulebase/rulebase_config.h" -#include "generic_rulebase/zones_config.h" -#include "i_environment.h" -#include "singleton.h" -#include "config.h" -#include "debug.h" -#include "enum_range.h" - -using namespace std; - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -QueryMatcher::QueryMatcher(const vector ¶ms) -{ - if (params.size() < 1) reportWrongNumberOfParams(QueryMatcher::getName(), params.size(), 1); - - key = params.front(); - if (key == "any") { - is_any = true; - } else { - values.reserve(params.size() - 1); - for (uint i = 1; i < params.size() ; i++) { - if (params[i] == "any") { - values.clear(); - break; - } - values.insert(params[i]); - } - } -} - -const string -QueryMatcher::contextKeyToString(Context::MetaDataType type) -{ - if (type == Context::MetaDataType::SubjectIpAddr || type == Context::MetaDataType::OtherIpAddr) return "ip"; - return Context::convertToString(type); -} - -class QueryMatchSerializer -{ -public: - static const string req_attr_ctx_key; - - template - void - serialize(Archive &ar) - { - I_Environment *env = Singleton::Consume::by(); - auto req_attr = env->get(req_attr_ctx_key); - if (!req_attr.ok()) return; - - try { - ar(cereal::make_nvp(*req_attr, value)); - dbgDebug(D_RULEBASE_CONFIG) - << "Found value for requested attribute. Tag: " - << *req_attr - << ", Value: " - << value; - } catch (exception &e) { - dbgDebug(D_RULEBASE_CONFIG) << "Could not find values for requested attribute. Tag: " << *req_attr; - ar.finishNode(); - } - } - - template - bool - matchValues(const Values &requested_vals) const - { - return value != "" && (requested_vals.empty() || requested_vals.count(value) > 0); - } - -private: - string value; -}; - -const string QueryMatchSerializer::req_attr_ctx_key = "requested attribute key"; - -Maybe -QueryMatcher::evalVariable() const -{ - if (is_any) return true; - - I_Environment *env = Singleton::Consume::by(); - auto local_asset_ctx = env->get("is local asset"); - bool is_remote_asset = local_asset_ctx.ok() && !(*local_asset_ctx); - - QueryRequest request; - for (Context::MetaDataType name : makeRange()) { - auto val = env->get(name); - if (val.ok()) { - if ((name == Context::MetaDataType::SubjectIpAddr && is_remote_asset) || - (name == Context::MetaDataType::OtherIpAddr && !is_remote_asset)) { - continue; - } - - request.addCondition(Condition::EQUALS, contextKeyToString(name), *val); - } - } - if (request.empty()) return false; - - request.setRequestedAttr(key); - ScopedContext req_attr_key; - req_attr_key.registerValue(QueryMatchSerializer::req_attr_ctx_key, key); - - I_Intelligence_IS_V2 *intelligence = Singleton::Consume::by(); - auto query_res = intelligence->queryIntelligence(request); - if (!query_res.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to perform intelligence query. Error: " << query_res.getErr(); - return false; - } - - for (const AssetReply &asset : query_res.unpack()) { - if (asset.matchValues>(values)) return true; - } - - return false; -} diff --git a/components/generic_rulebase/evaluators/trigger_eval.cc b/components/generic_rulebase/evaluators/trigger_eval.cc deleted file mode 100755 index cd26ac4..0000000 --- a/components/generic_rulebase/evaluators/trigger_eval.cc +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/evaluators/trigger_eval.h" - -#include -#include - -#include "generic_rulebase/rulebase_config.h" -#include "config.h" -#include "debug.h" - -using namespace std; - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -string TriggerMatcher::ctx_key = "triggers"; - -TriggerMatcher::TriggerMatcher(const vector ¶ms) -{ - if (params.size() != 1) reportWrongNumberOfParams(TriggerMatcher::getName(), params.size(), 1, 1); - trigger_id = params[0]; -} - -Maybe -TriggerMatcher::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto ac_bc_trigger_id_ctx = env->get>("ac_trigger_id"); - dbgTrace(D_RULEBASE_CONFIG) - << "Trying to match trigger for access control rule. ID: " - << trigger_id << ", Current set IDs: " - << makeSeparatedStr(ac_bc_trigger_id_ctx.ok() ? *ac_bc_trigger_id_ctx : set(), ", "); - if (ac_bc_trigger_id_ctx.ok()) { - return ac_bc_trigger_id_ctx.unpack().count(trigger_id) > 0; - } - - auto bc_trigger_id_ctx = env->get>(TriggerMatcher::ctx_key); - dbgTrace(D_RULEBASE_CONFIG) - << "Trying to match trigger. ID: " - << trigger_id << ", Current set IDs: " - << makeSeparatedStr(bc_trigger_id_ctx.ok() ? *bc_trigger_id_ctx : set(), ", "); - if (bc_trigger_id_ctx.ok() && bc_trigger_id_ctx.unpack().count(trigger_id) > 0 ) return true; - - auto rule = getConfiguration("rulebase", "rulesConfig"); - return rule.ok() && rule.unpack().isTriggerActive(trigger_id); -} diff --git a/components/generic_rulebase/evaluators/zone_eval.cc b/components/generic_rulebase/evaluators/zone_eval.cc deleted file mode 100755 index 17eb328..0000000 --- a/components/generic_rulebase/evaluators/zone_eval.cc +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/evaluators/zone_eval.h" - -#include -#include - -#include "generic_rulebase/zone.h" -#include "generic_rulebase/rulebase_config.h" -#include "config.h" - -using namespace std; - -string ZoneMatcher::ctx_key = "zone_id"; - -ZoneMatcher::ZoneMatcher(const vector ¶ms) -{ - if (params.size() != 1) reportWrongNumberOfParams(ZoneMatcher::getName(), params.size(), 1, 1); - zone_id = params[0]; -} - -Maybe -ZoneMatcher::evalVariable() const -{ - I_Environment *env = Singleton::Consume::by(); - auto bc_zone_id_ctx = env->get(ZoneMatcher::ctx_key); - if (bc_zone_id_ctx.ok() && *bc_zone_id_ctx == zone_id) return true; - - if (!getProfileAgentSettingWithDefault(false, "rulebase.enableQueryBasedMatch")) return false; - - auto zone = getConfiguration("rulebase", "zones"); - return zone.ok() && zone.unpack().getId() == zone_id; -} diff --git a/components/generic_rulebase/generic_rulebase.cc b/components/generic_rulebase/generic_rulebase.cc deleted file mode 100755 index d4fd8f9..0000000 --- a/components/generic_rulebase/generic_rulebase.cc +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/generic_rulebase.h" - -#include - -#include "generic_rulebase/evaluators/trigger_eval.h" -#include "generic_rulebase/evaluators/practice_eval.h" -#include "generic_rulebase/evaluators/parameter_eval.h" -#include "generic_rulebase/evaluators/zone_eval.h" -#include "generic_rulebase/evaluators/asset_eval.h" -#include "generic_rulebase/evaluators/query_eval.h" -#include "generic_rulebase/evaluators/connection_eval.h" -#include "generic_rulebase/evaluators/http_transaction_data_eval.h" -#include "generic_rulebase/zone.h" -#include "generic_rulebase/triggers_config.h" -#include "singleton.h" -#include "common.h" -#include "debug.h" -#include "cache.h" -#include "config.h" - -using namespace std; - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -class GenericRulebase::Impl : Singleton::Provide::From -{ -public: - void init() {} - void fini() {} - - void preload(); - - Maybe getLocalZone() const override { return getZoneConfig(true); } - Maybe getOtherZone() const override { return getZoneConfig(false); } - - set getBehavior(const ParameterKeyValues &key_value_pairs) const override; - -private: - Maybe - getZoneConfig(bool is_local_zone) const - { - ScopedContext asset_location_ctx; - asset_location_ctx.registerValue("is local asset", is_local_zone); - return getConfiguration("rulebase", "zones"); - } -}; - -void -GenericRulebase::Impl::preload() -{ - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - addMatcher(); - BasicRuleConfig::preload(); - LogTriggerConf::preload(); - ParameterException::preload(); - registerExpectedConfiguration("rulebase", "zones"); - registerExpectedConfigFile("zones", Config::ConfigFileType::Policy); - registerExpectedConfigFile("triggers", Config::ConfigFileType::Policy); - registerExpectedConfigFile("rules", Config::ConfigFileType::Policy); - registerExpectedConfigFile("parameters", Config::ConfigFileType::Policy); - registerExpectedConfigFile("exceptions", Config::ConfigFileType::Policy); - -} - -set -GenericRulebase::Impl::getBehavior(const ParameterKeyValues &key_value_pairs) const -{ - auto &exceptions = getConfiguration("rulebase", "exception"); - - if (!exceptions.ok()) { - dbgTrace(D_RULEBASE_CONFIG) << "Could not find any exception with the current rule's context"; - return {}; - } - return (*exceptions).getBehavior(key_value_pairs); -} - -GenericRulebase::GenericRulebase() : Component("GenericRulebase"), pimpl(make_unique()) {} - -GenericRulebase::~GenericRulebase() {} - -void -GenericRulebase::init() -{ - pimpl->init(); -} - -void -GenericRulebase::fini() -{ - pimpl->fini(); -} - -void -GenericRulebase::preload() -{ - pimpl->preload(); -} diff --git a/components/generic_rulebase/generic_rulebase_context.cc b/components/generic_rulebase/generic_rulebase_context.cc deleted file mode 100755 index 7dd98cb..0000000 --- a/components/generic_rulebase/generic_rulebase_context.cc +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/generic_rulebase_context.h" - -#include - -#include "context.h" -#include "config.h" -#include "generic_rulebase/evaluators/trigger_eval.h" -#include "generic_rulebase/evaluators/parameter_eval.h" -#include "generic_rulebase/evaluators/practice_eval.h" -#include "generic_rulebase/evaluators/zone_eval.h" -#include "generic_rulebase/evaluators/asset_eval.h" - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -using namespace std; - -template -set -extractIds(const vector &configurations) -{ - set ids; - for (const Configs &conf : configurations) { - ids.insert(conf.getId()); - } - return ids; -} - -void -GenericRulebaseContext::activate(const BasicRuleConfig &rule) -{ - switch(registration_state) { - case RuleRegistrationState::UNINITIALIZED: { - registration_state = RuleRegistrationState::REGISTERED; - ctx.registerValue>( - TriggerMatcher::ctx_key, - extractIds(rule.getTriggers()) - ); - ctx.registerValue>( - PracticeMatcher::ctx_key, - extractIds(rule.getPractices()) - ); - dbgTrace(D_RULEBASE_CONFIG) - << "Activating current practices. Current practice IDs: " - << makeSeparatedStr(extractIds(rule.getPractices()), ", "); - - ctx.registerValue>( - ParameterMatcher::ctx_key, - extractIds(rule.getParameters()) - ); - ctx.registerValue( - ZoneMatcher::ctx_key, - rule.getZoneId() - ); - ctx.registerValue( - AssetMatcher::ctx_key, - rule.getAssetId() - ); - ctx.activate(); - break; - } - case RuleRegistrationState::REGISTERED: { - dbgTrace(D_RULEBASE_CONFIG) << "Activating registered rule values"; - ctx.activate(); - break; - } - case RuleRegistrationState::UNREGISTERED: { - dbgTrace(D_RULEBASE_CONFIG) << "Failed to register rule values"; - } - } -} - -void -GenericRulebaseContext::activate() -{ - switch(registration_state) { - case RuleRegistrationState::UNINITIALIZED: { - auto maybe_rule = getConfiguration("rulebase", "rulesConfig"); - if (!maybe_rule.ok()) { - registration_state = RuleRegistrationState::UNREGISTERED; - return; - } - dbgTrace(D_RULEBASE_CONFIG) << "Registering new rule values"; - activate(maybe_rule.unpack()); - registration_state = RuleRegistrationState::REGISTERED; - break; - } - case RuleRegistrationState::REGISTERED: { - dbgTrace(D_RULEBASE_CONFIG) << "Activating registered rule values"; - ctx.activate(); - break; - } - case RuleRegistrationState::UNREGISTERED: { - dbgTrace(D_RULEBASE_CONFIG) << "Failed to register rule values"; - } - } -} diff --git a/components/generic_rulebase/match_query.cc b/components/generic_rulebase/match_query.cc deleted file mode 100755 index ef985dc..0000000 --- a/components/generic_rulebase/match_query.cc +++ /dev/null @@ -1,347 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/match_query.h" - -#include "cereal/types/set.hpp" - -#include "generic_rulebase/generic_rulebase_utils.h" -#include "config.h" -#include "ip_utilities.h" -#include "agent_core_utilities.h" - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -using namespace std; - -static const unordered_map string_to_match_type = { - { "condition", MatchQuery::MatchType::Condition }, - { "operator", MatchQuery::MatchType::Operator } -}; - -static const unordered_map string_to_operator = { - { "and", MatchQuery::Operators::And }, - { "or", MatchQuery::Operators::Or } -}; - -static const unordered_map string_to_condition = { - { "equals", MatchQuery::Conditions::Equals }, - { "not-equals", MatchQuery::Conditions::NotEquals }, - { "not equals", MatchQuery::Conditions::NotEquals }, - { "in", MatchQuery::Conditions::In }, - { "not-in", MatchQuery::Conditions::NotIn }, - { "not in", MatchQuery::Conditions::NotIn }, - { "exist", MatchQuery::Conditions::Exist } -}; - -static const string ip_addr_type_name = "IP address"; -static const string port_type_name = "port"; -static const string ip_proto_type_name = "IP protocol"; - -static const unordered_map string_to_key = { - { "sourceIP", MatchQuery::StaticKeys::SrcIpAddress }, - { "sourceIpAddr", MatchQuery::StaticKeys::SrcIpAddress }, - { "destinationIP", MatchQuery::StaticKeys::DstIpAddress }, - { "destinationIpAddr", MatchQuery::StaticKeys::DstIpAddress }, - { "ipAddress", MatchQuery::StaticKeys::IpAddress }, - { "sourcePort", MatchQuery::StaticKeys::SrcPort }, - { "listeningPort", MatchQuery::StaticKeys::ListeningPort }, - { "ipProtocol", MatchQuery::StaticKeys::IpProtocol }, - { "domain", MatchQuery::StaticKeys::Domain } -}; - -MatchQuery::MatchQuery(const string &match) : is_specific_label(false), is_ignore_keyword(false) -{ - try { - stringstream ss; - ss.str(match); - cereal::JSONInputArchive archive_in(ss); - load(archive_in); - } catch (const exception &e) { - dbgWarning(D_RULEBASE_CONFIG) - << "Unable to load match query JSON. JSON content: " - << match - << ", Error: " - << e.what(); - } -} - -void -MatchQuery::load(cereal::JSONInputArchive &archive_in) -{ - string type_as_string; - archive_in(cereal::make_nvp("type", type_as_string)); - - string op_as_string; - archive_in(cereal::make_nvp("op", op_as_string)); - - auto maybe_type = string_to_match_type.find(type_as_string); - if (maybe_type == string_to_match_type.end()) { - reportConfigurationError("Illegal Zone match query type. Provided type in configuration: " + type_as_string); - } - - type = maybe_type->second; - switch (type) { - case (MatchType::Condition): { - auto maybe_condition = string_to_condition.find(op_as_string); - if (maybe_condition == string_to_condition.end()) { - reportConfigurationError( - "Illegal op provided for condition. Provided op in configuration: " + - op_as_string - ); - } - condition_type = maybe_condition->second; - operator_type = Operators::None; - archive_in(cereal::make_nvp("key", key)); - key_type = getKeyByName(key); - if (key_type == StaticKeys::NotStatic) { - if (key.rfind("containerLabels.", 0) == 0) { - is_specific_label = true; - } else { - is_specific_label = false; - } - } - is_ignore_keyword = (key == "indicator"); - - if (condition_type != Conditions::Exist) { - archive_in(cereal::make_nvp("value", value)); - for(const auto &val: value) { - if (isKeyTypeIp()) { - auto ip_range = IPUtilities::createRangeFromString(val, ip_addr_type_name); - if (ip_range.ok()) { - ip_addr_value.push_back(ip_range.unpack()); - } else { - dbgWarning(D_RULEBASE_CONFIG) - << "Failed to parse IP address range. Error: " - << ip_range.getErr(); - } - } else if (isKeyTypePort()) { - auto port_range = IPUtilities::createRangeFromString( - val, - port_type_name - ); - if (port_range.ok()) { - port_value.push_back(port_range.unpack()); - } else { - dbgWarning(D_RULEBASE_CONFIG) - << "Failed to parse port range. Error: " - << port_range.getErr(); - } - } else if (isKeyTypeProtocol()) { - auto proto_range = IPUtilities::createRangeFromString( - val, - ip_proto_type_name - ); - if (proto_range.ok()) { - ip_proto_value.push_back(proto_range.unpack()); - } else { - dbgWarning(D_RULEBASE_CONFIG) - << "Failed to parse IP protocol range. Error: " - << proto_range.getErr(); - } - } - - try { - regex_values.insert(boost::regex(val)); - } catch (const exception &e) { - dbgDebug(D_RULEBASE_CONFIG) << "Failed to compile regex. Error: " << e.what(); - } - } - first_value = *(value.begin()); - } - break; - } - case (MatchType::Operator): { - auto maybe_operator = string_to_operator.find(op_as_string); - if (maybe_operator == string_to_operator.end()) { - reportConfigurationError( - "Illegal op provided for operator. Provided op in configuration: " + - op_as_string - ); - } - operator_type = maybe_operator->second; - condition_type = Conditions::None; - archive_in(cereal::make_nvp("items", items)); - break; - } - } -} - -MatchQuery::StaticKeys -MatchQuery::getKeyByName(const string &key_type_name) -{ - auto key = string_to_key.find(key_type_name); - if (key == string_to_key.end()) return StaticKeys::NotStatic; - return key->second; -} - -bool -MatchQuery::isKeyTypeIp() const -{ - return (key_type >= StaticKeys::IpAddress && key_type <= StaticKeys::DstIpAddress); -} - -bool -MatchQuery::isKeyTypePort() const -{ - return (key_type == StaticKeys::SrcPort || key_type == StaticKeys::ListeningPort); -} - -bool -MatchQuery::isKeyTypeProtocol() const -{ - return (key_type == StaticKeys::IpProtocol); -} - -bool -MatchQuery::isKeyTypeDomain() const -{ - return (key_type == StaticKeys::Domain); -} - -bool -MatchQuery::isKeyTypeSpecificLabel() const -{ - return is_specific_label; -} - -bool -MatchQuery::isKeyTypeStatic() const -{ - return (key_type != StaticKeys::NotStatic); -} - -set -MatchQuery::getAllKeys() const -{ - set keys; - if (type == MatchType::Condition) { - if (!key.empty()) keys.insert(key); - return keys; - } - - for (const MatchQuery &inner_match: items) { - set iner_keys = inner_match.getAllKeys(); - keys.insert(iner_keys.begin(), iner_keys.end()); - } - - return keys; -} - -bool -MatchQuery::matchAttributes( - const unordered_map> &key_value_pairs, - set &matched_override_keywords ) const -{ - - if (type == MatchType::Condition) { - auto key_value_pair = key_value_pairs.find(key); - if (key_value_pair == key_value_pairs.end()) { - dbgTrace(D_RULEBASE_CONFIG) << "Ignoring irrelevant key: " << key; - return false; - } - return matchAttributes(key_value_pair->second, matched_override_keywords); - } else if (type == MatchType::Operator && operator_type == Operators::And) { - for (const MatchQuery &inner_match: items) { - if (!inner_match.matchAttributes(key_value_pairs, matched_override_keywords)) { - return false; - } - } - return true; - } else if (type == MatchType::Operator && operator_type == Operators::Or) { - // With 'or' condition, evaluate matched override keywords first and add the ones that were fully matched - set inner_override_keywords; - bool res = false; - for (const MatchQuery &inner_match: items) { - inner_override_keywords.clear(); - if (inner_match.matchAttributes(key_value_pairs, inner_override_keywords)) { - matched_override_keywords.insert(inner_override_keywords.begin(), inner_override_keywords.end()); - res = true; - } - } - return res; - } else { - dbgWarning(D_RULEBASE_CONFIG) << "Unsupported match query type"; - } - return false; -} - -MatchQuery::MatchResult -MatchQuery::getMatch( const unordered_map> &key_value_pairs) const -{ - MatchQuery::MatchResult matches; - matches.matched_keywords = make_shared>(); - matches.is_match = matchAttributes(key_value_pairs, *matches.matched_keywords); - return matches; -} - -bool -MatchQuery::matchAttributes( - const unordered_map> &key_value_pairs) const -{ - return getMatch(key_value_pairs).is_match; -} - -bool -MatchQuery::matchAttributes( - const set &values, - set &matched_override_keywords) const -{ - auto &type = condition_type; - bool negate = type == MatchQuery::Conditions::NotEquals || type == MatchQuery::Conditions::NotIn; - bool match = isRegEx() ? matchAttributesRegEx(values, matched_override_keywords) : matchAttributesString(values); - return negate ? !match : match; -} - -bool -MatchQuery::matchAttributesRegEx( - const set &values, - set &matched_override_keywords) const -{ - bool res = false; - boost::cmatch value_matcher; - for (const boost::regex &val_regex : regex_values) { - for (const string &requested_match_value : values) { - if (NGEN::Regex::regexMatch( - __FILE__, - __LINE__, - requested_match_value.c_str(), - value_matcher, - val_regex)) - { - res = true; - if (is_ignore_keyword) { - matched_override_keywords.insert(requested_match_value); - } else { - return res; - } - } - } - } - return res; -} - -bool -MatchQuery::matchAttributesString(const set &values) const -{ - for (const string &requested_value : values) { - if (value.find(requested_value) != value.end()) return true; - } - return false; -} - -bool -MatchQuery::isRegEx() const -{ - return key != "protectionName"; -} diff --git a/components/generic_rulebase/parameters_config.cc b/components/generic_rulebase/parameters_config.cc deleted file mode 100755 index 7677b86..0000000 --- a/components/generic_rulebase/parameters_config.cc +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/parameters_config.h" - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -using namespace std; - -bool ParameterException::is_geo_location_exception_exists(false); -bool ParameterException::is_geo_location_exception_being_loaded(false); - -void -ParameterOverrides::load(cereal::JSONInputArchive &archive_in) -{ - parseJSONKey>("parsedBehavior", parsed_behaviors, archive_in); -} - -void -ParameterTrustedSources::load(cereal::JSONInputArchive &archive_in) -{ - parseJSONKey("numOfSources", num_of_sources, archive_in); - parseJSONKey>("sourcesIdentifiers", sources_identidiers, archive_in); -} - -void -ParameterBehavior::load(cereal::JSONInputArchive &archive_in) -{ - string key_string; - string val_string; - parseJSONKey("id", id, archive_in); - parseJSONKey("key", key_string, archive_in); - parseJSONKey("value", val_string, archive_in); - if (string_to_behavior_key.find(key_string) == string_to_behavior_key.end()) { - dbgWarning(D_RULEBASE_CONFIG) << "Unsupported behavior key: " << key_string; - return; - } - key = string_to_behavior_key.at(key_string); - - if (string_to_behavior_val.find(val_string) == string_to_behavior_val.end()) { - dbgWarning(D_RULEBASE_CONFIG) << "Unsupported behavior value: " << val_string; - return; - } - value = string_to_behavior_val.at(val_string); -} - -void -ParameterAntiBot::load(cereal::JSONInputArchive &archive_in) -{ - parseJSONKey>("injected", injected, archive_in); - parseJSONKey>("validated", validated, archive_in); -} - -void -ParameterOAS::load(cereal::JSONInputArchive &archive_in) -{ - parseJSONKey("value", value, archive_in); -} - -void -ParameterException::MatchBehaviorPair::load(cereal::JSONInputArchive &archive_in) -{ - parseJSONKey("match", match, archive_in); - parseJSONKey("behavior", behavior, archive_in); -} - -void -ParameterException::load(cereal::JSONInputArchive &archive_in) -{ - try { - archive_in( - cereal::make_nvp("match", match), - cereal::make_nvp("behavior", behavior) - ); - } catch (...) { - parseJSONKey>("exceptions", match_queries, archive_in); - } - - function isGeoLocationExists = - [&](const MatchQuery &query) - { - if (query.getKey() == "countryCode" || query.getKey() == "countryName") { - is_geo_location_exception_being_loaded = true; - return true; - } - - for (const MatchQuery &query_item : query.getItems()) { - if (isGeoLocationExists(query_item)) return true; - } - - return false; - }; - - if (isGeoLocationExists(match)) return; - for (const MatchBehaviorPair &match_query : match_queries) { - if (isGeoLocationExists(match_query.match)) return; - } -} - -set -ParameterException::getBehavior( - const unordered_map> &key_value_pairs, - set &matched_override_keywords) const -{ - set matched_behaviors; - - matched_override_keywords.clear(); - dbgTrace(D_RULEBASE_CONFIG) << "Matching exception"; - for (const MatchBehaviorPair &match_behavior_pair: match_queries) { - MatchQuery::MatchResult match_res = match_behavior_pair.match.getMatch(key_value_pairs); - if (match_res.is_match) { - dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception from a list of matches."; - // When matching indicators with action=ignore, we expect no behavior override. - // Instead, a matched keywords list should be returned which will be later removed from score calculation - if (match_res.matched_keywords->size() > 0 && match_behavior_pair.behavior == action_ignore) { - matched_override_keywords.insert(match_res.matched_keywords->begin(), - match_res.matched_keywords->end()); - } else { - matched_behaviors.insert(match_behavior_pair.behavior); - } - } - } - - if (match_queries.empty()) { - MatchQuery::MatchResult match_res = match.getMatch(key_value_pairs); - if (match_res.is_match) { - dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception."; - // When matching indicators with action=ignore, we expect no behavior override. - // Instead, a matched keywords list should be returned which will be later removed from score calculation - if (match_res.matched_keywords->size() > 0 && behavior == action_ignore) { - matched_override_keywords.insert(match_res.matched_keywords->begin(), - match_res.matched_keywords->end()); - } else { - matched_behaviors.insert(behavior); - } - } - } - - return matched_behaviors; -} - -set -ParameterException::getBehavior(const unordered_map> &key_value_pairs) const -{ - set keywords; - return getBehavior(key_value_pairs, keywords); -} diff --git a/components/generic_rulebase/rulebase_config.cc b/components/generic_rulebase/rulebase_config.cc deleted file mode 100755 index 2dd018f..0000000 --- a/components/generic_rulebase/rulebase_config.cc +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/rulebase_config.h" - -#include "telemetry.h" -#include "config.h" - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -using namespace std; - -set BasicRuleConfig::assets_ids{}; -set BasicRuleConfig::assets_ids_aggregation{}; - -void -BasicRuleConfig::load(cereal::JSONInputArchive &ar) -{ - parseJSONKey>("practices", practices, ar); - parseJSONKey>("triggers", triggers, ar); - parseJSONKey>("parameters", parameters, ar); - parseJSONKey("priority", priority, ar); - parseJSONKey("ruleId", rule_id, ar); - parseJSONKey("ruleName", rule_name, ar); - parseJSONKey("assetId", asset_id, ar); - parseJSONKey("assetName", asset_name, ar); - parseJSONKey("zoneId", zone_id, ar); - parseJSONKey("zoneName", zone_name, ar); - - assets_ids_aggregation.insert(asset_id); -} - -void -BasicRuleConfig::updateCountMetric() -{ - BasicRuleConfig::assets_ids = BasicRuleConfig::assets_ids_aggregation; - AssetCountEvent(AssetType::ALL, BasicRuleConfig::assets_ids.size()).notify(); -} - -bool -BasicRuleConfig::isPracticeActive(const string &practice_id) const -{ - for (auto practice: practices) { - if (practice.getId() == practice_id) return true; - } - return false; -} - -bool -BasicRuleConfig::isTriggerActive(const string &trigger_id) const -{ - for (auto trigger: triggers) { - if (trigger.getId() == trigger_id) { - return true; - } - } - return false; -} - -bool -BasicRuleConfig::isParameterActive(const string ¶meter_id) const -{ - for (auto param: parameters) { - if (param.getId() == parameter_id) { - return true; - } - } - return false; -} diff --git a/components/generic_rulebase/triggers_config.cc b/components/generic_rulebase/triggers_config.cc deleted file mode 100755 index 01429ec..0000000 --- a/components/generic_rulebase/triggers_config.cc +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "generic_rulebase/triggers_config.h" -#include "generic_rulebase/generic_rulebase_utils.h" - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -using namespace std; - -WebTriggerConf::WebTriggerConf() : response_title(""), response_body(""), response_code(0) {} -WebTriggerConf::WebTriggerConf(const string &title, const string &body, uint code) - : - response_title(title), - response_body(body), - response_code(code) -{} - -WebTriggerConf WebTriggerConf::default_trigger_conf = WebTriggerConf( - "Attack blocked by web application protection", // title - "Check Point's Application Security has detected an attack and blocked it.", // body - 403 -); - -void -WebTriggerConf::load(cereal::JSONInputArchive &archive_in) -{ - try { - parseJSONKey("details level", details_level, archive_in); - if (details_level == "Redirect") { - parseJSONKey("redirect URL", redirect_url, archive_in); - parseJSONKey("xEventId", add_event_id_to_header, archive_in); - parseJSONKey("eventIdInHeader", add_event_id_to_header, archive_in); - return; - } - parseJSONKey("response code", response_code, archive_in); - if (response_code < 100 || response_code > 599) { - throw cereal::Exception( - "illegal web trigger response code: " + - to_string(response_code) + - " is out of range (100-599)" - ); - } - - if (details_level == "Response Code") return; - - parseJSONKey("response body", response_body, archive_in); - parseJSONKey("response title", response_title, archive_in); - } catch (const exception &e) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to parse the web trigger configuration: '" << e.what() << "'"; - archive_in.setNextName(nullptr); - } -} - -bool -WebTriggerConf::operator==(const WebTriggerConf &other) const -{ - return - response_code == other.response_code && - response_title == other.response_title && - response_body == other.response_body; -} - -LogTriggerConf::LogTriggerConf(string trigger_name, bool log_detect, bool log_prevent) : name(trigger_name) -{ - if (log_detect) should_log_on_detect.setAll(); - if (log_prevent) should_log_on_prevent.setAll(); - active_streams.setFlag(ReportIS::StreamType::JSON_FOG); - active_streams.setFlag(ReportIS::StreamType::JSON_LOG_FILE); -} - -ReportIS::Severity -LogTriggerConf::getSeverity(bool is_action_drop_or_prevent) const -{ - return is_action_drop_or_prevent ? ReportIS::Severity::MEDIUM : ReportIS::Severity::LOW; -} - -ReportIS::Priority -LogTriggerConf::getPriority(bool is_action_drop_or_prevent) const -{ - return is_action_drop_or_prevent ? ReportIS::Priority::HIGH : ReportIS::Priority::MEDIUM; -} - -Flags -LogTriggerConf::getStreams(SecurityType security_type, bool is_action_drop_or_prevent) const -{ - if (is_action_drop_or_prevent && should_log_on_prevent.isSet(security_type)) return active_streams; - if (!is_action_drop_or_prevent && should_log_on_detect.isSet(security_type)) return active_streams; - - return Flags(); -} - -Flags -LogTriggerConf::getEnrechments(SecurityType security_type) const -{ - Flags enreachments; - - if (log_geo_location.isSet(security_type)) enreachments.setFlag(ReportIS::Enreachments::GEOLOCATION); - if (should_format_output) enreachments.setFlag(ReportIS::Enreachments::BEAUTIFY_OUTPUT); - - return enreachments; -} - -template -static void -setTriggersFlag(const string &key, cereal::JSONInputArchive &ar, EnumClass flag, Flags &flags) -{ - bool value = false; - parseJSONKey(key, value, ar); - if (value) flags.setFlag(flag); -} - -static void -setLogConfiguration( - const ReportIS::StreamType &log_type, - const string &log_server_url = "", - const string &protocol = "" -) -{ - dbgTrace(D_RULEBASE_CONFIG) << "log server url:" << log_server_url; - if (log_server_url != "" && protocol != "") { - Singleton::Consume::by()->addStream(log_type, log_server_url, protocol); - } else { - Singleton::Consume::by()->addStream(log_type); - } -} - -static string -parseProtocolWithDefault( - const std::string &default_value, - const std::string &key_name, - cereal::JSONInputArchive &archive_in -) -{ - string value; - try { - archive_in(cereal::make_nvp(key_name, value)); - } catch (const cereal::Exception &e) { - return default_value; - } - return value; -} - -void -LogTriggerConf::load(cereal::JSONInputArchive& archive_in) -{ - try { - parseJSONKey("triggerName", name, archive_in); - parseJSONKey("verbosity", verbosity, archive_in); - parseJSONKey("urlForSyslog", url_for_syslog, archive_in); - parseJSONKey("urlForCef", url_for_cef, archive_in); - parseJSONKey("syslogProtocol", syslog_protocol, archive_in); - syslog_protocol = parseProtocolWithDefault("UDP", "syslogProtocol", archive_in); - cef_protocol = parseProtocolWithDefault("UDP", "cefProtocol", archive_in); - - setTriggersFlag("webBody", archive_in, WebLogFields::webBody, log_web_fields); - setTriggersFlag("webHeaders", archive_in, WebLogFields::webHeaders, log_web_fields); - setTriggersFlag("webRequests", archive_in, WebLogFields::webRequests, log_web_fields); - setTriggersFlag("webUrlPath", archive_in, WebLogFields::webUrlPath, log_web_fields); - setTriggersFlag("webUrlQuery", archive_in, WebLogFields::webUrlQuery, log_web_fields); - setTriggersFlag("logToAgent", archive_in, ReportIS::StreamType::JSON_LOG_FILE, active_streams); - setTriggersFlag("logToCloud", archive_in, ReportIS::StreamType::JSON_FOG, active_streams); - setTriggersFlag("logToK8sService", archive_in, ReportIS::StreamType::JSON_K8S_SVC, active_streams); - setTriggersFlag("logToSyslog", archive_in, ReportIS::StreamType::SYSLOG, active_streams); - setTriggersFlag("logToCef", archive_in, ReportIS::StreamType::CEF, active_streams); - setTriggersFlag("acAllow", archive_in, SecurityType::AccessControl, should_log_on_detect); - setTriggersFlag("acDrop", archive_in, SecurityType::AccessControl, should_log_on_prevent); - setTriggersFlag("tpDetect", archive_in, SecurityType::ThreatPrevention, should_log_on_detect); - setTriggersFlag("tpPrevent", archive_in, SecurityType::ThreatPrevention, should_log_on_prevent); - setTriggersFlag("complianceWarnings", archive_in, SecurityType::Compliance, should_log_on_detect); - setTriggersFlag("complianceViolations", archive_in, SecurityType::Compliance, should_log_on_prevent); - setTriggersFlag("acLogGeoLocation", archive_in, SecurityType::AccessControl, log_geo_location); - setTriggersFlag("tpLogGeoLocation", archive_in, SecurityType::ThreatPrevention, log_geo_location); - setTriggersFlag("complianceLogGeoLocation", archive_in, SecurityType::Compliance, log_geo_location); - - bool extend_logging = false; - parseJSONKey("extendLogging", extend_logging, archive_in); - if (extend_logging) { - setTriggersFlag("responseCode", archive_in, WebLogFields::responseCode, log_web_fields); - setTriggersFlag("responseBody", archive_in, WebLogFields::responseBody, log_web_fields); - - string severity; - static const map extend_logging_severity_strings = { - {"High", extendLoggingSeverity::High}, - {"Critical", extendLoggingSeverity::Critical} - }; - parseJSONKey("extendLoggingMinSeverity", severity, archive_in); - auto extended_severity = extend_logging_severity_strings.find(severity); - if (extended_severity != extend_logging_severity_strings.end()) { - extend_logging_severity = extended_severity->second; - } else { - dbgWarning(D_RULEBASE_CONFIG) - << "Failed to parse the extendLoggingMinSeverityfield: '" - << severity - << "'"; - } - } - - for (ReportIS::StreamType log_stream : makeRange()) { - if (!active_streams.isSet(log_stream)) continue; - switch (log_stream) { - case ReportIS::StreamType::JSON_DEBUG: - setLogConfiguration(ReportIS::StreamType::JSON_DEBUG); - break; - case ReportIS::StreamType::JSON_FOG: - setLogConfiguration(ReportIS::StreamType::JSON_FOG); - break; - case ReportIS::StreamType::JSON_LOG_FILE: - setLogConfiguration(ReportIS::StreamType::JSON_LOG_FILE); - break; - case ReportIS::StreamType::JSON_K8S_SVC: - setLogConfiguration(ReportIS::StreamType::JSON_K8S_SVC); - break; - case ReportIS::StreamType::SYSLOG: - setLogConfiguration(ReportIS::StreamType::SYSLOG, getUrlForSyslog(), syslog_protocol); - break; - case ReportIS::StreamType::CEF: - setLogConfiguration(ReportIS::StreamType::CEF, getUrlForCef(), cef_protocol); - break; - case ReportIS::StreamType::NONE: break; - case ReportIS::StreamType::COUNT: break; - } - } - - parseJSONKey("formatLoggingOutput", should_format_output, archive_in); - } catch (const exception &e) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to parse the log trigger configuration: '" << e.what() << "'"; - archive_in.setNextName(nullptr); - } -} diff --git a/components/generic_rulebase/zone.cc b/components/generic_rulebase/zone.cc deleted file mode 100755 index c9f902c..0000000 --- a/components/generic_rulebase/zone.cc +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/zone.h" - -#include -#include -#include - -using namespace std; - -static const unordered_map string_to_direction = { - { "to", Zone::Direction::To }, - { "from", Zone::Direction::From }, - { "bidirectional", Zone::Direction::Bidirectional } -}; - -class AdjacentZone -{ -public: - void - load(cereal::JSONInputArchive &archive_in) - { - string direction_as_string; - archive_in(cereal::make_nvp("direction", direction_as_string)); - archive_in(cereal::make_nvp("zoneId", id)); - auto maybe_direction = string_to_direction.find(direction_as_string); - if (maybe_direction == string_to_direction.end()) { - reportConfigurationError( - "Illegal direction provided for adjacency. Provided direction in configuration: " + - direction_as_string - ); - } - dir = maybe_direction->second; - } - - pair getValue() const { return make_pair(dir, id); } - -private: - Zone::Direction dir; - GenericConfigId id; -}; - -class TagsValues -{ -public: - static const string req_attrs_ctx_key; - - TagsValues() {} - - template - void - serialize(Archive &ar) - { - I_Environment *env = Singleton::Consume::by(); - auto req_attrs = env->get>(req_attrs_ctx_key); - if (!req_attrs.ok()) return; - - for (const string &req_attr : *req_attrs) { - try { - string data; - ar(cereal::make_nvp(req_attr, data)); - dbgDebug(D_RULEBASE_CONFIG) - << "Found value for requested attribute. Tag: " - << req_attr - << ", Value: " - << data; - - tags_set[req_attr].insert(data); - } catch (const exception &e) { - dbgDebug(D_RULEBASE_CONFIG) << "Could not find values for requested attribute. Tag: " << req_attr; - ar.setNextName(nullptr); - } - } - } - - bool - matchValueByKey(const string &requested_key, const unordered_set &possible_values) const - { - auto values = tags_set.find(requested_key); - if (values == tags_set.end()) return false; - - for (const string &val : possible_values) { - if (values->second.count(val)) return true; - } - return false; - } - - void - insert(const TagsValues &other) - { - for (auto &single_tags_value : other.getData()) { - tags_set[single_tags_value.first].insert(single_tags_value.second.begin(), single_tags_value.second.end()); - } - } - - const unordered_map> & getData() const { return tags_set; } - -private: - unordered_map> tags_set; -}; - -const string TagsValues::req_attrs_ctx_key = "requested attributes key"; - -void -Zone::load(cereal::JSONInputArchive &archive_in) -{ - archive_in(cereal::make_nvp("id", zone_id)); - archive_in(cereal::make_nvp("name", zone_name)); - vector adjacency; - try { - archive_in(cereal::make_nvp("adjacentZones", adjacency)); - } catch (const cereal::Exception &) { - dbgTrace(D_RULEBASE_CONFIG) - << "List of adjacentZones does not exist for current object. Zone id: " - << zone_id - << ", Zone name: " - << zone_name; - - archive_in.setNextName(nullptr); - } - - for (const AdjacentZone &zone : adjacency) { - adjacent_zones.push_back(zone.getValue()); - } - - archive_in(cereal::make_nvp("match", match_query)); - - is_any = - match_query.getType() == MatchQuery::MatchType::Condition && - match_query.getKey() == "any" && - match_query.getValue().count("any") > 0; - - set keys = match_query.getAllKeys(); -} - -const string -contextKeyToString(Context::MetaDataType type) -{ - if (type == Context::MetaDataType::SubjectIpAddr || type == Context::MetaDataType::OtherIpAddr) return "ip"; - return Context::convertToString(type); -} - -bool -Zone::contains(const Asset &asset) -{ - QueryRequest request; - - for (const auto &main_attr : asset.getAttrs()) { - request.addCondition(Condition::EQUALS, contextKeyToString(main_attr.first), main_attr.second); - } - - ScopedContext req_attrs_key; - req_attrs_key.registerValue>(TagsValues::req_attrs_ctx_key, match_query.getAllKeys()); - - I_Intelligence_IS_V2 *intelligence = Singleton::Consume::by(); - auto query_res = intelligence->queryIntelligence(request); - if (!query_res.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to perform intelligence query. Error: " << query_res.getErr(); - return false; - } - - for (const AssetReply &asset : query_res.unpack()) { - TagsValues tag_values = asset.mergeReplyData(); - - if (match_query.matchAttributes(tag_values.getData())) return true; - } - return false; -} diff --git a/components/generic_rulebase/zones_config.cc b/components/generic_rulebase/zones_config.cc deleted file mode 100755 index 97bd88e..0000000 --- a/components/generic_rulebase/zones_config.cc +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generic_rulebase/zones_config.h" - -#include -#include - -#include "generic_rulebase/generic_rulebase_utils.h" -#include "config.h" -#include "ip_utilities.h" -#include "connkey.h" -#include "i_generic_rulebase.h" - -USE_DEBUG_FLAG(D_RULEBASE_CONFIG); - -using namespace std; - -void -ZonesConfig::load(cereal::JSONInputArchive &archive_in) -{ - dbgFlow(D_RULEBASE_CONFIG) << "Saving active zones"; - set used_zones; - cereal::load(archive_in, used_zones); - - dbgTrace(D_RULEBASE_CONFIG) << "Loading all zones"; - auto all_zones_maybe = getSetting("rulebase", "zones"); - if (!all_zones_maybe.ok()) { - dbgWarning(D_RULEBASE_CONFIG) << "Failed to load zones"; - return; - } - - dbgTrace(D_RULEBASE_CONFIG) << "Creating cache of all zones by ID"; - map all_zones; - for (const auto &single_zone : all_zones_maybe.unpack().zones) { - if (used_zones.count(single_zone.getId()) > 0 && single_zone.isAnyZone()) { - dbgTrace(D_RULEBASE_CONFIG) << "Found used zone of type \"Any\": saving all zones as active zones"; - zones = all_zones_maybe.unpack().zones; - return; - } - - dbgDebug(D_RULEBASE_CONFIG) - << "Adding specific zone to cache. Zone ID: " - << single_zone.getId() - << ", name: " - << single_zone.getName(); - all_zones.emplace(single_zone.getId(), single_zone); - } - - dbgTrace(D_RULEBASE_CONFIG) << "Creating list of active zones"; - map active_zones_set; - for (const auto &single_used_zone_id : used_zones) { - const auto &found_zone = all_zones[single_used_zone_id]; - dbgTrace(D_RULEBASE_CONFIG) - << "Adding zone to list of active zones. Zone ID: " - << single_used_zone_id - << ", zone name: " - << found_zone.getName(); - active_zones_set.emplace(found_zone.getId(), found_zone); - - for (const auto &adjacent_zone : found_zone.getAdjacentZones()) { - const auto &adjacent_zone_obj = all_zones[adjacent_zone.second]; - dbgTrace(D_RULEBASE_CONFIG) - << "Adding adjacent zone to list of active zones. Zone ID: " - << adjacent_zone_obj.getId() - << ", zone name: " - << adjacent_zone_obj.getName(); - active_zones_set.emplace(adjacent_zone_obj.getId(), adjacent_zone_obj); - } - } - - vector implied_zones = { - "impliedAzure", - "impliedDNS", - "impliedSSH", - "impliedProxy", - "impliedFog" - }; - - GenericConfigId any_zone_id = ""; - for (const auto &single_zone : all_zones_maybe.unpack().zones) { - if (single_zone.isAnyZone()) any_zone_id = single_zone.getId(); - } - for (GenericConfigId &implied_id: implied_zones) { - if (all_zones.find(implied_id) != all_zones.end()) { - dbgDebug(D_RULEBASE_CONFIG) << "Adding implied zone to cache. Zone ID: " << implied_id; - active_zones_set.emplace(implied_id, all_zones[implied_id]); - if (any_zone_id != "" && active_zones_set.count(any_zone_id) == 0) { - active_zones_set.emplace(any_zone_id, all_zones[any_zone_id]); - } - } - } - - for (const auto &single_id_zone_pair : active_zones_set) { - zones.push_back(single_id_zone_pair.second); - } -} - -void -ZonesConfig::preload() -{ - registerExpectedSetting("rulebase", "zones"); - registerExpectedSetting("rulebase", "usedZones"); -} diff --git a/components/include/details_resolver.h b/components/include/details_resolver.h index 55e3b1a..98e6d1f 100644 --- a/components/include/details_resolver.h +++ b/components/include/details_resolver.h @@ -34,6 +34,7 @@ public: ~DetailsResolver(); void preload() override; + void init() override; private: class Impl; diff --git a/components/include/downloader.h b/components/include/downloader.h index 2217772..f7af462 100755 --- a/components/include/downloader.h +++ b/components/include/downloader.h @@ -21,6 +21,7 @@ #include "url_parser.h" #include "i_agent_details.h" #include "i_mainloop.h" +#include "i_environment.h" #include "singleton.h" #include "component.h" @@ -32,6 +33,7 @@ class Downloader Singleton::Consume, Singleton::Consume, Singleton::Consume, + Singleton::Consume, Singleton::Consume { public: diff --git a/components/include/i_orchestration_tools.h b/components/include/i_orchestration_tools.h index 692d513..20b9982 100755 --- a/components/include/i_orchestration_tools.h +++ b/components/include/i_orchestration_tools.h @@ -117,7 +117,7 @@ public: const std::string &conf_path) const = 0; virtual bool copyFile(const std::string &src_path, const std::string &dst_path) const = 0; virtual bool doesFileExist(const std::string &file_path) const = 0; - virtual void getClusterId() const = 0; + virtual void setClusterId() const = 0; virtual void fillKeyInJson( const std::string &filename, const std::string &_key, diff --git a/components/include/orchestration_status.h b/components/include/orchestration_status.h index 84879a4..dcf99db 100755 --- a/components/include/orchestration_status.h +++ b/components/include/orchestration_status.h @@ -40,7 +40,7 @@ public: ~OrchestrationStatus(); void init() override; - + private: class Impl; std::unique_ptr pimpl; diff --git a/components/include/package_handler.h b/components/include/package_handler.h index 6f852ca..b292655 100755 --- a/components/include/package_handler.h +++ b/components/include/package_handler.h @@ -17,6 +17,7 @@ #include "i_package_handler.h" #include "i_orchestration_tools.h" #include "i_shell_cmd.h" +#include "i_environment.h" #include "component.h" class PackageHandler @@ -24,7 +25,8 @@ class PackageHandler public Component, Singleton::Provide, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: PackageHandler(); diff --git a/components/include/url_parser.h b/components/include/url_parser.h index 554df67..3f65130 100755 --- a/components/include/url_parser.h +++ b/components/include/url_parser.h @@ -35,8 +35,10 @@ public: bool isOverSSL() const { return over_ssl; } std::string getPort() const { return port; } std::string getQuery() const { return query; } + std::string getHost() const; URLProtocol getProtocol() const { return protocol; } std::string toString() const; + void setHost(const std::string &new_host); void setQuery(const std::string &new_query); private: @@ -47,6 +49,7 @@ private: std::string base_url; std::string port; std::string query; + std::string host; URLProtocol protocol; }; diff --git a/components/security_apps/ips/include/ips_basic_policy.h b/components/security_apps/ips/include/ips_basic_policy.h index c57f586..b19566e 100644 --- a/components/security_apps/ips/include/ips_basic_policy.h +++ b/components/security_apps/ips/include/ips_basic_policy.h @@ -50,9 +50,13 @@ public: private: void readRules(cereal::JSONInputArchive &ar); + void readTriggerId(cereal::JSONInputArchive &ar); + void readExceptionId(cereal::JSONInputArchive &ar); void readDefaultAction(cereal::JSONInputArchive &ar); std::vector rules; + std::string trigger_id; + std::string exception_id; }; #endif // __IPS_BASIC_POLICY_H__ diff --git a/components/security_apps/ips/include/ips_signatures.h b/components/security_apps/ips/include/ips_signatures.h index f6347b2..cf78a81 100644 --- a/components/security_apps/ips/include/ips_signatures.h +++ b/components/security_apps/ips/include/ips_signatures.h @@ -27,6 +27,7 @@ #include "log_generator.h" #include "parsed_context.h" #include "pm_hook.h" +#include "i_generic_rulebase.h" /// \namespace IPSSignatureSubTypes /// \brief Namespace containing subtypes for IPS signatures. @@ -348,8 +349,16 @@ public: /// \brief Construct a SignatureAndAction object. /// \param _signature The complete signature. /// \param _action The signature action. - SignatureAndAction(std::shared_ptr _signature, SignatureAction _action) : - signature(_signature), action(_action) + SignatureAndAction( + std::shared_ptr _signature, + SignatureAction _action, + std::string _trigger_id, + std::string _exception_id) + : + signature(_signature), + action(_action), + trigger_id(_trigger_id), + exception_id(_exception_id) {} /// \brief Check if the signature is matched for prevention. @@ -375,6 +384,11 @@ public: return signature->getContext(); } + LogTriggerConf getTrigger() const; + + std::set + getBehavior(const std::unordered_map> &exceptions_dict) const; + private: /// \brief Get the action results for the IPS state. /// \param ips_state The IPS entry. @@ -382,6 +396,8 @@ private: std::shared_ptr signature; SignatureAction action; + std::string trigger_id; + std::string exception_id; }; } // namespace IPSSignatureSubTypes diff --git a/components/security_apps/ips/include/snort_basic_policy.h b/components/security_apps/ips/include/snort_basic_policy.h index 96d11b2..263fc2c 100644 --- a/components/security_apps/ips/include/snort_basic_policy.h +++ b/components/security_apps/ips/include/snort_basic_policy.h @@ -17,6 +17,8 @@ public: private: IPSSignatureSubTypes::SignatureAction action = IPSSignatureSubTypes::SignatureAction::IGNORE; std::vector file_names; + std::string trigger_id; + std::string exception_id; }; #endif // __SNORT_BASIC_POLICY_H__ diff --git a/components/security_apps/ips/ips_basic_policy.cc b/components/security_apps/ips/ips_basic_policy.cc index b285429..42dabe7 100644 --- a/components/security_apps/ips/ips_basic_policy.cc +++ b/components/security_apps/ips/ips_basic_policy.cc @@ -17,6 +17,8 @@ void RuleSelector::load(cereal::JSONInputArchive &ar) { readRules(ar); + readTriggerId(ar); + readExceptionId(ar); readDefaultAction(ar); } @@ -36,7 +38,7 @@ RuleSelector::selectSignatures() const if (rule.isSignaturedMatched(*signature)) { if (rule.getAction() != IPSSignatureSubTypes::SignatureAction::IGNORE) { signature->setIndicators("Check Point", signatures_version); - res.emplace_back(signature, rule.getAction()); + res.emplace_back(signature, rule.getAction(), trigger_id, exception_id); } break; } @@ -52,6 +54,28 @@ RuleSelector::readRules(cereal::JSONInputArchive &ar) ar(cereal::make_nvp("rules", rules)); } +void +RuleSelector::readTriggerId(cereal::JSONInputArchive &ar) +{ + try { + ar(cereal::make_nvp("triggers", trigger_id)); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + trigger_id = ""; + } +} + +void +RuleSelector::readExceptionId(cereal::JSONInputArchive &ar) +{ + try { + ar(cereal::make_nvp("exceptions", exception_id)); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + exception_id = ""; + } +} + void RuleSelector::readDefaultAction(cereal::JSONInputArchive &ar) { diff --git a/components/security_apps/ips/ips_signatures.cc b/components/security_apps/ips/ips_signatures.cc index 0412a60..565d5c0 100644 --- a/components/security_apps/ips/ips_signatures.cc +++ b/components/security_apps/ips/ips_signatures.cc @@ -280,8 +280,7 @@ SignatureAndAction::getAction(const IPSEntry &ips_state) const exceptions_dict["sourceIdentifier"].insert(*env_source_identifier); } - I_GenericRulebase *i_rulebase = Singleton::Consume::by(); - auto behaviors = i_rulebase->getBehavior(exceptions_dict); + auto behaviors = getBehavior(exceptions_dict); set override_actions; vector override_ids; @@ -315,6 +314,23 @@ static const auto url_query = LogTriggerConf::WebLogFields::webUrlQuery; static const auto res_body = LogTriggerConf::WebLogFields::responseBody; static const auto res_code = LogTriggerConf::WebLogFields::responseCode; +LogTriggerConf +SignatureAndAction::getTrigger() const +{ + if (trigger_id.empty()) return getConfigurationWithDefault(LogTriggerConf(), "rulebase", "log"); + + return Singleton::Consume::by()->getLogTriggerConf(trigger_id); +} + +set +SignatureAndAction::getBehavior(const unordered_map> &exceptions_dict) const +{ + I_GenericRulebase *i_rulebase = Singleton::Consume::by(); + if (exception_id.empty()) return i_rulebase->getBehavior(exceptions_dict); + + return i_rulebase->getParameterException(exception_id).getBehavior(exceptions_dict); +} + bool SignatureAndAction::matchSilent(const Buffer &sample) const { @@ -398,7 +414,7 @@ SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set(override_action) == IPSSignatureSubTypes::SignatureAction::PREVENT; auto severity = signature->getSeverity() < IPSLevel::HIGH ? Severity::HIGH : Severity::CRITICAL; diff --git a/components/security_apps/ips/ips_ut/component_ut.cc b/components/security_apps/ips/ips_ut/component_ut.cc index dd6be98..70f64bf 100644 --- a/components/security_apps/ips/ips_ut/component_ut.cc +++ b/components/security_apps/ips/ips_ut/component_ut.cc @@ -596,6 +596,8 @@ TEST_F(ComponentTest, check_filtering_by_year) TEST_F(ComponentTest, log_fields) { + generic_rulebase.preload(); + generic_rulebase.init(); string config = "{" "\"IPS\": {" @@ -632,6 +634,8 @@ TEST_F(ComponentTest, log_fields) "\"assetId\": \"1-1-1\"," "\"practiceId\": \"2-2-2\"," "\"practiceName\": \"practice1\"," + "\"triggers\": \"5eaeefde6765c30010bae8b6\"," + "\"exceptions\": \"\"," "\"defaultAction\": \"Detect\"," "\"rules\": [" "{" @@ -643,10 +647,36 @@ TEST_F(ComponentTest, log_fields) "]" "}" "]" + "}," + "\"rulebase\": {" + "\"log\": [" + "{" + "\"context\": \"triggerId(5eaeefde6765c30010bae8b6)\"," + "\"triggerName\": \"Logging Trigger\"," + "\"triggerType\": \"log\"," + "\"urlForSyslog\": \"\"," + "\"urlForCef\": \"128.1.1.1:333\"," + "\"acAllow\": false," + "\"acDrop\": true," + "\"complianceViolations\": true," + "\"complianceWarnings\": true," + "\"logToAgent\": true," + "\"logToCloud\": true," + "\"logToSyslog\": false," + "\"logToCef\": true," + "\"tpDetect\": true," + "\"tpPrevent\": true," + "\"verbosity\": \"Standard\"," + "\"webBody\": true," + "\"webHeaders\": true," + "\"webRequests\": true," + "\"webUrlPath\": true," + "\"webUrlQuery\": true" + "}" + "]" "}" "}"; loadPolicy(config); - setTrigger(); EXPECT_CALL(table, createStateRValueRemoved(_, _)); EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry)); @@ -829,6 +859,8 @@ TEST_F(ComponentTest, prxeem_exception_bug) " \"practiceId\": \"2-2-2\"," " \"practiceName\": \"practice1\"," " \"defaultAction\": \"Prevent\"," + " \"triggers\": \"\"," + " \"exceptions\": \"6c3867be-4da5-42c2-93dc-8f509a764004\"," " \"rules\": []" " }" " ]" @@ -847,6 +879,11 @@ TEST_F(ComponentTest, prxeem_exception_bug) " \"parameterId\": \"6c3867be-4da5-42c2-93dc-8f509a764003\"," " \"parameterType\": \"exceptions\"," " \"parameterName\": \"exception\"" + " }," + " {" + " \"parameterId\": \"6c3867be-4da5-42c2-93dc-8f509a764004\"," + " \"parameterType\": \"exceptions\"," + " \"parameterName\": \"exception\"" " }" " ]," " \"zoneId\": \"\"," @@ -855,7 +892,7 @@ TEST_F(ComponentTest, prxeem_exception_bug) " ]," " \"exception\": [" " {" - " \"context\": \"parameterId(6c3867be-4da5-42c2-93dc-8f509a764003)\"," + " \"context\": \"parameterId(6c3867be-4da5-42c2-93dc-8f509a764004)\"," " \"match\": {" " \"type\": \"operator\"," " \"op\": \"and\"," diff --git a/components/security_apps/ips/snort_basic_policy.cc b/components/security_apps/ips/snort_basic_policy.cc index 592fbc7..b89560a 100644 --- a/components/security_apps/ips/snort_basic_policy.cc +++ b/components/security_apps/ips/snort_basic_policy.cc @@ -16,6 +16,19 @@ using namespace std; void SnortRuleSelector::load(cereal::JSONInputArchive &ar) { + try { + ar(cereal::make_nvp("triggers", trigger_id)); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + trigger_id = ""; + } + + try { + ar(cereal::make_nvp("exceptions", exception_id)); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + exception_id = ""; + } string mode; ar(cereal::make_nvp("mode", mode), cereal::make_nvp("files", file_names)); @@ -38,7 +51,7 @@ SnortRuleSelector::selectSignatures() const for (auto &file : file_names) { for (auto &signature : (*signatures).getSignatures(file)) { - res.emplace_back(signature, action); + res.emplace_back(signature, action, trigger_id, exception_id); } } return res; diff --git a/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc b/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc index 08d46d2..77a3eaf 100755 --- a/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc +++ b/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc @@ -228,6 +228,7 @@ AccessControlPracticeSpec::load(cereal::JSONInputArchive &archive_in) dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec"; parseAppsecJSONKey("name", practice_name, archive_in); + parseAppsecJSONKey("practiceMode", mode, archive_in); parseAppsecJSONKey("appsecClassName", appsec_class_name, archive_in); parseMandatoryAppsecJSONKey("rateLimit", rate_limit, archive_in); } @@ -255,4 +256,10 @@ AccessControlPracticeSpec::getName() const { return practice_name; } + +const string & +AccessControlPracticeSpec::getMode(const std::string &default_mode) const +{ + return isModeInherited(mode) ? default_mode : mode; +} // LCOV_EXCL_STOP diff --git a/components/security_apps/local_policy_mgmt_gen/include/access_control_practice.h b/components/security_apps/local_policy_mgmt_gen/include/access_control_practice.h index 9050e6e..eaa6843 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/access_control_practice.h +++ b/components/security_apps/local_policy_mgmt_gen/include/access_control_practice.h @@ -181,12 +181,14 @@ public: const AccessControlRateLimit &getRateLimit() const; const std::string & getAppSecClassName() const; const std::string & getName() const; + const std::string & getMode(const std::string &default_mode = "inactive") const; void setName(const std::string &_name); private: AccessControlRateLimit rate_limit; std::string appsec_class_name; std::string practice_name; + std::string mode; }; #endif // __ACCESS_CONTROL_PRACTICE_H__ diff --git a/components/security_apps/local_policy_mgmt_gen/include/new_practice.h b/components/security_apps/local_policy_mgmt_gen/include/new_practice.h index 9b098b1..e8585fe 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/new_practice.h +++ b/components/security_apps/local_policy_mgmt_gen/include/new_practice.h @@ -587,6 +587,7 @@ public: const NewFileSecurity & getFileSecurity() const; const std::string & getAppSecClassName() const; const std::string & getName() const; + const std::string & getMode(const std::string &default_mode = "inactive") const; void setName(const std::string &_name); private: @@ -598,6 +599,7 @@ private: NewAppSecPracticeAntiBot anti_bot; std::string appsec_class_name; std::string practice_name; + std::string mode; }; #endif // __NEW_PRACTICE_H__ diff --git a/components/security_apps/local_policy_mgmt_gen/new_practice.cc b/components/security_apps/local_policy_mgmt_gen/new_practice.cc index 572cec6..8e266fd 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_practice.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_practice.cc @@ -1025,6 +1025,7 @@ NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in) parseMandatoryAppsecJSONKey("webAttacks", web_attacks, archive_in); parseAppsecJSONKey("antiBot", anti_bot, archive_in); parseAppsecJSONKey("name", practice_name, archive_in); + parseAppsecJSONKey("practiceMode", mode, archive_in, "inherited"); } void @@ -1080,4 +1081,11 @@ NewAppSecPracticeSpec::getName() const { return practice_name; } + +const string & +NewAppSecPracticeSpec::getMode(const string &default_mode) const +{ + return isModeInherited(mode) ? default_mode : mode; +} + // LCOV_EXCL_STOP diff --git a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc index 89927c7..739a1cd 100755 --- a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc @@ -1002,8 +1002,9 @@ PolicyMakerUtils::createIpsSections( auto apssec_practice = getAppsecPracticeSpec( rule_annotations[AnnotationTypes::PRACTICE], policy); + const string &override_mode = + apssec_practice.getIntrusionPrevention().getMode(apssec_practice.getMode(default_mode)); - const string &override_mode = apssec_practice.getIntrusionPrevention().getMode(default_mode); if (override_mode == "Inactive" || override_mode == "Disabled") return; IpsProtectionsSection ips_section = IpsProtectionsSection( @@ -1076,8 +1077,9 @@ PolicyMakerUtils::createSnortSections( auto apssec_practice = getAppsecPracticeSpec( rule_annotations[AnnotationTypes::PRACTICE], policy); + const string &override_mode = + apssec_practice.getSnortSignatures().getOverrideMode(apssec_practice.getMode(default_mode)); - const string &override_mode = apssec_practice.getSnortSignatures().getOverrideMode(default_mode); if (override_mode == "Inactive" || override_mode == "Disabled" || apssec_practice.getSnortSignatures().getFiles().size() == 0) { @@ -1129,7 +1131,7 @@ PolicyMakerUtils::createFileSecuritySections( asset_id, practice_name, practice_id, - default_mode + apssec_practice.getMode(default_mode) ); file_security[asset_name] = file_security_section; @@ -1171,7 +1173,7 @@ PolicyMakerUtils::createRateLimitSection( asset_name, url, uri, - access_control_practice.getRateLimit().getMode(default_mode), + access_control_practice.getRateLimit().getMode(access_control_practice.getMode(default_mode)), practice_id, rule_annotations[AnnotationTypes::ACCESS_CONTROL_PRACTICE], rules @@ -1191,6 +1193,8 @@ PolicyMakerUtils::createWebAppSection( rule_annotations[AnnotationTypes::PRACTICE], policy ); + const string &practice_mode = apssec_practice.getMode(default_mode); + PracticeAdvancedConfig practice_advance_config( apssec_practice.getWebAttacks().getMaxHeaderSizeBytes(), apssec_practice.getWebAttacks().getMaxBodySizeKb(), @@ -1206,8 +1210,8 @@ PolicyMakerUtils::createWebAppSection( practice_id, rule_annotations[AnnotationTypes::PRACTICE], rule_config.getContext(), - apssec_practice.getWebAttacks().getMinimumConfidence(default_mode), - apssec_practice.getWebAttacks().getMode(default_mode), + apssec_practice.getWebAttacks().getMinimumConfidence(practice_mode), + apssec_practice.getWebAttacks().getMode(practice_mode), practice_advance_config, apssec_practice.getAntiBot(), log_triggers[rule_annotations[AnnotationTypes::TRIGGER]], diff --git a/components/security_apps/orchestration/details_resolver/details_resolver.cc b/components/security_apps/orchestration/details_resolver/details_resolver.cc index 8ef831a..a135485 100644 --- a/components/security_apps/orchestration/details_resolver/details_resolver.cc +++ b/components/security_apps/orchestration/details_resolver/details_resolver.cc @@ -32,6 +32,7 @@ class DetailsResolver::Impl Singleton::Provide::From { public: + void init() { handler.init(); } Maybe getHostname() override; Maybe getPlatform() override; Maybe getArch() override; @@ -290,6 +291,12 @@ DetailsResolver::DetailsResolver() : Component("DetailsResolver"), pimpl(make_un DetailsResolver::~DetailsResolver() {} +void +DetailsResolver::init() +{ + pimpl->init(); +} + void DetailsResolver::preload() { diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index 55a3606..3b8d297 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -223,7 +223,7 @@ FILE_CONTENT_HANDLER( FILE_CONTENT_HANDLER("os_release", "/etc/os-release", getOsRelease) #endif // gaia || smb -FILE_CONTENT_HANDLER("AppSecModelVersion", "/etc/cp/conf/waap/waap.data", getWaapModelVersion) +FILE_CONTENT_HANDLER("AppSecModelVersion", "/conf/waap/waap.data", getWaapModelVersion) #endif // FILE_CONTENT_HANDLER diff --git a/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc b/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc index e5e486c..574d299 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc +++ b/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc @@ -36,9 +36,12 @@ using FileContentHandler = function(shared_ptr file_otput #include "checkpoint_product_handlers.h" +static const string filesystem_place_holder = ""; + class DetailsResolvingHanlder::Impl { public: + void init(); map getResolvedDetails() const; static Maybe getCommandOutput(const string &cmd); @@ -70,6 +73,20 @@ private: }; #undef SHELL_POST_CMD +void +DetailsResolvingHanlder::Impl::init() +{ + string actual_filesystem_prefix = getFilesystemPathConfig(); + + for (auto &file_handler : file_content_handlers) { + string &path = file_handler.second.first; + size_t place_holder_size = filesystem_place_holder.size(); + if (path.substr(0, place_holder_size) == filesystem_place_holder) { + path = actual_filesystem_prefix + path.substr(place_holder_size); + } + } +} + map DetailsResolvingHanlder::Impl::getResolvedDetails() const { @@ -155,6 +172,12 @@ DetailsResolvingHanlder::Impl::getCommandOutput(const string &cmd) DetailsResolvingHanlder::DetailsResolvingHanlder() : pimpl(make_unique()) {} DetailsResolvingHanlder::~DetailsResolvingHanlder() {} +void +DetailsResolvingHanlder::init() +{ + return pimpl->init(); +} + map DetailsResolvingHanlder::getResolvedDetails() const { diff --git a/components/security_apps/orchestration/details_resolver/details_resolving_handler.h b/components/security_apps/orchestration/details_resolver/details_resolving_handler.h index 0e70bce..5f6463f 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolving_handler.h +++ b/components/security_apps/orchestration/details_resolver/details_resolving_handler.h @@ -31,6 +31,7 @@ public: DetailsResolvingHanlder(); ~DetailsResolvingHanlder(); + void init(); std::map getResolvedDetails() const; static Maybe getCommandOutput(const std::string &cmd); diff --git a/components/security_apps/orchestration/downloader/CMakeLists.txt b/components/security_apps/orchestration/downloader/CMakeLists.txt index c050241..7a93e25 100755 --- a/components/security_apps/orchestration/downloader/CMakeLists.txt +++ b/components/security_apps/orchestration/downloader/CMakeLists.txt @@ -1,5 +1,5 @@ ADD_DEFINITIONS(-Wno-deprecated-declarations -Dalpine) -add_library(orchestration_downloader curl_client.cc downloader.cc http_client.cc https_client.cc) +add_library(orchestration_downloader curl_client.cc downloader.cc http_client.cc https_client.cc https_client_helper.cc) #add_subdirectory(downloader_ut) diff --git a/components/security_apps/orchestration/downloader/downloader.cc b/components/security_apps/orchestration/downloader/downloader.cc index 71da685..09670bb 100755 --- a/components/security_apps/orchestration/downloader/downloader.cc +++ b/components/security_apps/orchestration/downloader/downloader.cc @@ -121,6 +121,11 @@ Downloader::Impl::init() "Default file download path" ); + auto maybe_vs_id = Singleton::Consume::by()->get("VS ID"); + if (maybe_vs_id.ok()) { + dir_path = dir_path + "/vs" + maybe_vs_id.unpack(); + } + Singleton::Consume::by()->createDirectory(dir_path); } diff --git a/components/security_apps/orchestration/downloader/http_client.cc b/components/security_apps/orchestration/downloader/http_client.cc index 49a5475..cfa03c3 100755 --- a/components/security_apps/orchestration/downloader/http_client.cc +++ b/components/security_apps/orchestration/downloader/http_client.cc @@ -189,14 +189,12 @@ HTTPClient::getFile(const URLParser &url, ofstream &out_file, bool auth_required } if (url.isOverSSL()) { - auto get_file_over_ssl_res = getFileSSL(url, out_file, token); - if (!get_file_over_ssl_res.ok()) - { - //CURL fallback - dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Trying via CURL (SSL)."; - return curlGetFileOverSSL(url, out_file, token); - } - return get_file_over_ssl_res; + if (getFileSSLDirect(url, out_file, token).ok()) return Maybe(); + dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL directly. Trying indirectly."; + if (getFileSSL(url, out_file, token).ok()) return Maybe(); + //CURL fallback + dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Trying via CURL (SSL)."; + return curlGetFileOverSSL(url, out_file, token); } auto get_file_http_res = getFileHttp(url, out_file, token); if (!get_file_http_res.ok()) diff --git a/components/security_apps/orchestration/downloader/http_client.h b/components/security_apps/orchestration/downloader/http_client.h index ed01606..fb99007 100755 --- a/components/security_apps/orchestration/downloader/http_client.h +++ b/components/security_apps/orchestration/downloader/http_client.h @@ -34,6 +34,7 @@ public: private: std::string loadCAChainDir(); Maybe getFileSSL(const URLParser &url, std::ofstream &out_file, const std::string &_token); + Maybe getFileSSLDirect(const URLParser &url, std::ofstream &out_file, const std::string &_token); Maybe getFileHttp(const URLParser &url, std::ofstream &out_file, const std::string &_token); Maybe curlGetFileOverHttp(const URLParser &url, std::ofstream &out_file, const std::string &_token); Maybe curlGetFileOverSSL(const URLParser &url, std::ofstream &out_file, const std::string &_token); diff --git a/components/security_apps/orchestration/downloader/https_client.cc b/components/security_apps/orchestration/downloader/https_client.cc index fb96a80..522521d 100755 --- a/components/security_apps/orchestration/downloader/https_client.cc +++ b/components/security_apps/orchestration/downloader/https_client.cc @@ -90,7 +90,7 @@ public: ostream request_stream(&request_); stringstream http_request; http_request << "GET " << url.getQuery() << " HTTP/1.1\r\n"; - string host = url.getBaseURL().unpack(); + string host = url.getHost(); string port = url.getPort(); int port_int; try { diff --git a/components/security_apps/orchestration/downloader/https_client_helper.cc b/components/security_apps/orchestration/downloader/https_client_helper.cc new file mode 100644 index 0000000..975054e --- /dev/null +++ b/components/security_apps/orchestration/downloader/https_client_helper.cc @@ -0,0 +1,20 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "http_client.h" + +Maybe +HTTPClient::getFileSSLDirect(const URLParser &, std::ofstream &, const std::string &) +{ + return genError("No direct downloading in open-source"); +} diff --git a/components/security_apps/orchestration/hybrid_mode_telemetry.cc b/components/security_apps/orchestration/hybrid_mode_telemetry.cc index b49ecb1..800f253 100755 --- a/components/security_apps/orchestration/hybrid_mode_telemetry.cc +++ b/components/security_apps/orchestration/hybrid_mode_telemetry.cc @@ -47,6 +47,7 @@ HybridModeMetric::upon(const HybridModeMetricEvent &) string cmd_output = maybe_cmd_output.unpack(); trim(cmd_output); dbgDebug(D_ORCHESTRATOR) << "Watchdog process counter: " << cmd_output; + if (cmd_output.empty()) return; try { wd_process_restart.report(stoi(cmd_output)); diff --git a/components/security_apps/orchestration/include/namespace_data.h b/components/security_apps/orchestration/include/namespace_data.h index a895467..403701b 100644 --- a/components/security_apps/orchestration/include/namespace_data.h +++ b/components/security_apps/orchestration/include/namespace_data.h @@ -26,7 +26,7 @@ class NamespaceData : public ClientRest { public: bool loadJson(const std::string &json); - Maybe getNamespaceUidByName(const std::string &name); + Maybe getNamespaceUidByName(const std::string &name) const; private: std::map ns_name_to_uid; diff --git a/components/security_apps/orchestration/include/orchestration_policy.h b/components/security_apps/orchestration/include/orchestration_policy.h index 581cfaf..867aa43 100755 --- a/components/security_apps/orchestration/include/orchestration_policy.h +++ b/components/security_apps/orchestration/include/orchestration_policy.h @@ -21,8 +21,8 @@ class OrchestrationPolicy { public: const std::string & getFogAddress() const; - const unsigned long & getSleepInterval() const; - const unsigned long & getErrorSleepInterval() const; + unsigned int getSleepInterval() const; + unsigned int getErrorSleepInterval() const; void serialize(cereal::JSONInputArchive & archive); @@ -31,8 +31,8 @@ public: private: std::string fog_address; - unsigned long sleep_interval; - unsigned long error_sleep_interval; + unsigned int sleep_interval; + unsigned int error_sleep_interval; }; #endif // __ORCHESTRATION_POLICY_H__ diff --git a/components/security_apps/orchestration/modules/modules_ut/orchestration_policy_ut.cc b/components/security_apps/orchestration/modules/modules_ut/orchestration_policy_ut.cc index fdd1454..5b89b5c 100755 --- a/components/security_apps/orchestration/modules/modules_ut/orchestration_policy_ut.cc +++ b/components/security_apps/orchestration/modules/modules_ut/orchestration_policy_ut.cc @@ -43,8 +43,8 @@ TEST_F(PolicyTest, serialization) ASSERT_TRUE(false) << "Cereal threw an exception: " << e.what(); } - EXPECT_EQ(15u, orchestration_policy.getErrorSleepInterval()); - EXPECT_EQ(20u, orchestration_policy.getSleepInterval()); + EXPECT_EQ(15, orchestration_policy.getErrorSleepInterval()); + EXPECT_EQ(20, orchestration_policy.getSleepInterval()); EXPECT_EQ("http://10.0.0.18:81/control/", orchestration_policy.getFogAddress()); } @@ -63,8 +63,8 @@ TEST_F(PolicyTest, noAgentType) ASSERT_TRUE(false) << "Cereal threw an exception: " << e.what(); } - EXPECT_EQ(15u, orchestration_policy.getErrorSleepInterval()); - EXPECT_EQ(20u, orchestration_policy.getSleepInterval()); + EXPECT_EQ(15, orchestration_policy.getErrorSleepInterval()); + EXPECT_EQ(20, orchestration_policy.getSleepInterval()); EXPECT_EQ("http://10.0.0.18:81/control/", orchestration_policy.getFogAddress()); } @@ -83,8 +83,8 @@ TEST_F(PolicyTest, zeroSleepIntervels) ASSERT_TRUE(false) << "Cereal threw an exception: " << e.what(); } - EXPECT_EQ(0u, orchestration_policy.getErrorSleepInterval()); - EXPECT_EQ(0u, orchestration_policy.getSleepInterval()); + EXPECT_EQ(0, orchestration_policy.getErrorSleepInterval()); + EXPECT_EQ(0, orchestration_policy.getSleepInterval()); EXPECT_EQ("http://10.0.0.18:81/control/", orchestration_policy.getFogAddress()); } @@ -152,7 +152,7 @@ TEST_F(PolicyTest, newOptionalFields) ASSERT_TRUE(false) << "Cereal threw an exception: " << e.what(); } - EXPECT_EQ(10u, orchestration_policy.getErrorSleepInterval()); - EXPECT_EQ(30u, orchestration_policy.getSleepInterval()); + EXPECT_EQ(10, orchestration_policy.getErrorSleepInterval()); + EXPECT_EQ(30, orchestration_policy.getSleepInterval()); EXPECT_EQ("https://fog-api-gw-agents.cloud.ngen.checkpoint.com", orchestration_policy.getFogAddress()); } diff --git a/components/security_apps/orchestration/modules/modules_ut/url_parser_ut.cc b/components/security_apps/orchestration/modules/modules_ut/url_parser_ut.cc index eaa465e..2f8ac97 100755 --- a/components/security_apps/orchestration/modules/modules_ut/url_parser_ut.cc +++ b/components/security_apps/orchestration/modules/modules_ut/url_parser_ut.cc @@ -59,6 +59,15 @@ TEST_F(URLParserTest, parseAWSWithoutSlash) EXPECT_EQ("", link.getQuery()); } +TEST_F(URLParserTest, setHost) +{ + URLParser link("http://172.23.92.180:180/something"); + + EXPECT_EQ(link.getHost(), "172.23.92.180"); + link.setHost("my.domain"); + EXPECT_EQ(link.getHost(), "my.domain"); +} + TEST_F(URLParserTest, protocolIsMissing) { // HTTPS is set by default when protocol is not present in URL. diff --git a/components/security_apps/orchestration/modules/orchestration_policy.cc b/components/security_apps/orchestration/modules/orchestration_policy.cc index d6ce8cb..85ee599 100755 --- a/components/security_apps/orchestration/modules/orchestration_policy.cc +++ b/components/security_apps/orchestration/modules/orchestration_policy.cc @@ -22,13 +22,13 @@ OrchestrationPolicy::getFogAddress() const return fog_address; } -const unsigned long & +unsigned int OrchestrationPolicy::getSleepInterval() const { return sleep_interval; } -const unsigned long & +unsigned int OrchestrationPolicy::getErrorSleepInterval() const { return error_sleep_interval; @@ -37,10 +37,13 @@ OrchestrationPolicy::getErrorSleepInterval() const void OrchestrationPolicy::serialize(JSONInputArchive &archive) { - // Split it, so the order doesn't matter. - archive(make_nvp("fog-address", fog_address)); - archive(make_nvp("pulling-interval", sleep_interval)); - archive(make_nvp("error-pulling-interval", error_sleep_interval)); + try { + archive(make_nvp("fog-address", fog_address)); + archive(make_nvp("pulling-interval", sleep_interval)); + archive(make_nvp("error-pulling-interval", error_sleep_interval)); + } catch (const cereal::Exception&) { + archive(make_nvp("orchestration", *this)); + } } bool diff --git a/components/security_apps/orchestration/modules/orchestration_status.cc b/components/security_apps/orchestration/modules/orchestration_status.cc index f2839e6..112f868 100755 --- a/components/security_apps/orchestration/modules/orchestration_status.cc +++ b/components/security_apps/orchestration/modules/orchestration_status.cc @@ -399,7 +399,6 @@ public: if (!write_result) { dbgWarning(D_ORCHESTRATOR) << "Failed to write Orchestration status. File: " << orchestration_status_path; } - dbgTrace(D_ORCHESTRATOR) << "Orchestration status file has been updated. File: " << orchestration_status_path; } void @@ -459,7 +458,6 @@ public: seconds(5), [this] () { - dbgTrace(D_ORCHESTRATOR) << "Write Orchestration status file "; writeStatusToFile(); }, "Write Orchestration status file" diff --git a/components/security_apps/orchestration/modules/url_parser.cc b/components/security_apps/orchestration/modules/url_parser.cc index b2fbfc0..99e6bea 100755 --- a/components/security_apps/orchestration/modules/url_parser.cc +++ b/components/security_apps/orchestration/modules/url_parser.cc @@ -129,6 +129,18 @@ URLParser::parseProtocol(const string &url) const return URLProtocol::HTTPS; } +string +URLParser::getHost() const +{ + return host.empty() ? base_url : host; +} + +void +URLParser::setHost(const string &new_host) +{ + host = new_host; +} + void URLParser::setQuery(const string &new_query) { diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 10c4b91..4a87867 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -168,164 +168,96 @@ public: void init() { + i_agent_details = Singleton::Consume::by(); + i_service_controller = Singleton::Consume::by(); + i_orchestration_tools = Singleton::Consume::by(); + i_orchestration_status = Singleton::Consume::by(); + i_time = Singleton::Consume::by(); + upgrade_delay_time = chrono::duration_cast(i_time->getMonotonicTime()); + filesystem_prefix = getFilesystemPathConfig(); dbgTrace(D_ORCHESTRATOR) << "Initializing Orchestration component, file system path prefix: " << filesystem_prefix; - Singleton::Consume::by()->readAgentDetails(); - setAgentDetails(); - doEncrypt(); - health_check_status_listener.registerListener(); + auto orch_policy = loadDefaultOrchestrationPolicy(); + if (!orch_policy.ok()) { + dbgWarning(D_ORCHESTRATOR) << "Failed to load Orchestration Policy. Error: " << orch_policy.getErr(); + return; + } + policy = orch_policy.unpack(); + + if (getAttribute("no-setting", "IGNORE_CLUSTER_ID") != "TRUE") i_orchestration_tools->setClusterId(); + + i_orchestration_tools->loadTenantsFromDir( + getConfigurationWithDefault(getFilesystemPathConfig() + "/conf/", "orchestration", "Conf dir") + ); + + i_agent_details->readAgentDetails(); + setOpenSSLCerts(); + doEncrypt(); curr_agent_data_report.disableReportSending(); - auto rest = Singleton::Consume::by(); - rest->addRestCall(RestAction::SHOW, "orchestration-status"); - rest->addRestCall(RestAction::ADD, "proxy"); - rest->addRestCall(RestAction::SET, "agent-uninstall"); - // Main loop of the Orchestration. + + registerRestCalls(); + registerListeners(); + loadFogAddress(); + Singleton::Consume::by()->addOneTimeRoutine( I_MainLoop::RoutineType::RealTime, [this] () { run(); }, "Orchestration runner", true ); - - auto orchestration_tools = Singleton::Consume::by(); - - if (getAttribute("no-setting", "IGNORE_CLUSTER_ID") != "TRUE") orchestration_tools->getClusterId(); - - hybrid_mode_metric.init( - "Watchdog Metrics", - ReportIS::AudienceTeam::AGENT_CORE, - ReportIS::IssuingEngine::AGENT_CORE, - chrono::minutes(10), - true, - ReportIS::Audience::INTERNAL - ); - hybrid_mode_metric.registerListener(); - orchestration_tools->loadTenantsFromDir( - getConfigurationWithDefault(getFilesystemPathConfig() + "/conf/", "orchestration", "Conf dir") - ); } void fini() { - Singleton::Consume::by()->writeStatusToFile(); + i_orchestration_status->writeStatusToFile(); curr_agent_data_report.disableReportSending(); } private: Maybe - start() + registerToTheFog() { - auto update_communication = Singleton::Consume::by(); - auto agent_mode = getOrchestrationMode(); - auto policy_mgmt_mode = getSettingWithDefault("management", "profileManagedMode"); - bool declarative = agent_mode == OrchestrationMode::HYBRID || policy_mgmt_mode == "declarative"; - - bool enforce_policy_flag = false; - Maybe maybe_policy = genError("Empty policy"); - string policy_version = ""; - auto orchestration_policy_file = getPolicyConfigPath("orchestration", Config::ConfigFileType::Policy); - auto orchestration_tools = Singleton::Consume::by(); - if (orchestration_tools->doesFileExist(orchestration_policy_file)) { - maybe_policy = loadOrchestrationPolicy(); - if (!maybe_policy.ok()) { - dbgWarning(D_ORCHESTRATOR) << "Failed to load Orchestration policy. Error: " << maybe_policy.getErr(); - enforce_policy_flag = true; - } - dbgDebug(D_ORCHESTRATOR) << "Orchestration is restarting"; - - auto policy_file_path = getConfigurationWithDefault( - filesystem_prefix + "/conf/policy.json", - "orchestration", - "Policy file path" - ); - auto settings_file_path = getConfigurationWithDefault( - filesystem_prefix + "/conf/settings.json", - "orchestration", - "Settings file path" - ); - - auto service_controller = Singleton::Consume::by(); - auto is_update_config = service_controller->updateServiceConfiguration( - policy_file_path, - settings_file_path - ); - if (!is_update_config.ok()) { - dbgWarning(D_ORCHESTRATOR) - << "Failed to load the policy and settings, Error: " - << is_update_config.getErr(); - } - - policy_version = service_controller->getPolicyVersion(); - if (!policy_version.empty()) { - Singleton::Consume::by()->setPolicyVersion(policy_version); - } - } else { - dbgDebug(D_ORCHESTRATOR) << "Orchestration is running for the first time"; - enforce_policy_flag = true; - } - - if (enforce_policy_flag) { - // Trying to create the Orchestration policy from the general policy file - maybe_policy = enforceOrchestrationPolicy(); - if (!maybe_policy.ok()) { - return genError(maybe_policy.getErr()); - } - reloadConfiguration(); - } - - char *fog_address_env = getenv("FOG_ADDRESS"); - string fog_address = fog_address_env ? string(fog_address_env) : maybe_policy.unpack().getFogAddress(); - - if (updateFogAddress(fog_address)) { - policy = maybe_policy.unpack(); - } else { - return genError("Failed to set fog address from policy"); - } - - auto authentication_res = update_communication->authenticateAgent(); + auto i_update_communication = Singleton::Consume::by(); + auto authentication_res = i_update_communication->authenticateAgent(); + auto policy_version = i_service_controller->getPolicyVersion(); if (authentication_res.ok() && !policy_version.empty()) { - auto service_controller = Singleton::Consume::by(); - const string &policy_versions = service_controller->getPolicyVersions(); - auto path_policy_version = update_communication->sendPolicyVersion(policy_version, policy_versions); + const string &policy_versions = i_service_controller->getPolicyVersions(); + auto path_policy_version = i_update_communication->sendPolicyVersion(policy_version, policy_versions); if (!path_policy_version.ok()) { dbgWarning(D_ORCHESTRATOR) << path_policy_version.getErr(); } } - if (declarative) { - Singleton::Consume::from()->turnOnApplyPolicyFlag(); - } - return authentication_res; } - Maybe - loadOrchestrationPolicy() - { - auto maybe_policy = loadDefaultOrchestrationPolicy(); - if (!maybe_policy.ok()) { - dbgWarning(D_ORCHESTRATOR) << "Failed to load Orchestration Policy. Trying to load from backup."; - maybe_policy = loadOrchestrationPolicyFromBackup(); - } - return maybe_policy; - } - Maybe loadDefaultOrchestrationPolicy() { auto orchestration_policy_file = getPolicyConfigPath("orchestration", Config::ConfigFileType::Policy); - - auto orchestration_tools = Singleton::Consume::by(); - auto maybe_policy = orchestration_tools->jsonFileToObject(orchestration_policy_file); + bool orch_policy_exists = i_orchestration_tools->doesFileExist(orchestration_policy_file); + if (!orch_policy_exists) { + orchestration_policy_file = getConfigurationWithDefault( + filesystem_prefix + "/conf/policy.json", + "orchestration", + "Policy file path" + ); + } + dbgTrace(D_ORCHESTRATOR) << "Orchestration policy file: " << orchestration_policy_file; + auto maybe_policy = i_orchestration_tools->jsonFileToObject(orchestration_policy_file); if (maybe_policy.ok()) { return maybe_policy; } - return genError("Failed to load default Orchestration policy. Error: " + maybe_policy.getErr()); + dbgWarning(D_ORCHESTRATOR) + << "Failed to load Orchestration Policy. Error: " + << maybe_policy.getErr() + << "Trying to load from backup."; + return loadOrchestrationPolicyFromBackup(); } Maybe @@ -334,8 +266,7 @@ private: auto orchestration_policy_file = getPolicyConfigPath("orchestration", Config::ConfigFileType::Policy); auto backup_ext = getConfigurationWithDefault(".bk", "orchestration", "Backup file extension"); - auto orchestration_tools = Singleton::Consume::by(); - auto maybe_policy = orchestration_tools->jsonFileToObject( + auto maybe_policy = i_orchestration_tools->jsonFileToObject( orchestration_policy_file + backup_ext ); @@ -352,67 +283,6 @@ private: return genError("Failed to load Orchestration policy from backup."); } - Maybe - enforceOrchestrationPolicy() - { - Maybe maybe_policy(genError("Failed to enforce Orchestration policy")); - auto policy_file_path = getConfigurationWithDefault( - filesystem_prefix + "/conf/policy.json", - "orchestration", - "Policy file path" - ); - auto orchestration_policy_file = getPolicyConfigPath("orchestration", Config::ConfigFileType::Policy); - - auto settings_file_path = getConfigurationWithDefault( - filesystem_prefix + "/conf/settings.json", - "orchestration", - "Settings file path" - ); - - dbgInfo(D_ORCHESTRATOR) - << "Enforcing new configuration. Policy file: " - << policy_file_path - << ", Settings file: " - << settings_file_path; - - auto orchestration_tools = Singleton::Consume::by(); - auto service_controller = Singleton::Consume::by(); - auto is_update_config = service_controller->updateServiceConfiguration(policy_file_path, settings_file_path); - if (is_update_config.ok()) { - maybe_policy = orchestration_tools->jsonFileToObject(orchestration_policy_file); - } else { - dbgWarning(D_ORCHESTRATOR) << is_update_config.getErr(); - } - - if (!maybe_policy.ok()) { - dbgDebug(D_ORCHESTRATOR) << "Enforcing policy file from backup"; - string backup_ext = getConfigurationWithDefault( - ".bk", - "orchestration", - "Backup file extension" - ); - - dbgInfo(D_ORCHESTRATOR) << "Recovering the policy file from backup."; - if (!orchestration_tools->copyFile(policy_file_path + backup_ext, policy_file_path)) { - return genError("Failed to copy orchestration policy from backup policy.json file."); - } - // Try to use the backup policy.json file and re-write the services's policies. - is_update_config = service_controller->updateServiceConfiguration(policy_file_path, settings_file_path); - if (is_update_config.ok()) { - maybe_policy = orchestration_tools->jsonFileToObject(orchestration_policy_file); - } else { - dbgWarning(D_ORCHESTRATOR) << is_update_config.getErr(); - } - } - - if (maybe_policy.ok()) { - return maybe_policy; - } - - dbgDebug(D_ORCHESTRATOR) << "Failed to load Orchestration policy. Error: " << maybe_policy.getErr(); - return genError("Failed to load Orchestration policy from the main policy."); - } - bool recoverBackupOrchestrationPolicy() { @@ -421,13 +291,19 @@ private: string backup_ext = getConfigurationWithDefault(".bk", "orchestration", "Backup file extension"); string backup_orchestration_conf_file = conf_path + backup_ext; - auto orchestration_tools = Singleton::Consume::by(); - return orchestration_tools->copyFile(backup_orchestration_conf_file, conf_path); + return i_orchestration_tools->copyFile(backup_orchestration_conf_file, conf_path); } Maybe handleManifestUpdate(const OrchManifest &orch_manifest) { + if (isUpgradeDelayed()) { + dbgTrace(D_ORCHESTRATOR) + << "The manifest update is delayed for another " + << (upgrade_delay_time - chrono::duration_cast(i_time->getMonotonicTime())).count() + << " minutes."; + return Maybe(); + } if (!orch_manifest.ok()) return Maybe(); // Handling manifest update. @@ -440,12 +316,9 @@ private: resource_file ); - auto orch_status = Singleton::Consume::by(); - auto service_controller = Singleton::Consume::by(); - auto agent_details = Singleton::Consume::by(); - static int service_to_port_size = service_controller->getServiceToPortMap().size(); + static int service_to_port_size = i_service_controller->getServiceToPortMap().size(); auto hostname = Singleton::Consume::by()->getHostname(); - string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + agent_details->getAgentId()) + "'"; + string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + i_agent_details->getAgentId()) + "'"; if (!new_manifest_file.ok()) { string install_error; if (!service_to_port_size) { @@ -464,7 +337,7 @@ private: << new_manifest_file.getErr() << " Presenting the next message to the user: " << install_error; - orch_status->setFieldStatus( + i_orchestration_status->setFieldStatus( OrchestrationStatusFieldType::MANIFEST, OrchestrationStatusResult::FAILED, install_error @@ -485,9 +358,9 @@ private: "Warning: Agent/Gateway " + err_hostname + " software update failed. Agent is running previous software. Contact Check Point support."; - string current_error = orch_status->getManifestError(); + string current_error = i_orchestration_status->getManifestError(); if (current_error.find("Gateway was not fully deployed") == string::npos) { - orch_status->setFieldStatus( + i_orchestration_status->setFieldStatus( OrchestrationStatusFieldType::MANIFEST, OrchestrationStatusResult::FAILED, install_error @@ -508,7 +381,7 @@ private: return genError(install_error); } - orch_status->setFieldStatus( + i_orchestration_status->setFieldStatus( OrchestrationStatusFieldType::MANIFEST, OrchestrationStatusResult::SUCCESS ); @@ -573,12 +446,9 @@ private: "Backup file extension" ); - auto orchestration_tools = Singleton::Consume::by(); - auto service_controller = Singleton::Consume::by(); - // Try to use the backup policy.json file and re-write the services's policies. dbgInfo(D_ORCHESTRATOR) << "Updating services with the new policy."; - auto is_update_config = service_controller->updateServiceConfiguration( + auto is_update_config = i_service_controller->updateServiceConfiguration( policy_file_path + backup_ext, settings_file_path ); @@ -587,7 +457,7 @@ private: return false; } dbgInfo(D_ORCHESTRATOR) << "Recovering the policy file from backup."; - if (!orchestration_tools->copyFile(policy_file_path + backup_ext, policy_file_path)) { + if (!i_orchestration_tools->copyFile(policy_file_path + backup_ext, policy_file_path)) { dbgWarning (D_ORCHESTRATOR) << "Failed to recover policy file from backup. File: " << policy_file_path + backup_ext; @@ -609,14 +479,13 @@ private: policy = orchestration_policy; - auto service_controller = Singleton::Consume::by(); - string new_policy_version = service_controller->getPolicyVersion(); + string new_policy_version = i_service_controller->getPolicyVersion(); if (!new_policy_version.empty()) { - Singleton::Consume::by()->setPolicyVersion(new_policy_version); + i_orchestration_status->setPolicyVersion(new_policy_version); } - auto update_communication = Singleton::Consume::by(); - const string &policy_versions = service_controller->getPolicyVersions(); - auto path_policy_version = update_communication->sendPolicyVersion(new_policy_version, policy_versions); + auto i_update_communication = Singleton::Consume::by(); + const string &policy_versions = i_service_controller->getPolicyVersions(); + auto path_policy_version = i_update_communication->sendPolicyVersion(new_policy_version, policy_versions); if (!path_policy_version.ok()) { dbgWarning(D_ORCHESTRATOR) << path_policy_version.getErr(); } @@ -641,28 +510,26 @@ private: return genError("Failed to download the new policy file. Error: " + new_policy_file.getErr()); } - auto orchestration_tools = Singleton::Consume::by(); auto conf_path = filesystem_prefix + "/conf/policy.json"; string last_ext = getConfigurationWithDefault( ".last", "orchestration", "last fog policy file extension" ); - if (!orchestration_tools->copyFile(new_policy_file.unpack(), conf_path + last_ext)) { + if (!i_orchestration_tools->copyFile(new_policy_file.unpack(), conf_path + last_ext)) { dbgWarning(D_ORCHESTRATOR) << "Failed to copy a new policy file to " << conf_path + last_ext; } // Calculate the changes between the existing policy to the new one. - auto service_controller = Singleton::Consume::by(); - string old_policy_version = service_controller->getPolicyVersion(); - auto res = service_controller->updateServiceConfiguration( + string old_policy_version = i_service_controller->getPolicyVersion(); + auto res = i_service_controller->updateServiceConfiguration( new_policy_file.unpack(), settings_path, data_updates ); if (!res.ok()) { - string updated_policy_version = service_controller->getUpdatePolicyVersion(); + string updated_policy_version = i_service_controller->getUpdatePolicyVersion(); string error_str = "Failed to update services' policy configuration files. Previous version: " + old_policy_version + @@ -682,12 +549,12 @@ private: "Settings file path" ); - set changed_policy_files = service_controller->moveChangedPolicies(); + set changed_policy_files = i_service_controller->moveChangedPolicies(); for (const string &changed_policy_file : changed_policy_files) { - orchestration_tools->writeFile("{}\n", changed_policy_file); + i_orchestration_tools->writeFile("{}\n", changed_policy_file); } - service_controller->updateServiceConfiguration(policy_file, setting_file, data_updates); + i_service_controller->updateServiceConfiguration(policy_file, setting_file, data_updates); LogGen( error_str, Audience::SECURITY, @@ -700,7 +567,7 @@ private: return genError(error_str); } - service_controller->moveChangedPolicies(); + i_service_controller->moveChangedPolicies(); // Reload the orchestration policy, in case of the policy updated auto orchestration_policy = loadDefaultOrchestrationPolicy(); @@ -713,9 +580,9 @@ private: return genError("Failed to load Orchestration new policy file."); } if (getProfileAgentSettingWithDefault(false, "agent.config.orchestration.reportAgentDetail")) { - service_controller->clearFailedServices(); + i_service_controller->clearFailedServices(); reportAgentDetailsMetaData(); - if(service_controller->doesFailedServicesExist()) { + if(i_service_controller->doesFailedServicesExist()) { dbgWarning(D_ORCHESTRATOR) << "Failed to enforce Orchestration policy."; updateServiceConfigurationFromBackup(); // Reload the orchestration policy, in case of the policy updated @@ -801,9 +668,8 @@ private: dbgInfo(D_ORCHESTRATOR) << "There is a new data file."; const string data_file_dir = filesystem_prefix + "/conf/data"; - auto orchestration_tools = Singleton::Consume::by(); - if (!orchestration_tools->doesDirectoryExist(data_file_dir)) { - orchestration_tools->createDirectory(data_file_dir); + if (!i_orchestration_tools->doesDirectoryExist(data_file_dir)) { + i_orchestration_tools->createDirectory(data_file_dir); } const auto data_file_path = getConfigurationWithDefault( filesystem_prefix + "/conf/data.json", @@ -820,7 +686,7 @@ private: if (!new_data_files.ok()) { return genError("Failed to download new data file, Error: " + new_data_files.getErr()); } - auto new_data_file_input = orchestration_tools->readFile(new_data_files.unpack()); + auto new_data_file_input = i_orchestration_tools->readFile(new_data_files.unpack()); if (!new_data_file_input.ok()) { return genError("Failed to read new data file, Error: " + new_data_file_input.getErr()); } @@ -866,13 +732,13 @@ private: dbgWarning(D_ORCHESTRATOR) << current_error.str(); return genError(current_error.str()); } - if (!orchestration_tools->copyFile(new_data_file.unpack(), data_file_save_path)) { + if (!i_orchestration_tools->copyFile(new_data_file.unpack(), data_file_save_path)) { dbgWarning(D_ORCHESTRATOR) << "Failed to copy a new data file to " << data_file_save_path; } data_updates.push_back(data_file.first); } - if (!orchestration_tools->copyFile(new_data_files.unpack(), data_file_path)) { + if (!i_orchestration_tools->copyFile(new_data_files.unpack(), data_file_path)) { dbgWarning(D_ORCHESTRATOR) << "Failed to copy a new agents' data file to " << data_file_path; } @@ -944,7 +810,7 @@ private: ) ); - auto policy_version = Singleton::Consume::by()->getPolicyVersion(); + auto policy_version = i_service_controller->getPolicyVersion(); dbgDebug(D_ORCHESTRATOR) << "Sending check update request"; @@ -995,25 +861,24 @@ private: request.setGreedyMode(); } - auto update_communication = Singleton::Consume::by(); - auto response = update_communication->getUpdate(request); + auto i_update_communication = Singleton::Consume::by(); + auto response = i_update_communication->getUpdate(request); - auto orch_status = Singleton::Consume::by(); - orch_status->setLastUpdateAttempt(); + i_orchestration_status->setLastUpdateAttempt(); auto upgrade_mode = getSetting("upgradeMode"); auto agent_type = getSetting("agentType"); if (upgrade_mode.ok()) { - orch_status->setUpgradeMode(upgrade_mode.unpack()); + i_orchestration_status->setUpgradeMode(upgrade_mode.unpack()); } if (agent_type.ok()) { - orch_status->setAgentType(agent_type.unpack()); + i_orchestration_status->setAgentType(agent_type.unpack()); } HybridModeMetricEvent().notify(); if (!response.ok()) { dbgWarning(D_ORCHESTRATOR) << "Failed to get the update. Error: " << response.getErr(); - orch_status->setFieldStatus( + i_orchestration_status->setFieldStatus( OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::FAILED, "Warning: Agent/Gateway failed during the update process. Contact Check Point support." @@ -1042,12 +907,17 @@ private: } } + bool + isUpgradeDelayed() + { + return upgrade_delay_time > chrono::duration_cast(i_time->getMonotonicTime()); + } + Maybe handleUpdate(const CheckUpdateRequest &response) { auto span_scope = Singleton::Consume::by()->startNewSpanScope(Span::ContextType::CHILD_OF); - auto agent_details = Singleton::Consume::by(); dbgDebug(D_ORCHESTRATOR) << "Starting to handle check update response"; OrchManifest orch_manifest = response.getManifest(); @@ -1055,12 +925,11 @@ private: OrchSettings orch_settings = response.getSettings(); OrchData orch_data = response.getData(); - auto orch_status = Singleton::Consume::by(); - orch_status->setFieldStatus( + i_orchestration_status->setFieldStatus( OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::SUCCESS ); - orch_status->setIsConfigurationUpdated( + i_orchestration_status->setIsConfigurationUpdated( EnumArray( orch_manifest.ok(), orch_policy.ok(), orch_settings.ok(), orch_data.ok() ) @@ -1074,8 +943,11 @@ private: vector data_updates; update_results[OrchestrationStatusConfigType::DATA] = handleDataUpdate(orch_data, data_updates); - auto orch_mode = agent_details->getOrchestrationMode(); - if ((!orch_manifest.ok() || orch_mode == OrchestrationMode::HYBRID) && orch_policy.ok()) { + auto orch_mode = i_agent_details->getOrchestrationMode(); + if ( + (!orch_manifest.ok() || isUpgradeDelayed() || orch_mode == OrchestrationMode::HYBRID) && + orch_policy.ok() + ) { update_results[OrchestrationStatusConfigType::POLICY] = handlePolicyUpdate( orch_policy, settings_path, @@ -1083,8 +955,7 @@ private: ); } if (!orch_policy.ok() && (!data_updates.empty() || !settings_path.empty())) { - auto service_controller = Singleton::Consume::by(); - auto res = service_controller->updateServiceConfiguration( + auto res = i_service_controller->updateServiceConfiguration( "", settings_path, data_updates @@ -1097,12 +968,12 @@ private: update_results[OrchestrationStatusConfigType::MANIFEST] = handleManifestUpdate(orch_manifest); if (!update_results[OrchestrationStatusConfigType::MANIFEST].ok()) { - string current_error = orch_status->getManifestError(); + string current_error = i_orchestration_status->getManifestError(); string recommended_fix; string msg; bool is_deploy_error = current_error.find("Critical") != string::npos; auto hostname = Singleton::Consume::by()->getHostname(); - string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + agent_details->getAgentId()) + "'"; + auto err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + i_agent_details->getAgentId()) + "'"; if (is_deploy_error) { msg = "Agent/Gateway was not fully deployed " + @@ -1258,7 +1129,6 @@ private: } } } - auto orchestration_tools = Singleton::Consume::by(); auto conf_dir = getConfigurationWithDefault( getFilesystemPathConfig() + "/conf/", "orchestration", @@ -1274,7 +1144,7 @@ private: << ", Profile ID: " << profile_id; tenant_manager->deactivateTenant(tenant_id, profile_id); - orchestration_tools->deleteVirtualTenantProfileFiles( + i_orchestration_tools->deleteVirtualTenantProfileFiles( tenant_id, profile_id, conf_dir @@ -1302,7 +1172,7 @@ private: bool last_iteration = false; if (next(it) == sorted_files.end()) last_iteration = true; - Singleton::Consume::by()->updateServiceConfiguration( + i_service_controller->updateServiceConfiguration( policy_file, setting_file, new_data_files, @@ -1323,10 +1193,9 @@ private: "Conf dir" ) + (tenant_id != "" ? "tenant_" + tenant_id + "_profile_" + profile_id + "_" : ""); - auto orchestration_tools = Singleton::Consume::by(); string settings_file_path = conf_dir + "settings.json"; dbgTrace(D_ORCHESTRATOR) << "The settings directory is " << settings_file_path; - if (!orchestration_tools->copyFile(new_settings_file, settings_file_path)) { + if (!i_orchestration_tools->copyFile(new_settings_file, settings_file_path)) { dbgWarning(D_ORCHESTRATOR) << "Failed to update the settings."; return genError("Failed to update the settings"); } @@ -1369,8 +1238,7 @@ private: string getChecksum(const string &file_path) { - auto orchestration_tools = Singleton::Consume::by(); - Maybe file_checksum = orchestration_tools->calculateChecksum( + Maybe file_checksum = i_orchestration_tools->calculateChecksum( I_OrchestrationTools::SELECTED_CHECKSUM_TYPE, file_path ); @@ -1383,8 +1251,7 @@ private: getVersion(const string &file_path) { string version; - auto orchestration_tools = Singleton::Consume::by(); - Maybe file_data = orchestration_tools->readFile(file_path); + Maybe file_data = i_orchestration_tools->readFile(file_path); if (file_data.ok()) { try { @@ -1403,16 +1270,15 @@ private: void encryptOldFile(const string &old_path, const string &new_path) { - auto orchestration_tools = Singleton::Consume::by(); - auto file_data = orchestration_tools->readFile(old_path); + auto file_data = i_orchestration_tools->readFile(old_path); if (file_data.ok()) { auto encryptor = Singleton::Consume::by(); auto decoded_data = encryptor->base64Decode(file_data.unpack()); - if (!orchestration_tools->writeFile(decoded_data, new_path)) { + if (!i_orchestration_tools->writeFile(decoded_data, new_path)) { dbgWarning(D_ORCHESTRATOR) << "Failed to encrypt files"; } else { // Removing clear data files after encrypting - orchestration_tools->removeFile(old_path); + i_orchestration_tools->removeFile(old_path); } } } @@ -1420,8 +1286,7 @@ private: void encryptToFile(const string &data, const string &file) { - auto orchestration_tools = Singleton::Consume::by(); - if (!orchestration_tools->writeFile(data, file)) { + if (!i_orchestration_tools->writeFile(data, file)) { dbgWarning(D_ORCHESTRATOR) << "Failed to encrypt files"; } } @@ -1477,7 +1342,6 @@ private: agent_data_report << AgentReportFieldWithLabel("isVersionAboveR8110", "true"); } - auto i_agent_details = Singleton::Consume::by(); if ( i_agent_details->getOrchestrationMode() == OrchestrationMode::HYBRID || getSettingWithDefault("management", "profileManagedMode") == "declarative" @@ -1545,16 +1409,70 @@ private: return sleep_interval * failure_multiplier; } + void + preformCheckUpdate() + { + auto check_update_result = checkUpdate(); + if (!check_update_result.ok()) { + is_new_success = false; + sleep_interval = calcSleepInterval(policy.getErrorSleepInterval()); + dbgWarning(D_ORCHESTRATOR) + << "Failed during check update from Fog. Error: " + << check_update_result.getErr() + << ", new check will be every: " + << sleep_interval << " seconds"; + + health_check_status_listener.setStatus( + HealthCheckStatus::UNHEALTHY, + OrchestrationStatusFieldType::LAST_UPDATE, + "Failed during check update from Fog. Error: " + check_update_result.getErr() + ); + return; + } + failure_count = 0; + dbgDebug(D_ORCHESTRATOR) << "Check update process completed successfully"; + health_check_status_listener.setStatus( + HealthCheckStatus::HEALTHY, + OrchestrationStatusFieldType::LAST_UPDATE + ); + sleep_interval = policy.getSleepInterval(); + if (!is_new_success) { + dbgInfo(D_ORCHESTRATOR) + << "Check update process completed successfully, new check will be every: " + << sleep_interval << " seconds"; + is_new_success = true; + } + } + + void + setUpgradeTime() + { + if (getConfigurationFlag("service_startup") != "true") return; + if (i_service_controller->getServiceToPortMap().empty()) return; + + try { + string upgrade_delay_interval_str = getAttribute("no-setting", "UPGRADE_DELAY_INTERVAL_MIN"); + int upgrade_delay_interval = upgrade_delay_interval_str != "" ? stoi(upgrade_delay_interval_str) : 30; + dbgInfo(D_ORCHESTRATOR) + << "Setting upgrade delay time to " + << upgrade_delay_interval + << " minutes from now."; + upgrade_delay_time += chrono::minutes(upgrade_delay_interval); + } catch (const exception& err) { + dbgInfo(D_ORCHESTRATOR) << "Failed to parse upgrade delay interval."; + } + } + void run() { - int sleep_interval = policy.getErrorSleepInterval(); - Maybe start_state(genError("Not running yet.")); - while (!(start_state = start()).ok()) { + sleep_interval = policy.getErrorSleepInterval(); + Maybe registration_status(genError("Not running yet.")); + while (!(registration_status = registerToTheFog()).ok()) { health_check_status_listener.setStatus( HealthCheckStatus::UNHEALTHY, OrchestrationStatusFieldType::REGISTRATION, - start_state.getErr() + registration_status.getErr() ); sleep_interval = getConfigurationWithDefault( 30, @@ -1564,12 +1482,13 @@ private: sleep_interval = calcSleepInterval(sleep_interval); dbgWarning(D_ORCHESTRATOR) << "Orchestration not started yet. Status: " - << start_state.getErr() + << registration_status.getErr() << " Next attempt to start the orchestration will be in: " << sleep_interval << " seconds"; Singleton::Consume::by()->yield(seconds(sleep_interval)); } + loadExistingPolicy(); failure_count = 0; Singleton::Consume::by()->yield(chrono::seconds(1)); @@ -1611,42 +1530,13 @@ private: ); } - bool is_new_success = false; + setUpgradeTime(); while (true) { Singleton::Consume::by()->startNewTrace(false); if (shouldReportAgentDetailsMetadata()) { reportAgentDetailsMetaData(); } - auto check_update_result = checkUpdate(); - if (!check_update_result.ok()) { - is_new_success = false; - sleep_interval = calcSleepInterval(policy.getErrorSleepInterval()); - dbgWarning(D_ORCHESTRATOR) - << "Failed during check update from Fog. Error: " - << check_update_result.getErr() - << ", new check will be every: " - << sleep_interval << " seconds"; - - health_check_status_listener.setStatus( - HealthCheckStatus::UNHEALTHY, - OrchestrationStatusFieldType::LAST_UPDATE, - "Failed during check update from Fog. Error: " + check_update_result.getErr() - ); - } else { - failure_count = 0; - dbgDebug(D_ORCHESTRATOR) << "Check update process completed successfully"; - health_check_status_listener.setStatus( - HealthCheckStatus::HEALTHY, - OrchestrationStatusFieldType::LAST_UPDATE - ); - sleep_interval = policy.getSleepInterval(); - if (!is_new_success) { - dbgInfo(D_ORCHESTRATOR) - << "Check update process completed successfully, new check will be every: " - << sleep_interval << " seconds"; - is_new_success = true; - } - } + preformCheckUpdate(); dbgDebug(D_ORCHESTRATOR) << "Next check for update will be in: " << sleep_interval << " seconds"; Singleton::Consume::by()->finishTrace(); @@ -1741,11 +1631,10 @@ private: auto backup_installation_file = current_installation_file + backup_ext; auto temp_ext = getConfigurationWithDefault("_temp", "orchestration", "Temp file extension"); - auto orchestration_tools = Singleton::Consume::by(); - dbgAssert(orchestration_tools->doesFileExist(backup_installation_file)) + dbgAssert(i_orchestration_tools->doesFileExist(backup_installation_file)) << "There is no backup installation package"; - dbgAssert(orchestration_tools->copyFile(backup_installation_file, current_installation_file)) + dbgAssert(i_orchestration_tools->copyFile(backup_installation_file, current_installation_file)) << "Failed to copy backup installation package"; // Copy the backup manifest file to the default manifest file path. @@ -1755,7 +1644,7 @@ private: "Manifest file path" ); - if (!orchestration_tools->copyFile(manifest_file_path + backup_ext, manifest_file_path + temp_ext)) { + if (!i_orchestration_tools->copyFile(manifest_file_path + backup_ext, manifest_file_path + temp_ext)) { dbgWarning(D_ORCHESTRATOR) << "Failed to restore manifest backup file."; } @@ -1774,27 +1663,24 @@ private: const uint16_t port, const bool is_secure) { - auto agent_details = Singleton::Consume::by(); return - agent_details->getAccessToken().empty() || - agent_details->getSSLFlag() != is_secure || - !agent_details->getFogPort().ok() || agent_details->getFogPort().unpack() != port || - !agent_details->getFogDomain().ok() || agent_details->getFogDomain().unpack() != fog; + i_agent_details->getAccessToken().empty() || + i_agent_details->getSSLFlag() != is_secure || + !i_agent_details->getFogPort().ok() || i_agent_details->getFogPort().unpack() != port || + !i_agent_details->getFogDomain().ok() || i_agent_details->getFogDomain().unpack() != fog; } bool updateFogAddress(const string &fog_addr) { dbgFlow(D_ORCHESTRATOR) << "Setting a fog address: " << fog_addr; - auto orch_status = Singleton::Consume::by(); - auto agent_details = Singleton::Consume::by(); auto orchestration_mode = getOrchestrationMode(); - agent_details->setOrchestrationMode(orchestration_mode); + i_agent_details->setOrchestrationMode(orchestration_mode); if (orchestration_mode == OrchestrationMode::OFFLINE) { - orch_status->setUpgradeMode("Offline upgrades"); - orch_status->setRegistrationStatus("Offline mode"); - orch_status->setFogAddress(""); - if (agent_details->writeAgentDetails()) { + i_orchestration_status->setUpgradeMode("Offline upgrades"); + i_orchestration_status->setRegistrationStatus("Offline mode"); + i_orchestration_status->setFogAddress(""); + if (i_agent_details->writeAgentDetails()) { dbgDebug(D_ORCHESTRATOR) << "Agent details was successfully saved"; } else { dbgWarning(D_ORCHESTRATOR) << "Failed to save agent details to a file"; @@ -1823,19 +1709,19 @@ private: } if (message->setFogConnection(fog_domain, fog_port, encrypted_fog_connection, MessageCategory::GENERIC)) { - agent_details->setFogPort(fog_port); - agent_details->setFogDomain(fog_domain); - agent_details->setSSLFlag(encrypted_fog_connection); + i_agent_details->setFogPort(fog_port); + i_agent_details->setFogDomain(fog_domain); + i_agent_details->setSSLFlag(encrypted_fog_connection); - if (agent_details->writeAgentDetails()) { + if (i_agent_details->writeAgentDetails()) { dbgDebug(D_ORCHESTRATOR) << "Agent details was successfully saved"; } else { dbgWarning(D_ORCHESTRATOR) << "Failed to save agent details to a file"; } - auto update_communication = Singleton::Consume::by(); - update_communication->setAddressExtenesion(fog_query); - orch_status->setFogAddress(fog_addr); + auto i_update_communication = Singleton::Consume::by(); + i_update_communication->setAddressExtenesion(fog_query); + i_orchestration_status->setFogAddress(fog_addr); return true; } @@ -1897,7 +1783,7 @@ private: } void - setAgentDetails() + setOpenSSLCerts() { static const string openssl_dir_cmd = "openssl version -d | cut -d\" \" -f2 | cut -d\"\\\"\" -f2"; auto i_shell_cmd = Singleton::Consume::by(); @@ -1914,10 +1800,9 @@ private: << "Adding OpenSSL default directory to agent details. Directory: " << val_openssl_dir; - auto agent_details = Singleton::Consume::by(); - agent_details->setOpenSSLDir(val_openssl_dir + "/certs"); - agent_details->setOrchestrationMode(getOrchestrationMode()); - agent_details->writeAgentDetails(); + i_agent_details->setOpenSSLDir(val_openssl_dir + "/certs"); + i_agent_details->setOrchestrationMode(getOrchestrationMode()); + i_agent_details->writeAgentDetails(); } else { dbgWarning(D_ORCHESTRATOR) << "Failed to load OpenSSL default certificate authority. Error: " @@ -1957,21 +1842,149 @@ private: void doCall() override { - auto agent_details = Singleton::Consume::by(); - agent_details->setProxy(proxy.get()); - agent_details->writeAgentDetails(); + auto i_agent_details = Singleton::Consume::by(); + i_agent_details->setProxy(proxy.get()); + i_agent_details->writeAgentDetails(); } private: C2S_PARAM(string, proxy); }; + void + registerRestCalls() + { + auto rest = Singleton::Consume::by(); + rest->addRestCall(RestAction::SHOW, "orchestration-status"); + rest->addRestCall(RestAction::ADD, "proxy"); + rest->addRestCall(RestAction::SET, "agent-uninstall"); + } + + void + registerListeners() + { + hybrid_mode_metric.init( + "Watchdog Metrics", + ReportIS::AudienceTeam::AGENT_CORE, + ReportIS::IssuingEngine::AGENT_CORE, + chrono::minutes(10), + true, + ReportIS::Audience::INTERNAL + ); + hybrid_mode_metric.registerListener(); + health_check_status_listener.registerListener(); + } + + void + loadFogDataToEnv(const string &fog_address) + { + auto maybe_fog_params = parseURLParams(fog_address); + if (!maybe_fog_params.ok()) { + dbgWarning(D_ORCHESTRATOR) << "Failed to update Fog address, Error: " << maybe_fog_params.getErr(); + return; + } + auto &fog_params = maybe_fog_params.unpack(); + i_agent_details->setFogDomain(std::get<0>(fog_params)); + i_agent_details->setFogPort(std::get<2>(fog_params)); + i_agent_details->setSSLFlag(std::get<3>(fog_params)); + dbgDebug(D_ORCHESTRATOR) + << "Extracted Fog details: " + << std::get<0>(fog_params) + << ":" + << std::get<2>(fog_params); + if (i_agent_details->writeAgentDetails()) { + dbgTrace(D_ORCHESTRATOR) << "Agent details was successfully saved"; + } else { + dbgWarning(D_ORCHESTRATOR) << "Failed to save agent details to a file"; + } + i_orchestration_status->setFogAddress(fog_address); + } + + void + loadFogAddress() + { + dbgTrace(D_ORCHESTRATOR) << "Extracting Fog address"; + auto orchestration_mode = getOrchestrationMode(); + i_agent_details->setOrchestrationMode(orchestration_mode); + if (orchestration_mode == OrchestrationMode::OFFLINE) { + i_orchestration_status->setUpgradeMode("Offline upgrades"); + i_orchestration_status->setRegistrationStatus("Offline mode"); + i_orchestration_status->setFogAddress(""); + if (i_agent_details->writeAgentDetails()) { + dbgDebug(D_ORCHESTRATOR) << "Agent details was successfully saved"; + } else { + dbgWarning(D_ORCHESTRATOR) << "Failed to save agent details to a file"; + } + return; + } + + auto maybe_fog_domain = i_agent_details->getFogDomain(); + if (maybe_fog_domain.ok()) { + dbgTrace(D_ORCHESTRATOR) << "Fog address already exists: " << maybe_fog_domain.unpack(); + return; + } + auto fog_address = getFogAddress(); + if (fog_address.empty()) { + dbgWarning(D_ORCHESTRATOR) << "Fog address could not be empty on online update mode"; + return; + } + loadFogDataToEnv(fog_address); + } + + string + getFogAddress() + { + auto fog_address = policy.getFogAddress(); + char *fog_address_env = getenv("FOG_ADDRESS"); + return fog_address_env ? string(fog_address_env) : fog_address; + } + + void + loadExistingPolicy() + { + auto policy_file_path = getConfigurationWithDefault( + filesystem_prefix + "/conf/policy.json", + "orchestration", + "Policy file path" + ); + auto settings_file_path = getConfigurationWithDefault( + filesystem_prefix + "/conf/settings.json", + "orchestration", + "Settings file path" + ); + + auto update_config = i_service_controller->updateServiceConfiguration( + policy_file_path, + settings_file_path + ); + if (!update_config.ok()) { + dbgWarning(D_ORCHESTRATOR) + << "Failed to load the existing policy and settings, Error: " + << update_config.getErr(); + return; + } + + auto policy_version = i_service_controller->getPolicyVersion(); + if (!policy_version.empty()) { + i_orchestration_status->setPolicyVersion(policy_version); + } + } + + I_OrchestrationStatus *i_orchestration_status = nullptr; + I_OrchestrationTools *i_orchestration_tools = nullptr; + I_ServiceController *i_service_controller = nullptr; + I_AgentDetails *i_agent_details = nullptr; + I_TimeGet *i_time = nullptr; + const uint16_t default_fog_dport = 443; int failure_count = 0; + unsigned int sleep_interval = 0; + bool is_new_success = false; OrchestrationPolicy policy; HealthCheckStatusListener health_check_status_listener; HybridModeMetric hybrid_mode_metric; EnvDetails env_details; + chrono::minutes upgrade_delay_time; string filesystem_prefix = ""; AgentDataReport curr_agent_data_report; diff --git a/components/security_apps/orchestration/orchestration_tools/namespace_data.cc b/components/security_apps/orchestration/orchestration_tools/namespace_data.cc index 9479ef3..93bc02e 100644 --- a/components/security_apps/orchestration/orchestration_tools/namespace_data.cc +++ b/components/security_apps/orchestration/orchestration_tools/namespace_data.cc @@ -108,7 +108,7 @@ NamespaceData::loadJson(const string &json) } Maybe -NamespaceData::getNamespaceUidByName(const string &name) +NamespaceData::getNamespaceUidByName(const string &name) const { if (ns_name_to_uid.find(name) == ns_name_to_uid.end()) { return genError("Namespace doesn't exist. Name: " + name); diff --git a/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc b/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc index b0bbd8f..929abfe 100755 --- a/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc +++ b/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc @@ -54,7 +54,7 @@ public: bool removeFile(const string &path) const override; bool copyFile(const string &src_path, const string &dst_path) const override; bool doesFileExist(const string &file_path) const override; - void getClusterId() const override; + void setClusterId() const override; void fillKeyInJson(const string &filename, const string &_key, const string &_val) const override; bool createDirectory(const string &directory_path) const override; bool doesDirectoryExist(const string &dir_path) const override; @@ -143,83 +143,55 @@ isPlaygroundEnv() } Maybe -getNamespaceDataFromCluster(const string &path) +getNamespaceDataFromCluster() { - NamespaceData name_space; string token = Singleton::Consume::by()->getToken(); - auto messaging = Singleton::Consume::by(); + string auth_header = "Authorization: Bearer " + token; + string connection_header = "Connection: close"; + string host = "https://kubernetes.default.svc:443/api/v1/namespaces/"; + string culr_cmd = "curl -s -k -H \"" + auth_header + "\" -H \"" + connection_header + "\" " + host + + " | /etc/cp/bin/cpnano_json"; - MessageMetadata get_ns_md("kubernetes.default.svc", 443); - get_ns_md.insertHeader("Authorization", "Bearer " + token); - get_ns_md.insertHeader("Connection", "close"); - get_ns_md.setConnectioFlag(MessageConnectionConfig::IGNORE_SSL_VALIDATION); - auto res = messaging->sendSyncMessage( - HTTPMethod::GET, - path, - name_space, - MessageCategory::GENERIC, - get_ns_md - ); + auto output_res = Singleton::Consume::by()->getExecOutput(culr_cmd); + if (!output_res.ok()) { + return genError("Failed to get namespace data from the cluster: " + output_res.getErr()); + } - if (res.ok()) return name_space; - - return genError(string("Was not able to get object form k8s cluser in path: " + path)); + dbgTrace(D_ORCHESTRATOR) << "Got the repsonse from the cluster: " << output_res.unpack(); + NamespaceData name_space; + if (name_space.loadJson(output_res.unpack())) return name_space; + return genError("Was not able to parse the object form k8s cluser"); } -bool -doesClusterIdExists() +void +OrchestrationTools::Impl::setClusterId() const { + auto env_type = Singleton::Consume::by()->getEnvType(); + if (env_type != EnvType::K8S) return; + + dbgTrace(D_ORCHESTRATOR) << "Setting cluster UID"; + + Maybe namespaces_data = getNamespaceDataFromCluster(); + if (!namespaces_data.ok()) { + dbgWarning(D_ORCHESTRATOR) << "Failed to retrieve namespace data. Error: " << namespaces_data.getErr(); + return; + } + + auto ns_uid = (*namespaces_data).getNamespaceUidByName("kube-system"); + if (!ns_uid.ok()) { + dbgWarning(D_ORCHESTRATOR) << "Failed to retrieve namespace UID. Error: " << ns_uid.getErr(); + return; + } + string playground_uid = isPlaygroundEnv() ? "playground-" : ""; - - dbgTrace(D_ORCHESTRATOR) << "Getting cluster UID"; - - auto maybe_namespaces_data = getNamespaceDataFromCluster("/api/v1/namespaces/"); - - if (!maybe_namespaces_data.ok()) { - dbgWarning(D_ORCHESTRATOR) - << "Failed to retrieve K8S namespace data. Error: " - << maybe_namespaces_data.getErr(); - return false; - } - - NamespaceData namespaces_data = maybe_namespaces_data.unpack(); - - Maybe maybe_ns_uid = namespaces_data.getNamespaceUidByName("kube-system"); - if (!maybe_ns_uid.ok()) { - dbgWarning(D_ORCHESTRATOR) << maybe_ns_uid.getErr(); - return false; - } - string uid = playground_uid + maybe_ns_uid.unpack(); + string uid = playground_uid + ns_uid.unpack(); dbgTrace(D_ORCHESTRATOR) << "Found k8s cluster UID: " << uid; - I_Environment *env = Singleton::Consume::by(); - env->getConfigurationContext().registerValue( + Singleton::Consume::by()->getConfigurationContext().registerValue( "k8sClusterId", uid, EnvKeyAttr::LogSection::SOURCE ); - I_AgentDetails *i_agent_details = Singleton::Consume::by(); - i_agent_details->setClusterId(uid); - return true; -} - -void -OrchestrationTools::Impl::getClusterId() const -{ - auto env_type = Singleton::Consume::by()->getEnvType(); - - if (env_type == EnvType::K8S) { - Singleton::Consume::by()->addOneTimeRoutine( - I_MainLoop::RoutineType::Offline, - [this] () - { - while(!doesClusterIdExists()) { - Singleton::Consume::by()->yield(chrono::seconds(1)); - } - return; - }, - "Get k8s cluster ID" - ); - } + Singleton::Consume::by()->setClusterId(uid); } bool diff --git a/components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc b/components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc index cbb5e85..430317f 100755 --- a/components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc +++ b/components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc @@ -77,30 +77,23 @@ TEST_F(OrchestrationToolsTest, doNothing) { } -TEST_F(OrchestrationToolsTest, getClusterId) +TEST_F(OrchestrationToolsTest, setClusterId) { EXPECT_CALL(mock_env_details, getToken()).WillOnce(Return("123")); EXPECT_CALL(mock_env_details, getEnvType()).WillOnce(Return(EnvType::K8S)); - I_MainLoop::Routine routine; - EXPECT_CALL( - mock_mainloop, - addOneTimeRoutine(I_MainLoop::RoutineType::Offline, _, "Get k8s cluster ID", _) - ).WillOnce(DoAll(SaveArg<1>(&routine), Return(1))); string namespaces = getResource("k8s_namespaces.json"); EXPECT_CALL( - mock_messaging, - sendSyncMessage( - HTTPMethod::GET, - "/api/v1/namespaces/", - _, - _, - _ + mock_shell_cmd, + getExecOutput( + "curl -s -k -H \"Authorization: Bearer 123\" -H \"Connection: close\" " + "https://kubernetes.default.svc:443/api/v1/namespaces/ | /etc/cp/bin/cpnano_json", + 200, + false ) - ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, namespaces))); + ).WillOnce(Return(namespaces)); - i_orchestration_tools->getClusterId(); - routine(); + i_orchestration_tools->setClusterId(); } TEST_F(OrchestrationToolsTest, writeReadTextToFile) diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc index 94de360..fcd314e 100644 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc @@ -24,6 +24,21 @@ using namespace testing; using namespace std; +string host_address = "1.2.3.5"; +string host_url = "https://" + host_address + "/"; +Maybe response( + string( + "{\n" + " \"fog-address\": \"" + host_url + "\",\n" + " \"agent-type\": \"test\",\n" + " \"pulling-interval\": 25,\n" + " \"error-pulling-interval\": 15\n" + "}" + ) +); +string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy"; +string orchestration_policy_file_path_bk = orchestration_policy_file_path + ".bk"; + class OrchestrationMultitenancyTest : public Test { public: @@ -54,6 +69,11 @@ public: void init() { + EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response)); + EXPECT_CALL(mock_status, setFogAddress(host_url)).WillRepeatedly(Return()); + EXPECT_CALL(mock_orchestration_tools, setClusterId()); + EXPECT_CALL(mock_service_controller, isServiceInstalled("Access Control")).WillRepeatedly(Return(false)); // This Holding the Main Routine of the Orchestration. @@ -62,8 +82,6 @@ public: addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Orchestration runner", true) ).WillOnce(DoAll(SaveArg<1>(&routine), Return(1))); - EXPECT_CALL(mock_orchestration_tools, getClusterId()); - EXPECT_CALL(mock_shell_cmd, getExecOutput("openssl version -d | cut -d\" \" -f2 | cut -d\"\\\"\" -f2", _, _)) .WillOnce(Return(string("OpenSSL certificates Directory"))); @@ -209,7 +227,6 @@ TEST_F(OrchestrationMultitenancyTest, init) TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource) { - string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy"; string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; @@ -237,22 +254,6 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource) init(); expectDetailsResolver(); - Maybe response( - string( - "{\n" - " \"fog-address\": \"" + host_url + "\",\n" - " \"agent-type\": \"test\",\n" - " \"pulling-interval\": 25,\n" - " \"error-pulling-interval\": 15\n" - "}" - ) - ); - - EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response)); - EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) - .WillOnce(Return(true)); - EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, manifest_file_path)) @@ -268,7 +269,11 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource) .WillOnce(Return(data_checksum)); EXPECT_CALL(mock_service_controller, getPolicyVersion()) - .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); + .Times(3).WillRepeatedly(ReturnRef(first_policy_version)); + + map empty_service_to_port_map; + EXPECT_CALL(mock_service_controller, getServiceToPortMap()).WillRepeatedly(Return(empty_service_to_port_map)); + set active_tenants = { "1236", "1235" }; map> old_tenant_profile_set; diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc index 4b7ebfd..4a2034d 100755 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc @@ -26,6 +26,21 @@ using namespace testing; using namespace std; +string host_address = "1.2.3.5"; +string host_url = "https://" + host_address + "/"; +Maybe response( + string( + "{\n" + " \"fog-address\": \"" + host_url + "\",\n" + " \"agent-type\": \"test\",\n" + " \"pulling-interval\": 25,\n" + " \"error-pulling-interval\": 15\n" + "}" + ) +); +string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy"; +string orchestration_policy_file_path_bk = orchestration_policy_file_path + ".bk"; + class OrchestrationTest : public testing::TestWithParam { public: @@ -48,14 +63,15 @@ public: void init() { - // This Holding the Main Routine of the Orchestration. + EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response)); + EXPECT_CALL(mock_status, setFogAddress(host_url)).WillRepeatedly(Return()); + EXPECT_CALL(mock_orchestration_tools, setClusterId()); EXPECT_CALL( mock_ml, addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Orchestration runner", true) ).WillOnce(DoAll(SaveArg<1>(&routine), Return(1))); - EXPECT_CALL(mock_orchestration_tools, getClusterId()); - EXPECT_CALL( mock_shell_cmd, getExecOutput("openssl version -d | cut -d\" \" -f2 | cut -d\"\\\"\" -f2", _, _) @@ -270,8 +286,6 @@ public: NiceMock mock_time_get; ::Environment env; string first_policy_version = ""; - string host_address = "1.2.3.5"; - string host_url = "https://" + host_address + "/"; ConfigComponent config_comp; StrictMock mock_encryptor; NiceMock mock_log; @@ -490,27 +504,12 @@ TEST_F(OrchestrationTest, check_sending_registration_data) env.init(); init(); - EXPECT_CALL(mock_orchestration_tools, doesFileExist(_)).WillOnce(Return(false)); - Maybe response( - string( - "{\n" - " \"fog-address\": \"" + host_url + "\",\n" - " \"agent-type\": \"test\",\n" - " \"pulling-interval\": 25,\n" - " \"error-pulling-interval\": 15\n" - "}" - ) - ); - EXPECT_CALL(mock_orchestration_tools, readFile(_)).WillOnce(Return(response)); EXPECT_CALL(mock_service_controller, updateServiceConfiguration(_, _, _, _, _, _)) .WillOnce(Return(Maybe())); - EXPECT_CALL(mock_message, setFogConnection(_, _, _, _)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, _)).WillRepeatedly(Return(string())); EXPECT_CALL(mock_service_controller, getPolicyVersion()).WillRepeatedly(ReturnRef(first_policy_version)); EXPECT_CALL(mock_shell_cmd, getExecOutput(_, _, _)).WillRepeatedly(Return(string())); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); - EXPECT_CALL(mock_update_communication, setAddressExtenesion(_)); - EXPECT_CALL(mock_status, setFogAddress(_)); EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); expectDetailsResolver(); EXPECT_CALL(mock_update_communication, getUpdate(_)); @@ -554,7 +553,6 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdatRollback) rest, mockRestCall(RestAction::ADD, "proxy", _) ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandler))); - EXPECT_CALL(mock_status, setFogAddress(host_url)).Times(2); string config_json = "{\n" @@ -592,17 +590,6 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdatRollback) string second_val = "12"; string third_val = "13"; - Maybe policy_response( - string( - "{\n" - " \"fog-address\": \"" + host_url + "\",\n" - " \"agent-type\": \"test\",\n" - " \"pulling-interval\": 25,\n" - " \"error-pulling-interval\": 15\n" - "}" - ) - ); - Maybe new_policy_response( string( "{\n" @@ -618,20 +605,21 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdatRollback) EXPECT_CALL(mock_service_controller, mockMoveChangedPolicies()).WillOnce(Return(expected_changed_policies)); EXPECT_CALL(mock_status, setFogAddress(new_host_url)); - EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)) + .WillOnce(Return(true)) + .WillOnce(Return(true)); // Rollback related test: The readFile function is called 3 times: // 1. Read the current policy file // 2. Read the new policy file - The one that should fail // 3. Read the current policy file again - The one that should be restored EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)) - .WillOnce(Return(policy_response)) .WillOnce(Return(new_policy_response)) - .WillOnce(Return(policy_response)); + .WillOnce(Return(response)); EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_path, policy_file_path + ".last")) .WillOnce(Return(true)); EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) - .Times(2).WillRepeatedly(Return(true)); - EXPECT_CALL(mock_update_communication, setAddressExtenesion("")).Times(2); + .WillOnce(Return(true)); + EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); expectDetailsResolver(); EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); @@ -649,7 +637,8 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdatRollback) // Rollback related test: After failing to update the policy file, the policy version should be restored EXPECT_CALL(mock_service_controller, getPolicyVersion()) - .Times(5) + .Times(6) + .WillOnce(ReturnRef(first_policy_version)) .WillOnce(ReturnRef(first_policy_version)) .WillOnce(ReturnRef(first_policy_version)) .WillOnce(ReturnRef(second_val)) @@ -772,7 +761,6 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate) rest, mockRestCall(RestAction::ADD, "proxy", _) ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandler))); - EXPECT_CALL(mock_status, setFogAddress(host_url)); init(); @@ -796,17 +784,6 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate) string second_val = "12"; string third_val = "13"; - Maybe policy_response( - string( - "{\n" - " \"fog-address\": \"" + host_url + "\",\n" - " \"agent-type\": \"test\",\n" - " \"pulling-interval\": 25,\n" - " \"error-pulling-interval\": 15\n" - "}" - ) - ); - Maybe new_policy_response( string( "{\n" @@ -824,13 +801,9 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate) EXPECT_CALL(mock_status, setFogAddress(new_host_url)); EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)) - .WillOnce(Return(policy_response)) .WillOnce(Return(new_policy_response)); EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_path, policy_file_path + ".last")) .WillOnce(Return(true)); - EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) - .WillOnce(Return(true)); - EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); expectDetailsResolver(); EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); @@ -847,7 +820,8 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate) .WillOnce(Return(data_checksum)); EXPECT_CALL(mock_service_controller, getPolicyVersion()) - .Times(4) + .Times(5) + .WillOnce(ReturnRef(first_policy_version)) .WillOnce(ReturnRef(first_policy_version)) .WillOnce(ReturnRef(first_policy_version)) .WillOnce(ReturnRef(second_val)) @@ -939,178 +913,52 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate) } catch (const invalid_argument& e) {} } -TEST_F(OrchestrationTest, startOrchestrationPoliceWithFailures) -{ - waitForRestCall(); - preload(); - Maybe msg_err = genError("Failed to send message"); - EXPECT_CALL(mock_status, setFogAddress(host_url)); - EXPECT_CALL( - rest, - mockRestCall(RestAction::ADD, "proxy", _) - ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandler))); - init(); - string orchestration_policy_file_path = getPolicyConfigPath("orchestration", Config::ConfigFileType::Policy); - string orchestration_policy_file_path_bk = orchestration_policy_file_path + ".bk"; - string manifest_file_path = "/etc/cp/conf/manifest.json"; - string setting_file_path = "/etc/cp/conf/settings.json"; - string policy_file_path = "/etc/cp/conf/policy.json"; - string last_policy_file_path = "/etc/cp/conf/policy.json.last"; - string data_file_path = "/etc/cp/conf/data.json"; - - string host_address = "1.2.3.5"; - string manifest_checksum = "manifest"; - string policy_checksum = "policy"; - string settings_checksum = "settings"; - string data_checksum = "data"; - - Maybe response( - string( - "{\n" - " \"fog-address\": \"" + host_url + "\",\n" - " \"agent-type\": \"test\",\n" - " \"pulling-interval\": 25,\n" - " \"error-pulling-interval\": 15\n" - "}" - ) - ); - EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)) - .WillOnce(Return(Maybe(genError("Failed")))) - .WillOnce(Return(response)); - - EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path_bk)).WillOnce( - Return(Maybe(genError("Failed"))) - ); - - vector expected_data_types = {}; - EXPECT_CALL( - mock_service_controller, - updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "", _) - ).Times(2).WillRepeatedly(Return(Maybe())); - - EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) - .WillOnce(Return(true)); - EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); - EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); - expectDetailsResolver(); - EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, manifest_file_path)) - .WillOnce(Return(manifest_checksum)); - - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, setting_file_path)) - .WillOnce(Return(settings_checksum)); - - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, policy_file_path)) - .WillOnce(Return(policy_checksum)); - - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, data_file_path)) - .WillOnce(Return(data_checksum)); - - EXPECT_CALL(mock_service_controller, getPolicyVersion()) - .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); - - EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce( - Invoke( - [&](CheckUpdateRequest &req) - { - EXPECT_THAT(req.getPolicy(), IsValue(policy_checksum)); - EXPECT_THAT(req.getSettings(), IsValue(settings_checksum)); - EXPECT_THAT(req.getManifest(), IsValue(manifest_checksum)); - EXPECT_THAT(req.getData(), IsValue(data_checksum)); - req = CheckUpdateRequest("", "", "", "", "", ""); - return Maybe(); - } - ) - ); - - EXPECT_CALL(mock_status, setLastUpdateAttempt()); - EXPECT_CALL( - mock_status, - setFieldStatus(OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::SUCCESS, "") - ); - - EXPECT_CALL(mock_status, setIsConfigurationUpdated(A>()) - ).WillOnce( - Invoke( - [](EnumArray arr) - { - EXPECT_EQ(arr[OrchestrationStatusConfigType::MANIFEST], false); - EXPECT_EQ(arr[OrchestrationStatusConfigType::POLICY], false); - EXPECT_EQ(arr[OrchestrationStatusConfigType::SETTINGS], false); - } - ) - ); - - EXPECT_CALL(mock_ml, yield(A())) - .WillOnce( - Invoke( - [] (chrono::microseconds microseconds) - { - EXPECT_EQ(1000000, microseconds.count()); - } - ) - ) - .WillOnce( - Invoke( - [] (chrono::microseconds microseconds) - { - EXPECT_EQ(25000000, microseconds.count()); - throw invalid_argument("stop while loop"); - } - ) - ); - EXPECT_CALL( - mock_shell_cmd, - getExecOutput(_, _, _) - ).WillRepeatedly(Return(string("daniel\n1\n"))); - try { - runRoutine(); - } catch (const invalid_argument& e) {} -} - TEST_F(OrchestrationTest, loadOrchestrationPolicyFromBackup) { EXPECT_CALL( rest, mockRestCall(RestAction::ADD, "proxy", _) - ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandler))); + ); waitForRestCall(); - init(); - string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy"; - string orchestration_policy_file_path_bk = orchestration_policy_file_path + ".bk"; - string manifest_file_path = "/etc/cp/conf/manifest.json"; - string setting_file_path = "/etc/cp/conf/settings.json"; - string policy_file_path = "/etc/cp/conf/policy.json"; - string last_policy_file_path = "/etc/cp/conf/policy.json.last"; - string data_file_path = "/etc/cp/conf/data.json"; - string host_address = "1.2.3.5"; - string manifest_checksum = "manifest"; - string policy_checksum = "policy"; - string settings_checksum = "settings"; - string data_checksum = "data"; + EXPECT_CALL( + mock_ml, + addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Orchestration runner", true) + ); - Maybe response( - string( - "{\n" - " \"fog-address\": \"https://1.2.3.5/\",\n" - " \"agent-type\": \"test\",\n" - " \"pulling-interval\": 25,\n" - " \"error-pulling-interval\": 15\n" - "}" + EXPECT_CALL( + mock_shell_cmd, + getExecOutput("openssl version -d | cut -d\" \" -f2 | cut -d\"\\\"\" -f2", _, _) + ).WillOnce(Return(string("OpenSSL certificates Directory"))); + + EXPECT_CALL(mock_service_controller, isServiceInstalled("Access Control")).WillRepeatedly( + InvokeWithoutArgs( + []() + { + static int count = 0; + if (count > 0) return false; + count++; + return true; + } ) ); - EXPECT_CALL(mock_status, setFogAddress(host_url)); + map empty_service_to_port_map; + EXPECT_CALL(mock_service_controller, getServiceToPortMap()).WillRepeatedly(Return(empty_service_to_port_map)); + + EXPECT_CALL(rest, mockRestCall(RestAction::SHOW, "orchestration-status", _)); - vector expected_data_types = {}; EXPECT_CALL( - mock_service_controller, - updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "", _) - ).WillOnce(Return(Maybe())); + rest, + mockRestCall(RestAction::SET, "agent-uninstall", _) + ); + doEncrypt(); + EXPECT_CALL(mock_orchestration_tools, loadTenantsFromDir(_)).Times(1); + + EXPECT_CALL(mock_status, setFogAddress(host_url)); EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, setClusterId()); EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)) .WillOnce(Return(Maybe(genError("Failed")))); EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path_bk)).WillOnce(Return(response)); @@ -1118,81 +966,8 @@ TEST_F(OrchestrationTest, loadOrchestrationPolicyFromBackup) mock_orchestration_tools, copyFile(orchestration_policy_file_path_bk, orchestration_policy_file_path) ).WillOnce(Return(true)); - EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) - .WillOnce(Return(true)); - EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); - EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); - expectDetailsResolver(); - EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, manifest_file_path)) - .WillOnce(Return(manifest_checksum)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, setting_file_path)) - .WillOnce(Return(settings_checksum)); - - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, policy_file_path)) - .WillOnce(Return(policy_checksum)); - - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, data_file_path)) - .WillOnce(Return(data_checksum)); - - EXPECT_CALL(mock_service_controller, getPolicyVersion()) - .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); - EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce( - Invoke( - [&](CheckUpdateRequest &req) - { - EXPECT_THAT(req.getPolicy(), IsValue(policy_checksum)); - EXPECT_THAT(req.getSettings(), IsValue(settings_checksum)); - EXPECT_THAT(req.getManifest(), IsValue(manifest_checksum)); - EXPECT_THAT(req.getData(), IsValue(data_checksum)); - req = CheckUpdateRequest("", "", "", "", "", ""); - return Maybe(); - } - ) - ); - - EXPECT_CALL(mock_status, setLastUpdateAttempt()); - EXPECT_CALL( - mock_status, - setFieldStatus(OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::SUCCESS, "") - ); - EXPECT_CALL(mock_status, setIsConfigurationUpdated(A>()) - ).WillOnce( - Invoke( - [](EnumArray arr) - { - EXPECT_EQ(arr[OrchestrationStatusConfigType::MANIFEST], false); - EXPECT_EQ(arr[OrchestrationStatusConfigType::POLICY], false); - EXPECT_EQ(arr[OrchestrationStatusConfigType::SETTINGS], false); - } - ) - ); - EXPECT_CALL(mock_ml, yield(A())) - .WillOnce( - Invoke( - [] (chrono::microseconds microseconds) - { - EXPECT_EQ(1000000, microseconds.count()); - } - ) - ) - .WillOnce( - Invoke( - [] (chrono::microseconds microseconds) - { - EXPECT_EQ(25000000, microseconds.count()); - throw invalid_argument("stop while loop"); - } - ) - ); - EXPECT_CALL( - mock_shell_cmd, - getExecOutput(_, _, _) - ).WillRepeatedly(Return(string("daniel\n1\n"))); - try { - runRoutine(); - } catch (const invalid_argument& e) {} + orchestration_comp.init(); } TEST_F(OrchestrationTest, newServicePolicyUpdate) @@ -1213,7 +988,6 @@ TEST_F(OrchestrationTest, manifestUpdate) ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandler))); waitForRestCall(); init(); - string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy"; string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; @@ -1226,30 +1000,12 @@ TEST_F(OrchestrationTest, manifestUpdate) string settings_checksum= "settings"; string data_checksum = "data"; - EXPECT_CALL(mock_status, setFogAddress(host_url)); - - Maybe response( - string( - "{\n" - " \"fog-address\": \"" + host_url + "\",\n" - " \"agent-type\": \"test\",\n" - " \"pulling-interval\": 25,\n" - " \"error-pulling-interval\": 15\n" - "}" - ) - ); - vector expected_data_types = {}; EXPECT_CALL( mock_service_controller, updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "", _) ).WillOnce(Return(Maybe())); - EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response)); - EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) - .WillOnce(Return(true)); - EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); expectDetailsResolver(); EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); @@ -1266,7 +1022,7 @@ TEST_F(OrchestrationTest, manifestUpdate) .WillOnce(Return(data_checksum)); EXPECT_CALL(mock_service_controller, getPolicyVersion()) - .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); + .Times(3).WillRepeatedly(ReturnRef(first_policy_version)); EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce( Invoke( [&](CheckUpdateRequest &req) @@ -1346,7 +1102,6 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandler))); waitForRestCall(); init(); - string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy"; string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; @@ -1359,18 +1114,6 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) string settings_checksum = "settings"; string data_checksum = "data"; - Maybe response( - string( - "{\n" - " \"fog-address\": \"" + host_url + "\",\n" - " \"agent-type\": \"test\",\n" - " \"pulling-interval\": 25,\n" - " \"error-pulling-interval\": 15\n" - "}" - ) - ); - EXPECT_CALL(mock_status, setFogAddress(host_url)); - vector expected_data_types = {}; EXPECT_CALL( @@ -1380,13 +1123,9 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) set expected_changed_policies = {}; EXPECT_CALL(mock_service_controller, mockMoveChangedPolicies()).WillOnce(Return(expected_changed_policies)); - EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response)); + EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_path, policy_file_path + ".last")) .WillOnce(Return(true)); - EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) - .WillOnce(Return(true)); - EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); expectDetailsResolver(); @@ -1421,7 +1160,8 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) string second_val = "12"; string third_val = "13"; EXPECT_CALL(mock_service_controller, getPolicyVersion()) - .Times(3) + .Times(4) + .WillOnce(ReturnRef(first_policy_version)) .WillOnce(ReturnRef(first_policy_version)) .WillOnce(ReturnRef(first_policy_version)) .WillOnce(ReturnRef(second_val) @@ -1456,7 +1196,7 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) ) ); - EXPECT_CALL(mock_service_controller, getUpdatePolicyVersion()).Times(1).WillOnce(ReturnRef(third_val)); + EXPECT_CALL(mock_service_controller, getUpdatePolicyVersion()).WillRepeatedly(ReturnRef(third_val)); EXPECT_CALL( mock_service_controller, @@ -1498,7 +1238,6 @@ TEST_F(OrchestrationTest, failedDownloadSettings) ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandler))); waitForRestCall(); init(); - string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy"; string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; @@ -1511,30 +1250,12 @@ TEST_F(OrchestrationTest, failedDownloadSettings) string settings_checksum = "settings-checksum"; string data_checksum = "data"; - Maybe response( - string( - "{\n" - " \"fog-address\": \"" + host_url + "\",\n" - " \"agent-type\": \"test\",\n" - " \"pulling-interval\": 25,\n" - " \"error-pulling-interval\": 15\n" - "}" - ) - ); - EXPECT_CALL(mock_status, setFogAddress(host_url)); - vector expected_data_types = {}; EXPECT_CALL( mock_service_controller, updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "", _) ).WillOnce(Return(Maybe())); - EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response)); - EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) - .WillOnce(Return(true)); - EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); - EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); expectDetailsResolver(); EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); @@ -1553,7 +1274,7 @@ TEST_F(OrchestrationTest, failedDownloadSettings) Maybe new_policy_checksum(string("111111")); EXPECT_CALL(mock_service_controller, getPolicyVersion()) - .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); + .Times(3).WillRepeatedly(ReturnRef(first_policy_version)); EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce( Invoke( [&](CheckUpdateRequest &req) @@ -1652,7 +1373,6 @@ TEST_P(OrchestrationTest, orchestrationFirstRun) ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandler))); waitForRestCall(); init(); - string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy"; string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; @@ -1669,26 +1389,6 @@ TEST_P(OrchestrationTest, orchestrationFirstRun) string policy = ""; string setting = ""; - Maybe response( - string( - "{\n" - " \"fog-address\": \"" + host_url + "\",\n" - " \"agent-type\": \"test\",\n" - " \"pulling-interval\": 25,\n" - " \"error-pulling-interval\": 15\n" - "}" - ) - ); - - EXPECT_CALL(mock_status, setFogAddress(host_url)); - EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(false)); - EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)) - .WillOnce(Return(response)); - EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)). - Times(1). - WillRepeatedly(Return(true)); - EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); - EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); expectDetailsResolver(); EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(true)); @@ -1721,7 +1421,7 @@ TEST_P(OrchestrationTest, orchestrationFirstRun) } ) ); - EXPECT_CALL(mock_service_controller, getPolicyVersion()).WillOnce(ReturnRef(first_policy_version)); + EXPECT_CALL(mock_service_controller, getPolicyVersion()).WillRepeatedly(ReturnRef(first_policy_version)); EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce( Invoke( [&](CheckUpdateRequest &req) @@ -1854,7 +1554,6 @@ TEST_F(OrchestrationTest, set_proxy) mockRestCall(RestAction::ADD, "proxy", _) ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandler))); waitForRestCall(); - init(); stringstream is; string proxy_url = "http://some-proxy.com:8080"; @@ -1873,7 +1572,7 @@ TEST_F(OrchestrationTest, dataUpdate) ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandler))); waitForRestCall(); init(); - string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy"; + string manifest_file_path = "/etc/cp/conf/manifest.json"; string setting_file_path = "/etc/cp/conf/settings.json"; string policy_file_path = "/etc/cp/conf/policy.json"; @@ -1890,19 +1589,6 @@ TEST_F(OrchestrationTest, dataUpdate) string data_checksum_type = "sha1sum"; string data_instance_checksum = "8d4a5709673a05b380ba7d6567e28910019118f5"; - EXPECT_CALL(mock_status, setFogAddress(host_url)); - - Maybe policy_response( - string( - "{\n" - " \"fog-address\": \"" + host_url + "\",\n" - " \"agent-type\": \"test\",\n" - " \"pulling-interval\": 25,\n" - " \"error-pulling-interval\": 15\n" - "}" - ) - ); - Maybe data_response( string( "{\n" @@ -1929,14 +1615,8 @@ TEST_F(OrchestrationTest, dataUpdate) ).After(expectation_set).WillOnce(Return(Maybe())); EXPECT_CALL(mock_orchestration_tools, doesDirectoryExist("/etc/cp/conf/data")).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(policy_response)); EXPECT_CALL(mock_orchestration_tools, readFile(data_file_path + ".download")).WillOnce(Return(data_response)); - - EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); - EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) - .WillOnce(Return(true)); - EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); expectDetailsResolver(); @@ -1955,8 +1635,7 @@ TEST_F(OrchestrationTest, dataUpdate) EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, "/path/ips")) .WillOnce(Return(data_instance_checksum)); - EXPECT_CALL(mock_service_controller, getPolicyVersion()) - .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); + EXPECT_CALL(mock_service_controller, getPolicyVersion()).WillRepeatedly(ReturnRef(first_policy_version)); EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce( Invoke( [&](CheckUpdateRequest &req) diff --git a/components/security_apps/orchestration/package_handler/package_handler.cc b/components/security_apps/orchestration/package_handler/package_handler.cc index fdddf8d..3679660 100755 --- a/components/security_apps/orchestration/package_handler/package_handler.cc +++ b/components/security_apps/orchestration/package_handler/package_handler.cc @@ -109,6 +109,10 @@ packageHandlerActionsToString(PackageHandlerActions action) installation_mode += " --certs-dir "; installation_mode += trusted_ca_directory.unpack(); } + + auto maybe_vs_id = Singleton::Consume::by()->get("VS ID"); + if (maybe_vs_id.ok()) installation_mode += " --vs_id " + *maybe_vs_id; + AdditionalFlagsConfiguration additional_flags = getConfigurationWithDefault( AdditionalFlagsConfiguration(), "orchestration", diff --git a/components/security_apps/orchestration/service_controller/service_controller.cc b/components/security_apps/orchestration/service_controller/service_controller.cc index b09e9b7..9457c86 100755 --- a/components/security_apps/orchestration/service_controller/service_controller.cc +++ b/components/security_apps/orchestration/service_controller/service_controller.cc @@ -33,7 +33,7 @@ using namespace std; using namespace ReportIS; -USE_DEBUG_FLAG(D_ORCHESTRATOR); +USE_DEBUG_FLAG(D_SERVICE_CONTROLLER); class SendConfigurations : public ClientRest { @@ -56,7 +56,7 @@ public: auto service_controller = Singleton::Consume::by(); if (!finished.get()) { service_controller->updateReconfStatus(id.get(), service_name.get(), ReconfStatus::IN_PROGRESS); - dbgTrace(D_ORCHESTRATOR) + dbgTrace(D_SERVICE_CONTROLLER) << "Request for service reconfiguration is still in progress. ID: " << id.get() << ", Service Name: " @@ -65,7 +65,7 @@ public: } if (error.get()) { service_controller->updateReconfStatus(id.get(), service_name.get(), ReconfStatus::FAILED); - dbgError(D_ORCHESTRATOR) + dbgError(D_SERVICE_CONTROLLER) << "Request for service reconfiguration failed to complete. ID: " << id.get() << ", Service Name: " @@ -75,7 +75,7 @@ public: return; } service_controller->updateReconfStatus(id.get(), service_name.get(), ReconfStatus::SUCCEEDED); - dbgInfo(D_ORCHESTRATOR) + dbgInfo(D_SERVICE_CONTROLLER) << "Request for service reconfiguration successfully accomplished. Reconf ID: " << id.get() << ", Service Name: " @@ -112,7 +112,7 @@ ServiceDetails::isServiceActive() const } } - dbgDebug(D_ORCHESTRATOR) + dbgDebug(D_SERVICE_CONTROLLER) << "Executing service status check via watchdog api. Service name: " << service_name << ", Watchdog command: " @@ -133,7 +133,7 @@ ServiceDetails::isServiceActive() const for (int current_attempt = 0; current_attempt < max_retry_attempts; ++current_attempt) { if (service_status.ok() || service_status.getErr().find("Reached timeout") == string::npos) break; - dbgWarning(D_ORCHESTRATOR) + dbgWarning(D_SERVICE_CONTROLLER) << "Retrying to execute service status check via watchdog API after getting timeout. Service name: " << service_name << ", Watchdog command: " @@ -146,7 +146,7 @@ ServiceDetails::isServiceActive() const } if (!service_status.ok()) { - dbgWarning(D_ORCHESTRATOR) + dbgWarning(D_SERVICE_CONTROLLER) << "Changing service status to inactive after failure to its status from watchdog. Service name: " << service_name << ", Watchdog output: " @@ -154,7 +154,7 @@ ServiceDetails::isServiceActive() const return false; } - dbgDebug(D_ORCHESTRATOR) + dbgDebug(D_SERVICE_CONTROLLER) << "Successfully retrieved service status from watchdog. Service name: " << service_name << ", Watchdog output: " @@ -166,7 +166,7 @@ ServiceDetails::isServiceActive() const bool is_registered = status.find("not-registered") == string::npos && status.find("registered") != string::npos; bool is_running = status.find("not-running") == string::npos && status.find("running") != string::npos; - dbgTrace(D_ORCHESTRATOR) + dbgTrace(D_SERVICE_CONTROLLER) << "Successfully set service status. Service name: " << service_name << ", Status: " @@ -189,7 +189,7 @@ ReconfStatus ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy_version) { if(!isServiceActive()) { - dbgDebug(D_ORCHESTRATOR) << "Service " << service_name << " is inactive"; + dbgDebug(D_SERVICE_CONTROLLER) << "Service " << service_name << " is inactive"; return ReconfStatus::INACTIVE; } @@ -210,7 +210,7 @@ ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy if (!res.ok()) { auto err = res.getErr(); - dbgDebug(D_ORCHESTRATOR) + dbgDebug(D_SERVICE_CONTROLLER) << "Service: " << service_name << " didn't get new configuration. Error: " @@ -223,7 +223,7 @@ ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy if (new_config.finished.get()) { if (!new_config.error.get()) { service_details->startReconfStatus(new_config.id.get(), ReconfStatus::SUCCEEDED, service_name, service_id); - dbgDebug(D_ORCHESTRATOR) << "Loading service configuration succeeded for service " << service_name; + dbgDebug(D_SERVICE_CONTROLLER) << "Loading service configuration succeeded for service " << service_name; return ReconfStatus::SUCCEEDED; } else { string log_name = "Agent could not update policy to version " + @@ -241,7 +241,7 @@ ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy << LogField("policyVersion", service_details->getPolicyVersion()); service_details->startReconfStatus(new_config.id.get(), ReconfStatus::FAILED, service_name, service_id); - dbgDebug(D_ORCHESTRATOR) + dbgDebug(D_SERVICE_CONTROLLER) << "Loading service configuration failed for service " << service_name << " with error: " @@ -249,7 +249,7 @@ ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy return ReconfStatus::FAILED; } } - dbgDebug(D_ORCHESTRATOR) << "Loading service configuration is in progress for service: " << service_name; + dbgDebug(D_SERVICE_CONTROLLER) << "Loading service configuration is in progress for service: " << service_name; service_details->startReconfStatus(new_config.id.get(), ReconfStatus::IN_PROGRESS, service_name, service_id); return ReconfStatus::IN_PROGRESS; } @@ -257,7 +257,7 @@ ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy void SetNanoServiceConfig::doCall() { - dbgFlow(D_ORCHESTRATOR) + dbgFlow(D_SERVICE_CONTROLLER) << "Received registration request from service. Service name: " << service_name.get() << ", service listening port: " @@ -402,12 +402,12 @@ ServiceController::Impl::getUpdatedReconfStatus() auto maybe_service = getServiceDetails(service_id); if (!maybe_service.ok()) { - dbgWarning(D_ORCHESTRATOR) << "Unable to get service details. Error: " << maybe_service.getErr(); + dbgWarning(D_SERVICE_CONTROLLER) << "Unable to get service details. Error: " << maybe_service.getErr(); continue; } if (!maybe_service.unpack().isServiceActive()) { - dbgInfo(D_ORCHESTRATOR) + dbgInfo(D_SERVICE_CONTROLLER) << "Service is not active, removing from registered services list. Service: " << services_reconf_names[service_and_reconf_status.first] << "ID: " @@ -490,7 +490,7 @@ ServiceController::Impl::loadRegisteredServicesFromFile() auto maybe_registered_services_str = Singleton::Consume::by()-> readFile(registered_services_file); if (!maybe_registered_services_str.ok()) { - dbgTrace(D_ORCHESTRATOR) + dbgTrace(D_SERVICE_CONTROLLER) << "could not read file. File: " << registered_services_file << " Error: " << maybe_registered_services_str.getErr(); @@ -501,7 +501,7 @@ ServiceController::Impl::loadRegisteredServicesFromFile() cereal::JSONInputArchive ar(ss); ar(cereal::make_nvp("Registered Services", pending_services)); - dbgInfo(D_ORCHESTRATOR) + dbgInfo(D_SERVICE_CONTROLLER) << "Orchestration pending services loaded from file." << " File: " << registered_services_file @@ -509,7 +509,7 @@ ServiceController::Impl::loadRegisteredServicesFromFile() for (const auto &id_service_pair : pending_services) { const auto &service = id_service_pair.second; - dbgInfo(D_ORCHESTRATOR) + dbgInfo(D_SERVICE_CONTROLLER) << "Service name: " << service.getServiceName() << ", Service ID: " @@ -522,7 +522,7 @@ ServiceController::Impl::loadRegisteredServicesFromFile() void ServiceController::Impl::writeRegisteredServicesToFile() { - dbgFlow(D_ORCHESTRATOR); + dbgFlow(D_SERVICE_CONTROLLER); auto registered_services_file = getConfigurationWithDefault( filesystem_prefix + "/conf/orchestrations_registered_services.json", "orchestration", @@ -533,14 +533,14 @@ ServiceController::Impl::writeRegisteredServicesToFile() cereal::JSONOutputArchive ar(ss); ar(cereal::make_nvp("Registered Services", registered_services)); - dbgInfo(D_ORCHESTRATOR) + dbgInfo(D_SERVICE_CONTROLLER) << "Orchestration registered services file has been updated. File: " << registered_services_file << ". Registered Services:"; for (const auto &id_service_pair : registered_services) { const auto &service = id_service_pair.second; - dbgInfo(D_ORCHESTRATOR) + dbgInfo(D_SERVICE_CONTROLLER) << "Service name: " << service.getServiceName() << ", Service ID: " @@ -626,6 +626,7 @@ ServiceController::Impl::registerServiceConfig( pending_services.erase(service_config.getServiceID()); pending_services.insert({service_config.getServiceID(), service_config}); + refreshPendingServices(); } bool @@ -639,12 +640,12 @@ ServiceController::Impl::isServiceInstalled(const string &service_name) void ServiceController::Impl::refreshPendingServices() { - dbgFlow(D_ORCHESTRATOR); + dbgFlow(D_SERVICE_CONTROLLER); if (pending_services.empty()) return; for (const auto &service : pending_services) { registered_services.erase(service.first); registered_services.insert({service.first, service.second}); - dbgDebug(D_ORCHESTRATOR) << "Successfully registered service. Name: " << service.first; + dbgDebug(D_SERVICE_CONTROLLER) << "Successfully registered service. Name: " << service.first; } pending_services.clear(); @@ -659,7 +660,7 @@ ServiceController::Impl::backupConfigurationFile(const string &config_file_path) string backup_file = config_file_path + backup_ext; if (!orchestration_tools->doesFileExist(config_file_path)) { - dbgTrace(D_ORCHESTRATOR) << "File does not exist. File: " << config_file_path; + dbgTrace(D_SERVICE_CONTROLLER) << "File does not exist. File: " << config_file_path; return true; } @@ -670,7 +671,7 @@ ServiceController::Impl::backupConfigurationFile(const string &config_file_path) mainloop->yield(false); } - dbgWarning(D_ORCHESTRATOR) << "Failed to back up the file. File: " << config_file_path; + dbgWarning(D_SERVICE_CONTROLLER) << "Failed to back up the file. File: " << config_file_path; return false; } @@ -692,12 +693,12 @@ ServiceController::Impl::createDirectoryForChildTenant( if (orchestration_tools->doesDirectoryExist(dir)) return true; if (!orchestration_tools->createDirectory(dir)) { - dbgError(D_ORCHESTRATOR) + dbgError(D_SERVICE_CONTROLLER) << "Failed to create configuration directory for tenant " << child_tenant_id; return false; } - dbgTrace(D_ORCHESTRATOR) << "Created new configuration directory for tenant " << child_tenant_id; + dbgTrace(D_SERVICE_CONTROLLER) << "Created new configuration directory for tenant " << child_tenant_id; return true; } @@ -716,7 +717,7 @@ getChecksum(const string &file_path) try { checksum = to_string(boost::uuids::random_generator()()); } catch (const boost::uuids::entropy_error &e) { - dbgDebug(D_ORCHESTRATOR) << "Couldn't generate random checksum"; + dbgDebug(D_SERVICE_CONTROLLER) << "Couldn't generate random checksum"; } return checksum; } @@ -734,7 +735,7 @@ ServiceController::Impl::updateServiceConfiguration( if (!child_tenant_id.empty()) { tenant_and_profile_ids = " Child tenant id: " + child_tenant_id + ", Child profile id: " + child_profile_id; } - dbgFlow(D_ORCHESTRATOR) + dbgFlow(D_SERVICE_CONTROLLER) << "new_policy_path: " << new_policy_path << ", new_settings_path: " @@ -758,9 +759,9 @@ ServiceController::Impl::updateServiceConfiguration( } for (const string &data : new_data_files) { - dbgTrace(D_ORCHESTRATOR) << "data: " << data; + dbgTrace(D_SERVICE_CONTROLLER) << "data: " << data; if (service.second.isConfigurationRelevant(data)) { - dbgTrace(D_ORCHESTRATOR) + dbgTrace(D_SERVICE_CONTROLLER) << "data has relevant configuration, will update the service: " << service.first; nano_services_to_update.insert(service.first); @@ -770,7 +771,8 @@ ServiceController::Impl::updateServiceConfiguration( } if (new_policy_path == "") { - dbgDebug(D_ORCHESTRATOR) << "Policy file was not updated. Sending reload command regarding settings and data"; + dbgDebug(D_SERVICE_CONTROLLER) + << "Policy file was not updated. Sending reload command regarding settings and data"; auto signal_services = sendSignalForServices(nano_services_to_update, ""); if (!signal_services.ok()) return signal_services.passErr(); Singleton::Consume::from()->turnOffApplyPolicyFlag(); @@ -779,7 +781,7 @@ ServiceController::Impl::updateServiceConfiguration( Maybe loaded_policy_json = orchestration_tools->readFile(new_policy_path); if (!loaded_policy_json.ok()) { - dbgWarning(D_ORCHESTRATOR) + dbgWarning(D_SERVICE_CONTROLLER) << "Failed to load new file: " << new_policy_path << ". Error: " @@ -795,7 +797,7 @@ ServiceController::Impl::updateServiceConfiguration( ); if (!all_security_policies.ok()) { - dbgWarning(D_ORCHESTRATOR) + dbgWarning(D_SERVICE_CONTROLLER) << "Failed to parse json file: " << new_policy_path << ". Error: " @@ -825,13 +827,13 @@ ServiceController::Impl::updateServiceConfiguration( if (child_tenant_id.empty() && single_policy.first == versions_param) { //In a multi-tenant env, only the parent should handle the versions parameter policy_versions = single_policy.second; - dbgWarning(D_ORCHESTRATOR) << "Found versions parameter in policy file:" << policy_versions; + dbgWarning(D_SERVICE_CONTROLLER) << "Found versions parameter in policy file:" << policy_versions; } - dbgDebug(D_ORCHESTRATOR) << "Starting to update policy file. Policy type: " << single_policy.first; + dbgDebug(D_SERVICE_CONTROLLER) << "Starting to update policy file. Policy type: " << single_policy.first; if (!createDirectoryForChildTenant(child_tenant_id, child_profile_id)) { - dbgWarning(D_ORCHESTRATOR) + dbgWarning(D_SERVICE_CONTROLLER) << "Failed to create directory for child. Tenant id: " << child_tenant_id << ", Profile id: " << child_profile_id; return genError("Failed to create directory for child tenant"); @@ -861,7 +863,7 @@ ServiceController::Impl::updateServiceConfiguration( } changed_policy_files.insert(policy_file_path); - dbgInfo(D_ORCHESTRATOR) << "Successfully updated policy file. Policy name: " << single_policy.first; + dbgInfo(D_SERVICE_CONTROLLER) << "Successfully updated policy file. Policy name: " << single_policy.first; auto orc_status = Singleton::Consume::by(); orc_status->setServiceConfiguration( @@ -878,7 +880,9 @@ ServiceController::Impl::updateServiceConfiguration( for (const auto &instance_id: instances) { auto relevant_service = registered_services.find(instance_id); if (relevant_service == registered_services.end()) { - dbgWarning(D_ORCHESTRATOR) << "Could not find registered service. Service Id: " << instance_id; + dbgWarning(D_SERVICE_CONTROLLER) + << "Could not find registered service. Service Id: " + << instance_id; continue; } if (relevant_service->second.isConfigurationRelevant(single_policy.first)) { @@ -902,7 +906,7 @@ ServiceController::Impl::updateServiceConfiguration( if (!is_send_signal_for_services.ok()) send_signal_for_services_err = is_send_signal_for_services.getErr(); } - dbgTrace(D_ORCHESTRATOR) << "was policy updated: " << (was_policy_updated ? "true" : "false"); + dbgTrace(D_SERVICE_CONTROLLER) << "was policy updated: " << (was_policy_updated ? "true" : "false"); if (was_policy_updated) { string base_path = @@ -916,14 +920,14 @@ ServiceController::Impl::updateServiceConfiguration( ); if (new_policy_path.compare(config_file_path) == 0) { - dbgDebug(D_ORCHESTRATOR) << "Enforcing the default policy file"; + dbgDebug(D_SERVICE_CONTROLLER) << "Enforcing the default policy file"; policy_version = version_value; Singleton::Consume::from()->turnOffApplyPolicyFlag(); return Maybe(); } if (!backupConfigurationFile(config_file_path)) { - dbgWarning(D_ORCHESTRATOR) << "Failed to backup the policy file."; + dbgWarning(D_SERVICE_CONTROLLER) << "Failed to backup the policy file."; return genError("Failed to backup the policy file."); } @@ -931,7 +935,7 @@ ServiceController::Impl::updateServiceConfiguration( // Save the new configuration file. if (!orchestration_tools->copyFile(new_policy_path, config_file_path)) { - dbgWarning(D_ORCHESTRATOR) << "Failed to save the policy file."; + dbgWarning(D_SERVICE_CONTROLLER) << "Failed to save the policy file."; return genError("Failed to save the policy file."); } } @@ -946,11 +950,11 @@ ServiceController::Impl::sendSignalForServices( const set &nano_services_to_update, const string &policy_version_to_update) { - dbgFlow(D_ORCHESTRATOR); + dbgFlow(D_SERVICE_CONTROLLER); for (auto &service_id : nano_services_to_update) { auto nano_service = registered_services.find(service_id); if (nano_service == registered_services.end()) { - dbgWarning(D_ORCHESTRATOR) << "Could not find registered service. Service Id: " << service_id; + dbgWarning(D_SERVICE_CONTROLLER) << "Could not find registered service. Service Id: " << service_id; continue; } @@ -958,13 +962,13 @@ ServiceController::Impl::sendSignalForServices( auto reconf_status = nano_service->second.sendNewConfigurations(configuration_id, policy_version_to_update); if (reconf_status == ReconfStatus::INACTIVE) { - dbgWarning(D_ORCHESTRATOR) << "Erasing details regarding inactive service " << service_id; + dbgWarning(D_SERVICE_CONTROLLER) << "Erasing details regarding inactive service " << service_id; registered_services.erase(service_id); writeRegisteredServicesToFile(); } if (reconf_status == ReconfStatus::FAILED) { - dbgDebug(D_ORCHESTRATOR) << "The reconfiguration failed for serivce: " << service_id; + dbgDebug(D_SERVICE_CONTROLLER) << "The reconfiguration failed for serivce: " << service_id; services_reconf_status.clear(); services_reconf_names.clear(); return genError("The reconfiguration failed for serivce: " + service_id); @@ -985,13 +989,14 @@ ServiceController::Impl::sendSignalForServices( while(timer->getMonotonicTime() < current_timeout) { switch (getUpdatedReconfStatus()) { case ReconfStatus::SUCCEEDED: { - dbgDebug(D_ORCHESTRATOR) << "The reconfiguration was successfully completed for all the services"; + dbgDebug(D_SERVICE_CONTROLLER) + << "The reconfiguration was successfully completed for all the services"; services_reconf_status.clear(); services_reconf_names.clear(); return Maybe(); } case ReconfStatus::IN_PROGRESS: { - dbgTrace(D_ORCHESTRATOR) << "Reconfiguration in progress..."; + dbgTrace(D_SERVICE_CONTROLLER) << "Reconfiguration in progress..."; Singleton::Consume::by()->yield(chrono::seconds(2)); break; } @@ -1000,7 +1005,7 @@ ServiceController::Impl::sendSignalForServices( for(auto &status : services_reconf_status) { if (status.second == ReconfStatus::FAILED) { failed_services_vec.push_back(services_reconf_names[status.first]); - dbgDebug(D_ORCHESTRATOR) + dbgDebug(D_SERVICE_CONTROLLER) << "The reconfiguration failed for serivce " << services_reconf_names[status.first]; } @@ -1013,7 +1018,7 @@ ServiceController::Impl::sendSignalForServices( return genError("The reconfiguration failed for serivces: " + failed_services); } case ReconfStatus::INACTIVE: { - dbgError(D_ORCHESTRATOR) << "Reached inactive state in the middle of reconfiguration!"; + dbgError(D_SERVICE_CONTROLLER) << "Reached inactive state in the middle of reconfiguration!"; services_reconf_status.clear(); services_reconf_names.clear(); return genError("Reached inactive state in the middle of reconfiguration!"); @@ -1021,7 +1026,7 @@ ServiceController::Impl::sendSignalForServices( } } - dbgDebug(D_ORCHESTRATOR) << "The reconfiguration has reached a timeout"; + dbgDebug(D_SERVICE_CONTROLLER) << "The reconfiguration has reached a timeout"; services_reconf_status.clear(); services_reconf_names.clear(); return genError("The reconfiguration has reached a timeout"); @@ -1033,17 +1038,17 @@ ServiceController::Impl::updateServiceConfigurationFile( const string &configuration_file_path, const string &new_configuration) { - dbgFlow(D_ORCHESTRATOR) << "Updating configuration. Config Name: " << configuration_name; + dbgFlow(D_SERVICE_CONTROLLER) << "Updating configuration. Config Name: " << configuration_name; if (orchestration_tools->doesFileExist(configuration_file_path)) { Maybe old_configuration = orchestration_tools->readFile(configuration_file_path); if (old_configuration.ok()) { bool service_changed = old_configuration.unpack().compare(new_configuration) != 0; if (service_changed == false) { - dbgDebug(D_ORCHESTRATOR) << "There is no update for policy file: " << configuration_file_path; + dbgDebug(D_SERVICE_CONTROLLER) << "There is no update for policy file: " << configuration_file_path; return Maybe(); } - dbgDebug(D_ORCHESTRATOR) + dbgDebug(D_SERVICE_CONTROLLER) << "Starting to update " << configuration_file_path << " to " << new_configuration; string old_configuration_backup_path = configuration_file_path + getConfigurationWithDefault( ".bk", @@ -1051,13 +1056,15 @@ ServiceController::Impl::updateServiceConfigurationFile( "Backup file extension" ); if (orchestration_tools->copyFile(configuration_file_path, old_configuration_backup_path)) { - dbgDebug(D_ORCHESTRATOR) << "Backup of policy file has been created in: " << configuration_file_path; + dbgDebug(D_SERVICE_CONTROLLER) + << "Backup of policy file has been created in: " + << configuration_file_path; } else { - dbgWarning(D_ORCHESTRATOR) << "Failed to backup policy file"; + dbgWarning(D_SERVICE_CONTROLLER) << "Failed to backup policy file"; return genError("Failed to backup policy file"); } } else { - dbgWarning(D_ORCHESTRATOR) + dbgWarning(D_SERVICE_CONTROLLER) << "Failed to read current policy file " << configuration_file_path << ". Error: " @@ -1073,13 +1080,13 @@ ServiceController::Impl::updateServiceConfigurationFile( } if (orchestration_tools->writeFile(new_configuration, configuration_file_path)) { - dbgDebug(D_ORCHESTRATOR) << "New policy file has been saved in: " << configuration_file_path; + dbgDebug(D_SERVICE_CONTROLLER) << "New policy file has been saved in: " << configuration_file_path; } else { - dbgWarning(D_ORCHESTRATOR) << "Failed to save new policy file"; + dbgWarning(D_SERVICE_CONTROLLER) << "Failed to save new policy file"; return genError("Failed to save new policy file"); } - dbgInfo(D_ORCHESTRATOR) << "Successfully updated policy file: " << configuration_file_path; + dbgInfo(D_SERVICE_CONTROLLER) << "Successfully updated policy file: " << configuration_file_path; return Maybe(); } @@ -1120,14 +1127,14 @@ ServiceController::Impl::updateReconfStatus(int id, const string &service_name, } if (services_reconf_status.find(id) == services_reconf_status.end()) { - dbgError(D_ORCHESTRATOR) + dbgError(D_SERVICE_CONTROLLER) << "Unable to find a mapping for reconfiguration ID:" << id << ". Service name: " << service_name; return; } - dbgTrace(D_ORCHESTRATOR) + dbgTrace(D_SERVICE_CONTROLLER) << "Updating reconf status for reconfiguration ID " << id << ", Service name: " @@ -1144,7 +1151,7 @@ ServiceController::Impl::startReconfStatus( const string &service_name, const string &service_id) { - dbgTrace(D_ORCHESTRATOR) + dbgTrace(D_SERVICE_CONTROLLER) << "Starting reconf status. Configuration ID: " << id << ", service name: " diff --git a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc index d027b0c..2effcaa 100755 --- a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc +++ b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc @@ -21,15 +21,13 @@ using namespace testing; using namespace std; +USE_DEBUG_FLAG(D_SERVICE_CONTROLLER); + class ServiceControllerTest : public Test { public: ServiceControllerTest() { - Debug::setUnitTestFlag(D_ORCHESTRATOR, Debug::DebugLevel::NOISE); - Debug::setNewDefaultStdout(&capture_debug); - - CPTestTempfile status_file; registered_services_file_path = status_file.fname; setConfiguration(registered_services_file_path, "orchestration", "Orchestration registered services"); @@ -116,28 +114,6 @@ public: Debug::setNewDefaultStdout(&cout); } - void - registerNewService() - { - stringstream new_service_registration; - new_service_registration - << "{" - << " \"service_name\": \"mock access control\"," - << " \"service_listening_port\":" + to_string(l4_firewall_service_port) + "," - << " \"expected_configurations\": [\"l4_firewall\", \"non updated capability\"]," - << " \"service_id\": \"family1_id2\"," - << " \"general_settings\": \"path_to_settings\"," - << " \"debug_settings\": \"path_to_debug\"" - << "}"; - - auto registration_res = set_nano_service_config->performRestCall(new_service_registration); - ASSERT_TRUE(registration_res.ok()); - - i_service_controller = Singleton::Consume::from(service_controller); - EXPECT_TRUE(i_service_controller->isServiceInstalled("family1_id2")); - EXPECT_FALSE(i_service_controller->isServiceInstalled("I am not installed")); - } - string orchestrationRegisteredServicesFileToString(const string &file_name) { @@ -159,6 +135,43 @@ public: return string_stream.str(); } + void + registerNewService() + { + stringstream new_service_registration; + new_service_registration + << "{" + << " \"service_name\": \"mock access control\"," + << " \"service_listening_port\":" + to_string(l4_firewall_service_port) + "," + << " \"expected_configurations\": [\"l4_firewall\", \"non updated capability\"]," + << " \"service_id\": \"family1_id2\"," + << " \"general_settings\": \"path_to_settings\"," + << " \"debug_settings\": \"path_to_debug\"" + << "}"; + + auto registration_res = set_nano_service_config->performRestCall(new_service_registration); + ASSERT_TRUE(registration_res.ok()); + + i_service_controller = Singleton::Consume::from(service_controller); + EXPECT_TRUE(i_service_controller->isServiceInstalled("family1_id2")); + EXPECT_FALSE(i_service_controller->isServiceInstalled("I am not installed")); + + string expected_json = "{\n" + " \"Registered Services\": {\n" + " \"family1_id2\": {\n" + " \"Service name\": \"mock access control\",\n" + " \"Service ID\": \"family1_id2\",\n" + " \"Service port\": " + to_string(l4_firewall_service_port) + ",\n" + " \"Relevant configs\": [\n" + " \"non updated capability\",\n" + " \"l4_firewall\"\n" + " ]\n" + " }\n" + " }\n" + "}"; + EXPECT_EQ(orchestrationRegisteredServicesFileToString(registered_services_file_path), expected_json); + } + void expectNewConfigRequest(const string &req_body, const string &response) { @@ -174,6 +187,7 @@ public: ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response))); } + CPTestTempfile status_file; const uint16_t l4_firewall_service_port = 8888; const uint16_t waap_service_port = 7777; ::Environment env; @@ -193,7 +207,7 @@ public: string services_port; StrictMock time; StrictMock mock_rest_api; - StrictMock mock_message; + StrictMock mock_message; StrictMock mock_ml; StrictMock mock_shell_cmd; StrictMock mock_orchestration_status; @@ -206,11 +220,10 @@ public: unique_ptr get_services_ports; unique_ptr set_reconf_status; unique_ptr set_new_configuration; - - I_MainLoop::Routine v_tenants_cleanup; + I_MainLoop::Routine v_tenants_cleanup; ostringstream capture_debug; - string version_value = "1.0.2"; - string old_version = "1.0.1"; + string version_value = "1.0.2"; + string old_version = "1.0.1"; }; TEST_F(ServiceControllerTest, doNothing) @@ -494,103 +507,6 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration) EXPECT_EQ(i_service_controller->getUpdatePolicyVersion(), version_value); } -TEST_F(ServiceControllerTest, writeRegisteredServicesFromFile) -{ - EXPECT_EQ(orchestrationRegisteredServicesFileToString(registered_services_file_path), string("")); - - string new_configuration = "{" - " \"version\": \"" + version_value + "\"" - " \"l4_firewall\":" - " {" - " \"app\": \"netfilter\"," - " \"l4_firewall_rules\": [" - " {" - " \"name\": \"allow_statefull_conns\"," - " \"flags\": [\"established\"]," - " \"action\": \"accept\"" - " }," - " {" - " \"name\": \"icmp drop\"," - " \"flags\": [\"log\"]," - " \"services\": [{\"name\":\"icmp\"}]," - " \"action\": \"drop\"" - " }" - " ]" - " }" - "}"; - - string l4_firewall = "{" - " \"app\": \"netfilter\"," - " \"l4_firewall_rules\": [" - " {" - " \"name\": \"allow_statefull_conns\"," - " \"flags\": [\"established\"]," - " \"action\": \"accept\"" - " }," - " {" - " \"name\": \"icmp drop\"," - " \"flags\": [\"log\"]," - " \"services\": [{\"name\":\"icmp\"}]," - " \"action\": \"drop\"" - " }" - " ]" - "}"; - string expected_json = "{\n" - " \"Registered Services\": {\n" - " \"family1_id2\": {\n" - " \"Service name\": \"mock access control\",\n" - " \"Service ID\": \"family1_id2\",\n" - " \"Service port\": 8888,\n" - " \"Relevant configs\": [\n" - " \"non updated capability\",\n" - " \"l4_firewall\"\n" - " ]\n" - " }\n" - " }\n" - "}"; - - Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); - EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration)); - EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) - .WillOnce(Return(json_parser_return)); - EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(false)); - EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path, false)) - .WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_status, - setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY)); - - EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); - - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - - EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) - .WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); - - string general_settings_path = "/my/settings/path"; - string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; - - expectNewConfigRequest("{\n \"id\": 1,\n \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg); - - EXPECT_CALL( - mock_shell_cmd, - getExecOutput( - "/etc/cp/watchdog/cp-nano-watchdog --status --verbose --service mock access control" - " --family family1 --id id2", - _, - _ - ) - ).WillRepeatedly(Return(string("registered and running"))); - - EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path).ok()); - EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value); - EXPECT_EQ(i_service_controller->getUpdatePolicyVersion(), version_value); - EXPECT_EQ(orchestrationRegisteredServicesFileToString(registered_services_file_path), expected_json); -} - TEST_F(ServiceControllerTest, readRegisteredServicesFromFile) { int family1_id3_port = 1111; @@ -1409,6 +1325,8 @@ TEST_F(ServiceControllerTest, failingWhileCopyingCurrentConfiguration) TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest) { + Debug::setUnitTestFlag(D_SERVICE_CONTROLLER, Debug::DebugLevel::NOISE); + Debug::setNewDefaultStdout(&capture_debug); string new_configuration = "{" " \"version\": \"" + version_value + "\"" " \"l4_firewall\":" diff --git a/core/config/config.cc b/core/config/config.cc index 8c0a0f4..1271882 100644 --- a/core/config/config.cc +++ b/core/config/config.cc @@ -17,11 +17,11 @@ #include #include #include -#include +#include +#include + #include "agent_core_utilities.h" - #include "cereal/archives/json.hpp" - #include "debug.h" #include "cereal/external/rapidjson/error/en.h" #include "include/profile_settings.h" @@ -158,6 +158,7 @@ private: vector fillMultiTenantConfigFiles(const map> &tenants); vector fillMultiTenantExpectedConfigFiles(const map> &tenants); map getProfileAgentSetting() const; + void resolveVsId() const; string getActiveTenant() const @@ -243,7 +244,7 @@ private: dbgTrace(D_CONFIG) << "File system path reloaded: " << config_directory_path; } - bool + void sendOrchestatorReloadStatusMsg(const LoadNewConfigurationStatus &status) { I_Messaging *messaging = Singleton::Consume::by(); @@ -261,7 +262,7 @@ private: MessageMetadata secondary_port_req_md("127.0.0.1", 7778); secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); - service_config_status = messaging->sendSyncMessageWithoutResponse( + messaging->sendSyncMessageWithoutResponse( HTTPMethod::POST, "/set-reconf-status", status, @@ -269,11 +270,6 @@ private: secondary_port_req_md ); } - if (!service_config_status) { - dbgWarning(D_CONFIG) << "Unsuccessful attempt to send configuration reload status"; - return false; - } - return true; } unordered_map, PerContextValue>> configuration_nodes; @@ -339,6 +335,9 @@ void ConfigComponent::Impl::init() { reloadFileSystemPaths(); + + resolveVsId(); + tenant_manager = Singleton::Consume::by(); if (!Singleton::exists()) return; @@ -354,8 +353,7 @@ ConfigComponent::Impl::init() } } -static -bool +static bool checkContext(const shared_ptr> &ctx) { if (ctx == nullptr) return true; @@ -937,19 +935,31 @@ ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version mainloop->stop(routine_id); LoadNewConfigurationStatus finished(id, service_name, !res, true); if (!res) finished.setError("Failed to reload configuration"); - I_TimeGet *time = Singleton::Consume::by(); - auto send_status_time_out = time->getMonotonicTime() + chrono::seconds(180); - while (time->getMonotonicTime() < send_status_time_out) { - if (sendOrchestatorReloadStatusMsg(finished)) break; - mainloop->yield(chrono::seconds(1)); - } - if (time->getMonotonicTime() >= send_status_time_out) { - dbgWarning(D_CONFIG) << "Failed to send configuration reload status(finish) to the orchestrator"; - } + sendOrchestatorReloadStatusMsg(finished); is_continuous_report = false; } +void +ConfigComponent::Impl::resolveVsId() const +{ + const string &path = getConfigurationFlag("filesystem_path"); + + size_t vs_pos = path.rfind("/vs"); + if (vs_pos == string::npos) return; + + string vs_id = path.substr(vs_pos + 3); + + if (!vs_id.empty() && all_of(vs_id.begin(), vs_id.end(), ::isdigit) && vs_id.size() < 6) { + dbgDebug(D_CONFIG) << "Identified VSX installation, VS ID: " << vs_id; + Singleton::Consume::by()->registerValue("VS ID", vs_id); + return; + } + + dbgWarning(D_CONFIG) << "Possible VSX installation but VS ID is invalid, VS ID: " << vs_id; + return; +} + ConfigComponent::ConfigComponent() : Component("ConfigComponent"), pimpl(make_unique()) {} ConfigComponent::~ConfigComponent() {} diff --git a/core/include/services_sdk/interfaces/messaging/messaging_enums.h b/core/include/services_sdk/interfaces/messaging/messaging_enums.h index 6caa60b..ef7b0e7 100644 --- a/core/include/services_sdk/interfaces/messaging/messaging_enums.h +++ b/core/include/services_sdk/interfaces/messaging/messaging_enums.h @@ -80,4 +80,13 @@ enum class HTTPStatusCode HTTP_SUSPEND = -2 }; +enum class BioConnectionStatus +{ + SUCCESS, + SHOULD_RETRY, + SHOULD_NOT_RETRY, + + COUNT +}; + #endif // __MESSAGING_ENUMS_H__ diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index cc07c30..027ad2d 100644 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -134,6 +134,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_AGENT_DETAILS, D_ORCHESTRATOR) DEFINE_FLAG(D_LOCAL_POLICY, D_ORCHESTRATOR) DEFINE_FLAG(D_NGINX_POLICY, D_ORCHESTRATOR) + DEFINE_FLAG(D_SERVICE_CONTROLLER, D_ORCHESTRATOR) DEFINE_FLAG(D_GRADUAL_DEPLOYMENT, D_COMPONENT) DEFINE_FLAG(D_SDWAN, D_COMPONENT) diff --git a/core/include/services_sdk/resources/intelligence_invalidation.h b/core/include/services_sdk/resources/intelligence_invalidation.h index 3d5d121..11c5799 100644 --- a/core/include/services_sdk/resources/intelligence_invalidation.h +++ b/core/include/services_sdk/resources/intelligence_invalidation.h @@ -18,6 +18,7 @@ #include #include #include +#include #include "maybe_res.h" #include "enum_array.h" @@ -31,26 +32,46 @@ enum class ClassifierType { CLASS, CATEGORY, FAMILY, GROUP, ORDER, KIND }; enum class ObjectType { ASSET, ZONE, POLICY_PACKAGE, CONFIGURATION, SESSION, SHORTLIVED }; enum class InvalidationType { ADD, DELETE, UPDATE }; +class StrAttributes +{ +public: + StrAttributes() = default; + StrAttributes & addStringAttr(const std::string &attr, const std::string &val); + StrAttributes & addStringSetAttr(const std::string &attr, const std::set &val); + Maybe getStringAttr(const std::string &attr) const; + Maybe, void> getStringSetAttr(const std::string &attr) const; + Maybe genObject() const; + bool isEmpty() const; + bool matches(const StrAttributes &other) const; + void serialize(cereal::JSONInputArchive &ar); + void performOutputingSchema(std::ostream &, int); + +private: + bool hasAttr(const std::string &key, const std::string &value) const; + + std::map string_attr; + std::map> set_string_attr; +}; + class Invalidation { public: Invalidation(const std::string &class_value); Invalidation & setClassifier(ClassifierType type, const std::string &val); - Invalidation & setStringAttr(const std::string &attr, const std::string &val, bool is_main = true); - Invalidation & setStringSetAttr(const std::string &attr, const std::set &val, bool is_main = true); + Invalidation & addMainAttr(const StrAttributes &attr); + Invalidation & addAttr(const StrAttributes &attr); Invalidation & setSourceId(const std::string &id); Invalidation & setObjectType(ObjectType type); Invalidation & setInvalidationType(InvalidationType type); std::string getClassifier(ClassifierType type) const { return classifiers[type]; } - Maybe getStringMainAttr(const std::string &attr) const; - Maybe, void> getStringSetMainAttr(const std::string &attr) const; - Maybe getStringAttr(const std::string &attr) const; - Maybe, void> getStringSetAttr(const std::string &attr) const; + std::vector getMainAttributes() const { return main_attributes; } + std::vector getAttributes() const { return attributes; } const Maybe & getSourceId() const { return source_id; } const Maybe & getObjectType() const { return object_type; } InvalidationType getInvalidationType() const { return invalidation_type; } + Maybe getRegistrationID() const; bool report(I_Intelligence_IS_V2 *interface) const; @@ -64,18 +85,16 @@ public: bool matches(const Invalidation &other) const; private: - bool hasMainAttr(const std::string &key, const std::string &value) const; - bool hasAttr(const std::string &key, const std::string &value) const; + bool attr_matches(const std::vector ¤t, const std::vector &other) const; EnumArray classifiers; - std::map string_main_attr; - std::map> set_string_main_attr; - std::map string_attr; - std::map> set_string_attr; + std::vector main_attributes; + std::vector attributes; Maybe source_id; Maybe object_type; InvalidationType invalidation_type = InvalidationType::ADD; Maybe listening_id; + Maybe registration_id; }; } // namespace Intelligence diff --git a/core/intelligence_is_v2/include/intelligence_request.h b/core/intelligence_is_v2/include/intelligence_request.h index cba0af3..820cf69 100644 --- a/core/intelligence_is_v2/include/intelligence_request.h +++ b/core/intelligence_is_v2/include/intelligence_request.h @@ -34,9 +34,7 @@ public: bool isPagingActivated() const; Maybe isPagingFinished() const; Maybe getPagingStatus() const; - bool loadJson(const std::string &json); Maybe genJson() const; - Maybe getResponseFromFog() const; size_t getSize() const { return queries.size(); } bool isBulk() const { return is_bulk; } diff --git a/core/intelligence_is_v2/include/intelligence_server.h b/core/intelligence_is_v2/include/intelligence_server.h index 41856b9..ec90933 100644 --- a/core/intelligence_is_v2/include/intelligence_server.h +++ b/core/intelligence_is_v2/include/intelligence_server.h @@ -33,7 +33,7 @@ private: Maybe sendQueryObjectToLocalServer(bool is_primary_port); Maybe sendQueryMessage(); Maybe sendMessage(); - Maybe createResponse(); + Maybe createResponse(const std::string &response_body); IntelligenceRequest request; Flags conn_flags; diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc index a75bcd2..17bba5f 100644 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -37,14 +37,12 @@ static const string registration_uri = "/api/v2/intelligence/invalidation/regist class I_InvalidationCallBack { public: - virtual void performCallBacks(const Invalidation &invalidation) const = 0; + virtual void performCallBacks(const Invalidation &invalidation, const string ®istration_id) const = 0; protected: virtual ~I_InvalidationCallBack() {} }; -using MainAttrTypes = SerializableMultiMap>; - static const map object_names = { { "asset", Intelligence::ObjectType::ASSET }, { "zone", Intelligence::ObjectType::ZONE }, @@ -54,6 +52,12 @@ static const map object_names = { { "shortLived", Intelligence::ObjectType::SHORTLIVED } }; +static const map invalidation_type_names = { + { "add", Intelligence::InvalidationType::ADD }, + { "delete", Intelligence::InvalidationType::DELETE }, + { "update", Intelligence::InvalidationType::UPDATE } +}; + class InvalidationRegistration { public: @@ -108,11 +112,22 @@ public: do { ++running_id; } while (callbacks.find(running_id) != callbacks.end()); + auto invalidation_reg_id = invalidation.getRegistrationID(); + if (invalidation_reg_id.ok()) registration_id_to_cb[*invalidation_reg_id] = cb; callbacks.emplace(running_id, make_pair(invalidation, cb)); return running_id; } - void erase(uint id) { callbacks.erase(id); } + void + erase(uint id) + { + auto actual_invalidation = callbacks.find(id); + if (actual_invalidation == callbacks.end()) return; + auto invalidation_reg_id = actual_invalidation->second.first.getRegistrationID(); + if (invalidation_reg_id.ok()) registration_id_to_cb.erase(*invalidation_reg_id); + callbacks.erase(id); + } + bool empty() const { return callbacks.empty(); } InvalidationRegistration::RestCall @@ -128,9 +143,13 @@ public: } void - performCallBacks(const Invalidation &invalidation) const override + performCallBacks(const Invalidation &invalidation, const string ®istration_id) const override { dbgDebug(D_INTELLIGENCE) << "Looking for callbacks for invalidation " << invalidation.genObject(); + if (registration_id != "") { + auto invalidation_cb = registration_id_to_cb.find(registration_id); + if (invalidation_cb != registration_id_to_cb.end()) return invalidation_cb->second(invalidation); + } for (auto ®isted_invalidation : callbacks) { dbgTrace(D_INTELLIGENCE) << "Checking against: " << registed_invalidation.second.first.genObject(); performCallBacksImpl(invalidation, registed_invalidation.second); @@ -140,20 +159,22 @@ public: private: void performCallBacksImpl( - const Invalidation &actual_invalidation, - const pair> &invalidation_and_cb + const Invalidation &actual_invalidation, + const pair> &invalidation_and_cb ) const { auto ®istereed_invalidation = invalidation_and_cb.first; auto &cb = invalidation_and_cb.second; - if (registereed_invalidation.matches(actual_invalidation)) cb(actual_invalidation); + if (!registereed_invalidation.matches(actual_invalidation)) return; + cb(actual_invalidation); } map>> callbacks; + map> registration_id_to_cb; uint running_id = 0; }; -class RecieveInvalidation : public ServerRest +class ReceiveInvalidation : public ServerRest { public: void @@ -168,14 +189,14 @@ public: if (kind.isActive()) invalidation.setClassifier(ClassifierType::KIND, kind.get()); if (mainAttributes.isActive()) { - auto strings = getMainAttr(); - for (const auto &value : strings) { - invalidation.setStringAttr(value.first, value.second); + for (auto &vec_entry : mainAttributes.get()) { + invalidation.addMainAttr(vec_entry); } + } - auto string_sets = getMainAttr>(); - for (const auto &value : string_sets) { - invalidation.setStringSetAttr(value.first, value.second); + if (attributes.isActive()) { + for (auto &vec_entry : attributes.get()) { + invalidation.addAttr(vec_entry); } } @@ -186,25 +207,14 @@ public: if (sourceId.isActive()) invalidation.setSourceId(sourceId.get()); + string registration_id = ""; + if (invalidationRegistrationId.isActive()) registration_id = invalidationRegistrationId.get(); + auto i_cb = Singleton::Consume::from(); - i_cb->performCallBacks(invalidation); + i_cb->performCallBacks(invalidation, registration_id); } private: - template - map - getMainAttr() - { - map res; - - for (auto &vec_entry : mainAttributes.get()) { - for (auto &attr : vec_entry.getMap()) { - res[attr.first] = attr.second; - } - } - - return res; - } C2S_LABEL_PARAM(string, class_name, "class"); C2S_OPTIONAL_PARAM(string, category); @@ -214,7 +224,9 @@ private: C2S_OPTIONAL_PARAM(string, kind); C2S_OPTIONAL_PARAM(string, objectType); C2S_OPTIONAL_PARAM(string, sourceId); - C2S_OPTIONAL_PARAM(vector, mainAttributes); + C2S_OPTIONAL_PARAM(string, invalidationRegistrationId); + C2S_OPTIONAL_PARAM(vector, mainAttributes); + C2S_OPTIONAL_PARAM(vector, attributes); }; class IntelligenceComponentV2::Impl @@ -238,7 +250,7 @@ public: ); auto rest_api = Singleton::Consume::by(); - rest_api->addRestCall(RestAction::SET, "new-invalidation/source/invalidation"); + rest_api->addRestCall(RestAction::SET, "new-invalidation/source/invalidation"); } bool @@ -256,8 +268,9 @@ public: registerInvalidation(const Invalidation &invalidation, const function &cb) override { if (!invalidation.isLegalInvalidation()) return genError("Attempting to register invalid invalidation"); - if (!sendRegistration(invalidation)) return genError("Failed to register for invalidation"); - return invalidations.emplace(invalidation, cb); + auto res = invalidations.emplace(invalidation, cb); + sendReccurringInvalidationRegistration(); + return res; } void diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc index 3881c18..a44c849 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc @@ -15,6 +15,98 @@ using namespace testing; static const string invalidation_uri = "/api/v2/intelligence/invalidation"; + +TEST(StringAttributesBasic, SettersAndGetters) +{ + StrAttributes string_attributes; + + EXPECT_TRUE(string_attributes.isEmpty()); + EXPECT_FALSE(string_attributes.getStringAttr("attr1").ok()); + EXPECT_FALSE(string_attributes.getStringSetAttr("attr2").ok()); + + set vals = { "2", "3" }; + string_attributes + .addStringAttr("attr1", "1") + .addStringSetAttr("attr2", vals); + + EXPECT_FALSE(string_attributes.isEmpty()); + EXPECT_EQ(string_attributes.getStringAttr("attr1").unpack(), "1"); + EXPECT_EQ(string_attributes.getStringSetAttr("attr2").unpack(), vals); +} + +TEST(StringAttributesBasic, attr_schema) +{ + set vals = { "2", "3" }; + auto string_attributes = StrAttributes() + .addStringAttr("attr1", "1") + .addStringSetAttr("attr2", vals); + stringstream ss; + string_attributes.performOutputingSchema(ss, 0); + string expected_schema = + "{\n" + " \"attr1\": \"1\",\n" + " \"attr2\": [\n" + " \"2\",\n" + " \"3\"\n" + " ]\n" + "}"; + EXPECT_EQ(ss.str(), expected_schema); +} + +TEST(StringAttributesBasic, Matching) +{ + set vals = { "2", "3" }; + auto base_string_attributes = StrAttributes() + .addStringAttr("attr1", "1") + .addStringAttr("attr2", "2") + .addStringAttr("attr3", "3") + .addStringSetAttr("attr4", vals); + + auto matching_string_attributes = StrAttributes() + .addStringAttr("attr1", "1") + .addStringAttr("attr2", "2") + .addStringAttr("attr3", "3") + .addStringSetAttr("attr4", vals) + .addStringAttr("attr5", "6") + .addStringSetAttr("attr6", vals); + + EXPECT_TRUE(base_string_attributes.matches(matching_string_attributes)); + + auto not_matching_string_attributes = StrAttributes() + .addStringAttr("attr1", "1") + .addStringAttr("attr2", "2") + .addStringSetAttr("attr4", vals) + .addStringAttr("attr3", "6"); + + EXPECT_FALSE(base_string_attributes.matches(not_matching_string_attributes)); + + auto missing_attr_string_attributes = StrAttributes() + .addStringAttr("attr1", "1") + .addStringSetAttr("attr2", vals); + + EXPECT_FALSE(base_string_attributes.matches(missing_attr_string_attributes)); + + set vals2 = { "1", "5", "2", "3" }; + auto has_extra_value_string_attributes = StrAttributes() + .addStringAttr("attr1", "1") + .addStringAttr("attr2", "2") + .addStringAttr("attr3", "3") + .addStringSetAttr("attr4", vals2); + + EXPECT_TRUE(base_string_attributes.matches(has_extra_value_string_attributes)); +} + +TEST(StringAttributesBasic, genObject) +{ + set vals = { "2", "3" }; + auto string_attributes = StrAttributes() + .addStringAttr("attr1", "1") + .addStringSetAttr("attr2", vals); + + string expected_json = "{ \"attr1\": \"1\", \"attr2\": [ \"2\", \"3\" ] }"; + EXPECT_EQ(string_attributes.genObject().unpack(), expected_json); +} + TEST(InvalidationBasic, SettersAndGetters) { Invalidation invalidation("aaa"); @@ -27,33 +119,37 @@ TEST(InvalidationBasic, SettersAndGetters) EXPECT_EQ(invalidation.getClassifier(ClassifierType::KIND), ""); EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::ADD); - EXPECT_FALSE(invalidation.getStringMainAttr("main_attr1").ok()); - EXPECT_FALSE(invalidation.getStringSetMainAttr("main_attr2").ok()); - EXPECT_FALSE(invalidation.getStringAttr("attr1").ok()); - EXPECT_FALSE(invalidation.getStringSetAttr("attr2").ok()); + EXPECT_TRUE(invalidation.getMainAttributes().empty()); + EXPECT_TRUE(invalidation.getAttributes().empty()); EXPECT_FALSE(invalidation.getSourceId().ok()); EXPECT_FALSE(invalidation.getObjectType().ok()); set main_vals = { "2", "3" }; set vals = { "5", "6" }; + auto main_attr = StrAttributes() + .addStringAttr("main_attr1", "1") + .addStringSetAttr("main_attr2", main_vals); + + auto attr = StrAttributes() + .addStringAttr("attr1", "4") + .addStringSetAttr("attr2", vals); + invalidation .setClassifier(ClassifierType::CATEGORY, "bbb") .setClassifier(ClassifierType::FAMILY, "ccc") - .setStringAttr("main_attr1", "1") - .setStringSetAttr("main_attr2", main_vals) - .setStringAttr("attr1", "4", false) - .setStringSetAttr("attr2", vals, false) + .addMainAttr(main_attr) + .addAttr(attr) .setSourceId("id") .setObjectType(Intelligence::ObjectType::ASSET) .setInvalidationType(InvalidationType::DELETE); EXPECT_EQ(invalidation.getClassifier(ClassifierType::CATEGORY), "bbb"); EXPECT_EQ(invalidation.getClassifier(ClassifierType::FAMILY), "ccc"); - EXPECT_EQ(invalidation.getStringMainAttr("main_attr1").unpack(), "1"); - EXPECT_EQ(invalidation.getStringSetMainAttr("main_attr2").unpack(), main_vals); - EXPECT_EQ(invalidation.getStringAttr("attr1").unpack(), "4"); - EXPECT_EQ(invalidation.getStringSetAttr("attr2").unpack(), vals); + EXPECT_EQ(invalidation.getMainAttributes().begin()->getStringAttr("main_attr1").unpack(), "1"); + EXPECT_EQ(invalidation.getMainAttributes().begin()->getStringSetAttr("main_attr2").unpack(), main_vals); + EXPECT_EQ(invalidation.getAttributes().begin()->getStringAttr("attr1").unpack(), "4"); + EXPECT_EQ(invalidation.getAttributes().begin()->getStringSetAttr("attr2").unpack(), vals); EXPECT_EQ(invalidation.getSourceId().unpack(), "id"); EXPECT_EQ(invalidation.getObjectType().unpack(), Intelligence::ObjectType::ASSET); EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::DELETE); @@ -63,84 +159,87 @@ TEST(InvalidationBasic, Matching) { set main_vals = { "2", "3" }; set vals = { "5", "6" }; + + auto main_attr = StrAttributes() + .addStringAttr("main_attr1", "1") + .addStringSetAttr("main_attr2", main_vals); + + auto attr = StrAttributes() + .addStringAttr("attr1", "4") + .addStringSetAttr("attr2", vals); + auto base_invalidation = Invalidation("aaa") .setClassifier(ClassifierType::CATEGORY, "bbb") .setClassifier(ClassifierType::FAMILY, "ccc") - .setStringAttr("main_attr1", "1") - .setStringSetAttr("main_attr2", main_vals) - .setStringAttr("attr1", "4", false) - .setStringSetAttr("attr2", vals, false); + .addMainAttr(main_attr) + .addAttr(attr); + auto matching_main_attr = StrAttributes() + .addStringAttr("main_attr1", "1") + .addStringSetAttr("main_attr2", main_vals) + .addStringAttr("main_attr3", "6"); + + auto matching_attr = StrAttributes() + .addStringAttr("attr1", "4") + .addStringSetAttr("attr2", vals) + .addStringAttr("attr3", "7"); auto matching_invalidation = Invalidation("aaa") .setClassifier(ClassifierType::CATEGORY, "bbb") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::GROUP, "ddd") - .setStringAttr("main_attr1", "1") - .setStringSetAttr("main_attr2", main_vals) - .setStringAttr("attr1", "4", false) - .setStringSetAttr("attr2", vals, false) - .setStringAttr("main_attr3", "6") - .setStringAttr("attr3", "7", false) + .addMainAttr(matching_main_attr) + .addAttr(matching_attr) .setSourceId("id") .setObjectType(Intelligence::ObjectType::ASSET) .setInvalidationType(InvalidationType::ADD); EXPECT_TRUE(base_invalidation.matches(matching_invalidation)); - auto not_matching_invalidation_type = Invalidation("aaa") - .setClassifier(ClassifierType::CATEGORY, "bbb") - .setClassifier(ClassifierType::FAMILY, "ccc") - .setClassifier(ClassifierType::GROUP, "ddd") - .setStringAttr("main_attr1", "1") - .setStringSetAttr("main_attr2", main_vals) - .setSourceId("id") - .setObjectType(Intelligence::ObjectType::ASSET) - .setInvalidationType(InvalidationType::DELETE); - - EXPECT_FALSE(base_invalidation.matches(not_matching_invalidation_type)); + auto missing_attr_main = StrAttributes() + .addStringAttr("main_attr1", "1") + .addStringAttr("main_attr2", "2") + .addStringAttr("main_attr3", "6"); auto missing_attr_invalidation_main = Invalidation("aaa") .setClassifier(ClassifierType::CATEGORY, "bbb") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::GROUP, "ddd") - .setStringAttr("main_attr1", "1") - .setStringAttr("main_attr2", "2") - .setStringAttr("main_attr3", "6") - .setStringAttr("attr1", "4", false) - .setStringSetAttr("attr2", vals, false) - .setStringAttr("attr3", "7", false) + .addMainAttr(missing_attr_main) + .addAttr(matching_attr) .setSourceId("id") .setObjectType(Intelligence::ObjectType::ASSET); EXPECT_FALSE(base_invalidation.matches(missing_attr_invalidation_main)); + auto missing_attr = StrAttributes() + .addStringAttr("attr1", "4") + .addStringAttr("attr2", "2") + .addStringAttr("attr3", "7"); + auto missing_attr_invalidation = Invalidation("aaa") .setClassifier(ClassifierType::CATEGORY, "bbb") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::GROUP, "ddd") - .setStringAttr("main_attr1", "1") - .setStringSetAttr("main_attr2", main_vals) - .setStringAttr("main_attr3", "6") - .setStringAttr("attr1", "4", false) - .setStringAttr("attr2", "2", false) - .setStringAttr("attr3", "7", false) + .addMainAttr(matching_main_attr) + .addAttr(missing_attr) .setSourceId("id") .setObjectType(Intelligence::ObjectType::ASSET); EXPECT_FALSE(base_invalidation.matches(missing_attr_invalidation)); set vals2 = { "1", "5" }; + auto extra_value_main_attr = StrAttributes() + .addStringSetAttr("main_attr1", vals2) + .addStringSetAttr("main_attr2", main_vals) + .addStringAttr("main_attr3", "6"); + auto has_extra_value_invalidation = Invalidation("aaa") .setClassifier(ClassifierType::CATEGORY, "bbb") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::GROUP, "ddd") - .setStringSetAttr("main_attr1", vals2) - .setStringSetAttr("main_attr2", main_vals) - .setStringAttr("main_attr3", "6") - .setStringAttr("attr1", "4", false) - .setStringSetAttr("attr2", vals, false) - .setStringAttr("attr3", "7", false) + .addMainAttr(extra_value_main_attr) + .addAttr(matching_attr) .setSourceId("id") .setObjectType(Intelligence::ObjectType::ASSET); @@ -180,6 +279,8 @@ public: conf.preload(); intelligence.preload(); intelligence.init(); + main_attr.addStringAttr("attr2", "2"); + attr.addStringAttr("attr3", "3"); } bool @@ -189,6 +290,8 @@ public: return true; } + StrAttributes main_attr; + StrAttributes attr; StrictMock messaging_mock; StrictMock mock_ml; NiceMock mock_time; @@ -208,7 +311,7 @@ public: TEST_F(IntelligenceInvalidation, sending_incomplete_invalidation) { auto invalidation = Invalidation("aaa") - .setStringAttr("attr2", "2") + .addMainAttr(main_attr) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setObjectType(Intelligence::ObjectType::ASSET); @@ -219,8 +322,8 @@ TEST_F(IntelligenceInvalidation, sending_incomplete_invalidation) TEST_F(IntelligenceInvalidation, sending_public_invalidation) { auto invalidation = Invalidation("aaa") - .setStringAttr("attr2", "2") - .setStringAttr("attr3", "3", false) + .addMainAttr(main_attr) + .addAttr(attr) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") @@ -254,11 +357,51 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation) EXPECT_FALSE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN)); } +TEST_F(IntelligenceInvalidation, multiple_assets_invalidation) +{ + auto main_attr_2 = StrAttributes() + .addStringAttr("attr2", "22") + .addStringSetAttr("attr3", {"33", "44"}); + + auto invalidation = Invalidation("aaa") + .addMainAttr(main_attr) + .addMainAttr(main_attr_2) + .addAttr(attr) + .setSourceId("id") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setObjectType(Intelligence::ObjectType::ASSET); + + string invalidation_json; + EXPECT_CALL( + messaging_mock, + sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(DoAll( + SaveArg<2>(&invalidation_json), + Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")) + )); + + EXPECT_TRUE(invalidation.report(i_intelligence)); + + string expected_json = + "{ \"invalidations\": [ { " + "\"class\": \"aaa\", " + "\"category\": \"bbb\", " + "\"family\": \"ccc\", " + "\"objectType\": \"asset\", " + "\"invalidationType\": \"add\", " + "\"sourceId\": \"id\", " + "\"mainAttributes\": [ { \"attr2\": \"2\" }, { \"attr2\": \"22\", \"attr3\": [ \"33\", \"44\" ] } ], " + "\"attributes\": [ { \"attr3\": \"3\" } ]" + " } ] }"; + EXPECT_EQ(invalidation_json, expected_json); +} + TEST_F(IntelligenceInvalidation, sending_private_invalidation) { auto invalidation = Invalidation("aaa") - .setStringAttr("attr2", "2") - .setStringAttr("attr3", "3", false) + .addMainAttr(main_attr) + .addAttr(attr) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") @@ -319,10 +462,10 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation) configuration << "}"; Singleton::Consume::from(conf)->loadConfiguration(configuration); - set vals = { "11", "55", "22" }; + auto invalidation = Invalidation("aaa") - .setStringAttr("attr2", "2") - .setStringSetAttr("attr3", vals, false) + .addMainAttr(main_attr) + .addAttr(attr) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") @@ -344,10 +487,89 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation) EXPECT_THAT(body, HasSubstr("\"url\": \"http://127.0.0.1:7000/set-new-invalidation\"")); EXPECT_THAT(body, HasSubstr("\"apiVersion\": \"v2\", \"communicationType\": \"sync\"")); EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]")); - EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"attr3\": [ \"11\", \"22\", \"55\" ] } ]")); + EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"attr3\": \"3\" } ]")); EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN)); } +TEST_F(IntelligenceInvalidation, register_for_multiple_assets_invalidation) +{ + stringstream configuration; + configuration << "{"; + configuration << " \"agentSettings\":["; + configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}"; + configuration << " ],"; + configuration << " \"intelligence\":{"; + configuration << " \"local intelligence server ip\":\"127.0.0.1\","; + configuration << " \"local intelligence server primary port\":9090"; + configuration << " }"; + configuration << "}"; + Singleton::Consume::from(conf)->loadConfiguration(configuration); + + auto multiple_assets_main_attr1 = StrAttributes() + .addStringAttr("attr2", "22"); + auto multiple_assets_main_attr2 = StrAttributes() + .addStringAttr("attr2", "222"); + auto multiple_assets_main_attr3 = StrAttributes() + .addStringAttr("attr2", "2222") + .addStringSetAttr("attr3", {"3333", "4444"}); + auto invalidation = Invalidation("aaa") + .addMainAttr(multiple_assets_main_attr1) + .addMainAttr(multiple_assets_main_attr2) + .addMainAttr(multiple_assets_main_attr3) + .addAttr(attr) + .setSourceId("id") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setObjectType(Intelligence::ObjectType::ASSET); + + string body; + EXPECT_CALL( + messaging_mock, + sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _) + ).WillOnce(DoAll( + SaveArg<2>(&body), + Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")) + )); + + EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0); + + EXPECT_THAT( + body, + HasSubstr( + "\"mainAttributes\": [ " + "{ \"attr2\": \"22\" }, " + "{ \"attr2\": \"222\" }, " + "{ \"attr2\": \"2222\", \"attr3\": [ \"3333\", \"4444\" ] } " + "]" + ) + ); +} + +TEST_F(IntelligenceInvalidation, register_incomplit_invalidation) +{ + stringstream configuration; + configuration << "{"; + configuration << " \"agentSettings\":["; + configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}"; + configuration << " ],"; + configuration << " \"intelligence\":{"; + configuration << " \"local intelligence server ip\":\"127.0.0.1\","; + configuration << " \"local intelligence server primary port\":9090"; + configuration << " }"; + configuration << "}"; + Singleton::Consume::from(conf)->loadConfiguration(configuration); + + + auto invalidation = Invalidation("aaa") + .addMainAttr(main_attr) + .addAttr(attr) + .setSourceId("id") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setObjectType(Intelligence::ObjectType::ASSET); + + EXPECT_FALSE(i_intelligence->registerInvalidation(invalidation, callback).ok()); +} + TEST_F(IntelligenceInvalidation, invalidation_callback) { stringstream configuration; @@ -363,7 +585,7 @@ TEST_F(IntelligenceInvalidation, invalidation_callback) Singleton::Consume::from(conf)->loadConfiguration(configuration); auto invalidation = Invalidation("aaa") - .setStringAttr("attr2", "2") + .addMainAttr(main_attr) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") @@ -377,8 +599,10 @@ TEST_F(IntelligenceInvalidation, invalidation_callback) EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0); set vals = { "1", "5", "2" }; + auto test_main_attr = StrAttributes() + .addStringSetAttr("attr2", vals); auto invalidation2 = Invalidation("aaa") - .setStringSetAttr("attr2", vals) + .addMainAttr(test_main_attr) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") @@ -389,7 +613,7 @@ TEST_F(IntelligenceInvalidation, invalidation_callback) mock_invalidation->performRestCall(json); EXPECT_EQ(recieved_invalidations.size(), 1); - EXPECT_EQ(recieved_invalidations[0].getStringSetMainAttr("attr2").unpack(), vals); + EXPECT_EQ(recieved_invalidations[0].getMainAttributes().begin()->getStringSetAttr("attr2").unpack(), vals); } TEST_F(IntelligenceInvalidation, delete_invalidation_callback) @@ -407,7 +631,7 @@ TEST_F(IntelligenceInvalidation, delete_invalidation_callback) Singleton::Consume::from(conf)->loadConfiguration(configuration); auto invalidation = Invalidation("aaa") - .setStringAttr("attr2", "2") + .addMainAttr(main_attr) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") @@ -421,9 +645,8 @@ TEST_F(IntelligenceInvalidation, delete_invalidation_callback) auto callback_id = i_intelligence->registerInvalidation(invalidation, callback); i_intelligence->unregisterInvalidation(*callback_id); - set vals = { "1", "5", "2" }; auto invalidation2 = Invalidation("aaa") - .setStringSetAttr("attr2", vals) + .addMainAttr(main_attr) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") @@ -451,7 +674,7 @@ TEST_F(IntelligenceInvalidation, invalidation_short_handling) Singleton::Consume::from(conf)->loadConfiguration(configuration); auto invalidation = Invalidation("aaa") - .setStringAttr("attr2", "2") + .addMainAttr(main_attr) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") @@ -466,9 +689,8 @@ TEST_F(IntelligenceInvalidation, invalidation_short_handling) invalidation.stopListening(i_intelligence); - set vals = { "1", "5", "2" }; auto invalidation2 = Invalidation("aaa") - .setStringSetAttr("attr2", vals) + .addMainAttr(main_attr) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") @@ -498,7 +720,7 @@ TEST_F(IntelligenceInvalidation, routine_registration) routine(); auto invalidation = Invalidation("aaa") - .setStringAttr("attr2", "2") + .addMainAttr(main_attr) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") @@ -526,3 +748,191 @@ TEST_F(IntelligenceInvalidation, routine_registration) EXPECT_THAT(body, HasSubstr("\"apiVersion\": \"v2\", \"communicationType\": \"sync\"")); EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]")); } + +TEST_F(IntelligenceInvalidation, invalidation_flow_with_multiple_assets) +{ + stringstream configuration; + configuration << "{"; + configuration << " \"agentSettings\":["; + configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}"; + configuration << " ],"; + configuration << " \"intelligence\":{"; + configuration << " \"local intelligence server ip\":\"127.0.0.1\","; + configuration << " \"local intelligence server primary port\":9090"; + configuration << " }"; + configuration << "}"; + Singleton::Consume::from(conf)->loadConfiguration(configuration); + + auto base_main_attr2 = StrAttributes() + .addStringAttr("attr3", "3"); + auto invalidation_to_register = Invalidation("aaa") + .addMainAttr(main_attr) + .addMainAttr(base_main_attr2) + .setSourceId("id") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setObjectType(Intelligence::ObjectType::ASSET); + + EXPECT_CALL( + messaging_mock, + sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); + + invalidation_to_register.startListening(i_intelligence, callback); + auto stop_listening = make_scope_exit([&] { invalidation_to_register.stopListening(i_intelligence); }); + + auto not_matching_main_attributes = StrAttributes() + .addStringAttr("attr3", "4"); + + auto not_matching_invalidation = Invalidation("aaa") + .addMainAttr(not_matching_main_attributes) + .setSourceId("id") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setObjectType(Intelligence::ObjectType::ASSET); + + stringstream json1; + json1 << not_matching_invalidation.genObject(); + mock_invalidation->performRestCall(json1); + + EXPECT_EQ(recieved_invalidations.size(), 0); + + auto matching_second_main_attribute = StrAttributes() + .addStringAttr("attr3", "3"); + + auto matching_invalidation = Invalidation("aaa") + .addMainAttr(matching_second_main_attribute) + .setSourceId("id") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setObjectType(Intelligence::ObjectType::ASSET); + + stringstream json2; + json2 << matching_invalidation.genObject(); + mock_invalidation->performRestCall(json2); + + EXPECT_EQ(recieved_invalidations.size(), 1); +} + +TEST_F(IntelligenceInvalidation, invalidation_cb_match_2_registred_assets) +{ + stringstream configuration; + configuration << "{"; + configuration << " \"agentSettings\":["; + configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}"; + configuration << " ],"; + configuration << " \"intelligence\":{"; + configuration << " \"local intelligence server ip\":\"127.0.0.1\","; + configuration << " \"local intelligence server primary port\":9090"; + configuration << " }"; + configuration << "}"; + Singleton::Consume::from(conf)->loadConfiguration(configuration); + + auto base_main_attr2 = StrAttributes() + .addStringAttr("attr3", "3"); + auto invalidation_to_register = Invalidation("aaa") + .addMainAttr(main_attr) + .addMainAttr(base_main_attr2) + .setSourceId("id") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setObjectType(Intelligence::ObjectType::ASSET); + + EXPECT_CALL( + messaging_mock, + sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _) + ).Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); + + invalidation_to_register.startListening(i_intelligence, callback); + auto stop_listening = make_scope_exit([&] { invalidation_to_register.stopListening(i_intelligence); }); + + auto matching_second_main_attribute = StrAttributes() + .addStringAttr("attr3", "3"); + + auto matching_invalidation = Invalidation("aaa") + .addMainAttr(matching_second_main_attribute) + .setSourceId("id") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setObjectType(Intelligence::ObjectType::ASSET); + + + auto invalidation_2_to_register = Invalidation("aaa") + .addMainAttr(base_main_attr2) + .setSourceId("id") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setObjectType(Intelligence::ObjectType::ASSET); + + invalidation_2_to_register.startListening(i_intelligence, callback); + auto stop_listening_2 = make_scope_exit([&] { invalidation_2_to_register.stopListening(i_intelligence); }); + + stringstream json; + json << matching_invalidation.genObject(); + mock_invalidation->performRestCall(json); + + EXPECT_EQ(recieved_invalidations.size(), 2); +} + +TEST_F(IntelligenceInvalidation, invalidation_cb_match_by_registration_id) +{ + stringstream configuration; + configuration << "{"; + configuration << " \"agentSettings\":["; + configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}"; + configuration << " ],"; + configuration << " \"intelligence\":{"; + configuration << " \"local intelligence server ip\":\"127.0.0.1\","; + configuration << " \"local intelligence server primary port\":9090"; + configuration << " }"; + configuration << "}"; + Singleton::Consume::from(conf)->loadConfiguration(configuration); + + auto base_main_attr2 = StrAttributes() + .addStringAttr("attr3", "3"); + auto invalidation_to_register = Invalidation("aaa") + .addMainAttr(main_attr) + .addMainAttr(base_main_attr2) + .setSourceId("id") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setObjectType(Intelligence::ObjectType::ASSET); + + EXPECT_CALL( + messaging_mock, + sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _) + ).Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); + + invalidation_to_register.startListening(i_intelligence, callback); + auto stop_listening = make_scope_exit([&] { invalidation_to_register.stopListening(i_intelligence); }); + + auto matching_second_main_attribute = StrAttributes() + .addStringAttr("attr3", "3"); + + auto matching_invalidation = Invalidation("aaa") + .addMainAttr(matching_second_main_attribute) + .setSourceId("id") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setObjectType(Intelligence::ObjectType::ASSET); + + + auto invalidation_2_to_register = Invalidation("aaa") + .addMainAttr(base_main_attr2) + .setSourceId("id") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setObjectType(Intelligence::ObjectType::ASSET); + + invalidation_2_to_register.startListening(i_intelligence, callback); + auto registration_id = invalidation_2_to_register.getRegistrationID(); + auto stop_listening_2 = make_scope_exit([&] { invalidation_2_to_register.stopListening(i_intelligence); }); + + string modifiedJsonString = matching_invalidation.genObject().substr(2); + stringstream json; + json << "{ \"invalidationRegistrationId\": \""<< *registration_id << "\", " << modifiedJsonString; + cout << json.str() << endl; + mock_invalidation->performRestCall(json); + + EXPECT_EQ(recieved_invalidations.size(), 1); +} diff --git a/core/intelligence_is_v2/intelligence_request.cc b/core/intelligence_is_v2/intelligence_request.cc index 3f3ee55..c6ca113 100644 --- a/core/intelligence_is_v2/intelligence_request.cc +++ b/core/intelligence_is_v2/intelligence_request.cc @@ -113,16 +113,3 @@ IntelligenceRequest::genJson() const return str_stream.str(); } - -bool -IntelligenceRequest::loadJson(const string &json) -{ - response_from_fog = json; - return true; -} - -Maybe -IntelligenceRequest::getResponseFromFog() const -{ - return response_from_fog; -} diff --git a/core/intelligence_is_v2/intelligence_server.cc b/core/intelligence_is_v2/intelligence_server.cc index ee46b3d..e7f7621 100644 --- a/core/intelligence_is_v2/intelligence_server.cc +++ b/core/intelligence_is_v2/intelligence_server.cc @@ -140,64 +140,49 @@ Sender::sendQueryMessage() Maybe Sender::sendMessage() { - if (server_ip.ok() || server_port.ok()) { - if (!server_ip.ok()) return genError("Can't send intelligence request. Server ip wasn't set"); - if (!server_port.ok()) return genError("Can't send intelligence request. Server port wasn't set"); - } else if (!server_ip.ok() && !server_port.ok()) { - auto req_status = i_message->sendSyncMessage( - HTTPMethod::POST, - request.isBulk() ? queries_uri : query_uri, - request, - MessageCategory::INTELLIGENCE - ); - if (req_status.ok()) { - return createResponse(); - }; - auto response_error = req_status.getErr().toString(); - dbgWarning(D_INTELLIGENCE) << "Failed to send intelligence request. Error:" << response_error; - return genError( - "Failed to send intelligence request. " - + req_status.getErr().getBody() - + " " - + req_status.getErr().toString() - ); + if (server_port.ok() && !server_ip.ok()) return genError("Can't send intelligence request. Server ip invalid"); + if (server_ip.ok() && !server_port.ok()) return genError("Can't send intelligence request. Server port invalid"); + auto req_md = server_ip.ok() ? MessageMetadata(*server_ip, *server_port, conn_flags) : MessageMetadata(); + + if (server_ip.ok()) { + dbgTrace(D_INTELLIGENCE) + << "Sending intelligence request with IP: " + << *server_ip + << " port: " + << *server_port + << " query_uri: " + << (request.isBulk() ? queries_uri : query_uri); } - dbgTrace(D_INTELLIGENCE) - << "Sending intelligence request with IP: " - << *server_ip - << " port: " - << *server_port - << " query_uri: " - << (request.isBulk() ? queries_uri : query_uri); - - MessageMetadata req_md(*server_ip, *server_port, conn_flags); - auto req_status = i_message->sendSyncMessage( + auto json_body = request.genJson(); + if (!json_body.ok()) return json_body.passErr(); + auto req_data = i_message->sendSyncMessage( HTTPMethod::POST, request.isBulk() ? queries_uri : query_uri, - request, + *json_body, MessageCategory::INTELLIGENCE, req_md ); - if (!req_status.ok()) { - auto response_error = req_status.getErr().toString(); + if (!req_data.ok()) { + auto response_error = req_data.getErr().toString(); dbgWarning(D_INTELLIGENCE) << "Failed to send intelligence request. Error:" << response_error; return genError( "Failed to send intelligence request. " - + req_status.getErr().getBody() + + req_data.getErr().getBody() + " " - + req_status.getErr().toString() + + req_data.getErr().toString() ); - }; - return createResponse(); + } else if (req_data->getHTTPStatusCode() != HTTPStatusCode::HTTP_OK) { + return genError("Intelligence response is invalid. " + req_data->toString()); + } + + return createResponse(req_data->getBody()); } Maybe -Sender::createResponse() +Sender::createResponse(const std::string &response_body) { - auto mb_json_body = request.getResponseFromFog(); - if (!mb_json_body.ok()) return mb_json_body.passErr(); - Response response(*mb_json_body, request.getSize(), request.isBulk()); + Response response(response_body, request.getSize(), request.isBulk()); auto load_status = response.load(); if (!load_status.ok()) return load_status.passErr(); return response; diff --git a/core/intelligence_is_v2/invalidation.cc b/core/intelligence_is_v2/invalidation.cc index 025e163..b70c6f6 100644 --- a/core/intelligence_is_v2/invalidation.cc +++ b/core/intelligence_is_v2/invalidation.cc @@ -14,6 +14,9 @@ #include "intelligence_invalidation.h" #include +#include +#include "boost/uuid/uuid_io.hpp" +#include "boost/uuid/uuid.hpp" #include "i_intelligence_is_v2.h" @@ -24,7 +27,8 @@ Invalidation::Invalidation(const string &class_value) : source_id(genError()), object_type(genError()), - listening_id(genError()) + listening_id(genError()), + registration_id(genError()) { setClassifier(ClassifierType::CLASS, class_value); } @@ -36,20 +40,6 @@ Invalidation::setClassifier(ClassifierType type, const string &val) return *this; } -Invalidation & -Invalidation::setStringAttr(const string &attr, const string &val, bool is_main) -{ - is_main ? string_main_attr[attr] = val : string_attr[attr] = val; - return *this; -} - -Invalidation & -Invalidation::setStringSetAttr(const string &attr, const set &val, bool is_main) -{ - is_main ? set_string_main_attr[attr] = val : set_string_attr[attr] = val; - return *this; -} - Invalidation & Invalidation::setSourceId(const string &id) { @@ -71,38 +61,6 @@ Invalidation::setInvalidationType(InvalidationType type) return *this; } -Maybe -Invalidation::getStringMainAttr(const string &attr) const -{ - auto val_ref = string_main_attr.find(attr); - if (val_ref == string_main_attr.end()) return genError(); - return val_ref->second; -} - -Maybe, void> -Invalidation::getStringSetMainAttr(const string &attr) const -{ - auto val_ref = set_string_main_attr.find(attr); - if (val_ref == set_string_main_attr.end()) return genError(); - return val_ref->second; -} - -Maybe -Invalidation::getStringAttr(const string &attr) const -{ - auto val_ref = string_attr.find(attr); - if (val_ref == string_attr.end()) return genError(); - return val_ref->second; -} - -Maybe, void> -Invalidation::getStringSetAttr(const string &attr) const -{ - auto val_ref = set_string_attr.find(attr); - if (val_ref == set_string_attr.end()) return genError(); - return val_ref->second; -} - bool Invalidation::report(I_Intelligence_IS_V2 *interface) const { @@ -113,6 +71,7 @@ Invalidation::report(I_Intelligence_IS_V2 *interface) const Maybe Invalidation::startListening(I_Intelligence_IS_V2 *interface, const function &cb) { + registration_id = to_string(boost::uuids::random_generator()()); auto res = interface->registerInvalidation(*this, cb); if (res.ok()) listening_id = *res; return res; @@ -176,53 +135,29 @@ Invalidation::genObject() const if (object_type.ok()) invalidation <<", \"objectType\": \"" << convertObjectType.at(*object_type) << '"'; invalidation << ", \"invalidationType\": \"" << convertInvalidationType.at(invalidation_type) << '"'; if (source_id.ok()) invalidation <<", \"sourceId\": \"" << *source_id << '"'; + if (registration_id.ok()) invalidation <<", \"invalidationRegistrationId\": \"" << *registration_id << '"'; - if (!string_main_attr.empty() || !set_string_main_attr.empty()) { + if (!main_attributes.empty()) { invalidation << ", \"mainAttributes\": [ "; bool first = true; - for (auto &attr : string_main_attr) { + for (auto &main_attr : main_attributes) { if (!first) invalidation << ", "; - invalidation << "{ \"" << attr.first << "\": \"" << attr.second << "\" }"; + auto val = main_attr.genObject(); + if (!val.ok()) continue; + invalidation << *val; first = false; } - - for (auto &attr : set_string_main_attr) { - if (!first) invalidation << ", "; - auto val = makeSeparatedStr(attr.second, ", "); - invalidation << "{ \"" << attr.first << "\": [ "; - bool internal_first = true; - for (auto &value : attr.second) { - if (!internal_first) invalidation << ", "; - invalidation << "\"" << value << "\""; - internal_first = false; - } - invalidation << " ] }"; - first = false; - } - invalidation << " ]"; } - if (!string_attr.empty() || !set_string_attr.empty()) { + if (!attributes.empty()) { invalidation << ", \"attributes\": [ "; bool first = true; - for (auto &attr : string_attr) { + for (auto &attr : attributes) { if (!first) invalidation << ", "; - invalidation << "{ \"" << attr.first << "\": \"" << attr.second << "\" }"; - first = false; - } - - for (auto &attr : set_string_attr) { - if (!first) invalidation << ", "; - auto val = makeSeparatedStr(attr.second, ", "); - invalidation << "{ \"" << attr.first << "\": [ "; - bool internal_first = true; - for (auto &value : attr.second) { - if (!internal_first) invalidation << ", "; - invalidation << "\"" << value << "\""; - internal_first = false; - } - invalidation << " ] }"; + auto val = attr.genObject(); + if (!val.ok()) continue; + invalidation << *val; first = false; } @@ -237,7 +172,7 @@ Invalidation::genObject() const bool Invalidation::isLegalInvalidation() const { - if (!set_string_main_attr.empty() || !string_main_attr.empty()) { + if (!main_attributes.empty() || !attributes.empty()) { if (classifiers[ClassifierType::FAMILY] == "") return false; } @@ -253,6 +188,18 @@ Invalidation::isLegalInvalidation() const template <> class EnumCount : public EnumCountSpecialization {}; +bool +Invalidation::attr_matches(const vector ¤t, const vector &other) const +{ + if (current.empty()) return true; + for (auto &attr : current) { + for(auto &other_attr : other) { + if (attr.matches(other_attr)) return true; + } + } + return false; +} + bool Invalidation::matches(const Invalidation &other) const { @@ -264,53 +211,104 @@ Invalidation::matches(const Invalidation &other) const if (!other.object_type.ok() || *object_type != *other.object_type) return false; } - if (invalidation_type != other.invalidation_type) return false; - if (source_id.ok()) { if (!other.source_id.ok() || *source_id != *other.source_id) return false; } - for (auto &key_value : string_main_attr) { - if (!other.hasMainAttr(key_value.first, key_value.second)) return false; - } + if (!attr_matches(main_attributes, other.getMainAttributes())) return false; - - for (auto &key_values : set_string_main_attr) { - for (auto &value : key_values.second) { - if (!other.hasMainAttr(key_values.first, value)) return false; - } - } - - for (auto &key_value : string_attr) { - if (!other.hasAttr(key_value.first, key_value.second)) return false; - } - - - for (auto &key_values : set_string_attr) { - for (auto &value : key_values.second) { - if (!other.hasAttr(key_values.first, value)) return false; - } - } + if(!attr_matches(attributes, other.getAttributes())) return false; return true; } -bool -Invalidation::hasMainAttr(const string &key, const string &value) const +Invalidation & +Invalidation::addAttr(const StrAttributes &attr) { - auto string_elem = string_main_attr.find(key); - if (string_elem != string_main_attr.end()) return string_elem->second == value; + attributes.emplace_back(attr); + return *this; +} - auto set_string_elem = set_string_main_attr.find(key); - if (set_string_elem != set_string_main_attr.end()) { - return set_string_elem->second.find(value) != set_string_elem->second.end(); +Invalidation & +Invalidation::addMainAttr(const StrAttributes &attr) +{ + main_attributes.emplace_back(attr); + return *this; +} + +Maybe +Invalidation::getRegistrationID() const{ + return registration_id; +} + +StrAttributes & +StrAttributes::addStringAttr(const std::string &attr, const std::string &val) +{ + string_attr[attr] = val; + return *this; +} + +StrAttributes & +StrAttributes::addStringSetAttr(const std::string &attr, const std::set &val) +{ + set_string_attr[attr] = val; + return *this; +} + +Maybe +StrAttributes::getStringAttr(const std::string &attr) const +{ + auto val_ref = string_attr.find(attr); + if (val_ref == string_attr.end()) return genError(); + return val_ref->second; +} + +Maybe, void> +StrAttributes::getStringSetAttr(const string &attr) const +{ + auto val_ref = set_string_attr.find(attr); + if (val_ref == set_string_attr.end()) return genError(); + return val_ref->second; +} + +Maybe +StrAttributes::genObject() const +{ + stringstream attributes_ss; + if (string_attr.empty() && set_string_attr.empty()) return genError(); + bool first = true; + attributes_ss << "{ "; + for (auto &attr : string_attr) { + if (!first) attributes_ss << ", "; + attributes_ss << "\"" << attr.first << "\": \"" << attr.second << "\""; + first = false; } - return false; + for (auto &attr : set_string_attr) { + if (!first) attributes_ss << ", "; + auto val = makeSeparatedStr(attr.second, ", "); + attributes_ss << "\"" << attr.first << "\": [ "; + bool internal_first = true; + for (auto &value : attr.second) { + if (!internal_first) attributes_ss << ", "; + attributes_ss << "\"" << value << "\""; + internal_first = false; + } + attributes_ss << " ]"; + first = false; + } + attributes_ss << " }"; + return attributes_ss.str(); } bool -Invalidation::hasAttr(const string &key, const string &value) const +StrAttributes::isEmpty() const +{ + return string_attr.empty() && set_string_attr.empty(); +} + +bool +StrAttributes::hasAttr(const string &key, const string &value) const { auto string_elem = string_attr.find(key); if (string_elem != string_attr.end()) return string_elem->second == value; @@ -322,3 +320,54 @@ Invalidation::hasAttr(const string &key, const string &value) const return false; } + +bool +StrAttributes::matches(const StrAttributes &other) const +{ + for (auto &key_value : string_attr) { + if (!other.hasAttr(key_value.first, key_value.second)) return false; + } + + for (auto &key_values : set_string_attr) { + for (auto &value : key_values.second) { + if (!other.hasAttr(key_values.first, value)) return false; + } + } + + return true; +} + +void +StrAttributes::serialize(cereal::JSONInputArchive &ar) +{ + SerializableMultiMap> attributes_map; + attributes_map.load(ar); + string_attr = attributes_map.getMap(); + set_string_attr = attributes_map.getMap>(); +} + +void +StrAttributes::performOutputingSchema(ostream &out, int level) { + bool first = true; + RestHelper::printIndent(out, level) << "{\n"; + for (auto &attr : string_attr) { + if (!first) out << ",\n"; + RestHelper::printIndent(out, level + 1) << "\"" << attr.first << "\": \"" << attr.second << "\""; + first = false; + } + + for (auto &attr : set_string_attr) { + if (!first) out << ",\n"; + RestHelper::printIndent(out, level + 1) << "\"" << attr.first << "\": [\n"; + bool internal_first = true; + for (auto &value : attr.second) { + if (!internal_first) out << ",\n"; + RestHelper::printIndent(out, level + 2) << "\"" << value << "\""; + internal_first = false; + } + out << "\n"; + RestHelper::printIndent(out, level + 1) << "]\n"; + first = false; + } + RestHelper::printIndent(out, level) << "}"; +} diff --git a/core/messaging/connection/connection.cc b/core/messaging/connection/connection.cc index 9ceac41..67ff250 100644 --- a/core/messaging/connection/connection.cc +++ b/core/messaging/connection/connection.cc @@ -438,6 +438,26 @@ private: } // LCOV_EXCL_STOP + BioConnectionStatus + tryToBioConnect(const string &full_address) + { + BIO_set_conn_hostname(bio.get(), full_address.c_str()); + BIO_set_nbio(bio.get(), 1); + auto bio_connect = BIO_do_connect(bio.get()); + + if (bio_connect > 0) return BioConnectionStatus::SUCCESS; + if (BIO_should_retry(bio.get())) return BioConnectionStatus::SHOULD_RETRY; + + string bio_error = ERR_error_string(ERR_get_error(), nullptr); + dbgWarning(D_CONNECTION) + << "Connection to: " + << full_address + << " failed and won't retry. Error: " + << bio_error; + return BioConnectionStatus::SHOULD_NOT_RETRY; + } + + Maybe connectToHost() { @@ -449,45 +469,43 @@ private: } dbgFlow(D_CONNECTION) << "Connecting to " << full_address; - BIO_set_conn_hostname(bio.get(), full_address.c_str()); - BIO_set_nbio(bio.get(), 1); I_MainLoop *i_mainloop = Singleton::Consume::by(); I_TimeGet *i_time = Singleton::Consume::by(); - auto bio_connect = BIO_do_connect(bio.get()); + BioConnectionStatus bio_connect = tryToBioConnect(full_address); uint attempts_count = 0; auto conn_end_time = i_time->getMonotonicTime() + getConnectionTimeout(); - while (i_time->getMonotonicTime() < conn_end_time && bio_connect <= 0) { - if (!BIO_should_retry(bio.get())) { - auto curr_time = chrono::duration_cast(i_time->getMonotonicTime()); - active = genError(curr_time + chrono::seconds(60)); - string bio_error = ERR_error_string(ERR_get_error(), nullptr); - return genError( - "Failed to connect to: " + - full_address + - ", error: " + - bio_error + - ". Connection suspended for 60 seconds"); - } - attempts_count++; - if (!isBioSocketReady()) { + while (i_time->getMonotonicTime() < conn_end_time && bio_connect == BioConnectionStatus::SHOULD_RETRY) { + if (isBioSocketReady()) { + bio_connect = tryToBioConnect(full_address); + } else { i_mainloop->yield((attempts_count % 10) == 0); - continue; } - bio_connect = BIO_do_connect(bio.get()); } - if (bio_connect > 0) { + + if (bio_connect == BioConnectionStatus::SUCCESS) { if (isUnsecure() || isOverProxy()) return Maybe(); return performHandshakeAndVerifyCert(i_time, i_mainloop); } + + if (bio_connect == BioConnectionStatus::SHOULD_NOT_RETRY) { + auto curr_time = chrono::duration_cast(i_time->getMonotonicTime()); + active = genError(curr_time + chrono::seconds(60)); + dbgWarning(D_CONNECTION) + << "Connection to: " + << full_address + << " failed and will be suspended for 60 seconds"; + return genError(full_address + ". There won't be a retry attempt."); + } + auto curr_time = chrono::duration_cast(i_time->getMonotonicTime()); active = genError(curr_time + chrono::seconds(60)); - return genError( - "Failed to establish new connection to: " + - full_address + - " after reaching timeout." + - " Connection suspended for 60 seconds"); + dbgWarning(D_CONNECTION) + << "Connection attempts to: " + << full_address + << " have reached timeout and will be suspended for 60 seconds"; + return genError(full_address + ". Connection has reached timeout."); } Maybe @@ -592,7 +610,7 @@ private: auto send_size = sendData(request, data_left_to_send); if (!send_size.ok()) return send_size.passErr(); data_left_to_send -= *send_size; - i_mainloop->yield(*send_size == 0); // We want to force waiting if we failed to send the data + i_mainloop->yield(*send_size == 0); } auto receiving_end_time = i_time->getMonotonicTime() + getConnectionTimeout(); diff --git a/core/messaging/messaging_comp/messaging_comp.cc b/core/messaging/messaging_comp/messaging_comp.cc index 6009075..157ffb6 100644 --- a/core/messaging/messaging_comp/messaging_comp.cc +++ b/core/messaging/messaging_comp/messaging_comp.cc @@ -62,7 +62,7 @@ MessagingComp::getConnection(MessageCategory category, const MessageMetadata &me auto maybe_conn = i_conn->establishConnection(metadata, category); if (!maybe_conn.ok()) { - dbgWarning(D_MESSAGING) << "Failed to establish connection: " << maybe_conn.getErr(); + dbgWarning(D_MESSAGING) << maybe_conn.getErr(); } return maybe_conn; } diff --git a/core/rest/rest_server.cc b/core/rest/rest_server.cc index 783b415..a751fc0 100644 --- a/core/rest/rest_server.cc +++ b/core/rest/rest_server.cc @@ -133,7 +133,7 @@ RestServer::Impl::init() auto is_primary = Singleton::Consume::by()->get("Is Rest primary routine"); id = mainloop->addFileRoutine( - I_MainLoop::RoutineType::RealTime, + I_MainLoop::RoutineType::Offline, fd, [&] () { this->startNewConnection(); }, "REST server listener", @@ -174,7 +174,7 @@ RestServer::Impl::startNewConnection() const RestConn conn(new_socket, mainloop, this); mainloop->addFileRoutine( - I_MainLoop::RoutineType::RealTime, + I_MainLoop::RoutineType::Offline, new_socket, [conn] () { conn.parseConn(); }, "REST server connection handler" diff --git a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc index 28d990c..4879c06 100755 --- a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc +++ b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc @@ -1287,6 +1287,8 @@ extractServices(const vector &args) services.push_back(Service::SDWAN); } else if (getServiceString(Service::LOGGER_SDWAN).find(maybe_service) == 0) { services.push_back(Service::LOGGER_SDWAN); + } else if (getServiceString(Service::CPVIEW_METRIC_PROVIDER).find(maybe_service) == 0) { + services.push_back(Service::CPVIEW_METRIC_PROVIDER); } else if (getServiceString(Service::IOT_WLP).find(maybe_service) == 0) { services.push_back(Service::IOT_WLP); } else if (getServiceString(Service::IDA).find(maybe_service) == 0) { diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index cf9a3cf..4213be4 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -7,6 +7,8 @@ SMB_LOG_FILE_PATH="/storage" USR_LIB_PATH="/usr/lib" USR_SBIN_PATH="/usr/sbin" INIT_D_PATH="/etc/init.d" +NANO_AGENT_SERVICE_NAME="nano_agent" +NANO_AGENT_SERVICE_FILE="${NANO_AGENT_SERVICE_NAME}.service" CONF_PATH="conf" CERTS_PATH="certs" DATA_PATH="data" @@ -19,6 +21,8 @@ ENV_DETAILS_FILE="${CONF_PATH}/environment-details.cfg" WATCHDOG_MAX_ROTATIONS=10 WATCHDOG_MAX_FILE_SIZE=4096 FORCE_CLEAN_FLAG='^(--force-clean|-f)$' +VS_ID="" +VS_LIB_SUB_FOLDER= is_wlp_orchestration="false" ORCHESTRATION_EXE_SOURCE_PATH="./bin/orchestration_comp" @@ -211,7 +215,6 @@ if [ -n "${CP_USR_SBIN_PATH}" ]; then export PATH=$PATH:$CP_USR_SBIN_PATH fi - while true; do if [ "$1" = "--arm32_openwrt" ]; then var_arch="arm" @@ -294,6 +297,14 @@ while true; do FILESYSTEM_PATH=$1 fi echo "Filesystem paths: ${FILESYSTEM_PATH}" + elif [ "$1" = "--vs_id" ]; then + shift + VS_ID=$1 + FILESYSTEM_PATH="/etc/cp/vs${VS_ID}" + NANO_AGENT_SERVICE_NAME="nano_agent_${VS_ID}" + NANO_AGENT_SERVICE_FILE="${NANO_AGENT_SERVICE_NAME}.service" + VS_LIB_SUB_FOLDER="/vs${VS_ID}" + LOG_FILE_PATH="${LOG_FILE_PATH}/vs${VS_ID}" elif [ "$1" = "--log_files_path" ]; then shift var=$1 @@ -324,6 +335,23 @@ while true; do shift done +# VS ID argument is available only on install, for other actions, extract it from the package location +if [ -z "$VS_ID" ]; then + parent_pid=$PPID + parent_cmdline=$(ps -o cmd= -p "$parent_pid") + parent_dir=$(dirname "$parent_cmdline") + packages_folder=$(dirname "$parent_dir") + vs_folder=$(dirname "$packages_folder") + VS_ID=`echo ${vs_folder} | grep -oE '/etc/cp/vs[0-9]+$' | grep -oE '[0-9]+$'` + if [ -n "$VS_ID" ]; then + FILESYSTEM_PATH="/etc/cp/vs${VS_ID}" + NANO_AGENT_SERVICE_NAME="nano_agent_${VS_ID}" + NANO_AGENT_SERVICE_FILE="${NANO_AGENT_SERVICE_NAME}.service" + VS_LIB_SUB_FOLDER="/vs${VS_ID}" + LOG_FILE_PATH="${LOG_FILE_PATH}/vs${VS_ID}" + fi +fi + if [ "$RUN_MODE" = "install" ] && [ $var_offline_mode = false ]; then if [ -n "$OTP_TOKEN" ] && [ -z "$var_token" ] && [ "$var_no_otp" = "false" ]; then var_token=$OTP_TOKEN @@ -464,23 +492,30 @@ update_cloudguard_appsec_manifest() install_watchdog_gaia() { + watchdog_pm_name="cp-nano-watchdog" + if [ -n "$VS_ID" ]; then + watchdog_pm_name="cp-nano-watchdog-vs${VS_ID}" + cp_exec "ln -s ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/${watchdog_pm_name}" + fi + # verify that DB is clean from cp-nano-watchdog - tellpm cp-nano-watchdog - dbset process:cp-nano-watchdog - dbset process:cp-nano-watchdog:path - dbset process:cp-nano-watchdog:arg:1 - dbset process:cp-nano-watchdog:runlevel + tellpm ${watchdog_pm_name} + dbset process:${watchdog_pm_name} + dbset process:${watchdog_pm_name}:path + dbset process:${watchdog_pm_name}:arg:1 + dbset process:${watchdog_pm_name}:runlevel # Add cp-nano-watchdog to DB - dbset process:cp-nano-watchdog t - dbset process:cp-nano-watchdog:path ${FILESYSTEM_PATH}/${WATCHDOG_PATH} - dbset process:cp-nano-watchdog:arg:1 --gaia - dbset process:cp-nano-watchdog:runlevel 1 + dbset process:${watchdog_pm_name} t + dbset process:${watchdog_pm_name}:path ${FILESYSTEM_PATH}/${WATCHDOG_PATH} + dbset process:${watchdog_pm_name}:arg:1 --gaia + dbset process:${watchdog_pm_name}:runlevel 1 dbset :save - tellpm cp-nano-watchdog t + tellpm ${watchdog_pm_name} t } install_watchdog() { + is_upgrade=$1 # Check if watchdog is updated/new old_cp_nano_watchdog_md5="" new_cp_nano_watchdog_md5=$(md5sum watchdog/watchdog | awk '{print$1}') @@ -489,8 +524,8 @@ install_watchdog() fi if [ "$old_cp_nano_watchdog_md5" = "$new_cp_nano_watchdog_md5" ]; then # Watchdog did not changed - cp_print "There is no update in watchdog. Everything is up to date. Reregistering services to be on the sae side." - cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration $var_arch_flag" + cp_print "There is no update in watchdog. Everything is up to date. Reregistering services to be on the same side." + cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register $is_upgrade ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration $var_arch_flag" if [ "$IS_K8S_ENV" = "true" ]; then cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh" fi @@ -502,10 +537,11 @@ install_watchdog() cp_exec "mkdir -p ${FILESYSTEM_PATH}/${WATCHDOG_PATH}" cp_copy watchdog/watchdog ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog cp_copy watchdog/wait-for-networking-inspection-modules.sh ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/wait-for-networking-inspection-modules.sh - cp_exec "chmod 700 ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog" - cp_exec "chmod 700 ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/wait-for-networking-inspection-modules.sh" + cp_exec "chmod 700 ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog" + cp_exec "chmod 700 ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/wait-for-networking-inspection-modules.sh" cp_exec "touch ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/wd.services" - cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration $var_arch_flag" + + cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register $is_upgrade ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration $var_arch_flag" if [ "$IS_K8S_ENV" = "true" ]; then cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh" fi @@ -533,11 +569,15 @@ install_watchdog() cp_exec "ln -s ${CPDIR}/bin/cpopenssl ${CPDIR}/bin/openssl" elif [ $var_startup_service = "systemd" ]; then cp_print "Install for systemd" - cp_copy service/x86/ubuntu16/nano_agent.service /etc/systemd/system/nano_agent.service - echo "ExecStart=${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog" >> /etc/systemd/system/nano_agent.service - echo "ExecStartPost=${FILESYSTEM_PATH}/${WATCHDOG_PATH}/wait-for-networking-inspection-modules.sh" >> /etc/systemd/system/nano_agent.service - echo "Environment=\"FILESYSTEM_PATH=${FILESYSTEM_PATH}\"" >> /etc/systemd/system/nano_agent.service - + cp_copy service/x86/ubuntu16/nano_agent.service /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} + if [ -z "$VS_ID" ]; then + echo "ExecStart=${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} + echo "ExecStartPost=${FILESYSTEM_PATH}/${WATCHDOG_PATH}/wait-for-networking-inspection-modules.sh" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} + else + echo "ExecStart=ip netns exec CTX0000${VS_ID} ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} + fi + echo "Environment=\"FILESYSTEM_PATH=${FILESYSTEM_PATH}\"" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} + cp_exec "systemctl daemon-reload" cp_exec "systemctl enable nano_agent" else @@ -562,7 +602,7 @@ install_watchdog() elif [ $var_arch = "gaia" ]; then install_watchdog_gaia else - cp_exec "service nano_agent start" + cp_exec "service $NANO_AGENT_SERVICE_NAME start" fi fi } @@ -601,7 +641,7 @@ install_cp_nano_ctl() cp_exec "rm -rf $USR_SBIN_PATH/${CP_NANO_CTL_DEPRECATED}" fi # Removing old CP-CTL - if [ -f ${FILESYSTEM_PATH}/${CONF_PATH}/CP_NANO_AGENT_CTL ]; then + if [ -f ${FILESYSTEM_PATH}/${CONF_PATH}/$CP_NANO_AGENT_CTL ]; then cp_exec "rm -rf ${FILESYSTEM_PATH}/${CONF_PATH}/$CP_NANO_AGENT_CTL" fi @@ -612,6 +652,9 @@ install_cp_nano_ctl() cp_exec "cp -f $CP_NANO_CLI ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/$CP_NANO_AGENT_CTL" cp_exec "chmod 700 ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/$CP_NANO_AGENT_CTL" + if [ -n "$VS_ID" ]; then + CP_NANO_CTL="${CP_NANO_CTL}-vs${VS_ID}" + fi cp_exec "ln -s ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/$CP_NANO_AGENT_CTL $USR_SBIN_PATH/${CP_NANO_CTL}" cp_exec "ln -s ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/${OPEN_APPSEC_CTL}.sh $USR_SBIN_PATH/${OPEN_APPSEC_CTL}" @@ -804,16 +847,17 @@ uninstall_messaging_proxy_if_needed() install_orchestration() { INSTALLATION_TIME=$(date) + if [ "$is_smb" != "1" ]; then - cp_exec "mkdir -p ${USR_LIB_PATH}/cpnano" + cp_exec "mkdir -p ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}" else cp_exec "mkdir -p /storage/nano_agent${USR_LIB_PATH}/cpnano" cp_exec "ln -sf /storage/nano_agent${USR_LIB_PATH}/cpnano ${USR_LIB_PATH}/cpnano" cp_exec "mkdir -p /storage/nano_agent/${FILESYSTEM_PATH}" cp_exec "ln -sf /storage/nano_agent/${FILESYSTEM_PATH} ${FILESYSTEM_PATH}" fi - ${INSTALL_COMMAND} lib/*.so* ${USR_LIB_PATH}/cpnano/ - ${INSTALL_COMMAND} lib/boost/*.so* ${USR_LIB_PATH}/cpnano/ + ${INSTALL_COMMAND} lib/*.so* ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/ + ${INSTALL_COMMAND} lib/boost/*.so* ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/ if [ $var_compact_mode = true ]; then [ -f /etc/environment ] && . "/etc/environment" @@ -882,16 +926,15 @@ install_orchestration() upgrade_conf_if_needed - install_watchdog + cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration $var_arch_flag" + if [ "$IS_K8S_ENV" = "true" ]; then + cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh" + fi + cp_print "Upgrade to latest" uninstall_messaging_proxy_if_needed - ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration "$var_arch_flag" > /dev/null 2>&1 - if [ "$IS_K8S_ENV" = "true" ]; then - ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh - fi - if [ ! -f ${FILESYSTEM_PATH}/${DEFAULT_SETTINGS_PATH} ]; then echo "{\"agentSettings\": []}" > ${FILESYSTEM_PATH}/${DEFAULT_SETTINGS_PATH} fi @@ -900,21 +943,18 @@ install_orchestration() copy_k8s_executable copy_nginx_metadata_script - ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration $var_arch_flag - if [ "$IS_K8S_ENV" = "true" ]; then - ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh - fi + install_watchdog "--upgrade" cp_print "Upgrade completed successfully" ${FORCE_STDOUT} - if [ -f /etc/systemd/system/nano_agent.service ]; then - cat "/etc/systemd/system/nano_agent.service" | grep -q "EnvironmentFile=/etc/environment" + if [ -f /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} ]; then + cat "/etc/systemd/system/${NANO_AGENT_SERVICE_FILE}" | grep -q "EnvironmentFile=/etc/environment" result=$? if [ $var_container_mode = false ] && [ $result -eq 0 ]; then - sed -i "$ d" /etc/systemd/system/nano_agent.service - echo "EnvironmentFile=/etc/environment" >> /etc/systemd/system/nano_agent.service - echo >> /etc/systemd/system/nano_agent.service + sed -i "$ d" /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} + echo "EnvironmentFile=/etc/environment" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} + echo >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} cp_exec "systemctl daemon-reload" cp_exec "systemctl restart nano_agent" fi @@ -953,6 +993,9 @@ install_orchestration() if [ -n "${FILESYSTEM_PATH}" ]; then echo "CP_ENV_FILESYSTEM=${FILESYSTEM_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} fi + if [ -n "${VS_ID}" ]; then + echo "CP_VS_ID=${VS_ID}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} + fi if [ -n "${LOG_FILE_PATH}" ]; then echo "CP_ENV_LOG_FILE=${LOG_FILE_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} fi @@ -1095,19 +1138,19 @@ run_pre_install_test() run_post_install_test() { if [ $var_is_alpine = false ]; then - if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_chrono.so* ]; then + if [ ! -f ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/libboost_chrono.so* ]; then cp_print "Error, libboost_chrono .so file is missing" ${FORCE_STDOUT} exit 1 fi - if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_context.so* ]; then + if [ ! -f ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/libboost_context.so* ]; then cp_print "Error, libboost_context .so file is missing" ${FORCE_STDOUT} exit 1 fi - if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_system.so* ]; then + if [ ! -f ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/libboost_system.so* ]; then cp_print "Error, libboost_system .so file is missing" ${FORCE_STDOUT} exit 1 fi - if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_thread.so* ]; then + if [ ! -f ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/libboost_thread.so* ]; then cp_print "Error, libboost_thread .so file is missing" ${FORCE_STDOUT} exit 1 fi