From 66ed4a8d817d9450ec9707acab3f9aa81f5b7a9f Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Sun, 21 Apr 2024 12:38:24 +0000 Subject: [PATCH] April 21th 2024 update --- build_system/docker/entry.sh | 1 + .../nginx_attachment/nginx_attachment.cc | 3 +- components/include/i_details_resolver.h | 1 + components/include/i_downloader.h | 2 +- components/include/i_update_communication.h | 1 + components/include/orchestration_comp.h | 4 +- .../local_policy_mgmt_gen/new_log_trigger.cc | 15 +- .../policy_maker_utils.cc | 4 +- .../local_policy_mgmt_gen/triggers_section.cc | 8 +- .../details_resolver/details_resolver.cc | 13 + .../checkpoint_product_handlers.h | 57 +- .../details_resolver_impl.h | 19 +- .../orchestration/downloader/CMakeLists.txt | 2 +- .../orchestration/downloader/downloader.cc | 19 +- .../downloader/downloader_ut/downloader_ut.cc | 8 +- .../orchestration/downloader/http_client.cc | 268 -------- .../orchestration/downloader/http_client.h | 44 -- .../orchestration/downloader/https_client.cc | 575 ++---------------- .../orchestration/downloader/https_client.h | 36 ++ .../downloader/https_client_helper.cc | 4 +- .../orchestration/env_details/env_details.cc | 10 +- .../orchestration/include/fog_authenticator.h | 10 +- .../include/hybrid_communication.h | 5 +- .../include/local_communication.h | 1 + .../include/mock/mock_details_resolver.h | 1 + .../include/mock/mock_downloader.h | 2 +- .../include/mock/mock_update_communication.h | 1 + .../orchestration/orchestration_comp.cc | 40 +- .../orchestration_multitenant_ut.cc | 10 + .../orchestration_ut/orchestration_ut.cc | 81 ++- .../service_controller/service_controller.cc | 26 +- .../service_controller_ut.cc | 361 +++++++---- .../declarative_policy_utils.cc | 18 +- .../update_communication/fog_authenticator.cc | 125 +++- .../update_communication/fog_communication.cc | 2 - .../hybrid_communication.cc | 2 - .../local_communication.cc | 6 + .../update_communication.cc | 5 + .../local_communication_ut.cc | 11 + .../waap/waap_clib/ParserJson.cc | 1 + .../security_apps/waap/waap_clib/Telemetry.cc | 1 + .../waap/waap_clib/TuningDecisions.h | 1 + components/signal_handler/signal_handler.cc | 4 +- core/config/config.cc | 19 +- core/include/attachments/attachment_types.h | 1 + .../services_sdk/interfaces/i_env_details.h | 2 +- .../interfaces/i_intelligence_is_v2.h | 21 +- .../services_sdk/interfaces/i_messaging.h | 4 +- .../intelligence_interface_impl.h | 17 +- .../interfaces/mock/mock_intelligence.h | 12 +- .../interfaces/mock/mock_messaging.h | 10 +- .../services_sdk/resources/debug_flags.h | 2 +- .../resources/intelligence_invalidation.h | 4 +- .../resources/report/report_enums.h | 1 + .../bulk_query_response_v2.cc | 6 +- .../include/intelligence_request.h | 13 +- .../intelligence_comp_v2.cc | 40 +- .../intelligence_comp_v2_ut.cc | 6 +- .../intelligence_query_v2_ut.cc | 10 +- .../intelligence_is_v2_ut/invalidation_ut.cc | 7 +- .../intelligence_is_v2/intelligence_server.cc | 9 +- core/intelligence_is_v2/invalidation.cc | 9 +- core/messaging/connection/connection.cc | 2 - core/messaging/include/messaging_comp.h | 4 +- core/messaging/messaging.cc | 4 +- core/messaging/messaging_comp/http_request.cc | 14 +- .../messaging_comp/messaging_comp.cc | 15 +- .../messaging_comp_ut/messaging_comp_ut.cc | 2 - core/report/tag_and_enum_management.cc | 6 +- nodes/orchestration/main.cc | 2 +- .../package/cpnano_debug/cpnano_debug.cc | 27 +- .../package/open-appsec-cloud-mgmt | 8 +- .../package/orchestration_package.sh | 75 ++- 73 files changed, 994 insertions(+), 1166 deletions(-) delete mode 100755 components/security_apps/orchestration/downloader/http_client.cc delete mode 100755 components/security_apps/orchestration/downloader/http_client.h create mode 100644 components/security_apps/orchestration/downloader/https_client.h diff --git a/build_system/docker/entry.sh b/build_system/docker/entry.sh index 5a93ab9..899e2e9 100644 --- a/build_system/docker/entry.sh +++ b/build_system/docker/entry.sh @@ -47,6 +47,7 @@ fi orchestration_service_installation_flags="--container_mode --skip_registration" if [ ! -z $var_token ]; then + export AGENT_TOKEN="$var_token" orchestration_service_installation_flags="$orchestration_service_installation_flags --token $var_token" fi if [ ! -z $var_fog_address ]; then diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc index 6174c65..9f52aa5 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc @@ -1136,8 +1136,7 @@ private: NginxAttachmentOpaque &opaque = i_transaction_table->getState(); uuid = opaque.getSessionUUID(); } - web_response_data.uuid_size = - string("Incident Id: ").length() + uuid.size(); + web_response_data.uuid_size = uuid.size(); if (web_trigger_conf.getDetailsLevel() == "Redirect") { web_response_data.response_data.redirect_data.redirect_location_size = diff --git a/components/include/i_details_resolver.h b/components/include/i_details_resolver.h index 02d9c65..9d9c6eb 100644 --- a/components/include/i_details_resolver.h +++ b/components/include/i_details_resolver.h @@ -29,6 +29,7 @@ public: virtual bool isGwNotVsx() = 0; virtual bool isVersionAboveR8110() = 0; virtual bool isReverseProxy() = 0; + virtual bool isCloudStorageEnabled() = 0; virtual Maybe> parseNginxMetadata() = 0; virtual std::map getResolvedDetails() = 0; #if defined(gaia) || defined(smb) diff --git a/components/include/i_downloader.h b/components/include/i_downloader.h index f85ec19..ab67ff4 100755 --- a/components/include/i_downloader.h +++ b/components/include/i_downloader.h @@ -22,7 +22,7 @@ class I_Downloader { public: - virtual Maybe downloadFileFromFog( + virtual Maybe downloadFile( const std::string &checksum, Package::ChecksumTypes, const GetResourceFile &resourse_file diff --git a/components/include/i_update_communication.h b/components/include/i_update_communication.h index 7193ff8..bc113e3 100755 --- a/components/include/i_update_communication.h +++ b/components/include/i_update_communication.h @@ -32,6 +32,7 @@ public: const std::string &policy_versions ) const = 0; virtual Maybe authenticateAgent() = 0; + virtual void registerLocalAgentToFog() = 0; virtual Maybe getUpdate(CheckUpdateRequest &request) = 0; virtual Maybe downloadAttributeFile( const GetResourceFile &resourse_file, diff --git a/components/include/orchestration_comp.h b/components/include/orchestration_comp.h index 5a02dcd..30f9987 100755 --- a/components/include/orchestration_comp.h +++ b/components/include/orchestration_comp.h @@ -31,6 +31,7 @@ #include "i_environment.h" #include "i_tenant_manager.h" #include "i_package_handler.h" +#include "i_proxy_configuration.h" #include "i_env_details.h" #include "component.h" @@ -54,7 +55,8 @@ class OrchestrationComp Singleton::Consume, Singleton::Consume, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: OrchestrationComp(); diff --git a/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc b/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc index 5727c80..00c18e4 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc @@ -27,7 +27,7 @@ NewAppsecTriggerAccessControlLogging::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger - Access Control Logging"; parseAppsecJSONKey("allowEvents", ac_allow_events, archive_in, false); - parseAppsecJSONKey("dropEvents", ac_drop_events, archive_in, false); + parseAppsecJSONKey("dropEvents", ac_drop_events, archive_in, true); } void @@ -36,8 +36,7 @@ NewAppsecTriggerAdditionalSuspiciousEventsLogging::load(cereal::JSONInputArchive dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger - Additional Suspicious Events Logging"; parseAppsecJSONKey("enabled", enabled, archive_in, true); parseAppsecJSONKey("responseBody", response_body, archive_in, false); - //the old code didn't parse the responsecode so ask Noam what is the currenct default value for it - parseAppsecJSONKey("responseCode", response_code, archive_in, false); + parseAppsecJSONKey("responseCode", response_code, archive_in, true); parseAppsecJSONKey("minSeverity", minimum_severity, archive_in, "high"); if (valid_severities.count(minimum_severity) == 0) { dbgWarning(D_LOCAL_POLICY) @@ -175,8 +174,12 @@ void NewAppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger LogDestination"; - // TBD: support "file" - parseAppsecJSONKey("cloud", cloud, archive_in, false); + if (getConfigurationFlag("orchestration-mode") != "hybrid_mode") { + // TBD: support "file" + parseAppsecJSONKey("cloud", cloud, archive_in, false); + } else { + cloud = false; + } auto mode = Singleton::Consume::by()->getOrchestrationMode(); auto env_type = Singleton::Consume::by()->getEnvType(); bool k8s_service_default = (mode == OrchestrationMode::HYBRID && env_type == EnvType::K8S); @@ -184,7 +187,7 @@ NewAppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in) NewStdoutLogging stdout_log; parseAppsecJSONKey("stdout", stdout_log, archive_in); - agent_local = !(stdout_log.getFormat().empty()); + parseAppsecJSONKey("logToAgent", agent_local, archive_in, true); beautify_logs = stdout_log.getFormat() == "json-formatted"; parseAppsecJSONKey("syslogService", syslog_service, archive_in); parseAppsecJSONKey("cefService", cef_service, archive_in); diff --git a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc index 739a1cd..2d54e7d 100755 --- a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc @@ -147,7 +147,7 @@ string PolicyMakerUtils::dumpPolicyToFile( const PolicyWrapper &policy, const string &policy_path, - const string &settings_path) + const string &) { clearElementsMaps(); @@ -170,6 +170,7 @@ PolicyMakerUtils::dumpPolicyToFile( cereal::JSONOutputArchive ar(settings_ss); policy.getSettings().save(ar); } +#if 0 string settings_str = settings_ss.str(); try { ofstream settings_file(settings_path); @@ -179,6 +180,7 @@ PolicyMakerUtils::dumpPolicyToFile( dbgDebug(D_NGINX_POLICY) << "Error while writing settings to " << settings_path << ", Error: " << e.what(); } dbgDebug(D_LOCAL_POLICY) << settings_path << " content: " << settings_str; +#endif return policy_str; } diff --git a/components/security_apps/local_policy_mgmt_gen/triggers_section.cc b/components/security_apps/local_policy_mgmt_gen/triggers_section.cc index 4720ee6..fc510c0 100755 --- a/components/security_apps/local_policy_mgmt_gen/triggers_section.cc +++ b/components/security_apps/local_policy_mgmt_gen/triggers_section.cc @@ -387,8 +387,12 @@ void AppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger LogDestination"; - // TBD: support "file" - parseAppsecJSONKey("cloud", cloud, archive_in, false); + if (getConfigurationFlag("orchestration-mode") != "hybrid_mode") { + // TBD: support "file" + parseAppsecJSONKey("cloud", cloud, archive_in, false); + } else { + cloud = false; + } auto mode = Singleton::Consume::by()->getOrchestrationMode(); auto env_type = Singleton::Consume::by()->getEnvType(); bool k8s_service_default = (mode == OrchestrationMode::HYBRID && env_type == EnvType::K8S); diff --git a/components/security_apps/orchestration/details_resolver/details_resolver.cc b/components/security_apps/orchestration/details_resolver/details_resolver.cc index a135485..c1a9331 100644 --- a/components/security_apps/orchestration/details_resolver/details_resolver.cc +++ b/components/security_apps/orchestration/details_resolver/details_resolver.cc @@ -44,6 +44,7 @@ public: bool isGwNotVsx() override; bool isVersionAboveR8110() override; bool isReverseProxy() override; + bool isCloudStorageEnabled() override; Maybe> parseNginxMetadata() override; #if defined(gaia) || defined(smb) bool compareCheckpointVersion(int cp_version, std::function compare_operator) const override; @@ -135,6 +136,18 @@ DetailsResolver::Impl::isReverseProxy() return getenv("DOCKER_RPM_ENABLED") && getenv("DOCKER_RPM_ENABLED") == string("true"); } +bool +DetailsResolver::Impl::isCloudStorageEnabled() +{ + auto cloud_storage_mode_override = getProfileAgentSetting("agent.cloudStorage.enabled"); + if (cloud_storage_mode_override.ok()) { + dbgInfo(D_ORCHESTRATOR) << "Received cloud-storage mode override: " << *cloud_storage_mode_override; + return *cloud_storage_mode_override; + } + + return getenv("CLOUD_STORAGE_ENABLED") && getenv("CLOUD_STORAGE_ENABLED") == string("true"); +} + bool DetailsResolver::Impl::isKernelVersion3OrHigher() { diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h index 99310e1..bf0ddca 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h @@ -32,6 +32,17 @@ checkSAMLSupportedBlade(const string &command_output) return genError("Current host does not have SAML capability"); } +Maybe +checkIDABlade(const string &command_output) +{ + string idaBlade = "identityServer"; + if (command_output.find(idaBlade) != string::npos) { + return string("true"); + } + + return genError("Current host does not have IDA installed"); +} + Maybe checkSAMLPortal(const string &command_output) { @@ -43,9 +54,19 @@ checkSAMLPortal(const string &command_output) } Maybe -getIDASSamlGaia(const string &command_output) +checkPepIdaIdnStatus(const string &command_output) { - return string("idaSaml_gaia"); + if (command_output.find("ida_idn_nano_service_enabled=1") != string::npos) { + return string("true"); + } + + return genError("Current host does not have PEP control IDA IDN enabled"); +} + +Maybe +getIDAGaiaPackages(const string &command_output) +{ + return string("idaSaml_gaia;idaIdn_gaia;idaIdnBg_gaia;"); } Maybe @@ -76,6 +97,18 @@ checkIsInstallHorizonTelemetrySucceeded(const string &command_output) return command_output; } +Maybe +getQUID(const string &command_output) +{ + if (command_output == "" ) return string("false"); + // validate valid QUID - contains exactly 4 '-' + if (std::count(command_output.begin(), command_output.end(), '-') != 4) { + return genError("not valid QUID"); + } + return command_output; +} + + Maybe checkHasSDWan(const string &command_output) { @@ -92,6 +125,26 @@ checkCanUpdateSDWanData(const string &command_output) return string("true"); } +Maybe +checkLsmProfileName(const string &command_output) +{ + if (!command_output.empty()) { + return command_output; + } + + return genError("LSM profile name was not found"); +} + +Maybe +checkLsmProfileUuid(const string &command_output) +{ + if (!command_output.empty()) { + return command_output; + } + + return genError("LSM profile uuid was not found"); +} + Maybe getMgmtObjType(const string &command_output) { diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index 3b8d297..682beab 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -46,6 +46,9 @@ SHELL_CMD_HANDLER("prerequisitesForHorizonTelemetry", "[ -f /var/log/nano_agent/cp-nano-horizon-telemetry-prerequisites.log ] " "&& head -1 /var/log/nano_agent/cp-nano-horizon-telemetry-prerequisites.log || echo ''", checkIsInstallHorizonTelemetrySucceeded) +SHELL_CMD_HANDLER("QUID", "[ -d /opt/CPquid ] " + "&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo ''", + getQUID) SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan) SHELL_CMD_HANDLER( "canUpdateSDWanData", @@ -56,6 +59,16 @@ SHELL_CMD_HANDLER( "isSdwanRunning", "[ -v $(pidof cp-nano-sdwan) ] && echo 'false' || echo 'true'", checkIfSdwanRunning) +SHELL_CMD_HANDLER( + "lsmProfileName", + "jq -r .lsm_profile_name /tmp/cpsdwan_getdata_orch.json", + checkLsmProfileName +) +SHELL_CMD_HANDLER( + "lsmProfileUuid", + "jq -r .lsm_profile_uuid /tmp/cpsdwan_getdata_orch.json", + checkLsmProfileUuid +) SHELL_CMD_HANDLER( "IP Address", "[ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" " @@ -84,8 +97,10 @@ SHELL_CMD_HANDLER( #if defined(gaia) SHELL_CMD_HANDLER("hasSAMLSupportedBlade", "enabled_blades", checkSAMLSupportedBlade) -SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status saml-vpn", checkSAMLPortal) -SHELL_CMD_HANDLER("requiredNanoServices", "ida_saml_gaia", getIDASSamlGaia) +SHELL_CMD_HANDLER("hasIDABlade", "enabled_blades", checkIDABlade) +SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status nac", checkSAMLPortal) +SHELL_CMD_HANDLER("hasIdaIdnEnabled", "pep control IDN_nano_Srv_support status", checkPepIdaIdnStatus) +SHELL_CMD_HANDLER("requiredNanoServices", "ida_packages", getIDAGaiaPackages) SHELL_CMD_HANDLER( "cpProductIntegrationMgmtParentObjectName", "cat $FWDIR/database/myself_objects.C " diff --git a/components/security_apps/orchestration/downloader/CMakeLists.txt b/components/security_apps/orchestration/downloader/CMakeLists.txt index 7a93e25..73bb13d 100755 --- a/components/security_apps/orchestration/downloader/CMakeLists.txt +++ b/components/security_apps/orchestration/downloader/CMakeLists.txt @@ -1,5 +1,5 @@ ADD_DEFINITIONS(-Wno-deprecated-declarations -Dalpine) -add_library(orchestration_downloader curl_client.cc downloader.cc http_client.cc https_client.cc https_client_helper.cc) +add_library(orchestration_downloader curl_client.cc downloader.cc https_client.cc https_client_helper.cc) #add_subdirectory(downloader_ut) diff --git a/components/security_apps/orchestration/downloader/downloader.cc b/components/security_apps/orchestration/downloader/downloader.cc index 09670bb..811648b 100755 --- a/components/security_apps/orchestration/downloader/downloader.cc +++ b/components/security_apps/orchestration/downloader/downloader.cc @@ -15,7 +15,7 @@ #include "i_orchestration_tools.h" #include "singleton.h" -#include "http_client.h" +#include "https_client.h" #include "debug.h" #include "config.h" #include "rest.h" @@ -63,7 +63,7 @@ class Downloader::Impl : Singleton::Provide::From public: void init(); - Maybe downloadFileFromFog( + Maybe downloadFile( const string &checksum, Package::ChecksumTypes checksum_type, const GetResourceFile &resourse_file @@ -87,7 +87,7 @@ public: string getProfileFromMap(const string &tenant_id) const override; private: - Maybe downloadFileFromFogByHTTP( + Maybe downloadAttributeFile( const GetResourceFile &resourse_file, const string &file_name ) const; @@ -130,12 +130,12 @@ Downloader::Impl::init() } Maybe -Downloader::Impl::downloadFileFromFog( +Downloader::Impl::downloadFile( const string &checksum, Package::ChecksumTypes checksum_type, const GetResourceFile &resourse_file) const { - auto file_path = downloadFileFromFogByHTTP(resourse_file, resourse_file.getFileName() + ".download"); + auto file_path = downloadAttributeFile(resourse_file, resourse_file.getFileName() + ".download"); if (!file_path.ok()) { return file_path; @@ -378,11 +378,11 @@ Downloader::Impl::validateChecksum( } Maybe -Downloader::Impl::downloadFileFromFogByHTTP(const GetResourceFile &resourse_file, const string &file_name) const +Downloader::Impl::downloadAttributeFile(const GetResourceFile &resourse_file, const string &file_name) const { string file_path = dir_path + "/" + file_name; - dbgInfo(D_ORCHESTRATOR) << "Downloading file from fog. File: " << resourse_file.getFileName(); + dbgInfo(D_ORCHESTRATOR) << "Downloading file. File: " << resourse_file.getFileName(); I_UpdateCommunication *update_communication = Singleton::Consume::by(); auto downloaded_file = update_communication->downloadAttributeFile(resourse_file, file_path); @@ -408,17 +408,14 @@ Downloader::Impl::getFileFromLocal(const string &local_file_path, const string & Maybe Downloader::Impl::getFileFromURL(const URLParser &url, const string &file_path, bool auth_required) const { - ofstream outFile(file_path, ofstream::out | ofstream::binary); - HTTPClient http_client; dbgInfo(D_ORCHESTRATOR) << "Downloading file. URL: " << url; - auto get_file_response = http_client.getFile(url, outFile, auth_required); + auto get_file_response = HTTPSClient().getFile(url, file_path, auth_required); if (!get_file_response.ok()) { Maybe error = genError("Failed to download file from " + url.getBaseURL().unpack() + ". Error: " + get_file_response.getErr()); dbgWarning(D_ORCHESTRATOR) << "Download failed"; return error; } - outFile.close(); dbgInfo(D_ORCHESTRATOR) << "Download completed. URL: " << url; return file_path; } diff --git a/components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc b/components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc index e29d9ee..747a4b4 100755 --- a/components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc +++ b/components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc @@ -38,7 +38,7 @@ TEST_F(DownloaderTest, doNothing) { } -TEST_F(DownloaderTest, downloadFileFromFog) +TEST_F(DownloaderTest, downloadFile) { string fog_response = "bla bla"; string checksum = "123"; @@ -55,7 +55,7 @@ TEST_F(DownloaderTest, downloadFileFromFog) EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile("/tmp/virtualSettings.download")).WillOnce(Return(true)); - Maybe downloaded_file = i_downloader->downloadFileFromFog( + Maybe downloaded_file = i_downloader->downloadFile( checksum, Package::ChecksumTypes::SHA256, resourse_file @@ -76,7 +76,7 @@ TEST_F(DownloaderTest, downloadFileFromFogFailure) downloadAttributeFile(resourse_file, "/tmp/settings.download") ).WillOnce(Return(fog_response)); - Maybe downloaded_file = i_downloader->downloadFileFromFog( + Maybe downloaded_file = i_downloader->downloadFile( checksum, Package::ChecksumTypes::SHA256, resourse_file @@ -241,7 +241,7 @@ TEST_F(DownloaderTest, downloadEmptyFileFromFog) calculateChecksum(Package::ChecksumTypes::SHA256, "/tmp/manifest.download") ).WillOnce(Return(checksum)); - Maybe downloaded_file = i_downloader->downloadFileFromFog( + Maybe downloaded_file = i_downloader->downloadFile( checksum, Package::ChecksumTypes::SHA256, resourse_file diff --git a/components/security_apps/orchestration/downloader/http_client.cc b/components/security_apps/orchestration/downloader/http_client.cc deleted file mode 100755 index cfa03c3..0000000 --- a/components/security_apps/orchestration/downloader/http_client.cc +++ /dev/null @@ -1,268 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "http_client.h" - -#include "curl_client.h" -#include "downloader.h" -#include "debug.h" -#include "i_encryptor.h" -#include "url_parser.h" -#include "config.h" -#include "i_environment.h" -#include "orchestration_comp.h" - -#include -#include -#include -#include -#include - -using boost::asio::ip::tcp; -using namespace std; - -USE_DEBUG_FLAG(D_ORCHESTRATOR); -USE_DEBUG_FLAG(D_HTTP_REQUEST); - -// LCOV_EXCL_START Reason: Depends on real download server. -class ClientConnection -{ -public: - ClientConnection( - const URLParser &_url, - const Maybe &_proxy_url, - const Maybe &_proxy_port, - const Maybe &_proxy_auth, - const string &_token) - : - url(_url), - proxy_url(_proxy_url), - proxy_port(_proxy_port), - proxy_auth(_proxy_auth), - token(_token) - { - } - - Maybe - handleConnect() - { - if (!url.getBaseURL().ok()) { - return genError("Failed to handle connection. Error: " + url.getBaseURL().getErr()); - } - string server_name = url.getBaseURL().unpack(); - string port = url.getPort(); - string query = url.getQuery(); - string host = server_name; - try { - if (stoi(port) != 80) { - host = host + ":" + port; - } - } catch (const exception &err) { - return genError("Failed to parse port to a number. Port: " + port ); - } - - chrono::duration> sleep_time(60); - io_stream.expires_from_now(sleep_time); - - if (proxy_url.ok()) { - if (!proxy_port.ok()) { - return genError( - "Failed to handle connection to server. proxy domain is defined with invalid port, Error: " + - proxy_port.getErr() - ); - } - io_stream.connect(proxy_url.unpack(), to_string(proxy_port.unpack())); - } else { - io_stream.connect(server_name, port); - } - - if (!io_stream) { - return genError("Failed to handle connection to server. Error: " + io_stream.error().message()); - } - - string request_url = query; - if (proxy_url.ok()) { - request_url = host + query; - } - - stringstream http_request; - http_request << "GET http://" << request_url << " HTTP/1.1\r\n"; - http_request << "Host: " << host << "\r\n"; - if (!token.empty()) { - http_request << "Authorization: " << "Bearer " << token << "\r\n"; - } - http_request << "User-Agent: Infinity Next (a7030abf93a4c13)\r\n"; - - auto i_trace_env = Singleton::Consume::by(); - http_request << i_trace_env->getCurrentHeaders(); - http_request << "Accept: */*\r\n"; - - if (proxy_url.ok()) { - http_request << "Accept-Encoding: identity"; - http_request << "Connection: close\r\n"; - http_request << "Proxy-Connection: Keep-Alive\r\n"; - - if (proxy_auth.ok()) { - I_Encryptor *encryptor = Singleton::Consume::by(); - http_request << "Proxy-Authorization: Basic " + encryptor->base64Encode(proxy_auth.unpack()) + "\r\n"; - } - http_request << "\r\n"; - } else { - http_request << "Connection: close\r\n\r\n"; - } - - dbgTrace(D_HTTP_REQUEST) << "Sending the following HTTP Request: " << endl << http_request.str(); - io_stream << http_request.str(); - return Maybe(); - } - - Maybe - handleResponse(ofstream &out_file) - { - string response_http_version; - io_stream >> response_http_version; - unsigned int status_code; - io_stream >> status_code; - string status_message; - getline(io_stream, status_message); - - if (!io_stream || response_http_version.substr(0, 5) != "HTTP/") { - return genError("Invalid response"); - } - - if (status_code != 200) { - return genError("HTTP response returned with status code " + status_code); - } - - string header; - vector headers; - while (getline(io_stream, header) && header != "\r") { - headers.push_back(header); - } - - out_file << io_stream.rdbuf(); - - dbgTrace(D_HTTP_REQUEST) - << "Received HTTP Response with the following data (downloaded file will not be printed):" - << endl - << response_http_version - << " " - << status_code - << " " - << status_message - << endl - << makeSeparatedStr(headers, "\n"); - - - return Maybe(); - } - -private: - const URLParser url; - const Maybe proxy_url; - const Maybe proxy_port; - const Maybe proxy_auth; - const string &token; - boost::asio::ip::tcp::iostream io_stream; -}; - -Maybe -HTTPClient::getFile(const URLParser &url, ofstream &out_file, bool auth_required) -{ - auto proxy_config = Singleton::Consume::by(); - auto load_env_proxy = proxy_config->loadProxy(); - if (!load_env_proxy.ok()) return load_env_proxy; - - string token = ""; - if (auth_required) { - token = Singleton::Consume::by()->getAccessToken(); - } - - if (url.isOverSSL()) { - if (getFileSSLDirect(url, out_file, token).ok()) return Maybe(); - dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL directly. Trying indirectly."; - if (getFileSSL(url, out_file, token).ok()) return Maybe(); - //CURL fallback - dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Trying via CURL (SSL)."; - return curlGetFileOverSSL(url, out_file, token); - } - auto get_file_http_res = getFileHttp(url, out_file, token); - if (!get_file_http_res.ok()) - { - //CURL fallback - dbgWarning(D_ORCHESTRATOR) << "Failed to get file over HTTP. Trying via CURL (HTTP)."; - return curlGetFileOverHttp(url, out_file, token); - } - - return get_file_http_res; -} - -Maybe -HTTPClient::curlGetFileOverHttp(const URLParser &url, ofstream &out_file, const string &token) -{ - try { - auto proxy_config = Singleton::Consume::by(); - - HttpCurl http_curl_client( - url, - out_file, - token, - proxy_config->getProxyDomain(ProxyProtocol::HTTPS), - proxy_config->getProxyPort(ProxyProtocol::HTTPS), - proxy_config->getProxyAuthentication(ProxyProtocol::HTTPS)); - - http_curl_client.setCurlOpts(); - bool connection_ok = http_curl_client.connect(); - if (!connection_ok) - { - stringstream url_s; - url_s << url; - string err_msg = string("Failed to get file over HTTP. URL: ") + url_s.str(); - return genError(err_msg); - } - - // As this class is a temporal solution catch all exception types is enabled. - } catch (const exception &e) { - string err_msg = "Failed to get file over HTTP. Exception: " + string(e.what()); - return genError(err_msg); - } - - return Maybe(); -} - -Maybe -HTTPClient::getFileHttp(const URLParser &url, ofstream &out_file, const string &token) -{ - try { - auto proxy_config = Singleton::Consume::by(); - ClientConnection client_connection( - url, - proxy_config->getProxyDomain(ProxyProtocol::HTTP), - proxy_config->getProxyPort(ProxyProtocol::HTTP), - proxy_config->getProxyAuthentication(ProxyProtocol::HTTP), - token - ); - auto handle_connect_res = client_connection.handleConnect(); - if (!handle_connect_res.ok()) return handle_connect_res; - - return client_connection.handleResponse(out_file); - - // As this class is a temporal solution catch all exception types is enabled. - } catch (const exception &e) { - string err_msg = "Failed to get file over HTTP. Exception: " + string(e.what()); - return genError(err_msg); - } - - return Maybe(); -} -// LCOV_EXCL_STOP diff --git a/components/security_apps/orchestration/downloader/http_client.h b/components/security_apps/orchestration/downloader/http_client.h deleted file mode 100755 index fb99007..0000000 --- a/components/security_apps/orchestration/downloader/http_client.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __HTTP_CLIENT_H__ -#define __HTTP_CLIENT_H__ - -#include -#include "maybe_res.h" -#include "url_parser.h" -#include "i_agent_details.h" -#include "i_proxy_configuration.h" - -// LCOV_EXCL_START Reason: Depends on real download server. -class HTTPClient - : - public Singleton::Consume, - public Singleton::Consume -{ -public: - HTTPClient() = default; - - Maybe getFile(const URLParser &url, std::ofstream &out_file, bool auth_required); - -private: - std::string loadCAChainDir(); - Maybe getFileSSL(const URLParser &url, std::ofstream &out_file, const std::string &_token); - Maybe getFileSSLDirect(const URLParser &url, std::ofstream &out_file, const std::string &_token); - Maybe getFileHttp(const URLParser &url, std::ofstream &out_file, const std::string &_token); - Maybe curlGetFileOverHttp(const URLParser &url, std::ofstream &out_file, const std::string &_token); - Maybe curlGetFileOverSSL(const URLParser &url, std::ofstream &out_file, const std::string &_token); -}; -// LCOV_EXCL_STOP - -#endif // __HTTP_CLIENT_H__ diff --git a/components/security_apps/orchestration/downloader/https_client.cc b/components/security_apps/orchestration/downloader/https_client.cc index 522521d..b0441c3 100755 --- a/components/security_apps/orchestration/downloader/https_client.cc +++ b/components/security_apps/orchestration/downloader/https_client.cc @@ -11,498 +11,51 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "http_client.h" -#include "curl_client.h" - -#include "debug.h" -#include "i_agent_details.h" -#include "i_encryptor.h" -#include "downloader.h" -#include "config.h" -#include "boost/uuid/uuid.hpp" -#include "boost/uuid/uuid_generators.hpp" -#include -#include "boost/uuid/uuid_io.hpp" +#include "https_client.h" +#include #include #include -#include -#include -#include -#include -#include -#include -#include +#include + +#include "config.h" +#include "curl_client.h" -using namespace boost::placeholders; -using boost::asio::ip::tcp; using namespace std; -USE_DEBUG_FLAG(D_COMMUNICATION); -USE_DEBUG_FLAG(D_HTTP_REQUEST); USE_DEBUG_FLAG(D_ORCHESTRATOR); +USE_DEBUG_FLAG(D_HTTP_REQUEST); // LCOV_EXCL_START Reason: Depends on real download server. -class BadResponseFromServer : public exception +Maybe +HTTPSClient::getFile(const URLParser &url, const string &out_file, bool auth_required) { -public: - BadResponseFromServer() : message("Bad response returned from server") {} - BadResponseFromServer(const string &msg) : message(msg) {} - const char * - what() const throw() - { - return message.c_str(); + auto proxy_config = Singleton::Consume::by(); + auto load_env_proxy = proxy_config->loadProxy(); + if (!load_env_proxy.ok()) return load_env_proxy; + + string token = ""; + if (auth_required) { + token = Singleton::Consume::by()->getAccessToken(); } -private: - string message; -}; + if (!url.isOverSSL()) return genError("URL is not over SSL."); -class Client -{ -public: - Client( - ofstream &out_file, - boost::asio::io_service &io_service, - boost::asio::ssl::context &context, - const URLParser &_url, - const Maybe &_proxy_url, - const Maybe &_proxy_port, - const Maybe &_proxy_auth, - const string &_token) - : - out_file(out_file), - url(_url), - proxy_url(_proxy_url), - proxy_port(_proxy_port), - proxy_auth(_proxy_auth), - resolver_(io_service), - deadline(io_service), - socket_(io_service), - ssl_socket(socket_, context), - token(_token) - { - } + if (getFileSSLDirect(url, out_file, token).ok()) return Maybe(); + dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL directly. Trying indirectly."; - Maybe - handleConnection() - { - ostream request_stream(&request_); - stringstream http_request; - http_request << "GET " << url.getQuery() << " HTTP/1.1\r\n"; - string host = url.getHost(); - string port = url.getPort(); - int port_int; - try { - port_int = stoi(port); - } catch (const exception &err) { - dbgWarning(D_COMMUNICATION) - << "Failed to convert port number from string. Port: " - << port - << ", Error: " - << err.what(); - return genError("Failed to parse port to a number. Port: " + port); - } - if (port_int != 443) { - host = host + ":" + port; - } + if (getFileSSL(url, out_file, token).ok()) return Maybe(); + dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Trying via CURL (SSL)."; - http_request << "Host: " << host << "\r\n"; - - if (!token.empty()) { - http_request << "Authorization: " << "Bearer " << token << "\r\n"; - } - http_request << "User-Agent: Infinity Next (a7030abf93a4c13)\r\n"; - boost::uuids::uuid correlation_id; - try { - correlation_id = uuid_random_gen(); - } catch (const boost::uuids::entropy_error &) { - dbgWarning(D_COMMUNICATION) << "Failed to generate random correlation id - entropy exception"; - } - http_request << "X-Trace-Id: " + boost::uuids::to_string(correlation_id) + "\r\n"; - http_request << "Accept: */*\r\n"; - http_request << "Connection: close\r\n\r\n"; - - request_stream << http_request.str(); - - deadline.expires_from_now(boost::posix_time::minutes(5)); - deadline.async_wait(boost::bind(&Client::checkDeadline, this, _1)); - - if (proxy_url.ok()) { - if (!proxy_port.ok()) { - dbgWarning(D_COMMUNICATION) - << "Failed to connect to proxy due to invalid port value, Error: " - << proxy_port.getErr(); - - return genError( - "Failed to handle connection to server. proxy port is invalid, Error: " + - proxy_port.getErr() - ); - } - if (port_int == 443) host = host + ":" + port; - ostream connect_request_stream(&connect_request); - stringstream proxy_request; - proxy_request << "CONNECT " << host << " HTTP/1.1\r\n"; - proxy_request << "Host: " << host << "\r\n"; - if (proxy_auth.ok()) { - I_Encryptor *encryptor = Singleton::Consume::by(); - proxy_request - << "Proxy-Authorization: Basic " - << encryptor->base64Encode(proxy_auth.unpack()) - << "\r\n"; - } - proxy_request << "\r\n"; - - dbgTrace(D_HTTP_REQUEST) << "Connecting to proxy: " << endl << proxy_request.str(); - connect_request_stream << proxy_request.str(); - - tcp::resolver::query query(proxy_url.unpack(), to_string(proxy_port.unpack())); - resolver_.async_resolve( - query, - boost::bind( - &Client::overProxyResolver, - this, - boost::asio::placeholders::error, - boost::asio::placeholders::iterator - ) - ); - } else { - tcp::resolver::query query(url.getBaseURL().unpack(), port); - resolver_.async_resolve( - query, - boost::bind( - &Client::handleResolve, - this, - boost::asio::placeholders::error, - boost::asio::placeholders::iterator - ) - ); - } - - dbgTrace(D_HTTP_REQUEST) << "Sending the following HTTP Request: " << endl << http_request.str(); - return Maybe(); - } - -private: - void - checkDeadline(const boost::system::error_code &err) - { - if (err) return; - if (deadline.expires_at() <= boost::asio::deadline_timer::traits_type::now()) { - boost::system::error_code ignored_ec = boost::asio::error::operation_aborted; - socket_.close(ignored_ec); - deadline.expires_at(boost::posix_time::pos_infin); - return; - } - deadline.async_wait(boost::bind(&Client::checkDeadline, this, _1)); - } - - void - overProxyResolver(const boost::system::error_code &err, tcp::resolver::iterator endpoint_iterator) - { - if (!err) { - boost::asio::async_connect(socket_, endpoint_iterator, - boost::bind(&Client::overProxyHandleConnect, this, - boost::asio::placeholders::error)); - } else { - string err_msg = "Failed to connect to proxy. Error: " + err.message(); - throw BadResponseFromServer(err_msg); - } - } - - void - overProxyHandleConnect(const boost::system::error_code &err) - { - if (!err) { - boost::asio::async_write(socket_, connect_request, - boost::bind(&Client::overProxyHandleWriteRequest, this, - boost::asio::placeholders::error)); - } else { - string err_msg = "Failed to connect to proxy. Error: " + err.message(); - throw BadResponseFromServer(err_msg); - } - } - - void - overProxyHandleWriteRequest(const boost::system::error_code &err) - { - if (!err) { - boost::asio::async_read_until( - socket_, - response_, - "\r\n", - boost::bind(&Client::overProxyHandleReadStatusLine, this, boost::asio::placeholders::error) - ); - } else { - string err_msg = "Failed to write over proxy. Error: " + err.message(); - throw BadResponseFromServer(err_msg); - } - } - - void - overProxyHandleReadStatusLine(const boost::system::error_code &err) - { - if (err) { - string err_msg = "Failed to read status line over proxy. Error: " + err.message(); - throw BadResponseFromServer(err_msg); - } - // Check that response is OK. - istream response_stream(&response_); - string response_http_version; - response_stream >> response_http_version; - unsigned int status_code; - response_stream >> status_code; - string status_message; - getline(response_stream, status_message); - if (!response_stream || response_http_version.substr(0, 5) != "HTTP/") { - throw BadResponseFromServer("Invalid response"); - return; - } - - if (status_code != 200) { - string err_msg = "Response returned with status code " + status_code; - throw BadResponseFromServer(err_msg); - } - - dbgTrace(D_HTTP_REQUEST) - << "Received HTTP Response over proxied connection with the following data:" - << endl - << response_http_version - << " " - << status_code - << " " - << status_message; - - if (getProfileAgentSettingWithDefault(false, "agent.config.message.ignoreSslValidation") == false) { - ssl_socket.set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert); - ssl_socket.set_verify_callback(boost::bind(&Client::verifyCertificate, this, _1, _2)); - } else { - dbgWarning(D_HTTP_REQUEST) << "Ignoring SSL validation"; - } - - ssl_socket.async_handshake( - boost::asio::ssl::stream_base::client, - boost::bind(&Client::handleHandshake, this, boost::asio::placeholders::error) - ); - } - - void - handleResolve(const boost::system::error_code &err, tcp::resolver::iterator endpoint_iterator) - { - if (!err) { - boost::asio::async_connect(ssl_socket.lowest_layer(), endpoint_iterator, - boost::bind(&Client::handleConnect, this, - boost::asio::placeholders::error)); - } else { - string message = "Failed to connect. Error: " + err.message(); - throw BadResponseFromServer(message); - } - } - - bool - verifyCertificate(bool preverified, boost::asio::ssl::verify_context &ctx) - { - if (!token.empty()) { - X509_STORE_CTX *cts = ctx.native_handle(); - - switch (X509_STORE_CTX_get_error(cts)) - { - case X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT: - dbgWarning(D_ORCHESTRATOR) << "SSL verification error: X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT"; - break; - case X509_V_ERR_CERT_NOT_YET_VALID: - case X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD: - dbgWarning(D_ORCHESTRATOR) << "SSL verification error: Certificate not yet valid"; - break; - case X509_V_ERR_CERT_HAS_EXPIRED: - case X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD: - dbgWarning(D_ORCHESTRATOR) << "Certificate expired"; - break; - case X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT: - case X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN: - dbgDebug(D_ORCHESTRATOR) << "Self signed certificate in chain"; - if (getConfigurationWithDefault(false, "orchestration", "Self signed certificates acceptable")) { - preverified = true; - } - break; - default: - if (!preverified) { - dbgWarning(D_ORCHESTRATOR) - << "Certificate verification error number: " - << X509_STORE_CTX_get_error(cts); - } - break; - } - return preverified; - } - return true; - } - - void - handleConnect(const boost::system::error_code &err) - { - if (!err) { - if (getProfileAgentSettingWithDefault(false, "agent.config.message.ignoreSslValidation") == false) { - ssl_socket.set_verify_mode( - boost::asio::ssl::verify_peer | - boost::asio::ssl::verify_fail_if_no_peer_cert - ); - ssl_socket.set_verify_callback(boost::bind(&Client::verifyCertificate, this, _1, _2)); - } else { - dbgWarning(D_HTTP_REQUEST) << "Ignoring SSL validation"; - } - - ssl_socket.async_handshake(boost::asio::ssl::stream_base::client, - boost::bind(&Client::handleHandshake, this, - boost::asio::placeholders::error)); - } else { - string err_message = "Failed to connect. Error: " + err.message(); - throw BadResponseFromServer(err_message); - } - } - - void - handleHandshake(const boost::system::error_code &error) - { - if (!error) { - boost::asio::buffer_cast(request_.data()); - - boost::asio::async_write(ssl_socket, request_, - boost::bind(&Client::handleWriteRequest, this, - boost::asio::placeholders::error)); - } else { - string err_message = "Handshake failed. Error: " + error.message(); - throw BadResponseFromServer(err_message); - } - } - - void - handleWriteRequest(const boost::system::error_code &err) - { - if (!err) { - boost::asio::async_read_until(ssl_socket, resp, "\r\n", - boost::bind(&Client::handleReadStatusLine, this, - boost::asio::placeholders::error)); - } else { - string err_message = "Failed to handle write request. Error: " + err.message(); - throw BadResponseFromServer(err_message); - } - } - - void - handleReadStatusLine(const boost::system::error_code &err) - { - if (!err) { - istream response_stream(&resp); - string http_version; - response_stream >> http_version; - unsigned int status_code; - response_stream >> status_code; - string status_message; - getline(response_stream, status_message); - dbgTrace(D_HTTP_REQUEST) - << "Received HTTP Response with the following data:" - << endl - << http_version - << " " - << status_code; - - if (!response_stream || http_version.substr(0, 5) != "HTTP/") { - string err_message = "Invalid response"; - throw BadResponseFromServer(err_message); - } - if (status_code != 200) { - string err_message = "HTTPS response returned with status code " + to_string(status_code) - + ". URL: " + url.toString(); - throw BadResponseFromServer(err_message); - } - - boost::asio::async_read_until(ssl_socket, resp, "\r\n\r\n", - boost::bind(&Client::handleReadHeaders, this, - boost::asio::placeholders::error)); - } else { - dbgWarning(D_COMMUNICATION) << "Failed to read response status. Error:" << err.message(); - string err_message = "Failed to read status. Error: " + err.message(); - throw BadResponseFromServer(err_message); - } - } - - void - handleReadHeaders(const boost::system::error_code &err) - { - if (!err) { - // Process the response headers. - istream response_stream(&resp); - string header; - vector headers; - while (getline(response_stream, header) && header != "\r") { - headers.push_back(header); - } - - dbgTrace(D_HTTP_REQUEST) << "Received Response headers:" << endl << makeSeparatedStr(headers, "\n"); - // Write whatever content we already have to output. - if (resp.size() > 0) - out_file << &resp; - - // Start reading remaining data until EOF. - boost::asio::async_read(ssl_socket, resp, - boost::asio::transfer_at_least(1), - boost::bind(&Client::handleReadContent, this, - boost::asio::placeholders::error)); - } else { - dbgWarning(D_COMMUNICATION) << "Failed to read response headers. Error:" << err.message(); - string err_message = "Failed to read headers. Error: " + err.message(); - throw BadResponseFromServer(err_message); - } - } - - void - handleReadContent(const boost::system::error_code &err) - { - if (!err) { - // Write all of the data that has been read so far. - out_file << &resp; - // Continue reading remaining data until EOF. - boost::asio::async_read( - ssl_socket, - resp, - boost::asio::transfer_at_least(1), - boost::bind(&Client::handleReadContent, this, boost::asio::placeholders::error) - ); - } else if (err != boost::asio::error::eof && err != boost::asio::ssl::error::stream_truncated) { - dbgWarning(D_COMMUNICATION) << "Failed to read response body. Error:" << err.message(); - string err_message = "Failed to read content. Error: " + err.message(); - throw BadResponseFromServer(err_message); - } else if (err == boost::asio::ssl::error::stream_truncated) { - dbgError(D_COMMUNICATION) << "Had SSL warning during reading response body stage. Error:" << err.message(); - deadline.cancel(); - } else { - deadline.cancel(); - } - } - - ofstream &out_file; - const URLParser &url; - const Maybe proxy_url; - const Maybe proxy_port; - const Maybe proxy_auth; - tcp::resolver resolver_; - boost::asio::deadline_timer deadline; - boost::asio::ip::tcp::socket socket_; - boost::asio::ssl::stream ssl_socket; - boost::asio::streambuf request_; - boost::asio::streambuf connect_request; - boost::asio::streambuf response_; - boost::asio::streambuf resp; - const string &token; - boost::uuids::random_generator uuid_random_gen; -}; + //CURL fallback + return curlGetFileOverSSL(url, out_file, token); +} string -HTTPClient::loadCAChainDir() +HTTPSClient::loadCAChainDir() { string ca_chain_dir; - auto agent_details = Singleton::Consume::by(); + auto agent_details = Singleton::Consume::by(); auto load_ca_chain_dir = agent_details->getOpenSSLDir(); if (load_ca_chain_dir.ok()) { ca_chain_dir = load_ca_chain_dir.unpack(); @@ -511,69 +64,26 @@ HTTPClient::loadCAChainDir() } Maybe -HTTPClient::getFileSSL(const URLParser &url, ofstream &out_file, const string &token) +HTTPSClient::getFileSSL(const URLParser &url, const string &out_file, const string &) { - try { - boost::asio::ssl::context ctx(boost::asio::ssl::context::sslv23); - if (!token.empty()) { - string cert_file_path = getConfigurationWithDefault( - getFilesystemPathConfig() + "/certs/fog.pem", - "message", - "Certificate chain file path" - ); - dbgTrace(D_ORCHESTRATOR) << "Http client, cert file path: " << cert_file_path; - auto trusted_ca_directory = getConfiguration("message", "Trusted CA directory"); - if (trusted_ca_directory.ok() && !trusted_ca_directory.unpack().empty()) { - ctx.add_verify_path(trusted_ca_directory.unpack()); - } else { - string cert_file_path = getConfigurationWithDefault( - getFilesystemPathConfig() + "/certs/fog.pem", - "message", - "Certificate chain file path" - ); - ctx.load_verify_file(cert_file_path); - } - } - boost::asio::io_service io_service; - auto proxy_config = Singleton::Consume::by(); - - Client client( - out_file, - io_service, - ctx, - url, - proxy_config->getProxyDomain(ProxyProtocol::HTTPS), - proxy_config->getProxyPort(ProxyProtocol::HTTPS), - proxy_config->getProxyAuthentication(ProxyProtocol::HTTPS), - token - ); - - auto connection_result = client.handleConnection(); - if (!connection_result.ok()) { - return connection_result; - }; - - auto mainloop = Singleton::Consume::by(); - while (!io_service.stopped()) { - io_service.poll_one(); - mainloop->yield(true); - } - } catch (const exception &e) { - dbgWarning(D_COMMUNICATION) << "Failed to get file over HTTPS. Error:" << string(e.what()); - string error_str = "Failed to get file over HTTPS, exception: " + string(e.what()); - return genError(error_str); + auto downlaod_file = Singleton::Consume::by()->downloadFile( + HTTPMethod::GET, + url.getQuery(), + out_file + ); + if (!downlaod_file.ok()) { + dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Error: " << downlaod_file.getErr().toString(); + return genError(downlaod_file.getErr().toString()); } - return Maybe(); } Maybe -HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const string &token) +HTTPSClient::curlGetFileOverSSL(const URLParser &url, const string &out_file, const string &token) { try { string cert_file_path; - if (!token.empty()) - { + if (!token.empty()) { cert_file_path = getConfigurationWithDefault( getFilesystemPathConfig() + "/certs/fog.pem", "message", @@ -581,11 +91,12 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s ); } - auto proxy_config = Singleton::Consume::by(); + auto proxy_config = Singleton::Consume::by(); + ofstream out_file_stream(out_file, ofstream::out | ofstream::binary); HttpsCurl ssl_curl_client( url, - out_file, + out_file_stream, token, proxy_config->getProxyDomain(ProxyProtocol::HTTPS), proxy_config->getProxyPort(ProxyProtocol::HTTPS), @@ -594,8 +105,7 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s ssl_curl_client.setCurlOpts(); bool connection_ok = ssl_curl_client.connect(); - if (!connection_ok) - { + if (!connection_ok) { stringstream url_s; url_s << url; string err_msg = string("Failed to get file over HTTPS. URL: ") + url_s.str(); @@ -603,12 +113,11 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s } } catch (const exception &e) { - dbgWarning(D_COMMUNICATION) << "Failed to get file over HTTPS. Error:" << string(e.what()); + dbgWarning(D_HTTP_REQUEST) << "Failed to get file over HTTPS. Error:" << string(e.what()); string error_str = "Failed to get file over HTTPS, exception: " + string(e.what()); return genError(error_str); } return Maybe(); } - // LCOV_EXCL_STOP diff --git a/components/security_apps/orchestration/downloader/https_client.h b/components/security_apps/orchestration/downloader/https_client.h new file mode 100644 index 0000000..b27b0a3 --- /dev/null +++ b/components/security_apps/orchestration/downloader/https_client.h @@ -0,0 +1,36 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __HTTPS_CLIENT_H__ +#define __HTTPS_CLIENT_H__ + +#include +#include "maybe_res.h" +#include "url_parser.h" +#include "orchestration_comp.h" + +// LCOV_EXCL_START Reason: Depends on real download server. +class HTTPSClient +{ +public: + Maybe getFile(const URLParser &url, const std::string &out_file, bool auth_required); + +private: + std::string loadCAChainDir(); + Maybe getFileSSL(const URLParser &url, const std::string &out_file, const std::string &_token); + Maybe getFileSSLDirect(const URLParser &url, const std::string &out_file, const std::string &_token); + Maybe curlGetFileOverSSL(const URLParser &url, const std::string &out_file, const std::string &_token); +}; +// LCOV_EXCL_STOP + +#endif // __HTTPS_CLIENT_H__ diff --git a/components/security_apps/orchestration/downloader/https_client_helper.cc b/components/security_apps/orchestration/downloader/https_client_helper.cc index 975054e..1f79489 100644 --- a/components/security_apps/orchestration/downloader/https_client_helper.cc +++ b/components/security_apps/orchestration/downloader/https_client_helper.cc @@ -11,10 +11,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "http_client.h" +#include "https_client.h" Maybe -HTTPClient::getFileSSLDirect(const URLParser &, std::ofstream &, const std::string &) +HTTPSClient::getFileSSLDirect(const URLParser &, const std::string &, const std::string &) { return genError("No direct downloading in open-source"); } diff --git a/components/security_apps/orchestration/env_details/env_details.cc b/components/security_apps/orchestration/env_details/env_details.cc index 10af97b..a33fa56 100644 --- a/components/security_apps/orchestration/env_details/env_details.cc +++ b/components/security_apps/orchestration/env_details/env_details.cc @@ -15,6 +15,7 @@ #include "config.h" #include "debug.h" +#include "orchestration_tools.h" using namespace std; @@ -22,10 +23,15 @@ USE_DEBUG_FLAG(D_LOCAL_POLICY); static const string k8s_service_account = "/var/run/secrets/kubernetes.io/serviceaccount"; // LCOV_EXCL_START Reason: can't use on the pipline environment -EnvDetails::EnvDetails() +EnvDetails::EnvDetails() : env_type(EnvType::LINUX) { + auto tools = Singleton::Consume::from(); + if (tools->doesFileExist("/.dockerenv")) env_type = EnvType::DOCKER; token = retrieveToken(); - token.empty() ? env_type = EnvType::LINUX : env_type = EnvType::K8S; + if (!token.empty()) { + auto env_res = getenv("deployment_type"); + env_type = env_res != nullptr && env_res == string("non_crd_k8s") ? EnvType::NON_CRD_K8S : EnvType::K8S; + } } EnvType diff --git a/components/security_apps/orchestration/include/fog_authenticator.h b/components/security_apps/orchestration/include/fog_authenticator.h index 16e448a..8ce7c1a 100755 --- a/components/security_apps/orchestration/include/fog_authenticator.h +++ b/components/security_apps/orchestration/include/fog_authenticator.h @@ -33,6 +33,7 @@ #include "i_rest_api.h" #include "i_time_get.h" #include "i_encryptor.h" +#include "i_shell_cmd.h" #include "maybe_res.h" class FogAuthenticator @@ -43,10 +44,13 @@ class FogAuthenticator Singleton::Consume, Singleton::Consume, Singleton::Consume, + Singleton::Consume, Singleton::Consume, Singleton::Consume, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume, + Singleton::Consume { class AccessToken { @@ -88,6 +92,8 @@ public: void serialize(cereal::JSONOutputArchive &out_ar) const; void serialize(cereal::JSONInputArchive &in_ar); + std::string getData() const; + private: AuthenticationType type; std::string data; @@ -103,6 +109,7 @@ public: Maybe authenticateAgent() override; void setAddressExtenesion(const std::string &extension) override; static std::string getUserEdition(); + void registerLocalAgentToFog() override; protected: class UserCredentials @@ -138,6 +145,7 @@ protected: bool saveCredentialsToFile(const UserCredentials &credentials) const; Maybe getCredentialsFromFile() const; + Maybe getRegistrationToken(); Maybe getRegistrationData(); std::string base64Encode(const std::string &in) const; diff --git a/components/security_apps/orchestration/include/hybrid_communication.h b/components/security_apps/orchestration/include/hybrid_communication.h index 407a1ad..46199a8 100755 --- a/components/security_apps/orchestration/include/hybrid_communication.h +++ b/components/security_apps/orchestration/include/hybrid_communication.h @@ -37,10 +37,7 @@ #include "maybe_res.h" #include "declarative_policy_utils.h" -class HybridCommunication - : - public FogAuthenticator, - Singleton::Consume +class HybridCommunication : public FogAuthenticator { public: void init() override; diff --git a/components/security_apps/orchestration/include/local_communication.h b/components/security_apps/orchestration/include/local_communication.h index 189a90f..056a1d4 100755 --- a/components/security_apps/orchestration/include/local_communication.h +++ b/components/security_apps/orchestration/include/local_communication.h @@ -29,6 +29,7 @@ public: void init(); Maybe authenticateAgent() override; + void registerLocalAgentToFog() override; Maybe getUpdate(CheckUpdateRequest &request) override; Maybe downloadAttributeFile( diff --git a/components/security_apps/orchestration/include/mock/mock_details_resolver.h b/components/security_apps/orchestration/include/mock/mock_details_resolver.h index d08304d..72d946d 100644 --- a/components/security_apps/orchestration/include/mock/mock_details_resolver.h +++ b/components/security_apps/orchestration/include/mock/mock_details_resolver.h @@ -35,6 +35,7 @@ public: MOCK_METHOD0(getPlatform, Maybe()); MOCK_METHOD0(getArch, Maybe()); MOCK_METHOD0(getAgentVersion, std::string()); + MOCK_METHOD0(isCloudStorageEnabled, bool()); MOCK_METHOD0(isReverseProxy, bool()); MOCK_METHOD0(isKernelVersion3OrHigher, bool()); MOCK_METHOD0(isGwNotVsx, bool()); diff --git a/components/security_apps/orchestration/include/mock/mock_downloader.h b/components/security_apps/orchestration/include/mock/mock_downloader.h index 7b3fa94..4e9baa6 100755 --- a/components/security_apps/orchestration/include/mock/mock_downloader.h +++ b/components/security_apps/orchestration/include/mock/mock_downloader.h @@ -24,7 +24,7 @@ class MockDownloader : { public: MOCK_CONST_METHOD3( - downloadFileFromFog, + downloadFile, Maybe(const std::string &, Package::ChecksumTypes, const GetResourceFile &) ); diff --git a/components/security_apps/orchestration/include/mock/mock_update_communication.h b/components/security_apps/orchestration/include/mock/mock_update_communication.h index c5e9649..382720c 100755 --- a/components/security_apps/orchestration/include/mock/mock_update_communication.h +++ b/components/security_apps/orchestration/include/mock/mock_update_communication.h @@ -29,6 +29,7 @@ class MockUpdateCommunication : public: void init() {} MOCK_METHOD0(authenticateAgent, Maybe()); + MOCK_METHOD0(registerLocalAgentToFog, void()); MOCK_METHOD1(getUpdate, Maybe(CheckUpdateRequest &)); MOCK_METHOD2( downloadAttributeFile, diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 8a93433..cbf8455 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -310,7 +310,7 @@ private: dbgInfo(D_ORCHESTRATOR) << "There is a new manifest file."; GetResourceFile resource_file(GetResourceFile::ResourceFileType::MANIFEST); Maybe new_manifest_file = - Singleton::Consume::by()->downloadFileFromFog( + Singleton::Consume::by()->downloadFile( orch_manifest.unpack(), I_OrchestrationTools::SELECTED_CHECKSUM_TYPE, resource_file @@ -394,7 +394,6 @@ private: if (restart_watchdog_orch.good()) { ofstream restart_watchdog("/tmp/restart_watchdog", ofstream::out); restart_watchdog.close(); - remove((filesystem_prefix + "/orchestration/restart_watchdog").c_str()); restart_watchdog_orch.close(); } @@ -501,7 +500,7 @@ private: dbgInfo(D_ORCHESTRATOR) << "There is a new policy file."; GetResourceFile resource_file(GetResourceFile::ResourceFileType::POLICY); Maybe new_policy_file = - Singleton::Consume::by()->downloadFileFromFog( + Singleton::Consume::by()->downloadFile( new_policy.unpack(), I_OrchestrationTools::SELECTED_CHECKSUM_TYPE, resource_file @@ -677,7 +676,7 @@ private: "Data file path" ); GetResourceFile resource_file(GetResourceFile::ResourceFileType::DATA); - Maybe new_data_files = Singleton::Consume::by()->downloadFileFromFog( + Maybe new_data_files = Singleton::Consume::by()->downloadFile( orch_data.unpack(), I_OrchestrationTools::SELECTED_CHECKSUM_TYPE, resource_file @@ -753,7 +752,7 @@ private: dbgInfo(D_ORCHESTRATOR) << "There is a new settings file."; GetResourceFile resource_file(GetResourceFile::ResourceFileType::SETTINGS); Maybe new_settings_file = - Singleton::Consume::by()->downloadFileFromFog( + Singleton::Consume::by()->downloadFile( orch_settings.unpack(), I_OrchestrationTools::SELECTED_CHECKSUM_TYPE, resource_file @@ -1330,6 +1329,12 @@ private: agent_data_report << AgentReportFieldWithLabel("reverse_proxy", "true"); } + if (i_details_resolver->isCloudStorageEnabled()) { + agent_data_report << AgentReportFieldWithLabel("cloud_storage_service", "true"); + } else { + agent_data_report << AgentReportFieldWithLabel("cloud_storage_service", "false"); + } + if (i_details_resolver->isKernelVersion3OrHigher()) { agent_data_report << AgentReportFieldWithLabel("isKernelVersion3OrHigher", "true"); } @@ -1417,7 +1422,7 @@ private: is_new_success = false; sleep_interval = calcSleepInterval(policy.getErrorSleepInterval()); dbgWarning(D_ORCHESTRATOR) - << "Failed during check update from Fog. Error: " + << "Failed during check update. Error: " << check_update_result.getErr() << ", new check will be every: " << sleep_interval << " seconds"; @@ -1425,7 +1430,7 @@ private: health_check_status_listener.setStatus( HealthCheckStatus::UNHEALTHY, OrchestrationStatusFieldType::LAST_UPDATE, - "Failed during check update from Fog. Error: " + check_update_result.getErr() + "Failed during check update. Error: " + check_update_result.getErr() ); return; } @@ -1507,12 +1512,24 @@ private: << LogField("agentType", "Orchestration") << LogField("agentVersion", Version::get()); - Singleton::Consume::by()->addOneTimeRoutine( + auto mainloop = Singleton::Consume::by(); + mainloop->addOneTimeRoutine( I_MainLoop::RoutineType::Offline, sendRegistrationData, "Send registration data" ); + if (getOrchestrationMode() == OrchestrationMode::HYBRID) { + Singleton::Consume::by()->addRecurringRoutine( + I_MainLoop::RoutineType::Offline, + chrono::seconds(60), + [&] () { + Singleton::Consume::by()->registerLocalAgentToFog(); + }, + "Check For Environment Registration Token" + ); + } + reportAgentDetailsMetaData(); if (!Singleton::Consume::by()->loadAfterSelfUpdate()) { @@ -1557,6 +1574,11 @@ private: tags.insert(Tags::DEPLOYMENT_EMBEDDED); break; } + case EnvType::DOCKER: { + tags.insert(Tags::DEPLOYMENT_DOCKER); + break; + } + case EnvType::NON_CRD_K8S: case EnvType::K8S: { tags.insert(Tags::DEPLOYMENT_K8S); break; @@ -1760,7 +1782,7 @@ private: string orchestration_mode = getConfigurationFlag("orchestration-mode"); if ( orchestration_mode == "online_mode" || - orchestration_mode == "hybrid_mode" || + orchestration_mode == "hybrid_mode" || orchestration_mode == "offline_mode" ) { dbgTrace(D_ORCHESTRATOR) << "Orchestraion mode: " << orchestration_mode; diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc index fcd314e..e2cd4a0 100644 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc @@ -39,6 +39,15 @@ Maybe response( string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy"; string orchestration_policy_file_path_bk = orchestration_policy_file_path + ".bk"; +class ExpectInitializer +{ +public: + ExpectInitializer(StrictMock &mock_orchestration_tools) + { + EXPECT_CALL(mock_orchestration_tools, doesFileExist("/.dockerenv")).WillRepeatedly(Return(false)); + } +}; + class OrchestrationMultitenancyTest : public Test { public: @@ -200,6 +209,7 @@ public: NiceMock mock_agent_reporter; NiceMock mock_log; + ExpectInitializer initializer{mock_orchestration_tools}; OrchestrationComp orchestration_comp; private: diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc index 4a2034d..316fb34 100755 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc @@ -22,6 +22,7 @@ #include "agent_details.h" #include "customized_cereal_map.h" #include "health_check_status/health_check_status.h" +#include "declarative_policy_utils.h" using namespace testing; using namespace std; @@ -41,6 +42,15 @@ Maybe response( string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy"; string orchestration_policy_file_path_bk = orchestration_policy_file_path + ".bk"; +class ExpectInitializer +{ +public: + ExpectInitializer(StrictMock &mock_orchestration_tools) + { + EXPECT_CALL(mock_orchestration_tools, doesFileExist("/.dockerenv")).WillRepeatedly(Return(false)); + } +}; + class OrchestrationTest : public testing::TestWithParam { public: @@ -150,6 +160,7 @@ public: EXPECT_CALL(mock_details_resolver, getPlatform()).WillRepeatedly(Return(string("linux"))); EXPECT_CALL(mock_details_resolver, getArch()).WillRepeatedly(Return(string("x86_64"))); EXPECT_CALL(mock_details_resolver, isReverseProxy()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, isCloudStorageEnabled()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isKernelVersion3OrHigher()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isGwNotVsx()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isVersionAboveR8110()).WillRepeatedly(Return(false)); @@ -244,6 +255,17 @@ public: routine(); } + void + runRegTokenRoutine() + { + EXPECT_CALL( + mock_ml, + addRecurringRoutine(I_MainLoop::RoutineType::Offline, _, _, "Check For Environment Registration Token", _) + ).WillOnce(DoAll(SaveArg<2>(®_token_routine), Return(0))); + + routine(); + } + void runStatusRoutine() { @@ -303,9 +325,12 @@ public: StrictMock mock_details_resolver; NiceMock mock_agent_reporter; NiceMock tenant_manager; + ExpectInitializer initializer{mock_orchestration_tools}; OrchestrationComp orchestration_comp; + DeclarativePolicyUtils declarative_policy_utils; AgentDetails agent_details; I_MainLoop::Routine sending_routine; + I_MainLoop::Routine reg_token_routine; string message_body; private: @@ -326,6 +351,48 @@ private: I_MainLoop::Routine status_routine; }; + +TEST_F(OrchestrationTest, hybridModeRegisterLocalAgentRoutine) +{ + EXPECT_CALL(rest, mockRestCall(_, _, _)).WillRepeatedly(Return(true)); + Singleton::Consume::from(config_comp)->loadConfiguration( + vector{"--orchestration-mode=hybrid_mode"} + ); + + preload(); + env.init(); + init(); + + EXPECT_CALL(mock_service_controller, updateServiceConfiguration(_, _, _, _, _, _)) + .WillOnce(Return(Maybe())); + EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, _)).WillRepeatedly(Return(string())); + EXPECT_CALL(mock_service_controller, getPolicyVersion()).WillRepeatedly(ReturnRef(first_policy_version)); + EXPECT_CALL(mock_shell_cmd, getExecOutput(_, _, _)).WillRepeatedly(Return(string())); + EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); + EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); + expectDetailsResolver(); + EXPECT_CALL(mock_update_communication, getUpdate(_)); + EXPECT_CALL(mock_status, setLastUpdateAttempt()); + EXPECT_CALL(mock_status, setFieldStatus(_, _, _)); + EXPECT_CALL(mock_status, setIsConfigurationUpdated(_)); + + EXPECT_CALL(mock_ml, yield(A())) + .WillOnce(Return()) + .WillOnce(Invoke([] (chrono::microseconds) { throw invalid_argument("stop while loop"); })); + + EXPECT_CALL( + mock_ml, + addOneTimeRoutine(I_MainLoop::RoutineType::Offline, _, "Send registration data", false) + ).WillOnce(Return(1)); + + try { + runRegTokenRoutine(); + } catch (const invalid_argument& e) {} + + EXPECT_CALL(mock_update_communication, registerLocalAgentToFog()); + reg_token_routine(); +} + TEST_F(OrchestrationTest, testAgentUninstallRest) { EXPECT_CALL( @@ -671,7 +738,7 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdatRollback) GetResourceFile policy_file(GetResourceFile::ResourceFileType::POLICY); EXPECT_CALL( mock_downloader, - downloadFileFromFog(new_policy_checksum, Package::ChecksumTypes::SHA256, policy_file) + downloadFile(new_policy_checksum, Package::ChecksumTypes::SHA256, policy_file) ).WillOnce(Return(Maybe(new_policy_path))); vector expected_data_types = {}; @@ -850,7 +917,7 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate) GetResourceFile policy_file(GetResourceFile::ResourceFileType::POLICY); EXPECT_CALL( mock_downloader, - downloadFileFromFog(new_policy_checksum, Package::ChecksumTypes::SHA256, policy_file) + downloadFile(new_policy_checksum, Package::ChecksumTypes::SHA256, policy_file) ).WillOnce(Return(Maybe(new_policy_path))); vector expected_data_types = {}; @@ -1060,7 +1127,7 @@ TEST_F(OrchestrationTest, manifestUpdate) GetResourceFile manifest_file(GetResourceFile::ResourceFileType::MANIFEST); EXPECT_CALL(mock_downloader, - downloadFileFromFog( + downloadFile( string("new check sum"), Package::ChecksumTypes::SHA256, manifest_file @@ -1147,7 +1214,7 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) GetResourceFile policy_file(GetResourceFile::ResourceFileType::POLICY); EXPECT_CALL( mock_downloader, - downloadFileFromFog( + downloadFile( string("111111"), Package::ChecksumTypes::SHA256, policy_file @@ -1324,14 +1391,14 @@ TEST_F(OrchestrationTest, failedDownloadSettings) GetResourceFile manifest_file(GetResourceFile::ResourceFileType::MANIFEST); EXPECT_CALL(mock_downloader, - downloadFileFromFog( + downloadFile( string("manifest-checksum"), Package::ChecksumTypes::SHA256, manifest_file ) ).WillOnce(Return(download_error)); EXPECT_CALL(mock_downloader, - downloadFileFromFog( + downloadFile( string("settings-checksum"), Package::ChecksumTypes::SHA256, settings_file @@ -1690,7 +1757,7 @@ TEST_F(OrchestrationTest, dataUpdate) string new_data_file_path = data_file_path + ".download"; GetResourceFile data_file(GetResourceFile::ResourceFileType::DATA); EXPECT_CALL(mock_downloader, - downloadFileFromFog( + downloadFile( string("new data"), Package::ChecksumTypes::SHA256, data_file diff --git a/components/security_apps/orchestration/service_controller/service_controller.cc b/components/security_apps/orchestration/service_controller/service_controller.cc index 9457c86..9cca9f4 100755 --- a/components/security_apps/orchestration/service_controller/service_controller.cc +++ b/components/security_apps/orchestration/service_controller/service_controller.cc @@ -702,26 +702,6 @@ ServiceController::Impl::createDirectoryForChildTenant( return true; } -static string -getChecksum(const string &file_path) -{ - auto orchestration_tools = Singleton::Consume::by(); - Maybe file_checksum = orchestration_tools->calculateChecksum( - Package::ChecksumTypes::MD5, - file_path - ); - - if (file_checksum.ok()) return file_checksum.unpack(); - - string checksum = "unknown version"; - try { - checksum = to_string(boost::uuids::random_generator()()); - } catch (const boost::uuids::entropy_error &e) { - dbgDebug(D_SERVICE_CONTROLLER) << "Couldn't generate random checksum"; - } - return checksum; -} - Maybe ServiceController::Impl::updateServiceConfiguration( const string &new_policy_path, @@ -827,7 +807,7 @@ ServiceController::Impl::updateServiceConfiguration( if (child_tenant_id.empty() && single_policy.first == versions_param) { //In a multi-tenant env, only the parent should handle the versions parameter policy_versions = single_policy.second; - dbgWarning(D_SERVICE_CONTROLLER) << "Found versions parameter in policy file:" << policy_versions; + dbgTrace(D_SERVICE_CONTROLLER) << "Found versions parameter in policy file:" << policy_versions; } dbgDebug(D_SERVICE_CONTROLLER) << "Starting to update policy file. Policy type: " << single_policy.first; @@ -901,7 +881,7 @@ ServiceController::Impl::updateServiceConfiguration( // In a multi-tenant env, we send the signal to the services only on the last iteration if (!is_multi_tenant_env || last_iteration) { auto is_send_signal_for_services = - sendSignalForServices(nano_services_to_update, version_value + ',' + getChecksum(new_policy_path)); + sendSignalForServices(nano_services_to_update, version_value + ',' + policy_versions); was_policy_updated &= is_send_signal_for_services.ok(); if (!is_send_signal_for_services.ok()) send_signal_for_services_err = is_send_signal_for_services.getErr(); } @@ -950,7 +930,7 @@ ServiceController::Impl::sendSignalForServices( const set &nano_services_to_update, const string &policy_version_to_update) { - dbgFlow(D_SERVICE_CONTROLLER); + dbgFlow(D_SERVICE_CONTROLLER) << "Policy version to update: " << policy_version_to_update; for (auto &service_id : nano_services_to_update) { auto nano_service = registered_services.find(service_id); if (nano_service == registered_services.end()) { diff --git a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc index 2effcaa..5b50565 100755 --- a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc +++ b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc @@ -98,9 +98,12 @@ public: "orchestration", "Settings file path" ); + } + void + init() + { service_controller.init(); - registerNewService(); } @@ -173,14 +176,14 @@ public: } void - expectNewConfigRequest(const string &req_body, const string &response) + expectNewConfigRequest(const string &response) { EXPECT_CALL( mock_message, sendSyncMessage( HTTPMethod::POST, "/set-new-configuration", - req_body, + HasSubstr("1.0.2"), _, _ ) @@ -224,16 +227,37 @@ public: ostringstream capture_debug; string version_value = "1.0.2"; string old_version = "1.0.1"; + + string versions = + "[" + " {" + " \"id\": \"d8c3cc3c-f9df-83c8-f875-322dd8a0c161\"," + " \"name\": \"Linux Embedded Agents\"," + " \"version\": \"1.0.2\"" + " }" + "]"; + string old_versions = + "[" + " {" + " \"id\": \"d8c3cc3c-f9df-83c8-f875-322dd8a0c161\"," + " \"name\": \"Linux Embedded Agents\"," + " \"version\": \"1.0.1\"" + " }" + "]"; + }; TEST_F(ServiceControllerTest, doNothing) { + init(); } TEST_F(ServiceControllerTest, UpdateConfiguration) { + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -271,7 +295,13 @@ TEST_F(ServiceControllerTest, UpdateConfiguration) "}"; Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration)); EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) .WillOnce(Return(json_parser_return)); @@ -284,9 +314,6 @@ TEST_F(ServiceControllerTest, UpdateConfiguration) EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); EXPECT_EQ(i_service_controller->getPolicyVersions(), ""); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -295,7 +322,7 @@ TEST_F(ServiceControllerTest, UpdateConfiguration) string general_settings_path = "/my/settings/path"; string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; - expectNewConfigRequest("{\n \"id\": 1,\n \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg); + expectNewConfigRequest(reply_msg); EXPECT_CALL( mock_shell_cmd, @@ -309,25 +336,13 @@ TEST_F(ServiceControllerTest, UpdateConfiguration) EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path).ok()); EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value); - EXPECT_EQ(i_service_controller->getPolicyVersions(), ""); + EXPECT_EQ(i_service_controller->getPolicyVersions(), versions); EXPECT_EQ(i_service_controller->getUpdatePolicyVersion(), version_value); } TEST_F(ServiceControllerTest, supportVersions) { - string versions = "[" - " {" - " \"id\" : \"40c4a460-eb24-f002-decb-f4a7f00423fc\"," - " \"name\" : \"Linux Embedded Agents\"," - " \"version\" : 1" - " }," - " {" - " \"id\" : \"93788960-6969-11ee-be56-0242ac120002\"," - " \"name\" : \"Linux SUPER Embedded Agents\"," - " \"version\" : 420" - " }" - "]"; - + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" " \"versions\": " + versions + @@ -386,9 +401,6 @@ TEST_F(ServiceControllerTest, supportVersions) EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); EXPECT_EQ(i_service_controller->getPolicyVersions(), ""); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -397,7 +409,7 @@ TEST_F(ServiceControllerTest, supportVersions) string general_settings_path = "/my/settings/path"; string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; - expectNewConfigRequest("{\n \"id\": 1,\n \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg); + expectNewConfigRequest(reply_msg); EXPECT_CALL( mock_shell_cmd, @@ -417,8 +429,10 @@ TEST_F(ServiceControllerTest, supportVersions) TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration) { + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -456,7 +470,13 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration) "}"; Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration)); EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) .WillOnce(Return(json_parser_return)); @@ -468,9 +488,6 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration) EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -500,7 +517,7 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration) string general_settings_path = "/my/settings/path"; string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; - expectNewConfigRequest("{\n \"id\": 1,\n \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg); + expectNewConfigRequest(reply_msg); EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path).ok()); EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value); @@ -509,6 +526,7 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration) TEST_F(ServiceControllerTest, readRegisteredServicesFromFile) { + init(); int family1_id3_port = 1111; string registered_services_json = "{\n" " \"Registered Services\": {\n" @@ -560,8 +578,10 @@ TEST_F(ServiceControllerTest, readRegisteredServicesFromFile) TEST_F(ServiceControllerTest, noPolicyUpdate) { + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -599,7 +619,13 @@ TEST_F(ServiceControllerTest, noPolicyUpdate) "}"; Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration)); EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) .WillOnce(Return(json_parser_return)); @@ -612,11 +638,8 @@ TEST_F(ServiceControllerTest, noPolicyUpdate) EXPECT_CALL(mock_orchestration_status, setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; - expectNewConfigRequest("{\n \"id\": 1,\n \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg); + expectNewConfigRequest(reply_msg); EXPECT_CALL( mock_shell_cmd, @@ -634,8 +657,10 @@ TEST_F(ServiceControllerTest, noPolicyUpdate) TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations) { + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -673,7 +698,12 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations) "}"; Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration)); EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) .WillOnce(Return(json_parser_return)); @@ -685,9 +715,6 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations) EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -705,7 +732,7 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations) string general_settings_path = "/my/settings/path"; string reply_msg1 = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; - expectNewConfigRequest("{\n \"id\": 1,\n \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg1); + expectNewConfigRequest(reply_msg1); // both policy and settings now being updated EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path).ok()); @@ -725,13 +752,15 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations) EXPECT_CALL(mock_orchestration_status, setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); general_settings_path += "/new"; string reply_msg2 = "{\"id\": 2, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; - expectNewConfigRequest("{\n \"id\": 2,\n \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg2); + expectNewConfigRequest(reply_msg2); EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path).ok()); EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value); @@ -739,8 +768,10 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations) TEST_F(ServiceControllerTest, backup) { + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -779,6 +810,7 @@ TEST_F(ServiceControllerTest, backup) string old_configuration = "{" " \"version\": \"" + old_version + "\"" + " \"versions\": \"" + old_versions + "\"" " \"app\": \"netfilter\"," " \"l4_firewall_rules\": [" " {" @@ -796,7 +828,13 @@ TEST_F(ServiceControllerTest, backup) "}"; Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration)); EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) .WillOnce(Return(json_parser_return)); @@ -805,9 +843,6 @@ TEST_F(ServiceControllerTest, backup) EXPECT_CALL(mock_orchestration_status, setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - EXPECT_CALL( mock_orchestration_tools, copyFile(l4_firewall_policy_path, l4_firewall_policy_path + backup_extension) @@ -842,8 +877,10 @@ TEST_F(ServiceControllerTest, backup) TEST_F(ServiceControllerTest, backup_file_doesnt_exist) { + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -882,6 +919,7 @@ TEST_F(ServiceControllerTest, backup_file_doesnt_exist) string old_configuration = "{" " \"version\": \"" + old_version + "\"" + " \"versions\": \"" + old_versions + "\"" " \"app\": \"netfilter\"," " \"l4_firewall_rules\": [" " {" @@ -899,7 +937,14 @@ TEST_F(ServiceControllerTest, backup_file_doesnt_exist) "}"; Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); + + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration)); EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) .WillOnce(Return(json_parser_return)); @@ -908,9 +953,6 @@ TEST_F(ServiceControllerTest, backup_file_doesnt_exist) EXPECT_CALL(mock_orchestration_status, setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - EXPECT_CALL( mock_orchestration_tools, copyFile(l4_firewall_policy_path, l4_firewall_policy_path + backup_extension) @@ -937,7 +979,7 @@ TEST_F(ServiceControllerTest, backup_file_doesnt_exist) ).WillRepeatedly(Return(string("registered and running"))); string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; - expectNewConfigRequest("{\n \"id\": 1,\n \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg); + expectNewConfigRequest(reply_msg); EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, "").ok()); @@ -946,8 +988,10 @@ TEST_F(ServiceControllerTest, backup_file_doesnt_exist) TEST_F(ServiceControllerTest, backupAttempts) { + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -986,6 +1030,7 @@ TEST_F(ServiceControllerTest, backupAttempts) string old_configuration = "{" " \"version\": \"" + old_version + "\"" + " \"versions\": \"" + old_versions + "\"" " \"app\": \"netfilter\"," " \"l4_firewall_rules\": [" " {" @@ -1003,7 +1048,14 @@ TEST_F(ServiceControllerTest, backupAttempts) "}"; Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); + + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration)); EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) .WillOnce(Return(json_parser_return)); @@ -1012,9 +1064,6 @@ TEST_F(ServiceControllerTest, backupAttempts) EXPECT_CALL(mock_orchestration_status, setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - EXPECT_CALL( mock_orchestration_tools, copyFile(l4_firewall_policy_path, l4_firewall_policy_path + backup_extension) @@ -1041,7 +1090,7 @@ TEST_F(ServiceControllerTest, backupAttempts) ).WillRepeatedly(Return(string("registered and running"))); string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; - expectNewConfigRequest("{\n \"id\": 1,\n \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg); + expectNewConfigRequest(reply_msg); EXPECT_CALL(mock_ml, yield(false)).Times(2); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -1053,8 +1102,10 @@ TEST_F(ServiceControllerTest, backupAttempts) TEST_F(ServiceControllerTest, MultiUpdateConfiguration) { + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -1110,8 +1161,16 @@ TEST_F(ServiceControllerTest, MultiUpdateConfiguration) Maybe> json_parser_return = map({ {"version", version_value}, {"l4_firewall", l4_firewall}, - {"orchestration", orchestration} + {"orchestration", orchestration}, + {"versions", versions} }); + + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + string orchestration_policy_path = configuration_dir + "/orchestration/orchestration" + policy_extension; string orchestration_settings_path = configuration_dir + "/orchestration/orchestration" + settings_extension; @@ -1125,9 +1184,6 @@ TEST_F(ServiceControllerTest, MultiUpdateConfiguration) EXPECT_CALL(mock_orchestration_status, setServiceConfiguration("orchestration", orchestration_policy_path, OrchestrationStatusConfigType::POLICY)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path, false)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, writeFile(orchestration, orchestration_policy_path, false)) @@ -1148,12 +1204,13 @@ TEST_F(ServiceControllerTest, MultiUpdateConfiguration) ).WillRepeatedly(Return(string("registered and running"))); string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; - expectNewConfigRequest("{\n \"id\": 1,\n \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg); + expectNewConfigRequest(reply_msg); EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, "").ok()); set changed_policies = { "/etc/cp/conf/l4_firewall/l4_firewall.policy", - "/etc/cp/conf/orchestration/orchestration.policy" + "/etc/cp/conf/orchestration/orchestration.policy", + policy_versions_path }; EXPECT_EQ(i_service_controller->moveChangedPolicies(), changed_policies); } @@ -1166,6 +1223,7 @@ public: TEST_F(ServiceControllerTest, badJsonFile) { + init(); Maybe err = genError("Error"); EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).Times(1).WillRepeatedly(Return(err)); EXPECT_FALSE(i_service_controller->updateServiceConfiguration(file_name, "").ok()); @@ -1173,6 +1231,7 @@ TEST_F(ServiceControllerTest, badJsonFile) TEST_F(ServiceControllerTest, emptyServices) { + init(); Maybe> json_parser_return = map(); string empty_string = ""; EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).Times(1).WillRepeatedly(Return(empty_string)); @@ -1185,16 +1244,15 @@ TEST_F(ServiceControllerTest, emptyServices) EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, "").ok()); } TEST_F(ServiceControllerTest, failingWhileLoadingCurrentConfiguration) { + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -1232,23 +1290,31 @@ TEST_F(ServiceControllerTest, failingWhileLoadingCurrentConfiguration) "}"; Maybe err = genError("Error"); + Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); + + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration)); EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) .WillOnce(Return(json_parser_return)); EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(err)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); EXPECT_FALSE(i_service_controller->updateServiceConfiguration(file_name, "").ok()); } TEST_F(ServiceControllerTest, failingWhileCopyingCurrentConfiguration) { + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -1287,6 +1353,7 @@ TEST_F(ServiceControllerTest, failingWhileCopyingCurrentConfiguration) string old_configuration = "{" " \"version\": \"" + old_version + "\"" + " \"versions\": \"" + old_versions + "\"" " \"app\": \"netfilter\"," " \"l4_firewall_rules\": [" " {" @@ -1304,15 +1371,20 @@ TEST_F(ServiceControllerTest, failingWhileCopyingCurrentConfiguration) "}"; Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); + + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).Times(1).WillRepeatedly(Return(new_configuration)); EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)).Times(1).WillRepeatedly( Return(json_parser_return) ); EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(old_configuration)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); EXPECT_CALL( mock_orchestration_tools, copyFile(l4_firewall_policy_path, l4_firewall_policy_path + backup_extension) @@ -1325,10 +1397,12 @@ TEST_F(ServiceControllerTest, failingWhileCopyingCurrentConfiguration) TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest) { + init(); Debug::setUnitTestFlag(D_SERVICE_CONTROLLER, Debug::DebugLevel::NOISE); Debug::setNewDefaultStdout(&capture_debug); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -1368,7 +1442,14 @@ TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest) EXPECT_CALL(time, getWalltime()).WillRepeatedly(Return(chrono::microseconds(0))); Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); + + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration)); EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) .WillOnce(Return(json_parser_return)); @@ -1384,9 +1465,6 @@ TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest) setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY) ); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); EXPECT_TRUE(i_service_controller->isServiceInstalled("family1_id2")); @@ -1417,8 +1495,10 @@ TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest) TEST_F(ServiceControllerTest, errorWhileWrtingNewConfiguration) { + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -1457,6 +1537,7 @@ TEST_F(ServiceControllerTest, errorWhileWrtingNewConfiguration) string old_configuration = "{" " \"version\": \"" + old_version + "\"" + " \"versions\": \"" + old_versions + "\"" " \"app\": \"netfilter\"," " \"l4_firewall_rules\": [" " {" @@ -1474,15 +1555,20 @@ TEST_F(ServiceControllerTest, errorWhileWrtingNewConfiguration) "}"; Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); + + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).Times(1).WillRepeatedly(Return(new_configuration)); EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)).Times(1).WillRepeatedly( Return(json_parser_return) ); EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(old_configuration)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); EXPECT_CALL( mock_orchestration_tools, copyFile(l4_firewall_policy_path, l4_firewall_policy_path + backup_extension) @@ -1498,6 +1584,7 @@ TEST_F(ServiceControllerTest, errorWhileWrtingNewConfiguration) TEST_F(ServiceControllerTest, testPortsRest) { + init(); stringstream empty_json; empty_json << "{}"; auto res = get_services_ports->performRestCall(empty_json); @@ -1507,7 +1594,12 @@ TEST_F(ServiceControllerTest, testPortsRest) TEST_F(ServiceControllerTest, testMultitenantConfFiles) { + setSetting("VirtualNSaaS", "agentType"); + init(); + map, pair> tenant_files_input = { + {make_pair("", ""), + make_pair("/etc/cp/conf/policy.json", "")}, {make_pair("tenant1", "1234"), make_pair("/etc/cp/conf/tenant1_profile_1234_policy.json", "/etc/cp/conf/tenant1_profile_1234_settings.json")}, {make_pair("tenant2", "1235"), @@ -1517,11 +1609,11 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles) set ids = {"family1_id2"}; set empty_ids; - EXPECT_CALL(tenant_manager, getInstances("tenant1", "1234")).WillOnce(Return(ids)); - EXPECT_CALL(tenant_manager, getInstances("tenant2", "1235")).WillOnce(Return(empty_ids)); + EXPECT_CALL(tenant_manager, getInstances("tenant1", "1234")).WillRepeatedly(Return(ids)); + EXPECT_CALL(tenant_manager, getInstances("tenant2", "1235")).WillRepeatedly(Return(empty_ids)); string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; - expectNewConfigRequest("{\n \"id\": 1,\n \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg); + expectNewConfigRequest(reply_msg); for(auto entry : tenant_files_input) { auto tenant = entry.first.first; @@ -1532,6 +1624,7 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles) string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": \"" + versions + "\"" " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -1568,44 +1661,72 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles) " ]" "}"; - string l4_firewall_policy_path_new = - configuration_dir + "/tenant_" + tenant + - "_profile_" + profile +"/l4_firewall/l4_firewall" + policy_extension; Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); - + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); EXPECT_CALL(mock_orchestration_tools, readFile(conf_file_name)).WillOnce(Return(new_configuration)); - EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, tenant, profile)) .WillOnce(Return(json_parser_return)); + if (!tenant.empty()) { + string l4_firewall_policy_path_new = + configuration_dir + "/tenant_" + tenant + + "_profile_" + profile +"/l4_firewall/l4_firewall" + policy_extension; + string policy_versions_path_new = + configuration_dir + "/tenant_" + tenant + + "_profile_" + profile +"/versions/versions" + policy_extension; - EXPECT_CALL( - mock_orchestration_tools, - doesDirectoryExist(configuration_dir + "/tenant_" + tenant + "_profile_" + profile) - ).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path_new)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path_new, false)) + .WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path_new, OrchestrationStatusConfigType::POLICY)); - EXPECT_CALL( - mock_orchestration_tools, - createDirectory(configuration_dir + "/tenant_" + tenant + "_profile_" + profile) - ).WillOnce(Return(true)); + EXPECT_CALL( + mock_orchestration_tools, + doesDirectoryExist(configuration_dir + "/tenant_" + tenant + "_profile_" + profile) + ).WillOnce(Return(false)).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path_new)).WillOnce(Return(false)); + EXPECT_CALL( + mock_orchestration_tools, + createDirectory(configuration_dir + "/tenant_" + tenant + "_profile_" + profile) + ).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path_new, false)) - .WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path_new)).WillOnce(Return(false)); - EXPECT_CALL(mock_orchestration_status, setServiceConfiguration( - "l4_firewall", l4_firewall_policy_path_new, OrchestrationStatusConfigType::POLICY) - ); + EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path_new, false)) + .WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, conf_file_name)) - .WillRepeatedly(Return(version_value)); + string new_policy_file_path = + "/etc/cp/conf/tenant_" + tenant + "_profile_" + profile + "/" + "policy.json"; + EXPECT_CALL( + mock_orchestration_tools, + copyFile(new_policy_file_path, new_policy_file_path + backup_extension) + ).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, copyFile(conf_file_name, new_policy_file_path)) + .WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(new_policy_file_path)).WillOnce(Return(true)); - string new_policy_file_path = "/etc/cp/conf/tenant_" + tenant + "_profile_" + profile + "/" + "policy.json"; - EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_file_path, new_policy_file_path + backup_extension)) - .WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, copyFile(conf_file_name, new_policy_file_path)).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, doesFileExist(new_policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, setServiceConfiguration( + "l4_firewall", l4_firewall_policy_path_new, OrchestrationStatusConfigType::POLICY) + ); + } else { + EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path, false)). + WillOnce(Return(true)); + EXPECT_CALL( + mock_orchestration_status, + setServiceConfiguration( + "l4_firewall", + l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY + ) + ); + + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL( + mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + } EXPECT_CALL( mock_shell_cmd, @@ -1623,7 +1744,8 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles) settings_file_name, {}, tenant, - profile + profile, + tenant.empty() ).ok() ); } @@ -1631,6 +1753,7 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles) TEST_F(ServiceControllerTest, cleanup_virtual_files) { + init(); string agent_tenants_files = "111111\n" "222222\n" @@ -1653,9 +1776,11 @@ TEST_F(ServiceControllerTest, cleanup_virtual_files) TEST_F(ServiceControllerTest, test_delayed_reconf) { + init(); string new_configuration = "{" " \"version\": \"" + version_value + "\"" + " \"versions\": " + versions + " \"l4_firewall\":" " {" " \"app\": \"netfilter\"," @@ -1696,7 +1821,14 @@ TEST_F(ServiceControllerTest, test_delayed_reconf) setConfiguration(60, "orchestration", "Reconfiguration timeout seconds"); Maybe> json_parser_return = - map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + map({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}}); + + string policy_versions_path = "/etc/cp/conf/versions/versions.policy"; + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY)); + EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration)); EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) .WillOnce(Return(json_parser_return)); @@ -1706,9 +1838,6 @@ TEST_F(ServiceControllerTest, test_delayed_reconf) EXPECT_CALL(mock_orchestration_status, setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY)); - EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name)) - .WillOnce(Return(version_value)); - EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); @@ -1737,7 +1866,7 @@ TEST_F(ServiceControllerTest, test_delayed_reconf) << " \"error_message\": \"\"" << "}"; - expectNewConfigRequest("{\n \"id\": 1,\n \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg); + expectNewConfigRequest(reply_msg); auto func = [&] (chrono::microseconds) { set_reconf_status->performRestCall(reconf_status); }; EXPECT_CALL(mock_ml, yield(chrono::microseconds(2000000))).WillOnce(Invoke(func)); diff --git a/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc b/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc index fcfce84..1840fcc 100644 --- a/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc +++ b/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc @@ -100,11 +100,19 @@ DeclarativePolicyUtils::updateCurrentPolicy(const string &policy_checksum) { string clean_policy_checksum = getCleanChecksum(policy_checksum); auto env = Singleton::Consume::by()->getEnvType(); - curr_policy = Singleton::Consume::by()->generateAppSecLocalPolicy( - env, - clean_policy_checksum, - local_policy_path - ); + string maybe_policy = + Singleton::Consume::by()->generateAppSecLocalPolicy( + env, + clean_policy_checksum, + local_policy_path + ); + + if (maybe_policy.empty()) { + dbgWarning(D_ORCHESTRATOR) << "Could not generate appsec local policy"; + return; + } + + curr_policy = maybe_policy; } string diff --git a/components/security_apps/orchestration/update_communication/fog_authenticator.cc b/components/security_apps/orchestration/update_communication/fog_authenticator.cc index e1ab0ff..e2c610c 100755 --- a/components/security_apps/orchestration/update_communication/fog_authenticator.cc +++ b/components/security_apps/orchestration/update_communication/fog_authenticator.cc @@ -68,6 +68,12 @@ FogAuthenticator::RegistrationData::RegistrationData(const string &token) { } +string +FogAuthenticator::RegistrationData::getData() const +{ + return data; +} + FogAuthenticator::UserCredentials::UserCredentials(const string &_client_id, const string &_shared_secret) : client_id(_client_id), @@ -193,6 +199,10 @@ FogAuthenticator::registerAgent( request << make_pair("reverse_proxy", "true"); } + if (details_resolver->isCloudStorageEnabled()) { + request << make_pair("cloud_storage_service", "true"); + } + if (details_resolver->isKernelVersion3OrHigher()) { request << make_pair("isKernelVersion3OrHigher", "true"); } @@ -212,6 +222,10 @@ FogAuthenticator::registerAgent( if (details_resolver->compareCheckpointVersion(8200, std::greater_equal())) { request << make_pair("isCheckpointVersionGER82", "true"); } + auto maybe_vs_id = Singleton::Consume::by()->get("VS ID"); + if (maybe_vs_id.ok()) { + request << make_pair("virtualSystemId", maybe_vs_id.unpack()); + } #endif // gaia || smb dbgDebug(D_ORCHESTRATOR) << "Sending registration request to fog"; @@ -297,36 +311,88 @@ FogAuthenticator::getRegistrationData() return reg_data; } - const char *env_otp = getenv("NANO_AGENT_TOKEN"); - if (env_otp) { - dbgInfo(D_ORCHESTRATOR) << "Loading registration token from environment"; - return RegistrationData(env_otp); - } if (reg_data.ok()) { dbgInfo(D_ORCHESTRATOR) << "Loading registration token from cache"; return reg_data; } + auto local_env_token = getRegistrationToken(); + if (local_env_token.ok()) return local_env_token; + + return genError("Failed to load registration token from the environment."); +} + +Maybe +FogAuthenticator::getRegistrationToken() +{ auto reg_data_path = getConfigurationWithDefault( filesystem_prefix + "/conf/registration-data.json", "orchestration", "Registration data Path" ); + dbgTrace(D_ORCHESTRATOR) << "Getting registration token from " << reg_data_path; - dbgDebug(D_ORCHESTRATOR) << "Loading registration data from " << reg_data_path; auto orchestration_tools = Singleton::Consume::by(); auto raw_reg_data = orchestration_tools->readFile(reg_data_path); - if (!raw_reg_data.ok()) return genError(raw_reg_data.getErr()); + if (raw_reg_data.ok()) { + auto decoded_reg_data = orchestration_tools->base64Decode(raw_reg_data.unpack()); + reg_data = orchestration_tools->jsonStringToObject(decoded_reg_data); - dbgTrace(D_ORCHESTRATOR) << "Successfully loaded the registration data"; - auto decoded_reg_data = orchestration_tools->base64Decode(raw_reg_data.unpack()); - reg_data = orchestration_tools->jsonStringToObject(decoded_reg_data); - - if (reg_data.ok()) { - dbgTrace(D_ORCHESTRATOR) << "Registration token has been converted to an object"; + if (reg_data.ok()) { + dbgInfo(D_ORCHESTRATOR) << "Registration token has been loaded from: " << reg_data_path; + return reg_data; + } } - return reg_data; + dbgTrace(D_ORCHESTRATOR) << "Getting registration token from container environment."; + const char *container_otp = getenv("AGENT_TOKEN"); + if (container_otp) { + dbgInfo(D_ORCHESTRATOR) << "Registration token has been loaded from container environment"; + return RegistrationData(container_otp); + } + + dbgTrace(D_ORCHESTRATOR) << "Getting registration token from the environment."; + const char *env_otp = getenv("NANO_AGENT_TOKEN"); + if (env_otp) { + dbgInfo(D_ORCHESTRATOR) << "Registration token has been loaded from the environment"; + return RegistrationData(env_otp); + } + + return genError("No registration token in the environment"); +} + +void +FogAuthenticator::registerLocalAgentToFog() +{ + auto local_reg_token = getRegistrationToken(); + if (!local_reg_token.ok()) return; + dbgInfo(D_ORCHESTRATOR) << "Start local agent registration to the fog"; + + string exec_command = "open-appsec-ctl --set-mode --online_mode --token " + local_reg_token.unpack().getData(); + + auto i_agent_details = Singleton::Consume::by(); + auto fog_address = i_agent_details->getFogDomain(); + if (fog_address.ok()) exec_command = exec_command + " --fog https://" + fog_address.unpack(); + + auto shell_cmd = Singleton::Consume::by(); + auto maybe_cmd_output = shell_cmd->getExecOutputAndCode( + exec_command, + 300000, + true + ); + if (!maybe_cmd_output.ok()){ + dbgWarning(D_ORCHESTRATOR) + << "Failed in local agent registertion to the fog. Error: " + << maybe_cmd_output.getErr(); + return; + } + + if (maybe_cmd_output.unpack().second != 0) { + dbgWarning(D_ORCHESTRATOR) + << "Failed in local agent registertion to the fog. Error: " + << maybe_cmd_output.unpack().first; + return; + } } bool @@ -378,6 +444,22 @@ FogAuthenticator::getCredentialsFromFile() const return orchestration_tools->jsonStringToObject(encrypted_cred.unpack()); } +static string +getDeplymentType() +{ + auto deplyment_type = Singleton::Consume::by()->getEnvType(); + switch (deplyment_type) { + case EnvType::LINUX: return "Embedded"; + case EnvType::DOCKER: return "Docker"; + case EnvType::NON_CRD_K8S: + case EnvType::K8S: return "K8S"; + case EnvType::COUNT: break; + } + + dbgAssert(false) << "Failed to get a legitimate deplyment type: " << static_cast(deplyment_type); + return "Embedded"; +} + Maybe FogAuthenticator::getCredentials() { @@ -386,7 +468,7 @@ FogAuthenticator::getCredentials() return maybe_credentials; } - dbgTrace(D_ORCHESTRATOR) << "Credentials were not not receoived from the file. Getting registration data."; + dbgTrace(D_ORCHESTRATOR) << "Credentials were not not received from the file. Getting registration data."; auto reg_data = getRegistrationData(); if (!reg_data.ok()) { return genError("Failed to load a valid registration token, Error: " + reg_data.getErr()); @@ -396,17 +478,24 @@ FogAuthenticator::getCredentials() Maybe name = details_resolver->getHostname(); if (!name.ok()) return name.passErr(); + auto maybe_vs_id = Singleton::Consume::by()->get("VS ID"); + string host_name = *name; + if (maybe_vs_id.ok()) { + host_name.append(":"); + host_name.append(maybe_vs_id.unpack()); + } + Maybe platform = details_resolver->getPlatform(); if (!platform.ok()) return platform.passErr(); Maybe arch = details_resolver->getArch(); if (!arch.ok()) return arch.passErr(); - string type = getConfigurationWithDefault("Embedded", "orchestration", "Agent type"); - maybe_credentials = registerAgent(reg_data.unpack(), *name, type, *platform, *arch); + string type = getSettingWithDefault(getDeplymentType(), "orchestration", "Agent type"); + maybe_credentials = registerAgent(reg_data.unpack(), host_name, type, *platform, *arch); auto orc_status = Singleton::Consume::by(); - orc_status->setRegistrationDetails(*name, type, *platform, *arch); + orc_status->setRegistrationDetails(host_name, type, *platform, *arch); if (!maybe_credentials.ok()) return maybe_credentials; diff --git a/components/security_apps/orchestration/update_communication/fog_communication.cc b/components/security_apps/orchestration/update_communication/fog_communication.cc index 30731ef..e62349b 100755 --- a/components/security_apps/orchestration/update_communication/fog_communication.cc +++ b/components/security_apps/orchestration/update_communication/fog_communication.cc @@ -96,8 +96,6 @@ FogCommunication::downloadAttributeFile(const GetResourceFile &resourse_file, co { if (!access_token.ok()) return genError("Acccess Token not available."); - auto unpacked_access_token = access_token.unpack().getToken(); - string policy_mgmt_mode = getSettingWithDefault("management", "profileManagedMode"); if (policy_mgmt_mode == "declarative" && resourse_file.getFileName() =="policy") { dbgDebug(D_ORCHESTRATOR) << "Download policy on declarative mode - returning the local policy"; diff --git a/components/security_apps/orchestration/update_communication/hybrid_communication.cc b/components/security_apps/orchestration/update_communication/hybrid_communication.cc index 1b2fd27..1eac575 100755 --- a/components/security_apps/orchestration/update_communication/hybrid_communication.cc +++ b/components/security_apps/orchestration/update_communication/hybrid_communication.cc @@ -104,8 +104,6 @@ HybridCommunication::downloadAttributeFile(const GetResourceFile &resourse_file, if (resourse_file.getFileName() == "manifest") { if (!access_token.ok()) return genError("Acccess Token not available."); - auto unpacked_access_token = access_token.unpack().getToken(); - auto attribute_file = Singleton::Consume::by()->downloadFile( resourse_file.getRequestMethod(), agent_resource_api + '/' + resourse_file.getFileName(), diff --git a/components/security_apps/orchestration/update_communication/local_communication.cc b/components/security_apps/orchestration/update_communication/local_communication.cc index 373d5dd..17daf47 100755 --- a/components/security_apps/orchestration/update_communication/local_communication.cc +++ b/components/security_apps/orchestration/update_communication/local_communication.cc @@ -40,6 +40,12 @@ LocalCommunication::authenticateAgent() return Maybe(); } +void +LocalCommunication::registerLocalAgentToFog() +{ + return; +} + string LocalCommunication::getChecksum(const string &file_path) { diff --git a/components/security_apps/orchestration/update_communication/update_communication.cc b/components/security_apps/orchestration/update_communication/update_communication.cc index e3a52f0..95a3390 100755 --- a/components/security_apps/orchestration/update_communication/update_communication.cc +++ b/components/security_apps/orchestration/update_communication/update_communication.cc @@ -69,6 +69,11 @@ public: return i_update_comm_impl->authenticateAgent(); } + void registerLocalAgentToFog() + { + i_update_comm_impl->registerLocalAgentToFog(); + } + Maybe getUpdate(CheckUpdateRequest &request) override { diff --git a/components/security_apps/orchestration/update_communication/update_communication_ut/local_communication_ut.cc b/components/security_apps/orchestration/update_communication/update_communication_ut/local_communication_ut.cc index 2bae44a..cf49c62 100755 --- a/components/security_apps/orchestration/update_communication/update_communication_ut/local_communication_ut.cc +++ b/components/security_apps/orchestration/update_communication/update_communication_ut/local_communication_ut.cc @@ -38,6 +38,12 @@ public: return local_communication.authenticateAgent(); } + void + registerLocalAgentToFog() + { + local_communication.registerLocalAgentToFog(); + } + Maybe sendPolicyVersion(const string &version, const string &policy_versions) { @@ -122,6 +128,11 @@ TEST_F(LocalCommunicationTest, authenticateAgent) EXPECT_TRUE(authenticat_res.ok()); } +TEST_F(LocalCommunicationTest, registerLocalAgentToFog) +{ + registerLocalAgentToFog(); +} + TEST_F(LocalCommunicationTest, downloadManifest) { string new_manifest_string = "new manifest"; diff --git a/components/security_apps/waap/waap_clib/ParserJson.cc b/components/security_apps/waap/waap_clib/ParserJson.cc index 8fb930b..b98d720 100755 --- a/components/security_apps/waap/waap_clib/ParserJson.cc +++ b/components/security_apps/waap/waap_clib/ParserJson.cc @@ -23,6 +23,7 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_JSON); USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); const std::string ParserJson::m_parserName = "jsonParser"; +static const std::string OPERATION_NAME_GRAPHQL = "operationName"; int ParserJson::cb_null() diff --git a/components/security_apps/waap/waap_clib/Telemetry.cc b/components/security_apps/waap/waap_clib/Telemetry.cc index 55e9386..c12d40c 100755 --- a/components/security_apps/waap/waap_clib/Telemetry.cc +++ b/components/security_apps/waap/waap_clib/Telemetry.cc @@ -44,6 +44,7 @@ WaapTelemetryBase::sendLog(const LogRest &metric_client_rest) const "X-Tenant-Id", Singleton::Consume::by()->getTenantId() ); + req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); Singleton::Consume::by()->sendSyncMessageWithoutResponse( HTTPMethod::POST, fog_metric_uri, diff --git a/components/security_apps/waap/waap_clib/TuningDecisions.h b/components/security_apps/waap/waap_clib/TuningDecisions.h index 00ff8a3..a6f9ee4 100755 --- a/components/security_apps/waap/waap_clib/TuningDecisions.h +++ b/components/security_apps/waap/waap_clib/TuningDecisions.h @@ -59,6 +59,7 @@ private: if (agentDetails->getOrchestrationMode() != OrchestrationMode::ONLINE) { MessageMetadata req_md(getSharedStorageHost(), 80); req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); + req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); auto req_status = messaging->sendSyncMessage( method, uri, diff --git a/components/signal_handler/signal_handler.cc b/components/signal_handler/signal_handler.cc index 994ba98..991c2b7 100755 --- a/components/signal_handler/signal_handler.cc +++ b/components/signal_handler/signal_handler.cc @@ -136,8 +136,10 @@ private: string service_underscore_name = service_name; replace(service_underscore_name.begin(), service_underscore_name.end(), ' ', '_'); + string logFilesPath = getLogFilesPathConfig(); + trace_file_path = getConfigurationWithDefault( - "/var/log/nano_agent/trace_export_files/" + service_underscore_name + "_trace_file.dbg", + logFilesPath + "/nano_agent/trace_export_files/" + service_underscore_name + "_trace_file.dbg", "SignalHandler", "outputFilePath" ); diff --git a/core/config/config.cc b/core/config/config.cc index 1271882..84b5c6e 100644 --- a/core/config/config.cc +++ b/core/config/config.cc @@ -244,7 +244,7 @@ private: dbgTrace(D_CONFIG) << "File system path reloaded: " << config_directory_path; } - void + bool sendOrchestatorReloadStatusMsg(const LoadNewConfigurationStatus &status) { I_Messaging *messaging = Singleton::Consume::by(); @@ -262,7 +262,7 @@ private: MessageMetadata secondary_port_req_md("127.0.0.1", 7778); secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); - messaging->sendSyncMessageWithoutResponse( + service_config_status = messaging->sendSyncMessageWithoutResponse( HTTPMethod::POST, "/set-reconf-status", status, @@ -270,6 +270,11 @@ private: secondary_port_req_md ); } + if (!service_config_status) { + dbgWarning(D_CONFIG) << "Unsuccessful attempt to send configuration reload status"; + return false; + } + return true; } unordered_map, PerContextValue>> configuration_nodes; @@ -935,7 +940,15 @@ ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version mainloop->stop(routine_id); LoadNewConfigurationStatus finished(id, service_name, !res, true); if (!res) finished.setError("Failed to reload configuration"); - sendOrchestatorReloadStatusMsg(finished); + I_TimeGet *time = Singleton::Consume::by(); + auto send_status_time_out = time->getMonotonicTime() + chrono::seconds(180); + while (time->getMonotonicTime() < send_status_time_out) { + if (sendOrchestatorReloadStatusMsg(finished)) break; + mainloop->yield(chrono::seconds(1)); + } + if (time->getMonotonicTime() >= send_status_time_out) { + dbgWarning(D_CONFIG) << "Failed to send configuration reload status(finish) to the orchestrator"; + } is_continuous_report = false; } diff --git a/core/include/attachments/attachment_types.h b/core/include/attachments/attachment_types.h index 49af0df..206c80a 100644 --- a/core/include/attachments/attachment_types.h +++ b/core/include/attachments/attachment_types.h @@ -24,6 +24,7 @@ enum AttachmentType NGINX_ATT_ID, PRELOAD_ATT_ID, SQUID_ATT_ID, + ENVOY_ATT_ID, #ifdef __cplusplus COUNT #endif diff --git a/core/include/services_sdk/interfaces/i_env_details.h b/core/include/services_sdk/interfaces/i_env_details.h index 5348436..f1b5d04 100644 --- a/core/include/services_sdk/interfaces/i_env_details.h +++ b/core/include/services_sdk/interfaces/i_env_details.h @@ -17,7 +17,7 @@ #include #include -enum class EnvType { LINUX, K8S, COUNT }; +enum class EnvType { LINUX, DOCKER, K8S, NON_CRD_K8S, COUNT }; class I_EnvDetails { diff --git a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h index 0044339..a7d8dbf 100644 --- a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h +++ b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h @@ -21,6 +21,8 @@ #include "intelligence_is_v2/intelligence_response.h" #include "intelligence_is_v2/intelligence_types_v2.h" #include "intelligence_is_v2/query_request_v2.h" +#include "messaging/messaging_enums.h" +#include "messaging/messaging_metadata.h" #include "maybe_res.h" namespace Intelligence { @@ -43,18 +45,29 @@ public: getResponse( const std::vector &query_requests, bool is_pretty, - bool is_bulk + bool is_bulk, + const MessageMetadata &req_md ) const = 0; - virtual Maybe getResponse(const QueryRequest &query_request, bool is_pretty) const = 0; + virtual Maybe + getResponse(const QueryRequest &query_request, bool is_pretty, const MessageMetadata &req_md) const = 0; template Maybe>> - queryIntelligence(QueryRequest &query_request, bool ignore_in_progress = false, bool is_pretty = true); + queryIntelligence( + QueryRequest &query_request, + bool ignore_in_progress = false, + bool is_pretty = true, + MessageMetadata req_md = MessageMetadata("", 0) + ); template Maybe>>>> - queryIntelligence(std::vector &query_requests, bool is_pretty = true); + queryIntelligence( + std::vector &query_requests, + bool is_pretty = true, + MessageMetadata req_md = MessageMetadata("", 0) + ); protected: virtual ~I_Intelligence_IS_V2() {} diff --git a/core/include/services_sdk/interfaces/i_messaging.h b/core/include/services_sdk/interfaces/i_messaging.h index f8e7171..e8ba7e6 100644 --- a/core/include/services_sdk/interfaces/i_messaging.h +++ b/core/include/services_sdk/interfaces/i_messaging.h @@ -76,7 +76,7 @@ public: MessageMetadata message_metadata = MessageMetadata() ) = 0; - virtual Maybe downloadFile( + virtual Maybe downloadFile( const HTTPMethod method, const std::string &uri, const std::string &download_file_path, @@ -84,7 +84,7 @@ public: MessageMetadata message_metadata = MessageMetadata() ) = 0; - virtual Maybe uploadFile( + virtual Maybe uploadFile( const std::string & uri, const std::string & upload_file_path, const MessageCategory category = MessageCategory::GENERIC, diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_interface_impl.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_interface_impl.h index 9880de8..66bfaf9 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_interface_impl.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_interface_impl.h @@ -20,9 +20,14 @@ template Maybe>> -I_Intelligence_IS_V2::queryIntelligence(QueryRequest &query_request, bool ignore_in_progress, bool is_pretty) +I_Intelligence_IS_V2::queryIntelligence( + QueryRequest &query_request, + bool ignore_in_progress, + bool is_pretty, + MessageMetadata req_md +) { - auto response = getResponse(query_request, is_pretty); + auto response = getResponse(query_request, is_pretty, req_md); if (!response.ok()) return response.passErr(); auto serializable_response = response->getSerializableResponse(); @@ -39,9 +44,13 @@ I_Intelligence_IS_V2::queryIntelligence(QueryRequest &query_request, bool ignore template Maybe>>>> -I_Intelligence_IS_V2::queryIntelligence(std::vector &query_requests, bool is_pretty) +I_Intelligence_IS_V2::queryIntelligence( + std::vector &query_requests, + bool is_pretty, + MessageMetadata req_md +) { - auto res = getResponse(query_requests, is_pretty, true); + auto res = getResponse(query_requests, is_pretty, true, req_md); if (!res.ok()) return res.passErr(); return res->getBulkData(); diff --git a/core/include/services_sdk/interfaces/mock/mock_intelligence.h b/core/include/services_sdk/interfaces/mock/mock_intelligence.h index 4a19b4f..dade4ee 100644 --- a/core/include/services_sdk/interfaces/mock/mock_intelligence.h +++ b/core/include/services_sdk/interfaces/mock/mock_intelligence.h @@ -26,11 +26,19 @@ public: MOCK_CONST_METHOD1(sendInvalidation, bool(const Invalidation &invalidation)); MOCK_METHOD2(registerInvalidation, Maybe(const Invalidation &invalidation, const InvalidationCb &callback)); MOCK_METHOD1(unregisterInvalidation, void(uint id)); + MOCK_CONST_METHOD4( + getResponse, + Maybe( + const std::vector &query_requests, + bool is_pretty, + bool is_bulk, + const MessageMetadata &req_md + ) + ); MOCK_CONST_METHOD3( getResponse, - Maybe(const std::vector &query_requests, bool is_pretty, bool is_bulk) + Maybe(const QueryRequest &query_request, bool is_pretty, const MessageMetadata &req_md) ); - MOCK_CONST_METHOD2(getResponse, Maybe(const QueryRequest &query_request, bool is_pretty)); MOCK_CONST_METHOD0(getIsOfflineOnly, bool(void)); MOCK_CONST_METHOD1(getOfflineInfoString, Maybe(const SerializableQueryFilter &query)); }; diff --git a/core/include/services_sdk/interfaces/mock/mock_messaging.h b/core/include/services_sdk/interfaces/mock/mock_messaging.h index 061a0e2..8cb38af 100644 --- a/core/include/services_sdk/interfaces/mock/mock_messaging.h +++ b/core/include/services_sdk/interfaces/mock/mock_messaging.h @@ -32,7 +32,7 @@ public: MOCK_METHOD5( downloadFile, - Maybe ( + Maybe ( HTTPMethod, const string &, const string &, @@ -43,7 +43,7 @@ public: MOCK_METHOD4( uploadFile, - Maybe ( + Maybe ( const string &, const string &, MessageCategory, @@ -62,10 +62,4 @@ operator<<(std::ostream &os, const HTTPResponse &) return os; } -static std::ostream & -operator<<(std::ostream &os, const HTTPStatusCode &) -{ - return os; -} - #endif // __MOCK_MESSAGING_H__ diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index 027ad2d..22cdf60 100644 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -149,7 +149,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_UPSTREAM_KEEPALIVE, D_REVERSE_PROXY) DEFINE_FLAG(D_FORWARD_PROXY, D_REVERSE_PROXY) - DEFINE_FLAG(D_IDA_SAML, D_COMPONENT) + DEFINE_FLAG(D_IDA, D_COMPONENT) DEFINE_FLAG(D_IOT_NEXT, D_COMPONENT) DEFINE_FLAG(D_IOT_AUXILIARY, D_IOT_NEXT) diff --git a/core/include/services_sdk/resources/intelligence_invalidation.h b/core/include/services_sdk/resources/intelligence_invalidation.h index 11c5799..985c1b4 100644 --- a/core/include/services_sdk/resources/intelligence_invalidation.h +++ b/core/include/services_sdk/resources/intelligence_invalidation.h @@ -70,7 +70,7 @@ public: std::vector getAttributes() const { return attributes; } const Maybe & getSourceId() const { return source_id; } const Maybe & getObjectType() const { return object_type; } - InvalidationType getInvalidationType() const { return invalidation_type; } + const Maybe & getInvalidationType() const { return invalidation_type; } Maybe getRegistrationID() const; bool report(I_Intelligence_IS_V2 *interface) const; @@ -92,7 +92,7 @@ private: std::vector attributes; Maybe source_id; Maybe object_type; - InvalidationType invalidation_type = InvalidationType::ADD; + Maybe invalidation_type; Maybe listening_id; Maybe registration_id; }; diff --git a/core/include/services_sdk/resources/report/report_enums.h b/core/include/services_sdk/resources/report/report_enums.h index 796e0ca..7923ad3 100644 --- a/core/include/services_sdk/resources/report/report_enums.h +++ b/core/include/services_sdk/resources/report/report_enums.h @@ -68,6 +68,7 @@ enum class Tags { API_DISCOVERY, NGINX_PROXY_MANAGER, WEB_SERVER_APISIX, + DEPLOYMENT_DOCKER, COUNT }; diff --git a/core/intelligence_is_v2/bulk_query_response_v2.cc b/core/intelligence_is_v2/bulk_query_response_v2.cc index 148c994..b07d578 100644 --- a/core/intelligence_is_v2/bulk_query_response_v2.cc +++ b/core/intelligence_is_v2/bulk_query_response_v2.cc @@ -34,6 +34,10 @@ ValidBulkQueryResponse::serialize(cereal::JSONInputArchive &ar) void IntelligenceQueryBulkResponse::serialize(cereal::JSONInputArchive &ar) { - ar(cereal::make_nvp("errors", errors)); ar(cereal::make_nvp("queriesResponse", valid_responses)); + try { + ar(cereal::make_nvp("errors", errors)); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + } } diff --git a/core/intelligence_is_v2/include/intelligence_request.h b/core/intelligence_is_v2/include/intelligence_request.h index 820cf69..6725cce 100644 --- a/core/intelligence_is_v2/include/intelligence_request.h +++ b/core/intelligence_is_v2/include/intelligence_request.h @@ -14,6 +14,8 @@ #ifndef __INTELLIGENCE_REQUEST_H__ #define __INTELLIGENCE_REQUEST_H__ #include "intelligence_is_v2/query_request_v2.h" +#include "messaging/messaging_enums.h" +#include "messaging/messaging_metadata.h" #include #include "maybe_res.h" @@ -23,9 +25,14 @@ namespace Intelligence { class IntelligenceRequest : ClientRest { public: - IntelligenceRequest(const std::vector &queries, bool is_pretty, bool is_bulk) + IntelligenceRequest( + const std::vector &queries, + bool is_pretty, + bool is_bulk, + const MessageMetadata &req_md + ) : - queries(queries), is_pretty(is_pretty), is_bulk(is_bulk) + queries(queries), is_pretty(is_pretty), is_bulk(is_bulk), req_md(req_md) {} Maybe checkAssetsLimit() const; @@ -38,12 +45,14 @@ public: size_t getSize() const { return queries.size(); } bool isBulk() const { return is_bulk; } + const MessageMetadata & getReqMD() const { return req_md; } private: const std::vector &queries; bool is_pretty = true; bool is_bulk = false; Maybe response_from_fog = genError("Uninitialized"); + const MessageMetadata &req_md; }; } diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc index 17bba5f..6b6de25 100644 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -202,11 +202,26 @@ public: if (objectType.isActive()) { auto type = object_names.find(objectType.get()); - if (type != object_names.end()) invalidation.setObjectType(type->second); + if (type != object_names.end()) { + invalidation.setObjectType(type->second); + } + else { + dbgWarning(D_INTELLIGENCE) << "Received invalid object type: " << objectType.get(); + } } if (sourceId.isActive()) invalidation.setSourceId(sourceId.get()); + if (invalidationType.isActive()) { + auto type = invalidation_type_names.find(invalidationType.get()); + if (type != invalidation_type_names.end()) { + invalidation.setInvalidationType(type->second); + } + else { + dbgWarning(D_INTELLIGENCE) << "Received invalid invalidation type: " << invalidationType.get(); + } + } + string registration_id = ""; if (invalidationRegistrationId.isActive()) registration_id = invalidationRegistrationId.get(); @@ -227,6 +242,7 @@ private: C2S_OPTIONAL_PARAM(string, invalidationRegistrationId); C2S_OPTIONAL_PARAM(vector, mainAttributes); C2S_OPTIONAL_PARAM(vector, attributes); + C2S_OPTIONAL_PARAM(string, invalidationType); }; class IntelligenceComponentV2::Impl @@ -280,9 +296,14 @@ public: } Maybe - getResponse(const vector &query_requests, bool is_pretty, bool is_bulk) const override + getResponse( + const vector &query_requests, + bool is_pretty, + bool is_bulk, + const MessageMetadata &req_md + ) const override { - IntelligenceRequest intelligence_req(query_requests, is_pretty, is_bulk); + IntelligenceRequest intelligence_req(query_requests, is_pretty, is_bulk, req_md); if (!intelligence_req.checkAssetsLimit().ok()) return intelligence_req.checkAssetsLimit().passErr(); if (!intelligence_req.checkMinConfidence().ok()) return intelligence_req.checkMinConfidence().passErr(); if (intelligence_req.isPagingActivated()) { @@ -297,10 +318,10 @@ public: } Maybe - getResponse(const QueryRequest &query_request, bool is_pretty) const override + getResponse(const QueryRequest &query_request, bool is_pretty, const MessageMetadata &req_md) const override { vector queries = {query_request}; - return getResponse(queries, is_pretty, false); + return getResponse(queries, is_pretty, false, req_md); } private: @@ -312,6 +333,11 @@ private: if (!is_supported) { is_supported = getProfileAgentSettingWithDefault(false, "agent.config.supportInvalidation"); } + + if (!is_supported) { + is_supported = getConfigurationWithDefault(false, "intelligence", "support Invalidation"); + } + return is_supported; } @@ -390,7 +416,8 @@ private: auto tenant = details->getTenantId(); if (tenant == "") tenant = "Global"; headers["X-Tenant-Id"] = tenant; - auto agent = details->getAgentId(); + auto rest = Singleton::Consume::by(); + auto agent = details->getAgentId() + ":" + to_string(rest->getListeningPort()); headers["X-Source-Id"] = agent; return headers; @@ -474,6 +501,7 @@ IntelligenceComponentV2::preload() { registerExpectedConfiguration("intelligence", "maximum request overall time"); registerExpectedConfiguration("intelligence", "maximum request lap time"); + registerExpectedConfiguration("intelligence", "support Invalidation"); registerExpectedSetting("intelligence", "local intelligence server ip"); registerExpectedSetting("intelligence", primary_port_setting); registerExpectedSetting("intelligence", secondary_port_setting); diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc index d47486f..997da6f 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc @@ -126,7 +126,7 @@ TEST_F(IntelligenceComponentMockTest, getResponseErrorTest) QueryRequest request(Condition::EQUALS, "category", "cloud", true); Maybe res_error = genError("Test error"); - EXPECT_CALL(intelligence_mock, getResponse(_, _) + EXPECT_CALL(intelligence_mock, getResponse(_, _, _) ).WillOnce(Return(res_error)); auto maybe_ans = intell->queryIntelligence(request); @@ -180,7 +180,7 @@ TEST_F(IntelligenceComponentMockTest, getResponseTest) Intelligence::Response response(response_str, 1, false); - EXPECT_CALL(intelligence_mock, getResponse(_, _) + EXPECT_CALL(intelligence_mock, getResponse(_, _, _) ).WillOnce(Return(response)); auto maybe_ans = intell->queryIntelligence(request); @@ -341,7 +341,7 @@ TEST_F(IntelligenceComponentMockTest, bulkOnlineIntelligenceMockTest) ); Intelligence::Response response(response_str, 4, true); - EXPECT_CALL(intelligence_mock, getResponse(_, _, _) + EXPECT_CALL(intelligence_mock, getResponse(_, _, _, _) ).WillOnce(Return(response)); auto maybe_ans = intell->queryIntelligence(requests); diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc index 17e93a2..e245a10 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc @@ -24,7 +24,7 @@ USE_DEBUG_FLAG(D_INTELLIGENCE); TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequest) { QueryRequest request(Condition::EQUALS, "phase", "testing", true); vector requests = {request}; - Intelligence::IntelligenceRequest query(requests, true, false); + Intelligence::IntelligenceRequest query(requests, true, false, MessageMetadata("", 0)); std::string expected = "{\n" " \"limit\": 20,\n" @@ -42,7 +42,7 @@ TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequest) { TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) { QueryRequest request(Condition::EQUALS, "phase", "testing", true); vector requests = {request}; - Intelligence::IntelligenceRequest query(requests, false, false); + Intelligence::IntelligenceRequest query(requests, false, false, MessageMetadata("", 0)); std::string expected = "{" "\"limit\":20," @@ -59,7 +59,7 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) { TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequestSpaces) { QueryRequest request(Condition::EQUALS, "ph ase", "te sti\" n g\\", true); vector requests = {request}; - Intelligence::IntelligenceRequest query(requests, false, false); + Intelligence::IntelligenceRequest query(requests, false, false, MessageMetadata("", 0)); std::string expected = "{" "\"limit\":20," "\"fullResponse\":true," @@ -76,7 +76,7 @@ TEST(IntelligenceQueryTestV2, genJsonPrettyBulkRequests) { QueryRequest request1(Condition::EQUALS, "phase", "testing", true); QueryRequest request2(Condition::EQUALS, "height", "testing", 25); std::vector requests = {request1, request2}; - Intelligence::IntelligenceRequest query(requests, true, true); + Intelligence::IntelligenceRequest query(requests, true, true, MessageMetadata("", 0)); std::string expected = "{\n" " \"queries\": [\n" @@ -114,7 +114,7 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettyBulkRequest) { QueryRequest request1(Condition::EQUALS, "phase", "testing", true); QueryRequest request2(Condition::EQUALS, "height", "testing", 25); std::vector requests = {request1, request2}; - Intelligence::IntelligenceRequest query(requests, false, true); + Intelligence::IntelligenceRequest query(requests, false, true, MessageMetadata("", 0)); std::string expected = "{" "\"queries\":[{" diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc index a44c849..11112b8 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc @@ -117,12 +117,12 @@ TEST(InvalidationBasic, SettersAndGetters) EXPECT_EQ(invalidation.getClassifier(ClassifierType::GROUP), ""); EXPECT_EQ(invalidation.getClassifier(ClassifierType::ORDER), ""); EXPECT_EQ(invalidation.getClassifier(ClassifierType::KIND), ""); - EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::ADD); EXPECT_TRUE(invalidation.getMainAttributes().empty()); EXPECT_TRUE(invalidation.getAttributes().empty()); EXPECT_FALSE(invalidation.getSourceId().ok()); EXPECT_FALSE(invalidation.getObjectType().ok()); + EXPECT_FALSE(invalidation.getInvalidationType().ok()); set main_vals = { "2", "3" }; set vals = { "5", "6" }; @@ -152,7 +152,7 @@ TEST(InvalidationBasic, SettersAndGetters) EXPECT_EQ(invalidation.getAttributes().begin()->getStringSetAttr("attr2").unpack(), vals); EXPECT_EQ(invalidation.getSourceId().unpack(), "id"); EXPECT_EQ(invalidation.getObjectType().unpack(), Intelligence::ObjectType::ASSET); - EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::DELETE); + EXPECT_EQ(invalidation.getInvalidationType().unpack(), InvalidationType::DELETE); } TEST(InvalidationBasic, Matching) @@ -348,7 +348,6 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation) "\"category\": \"bbb\", " "\"family\": \"ccc\", " "\"objectType\": \"asset\", " - "\"invalidationType\": \"add\", " "\"sourceId\": \"id\", " "\"mainAttributes\": [ { \"attr2\": \"2\" } ], " "\"attributes\": [ { \"attr3\": \"3\" } ]" @@ -389,7 +388,6 @@ TEST_F(IntelligenceInvalidation, multiple_assets_invalidation) "\"category\": \"bbb\", " "\"family\": \"ccc\", " "\"objectType\": \"asset\", " - "\"invalidationType\": \"add\", " "\"sourceId\": \"id\", " "\"mainAttributes\": [ { \"attr2\": \"2\" }, { \"attr2\": \"22\", \"attr3\": [ \"33\", \"44\" ] } ], " "\"attributes\": [ { \"attr3\": \"3\" } ]" @@ -439,7 +437,6 @@ TEST_F(IntelligenceInvalidation, sending_private_invalidation) "\"category\": \"bbb\", " "\"family\": \"ccc\", " "\"objectType\": \"asset\", " - "\"invalidationType\": \"add\", " "\"sourceId\": \"id\", " "\"mainAttributes\": [ { \"attr2\": \"2\" } ], " "\"attributes\": [ { \"attr3\": \"3\" } ]" diff --git a/core/intelligence_is_v2/intelligence_server.cc b/core/intelligence_is_v2/intelligence_server.cc index e7f7621..5386404 100644 --- a/core/intelligence_is_v2/intelligence_server.cc +++ b/core/intelligence_is_v2/intelligence_server.cc @@ -142,7 +142,14 @@ Sender::sendMessage() { if (server_port.ok() && !server_ip.ok()) return genError("Can't send intelligence request. Server ip invalid"); if (server_ip.ok() && !server_port.ok()) return genError("Can't send intelligence request. Server port invalid"); - auto req_md = server_ip.ok() ? MessageMetadata(*server_ip, *server_port, conn_flags) : MessageMetadata(); + + MessageMetadata req_md; + if (server_ip.ok()) { + req_md = MessageMetadata(*server_ip, *server_port, conn_flags); + } + else { + req_md = request.getReqMD().getHostName().empty() ? MessageMetadata() : request.getReqMD(); + } if (server_ip.ok()) { dbgTrace(D_INTELLIGENCE) diff --git a/core/intelligence_is_v2/invalidation.cc b/core/intelligence_is_v2/invalidation.cc index b70c6f6..21244b4 100644 --- a/core/intelligence_is_v2/invalidation.cc +++ b/core/intelligence_is_v2/invalidation.cc @@ -27,6 +27,7 @@ Invalidation::Invalidation(const string &class_value) : source_id(genError()), object_type(genError()), + invalidation_type(genError()), listening_id(genError()), registration_id(genError()) { @@ -133,7 +134,9 @@ Invalidation::genObject() const } if (object_type.ok()) invalidation <<", \"objectType\": \"" << convertObjectType.at(*object_type) << '"'; - invalidation << ", \"invalidationType\": \"" << convertInvalidationType.at(invalidation_type) << '"'; + if (invalidation_type.ok()) { + invalidation << ", \"invalidationType\": \"" << convertInvalidationType.at(*invalidation_type) << '"'; + } if (source_id.ok()) invalidation <<", \"sourceId\": \"" << *source_id << '"'; if (registration_id.ok()) invalidation <<", \"invalidationRegistrationId\": \"" << *registration_id << '"'; @@ -211,6 +214,10 @@ Invalidation::matches(const Invalidation &other) const if (!other.object_type.ok() || *object_type != *other.object_type) return false; } + if (invalidation_type.ok()) { + if (!other.invalidation_type.ok() || *invalidation_type != *other.invalidation_type) return false; + } + if (source_id.ok()) { if (!other.source_id.ok() || *source_id != *other.source_id) return false; } diff --git a/core/messaging/connection/connection.cc b/core/messaging/connection/connection.cc index 67ff250..ac0e5e5 100644 --- a/core/messaging/connection/connection.cc +++ b/core/messaging/connection/connection.cc @@ -513,7 +513,6 @@ private: { if (!isBioSocketReady()) return 0; - dbgTrace(D_MESSAGING) << "Sending request: " << printOut(request); size_t offset = request.length() - data_left_to_send; auto curr_data_to_send = request.c_str() + offset; int data_sent_len = BIO_write(bio.get(), curr_data_to_send, data_left_to_send); @@ -544,7 +543,6 @@ private: int receive_len = BIO_read(bio.get(), buffer, sizeof(buffer) - 1); if (receive_len > 0) { - dbgTrace(D_CONNECTION) << "Received " << receive_len << " bytes"; return string(buffer, receive_len); } diff --git a/core/messaging/include/messaging_comp.h b/core/messaging/include/messaging_comp.h index 210254a..5d68834 100644 --- a/core/messaging/include/messaging_comp.h +++ b/core/messaging/include/messaging_comp.h @@ -54,7 +54,7 @@ public: bool force_buffering = true ); - Maybe downloadFile( + Maybe downloadFile( HTTPMethod method, const std::string &uri, const std::string &download_file_path, @@ -62,7 +62,7 @@ public: const MessageMetadata &message_metadata = MessageMetadata() ); - Maybe uploadFile( + Maybe uploadFile( const std::string &uri, const std::string &upload_file_path, MessageCategory category, diff --git a/core/messaging/messaging.cc b/core/messaging/messaging.cc index 5e200d4..0a8398b 100644 --- a/core/messaging/messaging.cc +++ b/core/messaging/messaging.cc @@ -62,7 +62,7 @@ public: return messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata, force_buffering); } - Maybe + Maybe downloadFile( const HTTPMethod method, const std::string &uri, @@ -74,7 +74,7 @@ public: return messaging_comp.downloadFile(method, uri, download_file_path, category, message_metadata); } - Maybe + Maybe uploadFile( const std::string &uri, const std::string &upload_file_path, diff --git a/core/messaging/messaging_comp/http_request.cc b/core/messaging/messaging_comp/http_request.cc index 27451f2..6bded0a 100644 --- a/core/messaging/messaging_comp/http_request.cc +++ b/core/messaging/messaging_comp/http_request.cc @@ -61,7 +61,9 @@ HTTPRequest::setConnectionHeaders(const Connection &conn) } } - insertHeader("Host", host); + if (headers.find("Host") == headers.end()) { + insertHeader("Host", host); + } insertHeader("Content-Length", to_string(body.size())); insertHeader("Content-type", "application/json"); insertHeader("Accept-Encoding", "identity"); @@ -82,10 +84,12 @@ HTTPRequest::prepareRequest( { HTTPRequest req(method, uri, headers, body); - if (!req.setConnectionHeaders(conn)) return genError("Failed to identify the HTTP method"); - - string agent_registration_query = R"("authenticationMethod": "token")"; bool dont_add_access_token = false; + if (headers.find("Host") != headers.end()) { + dont_add_access_token = true; + dbgTrace(D_MESSAGING) << "Request is not for FOG"; + } + string agent_registration_query = R"("authenticationMethod": "token")"; if (method == HTTPMethod::CONNECT || body.find(agent_registration_query) != string::npos) { dont_add_access_token = true; dbgTrace(D_MESSAGING) << "Request is for agent authentication"; @@ -93,6 +97,8 @@ HTTPRequest::prepareRequest( auto res = req.addAccessToken(conn, dont_add_access_token); if (!res.ok()) return res.passErr(); + if (!req.setConnectionHeaders(conn)) return genError("Failed to identify the HTTP method"); + if (conn.isOverProxy()) { auto res = req.addProxyAuthorization(conn); if (!res.ok()) return res.passErr(); diff --git a/core/messaging/messaging_comp/messaging_comp.cc b/core/messaging/messaging_comp/messaging_comp.cc index 157ffb6..4aaf040 100644 --- a/core/messaging/messaging_comp/messaging_comp.cc +++ b/core/messaging/messaging_comp/messaging_comp.cc @@ -145,7 +145,7 @@ MessagingComp::sendAsyncMessage( i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, new_message_metadata, false); } -Maybe +Maybe MessagingComp::downloadFile( HTTPMethod method, const string &uri, @@ -166,7 +166,9 @@ MessagingComp::downloadFile( auto response = sendSyncMessage(method, uri, "", category, message_metadata); if (!response.ok()) return response.passErr(); - + if (response.unpack().getHTTPStatusCode() != HTTPStatusCode::HTTP_OK) { + return genError(HTTPResponse(response.unpack().getHTTPStatusCode(), response.unpack().getBody())); + } ofstream file_stream(download_file_path); if (!file_stream.is_open()) { string open_err = "Failed to open the destination file. Path: " + download_file_path; @@ -177,10 +179,10 @@ MessagingComp::downloadFile( file_stream.close(); dbgTrace(D_MESSAGING) << "Successfully downloaded and save file to: " << download_file_path; - return HTTPStatusCode::HTTP_OK; + return Maybe(); } -Maybe +Maybe MessagingComp::uploadFile( const string &uri, const string &upload_file_path, @@ -205,9 +207,12 @@ MessagingComp::uploadFile( sendSyncMessage(HTTPMethod::PUT, uri, buffer.str(), category, message_metadata); if (!response.ok()) return response.passErr(); + if (response.unpack().getHTTPStatusCode() != HTTPStatusCode::HTTP_OK) { + return genError(HTTPResponse(response.unpack().getHTTPStatusCode(), response.unpack().getBody())); + } dbgTrace(D_MESSAGING) << "Successfully upload file from: " << upload_file_path; - return HTTPStatusCode::HTTP_OK; + return Maybe(); } bool diff --git a/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc index 155e90c..84c8ea2 100644 --- a/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc +++ b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc @@ -183,7 +183,6 @@ TEST_F(TestMessagingComp, testUploadFile) EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res)); auto upload_res = messaging_comp.uploadFile(uri, path, category, conn_metadata); ASSERT_TRUE(upload_res.ok()); - EXPECT_EQ(upload_res.unpack(), HTTPStatusCode::HTTP_OK); } TEST_F(TestMessagingComp, testDownloadFile) @@ -207,7 +206,6 @@ TEST_F(TestMessagingComp, testDownloadFile) EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res)); auto upload_res = messaging_comp.downloadFile(method, uri, "/tmp/test.txt", category, conn_metadata); ASSERT_TRUE(upload_res.ok()); - EXPECT_EQ(upload_res.unpack(), HTTPStatusCode::HTTP_OK); } bool diff --git a/core/report/tag_and_enum_management.cc b/core/report/tag_and_enum_management.cc index fd9aaaf..80f0f3a 100644 --- a/core/report/tag_and_enum_management.cc +++ b/core/report/tag_and_enum_management.cc @@ -110,7 +110,8 @@ TagAndEnumManagement::convertStringToTag(const string &tag) {"apiDiscoveryCloudMessaging", ReportIS::Tags::API_DISCOVERY}, {"Playground", ReportIS::Tags::PLAYGROUND}, {"Nginx Proxy Manager", ReportIS::Tags::NGINX_PROXY_MANAGER}, - {"APISIX Server", ReportIS::Tags::WEB_SERVER_APISIX} + {"APISIX Server", ReportIS::Tags::WEB_SERVER_APISIX}, + {"Docker Deployment", ReportIS::Tags::DEPLOYMENT_DOCKER} }; auto report_is_tag = strings_to_tags.find(tag); @@ -318,7 +319,8 @@ EnumArray TagAndEnumManagement::tags_translation_arr { "Playground", "apiDiscoveryCloudMessaging", "Nginx Proxy Manager", - "APISIX Server" + "APISIX Server", + "Docker Deployment" }; EnumArray TagAndEnumManagement::audience_team_translation { diff --git a/nodes/orchestration/main.cc b/nodes/orchestration/main.cc index 213a66b..8105c51 100755 --- a/nodes/orchestration/main.cc +++ b/nodes/orchestration/main.cc @@ -61,7 +61,6 @@ main(int argc, char **argv) { NodeComponents< OrchestrationStatus, - OrchestrationTools, PackageHandler, Downloader, ServiceController, @@ -70,6 +69,7 @@ main(int argc, char **argv) AgentDetailsReporter, DetailsResolver, OrchestrationComp, + OrchestrationTools, HealthChecker, HealthCheckManager, LocalPolicyMgmtGenerator diff --git a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc index 4879c06..39bbc37 100755 --- a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc +++ b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc @@ -92,8 +92,9 @@ enum class Service { HTTP_TRANSACTION_HANDLER, DEDICATED_NETWORK_HANDLER, HELLO_WORLD, - IDA, IDA_SAML, + IDA_IDN, + IDA_IDN_BG, IOT_ACCESS_CONTROL, HORIZON_TELEMETRY, @@ -169,8 +170,9 @@ getServiceString(const Service service) case (Service::LOGGER_SDWAN): return "logger-sdwan"; case (Service::IOT_WLP): return "workload-protection"; case (Service::HELLO_WORLD): return "hello-world"; - case (Service::IDA): return "identity-awareness"; case (Service::IDA_SAML): return "ida-saml"; + case (Service::IDA_IDN): return "ida-idn"; + case (Service::IDA_IDN_BG): return "ida-idn-bg"; case (Service::IOT_ACCESS_CONTROL): return "iot-access-control"; case (Service::HORIZON_TELEMETRY): return "horizon-telemetry"; default: @@ -341,16 +343,21 @@ getServiceConfig (const Service service) filesystem_path + "/conf/cp-nano-cpview-metric-provider-debug-conf.json", log_files_path + "/nano_agent/cp-nano-cpview-metric-provider.dbg" ); - case (Service::IDA): - return ServiceConfig( - filesystem_path + "/conf/cp-nano-ida-debug-conf.json", - log_files_path + "/nano_agent/cp-nano-ida.dbg" - ); case (Service::IDA_SAML): return ServiceConfig( filesystem_path + "/conf/cp-nano-ida-saml-debug-conf.json", log_files_path + "/nano_agent/cp-nano-ida-saml.dbg" ); + case (Service::IDA_IDN): + return ServiceConfig( + filesystem_path + "/conf/cp-nano-ida-idn-debug-conf.json", + log_files_path + "/nano_agent/cp-nano-ida-idn.dbg" + ); + case (Service::IDA_IDN_BG): + return ServiceConfig( + filesystem_path + "/conf/cp-nano-ida-idn-bg-debug-conf.json", + log_files_path + "/nano_agent/cp-nano-ida-idn-bg.dbg" + ); case (Service::HELLO_WORLD): return ServiceConfig( filesystem_path + "/conf/cp-nano-hello-world-conf.json", @@ -1291,10 +1298,12 @@ extractServices(const vector &args) services.push_back(Service::CPVIEW_METRIC_PROVIDER); } else if (getServiceString(Service::IOT_WLP).find(maybe_service) == 0) { services.push_back(Service::IOT_WLP); - } else if (getServiceString(Service::IDA).find(maybe_service) == 0) { - services.push_back(Service::IDA); } else if (getServiceString(Service::IDA_SAML).find(maybe_service) == 0) { services.push_back(Service::IDA_SAML); + } else if (getServiceString(Service::IDA_IDN).find(maybe_service) == 0) { + services.push_back(Service::IDA_IDN); + } else if (getServiceString(Service::IDA_IDN_BG).find(maybe_service) == 0) { + services.push_back(Service::IDA_IDN_BG); } else if (getServiceString(Service::IOT_ACCESS_CONTROL).find(maybe_service) == 0) { services.push_back(Service::IOT_ACCESS_CONTROL); } else if (getServiceString(Service::HORIZON_TELEMETRY).find(maybe_service) == 0) { diff --git a/nodes/orchestration/package/open-appsec-cloud-mgmt b/nodes/orchestration/package/open-appsec-cloud-mgmt index 6fdd2c1..4d1dc2d 100755 --- a/nodes/orchestration/package/open-appsec-cloud-mgmt +++ b/nodes/orchestration/package/open-appsec-cloud-mgmt @@ -153,7 +153,7 @@ poll_for_status_file() until [ ${attempt_counter} -eq ${max_attempts} ]; do if [ ${attempt_counter} -eq ${max_attempts} ];then echo "Max attempts reached" - exit 1 + return 1 fi file_exists="$(curl -s -w "%{http_code}\n" --request GET -H \ "user-agent: Infinity Next (a7030abf93a4c13)" -H \ @@ -165,7 +165,7 @@ poll_for_status_file() FAILURE=$(echo $file_exists | grep "false") if [ ! -z "$FAILURE" ]; then echo "Failed creating the Assets: $(echo $file_exists | cut -c27- | cut -d '"' -f 1)" - exit 1 + return 1 else echo "." return 0 @@ -177,7 +177,7 @@ poll_for_status_file() fi done echo "Error: Status file was not generated" - exit 1 + return 1 } upload_policy_to_the_cloud() @@ -215,7 +215,7 @@ upload_policy_to_the_cloud() STATUS="SUCCESS" exit 0 fi - if [ "$STATUS" = "FAILURE" ]; then + if [ "$STATUS" = "FAILURE" ]; then echo "Failed to upload policy to the cloud" exit 1 fi diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index 4213be4..9bb0b98 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -18,6 +18,7 @@ WATCHDOG_PATH="watchdog" SERVICE_PATH="orchestration" DBG_FILE_PATH="${LOG_PATH}/cp-nano-orchestration.dbg" ENV_DETAILS_FILE="${CONF_PATH}/environment-details.cfg" +TMP_FOLDER="/tmp" WATCHDOG_MAX_ROTATIONS=10 WATCHDOG_MAX_FILE_SIZE=4096 FORCE_CLEAN_FLAG='^(--force-clean|-f)$' @@ -76,6 +77,7 @@ var_gaia_release=1 var_mds_release=1 var_alpine_release=1 var_which_cmd_exists=0 +var_cloud_storage= if [ -f /.dockerenv ]; then var_container_mode=true @@ -305,6 +307,7 @@ while true; do NANO_AGENT_SERVICE_FILE="${NANO_AGENT_SERVICE_NAME}.service" VS_LIB_SUB_FOLDER="/vs${VS_ID}" LOG_FILE_PATH="${LOG_FILE_PATH}/vs${VS_ID}" + TMP_FOLDER="${TMP_FOLDER}/vs${VS_ID}" elif [ "$1" = "--log_files_path" ]; then shift var=$1 @@ -321,6 +324,9 @@ while true; do continue elif [ "$1" = "--skip_registration" ]; then var_skip_registration=true + elif [ "$1" = "--cloud-storage" ]; then + shift + var_cloud_storage=$1 elif echo "$1" | grep -q ${FORCE_CLEAN_FLAG}; then var_upgrade_mode= elif echo "$1" | grep -q ${DEBUG_FLAG}; then @@ -349,6 +355,7 @@ if [ -z "$VS_ID" ]; then NANO_AGENT_SERVICE_FILE="${NANO_AGENT_SERVICE_NAME}.service" VS_LIB_SUB_FOLDER="/vs${VS_ID}" LOG_FILE_PATH="${LOG_FILE_PATH}/vs${VS_ID}" + TMP_FOLDER="${TMP_FOLDER}/vs${VS_ID}" fi fi @@ -474,9 +481,9 @@ update_cloudguard_appsec_manifest() return fi - selected_cloudguard_appsec_manifest_path="/tmp/cloudguard_appsec_manifest.json" + selected_cloudguard_appsec_manifest_path="${TMP_FOLDER}/cloudguard_appsec_manifest.json" if [ "${DOCKER_RPM_ENABLED}" = "false" ]; then - selected_cloudguard_appsec_manifest_path="/tmp/self_managed_cloudguard_appsec_manifest.json" + selected_cloudguard_appsec_manifest_path="${TMP_FOLDER}/self_managed_cloudguard_appsec_manifest.json" fi if [ ! -f "$selected_cloudguard_appsec_manifest_path" ]; then @@ -490,6 +497,43 @@ update_cloudguard_appsec_manifest() sed "s/namespace/${fog_host}/g" ${cloudguard_appsec_manifest_path} > "${FILESYSTEM_PATH}/${CONF_PATH}/manifest.json" } +set_cloud_storage() +{ + CP_NANO_CLOUD_STORAGE_CONFIG_PATH="${TMP_FOLDER}/cp-nano-cloud-storage.conf" + if [ ! -f "${CP_NANO_CLOUD_STORAGE_CONFIG_PATH}" ] && [ -z "${var_cloud_storage}" ]; then + sed -i '/CLOUD_STORAGE_ENABLED/d' ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} + return + fi + + touch ${CP_NANO_CLOUD_STORAGE_CONFIG_PATH} + + if [ -n "${var_token}" ]; then + if ! command -v "openssl" 1> /dev/null 2> /dev/null && command -v "cpopenssl" 1> /dev/null 2> /dev/null; then + ln -s "$(command -v cpopenssl)" "$(dirname $(command -v cpopenssl))/openssl" + fi + HASHED_TOKEN=$(openssl passwd -6 -salt cp-cloud-key ${var_token}) + if grep -q "HASHED_TOKEN" "${CP_NANO_CLOUD_STORAGE_CONFIG_PATH}"; then + sed -i "/HASHED_TOKEN/c\HASHED_TOKEN='${HASHED_TOKEN}'" "${CP_NANO_CLOUD_STORAGE_CONFIG_PATH}" + else + echo "HASHED_TOKEN='${HASHED_TOKEN}'" >> ${CP_NANO_CLOUD_STORAGE_CONFIG_PATH} + fi + fi + + if [ -n "${var_cloud_storage}" ]; then + if grep -q "CLOUD_STORAGE" "${CP_NANO_CLOUD_STORAGE_CONFIG_PATH}"; then + sed -i "/CLOUD_STORAGE/c\CLOUD_STORAGE=${var_cloud_storage}" "${CP_NANO_CLOUD_STORAGE_CONFIG_PATH}" + else + echo "CLOUD_STORAGE=${var_cloud_storage}" >> "${CP_NANO_CLOUD_STORAGE_CONFIG_PATH}" + fi + fi + + if grep -q "CLOUD_STORAGE_ENABLED" "${FILESYSTEM_PATH}/${ENV_DETAILS_FILE}"; then + sed -i "/CLOUD_STORAGE_ENABLED/c\export CLOUD_STORAGE_ENABLED=true" "${FILESYSTEM_PATH}/${ENV_DETAILS_FILE}" + else + echo "export CLOUD_STORAGE_ENABLED=true" >> "${FILESYSTEM_PATH}/${ENV_DETAILS_FILE}" + fi +} + install_watchdog_gaia() { watchdog_pm_name="cp-nano-watchdog" @@ -497,7 +541,7 @@ install_watchdog_gaia() watchdog_pm_name="cp-nano-watchdog-vs${VS_ID}" cp_exec "ln -s ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/${watchdog_pm_name}" fi - + # verify that DB is clean from cp-nano-watchdog tellpm ${watchdog_pm_name} dbset process:${watchdog_pm_name} @@ -529,7 +573,6 @@ install_watchdog() if [ "$IS_K8S_ENV" = "true" ]; then cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh" fi - return fi cp_print "Installing the watchdog" ${FORCE_STDOUT} @@ -577,7 +620,7 @@ install_watchdog() echo "ExecStart=ip netns exec CTX0000${VS_ID} ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} fi echo "Environment=\"FILESYSTEM_PATH=${FILESYSTEM_PATH}\"" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} - + cp_exec "systemctl daemon-reload" cp_exec "systemctl enable nano_agent" else @@ -847,7 +890,6 @@ uninstall_messaging_proxy_if_needed() install_orchestration() { INSTALLATION_TIME=$(date) - if [ "$is_smb" != "1" ]; then cp_exec "mkdir -p ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}" else @@ -899,6 +941,11 @@ install_orchestration() add_uninstall_script cp_exec "cp -f certificate/ngen.body.crt ${FILESYSTEM_PATH}/${CERTS_PATH}/fog.pem" + if [ -n ${OTP_TOKEN} ]; then + cp_print "Saving authentication token to file" + printf '{\n "registration type": "token",\n "registration data": "%b"\n}' "$OTP_TOKEN" | ${FILESYSTEM_PATH}/${BIN_PATH}/${CP_NANO_BASE64} -e > ${FILESYSTEM_PATH}/${CONF_PATH}/registration-data.json + fi + [ -f "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" ] && . "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" previous_mode=$(cat ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg | grep "orchestration-mode" | cut -d = -f 3 | sed 's/"//') @@ -931,6 +978,8 @@ install_orchestration() cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh" fi + set_cloud_storage + cp_print "Upgrade to latest" uninstall_messaging_proxy_if_needed @@ -1016,11 +1065,15 @@ install_orchestration() done fi - cp_print "Building the default policy json" - echo '{"'$ORCHESTRATION_NAME'": { "fog-address":"'$var_fog_address'", ' > ${FILESYSTEM_PATH}/${CONF_PATH}/policy.json - echo '"pulling-interval":'$var_sleep_interval', ' >> ${FILESYSTEM_PATH}/${CONF_PATH}/policy.json - echo '"error-pulling-interval":'$var_error_sleep_interval'},' >> ${FILESYSTEM_PATH}/${CONF_PATH}/policy.json - echo '"registration-data": { "email-address": "'$var_email'", "registered-server": "'$var_server'"}}' >> ${FILESYSTEM_PATH}/${CONF_PATH}/policy.json + if [ ! -f ${FILESYSTEM_PATH}/${CONF_PATH}/policy.json ] ; then + cp_print "Building the default policy json" + echo '{"'$ORCHESTRATION_NAME'": { "fog-address":"'$var_fog_address'", ' > ${FILESYSTEM_PATH}/${CONF_PATH}/policy.json + echo '"pulling-interval":'$var_sleep_interval', ' >> ${FILESYSTEM_PATH}/${CONF_PATH}/policy.json + echo '"error-pulling-interval":'$var_error_sleep_interval'},' >> ${FILESYSTEM_PATH}/${CONF_PATH}/policy.json + echo '"registration-data": { "email-address": "'$var_email'", "registered-server": "'$var_server'"}}' >> ${FILESYSTEM_PATH}/${CONF_PATH}/policy.json + fi + + set_cloud_storage copy_orchestration_executable copy_k8s_executable