From a59f079ef7a53f556301b807f0bba4be594e1440 Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Wed, 5 Jul 2023 23:32:39 +0000 Subject: [PATCH 1/3] Jul 5th update --- .../core/include/common_is/kdebug_flags.h | 1 + components/CMakeLists.txt | 1 - .../nginx_attachment_opaque.cc | 4 +- components/include/http_transaction_data.h | 21 + components/include/i_service_controller.h | 2 + components/include/orchestration_comp.h | 2 - .../messaging_downloader/CMakeLists.txt | 4 - .../messaging_downloader_client.cc | 230 ----------- .../messaging_downloader_server.cc | 375 ----------------- .../messaging_downloader_ut/CMakeLists.txt | 2 - .../downloader_client_ut/CMakeLists.txt | 9 - .../downloader_client_ut.cc | 113 ----- .../downloader_server_ut/CMakeLists.txt | 9 - .../downloader_server_ut.cc | 304 -------------- .../layer_7_access_control.cc | 3 +- .../layer_7_access_control_ut.cc | 6 +- .../checkpoint_product_handlers.h | 16 + .../details_resolver_impl.h | 5 + .../orchestration/include/get_status_rest.h | 1 - .../include/mock/mock_messaging_downloader.h | 39 -- .../include/mock/mock_service_controller.h | 4 + .../manifest_controller_ut.cc | 1 + .../manifest_controller/manifest_handler.cc | 49 ++- .../orchestration/orchestration_comp.cc | 176 ++++++-- .../orchestration_multitenant_ut.cc | 29 +- .../orchestration_ut/orchestration_ut.cc | 255 +++++++++++- .../service_controller/service_controller.cc | 89 ++-- .../service_controller_ut.cc | 199 +++++++-- .../waap/waap_clib/ConfidenceCalculator.cc | 6 +- .../waap_clib/TrustedSourcesConfidence.cc | 2 +- .../security_apps/waap/waap_clib/Waf2Util.h | 18 + .../security_apps/waap/waap_component_impl.cc | 4 +- components/signal_handler/signal_handler.cc | 64 +-- .../http_transaction_data.cc | 71 +++- .../http_transaction_data_ut.cc | 44 ++ core/config/config.cc | 58 ++- core/config/config_globals.cc | 6 + .../attachments/nginx_attachment_common.h | 12 + core/include/general/intelligence_comp_v2.h | 4 + core/include/general/tenant_manager.h | 4 +- .../interfaces/i_intelligence_is_v2.h | 24 +- .../interfaces/i_messaging_downloader.h | 38 -- .../services_sdk/interfaces/i_rest_api.h | 2 + .../interfaces/i_tenant_manager.h | 8 +- .../intelligence_query_v2.h | 11 +- .../intelligence_query_v2_impl.h | 11 +- .../intelligence_types_v2.h | 4 +- .../intelligence_is_v2/json_stream.h | 35 ++ .../intelligence_is_v2/query_filter_v2.h | 29 +- .../intelligence_is_v2/query_request_v2.h | 15 + .../interfaces/mock/mock_rest_api.h | 1 + .../interfaces/mock/mock_tenant_manager.h | 6 +- core/include/services_sdk/resources/config.h | 1 + .../resources/config/config_impl.h | 2 +- .../services_sdk/resources/config/i_config.h | 4 +- .../services_sdk/resources/debug_flags.h | 2 + .../resources/intelligence_invalidation.h | 74 ++++ .../resources/report/base_field.h | 57 ++- .../resources/report/report_enums.h | 2 + .../utilities/customized_cereal_multimap.h | 44 +- core/intelligence_is_v2/CMakeLists.txt | 14 +- .../intelligence_comp_v2.cc | 386 +++++++++++++++++- .../intelligence_is_v2_ut/CMakeLists.txt | 14 +- .../intelligence_comp_v2_ut.cc | 188 ++++++++- .../query_request_v2_ut.cc | 151 ++++++- .../intelligence_types_v2.cc | 2 + core/intelligence_is_v2/invalidation.cc | 240 +++++++++++ core/intelligence_is_v2/json_stream.cc | 50 +++ core/intelligence_is_v2/query_filter_v2.cc | 114 ++++-- core/intelligence_is_v2/query_request_v2.cc | 22 + core/logging/cef_stream.cc | 3 + core/logging/logging_ut/logging_ut.cc | 47 +++ core/logging/syslog_stream.cc | 3 + core/message/message.cc | 32 ++ core/report/tag_and_enum_management.cc | 8 +- core/rest/rest_conn.cc | 2 +- core/rest/rest_server.cc | 5 +- core/rest/rest_ut/rest_config_ut.cc | 2 + core/rest/rest_ut/rest_schema_ut.cc | 35 ++ core/tenant_manager/tenant_manager.cc | 159 ++------ nodes/orchestration/CMakeLists.txt | 1 - nodes/orchestration/main.cc | 4 - .../package/cpnano_debug/cpnano_debug.cc | 9 - .../orchestration/package/open-appsec-ctl.sh | 105 +---- .../package/orchestration_package.sh | 34 +- 85 files changed, 2488 insertions(+), 1754 deletions(-) delete mode 100755 components/messaging_downloader/CMakeLists.txt delete mode 100755 components/messaging_downloader/messaging_downloader_client.cc delete mode 100755 components/messaging_downloader/messaging_downloader_server.cc delete mode 100755 components/messaging_downloader/messaging_downloader_ut/CMakeLists.txt delete mode 100755 components/messaging_downloader/messaging_downloader_ut/downloader_client_ut/CMakeLists.txt delete mode 100755 components/messaging_downloader/messaging_downloader_ut/downloader_client_ut/downloader_client_ut.cc delete mode 100755 components/messaging_downloader/messaging_downloader_ut/downloader_server_ut/CMakeLists.txt delete mode 100755 components/messaging_downloader/messaging_downloader_ut/downloader_server_ut/downloader_server_ut.cc delete mode 100755 components/security_apps/orchestration/include/mock/mock_messaging_downloader.h delete mode 100755 core/include/services_sdk/interfaces/i_messaging_downloader.h create mode 100644 core/include/services_sdk/interfaces/intelligence_is_v2/json_stream.h create mode 100644 core/include/services_sdk/resources/intelligence_invalidation.h create mode 100644 core/intelligence_is_v2/invalidation.cc create mode 100644 core/intelligence_is_v2/json_stream.cc diff --git a/attachments/kernel_modules/core/include/common_is/kdebug_flags.h b/attachments/kernel_modules/core/include/common_is/kdebug_flags.h index ba8d61c..cb51568 100755 --- a/attachments/kernel_modules/core/include/common_is/kdebug_flags.h +++ b/attachments/kernel_modules/core/include/common_is/kdebug_flags.h @@ -31,5 +31,6 @@ DEFINE_KDEBUG_FLAG(statefulValidation) DEFINE_KDEBUG_FLAG(statelessValidation) DEFINE_KDEBUG_FLAG(kernelMetric) DEFINE_KDEBUG_FLAG(tproxy) +DEFINE_KDEBUG_FLAG(tenantStats) #endif // DEFINE_KDEBUG_FLAG diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index 7d8bd27..99be4a8 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -5,7 +5,6 @@ add_subdirectory(signal_handler) add_subdirectory(gradual_deployment) add_subdirectory(packet) add_subdirectory(pending_key) -add_subdirectory(messaging_downloader) add_subdirectory(health_check_manager) add_subdirectory(utils) diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc b/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc index 2e5d22f..475a688 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc @@ -52,14 +52,14 @@ NginxAttachmentOpaque::NginxAttachmentOpaque(HttpTransactionData _transaction_da ctx.registerValue("eventReferenceId", uuid, EnvKeyAttr::LogSection::DATA); ctx.registerValue(HttpTransactionData::http_proto_ctx, transaction_data.getHttpProtocol()); ctx.registerValue(HttpTransactionData::method_ctx, transaction_data.getHttpMethod()); - ctx.registerValue(HttpTransactionData::host_name_ctx, transaction_data.getDestinationHost()); + ctx.registerValue(HttpTransactionData::host_name_ctx, transaction_data.getParsedHost()); ctx.registerValue(HttpTransactionData::listening_port_ctx, transaction_data.getListeningPort()); ctx.registerValue(HttpTransactionData::listening_ip_ctx, transaction_data.getListeningIP()); ctx.registerValue(HttpTransactionData::client_ip_ctx, transaction_data.getSourceIP()); ctx.registerValue(HttpTransactionData::client_port_ctx, transaction_data.getSourcePort()); ctx.registerFunc(HttpTransactionData::source_identifier, [this](){ return source_identifier; }); - ctx.registerValue(HttpTransactionData::uri_ctx, transaction_data.getURI()); + ctx.registerValue(HttpTransactionData::uri_ctx, transaction_data.getParsedURI()); auto decoder = makeVirtualContainer>(transaction_data.getURI()); string decoded_url(decoder.begin(), decoder.end()); auto question_mark_location = decoded_url.find('?'); diff --git a/components/include/http_transaction_data.h b/components/include/http_transaction_data.h index 5fff08b..33ad324 100755 --- a/components/include/http_transaction_data.h +++ b/components/include/http_transaction_data.h @@ -43,6 +43,19 @@ public: uint16_t client_port ); + HttpTransactionData ( + std::string http_proto, + std::string method, + std::string host_name, + std::string parsed_host, + IPAddr listening_ip, + uint16_t listening_port, + std::string uri, + std::string parsed_uri, + IPAddr client_ip, + uint16_t client_port + ); + // LCOV_EXCL_START - sync functions, can only be tested once the sync module exists template void @@ -52,9 +65,11 @@ public: http_proto, method, host_name, + parsed_host, listening_ip, listening_port, uri, + parsed_uri, client_ip, client_port, response_content_encoding @@ -69,9 +84,11 @@ public: http_proto, method, host_name, + parsed_host, listening_ip, listening_port, uri, + parsed_uri, client_ip, client_port, response_content_encoding @@ -86,8 +103,10 @@ public: const IPAddr & getListeningIP() const { return listening_ip; } uint16_t getListeningPort() const { return listening_port; } const std::string & getDestinationHost() const { return host_name; } + const std::string & getParsedHost() const { return parsed_host; } const std::string & getHttpProtocol() const { return http_proto; } const std::string & getURI() const { return uri; } + const std::string & getParsedURI() const { return parsed_uri; } const std::string & getHttpMethod() const { return method; } void print(std::ostream &out_stream) const; @@ -124,9 +143,11 @@ private: std::string http_proto; std::string method = "GET"; std::string host_name; + std::string parsed_host; IPAddr listening_ip; uint16_t listening_port; std::string uri; + std::string parsed_uri; IPAddr client_ip; uint16_t client_port; bool is_request; diff --git a/components/include/i_service_controller.h b/components/include/i_service_controller.h index a2f5070..e32786c 100755 --- a/components/include/i_service_controller.h +++ b/components/include/i_service_controller.h @@ -51,6 +51,8 @@ public: virtual void clearFailedServices() = 0; + virtual std::set && moveChangedPolicies() = 0; + virtual bool isServiceInstalled(const std::string &service_name) = 0; virtual void registerServiceConfig( diff --git a/components/include/orchestration_comp.h b/components/include/orchestration_comp.h index e5c8c16..2ab061d 100755 --- a/components/include/orchestration_comp.h +++ b/components/include/orchestration_comp.h @@ -14,7 +14,6 @@ #ifndef __ORCHESTRATION_COMP_H__ #define __ORCHESTRATION_COMP_H__ -#include "i_messaging_downloader.h" #include "i_messaging.h" #include "i_mainloop.h" #include "i_shell_cmd.h" @@ -49,7 +48,6 @@ class OrchestrationComp Singleton::Consume, Singleton::Consume, Singleton::Consume, - Singleton::Consume, Singleton::Consume, Singleton::Consume, Singleton::Consume, diff --git a/components/messaging_downloader/CMakeLists.txt b/components/messaging_downloader/CMakeLists.txt deleted file mode 100755 index 11acfc3..0000000 --- a/components/messaging_downloader/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_library(messaging_downloader_server messaging_downloader_server.cc) -add_library(messaging_downloader_client messaging_downloader_client.cc) - -add_subdirectory(messaging_downloader_ut) diff --git a/components/messaging_downloader/messaging_downloader_client.cc b/components/messaging_downloader/messaging_downloader_client.cc deleted file mode 100755 index 1f2887b..0000000 --- a/components/messaging_downloader/messaging_downloader_client.cc +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "messaging_downloader_client.h" -#include "i_messaging.h" -#include "config.h" -#include "rest.h" - -USE_DEBUG_FLAG(D_COMMUNICATION); - -using namespace std; - -class MessagingDownloaderClientRequest : public ClientRest -{ -public: - MessagingDownloaderClientRequest() - { - file_name = ""; - url = ""; - port = 80; - response_port = 0; - status = false; - } - - MessagingDownloaderClientRequest( - const string &_file_name, - const string &_url, - const unsigned int _port, - const unsigned int _response_port - ) : - file_name(_file_name), - url(_url), - port(_port), - response_port(_response_port), - status(false) - {} - - bool getStatus() const { return status.get(); } - const string & getUuid() const { return uuid.get(); } - - C2S_PARAM(string, file_name); - C2S_PARAM(string, url); - C2S_PARAM(unsigned int, port); - C2S_PARAM(unsigned int, response_port); - - S2C_PARAM(string, uuid); - S2C_PARAM(bool, status); -}; - -class DownloaderCbHandler -{ -public: - void - addCallback(const string &uuid, I_MessagingDownloader::OnCompleteCB &cb) - { - DownloaderCbHandler::uuid_to_cb[uuid] = cb; - } - - static void - handleDownloadCB(const string &uuid, Maybe &downloaded_file) - { - dbgTrace(D_COMMUNICATION) << "Handling downloading complete callback. UUID: " << uuid; - if(DownloaderCbHandler::uuid_to_cb.find(uuid) == DownloaderCbHandler::uuid_to_cb.end()) { - dbgWarning(D_COMMUNICATION) << "Failed to execute download completion callback."; - return; - } - if (DownloaderCbHandler::uuid_to_cb.at(uuid) != nullptr) { - DownloaderCbHandler::uuid_to_cb.at(uuid)(downloaded_file); - DownloaderCbHandler::uuid_to_cb.erase(uuid); - } else { - string curr_status; - if (downloaded_file.ok()) { - curr_status = ". File path: " + downloaded_file.unpack(); - } else { - curr_status = ". Error: " + downloaded_file.getErr(); - } - dbgWarning(D_COMMUNICATION) - << "Illegal download completion callback for downloading process with UUID: " - << uuid - << curr_status; - } - dbgTrace(D_COMMUNICATION) << "Successfully handled the downloading complete callback. UUID: " << uuid; - } - - static unordered_map uuid_to_cb; -}; - -unordered_map DownloaderCbHandler::uuid_to_cb; - -class MessagingDownloaderClientRes : public ServerRest -{ -public: - void - doCall() override - { - dbgTrace(D_COMMUNICATION) << "Received response from the downloading server."; - if (status.get() && filepath.isActive()) { - Maybe response(filepath.get()); - DownloaderCbHandler::handleDownloadCB(uuid.get(), response); - } else { - if (!error.isActive()) error = "unknown error"; - dbgWarning(D_COMMUNICATION) << "Failed to download. Error: " << error.get(); - Maybe response = genError(error.get()); - DownloaderCbHandler::handleDownloadCB(uuid.get(), response); - } - } - - C2S_PARAM(string, uuid); - C2S_PARAM(bool, status); - C2S_OPTIONAL_PARAM(string, filepath); - C2S_OPTIONAL_PARAM(string, error); -}; - -class MessagingDownloaderClient::Impl : Singleton::Provide::From -{ -public: - void - init() - { - i_msg = Singleton::Consume::by(); - Singleton::Consume::by()->addRestCall( - RestAction::SHOW, - "download-status" - ); - } - - void - fini() - { - i_msg = nullptr; - } - - bool - downloadFile( - const string &file_name, - const string &url, - I_MessagingDownloader::OnCompleteCB cb = nullptr, - const unsigned int port = 0 - ) override - { - dbgTrace(D_COMMUNICATION) - << "Processing new download request." - << "File name: " - << file_name - << "URL: " - << url; - - auto response_port = Singleton::Consume::by()->get( - "Listening Port" - ); - - if (!response_port.ok()) { - dbgWarning(D_COMMUNICATION) << "Failed to get the service listening port."; - return false; - } - - vector download_ports = { - getConfigurationWithDefault(8164, "Downloader", "Downloader Primary Port"), - getConfigurationWithDefault(8167, "Downloader", "Downloader Secondary Port") - }; - - MessagingDownloaderClientRequest download_obj( - file_name, - url, - port, - response_port.unpack() - ); - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::EXPECT_REPLY); - if (i_msg != nullptr) { - dbgTrace(D_COMMUNICATION) << "Sending request to the downloading service."; - bool res = false; - for (int port: download_ports) { - dbgTrace(D_COMMUNICATION) << "Trying to request downloading with downloading service port " << port; - res = i_msg->sendObject( - download_obj, - I_Messaging::Method::POST, - "127.0.0.1", - port, - conn_flags, - "/add-download-file" - ); - if (res) break; - } - - if (!res) { - dbgInfo(D_COMMUNICATION) << "Failed to request for file downloading"; - return false; - } - dbgTrace(D_COMMUNICATION) << "Successfully requested for downloading."; - cb_handler.addCallback(download_obj.getUuid(), cb); - } else { - dbgDebug(D_COMMUNICATION) << "Failed to request downloading. Illegal messaging infrastructure."; - } - return download_obj.getStatus(); - } - -private: - I_Messaging *i_msg; - DownloaderCbHandler cb_handler; -}; - -MessagingDownloaderClient::MessagingDownloaderClient() - : - Component("MessagingDownloaderClient"), - pimpl(make_unique()) -{} -MessagingDownloaderClient::~MessagingDownloaderClient() {} - -void MessagingDownloaderClient::init() { pimpl->init(); } -void MessagingDownloaderClient::fini() { pimpl->fini(); } - -void -MessagingDownloaderClient::preload() -{ - registerExpectedConfiguration("Downloader", "Downloader Primary Port"); - registerExpectedConfiguration("Downloader", "Downloader Secondary Port"); -}; diff --git a/components/messaging_downloader/messaging_downloader_server.cc b/components/messaging_downloader/messaging_downloader_server.cc deleted file mode 100755 index ffb50b4..0000000 --- a/components/messaging_downloader/messaging_downloader_server.cc +++ /dev/null @@ -1,375 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "messaging_downloader_server.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "i_messaging.h" -#include "rest.h" -#include "config.h" -#include "url_parser.h" -#include "agent_core_utilities.h" - -USE_DEBUG_FLAG(D_COMMUNICATION); - -using namespace std; - -class MessagingDownloaderResponser : public ClientRest -{ -public: - MessagingDownloaderResponser()=delete; - - MessagingDownloaderResponser(string &_uuid, const Maybe &_filepath) - : - uuid(_uuid), - status(_filepath.ok()) - { - if (_filepath.ok()) { - filepath = _filepath.unpack(); - } else { - error = _filepath.getErr(); - } - } - C2S_PARAM(string, uuid); - C2S_PARAM(bool, status); - C2S_OPTIONAL_PARAM(string, filepath) - C2S_OPTIONAL_PARAM(string, error) -}; - -class MessagingDownloaderReceiver : public ServerRest -{ -public: - void - doCall() override - { - dbgTrace(D_COMMUNICATION) << "Received new downloading request."; - - stringstream uuid_ss; - uuid_ss << boost::uuids::random_generator()(); - uuid = uuid_ss.str(); - - if (!port.isActive()) { - dbgTrace(D_COMMUNICATION) << "Request does not contain explicit port."; - port = 0; - } - - dbgInfo(D_COMMUNICATION) - << "Downloading a file and using the next parameters: " - << "file_name: " - << file_name.get() - << ", url: " - << url.get() - << ", uuid: " - << uuid.get() - << ", port: " - << port.get() - << ", notification port: " - << response_port.get(); - - unsigned int response_port_cap = response_port.get(); - string uuid_capture = uuid.get(); - status = Singleton::Consume::from()->downloadFile( - file_name.get(), - url.get(), - [uuid_capture, response_port_cap](const Maybe &downloaded_file) mutable - { - Flags conn_flags; - MessagingDownloaderResponser res(uuid_capture, downloaded_file); - dbgTrace(D_COMMUNICATION) << "Sending the download status to the client."; - bool res_status = Singleton::Consume::by()->sendNoReplyObject( - res, - I_Messaging::Method::POST, - "127.0.0.1", - response_port_cap, - conn_flags, - "/show-download-status" - ); - if (!res_status) { - dbgInfo(D_COMMUNICATION) << "Failed to send the download status."; - } else { - dbgDebug(D_COMMUNICATION) - << "Successfully sent the download status. Notification port: " - << response_port_cap - << ", Status: " - << downloaded_file.ok(); - } - }, - port.get() - ); - } - - C2S_PARAM(string, file_name); - C2S_PARAM(string, url); - C2S_PARAM(int, response_port); - C2S_PARAM(int, port); - S2C_PARAM(string, uuid); - S2C_PARAM(bool, status); -}; - -class DownloadingInstance -{ -public: - DownloadingInstance()=default; - - DownloadingInstance( - const string &_file_name, - const string &_url, - const unsigned int _port - ) : - file_name(_file_name), - url(_url), - port(_port), - url_parser(_url) - { - parseURL(); - } - - Maybe - genJson() const - { - return string(""); - } - - bool - loadJson(const string &_body) - { - body = vector(_body.begin(), _body.end()); - return true; - } - - const vector & - getResponse() const - { - return body; - } - - bool - operator==(const DownloadingInstance &other) const - { - return file_name == other.file_name && - host == other.host && - url == other.url && - port == other.port && - is_secure == other.is_secure && - origin_is_fog == other.origin_is_fog; - } - - bool - operator<(const DownloadingInstance &other) const - { - return file_name < other.file_name || - host < other.host || - url < other.url || - port < other.port || - is_secure < other.is_secure || - origin_is_fog < other.origin_is_fog; - } - - const string & getFileName() const { return file_name; } - const string & getHost() const { return host; } - const string & getUrl() const { return url; } - unsigned int getPort() const { return port; } - bool getIsSecure() const { return is_secure; } - bool getIsFogOrigin() const { return origin_is_fog; } - -private: - void - parseURL() - { - dbgTrace(D_COMMUNICATION) << "Parsing the URL to extract the relevant info. URL: " << url; - origin_is_fog = false; - auto maybe_host = url_parser.getBaseURL(); - if (!maybe_host.ok()) { - dbgWarning(D_COMMUNICATION) << "Failed to parse the URL"; - return; - } - host = maybe_host.unpack(); - is_secure = url_parser.isOverSSL(); - if (port == 0 && url_parser.getPort() != "") { - try { - port = stoi(url_parser.getPort()); - } catch (exception &e) { - port = 443; - dbgInfo(D_COMMUNICATION) - << "Failed to parse the port for the downloading request. Error " - << e.what() - << ". Using the default port " - << port; - } - } else { - dbgTrace(D_COMMUNICATION) << "Using explicitly defined port. Port: " << port; - } - - I_AgentDetails *agent_details = Singleton::Consume::by(); - if (agent_details->getFogDomain().ok()) { - string fog_domain = agent_details->getFogDomain().unpack(); - if (host.find(fog_domain) != string::npos) { - origin_is_fog = true; - } - } else { - dbgTrace(D_COMMUNICATION) << "Failed to receive fog domain."; - } - } - - string file_name = ""; - string url = ""; - unsigned int port = 0; - URLParser url_parser; - vector body = {}; - string host = ""; - bool is_secure = true; - bool origin_is_fog = true; -}; - -class MessagingDownloaderServer::Impl : Singleton::Provide::From -{ -public: - void - init() - { - i_msg = Singleton::Consume::by(); - i_mainloop = Singleton::Consume::by(); - auto rest = Singleton::Consume::by(); - rest->addRestCall(RestAction::ADD, "download-file"); - string default_downloading_dir = "/tmp/cp_nano_downloader/"; - download_dir = getConfigurationWithDefault( - default_downloading_dir, - "Downloader", - "Downloading Directory" - ); - NGEN::Filesystem::makeDirRecursive(download_dir); - } - - void - fini() - { - i_msg = nullptr; - i_mainloop = nullptr; - } - - bool - downloadFile( - const string &file_name, - const string &url, - OnCompleteCB on_complete_func = nullptr, - const unsigned int port = 443 - ) override - { - dbgTrace(D_COMMUNICATION) << "Handling new download request. URL: " << url << ". File name: " << file_name; - DownloadingInstance req(file_name, url, port); - if (downloading_queue.find(req) != downloading_queue.end()) { - dbgInfo(D_COMMUNICATION) << "Failed to download the file. Similar download request already exists."; - return false; - } - if (!isValidPath(file_name)) { - dbgInfo(D_COMMUNICATION) << "Failed to validate the download path. Path: " << download_dir + file_name; - return false; - } - downloading_queue.insert(req); - - i_mainloop->addOneTimeRoutine( - I_MainLoop::RoutineType::RealTime, - [this, req, on_complete_func]() mutable - { - Flags conn_flags; - if (req.getIsSecure()) conn_flags.setFlag(MessageConnConfig::SECURE_CONN); - if (!req.getIsFogOrigin()) conn_flags.setFlag(MessageConnConfig::EXTERNAL); - auto on_exit = make_scope_exit([this, &req]() { downloading_queue.erase(req); } ); - bool response = i_msg->sendObject( - req, - I_Messaging::Method::GET, - req.getHost(), - req.getPort(), - conn_flags, - req.getUrl() - ); - if (response) { - dbgTrace(D_COMMUNICATION) << "Successfully received a response from the downloading file host."; - std::ofstream downloaded_file; - downloaded_file.open(download_dir + req.getFileName()); - if (!downloaded_file.is_open()) { - dbgInfo(D_COMMUNICATION) - << "Failed to download file. Error: Failed to open the file " - << req.getFileName(); - Maybe err = genError("Failed to open the file"); - on_complete_func(err); - if (i_mainloop != nullptr) i_mainloop->yield(true); - } - auto &res_body = req.getResponse(); - downloaded_file.write(res_body.data(), res_body.size()); - downloaded_file.close(); - dbgInfo(D_COMMUNICATION) << "Successfully downloaded the file. File name: " << req.getFileName(); - Maybe filepath = download_dir + req.getFileName(); - on_complete_func(filepath); - } else { - dbgInfo(D_COMMUNICATION) << "Failed to download file. File name: " << req.getFileName(); - Maybe err = genError("Failed during the downloading process."); - on_complete_func(err); - } - }, - "Download file routine for '" + file_name + "'", - false - ); - return true; - } - -private: - bool - isValidPath(const string &file_name) - { - struct stat info; - string file_to_download = download_dir + file_name; - dbgTrace(D_COMMUNICATION) << "Validating the downloading file path. Path: " << file_to_download; - if (stat(download_dir.c_str(), &info) != 0) { - dbgDebug(D_COMMUNICATION) << "Failed to access the downloading directory"; - return false; - } - if (stat(file_to_download.c_str(), &info) == 0) { - dbgDebug(D_COMMUNICATION) - << "The file with the name '" - << file_name - << "' is already exist in the downloading directory"; - return false; - } - return true; - } - - I_Messaging *i_msg; - I_MainLoop *i_mainloop; - string download_dir; - set downloading_queue; -}; - -MessagingDownloaderServer::MessagingDownloaderServer() - : - Component("MessagingDownloaderServer"), - pimpl(make_unique()) -{} - -MessagingDownloaderServer::~MessagingDownloaderServer() {} - -void MessagingDownloaderServer::init() { pimpl->init(); } -void MessagingDownloaderServer::fini() { pimpl->fini(); } - -void -MessagingDownloaderServer::preload() -{ - registerExpectedConfiguration("Downloader", "Downloading Directory"); -}; diff --git a/components/messaging_downloader/messaging_downloader_ut/CMakeLists.txt b/components/messaging_downloader/messaging_downloader_ut/CMakeLists.txt deleted file mode 100755 index 21833c3..0000000 --- a/components/messaging_downloader/messaging_downloader_ut/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_subdirectory(downloader_server_ut) -add_subdirectory(downloader_client_ut) \ No newline at end of file diff --git a/components/messaging_downloader/messaging_downloader_ut/downloader_client_ut/CMakeLists.txt b/components/messaging_downloader/messaging_downloader_ut/downloader_client_ut/CMakeLists.txt deleted file mode 100755 index 48a57dc..0000000 --- a/components/messaging_downloader/messaging_downloader_ut/downloader_client_ut/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -link_directories(${BOOST_ROOT}/lib) -include_directories(${CMAKE_SOURCE_DIR}/core/include) -link_directories(${CMAKE_BINARY_DIR}/core/include) - -add_unit_test( - downloader_client_ut - "downloader_client_ut.cc" - "singleton;messaging_downloader_client;time_proxy;mainloop;rest;metric;event_is;message;-lboost_context;agent_core_utilities;orchestration_modules;connkey;-lboost_regex;-lboost_filesystem;-lboost_system" -) diff --git a/components/messaging_downloader/messaging_downloader_ut/downloader_client_ut/downloader_client_ut.cc b/components/messaging_downloader/messaging_downloader_ut/downloader_client_ut/downloader_client_ut.cc deleted file mode 100755 index 1ceaa44..0000000 --- a/components/messaging_downloader/messaging_downloader_ut/downloader_client_ut/downloader_client_ut.cc +++ /dev/null @@ -1,113 +0,0 @@ -#include "messaging_downloader_client.h" - -#include - -#include "environment.h" -#include "singleton.h" -#include "config.h" -#include "config_component.h" -#include "mainloop.h" -#include "cptest.h" -#include "mock/mock_mainloop.h" -#include "mock/mock_messaging.h" -#include "mock/mock_rest_api.h" -#include "mock/mock_agent_details.h" -#include "mock/mock_time_get.h" - -using namespace std; -using namespace testing; - -class MessagingDownloaderClientTest : public Test -{ -public: - MessagingDownloaderClientTest() - { - EXPECT_CALL( - rest, - mockRestCall(RestAction::SHOW, "download-status", _) - ).WillOnce(WithArg<2>(Invoke(this, &MessagingDownloaderClientTest::restHandler))); - - EXPECT_CALL(rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true)); - - Debug::setUnitTestFlag(D_COMMUNICATION, Debug::DebugLevel::TRACE); - Debug::setNewDefaultStdout(&capture_debug); - - messaging_downloader.preload(); - env.preload(); - env.init(); - messaging_downloader.init(); - } - - ~MessagingDownloaderClientTest() - { - boost::filesystem::remove_all("/tmp/test_download_dir/"); - messaging_downloader.fini(); - } - - bool - restHandler(const unique_ptr &rest_ptr) - { - rest_handler = rest_ptr->getRest(); - return true; - } - - unique_ptr rest_handler; - ostringstream capture_debug; - I_MainLoop::Routine downloading_routine; - MessagingDownloaderClient messaging_downloader; - NiceMock mock_time; - NiceMock mock_agent_details; - StrictMock mock_msg; - StrictMock rest; - StrictMock mock_ml; - ::Environment env; - ConfigComponent conf; -}; - -TEST_F(MessagingDownloaderClientTest, do_nothing) -{ -} - -TEST_F(MessagingDownloaderClientTest, request_download) -{ - string file_name = "test_file"; - string url = "https://download_test.com/test_download"; - Singleton::Consume::by()->registerValue("Listening Port", 6464); - - stringstream ss; - ss << "{\n \"file_name\": \"" << file_name << "\"," - << "\n \"url\": \"" << url << "\"," - << "\n \"port\": 0,\n \"response_port\": 6464\n}"; - - EXPECT_CALL(mock_msg, sendMessage( - true, - ss.str(), - I_Messaging::Method::POST, - "127.0.0.1", - 8164, - _, - "/add-download-file", - _, - _, - _ - )).WillOnce(Return(Maybe(string("{\"uuid\": \"111\", \"status\": true}")))); - - bool is_cb_run = false; - bool res = Singleton::Consume::from()->downloadFile( - file_name, - url, - [&is_cb_run](const Maybe& filepath) - { - is_cb_run = true; - EXPECT_TRUE(filepath.ok()); - EXPECT_EQ(filepath.unpack(), "/tmp/test_download_dir/test_file"); - } - ); - EXPECT_TRUE(res); - - stringstream is; - is << "{\"uuid\": \"111\", \"status\": true, \"filepath\": \"/tmp/test_download_dir/test_file\"}"; - EXPECT_FALSE(is_cb_run); - rest_handler->performRestCall(is); - EXPECT_TRUE(is_cb_run); -} diff --git a/components/messaging_downloader/messaging_downloader_ut/downloader_server_ut/CMakeLists.txt b/components/messaging_downloader/messaging_downloader_ut/downloader_server_ut/CMakeLists.txt deleted file mode 100755 index 214fb3c..0000000 --- a/components/messaging_downloader/messaging_downloader_ut/downloader_server_ut/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -link_directories(${BOOST_ROOT}/lib) -include_directories(${CMAKE_SOURCE_DIR}/core/include) -link_directories(${CMAKE_BINARY_DIR}/core/include) - -add_unit_test( - downloader_server_ut - "downloader_server_ut.cc" - "singleton;messaging_downloader_server;time_proxy;mainloop;rest;metric;event_is;message;-lboost_context;agent_core_utilities;orchestration_modules;agent_details;connkey;-lboost_regex;-lboost_filesystem;-lboost_system" -) diff --git a/components/messaging_downloader/messaging_downloader_ut/downloader_server_ut/downloader_server_ut.cc b/components/messaging_downloader/messaging_downloader_ut/downloader_server_ut/downloader_server_ut.cc deleted file mode 100755 index 9b1d262..0000000 --- a/components/messaging_downloader/messaging_downloader_ut/downloader_server_ut/downloader_server_ut.cc +++ /dev/null @@ -1,304 +0,0 @@ -#include "messaging_downloader_server.h" - -#include - -#include "environment.h" -#include "singleton.h" -#include "config.h" -#include "config_component.h" -#include "mainloop.h" -#include "cptest.h" -#include "mock/mock_mainloop.h" -#include "mock/mock_messaging.h" -#include "mock/mock_rest_api.h" -#include "mock/mock_agent_details.h" -#include "mock/mock_time_get.h" - -using namespace std; -using namespace testing; - -class MessagingDownloaderServerTest : public Test -{ -public: - MessagingDownloaderServerTest() - { - setConfiguration(string("/tmp/test_download_dir/"), "Downloader", "Downloading Directory"); - EXPECT_CALL( - rest, - mockRestCall(RestAction::ADD, "download-file", _) - ).WillOnce(WithArg<2>(Invoke(this, &MessagingDownloaderServerTest::restHandler))); - - Maybe fog_addr(string("test.fog.com")); - EXPECT_CALL( - mock_agent_details, - getFogDomain() - ).WillRepeatedly(Return(fog_addr)); - - Debug::setUnitTestFlag(D_COMMUNICATION, Debug::DebugLevel::TRACE); - Debug::setNewDefaultStdout(&capture_debug); - - messaging_downloader.preload(); - messaging_downloader.init(); - } - - ~MessagingDownloaderServerTest() - { - boost::filesystem::remove_all("/tmp/test_download_dir/"); - messaging_downloader.fini(); - } - - bool - restHandler(const unique_ptr &rest_ptr) - { - rest_handler = rest_ptr->getRest(); - return true; - } - - void - expectRequestSuccess( - string &test_file_name, - string &host, - string &url, - string &uuid, - unsigned int port, - unsigned int response_port, - string &success_msg - ) - { - EXPECT_CALL( - mock_ml, - addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, _, false) - ).WillOnce(DoAll(SaveArg<1>(&downloading_routine), Return(0))); - - EXPECT_CALL( - mock_msg, - sendMessage(true, "", I_Messaging::Method::GET, host, port, _, url, _, _, _) - ).WillOnce(Return(Maybe(string("test_body")))); - - stringstream expected_response; - expected_response - << "\n \"status\": true," - << "\n \"filepath\": \"/tmp/test_download_dir/" << test_file_name << "\"\n}"; - - string saved_response; - - EXPECT_CALL(mock_msg, sendMessage( - false, - _, - I_Messaging::Method::POST, - "127.0.0.1", - response_port, - _, - "/show-download-status", - _, - _, - _ - )).WillOnce(DoAll(SaveArg<1>(&saved_response), Return(Maybe(string())))); - - - stringstream is; - is << "{\"file_name\": \"" << test_file_name << "\"," - << "\"response_port\": " << response_port << "," - << "\"url\": \"" << url << "\"," - << "\"port\": " << port << "," - << "\"uuid\": \"" << uuid << "\"}"; - - rest_handler->performRestCall(is); - downloading_routine(); - EXPECT_THAT(saved_response, HasSubstr(expected_response.str())); - EXPECT_THAT(capture_debug.str(), HasSubstr(success_msg)); - } - - unique_ptr rest_handler; - ostringstream capture_debug; - I_MainLoop::Routine downloading_routine; - MessagingDownloaderServer messaging_downloader; - NiceMock mock_time; - StrictMock mock_agent_details; - StrictMock mock_msg; - StrictMock rest; - StrictMock mock_ml; - ::Environment env; - ConfigComponent conf; -}; - -TEST_F(MessagingDownloaderServerTest, do_nothing) -{ -} - -TEST_F(MessagingDownloaderServerTest, add_one_secured_request) -{ - string test_file_name = "test_file_name"; - string host = "test_host"; - string url = "https://test_host/test_url"; - string uuid = "111"; - string success_msg = "Successfully downloaded the file. File name: " + test_file_name; - unsigned int port = 443; - unsigned int response_port = 123; - expectRequestSuccess(test_file_name, host, url, uuid, port, response_port, success_msg); -} - -TEST_F(MessagingDownloaderServerTest, add_one_non_secured_request) -{ - string test_file_name = "test_file_name"; - string host = "test_host"; - string url = "http://test_host/test_url"; - string uuid = "111"; - string success_msg = "Successfully downloaded the file. File name: " + test_file_name; - unsigned int port = 80; - unsigned int response_port = 123; - expectRequestSuccess(test_file_name, host, url, uuid, port, response_port, success_msg); -} - -TEST_F(MessagingDownloaderServerTest, add_multiple_requests) -{ - string test_file_name1 = "test_file_name1"; - string test_file_name2 = "test_file_name2"; - string host = "test_host"; - string url = "https://test_host/test_url"; - string uuid = "111"; - string success_msg1 = "Successfully downloaded the file. File name: " + test_file_name1; - string success_msg2 = "Successfully downloaded the file. File name: " + test_file_name2; - unsigned int port = 443; - unsigned int response_port = 123; - expectRequestSuccess(test_file_name1, host, url, uuid, port, response_port, success_msg1); - expectRequestSuccess(test_file_name2, host, url, uuid, port, response_port, success_msg2); -} - -TEST_F(MessagingDownloaderServerTest, add_same_request_twice) -{ - string test_file_name = "test_file_name"; - string host = "test_host"; - string url = "https://test_host/test_url"; - string uuid = "111"; - unsigned int response_port = 123; - - EXPECT_CALL( - mock_ml, - addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, _, false) - ).WillOnce(DoAll(SaveArg<1>(&downloading_routine), Return(0))); - - EXPECT_CALL( - mock_msg, - sendMessage(true, "", I_Messaging::Method::GET, host, 442, _, url, _, _, _) - ).WillOnce(Return(Maybe(string("test_body")))); - - stringstream expected_response; - expected_response - << "\n \"status\": true," - << "\n \"filepath\": \"/tmp/test_download_dir/" << test_file_name << "\"\n}"; - - string saved_response; - - EXPECT_CALL(mock_msg, sendMessage( - false, - _, - I_Messaging::Method::POST, - "127.0.0.1", - response_port, - _, - "/show-download-status", - _, - _, - _ - )).WillOnce(DoAll(SaveArg<1>(&saved_response), Return(Maybe(string())))); - - stringstream is; - is - << "{\"file_name\": \"" << test_file_name << "\"," - << "\"response_port\": " << response_port << "," - << "\"uuid\": \"" << uuid << "\"," - << "\"port\": 442," - << "\"url\": \"" << url << "\"}"; - - rest_handler->performRestCall(is); - rest_handler->doCall(); - downloading_routine(); - - EXPECT_THAT(saved_response, HasSubstr(expected_response.str())); - EXPECT_THAT( - capture_debug.str(), - HasSubstr("Failed to download the file. Similar download request already exists.") - ); -} - -TEST_F(MessagingDownloaderServerTest, add_request_that_fails) -{ - string test_file_name = "test_file_name"; - string host = "test_host"; - string url = "https://test_host/test_url"; - string uuid = "111"; - unsigned int response_port = 123; - unsigned int additional_port_test = 123; - - Maybe err = genError("no"); - - EXPECT_CALL( - mock_ml, - addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, _, false) - ).WillOnce(DoAll(SaveArg<1>(&downloading_routine), Return(0))); - - EXPECT_CALL( - mock_msg, - sendMessage(true, "", I_Messaging::Method::GET, host, additional_port_test, _, url, _, _, _) - ).WillOnce(Return(err)); - - stringstream expected_response; - expected_response - << "\n \"status\": false," - << "\n \"error\": \"Failed during the downloading process.\"\n}"; - - string saved_response; - - EXPECT_CALL(mock_msg, sendMessage( - false, - _, - I_Messaging::Method::POST, - "127.0.0.1", - response_port, - _, - "/show-download-status", - _, - _, - _ - )).WillOnce(DoAll(SaveArg<1>(&saved_response), Return(Maybe(string())))); - - stringstream is; - is - << "{\"file_name\": \"" << test_file_name << "\"," - << "\"response_port\": " << response_port << "," - << "\"url\": \"" << url << "\"," - << "\"port\": " << additional_port_test << "," - << "\"uuid\": \"" << uuid << "\"}"; - - rest_handler->performRestCall(is); - downloading_routine(); - EXPECT_THAT(saved_response, HasSubstr(expected_response.str())); - EXPECT_THAT(capture_debug.str(), HasSubstr("Failed to download file. File name: test_file_name")); -} - -TEST_F(MessagingDownloaderServerTest, download_with_same_filename) -{ - string test_file_name = "test_file_name"; - string host = "test_host"; - string url1 = "https://test_host/test_url1"; - string url2 = "https://test_host/test_url2"; - string uuid = "111"; - unsigned int port = 443; - string success_msg = "Successfully downloaded the file. File name: " + test_file_name; - unsigned int response_port = 123; - expectRequestSuccess(test_file_name, host, url1, uuid, port, response_port, success_msg); - - stringstream is; - is - << "{\"file_name\": \"" << test_file_name << "\"," - << "\"response_port\": " << response_port << "," - << "\"port\": " << port << "," - << "\"url\": \"" << url2 << "\"}"; - - rest_handler->performRestCall(is); - EXPECT_THAT( - capture_debug.str(), - HasSubstr("The file with the name 'test_file_name' is already exist in the downloading directory") - ); -} diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control.cc b/components/security_apps/layer_7_access_control/layer_7_access_control.cc index 189b3b6..e254a5e 100644 --- a/components/security_apps/layer_7_access_control/layer_7_access_control.cc +++ b/components/security_apps/layer_7_access_control/layer_7_access_control.cc @@ -278,7 +278,8 @@ Layer7AccessControl::Impl::generateLog(const string &source_ip, const Intelligen << genLogIPField("destinationIP", HttpTransactionData::listening_ip_ctx) << LogField("securityAction", security_action) << LogField("sourceIP", source_ip) - << LogField("externalVendorName", "crowdsec") + << LogField("externalVendorName", "CrowdSec") + << LogField("waapIncidentType", "CrowdSec") << ip_reputation.getCrowdsecEventId() << ip_reputation.getType() << ip_reputation.getOrigin() diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc index 13de087..361438a 100644 --- a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc +++ b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc @@ -7,6 +7,7 @@ #include "mock/mock_http_manager.h" #include "mock/mock_logging.h" #include "mock/mock_messaging.h" +#include "mock/mock_rest_api.h" #include "intelligence_comp_v2.h" #include "agent_details.h" @@ -27,6 +28,7 @@ public: EXPECT_CALL(mock_time, getMonotonicTime()).WillRepeatedly(Return(chrono::seconds(60))); EXPECT_CALL(mock_ml, doesRoutineExist(_)).WillRepeatedly(Return(true)); EXPECT_CALL(mock_ml, stop(_)).WillRepeatedly(Return()); + EXPECT_CALL(mock_ml, addRecurringRoutine(_, _, _, "Sending intelligence invalidation", _)); env.preload(); env.init(); config.preload(); @@ -57,6 +59,7 @@ public: StrictMock mock_time; StrictMock mock_ml; StrictMock messaging_mock; + NiceMock mock_rest; AgentDetails agent_details; IntelligenceComponentV2 intelligence_comp; Context ctx; @@ -243,7 +246,8 @@ Layer7AccessControlTest::verifyReport( EXPECT_THAT(log, HasSubstr("\"httpMethod\": \"POST\"")); EXPECT_THAT(log, HasSubstr("\"ipProtocol\": \"http\"")); EXPECT_THAT(log, HasSubstr("\"destinationIP\": \"5.6.7.8\"")); - EXPECT_THAT(log, HasSubstr("\"externalVendorName\": \"crowdsec\"")); + EXPECT_THAT(log, HasSubstr("\"externalVendorName\": \"CrowdSec\"")); + EXPECT_THAT(log, HasSubstr("\"waapIncidentType\": \"CrowdSec\"")); EXPECT_THAT(log, HasSubstr("\"externalVendorRecommendationId\": 2253734")); EXPECT_THAT(log, HasSubstr("\"externalVendorRecommendedAction\": \"ban\"")); EXPECT_THAT(log, HasSubstr("\"externalVendorRecommendationOrigin\": \"cscli\"")); diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h index f8293a4..37014fb 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h @@ -70,6 +70,14 @@ checkHasSDWan(const string &command_output) return genError("Current host does not have SDWAN capability"); } +Maybe +checkCanUpdateSDWanData(const string &command_output) +{ + if (command_output == "true" || command_output == "false") return command_output; + + return string("true"); +} + Maybe getMgmtObjType(const string &command_output) { @@ -157,6 +165,14 @@ getGWVersion(shared_ptr file_stream) return getMgmtObjAttr(file_stream, "svn_version_name "); } +Maybe +checkIfSdwanRunning(const string &command_output) +{ + if (command_output == "true" || command_output == "false") return command_output; + + return genError("Could not determine if sd-wan is running or not"); +} + Maybe getSmbObjectName(const string &command_output) { diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index 50f1f69..111da0c 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -31,6 +31,11 @@ #if defined(gaia) || defined(smb) SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType) SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan) +SHELL_CMD_HANDLER("canUpdateSDWanData", "cpsdwan get_data | jq -r .can_update_sdwan_data", checkCanUpdateSDWanData) +SHELL_CMD_HANDLER( + "isSdwanRunning", + "[ -v $(pidof cp-nano-sdwan) ] && echo 'false' || echo 'true'", + checkIfSdwanRunning) #endif //gaia || smb #if defined(gaia) diff --git a/components/security_apps/orchestration/include/get_status_rest.h b/components/security_apps/orchestration/include/get_status_rest.h index ba3978d..655389f 100755 --- a/components/security_apps/orchestration/include/get_status_rest.h +++ b/components/security_apps/orchestration/include/get_status_rest.h @@ -14,7 +14,6 @@ #ifndef __GET_STATUS_RES_H__ #define __GET_STATUS_RES_H__ -#include "i_messaging_downloader.h" #include "i_messaging.h" #include "i_mainloop.h" #include "i_shell_cmd.h" diff --git a/components/security_apps/orchestration/include/mock/mock_messaging_downloader.h b/components/security_apps/orchestration/include/mock/mock_messaging_downloader.h deleted file mode 100755 index 61d405a..0000000 --- a/components/security_apps/orchestration/include/mock/mock_messaging_downloader.h +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __MOCK_MESSAGING_DOWNLOADER_H__ -#define __MOCK_MESSAGING_DOWNLOADER_H__ - -#include "cptest.h" -#include - -#include "i_messaging_downloader.h" - -class MockMessagingDownloader - : - public Singleton::Provide::From> -{ -public: - MOCK_METHOD4( - downloadFile, - bool( - const std::string &, - const std::string &, - OnCompleteCB, - const unsigned int - ) - ); -}; - - -#endif // __MOCK_MESSAGING_DOWNLOADER_H__ diff --git a/components/security_apps/orchestration/include/mock/mock_service_controller.h b/components/security_apps/orchestration/include/mock/mock_service_controller.h index 8d8e104..e425d0b 100755 --- a/components/security_apps/orchestration/include/mock/mock_service_controller.h +++ b/components/security_apps/orchestration/include/mock/mock_service_controller.h @@ -26,6 +26,10 @@ class MockServiceController : public: MOCK_METHOD0(refreshPendingServices, void()); + MOCK_METHOD0(mockMoveChangedPolicies, std::set()); + std::set tmp; + std::set && moveChangedPolicies() override { tmp = mockMoveChangedPolicies(); return std::move(tmp); } + MOCK_METHOD0(doesFailedServicesExist, bool()); MOCK_METHOD0(clearFailedServices, void()); diff --git a/components/security_apps/orchestration/manifest_controller/manifest_controller_ut/manifest_controller_ut.cc b/components/security_apps/orchestration/manifest_controller/manifest_controller_ut/manifest_controller_ut.cc index 1de6dde..83aeb8a 100755 --- a/components/security_apps/orchestration/manifest_controller/manifest_controller_ut/manifest_controller_ut.cc +++ b/components/security_apps/orchestration/manifest_controller/manifest_controller_ut/manifest_controller_ut.cc @@ -740,6 +740,7 @@ TEST_F(ManifestControllerTest, selfUpdateWithOldCopyWithError) orch_service_name; EXPECT_CALL(mock_orchestration_tools, doesFileExist(path)).WillOnce(Return(false)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(path, path + backup_ext + temp_ext)).WillOnce(Return(false)); + EXPECT_CALL(mock_details_resolver, getHostname()).WillOnce(Return(hostname)); EXPECT_FALSE(i_manifest_controller->updateManifest(file_name)); } diff --git a/components/security_apps/orchestration/manifest_controller/manifest_handler.cc b/components/security_apps/orchestration/manifest_controller/manifest_handler.cc index 6c8e96b..02f9a48 100755 --- a/components/security_apps/orchestration/manifest_controller/manifest_handler.cc +++ b/components/security_apps/orchestration/manifest_controller/manifest_handler.cc @@ -145,23 +145,21 @@ ManifestHandler::downloadPackages( } } downloaded_packages.clear(); + + auto agent_details = Singleton::Consume::by(); + auto hostname = Singleton::Consume::by()->getHostname(); + string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + agent_details->getAgentId()) + "'"; string install_error; if (is_clean_installation) { - string error_hostname_addition = ""; - auto maybe_hostname = Singleton::Consume::by()->getHostname(); - if (maybe_hostname.ok()) { - error_hostname_addition = " on host '" + maybe_hostname.unpack() + "'"; - } install_error = - "Critical Error: Agent/Gateway was not fully deployed" + - error_hostname_addition + + "Critical Error: Agent/Gateway was not fully deployed " + + err_hostname + " and is not enforcing a security policy. Retry installation or contact Check Point support."; } else { - auto agent_details = Singleton::Consume::by(); install_error = - "Warning: Agent/Gateway '" + - agent_details->getAgentId() + - "' software update failed. Agent is running previous software. Contact Check Point support."; + "Warning: Agent/Gateway " + + err_hostname + + " software update failed. Agent is running previous software. Contact Check Point support."; } auto orchestration_status = Singleton::Consume::by(); @@ -221,11 +219,13 @@ ManifestHandler::installPackages( orchestration_status->writeStatusToFile(); bool self_update_status = selfUpdate(package, current_packages, package_handler_path); if (!self_update_status) { - auto agent_details = Singleton::Consume::by(); + auto details = Singleton::Consume::by(); + auto hostname = Singleton::Consume::by()->getHostname(); + string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + details->getAgentId()) + "'"; string install_error = - "Warning: Agent/Gateway '" + - agent_details->getAgentId() + - "' software update failed. Agent is running previous software. Contact Check Point support."; + "Warning: Agent/Gateway " + + err_hostname + + " software update failed. Agent is running previous software. Contact Check Point support."; if (orchestration_status->getManifestError().find("Gateway was not fully deployed") == string::npos) { orchestration_status->setFieldStatus( OrchestrationStatusFieldType::MANIFEST, @@ -278,23 +278,20 @@ ManifestHandler::installPackages( } if (!current_result) { + auto agent_details = Singleton::Consume::by(); + auto hostname = Singleton::Consume::by()->getHostname(); + string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" +agent_details->getAgentId()) + "'"; string install_error; if (is_clean_installation) { - string error_hostname_addition = ""; - auto maybe_hostname = Singleton::Consume::by()->getHostname(); - if (maybe_hostname.ok()) { - error_hostname_addition = " on host '" + maybe_hostname.unpack() + "'"; - } install_error = - "Critical Error: Agent/Gateway was not fully deployed" + - error_hostname_addition + + "Critical Error: Agent/Gateway was not fully deployed " + + err_hostname + " and is not enforcing a security policy. Retry installation or contact Check Point support."; } else { - auto agent_details = Singleton::Consume::by(); install_error = - "Warning: Agent/Gateway '" + - agent_details->getAgentId() + - "' software update failed. Agent is running previous software. Contact Check Point support."; + "Warning: Agent/Gateway " + + err_hostname + + " software update failed. Agent is running previous software. Contact Check Point support."; } corrupted_packages.insert(make_pair(package_name, package)); dbgWarning(D_ORCHESTRATOR) << "Failed to install package. Package: " << package_name; diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 7941e7c..49cb0af 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -41,6 +41,7 @@ #include "tenant_profile_pair.h" #include "env_details.h" #include "hybrid_communication.h" +#include "agent_core_utilities.h" using namespace std; using namespace chrono; @@ -48,6 +49,14 @@ using namespace ReportIS; USE_DEBUG_FLAG(D_ORCHESTRATOR); + +static const string ls_prefix = "ls "; +static const string extract_tenant_profile_suffix = + "| grep tenant " + "| cut -d '_' -f 2,4 " + "| sort --unique " + "| awk -F '_' '{ printf \"%s %s \",$1,$2 }'"; + class HealthCheckStatusListener : public Listener { public: @@ -193,6 +202,7 @@ public: ReportIS::Audience::INTERNAL ); hybrid_mode_metric.registerListener(); + loadExistingTenantsFromConfDir(); } void @@ -237,7 +247,9 @@ private: } policy_version = service_controller->getPolicyVersion(); - Singleton::Consume::by()->setPolicyVersion(policy_version); + if (!policy_version.empty()) { + Singleton::Consume::by()->setPolicyVersion(policy_version); + } } else { dbgDebug(D_ORCHESTRATOR) << "Orchestration is running for the first time"; enforce_policy_flag = true; @@ -277,6 +289,59 @@ private: return authentication_res; } + void + loadExistingTenantsFromConfDir() + { + dbgTrace(D_ORCHESTRATOR) << "Load existing tenants and profiles from the configuration folder"; + + string global_conf_dir = getConfigurationWithDefault( + getFilesystemPathConfig()+ "/conf/", + "orchestration", + "Conf dir" + ); + string shell_cmd_string = ls_prefix + global_conf_dir + extract_tenant_profile_suffix; + auto shell = Singleton::Consume::by(); + Maybe output_res = shell->getExecOutput(shell_cmd_string); + + if (!output_res.ok()) { + dbgWarning(D_ORCHESTRATOR) + << "Failed to load existing tenants from configuration folder: " + output_res.getErr(); + return; + } + + auto tenant_manager = Singleton::Consume::by(); + stringstream ss(output_res.unpack()); + string tenant_id; + string profile_id; + while (!ss.eof() && getline(ss, tenant_id, ' ') && !ss.eof() && getline(ss, profile_id, ' ')) { + dbgTrace(D_ORCHESTRATOR) << "Add existing tenant_" + tenant_id + "_profile_" + profile_id; + tenant_manager->addActiveTenantAndProfile(tenant_id, profile_id); + } + } + + void + deleteInactiveTenantProfileFiles(const string &tenant_id, const string &profile_id) + { + string global_conf_dir = getConfigurationWithDefault( + getFilesystemPathConfig()+ "/conf/", + "orchestration", + "Conf dir" + ); + string tenant_and_profile_suffix = "tenant_" + tenant_id + "_profile_" + profile_id; + string virtual_policy_dir = global_conf_dir + tenant_and_profile_suffix; + + dbgTrace(D_ORCHESTRATOR) << "Delete virtual policy folder : " << virtual_policy_dir; + if (!NGEN::Filesystem::deleteDirectory(virtual_policy_dir, true)) { + dbgWarning(D_ORCHESTRATOR) << "Failed to delete virtual policy folder : " << virtual_policy_dir; + } + + string settings_file_path = virtual_policy_dir + "_settings.json"; + dbgTrace(D_ORCHESTRATOR) << "Delete settings file " << settings_file_path; + if (!NGEN::Filesystem::deleteFile(settings_file_path)) { + dbgWarning(D_ORCHESTRATOR) << "Failed to delete virtual policy settings file : " << settings_file_path; + } + } + Maybe loadOrchestrationPolicy() { @@ -404,33 +469,30 @@ private: dbgInfo(D_ORCHESTRATOR) << "There is a new manifest file."; GetResourceFile resource_file(GetResourceFile::ResourceFileType::MANIFEST); Maybe new_manifest_file = - Singleton::Consume::by()->downloadFileFromFog( - orch_manifest.unpack(), - I_OrchestrationTools::SELECTED_CHECKSUM_TYPE, - resource_file - ); + Singleton::Consume::by()->downloadFileFromFog( + orch_manifest.unpack(), + I_OrchestrationTools::SELECTED_CHECKSUM_TYPE, + resource_file + ); auto orch_status = Singleton::Consume::by(); auto service_controller = Singleton::Consume::by(); auto agent_details = Singleton::Consume::by(); static int service_to_port_size = service_controller->getServiceToPortMap().size(); + auto hostname = Singleton::Consume::by()->getHostname(); + string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + agent_details->getAgentId()) + "'"; if (!new_manifest_file.ok()) { string install_error; if (!service_to_port_size) { - string error_hostname_addition = ""; - auto maybe_hostname = Singleton::Consume::by()->getHostname(); - if (maybe_hostname.ok()) { - error_hostname_addition = " on host '" + maybe_hostname.unpack() + "'"; - } install_error = - "Critical Error: Agent/Gateway was not fully deployed" + - error_hostname_addition + + "Critical Error: Agent/Gateway was not fully deployed " + + err_hostname + " and is not enforcing a security policy. Retry installation or contact Check Point support."; } else { install_error = - "Warning: Agent/Gateway '" + - agent_details->getAgentId() + - "' software update failed. Agent is running previous software. Contact Check Point support."; + "Warning: Agent/Gateway " + + err_hostname + + " software update failed. Agent is running previous software. Contact Check Point support."; } dbgTrace(D_ORCHESTRATOR) << "Manifest failed to be updated. Error: " @@ -455,9 +517,9 @@ private: auto manifest_controller = Singleton::Consume::by(); if (!manifest_controller->updateManifest(new_manifest_file.unpack())) { string install_error = - "Warning: Agent/Gateway '" + - agent_details->getAgentId() + - "' software update failed. Agent is running previous software. Contact Check Point support."; + "Warning: Agent/Gateway " + + err_hostname + + " software update failed. Agent is running previous software. Contact Check Point support."; string current_error = orch_status->getManifestError(); if (current_error.find("Gateway was not fully deployed") == string::npos) { orch_status->setFieldStatus( @@ -499,9 +561,9 @@ private: } string manifest_success_notification_message( - "Agent/Gateway '" + - agent_details->getAgentId() + - "' software update succeeded. Agent is running latest software." + "Agent/Gateway " + + err_hostname + + " software update succeeded. Agent is running latest software." ); LogGen manifest_success_notification( manifest_success_notification_message, @@ -517,7 +579,6 @@ private: return Maybe(); } - // LCOV_EXCL_START Reason: future changes will be done bool updateServiceConfigurationFromBackup() { @@ -565,7 +626,6 @@ private: dbgWarning (D_ORCHESTRATOR) << "Failed to load Orchestration policy."; return false; } - // LCOV_EXCL_STOP string updatePolicyAndFogAddress(const OrchestrationPolicy &orchestration_policy) @@ -582,7 +642,9 @@ private: auto service_controller = Singleton::Consume::by(); string new_policy_version = service_controller->getPolicyVersion(); - Singleton::Consume::by()->setPolicyVersion(new_policy_version); + if (!new_policy_version.empty()) { + Singleton::Consume::by()->setPolicyVersion(new_policy_version); + } auto update_communication = Singleton::Consume::by(); auto path_policy_version = update_communication->sendPolicyVersion(new_policy_version); if (!path_policy_version.ok()) { @@ -617,7 +679,7 @@ private: "last fog policy file extension" ); if (!orchestration_tools->copyFile(new_policy_file.unpack(), conf_path + last_ext)) { - dbgWarning(D_ORCHESTRATOR) << "Failed to copy a new policy file to " << conf_path + last_ext; + dbgWarning(D_ORCHESTRATOR) << "Failed to copy a new policy file to " << conf_path + last_ext; } // Calculate the changes between the existing policy to the new one. @@ -648,6 +710,11 @@ private: "Settings file path" ); + set changed_policy_files = service_controller->moveChangedPolicies(); + for (const string &changed_policy_file : changed_policy_files) { + orchestration_tools->writeFile("{}\n", changed_policy_file); + } + service_controller->updateServiceConfiguration(policy_file, setting_file, data_updates); LogGen( error_str, @@ -661,6 +728,7 @@ private: return genError(error_str); } + service_controller->moveChangedPolicies(); // Reload the orchestration policy, in case of the policy updated auto orchestration_policy = loadDefaultOrchestrationPolicy(); @@ -672,8 +740,6 @@ private: if (new_policy_version.empty()) { return genError("Failed to load Orchestration new policy file."); } - - reloadConfiguration(); if (getProfileAgentSettingWithDefault(false, "agent.config.orchestration.reportAgentDetail")) { service_controller->clearFailedServices(); reportAgentDetailsMetaData(); @@ -1021,12 +1087,13 @@ private: vector data_updates; update_results[OrchestrationStatusConfigType::DATA] = handleDataUpdate(orch_data, data_updates); - update_results[OrchestrationStatusConfigType::POLICY] = handlePolicyUpdate( - orch_policy, - settings_path, - data_updates - ); - + if (!orch_manifest.ok() && orch_policy.ok()) { + update_results[OrchestrationStatusConfigType::POLICY] = handlePolicyUpdate( + orch_policy, + settings_path, + data_updates + ); + } if (!orch_policy.ok() && (data_updates.size() > 0 || settings_path != "")) { auto service_controller = Singleton::Consume::by(); bool res = service_controller->updateServiceConfiguration( @@ -1046,22 +1113,19 @@ private: string recommended_fix; string msg; bool is_deploy_error = current_error.find("Critical") != string::npos; + auto hostname = Singleton::Consume::by()->getHostname(); + string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + agent_details->getAgentId()) + "'"; if (is_deploy_error) { - string error_hostname_addition = ""; - auto maybe_hostname = Singleton::Consume::by()->getHostname(); - if (maybe_hostname.ok()) { - error_hostname_addition = " on host '" + maybe_hostname.unpack() + "'"; - } msg = - "Agent/Gateway was not fully deployed" + - error_hostname_addition + + "Agent/Gateway was not fully deployed " + + err_hostname + " and is not enforcing a security policy."; recommended_fix = "Retry installation or contact Check Point support."; } else if (current_error.find("Warning") != string::npos) { msg = - "Agent/Gateway '" + - agent_details->getAgentId() + - "' software update failed. Agent is running previous software."; + "Agent/Gateway " + + err_hostname + + " software update failed. Agent is running previous software."; recommended_fix = "Contact Check Point support."; } if (!msg.empty() && !recommended_fix.empty()) { @@ -1114,7 +1178,11 @@ private: bool is_empty = true; GetResourceFile resource_v_policy_file(GetResourceFile::ResourceFileType::VIRTUAL_POLICY); I_Downloader *downloader = Singleton::Consume::by(); + auto tenant_manager = Singleton::Consume::by(); + map> profiles_to_be_deleted = + tenant_manager->fetchAndUpdateActiveTenantsAndProfiles(false); for (const auto &tenant: *updated_policy_tenants) { + profiles_to_be_deleted[tenant.getTenantID()].erase(tenant.getProfileID()); if (!tenant.getVersion().empty()) { is_empty = false; @@ -1127,7 +1195,6 @@ private: << tenant.getTenantID() << " Profile: " << profile_to_use; - auto tenant_manager = Singleton::Consume::by(); tenant_manager->addActiveTenantAndProfile(tenant.getTenantID(), profile_to_use); resource_v_policy_file.addTenant( @@ -1204,6 +1271,22 @@ private: } } + for (const auto &tenant_profile_set : profiles_to_be_deleted) { + auto tenant_id = tenant_profile_set.first; + for (const auto &profile_id: tenant_profile_set.second) { + dbgTrace(D_ORCHESTRATOR) + << "Delete configuration files for inactive profile: " + << "Tenant ID: " + << tenant_id + << ", Profile ID: " + << profile_id; + tenant_manager->deactivateTenant(tenant_id, profile_id); + deleteInactiveTenantProfileFiles(tenant_id, profile_id); + } + } + + clearOldTenants(); + for (auto it = sorted_files.begin(); it != sorted_files.end(); it++) { const auto &downloaded_files = *it; auto files = downloaded_files.second; @@ -1574,6 +1657,9 @@ private: auto server = TagAndEnumManagement::convertStringToTag(server_name); if (server.ok()) tags.insert(*server); + if (getAttribute("no-setting", "CROWDSEC_ENABLED") == "true") tags.insert(Tags::CROWDSEC); + if (getAttribute("no-setting", "PLAYGROUND") == "true") tags.insert(Tags::PLAYGROUND); + Report registration_report( "Local Agent Data", Singleton::Consume::by()->getWalltime(), diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc index f925d2a..0eb2efe 100644 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc @@ -17,7 +17,6 @@ #include "mock/mock_time_get.h" #include "mock/mock_rest_api.h" #include "mock/mock_tenant_manager.h" -#include "mock/mock_messaging_downloader.h" #include "config.h" #include "config_component.h" #include "agent_details.h" @@ -35,7 +34,6 @@ public: mockRestCall(RestAction::SET, "new-configuration", _) ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationMultitenancyTest::setNewConfiguration))); - EXPECT_CALL(tenant_manager, getTimeoutVal()).WillOnce(Return(chrono::microseconds(0))); EXPECT_CALL( mock_ml, addRecurringRoutine(I_MainLoop::RoutineType::System, _, _, _, _) @@ -76,6 +74,17 @@ public: ).WillOnce(Return(true)); doEncrypt(); + EXPECT_CALL( + mock_shell_cmd, + getExecOutput( + "ls /etc/cp/conf/" + "| grep tenant " + "| cut -d '_' -f 2,4 " + "| sort --unique " + "| awk -F '_' '{ printf \"%s %s \",$1,$2 }'", + _, + _ + )).WillOnce(Return(Maybe(string("")))); orchestration_comp.init(); } @@ -173,7 +182,6 @@ public: StrictMock mock_service_controller; StrictMock mock_manifest_controller; StrictMock mock_update_communication; - StrictMock mock_messaging_downloader; StrictMock tenant_manager; NiceMock mock_status; @@ -225,6 +233,10 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource) string first_policy_version = ""; string host_url = "https://" + host_address + "/"; + stringstream debug_output; + Debug::setNewDefaultStdout(&debug_output); + Debug::setUnitTestFlag(D_ORCHESTRATOR, Debug::DebugLevel::TRACE); + EXPECT_CALL( rest, mockRestCall(RestAction::ADD, "proxy", _) @@ -266,7 +278,13 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource) .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); set active_tenants = { "1236", "1235" }; + map> old_tenant_profile_set; + set old_profiles = { "123123" }; + old_tenant_profile_set["321321"] = old_profiles; EXPECT_CALL(tenant_manager, fetchActiveTenants()).WillOnce(Return(active_tenants)); + EXPECT_CALL(tenant_manager, fetchAndUpdateActiveTenantsAndProfiles(false)) + .WillOnce(Return(old_tenant_profile_set)); + EXPECT_CALL(tenant_manager, deactivateTenant("321321", "123123")).Times(1); EXPECT_CALL(tenant_manager, addActiveTenantAndProfile("1235", "2311")); EXPECT_CALL(tenant_manager, addActiveTenantAndProfile("1236", "2611")); @@ -479,4 +497,9 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource) try { runRoutine(); } catch (const invalid_argument& e) {} + string debug_str_folder = "Delete virtual policy folder : /etc/cp/conf/tenant_321321_profile_123123"; + string debug_str_settings = "Delete settings file /etc/cp/conf/tenant_321321_profile_123123_settings.json"; + EXPECT_THAT(debug_output.str(), HasSubstr(debug_str_folder)); + EXPECT_THAT(debug_output.str(), HasSubstr(debug_str_settings)); + Debug::setNewDefaultStdout(&cout); } diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc index 94910af..d44a6f7 100755 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc @@ -16,7 +16,6 @@ #include "mock/mock_messaging.h" #include "mock/mock_time_get.h" #include "mock/mock_rest_api.h" -#include "mock/mock_messaging_downloader.h" #include "mock/mock_tenant_manager.h" #include "config.h" #include "config_component.h" @@ -95,6 +94,17 @@ public: )).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); doEncrypt(); + EXPECT_CALL( + mock_shell_cmd, + getExecOutput( + "ls /etc/cp/conf/" + "| grep tenant " + "| cut -d '_' -f 2,4 " + "| sort --unique " + "| awk -F '_' '{ printf \"%s %s \",$1,$2 }'", + _, + _ + )).WillOnce(Return(Maybe(string("")))); orchestration_comp.init(); } @@ -138,6 +148,7 @@ public: EXPECT_CALL(mock_details_resolver, isVersionEqualOrAboveR8110()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, parseNginxMetadata()).WillRepeatedly(Return(no_nginx)); EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("1.1.1")); + EXPECT_CALL(mock_details_resolver, getHostname()).WillRepeatedly(Return(string("hostname"))); map resolved_mgmt_details({{"kernel_version", "4.4.0-87-generic"}}); EXPECT_CALL(mock_details_resolver, getResolvedDetails()).WillRepeatedly(Return(resolved_mgmt_details)); @@ -277,7 +288,6 @@ public: unique_ptr rest_status; StrictMock mock_orchestration_tools; StrictMock mock_downloader; - StrictMock mock_messaging_downloader; StrictMock mock_shell_cmd; StrictMock mock_message; StrictMock rest; @@ -538,6 +548,222 @@ TEST_F(OrchestrationTest, check_sending_registration_data) EXPECT_THAT(message_body, HasSubstr("\"NGINX Server\"")); } +TEST_F(OrchestrationTest, orchestrationPolicyUpdatRollback) +{ + Debug::setUnitTestFlag(D_CONFIG, Debug::DebugLevel::TRACE); + waitForRestCall(); + preload(); + + EXPECT_CALL( + mock_ml, + addOneTimeRoutine(I_MainLoop::RoutineType::Offline, _, "Send policy update report", _) + ).WillOnce(Return(1)); + EXPECT_CALL( + rest, + mockRestCall(RestAction::ADD, "proxy", _) + ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandler))); + EXPECT_CALL(mock_status, setFogAddress(host_url)).Times(2); + + string config_json = + "{\n" + "\"agentSettings\": [{\n" + "\"key\": \"agent.config.orchestration.reportAgentDetail\",\n" + "\"id\": \"id1\",\n" + "\"value\": \"true\"\n" + "}]\n" + "}\n"; + + istringstream ss(config_json); + Singleton::Consume::from(config_comp)->loadConfiguration(ss); + + init(); + + // All duplicates should be removed in INXT-35947 + string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy"; + string manifest_file_path = "/etc/cp/conf/manifest.json"; + string setting_file_path = "/etc/cp/conf/settings.json"; + string policy_file_path = "/etc/cp/conf/policy.json"; + string policy_file_path_bk = "/etc/cp/conf/policy.json.bk"; + string last_policy_file_path = "/etc/cp/conf/policy.json.last"; + string data_file_path = "/etc/cp/conf/data.json"; + string host_address = "1.2.3.5"; + string new_host_address = "6.2.3.5"; + string new_host_url = "https://" + new_host_address + "/test/"; + string new_policy_path = "/some-path"; + + string manifest_checksum = "manifest"; + string policy_checksum = "policy"; + string settings_checksum = "settings"; + string data_checksum = "data"; + string new_policy_checksum= "111111"; + + string second_val = "12"; + string third_val = "13"; + + Maybe policy_response( + string( + "{\n" + " \"fog-address\": \"" + host_url + "\",\n" + " \"agent-type\": \"test\",\n" + " \"pulling-interval\": 25,\n" + " \"error-pulling-interval\": 15\n" + "}" + ) + ); + + Maybe new_policy_response( + string( + "{\n" + " \"fog-address\": \"" + new_host_url + "\",\n" + " \"agent-type\": \"test\",\n" + " \"pulling-interval\": 25,\n" + " \"error-pulling-interval\": 15\n" + "}" + ) + ); + + set expected_changed_policies = {}; + EXPECT_CALL(mock_service_controller, mockMoveChangedPolicies()).WillOnce(Return(expected_changed_policies)); + + EXPECT_CALL(mock_status, setFogAddress(new_host_url)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); + // Rollback related test: The readFile function is called 3 times: + // 1. Read the current policy file + // 2. Read the new policy file - The one that should fail + // 3. Read the current policy file again - The one that should be restored + EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)) + .WillOnce(Return(policy_response)) + .WillOnce(Return(new_policy_response)) + .WillOnce(Return(policy_response)); + EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_path, policy_file_path + ".last")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)) + .Times(2).WillRepeatedly(Return(true)); + EXPECT_CALL(mock_update_communication, setAddressExtenesion("")).Times(2); + EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); + expectDetailsResolver(); + EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, manifest_file_path)) + .WillOnce(Return(manifest_checksum)); + + EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, setting_file_path)) + .WillOnce(Return(settings_checksum)); + + EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, policy_file_path)) + .WillOnce(Return(policy_checksum)); + + EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, data_file_path)) + .WillOnce(Return(data_checksum)); + + // Rollback related test: After failing to update the policy file, the policy version should be restored + EXPECT_CALL(mock_service_controller, getPolicyVersion()) + .Times(5) + .WillOnce(ReturnRef(first_policy_version)) + .WillOnce(ReturnRef(first_policy_version)) + .WillOnce(ReturnRef(second_val)) + .WillOnce(ReturnRef(third_val)) + .WillOnce(ReturnRef(second_val) + ); + EXPECT_CALL(mock_status, setPolicyVersion(third_val)); + EXPECT_CALL(mock_status, setPolicyVersion(second_val)); + EXPECT_CALL(mock_update_communication, sendPolicyVersion("13")).Times(1).WillOnce(Return(Maybe())); + // Rollback related test: The old policy version 12 is restored + EXPECT_CALL(mock_update_communication, sendPolicyVersion("12")).Times(1).WillOnce(Return(Maybe())); + + EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce( + Invoke( + [&](CheckUpdateRequest &req) + { + EXPECT_THAT(req.getPolicy(), IsValue(policy_checksum)); + EXPECT_THAT(req.getSettings(), IsValue(settings_checksum)); + EXPECT_THAT(req.getManifest(), IsValue(manifest_checksum)); + EXPECT_THAT(req.getData(), IsValue(data_checksum)); + req = CheckUpdateRequest("", new_policy_checksum, "", "", "", ""); + return Maybe(); + } + ) + ); + + GetResourceFile policy_file(GetResourceFile::ResourceFileType::POLICY); + EXPECT_CALL( + mock_downloader, + downloadFileFromFog(new_policy_checksum, Package::ChecksumTypes::SHA256, policy_file) + ).WillOnce(Return(Maybe(new_policy_path))); + + vector expected_data_types = {}; + EXPECT_CALL( + mock_service_controller, + updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "", _) + ).WillOnce(Return(true)); + + EXPECT_CALL( + mock_service_controller, + updateServiceConfiguration(new_policy_path, "", expected_data_types, "", "", _) + ).WillOnce(Return(true)); + + EXPECT_CALL( + mock_message, + setActiveFog(new_host_address, 443, true, MessageTypeTag::GENERIC) + ).WillOnce(Return(true)); + EXPECT_CALL(mock_update_communication, setAddressExtenesion("/test")); + EXPECT_CALL(mock_status, setLastUpdateAttempt()); + EXPECT_CALL( + mock_status, + setFieldStatus(OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::SUCCESS, "") + ); + EXPECT_CALL(mock_status, setIsConfigurationUpdated(A>()) + ).WillOnce( + Invoke( + [](EnumArray arr) + { + EXPECT_EQ(arr[OrchestrationStatusConfigType::MANIFEST], false); + EXPECT_EQ(arr[OrchestrationStatusConfigType::POLICY], true); + EXPECT_EQ(arr[OrchestrationStatusConfigType::SETTINGS], false); + } + ) + ); + + EXPECT_CALL(mock_ml, yield(A())) + .WillOnce( + Invoke( + [] (chrono::microseconds microseconds) + { + EXPECT_EQ(1000000, microseconds.count()); + } + ) + ) + .WillOnce( + Invoke( + [] (chrono::microseconds microseconds) + { + EXPECT_EQ(25000000, microseconds.count()); + throw invalid_argument("stop while loop"); + } + ) + ); + EXPECT_CALL( + mock_shell_cmd, + getExecOutput(_, _, _) + ).WillRepeatedly(Return(string("daniel\n1\n"))); + + EXPECT_CALL(mock_service_controller, clearFailedServices()); + EXPECT_CALL(mock_service_controller, doesFailedServicesExist()).WillOnce(Return(true)); + EXPECT_CALL( + mock_service_controller, + updateServiceConfiguration(policy_file_path_bk, _, _, _, _, _) + ).WillOnce(Return(true)); + + EXPECT_CALL( + mock_orchestration_tools, + copyFile(policy_file_path_bk, policy_file_path) + ).WillOnce(Return(true)); + + + try { + runRoutine(); + } catch (const invalid_argument& e) {} +} + TEST_F(OrchestrationTest, orchestrationPolicyUpdate) { waitForRestCall(); @@ -597,6 +823,9 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate) ) ); + set expected_changed_policies = {}; + EXPECT_CALL(mock_service_controller, mockMoveChangedPolicies()).WillOnce(Return(expected_changed_policies)); + EXPECT_CALL(mock_status, setFogAddress(new_host_url)); EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)) @@ -628,7 +857,6 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate) .WillOnce(ReturnRef(second_val)) .WillOnce(ReturnRef(third_val) ); - EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version)); EXPECT_CALL(mock_status, setPolicyVersion(third_val)); EXPECT_CALL(mock_update_communication, sendPolicyVersion("13")).Times(1).WillOnce(Return(Maybe())); @@ -781,7 +1009,6 @@ TEST_F(OrchestrationTest, startOrchestrationPoliceWithFailures) EXPECT_CALL(mock_service_controller, getPolicyVersion()) .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); - EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version)); EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce( Invoke( @@ -910,7 +1137,6 @@ TEST_F(OrchestrationTest, loadOrchestrationPolicyFromBackup) EXPECT_CALL(mock_service_controller, getPolicyVersion()) .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); - EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version)); EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce( Invoke( [&](CheckUpdateRequest &req) @@ -1039,7 +1265,6 @@ TEST_F(OrchestrationTest, manifestUpdate) EXPECT_CALL(mock_service_controller, getPolicyVersion()) .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); - EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version)); EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce( Invoke( [&](CheckUpdateRequest &req) @@ -1165,11 +1390,14 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) EXPECT_CALL(mock_status, setFogAddress(host_url)); vector expected_data_types = {}; + EXPECT_CALL( mock_service_controller, updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "", _) ).Times(2).WillRepeatedly(Return(true)); + set expected_changed_policies = {}; + EXPECT_CALL(mock_service_controller, mockMoveChangedPolicies()).WillOnce(Return(expected_changed_policies)); EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response)); EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_path, policy_file_path + ".last")) @@ -1215,7 +1443,6 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) .WillOnce(ReturnRef(first_policy_version)) .WillOnce(ReturnRef(second_val) ); - EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version)); EXPECT_CALL(mock_status, setLastUpdateAttempt()); EXPECT_CALL( mock_status, @@ -1250,8 +1477,8 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) EXPECT_CALL( mock_service_controller, - updateServiceConfiguration(string("policy path"), "", expected_data_types, "", "", _)).WillOnce(Return(false) - ); + updateServiceConfiguration(string("policy path"), "", expected_data_types, "", "", _) + ).WillOnce(Return(false)); EXPECT_CALL(mock_ml, yield(A())) .WillOnce( @@ -1343,7 +1570,6 @@ TEST_F(OrchestrationTest, failedDownloadSettings) EXPECT_CALL(mock_service_controller, getPolicyVersion()) .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); - EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version)); EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce( Invoke( [&](CheckUpdateRequest &req) @@ -1374,7 +1600,6 @@ TEST_F(OrchestrationTest, failedDownloadSettings) manifest_err ) ).Times(1); - EXPECT_CALL(mock_details_resolver, getHostname()).Times(2).WillRepeatedly(Return(string("hostname"))); EXPECT_CALL(mock_status, getManifestError()).WillOnce(ReturnRef(manifest_err)); EXPECT_CALL(mock_status, setIsConfigurationUpdated(A>()) @@ -1400,13 +1625,6 @@ TEST_F(OrchestrationTest, failedDownloadSettings) manifest_file ) ).WillOnce(Return(download_error)); - EXPECT_CALL(mock_downloader, - downloadFileFromFog( - string("policy-checksum"), - Package::ChecksumTypes::SHA256, - policy_file - ) - ).WillOnce(Return(download_error)); EXPECT_CALL(mock_downloader, downloadFileFromFog( string("settings-checksum"), @@ -1754,7 +1972,6 @@ TEST_F(OrchestrationTest, dataUpdate) EXPECT_CALL(mock_service_controller, getPolicyVersion()) .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); - EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version)); EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce( Invoke( [&](CheckUpdateRequest &req) diff --git a/components/security_apps/orchestration/service_controller/service_controller.cc b/components/security_apps/orchestration/service_controller/service_controller.cc index 5c90440..7e466d0 100755 --- a/components/security_apps/orchestration/service_controller/service_controller.cc +++ b/components/security_apps/orchestration/service_controller/service_controller.cc @@ -181,6 +181,11 @@ ServiceDetails::serialize(Archive &ar) ReconfStatus ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy_version) { + if(!isServiceActive()) { + dbgDebug(D_ORCHESTRATOR) << "Service " << service_name << " is inactive"; + return ReconfStatus::INACTIVE; + } + SendConfigurations new_config(configuration_id, policy_version); I_Messaging *messaging = Singleton::Consume::by(); @@ -194,14 +199,12 @@ ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy conn_flags, "/set-new-configuration" ); + if (!res) { - if(!isServiceActive()) { - dbgDebug(D_ORCHESTRATOR) << "Service " << service_name << " is inactive"; - return ReconfStatus::INACTIVE; - } dbgDebug(D_ORCHESTRATOR) << "Service " << service_name << " didn't respond to new configuration request"; return ReconfStatus::FAILED; } + auto service_details = Singleton::Consume::by(); if (new_config.finished.get()) { @@ -300,6 +303,8 @@ public: void clearFailedServices() override; + set && moveChangedPolicies() override; + private: void cleanUpVirtualFiles(); @@ -320,6 +325,8 @@ private: void loadRegisteredServicesFromFile(); void writeRegisteredServicesToFile(); + bool backupConfigurationFile(const string &configuration_file_path); + int configuration_id = 0; map registered_services; map pending_services; @@ -332,6 +339,10 @@ private: map services_reconf_ids; string filesystem_prefix; bool is_multi_tenant_env = false; + set changed_policy_files; + + I_OrchestrationTools *orchestration_tools = nullptr; + I_MainLoop *mainloop = nullptr; }; class GetServicesPorts : public ServerRest @@ -410,6 +421,12 @@ ServiceController::Impl::doesFailedServicesExist() } // LCOV_EXCL_STOP +set && +ServiceController::Impl::moveChangedPolicies() +{ + return move(changed_policy_files); +} + void ServiceController::Impl::init() { @@ -418,6 +435,9 @@ ServiceController::Impl::init() rest->addRestCall(RestAction::SHOW, "all-service-ports"); rest->addRestCall(RestAction::SET, "reconf-status"); + orchestration_tools = Singleton::Consume::by(); + mainloop = Singleton::Consume::by(); + Singleton::Consume::by()->addRecurringRoutine( I_MainLoop::RoutineType::System, chrono::seconds( @@ -613,6 +633,29 @@ ServiceController::Impl::refreshPendingServices() writeRegisteredServicesToFile(); } +bool +ServiceController::Impl::backupConfigurationFile(const string &config_file_path) +{ + uint max_backup_attempts = 3; + string backup_ext = getConfigurationWithDefault(".bk", "orchestration", "Backup file extension"); + string backup_file = config_file_path + backup_ext; + + if (!orchestration_tools->doesFileExist(config_file_path)) { + dbgTrace(D_ORCHESTRATOR) << "File does not exist. File: " << config_file_path; + return true; + } + + for (size_t i = 0; i < max_backup_attempts; i++) { + if (orchestration_tools->copyFile(config_file_path, backup_file)) { + return true; + } + mainloop->yield(false); + } + + dbgWarning(D_ORCHESTRATOR) << "Failed to back up the file. File: " << config_file_path; + return false; +} + bool ServiceController::Impl::updateServiceConfiguration( const string &new_policy_path, @@ -665,19 +708,23 @@ ServiceController::Impl::updateServiceConfiguration( return sendSignalForServices(nano_services_to_update, ""); } - I_OrchestrationTools *orchestration_tools = Singleton::Consume::by(); - Maybe loaded_json = orchestration_tools->readFile(new_policy_path); - if (!loaded_json.ok()) { + Maybe loaded_policy_json = orchestration_tools->readFile(new_policy_path); + if (!loaded_policy_json.ok()) { dbgWarning(D_ORCHESTRATOR) << "Failed to load new file: " << new_policy_path << ". Error: " - << loaded_json.getErr(); + << loaded_policy_json.getErr(); return false; } - auto all_security_policies = orchestration_tools->jsonObjectSplitter(loaded_json.unpack(), tenant_id, profile_id); + + auto all_security_policies = orchestration_tools->jsonObjectSplitter( + loaded_policy_json.unpack(), + tenant_id, + profile_id + ); if (!all_security_policies.ok()) { dbgWarning(D_ORCHESTRATOR) @@ -740,6 +787,7 @@ ServiceController::Impl::updateServiceConfiguration( was_policy_updated = false; continue; } + changed_policy_files.insert(policy_file_path); dbgInfo(D_ORCHESTRATOR) << "Successfully updated policy file. Policy name: " << single_policy.first; @@ -779,7 +827,7 @@ ServiceController::Impl::updateServiceConfiguration( true : sendSignalForServices(nano_services_to_update, version_value); - dbgTrace(D_ORCHESTRATOR) << "was_policy_updated: " << (was_policy_updated ? "true" : "false"); + dbgTrace(D_ORCHESTRATOR) << "was policy updated: " << (was_policy_updated ? "true" : "false"); if (was_policy_updated) { string config_file_path; @@ -798,24 +846,8 @@ ServiceController::Impl::updateServiceConfiguration( return true; } - string backup_ext = getConfigurationWithDefault(".bk", "orchestration", "Backup file extension"); - - // Backup the current configuration file. - uint max_backup_attempts = 3; - bool is_backup_succeed = false; - string backup_file = config_file_path + backup_ext; - I_MainLoop *mainloop = Singleton::Consume::by(); - - for (size_t i = 0; i < max_backup_attempts; i++) { - if (orchestration_tools->copyFile(config_file_path, backup_file)) { - is_backup_succeed = true; - break; - } - mainloop->yield(false); - } - - if (!is_backup_succeed) { - dbgWarning(D_ORCHESTRATOR) << "Failed to back up the policy file."; + if (!backupConfigurationFile(config_file_path)) { + dbgWarning(D_ORCHESTRATOR) << "Failed to backup the policy file."; return false; } @@ -921,7 +953,6 @@ ServiceController::Impl::updateServiceConfigurationFile( dbgFlow(D_ORCHESTRATOR) << "Updating configuration. Config Name: " << configuration_name; - I_OrchestrationTools *orchestration_tools = Singleton::Consume::by(); if (orchestration_tools->doesFileExist(configuration_file_path)) { Maybe old_configuration = orchestration_tools->readFile(configuration_file_path); if (old_configuration.ok()) { diff --git a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc index 6639b43..68530f9 100755 --- a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc +++ b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc @@ -42,9 +42,7 @@ public: EXPECT_CALL(mock_rest_api, mockRestCall(RestAction::SET, "new-configuration", _)).WillOnce( WithArg<2>(Invoke(this, &ServiceControllerTest::setNanoServiceConfig)) ); - EXPECT_CALL(mock_ml, addRecurringRoutine(I_MainLoop::RoutineType::System, _, _, _, _)).WillOnce(Return(2)); - EXPECT_CALL(tenant_manager, getTimeoutVal()).WillOnce(Return(chrono::seconds(1))); EXPECT_CALL(mock_ml, addOneTimeRoutine(I_MainLoop::RoutineType::System, _, _, false)).WillOnce(Return(1)); config.init(); @@ -257,6 +255,7 @@ TEST_F(ServiceControllerTest, UpdateConfiguration) EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); string general_settings_path = "/my/settings/path"; string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; @@ -349,6 +348,29 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration) EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); + + EXPECT_CALL( + mock_shell_cmd, + getExecOutput( + "/etc/cp/watchdog/cp-nano-watchdog --status --verbose --service mock access control" + " --family family1 --id id2", + _, + _ + ) + ).Times(4).WillRepeatedly( + InvokeWithoutArgs( + [&]() -> Maybe + { + static int counter = 0; + if (counter++ < 2) { + return genError("Reached timeout while executing shell command:"); + } + + return string("registered and running"); + } + ) + ); string general_settings_path = "/my/settings/path"; string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; @@ -371,28 +393,6 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration) ) ).WillOnce(Return(Maybe(reply_msg))); - EXPECT_CALL( - mock_shell_cmd, - getExecOutput( - "/etc/cp/watchdog/cp-nano-watchdog --status --verbose --service mock access control" - " --family family1 --id id2", - _, - _ - ) - ).Times(3).WillRepeatedly( - InvokeWithoutArgs( - [&]() -> Maybe - { - static int counter = 0; - if (counter++ < 2) { - return genError("Reached timeout while executing shell command:"); - } - - return string("registered and running"); - } - ) - ); - EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path)); EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value); EXPECT_EQ(i_service_controller->getUpdatePolicyVersion(), version_value); @@ -468,6 +468,7 @@ TEST_F(ServiceControllerTest, writeRegisteredServicesFromFile) EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); string general_settings_path = "/my/settings/path"; string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; @@ -529,9 +530,7 @@ TEST_F(ServiceControllerTest, readRegisteredServicesFromFile) EXPECT_CALL(mock_rest_api, mockRestCall(RestAction::SET, "new-configuration", _)).WillOnce( WithArg<2>(Invoke(this, &ServiceControllerTest::setNanoServiceConfig)) ); - EXPECT_CALL(mock_ml, addRecurringRoutine(I_MainLoop::RoutineType::System, _, _, _, _)).WillOnce(Return(2)); - EXPECT_CALL(tenant_manager, getTimeoutVal()).WillOnce(Return(chrono::seconds(1))); EXPECT_CALL(mock_ml, addOneTimeRoutine(I_MainLoop::RoutineType::System, _, _, false)).WillOnce(Return(1)); config.init(); @@ -609,6 +608,7 @@ TEST_F(ServiceControllerTest, noPolicyUpdate) EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_status, setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY)); @@ -700,6 +700,7 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations) EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); EXPECT_CALL( mock_shell_cmd, @@ -742,6 +743,7 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations) EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) .WillOnce(Return(json_parser_return)); EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(l4_firewall)); EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); @@ -852,6 +854,122 @@ TEST_F(ServiceControllerTest, backup) EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); + + EXPECT_CALL( + mock_shell_cmd, + getExecOutput( + "/etc/cp/watchdog/cp-nano-watchdog --status --verbose --service mock access control" + " --family family1 --id id2", + _, + _ + ) + ).WillRepeatedly(Return(string("registered and running"))); + + string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}"; + EXPECT_CALL( + mock_message, + sendMessage( + _, + _, + _, + "127.0.0.1", + l4_firewall_service_port, + _, + "/set-new-configuration", + _, + _, + MessageTypeTag::GENERIC + ) + ).WillOnce(Return(Maybe(reply_msg))); + + EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); + EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, "")); + EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value); +} + +TEST_F(ServiceControllerTest, backup_file_doesnt_exist) +{ + string new_configuration = "{" + " \"version\": \"" + version_value + "\"" + " \"l4_firewall\":" + " {" + " \"app\": \"netfilter\"," + " \"l4_firewall_rules\": [" + " {" + " \"name\": \"allow_statefull_conns\"," + " \"flags\": [\"established\"]," + " \"action\": \"accept\"" + " }," + " {" + " \"name\": \"icmp drop\"," + " \"flags\": [\"log\"]," + " \"services\": [{\"name\":\"icmp\"}]," + " \"action\": \"drop\"" + " }" + " ]" + " }" + "}"; + + string l4_firewall = "{" + " \"app\": \"netfilter\"," + " \"l4_firewall_rules\": [" + " {" + " \"name\": \"allow_statefull_conns\"," + " \"flags\": [\"established\"]," + " \"action\": \"accept\"" + " }," + " {" + " \"name\": \"icmp drop\"," + " \"flags\": [\"log\"]," + " \"services\": [{\"name\":\"icmp\"}]," + " \"action\": \"drop\"" + " }" + " ]" + "}"; + + string old_configuration = "{" + " \"version\": \"" + old_version + "\"" + " \"app\": \"netfilter\"," + " \"l4_firewall_rules\": [" + " {" + " \"name\": \"allow_statefull_conns\"," + " \"flags\": [\"established\"]," + " \"action\": \"reject\"" + " }," + " {" + " \"name\": \"icmp drop\"," + " \"flags\": [\"log\"]," + " \"services\": [{\"name\":\"icmp\"}]," + " \"action\": \"drop\"" + " }" + " ]" + "}"; + + Maybe> json_parser_return = + map({{"l4_firewall", l4_firewall}, {"version", version_value}}); + EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration)); + EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)) + .WillOnce(Return(json_parser_return)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(old_configuration)); + EXPECT_CALL(mock_orchestration_status, + setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY)); + + EXPECT_CALL( + mock_orchestration_tools, + copyFile(l4_firewall_policy_path, l4_firewall_policy_path + backup_extension) + ).WillOnce(Return(true)); + EXPECT_CALL( + mock_orchestration_tools, + writeFile(l4_firewall, l4_firewall_policy_path)).WillOnce(Return(true) + ); + + // backup file doesn't exist so the copyFile function should be called 0 times + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)).Times(0); + + EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); EXPECT_CALL( mock_shell_cmd, @@ -997,6 +1115,7 @@ TEST_F(ServiceControllerTest, backupAttempts) EXPECT_CALL(mock_ml, yield(false)).Times(2); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, "")); EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value); @@ -1081,6 +1200,7 @@ TEST_F(ServiceControllerTest, MultiUpdateConfiguration) EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); EXPECT_CALL( mock_shell_cmd, @@ -1112,6 +1232,11 @@ TEST_F(ServiceControllerTest, MultiUpdateConfiguration) ).WillOnce(Return(Maybe(reply_msg))); EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, "")); + set changed_policies = { + "/etc/cp/conf/l4_firewall/l4_firewall.policy", + "/etc/cp/conf/orchestration/orchestration.policy" + }; + EXPECT_EQ(i_service_controller->moveChangedPolicies(), changed_policies); } class TestSendRequestToService : public ClientRest @@ -1139,6 +1264,7 @@ TEST_F(ServiceControllerTest, emptyServices) EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, "")); } @@ -1326,24 +1452,6 @@ TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest) EXPECT_EQ(i_service_controller->getPolicyVersion(), ""); - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN); - EXPECT_CALL( - mock_message, - sendMessage( - true, - "{\n \"id\": 1,\n \"policy_version\": \"1.0.2\"\n}", - I_Messaging::Method::POST, - string("127.0.0.1"), - l4_firewall_service_port, - conn_flags, - string("/set-new-configuration"), - string(), - _, - MessageTypeTag::GENERIC - ) - ).WillOnce(Return(Maybe(genError("")))); - EXPECT_TRUE(i_service_controller->isServiceInstalled("family1_id2")); EXPECT_CALL( @@ -1358,6 +1466,7 @@ TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest) EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, "")); EXPECT_THAT( @@ -1568,6 +1677,7 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles) EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_file_path, new_policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(conf_file_name, new_policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(new_policy_file_path)).WillOnce(Return(true)); EXPECT_CALL( mock_shell_cmd, @@ -1665,6 +1775,7 @@ TEST_F(ServiceControllerTest, test_delayed_reconf) EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)) .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_ml, yield(false)).Times(AnyNumber()); EXPECT_CALL( diff --git a/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc b/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc index 8003844..fe4648a 100755 --- a/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc +++ b/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc @@ -208,12 +208,14 @@ void ConfidenceCalculator::pullProcessedData(const std::vector& fil { dbgTrace(D_WAAP) << "Fetching the confidence set object"; bool is_first_pull = true; + bool is_ok = false; for (auto file : files) { ConfidenceFileDecryptor getConfFile; bool res = sendObjectWithRetry(getConfFile, I_Messaging::Method::GET, getUri() + "/" + file); + is_ok |= res; if (res && getConfFile.getConfidenceSet().ok()) { mergeFromRemote(getConfFile.getConfidenceSet().unpack(), is_first_pull); @@ -224,8 +226,8 @@ void ConfidenceCalculator::pullProcessedData(const std::vector& fil m_confidence_level = getConfFile.getConfidenceLevels().unpackMove(); } } - // is_first_pull = false -> at least one file was downloaded and merged - if (is_first_pull) { + // is_ok = false -> no file was downloaded and merged + if (!is_ok) { dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the remote state"; } } diff --git a/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc b/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc index 6822560..2a64ac4 100755 --- a/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc +++ b/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc @@ -151,9 +151,9 @@ void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector value.size()) { + return false; + } + + return value.compare(0, prefix.size(), prefix) == 0; +} + +inline bool str_ends_with(const std::string& value, const std::string& ending) +{ + if (ending.size() > value.size()) { + return false; + } + + return value.compare(value.size() - ending.size(), ending.size(), ending) == 0; +} + template _IT unquote_plus(_IT first, _IT last, bool decodeUrl=true, bool decodePlus=true) { _IT result = first; diff --git a/components/security_apps/waap/waap_component_impl.cc b/components/security_apps/waap/waap_component_impl.cc index 2d84415..fa0cedd 100755 --- a/components/security_apps/waap/waap_component_impl.cc +++ b/components/security_apps/waap/waap_component_impl.cc @@ -89,6 +89,8 @@ WaapComponent::Impl::init(const std::string &waapDataFileName) reputationAggregator.init(); + waapStateTable = Singleton::Consume::by(); + bool success = waf2_proc_start(waapDataFileName); if (!success) { dbgWarning(D_WAAP) << "WAF2 engine FAILED to initialize (probably failed to load signatures). Aborting!"; @@ -101,8 +103,6 @@ WaapComponent::Impl::init(const std::string &waapDataFileName) I_StaticResourcesHandler *static_resources = Singleton::Consume::by(); static_resources->registerStaticResource("cp-ab.js", "/etc/cp/conf/waap/cp-ab.js"); static_resources->registerStaticResource("cp-csrf.js", "/etc/cp/conf/waap/cp-csrf.js"); - - waapStateTable = Singleton::Consume::by(); } // Called when component is shut down diff --git a/components/signal_handler/signal_handler.cc b/components/signal_handler/signal_handler.cc index 2c484db..1f09ccd 100755 --- a/components/signal_handler/signal_handler.cc +++ b/components/signal_handler/signal_handler.cc @@ -28,17 +28,11 @@ #include #include -#if defined(use_unwind) -#if defined(alpine) || defined(PLATFORM_x86) +#ifdef UNWIND_LIBRARY #define UNW_LOCAL_ONLY #include #include -#endif // defined(alpine) || defined(PLATFORM_x86) -#endif // defined(use_unwind) - -#if !defined(alpine) && !defined(arm32_musl) -#include -#endif // not alpine && not arm32_musl +#endif // UNWIND_LIBRARY #include "debug.h" #include "common.h" @@ -82,11 +76,9 @@ public: Maybe> getBacktrace() override { +#ifdef UNWIND_LIBRARY + vector symbols; -#if defined(_UCLIBC_) || defined(arm32_musl) || !defined(use_unwind) - return genError("Could not print any backtrace entries using uclibc (backtrace_symbols not supported)"); -#else // not (_UCLIBC_ || arm32_musl) -#if defined(alpine) || defined(PLATFORM_x86) unw_cursor_t cursor; unw_context_t context; @@ -116,24 +108,14 @@ public: } symbols.push_back(buf); } -#else // not (alpine || PLATFORM_x86) - auto stack_trace_list = vector(stack_trace_max_len); - uint trace_len = backtrace(stack_trace_list.data(), stack_trace_list.size()); - if (trace_len == 0 ) return genError("Could not find any backtrace entries in the current process"); - - char **trace_prints = backtrace_symbols(stack_trace_list.data(), trace_len); - if (trace_prints == nullptr) return genError("Could not convert backtrace entries to symbol strings"); - - symbols.reserve(trace_len); - for (uint i = 0; i < trace_len; ++i) { - symbols.emplace_back(trace_prints[i]); - } - free(trace_prints); - -#endif // alpine || PLATFORM_x86 -#endif // _UCLIBC_ return symbols; + +#else // UNWIND_LIBRARY + + return genError("Could not print any backtrace entries using uclibc (backtrace_symbols not supported)"); + +#endif // UNWIND_LIBRARY } private: @@ -388,18 +370,13 @@ private: static void printStackTrace() { +#ifdef UNWIND_LIBRARY + if (out_trace_file_fd == -1) return; const char *stack_trace_title = "Stack trace:\n"; writeData(stack_trace_title, strlen(stack_trace_title)); -#if defined(_UCLIBC_) || defined(arm32_musl) || !defined(use_unwind) - const char *uclibc_error = - "Could not print any backtrace entries using uclibc (backtrace_symbols not supported)\n"; - writeData(uclibc_error, strlen(uclibc_error)); - return; -#else // not (_UCLIBC_ || arm32_musl) -#ifdef alpine unw_cursor_t cursor; unw_context_t uc; unw_getcontext(&uc); @@ -426,19 +403,14 @@ private: if (unw_step(&cursor) <= 0) return; } -#else // not alpine - void *stack_trace_list[stack_trace_max_len]; - uint actual_trace_len = backtrace(stack_trace_list, stack_trace_max_len); - if (actual_trace_len == 0 ) { - const char *no_bt_found_error = "Could not find any backtrace entries in the current process\n"; - writeData(no_bt_found_error, strlen(no_bt_found_error)); - return; - } +#else // UNWIND_LIBRARY - backtrace_symbols_fd(stack_trace_list, actual_trace_len, out_trace_file_fd); -#endif // alpine -#endif // _UCLIBC_ || arm32_musl + const char *uclibc_error = + "Could not print any backtrace entries using uclibc (backtrace_symbols not supported)\n"; + writeData(uclibc_error, strlen(uclibc_error)); + +#endif // UNWIND_LIBRARY } // LCOV_EXCL_STOP diff --git a/components/utils/http_transaction_data/http_transaction_data.cc b/components/utils/http_transaction_data/http_transaction_data.cc index ff4ace4..d1acb4d 100644 --- a/components/utils/http_transaction_data/http_transaction_data.cc +++ b/components/utils/http_transaction_data/http_transaction_data.cc @@ -101,7 +101,7 @@ deserializeStrParam(const Buffer &data, uint &cur_pos) << to_string(*str_size); cur_pos += *str_size; - + return move(res); } @@ -185,6 +185,39 @@ HttpTransactionData::createTransactionData(const Buffer &transaction_raw_data) dbgTrace(D_NGINX_ATTACHMENT) << "Successfully deserialized client port: " << client_port.unpack(); } + if (cur_pos == transaction_raw_data.size()) { + dbgDebug(D_NGINX_ATTACHMENT) + << "No extra data to read from buffer. This agent is working with an old " + << "attachment version that does not contain the parsed host and parsed uri elements."; + + HttpTransactionData transaction( + http_protocol.unpackMove(), + http_method.unpackMove(), + host_name.unpackMove(), + listening_addr.unpackMove(), + listening_port.unpackMove(), + uri.unpackMove(), + client_addr.unpackMove(), + client_port.unpackMove() + ); + + return transaction; + } + + Maybe ngx_parsed_host = deserializeStrParam(transaction_raw_data, cur_pos); + if (!ngx_parsed_host.ok()) { + return genError("Could not deserialize nginx host: " + ngx_parsed_host.getErr()); + } else { + dbgTrace(D_NGINX_ATTACHMENT) << "Successfully deserialized nginx_host: " << ngx_parsed_host.unpack(); + } + + Maybe ngx_parsed_uri = deserializeStrParam(transaction_raw_data, cur_pos); + if (!ngx_parsed_uri.ok()) { + return genError("Could not deserialize parsed URI: " + ngx_parsed_uri.getErr()); + } else { + dbgTrace(D_NGINX_ATTACHMENT) << "Successfully deserialized parsed URI: " << ngx_parsed_uri.unpack(); + } + // Fail if after parsing exact number of items, we didn't exactly consume whole buffer if (cur_pos != transaction_raw_data.size()) { dbgWarning(D_NGINX_ATTACHMENT) << "Nothing to deserialize, but raw data still remain"; @@ -195,14 +228,16 @@ HttpTransactionData::createTransactionData(const Buffer &transaction_raw_data) http_protocol.unpackMove(), http_method.unpackMove(), host_name.unpackMove(), + ngx_parsed_host.unpackMove(), listening_addr.unpackMove(), listening_port.unpackMove(), uri.unpackMove(), + ngx_parsed_uri.unpackMove(), client_addr.unpackMove(), client_port.unpackMove() ); - return move(transaction); + return transaction; } HttpTransactionData::HttpTransactionData ( @@ -214,14 +249,44 @@ HttpTransactionData::HttpTransactionData ( string _uri, IPAddr _client_ip, uint16_t _client_port +) + : + HttpTransactionData::HttpTransactionData( + _http_proto, + _method, + _host_name, + _host_name, + _listening_ip, + _listening_port, + _uri, + _uri, + _client_ip, + _client_port + ) +{ +} + +HttpTransactionData::HttpTransactionData ( + string _http_proto, + string _method, + string _host_name, + string _parsed_host, + IPAddr _listening_ip, + uint16_t _listening_port, + string _uri, + string _parsed_uri, + IPAddr _client_ip, + uint16_t _client_port ) : http_proto(move(_http_proto)), method(move(_method)), host_name(move(_host_name)), + parsed_host(move(_parsed_host)), listening_ip(move(_listening_ip)), listening_port(move(_listening_port)), uri(move(_uri)), + parsed_uri(move(_parsed_uri)), client_ip(move(_client_ip)), client_port(move(_client_port)), is_request(true), @@ -235,9 +300,11 @@ HttpTransactionData::HttpTransactionData() "", "GET", "", + "", IPAddr(), -1, "", + "", IPAddr(), -1 ) diff --git a/components/utils/http_transaction_data/http_transaction_data_ut/http_transaction_data_ut.cc b/components/utils/http_transaction_data/http_transaction_data_ut/http_transaction_data_ut.cc index f090b73..e15bc60 100644 --- a/components/utils/http_transaction_data/http_transaction_data_ut/http_transaction_data_ut.cc +++ b/components/utils/http_transaction_data/http_transaction_data_ut/http_transaction_data_ut.cc @@ -102,6 +102,8 @@ TEST_F(HttpTransactionTest, TestTransactionDataFromBuf) EXPECT_EQ(data.getHttpProtocol(), "HTTP/1.1"); EXPECT_EQ(data.getURI(), "/user-app/"); EXPECT_EQ(data.getHttpMethod(), "GET"); + EXPECT_EQ(data.getParsedURI(), "/user-app/"); + EXPECT_EQ(data.getParsedHost(), "localhost"); } TEST_F(HttpTransactionTest, TestTransactionDataBadVer) @@ -125,3 +127,45 @@ TEST_F(HttpTransactionTest, TestTransactionDataBadAddress) "Could not parse IP Address: String 'this.is.not.IP' is not a valid IPv4/IPv6 address" ); } + +TEST_F(HttpTransactionTest, TestTransactionDataFromBufWIthParsedHostAndParsedUri) +{ + Buffer meta_data = + Buffer(encodeInt16(strlen("HTTP/1.1"))) + + Buffer("HTTP/1.1") + + encodeInt16(3) + + Buffer("GET") + + encodeInt16(9) + + Buffer("localhost") + + encodeInt16(7) + + Buffer("0.0.0.0") + + encodeInt16(443) + + encodeInt16(11) + + Buffer("//user-app/") + + encodeInt16(9) + + Buffer("127.0.0.1") + + encodeInt16(47423) + + encodeInt16(10) + + Buffer("localhost2") + + encodeInt16(10) + + Buffer("/user-app/"); + + HttpTransactionData data = HttpTransactionData::createTransactionData(meta_data).unpack(); + stringstream data_stream; + data.print(data_stream); + string data_string( + "HTTP/1.1 GET\nFrom: 127.0.0.1:47423\nTo: localhost//user-app/ (listening on 0.0.0.0:443)\n" + ); + EXPECT_EQ(data_stream.str(), data_string); + + EXPECT_EQ(data.getSourceIP(), IPAddr::createIPAddr("127.0.0.1").unpack()); + EXPECT_EQ(data.getSourcePort(), 47423); + EXPECT_EQ(data.getListeningIP(), IPAddr::createIPAddr("0.0.0.0").unpack()); + EXPECT_EQ(data.getListeningPort(), 443); + EXPECT_EQ(data.getDestinationHost(), "localhost"); + EXPECT_EQ(data.getHttpProtocol(), "HTTP/1.1"); + EXPECT_EQ(data.getURI(), "//user-app/"); + EXPECT_EQ(data.getHttpMethod(), "GET"); + EXPECT_EQ(data.getParsedURI(), "/user-app/"); + EXPECT_EQ(data.getParsedHost(), "localhost2"); +} diff --git a/core/config/config.cc b/core/config/config.cc index 82b8caf..64ab463 100644 --- a/core/config/config.cc +++ b/core/config/config.cc @@ -35,6 +35,8 @@ using namespace Config; USE_DEBUG_FLAG(D_CONFIG); +static const string not_found = ""; + AgentProfileSettings AgentProfileSettings::default_profile_settings = AgentProfileSettings(); class registerExpectedConfigUpdates : public ClientRest @@ -107,7 +109,7 @@ public: PerContextValue getAllConfiguration(const std::vector &paths) const; const TypeWrapper & getResource(const vector &paths) const override; const TypeWrapper & getSetting(const vector &paths) const override; - const string & getProfileAgentSetting(const string &setting_name) const override; + string getProfileAgentSetting(const string &setting_name) const override; vector getProfileAgentSettings(const string ®ex) const override; const string & getConfigurationFlag(const string &flag_name) const override; @@ -139,9 +141,9 @@ public: void registerConfigPrepareCb(ConfigCb) override; void registerConfigLoadCb(ConfigCb) override; void registerConfigAbortCb(ConfigCb) override; + void clearOldTenants() override; private: - void clearOldTenants(); bool areTenantAndProfileActive(const TenantProfilePair &tenant_profile) const; void periodicRegistrationRefresh(); @@ -152,6 +154,7 @@ private: void reloadConfigurationContinuesWrapper(const string &version, uint id); vector fillMultiTenantConfigFiles(const map> &tenants); vector fillMultiTenantExpectedConfigFiles(const map> &tenants); + map getProfileAgentSetting() const; string getActiveTenant() const @@ -262,7 +265,6 @@ private: unordered_map, PerContextValue>> configuration_nodes; unordered_map, TypeWrapper>> settings_nodes; - map profile_settings; unordered_map config_flags; map, TypeWrapper> new_resource_nodes; @@ -337,13 +339,6 @@ ConfigComponent::Impl::init() false ); } - - mainloop->addRecurringRoutine( - I_MainLoop::RoutineType::System, - tenant_manager->getTimeoutVal(), - [this] () { clearOldTenants(); }, - "Config comp old tenant cleanup" - ); } static @@ -428,13 +423,13 @@ ConfigComponent::Impl::getSetting(const vector &paths) const return empty; } -const string & +string ConfigComponent::Impl::getProfileAgentSetting(const string &setting_name) const { + auto profile_settings = getProfileAgentSetting(); auto setting_raw_val = profile_settings.find(setting_name); if (setting_raw_val != profile_settings.end()) return setting_raw_val->second; - const static string not_found = ""; return not_found; } @@ -447,7 +442,6 @@ ConfigComponent::Impl::getConfigurationFlag(const string &flag_name) const flag = config_flags.find(flag_name); if (flag != config_flags.end()) return flag->second; - const static string not_found = ""; return not_found; } @@ -535,6 +529,7 @@ ConfigComponent::Impl::getProfileAgentSettings(const string ®ex) const { vector setting_raw_values; boost::regex reg(regex); + auto profile_settings = getProfileAgentSetting(); for (auto &setting : profile_settings) { if (NGEN::Regex::regexMatch(__FILE__, __LINE__, setting.first, reg)) { setting_raw_values.push_back(setting.second); @@ -781,23 +776,6 @@ ConfigComponent::Impl::commitSuccess() configuration_nodes = move(new_configuration_nodes); settings_nodes = move(new_settings_nodes); - AgentProfileSettings profile_agent_settings = getSettingWithDefault( - AgentProfileSettings::default_profile_settings, - "agentSettings" - ); - - AgentProfileSettings general_agent_settings = getSettingWithDefault( - AgentProfileSettings::default_profile_settings, - "generalAgentSettings" - ); - - auto tmp_general_settings = general_agent_settings.getSettings(); - - for (const pair &profile_setting : profile_agent_settings.getSettings()) { - tmp_general_settings.insert(profile_setting); - } - - profile_settings = tmp_general_settings; reloadFileSystemPaths(); for (auto &cb : configuration_commit_cbs) { @@ -868,14 +846,16 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as map> files; for (const auto &path : config_file_paths) { + dbgTrace(D_CONFIG) << "Inserting " << path << " to the list of files to be handled"; auto fullpath = config_directory_path + path; files.emplace(fullpath, make_shared(fullpath)); } map> active_tenants = - tenant_manager ? tenant_manager->fetchActiveTenantsAndProfiles() : map>(); + tenant_manager ? tenant_manager->fetchAndUpdateActiveTenantsAndProfiles(true) : map>(); dbgTrace(D_CONFIG) << "Number of active tenants found while reloading configuration: " << active_tenants.size(); + clearOldTenants(); const vector &config_files = fillMultiTenantConfigFiles(active_tenants); const vector &expected_config_files = fillMultiTenantExpectedConfigFiles(active_tenants); @@ -903,6 +883,22 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as return res; } +map +ConfigComponent::Impl::getProfileAgentSetting() const +{ + auto general_sets = getSettingWithDefault(AgentProfileSettings::default_profile_settings, "generalAgentSettings"); + + auto settings = general_sets.getSettings(); + + auto profile_sets = getSettingWithDefault(AgentProfileSettings::default_profile_settings, "agentSettings"); + auto profile_settings = profile_sets.getSettings(); + for (const auto &profile_setting : profile_settings) { + settings.insert(profile_setting); + } + + return settings; +} + void ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version, uint id) { diff --git a/core/config/config_globals.cc b/core/config/config_globals.cc index bb31a92..cb8f607 100644 --- a/core/config/config_globals.cc +++ b/core/config/config_globals.cc @@ -77,6 +77,12 @@ getFilesystemPathConfig() return Singleton::Consume::from()->getFilesystemPathConfig(); } +void +clearOldTenants() +{ + Singleton::Consume::from()->clearOldTenants(); +} + const string & getLogFilesPathConfig() { diff --git a/core/include/attachments/nginx_attachment_common.h b/core/include/attachments/nginx_attachment_common.h index 01fa278..89697b4 100755 --- a/core/include/attachments/nginx_attachment_common.h +++ b/core/include/attachments/nginx_attachment_common.h @@ -18,6 +18,7 @@ #include #include #include +#include #define MAX_NGINX_UID_LEN 32 #define NUM_OF_NGINX_IPC_ELEMENTS 200 @@ -183,6 +184,10 @@ typedef enum ngx_http_meta_data CLIENT_ADDR_SIZE, CLIENT_ADDR_DATA, CLIENT_PORT, + PARSED_HOST_SIZE, + PARSED_HOST_DATA, + PARSED_URI_SIZE, + PARSED_URI_DATA, META_DATA_COUNT } ngx_http_meta_data_e; @@ -242,6 +247,7 @@ typedef struct __attribute__((__packed__)) ngx_http_cp_web_response_data { } custom_response_data; struct __attribute__((__packed__)) ngx_http_cp_redirect_data { + uint8_t unused_dummy; uint8_t add_event_id; uint16_t redirect_location_size; char redirect_location[0]; @@ -249,6 +255,12 @@ typedef struct __attribute__((__packed__)) ngx_http_cp_web_response_data { } response_data; } ngx_http_cp_web_response_data_t; +static_assert( + sizeof(((ngx_http_cp_web_response_data_t*)0)->response_data.custom_response_data) == + sizeof(((ngx_http_cp_web_response_data_t*)0)->response_data.redirect_data), + "custom_response_data must be equal to redirect_data in size" +); + typedef union __attribute__((__packed__)) ngx_http_cp_modify_data { ngx_http_cp_inject_data_t inject_data[0]; ngx_http_cp_web_response_data_t web_response_data[0]; diff --git a/core/include/general/intelligence_comp_v2.h b/core/include/general/intelligence_comp_v2.h index 105c879..796207a 100755 --- a/core/include/general/intelligence_comp_v2.h +++ b/core/include/general/intelligence_comp_v2.h @@ -20,6 +20,8 @@ #include "i_messaging.h" #include "i_mainloop.h" #include "i_time_get.h" +#include "i_agent_details.h" +#include "i_rest_api.h" #include "component.h" class IntelligenceComponentV2 @@ -28,6 +30,8 @@ class IntelligenceComponentV2 Singleton::Provide, Singleton::Consume, Singleton::Consume, + Singleton::Consume, + Singleton::Consume, Singleton::Consume { public: diff --git a/core/include/general/tenant_manager.h b/core/include/general/tenant_manager.h index c64a0f1..6b99d05 100644 --- a/core/include/general/tenant_manager.h +++ b/core/include/general/tenant_manager.h @@ -21,6 +21,7 @@ #include "i_time_get.h" #include "i_instance_awareness.h" #include "component.h" +#include "i_shell_cmd.h" enum class TenantManagerType { CLIENT, SERVER }; @@ -33,7 +34,8 @@ class TenantManager Singleton::Consume, Singleton::Consume, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: TenantManager(); diff --git a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h index f319eea..365120a 100755 --- a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h +++ b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h @@ -25,12 +25,25 @@ #include "intelligence_is_v2/intelligence_query_v2.h" #include "config.h" +namespace Intelligence { + +class Invalidation; + +} // namespace Intelligence + class I_Intelligence_IS_V2 { public: + virtual bool sendInvalidation(const Intelligence::Invalidation &invalidation) const = 0; + virtual Maybe registerInvalidation( + const Intelligence::Invalidation &invalidation, + const std::function &callback + ) = 0; + virtual void unregisterInvalidation(uint id) = 0; + template Maybe>> - queryIntelligence(QueryRequest &query_request) + queryIntelligence(QueryRequest &query_request, bool ignore_in_progress = false, bool is_pretty = true) { uint assets_limit = query_request.getAssetsLimit(); static const uint upper_assets_limit = 50; @@ -50,7 +63,7 @@ public: return genError("Paging is activated and already finished. No need for more queries."); } - IntelligenceQuery intelligence_query(query_request); + IntelligenceQuery intelligence_query(query_request, is_pretty); static const std::string query_uri = "/api/v2/intelligence/assets/query"; bool res = getIsOfflineOnly() ? false : sendQueryObject(intelligence_query, query_uri, assets_limit); @@ -67,12 +80,15 @@ public: } } + if (ignore_in_progress && intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS) { + return genError("Query intelligence response with InProgress status"); + } return intelligence_query.getData(); } template Maybe>>>> - queryIntelligence(std::vector &query_requests) + queryIntelligence(std::vector &query_requests, bool is_pretty = true) { static const uint upper_assets_limit = 50; static const uint upper_confidence_limit = 1000; @@ -95,7 +111,7 @@ public: ); } } - IntelligenceQuery intelligence_query(query_requests); + IntelligenceQuery intelligence_query(query_requests, is_pretty); static const std::string query_uri = "/api/v2/intelligence/assets/queries"; dbgTrace(D_INTELLIGENCE) << "Sending intelligence bulk request with " << query_requests.size() << " items"; diff --git a/core/include/services_sdk/interfaces/i_messaging_downloader.h b/core/include/services_sdk/interfaces/i_messaging_downloader.h deleted file mode 100755 index 2e2ea61..0000000 --- a/core/include/services_sdk/interfaces/i_messaging_downloader.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __I_MESSAGING_DOWNLOADER_H__ -#define __I_MESSAGING_DOWNLOADER_H__ - -#include -#include - -#include "maybe_res.h" - -class I_MessagingDownloader -{ -public: - using OnCompleteCB = std::function &)>; - - virtual bool downloadFile( - const std::string &file_name, - const std::string &url, - OnCompleteCB cb = nullptr, - const unsigned int port = 0 - ) = 0; - -protected: - virtual ~I_MessagingDownloader() {} -}; - -#endif // __I_MESSAGING_DOWNLOADER_H__ diff --git a/core/include/services_sdk/interfaces/i_rest_api.h b/core/include/services_sdk/interfaces/i_rest_api.h index 366bc2c..682c20d 100644 --- a/core/include/services_sdk/interfaces/i_rest_api.h +++ b/core/include/services_sdk/interfaces/i_rest_api.h @@ -48,6 +48,8 @@ public: return addRestCall(oper, uri, std::make_unique>()); } + virtual uint16_t getListeningPort() const = 0; + protected: ~I_RestApi() {} virtual bool addRestCall(RestAction oper, const std::string &uri, std::unique_ptr &&init) = 0; diff --git a/core/include/services_sdk/interfaces/i_tenant_manager.h b/core/include/services_sdk/interfaces/i_tenant_manager.h index 167eb92..444836d 100644 --- a/core/include/services_sdk/interfaces/i_tenant_manager.h +++ b/core/include/services_sdk/interfaces/i_tenant_manager.h @@ -21,12 +21,11 @@ #include #include +#include "tenant_profile_pair.h" + class I_TenantManager { public: - using newTenantCB = std::function &)>; - - virtual void uponNewTenants(const newTenantCB &cb) = 0; virtual bool areTenantAndProfileActive(const std::string &tenant_id, const std::string &profile_id) const = 0; virtual std::set fetchAllActiveTenants() const = 0; @@ -36,14 +35,13 @@ public: const std::string &profile_id ) const = 0; virtual std::map> fetchActiveTenantsAndProfiles() const = 0; + virtual std::map> fetchAndUpdateActiveTenantsAndProfiles(bool update) = 0; virtual std::set fetchProfileIds(const std::string &tenant_id) const = 0; virtual void deactivateTenant(const std::string &tenant_id, const std::string &profile_id) = 0; virtual void addActiveTenantAndProfile(const std::string &tenant_id, const std::string &profile_id) = 0; - virtual std::chrono::microseconds getTimeoutVal() const = 0; - virtual std::set getProfileIdsForRegionAccount( const std::string &tenant_id, const std::string ®ion, diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h index 4e72df2..66e7560 100755 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h @@ -27,20 +27,22 @@ template class IntelligenceQuery { public: - IntelligenceQuery(QueryRequest &filter) + IntelligenceQuery(QueryRequest &filter, bool is_pretty) : request(filter), response(), responses(), - is_bulk(false) + is_bulk(false), + is_pretty(is_pretty) {} - IntelligenceQuery(std::vector &filters) + IntelligenceQuery(std::vector &filters, bool is_pretty) : requests(filters), response(), responses(), - is_bulk(true) + is_bulk(true), + is_pretty(is_pretty) {} Maybe genJson() const; @@ -67,6 +69,7 @@ private: IntelligenceQueryResponse response; std::vector> responses; bool is_bulk; + bool is_pretty; }; #include "intelligence_query_v2_impl.h" diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h index e3659d3..eebbafd 100755 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h @@ -18,6 +18,9 @@ #error intelligence_query_impl_v2.h should not be included directly! #endif // __INTELLIGENCE_QUERY_V2_H__ +#include +#include "json_stream.h" + USE_DEBUG_FLAG(D_INTELLIGENCE); template @@ -32,9 +35,10 @@ Maybe IntelligenceQuery::genJson() const { { - std::stringstream out; + std::stringstream str_stream; + JsonStream json_stream(&str_stream, is_pretty); { - cereal::JSONOutputArchive out_ar(out); + cereal::JSONOutputArchive out_ar(json_stream); if (is_bulk) { std::vector bulk_requests; int index = 0; @@ -46,7 +50,8 @@ IntelligenceQuery::genJson() const request.saveToJson(out_ar); } } - return out.str(); + + return str_stream.str(); } } diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h index 542cdf2..dd2ba45 100755 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h @@ -43,7 +43,9 @@ enum class Condition STARTS_WITH, CONTAINS, IN, - NOT_IN + NOT_IN, + GREATER_THAN, + LESS_THAN }; enum class CursorState { diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/json_stream.h b/core/include/services_sdk/interfaces/intelligence_is_v2/json_stream.h new file mode 100644 index 0000000..21369b0 --- /dev/null +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/json_stream.h @@ -0,0 +1,35 @@ +// Copyright (C) 2023 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __JSON_STREAM_H__ +#define __JSON_STREAM_H__ + +#include + +class JsonStream : public std::streambuf, public std::ostream +{ +public: + JsonStream(std::ostream *os, bool is_pretty); + +private: + int overflow(int c) override; + void emplace(char c); + void add(char c); + + std::ostream *os; + bool is_prev_single_backslash = false; + bool is_pretty; + bool in_string = false; +}; + +#endif // __JSON_STREAM_H__ diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h index aa0029b..7fb173b 100755 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h @@ -14,6 +14,7 @@ #ifndef __QUERY_FILTER_V2_H__ #define __QUERY_FILTER_V2_H__ +#include #include #include #include @@ -29,6 +30,8 @@ using namespace Intelligence_IS_V2; class SerializableQueryCondition { public: + typedef boost::variant ValueVariant; + SerializableQueryCondition() {} SerializableQueryCondition(Condition _condition_type, std::string _key, std::string _value) @@ -36,18 +39,25 @@ public: condition_type(_condition_type), key(_key), value(_value) - {}; + {} + + SerializableQueryCondition(Condition _condition_type, std::string _key, int64_t _value) + : + condition_type(_condition_type), + key(_key), + value(_value) + {} void save(cereal::JSONOutputArchive &ar) const; Condition getConditionType() const { return condition_type; } const std::string & getKey() const { return key; } - const std::string & getValue() const { return value; } + const ValueVariant & getValue() const { return value; } private: Condition condition_type = Condition::EQUALS; std::string key = ""; - std::string value = ""; + ValueVariant value = ""; }; class SerializableQueryFilter @@ -55,22 +65,18 @@ class SerializableQueryFilter public: SerializableQueryFilter() {} SerializableQueryFilter(Condition condition_type, const std::string &key, const std::string &value); - SerializableQueryFilter( - Operator operator_type, - Condition condition_type, - const std::string &key, - const std::string &value - ); + SerializableQueryFilter(Condition condition_type, const std::string &key, const int64_t &value); void save(cereal::JSONOutputArchive &ar) const; void addCondition(Condition condition_type, const std::string &key, const std::string &value); + void addCondition(Condition condition_type, const std::string &key, const int64_t &value); Operator getOperator() const { return operator_type; } const std::vector & getConditionOperands() const { return condition_operands; } const std::vector & getQueriesOperands() const { return queries_operands; } - const std::string & getConditionValueByKey(const std::string &key) const; + Maybe getConditionValueByKey(const std::string &key) const; bool empty() const { return condition_operands.empty() && queries_operands.empty(); } @@ -80,7 +86,8 @@ public: private: void saveCondition(cereal::JSONOutputArchive &ar) const; void saveOperation(cereal::JSONOutputArchive &ar) const; - SerializableQueryFilter calcOperator(const SerializableQueryFilter &other_query, const Operator &operator_type); + bool isOperatorComp(const Operator &oper) const; + SerializableQueryFilter calcOperator(const SerializableQueryFilter &other_query, const Operator &oper); Operator operator_type = Operator::NONE; std::vector queries_operands = {}; diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h index c3653d0..ecf90bf 100755 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h @@ -37,6 +37,14 @@ public: AttributeKeyType type = AttributeKeyType::MAIN ); + QueryRequest( + Condition condition_type, + const std::string &key, + const int64_t &value, + bool full_response, + AttributeKeyType type = AttributeKeyType::MAIN + ); + void saveToJson(cereal::JSONOutputArchive &ar) const; void save(cereal::JSONOutputArchive &ar) const; @@ -51,6 +59,13 @@ public: AttributeKeyType attribute_type = AttributeKeyType::MAIN ); + void addCondition( + Condition condition_type, + const std::string &key, + const int64_t &value, + AttributeKeyType attribute_type = AttributeKeyType::MAIN + ); + void setRequestedAttr( const std::string &attr, AttributeKeyType attribute_type = AttributeKeyType::REGULAR diff --git a/core/include/services_sdk/interfaces/mock/mock_rest_api.h b/core/include/services_sdk/interfaces/mock/mock_rest_api.h index 28619e5..33acfe4 100644 --- a/core/include/services_sdk/interfaces/mock/mock_rest_api.h +++ b/core/include/services_sdk/interfaces/mock/mock_rest_api.h @@ -8,6 +8,7 @@ class MockRestApi : public Singleton::Provide::From> { public: + MOCK_CONST_METHOD0(getListeningPort, uint16_t()); // You can't mock a function with an R-value reference. So mock a slightly different one MOCK_METHOD3(mockRestCall, bool(RestAction, const std::string &, const std::unique_ptr &)); diff --git a/core/include/services_sdk/interfaces/mock/mock_tenant_manager.h b/core/include/services_sdk/interfaces/mock/mock_tenant_manager.h index ab9bfc1..5fb50fb 100644 --- a/core/include/services_sdk/interfaces/mock/mock_tenant_manager.h +++ b/core/include/services_sdk/interfaces/mock/mock_tenant_manager.h @@ -13,8 +13,6 @@ class MockTenantManager : public Singleton::Provide::From> { public: - MOCK_METHOD1(uponNewTenants, void(const I_TenantManager::newTenantCB &cb)); - MOCK_CONST_METHOD0(fetchActiveTenantsAndProfiles, std::map>()); MOCK_CONST_METHOD0(fetchActiveTenants, std::set()); MOCK_CONST_METHOD0(fetchAllActiveTenants, std::set()); @@ -26,13 +24,11 @@ public: MOCK_CONST_METHOD2(areTenantAndProfileActive, bool(const std::string &, const std::string &)); MOCK_METHOD2(addActiveTenantAndProfile, void(const std::string &, const std::string &)); MOCK_METHOD2(deactivateTenant, void(const std::string &, const std::string &)); + MOCK_METHOD1(fetchAndUpdateActiveTenantsAndProfiles, std::map>(bool)); MOCK_CONST_METHOD3( getProfileIdsForRegionAccount, std::set(const std::string &, const std::string &, const std::string &) ); - - MOCK_CONST_METHOD0(getTimeoutVal, std::chrono::microseconds()); - private: MOCK_METHOD3( addInstance, diff --git a/core/include/services_sdk/resources/config.h b/core/include/services_sdk/resources/config.h index 85e3d64..76179ca 100644 --- a/core/include/services_sdk/resources/config.h +++ b/core/include/services_sdk/resources/config.h @@ -90,6 +90,7 @@ std::string getConfigurationFlagWithDefault(const std::string &default_val, cons const std::string & getFilesystemPathConfig(); const std::string & getLogFilesPathConfig(); +void clearOldTenants(); std::string getPolicyConfigPath( const std::string &name, diff --git a/core/include/services_sdk/resources/config/config_impl.h b/core/include/services_sdk/resources/config/config_impl.h index 6d38274..9f318f8 100644 --- a/core/include/services_sdk/resources/config/config_impl.h +++ b/core/include/services_sdk/resources/config/config_impl.h @@ -155,7 +155,7 @@ Maybe getProfileAgentSetting(const std::string &setting) { auto i_config = Singleton::Consume::from(); - const std::string &value = i_config->getProfileAgentSetting(setting); + std::string value = i_config->getProfileAgentSetting(setting); if (value.empty()) return TypeWrapper::failMissing(); diff --git a/core/include/services_sdk/resources/config/i_config.h b/core/include/services_sdk/resources/config/i_config.h index 4ab3022..68e1586 100644 --- a/core/include/services_sdk/resources/config/i_config.h +++ b/core/include/services_sdk/resources/config/i_config.h @@ -39,7 +39,7 @@ public: virtual PerContextValue getAllConfiguration(const std::vector &paths) const = 0; virtual const TypeWrapper & getResource(const std::vector &paths) const = 0; virtual const TypeWrapper & getSetting(const std::vector &paths) const = 0; - virtual const string & getProfileAgentSetting(const string &setting_name) const = 0; + virtual string getProfileAgentSetting(const string &setting_name) const = 0; virtual vector getProfileAgentSettings(const string &setting_name_regex) const = 0; virtual const string & getConfigurationFlag(const string &flag_name) const = 0; @@ -104,6 +104,8 @@ public: virtual void registerConfigLoadCb(ConfigCb) = 0; virtual void registerConfigAbortCb(ConfigCb) = 0; + virtual void clearOldTenants() = 0; + protected: virtual ~I_Config() {} }; diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index 4ee860e..7d3578b 100755 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -148,6 +148,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_IOT_QUERY_INTELLIGENCE, D_IOT_AUXILIARY) DEFINE_FLAG(D_IOT_SAVE_PERSISTENT, D_IOT_AUXILIARY) DEFINE_FLAG(D_IOT_DOCKER, D_IOT_AUXILIARY) + DEFINE_FLAG(D_IOT_REST_AUTHENTICATION, D_IOT_AUXILIARY) DEFINE_FLAG(D_IOT_ENFORCE, D_IOT_NEXT) DEFINE_FLAG(D_IOT_ENFORCE_POLICY, D_IOT_ENFORCE) DEFINE_FLAG(D_IOT_ENFORCE_ASSETS, D_IOT_ENFORCE) @@ -156,6 +157,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_IOT_QUERY_ASSETS, D_IOT_RISK) DEFINE_FLAG(D_IOT_INDICATOR_DATA, D_IOT_RISK) DEFINE_FLAG(D_IOT_INDICATORS, D_IOT_RISK) + DEFINE_FLAG(D_IOT_DOCKER_WATCHDOG, D_IOT_RISK) DEFINE_FLAG(D_IOT_DISCOVERY, D_IOT_NEXT) DEFINE_FLAG(D_IOT_PROBE, D_IOT_DISCOVERY) DEFINE_FLAG(D_IOT_ASSETS_DATA, D_IOT_DISCOVERY) diff --git a/core/include/services_sdk/resources/intelligence_invalidation.h b/core/include/services_sdk/resources/intelligence_invalidation.h new file mode 100644 index 0000000..2580960 --- /dev/null +++ b/core/include/services_sdk/resources/intelligence_invalidation.h @@ -0,0 +1,74 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __INTELLIGENCE_INVALIDATION_H__ +#define __INTELLIGENCE_INVALIDATION_H__ + +#include +#include +#include +#include + +#include "maybe_res.h" +#include "enum_array.h" + +class I_Intelligence_IS_V2; + +namespace Intelligence +{ + +enum class ClassifierType { CLASS, CATEGORY, FAMILY, GROUP, ORDER, KIND }; +enum class ObjectType { ASSET, ZONE, POLICY_PACKAGE, CONFIGURATION, SESSION }; + +class Invalidation +{ +public: + Invalidation(const std::string &class_value); + + Invalidation & setClassifier(ClassifierType type, const std::string &val); + Invalidation & setStringAttr(const std::string &attr, const std::string &val); + Invalidation & setStringSetAttr(const std::string &attr, const std::set &val); + Invalidation & setSourceId(const std::string &id); + Invalidation & setObjectType(ObjectType type); + + std::string getClassifier(ClassifierType type) const { return classifiers[type]; } + Maybe getStringAttr(const std::string &attr) const; + Maybe, void> getStringSetAttr(const std::string &attr) const; + const Maybe & getSourceId() const { return source_id; } + const Maybe & getObjectType() const { return object_type; } + + bool report(I_Intelligence_IS_V2 *interface) const; + + Maybe startListening(I_Intelligence_IS_V2 *interface, const std::function &cb); + void stopListening(I_Intelligence_IS_V2 *interface); + + Maybe genJson() const; + std::string genObject() const; + bool isLegalInvalidation() const; + + bool matches(const Invalidation &other) const; + +private: + bool hasAttr(const std::string &key, const std::string &value) const; + + EnumArray classifiers; + std::map string_main_attr; + std::map> set_string_main_attr; + Maybe source_id; + Maybe object_type; + Maybe listening_id; +}; + +} // namespace Intelligence + +#endif // __INTELLIGENCE_INVALIDATION_H__ diff --git a/core/include/services_sdk/resources/report/base_field.h b/core/include/services_sdk/resources/report/base_field.h index d517eaa..90c4590 100755 --- a/core/include/services_sdk/resources/report/base_field.h +++ b/core/include/services_sdk/resources/report/base_field.h @@ -25,8 +25,6 @@ #include "debug.h" #include "flags.h" #include "config.h" -#include "virtual_container.h" -#include "Log_modifiers.h" enum class LogFieldOption { XORANDB64, COUNT }; @@ -111,14 +109,53 @@ class LogField : Singleton::Consume getSyslogAndCef() const override { std::string value(Details::getValueAsString(getValue())); - auto modifier1 = makeVirtualContainer(value); - auto modifier2 = makeVirtualContainer(modifier1); - auto modifier3 = makeVirtualContainer(modifier2); - auto modifier4 = makeVirtualContainer(modifier3); - auto modifier5 = makeVirtualContainer(modifier4); - auto modifier6 = makeVirtualContainer(modifier5); - auto modifier7 = makeVirtualContainer(modifier6); - return name + "=\"" + std::string(modifier7.begin(), modifier7.end()) + "\""; + + std::string encoded_value; + encoded_value.reserve(value.size() + 6); + for (char ch : value) { + switch (ch) { + case '\\': { + encoded_value.push_back('\\'); + encoded_value.push_back('\\'); + break; + } + case '\n': { + encoded_value.push_back('\\'); + encoded_value.push_back('n'); + break; + } + case '\r': { + encoded_value.push_back('\\'); + encoded_value.push_back('r'); + break; + } + case '"': { + encoded_value.push_back('\\'); + encoded_value.push_back('"'); + break; + } + case '\'': { + encoded_value.push_back('\\'); + encoded_value.push_back('\''); + break; + } + case ']': { + encoded_value.push_back('\\'); + encoded_value.push_back(']'); + break; + } + case '=': { + encoded_value.push_back('\\'); + encoded_value.push_back('='); + break; + } + default: { + encoded_value.push_back(ch); + } + } + } + + return name + "=\"" + encoded_value + "\""; } // LCOV_EXCL_START Reason: seems that assert prevent the LCOV from identifying that method was tested diff --git a/core/include/services_sdk/resources/report/report_enums.h b/core/include/services_sdk/resources/report/report_enums.h index 1e21246..6285eea 100755 --- a/core/include/services_sdk/resources/report/report_enums.h +++ b/core/include/services_sdk/resources/report/report_enums.h @@ -63,6 +63,8 @@ enum class Tags { DEPLOYMENT_K8S, LAYER_7_ACCESS_CONTROL, HORIZON_TELEMETRY_METRICS, + CROWDSEC, + PLAYGROUND, COUNT }; diff --git a/core/include/services_sdk/utilities/customized_cereal_multimap.h b/core/include/services_sdk/utilities/customized_cereal_multimap.h index 6a62766..0f19f10 100755 --- a/core/include/services_sdk/utilities/customized_cereal_multimap.h +++ b/core/include/services_sdk/utilities/customized_cereal_multimap.h @@ -20,6 +20,8 @@ #include "cereal/types/vector.hpp" #include "cereal/archives/json.hpp" +#include "rest/schema_printer.h" + template class SerializableMultiMap : std::map ... { @@ -40,6 +42,27 @@ public: } } + void clear() { clear(Types()...); } + + template + std::map & + getMap() + { + return static_cast &>(*this); + } + + void + performOutputingSchema(std::ostream &out, int level) + { + RestHelper::printIndent(out, level) << "\"additionalProperties\": {\n"; + RestHelper::printIndent(out, level + 1) << "\"anyOf\": ["; + printTypes(out, level +2, 0); + out << '\n'; + RestHelper::printIndent(out, level + 1) << "]\n"; + RestHelper::printIndent(out, level) << "}"; + } + +private: template void load(const std::string &key, Archive &archive, T t, More ... more) @@ -59,20 +82,29 @@ public: std::map::operator[](key) = t; } - - void clear() { clear(Types()...); } - template void clear(const T &t, const More & ... more) { clear(t); clear(more...); } template void clear(const T &) { std::map::clear(); } + template + void + printTypes(std::ostream &out, int level, uint) + { + printTypes(out, level, 0); + out << ","; + printTypes(out, level, 0); + } template - std::map & - getMap() + void + printTypes(std::ostream &out, int level, int) { - return static_cast &>(*this); + out << '\n'; + RestHelper::printIndent(out, level) << "{\n"; + TypeDector::type(out, level + 1); + RestHelper::printIndent(out, level) << "}"; } + }; diff --git a/core/intelligence_is_v2/CMakeLists.txt b/core/intelligence_is_v2/CMakeLists.txt index a9d942c..2fb0b8d 100755 --- a/core/intelligence_is_v2/CMakeLists.txt +++ b/core/intelligence_is_v2/CMakeLists.txt @@ -1,7 +1,7 @@ -include_directories(include) -add_library( - intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc - intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc -) - -add_subdirectory(intelligence_is_v2_ut) +include_directories(include) +add_library( + intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc + intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc json_stream.cc invalidation.cc +) + +add_subdirectory(intelligence_is_v2_ut) diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc index c714d17..ddf7bf2 100755 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -19,13 +19,200 @@ #include "config.h" #include "table.h" #include "intelligence_is_v2/query_response_v2.h" +#include "intelligence_invalidation.h" using namespace std; using namespace chrono; using namespace Intelligence_IS_V2; +using namespace Intelligence; USE_DEBUG_FLAG(D_INTELLIGENCE); +static const string primary_port_setting = "local intelligence server primary port"; +static const string secondary_port_setting = "local intelligence server secondary port"; +static const string invalidation_uri = "/api/v2/intelligence/invalidation"; +static const string registration_uri = "/api/v2/intelligence/invalidation/register"; + +class I_InvalidationCallBack +{ +public: + virtual void performCallBacks(const Invalidation &invalidation) const = 0; + +protected: + virtual ~I_InvalidationCallBack() {} +}; + +using MainAttrTypes = SerializableMultiMap>; + +static const map object_names = { + { "asset", Intelligence::ObjectType::ASSET }, + { "zone", Intelligence::ObjectType::ZONE }, + { "policyPackage", Intelligence::ObjectType::POLICY_PACKAGE }, + { "configuration", Intelligence::ObjectType::CONFIGURATION }, + { "session", Intelligence::ObjectType::SESSION } +}; + +class InvalidationRegistration +{ +public: + class RestCall + { + public: + RestCall(const stringstream &input) : rest_body(input.str()) {} + Maybe genJson() const { return rest_body; } + ostream & print(ostream &os) { return os << rest_body; } + + private: + string rest_body; + }; + + void + addInvalidation(const Invalidation &invalidation) + { + if (!first) stream << ','; + stream << ' ' << invalidation.genObject(); + first = false; + } + + RestCall + genJson() const + { + stringstream res; + + res << "{ \"apiVersion\": \"v2\", \"communicationType\": \"sync\", "; + auto details = Singleton::Consume::by(); + res << "\"name\": \"" << details->getAgentId() << "\", "; + auto rest = Singleton::Consume::by(); + res << "\"url\": \"http://127.0.0.1:" << rest->getListeningPort() <<"/set-new-invalidation\", "; + res << "\"dataMap\": ["; + res << stream.str(); + res << " ] }"; + + return res; + } + +private: + bool first = true; + stringstream stream; +}; + +class InvalidationCallBack : Singleton::Provide::SelfInterface +{ +public: + uint + emplace(const Invalidation &invalidation, function cb) + { + dbgDebug(D_INTELLIGENCE) << "registering " << invalidation.genObject(); + do { + ++running_id; + } while (callbacks.find(running_id) != callbacks.end()); + callbacks.emplace(running_id, make_pair(invalidation, cb)); + return running_id; + } + + void erase(uint id) { callbacks.erase(id); } + bool empty() const { return callbacks.empty(); } + + InvalidationRegistration::RestCall + getRegistration() const + { + InvalidationRegistration registration; + + for (auto ®isted_invalidation : callbacks) { + registration.addInvalidation(registed_invalidation.second.first); + } + + return registration.genJson(); + } + + void + performCallBacks(const Invalidation &invalidation) const override + { + for (auto ®isted_invalidation : callbacks) { + performCallBacksImpl(invalidation, registed_invalidation.second); + } + } + +private: + void + performCallBacksImpl( + const Invalidation &actual_invalidation, + const pair> &invalidation_and_cb + ) const + { + auto ®istereed_invalidation = invalidation_and_cb.first; + auto &cb = invalidation_and_cb.second; + if (registereed_invalidation.matches(actual_invalidation)) cb(actual_invalidation); + } + + map>> callbacks; + uint running_id = 0; +}; + +class RecieveInvalidation : public ServerRest +{ +public: + void + doCall() override + { + Invalidation invalidation(class_name); + + if (category.isActive()) invalidation.setClassifier(ClassifierType::CATEGORY, category.get()); + if (family.isActive()) invalidation.setClassifier(ClassifierType::FAMILY, family.get()); + if (group.isActive()) invalidation.setClassifier(ClassifierType::GROUP, group.get()); + if (order.isActive()) invalidation.setClassifier(ClassifierType::ORDER, order.get()); + if (kind.isActive()) invalidation.setClassifier(ClassifierType::KIND, kind.get()); + + if (mainAttributes.isActive()) { + auto strings = getMainAttr(); + for (const auto &value : strings) { + invalidation.setStringAttr(value.first, value.second); + } + + auto string_sets = getMainAttr>(); + for (const auto &value : string_sets) { + invalidation.setStringSetAttr(value.first, value.second); + } + } + + if (objectType.isActive()) { + auto type = object_names.find(objectType.get()); + if (type != object_names.end()) invalidation.setObjectType(type->second); + } + + if (sourceId.isActive()) invalidation.setSourceId(sourceId.get()); + + auto i_cb = Singleton::Consume::from(); + i_cb->performCallBacks(invalidation); + } + +private: + template + map + getMainAttr() + { + map res; + + for (auto &vec_entry : mainAttributes.get()) { + for (auto &attr : vec_entry.getMap()) { + res[attr.first] = attr.second; + } + } + + return res; + } + + C2S_LABEL_PARAM(string, class_name, "class"); + C2S_OPTIONAL_PARAM(string, category); + C2S_OPTIONAL_PARAM(string, family); + C2S_OPTIONAL_PARAM(string, group); + C2S_OPTIONAL_PARAM(string, order); + C2S_OPTIONAL_PARAM(string, kind); + C2S_OPTIONAL_PARAM(string, objectType); + C2S_OPTIONAL_PARAM(string, sourceId); + C2S_OPTIONAL_PARAM(vector, mainAttributes); +}; + class IntelligenceComponentV2::Impl : Singleton::Provide::From @@ -78,6 +265,37 @@ public: message = Singleton::Consume::by(); timer = Singleton::Consume::by(); mainloop = Singleton::Consume::by(); + + mainloop->addRecurringRoutine( + I_MainLoop::RoutineType::System, + chrono::minutes(12), + [this] () { sendReccurringInvalidationRegistration(); }, + "Sending intelligence invalidation" + ); + + auto rest_api = Singleton::Consume::by(); + rest_api->addRestCall(RestAction::SET, "new-invalidation/source/invalidation"); + } + + bool + sendInvalidation(const Invalidation &invalidation) const override + { + if (offline_mode_only) return false; + return hasLocalIntelligence() ? sendLocalInvalidation(invalidation) : sendGlobalInvalidation(invalidation); + } + + Maybe + registerInvalidation(const Invalidation &invalidation, const function &cb) override + { + if (!invalidation.isLegalInvalidation()) return genError("Attempting to register invalid invalidation"); + if (!sendRegistration(invalidation)) return genError("Failed to register for invalidation"); + return invalidations.emplace(invalidation, cb); + } + + void + unregisterInvalidation(uint id) override + { + invalidations.erase(id); } I_Messaging * @@ -102,11 +320,20 @@ public: getOfflineInfoString(const SerializableQueryFilter &query) const override { string ip_attr_key = "mainAttributes.ip"; - string identifier_value = move(query.getConditionValueByKey(ip_attr_key)); - if (identifier_value == "") { + auto valueVariant = query.getConditionValueByKey(ip_attr_key); + + if (!valueVariant.ok()) { return genError("could not find IP main attribute in the given query."); } - return offline_intelligence.getValueByIdentifier(identifier_value); + const SerializableQueryCondition::ValueVariant& value = valueVariant.unpack(); + if (const string* identifier_value = boost::get(&value)) { + if (*identifier_value == "") { + return genError("Could not find IP main attribute in the given query."); + } + return offline_intelligence.getValueByIdentifier(*identifier_value); + } + + return genError("Value is not of type string."); } bool @@ -116,8 +343,157 @@ public: } private: + bool + hasLocalIntelligence() const + { + return getProfileAgentSettingWithDefault(false, "agent.config.useLocalIntelligence"); + } + + bool + sendLocalInvalidation(const Invalidation &invalidation) const + { + dbgFlow(D_INTELLIGENCE) << "Starting local invalidation"; + return sendLocalInvalidationImpl(invalidation) || sendGlobalInvalidation(invalidation); + } + + bool + sendLocalInvalidationImpl(const Invalidation &invalidation) const + { + auto server = getSetting("intelligence", "local intelligence server ip"); + if (!server.ok()) { + dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured"; + return false; + } + + return + sendLocalInvalidationImpl(invalidation, *server, primary_port_setting) || + sendLocalInvalidationImpl(invalidation, *server, secondary_port_setting); + } + + bool + sendLocalInvalidationImpl(const Invalidation &invalidation, const string &server, const string &port_setting) const + { + dbgFlow(D_INTELLIGENCE) << "Sending to local intelligence"; + + auto port = getSetting("intelligence", port_setting); + if (!port.ok()) { + dbgWarning(D_INTELLIGENCE) << "Could not resolve port for " << port_setting; + return false; + } + + dbgTrace(D_INTELLIGENCE) + << "Invalidation value: " + << (invalidation.genJson().ok() ? invalidation.genJson().unpack() : invalidation.genJson().getErr()); + + return message->sendNoReplyObject( + invalidation, + I_Messaging::Method::POST, + server, + *port, + Flags(), + invalidation_uri, + getHTTPHeaders(), + nullptr, + MessageTypeTag::INTELLIGENCE + ); + } + + bool + sendGlobalInvalidation(const Invalidation &invalidation) const + { + dbgFlow(D_INTELLIGENCE) << "Starting global invalidation"; + + dbgTrace(D_INTELLIGENCE) + << "Invalidation value: " + << (invalidation.genJson().ok() ? invalidation.genJson().unpack() : invalidation.genJson().getErr()); + + return message->sendNoReplyObject( + invalidation, + I_Messaging::Method::POST, + invalidation_uri, + getHTTPHeaders(), + nullptr, + true, + MessageTypeTag::INTELLIGENCE + ); + } + + string + getHTTPHeaders() const + { + auto details = Singleton::Consume::by(); + auto tenant = details->getTenantId(); + if (tenant == "") tenant = "Global"; + auto agent = details->getAgentId(); + + return "X-Tenant-Id: " + tenant + "\r\nX-Source-Id: " + agent; + } + + bool + sendRegistration(const Invalidation &invalidation) const + { + if (offline_mode_only) return false; + + InvalidationRegistration registration; + registration.addInvalidation(invalidation); + + return sendLocalRegistrationImpl(registration.genJson()); + } + + bool + sendLocalRegistrationImpl(const InvalidationRegistration::RestCall ®istration) const + { + auto server = getSetting("intelligence", "local intelligence server ip"); + if (!server.ok()) { + dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured"; + return false; + } + return + sendLocalRegistrationImpl(registration, *server, primary_port_setting) || + sendLocalRegistrationImpl(registration, *server, secondary_port_setting); + } + + bool + sendLocalRegistrationImpl( + const InvalidationRegistration::RestCall ®istration, + const string &server, + const string &port_setting + ) const + { + dbgFlow(D_INTELLIGENCE) << "Sending to local registration"; + + auto port = getSetting("intelligence", port_setting); + if (!port.ok()) { + dbgWarning(D_INTELLIGENCE) << "Could not resolve port for " << port_setting; + return false; + } + + dbgTrace(D_INTELLIGENCE) << "Invalidation value: " << registration.genJson(); + + return message->sendNoReplyObject( + registration, + I_Messaging::Method::POST, + server, + *port, + Flags(), + registration_uri, + "", + nullptr, + MessageTypeTag::INTELLIGENCE + ); + } + + void + sendReccurringInvalidationRegistration() const + { + if (offline_mode_only || !hasLocalIntelligence() || invalidations.empty()) return; + + sendLocalRegistrationImpl(invalidations.getRegistration()); + } + OfflineIntelligeceHandler offline_intelligence; bool offline_mode_only = false; + InvalidationCallBack invalidations; I_Messaging *message = nullptr; I_TimeGet *timer = nullptr; I_MainLoop *mainloop = nullptr; @@ -141,8 +517,8 @@ IntelligenceComponentV2::preload() registerExpectedConfiguration("intelligence", "maximum request overall time"); registerExpectedConfiguration("intelligence", "maximum request lap time"); registerExpectedSetting("intelligence", "local intelligence server ip"); - registerExpectedSetting("intelligence", "local intelligence server secondary port"); - registerExpectedSetting("intelligence", "local intelligence server primary port"); + registerExpectedSetting("intelligence", primary_port_setting); + registerExpectedSetting("intelligence", secondary_port_setting); registerExpectedConfigFile("agent-intelligence", Config::ConfigFileType::Policy); } diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/CMakeLists.txt b/core/intelligence_is_v2/intelligence_is_v2_ut/CMakeLists.txt index 099c0e3..fc6fb63 100755 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/CMakeLists.txt +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/CMakeLists.txt @@ -1,7 +1,7 @@ -file(COPY offline_intelligence_files_v2 DESTINATION .) -link_directories(${BOOST_ROOT}/lib) - -add_unit_test( - intelligence_is_v2_ut - "query_request_v2_ut.cc;query_response_v2_ut.cc;intelligence_comp_v2_ut.cc" - "intelligence_is_v2;singleton;shell_cmd;event_is;metric;message;agent_details;connkey;-lboost_regex") +file(COPY offline_intelligence_files_v2 DESTINATION .) +link_directories(${BOOST_ROOT}/lib) + +add_unit_test( + intelligence_is_v2_ut + "query_request_v2_ut.cc;query_response_v2_ut.cc;intelligence_comp_v2_ut.cc;invalidation_ut.cc" + "intelligence_is_v2;singleton;shell_cmd;event_is;metric;message;agent_details;connkey;-lboost_regex") diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc index f89b348..0dc746f 100755 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc @@ -9,6 +9,7 @@ #include "mock/mock_messaging.h" #include "mock/mock_mainloop.h" #include "mock/mock_time_get.h" +#include "mock/mock_rest_api.h" using namespace std; using namespace testing; @@ -35,8 +36,18 @@ public: EXPECT_CALL( mock_ml, - addRecurringRoutine(I_MainLoop::RoutineType::System, chrono::microseconds(600000000), _, _, _)) - .WillRepeatedly(DoAll(SaveArg<2>(&routine), Return(0))); + addRecurringRoutine(I_MainLoop::RoutineType::System, chrono::microseconds(600000000), _, _, _) + ).WillRepeatedly(DoAll(SaveArg<2>(&routine), Return(0))); + + EXPECT_CALL( + mock_ml, + addRecurringRoutine(I_MainLoop::RoutineType::System, chrono::microseconds(720000000), _, _, _) + ).WillRepeatedly(Return(0)); + + EXPECT_CALL( + mock_rest, + mockRestCall(_, "new-invalidation/source/invalidation", _) + ).WillRepeatedly(Return(true)); string offline_intel_path = cptestFnameInExeDir("offline_intelligence_files_v2"); setConfiguration(offline_intel_path, string("intelligence"), string("offline intelligence path")); @@ -52,6 +63,7 @@ public: stringstream debug_output; StrictMock mock_ml; + StrictMock mock_rest; NiceMock mock_time; ::Environment env; ConfigComponent conf; @@ -853,3 +865,175 @@ TEST_F(IntelligenceComponentTestV2, bulkOnlineIntelligenceTest) EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry2"); EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing2"); } + +TEST_F(IntelligenceComponentTestV2, ignoreInProgressQueryTest_2) +{ + string paging_in_progress_response_str( + "{\n" + " \"assetCollections\": [\n" + " {\n" + " \"schemaVersion\": 1,\n" + " \"assetType\": \"workload-cloud-fake-online-test\",\n" + " \"assetTypeSchemaVersion\": 1,\n" + " \"permissionType\": \"tenant\",\n" + " \"permissionGroupId\": \"fake-online-test-group\",\n" + " \"name\": \"fake-online-test-asset1\",\n" + " \"class\": \"workload\",\n" + " \"category\": \"cloud\",\n" + " \"family\": \"fake-online-test1\",\n" + " \"mainAttributes\": {\n" + " \"deAssetId\": \"C0:3F:0E:A5:59:64_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n" + " },\n" + " \"sources\": [\n" + " {\n" + " \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n" + " \"sourceId\": \"fog-app-msrv-iot-assets\",\n" + " \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n" + " \"ttl\": 120,\n" + " \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n" + " \"confidence\": 500,\n" + " \"attributes\": {\n" + " \"phase\": \"fake online test1\",\n" + " \"user\": \"Omry\",\n" + " \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n" + " }\n" + " }\n" + " ]\n" + " },\n" + " {\n" + " \"schemaVersion\": 1,\n" + " \"assetType\": \"workload-cloud-fake-online-test\",\n" + " \"assetTypeSchemaVersion\": 1,\n" + " \"permissionType\": \"tenant\",\n" + " \"permissionGroupId\": \"fake-online-test-group\",\n" + " \"name\": \"fake-online-test-asset2\",\n" + " \"class\": \"workload\",\n" + " \"category\": \"cloud\",\n" + " \"family\": \"fake-online-test2\",\n" + " \"mainAttributes\": {\n" + " \"deAssetId\": \"20:F8:5E:2F:6D:4C_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n" + " },\n" + " \"sources\": [\n" + " {\n" + " \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n" + " \"sourceId\": \"fog-app-msrv-iot-assets\",\n" + " \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd2\",\n" + " \"ttl\": 120,\n" + " \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n" + " \"confidence\": 500,\n" + " \"attributes\": {\n" + " \"phase\": \"fake online test2\",\n" + " \"user\": \"Max\",\n" + " \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n" + " }\n" + " }\n" + " ]\n" + " }\n" + " ],\n" + " \"status\": \"inProgress\",\n" + " \"totalNumAssets\": 2,\n" + " \"cursor\": \"efgh\"\n" + "}\n" + ); + + string paging_done_response_str( + "{\n" + " \"assetCollections\": [\n" + " {\n" + " \"schemaVersion\": 1,\n" + " \"assetType\": \"workload-cloud-fake-online-test\",\n" + " \"assetTypeSchemaVersion\": 1,\n" + " \"permissionType\": \"tenant\",\n" + " \"permissionGroupId\": \"fake-online-test-group\",\n" + " \"name\": \"fake-online-test-asset1\",\n" + " \"class\": \"workload\",\n" + " \"category\": \"cloud\",\n" + " \"family\": \"fake-online-test1\",\n" + " \"mainAttributes\": {\n" + " \"deAssetId\": \"C0:3F:0E:A5:59:64_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n" + " },\n" + " \"sources\": [\n" + " {\n" + " \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n" + " \"sourceId\": \"fog-app-msrv-iot-assets\",\n" + " \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n" + " \"ttl\": 120,\n" + " \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n" + " \"confidence\": 500,\n" + " \"attributes\": {\n" + " \"phase\": \"fake online test1\",\n" + " \"user\": \"Omry\",\n" + " \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n" + " }\n" + " }\n" + " ]\n" + " },\n" + " {\n" + " \"schemaVersion\": 1,\n" + " \"assetType\": \"workload-cloud-fake-online-test\",\n" + " \"assetTypeSchemaVersion\": 1,\n" + " \"permissionType\": \"tenant\",\n" + " \"permissionGroupId\": \"fake-online-test-group\",\n" + " \"name\": \"fake-online-test-asset2\",\n" + " \"class\": \"workload\",\n" + " \"category\": \"cloud\",\n" + " \"family\": \"fake-online-test2\",\n" + " \"mainAttributes\": {\n" + " \"deAssetId\": \"20:F8:5E:2F:6D:4C_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n" + " },\n" + " \"sources\": [\n" + " {\n" + " \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n" + " \"sourceId\": \"fog-app-msrv-iot-assets\",\n" + " \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd2\",\n" + " \"ttl\": 120,\n" + " \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n" + " \"confidence\": 500,\n" + " \"attributes\": {\n" + " \"phase\": \"fake online test2\",\n" + " \"user\": \"Max\",\n" + " \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n" + " }\n" + " }\n" + " ]\n" + " }\n" + " ],\n" + " \"status\": \"done\",\n" + " \"totalNumAssets\": 2,\n" + " \"cursor\": \"efgh\"\n" + "}\n" + ); + + EXPECT_CALL( + messaging_mock, + sendMessage(true, _, I_Messaging::Method::POST, _, _, _, _, MessageTypeTag::INTELLIGENCE)) + .WillOnce(Return(paging_in_progress_response_str)) + .WillOnce(Return(paging_done_response_str)); + + I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); + QueryRequest request(Condition::EQUALS, "category", "cloud", true, AttributeKeyType::NONE); + request.activatePaging(); + request.setAssetsLimit(10); + vector> objects_reply; + vector objects_ids; + do { + auto object_result = intell->queryIntelligence(request, true); + if (!object_result.ok()) { + if (object_result.getErr() == "Query intelligence response with InProgress status") continue; + break; + } + + objects_reply = object_result.unpack(); + if (objects_reply.empty()) break; + + for (const AssetReply ¤t_object : objects_reply) { + if (current_object.getMainAttributes().empty()) { + continue; + } + const string &id = current_object.getMainAttributes().begin()->second[0]; + objects_ids.push_back(id); + } + } while (!request.isPagingFinished()); + + EXPECT_EQ(objects_ids.size(), 2); +} diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc index 8381482..a64f6da 100755 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc @@ -10,29 +10,8 @@ USE_DEBUG_FLAG(D_INTELLIGENCE); TEST(QueryRequestTestV2, QueryTest) { QueryRequest request(Condition::EQUALS, "phase", "testing", true); - SerializableQueryFilter request_query1 = request.getQuery(); - vector request_operands1 = request_query1.getConditionOperands(); - SerializableQueryCondition request_condition = *request_operands1.begin(); - - EXPECT_EQ(request_query1.getOperator(), Operator::NONE); - EXPECT_EQ(request_condition.getKey(), "mainAttributes.phase"); - EXPECT_EQ(request_condition.getValue(), "testing"); - request.addCondition(Condition::EQUALS, "user1", "Omry"); request.addCondition(Condition::EQUALS, "user2", "Max"); - SerializableQueryFilter request_query2 = request.getQuery(); - vector request_operands2 = request_query2.getConditionOperands(); - - vector::iterator it = request_operands2.begin(); - it++; - - EXPECT_EQ(request_query2.getOperator(), Operator::AND); - EXPECT_EQ(it->getKey(), "mainAttributes.user1"); - EXPECT_EQ(it->getValue(), "Omry"); - - it++; - EXPECT_EQ(it->getKey(), "mainAttributes.user2"); - EXPECT_EQ(it->getValue(), "Max"); string output_json = "{\n" @@ -67,6 +46,38 @@ TEST(QueryRequestTestV2, QueryTest) request.saveToJson(out_ar); } EXPECT_EQ(out.str(), output_json); + + QueryRequest request2(Condition::GREATER_THAN, "prev_time", 1676887025952, true); + request2.addCondition(Condition::LESS_THAN, "curr_time", 1676887025958); + + string output_json2= + "{\n" + " \"limit\": 20,\n" + " \"fullResponse\": true,\n" + " \"query\": {\n" + " \"operator\": \"and\",\n" + " \"operands\": [\n" + " {\n" + " \"operator\": \"greaterThan\",\n" + " \"key\": \"mainAttributes.prev_time\",\n" + " \"value\": 1676887025952\n" + " },\n" + " {\n" + " \"operator\": \"lessThan\",\n" + " \"key\": \"mainAttributes.curr_time\",\n" + " \"value\": 1676887025958\n" + " }\n" + " ]\n" + " }\n" + "}"; + + + stringstream out2; + { + cereal::JSONOutputArchive out_ar2(out2); + request2.saveToJson(out_ar2); + } + EXPECT_EQ(out2.str(), output_json2); } TEST(QueryRequestTestV2, AttributesTest) @@ -185,6 +196,104 @@ TEST(QueryRequestTestV2, OrQueryTest) EXPECT_EQ(out.str(), output_json); } +TEST(QueryRequestTestV2, AndQueryTestThree) +{ + QueryRequest request1(Condition::EQUALS, "phase", "testing1", true); + QueryRequest request2(Condition::EQUALS, "phase", "testing2", true); + QueryRequest request3(Condition::EQUALS, "phase", "testing3", true); + QueryRequest and_request_1_2 = request1 && (request2 && request3); + QueryRequest and_request_2_1 = (request1 && request2) && request3; + + stringstream out_1_2; + { + cereal::JSONOutputArchive out_1_2_ar(out_1_2); + and_request_1_2.saveToJson(out_1_2_ar); + } + + stringstream out_2_1; + { + cereal::JSONOutputArchive out_2_1_ar(out_2_1); + and_request_2_1.saveToJson(out_2_1_ar); + } + + string output_json = + "{\n" + " \"limit\": 20,\n" + " \"fullResponse\": true,\n" + " \"query\": {\n" + " \"operator\": \"and\",\n" + " \"operands\": [\n" + " {\n" + " \"operator\": \"equals\",\n" + " \"key\": \"mainAttributes.phase\",\n" + " \"value\": \"testing1\"\n" + " },\n" + " {\n" + " \"operator\": \"equals\",\n" + " \"key\": \"mainAttributes.phase\",\n" + " \"value\": \"testing2\"\n" + " },\n" + " {\n" + " \"operator\": \"equals\",\n" + " \"key\": \"mainAttributes.phase\",\n" + " \"value\": \"testing3\"\n" + " }\n" + " ]\n" + " }\n" + "}"; + EXPECT_EQ(out_1_2.str(), output_json); + EXPECT_EQ(out_2_1.str(), output_json); +} + +TEST(QueryRequestTestV2, OrQueryTestThree) +{ + QueryRequest request1(Condition::EQUALS, "phase", "testing1", true); + QueryRequest request2(Condition::EQUALS, "phase", "testing2", true); + QueryRequest request3(Condition::EQUALS, "phase", "testing3", true); + QueryRequest or_request_1_2 = request1 || (request2 || request3); + QueryRequest or_request_2_1 = (request1 || request2) || request3; + + stringstream out_1_2; + { + cereal::JSONOutputArchive out_1_2_ar(out_1_2); + or_request_1_2.saveToJson(out_1_2_ar); + } + + stringstream out_2_1; + { + cereal::JSONOutputArchive out_2_1_ar(out_2_1); + or_request_2_1.saveToJson(out_2_1_ar); + } + + string output_json = + "{\n" + " \"limit\": 20,\n" + " \"fullResponse\": true,\n" + " \"query\": {\n" + " \"operator\": \"or\",\n" + " \"operands\": [\n" + " {\n" + " \"operator\": \"equals\",\n" + " \"key\": \"mainAttributes.phase\",\n" + " \"value\": \"testing1\"\n" + " },\n" + " {\n" + " \"operator\": \"equals\",\n" + " \"key\": \"mainAttributes.phase\",\n" + " \"value\": \"testing2\"\n" + " },\n" + " {\n" + " \"operator\": \"equals\",\n" + " \"key\": \"mainAttributes.phase\",\n" + " \"value\": \"testing3\"\n" + " }\n" + " ]\n" + " }\n" + "}"; + EXPECT_EQ(out_1_2.str(), output_json); + EXPECT_EQ(out_2_1.str(), output_json); +} + TEST(QueryRequestTestV2, AndWithConditionQueryTest) { QueryRequest request1(Condition::EQUALS, "phase", "testing1", true); diff --git a/core/intelligence_is_v2/intelligence_types_v2.cc b/core/intelligence_is_v2/intelligence_types_v2.cc index c6238f8..0703b5e 100755 --- a/core/intelligence_is_v2/intelligence_types_v2.cc +++ b/core/intelligence_is_v2/intelligence_types_v2.cc @@ -36,6 +36,8 @@ Intelligence_IS_V2::convertConditionTypeToString(const Condition &condition_type {Condition::CONTAINS, "contains"}, {Condition::IN, "in"}, {Condition::NOT_IN, "notIn"}, + {Condition::GREATER_THAN, "greaterThan"}, + {Condition::LESS_THAN, "lessThan"}, }; auto condition_str = condition_type_to_string_map.find(condition_type); diff --git a/core/intelligence_is_v2/invalidation.cc b/core/intelligence_is_v2/invalidation.cc new file mode 100644 index 0000000..2cace7e --- /dev/null +++ b/core/intelligence_is_v2/invalidation.cc @@ -0,0 +1,240 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "intelligence_invalidation.h" + +#include + +#include "i_intelligence_is_v2.h" + +using namespace Intelligence; +using namespace std; + +Invalidation::Invalidation(const string &class_value) + : + source_id(genError()), + object_type(genError()), + listening_id(genError()) +{ + setClassifier(ClassifierType::CLASS, class_value); +} + +Invalidation & +Invalidation::setClassifier(ClassifierType type, const string &val) +{ + classifiers[type] = val; + return *this; +} + +Invalidation & +Invalidation::setStringAttr(const string &attr, const string &val) +{ + string_main_attr[attr] = val; + return *this; +} + +Invalidation & +Invalidation::setStringSetAttr(const string &attr, const set &val) +{ + set_string_main_attr[attr] = val; + return *this; +} + +Invalidation & +Invalidation::setSourceId(const string &id) +{ + source_id = id; + return *this; +} + +Invalidation & +Invalidation::setObjectType(ObjectType type) +{ + object_type = type; + return *this; +} + +Maybe +Invalidation::getStringAttr(const string &attr) const +{ + auto val_ref = string_main_attr.find(attr); + if (val_ref == string_main_attr.end()) return genError(); + return val_ref->second; +} + +Maybe, void> +Invalidation::getStringSetAttr(const string &attr) const +{ + auto val_ref = set_string_main_attr.find(attr); + if (val_ref == set_string_main_attr.end()) return genError(); + return val_ref->second; +} + +bool +Invalidation::report(I_Intelligence_IS_V2 *interface) const +{ + if (!isLegalInvalidation()) return false; + return interface->sendInvalidation(*this); +} + +Maybe +Invalidation::startListening(I_Intelligence_IS_V2 *interface, const function &cb) +{ + auto res = interface->registerInvalidation(*this, cb); + if (res.ok()) listening_id = *res; + return res; +} + +void +Invalidation::stopListening(I_Intelligence_IS_V2 *interface) +{ + if (listening_id.ok()) interface->unregisterInvalidation(*listening_id); +} + +static const map convertObjectType = { + { Intelligence::ObjectType::ASSET, "asset" }, + { Intelligence::ObjectType::ZONE, "zone" }, + { Intelligence::ObjectType::POLICY_PACKAGE, "policyPackage" }, + { Intelligence::ObjectType::CONFIGURATION, "configuration" }, + { Intelligence::ObjectType::SESSION, "session" } +}; + +Maybe +Invalidation::genJson() const +{ + if (!isLegalInvalidation()) return genError("Incomplete intelligence invalidation"); + + stringstream invalidation; + + invalidation << "{ \"invalidations\": [ " << genObject() <<" ] }"; + + return invalidation.str(); +} + +string +Invalidation::genObject() const +{ + stringstream invalidation; + + invalidation << "{ \"class\": \"" << classifiers[ClassifierType::CLASS] << '"'; + if (classifiers[ClassifierType::CATEGORY] != "") { + invalidation <<", \"category\": \"" << classifiers[ClassifierType::CATEGORY] << '"'; + } + if (classifiers[ClassifierType::FAMILY] != "") { + invalidation <<", \"family\": \"" << classifiers[ClassifierType::FAMILY] << '"'; + } + if (classifiers[ClassifierType::GROUP] != "") { + invalidation <<", \"group\": \"" << classifiers[ClassifierType::GROUP] << '"'; + } + if (classifiers[ClassifierType::ORDER] != "") { + invalidation <<", \"order\": \"" << classifiers[ClassifierType::ORDER] << '"'; + } + if (classifiers[ClassifierType::KIND] != "") { + invalidation <<", \"kind\": \"" << classifiers[ClassifierType::KIND] << '"'; + } + + if (object_type.ok()) invalidation <<", \"objectType\": \"" << convertObjectType.at(*object_type) << '"'; + if (source_id.ok()) invalidation <<", \"sourceId\": \"" << *source_id << '"'; + + if (!string_main_attr.empty() || !set_string_main_attr.empty()) { + invalidation << ", \"mainAttributes\": [ "; + bool first = true; + for (auto &attr : string_main_attr) { + if (!first) invalidation << ", "; + invalidation << "{ \"" << attr.first << "\": \"" << attr.second << "\" }"; + first = false; + } + + for (auto &attr : set_string_main_attr) { + if (!first) invalidation << ", "; + auto val = makeSeparatedStr(attr.second, ", "); + invalidation << "{ \"" << attr.first << "\": [ "; + bool internal_first = true; + for (auto &val : attr.second) { + if (!internal_first) invalidation << ", "; + invalidation << "\"" << val << "\""; + internal_first = false; + } + invalidation << " ] }"; + first = false; + } + + invalidation << " ]"; + } + + invalidation << " }"; + + return invalidation.str(); +} + +bool +Invalidation::isLegalInvalidation() const +{ + if (!set_string_main_attr.empty() || !string_main_attr.empty()) { + if (classifiers[ClassifierType::FAMILY] == "") return false; + } + + bool is_prev_empty = false; + for (auto &classifer : classifiers) { + if (is_prev_empty && classifer != "") return false; + is_prev_empty = classifer == ""; + } + + return true; +} + +template <> +class EnumCount : public EnumCountSpecialization {}; + +bool +Invalidation::matches(const Invalidation &other) const +{ + for (auto key : NGEN::Range()) { + if (classifiers[key] != "" && classifiers[key] != other.classifiers[key]) return false; + } + + if (object_type.ok()) { + if (!other.object_type.ok() || *object_type != *other.object_type) return false; + } + + if (source_id.ok()) { + if (!other.source_id.ok() || *source_id != *other.source_id) return false; + } + + for (auto &key_value : string_main_attr) { + if (!other.hasAttr(key_value.first, key_value.second)) return false; + } + + + for (auto &key_values : set_string_main_attr) { + for (auto &value : key_values.second) { + if (!other.hasAttr(key_values.first, value)) return false; + } + } + + return true; +} + +bool +Invalidation::hasAttr(const string &key, const string &value) const +{ + auto string_elem = string_main_attr.find(key); + if (string_elem != string_main_attr.end()) return string_elem->second == value; + + auto set_string_elem = set_string_main_attr.find(key); + if (set_string_elem != set_string_main_attr.end()) { + return set_string_elem->second.find(value) != set_string_elem->second.end(); + } + + return false; +} diff --git a/core/intelligence_is_v2/json_stream.cc b/core/intelligence_is_v2/json_stream.cc new file mode 100644 index 0000000..1440160 --- /dev/null +++ b/core/intelligence_is_v2/json_stream.cc @@ -0,0 +1,50 @@ +// Copyright (C) 2023 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "intelligence_is_v2/json_stream.h" + +JsonStream::JsonStream(std::ostream *os, bool is_pretty) : std::ostream(this), os(os), is_pretty(is_pretty) {} + +int +JsonStream::overflow(int c) +{ + if (c != std::streambuf::traits_type::eof()) { + add(std::streambuf::traits_type::to_char_type(c)); + } + return c; +} + +void +JsonStream::emplace(char c) +{ + *os << c; +} + +void +JsonStream::add(char c) +{ + if (is_pretty) { + emplace(c); + return; + } + + if (is_prev_single_backslash) { + emplace(c); + is_prev_single_backslash = false; + return; + } + + if (c == '"') in_string = !in_string; + if (c == '\\') is_prev_single_backslash = true; + if (in_string || !std::isspace(c)) emplace(c); +} diff --git a/core/intelligence_is_v2/query_filter_v2.cc b/core/intelligence_is_v2/query_filter_v2.cc index 0b48f20..8d2bec3 100755 --- a/core/intelligence_is_v2/query_filter_v2.cc +++ b/core/intelligence_is_v2/query_filter_v2.cc @@ -23,19 +23,46 @@ using namespace Intelligence_IS_V2; USE_DEBUG_FLAG(D_INTELLIGENCE); +struct Visitor : public boost::static_visitor +{ + Visitor(cereal::JSONOutputArchive& _ar) : ar(_ar) {} + + template + void operator()(const T& value) + { + ar(cereal::make_nvp("value", value)); + } + +private: + cereal::JSONOutputArchive &ar; +}; + void SerializableQueryCondition::save(cereal::JSONOutputArchive &ar) const { ar( cereal::make_nvp("operator", string(convertConditionTypeToString(condition_type))), - cereal::make_nvp("key", key), - cereal::make_nvp("value", value) + cereal::make_nvp("key", key) ); + + Visitor visitor(ar); + boost::apply_visitor(visitor, value); } -SerializableQueryFilter::SerializableQueryFilter(Condition condition_type, const string &key, const string &value) -{ - condition_operands.push_back(SerializableQueryCondition(condition_type, key, value)); +SerializableQueryFilter::SerializableQueryFilter( + Condition condition_type, + const std::string &key, + const std::string &value +) { + condition_operands.emplace_back(condition_type, key, value); +} + +SerializableQueryFilter::SerializableQueryFilter( + Condition condition_type, + const std::string &key, + const int64_t &value +) { + condition_operands.emplace_back(condition_type, key, value); } void @@ -52,31 +79,28 @@ void SerializableQueryFilter::addCondition(Condition condition_type, const string &key, const string &value) { if (queries_operands.size() > 0) { - SerializableQueryFilter new_query_filter(condition_type, key, value); - queries_operands.push_back(new_query_filter); + queries_operands.emplace_back(condition_type, key, value); return; } - if (condition_operands.size() == 1 && operator_type == Operator::NONE) { - operator_type = Operator::AND; + if (condition_operands.size() == 1 && operator_type == Operator::NONE) operator_type = Operator::AND; + condition_operands.emplace_back(condition_type, key, value); +} + +void +SerializableQueryFilter::addCondition(Condition condition_type, const string &key, const int64_t &value) +{ + if (queries_operands.size() > 0) { + queries_operands.emplace_back(condition_type, key, value); + return; } - SerializableQueryCondition cond(condition_type, key, value); - condition_operands.push_back(cond); + if (condition_operands.size() == 1 && operator_type == Operator::NONE) operator_type = Operator::AND; + condition_operands.emplace_back(condition_type, key, value); } void SerializableQueryFilter::saveCondition(cereal::JSONOutputArchive &ar) const { - SerializableQueryCondition cond = *condition_operands.begin(); - Condition condition_type = cond.getConditionType(); - string condition_str = convertConditionTypeToString(condition_type); - string filter_key = cond.getKey(); - string filter_value = cond.getValue(); - - ar( - cereal::make_nvp("operator", condition_str), - cereal::make_nvp("key", filter_key), - cereal::make_nvp("value", filter_value) - ); + condition_operands.begin()->save(ar); } void @@ -101,7 +125,7 @@ SerializableQueryFilter::saveOperation(cereal::JSONOutputArchive &ar) const } } -const string & +Maybe SerializableQueryFilter::getConditionValueByKey(const string &key) const { for (const SerializableQueryCondition &condition : condition_operands) { @@ -110,21 +134,49 @@ SerializableQueryFilter::getConditionValueByKey(const string &key) const } } - static string empty_str = ""; - return empty_str; + return genError("Key not found."); +} + +bool +SerializableQueryFilter::isOperatorComp(const Operator &oper) const +{ + return operator_type == Operator::NONE || operator_type == oper; } SerializableQueryFilter -SerializableQueryFilter::calcOperator(const SerializableQueryFilter &other_query, const Operator &operator_type) +SerializableQueryFilter::calcOperator(const SerializableQueryFilter &other_query, const Operator &oper) { SerializableQueryFilter query_filter_res; - vector new_queries_operands; - new_queries_operands.push_back(*this); - new_queries_operands.push_back(other_query); + query_filter_res.operator_type = oper; + + if (isOperatorComp(oper) && other_query.isOperatorComp(oper)) { + size_t queries_size = queries_operands.size() + other_query.queries_operands.size(); + size_t conditions_size = condition_operands.size() + other_query.condition_operands.size(); + query_filter_res.queries_operands.reserve(queries_size); + query_filter_res.condition_operands.reserve(conditions_size); + + for (const auto &subquery : queries_operands) { + query_filter_res.queries_operands.push_back(subquery); + } + + for (const auto &condition : condition_operands) { + query_filter_res.condition_operands.push_back(condition); + } + + for (const auto &subquery : other_query.queries_operands) { + query_filter_res.queries_operands.push_back(subquery); + } + + for (const auto &condition : other_query.condition_operands) { + query_filter_res.condition_operands.push_back(condition); + } + } else { + query_filter_res.queries_operands.reserve(2); + query_filter_res.queries_operands.push_back(*this); + query_filter_res.queries_operands.push_back(other_query); + } - query_filter_res.queries_operands = new_queries_operands; - query_filter_res.operator_type = operator_type; return query_filter_res; } diff --git a/core/intelligence_is_v2/query_request_v2.cc b/core/intelligence_is_v2/query_request_v2.cc index 2113311..38d3184 100755 --- a/core/intelligence_is_v2/query_request_v2.cc +++ b/core/intelligence_is_v2/query_request_v2.cc @@ -58,6 +58,18 @@ QueryRequest::QueryRequest( full_response = full_reponse; } +QueryRequest::QueryRequest( + Condition condition_type, + const string &key, + const int64_t &value, + bool full_reponse, + AttributeKeyType attribute_type +) { + query = SerializableQueryFilter(condition_type, createAttributeString(key, attribute_type), value); + assets_limit = default_assets_limit; + full_response = full_reponse; +} + Maybe QueryRequest::convertObjectTypeToString() const { @@ -139,6 +151,16 @@ QueryRequest::addCondition ( query.addCondition(condition_type, createAttributeString(key, attribute_type), value); } +void +QueryRequest::addCondition ( + Condition condition_type, + const string &key, + const int64_t &value, + AttributeKeyType attribute_type +) { + query.addCondition(condition_type, createAttributeString(key, attribute_type), value); +} + void QueryRequest::setRequestedAttr(const string &attr, AttributeKeyType attr_type) { diff --git a/core/logging/cef_stream.cc b/core/logging/cef_stream.cc index e115308..8dc2ddd 100755 --- a/core/logging/cef_stream.cc +++ b/core/logging/cef_stream.cc @@ -58,6 +58,9 @@ CefStream::sendLog(const Report &log) } dbgTrace(D_REPORT) << "Connected to socket."; string cef_report = log.getCef(); + if (protocol == I_Socket::SocketType::TCP) { + cef_report = to_string(cef_report.length()) + " " + cef_report; + } vector data(cef_report.begin(), cef_report.end()); for (size_t tries = 0; tries < 3; tries++) { if (i_socket->writeData(socket.unpack(), data)) { diff --git a/core/logging/logging_ut/logging_ut.cc b/core/logging/logging_ut/logging_ut.cc index 5b7cb11..eedf040 100755 --- a/core/logging/logging_ut/logging_ut.cc +++ b/core/logging/logging_ut/logging_ut.cc @@ -1,4 +1,5 @@ #include "log_generator.h" +#include "log_utils.h" #include #include @@ -466,6 +467,52 @@ TEST_F(LogTest, LogGen) ); EXPECT_THAT(getMessages(), HasSubstr(str3)); EXPECT_THAT(readLogFile(), HasSubstr(str3)); + + + enum class TestErrors { CPU, MEMORY, DISK }; + string str4( + "{\n" + " \"eventTime\": \"0:0:0\",\n" + " \"eventName\": \"Install policy\",\n" + " \"eventSeverity\": \"Info\",\n" + " \"eventPriority\": \"Low\",\n" + " \"eventType\": \"Event Driven\",\n" + " \"eventLevel\": \"Log\",\n" + " \"eventLogLevel\": \"info\",\n" + " \"eventAudience\": \"Internal\",\n" + " \"eventAudienceTeam\": \"\",\n" + " \"eventFrequency\": 0,\n" + " \"eventTags\": [\n" + " \"Policy Installation\"\n" + " ],\n" + " \"eventSource\": {\n" + " \"agentId\": \"Unknown\",\n" + " \"eventTraceId\": \"\",\n" + " \"eventSpanId\": \"\",\n" + " \"issuingEngineVersion\": \"" + Version::getFullVersion() + "\",\n" + " \"serviceName\": \"007\"\n" + " },\n" + " \"eventData\": {\n" + " \"logIndex\": 4,\n" + " \"eventCode\": \"015-0002\"\n" + " }\n" + "}" + ); + EXPECT_EQ( + toJson( + LogGen( + "Install policy", + Audience::INTERNAL, + Severity::INFO, + Priority::LOW, + tag1, + Enreachments::BEAUTIFY_OUTPUT + ) << ErrorCode::logError(TestErrors::DISK) + ), + str4 + ); + EXPECT_THAT(getMessages(), HasSubstr(str4)); + EXPECT_THAT(readLogFile(), HasSubstr(str4)); } TEST_F(LogTest, LogSpecificStream) diff --git a/core/logging/syslog_stream.cc b/core/logging/syslog_stream.cc index 4bcd8e9..2e9657f 100755 --- a/core/logging/syslog_stream.cc +++ b/core/logging/syslog_stream.cc @@ -51,6 +51,9 @@ void SyslogStream::sendLog(const Report &log) { string syslog_report = log.getSyslog(); + if (protocol == I_Socket::SocketType::TCP) { + syslog_report = to_string(syslog_report.length()) + " " + syslog_report; + } vector data(syslog_report.begin(), syslog_report.end()); mainloop->addOneTimeRoutine( I_MainLoop::RoutineType::Offline, diff --git a/core/message/message.cc b/core/message/message.cc index 278bb67..4fa2a74 100755 --- a/core/message/message.cc +++ b/core/message/message.cc @@ -160,6 +160,7 @@ private: bool setSocket(); bool connect(const string &host, const string &overwrite_port); bool shouldIgnoreSslValidation()const; + bool isBioSocketReady() const; Maybe calculatePublicKey(const BioUniquePtr &cert) const; Maybe getPinnedCertificate(); bool verifyCertPinning(const BioUniquePtr &cert); @@ -1431,6 +1432,11 @@ MessageConnection::doHandshake(const BioUniquePtr &bio) ); auto end_time = timer->getMonotonicTime() + timeout; while (timer->getMonotonicTime() < end_time) { + if (!isBioSocketReady()) { + dbgDebug(D_COMMUNICATION) << "Socket is not ready for use."; + if (mainloop != nullptr) mainloop->yield(true); + continue; + } if (BIO_do_handshake(bio.get()) > 0 || shouldIgnoreSslValidation()) { return Maybe(); } @@ -1533,6 +1539,17 @@ MessageConnection::setSocket() return true; } +bool +MessageConnection::isBioSocketReady() const +{ +#if OPENSSL_VERSION_NUMBER >= 0x30000000 && !defined(OPENSSL_NO_SOCK) + return BIO_socket_wait(BIO_get_fd(bio.get(), NULL), 0, time(NULL)); +#else // OPENSSL_VERSION_NUMBER >= 0x30000000 && !defined(OPENSSL_NO_SOCK) + return true; +#endif // OPENSSL_VERSION_NUMBER >= 0x30000000 && !defined(OPENSSL_NO_SOCK) +} + + bool MessageConnection::connect(const string &host, const string &overwrite_port) { @@ -1547,6 +1564,11 @@ MessageConnection::connect(const string &host, const string &overwrite_port) while (timer->getMonotonicTime() < end_time) { counter++; + if (!isBioSocketReady()) { + dbgDebug(D_COMMUNICATION) << "Socket is not ready for use."; + if (mainloop != nullptr) mainloop->yield(true); + continue; + } if (BIO_do_connect(bio.get()) > 0) { dbgDebug(D_COMMUNICATION) << "Successfully established new BIO connection. " @@ -1595,6 +1617,11 @@ MessageConnection::receiveResponse(I_MessageDecoder &decoder) uint counter = 0; char buf[1000]; while (timer->getMonotonicTime() < end_time) { + if (!isBioSocketReady()) { + dbgDebug(D_COMMUNICATION) << "Socket is not ready for use."; + if (mainloop != nullptr) mainloop->yield(true); + continue; + } int len_or_error_ret_val = BIO_read(bio.get(), buf, sizeof(buf) - 1); if (len_or_error_ret_val <= 0) { if (!BIO_should_retry(bio.get())) { @@ -1838,6 +1865,11 @@ MessageConnection::sendData(const string &data) const int remaining_data_len = data.length(); while (timer->getMonotonicTime() < end_time) { int offset = data.length() - remaining_data_len; + if (!isBioSocketReady()) { + dbgDebug(D_COMMUNICATION) << "Socket is not ready for use."; + if (mainloop != nullptr) mainloop->yield(true); + continue; + } int data_sent_len = BIO_write(bio.get(), data.c_str() + offset, remaining_data_len); if (data_sent_len > 0) { if (remaining_data_len - data_sent_len < 0) { diff --git a/core/report/tag_and_enum_management.cc b/core/report/tag_and_enum_management.cc index a480f82..81b3b9e 100755 --- a/core/report/tag_and_enum_management.cc +++ b/core/report/tag_and_enum_management.cc @@ -105,7 +105,9 @@ TagAndEnumManagement::convertStringToTag(const string &tag) {"Embedded Deployment", ReportIS::Tags::DEPLOYMENT_EMBEDDED}, {"Kubernetes Deployment", ReportIS::Tags::DEPLOYMENT_K8S}, {"Layer 7 Access Control", ReportIS::Tags::LAYER_7_ACCESS_CONTROL}, - {"Horizon Telemetry Metrics", ReportIS::Tags::HORIZON_TELEMETRY_METRICS} + {"Horizon Telemetry Metrics", ReportIS::Tags::HORIZON_TELEMETRY_METRICS}, + {"Crowdsec", ReportIS::Tags::CROWDSEC}, + {"Playground", ReportIS::Tags::PLAYGROUND} }; auto report_is_tag = strings_to_tags.find(tag); @@ -305,7 +307,9 @@ EnumArray TagAndEnumManagement::tags_translation_arr { "Embedded Deployment", "Kubernetes Deployment", "Layer 7 Access Control", - "Horizon Telemetry Metrics" + "Horizon Telemetry Metrics", + "Crowdsec", + "Playground" }; EnumArray TagAndEnumManagement::audience_team_translation { diff --git a/core/rest/rest_conn.cc b/core/rest/rest_conn.cc index 3c82268..9a7a721 100644 --- a/core/rest/rest_conn.cc +++ b/core/rest/rest_conn.cc @@ -70,7 +70,7 @@ RestConn::parseConn() const string uri; os >> uri; - string identifier = uri.substr(uri.find_last_of('/') + 1); + string identifier = uri.substr(uri.find_first_of('/') + 1); uint len = 0; while (true) { diff --git a/core/rest/rest_server.cc b/core/rest/rest_server.cc index 6db2bcb..db9cb87 100644 --- a/core/rest/rest_server.cc +++ b/core/rest/rest_server.cc @@ -47,6 +47,7 @@ public: bool bindRestServerSocket(struct sockaddr_in &addr, vector port_range); bool addRestCall(RestAction oper, const string &uri, unique_ptr &&init) override; + uint16_t getListeningPort() const override { return listening_port; } Maybe getSchema(const std::string &uri) const override; Maybe invokeRest(const std::string &uri, istream &in) const override; @@ -60,7 +61,7 @@ private: I_MainLoop::RoutineID id; I_MainLoop *mainloop; map> rest_calls; - uint16_t port_used = 0; + uint16_t listening_port = 0; vector port_range; }; @@ -138,7 +139,7 @@ RestServer::Impl::init() is_primary.ok() && *is_primary ); - uint16_t listening_port = ntohs(addr.sin_port); + listening_port = ntohs(addr.sin_port); dbgInfo(D_API) << "REST server started: " << listening_port; Singleton::Consume::by()->registerValue("Listening Port", listening_port); }; diff --git a/core/rest/rest_ut/rest_config_ut.cc b/core/rest/rest_ut/rest_config_ut.cc index d813b54..9a30bb3 100755 --- a/core/rest/rest_ut/rest_config_ut.cc +++ b/core/rest/rest_ut/rest_config_ut.cc @@ -99,6 +99,8 @@ TEST_F(RestConfigTest, alternative_port_used) ); mainloop->run(); + EXPECT_EQ(Singleton::Consume::from(rest_server)->getListeningPort(), *alternative_port); + sa.sin_port = htons(alternative_port.unpack()); EXPECT_EQ(bind(file_descriptor, reinterpret_cast(&sa), sizeof(struct sockaddr_in)), -1); diff --git a/core/rest/rest_ut/rest_schema_ut.cc b/core/rest/rest_ut/rest_schema_ut.cc index 6995416..08b1a30 100644 --- a/core/rest/rest_ut/rest_schema_ut.cc +++ b/core/rest/rest_ut/rest_schema_ut.cc @@ -19,6 +19,7 @@ #include #include #include "customized_cereal_map.h" +#include "customized_cereal_multimap.h" using namespace std; @@ -175,6 +176,14 @@ class MustMapInt : public ServerRest C2S_PARAM(mapStringInt, must_map_int); }; +class MustMultiMap : public ServerRest +{ + void doCall() override {} + + using mapStringInt = SerializableMultiMap; + C2S_PARAM(mapStringInt, must_multimap); +}; + TEST(RestSchema, must_map) { stringstream string_map_schema; @@ -214,6 +223,32 @@ TEST(RestSchema, must_map) " ]\n" "}" ); + + stringstream multi_map_schema; + MustMultiMap().performOutputingSchema(multi_map_schema); + EXPECT_EQ( + multi_map_schema.str(), + "{\n" + " \"properties\": {\n" + " \"must_multimap\": {\n" + " \"type\": \"object\",\n" + " \"additionalProperties\": {\n" + " \"anyOf\": [\n" + " {\n" + " \"type\": \"string\"\n" + " },\n" + " {\n" + " \"type\": \"integer\"\n" + " }\n" + " ]\n" + " }\n" + " }\n" + " },\n" + " \"required\": [\n" + " \"must_multimap\"\n" + " ]\n" + "}" + ); } class MustObject : public ServerRest diff --git a/core/tenant_manager/tenant_manager.cc b/core/tenant_manager/tenant_manager.cc index fd01494..14b2acd 100644 --- a/core/tenant_manager/tenant_manager.cc +++ b/core/tenant_manager/tenant_manager.cc @@ -24,6 +24,7 @@ #include "hash_combine.h" using namespace std; +using ProfilesPerTenantMap = map>; USE_DEBUG_FLAG(D_TENANT_MANAGER); @@ -75,10 +76,10 @@ public: void init(); void fini(); - void uponNewTenants(const newTenantCB &cb) override; bool areTenantAndProfileActive(const string &tenant_id, const string &profile_id) const override; - map> fetchActiveTenantsAndProfiles() const override; + ProfilesPerTenantMap fetchActiveTenantsAndProfiles() const override; + ProfilesPerTenantMap fetchAndUpdateActiveTenantsAndProfiles(bool update) override; set fetchAllActiveTenants() const override; set fetchActiveTenants() const override; set getInstances(const string &tenant_id, const string &profile_id) const override; @@ -88,8 +89,6 @@ public: void deactivateTenant(const string &tenant_id, const string &profile_id) override; - chrono::microseconds getTimeoutVal() const override; - set getProfileIdsForRegionAccount( const string &tenant_id, const string ®ion, @@ -97,33 +96,24 @@ public: ) const override; void - addInstance(const string &tenant_id, const string &profile_id, const string &instace_id) + addInstance(const string &tenant_id, const string &profile_id, const string &instace_id) override { auto tenant_profile_pair = TenantProfilePair(tenant_id, profile_id); auto tenant_cache = mapper.find(tenant_profile_pair); if (tenant_cache == mapper.end()) { tenant_cache = mapper.insert(make_pair(tenant_profile_pair, TemporaryCache())).first; - tenant_cache->second.startExpiration( - getTimeoutVal(), - Singleton::Consume::by(), - Singleton::Consume::by() - ); } tenant_cache->second.createEntry(instace_id); } private: - void runUponNewTenants(const vector &new_tenants); - void sendTenantAndProfile(const string &tenant_id, const string &profile_id); set getAllTenants() const; set fetchAllProfileIds(const string &tenant_id) const; set getProfileIds(const string &tenant_id) const; - bool sendWithCustomPort(const string &tenant_id, const string &profile_id, const uint16_t port); TemporaryCache active_tenants; map> mapper; - vector upon_cb; I_Messaging *i_messaging = nullptr; TenantManagerType type; @@ -218,111 +208,20 @@ TenantManager::Impl::init() conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN); i_messaging = Singleton::Consume::by(); - auto cache_timeout = getTimeoutVal(); - - active_tenants.startExpiration( - cache_timeout, - Singleton::Consume::by(), - Singleton::Consume::by() - ); - if (type == TenantManagerType::SERVER) { auto rest = Singleton::Consume::by(); rest->addRestCall(RestAction::SET, "tenant-id"); rest->addRestCall(RestAction::SHOW, "active-tenants"); rest->addRestCall(RestAction::SHOW, "profile-ids"); } - - if (type == TenantManagerType::CLIENT) { - auto interval = chrono::seconds( - getProfileAgentSettingWithDefault(600, "agentConfig.tenantReportIntervalSeconds") - ); - interval = chrono::seconds( - getConfigurationWithDefault(interval.count(), "Tenant Manager", "Report interval") - ); - - Singleton::Consume::by()->addRecurringRoutine( - I_MainLoop::RoutineType::System, - interval, - [this] () - { - auto tenants_ids = fetchActiveTenants(); - for (auto tenant_id : tenants_ids) { - auto profile_ids = fetchAllProfileIds(tenant_id); - for (auto profile_id : profile_ids) { - sendTenantAndProfile(tenant_id, profile_id); - } - } - }, - "Tenant manager client reporter" - ); - } } void TenantManager::Impl::fini() { - active_tenants.endExpiration(); i_messaging = nullptr; } -bool -TenantManager::Impl::sendWithCustomPort(const string &tenant_id, const string &profile_id, const uint16_t port) -{ - if (i_messaging == nullptr) { - i_messaging = Singleton::Consume::by(); - } - - SendNewTenants new_tenant_and_profile(tenant_id, profile_id); - - return i_messaging->sendNoReplyObject( - new_tenant_and_profile, - I_Messaging::Method::POST, - "127.0.0.1", - port, - conn_flags, - "/set-tenant-id" - ); -} - -void -TenantManager::Impl::runUponNewTenants(const vector &new_tenants) -{ - for (auto &cb: upon_cb) { - Singleton::Consume::by()->addOneTimeRoutine( - I_MainLoop::RoutineType::System, - [this, cb, new_tenants] () { cb(new_tenants); }, - "New tenant event handler" - ); - } -} - -void -TenantManager::Impl::sendTenantAndProfile(const string &tenant_id, const string &profile_id) -{ - auto res = sendWithCustomPort( - tenant_id, - profile_id, - getConfigurationWithDefault( - 7777, - "Tenant Manager", - "Orchestrator's primary port" - ) - ); - - if (!res) { - sendWithCustomPort( - tenant_id, - profile_id, - getConfigurationWithDefault( - 7778, - "Tenant Manager", - "Orchestrator's secondary port" - ) - ); - } -} - set TenantManager::Impl::getAllTenants() const { @@ -459,12 +358,6 @@ TenantManager::Impl::getProfileIdsForRegionAccount( return set(); } -void -TenantManager::Impl::uponNewTenants(const newTenantCB &cb) -{ - upon_cb.push_back(cb); -} - bool TenantManager::Impl::areTenantAndProfileActive(const string &tenant_id, const string &profile_id) const { @@ -485,24 +378,40 @@ TenantManager::Impl::addActiveTenantAndProfile(const string &tenant_id, const st << ", Profile ID: " << profile_id; active_tenants.createEntry(tenant_profile); - if (type == TenantManagerType::CLIENT) { - sendTenantAndProfile(tenant_id, profile_id); - } else { - runUponNewTenants({tenant_id}); - } } void TenantManager::Impl::deactivateTenant(const string &tenant_id, const string &profile_id) { + dbgTrace(D_TENANT_MANAGER) + << "Deactivate tenant and profile. Tenant ID: " + << tenant_id + << ", Profile ID: " + << profile_id; active_tenants.deleteEntry(TenantProfilePair(tenant_id, profile_id)); } -map> +ProfilesPerTenantMap +TenantManager::Impl::fetchAndUpdateActiveTenantsAndProfiles(bool update) +{ + if (!update) return fetchActiveTenantsAndProfiles(); + + active_tenants.clear(); + ProfilesPerTenantMap update_active_tenants = fetchActiveTenantsAndProfiles(); + for (const auto &tenant_profile_set : update_active_tenants) { + auto tenant_id = tenant_profile_set.first; + for (const auto &profile_id : tenant_profile_set.second) { + active_tenants.createEntry(TenantProfilePair(tenant_id, profile_id)); + } + } + return update_active_tenants; +} + +ProfilesPerTenantMap TenantManager::Impl::fetchActiveTenantsAndProfiles() const { dbgFlow(D_TENANT_MANAGER) << "Fetching active teants and profiles map"; - map> active_tenants_and_profiles; + ProfilesPerTenantMap active_tenants_and_profiles; set tenants = fetchAllActiveTenants(); for (const string &tenant : tenants) { active_tenants_and_profiles[tenant] = fetchProfileIds(tenant); @@ -567,19 +476,6 @@ TenantManager::Impl::fetchProfileIds(const string &tenant_id) const return (type == TenantManagerType::CLIENT) ? getProfileIds(tenant_id) : fetchAllProfileIds(tenant_id); } -chrono::microseconds -TenantManager::Impl::getTimeoutVal() const -{ - auto cache_timeout = chrono::seconds( - getProfileAgentSettingWithDefault(900, "Orchestration.TenantTimeoutSeconds") - ); - cache_timeout = chrono::seconds( - getConfigurationWithDefault(cache_timeout.count(), "Tenant Manager", "Tenant timeout") - ); - - return chrono::duration_cast(cache_timeout); -} - TenantManager::TenantManager() : Component("TenantManager"), @@ -604,7 +500,6 @@ TenantManager::fini() void TenantManager::preload() { - registerExpectedConfiguration("Tenant Manager", "Tenant timeout"); registerExpectedConfiguration("Tenant Manager", "Tenant manager type"); registerExpectedSetting("accountRegionSet"); registerExpectedSetting("region"); diff --git a/nodes/orchestration/CMakeLists.txt b/nodes/orchestration/CMakeLists.txt index 4bc010a..086beae 100755 --- a/nodes/orchestration/CMakeLists.txt +++ b/nodes/orchestration/CMakeLists.txt @@ -27,7 +27,6 @@ target_link_libraries( update_communication orchestration_tools env_details - messaging_downloader_client local_policy_mgmt_gen curl diff --git a/nodes/orchestration/main.cc b/nodes/orchestration/main.cc index 002d0de..f524c81 100755 --- a/nodes/orchestration/main.cc +++ b/nodes/orchestration/main.cc @@ -17,7 +17,6 @@ #include #include -#include "messaging_downloader_client.h" #include "time_proxy.h" #include "shell_cmd.h" #include "debug.h" @@ -58,8 +57,6 @@ using namespace std; #include "components_list.h" -#include "messaging_downloader_server.h" -#include "external_sdk_server.h" int main(int argc, char **argv) @@ -77,7 +74,6 @@ main(int argc, char **argv) OrchestrationComp, HealthChecker, HealthCheckManager, - MessagingDownloaderClient, LocalPolicyMgmtGenerator > comps; diff --git a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc index b118499..dcc0374 100755 --- a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc +++ b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc @@ -91,7 +91,6 @@ enum class Service { CPVIEW_METRIC_PROVIDER, HTTP_TRANSACTION_HANDLER, DEDICATED_NETWORK_HANDLER, - MESSAGING_PROXY, HELLO_WORLD, IDA, IOT_ACCESS_CONTROL, @@ -165,7 +164,6 @@ getServiceString(const Service service) case (Service::ATTACHMENT_REGISTRATOR): return "attachment-registrator"; case (Service::CPVIEW_METRIC_PROVIDER): return "cpview-metric-provider"; case (Service::DEDICATED_NETWORK_HANDLER): return "dedicated-network-handler"; - case (Service::MESSAGING_PROXY): return "messaging-proxy"; case (Service::SDWAN): return "sdwan"; case (Service::LOGGER_SDWAN): return "logger-sdwan"; case (Service::IOT_WLP): return "workload-protection"; @@ -321,11 +319,6 @@ getServiceConfig (const Service service) filesystem_path + "/conf/cp-nano-dedicated-network-handler-debug-conf.json", log_files_path + "/nano_agent/cp-nano-dedicated-network-handler.dbg" ); - case (Service::MESSAGING_PROXY): - return ServiceConfig( - filesystem_path + "/conf/cp-nano-messaging-proxy-debug-conf.json", - log_files_path + "/nano_agent/cp-nano-messaging-proxy.dbg" - ); case (Service::SDWAN): return ServiceConfig( filesystem_path + "/conf/cp-nano-sdwan-debug-conf.json", @@ -1282,8 +1275,6 @@ extractServices(const vector &args) services.push_back(Service::HTTP_TRANSACTION_HANDLER); } else if (getServiceString(Service::DEDICATED_NETWORK_HANDLER).find(maybe_service) == 0) { services.push_back(Service::DEDICATED_NETWORK_HANDLER); - } else if (getServiceString(Service::MESSAGING_PROXY).find(maybe_service) == 0) { - services.push_back(Service::MESSAGING_PROXY); } else if (getServiceString(Service::SDWAN).find(maybe_service) == 0) { services.push_back(Service::SDWAN); } else if (getServiceString(Service::LOGGER_SDWAN).find(maybe_service) == 0) { diff --git a/nodes/orchestration/package/open-appsec-ctl.sh b/nodes/orchestration/package/open-appsec-ctl.sh index 0e20fa3..ef6e837 100644 --- a/nodes/orchestration/package/open-appsec-ctl.sh +++ b/nodes/orchestration/package/open-appsec-ctl.sh @@ -26,6 +26,8 @@ DEFAULT_HEALTH_CHECK_TMP_FILE_PATH="/tmp/cpnano_health_check_output.txt" var_default_fog_address="i2-agents.cloud.ngen.checkpoint.com/" var_default_gem_fog_address="inext-agents.cloud.ngen.checkpoint.com" var_default_us_fog_address="inext-agents-us.cloud.ngen.checkpoint.com" +var_default_au_fog_address="inext-agents-aus1.cloud.ngen.checkpoint.com" +var_default_in_fog_address="inext-agents-ind1.cloud.ngen.checkpoint.com" #NOTE: open-appsec-ctl only supports nano services with name of the format cp-nano- cp_nano_service_name_prefix="cp-nano" @@ -92,7 +94,8 @@ fi get_basename() { - if command -v basename &>/dev/null; then + is_basename_exist=$(command -v basename) + if [ -n $is_basename_exist ]; then echo $(basename $1) else echo $(echo $1 | rev | cut -d / -f 1 | rev) @@ -267,7 +270,7 @@ usage() uninstall_option="-u, --uninstall" load_config_option="-lc, --load-config <$(get_installed_services '|')>" display_config_option="-dc, --display-config [$(get_installed_services '|')]" - cp_agent_info_option="-ai, --cp-agent-info [-wd|--with_dump|-u|--upload|-fms|--file_max_size|-an|--additional_name]" + cp_agent_info_option="-ai, --cp-agent-info [-wd|--with_dump|-fms|--file_max_size|-an|--additional_name]" display_policy_option="-dp, --display-policy" set_gradual_policy_option="-gp, --set-gradual-policy [access-control|http-manager] " delete_gradual_policy_option="-dg, --delete-gradual-policy [access-control|http-manager]" @@ -1217,16 +1220,11 @@ run_ai() # Initials - ra ra_tenant_id= ra_agent_id= ra_token= - ra_upload_to_fog=false # we use this address as default and replace later if needed ra_fog_address="inext-agents.cloud.ngen.checkpoint.com" for arg; do - if [ "$arg" = "--upload" ] || [ "$arg" = "-u" ]; then - ra_upload_to_fog=true - shift - continue - elif [ "$arg" = "--verbose" ] || [ "$arg" = "-v" ]; then + if [ "$arg" = "--verbose" ] || [ "$arg" = "-v" ]; then AI_VERBOSE=true elif [ -z "$1" ]; then break @@ -1235,14 +1233,6 @@ run_ai() # Initials - ra shift done - if [ "$ra_upload_to_fog" = "false" ]; then - printf "Should upload to Checkpoints' cloud? [y/n] " && read -r ra_should_upload - case $ra_should_upload in - [Yy] | [Yy][Ee][Ss]) ra_upload_to_fog=true ;; - *) ;; - esac - fi - ra_https_prefix="https://" ra_agent_details=$(cat ${FILESYSTEM_PATH}/$cp_nano_conf_location/agent_details.json) if echo "$ra_agent_details" | grep -q "Fog domain"; then @@ -1274,49 +1264,6 @@ run_ai() # Initials - ra echo "Failed to calculate agent-info data." exit 1 fi - if [ "$ra_upload_to_fog" = "true" ]; then - ra_token_data="$(curl_func "$(extract_api_port orchestration)"/show-access-token)" - ra_token_hex=$(echo "$ra_token_data" | grep "token" | cut -d '"' -f4 | base64 -d | od -t x1 -An) - ra_token_hex_formatted=$(echo $ra_token_hex | tr -d ' ') - ra_token="$(xor_decrypt "${ra_token_hex_formatted}")" - - ra_proxy_val="" - if [ -n "${is_gaia}" ]; then - ra_gaia_proxy_address=$(dbget proxy:ip-address | tr -d '\n') - ra_gaia_proxy_ip=$(dbget proxy:port | tr -d '\n') - - if [ -n "$ra_gaia_proxy_address" ] && [ -n "$ra_gaia_proxy_ip" ]; then - ra_proxy_val="--proxy http://${ra_gaia_proxy_address}:${ra_gaia_proxy_ip}" - fi - fi - if [ "$is_smb_release" = "1" ]; then - is_proxy_enabled=$(pt proxySettings | awk '{if ($1 == "useProxy") printf("%s", $3)}') - if [ "$is_proxy_enabled" = "true" ]; then - ra_smb_proxy_address=$(pt proxySettings | awk '{if ($1 == "ipAddress") printf("%s", $3)}') - ra_smb_proxy_port=$(pt proxySettings | awk '{if ($1 == "port") printf("%s", $3)}') - - if [ ! -z $ra_smb_proxy_address ] && [ ! -z $ra_smb_proxy_port ]; then - ra_proxy_val="--proxy http://${ra_smb_proxy_address}:${ra_smb_proxy_port}" - fi - fi - fi - - echo "---- Uploading agent information to Check Point ----" - sleep 1 - - upload_ai "$ra_cp_info_path" "$ra_token" "$ra_fog_address" "$ra_tenant_id" "$ra_agent_id" "$ra_current_time" "$ra_file_dir" - if [ "$AI_UPLOAD_TOO_LARGE_FLAG" = "true" ]; then - echo "Files are too large - splitting to files of size of $SPLIT_FILE_SMALL_SIZE" - cat "$ra_cp_info_path"/* >"$ra_cp_info_path"/temp_reassembled_files - rm "$ra_cp_info_path"/*.* - split -b "$SPLIT_FILE_SMALL_SIZE" "$ra_cp_info_path"/temp_reassembled_files "$ra_cp_info_path"/cp-nano-info-"$ra_agent_id"-"$ra_current_time".tar.gz - rm "$ra_cp_info_path"/temp_reassembled_files - upload_ai "$ra_cp_info_path" "$ra_token" "$ra_fog_address" "$ra_tenant_id" "$ra_agent_id" "$ra_current_time" "$ra_file_dir" - fi - echo "File upload to cloud: Succeeded" - else - echo "ignore uploading file to the Fog." - fi } create_entries_file() # Initials - cef @@ -1346,38 +1293,6 @@ create_entries_file() # Initials - cef } >>"$cef_entries_file_path" } -upload_ai() # Initials - uai -{ - uai_cp_info_path="$1" - uai_token="$2" - uai_fog_address="$3" - uai_tenant_id="$4" - uai_agent_id="$5" - uai_current_time="$6" - uai_file_dir="$7" - create_entries_file "$uai_cp_info_path" - for file in "$uai_cp_info_path"/*; do - if [ "$AI_VERBOSE" = "true" ]; then - echo "Uploading file $file" - fi - if [ -z "${is_gaia}" -o "$is_smb_release" = "1" ]; then - uai_curl_output=$(${curl_cmd} -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1) - elif [ "${remove_curl_ld_path}" = "true" ]; then - uai_curl_output=$(LD_LIBRARY_PATH="" ${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1) - else - uai_curl_output=$(${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1) - fi - if [ "$AI_UPLOAD_TOO_LARGE_FLAG" = "false" ] && [ "$uai_curl_output" = "413" ]; then - AI_UPLOAD_TOO_LARGE_FLAG=true - return - fi - if test "$uai_curl_output" != "200"; then - echo "File upload to cloud: Failed Error code ${uai_curl_output}" - exit 1 - fi - done -} - set_mode_usage_message() { echo "Usage:" @@ -1464,9 +1379,17 @@ set_mode() gem_prefix_uppercase="CP-" us_prefix="cp-us-" us_prefix_uppercase="CP-US-" + au_prefix="cp-au-" + au_prefix_uppercase="CP-AU-" + in_prefix="cp-in-" + in_prefix_uppercase="CP-IN-" if [ "${var_token#"$us_prefix"}" != "${var_token}" ] || [ "${var_token#"$us_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_us_fog_address" + elif [ "${var_token#$au_prefix}" != "${var_token}" ] || [ "${var_token#"$au_prefix_uppercase"}" != "${var_token}" ]; then + var_fog_address="$var_default_au_fog_address" + elif [ "${var_token#$in_prefix}" != "${var_token}" ] || [ "${var_token#"$in_prefix_uppercase"}" != "${var_token}" ]; then + var_fog_address="$var_default_in_fog_address" elif [ "${var_token#"$gem_prefix"}" != "${var_token}" ] || [ "${var_token#"$gem_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_gem_fog_address" else diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index 980fb9d..b2faa67 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -43,6 +43,8 @@ DEFAULT_SETTINGS_PATH="${CONF_PATH}/settings.json" var_default_fog_address="https://i2-agents.cloud.ngen.checkpoint.com/" var_default_gem_fog_address="https://inext-agents.cloud.ngen.checkpoint.com" var_default_us_fog_address="https://inext-agents-us.cloud.ngen.checkpoint.com" +var_default_au_fog_address="https://inext-agents-aus1.cloud.ngen.checkpoint.com" +var_default_in_fog_address="https://inext-agents-ind1.cloud.ngen.checkpoint.com" var_fog_address= var_certs_dir= var_public_key= @@ -330,9 +332,17 @@ if [ "$RUN_MODE" = "install" ] && [ $var_offline_mode = false ]; then gem_prefix_uppercase="CP-" us_prefix="cp-us-" us_prefix_uppercase="CP-US-" + au_prefix="cp-au-" + au_prefix_uppercase="CP-AU-" + in_prefix="cp-in-" + in_prefix_uppercase="CP-IN-" if [ "${var_token#"$us_prefix"}" != "${var_token}" ] || [ "${var_token#"$us_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_us_fog_address" + elif [ "${var_token#$au_prefix}" != "${var_token}" ] || [ "${var_token#"$au_prefix_uppercase"}" != "${var_token}" ]; then + var_fog_address="$var_default_au_fog_address" + elif [ "${var_token#$in_prefix}" != "${var_token}" ] || [ "${var_token#"$in_prefix_uppercase"}" != "${var_token}" ]; then + var_fog_address="$var_default_in_fog_address" elif [ "${var_token#"$gem_prefix"}" != "${var_token}" ] || [ "${var_token#"$gem_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_gem_fog_address" else @@ -567,7 +577,7 @@ install_cp_nano_ctl() if [ -f ${FILESYSTEM_PATH}/${CONF_PATH}/CP_NANO_AGENT_CTL ]; then cp_exec "rm -rf ${FILESYSTEM_PATH}/${CONF_PATH}/$CP_NANO_AGENT_CTL" fi - + if [ -f ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/${CP_NANO_AGENT_CTL_DEPRECATED} ]; then cp_exec "rm -f ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/${CP_NANO_AGENT_CTL_DEPRECATED} $USR_SBIN_PATH/${CP_NANO_CTL_DEPRECATED}" fi @@ -602,7 +612,7 @@ set_conf_temp_location() if ! cat ${FILESYSTEM_PATH}/${ORCHESTRATION_CONF_FILE} | grep -q "\"orchestration\":"; then sed -i -e "1 s/{/{\n\"orchestration\": {\"Default file download path\": [{\"value\":\""${escaped_temp_location}"\"}]},/" ${FILESYSTEM_PATH}/${ORCHESTRATION_CONF_FILE} else - sed -i -e "/\"orchestration\"/ s/\"orchestration\".*:.*{/\"orchestration\":{\"Default file download path\": [{\"value\":\""${escaped_temp_location}"\"}],/" ${FILESYSTEM_PATH}/${ORCHESTRATION_CONF_FILE} + sed -i -e "/\"orchestration\"/ s/\"orchestration\".*:.*{/\"orchestration\":{\"Default file download path\": [{\"value\":\""${escaped_temp_location}"\"}],/" ${FILESYSTEM_PATH}/${ORCHESTRATION_CONF_FILE} fi fi } @@ -753,6 +763,16 @@ install_public_key() fi } +uninstall_messaging_proxy_if_needed() +{ + messaging_exec_path="${FILESYSTEM_PATH}/packages/messagingProxy/messagingProxy" + if [ -f ${messaging_exec_path} ]; then + chmod +x ${messaging_exec_path} + ${messaging_exec_path} --uninstall + rm -rf ${FILESYSTEM_PATH}/packages/messagingProxy + fi +} + install_orchestration() { INSTALLATION_TIME=$(date) @@ -837,6 +857,8 @@ install_orchestration() install_watchdog cp_print "Upgrade to latest" + uninstall_messaging_proxy_if_needed + ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration "$var_arch_flag" > /dev/null 2>&1 if [ "$IS_K8S_ENV" = "true" ]; then ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh @@ -1043,19 +1065,19 @@ run_pre_install_test() run_post_install_test() { if [ $var_is_alpine = false ]; then - if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_chrono.so ]; then + if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_chrono.so* ]; then cp_print "Error, libboost_chrono .so file is missing" ${FORCE_STDOUT} exit 1 fi - if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_context.so ]; then + if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_context.so* ]; then cp_print "Error, libboost_context .so file is missing" ${FORCE_STDOUT} exit 1 fi - if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_system.so ]; then + if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_system.so* ]; then cp_print "Error, libboost_system .so file is missing" ${FORCE_STDOUT} exit 1 fi - if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_thread.so ]; then + if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_thread.so* ]; then cp_print "Error, libboost_thread .so file is missing" ${FORCE_STDOUT} exit 1 fi From c89001b6e0f982bdb012ad4aa81a2ef1384bac06 Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Thu, 6 Jul 2023 09:34:00 +0000 Subject: [PATCH 2/3] Change library locating --- nodes/http_transaction_handler/CMakeLists.txt | 6 +-- nodes/orchestration/CMakeLists.txt | 41 +++++++++++++------ 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/nodes/http_transaction_handler/CMakeLists.txt b/nodes/http_transaction_handler/CMakeLists.txt index f774c97..70f74e5 100755 --- a/nodes/http_transaction_handler/CMakeLists.txt +++ b/nodes/http_transaction_handler/CMakeLists.txt @@ -42,19 +42,19 @@ install(TARGETS cp-nano-http-transaction-handler DESTINATION bin) install(TARGETS cp-nano-http-transaction-handler DESTINATION http_transaction_handler_service/bin) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libpcre2-8.so/{sub(/[^/]*$/,\"\", $NF); path=$NF}END{system(\"ls \"path\"libpcre2-8.so*\")}' | awk '{printf $1\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libpcre2-8.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE pcre2-8 ) install(FILES ${pcre2-8} DESTINATION http_transaction_handler_service/lib) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libpcre2-posix.so/{sub(/[^/]*$/,\"\", $NF); path=$NF}END{system(\"ls \"path\"libpcre2-posix.so*\")}' | awk '{printf $1\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libpcre2-posix.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE pcre2-posix ) install(FILES ${pcre2-posix} DESTINATION http_transaction_handler_service/lib) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libxml2.so/{sub(/[^/]*$/,\"\", $NF); path=$NF}END{system(\"ls \"path\"libxml2.so*\")}' | awk '{printf $1\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libxml2.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE xml2 ) install(FILES ${xml2} DESTINATION http_transaction_handler_service/lib) diff --git a/nodes/orchestration/CMakeLists.txt b/nodes/orchestration/CMakeLists.txt index 086beae..836d8af 100755 --- a/nodes/orchestration/CMakeLists.txt +++ b/nodes/orchestration/CMakeLists.txt @@ -43,79 +43,94 @@ install(FILES package/certificate/public-keys/i2.pem DESTINATION orchestration/c install(FILES package/certificate/public-keys/stg-i2.pem DESTINATION orchestration/certificate/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libboost_regex.so/{printf $NF\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libboost_regex.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE boost_regex ) install(FILES ${boost_regex} DESTINATION orchestration/lib/boost) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libboost_atomic.so/{printf $NF\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libboost_atomic.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE boost_atomic ) install(FILES ${boost_atomic} DESTINATION orchestration/lib/boost) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libboost_chrono.so/{printf $NF\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libboost_chrono.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE boost_chrono ) install(FILES ${boost_chrono} DESTINATION orchestration/lib/boost) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libboost_context.so/{printf $NF\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libboost_context.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE boost_context ) install(FILES ${boost_context} DESTINATION orchestration/lib/boost) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libboost_filesystem.so/{printf $NF\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libboost_filesystem.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE boost_filesystem ) install(FILES ${boost_filesystem} DESTINATION orchestration/lib/boost) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libboost_iostreams.so/{printf $NF\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libboost_iostreams.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE boost_iostreams ) install(FILES ${boost_iostreams} DESTINATION orchestration/lib/boost) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libboost_system.so/{printf $NF\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libboost_system.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE boost_system ) install(FILES ${boost_system} DESTINATION orchestration/lib/boost) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libboost_system.so/{printf $NF\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libboost_system.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE boost_system ) install(FILES ${boost_system} DESTINATION orchestration/lib/boost) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libboost_thread.so/{printf $NF\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libboost_thread.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE boost_thread ) install(FILES ${boost_thread} DESTINATION orchestration/lib/boost) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libcrypto.so/{printf $NF\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libcrypto.so\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE crypto ) install(FILES ${crypto} DESTINATION orchestration/lib) +execute_process ( + COMMAND bash -c "find /lib -name \"libcrypto.so*\" | awk '{printf $0\";\"}'" + OUTPUT_VARIABLE crypto2 +) +install(FILES ${crypto2} DESTINATION orchestration/lib) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libssl.so/{printf $NF\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libssl.so\" | awk '{printf $1\";\"}'" OUTPUT_VARIABLE ssl ) install(FILES ${ssl} DESTINATION orchestration/lib) +execute_process ( + COMMAND bash -c "find /lib -name \"libssl.so*\" | awk '{printf $1\";\"}'" + OUTPUT_VARIABLE ssl2 +) +install(FILES ${ssl2} DESTINATION orchestration/lib) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libcurl.so/{sub(/[^/]*$/,\"\", $NF); path=$NF}END{system(\"ls \"path\"libcurl.so*\")}' | awk '{printf $1\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libcurl.so\" | awk '{printf $1\";\"}'" OUTPUT_VARIABLE curl ) install(FILES ${curl} DESTINATION orchestration/lib) +execute_process ( + COMMAND bash -c "find /lib -name \"libcurl.so*\" | awk '{printf $1\";\"}'" + OUTPUT_VARIABLE curl2 +) +install(FILES ${curl2} DESTINATION orchestration/lib) execute_process ( - COMMAND bash -c "ldconfig -p | awk '/libz.so/{sub(/[^/]*$/,\"\", $NF); path=$NF}END{system(\"ls \"path\"libz.so*\")}' | awk '{printf $1\";\"}'" + COMMAND bash -c "find /usr/lib -name \"libz.so*\" | awk '{printf $0\";\"}'" OUTPUT_VARIABLE z ) install(FILES ${z} DESTINATION orchestration/lib) From 01b6544ca5150567066afd7943810746f0ede156 Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Sun, 16 Jul 2023 12:51:20 +0000 Subject: [PATCH 3/3] Fix manifest handling --- .../manifest_controller/manifest_controller.cc | 11 ++++++++--- .../security_apps/orchestration/orchestration_comp.cc | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/components/security_apps/orchestration/manifest_controller/manifest_controller.cc b/components/security_apps/orchestration/manifest_controller/manifest_controller.cc index f48fa51..7d5ae76 100755 --- a/components/security_apps/orchestration/manifest_controller/manifest_controller.cc +++ b/components/security_apps/orchestration/manifest_controller/manifest_controller.cc @@ -160,16 +160,21 @@ ManifestController::Impl::updateManifest(const string &new_manifest_file) { auto i_env = Singleton::Consume::by(); auto span_scope = i_env->startNewSpanScope(Span::ContextType::CHILD_OF); + auto orchestration_tools = Singleton::Consume::by(); - if (isIgnoreFile(new_manifest_file)) return true; + if (isIgnoreFile(new_manifest_file)) { + if (!orchestration_tools->copyFile(new_manifest_file, manifest_file_path)) { + dbgWarning(D_ORCHESTRATOR) << "Failed to copy a new manifest file"; + return false; + } + return true; + } dbgDebug(D_ORCHESTRATOR) << "Starting to update manifest file"; auto ignored_settings_packages = getProfileAgentSetting("orchestration.IgnoredPackagesList"); set packages_to_ignore = ignore_packages; if (ignored_settings_packages.ok()) packages_to_ignore = *(*ignored_settings_packages); - auto orchestration_tools = Singleton::Consume::by(); - if (packages_to_ignore.count("all") > 0) { dbgTrace(D_ORCHESTRATOR) << "Nothing to update (\"ignore all\" turned on)"; diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 49cb0af..23d3c5a 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -1087,7 +1087,8 @@ private: vector data_updates; update_results[OrchestrationStatusConfigType::DATA] = handleDataUpdate(orch_data, data_updates); - if (!orch_manifest.ok() && orch_policy.ok()) { + auto orch_mode = agent_details->getOrchestrationMode(); + if ((!orch_manifest.ok() || orch_mode == OrchestrationMode::HYBRID) && orch_policy.ok()) { update_results[OrchestrationStatusConfigType::POLICY] = handlePolicyUpdate( orch_policy, settings_path,