diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index ee7bf7d..58aba03 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -1,4 +1,3 @@ -add_subdirectory(report_messaging) add_subdirectory(http_manager) add_subdirectory(signal_handler) add_subdirectory(gradual_deployment) diff --git a/components/attachment-intakers/attachment_registrator/attachment_registrator.cc b/components/attachment-intakers/attachment_registrator/attachment_registrator.cc index bd5abca..98adf25 100755 --- a/components/attachment-intakers/attachment_registrator/attachment_registrator.cc +++ b/components/attachment-intakers/attachment_registrator/attachment_registrator.cc @@ -39,6 +39,8 @@ USE_DEBUG_FLAG(D_ATTACHMENT_REGISTRATION); using namespace std; +static const AlertInfo alert(AlertTeam::CORE, "attachment registrator"); + class AttachmentRegistrator::Impl { public: @@ -163,7 +165,7 @@ private: break; } default: - dbgAssert(false) << "Unsupported Attachment " << static_cast(type); + dbgAssert(false) << alert << "Unsupported Attachment " << static_cast(type); } if (!family_id.empty()) handler_path << family_id << "_"; @@ -175,7 +177,9 @@ private: string genRegCommand(const string &family_id, const uint num_of_members, const AttachmentType type) const { - dbgAssert(num_of_members > 0) << "Failed to generate a registration command for an empty group of attachments"; + dbgAssert(num_of_members > 0) + << alert + << "Failed to generate a registration command for an empty group of attachments"; static const string registration_format = "/etc/cp/watchdog/cp-nano-watchdog --register "; stringstream registration_command; @@ -187,7 +191,7 @@ private: break; } default: - dbgAssert(false) << "Unsupported Attachment " << static_cast(type); + dbgAssert(false) << alert << "Unsupported Attachment " << static_cast(type); } if (!family_id.empty()) registration_command << " --family " << family_id; @@ -265,7 +269,7 @@ private: return -1; } - dbgAssert(new_socket.unpack() > 0) << "Generated socket is OK yet negative"; + dbgAssert(new_socket.unpack() > 0) << alert << "Generated socket is OK yet negative"; return new_socket.unpack(); } @@ -281,7 +285,7 @@ private: } I_Socket::socketFd client_socket = accepted_socket.unpack(); - dbgAssert(client_socket > 0) << "Generated client socket is OK yet negative"; + dbgAssert(client_socket > 0) << alert << "Generated client socket is OK yet negative"; auto close_socket_on_exit = make_scope_exit([&]() { i_socket->closeSocket(client_socket); }); Maybe attachment_id = readNumericParam(client_socket); @@ -375,7 +379,7 @@ private: } I_Socket::socketFd client_socket = accepted_socket.unpack(); - dbgAssert(client_socket > 0) << "Generated client socket is OK yet negative"; + dbgAssert(client_socket > 0) << alert << "Generated client socket is OK yet negative"; auto close_socket_on_exit = make_scope_exit([&]() { i_socket->closeSocket(client_socket); }); Maybe attachment_type = readAttachmentType(client_socket); diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc index 604fcab..fa07cfc 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc @@ -76,6 +76,7 @@ using namespace std; using ChunkType = ngx_http_chunk_type_e; static const uint32_t corrupted_session_id = CORRUPTED_SESSION_ID; +static const AlertInfo alert(AlertTeam::CORE, "nginx attachment"); class FailopenModeListener : public Listener { @@ -410,7 +411,10 @@ private: bool registerAttachmentProcess(uint32_t nginx_user_id, uint32_t nginx_group_id, I_Socket::socketFd new_socket) { - dbgAssert(server_sock > 0) << "Registration attempt occurred while registration socket is uninitialized"; + dbgAssert(server_sock > 0) + << alert + << "Registration attempt occurred while registration socket is uninitialized"; + #ifdef FAILURE_TEST bool did_fail_on_purpose = false; #endif @@ -802,10 +806,10 @@ private: case ChunkType::HOLD_DATA: return "HOLD_DATA"; case ChunkType::COUNT: - dbgAssert(false) << "Invalid 'COUNT' ChunkType"; + dbgAssert(false) << alert << "Invalid 'COUNT' ChunkType"; return ""; } - dbgAssert(false) << "ChunkType was not handled by the switch case"; + dbgAssert(false) << alert << "ChunkType was not handled by the switch case"; return ""; } @@ -1582,7 +1586,7 @@ private: case WAIT: return "WAIT"; } - dbgAssert(false) << "Invalid EventVerdict enum: " << static_cast(verdict.getVerdict()); + dbgAssert(false) << alert << "Invalid EventVerdict enum: " << static_cast(verdict.getVerdict()); return string(); } @@ -1633,13 +1637,14 @@ private: return false; } - dbgAssert(sock.unpack() > 0) << "The generated server socket is OK, yet negative"; + dbgAssert(sock.unpack() > 0) << alert << "The generated server socket is OK, yet negative"; server_sock = sock.unpack(); I_MainLoop::Routine accept_attachment_routine = [this] () { dbgAssert(inst_awareness->getUniqueID().ok()) + << alert << "NGINX attachment Initialized without Instance Awareness"; bool did_fail_on_purpose = false; @@ -1652,7 +1657,7 @@ private: << (did_fail_on_purpose ? "Intentional Failure" : new_sock.getErr()); return; } - dbgAssert(new_sock.unpack() > 0) << "The generated client socket is OK, yet negative"; + dbgAssert(new_sock.unpack() > 0) << alert << "The generated client socket is OK, yet negative"; I_Socket::socketFd new_attachment_socket = new_sock.unpack(); Maybe uid = getUidFromSocket(new_attachment_socket); @@ -1711,7 +1716,9 @@ private: Maybe getUidFromSocket(I_Socket::socketFd new_attachment_socket) { - dbgAssert(server_sock > 0) << "Registration attempt occurred while registration socket is uninitialized"; + dbgAssert(server_sock > 0) + << alert + << "Registration attempt occurred while registration socket is uninitialized"; bool did_fail_on_purpose = false; DELAY_IF_NEEDED(IntentionalFailureHandler::FailureType::ReceiveDataFromSocket); diff --git a/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc b/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc index 1e6af28..793505e 100755 --- a/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc +++ b/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc @@ -312,8 +312,6 @@ UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, Ext return; } NginxAttachmentOpaque &opaque = i_transaction_table->getState(); - opaque.setSavedData(HttpTransactionData::xff_vals_ctx, header.getValue()); - dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "xff found, value from header: " << static_cast(header.getValue()); auto value = parseXForwardedFor(header.getValue()); if (!value.ok()) { dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Could not extract source identifier from X-Forwarded-For header"; @@ -325,6 +323,10 @@ UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, Ext dbgDebug(D_NGINX_ATTACHMENT_PARSER) << "Added source identifir to XFF " << value.unpack(); + opaque.setSavedData(HttpTransactionData::xff_vals_ctx, header.getValue()); + dbgTrace(D_NGINX_ATTACHMENT_PARSER) + << "XFF found, set ctx with value from header: " + << static_cast(header.getValue()); } else { opaque.setSavedData(HttpTransactionData::proxy_ip_ctx, value.unpack()); } diff --git a/components/gradual_deployment/gradual_deployment.cc b/components/gradual_deployment/gradual_deployment.cc index 4b25b3b..ca0c4b1 100644 --- a/components/gradual_deployment/gradual_deployment.cc +++ b/components/gradual_deployment/gradual_deployment.cc @@ -128,7 +128,7 @@ private: break; } default: - dbgAssert(false) << "Unsupported IP type"; + dbgAssert(false) << AlertInfo(AlertTeam::CORE, "gradual deployment") << "Unsupported IP type"; } return address; } diff --git a/components/http_manager/http_manager.cc b/components/http_manager/http_manager.cc index 4d544b3..1821530 100755 --- a/components/http_manager/http_manager.cc +++ b/components/http_manager/http_manager.cc @@ -46,7 +46,10 @@ operator<<(ostream &os, const EventVerdict &event) case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT: return os << "Wait"; } - dbgAssert(false) << "Illegal Event Verdict value: " << static_cast(event.getVerdict()); + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "http manager") + << "Illegal Event Verdict value: " + << static_cast(event.getVerdict()); return os; } @@ -321,8 +324,11 @@ private: state.setApplicationVerdict(respond.first, respond.second.getVerdict()); } - - return state.getCurrVerdict(); + FilterVerdict aggregated_verdict = state.getCurrVerdict(); + if (aggregated_verdict.getVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) { + SecurityAppsDropEvent(state.getCurrentDropVerdictCausers()).notify(); + } + return aggregated_verdict; } static void diff --git a/components/http_manager/http_manager_opaque.cc b/components/http_manager/http_manager_opaque.cc index 7be0e25..eea3ed4 100644 --- a/components/http_manager/http_manager_opaque.cc +++ b/components/http_manager/http_manager_opaque.cc @@ -69,6 +69,7 @@ HttpManagerOpaque::getCurrVerdict() const break; default: dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "http manager") << "Received unknown verdict " << static_cast(app_verdic_pair.second); } @@ -77,6 +78,25 @@ HttpManagerOpaque::getCurrVerdict() const return accepted_apps == applications_verdicts.size() ? ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT : verdict; } +std::set +HttpManagerOpaque::getCurrentDropVerdictCausers() const +{ + std::set causers; + if (manager_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) { + causers.insert(HTTP_MANAGER_NAME); + } + for (const auto &app_verdic_pair : applications_verdicts) { + bool was_dropped = app_verdic_pair.second == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + dbgTrace(D_HTTP_MANAGER) + << "The verdict from: " << app_verdic_pair.first + << (was_dropped ? " is \"drop\"" : " is not \"drop\" "); + if (was_dropped) { + causers.insert(app_verdic_pair.first); + } + } + return causers; +} + void HttpManagerOpaque::saveCurrentDataToCache(const Buffer &full_data) { diff --git a/components/http_manager/http_manager_opaque.h b/components/http_manager/http_manager_opaque.h index 23a50f1..f9a3226 100644 --- a/components/http_manager/http_manager_opaque.h +++ b/components/http_manager/http_manager_opaque.h @@ -20,6 +20,8 @@ #include "table_opaque.h" #include "nginx_attachment_common.h" +static const std::string HTTP_MANAGER_NAME = "HTTP Manager"; + class HttpManagerOpaque : public TableOpaqueSerialize { public: @@ -30,6 +32,7 @@ public: void setManagerVerdict(ngx_http_cp_verdict_e verdict) { manager_verdict = verdict; } ngx_http_cp_verdict_e getManagerVerdict() const { return manager_verdict; } ngx_http_cp_verdict_e getCurrVerdict() const; + std::set getCurrentDropVerdictCausers() const; void saveCurrentDataToCache(const Buffer &full_data); void setUserDefinedValue(const std::string &value) { user_defined_value = value; } Maybe getUserDefinedValue() const { return user_defined_value; } diff --git a/components/include/health_checker.h b/components/include/health_checker.h index 80932c8..b0d37e3 100755 --- a/components/include/health_checker.h +++ b/components/include/health_checker.h @@ -21,6 +21,7 @@ #include "i_shell_cmd.h" #include "i_orchestration_status.h" #include "component.h" +#include "i_service_controller.h" class HealthChecker : @@ -29,7 +30,8 @@ class HealthChecker Singleton::Consume, Singleton::Consume, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: HealthChecker(); diff --git a/components/include/http_event_impl/i_http_event_impl.h b/components/include/http_event_impl/i_http_event_impl.h index 94b63c0..34633fb 100755 --- a/components/include/http_event_impl/i_http_event_impl.h +++ b/components/include/http_event_impl/i_http_event_impl.h @@ -50,9 +50,11 @@ public: position(mod_position) { dbgAssert(mod_type != ModificationType::APPEND || position == injection_pos_irrelevant) + << AlertInfo(AlertTeam::CORE, "http manager") << "Injection position is not applicable to a modification of type \"Append\""; dbgAssert(mod_type != ModificationType::INJECT || position >= 0) + << AlertInfo(AlertTeam::CORE, "http manager") << "Invalid injection position: must be non-negative. Position: " << position; } @@ -166,6 +168,7 @@ private: } default: dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "http manager") << "Unknown type of ModificationType: " << static_cast(modification_type); } diff --git a/components/include/http_inspection_events.h b/components/include/http_inspection_events.h index fb06278..c96e76f 100755 --- a/components/include/http_inspection_events.h +++ b/components/include/http_inspection_events.h @@ -183,4 +183,16 @@ class WaitTransactionEvent : public Event { }; +class SecurityAppsDropEvent : public Event +{ +public: + SecurityAppsDropEvent( + const std::set &apps_names) + : + apps_names(apps_names) {} + const std::set & getAppsNames() const { return apps_names; } + +private: + const std::set apps_names; +}; #endif // __HTTP_INSPECTION_EVENTS_H__ diff --git a/components/include/i_service_controller.h b/components/include/i_service_controller.h index 5deca00..9ec35f8 100755 --- a/components/include/i_service_controller.h +++ b/components/include/i_service_controller.h @@ -66,6 +66,8 @@ public: virtual std::map> getServiceToPortMap() = 0; + virtual bool getServicesPolicyStatus() const = 0; + protected: virtual ~I_ServiceController() {} }; diff --git a/components/include/orchestrator/rest_api/get_resource_file.h b/components/include/orchestrator/rest_api/get_resource_file.h index 6045b3b..cb19a79 100644 --- a/components/include/orchestrator/rest_api/get_resource_file.h +++ b/components/include/orchestrator/rest_api/get_resource_file.h @@ -115,7 +115,7 @@ public: case ResourceFileType::VIRTUAL_SETTINGS: return "virtualSettings"; case ResourceFileType::VIRTUAL_POLICY: return "virtualPolicy"; default: - dbgAssert(false) << "Unknown file type"; + dbgAssert(false) << AlertInfo(AlertTeam::CORE, "update process") << "Unknown file type"; } return std::string(); } diff --git a/components/include/package.h b/components/include/package.h index 7b479cc..9b5524d 100755 --- a/components/include/package.h +++ b/components/include/package.h @@ -56,7 +56,7 @@ private: if (mapped_type.second == type) return mapped_type.first; } - dbgAssert(false) << "Unsupported type " << static_cast(type); + dbgAssert(false) << AlertInfo(AlertTeam::CORE, "packaging") << "Unsupported type " << static_cast(type); // Just satisfying the compiler, this return never reached return std::string(); } diff --git a/components/include/reverse_proxy_defaults.h b/components/include/reverse_proxy_defaults.h index 252936b..0be2c3c 100644 --- a/components/include/reverse_proxy_defaults.h +++ b/components/include/reverse_proxy_defaults.h @@ -7,24 +7,28 @@ static const std::string product_name = getenv("DOCKER_RPM_ENABLED") ? "CloudGua static const std::string default_cp_cert_file = "/etc/cp/cpCert.pem"; static const std::string default_cp_key_file = "/etc/cp/cpKey.key"; static const std::string default_rpm_conf_path = "/etc/cp/conf/rpmanager/"; + static const std::string default_certificate_path = "/etc/cp/rpmanager/certs"; +static const std::string default_manual_certs_path = "/etc/cp/rpmanager/manualCerts/"; +static const std::string default_config_path = "/etc/cp/conf/rpmanager/servers"; +static const std::string default_rpm_prepare_path = "/etc/cp/conf/rpmanager/prepare/servers"; + +static const std::string default_nginx_log_files_path = "/var/log/nginx/"; static const std::string default_additional_files_path = "/etc/cp/conf/rpmanager/include"; static const std::string default_server_config = "additional_server_config.conf"; static const std::string default_location_config = "additional_location_config.conf"; static const std::string default_trusted_ca_suffix = "_user_ca_bundle.crt"; -static const std::string default_nginx_log_files_path = "/var/log/nginx/"; static const std::string default_log_files_host_path = "/var/log/nano_agent/rpmanager/nginx_log/"; -static const std::string default_config_path = "/etc/cp/conf/rpmanager/servers"; static const std::string default_template_path = "/etc/cp/conf/rpmanager/nginx-template-clear"; -static const std::string default_manual_certs_path = "/etc/cp/rpmanager/manualCerts/"; static const std::string default_server_certificate_path = "/etc/cp/rpmanager/certs/sslCertificate_"; static const std::string default_server_certificate_key_path = "/etc/cp/rpmanager/certs/sslPrivateKey_"; static const std::string default_container_name = "cp_nginx_gaia"; static const std::string default_docker_image = "cp_nginx_gaia"; static const std::string default_nginx_config_file = "/etc/cp/conf/rpmanager/nginx.conf"; +static const std::string default_prepare_nginx_config_file = "/etc/cp/conf/rpmanager/nginx_prepare.conf"; static const std::string default_global_conf_template = "/etc/cp/conf/rpmanager/nginx-conf-template"; static const std::string default_nginx_config_include_file = - "/etc/cp/conf/rpmanager/servers/nginx_conf_include"; + "/etc/cp/conf/rpmanager/servers/nginx_conf_include.conf"; static const std::string default_global_conf_include_template = "/etc/cp/conf/rpmanager/nginx-conf-include-template"; static const std::string default_global_conf_include_template_no_responses = diff --git a/components/include/service_health_status.h b/components/include/service_health_status.h new file mode 100644 index 0000000..0a53ed6 --- /dev/null +++ b/components/include/service_health_status.h @@ -0,0 +1,39 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __SERVICE_HEALTH_STATUS_H__ +#define __SERVICE_HEALTH_STATUS_H__ + +#include "singleton.h" +#include "i_rest_api.h" +#include "i_environment.h" +#include "component.h" + +class ServiceHealthStatus + : + public Component, + Singleton::Consume, + Singleton::Consume +{ +public: + ServiceHealthStatus(); + ~ServiceHealthStatus(); + + void init() override; + +private: + class Impl; + std::unique_ptr pimpl; +}; + +#endif // __SERVICE_HEALTH_STATUS_H__ diff --git a/components/include/waap.h b/components/include/waap.h index e4be504..4df31ba 100755 --- a/components/include/waap.h +++ b/components/include/waap.h @@ -34,6 +34,8 @@ class I_Messaging; class I_AgentDetails; class I_Encryptor; +const std::string WAAP_APPLICATION_NAME = "waap application"; + class WaapComponent : public Component, diff --git a/components/packet/packet.cc b/components/packet/packet.cc index 5a23cc3..b7ee225 100755 --- a/components/packet/packet.cc +++ b/components/packet/packet.cc @@ -563,7 +563,10 @@ Packet::parsePacket(PktType type, IPType proto) return parseFromL3v6(); } default: { - dbgAssert(false) << "Unknown (neither IPv4, nor IPv6), or uninitialized packet type: " << proto; + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "packet") + << "Unknown (neither IPv4, nor IPv6), or uninitialized packet type: " + << proto; } } diff --git a/components/pending_key/pending_key.cc b/components/pending_key/pending_key.cc index 6f61a9b..2924748 100755 --- a/components/pending_key/pending_key.cc +++ b/components/pending_key/pending_key.cc @@ -43,7 +43,9 @@ PendingKey::print(ostream &os) const size_t PendingKey::hash() const { - dbgAssert(src.type != IPType::UNINITIALIZED) << "PendingKey::hash was called on an uninitialized object"; + dbgAssert(src.type != IPType::UNINITIALIZED) + << AlertInfo(AlertTeam::CORE, "pending key") + << "PendingKey::hash was called on an uninitialized object"; size_t seed = 0; hashCombine(seed, static_cast(src.type)); hashCombine(seed, src.proto); diff --git a/components/report_messaging/report_messaging_ut/CMakeLists.txt b/components/report_messaging/report_messaging_ut/CMakeLists.txt deleted file mode 100644 index 8f263c7..0000000 --- a/components/report_messaging/report_messaging_ut/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -link_directories(${BOOST_ROOT}/lib) - -add_unit_test(report_messaging_ut "report_messaging_ut.cc" "report_messaging;report;messaging;singleton;-lboost_regex") \ No newline at end of file diff --git a/components/security_apps/http_geo_filter/http_geo_filter.cc b/components/security_apps/http_geo_filter/http_geo_filter.cc index a619b0e..4c621f8 100644 --- a/components/security_apps/http_geo_filter/http_geo_filter.cc +++ b/components/security_apps/http_geo_filter/http_geo_filter.cc @@ -67,18 +67,18 @@ public: dbgTrace(D_GEO_FILTER) << getListenerName() << " new transaction event"; if (!event.isLastHeader()) return EventVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT); - std::set xff_set; + std::set ip_set; auto env = Singleton::Consume::by(); auto maybe_xff = env->get(HttpTransactionData::xff_vals_ctx); if (!maybe_xff.ok()) { dbgTrace(D_GEO_FILTER) << "failed to get xff vals from env"; } else { - xff_set = split(maybe_xff.unpack(), ','); + ip_set = split(maybe_xff.unpack(), ','); } dbgDebug(D_GEO_FILTER) << getListenerName() << " last header, start lookup"; - if (xff_set.size() > 0) { - removeTrustedIpsFromXff(xff_set); + if (ip_set.size() > 0) { + removeTrustedIpsFromXff(ip_set); } else { dbgDebug(D_GEO_FILTER) << "xff not found in headers"; } @@ -90,14 +90,14 @@ public: } auto source_ip = convertIpAddrToString(maybe_source_ip.unpack()); - xff_set.insert(source_ip); + ip_set.insert(source_ip); - ngx_http_cp_verdict_e exception_verdict = getExceptionVerdict(xff_set); + ngx_http_cp_verdict_e exception_verdict = getExceptionVerdict(ip_set); if (exception_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) { return EventVerdict(exception_verdict); } - ngx_http_cp_verdict_e geo_lookup_verdict = getGeoLookupVerdict(xff_set); + ngx_http_cp_verdict_e geo_lookup_verdict = getGeoLookupVerdict(ip_set); if (geo_lookup_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) { return EventVerdict(geo_lookup_verdict); } @@ -469,5 +469,6 @@ void HttpGeoFilter::preload() { registerExpectedConfiguration("rulebase", "httpGeoFilter"); + registerExpectedConfiguration("rulebase", "usersIdentifiers"); registerConfigLoadCb([this]() { pimpl->loadDefaultAction(); }); } diff --git a/components/security_apps/ips/compound_protection.cc b/components/security_apps/ips/compound_protection.cc index 792385f..a6fca24 100644 --- a/components/security_apps/ips/compound_protection.cc +++ b/components/security_apps/ips/compound_protection.cc @@ -43,7 +43,10 @@ CompoundProtection::Impl::getMatch(const set &matched) const case Operation::ORDERED_AND: return getMatchOrderedAnd(matched); } - dbgAssert(false) << "Unknown compound operation: " << static_cast(operation); + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "ips") + << "Unknown compound operation: " + << static_cast(operation); return MatchType::NO_MATCH; } diff --git a/components/security_apps/ips/ips_configuration.cc b/components/security_apps/ips/ips_configuration.cc index 80f4bf1..68620a7 100644 --- a/components/security_apps/ips/ips_configuration.cc +++ b/components/security_apps/ips/ips_configuration.cc @@ -8,7 +8,9 @@ IPSConfiguration::Context::Context(ContextType _type, uint history) : type(_type uint IPSConfiguration::Context::getHistorySize() const { - dbgAssert(type == ContextType::HISTORY) << "Try to access history size for non-history context"; + dbgAssert(type == ContextType::HISTORY) + << AlertInfo(AlertTeam::CORE, "ips") + << "Try to access history size for non-history context"; return history_size; } @@ -69,6 +71,8 @@ uint IPSConfiguration::getHistorySize(const string &name) const { auto context = context_config.find(name); - dbgAssert(context != context_config.end()) << "Try to access history size for non-exiting context"; + dbgAssert(context != context_config.end()) + << AlertInfo(AlertTeam::CORE, "ips") + << "Try to access history size for non-exiting context"; return context->second.getHistorySize(); } diff --git a/components/security_apps/ips/ips_signatures.cc b/components/security_apps/ips/ips_signatures.cc index 565d5c0..36475e8 100644 --- a/components/security_apps/ips/ips_signatures.cc +++ b/components/security_apps/ips/ips_signatures.cc @@ -84,7 +84,7 @@ IPSSignatureMetaData::getSeverityString() const return "Critical"; } - dbgAssert(false) << "Illegal severity value: " << static_cast(severity); + dbgAssert(false) << AlertInfo(AlertTeam::CORE, "ips") << "Illegal severity value: " << static_cast(severity); return "Critical"; } @@ -116,7 +116,10 @@ IPSSignatureMetaData::getPerformanceString() const return "Critical"; } - dbgAssert(false) << "Illegal performance value: " << static_cast(performance); + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "ips") + << "Illegal performance value: " + << static_cast(performance); return "Critical"; } diff --git a/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc b/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc index 2d7adcd..b59ab4c 100755 --- a/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc +++ b/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc @@ -19,7 +19,14 @@ using namespace std; USE_DEBUG_FLAG(D_LOCAL_POLICY); // LCOV_EXCL_START Reason: no test exist -static const set valid_modes = {"prevent-learn", "detect-learn", "prevent", "detect", "inactive"}; +static const set valid_modes = { + "prevent-learn", + "detect-learn", + "prevent", + "detect", + "inactive", + "as-top-level" +}; static const set valid_confidences = {"medium", "high", "critical"}; void @@ -138,15 +145,11 @@ AppSecPracticeWebAttacks::load(cereal::JSONInputArchive &archive_in) dbgWarning(D_LOCAL_POLICY) << "AppSec practice override mode invalid: " << mode; } - if (getMode() == "Prevent") { - parseAppsecJSONKey("minimum-confidence", minimum_confidence, archive_in, "critical"); - if (valid_confidences.count(minimum_confidence) == 0) { - dbgWarning(D_LOCAL_POLICY) - << "AppSec practice override minimum confidence invalid: " - << minimum_confidence; - } - } else { - minimum_confidence = "Transparent"; + parseAppsecJSONKey("minimum-confidence", minimum_confidence, archive_in, "critical"); + if (valid_confidences.count(minimum_confidence) == 0) { + dbgWarning(D_LOCAL_POLICY) + << "AppSec practice override minimum confidence invalid: " + << minimum_confidence; } parseAppsecJSONKey("max-body-size-kb", max_body_size_kb, archive_in, 1000000); parseAppsecJSONKey("max-header-size-bytes", max_header_size_bytes, archive_in, 102400); @@ -189,7 +192,10 @@ AppSecPracticeWebAttacks::getMode(const string &default_mode) const { if (isModeInherited(mode) || (key_to_practices_val2.find(mode) == key_to_practices_val2.end())) { dbgError(D_LOCAL_POLICY) << "Couldn't find a value for key: " << mode << ". Returning " << default_mode; - return default_mode; + if(key_to_practices_val2.find(default_mode) == key_to_practices_val2.end()) { + return default_mode; + } + return key_to_practices_val2.at(default_mode); } return key_to_practices_val2.at(mode); } @@ -428,7 +434,6 @@ WebAppSection::WebAppSection( practice_id(_practice_id), practice_name(_practice_name), context(_context), - web_attack_mitigation_severity(parsed_appsec_spec.getWebAttacks().getMinimumConfidence()), web_attack_mitigation_mode(parsed_appsec_spec.getWebAttacks().getMode(default_mode)), csrf_protection_mode("Disabled"), open_redirect_mode("Disabled"), @@ -438,6 +443,9 @@ WebAppSection::WebAppSection( trusted_sources({ parsed_trusted_sources }) { web_attack_mitigation = web_attack_mitigation_mode != "Disabled"; + web_attack_mitigation_severity = + web_attack_mitigation_mode != "Prevent" ? "Transparent" : + parsed_appsec_spec.getWebAttacks().getMinimumConfidence(); web_attack_mitigation_action = web_attack_mitigation_mode != "Prevent" ? "Transparent" : web_attack_mitigation_severity == "critical" ? "low" : @@ -470,6 +478,7 @@ WebAppSection::WebAppSection( const string &_context, const string &_web_attack_mitigation_severity, const string &_web_attack_mitigation_mode, + const string &_bot_protection, const PracticeAdvancedConfig &_practice_advanced_config, const AppsecPracticeAntiBotSection &_anti_bots, const LogTriggerSection &parsed_log_trigger, @@ -486,6 +495,7 @@ WebAppSection::WebAppSection( context(_context), web_attack_mitigation_severity(_web_attack_mitigation_severity), web_attack_mitigation_mode(_web_attack_mitigation_mode), + bot_protection(_bot_protection), practice_advanced_config(_practice_advanced_config), anti_bots(_anti_bots), trusted_sources({ parsed_trusted_sources }) @@ -514,7 +524,6 @@ void WebAppSection::save(cereal::JSONOutputArchive &out_ar) const { string disabled_str = "Disabled"; - string detect_str = "Detect"; vector empty_list; out_ar( cereal::make_nvp("context", context), @@ -542,7 +551,7 @@ WebAppSection::save(cereal::JSONOutputArchive &out_ar) const cereal::make_nvp("waapParameters", empty_list), cereal::make_nvp("botProtection", false), cereal::make_nvp("antiBot", anti_bots), - cereal::make_nvp("botProtection_v2", detect_str) + cereal::make_nvp("botProtection_v2", bot_protection != "" ? bot_protection : string("Detect")) ); } diff --git a/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h b/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h index 9ab4589..b03b262 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h +++ b/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h @@ -290,6 +290,7 @@ public: const std::string &_context, const std::string &_web_attack_mitigation_severity, const std::string &_web_attack_mitigation_mode, + const std::string &_bot_protection, const PracticeAdvancedConfig &_practice_advanced_config, const AppsecPracticeAntiBotSection &_anti_bots, const LogTriggerSection &parsed_log_trigger, @@ -315,6 +316,7 @@ private: std::string csrf_protection_mode; std::string open_redirect_mode; std::string error_disclosure_mode; + std::string bot_protection; bool web_attack_mitigation; std::vector triggers; PracticeAdvancedConfig practice_advanced_config; diff --git a/components/security_apps/local_policy_mgmt_gen/include/new_practice.h b/components/security_apps/local_policy_mgmt_gen/include/new_practice.h index e8585fe..84ca770 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/new_practice.h +++ b/components/security_apps/local_policy_mgmt_gen/include/new_practice.h @@ -508,30 +508,20 @@ private: bool is_temporary; }; -class NewAppSecWebBotsURI -{ -public: - void load(cereal::JSONInputArchive &archive_in); - - const std::string & getURI() const; - -private: - std::string uri; -}; - class NewAppSecPracticeAntiBot { public: - std::vector getIjectedUris() const; - std::vector getValidatedUris() const; + const std::vector & getIjectedUris() const; + const std::vector & getValidatedUris() const; + const std::string & getMode() const; void load(cereal::JSONInputArchive &archive_in); void save(cereal::JSONOutputArchive &out_ar) const; private: std::string override_mode; - std::vector injected_uris; - std::vector validated_uris; + std::vector injected_uris; + std::vector validated_uris; }; class NewAppSecWebAttackProtections diff --git a/components/security_apps/local_policy_mgmt_gen/include/triggers_section.h b/components/security_apps/local_policy_mgmt_gen/include/triggers_section.h index 9e47a68..150a39d 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/triggers_section.h +++ b/components/security_apps/local_policy_mgmt_gen/include/triggers_section.h @@ -39,7 +39,7 @@ public: bool _logToAgent, bool _logToCef, bool _logToCloud, - bool _logToContainerService, + bool _logTolocalTuning, bool _logToSyslog, bool _responseBody, bool _tpDetect, @@ -73,7 +73,7 @@ private: bool logToAgent; bool logToCef; bool logToCloud; - bool logToContainerService; + bool logTolocalTuning; bool logToSyslog; bool responseBody; bool tpDetect; diff --git a/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc b/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc index 28972c2..12fcd21 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc @@ -180,12 +180,16 @@ NewAppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in) } else { cloud = false; } - auto mode = Singleton::Consume::by()->getOrchestrationMode(); - auto env_type = Singleton::Consume::by()->getEnvType(); - bool k8s_service_default = (mode == OrchestrationMode::HYBRID && env_type == EnvType::K8S); - // BC try load previous name. TODO: update CRD - parseAppsecJSONKey("k8s-service", container_service, archive_in, k8s_service_default); - parseAppsecJSONKey("container-service", container_service, archive_in, container_service); + bool local_tuning_default = false; + // check ENV VAR LOCAL_TUNING_ENABLED + char * tuning_enabled = getenv("LOCAL_TUNING_ENABLED"); + if (tuning_enabled != NULL) { + for (unsigned int i = 0; i < strlen(tuning_enabled); i++) { + tuning_enabled[i] = tolower(tuning_enabled[i]); + } + local_tuning_default = string(tuning_enabled) == "true"; + } + parseAppsecJSONKey("local-tuning", container_service, archive_in, local_tuning_default); NewStdoutLogging stdout_log; parseAppsecJSONKey("stdout", stdout_log, archive_in); diff --git a/components/security_apps/local_policy_mgmt_gen/new_practice.cc b/components/security_apps/local_policy_mgmt_gen/new_practice.cc index 8e266fd..6255bb0 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_practice.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_practice.cc @@ -50,6 +50,13 @@ static const std::unordered_map key_to_mode_val = { { "detect", "Detect"}, { "inactive", "Inactive"} }; +static const std::unordered_map anti_bot_key_to_mode_val = { + { "prevent-learn", "Prevent"}, + { "detect-learn", "Detect"}, + { "prevent", "Prevent"}, + { "detect", "Detect"}, + { "inactive", "Disabled"} +}; static const std::unordered_map unit_to_int = { { "bytes", 1}, { "KB", 1024}, @@ -81,57 +88,44 @@ getModeWithDefault( return key_to_val.at(mode); } -void -NewAppSecWebBotsURI::load(cereal::JSONInputArchive &archive_in) -{ - dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Web Bots URI"; - parseAppsecJSONKey("uri", uri, archive_in); -} - -const string & -NewAppSecWebBotsURI::getURI() const -{ - return uri; -} - -std::vector +const std::vector & NewAppSecPracticeAntiBot::getIjectedUris() const { - vector injected; - for (const NewAppSecWebBotsURI &uri : injected_uris) injected.push_back(uri.getURI()); - return injected; + return injected_uris; } -std::vector +const std::vector & NewAppSecPracticeAntiBot::getValidatedUris() const { - vector validated; - for (const NewAppSecWebBotsURI &uri : validated_uris) validated.push_back(uri.getURI()); - return validated; + return validated_uris; +} + +const std::string & +NewAppSecPracticeAntiBot::getMode() const +{ + return override_mode; } void NewAppSecPracticeAntiBot::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Web Bots"; - parseAppsecJSONKey>("injectedUris", injected_uris, archive_in); - parseAppsecJSONKey>("validatedUris", validated_uris, archive_in); - parseMandatoryAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); - if (valid_modes.count(override_mode) == 0) { - dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << override_mode; + string mode; + parseAppsecJSONKey>("injectedUris", injected_uris, archive_in); + parseAppsecJSONKey>("validatedUris", validated_uris, archive_in); + parseMandatoryAppsecJSONKey("overrideMode", mode, archive_in, "inactive"); + if (valid_modes.count(mode) == 0) { + dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << mode; } + override_mode = anti_bot_key_to_mode_val.at(mode); } void NewAppSecPracticeAntiBot::save(cereal::JSONOutputArchive &out_ar) const { - vector injected; - vector validated; - for (const NewAppSecWebBotsURI &uri : injected_uris) injected.push_back(uri.getURI()); - for (const NewAppSecWebBotsURI &uri : validated_uris) validated.push_back(uri.getURI()); out_ar( - cereal::make_nvp("injected", injected), - cereal::make_nvp("validated", validated) + cereal::make_nvp("injected", injected_uris), + cereal::make_nvp("validated", validated_uris) ); } diff --git a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc index eb256c7..493de1c 100755 --- a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc @@ -555,7 +555,7 @@ extractLogTriggerData(const string &trigger_annotation_name, const T &trigger_sp bool webHeaders = trigger_spec.getAppsecTriggerExtendedLogging().isHttpHeaders(); bool webBody = trigger_spec.getAppsecTriggerExtendedLogging().isRequestBody(); bool logToCloud = trigger_spec.getAppsecTriggerLogDestination().getCloud(); - bool logToContainerService = trigger_spec.getAppsecTriggerLogDestination().isContainerNeeded(); + bool logTolocalTuning = trigger_spec.getAppsecTriggerLogDestination().isContainerNeeded(); bool logToAgent = trigger_spec.getAppsecTriggerLogDestination().isAgentLocal(); bool beautify_logs = trigger_spec.getAppsecTriggerLogDestination().shouldBeautifyLogs(); bool logToCef = trigger_spec.getAppsecTriggerLogDestination().isCefNeeded(); @@ -582,7 +582,7 @@ extractLogTriggerData(const string &trigger_annotation_name, const T &trigger_sp logToAgent, logToCef, logToCloud, - logToContainerService, + logTolocalTuning, logToSyslog, responseBody, tpDetect, @@ -1236,6 +1236,7 @@ PolicyMakerUtils::createWebAppSection( rule_config.getContext(), apssec_practice.getWebAttacks().getMinimumConfidence(practice_mode), apssec_practice.getWebAttacks().getMode(practice_mode), + apssec_practice.getAntiBot().getMode(), practice_advance_config, apssec_practice.getAntiBot(), log_triggers[rule_annotations[AnnotationTypes::TRIGGER]], diff --git a/components/security_apps/local_policy_mgmt_gen/triggers_section.cc b/components/security_apps/local_policy_mgmt_gen/triggers_section.cc index 6659d52..860a6af 100755 --- a/components/security_apps/local_policy_mgmt_gen/triggers_section.cc +++ b/components/security_apps/local_policy_mgmt_gen/triggers_section.cc @@ -30,7 +30,7 @@ LogTriggerSection::LogTriggerSection( bool _logToAgent, bool _logToCef, bool _logToCloud, - bool _logToContainerService, + bool _logTolocalTuning, bool _logToSyslog, bool _responseBody, bool _tpDetect, @@ -55,7 +55,7 @@ LogTriggerSection::LogTriggerSection( logToAgent(_logToAgent), logToCef(_logToCef), logToCloud(_logToCloud), - logToContainerService(_logToContainerService), + logTolocalTuning(_logTolocalTuning), logToSyslog(_logToSyslog), responseBody(_responseBody), tpDetect(_tpDetect), @@ -101,7 +101,7 @@ LogTriggerSection::save(cereal::JSONOutputArchive &out_ar) const cereal::make_nvp("logToAgent", logToAgent), cereal::make_nvp("logToCef", logToCef), cereal::make_nvp("logToCloud", logToCloud), - cereal::make_nvp("logToContainerService", logToContainerService), + cereal::make_nvp("logTolocalTuning", logTolocalTuning), cereal::make_nvp("logToSyslog", logToSyslog), cereal::make_nvp("responseBody", responseBody), cereal::make_nvp("responseCode", false), @@ -393,12 +393,16 @@ AppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in) } else { cloud = false; } - auto mode = Singleton::Consume::by()->getOrchestrationMode(); - auto env_type = Singleton::Consume::by()->getEnvType(); - bool k8s_service_default = (mode == OrchestrationMode::HYBRID && env_type == EnvType::K8S); - // BC try load previous name. TODO: update CRD - parseAppsecJSONKey("k8s-service", container_service, archive_in, k8s_service_default); - parseAppsecJSONKey("container-service", container_service, archive_in, container_service); + // check ENV VAR LOCAL_TUNING_ENABLED + char * tuning_enabled = getenv("LOCAL_TUNING_ENABLED"); + if (tuning_enabled != NULL) { + for (unsigned int i = 0; i < strlen(tuning_enabled); i++) { + tuning_enabled[i] = tolower(tuning_enabled[i]); + } + container_service = string(tuning_enabled) == "true"; + } else { + container_service = false; + } StdoutLogging stdout_log; parseAppsecJSONKey("stdout", stdout_log, archive_in); diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h index 9bb89fa..cf8a80d 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h @@ -60,20 +60,16 @@ checkSAMLPortal(const string &command_output) Maybe checkPepIdaIdnStatus(const string &command_output) { - if (command_output.find("nac_pep_scaled_sharing_enabled = 1") != string::npos) { + if (command_output.find("nac_pep_identity_next_enabled = 1") != string::npos) { return string("true"); } return string("false"); } Maybe -getIDAGaiaPackages(const string &command_output) +getRequiredNanoServices(const string &command_output) { - string result = "idaSaml_gaia;idaIdn_gaia;idaIdnBg_gaia;"; - if (command_output.find("nac_pep_scaled_sharing_enabled = 1") != string::npos) { - result += "agentIntelligenceService_gaia;"; - } - return result; + return command_output; } Maybe @@ -191,26 +187,44 @@ getMgmtObjAttr(shared_ptr file_stream, const string &attr) } Maybe -getMgmtObjUid(shared_ptr file_stream) +getMgmtObjUid(const string &command_output) { + if (!command_output.empty()) { + return command_output; + } + + static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C"; + auto file_stream = std::make_shared(obj_path); + if (!file_stream->is_open()) { + return genError("Failed to open the object file"); + } return getMgmtObjAttr(file_stream, "uuid "); } Maybe -getMgmtObjName(shared_ptr file_stream) +getMgmtObjName(const string &command_output) { + if (!command_output.empty()) { + return command_output; + } + + static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C"; + auto file_stream = std::make_shared(obj_path); + if (!file_stream->is_open()) { + return genError("Failed to open the object file"); + } return getMgmtObjAttr(file_stream, "name "); } Maybe -getGWHardware(const string &command_output) +getHardware(const string &command_output) { if (!command_output.empty()) { if (command_output == "software") return string("Open server"); if (command_output == "Maestro Gateway") return string("Maestro"); return string(command_output); } - return genError("GW Hardware was not found"); + return genError("Hardware was not found"); } Maybe diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index 51b1118..3558abe 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -42,13 +42,29 @@ SHELL_PRE_CMD("gunzip local.cfg", "gunzip -c $FWDIR/state/local/FW1/local.cfg.gz #ifdef SHELL_CMD_HANDLER #if defined(gaia) || defined(smb) SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType) +SHELL_CMD_HANDLER( + "cpProductIntegrationMgmtObjectUid", + "mgmt_cli --format json -r true show-session | jq -r '.[\"connected-server\"].uid'", + getMgmtObjUid +) SHELL_CMD_HANDLER("prerequisitesForHorizonTelemetry", "FS_PATH=; [ -f ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log ] " "&& head -1 ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log || echo ''", checkIsInstallHorizonTelemetrySucceeded) -SHELL_CMD_HANDLER("QUID", "[ -d /opt/CPquid ] " +SHELL_CMD_HANDLER("GLOBAL_QUID", "[ -d /opt/CPquid ] " "&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo ''", getQUID) +SHELL_CMD_HANDLER("QUID", "FS_PATH=;" + "VS_ID=$(echo \"${FS_PATH}\" | grep -o -E \"vs[0-9]+\" | grep -o -E \"[0-9]+\");" + "[ -z \"${VS_ID}\" ] && " + "(python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo '');" + "[ -n \"${VS_ID}\" ] && " + "(sed \"s|###VS_ID###|${VS_ID}|g\" /opt/CPotelcol/quid_api/get_vs_quid.json" + " > /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID}); " + "[ -n \"${VS_ID}\" ] && [ -f /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} ] && " + "(python3 /opt/CPquid/Quid_Api.py -i " + "/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');", + getQUID) SHELL_CMD_HANDLER("SMO_QUID", "[ -d /opt/CPquid ] " "&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message || echo ''", getQUID) @@ -102,8 +118,13 @@ SHELL_CMD_HANDLER( SHELL_CMD_HANDLER("hasSAMLSupportedBlade", "enabled_blades", checkSAMLSupportedBlade) SHELL_CMD_HANDLER("hasIDABlade", "enabled_blades", checkIDABlade) SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status nac", checkSAMLPortal) -SHELL_CMD_HANDLER("hasIdaIdnEnabled", "fw ctl get int nac_pep_scaled_sharing_enabled", checkPepIdaIdnStatus) -SHELL_CMD_HANDLER("requiredNanoServices", "fw ctl get int nac_pep_scaled_sharing_enabled", getIDAGaiaPackages) +SHELL_CMD_HANDLER("hasIdaIdnEnabled", "fw ctl get int nac_pep_identity_next_enabled", checkPepIdaIdnStatus) +SHELL_CMD_HANDLER("requiredNanoServices", "echo 'idaSaml_gaia;idaIdn_gaia;'", getRequiredNanoServices) +SHELL_CMD_HANDLER( + "cpProductIntegrationMgmtObjectName", + "mgmt_cli --format json -r true show-session | jq -r '.[\"connected-server\"].name'", + getMgmtObjName +) SHELL_CMD_HANDLER( "cpProductIntegrationMgmtParentObjectName", "cat $FWDIR/database/myself_objects.C " @@ -118,8 +139,8 @@ SHELL_CMD_HANDLER( ) SHELL_CMD_HANDLER( "Hardware", - "cat $FWDIR/database/myself_objects.C | awk -F '[:()]' '/:appliance_type/ {print $3}' | head -n 1", - getGWHardware + "cat $FWDIR/database/myself_objects.C | awk -F '[:()]' '/:appliance_type/ {print $3}' | head -n 1 | sed 's/\"//g'", + getHardware ) SHELL_CMD_HANDLER( "Application Control", @@ -219,6 +240,7 @@ SHELL_CMD_HANDLER( SHELL_CMD_OUTPUT("kernel_version", "uname -r") SHELL_CMD_OUTPUT("helloWorld", "cat /tmp/agentHelloWorld 2>/dev/null") +SHELL_CMD_OUTPUT("report_timestamp", "date -u +\%s") #endif // SHELL_CMD_OUTPUT @@ -227,17 +249,11 @@ SHELL_CMD_OUTPUT("helloWorld", "cat /tmp/agentHelloWorld 2>/dev/null") #ifdef FILE_CONTENT_HANDLER #if defined(gaia) - FILE_CONTENT_HANDLER( "hasIdpConfigured", (getenv("SAMLPORTAL_HOME") ? string(getenv("SAMLPORTAL_HOME")) : "") + "/phpincs/spPortal/idpPolicy.xml", checkIDP ) -FILE_CONTENT_HANDLER( - "cpProductIntegrationMgmtObjectName", - (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C", - getMgmtObjName -) #endif //gaia #if defined(alpine) @@ -245,11 +261,6 @@ FILE_CONTENT_HANDLER("alpine_tag", "/usr/share/build/cp-alpine-tag", getCPAlpine #endif // alpine #if defined(gaia) || defined(smb) FILE_CONTENT_HANDLER("os_release", "/etc/cp-release", getOsRelease) -FILE_CONTENT_HANDLER( - "cpProductIntegrationMgmtObjectUid", - (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C", - getMgmtObjUid -) #else // !(gaia || smb) FILE_CONTENT_HANDLER("os_release", "/etc/os-release", getOsRelease) #endif // gaia || smb diff --git a/components/security_apps/orchestration/health_check/health_check.cc b/components/security_apps/orchestration/health_check/health_check.cc index 5d0918b..a7e55fa 100755 --- a/components/security_apps/orchestration/health_check/health_check.cc +++ b/components/security_apps/orchestration/health_check/health_check.cc @@ -40,6 +40,8 @@ public: i_mainloop = Singleton::Consume::by(); i_socket = Singleton::Consume::by(); i_orchestration_status = Singleton::Consume::by(); + i_service_controller = Singleton::Consume::by(); + initConfig(); initServerSocket(); @@ -270,18 +272,17 @@ private: } if (NGEN::Filesystem::exists(rpm_full_load_path)) { - dbgTrace(D_HEALTH_CHECK) << rpm_full_load_path << " exists, returning healthy status"; - return HealthCheckStatus::HEALTHY; + dbgTrace(D_HEALTH_CHECK) << "RPM is fully loaded"; + return i_service_controller->getServicesPolicyStatus() + ? HealthCheckStatus::HEALTHY + : HealthCheckStatus::UNHEALTHY; } - if (NGEN::Filesystem::exists(rpm_partial_load_path)) { - dbgTrace(D_HEALTH_CHECK) << rpm_partial_load_path << " exists, returning degraded status"; - return HealthCheckStatus::DEGRADED; - } - - if (!NGEN::Filesystem::exists(first_rpm_policy_load_path)) { - dbgTrace(D_HEALTH_CHECK) << "Could not load latest RPM policy, returning degraded status"; - return HealthCheckStatus::DEGRADED; + if (NGEN::Filesystem::exists(rpm_partial_load_path) || !NGEN::Filesystem::exists(first_rpm_policy_load_path)) { + dbgTrace(D_HEALTH_CHECK) << "RPM is partially loaded"; + return i_service_controller->getServicesPolicyStatus() + ? HealthCheckStatus::DEGRADED + : HealthCheckStatus::UNHEALTHY; } dbgTrace(D_HEALTH_CHECK) << "RPM is not loaded, returning unhealthy status"; @@ -442,6 +443,7 @@ private: I_Socket *i_socket = nullptr; I_Health_Check_Manager *i_health_check_manager = nullptr; I_OrchestrationStatus *i_orchestration_status = nullptr; + I_ServiceController *i_service_controller = nullptr; }; HealthChecker::HealthChecker() : Component("HealthChecker"), pimpl(make_unique()) {} diff --git a/components/security_apps/orchestration/health_check/health_check_ut/health_check_ut.cc b/components/security_apps/orchestration/health_check/health_check_ut/health_check_ut.cc index 0ad4bcd..91b8d32 100755 --- a/components/security_apps/orchestration/health_check/health_check_ut/health_check_ut.cc +++ b/components/security_apps/orchestration/health_check/health_check_ut/health_check_ut.cc @@ -9,6 +9,7 @@ #include "mock/mock_shell_cmd.h" #include "mock/mock_orchestration_status.h" #include "health_check_manager.h" +#include "mock/mock_service_controller.h" #include "config.h" #include "config_component.h" @@ -76,6 +77,7 @@ public: I_MainLoop::Routine handle_probe_routine; HealthCheckManager health_check_manager; I_Health_Check_Manager *i_health_check_manager; + StrictMock mock_service_controller; }; TEST_F(HealthCheckerTest, empty) @@ -342,3 +344,58 @@ TEST_F(HealthCheckerTest, FailedHealthCheck) connection_handler_routine(); setConfiguration(false, "Health Check", "Probe enabled"); } + +TEST_F(HealthCheckerTest, StandaloneHealthCheck) +{ + setenv("DOCKER_RPM_ENABLED", "true", 1); + + string ip = "1.2.3.4"; + setConfiguration(ip, "Health Check", "Probe IP"); + uint port = 11600; + setConfiguration(port, "Health Check", "Probe port"); + + NGEN::Filesystem::touchFile("/tmp/wd.all_running"); + NGEN::Filesystem::touchFile("/tmp/rpm_full_load"); + + auto on_exit = make_scope_exit( + []() { + NGEN::Filesystem::deleteFile("/tmp/wd.all_running"); + NGEN::Filesystem::deleteFile("/tmp/rpm_full_load"); + } + ); + + const string policy_version = "1"; + EXPECT_CALL(mock_orchestration_status, getPolicyVersion()).WillRepeatedly(ReturnRef(policy_version)); + EXPECT_CALL(mock_service_controller, getServicesPolicyStatus()).WillRepeatedly(Return(true)); + + EXPECT_CALL( + mock_mainloop, + addOneTimeRoutine(I_MainLoop::RoutineType::System, _, _, false) + ).WillOnce(DoAll(SaveArg<1>(&handle_probe_routine), Return(0))); + + EXPECT_CALL( + mock_socket, + genSocket(I_Socket::SocketType::TCP, false, true, _) + ).WillRepeatedly(Return(1)); + + EXPECT_CALL( + mock_mainloop, + addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true) + ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0))); + + EXPECT_CALL( + mock_mainloop, + addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Health check probe connection handler", true) + ).WillOnce(DoAll(SaveArg<1>(&connection_handler_routine), Return(0))); + + int socket = 1; + EXPECT_CALL(mock_socket, acceptSocket(1, false, ip)).WillOnce(Return(socket)); + EXPECT_CALL(mock_mainloop, getCurrentRoutineId()).WillRepeatedly(Return(0)); + EXPECT_CALL(mock_socket, receiveData(_, 1, false)).WillOnce(Return(vector())); + EXPECT_CALL(mock_socket, writeData(_, response_buffer)).WillOnce(Return(true)); + EXPECT_CALL(mock_socket, closeSocket(socket)).Times(2); + health_checker.init(); + handle_probe_routine(); + connection_handler_routine(); + connection_handler_routine(); +} diff --git a/components/security_apps/orchestration/health_check_manager/health_check_manager.cc b/components/security_apps/orchestration/health_check_manager/health_check_manager.cc index 7439acd..a683258 100644 --- a/components/security_apps/orchestration/health_check_manager/health_check_manager.cc +++ b/components/security_apps/orchestration/health_check_manager/health_check_manager.cc @@ -266,7 +266,9 @@ private: case OrchestrationStatusFieldType::COUNT : return "Count"; } - dbgAssert(false) << "Trying to convert unknown orchestration status field to string."; + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "orchestration health") + << "Trying to convert unknown orchestration status field to string."; return ""; } @@ -280,7 +282,9 @@ private: case UpdatesProcessResult::DEGRADED : return HealthCheckStatus::DEGRADED; } - dbgAssert(false) << "Trying to convert unknown update process result field to health check status."; + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "orchestration health") + << "Trying to convert unknown update process result field to health check status."; return HealthCheckStatus::IGNORED; } diff --git a/components/security_apps/orchestration/include/mock/mock_service_controller.h b/components/security_apps/orchestration/include/mock/mock_service_controller.h index 0495075..b226fc1 100755 --- a/components/security_apps/orchestration/include/mock/mock_service_controller.h +++ b/components/security_apps/orchestration/include/mock/mock_service_controller.h @@ -40,6 +40,8 @@ public: MOCK_CONST_METHOD0(getPolicyVersions, const std::string &()); + MOCK_CONST_METHOD0(getServicesPolicyStatus, bool()); + MOCK_METHOD6( updateServiceConfiguration, Maybe( diff --git a/components/security_apps/orchestration/modules/orchestration_status.cc b/components/security_apps/orchestration/modules/orchestration_status.cc index 22c440d..cbcdbe0 100755 --- a/components/security_apps/orchestration/modules/orchestration_status.cc +++ b/components/security_apps/orchestration/modules/orchestration_status.cc @@ -429,14 +429,16 @@ public: status.insertServiceSetting(service_name, path); return; case OrchestrationStatusConfigType::MANIFEST: - dbgAssert(false) << "Manifest is not a service configuration file type"; + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "sesrvice configuration") + << "Manifest is not a service configuration file type"; break; case OrchestrationStatusConfigType::DATA: return; case OrchestrationStatusConfigType::COUNT: break; } - dbgAssert(false) << "Unknown configuration file type"; + dbgAssert(false) << AlertInfo(AlertTeam::CORE, "sesrvice configuration") << "Unknown configuration file type"; } void diff --git a/components/security_apps/orchestration/modules/url_parser.cc b/components/security_apps/orchestration/modules/url_parser.cc index 99e6bea..80e1a02 100755 --- a/components/security_apps/orchestration/modules/url_parser.cc +++ b/components/security_apps/orchestration/modules/url_parser.cc @@ -43,7 +43,10 @@ operator<<(ostream &os, const URLProtocol &protocol) return os << "file://"; } default: { - dbgAssert(false) << "Unsupported protocol " << static_cast(protocol); + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "fog communication") + << "Unsupported protocol " + << static_cast(protocol); return os; } } @@ -91,7 +94,10 @@ URLParser::parseURL(const string &url) return; } default: { - dbgAssert(false) << "URL protocol is not supported. Protocol: " << static_cast(protocol); + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "fog communication") + << "URL protocol is not supported. Protocol: " + << static_cast(protocol); return; } } diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 16da91c..45e53cc 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -1631,6 +1631,7 @@ private: string server_name = getAttribute("registered-server", "registered_server"); auto server = TagAndEnumManagement::convertStringToTag(server_name); + if (server_name == "'SWAG'") server = Tags::WEB_SERVER_SWAG; if (server.ok()) tags.insert(*server); if (getAttribute("no-setting", "CROWDSEC_ENABLED") == "true") tags.insert(Tags::CROWDSEC); @@ -1652,6 +1653,8 @@ private: tags ); + if (server_name != "") registration_report.addToOrigin(LogField("eventCategory", server_name)); + auto email = getAttribute("email-address", "user_email"); if (email != "") registration_report << LogField("userDefinedId", email); @@ -1694,9 +1697,11 @@ private: auto temp_ext = getConfigurationWithDefault("_temp", "orchestration", "Temp file extension"); dbgAssert(i_orchestration_tools->doesFileExist(backup_installation_file)) + << AlertInfo(AlertTeam::CORE, "orchestration backup") << "There is no backup installation package"; dbgAssert(i_orchestration_tools->copyFile(backup_installation_file, current_installation_file)) + << AlertInfo(AlertTeam::CORE, "orchestration backup") << "Failed to copy backup installation package"; // Copy the backup manifest file to the default manifest file path. @@ -1713,8 +1718,10 @@ private: auto package_handler = Singleton::Consume::by(); // Install the backup orchestration service installation package. dbgAssert(package_handler->preInstallPackage(service_name, current_installation_file)) + << AlertInfo(AlertTeam::CORE, "orchestration backup") << "Failed to restore from backup, pre install test failed"; dbgAssert(package_handler->installPackage(service_name, current_installation_file, true)) + << AlertInfo(AlertTeam::CORE, "orchestration backup") << "Failed to restore from backup, installation failed"; } // LCOV_EXCL_STOP diff --git a/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc b/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc index 4c88c5c..3efee4e 100755 --- a/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc +++ b/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc @@ -386,7 +386,10 @@ OrchestrationTools::Impl::calculateChecksum(Package::ChecksumTypes checksum_type return genError("Error while reading file " + path + ", " + e.what()); } - dbgAssert(false) << "Checksum type is not supported. Checksum type: " << static_cast(checksum_type); + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "service configuration") + << "Checksum type is not supported. Checksum type: " + << static_cast(checksum_type); return genError("Unsupported checksum type"); } diff --git a/components/security_apps/orchestration/package_handler/package_handler.cc b/components/security_apps/orchestration/package_handler/package_handler.cc index 3679660..6071bd8 100755 --- a/components/security_apps/orchestration/package_handler/package_handler.cc +++ b/components/security_apps/orchestration/package_handler/package_handler.cc @@ -141,7 +141,10 @@ packageHandlerActionsToString(PackageHandlerActions action) } } - dbgAssert(false) << "Package handler action is not supported. Action: " << static_cast(action); + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "service configuration") + << "Package handler action is not supported. Action: " + << static_cast(action); return string(); } diff --git a/components/security_apps/orchestration/service_controller/service_controller.cc b/components/security_apps/orchestration/service_controller/service_controller.cc index 440c116..200af8e 100755 --- a/components/security_apps/orchestration/service_controller/service_controller.cc +++ b/components/security_apps/orchestration/service_controller/service_controller.cc @@ -71,7 +71,7 @@ public: UpdatesConfigType::GENERAL, UpdatesFailureReason::SERVISE_CONFIGURATION, string(service_name.get() + ", ID: " + to_string(id.get())), - (error_message.isActive() ? " Error: " + error_message.get() : "") + (error_message.isActive() ? " " + error_message.get() : "") ).notify(); dbgError(D_SERVICE_CONTROLLER) << "Request for service reconfiguration failed to complete. ID: " @@ -327,6 +327,8 @@ public: set && moveChangedPolicies() override; + bool getServicesPolicyStatus() const override; + private: void cleanUpVirtualFiles(); @@ -365,6 +367,7 @@ private: map services_reconf_ids; string filesystem_prefix; bool is_multi_tenant_env = false; + bool total_services_status = false; set changed_policy_files; ServiceDetails orchestration_service_details; @@ -459,6 +462,12 @@ ServiceController::Impl::moveChangedPolicies() return move(changed_policy_files); } +bool +ServiceController::Impl::getServicesPolicyStatus() const +{ + return total_services_status; +} + void ServiceController::Impl::init() { @@ -960,6 +969,8 @@ ServiceController::Impl::sendSignalForServices( const string &policy_version_to_update) { dbgFlow(D_SERVICE_CONTROLLER) << "Policy version to update: " << policy_version_to_update; + + total_services_status = false; for (auto &service_id : nano_services_to_update) { auto nano_service = registered_services.find(service_id); if (nano_service == registered_services.end()) { @@ -1002,6 +1013,7 @@ ServiceController::Impl::sendSignalForServices( << "The reconfiguration was successfully completed for all the services"; services_reconf_status.clear(); services_reconf_names.clear(); + total_services_status = true; return Maybe(); } case ReconfStatus::IN_PROGRESS: { diff --git a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc index cb1e4f0..a2da65d 100755 --- a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc +++ b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc @@ -790,6 +790,7 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations) EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path).ok()); EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value); + EXPECT_EQ(i_service_controller->getServicesPolicyStatus(), true); } TEST_F(ServiceControllerTest, backup) diff --git a/components/security_apps/orchestration/update_communication/fog_authenticator.cc b/components/security_apps/orchestration/update_communication/fog_authenticator.cc index c28ba86..e50601c 100755 --- a/components/security_apps/orchestration/update_communication/fog_authenticator.cc +++ b/components/security_apps/orchestration/update_communication/fog_authenticator.cc @@ -467,7 +467,10 @@ getDeplymentType() case EnvType::COUNT: break; } - dbgAssert(false) << "Failed to get a legitimate deplyment type: " << static_cast(deplyment_type); + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "fog communication") + << "Failed to get a legitimate deplyment type: " + << static_cast(deplyment_type); return "Embedded"; } diff --git a/components/security_apps/orchestration/updates_process_reporter/updates_process_reporter.cc b/components/security_apps/orchestration/updates_process_reporter/updates_process_reporter.cc index 1a1be31..b68aa66 100644 --- a/components/security_apps/orchestration/updates_process_reporter/updates_process_reporter.cc +++ b/components/security_apps/orchestration/updates_process_reporter/updates_process_reporter.cc @@ -18,6 +18,7 @@ #include "debug.h" #include "log_generator.h" +#include "service_health_update_event.h" using namespace std; @@ -34,6 +35,7 @@ UpdatesProcessReporter::upon(const UpdatesProcessEvent &event) if (event.getResult() == UpdatesProcessResult::SUCCESS && reports.empty()) { dbgTrace(D_UPDATES_PROCESS_REPORTER) << "Update proccess finished successfully"; report_failure_count_map.erase(version); + ServiceHealthUpdateEvent().notify(); return; } if (report_failure_count_map.find(version) == report_failure_count_map.end()) { @@ -62,6 +64,7 @@ UpdatesProcessReporter::upon(const UpdatesProcessEvent &event) reports.emplace_back( UpdatesProcessReport(event.getResult(), event.getType(), event.getReason(), event.parseDescription()) ); + ServiceHealthUpdateEvent(convertUpdatesConfigTypeToStr(event.getType()), event.parseDescription()).notify(); } void diff --git a/components/security_apps/rate_limit/rate_limit.cc b/components/security_apps/rate_limit/rate_limit.cc index 1c8cb19..8d5db5b 100755 --- a/components/security_apps/rate_limit/rate_limit.cc +++ b/components/security_apps/rate_limit/rate_limit.cc @@ -211,6 +211,8 @@ public: for (const auto &rule : rate_limit_config.getRateLimitRules()) { string full_rule_uri = application_uri + rule.getRateLimitUri(); + transform(full_rule_uri.begin(), full_rule_uri.end(), + full_rule_uri.begin(), [](unsigned char c) { return std::tolower(c); }); int full_rule_uri_length = full_rule_uri.length(); dbgTrace(D_RATE_LIMIT) diff --git a/components/security_apps/waap/waap_clib/DeepParser.cc b/components/security_apps/waap/waap_clib/DeepParser.cc index 3735778..9a10533 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.cc +++ b/components/security_apps/waap/waap_clib/DeepParser.cc @@ -390,6 +390,38 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f } } + if (flags & BUFFERED_RECEIVER_F_FIRST && offset < 0 && valueStats.hasPercent && + m_key.first().size() == 6 && m_key.first() == "cookie") { + dbgTrace(D_WAAP_DEEP_PARSER) + << "1st pass of createInternalParser() failed. " + << "Will try to decode percent-encoded data and repeate search for parser"; + orig_val.erase(unquote_plus(orig_val.begin(), orig_val.end()), orig_val.end()); + cur_val_html_escaped = orig_val; + cur_val_html_escaped.erase( + escape_html(cur_val_html_escaped.begin(), cur_val_html_escaped.end()), cur_val_html_escaped.end() + ); + offset = createInternalParser( + k, + k_len, + orig_val, + valueStats, + isBodyPayload, + isRefererPayload, + isRefererParamPayload, + isUrlPayload, + isUrlParamPayload, + flags, + parser_depth, + base64BinaryFileType + ); + if (offset >= 0) { + dbgTrace(D_WAAP_DEEP_PARSER) << "2nd pass of createInternalParser succeeded. Update values and proceed"; + ValueStatsAnalyzer valueStatsUpdated(cur_val_html_escaped); + cur_val.erase(unquote_plus(cur_val.begin(), cur_val.end()), cur_val.end()); + Waap::Util::decodeUtf16Value(valueStatsUpdated, cur_val); + } + } + // If there's a parser in parsers stack, push the value to the top parser if (!m_parsersDeque.empty() && offset >= 0 @@ -1183,6 +1215,7 @@ DeepParser::createInternalParser( offset = 0; } } + bool isCockiePapameter = m_key.depth() == 2 && m_key.first().size() == 6 && m_key.first() == "cookie"; if (offset < 0) { if (isPipesType) { dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse pipes, positional: " << isKeyValDelimited; @@ -1279,7 +1312,7 @@ DeepParser::createInternalParser( ); } else if (!Waap::Util::testUrlBareUtf8Evasion(cur_val)) { dbgTrace(D_WAAP_DEEP_PARSER) << "!Waap::Util::testUrlBareUtf8Evasion(cur_val)"; - if (!valueStats.hasSpace + if ((!valueStats.hasSpace || (valueStats.hasSpace && isCockiePapameter)) && valueStats.hasCharAmpersand && valueStats.hasTwoCharsEqual && !isBinaryData() @@ -1305,7 +1338,7 @@ DeepParser::createInternalParser( } } else if (!Waap::Util::testUrlBareUtf8Evasion(cur_val)) { dbgTrace(D_WAAP_DEEP_PARSER) << "!Waap::Util::testUrlBareUtf8Evasion(cur_val)"; - if (!valueStats.hasSpace + if ((!valueStats.hasSpace || (valueStats.hasSpace && isCockiePapameter)) && valueStats.hasCharAmpersand && valueStats.hasTwoCharsEqual && !isBinaryData() diff --git a/components/security_apps/waap/waap_clib/ParserUrlEncode.cc b/components/security_apps/waap/waap_clib/ParserUrlEncode.cc index 506ef1c..4b77428 100755 --- a/components/security_apps/waap/waap_clib/ParserUrlEncode.cc +++ b/components/security_apps/waap/waap_clib/ParserUrlEncode.cc @@ -103,6 +103,9 @@ ParserUrlEncode::push(const char *buf, size_t len) } case s_key_start: { dbgTrace(D_WAAP_PARSER_URLENCODE) << "ParserUrlEncode::push(): s_key_start"; + if (isspace(c)){ + break; + } mark = i; m_state = s_key; @@ -112,12 +115,6 @@ ParserUrlEncode::push(const char *buf, size_t len) case s_key: { dbgTrace(D_WAAP_PARSER_URLENCODE) << "ParserUrlEncode::push(): s_key"; - // skip leading spaces in the key - if (isspace(c)) { - m_state = s_key_start; // skip the space character without including it in the output - break; - } - if (c == '%' && should_decode_percent) { if (i - mark > 0) { if (m_receiver.onKey(buf + mark, i - mark) != 0) { diff --git a/components/security_apps/waap/waap_clib/WaapOverride.h b/components/security_apps/waap/waap_clib/WaapOverride.h index 2d5d3bd..80b7174 100755 --- a/components/security_apps/waap/waap_clib/WaapOverride.h +++ b/components/security_apps/waap/waap_clib/WaapOverride.h @@ -48,7 +48,7 @@ public: if (m_tag != "sourceip" && m_tag != "sourceidentifier" && m_tag != "url" && m_tag != "hostname" && m_tag != "keyword" && m_tag != "paramname" && m_tag != "paramvalue" && m_tag != "paramlocation" && - m_tag != "responsebody" && m_tag != "headername" && m_tag != "headervalue" ) { + m_tag != "responsebody" && m_tag != "headername" && m_tag != "headervalue" && m_tag != "method") { m_isValid = false; dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag; } diff --git a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc index 0bf93e0..0f2b5d6 100755 --- a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc +++ b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc @@ -51,7 +51,10 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& std::string tagLower = tag; std::transform(tagLower.begin(), tagLower.end(), tagLower.begin(), ::tolower); try { - if (tagLower == "url") { + if (tagLower == "method") { + return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getMethod().c_str(), what, rx); + } + else if (tagLower == "url") { return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUriStr().c_str(), what, rx); } else if (tagLower == "hostname") { diff --git a/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc b/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc index c84f865..7e94df0 100755 --- a/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc +++ b/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc @@ -95,7 +95,9 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val) canSplitPipe(true), hasSpace(false), isUrlEncoded(false), - hasCharLess(false) + hasCharLess(false), + hasDoubleQuote(false), + hasPercent(false) { unsigned int zerosSeq[2] = {0}; bool lastNul = false; // whether last processed character was ASCII NUL @@ -146,6 +148,9 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val) case '\"': hasDoubleQuote = true; break; + case '%': + hasPercent = true; + break; } if (isspace(ch)) { @@ -270,4 +275,6 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val) textual +=(hasCharLess ? "true" : "false"); textual.append("\nhasDoubleQuote = "); textual +=(hasDoubleQuote ? "true" : "false"); + textual.append("\nhasPercent = "); + textual +=(hasPercent ? "true" : "false"); } diff --git a/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.h b/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.h index 6b67cdd..e88cbe5 100755 --- a/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.h +++ b/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.h @@ -36,6 +36,7 @@ struct ValueStatsAnalyzer bool isUrlEncoded; bool hasCharLess; bool hasDoubleQuote; + bool hasPercent; std::string textual; }; diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.cc b/components/security_apps/waap/waap_clib/Waf2Engine.cc index bb5b40d..0a8a6a7 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.cc +++ b/components/security_apps/waap/waap_clib/Waf2Engine.cc @@ -867,7 +867,7 @@ void Waf2Transaction::parseCookie(const char* value, int value_len) if (value_len > 0) { dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] scanning the cookie value"; m_deepParser.m_key.push("cookie", 6); - ParserUrlEncode cookieValueParser(m_deepParserReceiver, 0, ';'); + ParserUrlEncode cookieValueParser(m_deepParserReceiver, 0, ';', false); cookieValueParser.push(value, value_len); cookieValueParser.finish(); m_deepParser.m_key.pop("cookie"); @@ -1077,6 +1077,9 @@ void Waf2Transaction::add_request_hdr(const char* name, int name_len, const char std::string header_name(name, name_len); boost::algorithm::to_lower(header_name); hdrs_map[header_name] = std::string(value, value_len); + if (header_name == "host") { + m_hostStr = hdrs_map[header_name]; + } } void Waf2Transaction::end_request_hdrs() { diff --git a/components/security_apps/waap/waap_component_impl.cc b/components/security_apps/waap/waap_component_impl.cc index f333c15..5549977 100755 --- a/components/security_apps/waap/waap_component_impl.cc +++ b/components/security_apps/waap/waap_component_impl.cc @@ -117,7 +117,7 @@ WaapComponent::Impl::fini() std::string WaapComponent::Impl::getListenerName() const { - return "waap application"; + return WAAP_APPLICATION_NAME; } // Start request (called before headers arrive). However, the method and URL path is known at this stage. diff --git a/components/utils/CMakeLists.txt b/components/utils/CMakeLists.txt index 1873694..3e6baa7 100644 --- a/components/utils/CMakeLists.txt +++ b/components/utils/CMakeLists.txt @@ -4,3 +4,4 @@ add_subdirectory(http_transaction_data) add_subdirectory(ip_utilities) add_subdirectory(keywords) add_subdirectory(pm) +add_subdirectory(service_health_status) diff --git a/components/utils/generic_rulebase/triggers_config.cc b/components/utils/generic_rulebase/triggers_config.cc index 2ad465f..62df576 100644 --- a/components/utils/generic_rulebase/triggers_config.cc +++ b/components/utils/generic_rulebase/triggers_config.cc @@ -173,7 +173,7 @@ LogTriggerConf::load(cereal::JSONInputArchive& archive_in) setTriggersFlag("webUrlQuery", archive_in, WebLogFields::webUrlQuery, log_web_fields); setTriggersFlag("logToAgent", archive_in, ReportIS::StreamType::JSON_LOG_FILE, active_streams); setTriggersFlag("logToCloud", archive_in, ReportIS::StreamType::JSON_FOG, active_streams); - setTriggersFlag("logToContainerService", archive_in, ReportIS::StreamType::JSON_CONTAINER_SVC, active_streams); + setTriggersFlag("logTolocalTuning", archive_in, ReportIS::StreamType::JSON_CONTAINER_SVC, active_streams); setTriggersFlag("logToSyslog", archive_in, ReportIS::StreamType::SYSLOG, active_streams); setTriggersFlag("logToCef", archive_in, ReportIS::StreamType::CEF, active_streams); setTriggersFlag("acAllow", archive_in, SecurityType::AccessControl, should_log_on_detect); diff --git a/components/utils/ip_utilities/ip_utilities.cc b/components/utils/ip_utilities/ip_utilities.cc index c467516..2b7d012 100644 --- a/components/utils/ip_utilities/ip_utilities.cc +++ b/components/utils/ip_utilities/ip_utilities.cc @@ -301,7 +301,7 @@ ConvertToIpAddress(const IPAddr &addr) { break; } default: - dbgAssert(false) << "Unsupported IP type"; + dbgAssert(false) << AlertInfo(AlertTeam::CORE, "ip utilities") << "Unsupported IP type"; } return address; } diff --git a/components/utils/keywords/data_keyword.cc b/components/utils/keywords/data_keyword.cc index 3681f91..b7e6802 100644 --- a/components/utils/keywords/data_keyword.cc +++ b/components/utils/keywords/data_keyword.cc @@ -74,7 +74,9 @@ private: uint moveOnNoMatch(uint offset_from_end, unsigned char first_unmatched_byte) const { - dbgAssert(shift.size() > offset_from_end) << "Shift table of the 'data' keyword is shorter than the offset"; + dbgAssert(shift.size() > offset_from_end) + << AlertInfo(AlertTeam::CORE, "keywords") + << "Shift table of the 'data' keyword is shorter than the offset"; uint skip_size; if (skip[first_unmatched_byte]>offset_from_end) { @@ -350,7 +352,9 @@ DataKeyword::bytesMatched(const Buffer &buf, uint offset) const MatchStatus DataKeyword::isMatch(const I_KeywordRuntimeState *prev) const { - dbgAssert(pattern.size()>0) << "Trying to run on an uninitialized keyword data"; + dbgAssert(pattern.size()>0) + << AlertInfo(AlertTeam::CORE, "keywords") + << "Trying to run on an uninitialized keyword data"; dbgDebug(D_KEYWORD) << "Searching for " << dumpHex(pattern); diff --git a/components/utils/keywords/jump_keyword.cc b/components/utils/keywords/jump_keyword.cc index 490040c..7df7db0 100644 --- a/components/utils/keywords/jump_keyword.cc +++ b/components/utils/keywords/jump_keyword.cc @@ -143,7 +143,7 @@ jumpKeyword::getStartOffset(uint buf_size, const I_KeywordRuntimeState *prev) co return prev->getOffset(ctx); } } - dbgAssert(false) << "Invalid jumping 'from' parameter"; + dbgAssert(false) << AlertInfo(AlertTeam::CORE, "keywords") << "Invalid jumping 'from' parameter"; return 0; } diff --git a/components/utils/keywords/keywords_rule.cc b/components/utils/keywords/keywords_rule.cc index c75a891..4e7bbc1 100644 --- a/components/utils/keywords/keywords_rule.cc +++ b/components/utils/keywords/keywords_rule.cc @@ -90,7 +90,7 @@ SentinelRuntimeState::getOffset(const std::string &) const uint SentinelRuntimeState::getVariable(uint var_id) const { - dbgAssert(false) << "Could not find the variable ID: " << var_id; + dbgAssert(false) << AlertInfo(AlertTeam::CORE, "keywords") << "Could not find the variable ID: " << var_id; return 0; } // LCOV_EXCL_STOP diff --git a/components/utils/keywords/pcre_keyword.cc b/components/utils/keywords/pcre_keyword.cc index c9ba9e3..6fed5e7 100644 --- a/components/utils/keywords/pcre_keyword.cc +++ b/components/utils/keywords/pcre_keyword.cc @@ -297,7 +297,9 @@ PCREKeyword::getStartOffsetAndLength(uint buf_size, const I_KeywordRuntimeState MatchStatus PCREKeyword::isMatch(const I_KeywordRuntimeState *prev) const { - dbgAssert(pcre_machine!=nullptr) << "Trying to run on an uninitialized keyword 'pcre'"; + dbgAssert(pcre_machine!=nullptr) + << AlertInfo(AlertTeam::CORE, "keywords") + << "Trying to run on an uninitialized keyword 'pcre'"; auto part = Singleton::Consume::by()->get(static_cast(ctx)); diff --git a/components/utils/keywords/single_keyword.cc b/components/utils/keywords/single_keyword.cc index a8437c0..a326196 100644 --- a/components/utils/keywords/single_keyword.cc +++ b/components/utils/keywords/single_keyword.cc @@ -250,7 +250,9 @@ ComparisonAttr::operator()(int first_val, int second_val) const return first_val >= second_val; } } - dbgAssert(false) << "ComparisonAttr::operator found an invalid comparison operator"; + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "keywords") + << "ComparisonAttr::operator found an invalid comparison operator"; return false; } diff --git a/components/utils/keywords/stateop_keyword.cc b/components/utils/keywords/stateop_keyword.cc index c6f5f78..4860cb6 100644 --- a/components/utils/keywords/stateop_keyword.cc +++ b/components/utils/keywords/stateop_keyword.cc @@ -131,7 +131,7 @@ StateopKeyword::isMatch(const I_KeywordRuntimeState *prev) const if (table->hasState()) table->getState().removeVariable(var_name); return runNext(prev); } else { - dbgAssert(false) << "Impossible 'stateop' keyword without operation"; + dbgAssert(false) << AlertInfo(AlertTeam::CORE, "keywords") << "Impossible 'stateop' keyword without operation"; } // If there was no matches and the keyword is effected by other keywords, then we know that the rule won't match diff --git a/components/utils/pm/kiss_patterns.cc b/components/utils/pm/kiss_patterns.cc index 0e7516e..80b6eb1 100644 --- a/components/utils/pm/kiss_patterns.cc +++ b/components/utils/pm/kiss_patterns.cc @@ -80,7 +80,7 @@ kiss_pmglob_string_s::kiss_pmglob_string_s(const char *buffer, size_t size, int kiss_pmglob_string_s::kiss_pmglob_string_s(const u_char *buffer, size_t size, int _pattern_id, u_int _flags) { - dbgAssert(buffer && size > 0) << "Illegal arguments"; + dbgAssert(buffer && size > 0) << AlertInfo(AlertTeam::CORE, "pattern matcher") << "Illegal arguments"; buf.resize(size); memcpy(buf.data(), buffer, size); pattern_id = _pattern_id; diff --git a/components/utils/pm/kiss_thin_nfa.cc b/components/utils/pm/kiss_thin_nfa.cc index 45642e6..6b14fa3 100644 --- a/components/utils/pm/kiss_thin_nfa.cc +++ b/components/utils/pm/kiss_thin_nfa.cc @@ -430,7 +430,9 @@ kiss_thin_nfa_exec(KissThinNFA *nfa_h, const Buffer& buf, std::vector &inputs) set PMHook::scanBuf(const Buffer &buf) const { - dbgAssert(handle != nullptr) << "Unusable Pattern Matcher"; + dbgAssert(handle != nullptr) << AlertInfo(AlertTeam::CORE, "pattern matcher") << "Unusable Pattern Matcher"; vector> pm_matches; kiss_thin_nfa_exec(handle.get(), buf, pm_matches); dbgTrace(D_PM) << pm_matches.size() << " raw matches found"; @@ -121,7 +121,7 @@ PMHook::scanBuf(const Buffer &buf) const set> PMHook::scanBufWithOffset(const Buffer &buf) const { - dbgAssert(handle != nullptr) << "Unusable Pattern Matcher"; + dbgAssert(handle != nullptr) << AlertInfo(AlertTeam::CORE, "pattern matcher") << "Unusable Pattern Matcher"; vector> pm_matches; kiss_thin_nfa_exec(handle.get(), buf, pm_matches); @@ -135,7 +135,7 @@ PMHook::scanBufWithOffset(const Buffer &buf) const void PMHook::scanBufWithOffsetLambda(const Buffer &buf, I_PMScan::CBFunction cb) const { - dbgAssert(handle != nullptr) << "Unusable Pattern Matcher"; + dbgAssert(handle != nullptr) << AlertInfo(AlertTeam::CORE, "pattern matcher") << "Unusable Pattern Matcher"; unordered_map match_counts; vector> pm_matches; diff --git a/components/utils/service_health_status/CMakeLists.txt b/components/utils/service_health_status/CMakeLists.txt new file mode 100755 index 0000000..a2a7518 --- /dev/null +++ b/components/utils/service_health_status/CMakeLists.txt @@ -0,0 +1,2 @@ +add_library(service_health_status service_health_status.cc) +add_subdirectory(service_health_status_ut) diff --git a/components/utils/service_health_status/service_health_status.cc b/components/utils/service_health_status/service_health_status.cc new file mode 100644 index 0000000..1170ac8 --- /dev/null +++ b/components/utils/service_health_status/service_health_status.cc @@ -0,0 +1,104 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "service_health_status.h" + +#include +#include + +#include "debug.h" +#include "rest.h" +#include "customized_cereal_map.h" +#include "service_health_update_event.h" + +using namespace std; + +USE_DEBUG_FLAG(D_SERVICE_HEALTH_STATUS); + +class I_ServiceHealthStatusImpl +{ +public: + virtual const map & getErrors() const = 0; + +protected: + virtual ~I_ServiceHealthStatusImpl() {} +}; + +class ServiceHealthStatus::Impl + : + public Singleton::Provide::SelfInterface, + public Listener +{ +public: + void init(); + const map & getErrors() const override { return errors_map; } + void upon(const ServiceHealthUpdateEvent &event) override; + +private: + map errors_map; +}; + +class ServiceHealthStatusRest + : + public ServerRest, + Singleton::Consume +{ + using ErrorsMap = map; + +public: + void + doCall() + { + errors = Singleton::Consume::by()->getErrors(); + healthy = errors.get().empty(); + dbgTrace(D_SERVICE_HEALTH_STATUS) + << "Heath status requested. " + << (healthy ? "Service is healthy." : "Service is not healthy."); + } + +private: + S2C_PARAM(bool, healthy); + S2C_PARAM(ErrorsMap, errors); +}; + +void +ServiceHealthStatus::Impl::init() +{ + if (!Singleton::exists()) return; + Singleton::Consume::by()->addRestCall( + RestAction::SHOW, + "health" + ); + registerListener(); +} + +void +ServiceHealthStatus::Impl::upon(const ServiceHealthUpdateEvent &event) +{ + dbgTrace(D_SERVICE_HEALTH_STATUS) + << "Service health update event. Error: " + << event.getComponent() + << " - " + << event.getError(); + + if (event.isHealthyUpdate()) { + errors_map.clear(); + } else { + errors_map[event.getComponent()] = event.getError(); + } +} + +ServiceHealthStatus::ServiceHealthStatus() : Component("ServiceHealthStatus"), pimpl(make_unique()) {} +ServiceHealthStatus::~ServiceHealthStatus() {} + +void ServiceHealthStatus::init() { pimpl->init(); } diff --git a/components/utils/service_health_status/service_health_status_ut/CMakeLists.txt b/components/utils/service_health_status/service_health_status_ut/CMakeLists.txt new file mode 100755 index 0000000..8b96a62 --- /dev/null +++ b/components/utils/service_health_status/service_health_status_ut/CMakeLists.txt @@ -0,0 +1,8 @@ +include_directories(${Boost_INCLUDE_DIRS}) +link_directories(${BOOST_ROOT}/lib) + +add_unit_test( + service_health_status_ut + "service_health_status_ut.cc" + "service_health_status;rest;event_is;connkey;mainloop;encryptor;messaging;-lz;metric;-lboost_regex;-lboost_context;-lcrypto;" +) diff --git a/components/utils/service_health_status/service_health_status_ut/service_health_status_ut.cc b/components/utils/service_health_status/service_health_status_ut/service_health_status_ut.cc new file mode 100644 index 0000000..ed297f1 --- /dev/null +++ b/components/utils/service_health_status/service_health_status_ut/service_health_status_ut.cc @@ -0,0 +1,73 @@ +#include "service_health_status.h" + +#include "cptest.h" +#include "environment.h" +#include "config.h" +#include "config_component.h" +#include "debug.h" +#include "connkey.h" +#include "rest.h" +#include "rest_server.h" +#include "mock/mock_rest_api.h" +#include "service_health_update_event.h" + +using namespace std; +using namespace testing; + +USE_DEBUG_FLAG(D_GEO_DB); + +class HealthCheckStatusTest : public Test +{ +public: + HealthCheckStatusTest() + { + EXPECT_CALL(mock_rest, mockRestCall(RestAction::SHOW, "health", _)) + .WillOnce(WithArg<2>(Invoke(this, &HealthCheckStatusTest::showHealthCheckStatus))); + health_check_status.init(); + } + + bool + showHealthCheckStatus(const unique_ptr &p) + { + show_health_check_status = p->getRest(); + return true; + } + + ::Environment env; + ConfigComponent config; + ServiceHealthStatus health_check_status; + NiceMock mock_rest; + unique_ptr show_health_check_status; +}; + +TEST_F(HealthCheckStatusTest, testHealthCheckStatus) +{ + ServiceHealthUpdateEvent().notify(); + + stringstream ss("{}"); + Maybe maybe_res = show_health_check_status->performRestCall(ss); + EXPECT_TRUE(maybe_res.ok()); + EXPECT_EQ(maybe_res.unpack(), + "{\n" + " \"healthy\": true,\n" + " \"errors\": {}\n" + "}" + ); +} + +TEST_F(HealthCheckStatusTest, testNotHealthyService) +{ + ServiceHealthUpdateEvent("test", "test description").notify(); + + stringstream ss("{}"); + Maybe maybe_res = show_health_check_status->performRestCall(ss); + EXPECT_TRUE(maybe_res.ok()); + EXPECT_EQ(maybe_res.unpack(), + "{\n" + " \"healthy\": false,\n" + " \"errors\": {\n" + " \"test\": \"test description\"\n" + " }\n" + "}" + ); +} diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 8ee29c4..f3a45c7 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -30,6 +30,7 @@ add_subdirectory(version) add_subdirectory(tenant_manager) add_subdirectory(compression) add_subdirectory(attachments) +add_subdirectory(report_messaging) add_library(ngen_core SHARED ".") target_link_libraries( @@ -38,6 +39,7 @@ target_link_libraries( "table;debug_is;shell_cmd;metric;tenant_manager;messaging;encryptor;time_proxy;singleton;mainloop;environment;logging;report;rest" "config;intelligence_is_v2;event_is;memory_consumption;connkey" "instance_awareness;socket_is;agent_details;agent_details_reporter;buffers;cpu;agent_core_utilities" + "report_messaging" -Wl,-no-whole-archive ) diff --git a/core/agent_core_utilities/agent_core_utilities.cc b/core/agent_core_utilities/agent_core_utilities.cc index 4cd74ee..e863f9a 100644 --- a/core/agent_core_utilities/agent_core_utilities.cc +++ b/core/agent_core_utilities/agent_core_utilities.cc @@ -183,6 +183,8 @@ copyFile(const string &src, const string &dest, bool overide_if_exists, mode_t p } dbgTrace(D_INFRA_UTILS) << "Finished attempt to copy file. Res: " << (bytes_copied != -1 ? "Success" : "Error"); + close(source_fd); + close(dest_fd); return bytes_copied != -1; } @@ -261,6 +263,66 @@ touchFile(const string &path) return true; } +bool +copyDirectory(const string &src_dir_path, const string &dst_dir_path) +{ + dbgFlow(D_INFRA_UTILS) + << "Trying to copy directory. Source: " + << src_dir_path + << ", Destination: " + << dst_dir_path; + + if (!exists(src_dir_path) || !isDirectory(src_dir_path)) { + dbgDebug(D_INFRA_UTILS) << "Failed to copy directory. Error: source directory does not exist"; + return false; + } + + if (!exists(dst_dir_path)) { + if (!makeDir(dst_dir_path, 0755)) { + dbgDebug(D_INFRA_UTILS) << "Failed to copy directory. Error: failed to create destination directory"; + return false; + } + } + + if (!isDirectory(dst_dir_path)) { + dbgDebug(D_INFRA_UTILS) << "Failed to copy directory. Error: destination path is not a directory"; + return false; + } + + struct dirent *entry = nullptr; + DIR *directory = opendir(src_dir_path.c_str()); + + if (!directory) { + dbgWarning(D_INFRA_UTILS) << "Fail to open directory. Path: " << src_dir_path; + return false; + } + + while ((entry = readdir(directory))) { + string entry_name = entry->d_name; + static const string curr_dir("."); + static const string parent_dir(".."); + if (entry_name == curr_dir || entry_name == parent_dir) { + dbgTrace(D_INFRA_UTILS) << "Skipping irrelevant directory entries. Entry name: " << entry_name; + continue; + } + + string src_entry_path = src_dir_path + (src_dir_path.back() == '/' ? "" : "/") + entry_name; + string dst_entry_path = dst_dir_path + (dst_dir_path.back() == '/' ? "" : "/") + entry_name; + struct stat statbuf; + if (!stat(src_entry_path.c_str(), &statbuf)) { + if (S_ISDIR(statbuf.st_mode)) { + copyDirectory(src_entry_path, dst_entry_path); + } else { + copyFile(src_entry_path, dst_entry_path, true); + } + } + } + + closedir(directory); + + return true; +} + string convertToHumanReadable(uint64_t size_in_bytes) { diff --git a/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc b/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc index 979f457..bf769b2 100644 --- a/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc +++ b/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc @@ -10,6 +10,8 @@ using namespace std; using namespace testing; +USE_DEBUG_FLAG(D_INFRA_UTILS); + auto contexts = make_pair(std::vector(), false); class AgentCoreUtilUT : public Test @@ -17,6 +19,7 @@ class AgentCoreUtilUT : public Test public: AgentCoreUtilUT() { + Debug::setUnitTestFlag(D_INFRA_UTILS, Debug::DebugLevel::TRACE); ON_CALL(mock_env, getActiveContexts()).WillByDefault(ReturnPointee(&contexts)); } @@ -123,6 +126,59 @@ TEST_F(AgentCoreUtilUT, isDirectoryTest) EXPECT_EQ(NGEN::Filesystem::isDirectory("./test"), true); } +TEST_F(AgentCoreUtilUT, copyDirectoryTest) +{ + string sourceDir = cptestFnameInExeDir("sourceDir1"); + string destDir = cptestFnameInExeDir("destDir1"); + cout << "sourceDir: " << sourceDir << endl; + NGEN::Filesystem::makeDir(sourceDir); + NGEN::Filesystem::makeDir(sourceDir + "/subdir1"); + NGEN::Filesystem::makeDir(sourceDir + "/subdir2"); + NGEN::Filesystem::makeDir(destDir); + + ofstream file_1(sourceDir + "/file1.txt"); + ASSERT_TRUE(file_1.is_open()); + file_1 << "File 1 content"; + file_1.close(); + ofstream file_2(sourceDir + "/subdir1/file2.txt"); + ASSERT_TRUE(file_2.is_open()); + file_2 << "File 2 content"; + file_2.close(); + ofstream file_3(sourceDir + "/subdir2/file3.txt"); + ASSERT_TRUE(file_3.is_open()); + file_3 << "File 3 content"; + file_3.close(); + + ASSERT_TRUE(NGEN::Filesystem::copyDirectory(sourceDir, destDir)); + + EXPECT_TRUE(NGEN::Filesystem::exists(destDir)); + EXPECT_TRUE(NGEN::Filesystem::exists(destDir + "/file1.txt")); + EXPECT_TRUE(NGEN::Filesystem::exists(destDir + "/subdir1/file2.txt")); + EXPECT_TRUE(NGEN::Filesystem::exists(destDir + "/subdir2/file3.txt")); + + ifstream file1(destDir + "/file1.txt"); + string content1((istreambuf_iterator(file1)), istreambuf_iterator()); + EXPECT_EQ(content1, "File 1 content"); + file1.close(); + + ifstream file2(destDir + "/subdir1/file2.txt"); + string content2((istreambuf_iterator(file2)), istreambuf_iterator()); + EXPECT_EQ(content2, "File 2 content"); + file2.close(); + + ifstream file3(destDir + "/subdir2/file3.txt"); + string content3((istreambuf_iterator(file3)), istreambuf_iterator()); + EXPECT_EQ(content3, "File 3 content"); + file3.close(); + + NGEN::Filesystem::deleteDirectory(sourceDir + "/subdir1", true); + NGEN::Filesystem::deleteDirectory(sourceDir + "/subdir2", true); + NGEN::Filesystem::deleteDirectory(sourceDir, true); + NGEN::Filesystem::deleteDirectory(destDir + "/subdir1", true); + NGEN::Filesystem::deleteDirectory(destDir + "/subdir2", true); + NGEN::Filesystem::deleteDirectory(destDir, true); +} + TEST_F(AgentCoreUtilUT, removeTrailingWhitespacesTest) { string str_with_trailing_whitespace = "str_with_trailing_whitespace\n\n\n\r \n\n\r"; diff --git a/core/agent_details/agent_details.cc b/core/agent_details/agent_details.cc index 0435aa0..fdbeabe 100644 --- a/core/agent_details/agent_details.cc +++ b/core/agent_details/agent_details.cc @@ -27,6 +27,7 @@ using namespace std; USE_DEBUG_FLAG(D_ORCHESTRATOR); +static const AlertInfo alert(AlertTeam::CORE, "agent details"); const map AgentDetails::machineTypes({ { "Amazon EC2", I_AgentDetails::MachineType::AWS }, @@ -381,7 +382,7 @@ AgentDetails::convertProxyProtocolToString(ProxyProtocol proto) const case ProxyProtocol::HTTP: return "http"; case ProxyProtocol::HTTPS: return "https"; } - dbgAssert(false) << "Unsupported Proxy Protocol " << static_cast(proto); + dbgAssert(false) << alert << "Unsupported Proxy Protocol " << static_cast(proto); return ""; } @@ -469,7 +470,9 @@ AgentDetails::loadProxyType(ProxyProtocol protocol) { dbgFlow(D_ORCHESTRATOR) << "Loading proxy type: " << convertProxyProtocolToString(protocol); dbgAssert(protocol == ProxyProtocol::HTTP || protocol == ProxyProtocol::HTTPS) - << "Unsupported Proxy Protocol " << static_cast(protocol); + << alert + << "Unsupported Proxy Protocol " + << static_cast(protocol); static const map env_var_name = { {ProxyProtocol::HTTPS, "https_proxy"}, diff --git a/core/buffers/buffer.cc b/core/buffers/buffer.cc index 41efb41..081a7ee 100644 --- a/core/buffers/buffer.cc +++ b/core/buffers/buffer.cc @@ -17,6 +17,8 @@ using namespace std; +static const AlertInfo alert(AlertTeam::CORE, "buffer i/s"); + Buffer::Buffer(vector &&vec) : len(vec.size()) @@ -142,7 +144,7 @@ Buffer::operator+(const Buffer &other) const Buffer Buffer::getSubBuffer(uint start, uint end) const { - dbgAssert(start<=end && end<=len) << "Buffer::getSubBuffer() returned: Illegal scoping of buffer"; + dbgAssert(start<=end && end<=len) << alert << "Buffer::getSubBuffer() returned: Illegal scoping of buffer"; if (start == end) return Buffer(); Buffer res; @@ -176,7 +178,7 @@ Buffer::getSubBuffer(uint start, uint end) const Maybe Buffer::findFirstOf(char ch, uint start) const { - dbgAssert(start <= len) << "Buffer::findFirstOf() returned: Cannot set a start point after buffer's end"; + dbgAssert(start <= len) << alert << "Buffer::findFirstOf() returned: Cannot set a start point after buffer's end"; for (; start < len; ++start) { if ((*this)[start] == ch) return start; @@ -187,7 +189,7 @@ Buffer::findFirstOf(char ch, uint start) const Maybe Buffer::findFirstOf(const Buffer &buf, uint start) const { - dbgAssert(start <= len) << "Buffer::findFirstOf() returned: Cannot set a start point after buffer's end"; + dbgAssert(start <= len) << alert << "Buffer::findFirstOf() returned: Cannot set a start point after buffer's end"; for (; start + buf.size() <= len; ++start) { auto sub_buffer = getSubBuffer(start, start + buf.size()); @@ -199,7 +201,9 @@ Buffer::findFirstOf(const Buffer &buf, uint start) const Maybe Buffer::findFirstNotOf(char ch, uint start) const { - dbgAssert(start <= len) << "Buffer::findFirstNotOf() returned: Cannot set a start point after buffer's end"; + dbgAssert(start <= len) + << alert + << "Buffer::findFirstNotOf() returned: Cannot set a start point after buffer's end"; for (; start < len; ++start) { if ((*this)[start] != ch) return start; } @@ -209,7 +213,7 @@ Buffer::findFirstNotOf(char ch, uint start) const Maybe Buffer::findLastOf(char ch, uint start) const { - dbgAssert(start <= len) << "Buffer::findLastOf() returned: Cannot set a start point after buffer's end"; + dbgAssert(start <= len) << alert << "Buffer::findLastOf() returned: Cannot set a start point after buffer's end"; for (; 0 < start; --start) { if ((*this)[start - 1] == ch) return start - 1; } @@ -219,7 +223,9 @@ Buffer::findLastOf(char ch, uint start) const Maybe Buffer::findLastNotOf(char ch, uint start) const { - dbgAssert(start <= len) << "Buffer::findLastNotOf() returned: Cannot set a start point after buffer's end"; + dbgAssert(start <= len) + << alert + << "Buffer::findLastNotOf() returned: Cannot set a start point after buffer's end"; for (; 0 < start; --start) { if ((*this)[start - 1] != ch) return start - 1; } @@ -229,7 +235,7 @@ Buffer::findLastNotOf(char ch, uint start) const void Buffer::truncateHead(uint size) { - dbgAssert(size <= len) << "Cannot set a new start of buffer after the buffer's end"; + dbgAssert(size <= len) << alert << "Cannot set a new start of buffer after the buffer's end"; if (size == 0) return; if (size == len) { clear(); @@ -255,7 +261,7 @@ Buffer::truncateHead(uint size) void Buffer::truncateTail(uint size) { - dbgAssert(size <= len) << "Cannot set a new end of buffer after the buffer's end"; + dbgAssert(size <= len) << alert << "Cannot set a new end of buffer after the buffer's end"; if (size == 0) return; if (size == len) { clear(); @@ -279,14 +285,14 @@ Buffer::truncateTail(uint size) void Buffer::keepHead(uint size) { - dbgAssert(size <= len) << "Cannot set a new end of buffer before the buffer's start"; + dbgAssert(size <= len) << alert << "Cannot set a new end of buffer before the buffer's start"; truncateTail(len - size); } void Buffer::keepTail(uint size) { - dbgAssert(size <= len) << "Cannot set a new start of buffer after the buffer's end"; + dbgAssert(size <= len) << alert << "Cannot set a new start of buffer after the buffer's end"; truncateHead(len - size); } @@ -376,7 +382,7 @@ Buffer::operator[](uint offset) const } return fast_path_ptr[offset]; } - dbgAssert(offset < len) << "Buffer::operator returned: attempted an access outside the buffer"; + dbgAssert(offset < len) << alert << "Buffer::operator returned: attempted an access outside the buffer"; return *(begin() + offset); } diff --git a/core/buffers/char_iterator.cc b/core/buffers/char_iterator.cc index b7b4032..1b7c16c 100644 --- a/core/buffers/char_iterator.cc +++ b/core/buffers/char_iterator.cc @@ -67,10 +67,11 @@ Buffer::CharIterator::operator==(const CharIterator &other) const return (cur_seg == other.cur_seg) && (offset == other.offset); } +static const AlertInfo alert(AlertTeam::CORE, "buffer i/s"); const u_char & Buffer::CharIterator::operator*() const { - dbgAssert(ptr != nullptr) << "Buffer::CharIterator is not pointing to a real value"; + dbgAssert(ptr != nullptr) << alert << "Buffer::CharIterator is not pointing to a real value"; return ptr[offset]; } diff --git a/core/compression/compression_utils_ut/compression_utils_ut.cc b/core/compression/compression_utils_ut/compression_utils_ut.cc index abec348..2f5782b 100644 --- a/core/compression/compression_utils_ut/compression_utils_ut.cc +++ b/core/compression/compression_utils_ut/compression_utils_ut.cc @@ -25,7 +25,10 @@ public: ); setCompressionDebugFunction( CompressionUtilsDebugLevel::COMPRESSION_DBG_LEVEL_ASSERTION, - [](const char *assert_message) { dbgAssert(false) << assert_message; } + [](const char *assert_message) + { + dbgAssert(false) << AlertInfo(AlertTeam::CORE, "testing") << assert_message; + } ); } diff --git a/core/connkey/connkey.cc b/core/connkey/connkey.cc index 5a7ce93..7788ca6 100644 --- a/core/connkey/connkey.cc +++ b/core/connkey/connkey.cc @@ -53,6 +53,7 @@ operator<<(ostream &os, const IPType &t) return os << "Invalid(" << static_cast(t) << ")"; } +static const AlertInfo alert(AlertTeam::CORE, "IP address"); // Format an IP address. Use a pair, becuase it depends on the type (v4/v6) ostream & IPAddr::print(ostream &os) const @@ -63,12 +64,12 @@ IPAddr::print(ostream &os) const switch (type) { case IPType::V4: { formatted_addr = inet_ntop(AF_INET, &v4, buf, sizeof(buf)); - dbgAssert(formatted_addr == buf) << "Failed to convert an IPv4 address"; + dbgAssert(formatted_addr == buf) << alert("conversion error") << "Failed to convert an IPv4 address"; break; } case IPType::V6: { formatted_addr = inet_ntop(AF_INET6, &v6, buf, sizeof(buf)); - dbgAssert(formatted_addr == buf) << "Failed to convert an IPv6 address"; + dbgAssert(formatted_addr == buf) << alert("conversion error") << "Failed to convert an IPv6 address"; break; } case IPType::UNINITIALIZED: { @@ -115,8 +116,10 @@ ConnKey::reverse() size_t ConnKey::hash() const { - dbgAssert(src.type != IPType::UNINITIALIZED) << "ConnKey::hash was called on an uninitialized object"; - size_t seed = 0; // XXX: random seed for security? + dbgAssert(src.type != IPType::UNINITIALIZED) + << alert("hashing") + << "ConnKey::hash was called on an uninitialized object"; + size_t seed = 0; hashCombine(seed, static_cast(src.type)); hashCombine(seed, src.proto); hashCombine(seed, src); diff --git a/core/cptest/cptest.cc b/core/cptest/cptest.cc index c21a17c..20858d4 100644 --- a/core/cptest/cptest.cc +++ b/core/cptest/cptest.cc @@ -80,7 +80,11 @@ cptestParseHex(const string &hex_text) size_t l = t.size(); if (l==0) continue; if (t[l-1]==':') continue; // tcpdump uses xxxx: to mark offsets, not data. So ignore it. - dbgAssert(t.size() %2 == 0) << "Expecting an even number of hex digits, " << t << " is invalid"; + dbgAssert(t.size() %2 == 0) + << AlertInfo(AlertTeam::CORE, "testing") + << "Expecting an even number of hex digits, " + << t + << " is invalid"; for (uint i=0; i &pkt) const while (optbuf.size()%4 != 0) { vec_append(optbuf, TCPOption::NOP); } - dbgAssert(optbuf.size() <= 40) << "too many tcp options. max is 40 bytes"; + dbgAssert(optbuf.size() <= 40) << AlertInfo(AlertTeam::CORE, "testing") << "too many tcp options. max is 40 bytes"; vec_append(pkt, optbuf); } diff --git a/core/cptest/cptest_ut/cptest_ut.cc b/core/cptest/cptest_ut/cptest_ut.cc index f3fdc61..9e50106 100644 --- a/core/cptest/cptest_ut/cptest_ut.cc +++ b/core/cptest/cptest_ut/cptest_ut.cc @@ -8,7 +8,7 @@ TEST(CPTest, PrepareToDie) { cptestPrepareToDie(); auto die = []() { - dbgAssert(false) << "You killed my father"; + dbgAssert(false) << AlertInfo(AlertTeam::CORE, "testing") << "You killed my father"; }; EXPECT_DEATH(die(), "You killed my father"); } diff --git a/core/debug_is/debug.cc b/core/debug_is/debug.cc index cbc603a..89ad565 100644 --- a/core/debug_is/debug.cc +++ b/core/debug_is/debug.cc @@ -26,6 +26,7 @@ #include "config.h" #include "i_instance_awareness.h" #include "i_signal_handler.h" +#include "hash_combine.h" using namespace std; @@ -283,6 +284,16 @@ private: S2C_PARAM(string, output); }; +void +AlertInfo::evalParams() +{ + id = 0; + hashCombine(id, family_id); + hashCombine(id, functionality); + hashCombine(id, description); + hashCombine(id, static_cast(team)); +} + // LCOV_EXCL_START - function is covered in unit-test, but not detected bt gcov Debug::Debug( const string &file_name, @@ -830,6 +841,14 @@ Debug::isCommunicationFlag(const DebugFlags &flag) ); } +void +Debug::sendAlert(const AlertInfo &alert) +{ + for (auto &added_stream : current_active_streams) { + added_stream->sendAlert(alert); + } +} + Debug::DebugLevel Debug::lowest_global_level = default_level; I_TimeGet *Debug::time = nullptr; I_MainLoop *Debug::mainloop = nullptr; diff --git a/core/debug_is/debug_ex.h b/core/debug_is/debug_ex.h index 2fcfda0..6cdf7b3 100644 --- a/core/debug_is/debug_ex.h +++ b/core/debug_is/debug_ex.h @@ -55,6 +55,7 @@ public: ); virtual void finishMessage() { *stream << std::endl; } + virtual void sendAlert(const AlertInfo &) {} std::ostream * getStream() const { return stream; } @@ -112,6 +113,7 @@ public: ) override; void finishMessage() override; + void sendAlert(const AlertInfo &_alert) override { possible_alert = _alert; } private: void sendBufferedMessages(); @@ -133,6 +135,7 @@ private: std::string trace_id; std::string span_id; uint line; + Maybe possible_alert = genError(""); }; #endif // __DEBUG_EX_H__ diff --git a/core/debug_is/debug_is_ut/debug_ut.cc b/core/debug_is/debug_is_ut/debug_ut.cc index c533a5a..66ea543 100644 --- a/core/debug_is/debug_is_ut/debug_ut.cc +++ b/core/debug_is/debug_is_ut/debug_ut.cc @@ -39,11 +39,32 @@ void doPMExecTrace() { dbgTrace(D_PM_EXEC) << "PM_EXEC trace message"; line = to template void doManyFlags(Args ...args) { dbgDebug(args...) << "stab"; line = to_string(__LINE__); } +TEST(DebugBaseTest, alert_obkect) +{ + AlertInfo alert1(AlertTeam::CORE, "testing"); + EXPECT_EQ(alert1.getTeam(), AlertTeam::CORE); + EXPECT_EQ(alert1.getFunctionality(), "testing"); + EXPECT_EQ(alert1.getDescription(), ""); + EXPECT_EQ(alert1.getFamilyId(), 0u); + EXPECT_NE(alert1.getId(), 0u); + + auto alert2 = alert1("additional data", 5); + EXPECT_EQ(alert2.getTeam(), AlertTeam::CORE); + EXPECT_EQ(alert2.getFunctionality(), "testing"); + EXPECT_EQ(alert2.getDescription(), "additional data"); + EXPECT_EQ(alert2.getFamilyId(), 5u); + EXPECT_NE(alert2.getId(), 0u); + EXPECT_NE(alert1.getId(), alert2.getId()); +} + TEST(DebugBaseTest, death_on_panic) { cptestPrepareToDie(); - EXPECT_DEATH(dbgAssert(1==2) << "Does your school teach otherwise?", "Does your school teach otherwise?"); + EXPECT_DEATH( + dbgAssert(1==2) << AlertInfo(AlertTeam::CORE, "testing") << "Does your school teach otherwise?", + "Does your school teach otherwise?" + ); } TEST(DebugBaseTest, default_levels) @@ -1014,7 +1035,7 @@ TEST(DebugFogTest, fog_stream) " \"agentId\": \"Unknown\",\n" " \"issuingFunction\": \"handleThresholdReach\",\n" " \"issuingFile\": \"debug_streams.cc\",\n" - " \"issuingLine\": 344,\n" + " \"issuingLine\": 364,\n" " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" @@ -1117,3 +1138,123 @@ TEST(DebugFogTest, fog_stream) EXPECT_CALL(mock_mainloop, stop(0)); Debug::fini(); } + +TEST(DebugFogTest, alert_fog_stream) +{ + ConfigComponent conf; + ::Environment env; + env.preload(); + env.init(); + stringstream capture_debug; + conf.preload(); + + StrictMock mock_mainloop; + StrictMock mock_time; + NiceMock mock_agent_details; + + ON_CALL(mock_agent_details, getFogDomain()).WillByDefault(Return(Maybe(string("fog_domain.com")))); + ON_CALL(mock_agent_details, getFogPort()).WillByDefault(Return(Maybe(443))); + + EXPECT_CALL(mock_agent_details, getAgentId()).WillRepeatedly(Return("Unknown")); + EXPECT_CALL(mock_agent_details, getOrchestrationMode()).WillRepeatedly(Return(OrchestrationMode::ONLINE)); + + EXPECT_CALL(mock_time, getWalltimeStr(_)).WillRepeatedly(Return(string("2016-11-13T17:31:24.087"))); + I_MainLoop::Routine send_debug_routine = nullptr; + + EXPECT_CALL(mock_mainloop, addRecurringRoutine(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&send_debug_routine), Return(0))); + + StrictMock messaging_mock; + string message_body; + + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events/bulk", + _, + _, + _, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); + + Singleton::Consume::from(conf)->loadConfiguration( + vector{"--orchestration-mode=online_mode"} + ); + Debug::preload(); + string config_json = + "{" + " \"Debug I/S\": {" + " \"Sent debug bulk size\": [" + " {" + " \"value\": 2" + " }" + " ]" + " }," + " \"Debug\": [{" + " \"Streams\": [" + " {" + " \"Output\": \"FOG\"" + " }," + " {" + " \"Output\": \"STDOUT\"" + " }" + " ]" + " }]" + "}"; + + istringstream ss(config_json); + Singleton::Consume::from(conf)->loadConfiguration(ss); + + Debug::DebugAlert("MockFile", "MockFunction", 0, Debug::DebugLevel::ERROR, D_FW).getStreamAggr() + << AlertInfo(AlertTeam::CORE, "testing") + << "Generic error message"; + + string expected_message = + "{\n" + " \"logs\": [\n" + " {\n" + " \"id\": 1,\n" + " \"log\": {\n" + " \"eventTime\": \"2016-11-13T17:31:24.087\",\n" + " \"eventName\": \"Debug message\",\n" + " \"eventSeverity\": \"High\",\n" + " \"eventPriority\": \"Low\",\n" + " \"eventType\": \"Code Related\",\n" + " \"eventLevel\": \"Log\",\n" + " \"eventLogLevel\": \"error\",\n" + " \"eventAudience\": \"Internal\",\n" + " \"eventAudienceTeam\": \"\",\n" + " \"eventFrequency\": 0,\n" + " \"eventTags\": [\n" + " \"Informational\"\n" + " ],\n" + " \"eventSource\": {\n" + " \"agentId\": \"Unknown\",\n" + " \"issuingFunction\": \"MockFunction\",\n" + " \"issuingFile\": \"MockFile\",\n" + " \"issuingLine\": 0,\n" + " \"eventTraceId\": \"\",\n" + " \"eventSpanId\": \"\",\n" + " \"issuingEngineVersion\": \"\",\n" + " \"serviceName\": \"Unnamed Nano Service\"\n" + " },\n" + " \"eventData\": {\n" + " \"eventMessage\": \"Generic error message\",\n" + " \"eventId\": 6255310698607853351,\n" + " \"eventFamilyId\": 0,\n" + " \"eventFunctionality\": \"testing\",\n" + " \"eventDescription\": \"\",\n" + " \"eventResponseTeam\": \"Core\"\n" + " }\n" + " }\n" + " }\n" + " ]\n" + "}"; + + send_debug_routine(); + + EXPECT_EQ(message_body, expected_message); + + EXPECT_CALL(mock_mainloop, doesRoutineExist(0)).WillOnce(Return(true)); + EXPECT_CALL(mock_mainloop, stop(0)); + Debug::fini(); +} diff --git a/core/debug_is/debug_streams.cc b/core/debug_is/debug_streams.cc index 503c64a..b28c235 100644 --- a/core/debug_is/debug_streams.cc +++ b/core/debug_is/debug_streams.cc @@ -230,6 +230,18 @@ DebugFogStream::printHeader( } } +static string +getTeam(const Maybe &alert) +{ + switch((*alert).getTeam()) { + case AlertTeam::CORE: return "Core"; + case AlertTeam::WAAP: return "Waap"; + case AlertTeam::SDWAN: return "SDWAN"; + case AlertTeam::IOT: return "IoT"; + } + return "Core"; +} + void DebugFogStream::finishMessage() { @@ -265,6 +277,14 @@ DebugFogStream::finishMessage() ); message_to_fog << LogField("eventMessage", message.str()); + if (possible_alert.ok()) { + message_to_fog << LogField("eventId", (*possible_alert).getId()); + message_to_fog << LogField("eventFamilyId", (*possible_alert).getFamilyId()); + message_to_fog << LogField("eventFunctionality", (*possible_alert).getFunctionality()); + message_to_fog << LogField("eventDescription", (*possible_alert).getDescription()); + message_to_fog << LogField("eventResponseTeam", getTeam(possible_alert)); + } + if (!getConfigurationWithDefault(true, "Debug I/S", "Enable bulk of debugs")) { LogRest rest(move(message_to_fog)); Singleton::Consume::by()->addOneTimeRoutine( @@ -370,18 +390,20 @@ DebugFogStream::getSeverity() const return Severity::CRITICAL; } +static const AlertInfo alert(AlertTeam::CORE, "debug configuration"); + LogLevel DebugFogStream::getLogLevel() const { switch (level) { - case Debug::DebugLevel::NOISE: dbgAssert(false) << "Impossible LogLevel 'Noise'"; break; + case Debug::DebugLevel::NOISE: dbgAssert(false) << alert << "Impossible LogLevel 'Noise'"; break; case Debug::DebugLevel::TRACE: return LogLevel::TRACE; case Debug::DebugLevel::DEBUG: return LogLevel::DEBUG; case Debug::DebugLevel::WARNING: return LogLevel::WARNING; case Debug::DebugLevel::INFO: return LogLevel::INFO; case Debug::DebugLevel::ERROR: return LogLevel::ERROR; case Debug::DebugLevel::ASSERTION: return LogLevel::ERROR; - case Debug::DebugLevel::NONE: dbgAssert(false) << "Impossible LogLevel 'None'"; break; + case Debug::DebugLevel::NONE: dbgAssert(false) << alert << "Impossible LogLevel 'None'"; break; } return LogLevel::INFO; diff --git a/core/environment/context.cc b/core/environment/context.cc index 503fc71..c34d44c 100644 --- a/core/environment/context.cc +++ b/core/environment/context.cc @@ -42,6 +42,8 @@ Context::getAllStrings(const EnvKeyAttr::ParamAttr ¶m) const return result; } +static const AlertInfo alert(AlertTeam::CORE, "environment contexts"); + const std::string Context::convertToString(MetaDataType type) { @@ -58,9 +60,9 @@ Context::convertToString(MetaDataType type) case MetaDataType::Direction: return "direction"; case MetaDataType::Email: return "email"; case MetaDataType::COUNT: - dbgAssert(false) << "COUNT is not a valid meta data type"; + dbgAssert(false) << alert << "COUNT is not a valid meta data type"; } - dbgAssert(false) << "Reached impossible case with type=" << static_cast(type); + dbgAssert(false) << alert << "Reached impossible case with type=" << static_cast(type); return ""; } diff --git a/core/environment/environment.cc b/core/environment/environment.cc index ec8a0e5..c65fcca 100644 --- a/core/environment/environment.cc +++ b/core/environment/environment.cc @@ -213,8 +213,9 @@ Environment::Impl::registerContext(Context *ptr) void Environment::Impl::unregisterContext(Context *ptr) { - dbgAssert(active_contexts.first.back() == ptr) << - "Contexts are supposed to unregister in reverse order to their registration"; + dbgAssert(active_contexts.first.back() == ptr) + << AlertInfo(AlertTeam::CORE, "environment contexts") + << "Contexts are supposed to unregister in reverse order to their registration"; active_contexts.first.pop_back(); } diff --git a/core/environment/span.cc b/core/environment/span.cc index 656e0d8..92915be 100644 --- a/core/environment/span.cc +++ b/core/environment/span.cc @@ -97,7 +97,7 @@ Span::convertSpanContextTypeToString(ContextType type) return "Follows from"; } } - dbgAssert(false) << "Span context not supported"; + dbgAssert(false) << AlertInfo(AlertTeam::CORE, "tracing") << "Span context not supported"; return string(); } diff --git a/core/include/general/buffer/internal_ptr.h b/core/include/general/buffer/internal_ptr.h index 36a9a1a..da6f408 100644 --- a/core/include/general/buffer/internal_ptr.h +++ b/core/include/general/buffer/internal_ptr.h @@ -32,7 +32,14 @@ public: } operator const T *() const { return ptr; } - const T & operator*() const { dbgAssert(ptr != nullptr) << "Accessing a moved pointer"; return *ptr; } + + const T & + operator*() const + { + dbgAssert(ptr != nullptr) << AlertInfo(AlertTeam::CORE, "buffer i/s") << "Accessing a moved pointer"; + return *ptr; + } + const T * operator->() const { return ptr; } private: diff --git a/core/include/general/debug.h b/core/include/general/debug.h index 100585b..a1c1153 100644 --- a/core/include/general/debug.h +++ b/core/include/general/debug.h @@ -36,6 +36,58 @@ class I_SignalHandler; namespace Config { enum class Errors; } std::ostream & operator<<(std::ostream &, const Config::Errors &); +enum class AlertTeam { CORE, WAAP, SDWAN, IOT }; + +class AlertInfo +{ +public: + template + AlertInfo(AlertTeam _team, const std::string &func, const Args & ... args) : team(_team), functionality(func) + { + evalParams(args ...); + } + + template + AlertInfo + operator()(const Args & ... args) const + { + AlertInfo res = *this; + res.evalParams(args ...); + return res; + } + + AlertTeam getTeam() const { return team; } + const std::string & getFunctionality() const { return functionality; } + const std::string & getDescription() const { return description; } + std::size_t getId() const { return id; } + std::size_t getFamilyId() const { return family_id; } + +private: + template + void + evalParams(const std::string &_description, const Args & ... args) + { + description = _description; + evalParams(args ...); + } + + template + void + evalParams(const std::size_t &fam_id, const Args & ... args) + { + family_id = fam_id; + evalParams(args ...); + } + + void evalParams(); + + AlertTeam team; + std::string functionality; + std::size_t id; + std::size_t family_id = 0; + std::string description; +}; + class Debug : Singleton::Consume, @@ -93,6 +145,8 @@ public: std::set streams; }; + class DebugAlert; + class DebugLockState { private: @@ -208,6 +262,7 @@ private: ); void isCommunicationFlag(const DebugFlags &flag); + void sendAlert(const AlertInfo &alert); static DebugLevel lowest_global_level; static I_TimeGet *time; @@ -225,6 +280,34 @@ private: std::set> current_active_streams; }; +class Debug::DebugAlert + { + class DebugAlertImpl + { + public: + DebugAlertImpl(Debug &_debug) : debug(_debug) {} + + DebugStreamAggr & + operator<<(const AlertInfo &alert) __attribute__((warn_unused_result)) + { + debug.sendAlert(alert); + return debug.getStreamAggr(); + } + + private: + Debug &debug; + }; + + public: + template DebugAlert(const Args & ... args) : debug(args...) {} + + DebugAlertImpl getStreamAggr() __attribute__((warn_unused_result)) { return DebugAlertImpl(debug); } + + private: + Debug debug; + }; + + #define USE_DEBUG_FLAG(x) extern const Debug::DebugFlags x // This function extract the base name from a full path. @@ -245,7 +328,7 @@ getBaseName(const char *iter, const char *base) #define dbgAssert(cond) \ if (CP_LIKELY(cond)) { \ - } else Debug(__FILENAME__, __FUNCTION__, __LINE__).getStreamAggr() + } else Debug::DebugAlert(__FILENAME__, __FUNCTION__, __LINE__).getStreamAggr() // Macros to allow simple debug messaging #define DBG_GENERIC(level, ...) \ diff --git a/core/include/general/maybe_res.h b/core/include/general/maybe_res.h index dad5dd2..c41863c 100644 --- a/core/include/general/maybe_res.h +++ b/core/include/general/maybe_res.h @@ -320,7 +320,7 @@ template const T & Maybe::unpack() const { - dbgAssert(set) << "Maybe value is not set"; + dbgAssert(set) << AlertInfo(AlertTeam::CORE, "maybe i/s") << "Maybe value is not set"; return val; } @@ -328,7 +328,7 @@ template T && Maybe::unpackMove() { - dbgAssert(set) << "No value to be moved"; + dbgAssert(set) << AlertInfo(AlertTeam::CORE, "maybe i/s") << "No value to be moved"; return std::move(val); } @@ -336,7 +336,7 @@ template TErr Maybe::getErr() const { - dbgAssert(!set) << "Maybe value is set"; + dbgAssert(!set) << AlertInfo(AlertTeam::CORE, "maybe i/s") << "Maybe value is set"; return err.err; } @@ -344,7 +344,7 @@ template const Error & Maybe::passErr() const { - dbgAssert(!set) << "Maybe value is set"; + dbgAssert(!set) << AlertInfo(AlertTeam::CORE, "maybe i/s") << "Maybe value is set"; return err; } diff --git a/core/include/general/table/expiration_impl.h b/core/include/general/table/expiration_impl.h index a1b373d..927cdcb 100644 --- a/core/include/general/table/expiration_impl.h +++ b/core/include/general/table/expiration_impl.h @@ -110,7 +110,9 @@ template const Key & Table::Impl::ExpList::getEarliest() const { - dbgAssert(!list.empty()) << "Cannot access the earliest member of an empty list"; + dbgAssert(!list.empty()) + << AlertInfo(AlertTeam::CORE, "table") + << "Cannot access the earliest member of an empty list"; return list.back().getKey(); } diff --git a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h index b967217..6b1de2c 100644 --- a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h +++ b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h @@ -36,6 +36,7 @@ class I_Intelligence_IS_V2 { public: virtual bool sendInvalidation(const Intelligence::Invalidation &invalidation) const = 0; + virtual bool isIntelligenceHealthy() const = 0; virtual Maybe registerInvalidation( const Intelligence::Invalidation &invalidation, const std::function &callback diff --git a/core/include/services_sdk/interfaces/i_rest_api.h b/core/include/services_sdk/interfaces/i_rest_api.h index 682c20d..ca5e0af 100644 --- a/core/include/services_sdk/interfaces/i_rest_api.h +++ b/core/include/services_sdk/interfaces/i_rest_api.h @@ -48,6 +48,8 @@ public: return addRestCall(oper, uri, std::make_unique>()); } + virtual bool addGetCall(const std::string &uri, const std::function &callback) = 0; + virtual uint16_t getListeningPort() const = 0; protected: diff --git a/core/include/services_sdk/interfaces/mock/mock_intelligence.h b/core/include/services_sdk/interfaces/mock/mock_intelligence.h index b126997..ab8365e 100644 --- a/core/include/services_sdk/interfaces/mock/mock_intelligence.h +++ b/core/include/services_sdk/interfaces/mock/mock_intelligence.h @@ -24,6 +24,7 @@ public: using Response = Intelligence::Response; MOCK_CONST_METHOD1(sendInvalidation, bool(const Invalidation &invalidation)); + MOCK_CONST_METHOD0(isIntelligenceHealthy, bool(void)); MOCK_METHOD2(registerInvalidation, Maybe(const Invalidation &invalidation, const InvalidationCb &callback)); MOCK_METHOD1(unregisterInvalidation, void(uint id)); MOCK_CONST_METHOD5( diff --git a/core/include/services_sdk/interfaces/mock/mock_rest_api.h b/core/include/services_sdk/interfaces/mock/mock_rest_api.h index 33acfe4..5cd9887 100644 --- a/core/include/services_sdk/interfaces/mock/mock_rest_api.h +++ b/core/include/services_sdk/interfaces/mock/mock_rest_api.h @@ -9,6 +9,7 @@ class MockRestApi : public Singleton::Provide::From &)); // You can't mock a function with an R-value reference. So mock a slightly different one MOCK_METHOD3(mockRestCall, bool(RestAction, const std::string &, const std::unique_ptr &)); diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index cc992c8..196b7d9 100644 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -34,7 +34,7 @@ DEFINE_FLAG(D_INFRA, D_ALL) DEFINE_FLAG(D_SIGNAL_HANDLER, D_INFRA) DEFINE_FLAG(D_TENANT_MANAGER, D_INFRA) DEFINE_FLAG(D_MONITORING, D_INFRA) - DEFINE_FLAG(D_HEALTH_CHECK_MANAGER, D_INFRA) + DEFINE_FLAG(D_SERVICE_HEALTH_STATUS, D_INFRA) DEFINE_FLAG(D_REPORT, D_INFRA) DEFINE_FLAG(D_REPORT_BULK, D_REPORT) DEFINE_FLAG(D_TRACE, D_INFRA) @@ -137,6 +137,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_HTTP_MANAGER, D_COMPONENT) DEFINE_FLAG(D_ORCHESTRATOR, D_COMPONENT) + DEFINE_FLAG(D_HEALTH_CHECK_MANAGER, D_ORCHESTRATOR) DEFINE_FLAG(D_HEALTH_CHECK, D_ORCHESTRATOR) DEFINE_FLAG(D_AGENT_DETAILS, D_ORCHESTRATOR) DEFINE_FLAG(D_LOCAL_POLICY, D_ORCHESTRATOR) @@ -166,7 +167,6 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_IOT_QUERY_INTELLIGENCE, D_IOT_AUXILIARY) DEFINE_FLAG(D_IOT_SAVE_PERSISTENT, D_IOT_AUXILIARY) DEFINE_FLAG(D_IOT_DOCKER, D_IOT_AUXILIARY) - DEFINE_FLAG(D_IOT_REST_AUTHENTICATION, D_IOT_AUXILIARY) DEFINE_FLAG(D_IOT_ENFORCE, D_IOT_NEXT) DEFINE_FLAG(D_IOT_ENFORCE_POLICY, D_IOT_ENFORCE) DEFINE_FLAG(D_IOT_ENFORCE_ASSETS, D_IOT_ENFORCE) diff --git a/core/include/services_sdk/resources/generic_metric.h b/core/include/services_sdk/resources/generic_metric.h index 08b90b9..69771e9 100644 --- a/core/include/services_sdk/resources/generic_metric.h +++ b/core/include/services_sdk/resources/generic_metric.h @@ -16,7 +16,9 @@ #include #include +#include +#include "metric/metric_metadata.h" #include "metric/metric_calc.h" #include "metric/all_metric_event.h" #include "i_mainloop.h" @@ -26,6 +28,7 @@ #include "i_messaging.h" #include "i_rest_api.h" #include "report/report_enums.h" +#include "flags.h" namespace MetricCalculations { @@ -38,6 +41,10 @@ namespace MetricCalculations template class MetricMap; } // MetricCalculations +MetricMetadata::DotName operator"" _dot(const char *, std::size_t); +MetricMetadata::Units operator"" _unit(const char *, std::size_t); +MetricMetadata::Description operator"" _desc(const char *, std::size_t); + class LogRest; class GenericMetric @@ -51,6 +58,8 @@ class GenericMetric public Listener { public: + enum class Stream { FOG, DEBUG, PROMETHEUS, AIOPS, COUNT }; + void init( const std::string &_metric_name, @@ -79,7 +88,8 @@ public: static std::string getName() { return "GenericMetric"; } - std::string generateReport(bool with_reset); + std::string generateReport() const; + void resetMetrics(); void upon(const AllMetricEvent &) override; std::string respond(const AllMetricEvent &event) override; std::string getListenerName() const override; @@ -87,6 +97,9 @@ public: std::string getMetricName() const; std::chrono::seconds getReportInterval() const; + void turnOnStream(Stream stream) { active_streams.setFlag(stream); } + void turnOffStream(Stream stream) { active_streams.unsetFlag(stream); } + protected: virtual void sendLog(const LogRest &metric_client_rest) const; @@ -98,6 +111,7 @@ private: void handleMetricStreamSending(); void generateLog(); + void generateDebug(); I_MainLoop *i_mainloop; I_TimeGet *i_time; @@ -107,12 +121,14 @@ private: ReportIS::Audience audience; std::chrono::seconds report_interval; std::vector calcs; + Flags active_streams; bool reset; bool force_buffering = false; Context ctx; }; #include "metric/counter.h" +#include "metric/no_reset_counter.h" #include "metric/max.h" #include "metric/min.h" #include "metric/average.h" diff --git a/core/include/services_sdk/resources/metric/average.h b/core/include/services_sdk/resources/metric/average.h index 8f18b53..a904106 100644 --- a/core/include/services_sdk/resources/metric/average.h +++ b/core/include/services_sdk/resources/metric/average.h @@ -25,12 +25,18 @@ template class Average : public MetricCalc { public: - Average(GenericMetric *metric, const std::string &title) : MetricCalc(metric, title), sum(0), count(0) {} + template + Average(GenericMetric *metric, const std::string &title, const Args & ... args) + : + MetricCalc(metric, title, args ...), + sum(0), + count(0) + { + } void report(const T &new_value) { - was_once_reported = true; sum += new_value; count++; } @@ -38,7 +44,6 @@ public: void reset() override { - was_once_reported = false; sum = 0; count = 0; } @@ -46,19 +51,19 @@ public: double getAverage() const { - return (was_once_reported) ? double(sum)/count : 0; + return (count > 0) ? double(sum)/count : 0; } void save(cereal::JSONOutputArchive &ar) const override { - ar(cereal::make_nvp(calc_title, getAverage())); + ar(cereal::make_nvp(getMetricName(), getAverage())); } LogField getLogField() const override { - return LogField(calc_title, static_cast(getAverage())); + return LogField(getMetricName(), static_cast(getAverage())); } private: diff --git a/core/include/services_sdk/resources/metric/counter.h b/core/include/services_sdk/resources/metric/counter.h index 960fac2..c4c56a1 100644 --- a/core/include/services_sdk/resources/metric/counter.h +++ b/core/include/services_sdk/resources/metric/counter.h @@ -24,12 +24,17 @@ namespace MetricCalculations class Counter : public MetricCalc { public: - Counter(GenericMetric *metric, const std::string &title) : MetricCalc(metric, title), counter(0) {} + template + Counter(GenericMetric *metric, const std::string &title, const Args & ... args) + : + MetricCalc(metric, title, args ...), + counter(0) + { + } void reset() override { - was_once_reported = false; counter = 0; } @@ -42,20 +47,19 @@ public: void save(cereal::JSONOutputArchive &ar) const override { - ar(cereal::make_nvp(calc_title, getCounter())); + ar(cereal::make_nvp(getMetricName(), getCounter())); } void report(const uint64_t &new_value) { - was_once_reported = true; counter += new_value; } LogField getLogField() const override { - return LogField(calc_title, static_cast(getCounter())); + return LogField(getMetricName(), static_cast(getCounter())); } private: diff --git a/core/include/services_sdk/resources/metric/last_reported_value.h b/core/include/services_sdk/resources/metric/last_reported_value.h index f13898a..2e53ffb 100644 --- a/core/include/services_sdk/resources/metric/last_reported_value.h +++ b/core/include/services_sdk/resources/metric/last_reported_value.h @@ -25,12 +25,16 @@ template class LastReportedValue : public MetricCalc { public: - LastReportedValue(GenericMetric *metric, const std::string &title) : MetricCalc(metric, title) {} + template + LastReportedValue(GenericMetric *metric, const std::string &title, const Args & ... args) + : + MetricCalc(metric, title, args ...) + { + } void reset() override { - was_once_reported = false; last_reported = T(); } @@ -43,24 +47,23 @@ public: void save(cereal::JSONOutputArchive &ar) const override { - ar(cereal::make_nvp(calc_title, getLastReportedValue())); + ar(cereal::make_nvp(getMetricName(), getLastReportedValue())); } void report(const T &new_value) { - was_once_reported = true; last_reported = new_value; } LogField getLogField() const override { - return LogField(calc_title, static_cast(getLastReportedValue())); + return LogField(getMetricName(), static_cast(getLastReportedValue())); } private: - T last_reported; + T last_reported{}; }; } // namespace MetricCalculations diff --git a/core/include/services_sdk/resources/metric/max.h b/core/include/services_sdk/resources/metric/max.h index d274edf..277c73b 100644 --- a/core/include/services_sdk/resources/metric/max.h +++ b/core/include/services_sdk/resources/metric/max.h @@ -25,23 +25,29 @@ template class Max : public MetricCalc { public: - Max(GenericMetric *metric, const std::string &title) : Max(metric, title, std::numeric_limits::min()) {} - Max(GenericMetric *metric, const std::string &title, T min_val) + Max(GenericMetric *metric, const std::string &title) : Max(metric, title, 0) {} + + template + Max(GenericMetric *metric, const std::string &title, T min_val, const Args & ... args) : - MetricCalc(metric, title), max(min_val), reset_value(min_val) {} + MetricCalc(metric, title, args ...), + max(min_val), + reset_value(min_val) + { + } void report(const T &new_value) { - was_once_reported = true; - if (new_value > max) max = new_value; + if (new_value > max || first) max = new_value; + first = false; } void reset() override { - was_once_reported = false; max = reset_value; + first = true; } T @@ -53,18 +59,19 @@ public: void save(cereal::JSONOutputArchive &ar) const override { - ar(cereal::make_nvp(calc_title, max)); + ar(cereal::make_nvp(getMetricName(), max)); } LogField getLogField() const override { - return LogField(calc_title, static_cast(getMax())); + return LogField(getMetricName(), static_cast(getMax())); } private: T max; T reset_value; + bool first = true; }; } // namespace MetricCalculations diff --git a/core/include/services_sdk/resources/metric/metric_calc.h b/core/include/services_sdk/resources/metric/metric_calc.h index 8bd7821..8cd6b42 100644 --- a/core/include/services_sdk/resources/metric/metric_calc.h +++ b/core/include/services_sdk/resources/metric/metric_calc.h @@ -24,19 +24,53 @@ class GenericMetric; +enum class MetricType { GAUGE, COUNTER }; + class MetricCalc { public: - MetricCalc(GenericMetric *metric, const std::string &calc_title); + template + MetricCalc(GenericMetric *metric, const std::string &calc_title, const Args & ... args) + { + setMetadata("BaseName", calc_title); + addMetric(metric); + parseMetadata(args ...); + } + virtual void reset() = 0; virtual void save(cereal::JSONOutputArchive &) const = 0; virtual LogField getLogField() const = 0; - bool wasOnceReported() const { return was_once_reported; } + std::string getMetricName() const { return getMetadata("BaseName"); } + std::string getMetricDotName() const { return getMetadata("DotName"); } + std::string getMetircUnits() const { return getMetadata("Units"); } + std::string getMetircDescription() const { return getMetadata("Description"); } + std::string getMetadata(const std::string &metadata) const; + virtual MetricType getMetricType() const { return MetricType::GAUGE; } + + void setMetricDotName(const std::string &name) { setMetadata("DotName", name); } + void setMetircUnits(const std::string &units) { setMetadata("Units", units); } + void setMetircDescription(const std::string &description) { setMetadata("Description", description); } + void setMetadata(const std::string &metadata, const std::string &value); protected: - bool was_once_reported = false; - std::string calc_title; + void addMetric(GenericMetric *metric); + + template + void + parseMetadata(const Metadata &metadata, const OtherMetadata & ... other_metadata) + { + parseMetadata(metadata); + parseMetadata(other_metadata ...); + } + + void parseMetadata(const MetricMetadata::DotName &name) { setMetricDotName(name.val); } + void parseMetadata(const MetricMetadata::Units &units) { setMetircUnits(units.val); } + void parseMetadata(const MetricMetadata::Description &description) { setMetircDescription(description.val); } + void parseMetadata() {} + +private: + std::map metadata; }; #endif // __METRIC_CALC_H__ diff --git a/core/include/services_sdk/resources/metric/metric_map.h b/core/include/services_sdk/resources/metric/metric_map.h index d4cfd1e..845e54f 100644 --- a/core/include/services_sdk/resources/metric/metric_map.h +++ b/core/include/services_sdk/resources/metric/metric_map.h @@ -46,6 +46,14 @@ class MetricMap : public MetricCalc void clear() { inner_map.clear(); } + MetricType + getMetricType() const + { + auto first = begin(); + if (first == end()) return MetricType::GAUGE; + return first->second.getMetricType(); + } + typename std::map::const_iterator begin() const { return inner_map.begin(); } typename std::map::const_iterator end() const { return inner_map.end(); } @@ -54,27 +62,31 @@ class MetricMap : public MetricCalc }; public: - MetricMap(GenericMetric *metric, const std::string &title) : MetricCalc(metric, title) {} + template + MetricMap(GenericMetric *metric, const std::string &title, const Args & ... args) + : + MetricCalc(metric, title, args ...) + { + } void reset() override { - was_once_reported = false; - metric_map.clear(); + if (getMetricType() == MetricType::GAUGE) metric_map.clear(); } void save(cereal::JSONOutputArchive &ar) const override { - ar(cereal::make_nvp(calc_title, metric_map)); + ar(cereal::make_nvp(getMetricName(), metric_map)); } + MetricType getMetricType() const override { return metric_map.getMetricType(); } template void report(const PrintableKey &key, const Values & ... new_values) { - was_once_reported = true; std::stringstream string_key; string_key << key; auto metric = metric_map.emplace(string_key.str(), Metric(nullptr, string_key.str())).first; @@ -84,7 +96,7 @@ public: LogField getLogField() const override { - LogField field(calc_title); + LogField field(getMetricName()); for (auto &metric : metric_map) { field.addFields(metric.second.getLogField()); diff --git a/core/include/services_sdk/resources/metric/metric_metadata.h b/core/include/services_sdk/resources/metric/metric_metadata.h new file mode 100644 index 0000000..2d5650d --- /dev/null +++ b/core/include/services_sdk/resources/metric/metric_metadata.h @@ -0,0 +1,43 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __METRIC_METADATA_H__ +#define __METRIC_METADATA_H__ + +#ifndef __GENERIC_METRIC_H__ +#error metric/metric_metadata.h should not be included directly +#endif // __GENERIC_METRIC_H_ + +#include + +namespace MetricMetadata +{ + +struct DotName +{ + std::string val; +}; + +struct Units +{ + std::string val; +}; + +struct Description +{ + std::string val; +}; + +} //MetricMetadata + +#endif // __METRIC_METADATA_H__ diff --git a/core/include/services_sdk/resources/metric/min.h b/core/include/services_sdk/resources/metric/min.h index 436f102..5a2a160 100644 --- a/core/include/services_sdk/resources/metric/min.h +++ b/core/include/services_sdk/resources/metric/min.h @@ -25,21 +25,29 @@ template class Min : public MetricCalc { public: - Min(GenericMetric *metric, const std::string &title) : Min(metric, title, std::numeric_limits::max()) {} - Min(GenericMetric *metric, const std::string &title, T max_val) : MetricCalc(metric, title), min(max_val) {} + Min(GenericMetric *metric, const std::string &title) : Min(metric, title, 0) {} + + template + Min(GenericMetric *metric, const std::string &title, T max_val, const Args & ... args) + : + MetricCalc(metric, title, args ...), + min(max_val), + reset_value(max_val) + { + } void report(const T &new_value) { - was_once_reported = true; - if (new_value < min) min = new_value; + if (new_value < min || first) min = new_value; + first = false; } void reset() override { - was_once_reported = false; - min = std::numeric_limits::max(); + min = reset_value; + first = true; } T @@ -51,17 +59,19 @@ public: void save(cereal::JSONOutputArchive &ar) const override { - ar(cereal::make_nvp(calc_title, min)); + ar(cereal::make_nvp(getMetricName(), min)); } LogField getLogField() const override { - return LogField(calc_title, static_cast(getMin())); + return LogField(getMetricName(), static_cast(getMin())); } private: T min; + T reset_value; + bool first = true; }; } // namespace MetricCalculations diff --git a/core/include/services_sdk/resources/metric/no_reset_counter.h b/core/include/services_sdk/resources/metric/no_reset_counter.h new file mode 100644 index 0000000..e1538de --- /dev/null +++ b/core/include/services_sdk/resources/metric/no_reset_counter.h @@ -0,0 +1,68 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __NO_RESET_COUNTER_H__ +#define __NO_RESET_COUNTER_H__ + +#ifndef __GENERIC_METRIC_H__ +#error metric/no_reset_counter.h should not be included directly +#endif // __GENERIC_METRIC_H_ + +namespace MetricCalculations +{ + +class NoResetCounter : public MetricCalc +{ +public: + template + NoResetCounter(GenericMetric *metric, const std::string &title, const Args & ... args) + : + MetricCalc(metric, title, args ...), + counter(0) + { + } + + void reset() override {} + MetricType getMetricType() const override { return MetricType::COUNTER; } + + uint64_t + getCounter() const + { + return counter; + } + + void + save(cereal::JSONOutputArchive &ar) const override + { + ar(cereal::make_nvp(getMetricName(), getCounter())); + } + + void + report(const uint64_t &new_value) + { + counter += new_value; + } + + LogField + getLogField() const override + { + return LogField(getMetricName(), static_cast(getCounter())); + } + +private: + uint64_t counter; +}; + +} // namespace MetricCalculations + +#endif // __NO_RESET_COUNTER_H__ diff --git a/core/include/services_sdk/resources/metric/top_values.h b/core/include/services_sdk/resources/metric/top_values.h index c6e9c79..b33d02f 100644 --- a/core/include/services_sdk/resources/metric/top_values.h +++ b/core/include/services_sdk/resources/metric/top_values.h @@ -28,12 +28,17 @@ template class TopValues : public MetricCalc { public: - TopValues(GenericMetric *metric, const std::string &title) : MetricCalc(metric, title) { values.reserve(N); } + template + TopValues(GenericMetric *metric, const std::string &title, const Args & ... args) + : + MetricCalc(metric, title, args ...) + { + values.reserve(N); + } void report(const T &new_value) { - was_once_reported = true; if (values.size() < N) { values.push_back(new_value); return; @@ -51,7 +56,6 @@ public: void reset() override { - was_once_reported = false; values.clear(); } @@ -66,13 +70,13 @@ public: void save(cereal::JSONOutputArchive &ar) const override { - ar(cereal::make_nvp(calc_title, getTopValues())); + ar(cereal::make_nvp(getMetricName(), getTopValues())); } LogField getLogField() const override { - return LogField(calc_title, getTopValues()); + return LogField(getMetricName(), getTopValues()); } private: diff --git a/core/include/services_sdk/resources/report/base_field.h b/core/include/services_sdk/resources/report/base_field.h index 90c4590..bf33788 100644 --- a/core/include/services_sdk/resources/report/base_field.h +++ b/core/include/services_sdk/resources/report/base_field.h @@ -162,7 +162,9 @@ class LogField : Singleton::Consume void addFields(const LogField &) { - dbgAssert(false) << "Trying to add a log field to a 'type'ed field"; + dbgAssert(false) + << AlertInfo(AlertTeam::CORE, "report i/s") + << "Trying to add a log field to a 'type'ed field"; } // LCOV_EXCL_STOP diff --git a/core/include/services_sdk/resources/report/report_bulks.h b/core/include/services_sdk/resources/report/report_bulks.h index fe5f0ab..61515cb 100644 --- a/core/include/services_sdk/resources/report/report_bulks.h +++ b/core/include/services_sdk/resources/report/report_bulks.h @@ -26,7 +26,7 @@ public: void setBulkSize(uint size) { - dbgAssert(size > 0) << "Bulk size must be larger than 0"; + dbgAssert(size > 0) << AlertInfo(AlertTeam::CORE, "report i/s") << "Bulk size must be larger than 0"; dbgDebug(D_REPORT_BULK) << "Bulk size is set to " << size; bulk_size = size; } diff --git a/components/include/report_messaging.h b/core/include/services_sdk/resources/report_messaging.h old mode 100755 new mode 100644 similarity index 100% rename from components/include/report_messaging.h rename to core/include/services_sdk/resources/report_messaging.h diff --git a/core/include/services_sdk/resources/service_health_update_event.h b/core/include/services_sdk/resources/service_health_update_event.h new file mode 100644 index 0000000..15faf60 --- /dev/null +++ b/core/include/services_sdk/resources/service_health_update_event.h @@ -0,0 +1,43 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __SERVICE_HEALTH_UPDATE_EVENT_H__ +#define __SERVICE_HEALTH_UPDATE_EVENT_H__ + +#include "event.h" +#include "singleton.h" +#include "config.h" +#include "debug.h" +#include "customized_cereal_map.h" + +class ServiceHealthUpdateEvent : public Event +{ +public: + ServiceHealthUpdateEvent() {} + ServiceHealthUpdateEvent( + const std::string &_component, + const std::string &_error) + : + component(_component), + error(_error) {} + + bool isHealthyUpdate() const { return component.empty(); } + const std::string & getComponent() const { return component; } + const std::string & getError() const { return error; } + +private: + std::string component = ""; + std::string error = ""; +}; + +#endif // __SERVICE_HEALTH_UPDATE_EVENT_H__ diff --git a/core/include/services_sdk/resources/table/i_table_impl.h b/core/include/services_sdk/resources/table/i_table_impl.h index f073c74..2e129a7 100644 --- a/core/include/services_sdk/resources/table/i_table_impl.h +++ b/core/include/services_sdk/resources/table/i_table_impl.h @@ -48,7 +48,10 @@ Opaque & I_Table::getState() { Opaque *ptr = static_cast(getState(typeid(Opaque))); - dbgAssert(ptr != nullptr) << "Trying to access a non existing opaque " << typeid(Opaque).name(); + dbgAssert(ptr != nullptr) + << AlertInfo(AlertTeam::CORE, "table") + << "Trying to access a non existing opaque " + << typeid(Opaque).name(); return *ptr; } diff --git a/core/include/services_sdk/utilities/agent_core_utilities.h b/core/include/services_sdk/utilities/agent_core_utilities.h index 34ef904..446230a 100644 --- a/core/include/services_sdk/utilities/agent_core_utilities.h +++ b/core/include/services_sdk/utilities/agent_core_utilities.h @@ -48,6 +48,8 @@ std::string convertToHumanReadable(uint64_t size_in_bytes); std::string getFileName(const std::string &path); +bool copyDirectory(const std::string &src_dir_path, const std::string &dst_dir_path); + }// namespace Filesystem namespace Regex diff --git a/core/include/services_sdk/utilities/connkey.h b/core/include/services_sdk/utilities/connkey.h index 09843c5..e0d3532 100644 --- a/core/include/services_sdk/utilities/connkey.h +++ b/core/include/services_sdk/utilities/connkey.h @@ -88,6 +88,7 @@ public: operator==(const IPAddr &other) const { dbgAssert(type!=IPType::UNINITIALIZED && other.type!=IPType::UNINITIALIZED) + << AlertInfo(AlertTeam::CORE, "connkey") << "Called on an uninitialized IPType object"; // Always compairing as if IPv6, in case of Ipv4 the rest of the address is zeroed out. int ip_len = (other.type == IPType::V4) ? sizeof(v4.s_addr) : sizeof(v6.s6_addr); @@ -307,7 +308,9 @@ public: IPType getType() const { - dbgAssert(src.type == dst.type) << "Mismatch in connection types (Src and Dst types are not identical)"; + dbgAssert(src.type == dst.type) + << AlertInfo(AlertTeam::CORE, "connkey") + << "Mismatch in connection types (Src and Dst types are not identical)"; return src.type; } diff --git a/core/include/services_sdk/utilities/rest/rest_param.h b/core/include/services_sdk/utilities/rest/rest_param.h index 672a801..f815562 100644 --- a/core/include/services_sdk/utilities/rest/rest_param.h +++ b/core/include/services_sdk/utilities/rest/rest_param.h @@ -27,13 +27,13 @@ public: operator T &() { - dbgAssert(is_active) << "Tried to access a non-existing variable"; + dbgAssert(is_active) << AlertInfo(AlertTeam::CORE, "rest i/s") << "Tried to access a non-existing variable"; return val; } operator const T &() const { - dbgAssert(is_active) << "Tried to access a non-existing variable"; + dbgAssert(is_active) << AlertInfo(AlertTeam::CORE, "rest i/s") << "Tried to access a non-existing variable"; return val; } diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc index fad66d5..35bb241 100644 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -34,6 +34,8 @@ static const string invalidation_uri = "/api/v2/intelligence/invalidation"; static const string registration_uri = "/api/v2/intelligence/invalidation/register"; static const string query_uri = "/api/v2/intelligence/assets/query"; static const string queries_uri = "/api/v2/intelligence/assets/queries"; +static const string fog_health_uri = "/access-manager/health/live"; +static const string intelligence_health_uri = "/show-health"; class I_InvalidationCallBack { @@ -291,6 +293,16 @@ private: I_MainLoop *mainloop; }; + +class IntelligenceHealth : public ClientRest +{ +public: + bool isLocalHealthy() const { return healthy.isActive() && healthy.get(); } + +private: + S2C_PARAM(bool, healthy); +}; + class IntelligenceComponentV2::Impl : Singleton::Provide::From @@ -314,6 +326,35 @@ public: rest_api->addRestCall(RestAction::SET, "new-invalidation/source/invalidation"); } + bool + isIntelligenceHealthy() const override + { + dbgFlow(D_INTELLIGENCE) << "Checking intelligence health"; + IntelligenceHealth healthObj; + if (hasLocalIntelligenceSupport()) { + dbgDebug(D_INTELLIGENCE) << "Local intelligence supported"; + return sendLocalIntelligenceToLocalServer(healthObj).ok(); + } + dbgTrace(D_INTELLIGENCE) << "Checking connection to the FOG"; + auto response = message->sendSyncMessage( + HTTPMethod::GET, + fog_health_uri, + string(""), + MessageCategory::INTELLIGENCE + ); + + if (response.ok() && response.unpack().getHTTPStatusCode() == HTTPStatusCode::HTTP_OK) { + dbgTrace(D_INTELLIGENCE) << "Connected to the FOG"; + return true; + } + + dbgTrace(D_INTELLIGENCE) + << "No connection to the FOG. " + << (response.ok() ? response.unpack() : response.getErr()).toString(); + + return false; + } + bool sendInvalidation(const Invalidation &invalidation) const override { @@ -487,6 +528,36 @@ private: return genError("Could not send local intelligence invalidation"); } + Maybe + sendIntelligenceRequestImpl(const IntelligenceHealth &, const MessageMetadata &req_md) const + { + dbgFlow(D_INTELLIGENCE) << "Sending intelligence health check"; + IntelligenceHealth healthObj; + auto req_data = message->sendSyncMessage( + HTTPMethod::GET, + intelligence_health_uri, + healthObj, + MessageCategory::INTELLIGENCE, + req_md + ); + if (req_data.ok() && healthObj.isLocalHealthy()) { + dbgDebug(D_INTELLIGENCE) << "Intelligence is healthy."; + return Response(); + } + + if (!req_data.ok()) { + dbgDebug(D_INTELLIGENCE) + << "Intelligence is not healthy. Body: " + << req_data.getErr().getBody() + << " Error: " + << req_data.getErr().toString(); + } else { + dbgDebug(D_INTELLIGENCE) << "Intelligence return unhealthy status."; + } + + return genError("Intelligence is not healthy"); + } + Maybe sendIntelligenceRequestImpl( const InvalidationRegistration::RestCall ®istration, diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc index df12734..9957550 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc @@ -1341,3 +1341,59 @@ TEST_F(IntelligenceComponentTestV2, ignoreInProgressQueryTest_2) EXPECT_EQ(objects_ids.size(), 2u); } + +TEST_F(IntelligenceComponentTestV2, foghealthy) +{ + Debug::setUnitTestFlag(D_INTELLIGENCE, Debug::DebugLevel::TRACE); + I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); + + HTTPResponse fog_res( + HTTPStatusCode::HTTP_OK, + string( + "{" + " \"up\": true," + " \"timestamp\":\"\"" + "}" + ) + ); + + EXPECT_CALL( + messaging_mock, + sendSyncMessage(HTTPMethod::GET, "/access-manager/health/live", _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(fog_res)); + + EXPECT_TRUE(intell->isIntelligenceHealthy()); +} + +TEST_F(IntelligenceComponentTestV2, localIntelligenceHealthy) +{ + Debug::setUnitTestFlag(D_INTELLIGENCE, Debug::DebugLevel::TRACE); + stringstream configuration; + configuration << "{"; + configuration << " \"agentSettings\":["; + configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}"; + configuration << " ],"; + configuration << " \"intelligence\":{"; + configuration << " \"local intelligence server ip\":\"127.0.0.1\","; + configuration << " \"local intelligence server primary port\":9090"; + configuration << " }"; + configuration << "}"; + Singleton::Consume::from(conf)->loadConfiguration(configuration); + + I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); + + string localHealthy( + "{\n" + " \"healthy\": true\n" + "}\n" + ); + + EXPECT_CALL( + messaging_mock, + sendSyncMessage(HTTPMethod::GET, "/show-health", _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, localHealthy))); + + EXPECT_CALL(mock_rest, getListeningPort()).Times(1).WillRepeatedly(Return(8888)); + + EXPECT_TRUE(intell->isIntelligenceHealthy()); +} diff --git a/core/mainloop/coroutine.cc b/core/mainloop/coroutine.cc index 89a7486..c73dd6e 100644 --- a/core/mainloop/coroutine.cc +++ b/core/mainloop/coroutine.cc @@ -76,7 +76,10 @@ RoutineWrapper::resume() void RoutineWrapper::invoke(pull_type &pull, I_MainLoop::Routine func) { - dbgAssert(active != nullptr) << "Trying to invoke without an active routine"; + dbgAssert(active != nullptr) + << AlertInfo(AlertTeam::CORE, "mainloop i/s") + << "Trying to invoke without an active routine"; + active->pull = move(pull); // First invokation (other invokaction will start inside `func`), set the `pull` object func(); } diff --git a/core/mainloop/mainloop.cc b/core/mainloop/mainloop.cc index 0a15d70..36630b7 100644 --- a/core/mainloop/mainloop.cc +++ b/core/mainloop/mainloop.cc @@ -34,6 +34,8 @@ USE_DEBUG_FLAG(D_MAINLOOP); bool fini_signal_flag = false; +static const AlertInfo alert(AlertTeam::CORE, "mainloop i/s"); + class MainloopStop {}; class MainloopComponent::Impl : Singleton::Provide::From @@ -225,7 +227,7 @@ MainloopComponent::Impl::reportStartupEvent() void MainloopComponent::Impl::run() { - dbgAssert(!is_running) << "MainloopComponent::Impl::run was called while it was already running"; + dbgAssert(!is_running) << alert << "MainloopComponent::Impl::run was called while it was already running"; is_running = true; bool has_primary_routines = true; @@ -465,7 +467,7 @@ MainloopComponent::Impl::getCurrentRoutineId() const void MainloopComponent::Impl::yield(bool force) { - dbgAssert(curr_iter != routines.end()) << "Calling 'yield' without a running current routine"; + dbgAssert(curr_iter != routines.end()) << alert << "Calling 'yield' without a running current routine"; if (do_stop) throw MainloopStop(); if (!force && getTimer()->getMonotonicTime() < stop_time) return; @@ -506,7 +508,7 @@ MainloopComponent::Impl::stopAll() void MainloopComponent::Impl::stop() { - dbgAssert(curr_iter != routines.end()) << "Attempting to stop a routine when none is running"; + dbgAssert(curr_iter != routines.end()) << alert << "Attempting to stop a routine when none is running"; stop(curr_iter); } @@ -524,7 +526,7 @@ MainloopComponent::Impl::stop(RoutineID id) void MainloopComponent::Impl::halt() { - dbgAssert(curr_iter != routines.end()) << "Calling 'halt' without a running current routine"; + dbgAssert(curr_iter != routines.end()) << alert << "Calling 'halt' without a running current routine"; curr_iter->second.halt(); yield(true); } @@ -533,7 +535,7 @@ void MainloopComponent::Impl::halt(RoutineID id) { auto iter = routines.find(id); - dbgAssert(iter != routines.end()) << "No routine " << id << " to halt"; + dbgAssert(iter != routines.end()) << alert << "No routine " << id << " to halt"; iter->second.halt(); if (iter == curr_iter) yield(true); } @@ -542,7 +544,7 @@ void MainloopComponent::Impl::resume(RoutineID id) { auto iter = routines.find(id); - dbgAssert(iter != routines.end()) << "No routine " << id << " to resume"; + dbgAssert(iter != routines.end()) << alert << "No routine " << id << " to resume"; iter->second.resume(); } diff --git a/core/messaging/include/dummy_socket.h b/core/messaging/include/dummy_socket.h index 92cf4d2..af273a1 100644 --- a/core/messaging/include/dummy_socket.h +++ b/core/messaging/include/dummy_socket.h @@ -39,7 +39,7 @@ public: init() { server_fd = socket(AF_INET, SOCK_STREAM, 0); - dbgAssert(server_fd >= 0) << "Failed to open a socket"; + dbgAssert(server_fd >= 0) << AlertInfo(AlertTeam::CORE, "messaging i/s") << "Failed to open a socket"; int socket_enable = 1; setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)); diff --git a/core/messaging/messaging_comp/http_response.cc b/core/messaging/messaging_comp/http_response.cc index 3fcf2d9..37440fc 100644 --- a/core/messaging/messaging_comp/http_response.cc +++ b/core/messaging/messaging_comp/http_response.cc @@ -92,7 +92,10 @@ string HTTPResponse::toString() const { auto code = status_code_to_string.find(status_code); - dbgAssert(code != status_code_to_string.end()) << "Unknown status code " << int(status_code); + dbgAssert(code != status_code_to_string.end()) + << AlertInfo(AlertTeam::CORE, "messaging i/s") + << "Unknown status code " + << int(status_code); return "[Status-code]: " + code->second + ", [Body]: " + (body.empty() ? "{}" : body); } diff --git a/core/metric/generic_metric.cc b/core/metric/generic_metric.cc index 004953a..15f9098 100644 --- a/core/metric/generic_metric.cc +++ b/core/metric/generic_metric.cc @@ -26,7 +26,29 @@ using namespace ReportIS; USE_DEBUG_FLAG(D_METRICS); -MetricCalc::MetricCalc(GenericMetric *metric, const string &title) : calc_title(title) +MetricMetadata::DotName operator"" _dot(const char *str, size_t) { return MetricMetadata::DotName{str}; } +MetricMetadata::Units operator"" _unit(const char *str, size_t) { return MetricMetadata::Units{str}; } +MetricMetadata::Description operator"" _desc(const char *str, size_t) { return MetricMetadata::Description{str}; } + +string +MetricCalc::getMetadata(const string &key) const +{ + auto value = metadata.find(key); + return value != metadata.end() ? value->second : ""; +} + +void +MetricCalc::setMetadata(const string &key, const string &value) +{ + if (value.empty()) { + metadata.erase(key); + } else { + metadata[key] = value; + } +} + +void +MetricCalc::addMetric(GenericMetric *metric) { // Only top level metric should add themselves to the metric. Nested metrics will be served by their parent. if (metric != nullptr) metric->addCalc(this); @@ -69,6 +91,9 @@ GenericMetric::init( bool _force_buffering ) { + turnOnStream(Stream::FOG); + turnOnStream(Stream::DEBUG); + i_mainloop = Singleton::Consume::by(); i_time = Singleton::Consume::by(); metric_name = _metric_name; @@ -95,22 +120,10 @@ GenericMetric::init( void GenericMetric::handleMetricStreamSending() { - auto metric_debug = getConfigurationWithDefault(true, "metric", "debugMetricSendEnable"); - auto report_str = generateReport(false); - if (!report_str.empty() && metric_debug) { - dbgTrace(D_METRICS) << report_str; - } + if (active_streams.isSet(Stream::DEBUG)) generateDebug(); + if (active_streams.isSet(Stream::FOG)) generateLog(); - auto metric_fog = getConfigurationWithDefault(true, "metric", "fogMetricSendEnable"); - if (!report_str.empty() && metric_fog) { - generateLog(); - } - - if (reset) { - for(auto &calc : calcs) { - calc->reset(); - } - } + if (reset) resetMetrics(); } string @@ -126,24 +139,26 @@ GenericMetric::getReportInterval() const } string -GenericMetric::generateReport(bool with_reset) +GenericMetric::generateReport() const { stringstream ss; - bool any_reported_calc = false; - { cereal::JSONOutputArchive ar(ss); ar(cereal::make_nvp("Metric", metric_name)); ar(cereal::make_nvp("Reporting interval", report_interval.count())); for(auto &calc : calcs) { - if (calc->wasOnceReported()) { - calc->save(ar); - if (with_reset) calc->reset(); - any_reported_calc = true; - } + calc->save(ar); } } - return any_reported_calc ? ss.str() : ""; + return ss.str(); +} + +void +GenericMetric::resetMetrics() +{ + for(auto &calc : calcs) { + calc->reset(); + } } void @@ -155,16 +170,16 @@ GenericMetric::addCalc(MetricCalc *calc) void GenericMetric::upon(const AllMetricEvent &event) { - auto report_str = generateReport(event.getReset()); - if (!report_str.empty()) { - dbgTrace(D_METRICS) << report_str; - } + dbgTrace(D_METRICS) << generateReport(); + if (event.getReset()) resetMetrics(); } string GenericMetric::respond(const AllMetricEvent &event) { - return generateReport(event.getReset()); + auto res = generateReport(); + if (event.getReset()) resetMetrics(); + return res; } string GenericMetric::getListenerName() const { return metric_name; } @@ -172,6 +187,8 @@ string GenericMetric::getListenerName() const { return metric_name; } void GenericMetric::generateLog() { + if (!getConfigurationWithDefault(true, "metric", "fogMetricSendEnable")) return; + set tags; Report metric_to_fog( metric_name, @@ -191,7 +208,7 @@ GenericMetric::generateLog() ); for (auto &calc : calcs) { - if (calc->wasOnceReported()) metric_to_fog << calc->getLogField(); + metric_to_fog << calc->getLogField(); } if (Singleton::exists()) { @@ -227,6 +244,13 @@ GenericMetric::generateLog() sendLog(metric_client_rest); } +void +GenericMetric::generateDebug() +{ + if (!getConfigurationWithDefault(true, "metric", "debugMetricSendEnable")) return; + dbgTrace(D_METRICS) << generateReport(); +} + void GenericMetric::sendLog(const LogRest &metric_client_rest) const { diff --git a/core/metric/metric_ut/metric_ut.cc b/core/metric/metric_ut/metric_ut.cc index c71c27c..e96520c 100644 --- a/core/metric/metric_ut/metric_ut.cc +++ b/core/metric/metric_ut/metric_ut.cc @@ -20,6 +20,16 @@ using namespace MetricCalculations; USE_DEBUG_FLAG(D_METRICS); +TEST(BaseMetric, generic_metadata) +{ + Max test(nullptr, "cpuMax", 0, "cpu.max"_dot, "percent"_unit, "CPU utilization percentage"_desc); + + EXPECT_EQ(test.getMetricName(), "cpuMax"); + EXPECT_EQ(test.getMetricDotName(), "cpu.max"); + EXPECT_EQ(test.getMetircUnits(), "percent"); + EXPECT_EQ(test.getMetircDescription(), "CPU utilization percentage"); +} + class CPUEvent : public Event { public: @@ -53,6 +63,7 @@ public: last_report.report(event.getCPU()); avg.report(event.getCPU()); samples_counter.report(1); + total_samples_counter.report(1); top_usage.report(event.getCPU()); } @@ -61,6 +72,7 @@ public: Average avg{this, "cpuAvg"}; LastReportedValue last_report{this, "cpuCurrent"}; Counter samples_counter{this, "cpuCounter"}; + NoResetCounter total_samples_counter{this, "cpuTotalCounter"}; TopValues top_usage{this, "cpuTops"}; }; @@ -129,10 +141,12 @@ public: upon(const HttpTransaction &event) override { avg.report(event.getUrl(), event.getBytes()); + total.report(event.getUrl(), 1); } private: MetricMap> avg{this, "PerUrlAvg"}; + MetricMap total{this, "TotalRequests"}; }; class MetricTest : public Test @@ -189,7 +203,6 @@ TEST_F(MetricTest, basicMetricTest) EXPECT_EQ(cpu_mt.getReportInterval().count(), 5); routine(); - EXPECT_EQ(debug_output.str(), ""); CPUEvent cpu_event; cpu_event.setProcessCPU(89); @@ -204,6 +217,7 @@ TEST_F(MetricTest, basicMetricTest) " \"cpuAvg\": 89.0,\n" " \"cpuCurrent\": 89.0,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 1,\n" " \"cpuTops\": [\n" " 89.0\n" " ]\n" @@ -249,6 +263,7 @@ TEST_F(MetricTest, basicMetricTest) " \"cpuAvg\": 89,\n" " \"cpuCurrent\": 89,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 1,\n" " \"cpuTops\": [\n" " 89.0\n" " ]\n" @@ -273,6 +288,7 @@ TEST_F(MetricTest, basicMetricTest) " \"cpuAvg\": 89.5,\n" " \"cpuCurrent\": 90.0,\n" " \"cpuCounter\": 2,\n" + " \"cpuTotalCounter\": 2,\n" " \"cpuTops\": [\n" " 89.0,\n" " 90.0\n" @@ -318,6 +334,7 @@ TEST_F(MetricTest, basicMetricTest) " \"cpuAvg\": 89,\n" " \"cpuCurrent\": 90,\n" " \"cpuCounter\": 2,\n" + " \"cpuTotalCounter\": 2,\n" " \"cpuTops\": [\n" " 89.0,\n" " 90.0\n" @@ -343,6 +360,7 @@ TEST_F(MetricTest, basicMetricTest) " \"cpuAvg\": 93.0,\n" " \"cpuCurrent\": 100.0,\n" " \"cpuCounter\": 3,\n" + " \"cpuTotalCounter\": 3,\n" " \"cpuTops\": [\n" " 89.0,\n" " 90.0,\n" @@ -389,6 +407,7 @@ TEST_F(MetricTest, basicMetricTest) " \"cpuAvg\": 93,\n" " \"cpuCurrent\": 100,\n" " \"cpuCounter\": 3,\n" + " \"cpuTotalCounter\": 3,\n" " \"cpuTops\": [\n" " 89.0,\n" " 90.0,\n" @@ -449,6 +468,7 @@ TEST_F(MetricTest, printMetricsTest) " \"cpuAvg\": 89.0,\n" " \"cpuCurrent\": 89.0,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 1,\n" " \"cpuTops\": [\n" " 89.0\n" " ]\n" @@ -481,7 +501,6 @@ TEST_F(MetricTest, metricTestWithReset) EXPECT_EQ(cpu_mt.getReportInterval().count(), 5); routine(); - EXPECT_EQ(debug_output.str(), ""); CPUEvent cpu_event; cpu_event.setProcessCPU(89); @@ -496,6 +515,7 @@ TEST_F(MetricTest, metricTestWithReset) " \"cpuAvg\": 89.0,\n" " \"cpuCurrent\": 89.0,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 1,\n" " \"cpuTops\": [\n" " 89.0\n" " ]\n" @@ -542,6 +562,7 @@ TEST_F(MetricTest, metricTestWithReset) " \"cpuAvg\": 89,\n" " \"cpuCurrent\": 89,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 1,\n" " \"cpuTops\": [\n" " 89.0\n" " ]\n" @@ -566,6 +587,7 @@ TEST_F(MetricTest, metricTestWithReset) " \"cpuAvg\": 90.0,\n" " \"cpuCurrent\": 90.0,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 2,\n" " \"cpuTops\": [\n" " 90.0\n" " ]\n" @@ -610,6 +632,7 @@ TEST_F(MetricTest, metricTestWithReset) " \"cpuAvg\": 90,\n" " \"cpuCurrent\": 90,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 2,\n" " \"cpuTops\": [\n" " 90.0\n" " ]\n" @@ -634,6 +657,7 @@ TEST_F(MetricTest, metricTestWithReset) " \"cpuAvg\": 100.0,\n" " \"cpuCurrent\": 100.0,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 3,\n" " \"cpuTops\": [\n" " 100.0\n" " ]\n" @@ -678,6 +702,7 @@ TEST_F(MetricTest, metricTestWithReset) " \"cpuAvg\": 100,\n" " \"cpuCurrent\": 100,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 3,\n" " \"cpuTops\": [\n" " 100.0\n" " ]\n" @@ -707,7 +732,17 @@ TEST_F(MetricTest, generateReportWithReset) EXPECT_EQ(cpu_mt.getReportInterval().count(), 5); routine(); - EXPECT_EQ(debug_output.str(), ""); + auto init_report = cpu_mt.generateReport(); + + EXPECT_NE(init_report, ""); + + EXPECT_THAT(init_report, HasSubstr("\"Metric\": \"CPU usage\"")); + EXPECT_THAT(init_report, HasSubstr("\"Reporting interval\": 5,")); + EXPECT_THAT(init_report, HasSubstr("cpuMax")); + EXPECT_THAT(init_report, HasSubstr("cpuMin")); + EXPECT_THAT(init_report, HasSubstr("cpuAvg")); + EXPECT_THAT(init_report, HasSubstr("cpuCurrent")); + EXPECT_THAT(init_report, HasSubstr("cpuTops")); CPUEvent cpu_event; cpu_event.setProcessCPU(89); @@ -722,6 +757,7 @@ TEST_F(MetricTest, generateReportWithReset) " \"cpuAvg\": 89.0,\n" " \"cpuCurrent\": 89.0,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 1,\n" " \"cpuTops\": [\n" " 89.0\n" " ]\n" @@ -768,6 +804,7 @@ TEST_F(MetricTest, generateReportWithReset) " \"cpuAvg\": 89,\n" " \"cpuCurrent\": 89,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 1,\n" " \"cpuTops\": [\n" " 89.0\n" " ]\n" @@ -779,13 +816,28 @@ TEST_F(MetricTest, generateReportWithReset) EXPECT_EQ(message_body, expected_message); debug_output.str(""); - auto report = cpu_mt.generateReport(true); + auto report = cpu_mt.generateReport(); + cpu_mt.resetMetrics(); EXPECT_THAT(metric_str, HasSubstr(report)); - EXPECT_EQ(message_body, expected_message); + debug_output.str(""); + + report = cpu_mt.generateReport(); + metric_str = + "{\n" + " \"Metric\": \"CPU usage\",\n" + " \"Reporting interval\": 5,\n" + " \"cpuMax\": 0.0,\n" + " \"cpuMin\": 0.0,\n" + " \"cpuAvg\": 0.0,\n" + " \"cpuCurrent\": 0.0,\n" + " \"cpuCounter\": 0,\n" + " \"cpuTotalCounter\": 1,\n" + " \"cpuTops\": []\n" + "}"; + EXPECT_EQ(report, metric_str); debug_output.str(""); routine(); - EXPECT_EQ(debug_output.str(), ""); cpu_event.setProcessCPU(90); cpu_event.notify(); @@ -799,6 +851,7 @@ TEST_F(MetricTest, generateReportWithReset) " \"cpuAvg\": 90.0,\n" " \"cpuCurrent\": 90.0,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 2,\n" " \"cpuTops\": [\n" " 90.0\n" " ]\n" @@ -843,6 +896,7 @@ TEST_F(MetricTest, generateReportWithReset) " \"cpuAvg\": 90,\n" " \"cpuCurrent\": 90,\n" " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 2,\n" " \"cpuTops\": [\n" " 90.0\n" " ]\n" @@ -867,6 +921,7 @@ TEST_F(MetricTest, generateReportWithReset) " \"cpuAvg\": 95.0,\n" " \"cpuCurrent\": 100.0,\n" " \"cpuCounter\": 2,\n" + " \"cpuTotalCounter\": 3,\n" " \"cpuTops\": [\n" " 90.0,\n" " 100.0\n" @@ -912,6 +967,7 @@ TEST_F(MetricTest, generateReportWithReset) " \"cpuAvg\": 95,\n" " \"cpuCurrent\": 100,\n" " \"cpuCounter\": 2,\n" + " \"cpuTotalCounter\": 3,\n" " \"cpuTops\": [\n" " 90.0,\n" " 100.0\n" @@ -987,6 +1043,7 @@ TEST_F(MetricTest, allMetricTest) " \"cpuAvg\": 93.0,\n" " \"cpuCurrent\": 100.0,\n" " \"cpuCounter\": 3,\n" + " \"cpuTotalCounter\": 3,\n" " \"cpuTops\": [\n" " 89.0,\n" " 90.0,\n" @@ -1040,6 +1097,24 @@ TEST_F(MetricTest, testMapMetric) " \"PerUrlAvg\": {\n" " \"/index.html\": 25.0,\n" " \"/index2.html\": 20.0\n" + " },\n" + " \"TotalRequests\": {\n" + " \"/index.html\": 2,\n" + " \"/index2.html\": 1\n" + " }\n" + "}"; + EXPECT_THAT(debug_output.str(), HasSubstr(msg_str)); + + debug_output.str(""); + routine(); + msg_str = + "{\n" + " \"Metric\": \"Bytes per URL\",\n" + " \"Reporting interval\": 5,\n" + " \"PerUrlAvg\": {},\n" + " \"TotalRequests\": {\n" + " \"/index.html\": 2,\n" + " \"/index2.html\": 1\n" " }\n" "}"; EXPECT_THAT(debug_output.str(), HasSubstr(msg_str)); diff --git a/core/report/tag_and_enum_management.cc b/core/report/tag_and_enum_management.cc index 173301e..c6ee59f 100644 --- a/core/report/tag_and_enum_management.cc +++ b/core/report/tag_and_enum_management.cc @@ -19,6 +19,8 @@ using namespace ReportIS; #include +static const AlertInfo alert(AlertTeam::CORE, "report i/s"); + Maybe TagAndEnumManagement::convertStringToSeverity(const string &severity) { @@ -158,7 +160,7 @@ TagAndEnumManagement::convertToString(const StreamType &stream_type) case StreamType::COUNT: break; } - dbgAssert(false) << "Unknown log stream type. Type: " << static_cast(stream_type); + dbgAssert(false) << alert << "Unknown log stream type. Type: " << static_cast(stream_type); return ""; } @@ -173,7 +175,7 @@ TagAndEnumManagement::convertToString(const Severity &severity) case Severity::INFO: return "Info"; } - dbgAssert(false) << "Reached an impossible severity value of: " << static_cast(severity); + dbgAssert(false) << alert << "Reached an impossible severity value of: " << static_cast(severity); return ""; } @@ -186,7 +188,7 @@ TagAndEnumManagement::convertToString(const Type &type) case Type::CODE: return "Code Related"; } - dbgAssert(false) << "Reached an impossible type value of: " << static_cast(type); + dbgAssert(false) << alert << "Reached an impossible type value of: " << static_cast(type); return ""; } @@ -201,7 +203,7 @@ TagAndEnumManagement::convertToString(const Level &level) case Level::CUSTOM: return "Custom"; } - dbgAssert(false) << "Reached an impossible type value of: " << static_cast(level); + dbgAssert(false) << alert << "Reached an impossible type value of: " << static_cast(level); return ""; } @@ -216,7 +218,7 @@ TagAndEnumManagement::convertToString(const LogLevel &log_level) case LogLevel::ERROR: return "error"; } - dbgAssert(false) << "Reached an impossible type value of: " << static_cast(log_level); + dbgAssert(false) << alert << "Reached an impossible type value of: " << static_cast(log_level); return ""; } @@ -228,7 +230,7 @@ TagAndEnumManagement::convertToString(const Audience &audience) case Audience::INTERNAL: return "Internal"; } - dbgAssert(false) << "Reached an impossible audience value of: " << static_cast(audience); + dbgAssert(false) << alert << "Reached an impossible audience value of: " << static_cast(audience); return ""; } @@ -242,7 +244,7 @@ TagAndEnumManagement::convertToString(const Priority &priority) case Priority::LOW: return "Low"; } - dbgAssert(false) << "Reached impossible priority value of: " << static_cast(priority); + dbgAssert(false) << alert << "Reached impossible priority value of: " << static_cast(priority); return ""; } @@ -261,7 +263,7 @@ TagAndEnumManagement::convertToString(const Notification ¬ification) case Notification::SDWAN_POLICY_WARNING_LOG: return "c58d490e-6aa0-43da-bfaa-7edad0a57b7a"; } - dbgAssert(false) << "Reached impossible notification value of: " << static_cast(notification); + dbgAssert(false) << alert << "Reached impossible notification value of: " << static_cast(notification); return ""; } @@ -279,7 +281,7 @@ TagAndEnumManagement::convertToString(const IssuingEngine &issuing_engine) case IssuingEngine::HORIZON_TELEMETRY_METRICS: return "horizonTelemetryMetrics"; } - dbgAssert(false) << "Reached impossible engine value of: " << static_cast(issuing_engine); + dbgAssert(false) << alert << "Reached impossible engine value of: " << static_cast(issuing_engine); return ""; } diff --git a/components/report_messaging/CMakeLists.txt b/core/report_messaging/CMakeLists.txt old mode 100644 new mode 100755 similarity index 98% rename from components/report_messaging/CMakeLists.txt rename to core/report_messaging/CMakeLists.txt index 7c9597d..48d3c07 --- a/components/report_messaging/CMakeLists.txt +++ b/core/report_messaging/CMakeLists.txt @@ -1,3 +1,2 @@ add_library(report_messaging report_messaging.cc) - add_subdirectory(report_messaging_ut) diff --git a/components/report_messaging/report_messaging.cc b/core/report_messaging/report_messaging.cc old mode 100755 new mode 100644 similarity index 100% rename from components/report_messaging/report_messaging.cc rename to core/report_messaging/report_messaging.cc diff --git a/core/report_messaging/report_messaging_ut/CMakeLists.txt b/core/report_messaging/report_messaging_ut/CMakeLists.txt new file mode 100755 index 0000000..0786f1a --- /dev/null +++ b/core/report_messaging/report_messaging_ut/CMakeLists.txt @@ -0,0 +1,7 @@ +link_directories(${BOOST_ROOT}/lib) + +add_unit_test( + report_messaging_ut + "report_messaging_ut.cc" + "report_messaging;report;singleton;-lboost_regex;messaging;" +) diff --git a/components/report_messaging/report_messaging_ut/report_messaging_ut.cc b/core/report_messaging/report_messaging_ut/report_messaging_ut.cc similarity index 100% rename from components/report_messaging/report_messaging_ut/report_messaging_ut.cc rename to core/report_messaging/report_messaging_ut/report_messaging_ut.cc diff --git a/core/rest/i_rest_invoke.h b/core/rest/i_rest_invoke.h index b9f8877..4ae053b 100644 --- a/core/rest/i_rest_invoke.h +++ b/core/rest/i_rest_invoke.h @@ -25,6 +25,9 @@ public: virtual Maybe getSchema(const std::string &uri) const = 0; virtual Maybe invokeRest(const std::string &uri, std::istream &in) const = 0; + virtual bool isGetCall(const std::string &uri) const = 0; + virtual std::string invokeGet(const std::string &uri) const = 0; + protected: ~I_RestInvoke() {} }; diff --git a/core/rest/rest_conn.cc b/core/rest/rest_conn.cc index ff49807..4d06140 100644 --- a/core/rest/rest_conn.cc +++ b/core/rest/rest_conn.cc @@ -97,6 +97,10 @@ RestConn::parseConn() const stop(); } + if (method=="GET" && invoke->isGetCall(identifier)) { + return sendResponse("200 OK", invoke->invokeGet(identifier), false); + } + stringstream body; body.str(readSize(len)); @@ -152,15 +156,16 @@ RestConn::readSize(int len) const } void -RestConn::sendResponse(const string &status, const string &body) const +RestConn::sendResponse(const string &status, const string &body, bool add_newline) const { stringstream stream; stream << "HTTP/1.1 " << status << "\r\n" << "Content-Type: application/json\r\n" << - "Content-Length: " << (body.size() + 2) << "\r\n" << + "Content-Length: " << (body.size() + (add_newline ? 2 : 0)) << "\r\n" << "\r\n" << - body << "\r\n"; + body; + if (add_newline) stream << "\r\n"; string res = stream.str(); diff --git a/core/rest/rest_conn.h b/core/rest/rest_conn.h index 29bafa9..bb7ecfa 100644 --- a/core/rest/rest_conn.h +++ b/core/rest/rest_conn.h @@ -30,7 +30,7 @@ private: void stop() const; std::string readLine() const; std::string readSize(int len) const; - void sendResponse(const std::string &status, const std::string &body) const; + void sendResponse(const std::string &status, const std::string &body, bool add_newline = true) const; int fd; I_MainLoop *mainloop; diff --git a/core/rest/rest_server.cc b/core/rest/rest_server.cc index a751fc0..2ce76c8 100644 --- a/core/rest/rest_server.cc +++ b/core/rest/rest_server.cc @@ -31,6 +31,7 @@ USE_DEBUG_FLAG(D_API); static const int listen_limit = 100; static const chrono::milliseconds bind_retry_interval_msec = chrono::milliseconds(500); +static const AlertInfo alert(AlertTeam::CORE, "rest i/s"); #include @@ -47,9 +48,12 @@ public: bool bindRestServerSocket(struct sockaddr_in &addr, vector port_range); bool addRestCall(RestAction oper, const string &uri, unique_ptr &&init) override; + bool addGetCall(const string &uri, const function &cb) override; uint16_t getListeningPort() const override { return listening_port; } - Maybe getSchema(const std::string &uri) const override; - Maybe invokeRest(const std::string &uri, istream &in) const override; + Maybe getSchema(const string &uri) const override; + Maybe invokeRest(const string &uri, istream &in) const override; + bool isGetCall(const string &uri) const override; + string invokeGet(const string &uri) const override; private: void prepareConfiguration(); @@ -61,6 +65,7 @@ private: I_MainLoop::RoutineID id; I_MainLoop *mainloop; map> rest_calls; + map> get_calls; uint16_t listening_port = 0; vector port_range; }; @@ -96,8 +101,10 @@ RestServer::Impl::prepareConfiguration() } else { auto range_start = getPortConfig("Nano service API Port Range start"); auto range_end = getPortConfig("Nano service API Port Range end"); - dbgAssert(range_start.ok() && range_end.ok()) << "Rest port configuration was not provided"; - dbgAssert(*range_start < *range_end) << "Rest port range corrupted (lower bound higher then upper bound)"; + dbgAssert(range_start.ok() && range_end.ok()) << alert << "Rest port configuration was not provided"; + dbgAssert(*range_start < *range_end) + << alert + << "Rest port range corrupted (lower bound higher then upper bound)"; port_range.resize(*range_end - *range_start); for (uint16_t i = 0, port = *range_start; i < port_range.size(); i++, port++) { @@ -113,7 +120,7 @@ RestServer::Impl::init() auto init_connection = [this] () { fd = socket(AF_INET, SOCK_STREAM, 0); - dbgAssert(fd >= 0) << "Failed to open a socket"; + dbgAssert(fd >= 0) << alert << "Failed to open a socket"; int socket_enable = 1; if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)) < 0) { dbgWarning(D_API) << "Could not set the socket options"; @@ -185,11 +192,19 @@ bool RestServer::Impl::addRestCall(RestAction oper, const string &uri, unique_ptr &&rest) { string full_uri = changeActionToString(oper) + uri; + if (get_calls.find(full_uri) != get_calls.end()) return false; return rest_calls.emplace(make_pair(full_uri, move(rest))).second; } -Maybe -RestServer::Impl::getSchema(const std::string &uri) const +bool +RestServer::Impl::addGetCall(const string &uri, const function &callback) +{ + if (rest_calls.find(uri) != rest_calls.end()) return false; + return get_calls.emplace(uri, callback).second; +} + +Maybe +RestServer::Impl::getSchema(const string &uri) const { auto iter = rest_calls.find(uri); if (iter == rest_calls.end()) return genError("No matching REST call was found"); @@ -200,8 +215,8 @@ RestServer::Impl::getSchema(const std::string &uri) const return out.str(); } -Maybe -RestServer::Impl::invokeRest(const std::string &uri, istream &in) const +Maybe +RestServer::Impl::invokeRest(const string &uri, istream &in) const { auto iter = rest_calls.find(uri); if (iter == rest_calls.end()) return genError("No matching REST call was found"); @@ -209,6 +224,19 @@ RestServer::Impl::invokeRest(const std::string &uri, istream &in) const return instance->performRestCall(in); } +bool +RestServer::Impl::isGetCall(const string &uri) const +{ + return get_calls.find(uri) != get_calls.end(); +} + +string +RestServer::Impl::invokeGet(const string &uri) const +{ + auto instance = get_calls.find(uri); + return instance != get_calls.end() ? instance->second() : ""; +} + string RestServer::Impl::changeActionToString(RestAction oper) { @@ -226,7 +254,7 @@ RestServer::Impl::changeActionToString(RestAction oper) return "delete-"; } default: { - dbgAssert(false) << "Unknown REST action"; + dbgAssert(false) << alert << "Unknown REST action"; return ""; } } diff --git a/core/rest/rest_ut/rest_config_ut.cc b/core/rest/rest_ut/rest_config_ut.cc index a130cb0..e699f10 100644 --- a/core/rest/rest_ut/rest_config_ut.cc +++ b/core/rest/rest_ut/rest_config_ut.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include "cptest.h" #include "environment.h" @@ -143,9 +144,12 @@ TEST_F(RestConfigTest, basic_flow) auto i_rest = Singleton::Consume::from(rest_server); ASSERT_TRUE(i_rest->addRestCall(RestAction::ADD, "test")); + ASSERT_TRUE(i_rest->addGetCall("stuff", [] () { return string("blabla"); })); - int file_descriptor = socket(AF_INET, SOCK_STREAM, 0); - EXPECT_NE(file_descriptor, -1); + int file_descriptor1 = socket(AF_INET, SOCK_STREAM, 0); + EXPECT_NE(file_descriptor1, -1); + int file_descriptor2 = socket(AF_INET, SOCK_STREAM, 0); + EXPECT_NE(file_descriptor2, -1); auto primary_port = getConfiguration("connection", "Nano service API Port Alternative"); struct sockaddr_in sa; @@ -153,20 +157,34 @@ TEST_F(RestConfigTest, basic_flow) sa.sin_port = htons(primary_port.unpack()); sa.sin_addr.s_addr = inet_addr("127.0.0.1"); int socket_enable = 1; - EXPECT_EQ(setsockopt(file_descriptor, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0); + EXPECT_EQ(setsockopt(file_descriptor1, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0); + EXPECT_EQ(setsockopt(file_descriptor2, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0); EXPECT_CALL(messaging, sendSyncMessage(_, _, _, _, _)) .WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); auto mainloop = Singleton::Consume::from(mainloop_comp); I_MainLoop::Routine stop_routine = [&] () { - EXPECT_EQ(connect(file_descriptor, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0); - string msg = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}"; - EXPECT_EQ(write(file_descriptor, msg.data(), msg.size()), static_cast(msg.size())); + EXPECT_EQ(connect(file_descriptor1, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0); + string msg1 = "GET /stuff HTTP/1.1\r\n\r\n"; + EXPECT_EQ(write(file_descriptor1, msg1.data(), msg1.size()), static_cast(msg1.size())); + + EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0); + string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}"; + EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast(msg2.size())); while(!TestServer::g_num) { mainloop->yield(true); } + + struct pollfd s_poll; + s_poll.fd = file_descriptor1; + s_poll.events = POLLIN; + s_poll.revents = 0; + while(poll(&s_poll, 1, 0) <= 0) { + mainloop->yield(true); + } + mainloop->stopAll(); }; mainloop->addOneTimeRoutine( @@ -178,4 +196,11 @@ TEST_F(RestConfigTest, basic_flow) mainloop->run(); EXPECT_EQ(TestServer::g_num, 5); + + char respose[1000]; + EXPECT_EQ(read(file_descriptor1, respose, 1000), 76); + EXPECT_EQ( + string(respose, 76), + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 6\r\n\r\nblabla" + ); } diff --git a/core/singleton/singleton.cc b/core/singleton/singleton.cc index c7815db..8323002 100644 --- a/core/singleton/singleton.cc +++ b/core/singleton/singleton.cc @@ -29,8 +29,13 @@ Singleton::unregisterSingleton(std::type_index type, void *ptr) void * Singleton::get(const std::type_index &type) { - dbgAssert(singles[type].size() == 1) << "There is no single element from type '" << type.name() << "', " - "number of elements is " << singles[type].size(); + dbgAssert(singles[type].size() == 1) + << AlertInfo(AlertTeam::CORE, "singleton i/s") + << "There is no single element from type '" + << type.name() + << "', number of elements is " + << singles[type].size(); + return *(singles[type].begin()); } diff --git a/core/socket_is/socket_is.cc b/core/socket_is/socket_is.cc index aa95a24..9cd1e46 100644 --- a/core/socket_is/socket_is.cc +++ b/core/socket_is/socket_is.cc @@ -28,6 +28,7 @@ #include "debug.h" static const uint udp_max_packet_size = 1024 * 64; +static const AlertInfo alert(AlertTeam::CORE, "socket i/s"); USE_DEBUG_FLAG(D_SOCKET); @@ -174,8 +175,8 @@ public: Maybe> acceptConn(bool is_blocking, const string &authorized_ip = "") { - dbgAssert(is_server_socket) << "Failed to accept new connections from a client socket"; - dbgAssert(socket_int > 0) << "Called with uninitialized server socket"; + dbgAssert(is_server_socket) << alert << "Failed to accept new connections from a client socket"; + dbgAssert(socket_int > 0) << alert << "Called with uninitialized server socket"; dbgDebug(D_SOCKET) << "Attempt to accept new socket. Server Socket FD: " << socket_int; int client_socket; diff --git a/core/time_proxy/time_proxy.cc b/core/time_proxy/time_proxy.cc index 339e101..42d18e6 100644 --- a/core/time_proxy/time_proxy.cc +++ b/core/time_proxy/time_proxy.cc @@ -52,7 +52,9 @@ public: setMonotonicTime(microseconds new_time) override { if (is_monotomic_set) { - dbgAssert((new_time+monotonic_delta) >= monotonic_now) << "Monotonic time must not go back!"; + dbgAssert((new_time+monotonic_delta) >= monotonic_now) + << AlertInfo(AlertTeam::CORE, "time proxy") + << "Monotonic time must not go back!"; } else { // The first time that the monotonic time is been set, we take the current value to be the base line. // This is in order to avoid the clock going backwards. diff --git a/nodes/orchestration/CMakeLists.txt b/nodes/orchestration/CMakeLists.txt index 2c2c484..229208b 100755 --- a/nodes/orchestration/CMakeLists.txt +++ b/nodes/orchestration/CMakeLists.txt @@ -31,6 +31,7 @@ target_link_libraries( local_policy_mgmt_gen curl external_sdk_server + service_health_status -Wl,--end-group ) diff --git a/nodes/orchestration/main.cc b/nodes/orchestration/main.cc index 8e99890..da03cd3 100755 --- a/nodes/orchestration/main.cc +++ b/nodes/orchestration/main.cc @@ -52,6 +52,7 @@ #include "tenant_manager.h" #include "local_policy_mgmt_gen.h" #include "external_sdk_server.h" +#include "service_health_status.h" using namespace std; @@ -74,7 +75,8 @@ main(int argc, char **argv) HealthChecker, HealthCheckManager, LocalPolicyMgmtGenerator, - ExternalSdkServer + ExternalSdkServer, + ServiceHealthStatus > comps; comps.registerGlobalValue("Nano service API Port Primary", 7777); diff --git a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc index 39bbc37..3d43480 100755 --- a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc +++ b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc @@ -92,6 +92,7 @@ enum class Service { HTTP_TRANSACTION_HANDLER, DEDICATED_NETWORK_HANDLER, HELLO_WORLD, + PROMETHEUS, IDA_SAML, IDA_IDN, IDA_IDN_BG, @@ -170,6 +171,7 @@ getServiceString(const Service service) case (Service::LOGGER_SDWAN): return "logger-sdwan"; case (Service::IOT_WLP): return "workload-protection"; case (Service::HELLO_WORLD): return "hello-world"; + case (Service::PROMETHEUS): return "prometheus"; case (Service::IDA_SAML): return "ida-saml"; case (Service::IDA_IDN): return "ida-idn"; case (Service::IDA_IDN_BG): return "ida-idn-bg"; @@ -363,6 +365,11 @@ getServiceConfig (const Service service) filesystem_path + "/conf/cp-nano-hello-world-conf.json", log_files_path + "/nano_agent/cp-nano-hello-world.dbg" ); + case (Service::PROMETHEUS): + return ServiceConfig( + filesystem_path + "/conf/cp-nano-prometheus-debug-conf.json", + log_files_path + "/nano_agent/cp-nano-prometheus.dbg" + ); case (Service::IOT_ACCESS_CONTROL): return ServiceConfig( filesystem_path + "/conf/cp-nano-iot-access-control-debug-conf.json", @@ -1308,6 +1315,8 @@ extractServices(const vector &args) services.push_back(Service::IOT_ACCESS_CONTROL); } else if (getServiceString(Service::HORIZON_TELEMETRY).find(maybe_service) == 0) { services.push_back(Service::HORIZON_TELEMETRY); + } else if (getServiceString(Service::PROMETHEUS).find(maybe_service) == 0) { + services.push_back(Service::PROMETHEUS); } else { break; } diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index 92cd4c5..6813ed3 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -971,12 +971,15 @@ install_orchestration() if [ -n "${OTP_TOKEN}" ]; then cp_print "Saving authentication token to file" printf '{\n "registration type": "token",\n "registration data": "%b"\n}' "$OTP_TOKEN" | ${FILESYSTEM_PATH}/${BIN_PATH}/${CP_NANO_BASE64} -e > ${FILESYSTEM_PATH}/${CONF_PATH}/registration-data.json - rm ${FILESYSTEM_PATH}/${CONF_PATH}/agent_details.json - rm ${FILESYSTEM_PATH}/${CONF_PATH}/orchestration_status.json - echo '{}'>${FILESYSTEM_PATH}/${CONF_PATH}/policy.json + previous_mode=$(awk -F\" '/Orchestration mode/{print $4}' /etc/cp/conf/agent_details.json) + if [ "$previous_mode" = "hybrid_mode" ]; then + rm ${FILESYSTEM_PATH}/${CONF_PATH}/agent_details.json + rm ${FILESYSTEM_PATH}/${CONF_PATH}/orchestration_status.json + echo '{}'>${FILESYSTEM_PATH}/${CONF_PATH}/policy.json - if [ -f ${FILESYSTEM_PATH}/data/data5.a ]; then - rm ${FILESYSTEM_PATH}/data/data5.a + if [ -f ${FILESYSTEM_PATH}/data/data5.a ]; then + rm ${FILESYSTEM_PATH}/data/data5.a + fi fi fi