From 4dacd7d009db5c697ec4f3d080ca982a73b708fa Mon Sep 17 00:00:00 2001 From: Daniel-Eisenberg <59121493+Daniel-Eisenberg@users.noreply.github.com> Date: Thu, 5 Jun 2025 16:28:57 +0300 Subject: [PATCH] Prometheus support (#316) * Add prometheus support * Add prometheus support * Add prometheus support * Add prometheus support * Add prometheus support --------- Co-authored-by: avigailo --- build_system/docker/entry.sh | 5 + components/include/prometheus_comp.h | 30 +++ components/security_apps/CMakeLists.txt | 1 + .../security_apps/prometheus/CMakeLists.txt | 2 + .../prometheus/prometheus_comp.cc | 200 ++++++++++++++++++ .../prometheus/prometheus_metric_names.h | 143 +++++++++++++ .../prometheus/prometheus_ut/CMakeLists.txt | 8 + .../prometheus/prometheus_ut/prometheus_ut.cc | 79 +++++++ .../prometheus_ut/registered_services.json | 32 +++ core/metric/generic_metric.cc | 12 +- nodes/CMakeLists.txt | 1 + .../package/cp-nano-package-list | 1 + .../package/orchestration_package.sh | 3 + nodes/prometheus/CMakeLists.txt | 30 +++ nodes/prometheus/main.cc | 15 ++ nodes/prometheus/package/CMakeLists.txt | 4 + .../package/cp-nano-prometheus-conf.json | 21 ++ .../cp-nano-prometheus-debug-conf.json | 11 + .../prometheus/package/cp-nano-prometheus.cfg | 0 .../package/install-cp-nano-prometheus.sh | 164 ++++++++++++++ 20 files changed, 761 insertions(+), 1 deletion(-) create mode 100755 components/include/prometheus_comp.h create mode 100755 components/security_apps/prometheus/CMakeLists.txt create mode 100755 components/security_apps/prometheus/prometheus_comp.cc create mode 100755 components/security_apps/prometheus/prometheus_metric_names.h create mode 100755 components/security_apps/prometheus/prometheus_ut/CMakeLists.txt create mode 100755 components/security_apps/prometheus/prometheus_ut/prometheus_ut.cc create mode 100755 components/security_apps/prometheus/prometheus_ut/registered_services.json create mode 100755 nodes/prometheus/CMakeLists.txt create mode 100755 nodes/prometheus/main.cc create mode 100755 nodes/prometheus/package/CMakeLists.txt create mode 100755 nodes/prometheus/package/cp-nano-prometheus-conf.json create mode 100755 nodes/prometheus/package/cp-nano-prometheus-debug-conf.json create mode 100755 nodes/prometheus/package/cp-nano-prometheus.cfg create mode 100755 nodes/prometheus/package/install-cp-nano-prometheus.sh diff --git a/build_system/docker/entry.sh b/build_system/docker/entry.sh index e74c7a6..f698034 100644 --- a/build_system/docker/entry.sh +++ b/build_system/docker/entry.sh @@ -6,6 +6,7 @@ HTTP_TRANSACTION_HANDLER_SERVICE="install-cp-nano-service-http-transaction-handl ATTACHMENT_REGISTRATION_SERVICE="install-cp-nano-attachment-registration-manager.sh" ORCHESTRATION_INSTALLATION_SCRIPT="install-cp-nano-agent.sh" CACHE_INSTALLATION_SCRIPT="install-cp-nano-agent-cache.sh" +PROMETHEUS_INSTALLATION_SCRIPT="install-cp-nano-service-prometheus.sh" var_fog_address= var_proxy= @@ -81,6 +82,10 @@ fi /nano-service-installers/$CACHE_INSTALLATION_SCRIPT --install /nano-service-installers/$HTTP_TRANSACTION_HANDLER_SERVICE --install +if [ "$PROMETHEUS" == "true" ]; then + /nano-service-installers/$PROMETHEUS_INSTALLATION_SCRIPT --install +fi + if [ "$CROWDSEC_ENABLED" == "true" ]; then /nano-service-installers/$INTELLIGENCE_INSTALLATION_SCRIPT --install /nano-service-installers/$CROWDSEC_INSTALLATION_SCRIPT --install diff --git a/components/include/prometheus_comp.h b/components/include/prometheus_comp.h new file mode 100755 index 0000000..abb0bae --- /dev/null +++ b/components/include/prometheus_comp.h @@ -0,0 +1,30 @@ +#ifndef __PROMETHEUS_COMP_H__ +#define __PROMETHEUS_COMP_H__ + +#include + +#include "component.h" +#include "singleton.h" + +#include "i_rest_api.h" +#include "i_messaging.h" +#include "generic_metric.h" + +class PrometheusComp + : + public Component, + Singleton::Consume, + Singleton::Consume +{ +public: + PrometheusComp(); + ~PrometheusComp(); + + void init() override; + +private: + class Impl; + std::unique_ptr pimpl; +}; + +#endif // __PROMETHEUS_COMP_H__ diff --git a/components/security_apps/CMakeLists.txt b/components/security_apps/CMakeLists.txt index 206262c..0518698 100644 --- a/components/security_apps/CMakeLists.txt +++ b/components/security_apps/CMakeLists.txt @@ -3,6 +3,7 @@ add_subdirectory(ips) add_subdirectory(layer_7_access_control) add_subdirectory(local_policy_mgmt_gen) add_subdirectory(orchestration) +add_subdirectory(prometheus) add_subdirectory(rate_limit) add_subdirectory(waap) add_subdirectory(central_nginx_manager) diff --git a/components/security_apps/prometheus/CMakeLists.txt b/components/security_apps/prometheus/CMakeLists.txt new file mode 100755 index 0000000..9879b39 --- /dev/null +++ b/components/security_apps/prometheus/CMakeLists.txt @@ -0,0 +1,2 @@ +add_library(prometheus_comp prometheus_comp.cc) +add_subdirectory(prometheus_ut) diff --git a/components/security_apps/prometheus/prometheus_comp.cc b/components/security_apps/prometheus/prometheus_comp.cc new file mode 100755 index 0000000..5b06f2e --- /dev/null +++ b/components/security_apps/prometheus/prometheus_comp.cc @@ -0,0 +1,200 @@ +#include "prometheus_comp.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "report/base_field.h" +#include "report/report_enums.h" +#include "log_generator.h" +#include "debug.h" +#include "rest.h" +#include "customized_cereal_map.h" +#include "i_messaging.h" +#include "prometheus_metric_names.h" + +USE_DEBUG_FLAG(D_PROMETHEUS); + +using namespace std; +using namespace ReportIS; + +struct ServiceData +{ + template + void + serialize(Archive &ar) + { + ar(cereal::make_nvp("Service port", service_port)); + } + + int service_port; +}; + +class PrometheusMetricData +{ +public: + PrometheusMetricData(const string &n, const string &t, const string &d) : name(n), type(t), description(d) {} + + void + addElement(const string &labels, const string &value) + { + metric_labels_to_values[labels] = value; + } + + ostream & + print(ostream &os) + { + if (metric_labels_to_values.empty()) return os; + + string representative_name = ""; + if (!name.empty()) { + auto metric_name = convertMetricName(name); + !metric_name.empty() ? representative_name = metric_name : representative_name = name; + } + + if (!description.empty()) os << "# HELP " << representative_name << ' ' << description << '\n'; + if (!name.empty()) os << "# TYPE " << representative_name << ' ' << type << '\n'; + for (auto &entry : metric_labels_to_values) { + os << representative_name << entry.first << ' ' << entry.second << '\n'; + } + os << '\n'; + metric_labels_to_values.clear(); + + return os; + } + +private: + + string name; + string type; + string description; + map metric_labels_to_values; +}; + +static ostream & operator<<(ostream &os, PrometheusMetricData &metric) { return metric.print(os); } + +class PrometheusComp::Impl +{ +public: + void + init() + { + Singleton::Consume::by()->addGetCall( + "metrics", + [&] () { return getFormatedPrometheusMetrics(); } + ); + } + + void + addMetrics(const vector &metrics) + { + for(auto &metric : metrics) { + auto &metric_object = getDataObject( + metric.name, + metric.type, + metric.description + ); + metric_object.addElement(metric.label, metric.value); + } + } + +private: + PrometheusMetricData & + getDataObject(const string &name, const string &type, const string &description) + { + auto elem = prometheus_metrics.find(name); + if (elem == prometheus_metrics.end()) { + elem = prometheus_metrics.emplace(name, PrometheusMetricData(name, type, description)).first; + } + + return elem->second; + } + + map + getServiceDetails() + { + map registeredServices; + auto registered_services_file = getConfigurationWithDefault( + getFilesystemPathConfig() + "/conf/orchestrations_registered_services.json", + "orchestration", + "Orchestration registered services" + ); + ifstream file(registered_services_file); + if (!file.is_open()) { + dbgWarning(D_PROMETHEUS) << "Failed to open file: " << registered_services_file; + return registeredServices; + } + stringstream buffer; + buffer << file.rdbuf(); + try { + cereal::JSONInputArchive archive(buffer); + archive(cereal::make_nvp("Registered Services", registeredServices)); + } catch (const exception& e) { + dbgWarning(D_PROMETHEUS) << "Error parsing Registered Services JSON file: " << e.what(); + } + + return registeredServices; + } + + void + getServicesMetrics() + { + dbgTrace(D_PROMETHEUS) << "Get all registered services metrics"; + map service_names_to_ports = getServiceDetails(); + for (const auto &service : service_names_to_ports) { + I_Messaging *messaging = Singleton::Consume::by(); + MessageMetadata servie_metric_req_md("127.0.0.1", service.second.service_port); + servie_metric_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); + servie_metric_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + auto res = messaging->sendSyncMessage( + HTTPMethod::GET, + "/service-metrics", + string(""), + MessageCategory::GENERIC, + servie_metric_req_md + ); + if (!res.ok()) { + dbgWarning(D_PROMETHEUS) << "Failed to get service metrics. Service: " << service.first; + continue; + } + stringstream buffer; + buffer << res.unpack().getBody(); + cereal::JSONInputArchive archive(buffer); + vector metrics; + archive(cereal::make_nvp("metrics", metrics)); + addMetrics(metrics); + } + } + + string + getFormatedPrometheusMetrics() + { + MetricScrapeEvent().notify(); + getServicesMetrics(); + stringstream result; + for (auto &metric : prometheus_metrics) { + result << metric.second; + } + dbgTrace(D_PROMETHEUS) << "Prometheus metrics: " << result.str(); + return result.str(); + } + + map prometheus_metrics; +}; + +PrometheusComp::PrometheusComp() : Component("Prometheus"), pimpl(make_unique()) {} + +PrometheusComp::~PrometheusComp() {} + +void +PrometheusComp::init() +{ + pimpl->init(); +} diff --git a/components/security_apps/prometheus/prometheus_metric_names.h b/components/security_apps/prometheus/prometheus_metric_names.h new file mode 100755 index 0000000..1a5fdf0 --- /dev/null +++ b/components/security_apps/prometheus/prometheus_metric_names.h @@ -0,0 +1,143 @@ +#ifndef __PROMETHEUS_METRIC_NAMES_H__ +#define __PROMETHEUS_METRIC_NAMES_H__ + +#include +#include + +#include "debug.h" + +USE_DEBUG_FLAG(D_PROMETHEUS); + +std::string +convertMetricName(const std::string &original_metric_name) +{ + static const std::unordered_map original_to_representative_names = { + // HybridModeMetric + {"watchdogProcessStartupEventsSum", "nano_service_restarts_counter"}, + // nginxAttachmentMetric + {"inspectVerdictSum", "traffic_inspection_verdict_inspect_counter"}, + {"acceptVeridctSum", "traffic_inspection_verdict_accept_counter"}, + {"dropVerdictSum", "traffic_inspection_verdict_drop_counter"}, + {"injectVerdictSum", "traffic_inspection_verdict_inject_counter"}, + {"irrelevantVerdictSum", "traffic_inspection_verdict_irrelevant_counter"}, + {"irrelevantVerdictSum", "traffic_inspection_verdict_irrelevant_counter"}, + {"reconfVerdictSum", "traffic_inspection_verdict_reconf_counter"}, + {"responseInspection", "response_body_inspection_counter"}, + // nginxIntakerMetric + {"successfullInspectionTransactionsSum", "successful_Inspection_counter"}, + {"failopenTransactionsSum", "fail_open_Inspection_counter"}, + {"failcloseTransactionsSum", "fail_close_Inspection_counter"}, + {"transparentModeTransactionsSum", "transparent_mode_counter"}, + {"totalTimeInTransparentModeSum", "total_time_in_transparent_mode_counter"}, + {"reachInspectVerdictSum", "inspect_verdict_counter"}, + {"reachAcceptVerdictSum", "accept_verdict_counter"}, + {"reachDropVerdictSum", "drop_verdict_counter"}, + {"reachInjectVerdictSum", "inject_verdict_counter"}, + {"reachIrrelevantVerdictSum", "irrelevant_verdict_counter"}, + {"reachReconfVerdictSum", "reconf_verdict_counter"}, + {"requestCompressionFailureSum", "failed_requests_compression_counter"}, + {"responseCompressionFailureSum", "failed_response_compression_counter"}, + {"requestDecompressionFailureSum", "failed_requests_decompression_counter"}, + {"responseDecompressionFailureSum", "failed_response_decompression_counter"}, + {"requestCompressionSuccessSum", "successful_request_compression_counter"}, + {"responseCompressionSuccessSum", "successful_response_compression_counter"}, + {"requestDecompressionSuccessSum", "successful_request_decompression_counter"}, + {"responseDecompressionSuccessSum", "successful_response_decompression_counter"}, + {"skippedSessionsUponCorruptedZipSum", "corrupted_zip_skipped_session_counter"}, + {"attachmentThreadReachedTimeoutSum", "thread_exceeded_processing_time_counter"}, + {"registrationThreadReachedTimeoutSum", "failed_registration_thread_counter"}, + {"requestHeaderThreadReachedTimeoutSum", "request_headers_processing_thread_timeouts_counter"}, + {"requestBodyThreadReachedTimeoutSum", "request_body_processing_thread_timeouts_counter"}, + {"respondHeaderThreadReachedTimeoutSum", "response_headers_processing_thread_timeouts_counter"}, + {"respondBodyThreadReachedTimeoutSum", "response_body_processing_thread_timeouts_counter"}, + {"attachmentThreadFailureSum", "thread_failures_counter"}, + {"httpRequestProcessingReachedTimeoutSum", "request_processing_timeouts_counter"}, + {"httpRequestsSizeSum", "requests_total_size_counter"}, + {"httpResponsesSizeSum", "response_total_size_counter"}, + {"httpRequestFailedToReachWebServerUpstreamSum", "requests_failed_reach_upstram_counter"}, + {"overallSessionProcessTimeToVerdictAvgSample", "overall_processing_time_until_verdict_average"}, + {"overallSessionProcessTimeToVerdictMaxSample", "overall_processing_time_until_verdict_max"}, + {"overallSessionProcessTimeToVerdictMinSample", "overall_processing_time_until_verdict_min"}, + {"requestProcessTimeToVerdictAvgSample", "requests_processing_time_until_verdict_average"}, + {"requestProcessTimeToVerdictMaxSample", "requests_processing_time_until_verdict_max"}, + {"requestProcessTimeToVerdictMinSample", "requests_processing_time_until_verdict_min"}, + {"responseProcessTimeToVerdictAvgSample", "response_processing_time_until_verdict_average"}, + {"responseProcessTimeToVerdictMaxSample", "response_processing_time_until_verdict_max"}, + {"responseProcessTimeToVerdictMinSample", "response_processing_time_until_verdict_min"}, + {"requestBodySizeUponTimeoutAvgSample", "request_body_size_average"}, + {"requestBodySizeUponTimeoutMaxSample", "request_body_size_max"}, + {"requestBodySizeUponTimeoutMinSample", "request_body_size_min"}, + {"responseBodySizeUponTimeoutAvgSample", "response_body_size_average"}, + {"responseBodySizeUponTimeoutMaxSample", "response_body_size_max"}, + {"responseBodySizeUponTimeoutMinSample", "response_body_size_min"}, + // WaapTelemetrics + {"reservedNgenA", "total_requests_counter"}, + {"reservedNgenB", "unique_sources_counter"}, + {"reservedNgenC", "requests_blocked_by_force_and_exception_counter"}, + {"reservedNgenD", "requests_blocked_by_waf_counter"}, + {"reservedNgenE", "requests_blocked_by_open_api_counter"}, + {"reservedNgenF", "requests_blocked_by_bot_protection_counter"}, + {"reservedNgenG", "requests_threat_level_info_and_no_threat_counter"}, + {"reservedNgenH", "requests_threat_level_low_counter"}, + {"reservedNgenI", "requests_threat_level_medium_counter"}, + {"reservedNgenJ", "requests_threat_level_high_counter"}, + // WaapTrafficTelemetrics + {"reservedNgenA", "post_requests_counter"}, + {"reservedNgenB", "get_requests_counter"}, + {"reservedNgenC", "put_requests_counter"}, + {"reservedNgenD", "patch_requests_counter"}, + {"reservedNgenE", "delete_requests_counter"}, + {"reservedNgenF", "other_requests_counter"}, + {"reservedNgenG", "2xx_status_code_responses_counter"}, + {"reservedNgenH", "4xx_status_code_responses_counter"}, + {"reservedNgenI", "5xx_status_code_responses_counter"}, + {"reservedNgenJ", "requests_time_latency_average"}, + // WaapAttackTypesMetrics + {"reservedNgenA", "sql_injection_attacks_type_counter"}, + {"reservedNgenB", "vulnerability_scanning_attacks_type_counter"}, + {"reservedNgenC", "path_traversal_attacks_type_counter"}, + {"reservedNgenD", "ldap_injection_attacks_type_counter"}, + {"reservedNgenE", "evasion_techniques_attacks_type_counter"}, + {"reservedNgenF", "remote_code_execution_attacks_type_counter"}, + {"reservedNgenG", "xml_extern_entity_attacks_type_counter"}, + {"reservedNgenH", "cross_site_scripting_attacks_type_counter"}, + {"reservedNgenI", "general_attacks_type_counter"}, + // AssetsMetric + {"numberOfProtectedApiAssetsSample", "api_assets_counter"}, + {"numberOfProtectedWebAppAssetsSample", "web_api_assets_counter"}, + {"numberOfProtectedAssetsSample", "all_assets_counter"}, + // IPSMetric + {"preventEngineMatchesSample", "prevent_action_matches_counter"}, + {"detectEngineMatchesSample", "detect_action_matches_counter"}, + {"ignoreEngineMatchesSample", "ignore_action_matches_counter"}, + // CPUMetric + {"cpuMaxSample", "cpu_usage_percentage_max"}, + {"cpuAvgSample", "cpu_usage_percentage_average"}, + {"cpuSample", "cpu_usage_percentage_last_value"}, + // LogMetric + {"logQueueMaxSizeSample", "logs_queue_size_max"}, + {"logQueueAvgSizeSample", "logs_queue_size_average"}, + {"logQueueCurrentSizeSample", "logs_queue_size_last_value"}, + {"sentLogsSum", "logs_sent_counter"}, + {"sentLogsBulksSum", "bulk_logs_sent_counter"}, + // MemoryMetric + {"serviceVirtualMemorySizeMaxSample", "service_virtual_memory_size_kb_max"}, + {"serviceVirtualMemorySizeMinSample", "service_virtual_memory_size_kb_min"}, + {"serviceVirtualMemorySizeAvgSample", "service_virtual_memory_size_kb_average"}, + {"serviceRssMemorySizeMaxSample", "service_physical_memory_size_kb_max"}, + {"serviceRssMemorySizeMinSample", "service_physical_memory_size_kb_min"}, + {"serviceRssMemorySizeAvgSample", "service_physical_memory_size_kb_average"}, + {"generalTotalMemorySizeMaxSample", "general_total_used_memory_max"}, + {"generalTotalMemorySizeMinSample", "general_total_used_memory_min"}, + {"generalTotalMemorySizeAvgSample", "general_total_used_memory_average"}, + }; + + auto metric_names = original_to_representative_names.find(original_metric_name); + if (metric_names != original_to_representative_names.end()) return metric_names->second; + dbgDebug(D_PROMETHEUS) + << "Metric don't have a representative name, originl name: " + << original_metric_name; + return ""; +} + +#endif // __PROMETHEUS_METRIC_NAMES_H__ diff --git a/components/security_apps/prometheus/prometheus_ut/CMakeLists.txt b/components/security_apps/prometheus/prometheus_ut/CMakeLists.txt new file mode 100755 index 0000000..b190188 --- /dev/null +++ b/components/security_apps/prometheus/prometheus_ut/CMakeLists.txt @@ -0,0 +1,8 @@ +link_directories(${BOOST_ROOT}/lib) +link_directories(${BOOST_ROOT}/lib ${CMAKE_BINARY_DIR}/core/shmem_ipc) + +add_unit_test( + prometheus_ut + "prometheus_ut.cc" + "prometheus_comp;logging;agent_details;waap_clib;table;singleton;time_proxy;metric;event_is;connkey;http_transaction_data;generic_rulebase;generic_rulebase_evaluators;ip_utilities;intelligence_is_v2;-lboost_regex;messaging;" +) diff --git a/components/security_apps/prometheus/prometheus_ut/prometheus_ut.cc b/components/security_apps/prometheus/prometheus_ut/prometheus_ut.cc new file mode 100755 index 0000000..f021450 --- /dev/null +++ b/components/security_apps/prometheus/prometheus_ut/prometheus_ut.cc @@ -0,0 +1,79 @@ +#include "prometheus_comp.h" + +#include +#include +#include + +#include "cmock.h" +#include "cptest.h" +#include "maybe_res.h" +#include "debug.h" +#include "config.h" +#include "environment.h" +#include "config_component.h" +#include "agent_details.h" +#include "time_proxy.h" +#include "mock/mock_mainloop.h" +#include "mock/mock_rest_api.h" +#include "mock/mock_messaging.h" + +using namespace std; +using namespace testing; + +USE_DEBUG_FLAG(D_PROMETHEUS); + +class PrometheusCompTest : public Test +{ +public: + PrometheusCompTest() + { + EXPECT_CALL(mock_rest, mockRestCall(_, "declare-boolean-variable", _)).WillOnce(Return(false)); + env.preload(); + config.preload(); + env.init(); + + EXPECT_CALL( + mock_rest, + addGetCall("metrics", _) + ).WillOnce(DoAll(SaveArg<1>(&get_metrics_func), Return(true))); + + prometheus_comp.init(); + } + + ::Environment env; + ConfigComponent config; + PrometheusComp prometheus_comp; + StrictMock mock_rest; + StrictMock mock_ml; + NiceMock mock_messaging; + unique_ptr agent_uninstall; + function get_metrics_func; + CPTestTempfile status_file; + string registered_services_file_path; + +}; + +TEST_F(PrometheusCompTest, checkAddingMetric) +{ + registered_services_file_path = cptestFnameInSrcDir(string("registered_services.json")); + setConfiguration(registered_services_file_path, "orchestration", "Orchestration registered services"); + string metric_body = "{\n" + " \"metrics\": [\n" + " {\n" + " \"metric_name\": \"watchdogProcessStartupEventsSum\",\n" + " \"metric_type\": \"counter\",\n" + " \"metric_description\": \"\",\n" + " \"labels\": \"{method=\\\"post\\\",code=\\\"200\\\"}\",\n" + " \"value\": \"1534\"\n" + " }\n" + " ]\n" + "}"; + + string message_body; + EXPECT_CALL(mock_messaging, sendSyncMessage(_, "/service-metrics", _, _, _)) + .Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, metric_body))); + + string metric_str = "# TYPE nano_service_restarts_counter counter\n" + "nano_service_restarts_counter{method=\"post\",code=\"200\"} 1534\n\n"; + EXPECT_EQ(metric_str, get_metrics_func()); +} diff --git a/components/security_apps/prometheus/prometheus_ut/registered_services.json b/components/security_apps/prometheus/prometheus_ut/registered_services.json new file mode 100755 index 0000000..98a0c10 --- /dev/null +++ b/components/security_apps/prometheus/prometheus_ut/registered_services.json @@ -0,0 +1,32 @@ +{ + "Registered Services": { + "cp-nano-orchestration": { + "Service name": "cp-nano-orchestration", + "Service ID": "cp-nano-orchestration", + "Service port": 7777, + "Relevant configs": [ + "zones", + "triggers", + "rules", + "registration-data", + "parameters", + "orchestration", + "exceptions", + "agent-intelligence" + ] + }, + "cp-nano-prometheus": { + "Service name": "cp-nano-prometheus", + "Service ID": "cp-nano-prometheus", + "Service port": 7465, + "Relevant configs": [ + "zones", + "triggers", + "rules", + "parameters", + "exceptions", + "agent-intelligence" + ] + } + } +} diff --git a/core/metric/generic_metric.cc b/core/metric/generic_metric.cc index 271b81d..014a002 100644 --- a/core/metric/generic_metric.cc +++ b/core/metric/generic_metric.cc @@ -332,7 +332,17 @@ vector GenericMetric::getPromMetricsData() { vector all_metrics; - if (!getProfileAgentSettingWithDefault(false, "prometheus")) return all_metrics; + bool enable_prometheus = false; + auto prometheus_settings = getProfileAgentSetting("prometheus"); + if (prometheus_settings.ok()) { + enable_prometheus = prometheus_settings.unpack(); + } else { + const char *prometheus_env = getenv("PROMETHEUS"); + if (prometheus_env != nullptr) { + enable_prometheus = string(prometheus_env) == "true"; + } + } + if (!enable_prometheus) return all_metrics; dbgTrace(D_METRICS) << "Get prometheus metrics"; for (auto &calc : prometheus_calcs) { diff --git a/nodes/CMakeLists.txt b/nodes/CMakeLists.txt index 85ee401..c24c6da 100644 --- a/nodes/CMakeLists.txt +++ b/nodes/CMakeLists.txt @@ -11,6 +11,7 @@ set(COMMON_LIBRARIES "-lngen_core;-lcompression_utils;-lssl;-lcrypto;-lz;-lboost include(packaging.cmake) add_subdirectory(orchestration) +add_subdirectory(prometheus) add_subdirectory(agent_cache) add_subdirectory(http_transaction_handler) add_subdirectory(attachment_registration_manager) diff --git a/nodes/orchestration/package/cp-nano-package-list b/nodes/orchestration/package/cp-nano-package-list index 170f1b3..a6c317f 100644 --- a/nodes/orchestration/package/cp-nano-package-list +++ b/nodes/orchestration/package/cp-nano-package-list @@ -29,4 +29,5 @@ cpview_metric_provider="cpviewMetricProvider 8282" hello_world="hello_world" crowdsec_aux="crowdsecAux 8081" central_nginx_manager="centralNginxManager 7555" +prometheus="prometheus 7465" # ## Please do not remove this comment - newline at end of file required. diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index 64ae1cf..d81eaf5 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -647,6 +647,9 @@ install_watchdog() echo "ExecStart=ip netns exec CTX0000${VS_ID} ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} fi echo "Environment=\"FILESYSTEM_PATH=${FILESYSTEM_PATH}\"" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} + if [ -n "${PROMETHEUS}" ] ; then + echo "Environment=\"PROMETHEUS=${PROMETHEUS}\"" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} + fi cp_exec "systemctl daemon-reload" cp_exec "systemctl enable nano_agent" diff --git a/nodes/prometheus/CMakeLists.txt b/nodes/prometheus/CMakeLists.txt new file mode 100755 index 0000000..7f0ca0e --- /dev/null +++ b/nodes/prometheus/CMakeLists.txt @@ -0,0 +1,30 @@ +add_subdirectory(package) + +add_executable(prometheus main.cc) + +target_link_libraries(prometheus + -Wl,--start-group + ${COMMON_LIBRARIES} + + generic_rulebase + generic_rulebase_evaluators + ip_utilities + version + signal_handler + + prometheus_comp + http_transaction_data + -Wl,--end-group +) + +add_dependencies(prometheus ngen_core) + +install(TARGETS prometheus DESTINATION bin) +install(TARGETS prometheus DESTINATION prometheus_service/bin) + +gen_package( + install-cp-nano-service-prometheus.sh + prometheus_service + ./install-cp-nano-prometheus.sh + Check Point Prometheus Agent Version ${PACKAGE_VERSION} Install Package +) diff --git a/nodes/prometheus/main.cc b/nodes/prometheus/main.cc new file mode 100755 index 0000000..242e8fa --- /dev/null +++ b/nodes/prometheus/main.cc @@ -0,0 +1,15 @@ +#include "components_list.h" +#include "prometheus_comp.h" + +int +main(int argc, char **argv) +{ + NodeComponents comps; + + comps.registerGlobalValue("Is Rest primary routine", true); + comps.registerGlobalValue("Nano service API Port Primary", 7465); + comps.registerGlobalValue("Nano service API Port Alternative", 7466); + comps.registerGlobalValue("Nano service API Allow Get From External IP", true); + + return comps.run("Prometheus Service", argc, argv); +} diff --git a/nodes/prometheus/package/CMakeLists.txt b/nodes/prometheus/package/CMakeLists.txt new file mode 100755 index 0000000..3cb262a --- /dev/null +++ b/nodes/prometheus/package/CMakeLists.txt @@ -0,0 +1,4 @@ +install(FILES install-cp-nano-prometheus.sh DESTINATION prometheus_service/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) +install(FILES cp-nano-prometheus.cfg DESTINATION prometheus_service/conf PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) +install(FILES cp-nano-prometheus-conf.json DESTINATION prometheus_service/conf PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) +install(FILES cp-nano-prometheus-debug-conf.json DESTINATION prometheus_service/conf PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) diff --git a/nodes/prometheus/package/cp-nano-prometheus-conf.json b/nodes/prometheus/package/cp-nano-prometheus-conf.json new file mode 100755 index 0000000..fb5184b --- /dev/null +++ b/nodes/prometheus/package/cp-nano-prometheus-conf.json @@ -0,0 +1,21 @@ +{ + "connection": { + "Nano service API Port Primary": [ + { + "value": 7465 + } + ], + "Nano service API Port Alternative": [ + { + "value": 7466 + } + ] + }, + "message": { + "Connection timeout": [ + { + "value": 10000000 + } + ] + } +} diff --git a/nodes/prometheus/package/cp-nano-prometheus-debug-conf.json b/nodes/prometheus/package/cp-nano-prometheus-debug-conf.json new file mode 100755 index 0000000..df2394b --- /dev/null +++ b/nodes/prometheus/package/cp-nano-prometheus-debug-conf.json @@ -0,0 +1,11 @@ +{ + "Debug": [ + { + "Streams": [ + { + "Output": "nano_agent/cp-nano-prometheus.dbg" + } + ] + } + ] +} diff --git a/nodes/prometheus/package/cp-nano-prometheus.cfg b/nodes/prometheus/package/cp-nano-prometheus.cfg new file mode 100755 index 0000000..e69de29 diff --git a/nodes/prometheus/package/install-cp-nano-prometheus.sh b/nodes/prometheus/package/install-cp-nano-prometheus.sh new file mode 100755 index 0000000..dedbc01 --- /dev/null +++ b/nodes/prometheus/package/install-cp-nano-prometheus.sh @@ -0,0 +1,164 @@ +#!/bin/sh + +#Nano Service Details +NANO_SERVICE_NAME="prometheus" +NANO_SERVICE_BIN_NAME="cp-nano-prometheus" +NANO_SERVICE_INSTALLATION_FOLDER="prometheus" +ATTACHMENT_BIN_NAME="cp-nano-prometheus" + +#Installable Names +CFG_FILE_NAME="cp-nano-prometheus.cfg" +DBG_CONF_FILE_NAME="cp-nano-prometheus-debug-conf.json" +SERVICE_CONF_FILE_NAME="cp-nano-prometheus-conf.json" +NANO_SERVICE_BIN="prometheus" +ATTACHMENT_BIN="prometheus_attachment" + +#Const variables +FORCE_STDOUT=true +INSTALLATION_TIME=$(date) +CP_NANO_LOG_PATH="/var/log/nano_agent" +CP_NANO_CONF_PATH="/etc/cp/conf" +NANO_SERVICE_INSTALLATION_PATH="/etc/cp/${NANO_SERVICE_INSTALLATION_FOLDER}" +NANO_SERVICE_BIN_PATH=${NANO_SERVICE_INSTALLATION_PATH}/${NANO_SERVICE_BIN_NAME} +NANO_SERVICE_CFG_PATH=${NANO_SERVICE_BIN_PATH}.cfg +ATTACHMENT_BIN_PATH=${NANO_SERVICE_INSTALLATION_PATH}/${ATTACHMENT_BIN_NAME} +DBG_CONF_PATH=${CP_NANO_CONF_PATH}/${NANO_SERVICE_BIN_NAME}-debug-conf.json +SERVICE_CONF_PATH=${CP_NANO_CONF_PATH}/${NANO_SERVICE_BIN_NAME}-conf.json +DBG_FILE_PATH=${CP_NANO_LOG_PATH}/${NANO_SERVICE_BIN_NAME}.dbg +INSTALLATION_LOG_FILE=${CP_NANO_LOG_PATH}/${NANO_SERVICE_BIN_NAME}-install.log + +mkdir -p ${CP_NANO_LOG_PATH} +touch ${DBG_FILE_PATH} + +cp_print() +{ + var_text=$1 + var_std_out=$2 + touch $INSTALLATION_LOG_FILE + if [ -n "$var_std_out" ]; then + if [ "$var_std_out" = "true" ]; then + printf "%b\n" "$var_text" + fi + fi + printf "%b\n" "$var_text" >> $INSTALLATION_LOG_FILE +} + +cp_exec() +{ + var_cmd=$1 + var_std_out=$2 + # Send exec output to RES + RES=$($var_cmd 2>&1) + if [ -n "$RES" ]; then + cp_print "$RES" "$var_std_out" + fi +} + +set_configuration() +{ + cp_exec "cp -n conf/${DBG_CONF_FILE_NAME} $DBG_CONF_PATH" + cp_exec "/etc/cp/scripts/cpnano_debug --default --service prometheus" + cp_exec "cp -n conf/${SERVICE_CONF_FILE_NAME} $SERVICE_CONF_PATH" +} + +run_installation() +{ + cp_print "Starting installation of Check Point ${NANO_SERVICE_NAME} Nano service [$INSTALLATION_TIME]\n" $FORCE_STDOUT + + cp_exec "/etc/cp/watchdog/cp-nano-watchdog --un-register ${ATTACHMENT_BIN_PATH}" + cp_exec "/etc/cp/watchdog/cp-nano-watchdog --un-register ${NANO_SERVICE_BIN_PATH}" + + att_path=$ATTACHMENT_BIN_PATH + cmd_pid_att=$(ps -eo pid,cmd,args | awk -v srv=${att_path} '{if($2 ~ srv || $3 ~ srv) print $1}') + srv_path=$NANO_SERVICE_BIN_NAME + cmd_pid_srv=$(ps -eo pid,cmd,args | awk -v srv=${srv_path} '{if($2 ~ srv || $3 ~ srv) print $1}') + + if [ -n "$cmd_pid_att" ]; then + cp_print "Killing running instance(pid=$cmd_pid_att) of the prometheus attachment on installation" + kill -9 "$cmd_pid_att" + fi + if [ -n "$cmd_pid_srv" ]; then + cp_print "Killing running instance(pid=$cmd_pid_srv) of the prometheus service on installation" + kill -9 "$cmd_pid_srv" + fi + + cp_exec "mkdir -p ${NANO_SERVICE_INSTALLATION_PATH}" + cp_exec "cp -f bin/${NANO_SERVICE_BIN} ${NANO_SERVICE_BIN_PATH}" + cp_exec "chmod +x ${NANO_SERVICE_BIN_PATH}" + cp_exec "cp -f conf/${CFG_FILE_NAME} ${NANO_SERVICE_CFG_PATH}" + cp_exec "chmod 600 ${NANO_SERVICE_CFG_PATH}" + + set_configuration + + cp_exec "/etc/cp/watchdog/cp-nano-watchdog --register ${NANO_SERVICE_BIN_PATH}" + cp_exec "/etc/cp/watchdog/cp-nano-watchdog --register ${ATTACHMENT_BIN_PATH}" + + cp_print "Installation completed successfully." $FORCE_STDOUT +} + +usage() +{ + echo "Check Point: available flags are" + echo "--install : install ${NANO_SERVICE_NAME} Nano Service" + echo "--uninstall : remove ${NANO_SERVICE_NAME} Nano Service" + echo "--pre_install_test : run Pre-installation test for ${NANO_SERVICE_NAME} Nano Service install package" + echo "--post_install_test : run Post-installation test for ${NANO_SERVICE_NAME} Nano Service install package" + exit 255 +} + +run_uninstall() +{ + cp_exec "/etc/cp/watchdog/cp-nano-watchdog --un-register ${ATTACHMENT_BIN_PATH}" + cp_exec "/etc/cp/watchdog/cp-nano-watchdog --un-register ${NANO_SERVICE_BIN_PATH}" + + cp_exec "rm -rf ${NANO_SERVICE_INSTALLATION_PATH}" + cp_exec "rm -rf ${NANO_SERVICE_CONF_DIR}" +} + +run_pre_install_test() +{ + cp_print "Starting Pre-installation test of Check Point ${NANO_SERVICE_NAME} Nano service installation package [$INSTALLATION_TIME]\n" $FORCE_STDOUT + + cp_print "Successfully finished pre-installation test for Check Point ${NANO_SERVICE_NAME} Nano service installation package [$INSTALLATION_TIME]\n" $FORCE_STDOUT + exit 0 +} + +run_post_install_test() +{ + cp_print "Starting Post-installation test of Check Point ${NANO_SERVICE_NAME} Nano service installation package [$INSTALLATION_TIME]\n" $FORCE_STDOUT + + if ! cat /etc/cp/watchdog/wd.services | grep -q ${NANO_SERVICE_BIN_PATH}; then + cp_print "Failed to register ${NANO_SERVICE_NAME} Nano service to the watchdog\n" $FORCE_STDOUT + exit 255 + fi + + cp_print "Successfully finished post-installation test for Check Point ${NANO_SERVICE_NAME} Nano service installation package [$INSTALLATION_TIME]\n" $FORCE_STDOUT + exit 0 + +} + +run() +{ + if [ '--install' = "$1" ]; then + run_installation "${@}" + elif [ '--uninstall' = "$1" ]; then + run_uninstall + elif [ '--pre_install_test' = "$1" ]; then + run_pre_install_test + elif [ '--post_install_test' = "$1" ]; then + run_post_install_test + else + usage + exit 1 + fi +} + +if [ "$(id -u)" != "0" ]; then + echo "Administrative privileges required for this Package (use su or sudo)" + exit 1 +fi + +shift +run "${@}" + +exit 0