github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-30 19:47:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add prometheus support

Browse Source
This commit is contained in:
avigailo
2025-04-09 22:06:45 +03:00
parent 2c6b6baa3b
commit 9e85e5703c
18 changed files with 746 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
components/security_apps/prometheus/CMakeLists.txt Executable file
View File

@@ -0,0 +1,2 @@
add_library(prometheus_comp prometheus_comp.cc)
add_subdirectory(prometheus_ut)

200
components/security_apps/prometheus/prometheus_comp.cc Executable file
View File

@@ -0,0 +1,200 @@
#include "prometheus_comp.h"
#include <string>
#include <map>
#include <vector>
#include <cereal/archives/json.hpp>
#include <cereal/types/map.hpp>
#include <cereal/types/vector.hpp>
#include <cereal/types/string.hpp>
#include <iostream>
#include <fstream>
#include "common.h"
#include "report/base_field.h"
#include "report/report_enums.h"
#include "log_generator.h"
#include "debug.h"
#include "rest.h"
#include "customized_cereal_map.h"
#include "i_messaging.h"
#include "prometheus_metric_names.h"
USE_DEBUG_FLAG(D_PROMETHEUS);
using namespace std;
using namespace ReportIS;
struct ServiceData
{
template <typename Archive>
void
serialize(Archive &ar)
{
ar(cereal::make_nvp("Service port", service_port));
}
int service_port;
};
class PrometheusMetricData
{
public:
PrometheusMetricData(const string &n, const string &t, const string &d) : name(n), type(t), description(d) {}
void
addElement(const string &labels, const string &value)
{
metric_labels_to_values[labels] = value;
}
ostream &
print(ostream &os)
{
if (metric_labels_to_values.empty()) return os;
string representative_name = "";
if (!name.empty()) {
auto metric_name = convertMetricName(name);
!metric_name.empty() ? representative_name = metric_name : representative_name = name;
}
if (!description.empty()) os << "# HELP " << representative_name << ' ' << description << '\n';
if (!name.empty()) os << "# TYPE " << representative_name << ' ' << type << '\n';
for (auto &entry : metric_labels_to_values) {
os << representative_name << entry.first << ' ' << entry.second << '\n';
}
os << '\n';
metric_labels_to_values.clear();
return os;
}
private:
string name;
string type;
string description;
map<string, string> metric_labels_to_values;
};
static ostream & operator<<(ostream &os, PrometheusMetricData &metric) { return metric.print(os); }
class PrometheusComp::Impl
{
public:
void
init()
{
Singleton::Consume<I_RestApi>::by<PrometheusComp>()->addGetCall(
"metrics",
[&] () { return getFormatedPrometheusMetrics(); }
);
}
void
addMetrics(const vector<PrometheusData> &metrics)
{
for(auto &metric : metrics) {
auto &metric_object = getDataObject(
metric.name,
metric.type,
metric.description
);
metric_object.addElement(metric.label, metric.value);
}
}
private:
PrometheusMetricData &
getDataObject(const string &name, const string &type, const string &description)
{
auto elem = prometheus_metrics.find(name);
if (elem == prometheus_metrics.end()) {
elem = prometheus_metrics.emplace(name, PrometheusMetricData(name, type, description)).first;
}
return elem->second;
}
map<string, ServiceData>
getServiceDetails()
{
map<string, ServiceData> registeredServices;
auto registered_services_file = getConfigurationWithDefault<string>(
getFilesystemPathConfig() + "/conf/orchestrations_registered_services.json",
"orchestration",
"Orchestration registered services"
);
ifstream file(registered_services_file);
if (!file.is_open()) {
dbgWarning(D_PROMETHEUS) << "Failed to open file: " << registered_services_file;
return registeredServices;
}
stringstream buffer;
buffer << file.rdbuf();
try {
cereal::JSONInputArchive archive(buffer);
archive(cereal::make_nvp("Registered Services", registeredServices));
} catch (const exception& e) {
dbgWarning(D_PROMETHEUS) << "Error parsing Registered Services JSON file: " << e.what();
}
return registeredServices;
}
void
getServicesMetrics()
{
dbgTrace(D_PROMETHEUS) << "Get all registered services metrics";
map<string, ServiceData> service_names_to_ports = getServiceDetails();
for (const auto &service : service_names_to_ports) {
I_Messaging *messaging = Singleton::Consume<I_Messaging>::by<PrometheusComp>();
MessageMetadata servie_metric_req_md("127.0.0.1", service.second.service_port);
servie_metric_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
servie_metric_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
auto res = messaging->sendSyncMessage(
HTTPMethod::GET,
"/service-metrics",
string(""),
MessageCategory::GENERIC,
servie_metric_req_md
);
if (!res.ok()) {
dbgWarning(D_PROMETHEUS) << "Failed to get service metrics. Service: " << service.first;
continue;
}
stringstream buffer;
buffer << res.unpack().getBody();
cereal::JSONInputArchive archive(buffer);
vector<PrometheusData> metrics;
archive(cereal::make_nvp("metrics", metrics));
addMetrics(metrics);
}
}
string
getFormatedPrometheusMetrics()
{
MetricScrapeEvent().notify();
getServicesMetrics();
stringstream result;
for (auto &metric : prometheus_metrics) {
result << metric.second;
}
dbgTrace(D_PROMETHEUS) << "Prometheus metrics: " << result.str();
return result.str();
}
map<string, PrometheusMetricData> prometheus_metrics;
};
PrometheusComp::PrometheusComp() : Component("Prometheus"), pimpl(make_unique<Impl>()) {}
PrometheusComp::~PrometheusComp() {}
void
PrometheusComp::init()
{
pimpl->init();
}

143
components/security_apps/prometheus/prometheus_metric_names.h Executable file
View File

@@ -0,0 +1,143 @@
#ifndef __PROMETHEUS_METRIC_NAMES_H__
#define __PROMETHEUS_METRIC_NAMES_H__
#include <string>
#include <unordered_map>
#include "debug.h"
USE_DEBUG_FLAG(D_PROMETHEUS);
std::string
convertMetricName(const std::string &original_metric_name)
{
static const std::unordered_map<std::string, std::string> original_to_representative_names = {
// HybridModeMetric
{"watchdogProcessStartupEventsSum", "nano_service_restarts_counter"},
// nginxAttachmentMetric
{"inspectVerdictSum", "traffic_inspection_verdict_inspect_counter"},
{"acceptVeridctSum", "traffic_inspection_verdict_accept_counter"},
{"dropVerdictSum", "traffic_inspection_verdict_drop_counter"},
{"injectVerdictSum", "traffic_inspection_verdict_inject_counter"},
{"irrelevantVerdictSum", "traffic_inspection_verdict_irrelevant_counter"},
{"irrelevantVerdictSum", "traffic_inspection_verdict_irrelevant_counter"},
{"reconfVerdictSum", "traffic_inspection_verdict_reconf_counter"},
{"responseInspection", "response_body_inspection_counter"},
// nginxIntakerMetric
{"successfullInspectionTransactionsSum", "successful_Inspection_counter"},
{"failopenTransactionsSum", "fail_open_Inspection_counter"},
{"failcloseTransactionsSum", "fail_close_Inspection_counter"},
{"transparentModeTransactionsSum", "transparent_mode_counter"},
{"totalTimeInTransparentModeSum", "total_time_in_transparent_mode_counter"},
{"reachInspectVerdictSum", "inspect_verdict_counter"},
{"reachAcceptVerdictSum", "accept_verdict_counter"},
{"reachDropVerdictSum", "drop_verdict_counter"},
{"reachInjectVerdictSum", "inject_verdict_counter"},
{"reachIrrelevantVerdictSum", "irrelevant_verdict_counter"},
{"reachReconfVerdictSum", "reconf_verdict_counter"},
{"requestCompressionFailureSum", "failed_requests_compression_counter"},
{"responseCompressionFailureSum", "failed_response_compression_counter"},
{"requestDecompressionFailureSum", "failed_requests_decompression_counter"},
{"responseDecompressionFailureSum", "failed_response_decompression_counter"},
{"requestCompressionSuccessSum", "successful_request_compression_counter"},
{"responseCompressionSuccessSum", "successful_response_compression_counter"},
{"requestDecompressionSuccessSum", "successful_request_decompression_counter"},
{"responseDecompressionSuccessSum", "successful_response_decompression_counter"},
{"skippedSessionsUponCorruptedZipSum", "corrupted_zip_skipped_session_counter"},
{"attachmentThreadReachedTimeoutSum", "thread_exceeded_processing_time_counter"},
{"registrationThreadReachedTimeoutSum", "failed_registration_thread_counter"},
{"requestHeaderThreadReachedTimeoutSum", "request_headers_processing_thread_timeouts_counter"},
{"requestBodyThreadReachedTimeoutSum", "request_body_processing_thread_timeouts_counter"},
{"respondHeaderThreadReachedTimeoutSum", "response_headers_processing_thread_timeouts_counter"},
{"respondBodyThreadReachedTimeoutSum", "response_body_processing_thread_timeouts_counter"},
{"attachmentThreadFailureSum", "thread_failures_counter"},
{"httpRequestProcessingReachedTimeoutSum", "request_processing_timeouts_counter"},
{"httpRequestsSizeSum", "requests_total_size_counter"},
{"httpResponsesSizeSum", "response_total_size_counter"},
{"httpRequestFailedToReachWebServerUpstreamSum", "requests_failed_reach_upstram_counter"},
{"overallSessionProcessTimeToVerdictAvgSample", "overall_processing_time_until_verdict_average"},
{"overallSessionProcessTimeToVerdictMaxSample", "overall_processing_time_until_verdict_max"},
{"overallSessionProcessTimeToVerdictMinSample", "overall_processing_time_until_verdict_min"},
{"requestProcessTimeToVerdictAvgSample", "requests_processing_time_until_verdict_average"},
{"requestProcessTimeToVerdictMaxSample", "requests_processing_time_until_verdict_max"},
{"requestProcessTimeToVerdictMinSample", "requests_processing_time_until_verdict_min"},
{"responseProcessTimeToVerdictAvgSample", "response_processing_time_until_verdict_average"},
{"responseProcessTimeToVerdictMaxSample", "response_processing_time_until_verdict_max"},
{"responseProcessTimeToVerdictMinSample", "response_processing_time_until_verdict_min"},
{"requestBodySizeUponTimeoutAvgSample", "request_body_size_average"},
{"requestBodySizeUponTimeoutMaxSample", "request_body_size_max"},
{"requestBodySizeUponTimeoutMinSample", "request_body_size_min"},
{"responseBodySizeUponTimeoutAvgSample", "response_body_size_average"},
{"responseBodySizeUponTimeoutMaxSample", "response_body_size_max"},
{"responseBodySizeUponTimeoutMinSample", "response_body_size_min"},
// WaapTelemetrics
{"reservedNgenA", "total_requests_counter"},
{"reservedNgenB", "unique_sources_counter"},
{"reservedNgenC", "requests_blocked_by_force_and_exception_counter"},
{"reservedNgenD", "requests_blocked_by_waf_counter"},
{"reservedNgenE", "requests_blocked_by_open_api_counter"},
{"reservedNgenF", "requests_blocked_by_bot_protection_counter"},
{"reservedNgenG", "requests_threat_level_info_and_no_threat_counter"},
{"reservedNgenH", "requests_threat_level_low_counter"},
{"reservedNgenI", "requests_threat_level_medium_counter"},
{"reservedNgenJ", "requests_threat_level_high_counter"},
// WaapTrafficTelemetrics
{"reservedNgenA", "post_requests_counter"},
{"reservedNgenB", "get_requests_counter"},
{"reservedNgenC", "put_requests_counter"},
{"reservedNgenD", "patch_requests_counter"},
{"reservedNgenE", "delete_requests_counter"},
{"reservedNgenF", "other_requests_counter"},
{"reservedNgenG", "2xx_status_code_responses_counter"},
{"reservedNgenH", "4xx_status_code_responses_counter"},
{"reservedNgenI", "5xx_status_code_responses_counter"},
{"reservedNgenJ", "requests_time_latency_average"},
// WaapAttackTypesMetrics
{"reservedNgenA", "sql_injection_attacks_type_counter"},
{"reservedNgenB", "vulnerability_scanning_attacks_type_counter"},
{"reservedNgenC", "path_traversal_attacks_type_counter"},
{"reservedNgenD", "ldap_injection_attacks_type_counter"},
{"reservedNgenE", "evasion_techniques_attacks_type_counter"},
{"reservedNgenF", "remote_code_execution_attacks_type_counter"},
{"reservedNgenG", "xml_extern_entity_attacks_type_counter"},
{"reservedNgenH", "cross_site_scripting_attacks_type_counter"},
{"reservedNgenI", "general_attacks_type_counter"},
// AssetsMetric
{"numberOfProtectedApiAssetsSample", "api_assets_counter"},
{"numberOfProtectedWebAppAssetsSample", "web_api_assets_counter"},
{"numberOfProtectedAssetsSample", "all_assets_counter"},
// IPSMetric
{"preventEngineMatchesSample", "prevent_action_matches_counter"},
{"detectEngineMatchesSample", "detect_action_matches_counter"},
{"ignoreEngineMatchesSample", "ignore_action_matches_counter"},
// CPUMetric
{"cpuMaxSample", "cpu_usage_percentage_max"},
{"cpuAvgSample", "cpu_usage_percentage_average"},
{"cpuSample", "cpu_usage_percentage_last_value"},
// LogMetric
{"logQueueMaxSizeSample", "logs_queue_size_max"},
{"logQueueAvgSizeSample", "logs_queue_size_average"},
{"logQueueCurrentSizeSample", "logs_queue_size_last_value"},
{"sentLogsSum", "logs_sent_counter"},
{"sentLogsBulksSum", "bulk_logs_sent_counter"},
// MemoryMetric
{"serviceVirtualMemorySizeMaxSample", "service_virtual_memory_size_kb_max"},
{"serviceVirtualMemorySizeMinSample", "service_virtual_memory_size_kb_min"},
{"serviceVirtualMemorySizeAvgSample", "service_virtual_memory_size_kb_average"},
{"serviceRssMemorySizeMaxSample", "service_physical_memory_size_kb_max"},
{"serviceRssMemorySizeMinSample", "service_physical_memory_size_kb_min"},
{"serviceRssMemorySizeAvgSample", "service_physical_memory_size_kb_average"},
{"generalTotalMemorySizeMaxSample", "general_total_used_memory_max"},
{"generalTotalMemorySizeMinSample", "general_total_used_memory_min"},
{"generalTotalMemorySizeAvgSample", "general_total_used_memory_average"},
};
auto metric_names = original_to_representative_names.find(original_metric_name);
if (metric_names != original_to_representative_names.end()) return metric_names->second;
dbgDebug(D_PROMETHEUS)
<< "Metric don't have a representative name, originl name: "
<< original_metric_name;
return "";
}
#endif // __PROMETHEUS_METRIC_NAMES_H__

8
components/security_apps/prometheus/prometheus_ut/CMakeLists.txt Executable file
View File

@@ -0,0 +1,8 @@
link_directories(${BOOST_ROOT}/lib)
link_directories(${BOOST_ROOT}/lib ${CMAKE_BINARY_DIR}/core/shmem_ipc)
add_unit_test(
prometheus_ut
"prometheus_ut.cc"
"prometheus_comp;logging;agent_details;waap_clib;table;singleton;time_proxy;metric;event_is;connkey;http_transaction_data;generic_rulebase;generic_rulebase_evaluators;ip_utilities;intelligence_is_v2;-lboost_regex;messaging;"
)

79
components/security_apps/prometheus/prometheus_ut/prometheus_ut.cc Executable file
View File

@@ -0,0 +1,79 @@
#include "prometheus_comp.h"
#include <sstream>
#include <fstream>
#include <vector>
#include "cmock.h"
#include "cptest.h"
#include "maybe_res.h"
#include "debug.h"
#include "config.h"
#include "environment.h"
#include "config_component.h"
#include "agent_details.h"
#include "time_proxy.h"
#include "mock/mock_mainloop.h"
#include "mock/mock_rest_api.h"
#include "mock/mock_messaging.h"
using namespace std;
using namespace testing;
USE_DEBUG_FLAG(D_PROMETHEUS);
class PrometheusCompTest : public Test
{
public:
PrometheusCompTest()
{
EXPECT_CALL(mock_rest, mockRestCall(_, "declare-boolean-variable", _)).WillOnce(Return(false));
env.preload();
config.preload();
env.init();
EXPECT_CALL(
mock_rest,
addGetCall("metrics", _)
).WillOnce(DoAll(SaveArg<1>(&get_metrics_func), Return(true)));
prometheus_comp.init();
}
::Environment env;
ConfigComponent config;
PrometheusComp prometheus_comp;
StrictMock<MockRestApi> mock_rest;
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockMessaging> mock_messaging;
unique_ptr<ServerRest> agent_uninstall;
function<string()> get_metrics_func;
CPTestTempfile status_file;
string registered_services_file_path;
};
TEST_F(PrometheusCompTest, checkAddingMetric)
{
registered_services_file_path = cptestFnameInSrcDir(string("registered_services.json"));
setConfiguration(registered_services_file_path, "orchestration", "Orchestration registered services");
string metric_body = "{\n"
" \"metrics\": [\n"
" {\n"
" \"metric_name\": \"watchdogProcessStartupEventsSum\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{method=\\\"post\\\",code=\\\"200\\\"}\",\n"
" \"value\": \"1534\"\n"
" }\n"
" ]\n"
"}";
string message_body;
EXPECT_CALL(mock_messaging, sendSyncMessage(_, "/service-metrics", _, _, _))
.Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, metric_body)));
string metric_str = "# TYPE nano_service_restarts_counter counter\n"
"nano_service_restarts_counter{method=\"post\",code=\"200\"} 1534\n\n";
EXPECT_EQ(metric_str, get_metrics_func());
}

32
components/security_apps/prometheus/prometheus_ut/registered_services.json Executable file
View File

@@ -0,0 +1,32 @@
{
"Registered Services": {
"cp-nano-orchestration": {
"Service name": "cp-nano-orchestration",
"Service ID": "cp-nano-orchestration",
"Service port": 7777,
"Relevant configs": [
"zones",
"triggers",
"rules",
"registration-data",
"parameters",
"orchestration",
"exceptions",
"agent-intelligence"
]
},
"cp-nano-prometheus": {
"Service name": "cp-nano-prometheus",
"Service ID": "cp-nano-prometheus",
"Service port": 7465,
"Relevant configs": [
"zones",
"triggers",
"rules",
"parameters",
"exceptions",
"agent-intelligence"
]
}
}
}