github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feb 10 2025 dev (#255)

Browse Source 
* sync code

* sync code

* code sync

* code sync

---------

Co-authored-by: Ned Wright <nedwright@proton.me>
Co-authored-by: Daniel Eisenberg <danielei@checkpoint.com>
This commit is contained in:
Daniel-Eisenberg
2025-02-12 10:56:44 +02:00
committed by GitHub
parent 81433bac25
commit 4ddcd2462a
75 changed files with 1540 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/metric/CMakeLists.txt
View File

@@ -1,3 +1,3 @@
add_library(metric generic_metric.cc)
add_library(metric generic_metric.cc metric_scraper.cc)
add_subdirectory(metric_ut)

97
core/metric/generic_metric.cc
View File

@@ -49,7 +49,7 @@ MetricCalc::getAiopsMetrics() const
string description = getMetircDescription();
string type = getMetricType() == MetricType::GAUGE ? "Gauge" : "Counter";
return { AiopsMetricData(name, type, units, description, getBasicLabels(), value) };
return { AiopsMetricData(name, type, units, description, getBasicLabels(getMetricName()), value) };
}
string
@@ -77,7 +77,7 @@ MetricCalc::addMetric(GenericMetric *metric)
}
vector<PrometheusData>
MetricCalc::getPrometheusMetrics() const
MetricCalc::getPrometheusMetrics(const std::string &metric_name, const string &asset_id) const
{
float value = getValue();
if (isnan(value)) return {};
@@ -86,10 +86,10 @@ MetricCalc::getPrometheusMetrics() const
res.name = getMetricDotName() != "" ? getMetricDotName() : getMetricName();
res.type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter";
res.desc = getMetircDescription();
res.description = getMetircDescription();
stringstream labels;
const auto &label_pairs = getBasicLabels();
const auto &label_pairs = getBasicLabels(metric_name, asset_id);
bool first = true;
for (auto &pair : label_pairs) {
if (!first) labels << ',';
@@ -106,7 +106,7 @@ MetricCalc::getPrometheusMetrics() const
}
map<string, string>
MetricCalc::getBasicLabels() const
MetricCalc::getBasicLabels(const string &metric_name, const string &asset_id) const
{
map<string, string> res;
@@ -121,6 +121,9 @@ MetricCalc::getBasicLabels() const
auto executable = env->get<string>("Base Executable Name");
if (executable.ok()) res["process"] = *executable;
if (!asset_id.empty()) res["assetId"] = asset_id;
res["metricName"] = metric_name;
return res;
}
@@ -158,7 +161,8 @@ GenericMetric::init(
chrono::seconds _report_interval,
bool _reset,
Audience _audience,
bool _force_buffering
bool _force_buffering,
const string &_asset_id
)
{
turnOnStream(Stream::FOG);
@@ -173,6 +177,7 @@ GenericMetric::init(
issuing_engine = _issuing_engine;
audience = _audience;
force_buffering = _force_buffering;
asset_id = _asset_id;
i_mainloop->addRecurringRoutine(
I_MainLoop::RoutineType::System,
@@ -185,13 +190,13 @@ GenericMetric::init(
},
"Metric Fog stream messaging for " + _metric_name
);
registerListener();
}
void
GenericMetric::handleMetricStreamSending()
{
if (active_streams.isSet(Stream::DEBUG)) generateDebug();
if (active_streams.isSet(Stream::PROMETHEUS)) generatePrometheus();
if (active_streams.isSet(Stream::FOG)) generateLog();
if (active_streams.isSet(Stream::AIOPS)) generateAiopsLog();
@@ -237,6 +242,7 @@ void
GenericMetric::addCalc(MetricCalc *calc)
{
calcs.push_back(calc);
prometheus_calcs.push_back(calc);
}
void
@@ -254,6 +260,12 @@ GenericMetric::respond(const AllMetricEvent &event)
return res;
}
vector<PrometheusData>
GenericMetric::respond(const MetricScrapeEvent &)
{
return getPromMetricsData();
}
string GenericMetric::getListenerName() const { return metric_name; }
void
@@ -316,70 +328,19 @@ GenericMetric::generateLog()
sendLog(metric_client_rest);
}
class PrometheusRest : public ClientRest
vector<PrometheusData>
GenericMetric::getPromMetricsData()
{
class Metric : public ClientRest
{
public:
Metric(const string &n, const string &t, const string &d, const string &l, const string &v)
:
metric_name(n),
metric_type(t),
metric_description(d),
labels(l),
value(v)
{}
private:
C2S_PARAM(string, metric_name);
C2S_PARAM(string, metric_type);
C2S_PARAM(string, metric_description);
C2S_PARAM(string, labels);
C2S_PARAM(string, value);
};
public:
PrometheusRest() : metrics(vector<Metric>()) {}
void
addMetric(const vector<PrometheusData> &vec)
{
auto &metric_vec = metrics.get();
metric_vec.reserve(vec.size());
for (auto &metric : vec) {
metric_vec.emplace_back(metric.name, metric.type, metric.desc, "{" + metric.label + "}", metric.value);
}
}
private:
C2S_PARAM(vector<Metric>, metrics);
};
void
GenericMetric::generatePrometheus()
{
if (!getProfileAgentSettingWithDefault(false, "prometheus")) return;
dbgTrace(D_METRICS) << "Generate prometheus metric";
vector<PrometheusData> all_metrics;
for (auto &calc : calcs) {
const auto &cal_metrics = calc->getPrometheusMetrics();
all_metrics.insert(all_metrics.end(), cal_metrics.begin(), cal_metrics.end());
if (!getProfileAgentSettingWithDefault(false, "prometheus")) return all_metrics;
dbgTrace(D_METRICS) << "Get prometheus metrics";
for (auto &calc : prometheus_calcs) {
const auto &calc_prom_metrics = calc->getPrometheusMetrics(metric_name, asset_id);
all_metrics.insert(all_metrics.end(), calc_prom_metrics.begin(), calc_prom_metrics.end());
calc->reset();
}
PrometheusRest rest;
rest.addMetric(all_metrics);
MessageMetadata new_config_req_md("127.0.0.1", 7465);
new_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
Singleton::Consume<I_Messaging>::by<GenericMetric>()->sendSyncMessage(
HTTPMethod::POST,
"/add-metrics",
rest,
MessageCategory::GENERIC,
new_config_req_md
);
return all_metrics;
}
void

50
core/metric/metric_scraper.cc Normal file
View File

@@ -0,0 +1,50 @@
#include "metric/metric_scraper.h"
using namespace std;
USE_DEBUG_FLAG(D_METRICS);
class MetricScraper::Impl
{
public:
void
init()
{
Singleton::Consume<I_RestApi>::by<MetricScraper>()->addGetCall(
"service-metrics",
[&] () { return getAllPrometheusMetrics(); }
);
}
string
getAllPrometheusMetrics()
{
auto all_metrics_events_res = MetricScrapeEvent().query();
for (auto metric_vec : all_metrics_events_res) {
for (PrometheusData metric : metric_vec) {
metric.label = "{" + metric.label + "}";
all_metrics.emplace_back(metric);
}
}
stringstream ss;
{
cereal::JSONOutputArchive archive(ss);
archive(cereal::make_nvp("metrics", all_metrics));
}
all_metrics.clear();
return ss.str();
}
private:
vector<PrometheusData> all_metrics;
};
MetricScraper::MetricScraper() : Component("MetricScraper"), pimpl(make_unique<MetricScraper::Impl>()) {}
MetricScraper::~MetricScraper() {}
void
MetricScraper::init()
{
pimpl->init();
}

202
core/metric/metric_ut/metric_ut.cc
View File

@@ -14,6 +14,7 @@
#include "mock/mock_instance_awareness.h"
#include "config.h"
#include "config_component.h"
#include "metric/metric_scraper.h"
using namespace std;
using namespace chrono;
@@ -191,9 +192,11 @@ public:
MetricTest()
{
EXPECT_CALL(rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true));
env.init();
conf.preload();
ON_CALL(instance, getUniqueID()).WillByDefault(Return(string("87")));
ON_CALL(instance, getFamilyID()).WillByDefault(Return(string("")));
env.init();
Debug::setNewDefaultStdout(&debug_output);
Debug::setUnitTestFlag(D_METRICS, Debug::DebugLevel::TRACE);
setConfiguration<bool>(true, string("metric"), string("fogMetricSendEnable"));
@@ -531,9 +534,12 @@ TEST_F(MetricTest, printMetricsTest)
GenericMetric::fini();
}
TEST_F(MetricTest, printPromeathus)
TEST_F(MetricTest, getPromeathusMetric)
{
conf.preload();
MetricScraper metric_scraper;
function<string()> get_metrics_func;
EXPECT_CALL(rest, addGetCall("service-metrics", _)).WillOnce(DoAll(SaveArg<1>(&get_metrics_func), Return(true)));
metric_scraper.init();
stringstream configuration;
configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
@@ -546,20 +552,21 @@ TEST_F(MetricTest, printPromeathus)
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
false
false,
ReportIS::Audience::INTERNAL,
false,
"asset id"
);
cpu_mt.turnOffStream(GenericMetric::Stream::FOG);
cpu_mt.turnOffStream(GenericMetric::Stream::DEBUG);
cpu_mt.turnOnStream(GenericMetric::Stream::PROMETHEUS);
cpu_mt.registerListener();
CPUEvent cpu_event;
cpu_event.setProcessCPU(89);
cpu_event.notify();
string message_body;
EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/add-metrics", _, _, _))
.WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
string message_body = get_metrics_func();
routine();
string res =
@@ -569,42 +576,48 @@ TEST_F(MetricTest, printPromeathus)
" \"metric_name\": \"cpuMax\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuMin\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuAvg\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuCurrent\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuCounter\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuTotalCounter\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"1\"\n"
" }\n"
" ]\n"
@@ -613,9 +626,12 @@ TEST_F(MetricTest, printPromeathus)
EXPECT_EQ(message_body, res);
}
TEST_F(MetricTest, printPromeathusMultiMap)
TEST_F(MetricTest, getPromeathusMultiMap)
{
conf.preload();
MetricScraper metric_scraper;
function<string()> get_metrics_func;
EXPECT_CALL(rest, addGetCall("service-metrics", _)).WillOnce(DoAll(SaveArg<1>(&get_metrics_func), Return(true)));
metric_scraper.init();
stringstream configuration;
configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
@@ -628,18 +644,18 @@ TEST_F(MetricTest, printPromeathusMultiMap)
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
true
true,
ReportIS::Audience::INTERNAL,
false,
"asset id"
);
metric.turnOnStream(GenericMetric::Stream::PROMETHEUS);
metric.registerListener();
HttpTransaction("/index.html", "GET", 10).notify();
HttpTransaction("/index2.html", "GET", 20).notify();
HttpTransaction("/index.html", "POST", 40).notify();
string message_body;
EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/add-metrics", _, _, _))
.WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
string message_body = get_metrics_func();
routine();
string res =
@@ -649,24 +665,156 @@ TEST_F(MetricTest, printPromeathusMultiMap)
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"Bytes per URL\\\",method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"Bytes per URL\\\",method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"Bytes per URL\\\",method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n"
" \"value\": \"1\"\n"
" }\n"
" ]\n"
"}";
EXPECT_EQ(message_body, res);
}
TEST_F(MetricTest, getPromeathusTwoMetrics)
{
MetricScraper metric_scraper;
function<string()> get_metrics_func;
EXPECT_CALL(rest, addGetCall("service-metrics", _)).WillOnce(DoAll(SaveArg<1>(&get_metrics_func), Return(true)));
metric_scraper.init();
stringstream configuration;
configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
EXPECT_TRUE(Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration));
CPUMetric cpu_mt;
cpu_mt.init(
"CPU usage",
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
false,
ReportIS::Audience::INTERNAL,
false,
"asset id"
);
cpu_mt.turnOffStream(GenericMetric::Stream::FOG);
cpu_mt.turnOffStream(GenericMetric::Stream::DEBUG);
cpu_mt.registerListener();
CPUEvent cpu_event;
cpu_event.setProcessCPU(89);
cpu_event.notify();
UrlMetric2 metric;
metric.init(
"Bytes per URL",
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
true,
ReportIS::Audience::INTERNAL,
false,
"asset id"
);
metric.registerListener();
HttpTransaction("/index.html", "GET", 10).notify();
HttpTransaction("/index2.html", "GET", 20).notify();
HttpTransaction("/index.html", "POST", 40).notify();
string message_body = get_metrics_func();
routine();
string res =
"{\n"
" \"metrics\": [\n"
" {\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"Bytes per URL\\\",method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"Bytes per URL\\\",method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"Bytes per URL\\\",method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuMax\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuMin\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuAvg\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuCurrent\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuCounter\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuTotalCounter\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"1\"\n"
" }\n"
" ]\n"