github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feb 10 2025 dev (#255)

Browse Source 
* sync code

* sync code

* code sync

* code sync

---------

Co-authored-by: Ned Wright <nedwright@proton.me>
Co-authored-by: Daniel Eisenberg <danielei@checkpoint.com>
This commit is contained in:
Daniel-Eisenberg
2025-02-12 10:56:44 +02:00
committed by GitHub
parent 81433bac25
commit 4ddcd2462a
75 changed files with 1540 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
core/agent_details/agent_details.cc
View File

@@ -237,6 +237,12 @@ AgentDetails::getAgentId() const
return agent_id;
}
string
AgentDetails::getRegisteredServer() const
{
return server;
}
Maybe<string>
AgentDetails::getProxy() const
{

4
core/attachments/http_configuration/http_configuration.cc
View File

@@ -111,6 +111,8 @@ HttpAttachmentConfiguration::save(cereal::JSONOutputArchive &archive) const
cereal::make_nvp("keep_alive_interval_msec", getNumericalValue("keep_alive_interval_msec")),
cereal::make_nvp("min_retries_for_verdict", getNumericalValue("min_retries_for_verdict")),
cereal::make_nvp("max_retries_for_verdict", getNumericalValue("max_retries_for_verdict")),
cereal::make_nvp("hold_verdict_retries", getNumericalValue("hold_verdict_retries")),
cereal::make_nvp("hold_verdict_polling_time", getNumericalValue("hold_verdict_polling_time")),
cereal::make_nvp("body_size_trigger", getNumericalValue("body_size_trigger")),
cereal::make_nvp("remove_server_header", getNumericalValue("remove_server_header"))
);
@@ -167,6 +169,8 @@ HttpAttachmentConfiguration::load(cereal::JSONInputArchive &archive)
loadNumericalValue(archive, "keep_alive_interval_msec", DEFAULT_KEEP_ALIVE_INTERVAL_MSEC);
loadNumericalValue(archive, "min_retries_for_verdict", 3);
loadNumericalValue(archive, "max_retries_for_verdict", 15);
loadNumericalValue(archive, "hold_verdict_retries", 3);
loadNumericalValue(archive, "hold_verdict_polling_time", 1);
loadNumericalValue(archive, "body_size_trigger", 200000);
loadNumericalValue(archive, "remove_server_header", 0);
}

2
core/debug_is/debug.cc
View File

@@ -527,7 +527,7 @@ Debug::preload()
active_streams["FOG"] = make_shared<DebugFogStream>();
string branch = Version::getBranch();
if (branch == "open-source" || branch == "master" || branch.substr(0, 6) == "hotfix") {
if (branch == "master" || branch.substr(0, 6) == "hotfix") {
should_assert_optional = false;
} else {
should_assert_optional = true;

3
core/include/attachments/nginx_attachment_util.h
View File

@@ -42,6 +42,9 @@ unsigned int getFailOpenTimeout();
int isFailOpenHoldMode();
unsigned int getFailOpenHoldTimeout();
unsigned int getHoldVerdictPollingTime();
unsigned int getHoldVerdictRetries();
unsigned int getMaxSessionsPerMinute();
int isFailOpenOnSessionLimit();

2
core/include/services_sdk/interfaces/i_agent_details.h
View File

@@ -36,6 +36,7 @@ public:
virtual Maybe<std::string> getFogDomain() const = 0;
virtual std::string getTenantId() const = 0;
virtual std::string getProfileId() const = 0;
virtual std::string getRegisteredServer() const = 0;
// Agent Details
virtual Maybe<std::string> getProxy() const = 0;
@@ -43,6 +44,7 @@ public:
virtual void setAgentId(const std::string &_agent_id) = 0;
virtual std::string getAgentId() const = 0;
virtual void setOrchestrationMode(OrchestrationMode _orchstration_mode) = 0;
virtual void setRegisteredServer(const std::string &_server) = 0;
virtual OrchestrationMode getOrchestrationMode() const = 0;
virtual std::string getAccessToken() const = 0;
virtual void loadAccessToken() = 0;

16
core/include/services_sdk/interfaces/messaging/messaging_metadata.h
View File

@@ -75,9 +75,16 @@ public:
port_num(_port_num),
conn_flags(_conn_flags),
should_buffer(_should_buffer),
is_to_fog(_is_to_fog)
is_to_fog(_is_to_fog),
should_send_access_token(true)
{}
const bool &
shouldSendAccessToken() const
{
return should_send_access_token;
}
const std::string &
getHostName() const
{
@@ -90,6 +97,12 @@ public:
return port_num;
}
void
setShouldSendAccessToken(const bool &_should_send_access_token)
{
should_send_access_token = _should_send_access_token;
}
void
setConnectioFlag(MessageConnectionConfig flag)
{
@@ -300,6 +313,7 @@ private:
bool is_to_fog = false;
bool is_rate_limit_block = false;
uint rate_limit_block_time = 0;
bool should_send_access_token = true;
};
#endif // __MESSAGING_METADATA_H__

2
core/include/services_sdk/interfaces/mock/mock_agent_details.h
View File

@@ -20,11 +20,13 @@ public:
MOCK_CONST_METHOD0(getFogDomain, Maybe<std::string>());
MOCK_CONST_METHOD0(getTenantId, std::string());
MOCK_CONST_METHOD0(getProfileId, std::string());
MOCK_CONST_METHOD0(getRegisteredServer, std::string());
// Agent Details
MOCK_CONST_METHOD0(getProxy, Maybe<std::string>());
MOCK_METHOD1(setProxy, void(const std::string&));
MOCK_METHOD1(setAgentId, void(const std::string&));
MOCK_METHOD1(setRegisteredServer, void(const std::string&));
MOCK_CONST_METHOD0(getAgentId, std::string());
MOCK_METHOD0(loadAccessToken, void());
MOCK_CONST_METHOD0(getAccessToken, std::string());

3
core/include/services_sdk/resources/agent_details.h
View File

@@ -73,6 +73,7 @@ public:
Maybe<std::string> getOpenSSLDir() const;
std::string getClusterId() const;
OrchestrationMode getOrchestrationMode() const;
std::string getRegisteredServer() const;
bool isOpenAppsecAgent() const;
std::string getAccessToken() const;
void loadAccessToken();
@@ -86,6 +87,7 @@ public:
void setOpenSSLDir(const std::string &_openssl_dir) { openssl_dir = _openssl_dir; }
void setSSLFlag(const bool _encrypted_connection) { encrypted_connection = _encrypted_connection; }
void setOrchestrationMode(OrchestrationMode _orchstration_mode) { orchestration_mode = _orchstration_mode; }
void setRegisteredServer(const std::string &_server) { server = _server; }
bool getSSLFlag() const { return encrypted_connection; }
bool readAgentDetails();
@@ -117,6 +119,7 @@ private:
uint16_t fog_port = 0;
bool encrypted_connection = false;
OrchestrationMode orchestration_mode = OrchestrationMode::ONLINE;
std::string server = "Unknown";
bool is_proxy_configured_via_settings = false;
std::map<ProxyProtocol, ProxyData> proxies;

2
core/include/services_sdk/resources/component_is/components_list_impl.h
View File

@@ -48,6 +48,7 @@
#include "intelligence_comp_v2.h"
#include "messaging.h"
#include "env_details.h"
#include "metric/metric_scraper.h"
USE_DEBUG_FLAG(D_COMP_IS);
@@ -216,6 +217,7 @@ class ComponentListCore
Version,
Buffer,
ShellCmd,
MetricScraper,
GenericMetric,
Messaging,
MainloopComponent,

1
core/include/services_sdk/resources/debug_flags.h
View File

@@ -153,6 +153,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_SDWAN, D_COMPONENT)
DEFINE_FLAG(D_SDWAN_POLICY, D_SDWAN)
DEFINE_FLAG(D_SDWAN_DATA, D_SDWAN)
DEFINE_FLAG(D_SDWAN_FEATURE_FLAG, D_SDWAN)
DEFINE_FLAG(D_LOGGER_SDWAN, D_SDWAN)
DEFINE_FLAG(D_SDWAN_API, D_SDWAN)
DEFINE_FLAG(D_REVERSE_PROXY, D_COMPONENT)

14
core/include/services_sdk/resources/generic_metric.h
View File

@@ -59,10 +59,11 @@ class GenericMetric
Singleton::Consume<I_Messaging>,
Singleton::Consume<I_RestApi>,
Singleton::Consume<I_Encryptor>,
public Listener<AllMetricEvent>
public Listener<AllMetricEvent>,
public Listener<MetricScrapeEvent>
{
public:
enum class Stream { FOG, DEBUG, PROMETHEUS, AIOPS, COUNT };
enum class Stream { FOG, DEBUG, AIOPS, COUNT };
void
init(
@@ -72,7 +73,8 @@ public:
std::chrono::seconds _report_interval,
bool _reset,
ReportIS::Audience _audience = ReportIS::Audience::INTERNAL,
bool _force_buffering = false
bool _force_buffering = false,
const std::string &_asset_id = ""
);
template <typename Value>
@@ -96,6 +98,7 @@ public:
void resetMetrics();
void upon(const AllMetricEvent &) override;
std::string respond(const AllMetricEvent &event) override;
std::vector<PrometheusData> respond(const MetricScrapeEvent &event) override;
std::string getListenerName() const override;
std::string getMetricName() const;
@@ -113,9 +116,10 @@ private:
friend class MetricCalc;
void addCalc(MetricCalc *calc);
std::vector<PrometheusData> getPromMetricsData();
void handleMetricStreamSending();
void generateLog();
void generatePrometheus();
void generateDebug();
void generateAiopsLog();
@@ -127,10 +131,12 @@ private:
ReportIS::Audience audience;
std::chrono::seconds report_interval;
std::vector<MetricCalc *> calcs;
std::vector<MetricCalc *> prometheus_calcs;
Flags<Stream> active_streams;
bool reset;
bool force_buffering = false;
Context ctx;
std::string asset_id;
};
#include "metric/counter.h"

37
core/include/services_sdk/resources/metric/metric_calc.h
View File

@@ -25,6 +25,9 @@
#include "customized_cereal_map.h"
#include "compression_utils.h"
#include "i_encryptor.h"
#include "event.h"
USE_DEBUG_FLAG(D_METRICS);
class GenericMetric;
@@ -32,13 +35,35 @@ enum class MetricType { GAUGE, COUNTER };
struct PrometheusData
{
template <typename Archive>
void
serialize(Archive &ar)
{
try {
ar(cereal::make_nvp("metric_name", name));
ar(cereal::make_nvp("metric_type", type));
ar(cereal::make_nvp("metric_description", description));
ar(cereal::make_nvp("labels", label));
ar(cereal::make_nvp("value", value));
} catch (const cereal::Exception &e) {
dbgTrace(D_METRICS) << "Error in serialize Prometheus data: " << e.what();
}
}
std::string name;
std::string type;
std::string desc;
std::string description;
std::string label;
std::string value;
};
class MetricScrapeEvent : public Event<MetricScrapeEvent, std::vector<PrometheusData>>
{
public:
MetricScrapeEvent() {}
};
class AiopsMetricData
{
public:
@@ -228,7 +253,10 @@ public:
std::string getMetircDescription() const { return getMetadata("Description"); }
std::string getMetadata(const std::string &metadata) const;
virtual MetricType getMetricType() const { return MetricType::GAUGE; }
virtual std::vector<PrometheusData> getPrometheusMetrics() const;
virtual std::vector<PrometheusData> getPrometheusMetrics(
const std::string &metric_name,
const std::string &asset_id = ""
) const;
virtual float getValue() const = 0;
virtual std::vector<AiopsMetricData> getAiopsMetrics() const;
@@ -240,7 +268,10 @@ public:
protected:
void addMetric(GenericMetric *metric);
std::map<std::string, std::string> getBasicLabels() const;
std::map<std::string, std::string> getBasicLabels(
const std::string &metric_name,
const std::string &asset_id = ""
) const;
template <typename Metadata, typename ... OtherMetadata>
void

13
core/include/services_sdk/resources/metric/metric_map.h
View File

@@ -55,12 +55,17 @@ class MetricMap : public MetricCalc
}
std::vector<PrometheusData>
getPrometheusMetrics(const std::string &label, const std::string &name) const
getPrometheusMetrics(
const std::string &metric_name,
const std::string &label,
const std::string &name,
const std::string &asset_id
) const
{
std::vector<PrometheusData> res;
for (auto &metric : inner_map) {
auto sub_res = metric.second.getPrometheusMetrics();
auto sub_res = metric.second.getPrometheusMetrics(metric_name, asset_id);
for (auto &sub_metric : sub_res) {
sub_metric.label += "," + label + "=\"" + metric.first + "\"";
sub_metric.name = name;
@@ -155,9 +160,9 @@ public:
}
std::vector<PrometheusData>
getPrometheusMetrics() const override
getPrometheusMetrics(const std::string &metric_name, const std::string &asset_id) const override
{
return metric_map.getPrometheusMetrics(label, getMetricName());
return metric_map.getPrometheusMetrics(metric_name, label, getMetricName(), asset_id);
}
std::vector<AiopsMetricData>

45
core/include/services_sdk/resources/metric/metric_scraper.h Normal file
View File

@@ -0,0 +1,45 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __METRIC_SCRAPER_H__
#define __METRIC_SCRAPER_H__
#include <string>
#include <fstream>
#include <vector>
#include <streambuf>
#include "singleton.h"
#include "debug.h"
#include "component.h"
#include "event.h"
#include "i_rest_api.h"
#include "generic_metric.h"
class MetricScraper
:
public Component,
Singleton::Consume<I_RestApi>
{
public:
MetricScraper();
~MetricScraper();
void init();
private:
class Impl;
std::unique_ptr<Impl> pimpl;
};
#endif // __METRIC_SCRAPER_H__

10
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@@ -492,9 +492,9 @@ private:
return genError("Local intelligence server ip not configured");
}
auto res = sendLocalIntelligenceToLocalServer(rest_req, *server, primary_port_setting);
auto res = sendLocalIntelligenceToLocalServer(rest_req, *server, primary_port_setting, false);
if (res.ok()) return res;
return sendLocalIntelligenceToLocalServer(rest_req, *server, secondary_port_setting);
return sendLocalIntelligenceToLocalServer(rest_req, *server, secondary_port_setting, false);
}
template <typename IntelligenceRest>
@@ -502,8 +502,9 @@ private:
sendLocalIntelligenceToLocalServer(
const IntelligenceRest &rest_req,
const string &server,
const string &port_setting
) const
const string &port_setting,
const bool should_send_access_token = false
) const
{
auto port = getSetting<uint>("intelligence", port_setting);
if (!port.ok()) {
@@ -519,6 +520,7 @@ private:
req_md.insertHeaders(getHTTPHeaders());
req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
req_md.setShouldSendAccessToken(should_send_access_token);
return sendIntelligenceRequestImpl(rest_req, req_md);
}

43
core/messaging/connection/connection.cc
View File

@@ -184,6 +184,18 @@ public:
establishConnection()
{
dbgFlow(D_CONNECTION) << "Establishing a new connection";
// check if connection already established
I_MainLoop *i_mainloop = Singleton::Consume<I_MainLoop>::by<Messaging>();
while (lock) {
i_mainloop->yield(true);
}
lock = true;
auto unlock = make_scope_exit([&] () { lock = false; });
if (is_connected && !should_close_connection) {
dbgTrace(D_CONNECTION) << "Connection already established";
return Maybe<void>();
}
auto set_socket = setSocket();
if (!set_socket.ok()) {
dbgWarning(D_CONNECTION) << "Failed to set socket: " << set_socket.getErr();
@@ -221,6 +233,7 @@ public:
<< (isOverProxy() ? ", Over proxy: " + settings.getProxyHost() + ":" + to_string(key.getPort()) : "");
active = Maybe<void, chrono::seconds>();
should_close_connection = false;
is_connected = true;
return Maybe<void>();
}
@@ -583,11 +596,13 @@ private:
if (BIO_should_retry(bio.get())) return string();
auto fd = BIO_get_fd(bio.get(), nullptr);
char error_buf[256];
ERR_error_string(ERR_get_error(), error_buf);
string error = receive_len == 0 ?
"Connection closed by peer" :
"Failed to read data from BIO socket. Error: " + string(error_buf);
"Connection closed by peer (BIO fd: " + to_string(fd) + "). Error: " + string(error_buf) :
"Failed to read data from BIO socket (fd: " + to_string(fd) + "). Error: " + string(error_buf);
dbgWarning(D_CONNECTION) << error;
return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, error));
}
@@ -621,17 +636,28 @@ private:
Maybe<HTTPResponse, HTTPResponse>
sendAndReceiveData(const string &request, bool is_connect)
{
dbgFlow(D_CONNECTION) << "Sending and receiving data";
dbgFlow(D_CONNECTION) << "Sending and receiving data, lock: " << lock;
I_MainLoop *i_mainloop = Singleton::Consume<I_MainLoop>::by<Messaging>();
while (lock) {
while (lock && !is_connect) {
i_mainloop->yield(true);
}
lock = true;
auto unlock = make_scope_exit([&] () { lock = false; });
dbgTrace(D_CONNECTION) << "acquire lock";
auto unlock = make_scope_exit([&] () {
lock = false;
});
if (should_close_connection) {
dbgWarning(D_CONNECTION) << close_error.getBody();
return genError(close_error);
dbgTrace(D_CONNECTION) << "reconnect in progress";
while (lock) {
i_mainloop->yield(true);
}
if (!is_connected) {
dbgWarning(D_CONNECTION) << close_error.getBody();
return genError(close_error);
}
dbgTrace(D_CONNECTION) << "reconnected by other routine";
lock = true;
}
I_TimeGet *i_time = Singleton::Consume<I_TimeGet>::by<Messaging>();
@@ -651,11 +677,13 @@ private:
dbgTrace(D_CONNECTION) << "Sent the message, now waiting for response";
while (!http_parser.hasReachedError()) {
if (i_time->getMonotonicTime() > receiving_end_time) {
is_connected = false;
should_close_connection = true;
return genError(receving_timeout);
};
auto receieved = receiveData();
if (!receieved.ok()) {
is_connected = false;
should_close_connection = true;
return receieved.passErr();
}
@@ -706,6 +734,7 @@ private:
bool lock = false;
bool should_close_connection = false;
bool is_dual_auth = false;
bool is_connected = false;
Maybe<string> sni_hostname = genError<string>("Uninitialized");
Maybe<string> dn_host_name = genError<string>("Uninitialized");

2
core/messaging/connection/connection_comp.cc
View File

@@ -92,12 +92,12 @@ private:
<< metadata.getPort();
MessageConnectionKey conn_key(metadata.getHostName(), metadata.getPort(), category);
Connection conn(conn_key, metadata);
persistent_connections.emplace(conn_key, conn);
const auto &external_certificate = metadata.getExternalCertificate();
if (!external_certificate.empty()) conn.setExternalCertificate(external_certificate);
auto connected = conn.establishConnection();
persistent_connections.emplace(conn_key, conn);
if (!connected.ok()) {
string connection_err = "Failed to establish connection. Error: " + connected.getErr();

3
core/messaging/include/http_request.h
View File

@@ -30,7 +30,8 @@ public:
HTTPMethod method,
const std::string &uri,
const std::map<std::string, std::string> &headers,
const std::string &body
const std::string &body,
const bool should_send_access_token = true
);
Maybe<void> setConnectionHeaders(const Connection &conn, bool is_access_token_needed);

4
core/messaging/messaging_comp/http_request.cc
View File

@@ -79,7 +79,8 @@ HTTPRequest::prepareRequest(
HTTPMethod method,
const string &uri,
const map<string, string> &headers,
const string &body
const string &body,
const bool should_send_access_token
)
{
HTTPRequest req(method, uri, headers, body);
@@ -94,6 +95,7 @@ HTTPRequest::prepareRequest(
dont_add_access_token = true;
dbgTrace(D_MESSAGING) << "Request is for agent authentication";
}
if (!should_send_access_token) dont_add_access_token = true;
auto res = req.addAccessToken(conn, dont_add_access_token);
if (!res.ok()) return res.passErr();

8
core/messaging/messaging_comp/messaging_comp.cc
View File

@@ -142,7 +142,13 @@ MessagingComp::sendMessage(
metadata.insertHeaders(i_env->getCurrentHeadersMap());
}
auto req = HTTPRequest::prepareRequest(conn, method, uri, metadata.getHeaders(), body);
auto req = HTTPRequest::prepareRequest(
conn,
method,
uri,
metadata.getHeaders(),
body,
metadata.shouldSendAccessToken());
if (!req.ok()) return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, req.getErr()));
auto response = i_conn->sendRequest(conn, *req);

2
core/metric/CMakeLists.txt
View File

@@ -1,3 +1,3 @@
add_library(metric generic_metric.cc)
add_library(metric generic_metric.cc metric_scraper.cc)
add_subdirectory(metric_ut)

97
core/metric/generic_metric.cc
View File

@@ -49,7 +49,7 @@ MetricCalc::getAiopsMetrics() const
string description = getMetircDescription();
string type = getMetricType() == MetricType::GAUGE ? "Gauge" : "Counter";
return { AiopsMetricData(name, type, units, description, getBasicLabels(), value) };
return { AiopsMetricData(name, type, units, description, getBasicLabels(getMetricName()), value) };
}
string
@@ -77,7 +77,7 @@ MetricCalc::addMetric(GenericMetric *metric)
}
vector<PrometheusData>
MetricCalc::getPrometheusMetrics() const
MetricCalc::getPrometheusMetrics(const std::string &metric_name, const string &asset_id) const
{
float value = getValue();
if (isnan(value)) return {};
@@ -86,10 +86,10 @@ MetricCalc::getPrometheusMetrics() const
res.name = getMetricDotName() != "" ? getMetricDotName() : getMetricName();
res.type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter";
res.desc = getMetircDescription();
res.description = getMetircDescription();
stringstream labels;
const auto &label_pairs = getBasicLabels();
const auto &label_pairs = getBasicLabels(metric_name, asset_id);
bool first = true;
for (auto &pair : label_pairs) {
if (!first) labels << ',';
@@ -106,7 +106,7 @@ MetricCalc::getPrometheusMetrics() const
}
map<string, string>
MetricCalc::getBasicLabels() const
MetricCalc::getBasicLabels(const string &metric_name, const string &asset_id) const
{
map<string, string> res;
@@ -121,6 +121,9 @@ MetricCalc::getBasicLabels() const
auto executable = env->get<string>("Base Executable Name");
if (executable.ok()) res["process"] = *executable;
if (!asset_id.empty()) res["assetId"] = asset_id;
res["metricName"] = metric_name;
return res;
}
@@ -158,7 +161,8 @@ GenericMetric::init(
chrono::seconds _report_interval,
bool _reset,
Audience _audience,
bool _force_buffering
bool _force_buffering,
const string &_asset_id
)
{
turnOnStream(Stream::FOG);
@@ -173,6 +177,7 @@ GenericMetric::init(
issuing_engine = _issuing_engine;
audience = _audience;
force_buffering = _force_buffering;
asset_id = _asset_id;
i_mainloop->addRecurringRoutine(
I_MainLoop::RoutineType::System,
@@ -185,13 +190,13 @@ GenericMetric::init(
},
"Metric Fog stream messaging for " + _metric_name
);
registerListener();
}
void
GenericMetric::handleMetricStreamSending()
{
if (active_streams.isSet(Stream::DEBUG)) generateDebug();
if (active_streams.isSet(Stream::PROMETHEUS)) generatePrometheus();
if (active_streams.isSet(Stream::FOG)) generateLog();
if (active_streams.isSet(Stream::AIOPS)) generateAiopsLog();
@@ -237,6 +242,7 @@ void
GenericMetric::addCalc(MetricCalc *calc)
{
calcs.push_back(calc);
prometheus_calcs.push_back(calc);
}
void
@@ -254,6 +260,12 @@ GenericMetric::respond(const AllMetricEvent &event)
return res;
}
vector<PrometheusData>
GenericMetric::respond(const MetricScrapeEvent &)
{
return getPromMetricsData();
}
string GenericMetric::getListenerName() const { return metric_name; }
void
@@ -316,70 +328,19 @@ GenericMetric::generateLog()
sendLog(metric_client_rest);
}
class PrometheusRest : public ClientRest
vector<PrometheusData>
GenericMetric::getPromMetricsData()
{
class Metric : public ClientRest
{
public:
Metric(const string &n, const string &t, const string &d, const string &l, const string &v)
:
metric_name(n),
metric_type(t),
metric_description(d),
labels(l),
value(v)
{}
private:
C2S_PARAM(string, metric_name);
C2S_PARAM(string, metric_type);
C2S_PARAM(string, metric_description);
C2S_PARAM(string, labels);
C2S_PARAM(string, value);
};
public:
PrometheusRest() : metrics(vector<Metric>()) {}
void
addMetric(const vector<PrometheusData> &vec)
{
auto &metric_vec = metrics.get();
metric_vec.reserve(vec.size());
for (auto &metric : vec) {
metric_vec.emplace_back(metric.name, metric.type, metric.desc, "{" + metric.label + "}", metric.value);
}
}
private:
C2S_PARAM(vector<Metric>, metrics);
};
void
GenericMetric::generatePrometheus()
{
if (!getProfileAgentSettingWithDefault(false, "prometheus")) return;
dbgTrace(D_METRICS) << "Generate prometheus metric";
vector<PrometheusData> all_metrics;
for (auto &calc : calcs) {
const auto &cal_metrics = calc->getPrometheusMetrics();
all_metrics.insert(all_metrics.end(), cal_metrics.begin(), cal_metrics.end());
if (!getProfileAgentSettingWithDefault(false, "prometheus")) return all_metrics;
dbgTrace(D_METRICS) << "Get prometheus metrics";
for (auto &calc : prometheus_calcs) {
const auto &calc_prom_metrics = calc->getPrometheusMetrics(metric_name, asset_id);
all_metrics.insert(all_metrics.end(), calc_prom_metrics.begin(), calc_prom_metrics.end());
calc->reset();
}
PrometheusRest rest;
rest.addMetric(all_metrics);
MessageMetadata new_config_req_md("127.0.0.1", 7465);
new_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
Singleton::Consume<I_Messaging>::by<GenericMetric>()->sendSyncMessage(
HTTPMethod::POST,
"/add-metrics",
rest,
MessageCategory::GENERIC,
new_config_req_md
);
return all_metrics;
}
void

50
core/metric/metric_scraper.cc Normal file
View File

@@ -0,0 +1,50 @@
#include "metric/metric_scraper.h"
using namespace std;
USE_DEBUG_FLAG(D_METRICS);
class MetricScraper::Impl
{
public:
void
init()
{
Singleton::Consume<I_RestApi>::by<MetricScraper>()->addGetCall(
"service-metrics",
[&] () { return getAllPrometheusMetrics(); }
);
}
string
getAllPrometheusMetrics()
{
auto all_metrics_events_res = MetricScrapeEvent().query();
for (auto metric_vec : all_metrics_events_res) {
for (PrometheusData metric : metric_vec) {
metric.label = "{" + metric.label + "}";
all_metrics.emplace_back(metric);
}
}
stringstream ss;
{
cereal::JSONOutputArchive archive(ss);
archive(cereal::make_nvp("metrics", all_metrics));
}
all_metrics.clear();
return ss.str();
}
private:
vector<PrometheusData> all_metrics;
};
MetricScraper::MetricScraper() : Component("MetricScraper"), pimpl(make_unique<MetricScraper::Impl>()) {}
MetricScraper::~MetricScraper() {}
void
MetricScraper::init()
{
pimpl->init();
}

202
core/metric/metric_ut/metric_ut.cc
View File

@@ -14,6 +14,7 @@
#include "mock/mock_instance_awareness.h"
#include "config.h"
#include "config_component.h"
#include "metric/metric_scraper.h"
using namespace std;
using namespace chrono;
@@ -191,9 +192,11 @@ public:
MetricTest()
{
EXPECT_CALL(rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true));
env.init();
conf.preload();
ON_CALL(instance, getUniqueID()).WillByDefault(Return(string("87")));
ON_CALL(instance, getFamilyID()).WillByDefault(Return(string("")));
env.init();
Debug::setNewDefaultStdout(&debug_output);
Debug::setUnitTestFlag(D_METRICS, Debug::DebugLevel::TRACE);
setConfiguration<bool>(true, string("metric"), string("fogMetricSendEnable"));
@@ -531,9 +534,12 @@ TEST_F(MetricTest, printMetricsTest)
GenericMetric::fini();
}
TEST_F(MetricTest, printPromeathus)
TEST_F(MetricTest, getPromeathusMetric)
{
conf.preload();
MetricScraper metric_scraper;
function<string()> get_metrics_func;
EXPECT_CALL(rest, addGetCall("service-metrics", _)).WillOnce(DoAll(SaveArg<1>(&get_metrics_func), Return(true)));
metric_scraper.init();
stringstream configuration;
configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
@@ -546,20 +552,21 @@ TEST_F(MetricTest, printPromeathus)
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
false
false,
ReportIS::Audience::INTERNAL,
false,
"asset id"
);
cpu_mt.turnOffStream(GenericMetric::Stream::FOG);
cpu_mt.turnOffStream(GenericMetric::Stream::DEBUG);
cpu_mt.turnOnStream(GenericMetric::Stream::PROMETHEUS);
cpu_mt.registerListener();
CPUEvent cpu_event;
cpu_event.setProcessCPU(89);
cpu_event.notify();
string message_body;
EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/add-metrics", _, _, _))
.WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
string message_body = get_metrics_func();
routine();
string res =
@@ -569,42 +576,48 @@ TEST_F(MetricTest, printPromeathus)
" \"metric_name\": \"cpuMax\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuMin\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuAvg\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuCurrent\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuCounter\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuTotalCounter\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"1\"\n"
" }\n"
" ]\n"
@@ -613,9 +626,12 @@ TEST_F(MetricTest, printPromeathus)
EXPECT_EQ(message_body, res);
}
TEST_F(MetricTest, printPromeathusMultiMap)
TEST_F(MetricTest, getPromeathusMultiMap)
{
conf.preload();
MetricScraper metric_scraper;
function<string()> get_metrics_func;
EXPECT_CALL(rest, addGetCall("service-metrics", _)).WillOnce(DoAll(SaveArg<1>(&get_metrics_func), Return(true)));
metric_scraper.init();
stringstream configuration;
configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
@@ -628,18 +644,18 @@ TEST_F(MetricTest, printPromeathusMultiMap)
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
true
true,
ReportIS::Audience::INTERNAL,
false,
"asset id"
);
metric.turnOnStream(GenericMetric::Stream::PROMETHEUS);
metric.registerListener();
HttpTransaction("/index.html", "GET", 10).notify();
HttpTransaction("/index2.html", "GET", 20).notify();
HttpTransaction("/index.html", "POST", 40).notify();
string message_body;
EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/add-metrics", _, _, _))
.WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
string message_body = get_metrics_func();
routine();
string res =
@@ -649,24 +665,156 @@ TEST_F(MetricTest, printPromeathusMultiMap)
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"Bytes per URL\\\",method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"Bytes per URL\\\",method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"Bytes per URL\\\",method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n"
" \"value\": \"1\"\n"
" }\n"
" ]\n"
"}";
EXPECT_EQ(message_body, res);
}
TEST_F(MetricTest, getPromeathusTwoMetrics)
{
MetricScraper metric_scraper;
function<string()> get_metrics_func;
EXPECT_CALL(rest, addGetCall("service-metrics", _)).WillOnce(DoAll(SaveArg<1>(&get_metrics_func), Return(true)));
metric_scraper.init();
stringstream configuration;
configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
EXPECT_TRUE(Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration));
CPUMetric cpu_mt;
cpu_mt.init(
"CPU usage",
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
false,
ReportIS::Audience::INTERNAL,
false,
"asset id"
);
cpu_mt.turnOffStream(GenericMetric::Stream::FOG);
cpu_mt.turnOffStream(GenericMetric::Stream::DEBUG);
cpu_mt.registerListener();
CPUEvent cpu_event;
cpu_event.setProcessCPU(89);
cpu_event.notify();
UrlMetric2 metric;
metric.init(
"Bytes per URL",
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
true,
ReportIS::Audience::INTERNAL,
false,
"asset id"
);
metric.registerListener();
HttpTransaction("/index.html", "GET", 10).notify();
HttpTransaction("/index2.html", "GET", 20).notify();
HttpTransaction("/index.html", "POST", 40).notify();
string message_body = get_metrics_func();
routine();
string res =
"{\n"
" \"metrics\": [\n"
" {\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"Bytes per URL\\\",method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"Bytes per URL\\\",method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"request.total\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"Bytes per URL\\\",method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuMax\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuMin\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuAvg\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuCurrent\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuCounter\",\n"
" \"metric_type\": \"gauge\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"metric_name\": \"cpuTotalCounter\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\","
"metricName=\\\"CPU usage\\\"}\",\n"
" \"value\": \"1\"\n"
" }\n"
" ]\n"