github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

sync code

Browse Source
This commit is contained in:
Ned Wright
2024-10-14 14:51:28 +00:00
parent b58f7781e6
commit c2ea2cda6d
89 changed files with 2545 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
core/agent_details/agent_details.cc
View File

@@ -304,6 +304,12 @@ AgentDetails::getOrchestrationMode() const
return orchestration_mode;
}
bool
AgentDetails::isOpenAppsecAgent() const
{
return (orchestration_mode == OrchestrationMode::HYBRID) || (tenant_id.rfind("org_", 0) == 0);
}
string
AgentDetails::getAccessToken() const
{

11
core/config/config.cc
View File

@@ -315,22 +315,13 @@ void
ConfigComponent::Impl::preload()
{
I_Environment *environment = Singleton::Consume<I_Environment>::by<ConfigComponent>();
auto executable = environment->get<string>("Executable Name");
auto executable = environment->get<string>("Base Executable Name");
if (!executable.ok() || *executable == "") {
dbgWarning(D_CONFIG)
<< "Could not load nano service's settings since \"Executable Name\" in not found in the environment";
return;
}
executable_name = *executable;
auto file_path_end = executable_name.find_last_of("/");
if (file_path_end != string::npos) {
executable_name = executable_name.substr(file_path_end + 1);
}
auto file_sufix_start = executable_name.find_first_of(".");
if (file_sufix_start != string::npos) {
executable_name = executable_name.substr(0, file_sufix_start);
}
config_file_paths.insert(executable_name + "-conf.json");
config_file_paths.insert(executable_name + "-debug-conf.json");

18
core/debug_is/debug.cc
View File

@@ -766,22 +766,8 @@ Debug::findDebugFilePrefix(const string &file_name)
string
Debug::getExecutableName()
{
auto executable = env->get<string>("Executable Name");
if (!executable.ok() || *executable == "") {
return "";
}
string executable_name = *executable;
auto file_path_end = executable_name.find_last_of("/");
if (file_path_end != string::npos) {
executable_name = executable_name.substr(file_path_end + 1);
}
auto file_sufix_start = executable_name.find_first_of(".");
if (file_sufix_start != string::npos) {
executable_name = executable_name.substr(0, file_sufix_start);
}
return executable_name;
auto executable = env->get<string>("Base Executable Name");
return executable.ok() ? *executable : "";
}
void

2
core/debug_is/debug_is_ut/debug_ut.cc
View File

@@ -836,7 +836,7 @@ TEST_F(DebugConfigTest, testSetConfig)
EXPECT_CALL(mock_rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true));
env.preload();
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "debug-ut");
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "debug-ut");
env.init();
Debug::init();

13
core/environment/environment.cc
View File

@@ -265,18 +265,9 @@ Environment::Impl::getCurrentHeadersMap()
tracing_headers["X-Span-Id"] = span_id;
}
auto exec_name = get<string>("Executable Name");
auto exec_name = get<string>("Base Executable Name");
if (exec_name.ok() && *exec_name != "") {
string executable_name = *exec_name;
auto file_path_end = executable_name.find_last_of("/");
if (file_path_end != string::npos) {
executable_name = executable_name.substr(file_path_end + 1);
}
auto file_sufix_start = executable_name.find_first_of(".");
if (file_sufix_start != string::npos) {
executable_name = executable_name.substr(0, file_sufix_start);
}
tracing_headers["X-Calling-Service"] = executable_name;
tracing_headers["X-Calling-Service"] = *exec_name;
}
return tracing_headers;

2
core/environment/environment_ut/tracing_ut.cc
View File

@@ -346,7 +346,7 @@ TEST_F(TracingCompRoutinesTest, 2SpansDifFlow)
{
I_MainLoop::Routine routine = [&] () {
string service_name = "test-service-name";
i_env->registerValue("Executable Name", service_name);
i_env->registerValue("Base Executable Name", service_name);
i_env->startNewTrace(true, "a687b388-1108-4083-9852-07c33b1074e9");
trace_id = i_env->getCurrentTrace();

1
core/include/services_sdk/interfaces/i_agent_details.h
View File

@@ -46,6 +46,7 @@ public:
virtual OrchestrationMode getOrchestrationMode() const = 0;
virtual std::string getAccessToken() const = 0;
virtual void loadAccessToken() = 0;
virtual bool isOpenAppsecAgent() const = 0;
// OpenSSL
virtual void setOpenSSLDir(const std::string &openssl_dir) = 0;

1
core/include/services_sdk/interfaces/i_env_details.h
View File

@@ -24,6 +24,7 @@ class I_EnvDetails
public:
virtual EnvType getEnvType() = 0;
virtual std::string getToken() = 0;
virtual std::string getNameSpace() = 0;
protected:
virtual ~I_EnvDetails() {}

13
core/include/services_sdk/interfaces/messaging/messaging_metadata.h
View File

@@ -215,6 +215,18 @@ public:
return is_to_fog;
}
void
setSniHostName(const std::string &_host_name)
{
sni_host_name = _host_name;
}
Maybe<std::string>
getSniHostName() const
{
return sni_host_name;
}
template <class Archive>
void
serialize(Archive &ar)
@@ -237,6 +249,7 @@ public:
private:
std::string host_name = "";
Maybe<std::string> sni_host_name = genError("SNI host name not set");
std::string ca_path = "";
std::string client_cert_path = "";
std::string client_key_path = "";

1
core/include/services_sdk/interfaces/mock/mock_agent_details.h
View File

@@ -28,6 +28,7 @@ public:
MOCK_CONST_METHOD0(getAgentId, std::string());
MOCK_METHOD0(loadAccessToken, void());
MOCK_CONST_METHOD0(getAccessToken, std::string());
MOCK_CONST_METHOD0(isOpenAppsecAgent, bool());
// OpenSSL
MOCK_METHOD1(setOpenSSLDir, void(const std::string&));

1
core/include/services_sdk/resources/agent_details.h
View File

@@ -73,6 +73,7 @@ public:
Maybe<std::string> getOpenSSLDir() const;
std::string getClusterId() const;
OrchestrationMode getOrchestrationMode() const;
bool isOpenAppsecAgent() const;
std::string getAccessToken() const;
void loadAccessToken();

8
core/include/services_sdk/resources/component_is/components_list_impl.h
View File

@@ -165,6 +165,14 @@ public:
}
registerGlobalValue<std::string>("Executable Name", arg_vec.front());
auto file_path_end = arg_vec.front().find_last_of("/");
auto executable_name = arg_vec.front().substr(file_path_end + 1);
auto file_sufix_start = executable_name.find_first_of(".");
if (file_sufix_start != std::string::npos) {
executable_name = executable_name.substr(0, file_sufix_start);
}
registerGlobalValue<std::string>("Base Executable Name", executable_name);
}
void

2
core/include/services_sdk/resources/debug_flags.h
View File

@@ -76,6 +76,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_WAAP_SCORE_BUILDER, D_WAAP)
DEFINE_FLAG(D_WAAP_ULIMITS, D_WAAP)
DEFINE_FLAG(D_WAAP_SCANNER, D_WAAP)
DEFINE_FLAG(D_WAAP_MODEL_LOGGER, D_WAAP)
DEFINE_FLAG(D_WAAP_DEEP_PARSER, D_WAAP)
DEFINE_FLAG(D_WAAP_BASE64, D_WAAP)
DEFINE_FLAG(D_WAAP_JSON, D_WAAP)
@@ -190,6 +191,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_L7_ACCESS_CONTROL, D_COMPONENT)
DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT)
DEFINE_FLAG(D_HORIZON_TELEMETRY, D_COMPONENT)
DEFINE_FLAG(D_PROMETHEUS, D_COMPONENT)
DEFINE_FLAG(D_FLOW, D_ALL)
DEFINE_FLAG(D_DROP, D_FLOW)

4
core/include/services_sdk/resources/generic_metric.h
View File

@@ -24,6 +24,7 @@
#include "i_mainloop.h"
#include "i_time_get.h"
#include "i_agent_details.h"
#include "i_instance_awareness.h"
#include "i_environment.h"
#include "i_messaging.h"
#include "i_rest_api.h"
@@ -52,6 +53,7 @@ class GenericMetric
Singleton::Consume<I_MainLoop>,
Singleton::Consume<I_TimeGet>,
Singleton::Consume<I_AgentDetails>,
Singleton::Consume<I_InstanceAwareness>,
Singleton::Consume<I_Environment>,
Singleton::Consume<I_Messaging>,
Singleton::Consume<I_RestApi>,
@@ -111,7 +113,9 @@ private:
void handleMetricStreamSending();
void generateLog();
void generatePrometheus();
void generateDebug();
void generateAiopsLog();
I_MainLoop *i_mainloop;
I_TimeGet *i_time;

2
core/include/services_sdk/resources/log_generator.h
View File

@@ -102,6 +102,8 @@ public:
std::string getLogInsteadOfSending();
void addMarkerSuffix(const std::string &suffix);
private:
std::chrono::microseconds getCurrentTime() const;
void loadBaseLogFields();

6
core/include/services_sdk/resources/metric/average.h
View File

@@ -54,6 +54,12 @@ public:
return (count > 0) ? double(sum)/count : 0;
}
float
getValue() const override
{
return static_cast<float>(getAverage());
}
void
save(cereal::JSONOutputArchive &ar) const override
{

6
core/include/services_sdk/resources/metric/counter.h
View File

@@ -44,6 +44,12 @@ public:
return counter;
}
float
getValue() const override
{
return static_cast<float>(counter);
}
void
save(cereal::JSONOutputArchive &ar) const override
{

6
core/include/services_sdk/resources/metric/last_reported_value.h
View File

@@ -44,6 +44,12 @@ public:
return last_reported;
}
float
getValue() const override
{
return static_cast<float>(last_reported);
}
void
save(cereal::JSONOutputArchive &ar) const override
{

6
core/include/services_sdk/resources/metric/max.h
View File

@@ -56,6 +56,12 @@ public:
return max;
}
float
getValue() const override
{
return static_cast<float>(max);
}
void
save(cereal::JSONOutputArchive &ar) const override
{

115
core/include/services_sdk/resources/metric/metric_calc.h
View File

@@ -18,21 +18,129 @@
#error metric/metric_calc.h should not be included directly
#endif // __GENERIC_METRIC_H_
#include <cmath>
#include <cereal/archives/json.hpp>
#include "report/report.h"
#include "customized_cereal_map.h"
class GenericMetric;
enum class MetricType { GAUGE, COUNTER };
struct PrometheusData
{
std::string name;
std::string type;
std::string desc;
std::string label;
std::string value;
};
class AiopsMetricData
{
public:
AiopsMetricData(
const std::string &_name,
const std::string &_type,
const std::string &_units,
const std::string &_description,
std::map<std::string, std::string> _resource_attributes,
float _value)
:
name(_name),
type(_type),
units(_units),
description(_description),
resource_attributes(_resource_attributes),
value(_value)
{
timestamp = Singleton::Consume<I_TimeGet>::by<GenericMetric>()->getWalltimeStr();
asset_id = Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId();
}
void
serialize(cereal::JSONOutputArchive &ar) const
{
ar(cereal::make_nvp("Timestamp", timestamp));
ar(cereal::make_nvp("MetricName", name));
ar(cereal::make_nvp("MetricType", type));
ar(cereal::make_nvp("MetricUnit", units));
ar(cereal::make_nvp("MetricDescription", description));
ar(cereal::make_nvp("MetricValue", value));
ar(cereal::make_nvp("ResourceAttributes", resource_attributes));
ar(cereal::make_nvp("MetricAttributes", metric_attributes));
ar(cereal::make_nvp("AssetID", asset_id));
}
std::string
toString() const
{
std::stringstream ss;
{
cereal::JSONOutputArchive ar(ss);
serialize(ar);
}
return ss.str();
}
void
addMetricAttribute(const std::string &label, const std::string &value)
{
metric_attributes[label] = value;
}
private:
std::string timestamp = "";
std::string asset_id = "";
std::string name;
std::string type;
std::string units;
std::string description;
std::map<std::string, std::string> resource_attributes;
std::map<std::string, std::string> metric_attributes;
float value = 0;
};
class AiopsMetricList
{
public:
void
addMetrics(const std::vector<AiopsMetricData> &_metrics)
{
metrics.insert(metrics.end(), _metrics.begin(), _metrics.end());
}
void
serialize(cereal::JSONOutputArchive &ar) const
{
ar(cereal::make_nvp("Metrics", metrics));
}
// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
std::string
toString() const
{
std::stringstream ss;
{
cereal::JSONOutputArchive ar(ss);
serialize(ar);
}
return ss.str();
}
// LCOV_EXCL_STOP
private:
std::vector<AiopsMetricData> metrics;
};
class MetricCalc
{
public:
template<typename ... Args>
MetricCalc(GenericMetric *metric, const std::string &calc_title, const Args & ... args)
{
setMetadata("BaseName", calc_title);
setMetricName(calc_title);
addMetric(metric);
parseMetadata(args ...);
}
@@ -47,7 +155,11 @@ public:
std::string getMetircDescription() const { return getMetadata("Description"); }
std::string getMetadata(const std::string &metadata) const;
virtual MetricType getMetricType() const { return MetricType::GAUGE; }
virtual std::vector<PrometheusData> getPrometheusMetrics() const;
virtual float getValue() const = 0;
virtual std::vector<AiopsMetricData> getAiopsMetrics() const;
void setMetricName(const std::string &name) { setMetadata("BaseName", name); }
void setMetricDotName(const std::string &name) { setMetadata("DotName", name); }
void setMetircUnits(const std::string &units) { setMetadata("Units", units); }
void setMetircDescription(const std::string &description) { setMetadata("Description", description); }
@@ -55,6 +167,7 @@ public:
protected:
void addMetric(GenericMetric *metric);
std::map<std::string, std::string> getBasicLabels() const;
template <typename Metadata, typename ... OtherMetadata>
void

69
core/include/services_sdk/resources/metric/metric_map.h
View File

@@ -54,6 +54,37 @@ class MetricMap : public MetricCalc
return first->second.getMetricType();
}
std::vector<PrometheusData>
getPrometheusMetrics(const std::string &label, const std::string &name) const
{
std::vector<PrometheusData> res;
for (auto &metric : inner_map) {
auto sub_res = metric.second.getPrometheusMetrics();
for (auto &sub_metric : sub_res) {
sub_metric.label += "," + label + "=\"" + metric.first + "\"";
sub_metric.name = name;
}
res.insert(res.end(), sub_res.begin(), sub_res.end());
}
return res;
}
std::vector<AiopsMetricData>
getAiopsMetrics(const std::string &label) const
{
std::vector<AiopsMetricData> aiops_metrics;
for (auto &metric : inner_map) {
auto metric_data = metric.second.getAiopsMetrics();
for (auto &sub_metric : metric_data) {
sub_metric.addMetricAttribute(label, metric.first);
}
aiops_metrics.insert(aiops_metrics.end(), metric_data.begin(), metric_data.end());
}
return aiops_metrics;
}
typename std::map<std::string, Metric>::const_iterator begin() const { return inner_map.begin(); }
typename std::map<std::string, Metric>::const_iterator end() const { return inner_map.end(); }
@@ -63,9 +94,17 @@ class MetricMap : public MetricCalc
public:
template <typename ... Args>
MetricMap(GenericMetric *metric, const std::string &title, const Args & ... args)
MetricMap(
const Metric &sub_metric,
GenericMetric *metric,
const std::string &l,
const std::string &title,
const Args & ... args
)
:
MetricCalc(metric, title, args ...)
MetricCalc(metric, title, args ...),
base_metric(sub_metric),
label(l)
{
}
@@ -75,6 +114,14 @@ public:
if (getMetricType() == MetricType::GAUGE) metric_map.clear();
}
// LCOV_EXCL_START Reason: Covered by printPromeathusMultiMap unit-test, but misdetected by the coverage
float
getValue() const override
{
return std::nanf("");
}
// LCOV_EXCL_STOP
void
save(cereal::JSONOutputArchive &ar) const override
{
@@ -89,7 +136,9 @@ public:
{
std::stringstream string_key;
string_key << key;
auto metric = metric_map.emplace(string_key.str(), Metric(nullptr, string_key.str())).first;
auto new_metric = base_metric;
new_metric.setMetricName(string_key.str());
auto metric = metric_map.emplace(string_key.str(), std::move(new_metric)).first;
metric->second.report(new_values...);
}
@@ -105,8 +154,22 @@ public:
return field;
}
std::vector<PrometheusData>
getPrometheusMetrics() const override
{
return metric_map.getPrometheusMetrics(label, getMetricName());
}
std::vector<AiopsMetricData>
getAiopsMetrics() const
{
return metric_map.getAiopsMetrics(label);
}
private:
InnerMap metric_map;
Metric base_metric;
std::string label;
};
} // namespace MetricCalculations

6
core/include/services_sdk/resources/metric/min.h
View File

@@ -56,6 +56,12 @@ public:
return min;
}
float
getValue() const override
{
return static_cast<float>(min);
}
void
save(cereal::JSONOutputArchive &ar) const override
{

6
core/include/services_sdk/resources/metric/no_reset_counter.h
View File

@@ -41,6 +41,12 @@ public:
return counter;
}
float
getValue() const override
{
return static_cast<float>(counter);
}
void
save(cereal::JSONOutputArchive &ar) const override
{

6
core/include/services_sdk/resources/metric/top_values.h
View File

@@ -59,6 +59,12 @@ public:
values.clear();
}
float
getValue() const override
{
return std::nanf("");
}
std::vector<T>
getTopValues() const
{

1
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@@ -485,6 +485,7 @@ private:
MessageMetadata req_md(server, *port);
req_md.insertHeaders(getHTTPHeaders());
req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
return sendIntelligenceRequestImpl(rest_req, req_md);
}

9
core/logging/log_generator.cc
View File

@@ -123,3 +123,12 @@ LogGen::loadBaseLogFields()
log.getMarkers() = env->getAllStrings(EnvKeyAttr::LogSection::MARKER);
}
// Adding a suffix to log markers will allow for creating a unique log that won't be reduced
void
LogGen::addMarkerSuffix(const string &suffix)
{
for (auto &marker : log.getMarkers()) {
marker.second += suffix;
}
}

11
core/messaging/connection/connection.cc
View File

@@ -96,6 +96,8 @@ public:
client_key_path = metadata.getClientKeyPath();
is_dual_auth = true;
}
sni_hostname = metadata.getSniHostName();
}
void
@@ -329,6 +331,12 @@ private:
if (!SSL_set1_host(ssl_socket, host)) {
return genError("Failed to set host name verification. Host: " + string(host));
}
if (sni_hostname.ok()) {
host = sni_hostname->c_str();
}
dbgDebug(D_CONNECTION) << "Setting TLS host name extension. Host: " << host;
if (!SSL_set_tlsext_host_name(ssl_socket, host)) {
return genError("Failed to set TLS host name extension. Host: " + string(host));
}
@@ -656,7 +664,7 @@ private:
printOut(const string &data)
{
string type = getConfigurationWithDefault<string>("chopped", "message", "Data printout type");
uint length = getConfigurationWithDefault<uint>(10, "message", "Data printout length");
uint length = getConfigurationWithDefault<uint>(50, "message", "Data printout length");
if (type == "full") return data;
if (type == "size") return to_string(data.size()) + " bytes";
if (type == "none") return "";
@@ -689,6 +697,7 @@ private:
bool lock = false;
bool should_close_connection = false;
bool is_dual_auth = false;
Maybe<string> sni_hostname = genError<string>("Uninitialized");
};
Connection::Connection(const MessageConnectionKey &key, const MessageMetadata &metadata)

5
core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc
View File

@@ -112,9 +112,8 @@ MessagingBufferComponent::Impl::init()
auto sub_path = getProfileAgentSettingWithDefault<string>("nano_agent/event_buffer/", "eventBuffer.baseFolder");
buffer_root_path = getLogFilesPathConfig() + "/" + sub_path;
string full_executable_name =
Singleton::Consume<I_Environment>::by<Messaging>()->get<string>("Executable Name").unpack();
string executable_name = full_executable_name.substr(full_executable_name.find_last_of("/") + 1);
string executable_name =
Singleton::Consume<I_Environment>::by<Messaging>()->get<string>("Base Executable Name").unpack();
removeLegacyBuffer(buffer_root_path, executable_name);
mkdir(buffer_root_path.c_str(), 0644);

2
core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc
View File

@@ -26,7 +26,7 @@ public:
TestMessagingBuffer()
{
env.preload();
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "tmp_test_file");
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "tmp_test_file");
config.preload();
config.init();

170
core/metric/generic_metric.cc
View File

@@ -17,6 +17,7 @@
#include "debug.h"
#include "report/log_rest.h"
#include "config.h"
#include "report_messaging.h"
#include <fstream>
#include <sstream>
@@ -30,6 +31,24 @@ MetricMetadata::DotName operator"" _dot(const char *str, size_t) { return Metric
MetricMetadata::Units operator"" _unit(const char *str, size_t) { return MetricMetadata::Units{str}; }
MetricMetadata::Description operator"" _desc(const char *str, size_t) { return MetricMetadata::Description{str}; }
// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
static ostream & operator<<(ostream &os, const AiopsMetricList &metrics) { return os << metrics.toString(); }
// LCOV_EXCL_STOP
vector<AiopsMetricData>
MetricCalc::getAiopsMetrics() const
{
float value = getValue();
if (isnan(value)) return {};
string name = getMetricDotName() != "" ? getMetricDotName() : getMetricName();
string units = getMetircUnits();
string description = getMetircDescription();
string type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter";
return { AiopsMetricData(name, type, units, description, getBasicLabels(), value) };
}
string
MetricCalc::getMetadata(const string &key) const
{
@@ -54,6 +73,54 @@ MetricCalc::addMetric(GenericMetric *metric)
if (metric != nullptr) metric->addCalc(this);
}
vector<PrometheusData>
MetricCalc::getPrometheusMetrics() const
{
float value = getValue();
if (isnan(value)) return {};
PrometheusData res;
res.name = getMetricDotName() != "" ? getMetricDotName() : getMetricName();
res.type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter";
res.desc = getMetircDescription();
stringstream labels;
const auto &label_pairs = getBasicLabels();
bool first = true;
for (auto &pair : label_pairs) {
if (!first) labels << ',';
labels << pair.first << "=\"" << pair.second << '"';
first = false;
}
res.label = labels.str();
stringstream value_str;
value_str << value;
res.value = value_str.str();
return {res};
}
map<string, string>
MetricCalc::getBasicLabels() const
{
map<string, string> res;
auto i_instance = Singleton::Consume<I_InstanceAwareness>::by<GenericMetric>();
auto id = i_instance->getUniqueID();
if (id.ok()) res["id"] = *id;
auto details = Singleton::Consume<I_AgentDetails>::by<GenericMetric>();
res["agent"] = details->getAgentId();
auto env = Singleton::Consume<I_Environment>::by<GenericMetric>();
auto executable = env->get<string>("Base Executable Name");
if (executable.ok()) res["process"] = *executable;
return res;
}
static const string metric_file = "/tmp/metrics_output.txt";
class GenericMetric::MetricsRest : public ServerRest
@@ -121,7 +188,9 @@ void
GenericMetric::handleMetricStreamSending()
{
if (active_streams.isSet(Stream::DEBUG)) generateDebug();
if (active_streams.isSet(Stream::PROMETHEUS)) generatePrometheus();
if (active_streams.isSet(Stream::FOG)) generateLog();
if (active_streams.isSet(Stream::AIOPS)) generateAiopsLog();
if (reset) resetMetrics();
}
@@ -244,6 +313,106 @@ GenericMetric::generateLog()
sendLog(metric_client_rest);
}
class PrometheusRest : public ClientRest
{
class Metric : public ClientRest
{
public:
Metric(const string &n, const string &t, const string &d, const string &l, const string &v)
:
name(n),
type(t),
description(d),
labels(l),
value(v)
{}
private:
C2S_PARAM(string, name);
C2S_PARAM(string, type);
C2S_PARAM(string, description);
C2S_PARAM(string, labels);
C2S_PARAM(string, value);
};
public:
PrometheusRest() : metrics(vector<Metric>()) {}
void
addMetric(const vector<PrometheusData> &vec)
{
auto &metric_vec = metrics.get();
metric_vec.reserve(vec.size());
for (auto &metric : vec) {
metric_vec.emplace_back(metric.name, metric.type, metric.desc, "{" + metric.label + "}", metric.value);
}
}
private:
C2S_PARAM(vector<Metric>, metrics);
};
void
GenericMetric::generatePrometheus()
{
if (!getProfileAgentSettingWithDefault(false, "prometheus")) return;
vector<PrometheusData> all_metrics;
for (auto &calc : calcs) {
const auto &cal_metrics = calc->getPrometheusMetrics();
all_metrics.insert(all_metrics.end(), cal_metrics.begin(), cal_metrics.end());
}
PrometheusRest rest;
rest.addMetric(all_metrics);
MessageMetadata new_config_req_md("127.0.0.1", 7465);
new_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
Singleton::Consume<I_Messaging>::by<GenericMetric>()->sendSyncMessage(
HTTPMethod::POST,
"/set-prometheus-data",
rest,
MessageCategory::GENERIC,
new_config_req_md
);
}
void
GenericMetric::generateAiopsLog()
{
if (!getConfigurationWithDefault<bool>(true, "metric", "aiopsMetricSendEnable")) return;
AiopsMetricList aiops_metrics;
for (auto &calc : calcs) {
aiops_metrics.addMetrics(calc->getAiopsMetrics());
}
set<ReportIS::Tags> tags;
Report metric_to_fog(
"AIOPS Metric Data",
Singleton::Consume<I_TimeGet>::by<GenericMetric>()->getWalltime(),
Type::PERIODIC,
Level::LOG,
LogLevel::INFO,
audience,
team,
Severity::INFO,
Priority::LOW,
report_interval,
LogField("agentId", Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId()),
tags,
Tags::INFORMATIONAL,
issuing_engine
);
metric_to_fog << LogField("eventObject", aiops_metrics);
LogRest metric_client_rest(metric_to_fog);
sendLog(metric_client_rest);
}
void
GenericMetric::generateDebug()
{
@@ -270,6 +439,7 @@ GenericMetric::preload()
{
registerExpectedConfiguration<bool>("metric", "fogMetricSendEnable");
registerExpectedConfiguration<bool>("metric", "debugMetricSendEnable");
registerExpectedConfiguration<bool>("metric", "aiopsMetricSendEnable");
registerExpectedConfiguration<bool>("metric", "fogMetricUri");
registerExpectedConfiguration<string>("metric", "metricsOutputTmpFile");
}

539
core/metric/metric_ut/metric_ut.cc
View File

@@ -10,6 +10,7 @@
#include "mock/mock_rest_api.h"
#include "agent_details.h"
#include "mock/mock_messaging.h"
#include "mock/mock_instance_awareness.h"
#include "config.h"
#include "config_component.h"
@@ -67,6 +68,14 @@ public:
top_usage.report(event.getCPU());
}
void
setAiopsMetric()
{
turnOnStream(Stream::AIOPS);
max.setMetricDotName("cpu.max");
max.setMetircUnits("percrnt");
}
Max<double> max{this, "cpuMax"};
Min<double> min{this, "cpuMin"};
Average<double> avg{this, "cpuAvg"};
@@ -121,13 +130,15 @@ public:
class HttpTransaction : public Event<HttpTransaction>
{
public:
HttpTransaction(const string &_url, uint _bytes) : url(_url), bytes(_bytes) {}
HttpTransaction(const string &_url, const string &m, uint _bytes) : url(_url), method(m), bytes(_bytes) {}
const string & getUrl() const { return url;}
const string & getUrl() const { return url; }
const string & getMethod() const { return method; }
uint getBytes() const { return bytes; }
private:
string url;
string method;
uint bytes;
};
@@ -144,9 +155,33 @@ public:
total.report(event.getUrl(), 1);
}
void
setAiopsMetric()
{
turnOnStream(Stream::AIOPS);
}
private:
MetricMap<string, Average<double>> avg{this, "PerUrlAvg"};
MetricMap<string, NoResetCounter> total{this, "TotalRequests"};
MetricMap<string, Average<double>> avg{Average<double>{nullptr, ""}, this, "url", "PerUrlAvg"};
MetricMap<string, NoResetCounter> total{NoResetCounter{nullptr, ""}, this, "url", "TotalRequests"};
};
class UrlMetric2 : public GenericMetric, public Listener<HttpTransaction>
{
public:
void
upon(const HttpTransaction &event) override
{
total.report(event.getUrl(), event.getMethod(), 1);
}
private:
MetricMap<string, MetricMap<string, NoResetCounter>> total{
MetricMap<string, NoResetCounter>{NoResetCounter{nullptr, ""}, nullptr, "method", ""},
this,
"url",
"request.total"
};
};
class MetricTest : public Test
@@ -155,6 +190,8 @@ public:
MetricTest()
{
EXPECT_CALL(rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true));
ON_CALL(instance, getUniqueID()).WillByDefault(Return(string("87")));
ON_CALL(instance, getFamilyID()).WillByDefault(Return(string("")));
env.init();
Debug::setNewDefaultStdout(&debug_output);
Debug::setUnitTestFlag(D_METRICS, Debug::DebugLevel::TRACE);
@@ -177,6 +214,7 @@ public:
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> timer;
NiceMock<MockInstanceAwareness> instance;
StrictMock<MockRestApi> rest;
::Environment env;
ConfigComponent conf;
@@ -255,7 +293,9 @@ TEST_F(MetricTest, basicMetricTest)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 89,\n"
@@ -326,7 +366,9 @@ TEST_F(MetricTest, basicMetricTest)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 90,\n"
@@ -399,7 +441,9 @@ TEST_F(MetricTest, basicMetricTest)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 100,\n"
@@ -485,6 +529,150 @@ TEST_F(MetricTest, printMetricsTest)
GenericMetric::fini();
}
TEST_F(MetricTest, printPromeathus)
{
conf.preload();
stringstream configuration;
configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
EXPECT_TRUE(Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration));
CPUMetric cpu_mt;
cpu_mt.init(
"CPU usage",
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
false
);
cpu_mt.turnOffStream(GenericMetric::Stream::FOG);
cpu_mt.turnOffStream(GenericMetric::Stream::DEBUG);
cpu_mt.turnOnStream(GenericMetric::Stream::PROMETHEUS);
cpu_mt.registerListener();
CPUEvent cpu_event;
cpu_event.setProcessCPU(89);
cpu_event.notify();
string message_body;
EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _))
.WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
routine();
string res =
"{\n"
" \"metrics\": [\n"
" {\n"
" \"name\": \"cpuMax\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"name\": \"cpuMin\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"name\": \"cpuAvg\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"name\": \"cpuCurrent\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"89\"\n"
" },\n"
" {\n"
" \"name\": \"cpuCounter\",\n"
" \"type\": \"gauge\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"name\": \"cpuTotalCounter\",\n"
" \"type\": \"counter\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
" \"value\": \"1\"\n"
" }\n"
" ]\n"
"}";
EXPECT_EQ(message_body, res);
}
TEST_F(MetricTest, printPromeathusMultiMap)
{
conf.preload();
stringstream configuration;
configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
EXPECT_TRUE(Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration));
UrlMetric2 metric;
metric.init(
"Bytes per URL",
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
true
);
metric.turnOnStream(GenericMetric::Stream::PROMETHEUS);
metric.registerListener();
HttpTransaction("/index.html", "GET", 10).notify();
HttpTransaction("/index2.html", "GET", 20).notify();
HttpTransaction("/index.html", "POST", 40).notify();
string message_body;
EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _))
.WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
routine();
string res =
"{\n"
" \"metrics\": [\n"
" {\n"
" \"name\": \"request.total\",\n"
" \"type\": \"counter\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"name\": \"request.total\",\n"
" \"type\": \"counter\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n"
" \"value\": \"1\"\n"
" },\n"
" {\n"
" \"name\": \"request.total\",\n"
" \"type\": \"counter\",\n"
" \"description\": \"\",\n"
" \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
"method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n"
" \"value\": \"1\"\n"
" }\n"
" ]\n"
"}";
EXPECT_EQ(message_body, res);
}
TEST_F(MetricTest, metricTestWithReset)
{
CPUMetric cpu_mt;
@@ -554,7 +742,9 @@ TEST_F(MetricTest, metricTestWithReset)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 89,\n"
@@ -624,7 +814,9 @@ TEST_F(MetricTest, metricTestWithReset)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 90,\n"
@@ -694,7 +886,9 @@ TEST_F(MetricTest, metricTestWithReset)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 100,\n"
@@ -796,7 +990,9 @@ TEST_F(MetricTest, generateReportWithReset)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 89,\n"
@@ -888,7 +1084,9 @@ TEST_F(MetricTest, generateReportWithReset)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 90,\n"
@@ -959,7 +1157,9 @@ TEST_F(MetricTest, generateReportWithReset)
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"My named nano service\"\n"
" \"serviceName\": \"My named nano service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"cpuMax\": 100,\n"
@@ -1015,6 +1215,7 @@ TEST_F(MetricTest, allMetricTest)
seconds(5),
false
);
msg_size_mt.registerListener();
EXPECT_EQ(msg_size_mt.getMetricName(), "Message size");
@@ -1074,9 +1275,9 @@ TEST_F(MetricTest, testMapMetric)
);
url_mt.registerListener();
HttpTransaction("/index.html", 10).notify();
HttpTransaction("/index2.html", 20).notify();
HttpTransaction("/index.html", 40).notify();
HttpTransaction("/index.html", "GET", 10).notify();
HttpTransaction("/index2.html", "GET", 20).notify();
HttpTransaction("/index.html", "POST", 40).notify();
string message_body;
@@ -1191,3 +1392,309 @@ TEST_F(MetricTest, testManyValuesOutOfOrder)
EXPECT_THAT(AllMetricEvent().query(), ElementsAre(HasSubstr(cpu_str)));
}
TEST_F(MetricTest, basicAIOPSMetricTest)
{
EXPECT_CALL(timer, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087")));
CPUMetric cpu_mt;
cpu_mt.init(
"CPU usage",
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
false
);
cpu_mt.setAiopsMetric();
cpu_mt.registerListener();
EXPECT_EQ(cpu_mt.getMetricName(), "CPU usage");
EXPECT_EQ(cpu_mt.getReportInterval().count(), 5);
routine();
CPUEvent cpu_event;
cpu_event.setProcessCPU(89);
cpu_event.notify();
string metric_str =
"{\n"
" \"Metric\": \"CPU usage\",\n"
" \"Reporting interval\": 5,\n"
" \"cpuMax\": 89.0,\n"
" \"cpuMin\": 89.0,\n"
" \"cpuAvg\": 89.0,\n"
" \"cpuCurrent\": 89.0,\n"
" \"cpuCounter\": 1,\n"
" \"cpuTotalCounter\": 1,\n"
" \"cpuTops\": [\n"
" 89.0\n"
" ]\n"
"}";
string message_body;
EXPECT_CALL(messaging_mock, sendAsyncMessage(
_,
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
string expected_message =
"{\n"
" \"log\": {\n"
" \"eventTime\": \"2016-11-13T17:31:24.087\",\n"
" \"eventName\": \"AIOPS Metric Data\",\n"
" \"eventSeverity\": \"Info\",\n"
" \"eventPriority\": \"Low\",\n"
" \"eventType\": \"Periodic\",\n"
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"Agent Core\",\n"
" \"eventFrequency\": 5,\n"
" \"eventTags\": [\n"
" \"Informational\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"issuingEngine\": \"Agent Core\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"eventObject\": {\n"
" \"Metrics\": [\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpu.max\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"percrnt\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 89.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuMin\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 89.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuAvg\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 89.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuCurrent\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 89.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuCounter\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 1.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"cpuTotalCounter\",\n"
" \"MetricType\": \"counter\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 1.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {},\n"
" \"AssetID\": \"Unknown\"\n"
" }\n"
" ]\n"
" }\n"
" }\n"
" }\n"
"}";
routine();
EXPECT_THAT(debug_output.str(), HasSubstr(metric_str));
EXPECT_EQ(message_body, expected_message);
debug_output.str("");
}
TEST_F(MetricTest, testAIOPSMapMetric)
{
EXPECT_CALL(timer, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087")));
UrlMetric url_mt;
url_mt.init(
"Bytes per URL",
ReportIS::AudienceTeam::AGENT_CORE,
ReportIS::IssuingEngine::AGENT_CORE,
seconds(5),
true
);
url_mt.registerListener();
url_mt.setAiopsMetric();
HttpTransaction("/index.html", "GET", 10).notify();
HttpTransaction("/index2.html", "GET", 20).notify();
HttpTransaction("/index.html", "POST", 40).notify();
string message_body;
EXPECT_CALL(messaging_mock, sendAsyncMessage(
_,
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
routine();
string expected_message =
"{\n"
" \"log\": {\n"
" \"eventTime\": \"2016-11-13T17:31:24.087\",\n"
" \"eventName\": \"AIOPS Metric Data\",\n"
" \"eventSeverity\": \"Info\",\n"
" \"eventPriority\": \"Low\",\n"
" \"eventType\": \"Periodic\",\n"
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"Agent Core\",\n"
" \"eventFrequency\": 5,\n"
" \"eventTags\": [\n"
" \"Informational\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"issuingEngine\": \"Agent Core\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\",\n"
" \"serviceId\": \"87\",\n"
" \"serviceFamilyId\": \"\"\n"
" },\n"
" \"eventData\": {\n"
" \"eventObject\": {\n"
" \"Metrics\": [\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"/index.html\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 25.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {\n"
" \"url\": \"/index.html\"\n"
" },\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"/index2.html\",\n"
" \"MetricType\": \"gauge\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 20.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {\n"
" \"url\": \"/index2.html\"\n"
" },\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"/index.html\",\n"
" \"MetricType\": \"counter\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 2.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {\n"
" \"url\": \"/index.html\"\n"
" },\n"
" \"AssetID\": \"Unknown\"\n"
" },\n"
" {\n"
" \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
" \"MetricName\": \"/index2.html\",\n"
" \"MetricType\": \"counter\",\n"
" \"MetricUnit\": \"\",\n"
" \"MetricDescription\": \"\",\n"
" \"MetricValue\": 1.0,\n"
" \"ResourceAttributes\": {\n"
" \"agent\": \"Unknown\",\n"
" \"id\": \"87\"\n"
" },\n"
" \"MetricAttributes\": {\n"
" \"url\": \"/index2.html\"\n"
" },\n"
" \"AssetID\": \"Unknown\"\n"
" }\n"
" ]\n"
" }\n"
" }\n"
" }\n"
"}";
EXPECT_EQ(message_body, expected_message);
}

15
core/rest/rest_conn.cc
View File

@@ -23,11 +23,12 @@ using namespace std;
USE_DEBUG_FLAG(D_API);
RestConn::RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke)
RestConn::RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke, bool is_external)
:
fd(_fd),
mainloop(_mainloop),
invoke(_invoke)
fd(_fd),
mainloop(_mainloop),
invoke(_invoke),
is_external_ip(is_external)
{}
RestConn::~RestConn()
@@ -101,6 +102,12 @@ RestConn::parseConn() const
return sendResponse("200 OK", invoke->invokeGet(identifier), false);
}
if (is_external_ip) {
dbgWarning(D_API) << "External IP tried to POST";
sendResponse("500 Internal Server Error", "", false);
stop();
}
stringstream body;
body.str(readSize(len));

3
core/rest/rest_conn.h
View File

@@ -21,7 +21,7 @@
class RestConn
{
public:
RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke);
RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke, bool is_external = false);
~RestConn();
void parseConn() const;
@@ -35,6 +35,7 @@ private:
int fd;
I_MainLoop *mainloop;
const I_RestInvoke *invoke;
bool is_external_ip = false;
};
#endif // __REST_CONN_H__

109
core/rest/rest_server.cc
View File

@@ -47,6 +47,7 @@ public:
void startNewConnection() const;
bool bindRestServerSocket(struct sockaddr_in &addr, vector<uint16_t> port_range);
bool bindRestServerSocket(struct sockaddr_in6 &addr, vector<uint16_t> port_range);
bool addRestCall(RestAction oper, const string &uri, unique_ptr<RestInit> &&init) override;
bool addGetCall(const string &uri, const function<string()> &cb) override;
uint16_t getListeningPort() const override { return listening_port; }
@@ -73,10 +74,36 @@ private:
bool
RestServer::Impl::bindRestServerSocket(struct sockaddr_in &addr, vector<uint16_t> port_range)
{
dbgFlow(D_API) << "Binding IPv4 socket";
for (uint16_t port : port_range) {
addr.sin_port = htons(port);
if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) == 0) return true;
if (errno == EADDRINUSE) {
dbgDebug(D_API) << "Port " << port << " is already in use";
} else {
dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno);
}
}
return false;
}
bool
RestServer::Impl::bindRestServerSocket(struct sockaddr_in6 &addr, vector<uint16_t> port_range)
{
dbgFlow(D_API) << "Binding IPv6 socket";
for (uint16_t port : port_range) {
addr.sin6_port = htons(port);
if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in6)) == 0) return true;
if (errno == EADDRINUSE) {
dbgDebug(D_API) << "Port " << port << " is already in use";
} else {
dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno);
}
}
return false;
@@ -119,21 +146,55 @@ RestServer::Impl::init()
mainloop = Singleton::Consume<I_MainLoop>::by<RestServer>();
auto init_connection = [this] () {
fd = socket(AF_INET, SOCK_STREAM, 0);
auto allow_external_conn = "Nano service API Allow Get From External IP";
auto conf_value = getConfiguration<bool>("connection", allow_external_conn);
bool accept_get_from_external_ip = false;
if (conf_value.ok()) {
accept_get_from_external_ip = *conf_value;
} else {
auto env_value = Singleton::Consume<I_Environment>::by<RestServer>()->get<bool>(allow_external_conn);
if (env_value.ok()) {
accept_get_from_external_ip = *env_value;
}
}
if (accept_get_from_external_ip) {
fd = socket(AF_INET6, SOCK_STREAM, 0);
} else {
fd = socket(AF_INET, SOCK_STREAM, 0);
}
dbgAssert(fd >= 0) << alert << "Failed to open a socket";
int socket_enable = 1;
if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)) < 0) {
dbgWarning(D_API) << "Could not set the socket options";
}
struct sockaddr_in addr;
bzero(&addr, sizeof(addr));
if (accept_get_from_external_ip) {
int option = 0;
if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &option, sizeof(option)) < 0) {
dbgWarning(D_API) << "Could not set the IPV6_V6ONLY option";
}
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
struct sockaddr_in6 addr6;
bzero(&addr6, sizeof(addr6));
addr6.sin6_family = AF_INET6;
addr6.sin6_addr = in6addr_any;
while (!bindRestServerSocket(addr, port_range)) {
mainloop->yield(bind_retry_interval_msec);
while (!bindRestServerSocket(addr6, port_range)) {
mainloop->yield(bind_retry_interval_msec);
}
listening_port = ntohs(addr6.sin6_port);
} else {
struct sockaddr_in addr;
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
while (!bindRestServerSocket(addr, port_range)) {
mainloop->yield(bind_retry_interval_msec);
}
listening_port = ntohs(addr.sin_port);
}
listen(fd, listen_limit);
@@ -146,9 +207,12 @@ RestServer::Impl::init()
"REST server listener",
is_primary.ok() && *is_primary
);
listening_port = ntohs(addr.sin_port);
dbgInfo(D_API) << "REST server started: " << listening_port;
dbgInfo(D_API)
<< "REST server started: "
<< listening_port
<< ". Accepting: "
<< (accept_get_from_external_ip ? "external" : "loopback")
<< " connections";
Singleton::Consume<I_Environment>::by<RestServer>()->registerValue<int>("Listening Port", listening_port);
};
@@ -172,14 +236,30 @@ void
RestServer::Impl::startNewConnection() const
{
dbgFlow(D_API) << "Starting a new connection";
int new_socket = accept(fd, nullptr, nullptr);
struct sockaddr_storage addr;
socklen_t addr_len = sizeof(addr);
int new_socket = accept(fd, (struct sockaddr *)&addr, &addr_len);
if (new_socket < 0) {
dbgWarning(D_API) << "Failed to accept a new socket";
dbgWarning(D_API) << "Failed to accept a new socket: " << strerror(errno);
return;
}
dbgDebug(D_API) << "Starting a new socket: " << new_socket;
RestConn conn(new_socket, mainloop, this);
dbgDebug(D_API) << "Starting a new socket: " << new_socket;
bool is_external = false;
if (addr.ss_family == AF_INET6) {
struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *) &addr;
if (IN6_IS_ADDR_V4MAPPED(&addr_in6->sin6_addr)) {
struct in_addr ipv4_addr;
memcpy(&ipv4_addr, &addr_in6->sin6_addr.s6_addr[12], sizeof(ipv4_addr));
is_external = ipv4_addr.s_addr != htonl(INADDR_LOOPBACK);
} else {
is_external = memcmp(&addr_in6->sin6_addr, &in6addr_loopback, sizeof(in6addr_loopback)) != 0;
}
} else {
struct sockaddr_in *addr_in = (struct sockaddr_in *)&addr;
is_external = addr_in->sin_addr.s_addr != htonl(INADDR_LOOPBACK);
}
RestConn conn(new_socket, mainloop, this, is_external);
mainloop->addFileRoutine(
I_MainLoop::RoutineType::Offline,
new_socket,
@@ -283,4 +363,5 @@ RestServer::preload()
registerExpectedConfiguration<uint>("connection", "Nano service API Port Alternative");
registerExpectedConfiguration<uint>("connection", "Nano service API Port Range start");
registerExpectedConfiguration<uint>("connection", "Nano service API Port Range end");
registerExpectedConfiguration<bool>("connection", "Nano service API Allow Get From External IP");
}

224
core/rest/rest_ut/rest_config_ut.cc
View File

@@ -14,10 +14,49 @@
#include "agent_details.h"
#include "mock/mock_messaging.h"
#include "tenant_manager.h"
#include <netdb.h>
#include <arpa/inet.h>
using namespace std;
using namespace testing;
static const string config_json_allow_external =
"{\n"
" \"connection\": {\n"
" \"Nano service API Port Primary\": [\n"
" {\n"
" \"value\": 9777\n"
" }\n"
" ],\n"
" \"Nano service API Port Alternative\": [\n"
" {\n"
" \"value\": 9778\n"
" }\n"
" ],\n"
" \"Nano service API Allow Get From External IP\": [\n"
" {\n"
" \"value\": true\n"
" }\n"
" ]\n"
" }\n"
"}\n";
static const string config_json =
"{\n"
" \"connection\": {\n"
" \"Nano service API Port Primary\": [\n"
" {\n"
" \"value\": 9777\n"
" }\n"
" ],\n"
" \"Nano service API Port Alternative\": [\n"
" {\n"
" \"value\": 9778\n"
" }\n"
" ]\n"
" }\n"
"}\n";
USE_DEBUG_FLAG(D_API);
USE_DEBUG_FLAG(D_MAINLOOP);
@@ -31,22 +70,6 @@ public:
time_proxy.init();
mainloop_comp.init();
string config_json =
"{\n"
" \"connection\": {\n"
" \"Nano service API Port Primary\": [\n"
" {\n"
" \"value\": 9777\n"
" }\n"
" ],\n"
" \"Nano service API Port Alternative\": [\n"
" {\n"
" \"value\": 9778\n"
" }\n"
" ]\n"
" }\n"
"}\n";
istringstream ss(config_json);
Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss);
@@ -58,6 +81,9 @@ public:
~RestConfigTest()
{
Debug::setNewDefaultStdout(&cout);
auto mainloop = Singleton::Consume<I_MainLoop>::from(mainloop_comp);
mainloop->stopAll();
rest_server.fini();
time_proxy.fini();
mainloop_comp.fini();
}
@@ -133,7 +159,7 @@ int TestServer::g_num = 0;
TEST_F(RestConfigTest, basic_flow)
{
env.preload();
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "tmp_test_file");
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "tmp_test_file");
config.preload();
config.init();
@@ -165,11 +191,15 @@ TEST_F(RestConfigTest, basic_flow)
auto mainloop = Singleton::Consume<I_MainLoop>::from(mainloop_comp);
I_MainLoop::Routine stop_routine = [&] () {
EXPECT_EQ(connect(file_descriptor1, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0);
EXPECT_EQ(connect(file_descriptor1, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0)
<< "file_descriptor1 Error: "
<< strerror(errno);
string msg1 = "GET /stuff HTTP/1.1\r\n\r\n";
EXPECT_EQ(write(file_descriptor1, msg1.data(), msg1.size()), static_cast<int>(msg1.size()));
EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0);
EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0)
<< "file_descriptor2 Error: "
<< strerror(errno);
string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}";
EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast<int>(msg2.size()));
@@ -204,3 +234,159 @@ TEST_F(RestConfigTest, basic_flow)
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 6\r\n\r\nblabla"
);
}
string
getLocalIPAddress() {
char hostname[1024];
hostname[1024 - 1] = '\0';
// Get the hostname
if (gethostname(hostname, sizeof(hostname)) == -1) {
return "";
}
struct addrinfo hints, *info, *p;
int gai_result;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_INET; // Use AF_INET for IPv4
hints.ai_socktype = SOCK_STREAM;
// Get the address info
if ((gai_result = getaddrinfo(hostname, nullptr, &hints, &info)) != 0) {
return "";
}
std::string ip_address;
for (p = info; p != nullptr; p = p->ai_next) {
void *addr;
char ipstr[INET_ADDRSTRLEN];
struct sockaddr_in *ipv4 = (struct sockaddr_in *)p->ai_addr;
addr = &(ipv4->sin_addr);
// Convert the IP to a string and print it
inet_ntop(p->ai_family, addr, ipstr, sizeof(ipstr));
if (std::string(ipstr) != "127.0.0.1") {
ip_address = ipstr;
break;
}
}
freeaddrinfo(info); // Free the linked list
return ip_address;
}
TEST_F(RestConfigTest, not_loopback_flow)
{
env.preload();
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "tmp_test_file");
istringstream ss(config_json_allow_external);
Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss);
config.preload();
config.init();
rest_server.init();
time_proxy.init();
mainloop_comp.init();
auto i_rest = Singleton::Consume<I_RestApi>::from(rest_server);
ASSERT_TRUE(i_rest->addRestCall<TestServer>(RestAction::ADD, "test"));
ASSERT_TRUE(i_rest->addGetCall("stuff", [] () { return string("blabla"); }));
int file_descriptor1 = socket(AF_INET, SOCK_STREAM, 0);
EXPECT_NE(file_descriptor1, -1);
int file_descriptor2 = socket(AF_INET, SOCK_STREAM, 0);
EXPECT_NE(file_descriptor2, -1);
auto primary_port = getConfiguration<uint>("connection", "Nano service API Port Primary");
auto second_port = getConfiguration<uint>("connection", "Nano service API Port Alternative");
auto local_ip = getLocalIPAddress();
struct sockaddr_in sa_primary;
sa_primary.sin_family = AF_INET;
sa_primary.sin_port = htons(primary_port.unpack());
sa_primary.sin_addr.s_addr = inet_addr(local_ip.c_str());
struct sockaddr_in sa_second;
sa_second.sin_family = AF_INET;
sa_second.sin_port = htons(second_port.unpack());
sa_second.sin_addr.s_addr = inet_addr(local_ip.c_str());
int socket_enable = 1;
EXPECT_EQ(setsockopt(file_descriptor1, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0);
EXPECT_EQ(setsockopt(file_descriptor2, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0);
EXPECT_CALL(messaging, sendSyncMessage(_, _, _, _, _))
.WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")));
Debug::setNewDefaultStdout(&cout);
auto mainloop = Singleton::Consume<I_MainLoop>::from(mainloop_comp);
Debug::setNewDefaultStdout(&cout);
I_MainLoop::Routine stop_routine = [&] () {
int socket_client_2 = -1;
auto socket_client_1 = connect(file_descriptor1, (struct sockaddr*)&sa_primary, sizeof(struct sockaddr));
dbgDebug(D_API) << "socket_client_1: " << socket_client_1;
if (socket_client_1 == -1) {
dbgDebug(D_API) << "Error: " << strerror(errno);
socket_client_2 = connect(file_descriptor1, (struct sockaddr*)&sa_second, sizeof(struct sockaddr));
dbgDebug(D_API) << "socket_client_2: " << socket_client_2;
if (socket_client_2 == -1) {
dbgDebug(D_API) << "Error: " << strerror(errno) << endl;
} else {
EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa_second, sizeof(struct sockaddr)), 0);
string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}";
EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast<int>(msg2.size()));
}
} else {
EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa_primary, sizeof(struct sockaddr)), 0);
string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}";
EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast<int>(msg2.size()));
}
EXPECT_TRUE(socket_client_1 != -1 || socket_client_2 != -1);
string msg1 = "GET /stuff HTTP/1.1\r\n\r\n";
EXPECT_EQ(write(file_descriptor1, msg1.data(), msg1.size()), static_cast<int>(msg1.size()));
mainloop->yield(true);
struct pollfd s_poll;
s_poll.fd = file_descriptor1;
s_poll.events = POLLIN;
s_poll.revents = 0;
while(poll(&s_poll, 1, 0) <= 0) {
mainloop->yield(true);
}
struct pollfd s_poll2;
s_poll2.fd = file_descriptor2;
s_poll2.events = POLLIN;
s_poll2.revents = 0;
while(poll(&s_poll2, 1, 0) <= 0) {
mainloop->yield(true);
}
mainloop->stopAll();
};
mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::RealTime,
stop_routine,
"RestConfigTest-alternative_port_used stop routine",
true
);
mainloop->run();
char respose[1000];
EXPECT_EQ(read(file_descriptor1, respose, 1000), 76);
EXPECT_EQ(
string(respose, 76),
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 6\r\n\r\nblabla"
);
EXPECT_EQ(read(file_descriptor2, respose, 1000), 89);
EXPECT_EQ(
string(respose, 89),
"HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\nContent-Length: 0\r\n\r\n"
);
}

2
core/rest/rest_ut/rest_schema_ut.cc
View File

@@ -437,7 +437,7 @@ TEST(RestSchema, server_schema)
env.preload();
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "tmp_test_file");
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "tmp_test_file");
config.preload();
config.init();