sync code

This commit is contained in:
Ned Wright
2024-12-29 12:13:27 +00:00
parent 96ce290e5f
commit 64ebf013eb
43 changed files with 1058 additions and 406 deletions

View File

@@ -31,6 +31,7 @@ add_subdirectory(tenant_manager)
add_subdirectory(compression)
add_subdirectory(attachments)
add_subdirectory(report_messaging)
add_subdirectory(env_details)
add_library(ngen_core SHARED ".")
target_link_libraries(
@@ -39,7 +40,7 @@ target_link_libraries(
"table;debug_is;shell_cmd;metric;tenant_manager;messaging;encryptor;time_proxy;singleton;mainloop;environment;logging;report;rest"
"compression_utils;-lz;config;intelligence_is_v2;event_is;memory_consumption;connkey"
"instance_awareness;socket_is;agent_details;agent_details_reporter;buffers;cpu;agent_core_utilities"
"report_messaging"
"report_messaging;env_details"
-Wl,-no-whole-archive
)

View File

@@ -111,7 +111,8 @@ HttpAttachmentConfiguration::save(cereal::JSONOutputArchive &archive) const
cereal::make_nvp("keep_alive_interval_msec", getNumericalValue("keep_alive_interval_msec")),
cereal::make_nvp("min_retries_for_verdict", getNumericalValue("min_retries_for_verdict")),
cereal::make_nvp("max_retries_for_verdict", getNumericalValue("max_retries_for_verdict")),
cereal::make_nvp("body_size_trigger", getNumericalValue("body_size_trigger"))
cereal::make_nvp("body_size_trigger", getNumericalValue("body_size_trigger")),
cereal::make_nvp("remove_server_header", getNumericalValue("remove_server_header"))
);
}
@@ -167,6 +168,7 @@ HttpAttachmentConfiguration::load(cereal::JSONInputArchive &archive)
loadNumericalValue(archive, "min_retries_for_verdict", 3);
loadNumericalValue(archive, "max_retries_for_verdict", 15);
loadNumericalValue(archive, "body_size_trigger", 200000);
loadNumericalValue(archive, "remove_server_header", 0);
}
bool

View File

@@ -63,7 +63,8 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration)
"\"waiting_for_verdict_thread_timeout_msec\": 60,\n"
"\"req_header_thread_timeout_msec\": 10,\n"
"\"ip_ranges\": " + createIPRangesString(ip_ranges) + ",\n"
"\"static_resources_path\": \"" + static_resources_path + "\""
"\"static_resources_path\": \"" + static_resources_path + "\",\n"
"\"remove_server_header\": 0"
"}\n";
ofstream valid_configuration_file(attachment_configuration_file_name);
valid_configuration_file << valid_configuration;
@@ -89,6 +90,7 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration)
EXPECT_EQ(conf_data_out.getNumericalValue("res_body_thread_timeout_msec"), 80u);
EXPECT_EQ(conf_data_out.getNumericalValue("waiting_for_verdict_thread_timeout_msec"), 60u);
EXPECT_EQ(conf_data_out.getNumericalValue("nginx_inspection_mode"), 1u);
EXPECT_EQ(conf_data_out.getNumericalValue("remove_server_header"), 0u);
}
TEST_F(HttpAttachmentUtilTest, GetMalformedAttachmentConfiguration)

View File

@@ -57,6 +57,7 @@ unsigned int getResBodyThreadTimeout();
unsigned int getMinRetriesForVerdict();
unsigned int getMaxRetriesForVerdict();
unsigned int getReqBodySizeTrigger();
unsigned int getRemoveResServerHeader();
unsigned int getWaitingForVerdictThreadTimeout();

View File

@@ -228,6 +228,18 @@ public:
return sni_host_name;
}
void
setDnHostName(const std::string &_dn_host_name)
{
dn_host_name = _dn_host_name;
}
Maybe<std::string>
getDnHostName() const
{
return dn_host_name;
}
void
setRateLimitBlock(uint block_time)
{
@@ -273,6 +285,7 @@ public:
private:
std::string host_name = "";
Maybe<std::string> sni_host_name = genError("SNI host name not set");
Maybe<std::string> dn_host_name = genError("DN host name not set");
std::string ca_path = "";
std::string client_cert_path = "";
std::string client_key_path = "";

View File

@@ -47,6 +47,7 @@
#include "buffer.h"
#include "intelligence_comp_v2.h"
#include "messaging.h"
#include "env_details.h"
USE_DEBUG_FLAG(D_COMP_IS);
@@ -233,6 +234,7 @@ class ComponentListCore
MemoryCalculator,
TenantManager,
GenericRulebase,
EnvDetails,
Components...
>
{

View File

@@ -18,12 +18,14 @@
#include "i_mainloop.h"
#include "singleton.h"
#include "component.h"
#include "i_time_get.h"
class SocketIS
:
public Component,
Singleton::Provide<I_Socket>,
Singleton::Consume<I_MainLoop>
Singleton::Consume<I_MainLoop>,
Singleton::Consume<I_TimeGet>
{
public:
SocketIS();

View File

@@ -1,3 +1,3 @@
add_library(logging logging.cc log_generator.cc debug_stream.cc file_stream.cc fog_stream.cc syslog_stream.cc cef_stream.cc k8s_svc_stream.cc)
add_library(logging logging.cc log_generator.cc debug_stream.cc file_stream.cc fog_stream.cc syslog_stream.cc cef_stream.cc k8s_svc_stream.cc log_connector.cc)
add_subdirectory(logging_ut)

View File

@@ -24,22 +24,21 @@ USE_DEBUG_FLAG(D_REPORT);
static string lookup_cmd = "nslookup ";
static string line_selection_cmd = "| grep Address | sed -n 2p";
static string parsing_cmd = "| cut -f2 -d' ' | tr -d '\n'";
static string CEF_NAME = "CEF";
CefStream::CefStream(const string &_address, int _port, I_Socket::SocketType _protocol)
:
i_socket(Singleton::Consume<I_Socket>::by<LoggingComp>()),
address(_address),
port(_port),
protocol(_protocol)
LogStreamConnector(_address, _port, _protocol, CEF_NAME)
{
connect();
if (!socket.ok()) {
dbgWarning(D_REPORT) << "Failed to connect to the CEF server";
}
init();
socket = genError("Not set yet");
}
CefStream::~CefStream()
{
sendAllLogs();
if (mainloop != nullptr && mainloop->doesRoutineExist(connecting_routine)) mainloop->stop(connecting_routine);
if (socket.ok()) {
i_socket->closeSocket(const_cast<int &>(*socket));
socket = genError("Closed socket");
@@ -49,50 +48,60 @@ CefStream::~CefStream()
void
CefStream::sendLog(const Report &log)
{
if (!socket.ok()) {
connect();
if (!socket.ok()) {
dbgWarning(D_REPORT) << "Failed to connect to the CEF server, log will not be sent.";
return;
}
}
dbgTrace(D_REPORT) << "Connected to socket.";
string cef_report = log.getCef();
if (protocol == I_Socket::SocketType::TCP) {
cef_report = to_string(cef_report.length()) + " " + cef_report;
}
vector<char> data(cef_report.begin(), cef_report.end());
for (size_t tries = 0; tries < 3; tries++) {
if (i_socket->writeData(socket.unpack(), data)) {
dbgTrace(D_REPORT) << "log was sent to CEF server";
return;
} else {
dbgWarning(D_REPORT) << "Failed to send log to CEF server";
}
}
sendLogWithQueue(data);
}
void
CefStream::init() {
updateSettings();
maintainConnection();
auto ceflog_retry_interval = getProfileAgentSettingWithDefault<uint>(
RETRY_CONNECT_INTERVAL,
"agent.config.log.cefServer.connect_retry_interval");
dbgTrace(D_REPORT) << "retry interval: " << ceflog_retry_interval;
chrono::seconds connect_retry_interval = chrono::seconds(ceflog_retry_interval);
connecting_routine = mainloop->addRecurringRoutine(
I_MainLoop::RoutineType::Offline,
connect_retry_interval,
[this] ()
{
dbgTrace(D_REPORT) << CEF_CONNECT_NAME;
maintainConnection();
},
CEF_CONNECT_NAME
);
}
void
CefStream::connect()
{
auto cef_address = getProfileAgentSettingWithDefault<string>(address, "agent.config.log.cefServer.IP");
auto cef_port = getProfileAgentSettingWithDefault<uint>(port, "agent.config.log.cefServer.port");
if (cef_address.empty()) {
dbgDebug(D_REPORT)
<< "Connecting to CEF server"
<< " Address: "
<< address
<< " Port: "
<< port;
if (address.empty()) {
dbgWarning(D_REPORT) << "Cannot connect to CEF server, IP/Domain is not configured.";
return;
}
struct in_addr addr;
if (inet_pton(AF_INET, cef_address.data(), &addr) != 1) {
if (inet_pton(AF_INET, address.data(), &addr) != 1) {
I_ShellCmd *shell_cmd = Singleton::Consume<I_ShellCmd>::by<LoggingComp>();
string host_cmd = lookup_cmd + cef_address + line_selection_cmd + parsing_cmd;
string host_cmd = lookup_cmd + address + line_selection_cmd + parsing_cmd;
Maybe<string> res = shell_cmd->getExecOutput(host_cmd, 500);
if (!res.ok()) {
dbgWarning(D_REPORT)
<< "Failed to execute domain lookup command. "
<< "CEF Domain: "
<< cef_address
<< address
<< "Error: "
<< res.getErr();
return;
@@ -102,7 +111,7 @@ CefStream::connect()
dbgWarning(D_REPORT)
<< "Got en empty ip address from lookup command. "
<< "CEF Domain: "
<< cef_address
<< address
<< "Got bad ip address: "
<< res.unpack();
return;
@@ -113,19 +122,47 @@ CefStream::connect()
dbgWarning(D_REPORT)
<< "Got a faulty ip address from lookup command. "
<< "CEF Domain: "
<< cef_address
<< address
<< "Got bad ip address: "
<< res.unpack();
return;
}
cef_address = res.unpack();
address = res.unpack();
}
socket = i_socket->genSocket(
protocol,
false,
false,
cef_address + ":" + to_string(cef_port)
address + ":" + to_string(port)
);
}
void
CefStream::updateSettings()
{
max_logs_per_send = getProfileAgentSettingWithDefault<int>(
NUMBER_OF_LOGS_PER_SEND,
"agent.config.log.cefServer.MaxLogsPerSend"
);
if (max_logs_per_send < 0) {
max_logs_per_send = NUMBER_OF_LOGS_PER_SEND;
}
address = getProfileAgentSettingWithDefault<string>(address, "agent.config.log.cefServer.IP");
port = getProfileAgentSettingWithDefault<uint>(port, "agent.config.log.cefServer.port");
max_data_in_queue = getProfileAgentSettingWithDefault<uint>(
MAX_LOG_QUEUE,
"agent.config.log.cefServer.MaxDataInQueue"
);
dbgTrace(D_REPORT)
<< "CEF server settings updated. "
<< "Address: "
<< address
<< " Port: "
<< port
<< " Max logs per send: "
<< max_logs_per_send
<< " Max data in queue: "
<< max_data_in_queue;
}

View File

@@ -23,6 +23,13 @@
#include "logging_metric.h"
#include "i_logging.h"
#include "i_socket_is.h"
#include "logging_comp.h"
static const int RETRY_CONNECT_INTERVAL = 120;
static const std::string SYSLOG_CONNECT_NAME = "connecting to Syslog server";
static const std::string CEF_CONNECT_NAME = "connecting to CEF server";
static const int NUMBER_OF_LOGS_PER_SEND = 15;
static size_t MAX_LOG_QUEUE = 1000;
USE_DEBUG_FLAG(D_REPORT);
@@ -93,43 +100,77 @@ private:
I_Messaging *i_msg = nullptr;
};
class SyslogStream : public Stream
class LogStreamConnector : public Stream
{
public:
LogStreamConnector(
const std::string &_address,
int _port,
I_Socket::SocketType _protocol,
const std::string &_log_name) :
mainloop(Singleton::Consume<I_MainLoop>::by<LoggingComp>()),
i_socket(Singleton::Consume<I_Socket>::by<LoggingComp>()),
address(_address),
port(_port),
protocol(_protocol),
logs_in_queue(),
log_name(_log_name) {}
virtual ~LogStreamConnector() {}
protected:
virtual void connect() = 0;
virtual void updateSettings() = 0;
void maintainConnection();
void addLogToQueue(const std::vector<char> &data);
void writeFail();
bool basicWriteLog(const std::vector<char> &data);
void sendLogWithQueue(const std::vector<char> &data);
void sendAllLogs();
I_MainLoop *mainloop = nullptr;
I_Socket *i_socket = nullptr;
std::string address;
int port;
I_Socket::SocketType protocol = I_Socket::SocketType::UDP;
Maybe<I_Socket::socketFd> socket = genError("Not set yet");
bool did_write_fail_in_this_window = false;
std::vector<std::vector<char>> logs_in_queue;
I_MainLoop::RoutineID connecting_routine = -1;
int max_logs_per_send = NUMBER_OF_LOGS_PER_SEND;
std::string log_name;
uint max_data_in_queue = MAX_LOG_QUEUE;
};
class SyslogStream : public LogStreamConnector
{
public:
SyslogStream(const std::string &_address, int _port, I_Socket::SocketType protocol);
~SyslogStream();
void sendLog(const Report &log) override;
private:
void sendLog(const std::vector<char> &data);
void connect();
protected:
void connect() override;
void updateSettings() override;
I_Socket *i_socket = nullptr;
I_MainLoop *mainloop = nullptr;
std::string address;
int port;
I_Socket::SocketType protocol = I_Socket::SocketType::UDP;
private:
void init();
void sendLog(const std::vector<char> &data);
I_MainLoop::RoutineID log_send_routine = -1;
Maybe<I_Socket::socketFd> socket = genError("Not set yet");
};
class CefStream : public Stream
class CefStream : public LogStreamConnector
{
public:
CefStream(const std::string &_address, int _port, I_Socket::SocketType _protocol);
~CefStream();
void sendLog(const Report &log) override;
protected:
void connect() override;
void updateSettings() override;
private:
void connect();
I_Socket *i_socket = nullptr;
std::string address;
int port;
I_Socket::SocketType protocol = I_Socket::SocketType::UDP;
Maybe<I_Socket::socketFd> socket = genError("Not set yet");
void init();
};
#endif // __LOG_STREAMS_H__

View File

@@ -171,6 +171,7 @@ public:
} else {
LogEventLogsSent(true).notify();
for (auto &iter : streams) {
dbgTrace(D_REPORT) << "Sending log to stream: " << TagAndEnumManagement::convertToString(iter.first);
if (log.isStreamActive(iter.first)) iter.second->sendLog(log);
}
}

View File

@@ -24,6 +24,7 @@
#include "metric/all_metric_event.h"
#include "mock/mock_shell_cmd.h"
#include "version.h"
#include "../log_streams.h"
using namespace testing;
using namespace std;
@@ -104,6 +105,7 @@ public:
class LogTest : public testing::Test
{
public:
const ::Report CreateReport(ReportIS::Tags &tag1, ReportIS::Tags &tag2);
LogTest()
:
agent_details(),
@@ -132,6 +134,16 @@ public:
DoAll(SaveArg<1>(&sysog_routine), Return(0))
);
EXPECT_CALL(
mock_mainloop,
addRecurringRoutine(_, _, _, "connecting to Syslog server", _)
).WillRepeatedly(DoAll(SaveArg<2>(&connect_syslog_routine), Return(2)));
EXPECT_CALL(
mock_mainloop,
addRecurringRoutine(_, _, _, "connecting to CEF server", _)
).WillRepeatedly(DoAll(SaveArg<2>(&connect_cef_routine), Return(3)));
EXPECT_CALL(mock_socket_is, writeData(1, _)).WillRepeatedly(
WithArg<1>(
Invoke(
@@ -291,6 +303,8 @@ public:
ConfigComponent config;
vector<string> capture_syslog_cef_data;
I_MainLoop::Routine sysog_routine = nullptr;
I_MainLoop::Routine connect_syslog_routine = nullptr;
I_MainLoop::Routine connect_cef_routine = nullptr;
StrictMock<MockShellCmd> mock_shell_cmd;
bool is_domain;
@@ -1469,98 +1483,148 @@ TEST_F(LogTest, BulkModification)
EXPECT_EQ(local_body, str1);
}
TEST_F(LogTest, ObfuscationTest)
const ::Report
LogTest::CreateReport(Tags &tag1, Tags &tag2) {
LogField origin("String", "Another string");
const ::Report report(
"String=\"Another string\"",
chrono::microseconds(90000),
Type::EVENT,
Level::LOG,
LogLevel::INFO,
Audience::INTERNAL,
AudienceTeam::AGENT_CORE,
Severity::INFO,
Priority::LOW,
chrono::seconds(3600),
origin,
tag1,
tag2,
Notification::POLICY_UPDATE,
IssuingEngine::AGENT_CORE
);
return report;
}
TEST_F(LogTest, ObfuscationCefSysLogTest)
{
loadFakeConfiguration(false);
Tags tag1 = Tags::POLICY_INSTALLATION;
Tags tag2 = Tags::ACCESS_CONTROL;
std::string address = "172.28.1.6";
int port = 514;
I_Socket::SocketType protocol = I_Socket::SocketType::TCP;
// for cef
CefStream cef_stream(address, port, protocol);
ASSERT_NE(connect_cef_routine, nullptr);
connect_cef_routine();
cef_stream.sendLog(CreateReport(tag1, tag2));
EXPECT_EQ(capture_syslog_cef_data.size(), 1u);
// for syslog activate send log
SyslogStream syslog_stream(address, port, protocol);
static const string expected_obfuscated_log(
"{\n"
" \"log\": {\n"
" \"eventTime\": \"0:0:0\",\n"
" \"eventName\": \"Install policy\",\n"
" \"eventSeverity\": \"Info\",\n"
" \"eventPriority\": \"Low\",\n"
" \"eventType\": \"Event Driven\",\n"
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"\",\n"
" \"eventFrequency\": 0,\n"
" \"eventTags\": [\n"
" \"Access Control\",\n"
" \"Policy Installation\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" },\n"
" \"eventData\": {\n"
" \"logIndex\": 1,\n"
" \"String\": \"{XORANDB64}:mocked field\"\n"
" }\n"
" }\n"
"}"
);
StrictMock<MockEncryptor> mock_encrypt;
EXPECT_CALL(mock_encrypt, base64Encode(_)).WillOnce(Return("mocked field"));
static const string expected_clear_log(
"{\n"
" \"eventTime\": \"0:0:0\",\n"
" \"eventName\": \"Install policy\",\n"
" \"eventSeverity\": \"Info\",\n"
" \"eventPriority\": \"Low\",\n"
" \"eventType\": \"Event Driven\",\n"
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"\",\n"
" \"eventFrequency\": 0,\n"
" \"eventTags\": [\n"
" \"Access Control\",\n"
" \"Policy Installation\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" },\n"
" \"eventData\": {\n"
" \"logIndex\": 1,\n"
" \"String\": \"Another string\"\n"
" }\n"
"}"
);
{
LogGen log(
"Install policy",
Audience::INTERNAL,
Severity::INFO,
Priority::LOW,
tag1,
tag2,
Enreachments::BEAUTIFY_OUTPUT
);
log << LogField("String", "Another string", LogFieldOption::XORANDB64);
EXPECT_EQ(toJson(log), expected_clear_log);
}
EXPECT_THAT(getMessages(), HasSubstr(expected_clear_log));
EXPECT_THAT(readLogFile(), HasSubstr(expected_clear_log));
EXPECT_EQ(getBodyFogMessage(), expected_obfuscated_log);
// connection to socket before send log
ASSERT_NE(connect_syslog_routine, nullptr);
connect_syslog_routine();
syslog_stream.sendLog(CreateReport(tag1, tag2)); // send log in routine sysog_routine
ASSERT_NE(sysog_routine, nullptr);
sysog_routine();
EXPECT_EQ(capture_syslog_cef_data.size(), 2u);
EXPECT_EQ(capture_syslog_cef_data.size(), 2u); // 1 for CEF 1 for Syslog
for (const string &str : capture_syslog_cef_data) {
EXPECT_THAT(str, AnyOf(HasSubstr("String='Another string'"), HasSubstr("String=\"Another string\"")));
EXPECT_THAT(str, AnyOf(
HasSubstr("String='Another string'"),
HasSubstr(R"(String="Another string")"),
HasSubstr("String=\"Another string\"")));
}
}
TEST_F(LogTest, SysLogWriteFailTest)
{
loadFakeConfiguration(false);
capture_syslog_cef_data.clear();
Tags tag1 = Tags::POLICY_INSTALLATION;
Tags tag2 = Tags::ACCESS_CONTROL;
// for syslog activate send log
std::string address = "172.28.1.6";
int port = 514;
I_Socket::SocketType protocol = I_Socket::SocketType::TCP;
SyslogStream syslog_stream(address, port, protocol);
ASSERT_NE(connect_syslog_routine, nullptr);
connect_syslog_routine();
EXPECT_CALL(mock_socket_is, writeData(1, _))
.WillOnce(Return(false))
.WillOnce(Return(false))
.WillOnce(Return(false))
.WillRepeatedly(
WithArg<1>(
Invoke(
[this](const vector<char> &data)
{
capture_syslog_cef_data.emplace_back(data.begin(), data.end());
return true;
}
)
)
);
syslog_stream.sendLog(CreateReport(tag1, tag2));
ASSERT_NE(sysog_routine, nullptr);
EXPECT_EQ(capture_syslog_cef_data.size(), 0u); //before write
sysog_routine();
EXPECT_EQ(capture_syslog_cef_data.size(), 1u);
for (const string &str : capture_syslog_cef_data) {
EXPECT_THAT(str, AnyOf(
HasSubstr("String='Another string'"),
HasSubstr(R"(String="Another string")"),
HasSubstr("String=\"Another string\"")));
}
}
TEST_F(LogTest, CefWriteFailTest)
{
loadFakeConfiguration(false);
capture_syslog_cef_data.clear();
Tags tag1 = Tags::POLICY_INSTALLATION;
Tags tag2 = Tags::ACCESS_CONTROL;
// for syslog activate send log
std::string address = "172.28.1.6";
int port = 514;
I_Socket::SocketType protocol = I_Socket::SocketType::TCP;
CefStream cef_stream(address, port, protocol);
ASSERT_NE(connect_cef_routine, nullptr);
connect_cef_routine();
EXPECT_CALL(mock_socket_is, writeData(1, _))
.WillOnce(Return(false))
.WillOnce(Return(false))
.WillOnce(Return(false))
.WillRepeatedly(
WithArg<1>(
Invoke(
[this](const vector<char> &data)
{
capture_syslog_cef_data.emplace_back(data.begin(), data.end());
return true;
}
)
)
);
EXPECT_EQ(capture_syslog_cef_data.size(), 0u); //before write
cef_stream.sendLog(CreateReport(tag1, tag2));
EXPECT_EQ(capture_syslog_cef_data.size(), 1u);
for (const string &str : capture_syslog_cef_data) {
EXPECT_THAT(str, AnyOf(
HasSubstr("String='Another string'"),
HasSubstr(R"(String="Another string")"),
HasSubstr("String=\"Another string\"")));
}
}

View File

@@ -23,24 +23,22 @@ USE_DEBUG_FLAG(D_REPORT);
static string lookup_cmd = "nslookup ";
static string line_selection_cmd = "| grep Address | sed -n 2p";
static string parsing_cmd = "| cut -f2 -d' ' | tr -d '\n'";
static string SYSLOG_NAME = "Syslog";
SyslogStream::SyslogStream(const string &_address, int _port, I_Socket::SocketType _protocol)
:
i_socket(Singleton::Consume<I_Socket>::by<LoggingComp>()),
mainloop(Singleton::Consume<I_MainLoop>::by<LoggingComp>()),
address(_address),
port(_port),
protocol(_protocol)
LogStreamConnector(_address, _port, _protocol, SYSLOG_NAME)
{
connect();
if (!socket.ok()) {
dbgWarning(D_REPORT) << "Failed to connect to the syslog server";
}
socket = genError("Not set yet");
init();
}
SyslogStream::~SyslogStream()
{
sendAllLogs();
if (mainloop != nullptr && mainloop->doesRoutineExist(log_send_routine)) mainloop->stop(log_send_routine);
if (mainloop != nullptr && mainloop->doesRoutineExist(connecting_routine)) mainloop->stop(connecting_routine);
if (socket.ok()) {
i_socket->closeSocket(const_cast<int &>(*socket));
socket = genError("Closed socket");
@@ -55,7 +53,7 @@ SyslogStream::sendLog(const Report &log)
syslog_report = to_string(syslog_report.length()) + " " + syslog_report;
}
vector<char> data(syslog_report.begin(), syslog_report.end());
mainloop->addOneTimeRoutine(
log_send_routine = mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::Offline,
[this, data] () { sendLog(data); },
"Logging Syslog stream messaging"
@@ -65,45 +63,57 @@ SyslogStream::sendLog(const Report &log)
void
SyslogStream::sendLog(const vector<char> &data)
{
for (int tries = 0; tries < 3; ++tries) {
if (!socket.ok()) {
connect();
if (!socket.ok()) {
dbgWarning(D_REPORT) << "Failed to connect to the syslog server, Log will not be sent.";
return;
}
dbgTrace(D_REPORT) << "Successfully connect to the syslog server";
}
dbgTrace(D_REPORT) << "Sending Syslog log." << " Max logs per send: " << max_logs_per_send;
sendLogWithQueue(data);
}
if (i_socket->writeData(socket.unpack(), data)) {
dbgTrace(D_REPORT) << "log was sent to syslog server";
return;
}
}
dbgWarning(D_REPORT) << "Failed to send log to syslog server";
void
SyslogStream::init() {
updateSettings();
maintainConnection();
auto syslog_retry_interval = getProfileAgentSettingWithDefault<uint>(
RETRY_CONNECT_INTERVAL,
"agent.config.log.syslogServer.connect_retry_interval");
chrono::seconds connect_retry_interval = chrono::seconds(syslog_retry_interval);
connecting_routine = mainloop->addRecurringRoutine(
I_MainLoop::RoutineType::Offline,
connect_retry_interval,
[this] ()
{
dbgTrace(D_REPORT) << SYSLOG_CONNECT_NAME;
maintainConnection();
},
SYSLOG_CONNECT_NAME
);
}
void
SyslogStream::connect()
{
auto syslog_address = getProfileAgentSettingWithDefault<string>(address, "agent.config.log.syslogServer.IP");
auto syslog_port = getProfileAgentSettingWithDefault<uint>(port, "agent.config.log.syslogServer.port");
dbgDebug(D_REPORT)
<< "Connecting to Syslog server"
<< " Address: "
<< address
<< " Port: "
<< port;
if (syslog_address.empty()) {
if (address.empty()) {
dbgWarning(D_REPORT) << "Cannot connect to Syslog server, Address IP/Domain not configured.";
return;
}
struct in_addr addr;
if (inet_pton(AF_INET, syslog_address.data(), &addr) != 1) {
if (inet_pton(AF_INET, address.data(), &addr) != 1) {
I_ShellCmd *shell_cmd = Singleton::Consume<I_ShellCmd>::by<LoggingComp>();
string host_cmd = lookup_cmd + syslog_address + line_selection_cmd + parsing_cmd;
string host_cmd = lookup_cmd + address + line_selection_cmd + parsing_cmd;
Maybe<string> res = shell_cmd->getExecOutput(host_cmd, 500);
if (!res.ok()) {
dbgWarning(D_REPORT)
<< "Failed to execute domain lookup command. "
<< "SYSLOG Domain: "
<< syslog_address
<< address
<< "Error: "
<< res.getErr();
return;
@@ -113,7 +123,7 @@ SyslogStream::connect()
dbgWarning(D_REPORT)
<< "Got en empty ip address from lookup command. "
<< "SYSLOG Domain: "
<< syslog_address
<< address
<< "Got bad ip address: "
<< res.unpack();
return;
@@ -124,19 +134,46 @@ SyslogStream::connect()
dbgWarning(D_REPORT)
<< "Got a faulty ip address from lookup command. "
<< "SYSLOG Domain: "
<< syslog_address
<< address
<< "Got bad ip address: "
<< res.unpack();
return;
}
syslog_address = res.unpack();
address = res.unpack();
}
socket = i_socket->genSocket(
protocol,
false,
false,
syslog_address + ":" + to_string(syslog_port)
address + ":" + to_string(port)
);
}
void
SyslogStream::updateSettings()
{
max_logs_per_send = getProfileAgentSettingWithDefault<int>(
NUMBER_OF_LOGS_PER_SEND,
"agent.config.log.syslogServer.MaxLogsPerSend"
);
if (max_logs_per_send < 0) {
max_logs_per_send = NUMBER_OF_LOGS_PER_SEND;
}
address = getProfileAgentSettingWithDefault<string>(address, "agent.config.log.syslogServer.IP");
port = getProfileAgentSettingWithDefault<uint>(port, "agent.config.log.syslogServer.port");
max_data_in_queue =
getProfileAgentSettingWithDefault<uint>(MAX_LOG_QUEUE, "agent.config.log.syslogServer.MaxLogQueue");
dbgTrace(D_REPORT)
<< "Syslog server settings updated. "
<< "Address: "
<< address
<< " Port: "
<< port
<< " Max logs per send: "
<< max_logs_per_send
<< " Max data in queue: "
<< max_data_in_queue;
}

View File

@@ -98,6 +98,8 @@ public:
}
sni_hostname = metadata.getSniHostName();
dn_host_name = metadata.getDnHostName();
}
void
@@ -328,17 +330,24 @@ private:
SSL_set_hostflags(ssl_socket, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS);
auto host = key.getHostName().c_str();
if (!SSL_set1_host(ssl_socket, host)) {
return genError("Failed to set host name verification. Host: " + string(host));
auto dn = host;
auto sni = host;
if (dn_host_name.ok()) {
dn = dn_host_name->c_str();
}
dbgDebug(D_CONNECTION) << "Setting host DN: " << dn;
if (!SSL_set1_host(ssl_socket, dn)) {
return genError("Failed to set host name verification. Host: " + string(dn));
}
if (sni_hostname.ok()) {
host = sni_hostname->c_str();
sni = sni_hostname->c_str();
}
dbgDebug(D_CONNECTION) << "Setting TLS host name extension. Host: " << host;
if (!SSL_set_tlsext_host_name(ssl_socket, host)) {
return genError("Failed to set TLS host name extension. Host: " + string(host));
dbgDebug(D_CONNECTION) << "Setting TLS host name extension. Host: " << sni;
if (!SSL_set_tlsext_host_name(ssl_socket, sni)) {
return genError("Failed to set TLS host name extension. Host: " + string(sni));
}
return Maybe<void>();
@@ -698,6 +707,8 @@ private:
bool should_close_connection = false;
bool is_dual_auth = false;
Maybe<string> sni_hostname = genError<string>("Uninitialized");
Maybe<string> dn_host_name = genError<string>("Uninitialized");
};
Connection::Connection(const MessageConnectionKey &key, const MessageMetadata &metadata)
@@ -716,7 +727,7 @@ Connection::setProxySettings(const MessageProxySettings &settings)
map<string, string> headers;
auto i_encrypt = Singleton::Consume<I_Encryptor>::by<Messaging>();
if (!settings.getProxyAuth().empty()) {
headers["Proxy-Authorization"] = i_encrypt->base64Encode(settings.getProxyAuth());
headers["Proxy-Authorization"] = "Basic " + i_encrypt->base64Encode(settings.getProxyAuth());
}
auto req = HTTPRequest::prepareRequest(*this, HTTPMethod::CONNECT, "", headers, "");

View File

@@ -30,11 +30,15 @@ using namespace std;
USE_DEBUG_FLAG(D_MESSAGING_BUFFER);
#ifndef smb
static constexpr uint buffer_max_size_MB = 100;
static constexpr uint buffer_max_size_MB = 300;
#else
static constexpr uint buffer_max_size_MB = 3;
#endif
static const uint reservation_default_size = 32;
static const uint memory_messages_max_size_default = 1024;
static const uint additional_buffer_size_default = 128;
static bool
checkExistence(const string &path)
{
@@ -74,7 +78,7 @@ private:
void handleInMemoryMessages();
void writeToDisk(const BufferedMessage &message);
bool writeToDisk(const BufferedMessage &message);
static Maybe<uint32_t> seekStartOfMessage(FILE *file);
static bool readBytes(FILE *file, uint size_to_read, char *output_bytes);
@@ -94,6 +98,9 @@ private:
string buffer_output;
string buffer_root_path;
uint max_size_on_disk_MB = 0;
uint memory_messages_max_size = 0;
uint additional_buffer_size = 0;
uint memory_messages_reserve_size = reservation_default_size;
uint curr_no_retries = 0;
I_ShellCmd *shell_cmd = nullptr;
I_Encryptor *encryptor = nullptr;
@@ -105,6 +112,16 @@ void
MessagingBufferComponent::Impl::init()
{
max_size_on_disk_MB = getProfileAgentSettingWithDefault<uint>(buffer_max_size_MB, "eventBuffer.maxSizeOnDiskInMB");
memory_messages_max_size =
getProfileAgentSettingWithDefault<uint>(
memory_messages_max_size_default,
"eventBuffer.maxMemoryMessagesToStore"
);
additional_buffer_size =
getProfileAgentSettingWithDefault<uint>(
additional_buffer_size_default,
"eventBuffer.additionalBufferSize"
);
shell_cmd = Singleton::Consume<I_ShellCmd>::by<Messaging>();
encryptor = Singleton::Consume<I_Encryptor>::by<Messaging>();
mainloop = Singleton::Consume<I_MainLoop>::by<Messaging>();
@@ -121,7 +138,7 @@ MessagingBufferComponent::Impl::init()
string unique_id = instance_awareness->getInstanceID().ok() ? instance_awareness->getInstanceID().unpack() : "";
buffer_input = buffer_root_path + "/" + executable_name + unique_id + ".input";
buffer_output = buffer_root_path + "/" + executable_name + unique_id + ".output";
memory_messages.reserve(32);
memory_messages.reserve(memory_messages_reserve_size);
uint tmo = getConfigurationWithDefault<uint>(5, "message", "Send event retry in sec");
mainloop->addRecurringRoutine(
@@ -138,6 +155,26 @@ MessagingBufferComponent::Impl::init()
"Handling in-memory messages",
false
);
registerConfigLoadCb(
[this]() {
memory_messages_max_size =
getProfileAgentSettingWithDefault<uint>(
1000,
"eventBuffer.maxMemoryMessagesToStore"
);
max_size_on_disk_MB =
getProfileAgentSettingWithDefault<uint>(
buffer_max_size_MB,
"eventBuffer.maxSizeOnDiskInMB"
);
additional_buffer_size =
getProfileAgentSettingWithDefault<uint>(
100,
"eventBuffer.additionalBufferSize"
);
}
);
}
void
@@ -152,8 +189,10 @@ MessagingBufferComponent::Impl::pushNewBufferedMessage(
{
dbgTrace(D_MESSAGING_BUFFER) << "Pushing new message to buffer";
if (!force_immediate_writing) {
dbgDebug(D_MESSAGING_BUFFER) << "Holding message temporarily in memory";
if (!force_immediate_writing && memory_messages.size() < memory_messages_max_size + additional_buffer_size) {
dbgTrace(D_MESSAGING_BUFFER)
<< "Holding message temporarily in memory. Memory messages size: "
<< memory_messages.size();
memory_messages.emplace_back(body, method, uri, category, message_metadata);
return;
}
@@ -328,30 +367,80 @@ void
MessagingBufferComponent::Impl::handleInMemoryMessages()
{
auto messages = move(memory_messages);
memory_messages.reserve(32);
uint failed_messages = 0;
dbgDebug(D_MESSAGING_BUFFER) << "Handling " << to_string(messages.size()) <<" new in-memory messages";
for (const auto &message : messages) {
if (sendMessage(message) != HTTPStatusCode::HTTP_OK) {
if (message.getMessageMetadata().shouldBufferMessage()) writeToDisk(message);
memory_messages.reserve(memory_messages_reserve_size);
auto it = messages.begin();
for (; it != messages.end() && memory_messages.size() < memory_messages_max_size; ++it) {
if (sendMessage(*it) != HTTPStatusCode::HTTP_OK) {
if (it->getMessageMetadata().shouldBufferMessage()) {
if (!writeToDisk(*it)) ++failed_messages;
}
}
dbgTrace(D_MESSAGING_BUFFER)
<< "Processed "
<< (it - messages.begin() + 1)
<< " messages out of "
<< messages.size();
mainloop->yield();
}
if (it == messages.end()) {
memory_messages_reserve_size = reservation_default_size;
if (failed_messages > 0) {
dbgDebug(D_MESSAGING_BUFFER)
<< "Failed to handle "
<< to_string(failed_messages)
<< " messages out of "
<< to_string(messages.size());
}
return;
}
memory_messages_reserve_size =
min(memory_messages_reserve_size * 2, memory_messages_max_size + additional_buffer_size);
dbgDebug(D_MESSAGING_BUFFER) << "Heap buffer is full. Storing messages to disk";
auto it2 = messages.end() - 1;
do {
if (it2->getMessageMetadata().shouldBufferMessage() && !writeToDisk(*it2)) {
failed_messages += it2 - it + 1;
break;
}
} while(it2-- != it);
if (failed_messages > 0) {
dbgDebug(D_MESSAGING_BUFFER)
<< "Failed to handle "
<< to_string(failed_messages)
<< " messages out of "
<< to_string(messages.size());
}
}
void
bool
MessagingBufferComponent::Impl::writeToDisk(const BufferedMessage &message)
{
static uint full_buffer_failed_messages = 0;
auto serialized_message = message.toString();
if (!canWriteToDisk(serialized_message.size())) {
dbgWarning(D_MESSAGING_BUFFER) << "Buffer is full. Message will not be written to disk: " << message.getURI();
return;
full_buffer_failed_messages++;
if (full_buffer_failed_messages % 10 == 0) {
dbgWarning(D_MESSAGING_BUFFER)
<< "Buffer is full. "
<< full_buffer_failed_messages
<< " messages will not be written to disk";
}
dbgDebug(D_MESSAGING_BUFFER) << "Buffer is full. Message will not be written to disk: " << message.getURI();
return false;
}
full_buffer_failed_messages = 0;
ofstream file(buffer_input, ios::app);
if (!file.is_open()) {
dbgWarning(D_MESSAGING_BUFFER) << "Failed to open file for writing. File: " << buffer_input;
return;
return false;
}
uint32_t size = serialized_message.size();
@@ -359,6 +448,8 @@ MessagingBufferComponent::Impl::writeToDisk(const BufferedMessage &message)
file.write(reinterpret_cast<char *>(&size), sizeof(size));
char type = 0;
file.write(&type, 1);
return true;
}
Maybe<uint32_t>
@@ -538,7 +629,7 @@ MessagingBufferComponent::Impl::canWriteToDisk(size_t message_size) const
return true;
}
dbgWarning(D_MESSAGING_BUFFER)
dbgDebug(D_MESSAGING_BUFFER)
<< "Buffer size is full. Directry size: "
<< *maybe_directory_size
<< ", Message size: "

View File

@@ -282,3 +282,51 @@ TEST_F(TestMessagingBuffer, testRoutinInMemory)
msg = buffer_provider->peekMessage();
ASSERT_FALSE(msg.ok());
}
TEST_F(TestMessagingBuffer, testRoutinInMemoryOverflow)
{
string config_json =
"{"
" \"agentSettings\": [\n"
" {\n"
" \"id\": \"123\",\n"
" \"key\": \"eventBuffer.maxMemoryMessagesToStore\",\n"
" \"value\": \"5\"\n"
" },\n"
" {\n"
" \"id\": \"123\",\n"
" \"key\": \"eventBuffer.additionalBufferSize\",\n"
" \"value\": \"1\"\n"
" }]\n"
"}";
istringstream ss(config_json);
Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss);
MessageCategory category = MessageCategory::GENERIC;
MessageMetadata message_metadata = MessageMetadata();
message_metadata.setShouldBufferMessage(true);
HTTPMethod method = HTTPMethod::POST;
HTTPResponse res(HTTPStatusCode::HTTP_OK, "");
for (int i = 0; i < 6; i++) {
string body = "body" + to_string(i);
buffer_provider->pushNewBufferedMessage(body, method, "/" + to_string(i), category, message_metadata, false);
EXPECT_CALL(mock_messaging, sendSyncMessage(method, "/" + to_string(i), body, _, _)).WillOnce(Return(res));
}
for (int i = 0; i < 2; i++) {
string body = "body" + to_string(i);
buffer_provider->pushNewBufferedMessage(body, method, "/" + to_string(i), category, message_metadata, false);
}
memory_routine();
for (int i = 0; i < 2; i++) {
auto msg = buffer_provider->peekMessage();
ASSERT_TRUE(msg.ok());
buffer_provider->popMessage();
}
auto msg = buffer_provider->peekMessage();
ASSERT_FALSE(msg.ok());
}

View File

@@ -26,9 +26,16 @@
#include <sys/stat.h>
#include <errno.h>
#include "debug.h"
#include "config.h"
#include "singleton.h"
#include "i_time_get.h"
static const uint udp_max_packet_size = 1024 * 64;
static const AlertInfo alert(AlertTeam::CORE, "socket i/s");
static const uint CONNECT_TIMEOUT_MICROSECOUNDS(10000000); //10 seconds
static const uint CHECK_CONNECTION_INTERVAL_MICROSECONDS(250000); //0.25 seconds
static const std::chrono::microseconds CHRONO_CHECK_CONNECTION_INTERVAL =
std::chrono::microseconds(CHECK_CONNECTION_INTERVAL_MICROSECONDS);
USE_DEBUG_FLAG(D_SOCKET);
@@ -235,9 +242,117 @@ private:
}
};
int setNonBlocking(int socket) {
dbgTrace(D_SOCKET) << "Setting socket to non-blocking mode";
int flags = fcntl(socket, F_GETFL, 0);
if (flags == -1) return -1;
return fcntl(socket, F_SETFL, flags | O_NONBLOCK);
}
int setBlocking(int socket) {
dbgTrace(D_SOCKET) << "Setting socket to blocking mode";
int flags = fcntl(socket, F_GETFL, 0);
if (flags == -1) return -1;
return fcntl(socket, F_SETFL, flags & ~O_NONBLOCK);
}
class TCPSocket : public SocketInternal
{
public:
static Maybe<unique_ptr<TCPSocket>>
connectAsClient(unique_ptr<TCPSocket> &tcp_socket)
{
if (setNonBlocking(tcp_socket->getSocket()) < 0) {
dbgTrace(D_SOCKET) << "Failed to set the socket to non-blocking mode";
return genError("Failed to set the socket to non-blocking mode");
}
chrono::microseconds time_before_connect =
Singleton::Consume<I_TimeGet>::by<SocketIS>()->getWalltime();
if (connect(
tcp_socket->getSocket(),
reinterpret_cast<struct sockaddr *>(&tcp_socket->server),
sizeof(struct sockaddr_in)
) >= 0
) {
dbgTrace(D_SOCKET) << "Successfully connected to socket";
if (setBlocking(tcp_socket->getSocket()) < 0) {
dbgWarning(D_SOCKET) << "Failed to set the socket to blocking mode";
close(tcp_socket->getSocket());
return genError("Failed to set the socket to blocking mode");
}
return move(tcp_socket);
}
if (setBlocking(tcp_socket->getSocket()) < 0) {
dbgWarning(D_SOCKET) << "Failed to set the socket to blocking mode";
close(tcp_socket->getSocket());
return genError("Failed to set the socket to blocking mode");
}
auto connection_timeout_to_server = getProfileAgentSettingWithDefault<uint>(
CONNECT_TIMEOUT_MICROSECOUNDS,
"agent.config.log.TCP.connectTimeout");
dbgTrace(D_SOCKET)
<< "Waiting for the socket connection to be established"
<< " with a timeout of "
<< connection_timeout_to_server
<< " microseconds and each iteration in this timeout is "
<< CHECK_CONNECTION_INTERVAL_MICROSECONDS
<< " microseconds";
int ready_fds = 0; // parameters for select
int err;
socklen_t len;
fd_set writefds;
struct timeval timeout;
timeout.tv_sec = 0;
timeout.tv_usec = 0;
while (
Singleton::Consume<I_TimeGet>::by<SocketIS>()->getWalltime() - time_before_connect
< chrono::microseconds(connection_timeout_to_server))
{
dbgTrace(D_SOCKET) << "Iterating to check the connection status";
Singleton::Consume<I_MainLoop>::by<SocketIS>()->yield(CHRONO_CHECK_CONNECTION_INTERVAL);
FD_ZERO(&writefds);
FD_SET(tcp_socket->getSocket(), &writefds);
ready_fds = select(tcp_socket->getSocket() + 1, NULL, &writefds, NULL, &timeout);
if (ready_fds > 0) {
len = sizeof(err);
if (getsockopt(tcp_socket->getSocket(), SOL_SOCKET, SO_ERROR, &err, &len) >= 0) {
if (err == 0) {
dbgTrace(D_SOCKET) << "Connected to socket";
return move(tcp_socket);
}
}
}
}
if (ready_fds > 0) {
// there is at least one file descriptor ready for IO operation
if (getsockopt(tcp_socket->getSocket(), SOL_SOCKET, SO_ERROR, &err, &len) < 0) {
dbgWarning(D_SOCKET) << "Failed to get socket options";
close(tcp_socket->getSocket());
return genError("Failed to get socket options");
}
if (err != 0) {
dbgWarning(D_SOCKET) << "Failed to connect socket. Error number: " << err;
close(tcp_socket->getSocket());
return genError("Failed to connect socket");
}
}
dbgWarning(D_SOCKET) << "No file descriptor is ready for IO operation";
close(tcp_socket->getSocket());
return genError("Failed to connect socket");
}
static Maybe<unique_ptr<TCPSocket>>
connectSock(bool _is_blocking, bool _is_server, const string &_address)
{
@@ -262,15 +377,7 @@ public:
tcp_socket->server.sin_port = htons(port);
if (!tcp_socket->isServerSock()) {
if (connect(
tcp_socket->getSocket(),
reinterpret_cast<struct sockaddr *>(&tcp_socket->server),
sizeof(struct sockaddr_in)
) == -1
) {
return genError("Failed to connect socket");
}
return move(tcp_socket);
return tcp_socket->connectAsClient(tcp_socket);
}
static const int on = 1;
@@ -638,7 +745,10 @@ SocketIS::Impl::genSocket(
socketTypeName = "UNIXDG";
} else if (type == SocketType::TCP) {
Maybe<unique_ptr<SocketInternal>> tcp_sock = TCPSocket::connectSock(is_blocking, is_server, address);
if (!tcp_sock.ok()) return tcp_sock.passErr();
if (!tcp_sock.ok()) {
dbgWarning(D_SOCKET) << "Failed to initialize TCP socket. Error: " << tcp_sock.getErr();
return tcp_sock.passErr();
}
new_sock = tcp_sock.unpackMove();
socketTypeName = "TCP";
} else if (type == SocketType::UDP) {