github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feb 15th 2023 update

Browse Source
This commit is contained in:
Ned Wright
2023-02-15 19:09:38 +00:00
parent f7934cd09d
commit 6a9b33ff93
159 changed files with 16474 additions and 2096 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/logging/CMakeLists.txt
View File

@@ -1,3 +1,3 @@
add_library(logging logging.cc log_generator.cc debug_stream.cc file_stream.cc fog_stream.cc syslog_stream.cc cef_stream.cc)
add_library(logging logging.cc log_generator.cc debug_stream.cc file_stream.cc fog_stream.cc syslog_stream.cc cef_stream.cc k8s_svc_stream.cc)
add_subdirectory(logging_ut)

63
core/logging/cef_stream.cc
View File

@@ -11,6 +11,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <arpa/inet.h>
#include "logging_comp.h"
#include "log_streams.h"
@@ -19,11 +21,16 @@ using namespace cereal;
USE_DEBUG_FLAG(D_REPORT);
CefStream::CefStream(const string &_ip_address, int _port)
static string lookup_cmd = "nslookup ";
static string line_selection_cmd = "| grep Address | sed -n 2p";
static string parsing_cmd = "| cut -f2 -d' ' | tr -d '\n'";
CefStream::CefStream(const string &_address, int _port, I_Socket::SocketType _protocol)
:
i_socket(Singleton::Consume<I_Socket>::by<LoggingComp>()),
ip_address(_ip_address),
port(_port)
address(_address),
port(_port),
protocol(_protocol)
{
connect();
if (!socket.ok()) {
@@ -65,17 +72,57 @@ CefStream::sendLog(const Report &log)
void
CefStream::connect()
{
auto cef_ip_address = getProfileAgentSettingWithDefault<string>(ip_address, "agent.config.log.cefServer.IP");
auto cef_address = getProfileAgentSettingWithDefault<string>(address, "agent.config.log.cefServer.IP");
auto cef_port = getProfileAgentSettingWithDefault<uint>(port, "agent.config.log.cefServer.port");
if (cef_ip_address.empty()) {
dbgWarning(D_REPORT) << "Cannot connect to CEF server, IP is not configured.";
if (cef_address.empty()) {
dbgWarning(D_REPORT) << "Cannot connect to CEF server, IP/Domain is not configured.";
return;
}
struct in_addr addr;
if (inet_pton(AF_INET, cef_address.data(), &addr) != 1) {
I_ShellCmd *shell_cmd = Singleton::Consume<I_ShellCmd>::by<LoggingComp>();
string host_cmd = lookup_cmd + cef_address + line_selection_cmd + parsing_cmd;
Maybe<string> res = shell_cmd->getExecOutput(host_cmd, 500);
if (!res.ok()) {
dbgWarning(D_REPORT)
<< "Failed to execute domain lookup command. "
<< "CEF Domain: "
<< cef_address
<< "Error: "
<< res.getErr();
return;
}
if (res.unpack().empty()) {
dbgWarning(D_REPORT)
<< "Got en empty ip address from lookup command. "
<< "CEF Domain: "
<< cef_address
<< "Got bad ip address: "
<< res.unpack();
return;
}
dbgDebug(D_REPORT) << "CEF Domain lookup result: " << res.unpack();
if (inet_pton(AF_INET, res.unpack().data(), &addr) != 1) {
dbgWarning(D_REPORT)
<< "Got a faulty ip address from lookup command. "
<< "CEF Domain: "
<< cef_address
<< "Got bad ip address: "
<< res.unpack();
return;
}
cef_address = res.unpack();
}
socket = i_socket->genSocket(
I_Socket::SocketType::UDP,
protocol,
false,
false,
cef_ip_address + ":" + to_string(cef_port)
cef_address + ":" + to_string(cef_port)
);
}

97
core/logging/k8s_svc_stream.cc Normal file
View File

@@ -0,0 +1,97 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "log_streams.h"
#include "logging_comp.h"
using namespace std;
const static string default_host = "open-appsec-tuning-svc";
const static string default_bulk_uri = "/api/v1/agents/events/bulk";
const static string default_log_uri = "/api/v1/agents/events";
USE_DEBUG_FLAG(D_REPORT);
K8sSvcStream::K8sSvcStream()
:
i_msg(Singleton::Consume<I_Messaging>::by<LoggingComp>())
{
}
K8sSvcStream::~K8sSvcStream()
{
}
string
K8sSvcStream::genHeader()
{
return "X-Tenant-Id: " + Singleton::Consume<I_AgentDetails>::by<LoggingComp>()->getTenantId();
}
void
K8sSvcStream::sendLog(const Report &log)
{
auto svc_host = getConfigurationWithDefault(default_host, "Logging", "K8sSvc Log host");
auto K8sSvc_log_uri = getConfigurationWithDefault(default_log_uri, "Logging", "K8sSvc Log URI");
LogRest rest(log);
Flags<MessageConnConfig> conn_flags;
conn_flags.setFlag(MessageConnConfig::EXTERNAL);
bool ok = i_msg->sendNoReplyObject(
rest,
I_Messaging::Method::POST,
svc_host,
80,
conn_flags,
K8sSvc_log_uri,
genHeader(),
nullptr,
MessageTypeTag::LOG
);
if (!ok) {
dbgWarning(D_REPORT) << "failed to send log";
}
}
void
K8sSvcStream::sendLog(const LogBulkRest &logs, bool persistence_only)
{
dbgFlow(D_REPORT) << "send bulk logs";
if (persistence_only) {
dbgWarning(D_REPORT) << "Skipping logs due to persistence only setting";
return;
}
auto svc_host = getConfigurationWithDefault(default_host, "Logging", "K8sSvc Log host");
auto K8sSvc_log_uri = getConfigurationWithDefault(default_bulk_uri, "Logging", "K8sSvc Bulk Log URI");
Flags<MessageConnConfig> conn_flags;
conn_flags.setFlag(MessageConnConfig::EXTERNAL);
bool ok = i_msg->sendNoReplyObject(
logs,
I_Messaging::Method::POST,
svc_host,
80,
conn_flags,
K8sSvc_log_uri,
genHeader(),
nullptr,
MessageTypeTag::LOG
);
if (!ok) {
dbgWarning(D_REPORT) << "failed to send bulk logs";
}
}

24
core/logging/log_streams.h
View File

@@ -80,10 +80,24 @@ private:
I_Messaging *i_msg = nullptr;
};
class K8sSvcStream : public Stream
{
public:
K8sSvcStream();
~K8sSvcStream();
void sendLog(const Report &log) override;
void sendLog(const LogBulkRest &logs, bool persistance_only) override;
private:
std::string genHeader();
I_Messaging *i_msg = nullptr;
};
class SyslogStream : public Stream
{
public:
SyslogStream(const std::string &_ip_address, int _port);
SyslogStream(const std::string &_address, int _port, I_Socket::SocketType protocol);
~SyslogStream();
void sendLog(const Report &log) override;
@@ -93,8 +107,9 @@ private:
I_Socket *i_socket = nullptr;
I_MainLoop *mainloop = nullptr;
std::string ip_address;
std::string address;
int port;
I_Socket::SocketType protocol = I_Socket::SocketType::UDP;
I_MainLoop::RoutineID log_send_routine = -1;
Maybe<I_Socket::socketFd> socket = genError("Not set yet");
};
@@ -102,7 +117,7 @@ private:
class CefStream : public Stream
{
public:
CefStream(const std::string &_ip_address, int _port);
CefStream(const std::string &_address, int _port, I_Socket::SocketType _protocol);
~CefStream();
void sendLog(const Report &log) override;
@@ -111,8 +126,9 @@ private:
void connect();
I_Socket *i_socket = nullptr;
std::string ip_address;
std::string address;
int port;
I_Socket::SocketType protocol = I_Socket::SocketType::UDP;
Maybe<I_Socket::socketFd> socket = genError("Not set yet");
};

17
core/logging/logging.cc
View File

@@ -111,7 +111,11 @@ public:
}
bool
addStream(ReportIS::StreamType type, const string &log_server_url) override
addStream(
ReportIS::StreamType type,
const string &log_server_url,
const string &_protocol
) override
{
string log_type = TagAndEnumManagement::convertToString(type);
if (streams_preperation.find(type) != streams_preperation.end()) {
@@ -124,8 +128,9 @@ public:
string ip = log_server_url.substr(0, log_server_url.find(':'));
string port = log_server_url.substr(log_server_url.find(':') + 1, log_server_url.length());
int port_num = stoi(port);
auto protocol = (_protocol == "TCP") ? I_Socket::SocketType::TCP : I_Socket::SocketType::UDP;
streams_preperation[type] = makeStream(type, ip, port_num);
streams_preperation[type] = makeStream(type, ip, port_num, protocol);
dbgInfo(D_REPORT)
<< "Successfully added log stream. Stream type: "
<< log_type
@@ -265,6 +270,7 @@ private:
case StreamType::JSON_DEBUG: return make_shared<DebugStream>();
case StreamType::JSON_FOG: return make_shared<FogStream>();
case StreamType::JSON_LOG_FILE: return make_shared<LogFileStream>();
case StreamType::JSON_K8S_SVC: return make_shared<K8sSvcStream>();
case StreamType::SYSLOG: return nullptr;
case StreamType::CEF: return nullptr;
case StreamType::NONE: return nullptr;
@@ -275,12 +281,11 @@ private:
}
shared_ptr<Stream>
makeStream(StreamType type, const string &ip, int port)
makeStream(StreamType type, const string &ip, int port, I_Socket::SocketType protocol)
{
switch (type) {
case StreamType::SYSLOG:
return make_shared<SyslogStream>(ip, port);
case StreamType::CEF: return make_shared<CefStream>(ip, port);
case StreamType::SYSLOG: return make_shared<SyslogStream>(ip, port, protocol);
case StreamType::CEF: return make_shared<CefStream>(ip, port, protocol);
default:
dbgWarning(D_REPORT) << "Invalid stream type with url";
return NULL;

176
core/logging/logging_ut/logging_ut.cc
View File

@@ -21,6 +21,7 @@
#include "mock/mock_encryptor.h"
#include "mock/mock_agent_details.h"
#include "metric/all_metric_event.h"
#include "mock/mock_shell_cmd.h"
#include "version.h"
using namespace testing;
@@ -33,6 +34,7 @@ class TestEnd {};
static bool should_fail = false;
static bool should_load_file_stream = false;
static bool should_load_k8s_stream = false;
class fakeConfig : Singleton::Consume<I_Logging>
{
@@ -57,19 +59,45 @@ public:
}
void
load(cereal::JSONInputArchive &)
load(cereal::JSONInputArchive &ar)
{
if (should_fail) throw cereal::Exception("Should fail load");
if (should_load_file_stream) {
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(ReportIS::StreamType::JSON_LOG_FILE);
return;
}
if (should_load_k8s_stream) {
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(ReportIS::StreamType::JSON_K8S_SVC);
return;
}
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(ReportIS::StreamType::JSON_DEBUG);
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(ReportIS::StreamType::JSON_FOG);
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(ReportIS::StreamType::CEF, "1.3.3.0:123");
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(ReportIS::StreamType::SYSLOG, "1.2.3.4:567");
}
bool is_domain;
ar(cereal::make_nvp("IsDomain", is_domain));
if (is_domain) {
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(
ReportIS::StreamType::CEF,
"www.youtube.com:123",
"UDP"
);
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(
ReportIS::StreamType::SYSLOG,
"www.google.com:567",
"UDP"
);
} else {
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(
ReportIS::StreamType::CEF,
"1.3.3.0:123", "UDP"
);
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(
ReportIS::StreamType::SYSLOG,
"1.2.3.4:567",
"UDP"
);
}
}
};
class LogTest : public testing::TestWithParam<bool>
@@ -81,8 +109,10 @@ public:
i_agent_details(Singleton::Consume<I_AgentDetails>::from(agent_details)),
logger(Singleton::Consume<I_Logging>::from(log_comp))
{
is_domain = false;
should_fail = false;
should_load_file_stream = false;
should_load_k8s_stream = false;
env.preload();
log_comp.preload();
env.init();
@@ -143,8 +173,10 @@ public:
~LogTest()
{
is_domain = false;
should_fail = false;
should_load_file_stream = false;
should_load_k8s_stream = false;
env.fini();
log_comp.fini();
Debug::setUnitTestFlag(D_REPORT, Debug::DebugLevel::INFO);
@@ -208,15 +240,23 @@ public:
}
bool
loadFakeConfiguration(const bool &enable_bulk, const string &log_file_name = "", int bulks_size = -1)
loadFakeConfiguration(
bool enable_bulk,
bool domain = false,
const string &log_file_name = "",
int bulks_size = -1)
{
string is_enable_bulks = enable_bulk ? "true" : "false";
string is_domain = domain ? "true" : "false";
fakeConfig::preload();
output_filename = log_file_name == "" ? file.fname : log_file_name;
stringstream str_stream;
str_stream
<< "{\"fake config\": [{}], \"Logging\": {\"Log file name\": [{\"value\": \""
<< "{\"fake config\": [{\"IsDomain\": "
<< is_domain
<< "}],"
<< "\"Logging\": {\"Log file name\": [{\"value\": \""
<< output_filename
<< "\"}],"
<< "\"Enable bulk of logs\": [{\"value\": "
@@ -247,6 +287,8 @@ public:
ConfigComponent config;
vector<string> capture_syslog_cef_data;
I_MainLoop::Routine sysog_routine = nullptr;
StrictMock<MockShellCmd> mock_shell_cmd;
bool is_domain;
private:
string body;
@@ -259,6 +301,16 @@ TEST_F(LogTest, load_policy)
EXPECT_TRUE(loadFakeConfiguration(false));
}
TEST_F(LogTest, loadPolicyDomain)
{
is_domain = true;
string result = "172.28.1.6";
EXPECT_CALL(mock_shell_cmd, getExecOutput(_, _, _)).WillRepeatedly(Return(result));
EXPECT_TRUE(loadFakeConfiguration(false, true));
string failed_str = "Failed to connect to the CEF server";
EXPECT_THAT(getMessages(), Not(HasSubstr(failed_str)));
}
TEST_F(LogTest, loadPolicyFailure)
{
should_fail = true;
@@ -690,9 +742,117 @@ TEST_F(LogTest, FogBulkLogs)
EXPECT_EQ(local_body, str1);
}
TEST_F(LogTest, OfflineK8sSvcTest)
{
i_agent_details->setOrchestrationMode(OrchestrationMode::HYBRID);
should_load_k8s_stream = true;
loadFakeConfiguration(false);
Tags tag1 = Tags::POLICY_INSTALLATION;
Tags tag2 = Tags::ACCESS_CONTROL;
string local_body;
string res("[{\"id\": 1, \"code\": 400, \"message\": \"yes\"}]");
EXPECT_CALL(
mock_fog_msg,
sendMessage(_, _, _, "open-appsec-tuning-svc", _, _, "/api/v1/agents/events", _, _, MessageTypeTag::LOG)
).WillRepeatedly(DoAll(SaveArg<1>(&local_body), Return(res)));
string str1(
"{\n"
" \"log\": {\n"
" \"eventTime\": \"0:0:0\",\n"
" \"eventName\": \"Install policy\",\n"
" \"eventSeverity\": \"Info\",\n"
" \"eventPriority\": \"Low\",\n"
" \"eventType\": \"Event Driven\",\n"
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"\",\n"
" \"eventFrequency\": 0,\n"
" \"eventTags\": [\n"
" \"Access Control\",\n"
" \"Policy Installation\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" },\n"
" \"eventData\": {\n"
" \"logIndex\": 1\n"
" }\n"
" }\n"
"}"
);
LogGen("Install policy", Audience::INTERNAL, Severity::INFO, Priority::LOW, tag1, tag2);
EXPECT_EQ(local_body, str1);
}
TEST_F(LogTest, OfflineK8sSvcBulkLogs)
{
i_agent_details->setOrchestrationMode(OrchestrationMode::HYBRID);
should_load_k8s_stream = true;
loadFakeConfiguration(true);
string local_body;
string res("[{\"id\": 1, \"code\": 400, \"message\": \"yes\"}]");
EXPECT_CALL(
mock_fog_msg,
sendMessage(_, _, _, "open-appsec-tuning-svc", _, _, "/api/v1/agents/events/bulk", _, _, MessageTypeTag::LOG)
).WillRepeatedly(DoAll(SaveArg<1>(&local_body), Return(res)));
Tags tag1 = Tags::POLICY_INSTALLATION;
Tags tag2 = Tags::ACCESS_CONTROL;
string str1(
"{\n"
" \"logs\": [\n"
" {\n"
" \"id\": 1,\n"
" \"log\": {\n"
" \"eventTime\": \"0:0:0\",\n"
" \"eventName\": \"Install policy\",\n"
" \"eventSeverity\": \"Info\",\n"
" \"eventPriority\": \"Low\",\n"
" \"eventType\": \"Event Driven\",\n"
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"\",\n"
" \"eventFrequency\": 0,\n"
" \"eventTags\": [\n"
" \"Access Control\",\n"
" \"Policy Installation\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" },\n"
" \"eventData\": {\n"
" \"logIndex\": 1\n"
" }\n"
" }\n"
" }\n"
" ]\n"
"}"
);
{
LogGen("Install policy", Audience::INTERNAL, Severity::INFO, Priority::LOW, tag1, tag2);
}
bulk_routine();
EXPECT_EQ(local_body, str1);
}
TEST_P(LogTest, metrics_check)
{
loadFakeConfiguration(true, "", 3);
loadFakeConfiguration(true, false, "", 3);
Tags tag1 = Tags::POLICY_INSTALLATION;
Tags tag2 = Tags::ACCESS_CONTROL;
@@ -837,7 +997,7 @@ TEST_F(LogTest, ShouldRetryAfterFailedWriteToFile)
EXPECT_TRUE(logger->delStream(ReportIS::StreamType::JSON_LOG_FILE));
static const string invalid_file_path = "/proc/gibberish";
loadFakeConfiguration(false, invalid_file_path, -1);
loadFakeConfiguration(false, false, invalid_file_path, -1);
LogGen(
"Install policy",

80
core/logging/syslog_stream.cc
View File

@@ -11,6 +11,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <arpa/inet.h>
#include "log_streams.h"
#include "logging_comp.h"
@@ -18,12 +20,17 @@ using namespace std;
USE_DEBUG_FLAG(D_REPORT);
SyslogStream::SyslogStream(const string &_ip_address, int _port)
static string lookup_cmd = "nslookup ";
static string line_selection_cmd = "| grep Address | sed -n 2p";
static string parsing_cmd = "| cut -f2 -d' ' | tr -d '\n'";
SyslogStream::SyslogStream(const string &_address, int _port, I_Socket::SocketType _protocol)
:
i_socket(Singleton::Consume<I_Socket>::by<LoggingComp>()),
mainloop(Singleton::Consume<I_MainLoop>::by<LoggingComp>()),
ip_address(_ip_address),
port(_port)
address(_address),
port(_port),
protocol(_protocol)
{
connect();
if (!socket.ok()) {
@@ -43,21 +50,21 @@ SyslogStream::~SyslogStream()
void
SyslogStream::sendLog(const Report &log)
{
if (!socket.ok()) {
connect();
if (!socket.ok()) {
dbgWarning(D_REPORT) << "Failed to connect to the syslog server, Log will not be sent.";
return;
}
dbgTrace(D_REPORT) << "Successfully connect to the syslog server";
}
string syslog_report = log.getSyslog();
vector<char> data(syslog_report.begin(), syslog_report.end());
mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::Offline,
[this, data] ()
{
if (!socket.ok()) {
connect();
if (!socket.ok()) {
dbgWarning(D_REPORT) << "Failed to connect to the syslog server, Log will not be sent.";
return;
}
dbgTrace(D_REPORT) << "Successfully connect to the syslog server";
}
int tries = 1;
for (; tries <=3; tries++) {
if (i_socket->writeData(socket.unpack(), data)) {
@@ -75,18 +82,57 @@ SyslogStream::sendLog(const Report &log)
void
SyslogStream::connect()
{
auto syslog_ip_address = getProfileAgentSettingWithDefault<string>(ip_address, "agent.config.log.syslogServer.IP");
auto syslog_address = getProfileAgentSettingWithDefault<string>(address, "agent.config.log.syslogServer.IP");
auto syslog_port = getProfileAgentSettingWithDefault<uint>(port, "agent.config.log.syslogServer.port");
if (syslog_ip_address.empty()) {
dbgWarning(D_REPORT) << "Cannot connect to Syslog server, IP is not configured.";
if (syslog_address.empty()) {
dbgWarning(D_REPORT) << "Cannot connect to Syslog server, Address IP/Domain not configured.";
return;
}
struct in_addr addr;
if (inet_pton(AF_INET, syslog_address.data(), &addr) != 1) {
I_ShellCmd *shell_cmd = Singleton::Consume<I_ShellCmd>::by<LoggingComp>();
string host_cmd = lookup_cmd + syslog_address + line_selection_cmd + parsing_cmd;
Maybe<string> res = shell_cmd->getExecOutput(host_cmd, 500);
if (!res.ok()) {
dbgWarning(D_REPORT)
<< "Failed to execute domain lookup command. "
<< "SYSLOG Domain: "
<< syslog_address
<< "Error: "
<< res.getErr();
return;
}
if (res.unpack().empty()) {
dbgWarning(D_REPORT)
<< "Got en empty ip address from lookup command. "
<< "SYSLOG Domain: "
<< syslog_address
<< "Got bad ip address: "
<< res.unpack();
return;
}
dbgDebug(D_REPORT) << "SYSLOG Domain lookup result: " << res.unpack();
if (inet_pton(AF_INET, res.unpack().data(), &addr) != 1) {
dbgWarning(D_REPORT)
<< "Got a faulty ip address from lookup command. "
<< "SYSLOG Domain: "
<< syslog_address
<< "Got bad ip address: "
<< res.unpack();
return;
}
syslog_address = res.unpack();
}
socket = i_socket->genSocket(
I_Socket::SocketType::UDP,
protocol,
false,
false,
syslog_ip_address + ":" + to_string(syslog_port)
syslog_address + ":" + to_string(syslog_port)
);
}