github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feb 15th 2023 update

Browse Source
This commit is contained in:
Ned Wright
2023-02-15 19:09:38 +00:00
parent f7934cd09d
commit 6a9b33ff93
159 changed files with 16474 additions and 2096 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
core/config/config.cc
View File

@@ -14,6 +14,7 @@
#include "config.h"
#include "config_component.h"
#include <dirent.h>
#include <algorithm>
#include <fstream>
#include <boost/regex.hpp>
@@ -66,11 +67,24 @@ public:
void
doCall() override
{
auto i_config = Singleton::Consume<I_Config>::from<ConfigComponent>();
static const map<I_Config::AsyncLoadConfigStatus, string> status_map {
{I_Config::AsyncLoadConfigStatus::Success, "Success"},
{I_Config::AsyncLoadConfigStatus::InProgress, "In Progress"},
{I_Config::AsyncLoadConfigStatus::Error, "Error"}
};
auto i_config = Singleton::Consume<I_Config>::from<ConfigComponent>();
I_Config::AsyncLoadConfigStatus load_config_staus = i_config->reloadConfiguration(policy_version, true, id);
finished = load_config_staus == I_Config::AsyncLoadConfigStatus::InProgress;
finished = load_config_staus != I_Config::AsyncLoadConfigStatus::InProgress;
error = load_config_staus == I_Config::AsyncLoadConfigStatus::Error;
if (error) {
dbgWarning(D_CONFIG) << "Configuration reload status: " << status_map.at(load_config_staus);
} else {
dbgDebug(D_CONFIG) << "Configuration reload status: " << status_map.at(load_config_staus);
}
if (!finished) {
error_message = "Reload already in progress - can't start another one";
}
@@ -669,8 +683,8 @@ ConfigComponent::Impl::clearOldTenants()
bool
ConfigComponent::Impl::areTenantAndProfileActive(const TenantProfilePair &tenant_profile) const
{
return (tenant_profile.getTenantId() == default_tenant_id && tenant_profile.getPfofileId() == default_profile_id)
|| tenant_mananger->areTenantAndProfileActive(tenant_profile.getTenantId(), tenant_profile.getPfofileId());
return (tenant_profile.getTenantId() == default_tenant_id && tenant_profile.getProfileId() == default_profile_id)
|| tenant_mananger->areTenantAndProfileActive(tenant_profile.getTenantId(), tenant_profile.getProfileId());
}
void
@@ -723,12 +737,21 @@ ConfigComponent::Impl::loadConfiguration(vector<shared_ptr<JSONInputArchive>> &f
string curr_profile = default_profile_id;
try {
(*archive)(cereal::make_nvp("tenantID", curr_tenant));
dbgTrace(D_CONFIG) << "Found a tenant ID in the file: " << curr_tenant;
} catch (cereal::Exception &e) {}
try {
(*archive)(cereal::make_nvp("profileID", curr_profile));
} catch (cereal::Exception &e) {
}
dbgTrace(D_CONFIG) << "Found a profile ID in the file " << curr_profile;
} catch (cereal::Exception &e) {}
dbgTrace(D_CONFIG)
<< "Loading configuration for tenant: "
<< curr_tenant
<< " and profile: "
<< curr_profile
<< ", for the archive: "
<< (*archive).getNodeName();
dbgTrace(D_CONFIG) <<
"Loading configuration for tenant: " << curr_tenant << " and profile: " << curr_profile;
TenantProfilePair tenant_profile(curr_tenant, curr_profile);
for (auto &config : expected_configs) {
auto loaded = config->loadConfiguration(*archive);
@@ -800,6 +823,7 @@ ConfigComponent::Impl::commitFailure(const string &error)
bool
ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_async)
{
dbgFlow(D_CONFIG) << "Reloading configuration";
auto env = Singleton::Consume<I_Environment>::by<ConfigComponent>();
env->registerValue<string>("New Policy Version", version);
auto cleanup = make_scope_exit([env] () { env->unregisterKey<string>("New Policy Version"); } );
@@ -812,6 +836,8 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as
const auto &active_tenants = tenant_mananger ? tenant_mananger->fetchAllActiveTenants() : vector<string>();
dbgTrace(D_CONFIG) << "Number of active tenants found while reloading configuration: " << active_tenants.size();
for (const auto &config_file : expected_configuration_files) {
for (const auto &type : config_file.second) {
if (type == ConfigFileType::RawData) continue;
@@ -835,7 +861,9 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as
const vector<string> &profile_ids =
tenant_mananger ? tenant_mananger->fetchProfileIds(tenant) : vector<string>();
for (auto &profile_id : profile_ids) {
string settings_path = config_directory_path + "tenant_" + tenant + "_" + profile_id + "_settings.json";
string settings_path =
config_directory_path + "tenant_" + tenant + "_profile_"+ profile_id + "_settings.json";
dbgTrace(D_CONFIG) << "Inserting a settings path: " << settings_path;
files.emplace(settings_path, make_shared<ifstream>(settings_path));
}
}
@@ -843,10 +871,10 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as
vector<shared_ptr<JSONInputArchive>> archives;
for (const auto &file : files) {
if (file.second->is_open()) {
dbgDebug(D_CONFIG) << "Succesfully opened configuration file. File: " << file.first;
dbgTrace(D_CONFIG) << "Succesfully opened configuration file. File: " << file.first;
archives.push_back(make_shared<JSONInputArchive>(*file.second));
} else {
dbgDebug(D_CONFIG) << "Could not open configuration file. Path: " << file.first;
dbgTrace(D_CONFIG) << "Could not open configuration file. Path: " << file.first;
}
}

20
core/connkey/connkey.cc
View File

@@ -161,26 +161,6 @@ IPAddr::createIPAddr(const string &ip_text)
bool IPAddr::isValidIPAddr(const string &ip_text) { return createIPAddr(ip_text).ok(); }
IPAddressConfig::IPAddressConfig(const string &ip_string)
{
auto maybe_address = IPAddr::createIPAddr(ip_string);
if (maybe_address.ok()) address = maybe_address.unpack();
}
void
IPAddressConfig::load(cereal::JSONInputArchive &ar)
{
string ip_string;
ar(cereal::make_nvp("IPAddress", ip_string));
auto ip_address = IPAddr::createIPAddr(ip_string);
if (!ip_address.ok()) {
throw Config::ConfigException(
"Failed to create an IP address from " + ip_string + ": " + ip_address.getErr()
);
}
address = ip_address.unpack();
}
const string ConnKey::network_key = "NetworkKey";
template<typename Num>

4
core/include/general/logging_comp.h
View File

@@ -24,6 +24,7 @@
#include "i_socket_is.h"
#include "component.h"
#include "i_agent_details.h"
#include "i_shell_cmd.h"
class LoggingComp
:
@@ -35,7 +36,8 @@ class LoggingComp
Singleton::Consume<I_TimeGet>,
Singleton::Consume<I_Logging>,
Singleton::Consume<I_Socket>,
Singleton::Consume<I_AgentDetails>
Singleton::Consume<I_AgentDetails>,
Singleton::Consume<I_ShellCmd>
{
public:
LoggingComp();

2
core/include/general/tenant_profile_pair.h
View File

@@ -79,7 +79,7 @@ public:
}
std::string
getPfofileId() const
getProfileId() const
{
return profile_id;
}

6
core/include/services_sdk/interfaces/i_logging.h
View File

@@ -25,7 +25,11 @@ public:
using GeneralModifier = std::function<void(LogBulkRest &)>;
virtual bool addStream(ReportIS::StreamType type) = 0;
virtual bool addStream(ReportIS::StreamType type, const std::string &log_server_url) = 0;
virtual bool addStream(
ReportIS::StreamType type,
const std::string &log_server_url,
const std::string &protocol
) = 0;
virtual bool delStream(ReportIS::StreamType type) = 0;
virtual void sendLog(const Report &msg) = 0;

2
core/include/services_sdk/interfaces/i_tenant_manager.h
View File

@@ -41,6 +41,8 @@ public:
virtual std::chrono::microseconds getTimeoutVal() const = 0;
virtual std::string getProfileId(const std::string &tenant_id, const std::string &region) const = 0;
private:
friend class LoadNewTenants;
friend class LoadNewTenantsAndProfiles;

2
core/include/services_sdk/interfaces/mock/mock_logging.h
View File

@@ -10,7 +10,7 @@ class MockLogging : public Singleton::Provide<I_Logging>::From<MockProvider<I_Lo
public:
MOCK_METHOD1(sendLog, void (const Report &));
MOCK_METHOD1(addStream, bool (ReportIS::StreamType));
MOCK_METHOD2(addStream, bool (ReportIS::StreamType, const std::string &));
MOCK_METHOD3(addStream, bool (ReportIS::StreamType, const std::string &, const std::string &));
MOCK_METHOD1(delStream, bool (ReportIS::StreamType));
MOCK_METHOD0(getCurrentLogId, uint64_t ());
MOCK_METHOD1(addGeneralModifier, void (const GeneralModifier &));

1
core/include/services_sdk/interfaces/mock/mock_tenant_manager.h
View File

@@ -25,6 +25,7 @@ public:
MOCK_CONST_METHOD2(areTenantAndProfileActive, bool(const std::string &, const std::string &));
MOCK_METHOD2(addActiveTenantAndProfile, void(const std::string &, const std::string &));
MOCK_METHOD2(deactivateTenant, void(const std::string &, const std::string &));
MOCK_CONST_METHOD2(getProfileId, std::string(const std::string &, const std::string &));
MOCK_CONST_METHOD0(getTimeoutVal, std::chrono::microseconds());

2
core/include/services_sdk/resources/debug_flags.h
View File

@@ -74,6 +74,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_WAAP_SCANNER, D_WAAP)
DEFINE_FLAG(D_WAAP_DEEP_PARSER, D_WAAP)
DEFINE_FLAG(D_WAAP_BASE64, D_WAAP)
DEFINE_FLAG(D_WAAP_JSON, D_WAAP)
DEFINE_FLAG(D_WAAP_BOT_PROTECTION, D_WAAP)
DEFINE_FLAG(D_WAAP_PARSER, D_WAAP)
DEFINE_FLAG(D_WAAP_PARSER_XML, D_WAAP_PARSER)
@@ -122,6 +123,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_HEALTH_CHECK, D_ORCHESTRATOR)
DEFINE_FLAG(D_AGENT_DETAILS, D_ORCHESTRATOR)
DEFINE_FLAG(D_K8S_POLICY, D_ORCHESTRATOR)
DEFINE_FLAG(D_NGINX_POLICY, D_ORCHESTRATOR)
DEFINE_FLAG(D_GRADUAL_DEPLOYMENT, D_COMPONENT)
DEFINE_FLAG(D_C8_CONTROLLER, D_COMPONENT)

2
core/include/services_sdk/resources/report/report_enums.h
View File

@@ -23,6 +23,7 @@ enum class StreamType {
JSON_DEBUG,
JSON_FOG,
JSON_LOG_FILE,
JSON_K8S_SVC,
SYSLOG,
CEF,
@@ -55,6 +56,7 @@ enum class Tags {
HTTP_GEO_FILTER,
FILE_UPLOAD,
IDENTITY_AWARENESS,
RATE_LIMIT,
COUNT
};

15
core/include/services_sdk/utilities/connkey.h
View File

@@ -199,21 +199,6 @@ private:
RangeType end;
};
class IPAddressConfig
{
public:
IPAddressConfig() = default;
IPAddressConfig(const std::string &ip_string);
void load(cereal::JSONInputArchive &ar);
const IPAddr & getAddress() const { return address; }
operator IPAddr() const { return address; }
private:
IPAddr address;
};
// Specialization of std::hash<> for IPAddr
namespace std
{

6
core/instance_awareness/instance_awareness.cc
View File

@@ -33,7 +33,7 @@ public:
Maybe<string> instance_id = checkIfValueIsConfigured("id");
if (instance_id.ok()) return instance_id;
return genError("Instance Awareness isn't active");
return genError("Instance Awareness isn't active, Error: " + instance_id.getErr());
}
Maybe<string>
@@ -42,14 +42,14 @@ public:
Maybe<string> family_id = checkIfValueIsConfigured("family");
if (family_id.ok()) return family_id;
return genError("Family ID isn't active");
return genError("Family ID isn't active, Error: " + family_id.getErr());
}
Maybe<string>
getUniqueID() override
{
Maybe<string> instance_id(getInstanceID());
if (!instance_id.ok()) return genError("Instance Awareness isn't active");
if (!instance_id.ok()) return genError("Can't get instance ID, Error: " + instance_id.getErr());
Maybe<string> family_id(getFamilyID());
if (!family_id.ok()) return *instance_id;

34
core/instance_awareness/instance_awareness_ut/instance_awareness_ut.cc
View File

@@ -43,9 +43,12 @@ TEST_F(InstanceAwarenessTest, emptyInit)
init(args);
EXPECT_THAT(getInstanceID(), IsError("Instance Awareness isn't active"));
EXPECT_THAT(getFamilyID(), IsError("Family ID isn't active"));
EXPECT_THAT(getUniqueID(), IsError("Instance Awareness isn't active"));
EXPECT_THAT(getInstanceID(), IsError("Instance Awareness isn't active, Error: Flag not found"));
EXPECT_THAT(getFamilyID(), IsError("Family ID isn't active, Error: Flag not found"));
EXPECT_THAT(
getUniqueID(),
IsError("Can't get instance ID, Error: Instance Awareness isn't active, Error: Flag not found")
);
}
TEST_F(InstanceAwarenessTest, badFamilyID)
@@ -55,7 +58,7 @@ TEST_F(InstanceAwarenessTest, badFamilyID)
init(args);
EXPECT_THAT(getInstanceID(), IsValue("9"));
EXPECT_THAT(getFamilyID(), IsError("Family ID isn't active"));
EXPECT_THAT(getFamilyID(), IsError("Family ID isn't active, Error: Illegal flag: family"));
EXPECT_THAT(getUniqueID(), IsValue("9"));
}
@@ -65,9 +68,12 @@ TEST_F(InstanceAwarenessTest, badInstanceID)
init(args);
EXPECT_THAT(getInstanceID(), IsError("Instance Awareness isn't active"));
EXPECT_THAT(getInstanceID(), IsError("Instance Awareness isn't active, Error: Illegal flag: id"));
EXPECT_THAT(getFamilyID(), IsValue("073b8744b4c5"));
EXPECT_THAT(getUniqueID(), IsError("Instance Awareness isn't active"));
EXPECT_THAT(
getUniqueID(),
IsError("Can't get instance ID, Error: Instance Awareness isn't active, Error: Illegal flag: id")
);
}
TEST_F(InstanceAwarenessTest, emptyInstanceID)
@@ -76,9 +82,12 @@ TEST_F(InstanceAwarenessTest, emptyInstanceID)
init(args);
EXPECT_THAT(getInstanceID(), IsError("Instance Awareness isn't active"));
EXPECT_THAT(getInstanceID(), IsError("Instance Awareness isn't active, Error: Flag not found"));
EXPECT_THAT(getFamilyID(), IsValue("073b8744b4c5"));
EXPECT_THAT(getUniqueID(), IsError("Instance Awareness isn't active"));
EXPECT_THAT(
getUniqueID(),
IsError("Can't get instance ID, Error: Instance Awareness isn't active, Error: Flag not found")
);
}
TEST_F(InstanceAwarenessTest, noInstanceID)
@@ -87,9 +96,12 @@ TEST_F(InstanceAwarenessTest, noInstanceID)
init(args);
EXPECT_THAT(getInstanceID(), IsError("Instance Awareness isn't active"));
EXPECT_THAT(getInstanceID(), IsError("Instance Awareness isn't active, Error: Flag not found"));
EXPECT_THAT(getFamilyID(), IsValue("073b8744b4c5"));
EXPECT_THAT(getUniqueID(), IsError("Instance Awareness isn't active"));
EXPECT_THAT(
getUniqueID(),
IsError("Can't get instance ID, Error: Instance Awareness isn't active, Error: Flag not found")
);
}
TEST_F(InstanceAwarenessTest, init)
@@ -111,7 +123,7 @@ TEST_F(InstanceAwarenessTest, initIDOnly)
EXPECT_THAT(getUniqueID(), IsValue("9"));
EXPECT_THAT(getInstanceID(), IsValue("9"));
EXPECT_THAT(getFamilyID(), IsError("Family ID isn't active"));
EXPECT_THAT(getFamilyID(), IsError("Family ID isn't active, Error: Flag not found"));
}
TEST_F(InstanceAwarenessTest, defaultValues)

2
core/logging/CMakeLists.txt
View File

@@ -1,3 +1,3 @@
add_library(logging logging.cc log_generator.cc debug_stream.cc file_stream.cc fog_stream.cc syslog_stream.cc cef_stream.cc)
add_library(logging logging.cc log_generator.cc debug_stream.cc file_stream.cc fog_stream.cc syslog_stream.cc cef_stream.cc k8s_svc_stream.cc)
add_subdirectory(logging_ut)

63
core/logging/cef_stream.cc
View File

@@ -11,6 +11,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <arpa/inet.h>
#include "logging_comp.h"
#include "log_streams.h"
@@ -19,11 +21,16 @@ using namespace cereal;
USE_DEBUG_FLAG(D_REPORT);
CefStream::CefStream(const string &_ip_address, int _port)
static string lookup_cmd = "nslookup ";
static string line_selection_cmd = "| grep Address | sed -n 2p";
static string parsing_cmd = "| cut -f2 -d' ' | tr -d '\n'";
CefStream::CefStream(const string &_address, int _port, I_Socket::SocketType _protocol)
:
i_socket(Singleton::Consume<I_Socket>::by<LoggingComp>()),
ip_address(_ip_address),
port(_port)
address(_address),
port(_port),
protocol(_protocol)
{
connect();
if (!socket.ok()) {
@@ -65,17 +72,57 @@ CefStream::sendLog(const Report &log)
void
CefStream::connect()
{
auto cef_ip_address = getProfileAgentSettingWithDefault<string>(ip_address, "agent.config.log.cefServer.IP");
auto cef_address = getProfileAgentSettingWithDefault<string>(address, "agent.config.log.cefServer.IP");
auto cef_port = getProfileAgentSettingWithDefault<uint>(port, "agent.config.log.cefServer.port");
if (cef_ip_address.empty()) {
dbgWarning(D_REPORT) << "Cannot connect to CEF server, IP is not configured.";
if (cef_address.empty()) {
dbgWarning(D_REPORT) << "Cannot connect to CEF server, IP/Domain is not configured.";
return;
}
struct in_addr addr;
if (inet_pton(AF_INET, cef_address.data(), &addr) != 1) {
I_ShellCmd *shell_cmd = Singleton::Consume<I_ShellCmd>::by<LoggingComp>();
string host_cmd = lookup_cmd + cef_address + line_selection_cmd + parsing_cmd;
Maybe<string> res = shell_cmd->getExecOutput(host_cmd, 500);
if (!res.ok()) {
dbgWarning(D_REPORT)
<< "Failed to execute domain lookup command. "
<< "CEF Domain: "
<< cef_address
<< "Error: "
<< res.getErr();
return;
}
if (res.unpack().empty()) {
dbgWarning(D_REPORT)
<< "Got en empty ip address from lookup command. "
<< "CEF Domain: "
<< cef_address
<< "Got bad ip address: "
<< res.unpack();
return;
}
dbgDebug(D_REPORT) << "CEF Domain lookup result: " << res.unpack();
if (inet_pton(AF_INET, res.unpack().data(), &addr) != 1) {
dbgWarning(D_REPORT)
<< "Got a faulty ip address from lookup command. "
<< "CEF Domain: "
<< cef_address
<< "Got bad ip address: "
<< res.unpack();
return;
}
cef_address = res.unpack();
}
socket = i_socket->genSocket(
I_Socket::SocketType::UDP,
protocol,
false,
false,
cef_ip_address + ":" + to_string(cef_port)
cef_address + ":" + to_string(cef_port)
);
}

97
core/logging/k8s_svc_stream.cc Normal file
View File

@@ -0,0 +1,97 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "log_streams.h"
#include "logging_comp.h"
using namespace std;
const static string default_host = "open-appsec-tuning-svc";
const static string default_bulk_uri = "/api/v1/agents/events/bulk";
const static string default_log_uri = "/api/v1/agents/events";
USE_DEBUG_FLAG(D_REPORT);
K8sSvcStream::K8sSvcStream()
:
i_msg(Singleton::Consume<I_Messaging>::by<LoggingComp>())
{
}
K8sSvcStream::~K8sSvcStream()
{
}
string
K8sSvcStream::genHeader()
{
return "X-Tenant-Id: " + Singleton::Consume<I_AgentDetails>::by<LoggingComp>()->getTenantId();
}
void
K8sSvcStream::sendLog(const Report &log)
{
auto svc_host = getConfigurationWithDefault(default_host, "Logging", "K8sSvc Log host");
auto K8sSvc_log_uri = getConfigurationWithDefault(default_log_uri, "Logging", "K8sSvc Log URI");
LogRest rest(log);
Flags<MessageConnConfig> conn_flags;
conn_flags.setFlag(MessageConnConfig::EXTERNAL);
bool ok = i_msg->sendNoReplyObject(
rest,
I_Messaging::Method::POST,
svc_host,
80,
conn_flags,
K8sSvc_log_uri,
genHeader(),
nullptr,
MessageTypeTag::LOG
);
if (!ok) {
dbgWarning(D_REPORT) << "failed to send log";
}
}
void
K8sSvcStream::sendLog(const LogBulkRest &logs, bool persistence_only)
{
dbgFlow(D_REPORT) << "send bulk logs";
if (persistence_only) {
dbgWarning(D_REPORT) << "Skipping logs due to persistence only setting";
return;
}
auto svc_host = getConfigurationWithDefault(default_host, "Logging", "K8sSvc Log host");
auto K8sSvc_log_uri = getConfigurationWithDefault(default_bulk_uri, "Logging", "K8sSvc Bulk Log URI");
Flags<MessageConnConfig> conn_flags;
conn_flags.setFlag(MessageConnConfig::EXTERNAL);
bool ok = i_msg->sendNoReplyObject(
logs,
I_Messaging::Method::POST,
svc_host,
80,
conn_flags,
K8sSvc_log_uri,
genHeader(),
nullptr,
MessageTypeTag::LOG
);
if (!ok) {
dbgWarning(D_REPORT) << "failed to send bulk logs";
}
}

24
core/logging/log_streams.h
View File

@@ -80,10 +80,24 @@ private:
I_Messaging *i_msg = nullptr;
};
class K8sSvcStream : public Stream
{
public:
K8sSvcStream();
~K8sSvcStream();
void sendLog(const Report &log) override;
void sendLog(const LogBulkRest &logs, bool persistance_only) override;
private:
std::string genHeader();
I_Messaging *i_msg = nullptr;
};
class SyslogStream : public Stream
{
public:
SyslogStream(const std::string &_ip_address, int _port);
SyslogStream(const std::string &_address, int _port, I_Socket::SocketType protocol);
~SyslogStream();
void sendLog(const Report &log) override;
@@ -93,8 +107,9 @@ private:
I_Socket *i_socket = nullptr;
I_MainLoop *mainloop = nullptr;
std::string ip_address;
std::string address;
int port;
I_Socket::SocketType protocol = I_Socket::SocketType::UDP;
I_MainLoop::RoutineID log_send_routine = -1;
Maybe<I_Socket::socketFd> socket = genError("Not set yet");
};
@@ -102,7 +117,7 @@ private:
class CefStream : public Stream
{
public:
CefStream(const std::string &_ip_address, int _port);
CefStream(const std::string &_address, int _port, I_Socket::SocketType _protocol);
~CefStream();
void sendLog(const Report &log) override;
@@ -111,8 +126,9 @@ private:
void connect();
I_Socket *i_socket = nullptr;
std::string ip_address;
std::string address;
int port;
I_Socket::SocketType protocol = I_Socket::SocketType::UDP;
Maybe<I_Socket::socketFd> socket = genError("Not set yet");
};

17
core/logging/logging.cc
View File

@@ -111,7 +111,11 @@ public:
}
bool
addStream(ReportIS::StreamType type, const string &log_server_url) override
addStream(
ReportIS::StreamType type,
const string &log_server_url,
const string &_protocol
) override
{
string log_type = TagAndEnumManagement::convertToString(type);
if (streams_preperation.find(type) != streams_preperation.end()) {
@@ -124,8 +128,9 @@ public:
string ip = log_server_url.substr(0, log_server_url.find(':'));
string port = log_server_url.substr(log_server_url.find(':') + 1, log_server_url.length());
int port_num = stoi(port);
auto protocol = (_protocol == "TCP") ? I_Socket::SocketType::TCP : I_Socket::SocketType::UDP;
streams_preperation[type] = makeStream(type, ip, port_num);
streams_preperation[type] = makeStream(type, ip, port_num, protocol);
dbgInfo(D_REPORT)
<< "Successfully added log stream. Stream type: "
<< log_type
@@ -265,6 +270,7 @@ private:
case StreamType::JSON_DEBUG: return make_shared<DebugStream>();
case StreamType::JSON_FOG: return make_shared<FogStream>();
case StreamType::JSON_LOG_FILE: return make_shared<LogFileStream>();
case StreamType::JSON_K8S_SVC: return make_shared<K8sSvcStream>();
case StreamType::SYSLOG: return nullptr;
case StreamType::CEF: return nullptr;
case StreamType::NONE: return nullptr;
@@ -275,12 +281,11 @@ private:
}
shared_ptr<Stream>
makeStream(StreamType type, const string &ip, int port)
makeStream(StreamType type, const string &ip, int port, I_Socket::SocketType protocol)
{
switch (type) {
case StreamType::SYSLOG:
return make_shared<SyslogStream>(ip, port);
case StreamType::CEF: return make_shared<CefStream>(ip, port);
case StreamType::SYSLOG: return make_shared<SyslogStream>(ip, port, protocol);
case StreamType::CEF: return make_shared<CefStream>(ip, port, protocol);
default:
dbgWarning(D_REPORT) << "Invalid stream type with url";
return NULL;

176
core/logging/logging_ut/logging_ut.cc
View File

@@ -21,6 +21,7 @@
#include "mock/mock_encryptor.h"
#include "mock/mock_agent_details.h"
#include "metric/all_metric_event.h"
#include "mock/mock_shell_cmd.h"
#include "version.h"
using namespace testing;
@@ -33,6 +34,7 @@ class TestEnd {};
static bool should_fail = false;
static bool should_load_file_stream = false;
static bool should_load_k8s_stream = false;
class fakeConfig : Singleton::Consume<I_Logging>
{
@@ -57,19 +59,45 @@ public:
}
void
load(cereal::JSONInputArchive &)
load(cereal::JSONInputArchive &ar)
{
if (should_fail) throw cereal::Exception("Should fail load");
if (should_load_file_stream) {
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(ReportIS::StreamType::JSON_LOG_FILE);
return;
}
if (should_load_k8s_stream) {
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(ReportIS::StreamType::JSON_K8S_SVC);
return;
}
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(ReportIS::StreamType::JSON_DEBUG);
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(ReportIS::StreamType::JSON_FOG);
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(ReportIS::StreamType::CEF, "1.3.3.0:123");
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(ReportIS::StreamType::SYSLOG, "1.2.3.4:567");
}
bool is_domain;
ar(cereal::make_nvp("IsDomain", is_domain));
if (is_domain) {
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(
ReportIS::StreamType::CEF,
"www.youtube.com:123",
"UDP"
);
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(
ReportIS::StreamType::SYSLOG,
"www.google.com:567",
"UDP"
);
} else {
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(
ReportIS::StreamType::CEF,
"1.3.3.0:123", "UDP"
);
Singleton::Consume<I_Logging>::by<fakeConfig>()->addStream(
ReportIS::StreamType::SYSLOG,
"1.2.3.4:567",
"UDP"
);
}
}
};
class LogTest : public testing::TestWithParam<bool>
@@ -81,8 +109,10 @@ public:
i_agent_details(Singleton::Consume<I_AgentDetails>::from(agent_details)),
logger(Singleton::Consume<I_Logging>::from(log_comp))
{
is_domain = false;
should_fail = false;
should_load_file_stream = false;
should_load_k8s_stream = false;
env.preload();
log_comp.preload();
env.init();
@@ -143,8 +173,10 @@ public:
~LogTest()
{
is_domain = false;
should_fail = false;
should_load_file_stream = false;
should_load_k8s_stream = false;
env.fini();
log_comp.fini();
Debug::setUnitTestFlag(D_REPORT, Debug::DebugLevel::INFO);
@@ -208,15 +240,23 @@ public:
}
bool
loadFakeConfiguration(const bool &enable_bulk, const string &log_file_name = "", int bulks_size = -1)
loadFakeConfiguration(
bool enable_bulk,
bool domain = false,
const string &log_file_name = "",
int bulks_size = -1)
{
string is_enable_bulks = enable_bulk ? "true" : "false";
string is_domain = domain ? "true" : "false";
fakeConfig::preload();
output_filename = log_file_name == "" ? file.fname : log_file_name;
stringstream str_stream;
str_stream
<< "{\"fake config\": [{}], \"Logging\": {\"Log file name\": [{\"value\": \""
<< "{\"fake config\": [{\"IsDomain\": "
<< is_domain
<< "}],"
<< "\"Logging\": {\"Log file name\": [{\"value\": \""
<< output_filename
<< "\"}],"
<< "\"Enable bulk of logs\": [{\"value\": "
@@ -247,6 +287,8 @@ public:
ConfigComponent config;
vector<string> capture_syslog_cef_data;
I_MainLoop::Routine sysog_routine = nullptr;
StrictMock<MockShellCmd> mock_shell_cmd;
bool is_domain;
private:
string body;
@@ -259,6 +301,16 @@ TEST_F(LogTest, load_policy)
EXPECT_TRUE(loadFakeConfiguration(false));
}
TEST_F(LogTest, loadPolicyDomain)
{
is_domain = true;
string result = "172.28.1.6";
EXPECT_CALL(mock_shell_cmd, getExecOutput(_, _, _)).WillRepeatedly(Return(result));
EXPECT_TRUE(loadFakeConfiguration(false, true));
string failed_str = "Failed to connect to the CEF server";
EXPECT_THAT(getMessages(), Not(HasSubstr(failed_str)));
}
TEST_F(LogTest, loadPolicyFailure)
{
should_fail = true;
@@ -690,9 +742,117 @@ TEST_F(LogTest, FogBulkLogs)
EXPECT_EQ(local_body, str1);
}
TEST_F(LogTest, OfflineK8sSvcTest)
{
i_agent_details->setOrchestrationMode(OrchestrationMode::HYBRID);
should_load_k8s_stream = true;
loadFakeConfiguration(false);
Tags tag1 = Tags::POLICY_INSTALLATION;
Tags tag2 = Tags::ACCESS_CONTROL;
string local_body;
string res("[{\"id\": 1, \"code\": 400, \"message\": \"yes\"}]");
EXPECT_CALL(
mock_fog_msg,
sendMessage(_, _, _, "open-appsec-tuning-svc", _, _, "/api/v1/agents/events", _, _, MessageTypeTag::LOG)
).WillRepeatedly(DoAll(SaveArg<1>(&local_body), Return(res)));
string str1(
"{\n"
" \"log\": {\n"
" \"eventTime\": \"0:0:0\",\n"
" \"eventName\": \"Install policy\",\n"
" \"eventSeverity\": \"Info\",\n"
" \"eventPriority\": \"Low\",\n"
" \"eventType\": \"Event Driven\",\n"
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"\",\n"
" \"eventFrequency\": 0,\n"
" \"eventTags\": [\n"
" \"Access Control\",\n"
" \"Policy Installation\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" },\n"
" \"eventData\": {\n"
" \"logIndex\": 1\n"
" }\n"
" }\n"
"}"
);
LogGen("Install policy", Audience::INTERNAL, Severity::INFO, Priority::LOW, tag1, tag2);
EXPECT_EQ(local_body, str1);
}
TEST_F(LogTest, OfflineK8sSvcBulkLogs)
{
i_agent_details->setOrchestrationMode(OrchestrationMode::HYBRID);
should_load_k8s_stream = true;
loadFakeConfiguration(true);
string local_body;
string res("[{\"id\": 1, \"code\": 400, \"message\": \"yes\"}]");
EXPECT_CALL(
mock_fog_msg,
sendMessage(_, _, _, "open-appsec-tuning-svc", _, _, "/api/v1/agents/events/bulk", _, _, MessageTypeTag::LOG)
).WillRepeatedly(DoAll(SaveArg<1>(&local_body), Return(res)));
Tags tag1 = Tags::POLICY_INSTALLATION;
Tags tag2 = Tags::ACCESS_CONTROL;
string str1(
"{\n"
" \"logs\": [\n"
" {\n"
" \"id\": 1,\n"
" \"log\": {\n"
" \"eventTime\": \"0:0:0\",\n"
" \"eventName\": \"Install policy\",\n"
" \"eventSeverity\": \"Info\",\n"
" \"eventPriority\": \"Low\",\n"
" \"eventType\": \"Event Driven\",\n"
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"\",\n"
" \"eventFrequency\": 0,\n"
" \"eventTags\": [\n"
" \"Access Control\",\n"
" \"Policy Installation\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"\",\n"
" \"serviceName\": \"Unnamed Nano Service\"\n"
" },\n"
" \"eventData\": {\n"
" \"logIndex\": 1\n"
" }\n"
" }\n"
" }\n"
" ]\n"
"}"
);
{
LogGen("Install policy", Audience::INTERNAL, Severity::INFO, Priority::LOW, tag1, tag2);
}
bulk_routine();
EXPECT_EQ(local_body, str1);
}
TEST_P(LogTest, metrics_check)
{
loadFakeConfiguration(true, "", 3);
loadFakeConfiguration(true, false, "", 3);
Tags tag1 = Tags::POLICY_INSTALLATION;
Tags tag2 = Tags::ACCESS_CONTROL;
@@ -837,7 +997,7 @@ TEST_F(LogTest, ShouldRetryAfterFailedWriteToFile)
EXPECT_TRUE(logger->delStream(ReportIS::StreamType::JSON_LOG_FILE));
static const string invalid_file_path = "/proc/gibberish";
loadFakeConfiguration(false, invalid_file_path, -1);
loadFakeConfiguration(false, false, invalid_file_path, -1);
LogGen(
"Install policy",

80
core/logging/syslog_stream.cc
View File

@@ -11,6 +11,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <arpa/inet.h>
#include "log_streams.h"
#include "logging_comp.h"
@@ -18,12 +20,17 @@ using namespace std;
USE_DEBUG_FLAG(D_REPORT);
SyslogStream::SyslogStream(const string &_ip_address, int _port)
static string lookup_cmd = "nslookup ";
static string line_selection_cmd = "| grep Address | sed -n 2p";
static string parsing_cmd = "| cut -f2 -d' ' | tr -d '\n'";
SyslogStream::SyslogStream(const string &_address, int _port, I_Socket::SocketType _protocol)
:
i_socket(Singleton::Consume<I_Socket>::by<LoggingComp>()),
mainloop(Singleton::Consume<I_MainLoop>::by<LoggingComp>()),
ip_address(_ip_address),
port(_port)
address(_address),
port(_port),
protocol(_protocol)
{
connect();
if (!socket.ok()) {
@@ -43,21 +50,21 @@ SyslogStream::~SyslogStream()
void
SyslogStream::sendLog(const Report &log)
{
if (!socket.ok()) {
connect();
if (!socket.ok()) {
dbgWarning(D_REPORT) << "Failed to connect to the syslog server, Log will not be sent.";
return;
}
dbgTrace(D_REPORT) << "Successfully connect to the syslog server";
}
string syslog_report = log.getSyslog();
vector<char> data(syslog_report.begin(), syslog_report.end());
mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::Offline,
[this, data] ()
{
if (!socket.ok()) {
connect();
if (!socket.ok()) {
dbgWarning(D_REPORT) << "Failed to connect to the syslog server, Log will not be sent.";
return;
}
dbgTrace(D_REPORT) << "Successfully connect to the syslog server";
}
int tries = 1;
for (; tries <=3; tries++) {
if (i_socket->writeData(socket.unpack(), data)) {
@@ -75,18 +82,57 @@ SyslogStream::sendLog(const Report &log)
void
SyslogStream::connect()
{
auto syslog_ip_address = getProfileAgentSettingWithDefault<string>(ip_address, "agent.config.log.syslogServer.IP");
auto syslog_address = getProfileAgentSettingWithDefault<string>(address, "agent.config.log.syslogServer.IP");
auto syslog_port = getProfileAgentSettingWithDefault<uint>(port, "agent.config.log.syslogServer.port");
if (syslog_ip_address.empty()) {
dbgWarning(D_REPORT) << "Cannot connect to Syslog server, IP is not configured.";
if (syslog_address.empty()) {
dbgWarning(D_REPORT) << "Cannot connect to Syslog server, Address IP/Domain not configured.";
return;
}
struct in_addr addr;
if (inet_pton(AF_INET, syslog_address.data(), &addr) != 1) {
I_ShellCmd *shell_cmd = Singleton::Consume<I_ShellCmd>::by<LoggingComp>();
string host_cmd = lookup_cmd + syslog_address + line_selection_cmd + parsing_cmd;
Maybe<string> res = shell_cmd->getExecOutput(host_cmd, 500);
if (!res.ok()) {
dbgWarning(D_REPORT)
<< "Failed to execute domain lookup command. "
<< "SYSLOG Domain: "
<< syslog_address
<< "Error: "
<< res.getErr();
return;
}
if (res.unpack().empty()) {
dbgWarning(D_REPORT)
<< "Got en empty ip address from lookup command. "
<< "SYSLOG Domain: "
<< syslog_address
<< "Got bad ip address: "
<< res.unpack();
return;
}
dbgDebug(D_REPORT) << "SYSLOG Domain lookup result: " << res.unpack();
if (inet_pton(AF_INET, res.unpack().data(), &addr) != 1) {
dbgWarning(D_REPORT)
<< "Got a faulty ip address from lookup command. "
<< "SYSLOG Domain: "
<< syslog_address
<< "Got bad ip address: "
<< res.unpack();
return;
}
syslog_address = res.unpack();
}
socket = i_socket->genSocket(
I_Socket::SocketType::UDP,
protocol,
false,
false,
syslog_ip_address + ":" + to_string(syslog_port)
syslog_address + ":" + to_string(syslog_port)
);
}

7
core/report/tag_and_enum_management.cc
View File

@@ -98,7 +98,8 @@ TagAndEnumManagement::convertStringToTag(const string &tag)
{"Reverse Proxy", ReportIS::Tags::REVERSE_PROXY},
{"Http Geo Filter", ReportIS::Tags::HTTP_GEO_FILTER},
{"File Upload", ReportIS::Tags::FILE_UPLOAD},
{"Identity Awareness", ReportIS::Tags::IDENTITY_AWARENESS}
{"Identity Awareness", ReportIS::Tags::IDENTITY_AWARENESS},
{"Rate Limit", ReportIS::Tags::RATE_LIMIT}
};
auto report_is_tag = strings_to_tags.find(tag);
@@ -135,6 +136,7 @@ TagAndEnumManagement::convertToString(const StreamType &stream_type)
case StreamType::JSON_DEBUG: return "JSON Debug stream";
case StreamType::JSON_FOG: return "JSON FOG stream";
case StreamType::JSON_LOG_FILE: return "JSON File stream";
case StreamType::JSON_K8S_SVC: return "JSON K8S service stream";
case StreamType::SYSLOG: return "Syslog stream";
case StreamType::CEF: return "CEF stream";
@@ -307,7 +309,8 @@ EnumArray<Tags, string> TagAndEnumManagement::tags_translation_arr {
"Reverse Proxy",
"Http Geo Filter",
"File Upload",
"Identity Awareness"
"Identity Awareness",
"Rate Limit"
};
EnumArray<AudienceTeam, string> TagAndEnumManagement::audience_team_translation {

77
core/tenant_manager/tenant_manager.cc
View File

@@ -49,6 +49,8 @@ public:
chrono::microseconds getTimeoutVal() const override;
string getProfileId(const string &tenant_id, const string &region) const override;
void
addInstance(const string &tenant_id, const string &profile_id, const string &instace_id)
{
@@ -183,7 +185,7 @@ TenantManager::Impl::init()
auto rest = Singleton::Consume<I_RestApi>::by<TenantManager>();
rest->addRestCall<LoadNewTenants>(RestAction::SET, "tenant-id");
rest->addRestCall<FetchActiveTenants>(RestAction::SHOW, "active-tenants");
rest->addRestCall<FetchActiveTenants>(RestAction::SHOW, "profile-ids");
rest->addRestCall<FetchProfileIds>(RestAction::SHOW, "profile-ids");
}
if (type == TenantManagerType::CLIENT) {
@@ -307,14 +309,14 @@ TenantManager::Impl::getAllTenants() const
}
vector<string>
TenantManager::Impl::getProfileIds(const string &tenant_id) const
TenantManager::Impl::getProfileIds(const string &_tenant_id) const
{
dbgFlow(D_TENANT_MANAGER) << "Tenant Manager is a client. Requesting the active tenants";
dbgFlow(D_TENANT_MANAGER) << "Tenant Manager is a client. Requesting the active profiles";
GetProfileIds profile_id(tenant_id);
GetProfileIds tenant_id(_tenant_id);
auto res = i_messaging->sendObject(
profile_id,
tenant_id,
I_Messaging::Method::POST,
"127.0.0.1",
7777,
@@ -324,7 +326,7 @@ TenantManager::Impl::getProfileIds(const string &tenant_id) const
if (!res) {
i_messaging->sendObject(
profile_id,
tenant_id,
I_Messaging::Method::POST,
"127.0.0.1",
7778,
@@ -333,7 +335,53 @@ TenantManager::Impl::getProfileIds(const string &tenant_id) const
);
}
return profile_id.profile_ids.get();
return tenant_id.profile_ids.get();
}
string
TenantManager::Impl::getProfileId(const string &tenant_id, const string &region) const
{
if (region.empty()) {
dbgWarning(D_TENANT_MANAGER) << "Can't find the profile ID. Region is empty";
return "";
}
vector<string> profile_ids = fetchProfileIds(tenant_id);
dbgTrace(D_TENANT_MANAGER) << "Fetched " << profile_ids.size() << " profiles";
auto i_env = Singleton::Consume<I_Environment>::by<TenantManager>();
auto unset_tenant_on_exit = make_scope_exit([&]() { i_env->unsetActiveTenantAndProfile(); });
for (const string &profile_id : profile_ids) {
dbgDebug(D_TENANT_MANAGER)
<< "Checking if the profile ID: "
<< profile_id
<< " corresponds to the tenant ID: "
<< tenant_id
<< " and the region "
<< region;
i_env->setActiveTenantAndProfile(tenant_id, profile_id);
auto maybe_region = getSetting<string>("region");
if (maybe_region.ok() && region == maybe_region.unpack()) {
dbgDebug(D_TENANT_MANAGER) << "The region corresponds to profile ID " << profile_id;
return profile_id;
} else {
if (maybe_region.ok()) {
dbgTrace(D_TENANT_MANAGER)
<< "The region does not corresponds to profile ID "
<< profile_id
<< " region "
<< *maybe_region;
} else {
dbgDebug(D_TENANT_MANAGER) << "Failed to get region for profile ID " << profile_id;
}
}
}
dbgWarning(D_TENANT_MANAGER) << "Found no profile ID for tenant " << tenant_id << " and region " << region;
return "";
}
void
@@ -351,7 +399,16 @@ TenantManager::Impl::areTenantAndProfileActive(const string &tenant_id, const st
void
TenantManager::Impl::addActiveTenantAndProfile(const string &tenant_id, const string &profile_id)
{
if (tenant_id.empty() || profile_id.empty()) {
dbgWarning(D_TENANT_MANAGER) << "Tenant ID and Profile ID should not be empty.";
return;
}
auto tenant_profile = TenantProfilePair(tenant_id, profile_id);
dbgTrace(D_TENANT_MANAGER)
<< "Adding an active tenant and profile. Tenant ID: "
<< tenant_id
<< ", Profile ID: "
<< profile_id;
active_tenants.createEntry(tenant_profile);
if (type == TenantManagerType::CLIENT) {
sendTenantAndProfile(tenant_id, profile_id);
@@ -410,7 +467,8 @@ TenantManager::Impl::fetchAllProfileIds(const string &tenant_id) const
for (auto iter = begin(active_tenants); iter != end(active_tenants); iter++) {
if (iter->first.getTenantId() == tenant_id) {
tenant_profile_ids.push_back(iter->first.getPfofileId());
dbgTrace(D_TENANT_MANAGER) << "Returning a fetched profile ID: " << iter->first.getProfileId();
tenant_profile_ids.push_back(iter->first.getProfileId());
}
}
return tenant_profile_ids;
@@ -419,7 +477,7 @@ TenantManager::Impl::fetchAllProfileIds(const string &tenant_id) const
vector<string>
TenantManager::Impl::fetchProfileIds(const string &tenant_id) const
{
dbgFlow(D_TENANT_MANAGER) << "Fetching all profile ids for tenant " << tenant_id;
dbgFlow(D_TENANT_MANAGER) << "Fetching all profile IDs for tenant " << tenant_id;
return (type == TenantManagerType::CLIENT) ? getProfileIds(tenant_id) : fetchAllProfileIds(tenant_id);
}
@@ -462,4 +520,5 @@ TenantManager::preload()
{
registerExpectedConfiguration<uint32_t>("Tenant Manager", "Tenant timeout");
registerExpectedConfiguration<string>("Tenant Manager", "Tenant manager type");
registerExpectedSetting<string>("region");
}