github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Apr 27th Update

Browse Source
This commit is contained in:
Ned Wright
2023-04-27 19:05:49 +00:00
parent cd4fb6e3e8
commit fd2d9fa081
89 changed files with 2175 additions and 544 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
core/agent_details_reporter/agent_details_report.cc
View File

@@ -19,6 +19,7 @@ using namespace std;
AgentDataReport::~AgentDataReport()
{
if (!should_report) return;
Singleton::Consume<I_AgentDetailsReporter>::by<AgentDataReport>()->sendReport(
agent_details,
policy_version,
@@ -35,6 +36,17 @@ AgentDataReport::operator<<(const pair<string, string> &data)
return *this;
}
bool
AgentDataReport::operator==(const AgentDataReport &other) const
{
return policy_version == other.policy_version &&
platform == other.platform &&
architecture == other.architecture &&
agent_version == other.agent_version &&
agent_details == other.agent_details &&
attributes == other.attributes;
}
void
AgentDataReport::setPolicyVersion(const string &_policy_version)
{
@@ -58,3 +70,9 @@ AgentDataReport::setAgentVersion(const string &_agent_version)
{
agent_version = _agent_version;
}
void
AgentDataReport::disableReportSending()
{
should_report = false;
}

13
core/agent_details_reporter/agent_details_reporter.cc
View File

@@ -127,6 +127,7 @@ private:
const string &operation;
};
map<string, string> persistant_attributes;
map<string, string> new_attributes;
map<string, string> attributes;
@@ -141,6 +142,12 @@ metaDataReport::operator<<(const pair<string, string> &data)
return *this;
}
bool
metaDataReport::operator==(const metaDataReport &other) const
{
return agent_details == other.agent_details;
}
void
metaDataReport::serialize(cereal::JSONOutputArchive &out_ar) const
{
@@ -169,7 +176,12 @@ AgentDetailsReporter::Impl::addAttr(const string &key, const string &val, bool a
}
}
if (persistant_attributes[key] == val) {
dbgDebug(D_AGENT_DETAILS) << "Attribute " << key << " did not change. Value: " << val;
return true;
}
new_attributes[key] = val;
persistant_attributes[key] = val;
dbgDebug(D_AGENT_DETAILS) << "Successfully added new attribute";
return true;
@@ -194,6 +206,7 @@ AgentDetailsReporter::Impl::deleteAttr(const string &key)
dbgDebug(D_AGENT_DETAILS) << "Deleting existing attributes. Key: " << key;
attributes.erase(key);
new_attributes.erase(key);
persistant_attributes.erase(key);
}
bool

9
core/config/config.cc
View File

@@ -80,14 +80,11 @@ public:
error = load_config_staus == I_Config::AsyncLoadConfigStatus::Error;
if (error) {
error_message = "Reload already in progress - can't start another one";
dbgWarning(D_CONFIG) << "Configuration reload status: " << status_map.at(load_config_staus);
} else {
dbgDebug(D_CONFIG) << "Configuration reload status: " << status_map.at(load_config_staus);
}
if (!finished) {
error_message = "Reload already in progress - can't start another one";
}
}
private:
@@ -625,7 +622,7 @@ ConfigComponent::Impl::reloadConfiguration(const string &version, bool is_async,
{
if (is_continuous_report) {
dbgWarning(D_CONFIG) << "Cannot start another continuous reload while another is running.";
return AsyncLoadConfigStatus::InProgress;
return AsyncLoadConfigStatus::Error;
}
if (!is_async) {
@@ -643,7 +640,7 @@ ConfigComponent::Impl::reloadConfiguration(const string &version, bool is_async,
"A-Synchronize reload configuraion"
);
return AsyncLoadConfigStatus::Success;
return AsyncLoadConfigStatus::InProgress;
}
void

12
core/core_ut/cache_ut.cc
View File

@@ -78,6 +78,18 @@ TEST(TempCaching, value_emplace)
EXPECT_EQ(val, 9);
}
TEST(TempCaching, value_get_const)
{
TemporaryCache<int, Int> cache;
cache.emplaceEntry(3, 27);
auto &const_cache = const_cast<const TemporaryCache<int, Int> &>(cache);
EXPECT_FALSE(const_cache.getEntry(0).ok());
EXPECT_TRUE(const_cache.getEntry(3).ok());
EXPECT_EQ(const_cache.getEntry(3).unpack(), 27);
}
TEST(TempCaching, get_uninitialized_value)
{
TemporaryCache<int, Int> cache;

3
core/include/general/debug.h
View File

@@ -32,6 +32,9 @@ class I_Encryptor;
class I_AgentDetails;
class I_SignalHandler;
namespace Config { enum class Errors; }
std::ostream & operator<<(std::ostream &, const Config::Errors &);
class Debug
:
Singleton::Consume<I_TimeGet>,

1
core/include/services_sdk/interfaces/i_agent_details_reporter.h
View File

@@ -27,6 +27,7 @@ public:
metaDataReport(const metaDataReport &) = default;
metaDataReport & operator<<(const std::pair<std::string, std::string> &data);
bool operator==(const metaDataReport &other) const;
void serialize(cereal::JSONOutputArchive &out_ar) const;
private:

24
core/include/services_sdk/interfaces/i_intelligence_is_v2.h
View File

@@ -15,6 +15,7 @@
#define __I_INTELLIGENCE_IS_V2_H__
#include <chrono>
#include <string>
#include "maybe_res.h"
#include "i_messaging.h"
@@ -130,6 +131,14 @@ private:
);
}
dbgTrace(D_INTELLIGENCE)
<< "Sending intelligence request with IP: "
<< ip
<< " port: "
<< server_port
<< " query_uri: "
<< query_uri;
return i_message->sendObject(
intelligence_query,
I_Messaging::Method::POST,
@@ -248,10 +257,11 @@ private:
"intelligence",
is_primary_port ? primary_port_setting : secondary_port_setting
);
if (!server_port.ok()) return false;
conn_flags.reset();
if (intelligence_query.getPagingStatus().ok()) {
return sendPagingQueryMessage(
intelligence_query,
@@ -275,12 +285,22 @@ private:
auto i_message = getMessaging();
Flags<MessageConnConfig> conn_flags;
bool crowdsec_enabled = std::getenv("CROWDSEC_ENABLED") ?
std::string(std::getenv("CROWDSEC_ENABLED")) == "true" :
false;
crowdsec_enabled = getProfileAgentSettingWithDefault<bool>(
crowdsec_enabled,
"layer7AccessControl.crowdsec.enabled"
);
bool use_local_intelligence = getProfileAgentSettingWithDefault<bool>(
false,
"agent.config.useLocalIntelligence"
);
auto server_ip = getSetting<std::string>("intelligence", "local intelligence server ip");
if (server_ip.ok() && use_local_intelligence) {
if (server_ip.ok() && (use_local_intelligence || crowdsec_enabled)) {
if (sendQueryObjectToLocalServer(
intelligence_query,
query_uri,

2
core/include/services_sdk/interfaces/i_mainloop.h
View File

@@ -61,6 +61,8 @@ public:
virtual Maybe<I_MainLoop::RoutineID> getCurrentRoutineId() const = 0;
virtual void updateCurrentStress(bool is_busy) = 0;
virtual void run() = 0;
// When a routine yields the scheduler may choose to let it continue to run (in the case the routine didn't use

2
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h
View File

@@ -58,6 +58,8 @@ enum class ResponseStatus
IN_PROGRESS
};
enum class ObjectType { ASSET, ZONE, CONFIGURATION, COUNT };
const std::string & convertConditionTypeToString(const Condition &condition_type);
const std::string & convertOperationTypeToString(const Operator &operation_type);
std::string createAttributeString(const std::string &key, AttributeKeyType type);

4
core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h
View File

@@ -63,6 +63,8 @@ public:
);
void setTenantsList(const std::vector<std::string> tenants);
void setCrossTenantAssetDB(bool cross_tenant_asset_db);
void setObjectType(const ObjectType &obj_type);
void setAssetsLimit(uint _assets_limit);
bool checkMinConfidence(uint upper_confidence_limit);
@@ -83,11 +85,13 @@ public:
private:
uint assets_limit = default_assets_limit;
bool full_response = false;
Maybe<ObjectType> object_type = genError("uninitialized");
Maybe<RequestCursor> cursor = genError("Cursor not initialized");
SerializableQueryFilter query;
SerializableAttributesMap requested_attributes;
SerializableQueryTypes query_types;
QueryRequest calcQueryRequestOperator(const QueryRequest &other_query, const Operator &operator_type);
Maybe<std::string> convertObjectTypeToString() const;
};
class BulkQueryRequest

25
core/include/services_sdk/interfaces/intelligence_is_v2/query_types_v2.h
View File

@@ -19,35 +19,26 @@
#include "cereal/types/tuple.hpp"
#include "cereal/types/vector.hpp"
#include "intelligence_types_v2.h"
#include "maybe_res.h"
#include <vector>
#include <unordered_map>
class serializableTenantList
{
public:
serializableTenantList(const std::vector<std::string> &_tenants)
:
tenants(_tenants)
{}
void serialize(cereal::JSONOutputArchive &ar) const;
private:
std::vector<std::string> tenants;
};
class SerializableQueryTypes
{
public:
SerializableQueryTypes() {};
void save(cereal::JSONOutputArchive &ar) const;
void setSerializableTenantList(const std::vector<std::string> tenants);
void setSerializableTenantList(const std::vector<std::string> &tenant_list);
void setQueryCrossTenantAssetDB(bool query_cross_tenant_asset_db);
private:
std::vector<std::string> tenants;
bool is_nsaas = false;
void serializeMultiTenant(cereal::JSONOutputArchive &ar) const;
void serializeCrossTenantAssetDB(cereal::JSONOutputArchive &ar) const;
Maybe<std::vector<std::string>> tenants = genError("tenant list is uninitialized");
Maybe<bool> query_cross_tenant_asset_db = genError("cross tenant asset db query is uninitialized");
};
#endif // __QUERY_TYPES_V2_H__

2
core/include/services_sdk/interfaces/mock/mock_mainloop.h
View File

@@ -26,6 +26,8 @@ public:
MOCK_CONST_METHOD0(getCurrentRoutineId, Maybe<I_MainLoop::RoutineID> ());
MOCK_METHOD1(updateCurrentStress, void (bool));
MOCK_METHOD1(yield, void (bool));
MOCK_METHOD1(yield, void (std::chrono::microseconds));

13
core/include/services_sdk/interfaces/mock/mock_shell_cmd.h
View File

@@ -7,6 +7,13 @@
#include "singleton.h"
#include "cptest.h"
static std::ostream &
operator<<(std::ostream &os, const Maybe<std::pair<std::string, int>> &val)
{
if (val.ok()) return os << "<" << (*val).first << ", " << (*val).second << ">";
return os;
}
class MockShellCmd : public Singleton::Provide<I_ShellCmd>::From<MockProvider<I_ShellCmd>>
{
public:
@@ -15,10 +22,4 @@ public:
MOCK_METHOD3(getExecOutputAndCode, Maybe<std::pair<std::string, int>>(const std::string &, uint, bool));
};
static std::ostream &
operator<<(std::ostream &os, const std::pair<std::string, int> &val)
{
return os << "<" << val.first << ", " << val.second << ">";
}
#endif // __MOCK_SHELL_CMD_H__

2
core/include/services_sdk/interfaces/mock/mock_socket_is.h
View File

@@ -5,6 +5,8 @@
#include "cptest.h"
#include "common.h"
std::ostream & operator<<(std::ostream &os, const Maybe<std::vector<char>> &) { return os; }
class MockSocketIS : public Singleton::Provide<I_Socket>::From<MockProvider<I_Socket>>
{
public:

5
core/include/services_sdk/resources/agent_details_report.h
View File

@@ -30,17 +30,22 @@ class AgentDataReport
{
public:
AgentDataReport() = default;
AgentDataReport(bool disable_report_sending) { should_report = disable_report_sending; }
~AgentDataReport();
AgentDataReport & operator<<(const std::pair<std::string, std::string> &data);
bool operator==(const AgentDataReport& other) const;
void setPolicyVersion(const std::string &policy_version);
void setPlatform(const std::string &platform);
void setArchitecture(const std::string &architecture);
void setAgentVersion(const std::string &_agent_version);
void disableReportSending();
private:
metaDataReport agent_details;
bool should_report = true;
Maybe<std::string> policy_version = genError("Not set");
Maybe<std::string> platform = genError("Not set");
Maybe<std::string> architecture = genError("Not set");

6
core/include/services_sdk/resources/debug_flags.h
View File

@@ -131,6 +131,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_SDWAN_POLICY, D_SDWAN)
DEFINE_FLAG(D_SDWAN_DATA, D_SDWAN)
DEFINE_FLAG(D_LOGGER_SDWAN, D_SDWAN)
DEFINE_FLAG(D_SDWAN_API, D_SDWAN)
DEFINE_FLAG(D_REVERSE_PROXY, D_COMPONENT)
DEFINE_FLAG(D_PLATFORM, D_REVERSE_PROXY)
DEFINE_FLAG(D_NGINX_MESSAGE_READER, D_REVERSE_PROXY)
@@ -143,11 +144,15 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_IOT_AUXILIARY, D_IOT_NEXT)
DEFINE_FLAG(D_IOT_REPORT_STATUS, D_IOT_AUXILIARY)
DEFINE_FLAG(D_IOT_COLLECT_METADATA, D_IOT_AUXILIARY)
DEFINE_FLAG(D_IOT_QUERY_INTELLIGENCE, D_IOT_AUXILIARY)
DEFINE_FLAG(D_IOT_SAVE_PERSISTENT, D_IOT_AUXILIARY)
DEFINE_FLAG(D_IOT_DOCKER, D_IOT_AUXILIARY)
DEFINE_FLAG(D_IOT_ENFORCE, D_IOT_NEXT)
DEFINE_FLAG(D_IOT_ENFORCE_POLICY, D_IOT_ENFORCE)
DEFINE_FLAG(D_IOT_ENFORCE_ASSETS, D_IOT_ENFORCE)
DEFINE_FLAG(D_IOT_DOCTOR, D_IOT_NEXT)
DEFINE_FLAG(D_IOT_RISK, D_IOT_NEXT)
DEFINE_FLAG(D_IOT_QUERY_ASSETS, D_IOT_RISK)
DEFINE_FLAG(D_IOT_INDICATOR_DATA, D_IOT_RISK)
DEFINE_FLAG(D_IOT_INDICATORS, D_IOT_RISK)
DEFINE_FLAG(D_IOT_DISCOVERY, D_IOT_NEXT)
@@ -159,6 +164,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_CPVIEW_METRIC_PROVIDER, D_COMPONENT)
DEFINE_FLAG(D_GEO_FILTER, D_COMPONENT)
DEFINE_FLAG(D_URL_FILTERING, D_COMPONENT)
DEFINE_FLAG(D_L7_ACCESS_CONTROL, D_COMPONENT)
DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT)
DEFINE_FLAG(D_FLOW, D_ALL)

82
core/include/services_sdk/resources/report/Log_modifiers.h Normal file
View File

@@ -0,0 +1,82 @@
#ifndef __LOG_MODIFIERS_H__
#define __LOG_MODIFIERS_H__
#include <string>
#include "virtual_modifiers.h"
namespace LogModifiers
{
class ReplaceBackslash : public ReplaceSubContiners<std::string>
{
public:
ReplaceBackslash() { init(&src, &dst); }
private:
std::string src = "\\";
std::string dst = "\\\\";
};
class ReplaceCR : public ReplaceSubContiners<std::string>
{
public:
ReplaceCR() { init(&src, &dst); }
private:
std::string src = "\r";
std::string dst = "\\r";
};
class ReplaceLF : public ReplaceSubContiners<std::string>
{
public:
ReplaceLF() { init(&src, &dst); }
private:
std::string src = "\n";
std::string dst = "\\n";
};
class ReplaceDoubleOuotes : public ReplaceSubContiners<std::string>
{
public:
ReplaceDoubleOuotes() { init(&src, &dst); }
private:
std::string src = "\"";
std::string dst = "\\\"";
};
class ReplaceQuote : public ReplaceSubContiners<std::string>
{
public:
ReplaceQuote() { init(&src, &dst); }
private:
std::string src = "'";
std::string dst = "\\'";
};
class ReplaceClosingBrace : public ReplaceSubContiners<std::string>
{
public:
ReplaceClosingBrace() { init(&src, &dst); }
private:
std::string src = "]";
std::string dst = "\\]";
};
class ReplaceEqualSign : public ReplaceSubContiners<std::string>
{
public:
ReplaceEqualSign() { init(&src, &dst); }
private:
std::string src = "=";
std::string dst = "\\=";
};
} // namesapce LogModifiers
#endif // __LOG_MODIFIERS_H__

50
core/include/services_sdk/resources/report/base_field.h
View File

@@ -25,6 +25,8 @@
#include "debug.h"
#include "flags.h"
#include "config.h"
#include "virtual_container.h"
#include "Log_modifiers.h"
enum class LogFieldOption { XORANDB64, COUNT };
@@ -72,8 +74,7 @@ class LogField : Singleton::Consume<I_Environment>
virtual void serialize(cereal::JSONOutputArchive &ar) const = 0;
virtual void addFields(const LogField &log) = 0;
virtual std::string getSyslog() const = 0;
virtual std::string getCef() const = 0;
virtual std::string getSyslogAndCef() const = 0;
template <typename ... Strings>
Maybe<std::string, void>
@@ -107,15 +108,17 @@ class LogField : Singleton::Consume<I_Environment>
}
std::string
getSyslog() const override
getSyslogAndCef() const override
{
return name + "='" + Details::getValueAsString(getValue()) + "'";
}
std::string
getCef() const override
{
return name + "=" + Details::getValueAsString(getValue());
std::string value(Details::getValueAsString(getValue()));
auto modifier1 = makeVirtualContainer<LogModifiers::ReplaceBackslash>(value);
auto modifier2 = makeVirtualContainer<LogModifiers::ReplaceCR>(modifier1);
auto modifier3 = makeVirtualContainer<LogModifiers::ReplaceLF>(modifier2);
auto modifier4 = makeVirtualContainer<LogModifiers::ReplaceDoubleOuotes>(modifier3);
auto modifier5 = makeVirtualContainer<LogModifiers::ReplaceQuote>(modifier4);
auto modifier6 = makeVirtualContainer<LogModifiers::ReplaceClosingBrace>(modifier5);
auto modifier7 = makeVirtualContainer<LogModifiers::ReplaceEqualSign>(modifier6);
return name + "=\"" + std::string(modifier7.begin(), modifier7.end()) + "\"";
}
// LCOV_EXCL_START Reason: seems that assert prevent the LCOV from identifying that method was tested
@@ -180,27 +183,14 @@ class LogField : Singleton::Consume<I_Environment>
}
std::string
getSyslog() const override
getSyslogAndCef() const override
{
if (fields.size() == 0) return "";
std::string res;
for (auto &field : fields) {
if (res.size() > 0) res += " ";
res += field.getSyslog();
}
return res;
}
std::string
getCef() const override
{
if (fields.size() == 0) return "";
std::string res;
for (auto &field : fields) {
if (res.size() > 0) res += " ";
res += field.getCef();
res += field.getSyslogAndCef();
}
return res;
}
@@ -252,15 +242,9 @@ public:
}
std::string
getSyslog() const
getSyslogAndCef() const
{
return field->getSyslog();
}
std::string
getCef() const
{
return field->getCef();
return field->getSyslogAndCef();
}
void

1
core/include/services_sdk/resources/report/report_enums.h
View File

@@ -61,6 +61,7 @@ enum class Tags {
WEB_SERVER_KONG,
DEPLOYMENT_EMBEDDED,
DEPLOYMENT_K8S,
LAYER_7_ACCESS_CONTROL,
COUNT
};

1
core/include/services_sdk/utilities/cache.h
View File

@@ -74,6 +74,7 @@ public:
bool emplaceEntry(const Key &key, const Value &val);
bool emplaceEntry(const Key &key, Value &&val);
Value & getEntry(const Key &key);
Maybe<Value, void> getEntry(const Key &key) const;
microseconds getEntryTimeLeft(const Key &key);
};

8
core/include/services_sdk/utilities/caching/cache_impl.h
View File

@@ -232,6 +232,14 @@ TemporaryCache<Key, Value>::getEntry(const Key &key)
return entries.at(key).getValue();
}
template <typename Key, typename Value>
Maybe<Value, void>
TemporaryCache<Key, Value>::getEntry(const Key &key) const
{
if (!BaseTemporaryCache<Key, Value>::doesKeyExists(key)) return genError(0);
return entries.at(key).getValue();
}
template <typename Key, typename Value>
std::chrono::microseconds
TemporaryCache<Key, Value>::getEntryTimeLeft(const Key &key)

1
core/include/services_sdk/utilities/caching/cache_types.h
View File

@@ -50,6 +50,7 @@ public:
void setNewTime(I_TimeGet *timer) { timer != nullptr ? time = timer->getMonotonicTime() : microseconds(0); }
bool isExpired(const microseconds &expired) const { return time < expired; }
Value & getValue() { return val; }
const Value & getValue() const { return val; }
microseconds getTime() { return time; }
private:

3
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@@ -140,6 +140,9 @@ IntelligenceComponentV2::preload()
registerExpectedConfiguration<bool>("intelligence", "offline intelligence only");
registerExpectedConfiguration<uint>("intelligence", "maximum request overall time");
registerExpectedConfiguration<uint>("intelligence", "maximum request lap time");
registerExpectedSetting<string>("intelligence", "local intelligence server ip");
registerExpectedSetting<uint>("intelligence", "local intelligence server secondary port");
registerExpectedSetting<uint>("intelligence", "local intelligence server primary port");
registerExpectedConfigFile("agent-intelligence", Config::ConfigFileType::Policy);
}

103
core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc
View File

@@ -5,6 +5,8 @@
using namespace std;
using namespace testing;
USE_DEBUG_FLAG(D_INTELLIGENCE);
TEST(QueryRequestTestV2, QueryTest)
{
QueryRequest request(Condition::EQUALS, "phase", "testing", true);
@@ -438,3 +440,104 @@ TEST(QueryRequestTestV2, OneLinerComplexQueryTest)
"}";
EXPECT_EQ(out.str(), output_json);
}
TEST(QueryRequestTestV2, CrossTenantAssetDBTest)
{
QueryRequest request(Condition::EQUALS, "class", "risk", true);
request.setObjectType(ObjectType::CONFIGURATION);
request.setCrossTenantAssetDB(true);
string output_json =
"{\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.class\",\n"
" \"value\": \"risk\"\n"
" },\n"
" \"objectType\": \"configuration\",\n"
" \"queryTypes\": {\n"
" \"queryCrossTenantAssetDB\": true\n"
" }\n"
"}";
stringstream out;
{
cereal::JSONOutputArchive out_ar(out);
request.saveToJson(out_ar);
}
EXPECT_EQ(out.str(), output_json);
}
TEST(QueryRequestTestV2, IllegalObjectTypeTest)
{
QueryRequest request(Condition::EQUALS, "class", "risk", true);
stringstream debug_output;
Debug::setNewDefaultStdout(&debug_output);
Debug::setUnitTestFlag(D_INTELLIGENCE, Debug::DebugLevel::TRACE);
request.setObjectType(static_cast<ObjectType>(static_cast<int>(ObjectType::COUNT) + 1));
request.setCrossTenantAssetDB(true);
string output_json =
"{\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.class\",\n"
" \"value\": \"risk\"\n"
" },\n"
" \"queryTypes\": {\n"
" \"queryCrossTenantAssetDB\": true\n"
" }\n"
"}";
stringstream out;
{
cereal::JSONOutputArchive out_ar(out);
request.saveToJson(out_ar);
}
EXPECT_EQ(out.str(), output_json);
string debug_str = "Illegal Object Type.";
EXPECT_THAT(debug_output.str(), HasSubstr(debug_str));
Debug::setNewDefaultStdout(&cout);
}
TEST(QueryRequestTestV2, UninitializedObjectTypeTest)
{
QueryRequest request(Condition::EQUALS, "class", "risk", true);
stringstream debug_output;
Debug::setNewDefaultStdout(&debug_output);
Debug::setUnitTestFlag(D_INTELLIGENCE, Debug::DebugLevel::TRACE);
request.setCrossTenantAssetDB(true);
string output_json =
"{\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.class\",\n"
" \"value\": \"risk\"\n"
" },\n"
" \"queryTypes\": {\n"
" \"queryCrossTenantAssetDB\": true\n"
" }\n"
"}";
stringstream out;
{
cereal::JSONOutputArchive out_ar(out);
request.saveToJson(out_ar);
}
EXPECT_EQ(out.str(), output_json);
string debug_str = "uninitialized";
EXPECT_THAT(debug_output.str(), HasSubstr(debug_str));
Debug::setNewDefaultStdout(&cout);
}

40
core/intelligence_is_v2/query_request_v2.cc
View File

@@ -13,6 +13,7 @@
#include "intelligence_is_v2/query_request_v2.h"
#include "debug.h"
#include "enum_array.h"
const uint QueryRequest::default_min_confidence = 500;
const uint QueryRequest::default_assets_limit = 20;
@@ -22,6 +23,8 @@ using namespace Intelligence_IS_V2;
USE_DEBUG_FLAG(D_INTELLIGENCE);
static const EnumArray<ObjectType, string> object_type_to_string_array{ "asset", "zone", "configuration" };
BulkQueryRequest::BulkQueryRequest(QueryRequest &_request, int _index)
:
request(_request),
@@ -55,6 +58,17 @@ QueryRequest::QueryRequest(
full_response = full_reponse;
}
Maybe<string>
QueryRequest::convertObjectTypeToString() const
{
if (!object_type.ok()) return object_type.passErr();
if (static_cast<uint>(*object_type) < static_cast<uint>(ObjectType::COUNT)) {
return object_type_to_string_array[*object_type];
}
return genError("Illegal Object Type.");
}
void
QueryRequest::saveToJson(cereal::JSONOutputArchive &ar) const
{
@@ -64,6 +78,13 @@ QueryRequest::saveToJson(cereal::JSONOutputArchive &ar) const
cereal::make_nvp("query", query)
);
auto objTypeString = convertObjectTypeToString();
if (objTypeString.ok()) {
ar(cereal::make_nvp("objectType", *objTypeString));
} else {
dbgTrace(D_INTELLIGENCE) << objTypeString.getErr();
}
if (cursor.ok()) ar(cereal::make_nvp("cursor", cursor.unpack().second));
requested_attributes.save(ar);
query_types.save(ar);
@@ -78,6 +99,13 @@ QueryRequest::save(cereal::JSONOutputArchive &ar) const
cereal::make_nvp("query", query)
);
auto objTypeString = convertObjectTypeToString();
if (objTypeString.ok()) {
ar(cereal::make_nvp("objectType", *objTypeString));
} else {
dbgTrace(D_INTELLIGENCE) << objTypeString.getErr();
}
if (cursor.ok()) ar(cereal::make_nvp("cursor", cursor.unpack().second));
requested_attributes.save(ar);
query_types.save(ar);
@@ -129,6 +157,12 @@ QueryRequest::setTenantsList(const vector<string> tenants)
query_types.setSerializableTenantList(tenants);
}
void
QueryRequest::setCrossTenantAssetDB(bool cross_tenant_asset_db)
{
query_types.setQueryCrossTenantAssetDB(cross_tenant_asset_db);
}
void
QueryRequest::setAssetsLimit(uint _assets_limit)
{
@@ -173,6 +207,12 @@ QueryRequest::setCursor(CursorState state, const string &value)
cursor = RequestCursor(state, value);
}
void
QueryRequest::setObjectType(const ObjectType &obj_type)
{
object_type = obj_type;
}
QueryRequest
QueryRequest::calcQueryRequestOperator(const QueryRequest &other_query, const Operator &operator_type)
{

31
core/intelligence_is_v2/query_types_v2.cc
View File

@@ -17,22 +17,37 @@ using namespace std;
using namespace Intelligence_IS_V2;
void
serializableTenantList::serialize(cereal::JSONOutputArchive &ar) const
SerializableQueryTypes::serializeMultiTenant(cereal::JSONOutputArchive &ar) const
{
ar(cereal::make_nvp("multiTenant", tenants));
ar(cereal::make_nvp("multiTenant", *tenants));
}
void
SerializableQueryTypes::serializeCrossTenantAssetDB(cereal::JSONOutputArchive &ar) const
{
ar(cereal::make_nvp("queryCrossTenantAssetDB", *query_cross_tenant_asset_db));
}
void
SerializableQueryTypes::save(cereal::JSONOutputArchive &ar) const
{
if (!is_nsaas) return;
serializableTenantList serializable_tenants(tenants);
ar(cereal::make_nvp("queryTypes", serializable_tenants));
if (!tenants.ok() && !query_cross_tenant_asset_db.ok()) return;
ar.setNextName("queryTypes");
ar.startNode();
if (tenants.ok()) serializeMultiTenant(ar);
if (query_cross_tenant_asset_db.ok()) serializeCrossTenantAssetDB(ar);
ar.finishNode();
}
void
SerializableQueryTypes::setSerializableTenantList(const std::vector<std::string> _tenants)
SerializableQueryTypes::setSerializableTenantList(const vector<string> &tenant_list)
{
tenants = _tenants;
is_nsaas = true;
tenants = tenant_list;
};
void
SerializableQueryTypes::setQueryCrossTenantAssetDB(bool cross_tenant_asset_db)
{
query_cross_tenant_asset_db = cross_tenant_asset_db;
}

1
core/logging/log_streams.h
View File

@@ -103,6 +103,7 @@ public:
void sendLog(const Report &log) override;
private:
void sendLog(const std::vector<char> &data);
void connect();
I_Socket *i_socket = nullptr;

2
core/logging/logging_ut/logging_ut.cc
View File

@@ -1489,7 +1489,7 @@ TEST_F(LogTest, ObfuscationTest)
sysog_routine();
EXPECT_EQ(capture_syslog_cef_data.size(), 2);
for (const string &str : capture_syslog_cef_data) {
EXPECT_THAT(str, AnyOf(HasSubstr("String='Another string'"), HasSubstr("String=Another string")));
EXPECT_THAT(str, AnyOf(HasSubstr("String='Another string'"), HasSubstr("String=\"Another string\"")));
}
}

43
core/logging/syslog_stream.cc
View File

@@ -54,31 +54,32 @@ SyslogStream::sendLog(const Report &log)
vector<char> data(syslog_report.begin(), syslog_report.end());
mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::Offline,
[this, data] ()
{
if (!socket.ok()) {
connect();
if (!socket.ok()) {
dbgWarning(D_REPORT) << "Failed to connect to the syslog server, Log will not be sent.";
return;
}
dbgTrace(D_REPORT) << "Successfully connect to the syslog server";
}
int tries = 1;
for (; tries <=3; tries++) {
if (i_socket->writeData(socket.unpack(), data)) {
dbgTrace(D_REPORT) << "log was sent to syslog server";
return;
} else {
dbgWarning(D_REPORT) << "Failed to send log to syslog server";
}
}
},
[this, data] () { sendLog(data); },
"Logging Syslog stream messaging"
);
}
void
SyslogStream::sendLog(const vector<char> &data)
{
for (int tries = 0; tries < 3; ++tries) {
if (!socket.ok()) {
connect();
if (!socket.ok()) {
dbgWarning(D_REPORT) << "Failed to connect to the syslog server, Log will not be sent.";
return;
}
dbgTrace(D_REPORT) << "Successfully connect to the syslog server";
}
if (i_socket->writeData(socket.unpack(), data)) {
dbgTrace(D_REPORT) << "log was sent to syslog server";
return;
}
}
dbgWarning(D_REPORT) << "Failed to send log to syslog server";
}
void
SyslogStream::connect()
{

3
core/mainloop/mainloop.cc
View File

@@ -73,6 +73,8 @@ public:
Maybe<RoutineID> getCurrentRoutineId() const override;
void updateCurrentStress(bool is_busy) override;
void yield(bool force) override;
void yield(chrono::microseconds time) override;
void stopAll() override;
@@ -118,7 +120,6 @@ public:
private:
void reportStartupEvent();
void stop(const RoutineMap::iterator &iter);
void updateCurrentStress(bool is_busy);
uint32_t getCurrentTimeSlice(uint32_t current_stress);
RoutineID getNextID();

112
core/message/http_decoder.cc
View File

@@ -129,17 +129,7 @@ HTTPDecoder::handleBody()
auto maybe_transfer_encoding = unpacked_headers.getHeaderVal("transfer-encoding");
if (maybe_transfer_encoding.ok()) {
auto transfer_encoding_type = maybe_transfer_encoding.unpack();
if (transfer_encoding_type == "chunked") {
if (Singleton::exists<I_EnvDetails>()) {
I_EnvDetails *env_details = Singleton::Consume<I_EnvDetails>::by<HTTPDecoder>();
EnvType env_type = env_details->getEnvType();
if (env_type == EnvType::K8S) {
dbgDebug(D_COMMUNICATION) << "Getting Chunked Response in a k8s env";
return getChunkedResponseK8s();
}
}
return getChunkedResponse();
}
if (transfer_encoding_type == "chunked") return getChunkedResponse();
}
auto connection_header = unpacked_headers.getHeaderVal("connection");
@@ -157,91 +147,43 @@ HTTPDecoder::getChunkedResponse()
{
if(!isLegalChunkedResponse(response)) return false;
stringstream ss(response);
string line;
string res = response;
string chunk_body = "";
size_t chunk_length = 0;
while (getline(ss, line) && line != "\r") {
if (chunk_body.length() == chunk_length) {
body += chunk_body;
chunk_body = "";
try {
chunk_length = stoi(line, nullptr, 16);
} catch (const exception& err) {
dbgDebug(D_COMMUNICATION) << "Failed to convert chunk length to a number. Line: " << line;
return false;
}
} else if (chunk_body.length() > chunk_length) {
dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure.";
string CRLF = "\r\n";
size_t chunk_size = 0;
for (auto end_of_line = res.find(CRLF); end_of_line != string::npos; end_of_line = res.find(CRLF)) {
line = res.substr(0, end_of_line);
try {
chunk_size = stoi(line, nullptr, 16);
} catch (const exception& err) {
dbgDebug(D_COMMUNICATION) << "Failed to convert chunk length to a number. Line: " << line;
return false;
} else {
if (line.back() == '\r') {
line.pop_back();
}
if (!chunk_body.empty()) {
chunk_body += '\n';
}
chunk_body += line;
}
if (end_of_line + 2 + chunk_size > res.length()) {
dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure - chunk-size is bigger than chunk-data";
return false;
}
chunk_body = res.substr(end_of_line + 2, chunk_size);
res = res.substr(end_of_line + 2 + chunk_size);
if (res.find(CRLF) != 0) {
dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure - chunk-data missing final CRLF sequence";
return false;
}
res = res.substr(2);
body += chunk_body;
}
if (chunk_length != 0) {
dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure.";
if (chunk_size != 0) {
dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure - last-chunk of the body is not sized 0";
return false;
}
return true;
}
// LCOV_EXCL_START Reason: Will be deleted in INXT-31454
bool
HTTPDecoder::getChunkedResponseK8s()
{
if(!isLegalChunkedResponse(response)) return false;
stringstream ss(response);
string line;
string chunk_body = "";
size_t chunk_length = 0;
while (getline(ss, line)) {
if(line == "\r"){
continue;
}
if (chunk_body.length() == chunk_length) {
body += chunk_body;
chunk_body = "";
try {
chunk_length = stoi(line, nullptr, 16);
} catch (const exception& err) {
dbgDebug(D_COMMUNICATION) << "Failed to convert chunk length to a number. Line: " << line;
return false;
}
} else if (chunk_body.length() > chunk_length) {
dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure.";
return false;
} else {
if (line.back() == '\r') {
line.pop_back();
}
if (!chunk_body.empty()) {
chunk_body += '\n';
chunk_length = chunk_length + 1;
}
chunk_body += line;
}
}
if (chunk_length != 0) {
dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure.";
if (chunk_body.length() == chunk_length) {
body += chunk_body;
return true;
}
return false;
}
return true;
}
// LCOV_EXCL_STOP
bool
HTTPDecoder::isLegalChunkedResponse(const string &res)
{

1
core/message/http_decoder.h
View File

@@ -39,7 +39,6 @@ private:
bool handleBody();
bool getChunkedResponse();
bool getChunkedResponseK8s();
bool isLegalChunkedResponse(const std::string &res);
I_Messaging::Method method;

25
core/message/message.cc
View File

@@ -513,17 +513,22 @@ public:
http_status_code == HTTPStatusCode::HTTP_BAD_REQUEST;
};
pending_signatures.insert(req_sig);
auto res = sendMessage(get_reply, body, method, url, headers, fog_server_err, should_yield, tag);
pending_signatures.erase(req_sig);
if (res.ok()) return res;
try {
auto res = sendMessage(get_reply, body, method, url, headers, fog_server_err, should_yield, tag);
pending_signatures.erase(req_sig);
if (res.ok()) return res;
bool should_buffer_default = getProfileAgentSettingWithDefault<bool>(
true,
"eventBuffer.bufferFailedRequests"
);
if (!getConfigurationWithDefault<bool>(should_buffer_default, "message", "Buffer Failed Requests")) {
dbgWarning(D_COMMUNICATION) << "Failed to send Request.";
return res;
bool should_buffer_default = getProfileAgentSettingWithDefault<bool>(
true,
"eventBuffer.bufferFailedRequests"
);
if (!getConfigurationWithDefault<bool>(should_buffer_default, "message", "Buffer Failed Requests")) {
dbgWarning(D_COMMUNICATION) << "Failed to send Request.";
return res;
}
} catch (...) {
dbgWarning(D_COMMUNICATION) << "Can't send a persistent message, mainloop has been stopped";
return genError("mainloop has been stopped");
}
dbgWarning(D_COMMUNICATION) << "Failed to send Request. Buffering the request.";
}

15
core/report/report.cc
View File

@@ -149,8 +149,8 @@ Report::getSyslog() const
}
time_stamp += "Z";
string origin_syslog = origin.getSyslog();
string event_data_syslog = event_data.getSyslog();
string origin_syslog = origin.getSyslogAndCef();
string event_data_syslog = event_data.getSyslogAndCef();
string agent_id = "cpnano-agent-" + Singleton::Consume<I_AgentDetails>::by<Report>()->getAgentId();
auto service_name = Singleton::Consume<I_Environment>::by<Report>()->get<string>("Service Name");
@@ -189,6 +189,12 @@ Report::getCef() const
CefReport report;
auto service_name = Singleton::Consume<I_Environment>::by<Report>()->get<string>("Service Name");
auto i_time = Singleton::Consume<I_TimeGet>::by<Report>();
string time_stamp = i_time->getWalltimeStr(time);
if (time_stamp.size() > 7 && time_stamp[time_stamp.size() - 7] == '.') {
time_stamp.erase(time_stamp.size() - 3); // downscale micro-sec resollution to milli-sec
}
if (service_name.ok()) {
string tmp = service_name.unpack();
tmp.erase(remove(tmp.begin(), tmp.end(), ' '), tmp.end());
@@ -205,9 +211,10 @@ Report::getCef() const
report.pushMandatory(title);
report.pushMandatory(TagAndEnumManagement::convertToString(priority));
string origin_cef = origin.getCef();
string event_data_cef = event_data.getCef();
string origin_cef = origin.getSyslogAndCef();
string event_data_cef = event_data.getSyslogAndCef();
report.pushExtension("eventTime=" + time_stamp);
if (!origin_cef.empty()) {
report.pushExtension(origin_cef);
}

23
core/report/report_ut/report_ut.cc
View File

@@ -570,8 +570,8 @@ TEST_F(ReportTest, testSyslogWithoutServiceName)
EXPECT_EQ(
report.getSyslog(),
"<133>1 0:0:0.123Z cpnano-agent-001 UnnamedNanoService - 0 - "
"title='Log Test' agent='Secret' eventTraceId='' eventSpanId='' "
"issuingEngineVersion='' serviceName='Unnamed Nano Service' serviceId='' serviceFamilyId=''"
"title='Log Test' agent=\"Secret\" eventTraceId=\"\" eventSpanId=\"\" "
"issuingEngineVersion=\"\" serviceName=\"Unnamed Nano Service\" serviceId=\"\" serviceFamilyId=\"\""
);
}
@@ -604,13 +604,17 @@ TEST_F(ReportTest, testSyslog)
vector<vector<string>> f1 = { { "a", "b"}, {"1", "2"} };
report << LogField("ArrayOfArraies", f1);
report << LogField("DataWithNewLine", "new\r\nline");
report << LogField("DataWithQuote", "data'bla");
string result =
string("<133>1 0:0:0.123Z cpnano-agent-001 AccessControlApp - 1 - "
"title='Log Test' agent='Secret'") +
" eventTraceId='' eventSpanId='' issuingEngineVersion=''" +
" serviceName='Access Control App' serviceId='' serviceFamilyId=''" +
string(" ArrayOfArraies='[ [ a, b ], [ 1, 2 ] ]'");
"title='Log Test' agent=\"Secret\"") +
" eventTraceId=\"\" eventSpanId=\"\" issuingEngineVersion=\"\"" +
" serviceName=\"Access Control App\" serviceId=\"\" serviceFamilyId=\"\"" +
string(" ArrayOfArraies=\"[ [ a, b \\], [ 1, 2 \\] \\]\"") +
string(" DataWithNewLine=\"new\\r\\nline\"") +
string(" DataWithQuote=\"data\\'bla\"");
EXPECT_EQ(report.getSyslog(), result);
}
@@ -643,11 +647,14 @@ TEST_F(ReportTest, testCef)
);
report.addToOrigin(another_origin);
report << LogField("DataWithQuote", "data'bla");
EXPECT_EQ(
report.getCef(),
"CEF:0|Check Point|AccessControlApp||Event Driven|Log Test|Low|"
"agent=Secret eventTraceId= eventSpanId= issuingEngineVersion="
" serviceName=Access Control App serviceId= serviceFamilyId= Bond=1"
"eventTime=0:0:0.123 agent=\"Secret\" eventTraceId=\"\" eventSpanId=\"\" issuingEngineVersion=\"\""
" serviceName=\"Access Control App\" serviceId=\"\""
" serviceFamilyId=\"\" Bond=\"1\" DataWithQuote=\"data\\'bla\""
);
}

6
core/report/tag_and_enum_management.cc
View File

@@ -103,7 +103,8 @@ TagAndEnumManagement::convertStringToTag(const string &tag)
{"NGINX Server", ReportIS::Tags::WEB_SERVER_NGINX},
{"Kong Server", ReportIS::Tags::WEB_SERVER_KONG},
{"Embedded Deployment", ReportIS::Tags::DEPLOYMENT_EMBEDDED},
{"Kubernetes Deployment", ReportIS::Tags::DEPLOYMENT_K8S}
{"Kubernetes Deployment", ReportIS::Tags::DEPLOYMENT_K8S},
{"Layer 7 Access Control", ReportIS::Tags::LAYER_7_ACCESS_CONTROL}
};
auto report_is_tag = strings_to_tags.find(tag);
@@ -300,7 +301,8 @@ EnumArray<Tags, string> TagAndEnumManagement::tags_translation_arr {
"NGINX Server",
"Kong Server",
"Embedded Deployment",
"Kubernetes Deployment"
"Kubernetes Deployment",
"Layer 7 Access Control"
};
EnumArray<AudienceTeam, string> TagAndEnumManagement::audience_team_translation {