github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-06-28 16:41:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Jun 16th update

Browse Source
This commit is contained in:
noam 2023-01-17 11:34:09 +02:00
parent 90bcc544a2
commit ad04b8d063
168 changed files with 64034 additions and 932 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
attachments/kernel_modules/core/include/common_is/kdebug_flags.h
View File

@ -30,5 +30,6 @@ DEFINE_KDEBUG_FLAG(assetResolver)
DEFINE_KDEBUG_FLAG(statefulValidation)
DEFINE_KDEBUG_FLAG(statelessValidation)
DEFINE_KDEBUG_FLAG(kernelMetric)
DEFINE_KDEBUG_FLAG(tproxy)
#endif // DEFINE_KDEBUG_FLAG

7
components/attachment-intakers/attachment_registrator/attachment_registrator.cc
View File

@ -154,6 +154,10 @@ private:
stringstream handler_path;
handler_path << handler_path_format;
switch(type) {
case (AttachmentType::SQUID_ATT_ID): {
handler_path << "squid-http-transaction-handler-";
break;
}
case (AttachmentType::NGINX_ATT_ID): {
handler_path << "http-transaction-handler-";
break;
@ -177,7 +181,8 @@ private:
stringstream registration_command;
registration_command<< registration_format;
switch(type) {
case (AttachmentType::NGINX_ATT_ID): {
case (AttachmentType::SQUID_ATT_ID):
case (AttachmentType::NGINX_ATT_ID):{
registration_command << "/etc/cp/HttpTransactionHandler/cp-nano-http-transaction-handler";
break;
}

2
components/attachment-intakers/nginx_attachment/cidrs_data.cc
View File

@ -54,7 +54,7 @@ CIDRSData::CIDRSData(const string &str_cidr)
string str_prefix = pos != string::npos ? str_cidr.substr(0, pos) : str_cidr;
// get subnet mask from targetCidr or calculate it based on ipv4 / ipv6
string str_suffix;
if (pos != string::npos) {
if (pos != string::npos && (pos + 1) <= str_cidr.size()) {
str_suffix = str_cidr.substr(pos + 1);
} else if (str_cidr.find(':') == string::npos) {
str_suffix = "32";

5
components/attachment-intakers/nginx_attachment/nginx_attachment.cc
View File

@ -44,7 +44,6 @@
#include "buffer.h"
#include "enum_array.h"
#include "shmem_ipc.h"
#include "sasal.h"
#include "i_http_manager.h"
#include "http_transaction_common.h"
#include "nginx_attachment_common.h"
@ -68,8 +67,6 @@
#endif // FAILURE_TEST
SASAL_START // HTTP Manager main
USE_DEBUG_FLAG(D_NGINX_ATTACHMENT);
USE_DEBUG_FLAG(D_COMPRESSION);
USE_DEBUG_FLAG(D_METRICS_NGINX_ATTACHMENT);
@ -1781,5 +1778,3 @@ NginxAttachment::preload()
BasicRuleConfig::preload();
WebTriggerConf::preload();
}
SASAL_END

18
components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc
View File

@ -18,11 +18,8 @@
#include "boost/uuid/uuid_io.hpp"
#include "config.h"
#include "sasal.h"
#include "virtual_modifiers.h"
SASAL_START // HTTP Manager - Transaction data
using namespace std;
using namespace boost::uuids;
@ -34,6 +31,7 @@ NginxAttachmentOpaque::NginxAttachmentOpaque(HttpTransactionData _transaction_da
transaction_data(move(_transaction_data)),
ctx(),
session_tenant(),
session_profile(),
uuid()
{
try {
@ -65,10 +63,10 @@ NginxAttachmentOpaque::NginxAttachmentOpaque(HttpTransactionData _transaction_da
auto decoder = makeVirtualContainer<HexDecoder<'%'>>(transaction_data.getURI());
string decoded_url(decoder.begin(), decoder.end());
auto question_mark_location = decoded_url.find('?');
if (question_mark_location != string::npos) {
if (question_mark_location != string::npos && (question_mark_location + 1) <= decoded_url.size()) {
ctx.registerValue(HttpTransactionData::uri_query_decoded, decoded_url.substr(question_mark_location + 1));
}
ctx.registerValue(HttpTransactionData::uri_path_decoded, decoded_url.substr(0, question_mark_location));
ctx.registerValue(HttpTransactionData::uri_path_decoded, decoded_url);
}
NginxAttachmentOpaque::~NginxAttachmentOpaque()
@ -85,10 +83,14 @@ NginxAttachmentOpaque::prototype()
// LCOV_EXCL_STOP
void
NginxAttachmentOpaque::setSessionTenant(const string &tenant)
NginxAttachmentOpaque::setSessionTenantAndProfile(const string &tenant, const string &profile)
{
session_tenant = tenant;
Singleton::Consume<I_Environment>::by<NginxAttachmentOpaque>()->setActiveTenant(session_tenant);
session_profile = profile;
Singleton::Consume<I_Environment>::by<NginxAttachmentOpaque>()->setActiveTenantAndProfile(
session_tenant,
session_profile
);
}
void
@ -117,5 +119,3 @@ NginxAttachmentOpaque::setSavedData(const string &name, const string &data, EnvK
saved_data[name] = data;
ctx.registerValue(name, data, log_ctx);
}
SASAL_END

10
components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.h
View File

@ -38,7 +38,10 @@ public:
ctx.activate();
gen_ctx.activate();
if (session_tenant != "") {
Singleton::Consume<I_Environment>::by<NginxAttachmentOpaque>()->setActiveTenant(session_tenant);
Singleton::Consume<I_Environment>::by<NginxAttachmentOpaque>()->setActiveTenantAndProfile(
session_tenant,
session_profile
);
}
}
@ -47,7 +50,7 @@ public:
deactivateContext()
{
if (session_tenant != "") {
Singleton::Consume<I_Environment>::by<NginxAttachmentOpaque>()->unsetActiveTenant();
Singleton::Consume<I_Environment>::by<NginxAttachmentOpaque>()->unsetActiveTenantAndProfile();
}
gen_ctx.deactivate();
ctx.deactivate();
@ -66,7 +69,7 @@ public:
static uint minVer() { return 0; }
const std::string & getSessionTenant() const { return session_tenant; }
void setSessionTenant(const std::string &tenant);
void setSessionTenantAndProfile(const std::string &tenant, const std::string &profile);
void setSourceIdentifier(const std::string &header_key, const std::string &source_identifier);
const std::string & getSourceIdentifiersType() const;
@ -85,6 +88,7 @@ private:
GenericRulebaseContext gen_ctx;
Context ctx;
std::string session_tenant;
std::string session_profile;
std::string uuid;
std::string source_identifier;
std::string identifier_type;

27
components/attachment-intakers/nginx_attachment/nginx_parser.cc
View File

@ -153,6 +153,29 @@ genHeaders(const Buffer &raw_data)
return headers;
}
static vector<string>
getActivetenantAndProfile(const string &str, const string &deli = ",")
{
vector<string> elems;
elems.reserve(2);
int start = 0;
int end = str.find(deli);
while (end != -1) {
elems.push_back(str.substr(start, end - start));
start = end + deli.size();
end = str.find(deli, start);
}
elems.push_back(str.substr(start, end - start));
if (elems.size() == 1) {
elems.push_back("");
}
return elems;
}
Maybe<vector<HttpHeader>>
NginxParser::parseRequestHeaders(const Buffer &data)
{
@ -182,8 +205,8 @@ NginxParser::parseRequestHeaders(const Buffer &data)
<< ", Value: "
<< dumpHex(header.getValue());
string active_tenant(static_cast<string>(header.getValue()));
opaque.setSessionTenant(active_tenant);
auto active_tenant_and_profile = getActivetenantAndProfile(header.getValue());
opaque.setSessionTenantAndProfile(active_tenant_and_profile[0], active_tenant_and_profile[1]);
} else if (proxy_ip_header_key == header.getKey()) {
source_identifiers.setXFFValuesToOpaqueCtx(header, UsersAllIdentifiersConfig::ExtractType::PROXYIP);
}

5
components/generic_rulebase/evaluators/http_transaction_data_eval.cc
View File

@ -20,9 +20,6 @@
#include "environment/evaluator_templates.h"
#include "i_environment.h"
#include "singleton.h"
#include "sasal.h"
SASAL_START // HTTP Manager - Transaction data
using namespace std;
using namespace EnvironmentHelper;
@ -121,5 +118,3 @@ BeginWithUri::evalVariable() const
return lower_uri_ctx.find(lower_uri_prefix) == 0;
}
SASAL_END

7
components/http_manager/http_manager.cc
View File

@ -27,11 +27,8 @@
#include "table_opaque.h"
#include "http_manager_opaque.h"
#include "log_generator.h"
#include "sasal.h"
#include "http_inspection_events.h"
SASAL_START // HTTP Manager
USE_DEBUG_FLAG(D_HTTP_MANAGER);
using namespace std;
@ -208,7 +205,7 @@ public:
LogGen(
"Web AppSec Policy Loaded Successfully",
ReportIS::Audience::SECURITY,
ReportIS::Severity::LOW,
ReportIS::Severity::INFO,
ReportIS::Priority::LOW,
ReportIS::Tags::THREAT_PREVENTION
);
@ -375,5 +372,3 @@ HttpManager::preload()
registerExpectedConfiguration<string>("HTTP manager", "Response Size Limit Verdict");
registerConfigLoadCb([this] () { pimpl->sendPolicyLog(); });
}
SASAL_END

5
components/http_manager/http_manager_opaque.cc
View File

@ -14,9 +14,6 @@
#include "http_manager_opaque.h"
#include "config.h"
#include "sasal.h"
SASAL_START // HTTP Manager - Transaction data
using namespace std;
@ -99,5 +96,3 @@ HttpManagerOpaque::updatePayloadSize(const uint curr_payload_size)
{
aggregated_payload_size += curr_payload_size;
}
SASAL_END

5
components/http_transaction_data/http_transaction_data.cc
View File

@ -18,12 +18,9 @@
#include <boost/algorithm/string.hpp>
#include "enum_array.h"
#include "sasal.h"
#include "buffer.h"
#include "nginx_attachment_common.h"
SASAL_START // HTTP Manager - Transaction data
using namespace std;
USE_DEBUG_FLAG(D_NGINX_ATTACHMENT);
@ -261,5 +258,3 @@ HttpTransactionData::print(ostream &out_stream) const
<< ")"
<< endl;
}
SASAL_END

2
components/include/i_downloader.h
View File

@ -28,7 +28,7 @@ public:
const GetResourceFile &resourse_file
) const = 0;
virtual Maybe<std::map<std::string, std::string>>downloadVirtualFileFromFog(
virtual Maybe<std::map<std::pair<std::string, std::string>, std::string>>downloadVirtualFileFromFog(
const GetResourceFile &resourse_file,
Package::ChecksumTypes checksum_type
) const = 0;

1
components/include/i_local_policy_mgmt_gen.h
View File

@ -19,7 +19,6 @@ class I_LocalPolicyMgmtGen
public:
enum class LocalPolicyEnv { LINUX, K8S, COUNT };
virtual std::string parsePolicy(const std::string &policy_version) = 0;
virtual const std::string & getPolicyPath(void) const = 0;
virtual LocalPolicyEnv getEnvType() const = 0;

3
components/include/i_orchestration_tools.h
View File

@ -102,7 +102,8 @@ public:
virtual Maybe<std::map<packageName, packageDetails>> jsonObjectSplitter(
const std::string &json,
const std::string &tenant_id = "") const = 0;
const std::string &tenant_id = "",
const std::string &profile_id = "") const = 0;
virtual bool isNonEmptyFile(const std::string &path) const = 0;
virtual Maybe<std::string> readFile(const std::string &path) const = 0;

3
components/include/i_service_controller.h
View File

@ -41,7 +41,8 @@ public:
const std::string &new_policy_path,
const std::string &new_settings_path,
const std::vector<std::string> &new_data_files = {},
const std::string &tenant_id = ""
const std::string &tenant_id = "",
const std::string &profile_id = ""
) = 0;
virtual bool isServiceInstalled(const std::string &service_name) = 0;

25
components/include/orchestrator/rest_api/get_resource_file.h
View File

@ -24,24 +24,39 @@ class GetResourceFile : public ClientRest
class TenantResource : public ClientRest
{
public:
TenantResource(const std::string &_tenant_id, const std::string &_version, const std::string &_checksum)
TenantResource(
const std::string &_tenant_id,
const std::string &_profile_id,
const std::string &_version,
const std::string &_checksum)
:
tenant_id(_tenant_id),
profile_id(_profile_id),
version(_version),
checksum(_checksum)
{
}
TenantResource(const TenantResource &other)
{
tenant_id = other.tenant_id;
profile_id = other.profile_id;
version = other.version;
checksum = other.checksum;
}
bool
operator==(const TenantResource &other) const
{
return
tenant_id.get() == other.tenant_id.get() &&
profile_id.get() == other.profile_id.get() &&
version.get() == other.version.get() &&
checksum.get() == other.checksum.get();
}
C2S_LABEL_PARAM(std::string, tenant_id, "tenantId");
C2S_LABEL_PARAM(std::string, profile_id, "profileId");
C2S_LABEL_PARAM(std::string, version, "version");
C2S_LABEL_PARAM(std::string, checksum, "checksum");
};
@ -76,12 +91,16 @@ public:
}
void
addTenant(const std::string &tenant_id, const std::string &version, const std::string &checksum)
addTenant(
const std::string &tenant_id,
const std::string &profile_id,
const std::string &version,
const std::string &checksum)
{
if (!isVirtual()) return;
if (!tenants.isActive()) tenants = std::vector<TenantResource>();
tenants.get().emplace_back(tenant_id, version, checksum);
tenants.get().emplace_back(tenant_id, profile_id, version, checksum);
}
std::string

80
components/include/orchestrator/rest_api/orchestration_check_update.h
View File

@ -17,8 +17,40 @@
#include <string>
#include "rest.h"
#include "debug.h"
#include "maybe_res.h"
USE_DEBUG_FLAG(D_ORCHESTRATOR);
class TenantError : public ClientRest
{
public:
TenantError() = default;
bool
operator==(const TenantError &other) const
{
return
messageId.get() == other.messageId.get() &&
message.get() == other.message.get() &&
referenceId.get() == other.referenceId.get() &&
severity.get() == other.severity.get();
}
const std::string & getMessageID() const { return messageId.get(); }
const std::string & getMessage() const { return message.get(); }
// LCOV_EXCL_START Reason: Will be covered in INXT-33277
const std::string & getReferenceID() const { return referenceId.get(); }
// LCOV_EXCL_STOP
const std::string & getSeverity() const { return severity.get(); }
private:
BOTH_LABEL_PARAM(std::string, messageId, "messageId");
BOTH_LABEL_PARAM(std::string, message, "message");
BOTH_LABEL_PARAM(std::string, referenceId, "referenceId");
BOTH_LABEL_PARAM(std::string, severity, "severity");
};
class CheckUpdateRequest : public ClientRest
{
public:
@ -30,13 +62,20 @@ public:
Tenants(const Tenants &other)
{
tenant_id = other.tenant_id;
profile_id = other.profile_id;
checksum = other.checksum;
version = other.version;
error = other.error;
}
Tenants(const std::string &_tenant_id, const std::string &_checksum, const std::string &_version)
Tenants(
const std::string &_tenant_id,
const std::string &_profile_id,
const std::string &_checksum,
const std::string &_version)
:
tenant_id(_tenant_id),
profile_id(_profile_id),
checksum(_checksum),
version(_version)
{
@ -47,18 +86,24 @@ public:
{
return
tenant_id.get() == other.tenant_id.get() &&
profile_id.get() == other.profile_id.get() &&
checksum.get() == other.checksum.get() &&
version.get() == other.version.get();
version.get() == other.version.get() &&
error.get() == other.error.get();
}
const std::string & getTenantID() const { return tenant_id.get(); }
const std::string & getChecksum() const { return checksum.get(); }
const std::string & getVersion() const { return version.get(); }
const std::string & getProfileID() const { return profile_id.get(); }
const std::string & getChecksum() const { return checksum.get(); }
const std::string & getVersion() const { return version.get(); }
const TenantError & getError() const { return error.get(); }
private:
BOTH_LABEL_PARAM(std::string, tenant_id, "tenantId");
BOTH_LABEL_PARAM(std::string, checksum, "checksum");
BOTH_LABEL_PARAM(std::string, version, "version");
BOTH_LABEL_OPTIONAL_PARAM(std::string, tenant_id, "tenantId");
BOTH_LABEL_OPTIONAL_PARAM(std::string, profile_id, "profileId");
BOTH_LABEL_OPTIONAL_PARAM(std::string, checksum, "checksum");
BOTH_LABEL_OPTIONAL_PARAM(std::string, version, "version");
BOTH_LABEL_OPTIONAL_PARAM(TenantError, error, "error");
};
CheckUpdateRequest(
@ -157,7 +202,26 @@ private:
tenants.get().emplace_back(std::forward<Args>(args)...);
}
const std::vector<Tenants> & getTenants() const { return tenants.get(); }
const std::vector<Tenants>
getTenants() const
{
std::vector<Tenants> tenants_to_return;
for (const auto &tenant : tenants.get()) {
if (tenant.getError().getMessage().empty()) {
tenants_to_return.push_back(tenant);
continue;
}
dbgError(D_ORCHESTRATOR)
<< "Error getting the tenant information. Tenant ID: "
<< tenant.getTenantID()
<< ", Error message: "
<< tenant.getError().getMessage()
<< ", Reference ID: "
<< tenant.getError().getReferenceID();
}
return tenants_to_return;
}
private:
BOTH_LABEL_PARAM(std::vector<Tenants>, tenants, "tenants");

1
components/include/waap.h
View File

@ -29,6 +29,7 @@
class I_Telemetry;
class I_DeepAnalyzer;
class I_WaapAssetStatesManager;
class I_Messaging;
class I_AgentDetails;
class I_Encryptor;

84
components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h
View File

@ -17,7 +17,51 @@
#include <algorithm>
#include <boost/regex.hpp>
#if defined(gaia)
Maybe<string>
checkHasSupportedBlade(const string &command_output)
{
string supportedBlades[3] = {"identityServer", "vpn", "cvpn"};
for(const string &blade : supportedBlades) {
if (command_output.find(blade) != string::npos) {
return string("true");
}
}
return genError("Current host does not have IDA capability");
}
Maybe<string>
checkSamlPortal(const string &command_output)
{
if (command_output.find("Portal is running") != string::npos) {
return string("true");
}
return genError("Current host does not have SAML Portal configured");
}
Maybe<string>
checkIDP(shared_ptr<istream> file_stream)
{
string line;
while (getline(*file_stream, line)) {
if (line.find("<identity_portal/>") != string::npos) {
return string("false");
}
if (line.find("identity_provider") != string::npos) {
return string("true");
}
}
return genError("Identity Provider was not found");
}
#endif // gaia
#if defined(gaia) || defined(smb)
Maybe<string>
checkHasSDWan(const string &command_output)
{
@ -72,6 +116,23 @@ getMgmtObjName(shared_ptr<istream> file_stream)
return getMgmtObjAttr(file_stream, "name ");
}
Maybe<string>
getSmbObjectName(const string &command_output)
{
static const char centrally_managed_comd_output = '0';
if (command_output.empty() || command_output[0] != centrally_managed_comd_output) {
return genError("Object name was not found");
}
static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C";
auto ifs = std::make_shared<std::ifstream>(obj_path);
if (!ifs->is_open()) {
return genError("Failed to open the object file");
}
return getMgmtObjAttr(ifs, "name ");
}
Maybe<string>
getMgmtParentObjAttr(shared_ptr<istream> file_stream, const string &parent_obj, const string &attr)
{
@ -89,7 +150,9 @@ getMgmtParentObjAttr(shared_ptr<istream> file_stream, const string &parent_obj,
}
return genError("Parent object attribute was not found. Attr: " + attr);
}
#endif // gaia || smb
#if defined(gaia)
Maybe<string>
getMgmtParentObjUid(shared_ptr<istream> file_stream)
{
@ -117,7 +180,26 @@ getMgmtParentObjName(shared_ptr<istream> file_stream)
const string &unparsed_name = maybe_unparsed_name.unpack();
return chopHeadAndTail(unparsed_name, "(", ")");
}
#endif // gaia || smb
#elif defined(smb)
Maybe<string>
getMgmtParentObjUid(const string &command_output)
{
if (!command_output.empty()) {
return command_output;
}
return genError("Parent object uuid was not found.");
}
Maybe<string>
getMgmtParentObjName(const string &command_output)
{
if (!command_output.empty()) {
return command_output;
}
return genError("Parent object name was not found.");
}
#endif // end if gaia/smb
Maybe<string>
getOsRelease(shared_ptr<istream> file_stream)

59
components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
View File

@ -19,12 +19,33 @@
// to return a string value for an attribute key based on a logic executed in a handler that receives
// shell command execution output as its input
#ifdef SHELL_CMD_HANDLER
#if defined(gaia) || defined(smb)
SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType)
SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
#endif //gaia || smb
#if defined(gaia)
SHELL_CMD_HANDLER("hasSupportedBlade", "enabled_blades", checkHasSupportedBlade)
SHELL_CMD_HANDLER("hasSamlPortal", "mpclient status saml-vpn", checkSamlPortal)
#endif //gaia
#if defined(smb)
SHELL_CMD_HANDLER(
"cpProductIntegrationMgmtParentObjectName",
"cpsdwan get_data | jq -r .cluster_name",
getMgmtParentObjName
)
SHELL_CMD_HANDLER(
"cpProductIntegrationMgmtParentObjectUid",
"cpsdwan get_data | jq -r .cluster_uuid",
getMgmtParentObjUid
)
SHELL_CMD_HANDLER(
"cpProductIntegrationMgmtObjectName",
"cpprod_util FwIsLocalMgmt",
getSmbObjectName
)
#endif//smb
#endif // SHELL_CMD_HANDLER
@ -40,21 +61,10 @@ SHELL_CMD_OUTPUT("helloWorld", "cat /tmp/agentHelloWorld 2>/dev/null")
// to return a string value for an attribute key based on a logic executed in a handler that receives file as input
#ifdef FILE_CONTENT_HANDLER
#if defined(alpine)
FILE_CONTENT_HANDLER("alpine_tag", "/usr/share/build/cp-alpine-tag", getCPAlpineTag)
#endif // alpine
#if defined(gaia) || defined(smb)
FILE_CONTENT_HANDLER("os_release", "/etc/cp-release", getOsRelease)
FILE_CONTENT_HANDLER(
"cpProductIntegrationMgmtObjectUid",
(getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C",
getMgmtObjUid
)
FILE_CONTENT_HANDLER(
"cpProductIntegrationMgmtObjectName",
(getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C",
getMgmtObjName
)
#if defined(gaia)
FILE_CONTENT_HANDLER("hasIdpConfigured", "/opt/CPSamlPortal/phpincs/spPortal/idpPolicy.xml", checkIDP)
FILE_CONTENT_HANDLER(
"cpProductIntegrationMgmtParentObjectUid",
(getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myself_objects.C",
@ -65,6 +75,23 @@ FILE_CONTENT_HANDLER(
(getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myself_objects.C",
getMgmtParentObjName
)
FILE_CONTENT_HANDLER(
"cpProductIntegrationMgmtObjectName",
(getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C",
getMgmtObjName
)
#endif //gaia
#if defined(alpine)
FILE_CONTENT_HANDLER("alpine_tag", "/usr/share/build/cp-alpine-tag", getCPAlpineTag)
#endif // alpine
#if defined(gaia) || defined(smb)
FILE_CONTENT_HANDLER("os_release", "/etc/cp-release", getOsRelease)
FILE_CONTENT_HANDLER(
"cpProductIntegrationMgmtObjectUid",
(getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C",
getMgmtObjUid
)
#else // !(gaia || smb)
FILE_CONTENT_HANDLER("os_release", "/etc/os-release", getOsRelease)
#endif // gaia || smb

2
components/security_apps/orchestration/details_resolver/details_resolving_handler.cc
View File

@ -92,7 +92,7 @@ DetailsResolvingHanlder::Impl::getResolvedDetails() const
}
I_AgentDetailsReporter *reporter = Singleton::Consume<I_AgentDetailsReporter>::by<DetailsResolvingHanlder>();
reporter->addAttr(resolved_details);
reporter->addAttr(resolved_details, true);
return resolved_details;
}

4
components/security_apps/orchestration/downloader/curl_client.cc
View File

@ -33,14 +33,12 @@
#include "config.h"
#include "url_parser.h"
#include "debug.h"
#include "sasal.h"
#include "scope_exit.h"
USE_DEBUG_FLAG(D_HTTP_REQUEST);
using namespace std;
SASAL_START // Orchestration - Communication
// LCOV_EXCL_START Reason: Depends on real download server.
class CurlGlobalInit
@ -434,5 +432,3 @@ TraceIdGenerator::generateTraceId()
string part5 = generateRandomString(12);
return string(part1 + "-" + part2 + "-" + part3 + "-" + part4 + "-" + part5);
}
SASAL_END

4
components/security_apps/orchestration/downloader/curl_client.h
View File

@ -28,11 +28,9 @@
#include "i_encryptor.h"
#include "scope_exit.h"
#include "url_parser.h"
#include "sasal.h"
USE_DEBUG_FLAG(D_HTTP_REQUEST);
SASAL_START // Orchestration - Communication
// LCOV_EXCL_START Reason: Depends on real download server.
enum class HTTP_VERSION
@ -111,5 +109,3 @@ public:
private:
std::string ca_path;
};
SASAL_END

33
components/security_apps/orchestration/downloader/downloader.cc
View File

@ -19,7 +19,6 @@
#include "debug.h"
#include "config.h"
#include "rest.h"
#include "sasal.h"
#include "cereal/external/rapidjson/document.h"
#include <fstream>
@ -27,8 +26,6 @@
using namespace std;
using namespace rapidjson;
SASAL_START // Orchestration - Communication
USE_DEBUG_FLAG(D_ORCHESTRATOR);
class Downloader::Impl : Singleton::Provide<I_Downloader>::From<Downloader>
@ -42,7 +39,7 @@ public:
const GetResourceFile &resourse_file
) const override;
Maybe<map<string, string>> downloadVirtualFileFromFog(
Maybe<map<pair<string, string>, string>> downloadVirtualFileFromFog(
const GetResourceFile &resourse_file,
Package::ChecksumTypes checksum_type
) const override;
@ -114,18 +111,19 @@ Downloader::Impl::downloadFileFromFog(
return file_path;
}
Maybe<map<string, string>>
Maybe<map<pair<string, string>, string>>
Downloader::Impl::downloadVirtualFileFromFog(
const GetResourceFile &resourse_file,
Package::ChecksumTypes) const
{
static const string tenand_id_key = "tenantId";
static const string policy_key = "policy";
static const string settings_key = "settings";
static const string tenants_key = "tenants";
static const string error_text = "error";
static const string tenand_id_key = "tenantId";
static const string profile_id_key = "profileId";
static const string policy_key = "policy";
static const string settings_key = "settings";
static const string tenants_key = "tenants";
static const string error_text = "error";
map<string, string> res;
map<pair<string, string>, string> res;
I_UpdateCommunication *update_communication = Singleton::Consume<I_UpdateCommunication>::by<Downloader>();
auto downloaded_data = update_communication->downloadAttributeFile(resourse_file);
if (!downloaded_data.ok()) return downloaded_data.passErr();
@ -146,7 +144,14 @@ Downloader::Impl::downloadVirtualFileFromFog(
if (artifact_data == itr->MemberEnd()) artifact_data = itr->FindMember(settings_key.c_str());
if (artifact_data != itr->MemberEnd()) {
string file_path = dir_path + "/" + resourse_file.getFileName() + "_" + tenant_id + ".download";
auto profile_id_obj = itr->FindMember(profile_id_key.c_str());
if (profile_id_obj == itr->MemberEnd()) continue;
string profile_id = profile_id_obj->value.GetString();
string file_path =
dir_path + "/" + resourse_file.getFileName() + "_" +
tenant_id + "_profile_" + profile_id + ".download";
rapidjson::StringBuffer buffer;
rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
@ -154,7 +159,7 @@ Downloader::Impl::downloadVirtualFileFromFog(
I_OrchestrationTools *orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<Downloader>();
if (orchestration_tools->writeFile(buffer.GetString(), file_path)) {
res.insert({tenant_id, file_path});
res.insert({{tenant_id, profile_id}, file_path});
}
continue;
}
@ -383,5 +388,3 @@ Downloader::preload()
registerExpectedConfiguration<string>("orchestration", "Self signed certificates acceptable");
registerExpectedConfiguration<bool>("orchestration", "Add tenant suffix");
}
SASAL_END

42
components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc
View File

@ -295,8 +295,8 @@ TEST_F(DownloaderTest, download_virtual_policy)
{
GetResourceFile resourse_file(GetResourceFile::ResourceFileType::VIRTUAL_POLICY);
resourse_file.addTenant("0000", "1", "checksum0000");
resourse_file.addTenant("1111", "2", "checksum1111");
resourse_file.addTenant("0000", "1234", "1", "checksum0000");
resourse_file.addTenant("1111", "1235", "2", "checksum1111");
string tenant_0000_file =
"{"
@ -319,6 +319,7 @@ TEST_F(DownloaderTest, download_virtual_policy)
" \"tenants\": [\n"
" {\n"
" \"tenantId\": \"0000\",\n"
" \"profileId\": \"1234\",\n"
" \"policy\": {\n"
" \"waap\": \"108-005\",\n"
" \"accessControl\": \"Internal error, check logs\",\n"
@ -328,6 +329,7 @@ TEST_F(DownloaderTest, download_virtual_policy)
" },\n"
" {\n"
" \"tenantId\": \"1111\",\n"
" \"profileId\": \"1235\",\n"
" \"policy\": {\n"
" \"messageId\": \"108-005\",\n"
" \"message\": \"Internal error, check logs\",\n"
@ -340,16 +342,16 @@ TEST_F(DownloaderTest, download_virtual_policy)
EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file)).WillOnce(Return(fog_response));
EXPECT_CALL(mock_orchestration_tools, writeFile(tenant_0000_file, "/tmp/virtualPolicy_0000.download"))
EXPECT_CALL(mock_orchestration_tools, writeFile(tenant_0000_file, "/tmp/virtualPolicy_0000_profile_1234.download"))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, writeFile(tenant_1111_file, "/tmp/virtualPolicy_1111.download"))
EXPECT_CALL(mock_orchestration_tools, writeFile(tenant_1111_file, "/tmp/virtualPolicy_1111_profile_1235.download"))
.WillOnce(Return(true));
map<string, string> expected_downloaded_files =
map<pair<string, string>, string> expected_downloaded_files =
{
{ "0000", "/tmp/virtualPolicy_0000.download" },
{ "1111", "/tmp/virtualPolicy_1111.download" }
{ {"0000", "1234" }, "/tmp/virtualPolicy_0000_profile_1234.download" },
{ {"1111", "1235" }, "/tmp/virtualPolicy_1111_profile_1235.download" }
};
EXPECT_EQ(
@ -365,7 +367,12 @@ TEST_F(DownloaderTest, download_virtual_settings)
{
GetResourceFile resourse_file(GetResourceFile::ResourceFileType::VIRTUAL_SETTINGS);
resourse_file.addTenant("4c721b40-85df-4364-be3d-303a10ee9789", "1", "checksum0000");
resourse_file.addTenant(
"4c721b40-85df-4364-be3d-303a10ee9789",
"4c721b40-85df-4364-be3d-303a10ee9780",
"1",
"checksum0000"
);
string tenant_0000_file =
"{"
@ -389,6 +396,7 @@ TEST_F(DownloaderTest, download_virtual_settings)
" \"tenants\": [\n"
" {\n"
" \"tenantId\": \"4c721b40-85df-4364-be3d-303a10ee9789\",\n"
" \"profileId\": \"4c721b40-85df-4364-be3d-303a10ee9780\",\n"
" \"settings\": {\n"
" \"agentSettings\": [\n"
" {\n"
@ -410,14 +418,24 @@ TEST_F(DownloaderTest, download_virtual_settings)
EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file)).WillOnce(Return(fog_response));
stringstream tenant_0000_path;
tenant_0000_path << "/tmp/virtualSettings_4c721b40-85df-4364-be3d-303a10ee9789"
"_profile_4c721b40-85df-4364-be3d-303a10ee9780.download";
EXPECT_CALL(
mock_orchestration_tools,
writeFile(tenant_0000_file, "/tmp/virtualSettings_4c721b40-85df-4364-be3d-303a10ee9789.download")
writeFile(
tenant_0000_file,
tenant_0000_path.str()
)
).WillOnce(Return(true));
map<string, string> expected_downloaded_files = {
{ "4c721b40-85df-4364-be3d-303a10ee9789",
"/tmp/virtualSettings_4c721b40-85df-4364-be3d-303a10ee9789.download"
stringstream file_path;
file_path << "/tmp/virtualSettings_4c721b40-85df-4364-be3d-303a10ee9789"
"_profile_4c721b40-85df-4364-be3d-303a10ee9780.download";
map<pair<string, string>, string> expected_downloaded_files = {
{ {"4c721b40-85df-4364-be3d-303a10ee9789", "4c721b40-85df-4364-be3d-303a10ee9780"},
file_path.str()
}
};

5
components/security_apps/orchestration/downloader/http_client.cc
View File

@ -18,7 +18,6 @@
#include "debug.h"
#include "i_encryptor.h"
#include "url_parser.h"
#include "sasal.h"
#include "config.h"
#include "i_environment.h"
#include "orchestration_comp.h"
@ -32,8 +31,6 @@
using boost::asio::ip::tcp;
using namespace std;
SASAL_START // Orchestration - Communication
USE_DEBUG_FLAG(D_ORCHESTRATOR);
USE_DEBUG_FLAG(D_HTTP_REQUEST);
@ -272,5 +269,3 @@ HTTPClient::getFileHttp(const URLParser &url, ofstream &out_file, const string &
return Maybe<void>();
}
// LCOV_EXCL_STOP
SASAL_END

5
components/security_apps/orchestration/downloader/https_client.cc
View File

@ -19,7 +19,6 @@
#include "i_encryptor.h"
#include "downloader.h"
#include "config.h"
#include "sasal.h"
#include "boost/uuid/uuid.hpp"
#include "boost/uuid/uuid_generators.hpp"
#include <boost/asio/deadline_timer.hpp>
@ -39,8 +38,6 @@ using namespace boost::placeholders;
using boost::asio::ip::tcp;
using namespace std;
SASAL_START // Orchestration - Communication
USE_DEBUG_FLAG(D_COMMUNICATION);
USE_DEBUG_FLAG(D_HTTP_REQUEST);
USE_DEBUG_FLAG(D_ORCHESTRATOR);
@ -615,5 +612,3 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s
}
// LCOV_EXCL_STOP
SASAL_END

5
components/security_apps/orchestration/include/mock/mock_downloader.h
View File

@ -30,7 +30,10 @@ public:
MOCK_CONST_METHOD2(
downloadVirtualFileFromFog,
Maybe<std::map<std::string, std::string>>(const GetResourceFile &, Package::ChecksumTypes)
Maybe<std::map<std::pair<std::string, std::string>, std::string>>(
const GetResourceFile &,
Package::ChecksumTypes
)
);
MOCK_CONST_METHOD4(

4
components/security_apps/orchestration/include/mock/mock_orchestration_tools.h
View File

@ -44,9 +44,9 @@ public:
MOCK_CONST_METHOD1(removeFile, bool(const std::string &));
MOCK_CONST_METHOD2(copyFile, bool(const std::string &, const std::string &));
MOCK_CONST_METHOD2(calculateChecksum, Maybe<std::string>(Package::ChecksumTypes, const std::string &));
MOCK_CONST_METHOD2(
MOCK_CONST_METHOD3(
jsonObjectSplitter,
Maybe<std::map<std::string, std::string>>(const std::string &, const std::string &)
Maybe<std::map<std::string, std::string>>(const std::string &, const std::string &, const std::string &)
);
MOCK_CONST_METHOD1(doesFileExist, bool(const std::string &));
MOCK_CONST_METHOD1(createDirectory, bool(const std::string &));

5
components/security_apps/orchestration/include/mock/mock_service_controller.h
View File

@ -28,13 +28,14 @@ public:
MOCK_CONST_METHOD0(getUpdatePolicyVersion, const std::string &());
MOCK_METHOD4(
MOCK_METHOD5(
updateServiceConfiguration,
bool(
const std::string &new_policy_path,
const std::string &new_settings_path,
const std::vector<std::string> &new_data_files,
const std::string &tenant_id
const std::string &tenant_id,
const std::string &profile_id
)
);

41
components/security_apps/orchestration/local_policy_mgmt_gen/include/appsec_practice_section.h
View File

@ -14,10 +14,7 @@
#ifndef __APPSEC_PRACTICE_SECTION_H__
#define __APPSEC_PRACTICE_SECTION_H__
#include <list>
#include <cereal/archives/json.hpp>
#include <cereal/types/list.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>
@ -119,9 +116,9 @@ public:
load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_K8S_POLICY) << "Loading AppSec Web Attack Protections";
parseAppsecJSONKey<std::string>("csrf-enabled", csrf_protection, archive_in, "inactive");
parseAppsecJSONKey<std::string>("error-disclosure-enabled", error_disclosure, archive_in, "inactive");
parseAppsecJSONKey<std::string>("open-redirect-enabled", open_redirect, archive_in, "inactive");
parseAppsecJSONKey<std::string>("csrf-protection", csrf_protection, archive_in, "Inactive");
parseAppsecJSONKey<std::string>("error-disclosure", error_disclosure, archive_in, "Inactive");
parseAppsecJSONKey<std::string>("open-redirect", open_redirect, archive_in, "Inactive");
parseAppsecJSONKey<bool>("non-valid-http-methods", non_valid_http_methods, archive_in, false);
}
@ -185,12 +182,8 @@ public:
{
dbgTrace(D_K8S_POLICY) << "Loading AppSec practice spec";
parseAppsecJSONKey<AppSecWebAttackProtections>("protections", protections, archive_in);
parseAppsecJSONKey<std::string>("minimum-confidence", minimum_confidence, archive_in, "critical");
parseAppsecJSONKey<std::string>("override-mode", mode, archive_in, "Unset");
if (getMode() == "Prevent") {
parseAppsecJSONKey<std::string>("minimum-confidence", minimum_confidence, archive_in, "critical");
} else {
minimum_confidence = "Transparent";
}
parseAppsecJSONKey<int>("max-body-size-kb", max_body_size_kb, archive_in, 1000000);
parseAppsecJSONKey<int>("max-header-size-bytes", max_header_size_bytes, archive_in, 102400);
parseAppsecJSONKey<int>("max-object-depth", max_object_depth, archive_in, 40);
@ -219,8 +212,8 @@ private:
int max_header_size_bytes;
int max_object_depth;
int max_url_size_bytes;
std::string minimum_confidence;
std::string mode;
std::string minimum_confidence;
AppSecWebAttackProtections protections;
};
@ -253,7 +246,7 @@ public:
{
dbgTrace(D_K8S_POLICY) << "Loading AppSec Snort Signatures practice";
parseAppsecJSONKey<std::string>("override-mode", override_mode, archive_in, "Inactive");
parseAppsecJSONKey<std::vector<std::string>>("files", config_map, archive_in);
parseAppsecJSONKey<std::vector<std::string>>("configmap", config_map, archive_in);
}
const std::string & getOverrideMode() const { return override_mode; }
@ -285,7 +278,7 @@ public:
{
dbgTrace(D_K8S_POLICY) << "Loading AppSecPracticeOpenSchemaAPI practice";
parseAppsecJSONKey<std::string>("override-mode", override_mode, archive_in, "Inactive");
parseAppsecJSONKey<std::vector<std::string>>("files", config_map, archive_in);
parseAppsecJSONKey<std::vector<std::string>>("configmap", config_map, archive_in);
}
const std::string & getOverrideMode() const { return override_mode; }
@ -700,9 +693,7 @@ public:
const std::string & getMode() const { return mode; }
void setHost(const std::string &_host) { host = _host; }
void setMode(const std::string &_mode) { mode = _mode; }
void setMode(const std::string &_mode) { mode = _mode; };
const std::string & getCustomResponse() const { return custom_response; }
@ -754,18 +745,16 @@ public:
if (default_mode_annot.ok() && !default_mode_annot.unpack().empty() && default_rule.getMode().empty()) {
default_rule.setMode(default_mode_annot.unpack());
}
default_rule.setHost("*");
parseAppsecJSONKey<std::list<ParsedRule>>("specific-rules", specific_rules, archive_in);
specific_rules.push_front(default_rule);
parseAppsecJSONKey<std::vector<ParsedRule>>("specific-rules", specific_rules, archive_in);
}
const ParsedRule & getDefaultRule() const { return default_rule; }
const std::list<ParsedRule> & getSpecificRules() const { return specific_rules; }
const std::vector<ParsedRule> & getSpecificRules() const { return specific_rules; }
private:
ParsedRule default_rule;
std::list<ParsedRule> specific_rules;
std::vector<ParsedRule> specific_rules;
};
class AppsecLinuxPolicy : Singleton::Consume<I_Environment>
@ -777,12 +766,12 @@ public:
dbgTrace(D_K8S_POLICY) << "Loading AppSec policy spec";
parseAppsecJSONKey<AppsecPolicySpec>("policies", policies, archive_in);
parseAppsecJSONKey<std::vector<AppSecPracticeSpec>>("practices", practices, archive_in);
parseAppsecJSONKey<std::vector<AppsecTriggerSpec>>("log-triggers", log_triggers, archive_in);
parseAppsecJSONKey<std::vector<AppSecCustomResponseSpec>>("custom-responses", custom_responses, archive_in);
parseAppsecJSONKey<std::vector<AppsecTriggerSpec>>("logtriggers", log_triggers, archive_in);
parseAppsecJSONKey<std::vector<AppSecCustomResponseSpec>>("customresponses", custom_responses, archive_in);
parseAppsecJSONKey<std::vector<AppsecExceptionSpec>>("exceptions", exceptions, archive_in);
parseAppsecJSONKey<std::vector<TrustedSourcesSpec>>("trusted-sources", trusted_sources, archive_in);
parseAppsecJSONKey<std::vector<TrustedSourcesSpec>>("trustedsources", trusted_sources, archive_in);
parseAppsecJSONKey<std::vector<SourceIdentifierSpecWrapper>>(
"source-identifier",
"sourceidentifiers",
sources_identifier,
archive_in
);

1
components/security_apps/orchestration/local_policy_mgmt_gen/include/triggers_section.h
View File

@ -210,7 +210,6 @@ public:
parseAppsecJSONKey<int>("http-response-code", httpResponseCode, archive_in, 403);
parseAppsecJSONKey<std::string>("mode", mode, archive_in, "block-page");
parseAppsecJSONKey<std::string>("name", name, archive_in);
if (mode == "block-page") {
parseAppsecJSONKey<std::string>(
"message-body",

4
components/security_apps/orchestration/local_policy_mgmt_gen/include/trusted_sources_section.h
View File

@ -34,7 +34,7 @@ public:
{
dbgTrace(D_K8S_POLICY) << "Loading trusted sources spec";
parseAppsecJSONKey<int>("minNumOfSources", min_num_of_sources, archive_in, 3);
parseAppsecJSONKey<std::vector<std::string>>("sources-identifiers", sources_identifiers, archive_in);
parseAppsecJSONKey<std::vector<std::string>>("sourcesIdentifiers", sources_identifiers, archive_in);
parseAppsecJSONKey<std::string>("name", name, archive_in);
}
@ -110,7 +110,7 @@ public:
load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_K8S_POLICY) << "Loading trusted sources spec";
parseAppsecJSONKey<std::string>("source-identifier", source_identifier, archive_in);
parseAppsecJSONKey<std::string>("sourceIdentifier", source_identifier, archive_in);
parseAppsecJSONKey<std::vector<std::string>>("value", value, archive_in);
}

29
components/security_apps/orchestration/local_policy_mgmt_gen/local_policy_mgmt_gen.cc
View File

@ -15,10 +15,10 @@
#include <algorithm>
#include <cctype>
#include <iostream>
#include <map>
#include <set>
#include <string>
#include <iostream>
#include <fstream>
#include <streambuf>
#include <cereal/types/vector.hpp>
@ -28,7 +28,6 @@
#include <boost/uuid/uuid_generators.hpp>
#include "rest.h"
#include "report/report.h"
#include "debug.h"
#include "config.h"
#include "connkey.h"
@ -290,7 +289,7 @@ public:
return appsec_policy;
});
list<ParsedRule> specific_rules = appsec_policy.getAppsecPolicySpec().getSpecificRules();
vector<ParsedRule> specific_rules = appsec_policy.getAppsecPolicySpec().getSpecificRules();
ParsedRule default_rule = appsec_policy.getAppsecPolicySpec().getDefaultRule();
string asset;
@ -673,7 +672,7 @@ public:
AppsecSpecParser<AppsecPolicySpec> appsec_policy = maybe_appsec_policy.unpack();
dbgTrace(D_K8S_POLICY) << "Succeessfully retrieved AppSec policy: " << appsec_policy.getSpec();
list<ParsedRule> specific_rules = appsec_policy.getSpec().getSpecificRules();
vector<ParsedRule> specific_rules = appsec_policy.getSpec().getSpecificRules();
ParsedRule default_rule = appsec_policy.getSpec().getDefaultRule();
for (const ParsedRule &parsed_rule : specific_rules) {
@ -1227,10 +1226,30 @@ private:
LocalPolicyEnv env_type;
map<string, string> practice_name_to_id_map;
bool
isPlaygroundEnv()
{
string playground_variable = "PLAYGROUND";
const char *env_string = getenv(playground_variable.c_str());
if (env_string) {
string env_value = env_string;
transform(
env_value.begin(),
env_value.end(),
env_value.begin(),
[](unsigned char c) { return std::tolower(c); }
);
return env_value == "true";
}
return false;
}
bool
getClusterId()
{
string playground_uid = Report::isPlaygroundEnv() ? "playground-" : "";
string playground_uid = isPlaygroundEnv() ? "playground-" : "";
dbgTrace(D_K8S_POLICY) << "Getting cluster UID";
auto maybe_namespaces_data = getObjectFromCluster<NamespaceData>("/api/v1/namespaces/");

5
components/security_apps/orchestration/manifest_controller/manifest_controller.cc
View File

@ -15,7 +15,6 @@
#include "config.h"
#include "debug.h"
#include "sasal.h"
#include "environment.h"
#include "version.h"
#include "log_generator.h"
@ -24,8 +23,6 @@
using namespace std;
using namespace ReportIS;
SASAL_START // Orchestration - Manifest Handler
USE_DEBUG_FLAG(D_ORCHESTRATOR);
class IgnoredPackages
@ -441,5 +438,3 @@ ManifestController::init()
{
pimpl->init();
}
SASAL_END

5
components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc
View File

@ -15,12 +15,9 @@
#include "debug.h"
#include "config.h"
#include "sasal.h"
using namespace std;
SASAL_START // Orchestration - Manifest Handler
USE_DEBUG_FLAG(D_ORCHESTRATOR);
void
@ -140,5 +137,3 @@ ManifestDiffCalculator::buildInstallationQueue(
installation_queue.push_back(updated_package);
return true;
}
SASAL_END

5
components/security_apps/orchestration/manifest_controller/manifest_handler.cc
View File

@ -15,14 +15,11 @@
#include "debug.h"
#include "config.h"
#include "sasal.h"
#include "agent_details.h"
#include "orchestration_comp.h"
using namespace std;
SASAL_START // Orchestration - Manifest Handler
USE_DEBUG_FLAG(D_ORCHESTRATOR);
void
@ -380,5 +377,3 @@ ManifestHandler::selfUpdate(
package_handler->preInstallPackage(orch_service_name, current_installation_file) &&
package_handler->installPackage(orch_service_name, current_installation_file, false);
}
SASAL_END

6
components/security_apps/orchestration/modules/data.cc
View File

@ -15,10 +15,6 @@
#include <map>
#include "sasal.h"
SASAL_START // Orchestration - Modules
using namespace std;
using namespace cereal;
@ -48,5 +44,3 @@ Data::serialize(JSONInputArchive &in_archive)
make_nvp("version", version)
);
}
SASAL_END

6
components/security_apps/orchestration/modules/orchestration_policy.cc
View File

@ -13,10 +13,6 @@
#include "orchestration_policy.h"
#include "sasal.h"
SASAL_START // Orchestration - Modules
using namespace std;
using namespace cereal;
@ -60,5 +56,3 @@ OrchestrationPolicy::operator!=(const OrchestrationPolicy &other) const
{
return !((*this) == other);
}
SASAL_END

5
components/security_apps/orchestration/modules/orchestration_status.cc
View File

@ -19,7 +19,6 @@
#include "debug.h"
#include "config.h"
#include "sasal.h"
using namespace cereal;
using namespace std;
@ -27,8 +26,6 @@ using namespace chrono;
USE_DEBUG_FLAG(D_ORCHESTRATOR);
SASAL_START // Orchestration - Modules
class RegistrationDetails
{
public:
@ -681,5 +678,3 @@ OrchestrationStatus::init() { pimpl->init(); }
OrchestrationStatus::OrchestrationStatus() : Component("OrchestrationStatus"), pimpl(make_unique<Impl>()) {}
OrchestrationStatus::~OrchestrationStatus() {}
SASAL_END

6
components/security_apps/orchestration/modules/package.cc
View File

@ -15,10 +15,6 @@
#include <map>
#include "sasal.h"
SASAL_START // Orchestration - Modules
using namespace std;
using namespace cereal;
@ -129,5 +125,3 @@ Package::serialize(JSONInputArchive & in_archive)
}
package_type = package_type_value->second;
}
SASAL_END

5
components/security_apps/orchestration/modules/url_parser.cc
View File

@ -18,12 +18,9 @@
#include "singleton.h"
#include "common.h"
#include "maybe_res.h"
#include "sasal.h"
using namespace std;
SASAL_START // Orchestration - Modules
USE_DEBUG_FLAG(D_ORCHESTRATOR);
ostream &
@ -145,5 +142,3 @@ URLParser::toString() const
s_build << protocol << base_url << query << ":" << port;
return s_build.str();
}
SASAL_END

80
components/security_apps/orchestration/orchestration_comp.cc
View File

@ -29,7 +29,6 @@
#include "manifest_controller.h"
#include "url_parser.h"
#include "i_messaging.h"
#include "sasal.h"
#include "agent_details_report.h"
#include "maybe_res.h"
#include "customized_cereal_map.h"
@ -38,8 +37,7 @@
#include "get_status_rest.h"
#include "hybrid_mode_telemetry.h"
#include "telemetry.h"
SASAL_START // Orchestration - Main
#include "tenant_profile_pair.h"
using namespace std;
using namespace chrono;
@ -596,10 +594,6 @@ private:
auto team = i_env->get<AudienceTeam>("Audience Team");
if (team.ok()) audience_team = *team;
string agent_uid =
(Report::isPlaygroundEnv() ? "playground-" : "") +
Singleton::Consume<I_AgentDetails>::by<OrchestrationComp>()->getAgentId();
Report policy_update_message(
"Agent's policy has been updated",
curr_time,
@ -611,7 +605,7 @@ private:
Severity::INFO,
Priority::LOW,
chrono::seconds(0),
LogField("agentId", agent_uid),
LogField("agentId", Singleton::Consume<I_AgentDetails>::by<OrchestrationComp>()->getAgentId()),
Tags::ORCHESTRATOR
);
policy_update_message.addToOrigin(LogField("policyVersion", new_policy_version));
@ -808,14 +802,22 @@ private:
auto greedy_update = getProfileAgentSettingWithDefault<bool>(false, "orchestration.multitenancy.greedymode");
greedy_update = getConfigurationWithDefault<bool>(greedy_update, "orchestration", "Multitenancy Greedy mode");
if (!greedy_update) {
auto tenant_manager = Singleton::Consume<I_TenantManager>::by<OrchestrationComp>();
for (auto const &active_tenant: tenant_manager->fetchActiveTenants()) {
auto virtual_policy_data = getPolicyTenantData(active_tenant);
auto tenant_manager = Singleton::Consume<I_TenantManager>::by<OrchestrationComp>();
for (auto const &active_tenant: tenant_manager->fetchActiveTenants()) {
for (auto const &profile_id: tenant_manager->fetchProfileIds(active_tenant)) {
auto virtual_policy_data = getPolicyTenantData(active_tenant, profile_id);
request.addTenantPolicy(virtual_policy_data);
request.addTenantSettings(getSettingsTenantData(active_tenant, virtual_policy_data.getVersion()));
request.addTenantSettings(
getSettingsTenantData(
active_tenant,
profile_id,
virtual_policy_data.getVersion()
)
);
}
} else {
}
if (greedy_update) {
request.setGreedyMode();
}
@ -980,10 +982,11 @@ private:
const Maybe<vector<CheckUpdateRequest::Tenants>> &updated_policy_tenants,
const vector<string> &new_data_files)
{
dbgFlow(D_ORCHESTRATOR) << "Hanlding virtual files";
if (!updated_policy_tenants.ok()) return;
// Sorting files by tenant id;
unordered_map<string, vector<string>> sorted_files;
unordered_map<TenantProfilePair, vector<string>> sorted_files;
// Download virtual policy
bool is_empty = true;
@ -991,7 +994,17 @@ private:
for (const auto &tenant: *updated_policy_tenants) {
if (!tenant.getVersion().empty()) {
is_empty = false;
resource_v_policy_file.addTenant(tenant.getTenantID(), tenant.getVersion(), tenant.getChecksum());
dbgTrace(D_ORCHESTRATOR)
<< "Adding a tenant to the multi-tenant list. Tenant: "
<< tenant.getTenantID();
auto tenant_manager = Singleton::Consume<I_TenantManager>::by<OrchestrationComp>();
tenant_manager->addActiveTenantAndProfile(tenant.getTenantID(), tenant.getProfileID());
resource_v_policy_file.addTenant(
tenant.getTenantID(),
tenant.getProfileID(),
tenant.getVersion(),
tenant.getChecksum()
);
}
}
@ -1003,7 +1016,8 @@ private:
);
if (new_virtual_policy_files.ok()) {
for (const auto &tenant_file: *new_virtual_policy_files) {
sorted_files[tenant_file.first].push_back(tenant_file.second);
auto tenant_profile = TenantProfilePair(tenant_file.first.first, tenant_file.first.second);
sorted_files[tenant_profile].push_back(tenant_file.second);
}
}
}
@ -1017,6 +1031,7 @@ private:
is_empty = false;
resource_v_settings_file.addTenant(
tenant.getTenantID(),
tenant.getProfileID(),
tenant.getVersion(),
tenant.getChecksum()
);
@ -1031,7 +1046,8 @@ private:
);
if (new_virtual_settings_files.ok()) {
for (const auto &tenant_file: *new_virtual_settings_files) {
sorted_files[tenant_file.first].push_back(tenant_file.second);
auto tenant_profile = TenantProfilePair(tenant_file.first.first, tenant_file.first.second);
sorted_files[tenant_profile].push_back(tenant_file.second);
}
}
}
@ -1043,7 +1059,11 @@ private:
string setting_file = "";
if (files.size() > 1) {
setting_file = files[1];
auto handled_settings = updateSettingsFile(setting_file, downloade_files.first);
auto handled_settings = updateSettingsFile(
setting_file,
downloade_files.first.getTenantId(),
downloade_files.first.getPfofileId()
);
if (handled_settings.ok()) setting_file = *handled_settings;
}
@ -1051,21 +1071,23 @@ private:
policy_file,
setting_file,
new_data_files,
downloade_files.first
downloade_files.first.getTenantId(),
downloade_files.first.getPfofileId()
);
}
}
Maybe<string>
updateSettingsFile(const string &new_settings_file, const string &tenant_id = "")
updateSettingsFile(const string &new_settings_file, const string &tenant_id = "", const string &profile_id = "")
{
// Handling settings update.
auto conf_dir = getConfigurationWithDefault<string>(
filesystem_prefix + "/conf/",
"orchestration",
"Conf dir"
) + (tenant_id != "" ? "tenant_" + tenant_id + "_" : "");
) + (tenant_id != "" ? "tenant_" + tenant_id + "_profile_" + profile_id + "_" : "");
dbgTrace(D_ORCHESTRATOR) << "The settings directory is " << conf_dir;
auto orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<OrchestrationComp>();
string settings_file_path = conf_dir + "settings.json";
if (!orchestration_tools->copyFile(new_settings_file, settings_file_path)) {
@ -1077,7 +1099,7 @@ private:
}
CheckUpdateRequest::Tenants
getPolicyTenantData(const string &tenant_id)
getPolicyTenantData(const string &tenant_id, const string &profile_id)
{
string dir = getConfigurationWithDefault<string>(
filesystem_prefix + "/conf",
@ -1085,16 +1107,16 @@ private:
"Configuration directory"
);
string policy_file = dir + "/tenant_" + tenant_id + "/policy.json";
string policy_file = dir + "/tenant_" + tenant_id + "_profile_" + profile_id + "/policy.json";
string policy_file_checksum = getChecksum(policy_file);
string policy_file_version= getVersion(policy_file);
return CheckUpdateRequest::Tenants(tenant_id, policy_file_checksum, policy_file_version);
return CheckUpdateRequest::Tenants(tenant_id, profile_id, policy_file_checksum, policy_file_version);
}
CheckUpdateRequest::Tenants
getSettingsTenantData(const string &tenant_id, const string &policy_version)
getSettingsTenantData(const string &tenant_id, const string &profile_id, const string &policy_version)
{
string dir = getConfigurationWithDefault<string>(
filesystem_prefix + "/conf",
@ -1102,10 +1124,10 @@ private:
"Configuration directory"
);
string settings_file = dir + "/tenant_" + tenant_id + "_settings.json";
string settings_file = dir + "/tenant_" + tenant_id + "_profile_" + profile_id + "_settings.json";
string settings_file_checksum = getChecksum(settings_file);
return CheckUpdateRequest::Tenants(tenant_id, settings_file_checksum, policy_version);
return CheckUpdateRequest::Tenants(tenant_id, profile_id, settings_file_checksum, policy_version);
}
string
@ -1629,5 +1651,3 @@ OrchestrationComp::preload()
registerExpectedSetting<string>("upgradeMode");
registerExpectedConfigFile("orchestration", Config::ConfigFileType::Policy);
}
SASAL_END

22
components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc
View File

@ -23,10 +23,6 @@
#include <arpa/inet.h>
#include <sys/stat.h>
#include "sasal.h"
SASAL_START // Orchestration - Tools
using namespace std;
using namespace rapidjson;
@ -41,7 +37,8 @@ public:
Maybe<map<packageName, packageDetails>>
jsonObjectSplitter(
const string &json,
const string &tenant_id) const override;
const string &tenant_id,
const string &profile_id) const override;
Maybe<string> readFile(const string &path) const override;
bool writeFile(const string &text, const string &path) const override;
@ -238,7 +235,10 @@ OrchestrationTools::Impl::copyFile(const string &src_path, const string &dst_pat
}
Maybe<map<packageName, packageDetails>>
OrchestrationTools::Impl::jsonObjectSplitter(const string &json, const string &tenant_id) const
OrchestrationTools::Impl::jsonObjectSplitter(
const string &json,
const string &tenant_id,
const string &profile_id) const
{
Document document;
map<string, string> parsed;
@ -247,14 +247,18 @@ OrchestrationTools::Impl::jsonObjectSplitter(const string &json, const string &t
if (document.HasParseError()) return genError("JSON file is not valid.");
for (Value::MemberIterator itr = document.MemberBegin(); itr != document.MemberEnd(); ++itr) {
if (!tenant_id.empty() && itr->value.IsObject()) {
itr->value.AddMember(
Value("tenantID"),
Value(tenant_id.c_str(), tenant_id.size()),
document.GetAllocator()
);
itr->value.AddMember(
Value("profileID"),
Value(profile_id.c_str(), profile_id.size()),
document.GetAllocator()
);
}
rapidjson::StringBuffer buffer;
@ -471,5 +475,3 @@ OrchestrationTools::Impl::base64Decode(const string &input) const
OrchestrationTools::OrchestrationTools() : Component("OrchestrationTools"), pimpl(make_unique<Impl>()) {}
OrchestrationTools::~OrchestrationTools() {}
SASAL_END

4
components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc
View File

@ -158,7 +158,7 @@ TEST_F(OrchestrationToolsTest, jsonObjectSplitter)
" ]"
" }";
Maybe<map<string, string>> parsed = i_orchestration_tools->jsonObjectSplitter(update_text, "");
Maybe<map<string, string>> parsed = i_orchestration_tools->jsonObjectSplitter(update_text, "", "");
EXPECT_TRUE(parsed.ok());
cleanSpaces(manifest);
EXPECT_EQ(manifest, parsed.unpack().find("manifest")->second);
@ -174,7 +174,7 @@ TEST_F(OrchestrationToolsTest, jsonObjectSplitter)
" \"download-options\": ["
" \"http://172.23.92.135/manifest_file.txt\""
" ]";
parsed = i_orchestration_tools->jsonObjectSplitter(invalid_json, "");
parsed = i_orchestration_tools->jsonObjectSplitter(invalid_json, "", "");
EXPECT_FALSE(parsed.ok());
}

88
components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc
View File

@ -65,7 +65,7 @@ public:
EXPECT_CALL(rest, mockRestCall(RestAction::SHOW, "orchestration-status", _)).WillOnce(
WithArg<2>(Invoke(this, &OrchestrationMultitenancyTest::setRestStatus)));
doEncrypt();
orchestration_comp.init();
}
@ -83,9 +83,12 @@ public:
Maybe<string> err = genError("No file exist");
EXPECT_CALL(mock_orchestration_tools, readFile("/etc/cp/conf/user-cred.json")).WillOnce(Return(err));
EXPECT_CALL(mock_orchestration_tools, writeFile("This is fake", "/etc/cp/data/data1.a")).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, writeFile("0000 is fake", "/etc/cp/data/data4.a")).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, writeFile("This is 3333", "/etc/cp/data/data6.a")).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, writeFile("This is fake", "/etc/cp/data/data1.a")).WillOnce(
Return(true));
EXPECT_CALL(mock_orchestration_tools, writeFile("0000 is fake", "/etc/cp/data/data4.a")).WillOnce(
Return(true));
EXPECT_CALL(mock_orchestration_tools, writeFile("This is 3333", "/etc/cp/data/data6.a")).WillOnce(
Return(true));
}
void
@ -182,7 +185,8 @@ private:
return true;
}
bool setRestStatus(const unique_ptr<RestInit> &p)
bool
setRestStatus(const unique_ptr<RestInit> &p)
{
rest_status = p->getRest();
return true;
@ -256,23 +260,38 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource)
vector<string> active_tenants = { "1236", "1235" };
EXPECT_CALL(tenant_manager, fetchActiveTenants()).WillOnce(Return(active_tenants));
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, "/etc/cp/conf/tenant_1236/policy.json"))
EXPECT_CALL(tenant_manager, addActiveTenantAndProfile("1235", "2311"));
EXPECT_CALL(tenant_manager, addActiveTenantAndProfile("1236", "2611"));
vector<string> first_tenant_profiles = { "2611" };
vector<string> second_tenant_profiles = { "2311"};
EXPECT_CALL(
tenant_manager,
fetchProfileIds("1236")).WillRepeatedly(Return(first_tenant_profiles)
);
EXPECT_CALL(
tenant_manager,
fetchProfileIds("1235")).WillRepeatedly(Return(second_tenant_profiles)
);
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, "/etc/cp/conf/tenant_1236_profile_2611/policy.json"))
.WillOnce(Return(string("checksum_policy_tenant_1236")));
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, "/etc/cp/conf/tenant_1235/policy.json"))
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, "/etc/cp/conf/tenant_1235_profile_2311/policy.json"))
.WillOnce(Return(string("checksum_policy_tenant_1235")));
EXPECT_CALL(mock_orchestration_tools, readFile("/etc/cp/conf/tenant_1236/policy.json"))
EXPECT_CALL(mock_orchestration_tools, readFile("/etc/cp/conf/tenant_1236_profile_2611/policy.json"))
.WillOnce(Return(string("{}")));
EXPECT_CALL(mock_orchestration_tools, readFile("/etc/cp/conf/tenant_1235/policy.json"))
EXPECT_CALL(mock_orchestration_tools, readFile("/etc/cp/conf/tenant_1235_profile_2311/policy.json"))
.WillOnce(Return(string("{}")));
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, "/etc/cp/conf/tenant_1236_settings.json"))
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, "/etc/cp/conf/tenant_1236_profile_2611_settings.json"))
.WillOnce(Return(string("checksum_settings_tenant_1236")));
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, "/etc/cp/conf/tenant_1235_settings.json"))
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, "/etc/cp/conf/tenant_1235_profile_2311_settings.json"))
.WillOnce(Return(string("checksum_settings_tenant_1235")));
EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce(
@ -294,11 +313,13 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource)
" \"tenants\": [\n"
" {\n"
" \"tenantId\": \"1236\",\n"
" \"profileId\": \"2611\",\n"
" \"checksum\": \"new_checksum_policy_tenant_1236\",\n"
" \"version\": \"1\"\n"
" },\n"
" {\n"
" \"tenantId\": \"1235\",\n"
" \"profileId\": \"2311\",\n"
" \"checksum\": \"new_checksum_policy_tenant_1235\",\n"
" \"version\": \"1\"\n"
" }\n"
@ -308,11 +329,13 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource)
" \"tenants\": [\n"
" {\n"
" \"tenantId\": \"1236\",\n"
" \"profileId\": \"2611\",\n"
" \"checksum\": \"new_checksum_settings_tenant_1236\",\n"
" \"version\": \"1\"\n"
" },\n"
" {\n"
" \"tenantId\": \"1235\",\n"
" \"profileId\": \"2311\",\n"
" \"checksum\": \"new_checksum_settings_tenant_1235\",\n"
" \"version\": \"1\"\n"
" }\n"
@ -328,21 +351,21 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource)
);
GetResourceFile policy_file(GetResourceFile::ResourceFileType::VIRTUAL_POLICY);
policy_file.addTenant("1236", "1", "new_checksum_policy_tenant_1236");
policy_file.addTenant("1235", "1", "new_checksum_policy_tenant_1235");
policy_file.addTenant("1236", "2611", "1", "new_checksum_policy_tenant_1236");
policy_file.addTenant("1235", "2311", "1", "new_checksum_policy_tenant_1235");
map<string, string> download_policy_res = {
{ "1236", "/tmp/orchestration_downloads/virtualPolicy_1236.download" },
{ "1235", "/tmp/orchestration_downloads/virtualPolicy_1235.download" }
map<pair<string, string>, string> download_policy_res = {
{ {"1236", "2611" }, "/tmp/orchestration_downloads/virtualPolicy_1236_profile_2611.download" },
{ {"1235", "2311" }, "/tmp/orchestration_downloads/virtualPolicy_1235_profile_2311.download" }
};
GetResourceFile settings_file(GetResourceFile::ResourceFileType::VIRTUAL_SETTINGS);
settings_file.addTenant("1236", "1", "new_checksum_settings_tenant_1236");
settings_file.addTenant("1235", "1", "new_checksum_settings_tenant_1235");
settings_file.addTenant("1236", "2611", "1", "new_checksum_settings_tenant_1236");
settings_file.addTenant("1235", "2311", "1", "new_checksum_settings_tenant_1235");
map<string, string> download_settings_res = {
{ "1236", "/tmp/orchestration_downloads/virtualSettings_1236.download" },
{ "1235", "/tmp/orchestration_downloads/virtualSettings_1235.download" }
map<pair<string, string>, string> download_settings_res = {
{ {"1236", "2611" }, "/tmp/orchestration_downloads/virtualSettings_1236_profile_2611.download" },
{ {"1235", "2311" }, "/tmp/orchestration_downloads/virtualSettings_1235_profile_2311.download" }
};
EXPECT_CALL(
@ -373,16 +396,16 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource)
EXPECT_CALL(
mock_orchestration_tools,
copyFile(
"/tmp/orchestration_downloads/virtualSettings_1236.download",
"/etc/cp/conf/tenant_1236_settings.json"
"/tmp/orchestration_downloads/virtualSettings_1236_profile_2611.download",
"/etc/cp/conf/tenant_1236_profile_2611_settings.json"
)
).WillOnce(Return(true));
EXPECT_CALL(
mock_orchestration_tools,
copyFile(
"/tmp/orchestration_downloads/virtualSettings_1235.download",
"/etc/cp/conf/tenant_1235_settings.json"
"/tmp/orchestration_downloads/virtualSettings_1235_profile_2311.download",
"/etc/cp/conf/tenant_1235_profile_2311_settings.json"
)
).WillOnce(Return(true));
@ -393,6 +416,7 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource)
"/etc/cp/conf/policy.json",
"/etc/cp/conf/settings.json",
expected_data_types,
"",
""
)
).WillOnce(Return(true));
@ -400,20 +424,22 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource)
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(
"/tmp/orchestration_downloads/virtualPolicy_1236.download",
"/etc/cp/conf/tenant_1236_settings.json",
"/tmp/orchestration_downloads/virtualPolicy_1236_profile_2611.download",
"/etc/cp/conf/tenant_1236_profile_2611_settings.json",
expected_data_types,
"1236"
"1236",
"2611"
)
).WillOnce(Return(true));
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(
"/tmp/orchestration_downloads/virtualPolicy_1235.download",
"/etc/cp/conf/tenant_1235_settings.json",
"/tmp/orchestration_downloads/virtualPolicy_1235_profile_2311.download",
"/etc/cp/conf/tenant_1235_profile_2311_settings.json",
expected_data_types,
"1235"
"1235",
"2311"
)
).WillOnce(Return(true));

32
components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc
View File

@ -101,15 +101,19 @@ public:
return true;
}
void
doEncrypt()
{
Maybe<string> err = genError("No file exist");
EXPECT_CALL(mock_orchestration_tools, readFile("/etc/cp/conf/user-cred.json")).WillOnce(Return(err));
EXPECT_CALL(mock_orchestration_tools, writeFile("This is fake", "/etc/cp/data/data1.a")).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, writeFile("0000 is fake", "/etc/cp/data/data4.a")).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, writeFile("This is 3333", "/etc/cp/data/data6.a")).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, writeFile("This is fake", "/etc/cp/data/data1.a")).WillOnce(
Return(true));
EXPECT_CALL(mock_orchestration_tools, writeFile("0000 is fake", "/etc/cp/data/data4.a")).WillOnce(
Return(true));
EXPECT_CALL(mock_orchestration_tools, writeFile("This is 3333", "/etc/cp/data/data6.a")).WillOnce(
Return(true));
}
void
@ -526,12 +530,12 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate)
vector<string> expected_data_types = {};
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "")
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "")
).WillOnce(Return(true));
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(new_policy_path, "", expected_data_types, "")
updateServiceConfiguration(new_policy_path, "", expected_data_types, "", "")
).WillOnce(Return(true));
EXPECT_CALL(
@ -629,7 +633,7 @@ TEST_F(OrchestrationTest, startOrchestrationPoliceWithFailures)
vector<string> expected_data_types = {};
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "")
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "")
).Times(2).WillRepeatedly(Return(true));
EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)).WillOnce(Return(true));
@ -749,7 +753,7 @@ TEST_F(OrchestrationTest, loadOrchestrationPolicyFromBackup)
vector<string> expected_data_types = {};
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "")
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "")
).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true));
@ -883,7 +887,7 @@ TEST_F(OrchestrationTest, manifestUpdate)
vector<string> expected_data_types = {};
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "")
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "")
).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true));
@ -1033,7 +1037,7 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate)
vector<string> expected_data_types = {};
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "")
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "")
).Times(2).WillRepeatedly(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true));
@ -1114,7 +1118,7 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate)
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(string("policy path"), "", expected_data_types, "")).WillOnce(Return(false)
updateServiceConfiguration(string("policy path"), "", expected_data_types, "", "")).WillOnce(Return(false)
);
EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>()))
@ -1179,7 +1183,7 @@ TEST_F(OrchestrationTest, failedDownloadSettings)
vector<string> expected_data_types = {};
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "")
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "")
).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true));
@ -1397,7 +1401,7 @@ TEST_P(OrchestrationTest, orchestrationFirstRun)
vector<string> expected_data_types = {};
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "")
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "")
).WillOnce(Return(true));
EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>()))
@ -1578,13 +1582,13 @@ TEST_F(OrchestrationTest, dataUpdate)
vector<string> expected_empty_data_types = {};
ExpectationSet expectation_set = EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(policy_file_path, setting_file_path, expected_empty_data_types, "")
updateServiceConfiguration(policy_file_path, setting_file_path, expected_empty_data_types, "", "")
).WillOnce(Return(true));
vector<string> expected_ips_data_types = { "ips" };
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration("", "", expected_ips_data_types, "")
updateServiceConfiguration("", "", expected_ips_data_types, "", "")
).After(expectation_set).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesDirectoryExist("/etc/cp/conf/data")).WillOnce(Return(true));

5
components/security_apps/orchestration/package_handler/package_handler.cc
View File

@ -13,14 +13,11 @@
#include "package_handler.h"
#include "config.h"
#include "sasal.h"
#include "i_shell_cmd.h"
#include <sys/stat.h>
#include <vector>
SASAL_START // Orchestration - Updates Control
USE_DEBUG_FLAG(D_ORCHESTRATOR);
using namespace std;
@ -504,5 +501,3 @@ PackageHandler::Impl::updateSavedPackage(const string &package_name, const strin
PackageHandler::PackageHandler() : Component("PackageHandler"), pimpl(make_unique<Impl>()) {}
PackageHandler::~PackageHandler() {}
SASAL_END

74
components/security_apps/orchestration/service_controller/service_controller.cc
View File

@ -20,7 +20,6 @@
#include "config.h"
#include "debug.h"
#include "sasal.h"
#include "rest.h"
#include "connkey.h"
#include "i_messaging.h"
@ -29,8 +28,6 @@
#include "i_orchestration_tools.h"
#include "customized_cereal_map.h"
SASAL_START // Orchestration - Updates Control
using namespace std;
using namespace ReportIS;
@ -117,10 +114,36 @@ ServiceDetails::isServiceActive() const
I_ShellCmd *shell_cmd = Singleton::Consume<I_ShellCmd>::by<ServiceController>();
Maybe<string> service_status = shell_cmd->getExecOutput(watchdog_status_cmd.str());
int max_retry_attempts = getConfigurationWithDefault<int>(
5,
"orchestration",
"service controller attempts before timeout"
);
uint default_ms_tmout = 200;
uint ms_tmout = default_ms_tmout;
for (int current_attempt = 0; current_attempt < max_retry_attempts; ++current_attempt) {
if (service_status.ok() || service_status.getErr().find("Reached timeout") == string::npos) break;
dbgWarning(D_ORCHESTRATOR)
<< "Retrying to execute service status check via watchdog API after getting timeout. Service name: "
<< service_name
<< ", Watchdog command: "
<< watchdog_status_cmd.str()
<< ", retry number: "
<< (current_attempt + 1);
ms_tmout = default_ms_tmout*(current_attempt + 2);
service_status = shell_cmd->getExecOutput(watchdog_status_cmd.str(), ms_tmout);
}
if (!service_status.ok()) {
dbgWarning(D_ORCHESTRATOR)
<< "Changing service status to inactive after failure to its status from watchdog. Service name: "
<< service_name;
<< service_name
<< ", Watchdog output: "
<< service_status.getErr();
return false;
}
@ -248,7 +271,8 @@ public:
const string &new_policy_path,
const string &new_settings_path,
const vector<string> &new_data_files,
const string &tenant_id
const string &tenant_id,
const string &profile_id
) override;
bool isServiceInstalled(const string &service_name) override;
@ -567,7 +591,8 @@ ServiceController::Impl::updateServiceConfiguration(
const string &new_policy_path,
const string &new_settings_path,
const vector<string> &new_data_files,
const string &tenant_id)
const string &tenant_id,
const string &profile_id)
{
dbgFlow(D_ORCHESTRATOR)
<< "new_policy_path: "
@ -577,7 +602,9 @@ ServiceController::Impl::updateServiceConfiguration(
<< ", new_data_files: "
<< makeSeparatedStr(new_data_files, ",")
<< ". tenant_id: "
<< tenant_id;
<< tenant_id
<< ". profile_id: "
<< profile_id;
if (!new_settings_path.empty()) {
settings_path = new_settings_path;
@ -622,7 +649,7 @@ ServiceController::Impl::updateServiceConfiguration(
return false;
}
auto all_security_policies = orchestration_tools->jsonObjectSplitter(loaded_json.unpack(), tenant_id);
auto all_security_policies = orchestration_tools->jsonObjectSplitter(loaded_json.unpack(), tenant_id, profile_id);
if (!all_security_policies.ok()) {
dbgWarning(D_ORCHESTRATOR)
@ -655,7 +682,7 @@ ServiceController::Impl::updateServiceConfiguration(
);
if (tenant_id != "") {
dir = dir + "/tenant_" + tenant_id;
dir = dir + "/tenant_" + tenant_id + "_profile_" + profile_id;
if (!orchestration_tools->doesDirectoryExist(dir)) {
if (orchestration_tools->createDirectory(dir)) {
dbgTrace(D_ORCHESTRATOR) << "Created new configuration directory for tenant " << tenant_id;
@ -666,7 +693,13 @@ ServiceController::Impl::updateServiceConfiguration(
}
}
string policy_file_path = getPolicyConfigPath(single_policy.first, Config::ConfigFileType::Policy, tenant_id);
string policy_file_path =
getPolicyConfigPath(
single_policy.first,
Config::ConfigFileType::Policy,
tenant_id,
profile_id
);
auto update_config_result = updateServiceConfigurationFile(
single_policy.first,
@ -690,7 +723,10 @@ ServiceController::Impl::updateServiceConfiguration(
);
if (tenant_id != "") {
auto instances = Singleton::Consume<I_TenantManager>::by<ServiceController>()->getInstances(tenant_id);
auto instances = Singleton::Consume<I_TenantManager>::by<ServiceController>()->getInstances(
tenant_id,
profile_id
);
for (const auto &instance_id: instances) {
auto relevant_service = registered_services.find(instance_id);
if (relevant_service == registered_services.end()) {
@ -716,7 +752,9 @@ ServiceController::Impl::updateServiceConfiguration(
if (was_policy_updated) {
string config_file_path;
string base_path = filesystem_prefix + "/conf/" + (tenant_id != "" ? "tenant_" + tenant_id + "/" : "");
string base_path =
filesystem_prefix + "/conf/" +
(tenant_id != "" ? "tenant_" + tenant_id + "_profile_" + profile_id + "/" : "");
config_file_path = getConfigurationWithDefault<string>(
base_path + "policy.json",
"orchestration",
@ -793,7 +831,15 @@ ServiceController::Impl::sendSignalForServices(
}
}
int reconf_timeout = getConfigurationWithDefault(600, "orchestration", "Reconfiguration timeout seconds");
int profile_tmo_conf = getProfileAgentSettingWithDefault<int>(
600,
"orchestration.configTimeoutSeconds"
);
int reconf_timeout = getConfigurationWithDefault<int>(
profile_tmo_conf,
"orchestration",
"Reconfiguration timeout seconds"
);
auto timer = Singleton::Consume<I_TimeGet>::by<ServiceController>();
auto current_timeout = timer->getMonotonicTime() + chrono::seconds(reconf_timeout);
while(timer->getMonotonicTime() < current_timeout) {
@ -933,5 +979,3 @@ ServiceController::Impl::startReconfStatus(
services_reconf_names.emplace(id, service_name);
services_reconf_ids.emplace(id, service_id);
}
SASAL_END

170
components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc
View File

@ -245,7 +245,7 @@ TEST_F(ServiceControllerTest, UpdateConfiguration)
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _))
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path)).WillOnce(Return(true));
@ -294,6 +294,110 @@ TEST_F(ServiceControllerTest, UpdateConfiguration)
EXPECT_EQ(i_service_controller->getUpdatePolicyVersion(), version_value);
}
TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration)
{
string new_configuration = "{"
" \"version\": \"" + version_value + "\""
" \"l4_firewall\":"
" {"
" \"app\": \"netfilter\","
" \"l4_firewall_rules\": ["
" {"
" \"name\": \"allow_statefull_conns\","
" \"flags\": [\"established\"],"
" \"action\": \"accept\""
" },"
" {"
" \"name\": \"icmp drop\","
" \"flags\": [\"log\"],"
" \"services\": [{\"name\":\"icmp\"}],"
" \"action\": \"drop\""
" }"
" ]"
" }"
"}";
string l4_firewall = "{"
" \"app\": \"netfilter\","
" \"l4_firewall_rules\": ["
" {"
" \"name\": \"allow_statefull_conns\","
" \"flags\": [\"established\"],"
" \"action\": \"accept\""
" },"
" {"
" \"name\": \"icmp drop\","
" \"flags\": [\"log\"],"
" \"services\": [{\"name\":\"icmp\"}],"
" \"action\": \"drop\""
" }"
" ]"
"}";
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_status,
setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY));
EXPECT_EQ(i_service_controller->getPolicyVersion(), "");
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
string general_settings_path = "/my/settings/path";
string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
Flags<MessageConnConfig> conn_flags;
conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN);
EXPECT_CALL(
mock_message,
sendMessage(
true,
"{\n \"id\": 1,\n \"policy_version\": \"1.0.2\"\n}",
I_Messaging::Method::POST,
string("127.0.0.1"),
l4_firewall_service_port,
conn_flags,
string("/set-new-configuration"),
string(),
_,
MessageTypeTag::GENERIC
)
).WillOnce(Return(Maybe<string>(reply_msg)));
EXPECT_CALL(
mock_shell_cmd,
getExecOutput(
"/etc/cp/watchdog/cp-nano-watchdog --status --verbose --service mock access control"
" --family family1 --id id2",
_,
_
)
).Times(3).WillRepeatedly(
InvokeWithoutArgs(
[&]() -> Maybe<string>
{
static int counter = 0;
if (counter++ < 2) {
return genError("Reached timeout while executing shell command:");
}
return string("registered and running");
}
)
);
EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path));
EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value);
EXPECT_EQ(i_service_controller->getUpdatePolicyVersion(), version_value);
}
TEST_F(ServiceControllerTest, writeRegisteredServicesFromFile)
{
EXPECT_EQ(orchestrationRegisteredServicesFileToString(registered_services_file_path), string(""));
@ -352,7 +456,7 @@ TEST_F(ServiceControllerTest, writeRegisteredServicesFromFile)
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _))
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path)).WillOnce(Return(true));
@ -498,7 +602,7 @@ TEST_F(ServiceControllerTest, noPolicyUpdate)
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _))
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(l4_firewall));
@ -584,7 +688,7 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations)
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _))
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path)).WillOnce(Return(true));
@ -635,7 +739,7 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations)
// Only settings now being updated
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _))
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(l4_firewall));
@ -730,7 +834,7 @@ TEST_F(ServiceControllerTest, backup)
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _))
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(old_configuration));
@ -842,7 +946,7 @@ TEST_F(ServiceControllerTest, backupAttempts)
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _))
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(old_configuration));
@ -963,7 +1067,7 @@ TEST_F(ServiceControllerTest, MultiUpdateConfiguration)
string orchestration_settings_path = configuration_dir + "/orchestration/orchestration" + settings_extension;
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _))
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_path)).WillOnce(Return(false));
@ -1028,7 +1132,7 @@ TEST_F(ServiceControllerTest, emptyServices)
Maybe<map<string, string>> json_parser_return = map<string, string>();
string empty_string = "";
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).Times(1).WillRepeatedly(Return(empty_string));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(empty_string, _)).Times(1).WillRepeatedly(
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(empty_string, _, _)).Times(1).WillRepeatedly(
Return(json_parser_return)
);
@ -1083,7 +1187,7 @@ TEST_F(ServiceControllerTest, failingWhileLoadingCurrentConfiguration)
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _))
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(err));
@ -1151,7 +1255,7 @@ TEST_F(ServiceControllerTest, failingWhileCopyingCurrentConfiguration)
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).Times(1).WillRepeatedly(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _)).Times(1).WillRepeatedly(
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)).Times(1).WillRepeatedly(
Return(json_parser_return)
);
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true));
@ -1211,7 +1315,7 @@ TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest)
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _))
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path)).WillOnce(Return(true));
@ -1328,7 +1432,7 @@ TEST_F(ServiceControllerTest, errorWhileWrtingNewConfiguration)
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).Times(1).WillRepeatedly(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _)).Times(1).WillRepeatedly(
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)).Times(1).WillRepeatedly(
Return(json_parser_return)
);
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true));
@ -1357,16 +1461,18 @@ TEST_F(ServiceControllerTest, testPortsRest)
TEST_F(ServiceControllerTest, testMultitenantConfFiles)
{
map<string, pair<string, string>> tenant_files_input = {
{"tenant1", make_pair("/etc/cp/conf/tenant1_policy.json", "/etc/cp/conf/tenant1_settings.json")},
{"tenant2", make_pair("/etc/cp/conf/tenant2_policy.json", "")}
map<pair<string, string>, pair<string, string>> tenant_files_input = {
{make_pair("tenant1", "1234"),
make_pair("/etc/cp/conf/tenant1_profile_1234_policy.json", "/etc/cp/conf/tenant1_profile_1234_settings.json")},
{make_pair("tenant2", "1235"),
make_pair("/etc/cp/conf/tenant2_profile_1235_policy.json", "")}
};
vector<string> ids = {"family1_id2"};
vector<string> empty_ids;
EXPECT_CALL(tenant_manager, getInstances("tenant1")).WillOnce(Return(ids));
EXPECT_CALL(tenant_manager, getInstances("tenant2")).WillOnce(Return(empty_ids));
EXPECT_CALL(tenant_manager, getInstances("tenant1", "1234")).WillOnce(Return(ids));
EXPECT_CALL(tenant_manager, getInstances("tenant2", "1235")).WillOnce(Return(empty_ids));
string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
EXPECT_CALL(
@ -1386,7 +1492,8 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles)
).WillOnce(Return(Maybe<string>(reply_msg)));
for(auto entry : tenant_files_input) {
auto tenant = entry.first;
auto tenant = entry.first.first;
auto profile = entry.first.second;
auto files = entry.second;
string conf_file_name = files.first;
string settings_file_name = files.second;
@ -1430,20 +1537,25 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles)
"}";
string l4_firewall_policy_path_new =
configuration_dir + "/tenant_" + tenant + "/l4_firewall/l4_firewall" + policy_extension;
configuration_dir + "/tenant_" + tenant +
"_profile_" + profile +"/l4_firewall/l4_firewall" + policy_extension;
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(conf_file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, tenant))
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, tenant, profile))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesDirectoryExist(configuration_dir + "/tenant_" + tenant))
.WillOnce(Return(false));
EXPECT_CALL(
mock_orchestration_tools,
doesDirectoryExist(configuration_dir + "/tenant_" + tenant + "_profile_" + profile)
).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, createDirectory(configuration_dir + "/tenant_" + tenant))
.WillOnce(Return(true));
EXPECT_CALL(
mock_orchestration_tools,
createDirectory(configuration_dir + "/tenant_" + tenant + "_profile_" + profile)
).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path_new)).WillOnce(Return(false));
@ -1454,7 +1566,7 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles)
"l4_firewall", l4_firewall_policy_path_new, OrchestrationStatusConfigType::POLICY)
);
string new_policy_file_path = "/etc/cp/conf/tenant_" + tenant + "/" + "policy.json";
string new_policy_file_path = "/etc/cp/conf/tenant_" + tenant + "_profile_" + profile + "/" + "policy.json";
EXPECT_CALL(mock_orchestration_tools, copyFile(conf_file_name, new_policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(conf_file_name, new_policy_file_path)).WillOnce(Return(true));
@ -1469,7 +1581,9 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles)
)
).WillRepeatedly(Return(string("registered and running")));
EXPECT_TRUE(i_service_controller->updateServiceConfiguration(conf_file_name, settings_file_name, {}, tenant));
EXPECT_TRUE(
i_service_controller->updateServiceConfiguration(conf_file_name, settings_file_name, {}, tenant, profile)
);
}
}
@ -1542,7 +1656,7 @@ TEST_F(ServiceControllerTest, test_delayed_reconf)
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _))
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path)).

7
components/security_apps/orchestration/update_communication/fog_authenticator.cc
View File

@ -17,14 +17,11 @@
#include "log_generator.h"
#include "agent_details.h"
#include "version.h"
#include "sasal.h"
#include <algorithm>
#include <map>
#include <vector>
SASAL_START // Orchestration - Communication
using namespace std;
using namespace cereal;
using HTTPMethod = I_Messaging::Method;
@ -319,6 +316,7 @@ FogAuthenticator::saveCredentialsToFile(const UserCredentials &user_credentials)
return false;
}
return orchestration_tools->writeFile(cred_str.unpack(), data_path + user_cred_file_name);
}
@ -347,6 +345,7 @@ FogAuthenticator::getCredentialsFromFile() const
if (!encrypted_cred.ok()) return genError(encrypted_cred.getErr());
dbgTrace(D_ORCHESTRATOR) << "Read the user credentials from the file";
return orchestration_tools->jsonStringToObject<UserCredentials>(encrypted_cred.unpack());
}
@ -568,5 +567,3 @@ FogAuthenticator::init()
loadRequiredSecurityApps();
initRestAPI();
}
SASAL_END

5
components/security_apps/orchestration/update_communication/fog_communication.cc
View File

@ -17,14 +17,11 @@
#include "log_generator.h"
#include "agent_details.h"
#include "version.h"
#include "sasal.h"
#include <algorithm>
#include <map>
#include <vector>
SASAL_START // Orchestration - Communication
using namespace std;
using namespace cereal;
using HTTPMethod = I_Messaging::Method;
@ -133,5 +130,3 @@ FogCommunication::sendPolicyVersion(const string &policy_version) const
}
return genError("Failed to patch policy version");
}
SASAL_END

9
components/security_apps/orchestration/update_communication/hybrid_communication.cc
View File

@ -18,14 +18,11 @@
#include "log_generator.h"
#include "agent_details.h"
#include "version.h"
#include "sasal.h"
#include <algorithm>
#include <map>
#include <vector>
SASAL_START // Orchestration - Communication
using namespace std;
using HTTPMethod = I_Messaging::Method;
@ -138,10 +135,9 @@ HybridCommunication::downloadAttributeFile(const GetResourceFile &resourse_file)
<< "Downloading attribute file on hybrid mode, file name: "
<< resourse_file.getFileName();
if (resourse_file.getFileName() == "policy") {
if (resourse_file.getFileName() =="policy") {
return declarative_policy_utils.getCurrPolicy();
}
if (resourse_file.getFileName() == "manifest") {
if (!access_token.ok()) return genError("Acccess Token not available.");
@ -156,7 +152,6 @@ HybridCommunication::downloadAttributeFile(const GetResourceFile &resourse_file)
);
return attribute_file;
}
dbgTrace(D_ORCHESTRATOR) << "Unnecessary attribute files downloading on hybrid mode";
return string("");
}
@ -168,5 +163,3 @@ HybridCommunication::sendPolicyVersion(const string &policy_version) const
policy_version.empty();
return Maybe<void>();
}
SASAL_END

5
components/security_apps/orchestration/update_communication/local_communication.cc
View File

@ -13,9 +13,6 @@
#include "local_communication.h"
#include "config.h"
#include "sasal.h"
SASAL_START // Orchestration - Communication
using namespace std;
@ -183,5 +180,3 @@ LocalCommunication::sendPolicyVersion(const string &) const
dbgTrace(D_ORCHESTRATOR) << "Agent in offline mode, no need to send policy version";
return Maybe<void>();
}
SASAL_END

5
components/security_apps/orchestration/update_communication/update_communication.cc
View File

@ -22,15 +22,12 @@
#include "log_generator.h"
#include "agent_details.h"
#include "version.h"
#include "sasal.h"
#include "i_encryptor.h"
#include "fog_authenticator.h"
#include "fog_communication.h"
#include "local_communication.h"
#include "hybrid_communication.h"
SASAL_START // Orchestration - Communication
using namespace std;
USE_DEBUG_FLAG(D_ORCHESTRATOR);
@ -145,5 +142,3 @@ UpdateCommunication::fini()
{
pimpl->fini();
}
SASAL_END

2
components/security_apps/waap/include/i_serialize.h
View File

@ -15,7 +15,6 @@
#include <chrono>
#include <fstream>
#include "i_time_get.h"
#include "i_encryptor.h"
#include "rest.h"
#include "i_messaging.h"
#include "i_mainloop.h"
@ -92,7 +91,6 @@ public:
virtual void saveData();
virtual void restore();
virtual void setFilePath(const std::string &new_file_path);
protected:
// saved file name for testing

4
components/security_apps/waap/include/i_waap_asset_state.h
View File

@ -18,8 +18,8 @@
class I_WaapAssetState {
public:
virtual void updateScores() = 0;
virtual std::string getSignaturesScoresFilePath() const = 0;
virtual std::string getSignaturesFilterDir() const = 0;
virtual std::string getWaapDataFileName() const = 0;
virtual std::string getWaapDataDir() const = 0;
virtual bool isKeywordOfType(const std::string& keyword, ParamType type) const = 0;
virtual bool isBinarySampleType(const std::string & sample) const = 0;
virtual bool isWBXMLSampleType(const std::string & sample) const = 0;

59902
components/security_apps/waap/resources/waap.data Normal file
View File

File diff suppressed because it is too large Load Diff

2
components/security_apps/waap/waap_clib/CidrMatch.cc
View File

@ -80,7 +80,7 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr)
// get ip from targetCidr
std::string strPrefix = pos != std::string::npos ? strCIDR.substr(0, pos) : strCIDR;
// get subnet mask from targetCidr or calculate it based on ipv4 / ipv6
std::string strSuffix = pos != std::string::npos ? strCIDR.substr(pos + 1) :
std::string strSuffix = (pos != std::string::npos && (pos + 1) <= strCIDR.size()) ? strCIDR.substr(pos + 1) :
(strCIDR.find(':') == std::string::npos) ? "32" : "128";
int bits = -1;

6
components/security_apps/waap/waap_clib/ConfidenceCalculator.cc
View File

@ -642,7 +642,11 @@ void ConfidenceCalculator::calculateInterval()
double factor = 1.0;
if (m_tuning != nullptr)
{
std::string param_name = key.substr(key.find("#") + 1); // not always accurate but good enough
std::string param_name = key;
auto param_name_pos = key.find("#");
if (param_name_pos != std::string::npos && (param_name_pos + 1) <= key.size()) {
param_name = key.substr(param_name_pos + 1); // not always accurate but good enough
}
if (m_tuning->getDecision(param_name, PARAM_NAME) == BENIGN)
{
factor = BENIGN_PARAM_FACTOR;

4
components/security_apps/waap/waap_clib/CsrfPolicy.h
View File

@ -30,6 +30,10 @@ struct Policy {
enable(false),
enforce(false)
{
bool web_attack_on;
ar(cereal::make_nvp("webAttackMitigation", web_attack_on));
if (!web_attack_on) return;
std::string level;
ar(cereal::make_nvp("csrfProtection", level));
level = boost::algorithm::to_lower_copy(level);

1
components/security_apps/waap/waap_clib/DecisionFactory.cc
View File

@ -95,6 +95,7 @@ void DecisionFactory::initCsrfDecision()
}
}
void DecisionFactory::initOpenRedirectDecision()
{
DecisionType type = DecisionType::OPEN_REDIRECT_DECISION;

66
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@ -63,6 +63,7 @@ DeepParser::~DeepParser()
{
}
void DeepParser::setWaapAssetState(std::shared_ptr<WaapAssetState> pWaapAssetState)
{
m_pWaapAssetState = pWaapAssetState;
@ -284,18 +285,21 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i
if (flags & BUFFERED_RECEIVER_F_FIRST)
{
createInternalParser(orig_val,
createInternalParser(k, k_len, orig_val,
valueStats,
isBodyPayload,
isRefererPayload,
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload);
isUrlParamPayload,
flags);
}
// If there's a parser in parsers stack, push the value to the top parser
if (!m_parsersDeque.empty() && !m_parsersDeque.front()->getRecursionFlag())
{
ScopedContext ctx;
ctx.registerValue<IWaf2Transaction*>("waap_transaction", m_pTransaction);
rc = pushValueToTopParser(cur_val, flags, base64ParamFound);
if (rc != CONTINUE_PARSING)
{
@ -680,13 +684,32 @@ int DeepParser::pushValueToTopParser(std::string& cur_val, int flags, bool base6
return CONTINUE_PARSING;
}
void DeepParser::createInternalParser(std::string& cur_val,
class StubParserReceiver : public IParserReceiver {
int
onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags)
{
return 0;
}
};
static bool
validateJson(const char *v, size_t v_len)
{
StubParserReceiver rcvr;
ParserJson jsParser(rcvr);
jsParser.push(v, v_len);
dbgTrace(D_WAAP_DEEP_PARSER) << "json validation: " << (jsParser.error() ? "invalid" : "valid");
return !jsParser.error();
}
void DeepParser::createInternalParser(const char *k, size_t k_len, std::string& cur_val,
const ValueStatsAnalyzer &valueStats,
bool isBodyPayload,
bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload)
bool isUrlParamPayload,
int flags)
{
bool isPipesType = false, isSemicolonType = false, isAsteriskType = false,
isCommaType = false, isAmperType = false;
@ -795,12 +818,31 @@ void DeepParser::createInternalParser(std::string& cur_val,
}
// This flag is enabled when current value is either top level (depth==1), or one-level inside multipart-encoded
// container (depth==2 and type of top parser is )
// container (depth==2 and type of top parser is "ParserMultipartForm")
bool isTopData = m_depth == 1
|| (m_depth == 2 && !m_parsersDeque.empty() && m_parsersDeque.front()->name() == "ParserMultipartForm");
// GQL query can potentially be in one of three places in HTTP request:
// 1. In url parameter named "query"
// 2. In the body when Content-Type is "application/graphql"
// 3. In the JSON contained in body, where top-level JSON parameter is named "query"
// Note: we consider decoding Graphql format only if it is contained whole within the MAX_VALUE_SIZE (64k) buffer
// size (you can find the value of MAX_VALUE_SIZE defined in ParserBase.cc).
Waap::Util::ContentType requestContentType = m_pTransaction->getContentType();
bool isPotentialGqlQuery = false;
if (flags == BUFFERED_RECEIVER_F_BOTH) { // TODO:: should we limit ourselves to the 64k buffer?
static std::string strQuery("query");
bool isParamQuery = strQuery.size() == k_len && std::equal(k, k + k_len, strQuery.begin());
isPotentialGqlQuery |= isParamQuery && m_depth == 1 && (isUrlParamPayload || isRefererParamPayload);
isPotentialGqlQuery |= m_depth == 1 && isBodyPayload && requestContentType == Waap::Util::CONTENT_TYPE_GQL;
isPotentialGqlQuery |= isParamQuery && m_depth == 2 && isBodyPayload &&
requestContentType == Waap::Util::CONTENT_TYPE_JSON;
}
dbgTrace(D_WAAP_DEEP_PARSER)
<< "isTopData="
<< "isPotentialGqlQuery="
<< isPotentialGqlQuery
<< ";isTopData="
<< isTopData
<< ";depth="
<< m_depth
@ -838,6 +880,8 @@ void DeepParser::createInternalParser(std::string& cur_val,
// JSON value detected
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a JSON file";
// Send openApiReceiver as secondary receiver, but only if the JSON is passed in body and on the top level.
m_parsersDeque.push_front(std::make_shared<BufferedParser<ParserJson>>(*this));
}
}
@ -1020,6 +1064,16 @@ bool DeepParser::isBinaryData() const
return false;
}
const std::string DeepParser::getLastParser() const
{
if (m_parsersDeque.empty()) {
return "";
}
else {
return m_parsersDeque.front()->name();
}
}
bool DeepParser::isWBXmlData() const
{
return m_is_wbxml;

6
components/security_apps/waap/waap_clib/DeepParser.h
View File

@ -43,6 +43,7 @@ public:
void setMultipartBoundary(const std::string &boundary);
const std::string &getMultipartBoundary() const;
bool isBinaryData() const;
const std::string getLastParser() const;
bool isWBXmlData() const;
Maybe<std::string> getSplitType() const;
std::vector<std::pair<std::string, std::string> > kv_pairs;
@ -114,13 +115,14 @@ private:
// note: This function calls onKv(), and the call can be recursive!
// TODO:: maybe convert this splitter to Parser-derived class?!
bool splitByRegex(const std::string &val, const Regex &r, const char *keyPrefix);
void createInternalParser(std::string& cur_val,
void createInternalParser(const char *k, size_t k_len, std::string& cur_val,
const ValueStatsAnalyzer &valueStats,
bool isBodyPayload,
bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload);
bool isUrlParamPayload,
int flags);
int pushValueToTopParser(std::string& cur_val, int flags, bool base64ParamFound);
int parseBuffer(ValueStatsAnalyzer& valueStats, const std::string &cur_val, bool base64ParamFound,
bool shouldUpdateKeyStack);

8
components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc
View File

@ -23,13 +23,13 @@
IndicatorsFiltersManager::IndicatorsFiltersManager(const std::string& remotePath, const std::string &assetId,
I_WaapAssetState* pWaapAssetState)
:
SerializeToFileBase(pWaapAssetState->getSignaturesFilterDir() + "/6.data"),
m_ignoreSources(pWaapAssetState->getSignaturesFilterDir(), remotePath, assetId),
SerializeToFileBase(pWaapAssetState->getWaapDataDir() + "/6.data"),
m_ignoreSources(pWaapAssetState->getWaapDataDir(), remotePath, assetId),
m_tuning(remotePath)
{
restore();
m_keywordsFreqFilter = std::make_unique<KeywordIndicatorFilter>(
pWaapAssetState->getSignaturesFilterDir(),
pWaapAssetState->getWaapDataDir(),
remotePath,
assetId,
&m_ignoreSources,
@ -206,7 +206,7 @@ std::string IndicatorsFiltersManager::extractUri(const std::string& referer, con
std::string url;
size_t pos = referer.find("://");
if (pos == std::string::npos)
if (pos == std::string::npos || (pos + 3) > referer.size())
{
url = referer;
}

35
components/security_apps/waap/waap_clib/KeywordTypeValidator.cc
View File

@ -16,13 +16,13 @@
#include <cereal/types/unordered_map.hpp>
#include <cereal/types/unordered_set.hpp>
#include "debug.h"
#include "Waf2Util.h"
USE_DEBUG_FLAG(D_WAAP);
KeywordTypeValidator::KeywordTypeValidator(const std::string& mapFilePath) :
SerializeToFileBase(mapFilePath),
m_keywordTypeMap()
m_serializedData(),
m_keywordTypeMap(m_serializedData.m_keywordTypeMap)
{
restore();
}
@ -44,25 +44,22 @@ void KeywordTypeValidator::saveData()
void KeywordTypeValidator::deserialize(std::istream& stream)
{
cereal::JSONInputArchive archive(stream);
std::unordered_map<std::string, std::unordered_set<std::string>> typesStrToKeysMap;
archive(cereal::make_nvp("keywordsTypeMap", typesStrToKeysMap));
for (auto typeStrItr : typesStrToKeysMap)
try
{
ParamType type = Waap::Util::convertTypeStrToEnum(typeStrItr.first);
for (auto keyword : typeStrItr.second)
{
if (m_keywordTypeMap.find(keyword) == m_keywordTypeMap.end())
{
// initialize type set
m_keywordTypeMap[keyword];
}
m_keywordTypeMap[keyword].insert(type);
}
cereal::JSONInputArchive archive(stream);
archive(
cereal::make_nvp("waap_kw_type_map", m_serializedData)
);
}
catch (std::runtime_error & e) {
dbgWarning(D_WAAP) << "failed to deserialize keyword types validator file. Error: " << e.what();
}
}
void KeywordTypeValidator::operator=(const KeywordTypeValidator &other) {
m_serializedData.m_keywordTypeMap = other.m_serializedData.m_keywordTypeMap;
}
bool KeywordTypeValidator::isKeywordOfType(const std::string& keyword, ParamType type) const

31
components/security_apps/waap/waap_clib/KeywordTypeValidator.h
View File

@ -16,6 +16,7 @@
#include <unordered_map>
#include <unordered_set>
#include "WaapEnums.h"
#include "Waf2Util.h"
#include "i_serialize.h"
class KeywordTypeValidator : public SerializeToFileBase
@ -30,6 +31,34 @@ public:
virtual void deserialize(std::istream& stream);
virtual void saveData();
void operator=(const KeywordTypeValidator &other);
private:
std::unordered_map<std::string, std::unordered_set<ParamType>> m_keywordTypeMap;
struct SerializedData {
template <class Archive>
void serialize(Archive& archive) {
std::unordered_map<std::string, std::unordered_set<std::string>> typesStrToKeysMap;
archive(cereal::make_nvp("keywordsTypeMap", typesStrToKeysMap));
for (auto typeStrItr : typesStrToKeysMap)
{
ParamType type = Waap::Util::convertTypeStrToEnum(typeStrItr.first);
for (auto keyword : typeStrItr.second)
{
if (m_keywordTypeMap.find(keyword) == m_keywordTypeMap.end())
{
// initialize type set
m_keywordTypeMap[keyword];
}
m_keywordTypeMap[keyword].insert(type);
}
}
}
std::unordered_map<std::string, std::unordered_set<ParamType>> m_keywordTypeMap;
};
SerializedData m_serializedData;
std::unordered_map<std::string, std::unordered_set<ParamType>> &m_keywordTypeMap;
};

5
components/security_apps/waap/waap_clib/ParserJson.cc
View File

@ -102,11 +102,12 @@ int ParserJson::cb_string(const unsigned char* s, yajl_size_t slen) {
int ParserJson::cb_map_key(const unsigned char* s, yajl_size_t slen) {
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_map_key(): '" << std::string((const char*)s, slen) << "'";
m_key.push((char*)s, slen);
if (m_receiver2) {
m_receiver2->onMapKey((const char*)s, slen);
m_receiver2->onMapKey(m_key.c_str(), m_key.size());
}
m_key.push((char*)s, slen);
return 1;
}

11
components/security_apps/waap/waap_clib/ParserXML.cc
View File

@ -15,6 +15,8 @@
#include "Waf2Util.h"
#include "debug.h"
#include <assert.h>
#include <boost/algorithm/string/case_conv.hpp>
#include <string>
USE_DEBUG_FLAG(D_WAAP_PARSER_XML);
@ -34,6 +36,15 @@ void ParserXML::onStartElementNs(
ParserXML* p = (ParserXML*)ctx;
dbgTrace(D_WAAP_PARSER_XML) << "XML OPEN: '" << localname << "'";
std::string aux_localname((const char*)localname, xmlStrlen(localname));
boost::algorithm::to_lower(aux_localname);
if (aux_localname == "script") {
dbgTrace(D_WAAP_PARSER_XML) << "Failing parser on <script> tag";
p->m_state = s_error;
}
p->m_key.push((const char*)localname, xmlStrlen(localname));
int i;

85
components/security_apps/waap/waap_clib/ScoreBuilder.cc
View File

@ -99,10 +99,10 @@ void KeywordsScorePool::mergeScores(const KeywordsScorePool& baseScores)
ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState) :
SerializeToFilePeriodically(duration_cast<seconds>(minutes(10)), pWaapAssetState->getSignaturesScoresFilePath()),
m_scoreTrigger(0),
m_fpStore(),
m_keywordsScorePools(),
m_serializedData(),
m_keywordsScorePools(m_serializedData.m_keywordsScorePools),
m_falsePositivesSetsIntersection(),
m_pWaapAssetState(pWaapAssetState)
{
@ -110,10 +110,10 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState) :
}
ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& baseScores) :
SerializeToFilePeriodically(duration_cast<seconds>(minutes(10)), pWaapAssetState->getSignaturesScoresFilePath()),
m_scoreTrigger(0),
m_fpStore(),
m_keywordsScorePools(),
m_serializedData(),
m_keywordsScorePools(m_serializedData.m_keywordsScorePools),
m_falsePositivesSetsIntersection(),
m_pWaapAssetState(pWaapAssetState)
{
@ -123,44 +123,52 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& base
mergeScores(baseScores);
}
void ScoreBuilder::serialize(std::ostream& stream) {
cereal::JSONOutputArchive archive(stream);
static const size_t version = 1;
archive(
cereal::make_nvp("version", version),
cereal::make_nvp("scorePools", m_keywordsScorePools)
);
}
void ScoreBuilder::restore()
{
const std::string filePath = this->m_pWaapAssetState->getWaapDataFileName();
void ScoreBuilder::deserialize(std::istream& stream) {
cereal::JSONInputArchive iarchive(stream);
dbgTrace(D_WAAP_SCORE_BUILDER) << "loadFromFile() file: " << filePath;
std::fstream filestream;
size_t version = 0;
try {
iarchive(cereal::make_nvp("version", version));
filestream.open(filePath, std::fstream::in);
if (filestream.is_open() == false) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "failed to open file: " << filePath << " Error: " << errno;
return;
}
dbgTrace(D_WAAP_SCORE_BUILDER) << "loading from file: " << filePath;
int length;
filestream.seekg(0, std::ios::end); // go to the end
length = filestream.tellg(); // report location (this is the length)
dbgTrace(D_WAAP_SCORE_BUILDER) << "file length: " << length;
assert(length >= 0); // length -1 really happens if filePath is a directory (!)
char* buffer = new char[length]; // allocate memory for a buffer of appropriate dimension
filestream.seekg(0, std::ios::beg); // go back to the beginning
if (!filestream.read(buffer, length)) // read the whole file into the buffer
{
filestream.close();
delete[] buffer;
dbgWarning(D_WAAP_SCORE_BUILDER) << "Failed to read file, file: " << filePath;
return;
}
filestream.close();
std::stringstream ss(std::string(buffer, length));
delete[] buffer;
try
{
cereal::JSONInputArchive iarchive(ss);
iarchive(
cereal::make_nvp("waap_scores", m_serializedData)
);
}
catch (std::runtime_error & e) {
iarchive.setNextName(nullptr);
version = 0;
dbgDebug(D_WAAP_SCORE_BUILDER) << "ScoreBuilder version absent, using version " << version <<
" e.what() is " << e.what();
}
dbgDebug(D_WAAP_SCORE_BUILDER) << "Loading scores from file version " << version << "...";
switch (version)
{
case 1: {
iarchive(cereal::make_nvp("scorePools", m_keywordsScorePools));
break;
}
case 0: {
m_keywordsScorePools[KEYWORDS_SCORE_POOL_BASE] = KeywordsScorePool(iarchive);
break;
}
default: {
dbgDebug(D_WAAP_SCORE_BUILDER) << "Unknown scores file version: " << version;
}
dbgWarning(D_WAAP_SCORE_BUILDER) << "failed to deserialize file: " << filePath << ", error: " <<
e.what();
}
}
@ -279,7 +287,6 @@ void ScoreBuilder::pumpKeywordScore(ScoreBuilderData& data, const std::string &p
{
m_pWaapAssetState->updateScores();
}
backupWorker();
}
}

40
components/security_apps/waap/waap_clib/ScoreBuilder.h
View File

@ -24,6 +24,7 @@
#include <cereal/archives/json.hpp>
#include <cereal/types/unordered_map.hpp>
#include <cereal/types/string.hpp>
#include "WaapDefines.h"
struct ScoreBuilderData {
std::string m_sourceIdentifier;
@ -131,7 +132,7 @@ struct KeywordsScorePool {
void mergeScores(const KeywordsScorePool& baseScores);
};
class ScoreBuilder : public SerializeToFilePeriodically {
class ScoreBuilder {
public:
ScoreBuilder(I_WaapAssetState* pWaapAssetState);
ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& baseScores);
@ -152,13 +153,43 @@ public:
keywords_set getUaItemKeywordsSet(std::string userAgent);
unsigned int getFpStoreCount();
virtual void serialize(std::ostream& stream);
virtual void deserialize(std::istream& stream);
void restore();
void mergeScores(const ScoreBuilder& baseScores);
protected:
typedef std::map<std::string, double> KeywordScoreMap;
struct SerializedData {
template <class Archive>
void serialize(Archive& ar) {
size_t version = 0;
try {
ar(cereal::make_nvp("version", version));
}
catch (std::runtime_error & e) {
ar.setNextName(nullptr);
version = 0;
}
switch (version)
{
case 1: {
ar(cereal::make_nvp("scorePools", m_keywordsScorePools));
break;
}
case 0: {
m_keywordsScorePools[KEYWORDS_SCORE_POOL_BASE] = KeywordsScorePool(ar);
break;
}
default: {
break;
}
}
}
std::map<std::string, KeywordsScorePool> m_keywordsScorePools; // live data continuously updated during traffic
};
void pumpKeywordScorePerKeyword(ScoreBuilderData& data,
const std::string& keyword,
KeywordType keywordSource,
@ -166,7 +197,8 @@ protected:
unsigned int m_scoreTrigger;
FalsePoisitiveStore m_fpStore;
std::map<std::string, KeywordsScorePool> m_keywordsScorePools; // live data continuously updated during traffic
SerializedData m_serializedData;
std::map<std::string, KeywordsScorePool> &m_keywordsScorePools; // live data continuously updated during traffic
std::map<std::string, KeywordScoreMap> m_snapshotKwScoreMap; // the snapshot is updated only by a call to snap()
std::list<std::string> m_falsePositivesSetsIntersection;
I_WaapAssetState* m_pWaapAssetState;

12
components/security_apps/waap/waap_clib/Serializator.cc
View File

@ -53,7 +53,8 @@ isGZipped(const std::string &stream)
bool RestGetFile::loadJson(const std::string& json)
{
std::string json_str = json;
std::string json_str = json;
if (isGZipped(json_str) == 0)
{
return ClientRest::loadJson(json_str);
@ -94,6 +95,7 @@ Maybe<std::string> RestGetFile::genJson() const
return genError("Failed to compress data");
}
data = std::string((const char *)res.output, res.num_output_bytes);
json = data;
if (res.output) free(res.output);
@ -175,6 +177,7 @@ void SerializeToFileBase::saveData()
}
serialize(ss);
filestream << ss.str();
filestream.close();
}
@ -235,6 +238,7 @@ void SerializeToFileBase::loadFromFile(std::string filePath)
delete[] buffer;
std::stringstream ss(dataObfuscated);
try
@ -252,12 +256,6 @@ void SerializeToFileBase::restore()
loadFromFile(m_filePath);
}
void SerializeToFileBase::setFilePath(const std::string& new_file_path)
{
m_filePath = new_file_path;
}
RemoteFilesList::RemoteFilesList() : files(), filesPathsList()
{

14
components/security_apps/waap/waap_clib/Signatures.cc
View File

@ -12,7 +12,6 @@
// limitations under the License.
#include "Signatures.h"
#include "i_encryptor.h"
#include "waap.h"
#include <fstream>
@ -242,13 +241,13 @@ bool Signatures::fail()
return error;
}
picojson::value::object Signatures::loadSource(const std::string& sigsFname)
picojson::value::object Signatures::loadSource(const std::string& waapDataFileName)
{
picojson::value doc;
std::ifstream f(sigsFname.c_str());
std::ifstream f(waapDataFileName);
if (f.fail()) {
dbgError(D_WAAP) << "Failed to open json data file '" << sigsFname << "'!";
dbgError(D_WAAP) << "Failed to open json data file '" << waapDataFileName << "'!";
error = true; // flag an error
return picojson::value::object();
}
@ -264,15 +263,18 @@ picojson::value::object Signatures::loadSource(const std::string& sigsFname)
std::string dataObfuscated(buffer, length);
delete[] buffer;
std::stringstream ss(dataObfuscated);
ss >> doc;
if (!picojson::get_last_error().empty()) {
dbgError(D_WAAP) << "WaapAssetState::loadSource('" << sigsFname << "') failed (parse error: '" <<
dbgError(D_WAAP) << "WaapAssetState::loadSource('" << waapDataFileName << "') failed (parse error: '" <<
picojson::get_last_error() << "').";
error = true; // flag an error
return picojson::value::object();
}
return doc.get<picojson::value::object>();
return doc.get<picojson::value::object>()["waap_signatures"].get<picojson::value::object>();
}

2
components/security_apps/waap/waap_clib/Signatures.h
View File

@ -87,7 +87,7 @@ public:
const boost::regex wbxml_data_kw_filter;
private:
picojson::value::object loadSource(const std::string& sigsFname);
picojson::value::object loadSource(const std::string& waapDataFileName);
};
#endif

4
components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc
View File

@ -36,8 +36,8 @@ TypeIndicatorFilter::TypeIndicatorFilter(I_WaapAssetState* pWaapAssetState,
size_t minIntervals,
std::chrono::minutes intervalDuration,
double ratioThreshold) :
IndicatorFilterBase(TYPES_FILTER_PATH(pWaapAssetState->getSignaturesFilterDir()),
TYPES_FILTER_TRUST_PATH(pWaapAssetState->getSignaturesFilterDir()),
IndicatorFilterBase(TYPES_FILTER_PATH(pWaapAssetState->getWaapDataDir()),
TYPES_FILTER_TRUST_PATH(pWaapAssetState->getWaapDataDir()),
(remotePath == "") ? remotePath : remotePath + "/Type",
assetId,
minSources,

163
components/security_apps/waap/waap_clib/WaapAssetState.cc
View File

@ -75,21 +75,38 @@ static void print_found_patterns(const Waap::Util::map_of_stringlists_t& m) {
#endif
static bool err_hex = false;
static const std::string path_traversal_chars_regex = "[\\w.%?*\\/\\\\]";
static const std::string evasion_hex_regex_unallowed_prefix_helper =
"(?:(?<!(?<!0x|%u)[0-9a-f][0-9a-f])|(?<!(?<!%)[0-9a-f][0-9a-f]))";
static const std::string evasion_hex_regex_helper = "(0x[0-9a-f][0-9a-f])";
static const SingleRegex evasion_hex_regex(
"(0x[0-9a-f][0-9a-f])[\\w.%?*\\/\\\\]|[\\w.%?*\\/\\\\](0x[0-9a-f][0-9a-f])",
evasion_hex_regex_unallowed_prefix_helper + evasion_hex_regex_helper + path_traversal_chars_regex +
"|" + path_traversal_chars_regex + evasion_hex_regex_unallowed_prefix_helper + evasion_hex_regex_helper,
err_hex,
"evasion_hex_regex");
static const boost::regex bad_hex_regex = boost::regex("%[cC]1%[19][cC]");
static const std::string bad_hex_regex_helper = "(%[cC]1%(([19][cC])|([pP][cC])|(8[sS])))";
static const boost::regex bad_hex_regex(bad_hex_regex_helper);
static const SingleRegex evasion_bad_hex_regex(
"(%[cC]1%[19][cC])[\\w.%?*\\/\\\\]|[\\w.%?*\\/\\\\](%[cC]1%[19][cC])",
bad_hex_regex_helper + path_traversal_chars_regex +
"|" + path_traversal_chars_regex + bad_hex_regex_helper,
err_hex,
"evasion_bad_hex_regex");
static const std::string utf_evasion_for_dot_helper =
"(%[cC]0%[562aAfFeE][eE])";
static const SingleRegex utf_evasion_for_dot(
utf_evasion_for_dot_helper + path_traversal_chars_regex +
"|" + path_traversal_chars_regex + utf_evasion_for_dot_helper,
err_hex,
"utf_evasion_for_dot");
static const boost::regex utf_evasion_for_dot_regex(utf_evasion_for_dot_helper);
static const std::string sqli_comma_evasion_regex_helper = "\"\\s*,\\s*\"";
static const boost::regex sqli_comma_evasion_regex(sqli_comma_evasion_regex_helper);
WaapAssetState::WaapAssetState(const std::shared_ptr<WaapAssetState>& pWaapAssetState,
const std::string& sigScoresFname,
const std::string& waapDataFileName,
const std::string& id) :
WaapAssetState(pWaapAssetState->m_Signatures,
sigScoresFname,
waapDataFileName,
pWaapAssetState->m_cleanValuesCache.capacity(),
pWaapAssetState->m_suspiciousValuesCache.capacity(),
pWaapAssetState->m_sampleTypeCache.capacity(),
@ -110,20 +127,22 @@ WaapAssetState::WaapAssetState(const std::shared_ptr<WaapAssetState>& pWaapAsset
}
WaapAssetState::WaapAssetState(std::shared_ptr<Signatures> signatures,
const std::string& sigScoresFname,
const std::string& waapDataFileName,
size_t cleanValuesCacheCapacity,
size_t suspiciousValuesCacheCapacity,
size_t sampleTypeCacheCapacity,
const std::string& assetId) :
m_Signatures(signatures),
m_SignaturesScoresFilePath(sigScoresFname),
m_waapDataFileName(waapDataFileName),
m_assetId(assetId),
scoreBuilder(this),
m_rateLimitingState(nullptr),
m_errorLimitingState(nullptr),
m_securityHeadersState(nullptr),
m_filtersMngr(nullptr),
m_typeValidator(getSignaturesFilterDir() + "/8.data"),
m_typeValidator(getWaapDataDir() + "/waap.data"),
m_cleanValuesCache(cleanValuesCacheCapacity),
m_suspiciousValuesCache(suspiciousValuesCacheCapacity),
m_sampleTypeCache(sampleTypeCacheCapacity)
@ -1039,7 +1058,7 @@ WaapAssetState::apply(
if (found != std::string::npos)
{
unescaped += res.unescaped_line.substr(pos, found-pos);
if (found < res.unescaped_line.size() - 3 &&
if (found + 3 < res.unescaped_line.size() &&
res.unescaped_line[found+1] == res.unescaped_line[found+2] && res.unescaped_line[found+3] == ']')
{
unescaped += res.unescaped_line[found+1];
@ -1145,16 +1164,11 @@ WaapAssetState::apply(
}
}
std::string *utf8_broken_line_ptr = nullptr;
if (Waap::Util::containsBrokenUtf8(unquote_line)) {
utf8_broken_line_ptr = &unquote_line;
} else if (Waap::Util::containsBrokenUtf8(line)) {
utf8_broken_line_ptr = (std::string*)&line;
}
Maybe<std::string> broken_utf8_line = Waap::Util::containsBrokenUtf8(line, unquote_line);
if (utf8_broken_line_ptr) {
if (broken_utf8_line.ok()) {
dbgTrace(D_WAAP_EVASIONS) << "broken-down utf-8 evasion found";
std::string unescaped = Waap::Util::unescapeBrokenUtf8(*utf8_broken_line_ptr);
std::string unescaped = Waap::Util::unescapeBrokenUtf8(broken_utf8_line.unpack());
size_t kwCount = res.keyword_matches.size();
unescaped = unescape(unescaped);
@ -1283,6 +1297,35 @@ WaapAssetState::apply(
}
}
boost::cmatch what;
if (boost::regex_search(res.unescaped_line.c_str(), what, sqli_comma_evasion_regex)) {
// Possible SQLi evasion detected (","): - clean up and scan with regexes again.
dbgTrace(D_WAAP_EVASIONS) << "Possible SQLi evasion detected (\",\"): - clean up and scan with regexes again.";
std::string unescaped = res.unescaped_line;
unescaped = boost::regex_replace(unescaped, sqli_comma_evasion_regex, "");
unescaped = unescape(unescaped);
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
}
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
newWordsCount);
// Take minimal words count because empirically it means evasion was probably succesfully decoded
wordsCount = std::min(wordsCount, newWordsCount);
}
if ((res.unescaped_line.find("0x") != std::string::npos) && evasion_hex_regex.hasMatch(res.unescaped_line)) {
dbgTrace(D_WAAP_EVASIONS) << "hex evasion found (in unescaped line)";
@ -1306,9 +1349,10 @@ WaapAssetState::apply(
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
for (const auto &kw : res.keyword_matches) {
if (kw.size() < 2 || str_contains(kw, "os_cmd_high_acuracy_fast_reg") ||
kw == "os_cmd_sep_medium_acuracy" || str_contains(kw, "regex_code_execution") ||
str_contains(kw, "regex_code_execution") || kw == "character_encoding" ||
str_contains(kw, "quotes_ev_fast_reg") || str_contains(kw, "encoded_") ||
str_contains(kw, "medium_acuracy") || str_contains(kw, "high_acuracy_fast_reg_xss"))
str_contains(kw, "quotes_ev_fast_reg") || str_contains(kw, "encoded_") ||
str_contains(kw, "medium_acuracy") || str_contains(kw, "high_acuracy_fast_reg_xss"))
{
keywordsToRemove.push_back(kw);
}
@ -1333,7 +1377,7 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
if (line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, false, binaryDataFound);
@ -1346,9 +1390,10 @@ WaapAssetState::apply(
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
for (const auto &kw : res.keyword_matches) {
if (kw.size() < 2 || str_contains(kw, "os_cmd_high_acuracy_fast_reg") ||
kw == "os_cmd_sep_medium_acuracy" || str_contains(kw, "regex_code_execution") ||
str_contains(kw, "regex_code_execution") || kw == "character_encoding" ||
str_contains(kw, "quotes_ev_fast_reg") || str_contains(kw, "encoded_") ||
str_contains(kw, "medium_acuracy") || str_contains(kw, "high_acuracy_fast_reg_xss"))
str_contains(kw, "quotes_ev_fast_reg") || str_contains(kw, "encoded_") ||
str_contains(kw, "medium_acuracy") || str_contains(kw, "high_acuracy_fast_reg_xss"))
{
keywordsToRemove.push_back(kw);
}
@ -1405,6 +1450,37 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
newWordsCount);
// Take minimal words count because empirically it means evasion was probably succesfully decoded
wordsCount = std::min(wordsCount, newWordsCount);
}
}
if ((res.unescaped_line.find("%") != std::string::npos) && utf_evasion_for_dot.hasMatch(res.unescaped_line)) {
dbgTrace(D_WAAP_EVASIONS) <<
"UTF evasion for dot found (%c0%*e) in unescaped line";
std::string unescaped = res.unescaped_line;
unescaped = boost::regex_replace(unescaped, utf_evasion_for_dot_regex, ".");
unescaped = unescape(unescaped);
dbgTrace(D_WAAP_EVASIONS) << "unescaped == '" << unescaped << "'";
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
@ -1425,6 +1501,38 @@ WaapAssetState::apply(
}
}
if ((line.find("%") != std::string::npos) && utf_evasion_for_dot.hasMatch(line)) {
dbgTrace(D_WAAP_EVASIONS) << "UTF evasion for dot found (%c0%*e) in raw line";
std::string unescaped = line;
unescaped = boost::regex_replace(unescaped, utf_evasion_for_dot_regex, ".");
unescaped = unescape(unescaped);
dbgTrace(D_WAAP_EVASIONS) << "unescaped == '" << unescaped << "'";
size_t kwCount = res.keyword_matches.size();
if (line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
newWordsCount);
// Take minimal words count because empirically it means evasion was probably succesfully decoded
wordsCount = std::min(wordsCount, newWordsCount);
}
}
// python: escape ='hi_acur_fast_reg_evasion' in found_patterns
bool escape = Waap::Util::find_in_map_of_stringlists_keys("evasion", res.found_patterns);
@ -1690,8 +1798,8 @@ void WaapAssetState::updateScores()
scoreBuilder.snap();
}
std::string WaapAssetState::getSignaturesScoresFilePath() const {
return m_SignaturesScoresFilePath;
std::string WaapAssetState::getWaapDataFileName() const {
return m_waapDataFileName;
}
std::map<std::string, std::vector<std::string>>& WaapAssetState::getFilterVerbose()
@ -1699,10 +1807,10 @@ std::map<std::string, std::vector<std::string>>& WaapAssetState::getFilterVerbos
return m_filtered_keywords_verbose;
}
std::string WaapAssetState::getSignaturesFilterDir() const {
size_t lastSlash = m_SignaturesScoresFilePath.find_last_of('/');
std::string WaapAssetState::getWaapDataDir() const {
size_t lastSlash = m_waapDataFileName.find_last_of('/');
std::string sigsFilterDir = ((lastSlash == std::string::npos) ?
m_SignaturesScoresFilePath : m_SignaturesScoresFilePath.substr(0, lastSlash));
m_waapDataFileName : m_waapDataFileName.substr(0, lastSlash));
dbgTrace(D_WAAP_ASSET_STATE) << " signatures filters directory: " << sigsFilterDir;
return sigsFilterDir;
}
@ -1948,6 +2056,7 @@ std::shared_ptr<Waap::SecurityHeaders::State>& WaapAssetState::getSecurityHeader
return m_securityHeadersState;
}
void WaapAssetState::clearRateLimitingState()
{
m_rateLimitingState.reset();

14
components/security_apps/waap/waap_clib/WaapAssetState.h
View File

@ -24,7 +24,6 @@
#include <set>
#include <boost/noncopyable.hpp>
#include "ScoreBuilder.h"
#include "i_encryptor.h"
#include "i_waap_asset_state.h"
#include "RateLimiting.h"
#include "SecurityHeadersPolicy.h"
@ -41,7 +40,7 @@ class WaapAssetState : public boost::noncopyable, public I_WaapAssetState
{
private: //ugly but needed for build
std::shared_ptr<Signatures> m_Signatures;
std::string m_SignaturesScoresFilePath;
std::string m_waapDataFileName;
std::map<std::string, std::vector<std::string>> m_filtered_keywords_verbose;
void checkRegex(const SampleValue &sample, const Regex & pattern, std::vector<std::string>& keyword_matches,
@ -51,13 +50,13 @@ private: //ugly but needed for build
public:
// Load and compile signatures from file
explicit WaapAssetState(std::shared_ptr<Signatures> signatures, const std::string& sigScoresFname,
explicit WaapAssetState(std::shared_ptr<Signatures> signatures, const std::string& waapDataFileName,
size_t cleanCacheCapacity = SIGS_APPLY_CLEAN_CACHE_CAPACITY,
size_t suspiciousCacheCapacity = SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY,
size_t sampleTypeCacheCapacity = SIGS_SAMPLE_TYPE_CACHE_CAPACITY,
const std::string& assetId = "");
explicit WaapAssetState(const std::shared_ptr<WaapAssetState>& pWaapAssetState, const std::string& sigScoresFname,
const std::string& assetId);
explicit WaapAssetState(const std::shared_ptr<WaapAssetState>& pWaapAssetState,
const std::string& waapDataFileName, const std::string& assetId);
virtual ~WaapAssetState();
std::shared_ptr<Signatures> getSignatures() const;
@ -76,8 +75,8 @@ public:
const Maybe<std::string> splitType=genError("not splitted")) const;
virtual void updateScores();
virtual std::string getSignaturesScoresFilePath() const;
virtual std::string getSignaturesFilterDir() const;
virtual std::string getWaapDataFileName() const;
virtual std::string getWaapDataDir() const;
std::map<std::string, std::vector<std::string>>& getFilterVerbose();
void updateFilterManagerPolicy(IWaapConfig* pConfig);
@ -105,6 +104,7 @@ public:
void clearErrorLimitingState();
void clearSecurityHeadersState();
std::shared_ptr<Waap::RateLimiting::State>& getRateLimitingState();
std::shared_ptr<Waap::RateLimiting::State>& getErrorLimitingState();
std::shared_ptr<Waap::SecurityHeaders::State>& getSecurityHeadersState();

16
components/security_apps/waap/waap_clib/WaapAssetStatesManager.cc
View File

@ -33,9 +33,9 @@ void WaapAssetStatesManager::preload()
registerExpectedConfiguration<std::string>("waap data", "base folder");
}
bool WaapAssetStatesManager::initBasicWaapSigs(const std::string& sigsFname, const std::string& sigScoresFname)
bool WaapAssetStatesManager::initBasicWaapSigs(const std::string& waapDataFileName)
{
return pimpl->initBasicWaapSigs(sigsFname, sigScoresFname);
return pimpl->initBasicWaapSigs(waapDataFileName);
}
std::shared_ptr<WaapAssetState> WaapAssetStatesManager::getWaapAssetStateGlobal()
@ -65,7 +65,7 @@ WaapAssetStatesManager::Impl::~Impl()
{
}
bool WaapAssetStatesManager::Impl::initBasicWaapSigs(const std::string& sigsFname, const std::string& sigScoresFname)
bool WaapAssetStatesManager::Impl::initBasicWaapSigs(const std::string& waapDataFileName)
{
if (m_signatures && !m_signatures->fail() && m_basicWaapSigs)
{
@ -73,18 +73,18 @@ bool WaapAssetStatesManager::Impl::initBasicWaapSigs(const std::string& sigsFnam
return true;
}
try {
m_signatures = std::make_shared<Signatures>(sigsFname);
m_signatures = std::make_shared<Signatures>(waapDataFileName);
m_basicWaapSigs = std::make_shared<WaapAssetState>(
m_signatures,
sigScoresFname,
waapDataFileName,
SIGS_APPLY_CLEAN_CACHE_CAPACITY,
SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY);
}
catch (std::runtime_error & e) {
// TODO:: properly handle component initialization failure
dbgTrace(D_WAAP) <<
"WaapAssetStatesManager::initBasicWaapSigs(): " << e.what() << ". Failed to read signature files"
" "<< sigsFname << " and " << sigScoresFname << ".";
"WaapAssetStatesManager::initBasicWaapSigs(): " << e.what() << ". Failed to read data file '" <<
waapDataFileName << "'";
m_basicWaapSigs.reset();
return false;
}
@ -178,7 +178,7 @@ WaapAssetStatesManager::Impl::CreateWaapSigsForAsset(const std::shared_ptr<WaapA
}
std::string basePath = pWaapAssetState->getSignaturesScoresFilePath();
std::string basePath = pWaapAssetState->getWaapDataFileName();
size_t lastSlash = basePath.find_last_of('/');
std::string assetScoresPath = assetPath +
((lastSlash == std::string::npos) ? basePath : basePath.substr(lastSlash));

6
components/security_apps/waap/waap_clib/WaapAssetStatesManager.h
View File

@ -24,7 +24,7 @@ class WaapAssetState;
class I_WaapAssetStatesManager {
public:
virtual bool initBasicWaapSigs(const std::string& sigsFname, const std::string& sigScoresFname) = 0;
virtual bool initBasicWaapSigs(const std::string& waapDataFileName) = 0;
virtual std::shared_ptr<WaapAssetState> getWaapAssetStateGlobal() = 0;
virtual std::shared_ptr<WaapAssetState> getWaapAssetStateById(const std::string& assetId) = 0;
virtual void setAssetDirectoryPath(const std::string &assetDirectoryPath) = 0;
@ -36,7 +36,7 @@ public:
virtual ~WaapAssetStatesManager();
void preload();
virtual bool initBasicWaapSigs(const std::string& sigsFname, const std::string& sigScoresFname);
virtual bool initBasicWaapSigs(const std::string& waapDataFileName);
virtual std::shared_ptr<WaapAssetState> getWaapAssetStateGlobal();
virtual std::shared_ptr<WaapAssetState> getWaapAssetStateById(const std::string& assetId);
@ -53,7 +53,7 @@ public:
Impl();
virtual ~Impl();
virtual bool initBasicWaapSigs(const std::string& sigsFname, const std::string& sigScoresFname);
virtual bool initBasicWaapSigs(const std::string& waapDataFileName);
virtual std::shared_ptr<WaapAssetState> getWaapAssetStateGlobal();
virtual std::shared_ptr<WaapAssetState> getWaapAssetStateById(const std::string& assetId);
virtual void setAssetDirectoryPath(const std::string &assetDirectoryPath);

5
components/security_apps/waap/waap_clib/WaapConfigApi.cc
View File

@ -66,7 +66,7 @@ WaapConfigAPI::WaapConfigAPI(
string practiceName,
string ruleId,
string ruleName,
bool schemaValidation) :
bool schemaValidation) :
WaapConfigBase(
autonomousSecurity,
autonomousSecurityLevel,
@ -84,8 +84,10 @@ WaapConfigAPI::WaapConfigAPI(
void WaapConfigAPI::load(cereal::JSONInputArchive& ar)
{
// order has affect - we need to call base last because of triggers and overrides
readJSONByCereal(ar);
WaapConfigBase::load(ar);
assets_ids_aggregation.insert(m_assetId);
}
@ -94,6 +96,7 @@ void WaapConfigAPI::readJSONByCereal(cereal::JSONInputArchive &ar)
{
}
bool WaapConfigAPI::operator==(const WaapConfigAPI& other) const
{
const WaapConfigBase* configBase = this;

2
components/security_apps/waap/waap_clib/WaapConfigApi.h
View File

@ -18,6 +18,7 @@
#include <set>
#include "WaapConfigBase.h"
#include "log_generator.h"
#include "debug.h"
@ -50,6 +51,7 @@ public:
private:
void readJSONByCereal(cereal::JSONInputArchive&ar);
std::string m_schemaValidationPoicyStatusMessage;
static const std::string s_PracticeSubType;

4
components/security_apps/waap/waap_clib/WaapConfigBase.cc
View File

@ -79,6 +79,7 @@ void WaapConfigBase::readJSONByCereal(cereal::JSONInputArchive& ar)
m_blockingLevel = blockingLevelBySensitivityStr(m_autonomousSecurityLevel);
}
void WaapConfigBase::loadCsrfPolicy(cereal::JSONInputArchive& ar)
{
std::string failMessage = "Failed to load the CSRF policy of the current rule: " +
@ -240,6 +241,7 @@ void WaapConfigBase::loadOpenRedirectPolicy(cereal::JSONInputArchive& ar)
}
}
void WaapConfigBase::loadErrorDisclosurePolicy(cereal::JSONInputArchive& ar)
{
std::string failMessage = "Failed to load the WAAP Information Disclosure policy";
@ -386,6 +388,7 @@ const std::shared_ptr<Waap::TrustedSources::TrustedSourcesParameter>& WaapConfig
return m_trustedSourcesPolicy;
}
const std::shared_ptr<Waap::Csrf::Policy>& WaapConfigBase::get_CsrfPolicy() const
{
return m_csrfPolicy;
@ -411,6 +414,7 @@ const std::shared_ptr<Waap::OpenRedirect::Policy>& WaapConfigBase::get_OpenRedir
return m_openRedirectPolicy;
}
const std::shared_ptr<Waap::ErrorDisclosure::Policy>& WaapConfigBase::get_ErrorDisclosurePolicy() const
{
return m_errorDisclosurePolicy;

1
components/security_apps/waap/waap_clib/WaapDecision.cc
View File

@ -13,6 +13,7 @@
#include "WaapDecision.h"
#include "OpenRedirectDecision.h"
#include "debug.h"
#include <algorithm>
#include <type_traits>

1
components/security_apps/waap/waap_clib/WaapDecision.h
View File

@ -24,6 +24,7 @@
#include "AutonomousSecurityDecision.h"
#include <iterator>
std::ostream& operator<<(std::ostream& os, const std::list<std::shared_ptr<SingleDecision>>& decisions);
std::ostream& operator<<(std::ostream& os, const DecisionsArr& decisions);
typedef std::list<std::shared_ptr<SingleDecision>> decision_list;

4
components/security_apps/waap/waap_clib/WaapErrorDisclosurePolicy.h
View File

@ -28,6 +28,10 @@ struct Policy {
enable(false),
enforce(false)
{
bool web_attack_on;
ar(cereal::make_nvp("webAttackMitigation", web_attack_on));
if (!web_attack_on) return;
std::string level;
ar(cereal::make_nvp("errorDisclosure", level));
level = boost::algorithm::to_lower_copy(level);

4
components/security_apps/waap/waap_clib/WaapOpenRedirectPolicy.h
View File

@ -30,6 +30,10 @@ struct Policy {
enable(false),
enforce(false)
{
bool web_attack_on;
ar(cereal::make_nvp("webAttackMitigation", web_attack_on));
if (!web_attack_on) return;
std::string level;
ar(cereal::make_nvp("openRedirect", level));
level = boost::algorithm::to_lower_copy(level);

3
components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
View File

@ -96,6 +96,9 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
}
return false;
}
else if (tag == "paramlocation" || tag == "paramLocation") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getLocation().c_str(), what, rx);
}
}
catch (std::runtime_error & e) {
dbgDebug(D_WAAP_OVERRIDE) << "RegEx match for tag " << tag << " failed due to: " << e.what();

32
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@ -105,7 +105,22 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
return Waap::Scores::calcArrayScore(res.scoreArray);
}
bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, const std::string& location, const std::string& param_name) {
// Ignore scan results from specific fields on csp-report json in case those are not filtered by learning
bool Waap::Scanner::isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp)
{
if (res.score < 8.0f && res.location == "body" && dp.getLastParser() == "jsonParser") {
if (key == "csp-report.blocked-uri" || key == "csp-report.script-sample" ||
(key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) {
dbgTrace(D_WAAP_SCANNER) << "CSP report detected, ignoring.";
return true;
}
}
return false;
}
bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, DeepParser &dp,
const std::string& location, const std::string& param_name, const std::string& key)
{
dbgTrace(D_WAAP_SCANNER) << "suspiciousHit processing for parameter: " << param_name << " at " << location <<
" num of keywords " << res.keyword_matches.size();
@ -130,7 +145,7 @@ bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, const std::string& locati
);
}
if (m_transaction->shouldIgnoreOverride(res)) {
if (isKeyCspReport(key, res, dp) || m_transaction->shouldIgnoreOverride(res)) {
dbgTrace(D_WAAP_SCANNER) << "Ignoring parameter key/value " << res.param_name <<
" due to ignore action in override";
m_bIgnoreOverride = true;
@ -182,6 +197,15 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len
dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: skip scanning our own anti-bot cookie, by name";
return 0;
}
// Do not scan google analytics cookie
if (isCookiePayload &&
(fullKeyStr.find("_ga") != std::string::npos ||
fullKeyStr.find("_gid") != std::string::npos ||
fullKeyStr.find("_gat") != std::string::npos)) {
dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: skip scanning google analytics cookie";
return 0;
}
// scan for csrf token.
if (isCookiePayload && fullKeyStr == "x-chkp-csrf-token") {
m_transaction->getCsrfState().set_CsrfToken(v, v_len);
@ -263,7 +287,7 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len
// Deep-scan parameter names
if (m_transaction->getAssetState()->apply(key, res, dp.m_key.first())) {
if (suspiciousHit(res, dp.m_key.first(), dp.m_key.str())) {
if (suspiciousHit(res, dp, dp.m_key.first(), dp.m_key.str(), key)) {
// Scanner found enough evidence to report this res
dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: SUSPICIOUS PARAM NAME: k='" <<
key << "' v='" << value << "'";
@ -295,7 +319,7 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len
res.mergeFrom(param_name_res);
}
if (suspiciousHit(res, dp.m_key.first(), dp.m_key.str())) {
if (suspiciousHit(res, dp, dp.m_key.first(), dp.m_key.str(), key)) {
// Scanner found enough evidence to report this res
dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: SUSPICIOUS VALUE: k='" << key <<
"' v='" << value << "'";

4
components/security_apps/waap/waap_clib/WaapScanner.h
View File

@ -32,7 +32,8 @@ namespace Waap {
m_bIgnoreOverride(false)
{
}
bool suspiciousHit(Waf2ScanResult &res, const std::string &location, const std::string &param_name);
bool suspiciousHit(Waf2ScanResult &res, DeepParser &dp,
const std::string &location, const std::string &param_name, const std::string &key);
int onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags) override;
const std::string &getAntibotCookie() const { return m_antibotCookie; }
@ -41,6 +42,7 @@ namespace Waap {
private:
double getScoreData(Waf2ScanResult& res, const std::string &poolName);
bool shouldIgnoreOverride(const Waf2ScanResult &res);
bool isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp);
Waf2ScanResult m_lastScanResult;
IWaf2Transaction *m_transaction;

8
components/security_apps/waap/waap_clib/WaapScores.cc
View File

@ -68,6 +68,7 @@ calcCombinations(
std::vector<std::string>& keyword_combinations)
{
keyword_combinations.clear();
static const double max_combi_score = 1.0f;
for (size_t i = 0; i < keyword_matches.size(); ++i) {
std::vector<std::string> combinations;
@ -79,6 +80,8 @@ calcCombinations(
// from signature_scores database.
std::sort(combinations.begin(), combinations.end());
std::string combination;
double default_score = 0.0f;
// note that std::set<> container output sorted data when iterated.
for (auto it = combinations.begin(); it != combinations.end(); it++) {
// add space between all items, except the first one
@ -86,8 +89,11 @@ calcCombinations(
combination += " ";
}
combination += *it;
default_score += scoreBuilder.getSnapshotKeywordScore(*it, 0.0f, poolName);
}
addKeywordScore(scoreBuilder, poolName, combination, 1.0f, scoresArray);
// set default combination score to be the sum of its keywords, bounded by 1
default_score = std::min(default_score, max_combi_score);
addKeywordScore(scoreBuilder, poolName, combination, default_score, scoresArray);
keyword_combinations.push_back(combination);
}
}

81
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@ -129,7 +129,7 @@ void Waf2Transaction::add_response_hdr(const char* name, int name_len, const cha
dbgTrace(D_WAAP) << "[transaction:" << this << "] add_response_hdr(name='" << std::string(name, name_len) <<
"', value='" << std::string(value, value_len) << "')";
// Detect location header and remember its value
// Detect location header and remember it's value
static const char location[] = "location";
auto openRedirectPolicy = m_siteConfig ? m_siteConfig->get_OpenRedirectPolicy() : NULL;
@ -260,6 +260,7 @@ void Waf2Transaction::end_response()
dbgTrace(D_WAAP) << "[transaction:" << this << "] end_response";
}
void Waf2Transaction::setCurrentAssetState(IWaapConfig* sitePolicy)
{
I_WaapAssetStatesManager* pWaapAssetStatesManager =
@ -593,10 +594,14 @@ bool Waf2Transaction::setCurrentAssetContext()
return result;
}
void Waf2Transaction::processUri(const char* uri, const std::string& scanStage) {
void Waf2Transaction::processUri(const std::string &uri, const std::string& scanStage) {
m_processedUri = true;
const char* p = uri;
size_t uriSize = uri.length();
const char* p = uri.c_str();
const char* uriEnd = p+uriSize;
std::string baseUri;
char querySep = '?';
char paramSep = '&';
// TODO:: refactor out this block to method, and the next block (parsing url parameters), too.
{
@ -606,27 +611,40 @@ void Waf2Transaction::processUri(const char* uri, const std::string& scanStage)
// Parse URL
ParserRaw urlParser(m_deepParserReceiver, scanStage);
// Scan the uri until '?' character found (or until end of the uri string).
// Scan the uri until '?' or ';' character found, whichever comes first (or until end of the uri string),
// Do not account for last character as valid separator
do {
const char* q = strchr(p, '?');
const char* q = strpbrk(p, "?;");
if (q == NULL) {
// Handle special case found in customer traffic where instead of '?' there was ';' character.
q = strchr(p, ';');
if (q) {
if (q != NULL && q < uriEnd-1) {
querySep = *q;
// Handle special case found in customer traffic where instead of '?' there was a ';' character.
if (querySep == ';') {
// Check that after ';' the parameter name is valid and terminated with '='. This would normally be
// the case in legit traffic, but not in attacks. This covers a case of "sap login".
const char *qq;
for (qq = q + 1; isalpha(*qq) || isdigit(*qq) || *qq=='-' || *qq=='_' || *qq=='*'; ++qq);
for (qq = q + 1;
qq < uriEnd && (isalpha(*qq) || isdigit(*qq) || *qq=='-' || *qq=='_' || *qq=='*');
++qq);
if (*qq != '=') {
// Assume it might be attack and cancel the separation by the ';' character (scan whole URL)
q = NULL;
}
else {
const char *qqSep = strpbrk(qq, "&;");
// Handle special case (deprecated standard) where instead of '&' there was a ';' separator,
// Do not account for last character as valid separator
if (qqSep && qqSep < uriEnd-1) {
paramSep = *qqSep;
}
}
}
}
if (q == NULL) {
baseUri = std::string(p);
dbgTrace(D_WAAP) << "Query separator not found, use entire uri as baseUri";
baseUri = std::string(uri);
if (scanStage == "url") {
m_uriPath = baseUri;
}
@ -637,7 +655,7 @@ void Waf2Transaction::processUri(const char* uri, const std::string& scanStage)
// Push the last piece to URL scanner
pushed = true;
std::string url(p, strlen(p));
std::string url(uri);
Waap::Util::decodePercentEncoding(url);
urlParser.push(url.data(), url.size());
@ -670,7 +688,7 @@ void Waf2Transaction::processUri(const char* uri, const std::string& scanStage)
// parameters from the character next to '?'
p = q + 1;
break;
} while (1);
} while (p && p < uriEnd);
if (pushed) {
urlParser.finish();
@ -721,7 +739,7 @@ void Waf2Transaction::processUri(const char* uri, const std::string& scanStage)
// at this point, p can either be NULL (if there are no URL parameters),
// or point to the parameters string (right after the '?' character)
if (p && *p) {
if (p && p < uriEnd && *p) {
// Decode URLEncoded data and send decoded key/value pairs to deep inspection
dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the " << scanStage.c_str() << " parameters";
@ -729,10 +747,12 @@ void Waf2Transaction::processUri(const char* uri, const std::string& scanStage)
m_uriQuery = std::string(p);
}
dbgTrace(D_WAAP) << "Query separator='" << querySep << "', " << "Param separator='" << paramSep << "'";
std::string tag = scanStage + "_param";
m_deepParser.m_key.push(tag.data(), tag.size());
size_t buff_len = strlen(p);
ParserUrlEncode up(m_deepParserReceiver, '&', checkUrlEncoded(p, buff_len));
size_t buff_len = uriEnd - p;
ParserUrlEncode up(m_deepParserReceiver, paramSep, checkUrlEncoded(p, buff_len));
up.push(p, buff_len);
up.finish();
m_deepParser.m_key.pop(tag.c_str());
@ -807,7 +827,7 @@ void Waf2Transaction::parseReferer(const char* value, int value_len)
// Parse referer value as if it was a URL
if (value_len > 0)
{
processUri(std::string(value, value_len).c_str(), "referer");
processUri(std::string(value, value_len), "referer");
}
}
@ -1011,13 +1031,14 @@ void Waf2Transaction::end_request_hdrs() {
}
// Scan URL and url query
if (m_isScanningRequired && !m_processedUri) {
processUri(m_uriStr.c_str(), "url");
processUri(m_uriStr, "url");
}
// Scan relevant headers for attacks
if (m_isScanningRequired && !m_processedHeaders) {
scanHeaders();
}
if(m_siteConfig != NULL) {
// Create rate limiting policy (lazy, on first request)
const std::shared_ptr<Waap::RateLimiting::Policy> rateLimitingPolicy = m_siteConfig->get_RateLimitingPolicy();
@ -1059,6 +1080,7 @@ void Waf2Transaction::start_request_body() {
clearRequestParserState();
m_requestBodyParser = new ParserRaw(m_deepParserReceiver, "body");
m_request_body_bytes_received = 0;
@ -1362,7 +1384,7 @@ Waf2Transaction::checkShouldInject()
{
dbgTrace(D_WAAP) << "Waf2Transaction::checkShouldInject(): Should not inject CSRF scripts.";
}
if(csrf) {
dbgTrace(D_WAAP) << "Waf2Transaction::checkShouldInject(): Should inject CSRF script";
m_responseInjectReasons.setCsrf(true);
@ -1370,6 +1392,7 @@ Waf2Transaction::checkShouldInject()
return;
}
bool
Waf2Transaction::decideAfterHeaders()
{
@ -1544,7 +1567,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
const auto& autonomousSecurityDecision = std::dynamic_pointer_cast<AutonomousSecurityDecision>(
m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
bool send_extended_log = shouldSendExtendedLog(triggerLog);
if (send_extended_log || triggerLog->webUrlPath || autonomousSecurityDecision->getOverridesLog()) {
if (triggerLog->webUrlPath || autonomousSecurityDecision->getOverridesLog()) {
std::string httpUriPath = m_uriPath;
if (httpUriPath.length() > MAX_LOG_FIELD_SIZE)
@ -1554,7 +1577,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
waapLog << LogField("httpUriPath", httpUriPath, LogFieldOption::XORANDB64);
}
if (send_extended_log || triggerLog->webUrlQuery || autonomousSecurityDecision->getOverridesLog()) {
if (triggerLog->webUrlQuery || autonomousSecurityDecision->getOverridesLog()) {
std::string uriQuery = m_uriQuery;
if (uriQuery.length() > MAX_LOG_FIELD_SIZE)
{
@ -1562,7 +1585,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
}
waapLog << LogField("httpUriQuery", uriQuery, LogFieldOption::XORANDB64);
}
if (send_extended_log || triggerLog->webHeaders || autonomousSecurityDecision->getOverridesLog()) {
if (triggerLog->webHeaders || autonomousSecurityDecision->getOverridesLog()) {
waapLog << LogField("httpRequestHeaders", logHeadersStr(), LogFieldOption::XORANDB64);
}
// Log http response code if it is known
@ -1627,6 +1650,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
}
}
void
Waf2Transaction::sendLog()
{
@ -1840,6 +1864,7 @@ Waf2Transaction::sendLog()
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType);
waap_log << LogField("waapIncidentDetails", incidentDetails);
if (grace_period) {
dbgTrace(D_WAAP)
@ -1871,7 +1896,7 @@ Waf2Transaction::sendLog()
<< max_grace_logs;
}
break;
}
}
case AUTONOMOUS_SECURITY_DECISION: {
if (triggerLog->webRequests ||
send_extended_log ||
@ -1929,7 +1954,7 @@ Waf2Transaction::decideAutonomousSecurity(
if (!m_processedUri) {
dbgWarning(D_WAAP) << "decideAutonomousSecurity(): processing URI although is was supposed "
"to be processed earlier ...";
processUri(m_uriStr.c_str(), "url");
processUri(m_uriStr, "url");
}
if (!m_processedHeaders) {
@ -2020,6 +2045,8 @@ Waf2Transaction::decideAutonomousSecurity(
}
}
if(decision->getThreatLevel() <= ThreatLevel::THREAT_INFO) {
decision->setLog(false);
} else {
@ -2103,7 +2130,6 @@ bool Waf2Transaction::decideResponse()
<< "Setting flag for collection of respond content logging to: "
<< (should_send_extended_log ? "True": "False");
m_responseInspectReasons.setCollectResponseForLog(should_send_extended_log);
}
dbgTrace(D_WAAP) << "Waf2Transaction::decideResponse: returns true (accept)";
@ -2126,8 +2152,6 @@ Waf2Transaction::reportScanResult(const Waf2ScanResult &res) {
bool
Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
dbgTrace(D_WAAP) << "reading exceptions";
auto exceptions = getConfiguration<ParameterException>("rulebase", "exception");
if (!exceptions.ok()) return false;
@ -2147,6 +2171,9 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
// collect param value
exceptions_dict["paramValue"].insert(res.unescaped_line);
// collect param location
exceptions_dict["paramLocation"].insert(res.location);
ScopedContext ctx;
ctx.registerValue<std::string>("paramValue", res.unescaped_line);
ctx.registerValue<std::set<std::string>>("paramName", param_name_set);

4
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@ -184,6 +184,7 @@ public:
void handleSecurityHeadersInjection(std::vector<std::pair<std::string, std::string>>& injectHeaderStrs);
void disableShouldInjectSecurityHeaders();
bool shouldSendExtendedLog(const std::shared_ptr<Waap::Trigger::Log> &trigger_log) const;
// query
@ -236,6 +237,7 @@ private:
std::string getUserReputationStr(double relativeReputation) const;
bool isTrustedSource() const;
void setCurrentAssetState(IWaapConfig* sitePolicy);
bool setCurrentAssetContext();
bool checkIsScanningRequired();
@ -254,7 +256,7 @@ private:
size_t getViolatingUserLimitSize() const;
// Internal
void processUri(const char *uri, const std::string &scanStage);
void processUri(const std::string &uri, const std::string &scanStage);
void parseContentType(const char* value, int value_len);
void parseCookie(const char* value, int value_len);
void parseReferer(const char* value, int value_len);

2
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@ -265,6 +265,8 @@ ParserBase* Waf2Transaction::getRequestBodyParser()
{
return m_requestBodyParser;
}
const std::string Waf2Transaction::getMethod() const
{
return m_methodStr;

28
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@ -1128,6 +1128,11 @@ static const SingleRegex broken_utf_evasion_re(
err,
"broken_utf_evasion"
);
static const SingleRegex csp_report_policy_re(
"default-src\\s+[^\\w]+.*report-uri\\s+[^\\w]+",
err,
"csp_report_policy"
);
static void b64TestChunk(const string &s,
string::const_iterator chunkStart,
@ -1391,10 +1396,16 @@ unescapeInvalidUtf8(const string &payload)
return unescaped_text;
}
bool
containsBrokenUtf8(const string &payload)
Maybe<std::string>
containsBrokenUtf8(const string &payload, const string &unquoted_payload)
{
return broken_utf_evasion_re.hasMatch(payload);
if (broken_utf_evasion_re.hasMatch(unquoted_payload)) {
return unquoted_payload;
} else if (broken_utf_evasion_re.hasMatch(payload)) {
return payload;
} else {
return genError("does not contain broken-down UTF8");
}
}
string
@ -1426,6 +1437,12 @@ unescapeBrokenUtf8(const string &payload)
return unescaped_text;
}
bool
containsCspReportPolicy(const string &payload)
{
return csp_report_policy_re.hasMatch(payload);
}
string
charToString(const char* s, int slen)
{
@ -1825,6 +1842,11 @@ ContentType detectContentType(const char* hdr_value) {
return CONTENT_TYPE_JSON;
}
// Detect Graphql content type if Content-Type header value is application/graphql
if (my_stristarts_with(hdr_value, "application/graphql")) {
return CONTENT_TYPE_GQL;
}
// Detect HTML content type
if (my_stristarts_with(hdr_value, "text/html")) {
return CONTENT_TYPE_HTML;

12
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@ -757,7 +757,8 @@ inline char convertFromUnicodeHalfAndFullWidthRange(uint32_t code) {
inline bool isSpecialUnicode(uint32_t code) {
return isUnicodeHalfAndFullWidthRange(code)
|| 0x2028 == code || 0x2029 == code
|| 0x2216 == code || 0xEFC8 == code || 0xF025 == code;
|| 0x2215 == code || 0x2216 == code
|| 0xEFC8 == code || 0xF025 == code;
}
inline char convertSpecialUnicode(uint32_t code) {
@ -768,6 +769,10 @@ inline char convertSpecialUnicode(uint32_t code) {
{
return '\\';
}
else if (0x2215 == code)
{
return '/';
}
// assuming 0x2028 == code || 0x2029 == code
else
{
@ -968,6 +973,7 @@ namespace Util {
CONTENT_TYPE_UNKNOWN,
CONTENT_TYPE_XML,
CONTENT_TYPE_JSON,
CONTENT_TYPE_GQL,
CONTENT_TYPE_HTML,
CONTENT_TYPE_MULTIPART_FORM,
CONTENT_TYPE_URLENCODED,
@ -1037,9 +1043,11 @@ namespace Util {
// based on invalid utf-8 evasion from here: https://www.cgisecurity.com/lib/URLEmbeddedAttacks.html
std::string unescapeInvalidUtf8(const std::string &text);
bool containsBrokenUtf8(const std::string &payload);
Maybe<std::string> containsBrokenUtf8(const std::string &payload, const std::string &unquoted_payload);
std::string unescapeBrokenUtf8(const std::string &text);
bool containsCspReportPolicy(const std::string &payload);
bool testUrlBareUtf8Evasion(const std::string &line);
bool testUrlBadUtf8Evasion(const std::string &line);

1
components/security_apps/waap/waap_component.cc
View File

@ -52,7 +52,6 @@ WaapComponent::preload()
// TODO:: call stuff like registerExpectedCofiguration here..
registerExpectedConfiguration<WaapConfigApplication>("WAAP", "WebApplicationSecurity");
registerExpectedConfiguration<WaapConfigAPI>("WAAP", "WebAPISecurity");
registerExpectedConfiguration<std::string>("WAAP", "Sigs score file path");
registerExpectedConfiguration<std::string>("WAAP", "Sigs file path");
registerExpectedConfigFile("waap", Config::ConfigFileType::Policy);
registerConfigLoadCb(

24
components/security_apps/waap/waap_component_impl.cc
View File

@ -60,18 +60,12 @@ WaapComponent::Impl::~Impl()
void
WaapComponent::Impl::init()
{
std::string sigs_file_path = getConfigurationWithDefault<string>(
"/etc/cp/conf/waap/1.data",
std::string waapDataFileName = getConfigurationWithDefault<string>(
"/etc/cp/conf/waap/waap.data",
"WAAP",
"Sigs file path"
);
std::string sigs_score_file_path = getConfigurationWithDefault<string>(
"/etc/cp/conf/waap/2.data",
"WAAP",
"Sigs score file path"
);
assets_metric.init(
"Assets Count",
ReportIS::AudienceTeam::AGENT_CORE,
@ -84,18 +78,20 @@ WaapComponent::Impl::init()
registerListener();
waap_metric.registerListener();
init(sigs_file_path, sigs_score_file_path);
init(waapDataFileName);
}
void
WaapComponent::Impl::init(const std::string &sigs_file_path, const std::string &sigs_scores_file_path)
WaapComponent::Impl::init(const std::string &waapDataFileName)
{
//waf2_set_log_target(WAF2_LOGTARGET_STDERR);
dbgTrace(D_WAAP) << "WaapComponent::Impl::init() ...";
reputationAggregator.init();
bool success = waf2_proc_start(sigs_file_path, sigs_scores_file_path);
waapStateTable = Singleton::Consume<I_Table>::by<WaapComponent>();
bool success = waf2_proc_start(waapDataFileName);
if (!success) {
dbgWarning(D_WAAP) << "WAF2 engine FAILED to initialize (probably failed to load signatures). Aborting!";
waf2_proc_exit();
@ -107,8 +103,6 @@ WaapComponent::Impl::init(const std::string &sigs_file_path, const std::string &
I_StaticResourcesHandler *static_resources = Singleton::Consume<I_StaticResourcesHandler>::by<WaapComponent>();
static_resources->registerStaticResource("cp-ab.js", "/etc/cp/conf/waap/cp-ab.js");
static_resources->registerStaticResource("cp-csrf.js", "/etc/cp/conf/waap/cp-csrf.js");
waapStateTable = Singleton::Consume<I_Table>::by<WaapComponent>();
}
// Called when component is shut down
@ -730,7 +724,7 @@ void WaapComponent::Impl::sendNotificationForFirstRequest(
}
bool
WaapComponent::Impl::waf2_proc_start(const std::string& sigsFname, const std::string& scoresFname)
WaapComponent::Impl::waf2_proc_start(const std::string& waapDataFileName)
{
// WAAP uses libxml library, which requires process-level initialization when process starts
#if 0 // TODO:: silence the error messages printed by libxml2
@ -740,7 +734,7 @@ WaapComponent::Impl::waf2_proc_start(const std::string& sigsFname, const std::st
::xmlInitParser();
return
Singleton::Consume<I_WaapAssetStatesManager>::by<WaapComponent>()->initBasicWaapSigs(sigsFname, scoresFname);
Singleton::Consume<I_WaapAssetStatesManager>::by<WaapComponent>()->initBasicWaapSigs(waapDataFileName);
}
void

4
components/security_apps/waap/waap_component_impl.h
View File

@ -55,13 +55,13 @@ public:
EventVerdict respond(const EndTransactionEvent &) override;
private:
void init(const std::string &sigs_file_path, const std::string &sigs_scores_file_path);
void init(const std::string &waapDataFileName);
EventVerdict waapDecisionAfterHeaders(IWaf2Transaction& waf2Transaction);
EventVerdict waapDecision(IWaf2Transaction& waf2Transaction);
void finishTransaction(IWaf2Transaction& waf2Transaction);
bool waf2_proc_start(const std::string& sigsFname, const std::string& scoresFname);
bool waf2_proc_start(const std::string& waapDataFileName);
void waf2_proc_exit();
void validateFirstRequestForAsset(const ReportIS::Severity severity);
void sendNotificationForFirstRequest(

18
components/signal_handler/signal_handler.cc
View File

@ -207,9 +207,6 @@ private:
}
set<ReportIS::Tags> tags;
string agent_uid =
(Report::isPlaygroundEnv() ? "playground-" : "") +
Singleton::Consume<I_AgentDetails>::by<SignalHandler>()->getAgentId();
Report message_to_fog(
"Nano service startup after crash",
curr_time,
@ -221,7 +218,7 @@ private:
Severity::HIGH,
Priority::HIGH,
chrono::seconds(0),
LogField("agentId", agent_uid),
LogField("agentId", Singleton::Consume<I_AgentDetails>::by<SignalHandler>()->getAgentId()),
tags,
Tags::INFORMATIONAL
);
@ -262,7 +259,12 @@ private:
<< ", Error: "
<< strerror(errno_copy);
}
setHandlerPerSignalNum();
}
void
setHandlerPerSignalNum()
{
static const vector<int> signals = {
SIGABRT,
SIGKILL,
@ -302,6 +304,9 @@ private:
{
const char *signal_name = "";
char signal_num[3];
reset_signal_handler = true;
switch(_signal) {
case SIGABRT: {
signal_name = "SIGABRT";
@ -445,6 +450,9 @@ private:
[&] ()
{
while (true) {
if (reset_signal_handler) {
setHandlerPerSignalNum();
}
if (reload_settings_flag == true) {
reload_settings_flag = false;
if (reloadConfiguration("")) {
@ -462,11 +470,13 @@ private:
static string trace_file_path;
static bool reload_settings_flag;
static bool reset_signal_handler;
static int out_trace_file_fd;
};
string SignalHandler::Impl::trace_file_path;
bool SignalHandler::Impl::reload_settings_flag = false;
bool SignalHandler::Impl::reset_signal_handler = false;
int SignalHandler::Impl::out_trace_file_fd = -1;
SignalHandler::SignalHandler() : Component("SignalHandler"), pimpl(make_unique<Impl>()) {}

3
components/utils/pm/debugpm.cc
View File

@ -18,13 +18,11 @@
#include <string.h>
#include <string>
#include <iostream>
#include "sasal.h"
using namespace std;
USE_DEBUG_FLAG(D_PM);
SASAL_START // Multiple Pattern Matcher
// Helper class for printing C format string
class CFmtPrinter
{
@ -60,4 +58,3 @@ debugPrtCFmt(const char *func, uint line, const char *fmt, ...)
Debug("PM", func, line, Debug::DebugLevel::TRACE, D_PM).getStreamAggr() << CFmtPrinter(fmt, va);
va_end(va);
}
SASAL_END

Some files were not shown because too many files have changed in this diff Show More