github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

sync code

Browse Source
This commit is contained in:
Ned Wright
2024-12-29 12:13:27 +00:00
parent 96ce290e5f
commit 64ebf013eb
43 changed files with 1058 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
components/attachment-intakers/nginx_attachment/nginx_attachment_config.cc
View File

@@ -203,6 +203,13 @@ HttpAttachmentConfig::setFailOpenTimeout()
"NGINX wait thread timeout msec"
));
conf_data.setNumericalValue("remove_server_header", getAttachmentConf<uint>(
0,
"agent.removeServerHeader.nginxModule",
"HTTP manager",
"Response server header removal"
));
uint inspection_mode = getAttachmentConf<uint>(
static_cast<uint>(ngx_http_inspection_mode_e::NON_BLOCKING_THREAD),
"agent.inspectionMode.nginxModule",

28
components/http_manager/http_manager.cc
View File

@@ -18,7 +18,9 @@
#include <sys/stat.h>
#include <climits>
#include <unordered_map>
#include <unordered_set>
#include <boost/range/iterator_range.hpp>
#include <boost/algorithm/string.hpp>
#include <fstream>
#include <algorithm>
@@ -28,6 +30,7 @@
#include "http_manager_opaque.h"
#include "log_generator.h"
#include "http_inspection_events.h"
#include "agent_core_utilities.h"
USE_DEBUG_FLAG(D_HTTP_MANAGER);
@@ -66,6 +69,22 @@ public:
i_transaction_table = Singleton::Consume<I_Table>::by<HttpManager>();
Singleton::Consume<I_Logging>::by<HttpManager>()->addGeneralModifier(compressAppSecLogs);
const char* ignored_headers_env = getenv("SAAS_IGNORED_UPSTREAM_HEADERS");
if (ignored_headers_env) {
string ignored_headers_str = ignored_headers_env;
ignored_headers_str = NGEN::Strings::removeTrailingWhitespaces(ignored_headers_str);
if (!ignored_headers_str.empty()) {
dbgInfo(D_HTTP_MANAGER)
<< "Ignoring SAAS_IGNORED_UPSTREAM_HEADERS environment variable: "
<< ignored_headers_str;
vector<string> ignored_headers_vec;
boost::split(ignored_headers_vec, ignored_headers_str, boost::is_any_of(";"));
for (const string &header : ignored_headers_vec) ignored_headers.insert(header);
}
}
}
FilterVerdict
@@ -90,6 +109,14 @@ public:
return FilterVerdict(default_verdict);
}
if (is_request && ignored_headers.find(static_cast<string>(event.getKey())) != ignored_headers.end()) {
dbgTrace(D_HTTP_MANAGER)
<< "Ignoring header key - "
<< static_cast<string>(event.getKey())
<< " - as it is in the ignored headers list";
return FilterVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT);
}
ScopedContext ctx;
ctx.registerValue(app_sec_marker_key, i_transaction_table->keyToString(), EnvKeyAttr::LogSection::MARKER);
@@ -394,6 +421,7 @@ private:
I_Table *i_transaction_table;
static const ngx_http_cp_verdict_e default_verdict;
static const string app_sec_marker_key;
unordered_set<string> ignored_headers;
};
const ngx_http_cp_verdict_e HttpManager::Impl::default_verdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP);

44
components/include/env_details.h
View File

@@ -1,44 +0,0 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __ENV_DETAILS_H__
#define __ENV_DETAILS_H__
#include <string>
#include <fstream>
#include <streambuf>
#include "i_env_details.h"
#include "singleton.h"
#include "debug.h"
class EnvDetails : Singleton::Provide<I_EnvDetails>::SelfInterface
{
public:
EnvDetails();
virtual EnvType getEnvType() override;
virtual std::string getToken() override;
virtual std::string getNameSpace() override;
private:
std::string retrieveToken();
std::string retrieveNamespace();
std::string readFileContent(const std::string &file_path);
std::string token;
std::string agent_namespace;
EnvType env_type;
};
#endif // __ENV_DETAILS_H__

8
components/include/rate_limit.h
View File

@@ -7,15 +7,21 @@
#include "singleton.h"
#include "i_mainloop.h"
#include "i_environment.h"
#include "i_geo_location.h"
#include "i_generic_rulebase.h"
#include "i_shell_cmd.h"
#include "i_env_details.h"
class RateLimit
:
public Component,
Singleton::Consume<I_MainLoop>,
Singleton::Consume<I_TimeGet>,
Singleton::Consume<I_GeoLocation>,
Singleton::Consume<I_Environment>,
Singleton::Consume<I_GenericRulebase>
Singleton::Consume<I_GenericRulebase>,
Singleton::Consume<I_ShellCmd>,
Singleton::Consume<I_EnvDetails>
{
public:
RateLimit();

4
components/include/waap.h
View File

@@ -33,6 +33,7 @@ class I_WaapAssetStatesManager;
class I_Messaging;
class I_AgentDetails;
class I_Encryptor;
class I_WaapModelResultLogger;
const std::string WAAP_APPLICATION_NAME = "waap application";
@@ -50,7 +51,8 @@ class WaapComponent
Singleton::Consume<I_AgentDetails>,
Singleton::Consume<I_Messaging>,
Singleton::Consume<I_Encryptor>,
Singleton::Consume<I_Environment>
Singleton::Consume<I_Environment>,
Singleton::Consume<I_WaapModelResultLogger>
{
public:
WaapComponent();

14
components/security_apps/http_geo_filter/http_geo_filter.cc
View File

@@ -88,9 +88,17 @@ public:
dbgWarning(D_GEO_FILTER) << "failed to get source ip from env";
return EventVerdict(default_action);
}
auto source_ip = convertIpAddrToString(maybe_source_ip.unpack());
ip_set.insert(source_ip);
// saas profile setting
bool ignore_source_ip =
getProfileAgentSettingWithDefault<bool>(false, "agent.geoProtaction.ignoreSourceIP");
if (ignore_source_ip){
dbgDebug(D_GEO_FILTER) << "Geo protection ignoring source ip: " << source_ip;
} else {
ip_set.insert(convertIpAddrToString(maybe_source_ip.unpack()));
}
ngx_http_cp_verdict_e exception_verdict = getExceptionVerdict(ip_set);
if (exception_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) {
@@ -343,7 +351,7 @@ private:
auto asset_location = i_geo_location->lookupLocation(maybe_source_ip.unpack());
if (!asset_location.ok()) {
dbgWarning(D_GEO_FILTER) << "Lookup location failed for source: " <<
dbgDebug(D_GEO_FILTER) << "Lookup location failed for source: " <<
source <<
", Error: " <<
asset_location.getErr();

7
components/security_apps/ips/include/ips_signatures.h
View File

@@ -336,9 +336,16 @@ public:
return metadata.getYear();
}
bool
isOk() const
{
return is_loaded;
}
private:
IPSSignatureMetaData metadata;
std::shared_ptr<BaseSignature> rule;
bool is_loaded;
};
/// \class SignatureAndAction

58
components/security_apps/ips/ips_signatures.cc
View File

@@ -219,10 +219,16 @@ IPSSignatureMetaData::getYear() const
void
CompleteSignature::load(cereal::JSONInputArchive &ar)
{
ar(cereal::make_nvp("protectionMetadata", metadata));
RuleDetection rule_detection(metadata.getName());
ar(cereal::make_nvp("detectionRules", rule_detection));
rule = rule_detection.getRule();
try {
ar(cereal::make_nvp("protectionMetadata", metadata));
RuleDetection rule_detection(metadata.getName());
ar(cereal::make_nvp("detectionRules", rule_detection));
rule = rule_detection.getRule();
is_loaded = true;
} catch (cereal::Exception &e) {
is_loaded = false;
dbgWarning(D_IPS) << "Failed to load signature: " << e.what();
}
}
MatchType
@@ -367,7 +373,16 @@ SignatureAndAction::matchSilent(const Buffer &sample) const
if (method.ok()) log << LogField("httpMethod", method.unpack());
auto path = env->get<Buffer>("HTTP_PATH_DECODED");
if (path.ok()) log << LogField("httpUriPath", getSubString(path, 1536), LogFieldOption::XORANDB64);
if (path.ok()) {
log << LogField("httpUriPath", getSubString(path, 1536), LogFieldOption::XORANDB64);
} else {
auto transaction_path = env->get<string>(HttpTransactionData::uri_path_decoded);
if (transaction_path.ok()) {
auto uri_path = transaction_path.unpack();
auto question_mark = uri_path.find('?');
log << LogField("httpUriPath", uri_path.substr(0, question_mark), LogFieldOption::XORANDB64);
}
}
auto req_header = ips_state.getTransactionData(IPSCommonTypes::requests_header_for_log);
if (req_header.ok()) log << LogField("httpRequestHeaders", getSubString(req_header), LogFieldOption::XORANDB64);
@@ -485,13 +500,30 @@ SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set<PMP
auto method = env->get<string>(HttpTransactionData::method_ctx);
if (method.ok()) log << LogField("httpMethod", method.unpack());
uint max_size = getConfigurationWithDefault<uint>(1536, "IPS", "Max Field Size");
auto path = env->get<Buffer>("HTTP_PATH_DECODED");
if (path.ok() && trigger.isWebLogFieldActive(url_path)) {
log << LogField("httpUriPath", getSubString(path, max_size), LogFieldOption::XORANDB64);
if (trigger.isWebLogFieldActive(url_path)) {
auto path = env->get<Buffer>("HTTP_PATH_DECODED");
if (path.ok()) {
log << LogField("httpUriPath", getSubString(path, max_size), LogFieldOption::XORANDB64);
} else {
auto transaction_path = env->get<string>(HttpTransactionData::uri_path_decoded);
if (transaction_path.ok()) {
auto uri_path = transaction_path.unpack();
auto question_mark = uri_path.find('?');
log << LogField("httpUriPath", uri_path.substr(0, question_mark), LogFieldOption::XORANDB64);
}
}
}
auto query = env->get<Buffer>("HTTP_QUERY_DECODED");
if (query.ok() && trigger.isWebLogFieldActive(url_query)) {
log << LogField("httpUriQuery", getSubString(query, max_size), LogFieldOption::XORANDB64);
if (trigger.isWebLogFieldActive(url_query)) {
auto query = env->get<Buffer>("HTTP_QUERY_DECODED");
if (query.ok()) {
log << LogField("httpUriQuery", getSubString(query, max_size), LogFieldOption::XORANDB64);
} else {
auto transaction_query = env->get<string>(HttpTransactionData::uri_query_decoded);
if (transaction_query.ok()) {
log << LogField("httpUriQuery", transaction_query.unpack());
}
}
}
auto res_code = env->get<Buffer>("HTTP_RESPONSE_CODE");
@@ -533,7 +565,9 @@ IPSSignaturesResource::load(cereal::JSONInputArchive &ar)
all_signatures.reserve(sigs.size());
for (auto &sig : sigs) {
all_signatures.emplace_back(make_shared<CompleteSignature>(move(sig)));
if (sig.isOk()) {
all_signatures.emplace_back(make_shared<CompleteSignature>(move(sig)));
}
}
}

38
components/security_apps/ips/ips_ut/signatures_ut.cc
View File

@@ -104,6 +104,12 @@ public:
cereal::JSONInputArchive ar(ss);
high_medium_confidance_signatures.load(ar);
}
{
stringstream ss;
ss << "[" << signature_performance_high << ", " << signature_broken << "]";
cereal::JSONInputArchive ar(ss);
single_broken_signature.load(ar);
}
}
~SignatureTest()
@@ -250,6 +256,7 @@ public:
IPSSignaturesResource performance_signatures1;
IPSSignaturesResource performance_signatures2;
IPSSignaturesResource performance_signatures3;
IPSSignaturesResource single_broken_signature;
NiceMock<MockTable> table;
MockAgg mock_agg;
@@ -483,6 +490,26 @@ private:
"\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]"
"}"
"}";
string signature_broken =
"{"
"\"protectionMetadata\": {"
"\"protectionName\": \"BrokenTest\","
"\"maintrainId\": \"101\","
"\"severity\": \"Medium High\","
"\"confidenceLevel\": \"Low\","
"\"performanceImpact\": \"High\","
"\"lastUpdate\": \"20210420\","
"\"tags\": [],"
"\"cveList\": []"
"},"
"\"detectionRules\": {"
"\"type\": \"simple\","
"\"SSM\": \"\","
"\"keywosrds\": \"data: \\\"www\\\";\","
"\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]"
"}"
"}";
};
TEST_F(SignatureTest, basic_load_of_signatures)
@@ -665,3 +692,14 @@ TEST_F(SignatureTest, high_confidance_signatures_matching)
expectLog("\"protectionId\": \"Test4\"", "\"matchedSignatureConfidence\": \"Medium\"");
EXPECT_FALSE(checkData("mmm"));
}
TEST_F(SignatureTest, broken_signature)
{
load(single_broken_signature, "Low or above", "Low");
EXPECT_FALSE(checkData("ggg"));
expectLog("\"matchedSignaturePerformance\": \"High\"");
EXPECT_TRUE(checkData("fff"));
EXPECT_FALSE(checkData("www"));
}

1
components/security_apps/local_policy_mgmt_gen/CMakeLists.txt
View File

@@ -22,4 +22,5 @@ add_library(local_policy_mgmt_gen
access_control_practice.cc
configmaps.cc
reverse_proxy_section.cc
policy_activation_data.cc
)

10
components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h
View File

@@ -48,7 +48,7 @@ public:
void init();
std::tuple<std::map<std::string, AppsecLinuxPolicy>, std::map<std::string, V1beta2AppsecLinuxPolicy>>
createAppsecPoliciesFromIngresses();
createAppsecPolicies();
void getClusterId() const;
private:
@@ -101,12 +101,18 @@ private:
) const;
template<class T, class K>
void createPolicy(
void createPolicyFromIngress(
T &appsec_policy,
std::map<std::string, T> &policies,
std::map<AnnotationKeys, std::string> &annotations_values,
const SingleIngressData &item) const;
template<class T, class K>
void createPolicyFromActivation(
T &appsec_policy,
std::map<std::string, T> &policies,
const EnabledPolicy &policy) const;
std::tuple<Maybe<AppsecLinuxPolicy>, Maybe<V1beta2AppsecLinuxPolicy>> createAppsecPolicyK8s(
const std::string &policy_name,
const std::string &ingress_mode

1
components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h
View File

@@ -32,6 +32,7 @@
#include "i_messaging.h"
#include "appsec_practice_section.h"
#include "ingress_data.h"
#include "policy_activation_data.h"
#include "settings_section.h"
#include "triggers_section.h"
#include "local_policy_common.h"

70
components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc
View File

@@ -577,7 +577,7 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i
template<class T, class K>
void
K8sPolicyUtils::createPolicy(
K8sPolicyUtils::createPolicyFromIngress(
T &appsec_policy,
map<std::string, T> &policies,
map<AnnotationKeys, string> &annotations_values,
@@ -615,10 +615,35 @@ K8sPolicyUtils::createPolicy(
}
}
std::tuple<map<string, AppsecLinuxPolicy>, map<string, V1beta2AppsecLinuxPolicy>>
K8sPolicyUtils::createAppsecPoliciesFromIngresses()
template<class T, class K>
void
K8sPolicyUtils::createPolicyFromActivation(
T &appsec_policy,
map<std::string, T> &policies,
const EnabledPolicy &policy) const
{
dbgFlow(D_LOCAL_POLICY) << "Getting all policy object from Ingresses";
if (policies.find(policy.getName()) == policies.end()) {
policies[policy.getName()] = appsec_policy;
}
auto default_mode = appsec_policy.getAppsecPolicySpec().getDefaultRule().getMode();
for (const string &host : policy.getHosts()) {
if (!appsec_policy.getAppsecPolicySpec().isAssetHostExist(host)) {
dbgTrace(D_LOCAL_POLICY)
<< "Inserting Host data to the specific asset set:"
<< "URL: '"
<< host
<< "'";
K ingress_rule = K(host, default_mode);
policies[policy.getName()].addSpecificRule(ingress_rule);
}
}
}
std::tuple<map<string, AppsecLinuxPolicy>, map<string, V1beta2AppsecLinuxPolicy>>
K8sPolicyUtils::createAppsecPolicies()
{
dbgFlow(D_LOCAL_POLICY) << "Getting all policy object from Ingresses and PolicyActivation";
map<string, AppsecLinuxPolicy> v1bet1_policies;
map<string, V1beta2AppsecLinuxPolicy> v1bet2_policies;
auto maybe_ingress = getObjectFromCluster<IngressData>("/apis/networking.k8s.io/v1/ingresses");
@@ -628,7 +653,7 @@ K8sPolicyUtils::createAppsecPoliciesFromIngresses()
dbgWarning(D_LOCAL_POLICY)
<< "Failed to retrieve K8S Ingress configurations. Error: "
<< maybe_ingress.getErr();
return make_tuple(v1bet1_policies, v1bet2_policies);
maybe_ingress = IngressData{};
}
@@ -658,19 +683,50 @@ K8sPolicyUtils::createAppsecPoliciesFromIngresses()
if (!std::get<0>(maybe_appsec_policy).ok()) {
auto appsec_policy=std::get<1>(maybe_appsec_policy).unpack();
createPolicy<V1beta2AppsecLinuxPolicy, NewParsedRule>(
createPolicyFromIngress<V1beta2AppsecLinuxPolicy, NewParsedRule>(
appsec_policy,
v1bet2_policies,
annotations_values,
item);
} else {
auto appsec_policy=std::get<0>(maybe_appsec_policy).unpack();
createPolicy<AppsecLinuxPolicy, ParsedRule>(
createPolicyFromIngress<AppsecLinuxPolicy, ParsedRule>(
appsec_policy,
v1bet1_policies,
annotations_values,
item);
}
}
auto maybe_policy_activation =
getObjectFromCluster<PolicyActivationData>("/apis/openappsec.io/v1beta2/policyactivations");
if (!maybe_policy_activation.ok()) {
dbgWarning(D_LOCAL_POLICY)
<< "Failed to retrieve K8S PolicyActivation configurations. Error: "
<< maybe_policy_activation.getErr();
return make_tuple(v1bet1_policies, v1bet2_policies);
}
PolicyActivationData policy_activation = maybe_policy_activation.unpack();
for (const SinglePolicyActivationData &item : policy_activation.getItems()) {
for (const auto &policy : item.getSpec().getPolicies()) {
auto maybe_appsec_policy = createAppsecPolicyK8s(policy.getName(), "");
if (!std::get<1>(maybe_appsec_policy).ok()) {
dbgWarning(D_LOCAL_POLICY)
<< "Failed to create appsec policy. v1beta2 Error: "
<< std::get<1>(maybe_appsec_policy).getErr();
continue;
} else {
auto appsec_policy=std::get<1>(maybe_appsec_policy).unpack();
createPolicyFromActivation<V1beta2AppsecLinuxPolicy, NewParsedRule>(
appsec_policy,
v1bet2_policies,
policy);
}
}
}
return make_tuple(v1bet1_policies, v1bet2_policies);
}

3
components/security_apps/local_policy_mgmt_gen/local_policy_mgmt_gen.cc
View File

@@ -36,6 +36,7 @@
#include "customized_cereal_map.h"
#include "include/appsec_practice_section.h"
#include "include/ingress_data.h"
#include "include/policy_activation_data.h"
#include "include/settings_section.h"
#include "include/triggers_section.h"
#include "include/local_policy_common.h"
@@ -85,7 +86,7 @@ public:
K8sPolicyUtils k8s_policy_utils;
k8s_policy_utils.init();
auto appsec_policies = k8s_policy_utils.createAppsecPoliciesFromIngresses();
auto appsec_policies = k8s_policy_utils.createAppsecPolicies();
if (!std::get<0>(appsec_policies).empty()) {
return policy_maker_utils.proccesMultipleAppsecPolicies<AppsecLinuxPolicy, ParsedRule>(
std::get<0>(appsec_policies),

1
components/security_apps/orchestration/CMakeLists.txt
View File

@@ -14,7 +14,6 @@ add_subdirectory(details_resolver)
add_subdirectory(health_check)
add_subdirectory(health_check_manager)
add_subdirectory(updates_process_reporter)
add_subdirectory(env_details)
add_subdirectory(external_sdk_server)
#add_subdirectory(orchestration_ut)

38
components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
View File

@@ -29,7 +29,7 @@
// shell command execution output as its input
#ifdef SHELL_PRE_CMD
#if defined(gaia) || defined(smb)
#if defined(gaia) || defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1)
SHELL_PRE_CMD("read sdwan data",
"(cpsdwan get_data > /tmp/cpsdwan_getdata_orch.json~) "
"&& (mv /tmp/cpsdwan_getdata_orch.json~ /tmp/cpsdwan_getdata_orch.json)")
@@ -40,7 +40,7 @@ SHELL_PRE_CMD("gunzip local.cfg", "gunzip -c $FWDIR/state/local/FW1/local.cfg.gz
#endif
#ifdef SHELL_CMD_HANDLER
#if defined(gaia) || defined(smb)
#if defined(gaia) || defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1)
SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType)
SHELL_CMD_HANDLER(
"cpProductIntegrationMgmtObjectUid",
@@ -51,6 +51,14 @@ SHELL_CMD_HANDLER("prerequisitesForHorizonTelemetry",
"FS_PATH=<FILESYSTEM-PREFIX>; [ -f ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log ] "
"&& head -1 ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log || echo ''",
checkIsInstallHorizonTelemetrySucceeded)
SHELL_CMD_HANDLER(
"IS_AIOPS_RUNNING",
"FS_PATH=<FILESYSTEM-PREFIX>; "
"PID=$(ps auxf | grep -v grep | grep -E ${FS_PATH}.*cp-nano-horizon-telemetry | awk -F' ' '{printf $2}'); "
"[ -z \"${PID}\" ] && echo 'false' || echo 'true'",
getIsAiopsRunning)
#endif
#if defined(gaia)
SHELL_CMD_HANDLER("GLOBAL_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo ''",
getQUID)
@@ -76,12 +84,21 @@ SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] "
SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "[ -d /opt/CPOtlpAgent/custom_scripts ] "
"&& ENV_NO_FORMAT=1 /opt/CPOtlpAgent/custom_scripts/agent_role.sh",
getOtlpAgentGaiaOsRole)
SHELL_CMD_HANDLER(
"IS_AIOPS_RUNNING",
"FS_PATH=<FILESYSTEM-PREFIX>; "
"PID=$(ps auxf | grep -v grep | grep -E ${FS_PATH}.*cp-nano-horizon-telemetry | awk -F' ' '{printf $2}'); "
"[ -z \"{PID}\" ] && echo 'false' || echo 'true'",
getIsAiopsRunning)
#endif
#if defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1)
SHELL_CMD_HANDLER("GLOBAL_QUID",
"cat $FWDIR/database/myown.C "
"| awk -F'[()]' '/:name/ { found=1; next } found && /:uuid/ { uid=tolower($2); print uid; exit }'",
getQUID)
SHELL_CMD_HANDLER("QUID",
"cat $FWDIR/database/myown.C "
"| awk -F'[()]' '/:name/ { found=1; next } found && /:uuid/ { uid=tolower($2); print uid; exit }'",
getQUID)
SHELL_CMD_HANDLER("SMO_QUID", "echo ''", getQUID)
SHELL_CMD_HANDLER("MGMT_QUID", "echo ''", getQUID)
SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "echo 'SMB'", getOtlpAgentGaiaOsRole)
#endif
#if defined(gaia) || defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1)
SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
SHELL_CMD_HANDLER(
"canUpdateSDWanData",
@@ -194,7 +211,7 @@ SHELL_CMD_HANDLER(
)
#endif //gaia
#if defined(smb)
#if defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1)
SHELL_CMD_HANDLER(
"cpProductIntegrationMgmtParentObjectName",
"jq -r .cluster_name /tmp/cpsdwan_getdata_orch.json",
@@ -252,7 +269,6 @@ SHELL_CMD_HANDLER(
SHELL_CMD_OUTPUT("kernel_version", "uname -r")
SHELL_CMD_OUTPUT("helloWorld", "cat /tmp/agentHelloWorld 2>/dev/null")
SHELL_CMD_OUTPUT("report_timestamp", "date -u +\%s")
#endif // SHELL_CMD_OUTPUT
@@ -282,7 +298,7 @@ FILE_CONTENT_HANDLER("AppSecModelVersion", "<FILESYSTEM-PREFIX>/conf/waap/waap.d
#endif // FILE_CONTENT_HANDLER
#ifdef SHELL_POST_CMD
#if defined(smb)
#if defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1)
SHELL_POST_CMD("remove local.cfg", "rm -rf /tmp/local.cfg")
#endif //smb
#endif

1
components/security_apps/orchestration/env_details/CMakeLists.txt
View File

@@ -1 +0,0 @@
add_library(env_details env_details.cc)

85
components/security_apps/orchestration/env_details/env_details.cc
View File

@@ -1,85 +0,0 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "env_details.h"
#include "config.h"
#include "debug.h"
#include "orchestration_tools.h"
using namespace std;
USE_DEBUG_FLAG(D_LOCAL_POLICY);
static const string k8s_service_account = "/var/run/secrets/kubernetes.io/serviceaccount";
// LCOV_EXCL_START Reason: can't use on the pipline environment
EnvDetails::EnvDetails() : env_type(EnvType::LINUX)
{
auto tools = Singleton::Consume<I_OrchestrationTools>::from<OrchestrationTools>();
if (tools->doesFileExist("/.dockerenv")) env_type = EnvType::DOCKER;
token = retrieveToken();
agent_namespace = retrieveNamespace();
if (!token.empty()) {
auto env_res = getenv("deployment_type");
env_type = env_res != nullptr && env_res == string("non_crd_k8s") ? EnvType::NON_CRD_K8S : EnvType::K8S;
}
}
EnvType
EnvDetails::getEnvType()
{
return env_type;
}
string
EnvDetails::getToken()
{
return token;
}
string
EnvDetails::getNameSpace()
{
return agent_namespace;
}
string
EnvDetails::retrieveToken()
{
return readFileContent(k8s_service_account + "/token");
}
string
EnvDetails::retrieveNamespace()
{
return readFileContent(k8s_service_account + "/namespace");
}
string
EnvDetails::readFileContent(const string &file_path)
{
try {
ifstream file(file_path);
stringstream buffer;
buffer << file.rdbuf();
return buffer.str();
} catch (ifstream::failure &f) {
dbgWarning(D_LOCAL_POLICY)
<< "Cannot read the file"
<< " File: " << file_path
<< " Error: " << f.what();
return "";
}
}
// LCOV_EXCL_STOP

3
components/security_apps/orchestration/orchestration_comp.cc
View File

@@ -1630,7 +1630,7 @@ private:
string server_name = getAttribute("registered-server", "registered_server");
auto server = TagAndEnumManagement::convertStringToTag(server_name);
if (server_name == "'SWAG'") server = Tags::WEB_SERVER_SWAG;
if (server_name == "'SWAG'" || server_name == "'SWAG Server'") server = Tags::WEB_SERVER_SWAG;
if (server.ok()) tags.insert(*server);
if (getAttribute("no-setting", "CROWDSEC_ENABLED") == "true") tags.insert(Tags::CROWDSEC);
@@ -2055,7 +2055,6 @@ private:
OrchestrationPolicy policy;
UpdatesProcessReporter updates_process_reporter_listener;
HybridModeMetric hybrid_mode_metric;
EnvDetails env_details;
chrono::minutes upgrade_delay_time;
string filesystem_prefix = "";

4
components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc
View File

@@ -28,6 +28,7 @@ std::ostream & operator<<(std::ostream &os, const Package &) { return os; }
#include "health_check_status/health_check_status.h"
#include "updates_process_event.h"
#include "declarative_policy_utils.h"
#include "mock/mock_env_details.h"
using namespace testing;
using namespace std;
@@ -324,6 +325,7 @@ public:
StrictMock<MockOrchestrationTools> mock_orchestration_tools;
StrictMock<MockDownloader> mock_downloader;
StrictMock<MockShellCmd> mock_shell_cmd;
StrictMock<EnvDetailsMocker> mock_env_details;
StrictMock<MockMessaging> mock_message;
StrictMock<MockRestApi> rest;
StrictMock<MockServiceController> mock_service_controller;
@@ -583,6 +585,8 @@ TEST_F(OrchestrationTest, check_sending_registration_data)
env.init();
init();
EXPECT_CALL(mock_env_details, getEnvType()).WillRepeatedly(Return(EnvType::LINUX));
EXPECT_CALL(mock_service_controller, updateServiceConfiguration(_, _, _, _, _, _))
.WillOnce(Return(Maybe<void>()));
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, _)).WillRepeatedly(Return(string()));

124
components/security_apps/rate_limit/rate_limit.cc
View File

@@ -246,6 +246,27 @@ public:
return matched_rule;
}
void
fetchReplicaCount()
{
string curl_cmd =
"curl -H \"Authorization: Bearer " + kubernetes_token + "\" "
"https://kubernetes.default.svc.cluster.local/apis/apps/v1/namespaces/" + kubernetes_namespace +
"/deployments/${AGENT_DEPLOYMENT_NAME} -k -s | jq .status.replicas";
auto maybe_replicas = i_shell_cmd->getExecOutput(curl_cmd);
if (maybe_replicas.ok()) {
try {
replicas = std::stoi(maybe_replicas.unpack());
} catch (const std::exception &e) {
dbgWarning(D_RATE_LIMIT) << "error while converting replicas: " << e.what();
}
}
if (replicas == 0) {
dbgWarning(D_RATE_LIMIT) << "replicas is set to 0, setting replicas to 1";
replicas = 1;
}
}
EventVerdict
respond(const HttpRequestHeaderEvent &event) override
{
@@ -271,10 +292,72 @@ public:
dbgDebug(D_RATE_LIMIT) << "source identifier value: " << source_identifier;
auto maybe_source_ip = env->get<IPAddr>(HttpTransactionData::client_ip_ctx);
set<string> ip_set;
string source_ip = "";
if (maybe_source_ip.ok()) source_ip = ipAddrToStr(maybe_source_ip.unpack());
if (maybe_source_ip.ok()) {
source_ip = ipAddrToStr(maybe_source_ip.unpack());
unordered_map<string, set<string>> condition_map = createConditionMap(uri, source_ip, source_identifier);
if (getProfileAgentSettingWithDefault<bool>(false, "agent.rateLimit.ignoreSourceIP")) {
dbgDebug(D_RATE_LIMIT) << "Rate limit ignoring source ip: " << source_ip;
} else {
ip_set.insert(source_ip);
}
}
auto maybe_xff = env->get<string>(HttpTransactionData::xff_vals_ctx);
if (!maybe_xff.ok()) {
dbgTrace(D_RATE_LIMIT) << "Rate limit failed to get xff vals from env";
} else {
auto ips = split(maybe_xff.unpack(), ',');
ip_set.insert(ips.begin(), ips.end());
}
EnumArray<I_GeoLocation::GeoLocationField, string> geo_location_data;
set<string> country_codes;
set<string> country_names;
for (const string& source : ip_set) {
Maybe<IPAddr> maybe_source_ip = IPAddr::createIPAddr(source);
if (!maybe_source_ip.ok()){
dbgWarning(D_RATE_LIMIT)
<< "Rate limit failed to create ip address from source: "
<< source
<< ", Error: "
<< maybe_source_ip.getErr();
continue;
}
auto asset_location =
Singleton::Consume<I_GeoLocation>::by<RateLimit>()->lookupLocation(maybe_source_ip.unpack());
if (!asset_location.ok()) {
dbgWarning(D_RATE_LIMIT)
<< "Rate limit lookup location failed for source: "
<< source_ip
<< ", Error: "
<< asset_location.getErr();
continue;
}
geo_location_data = asset_location.unpack();
auto code = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE];
auto name = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_NAME];
country_codes.insert(code);
country_names.insert(name);
dbgTrace(D_RATE_LIMIT)
<< "Rate limit found "
<< "country code: "
<< code
<< ", country name: "
<< name
<< ", source ip address: "
<< source;
}
unordered_map<string, set<string>> condition_map = createConditionMap(
uri,
source_ip,
source_identifier,
country_codes,
country_names
);
if (shouldApplyException(condition_map)) {
dbgDebug(D_RATE_LIMIT) << "found accept exception, not enforcing rate limit on this URI: " << uri;
return ACCEPT;
@@ -293,11 +376,6 @@ public:
return ACCEPT;
}
auto replicas = getenv("REPLICA_COUNT") ? std::stoi(getenv("REPLICA_COUNT")) : 1;
if (replicas == 0) {
dbgWarning(D_RATE_LIMIT) << "REPLICA_COUNT environment variable is set to 0, setting REPLICA_COUNT to 1";
replicas = 1;
}
burst = static_cast<float>(rule.getRateLimit()) / replicas;
limit = static_cast<float>(calcRuleLimit(rule)) / replicas;
@@ -476,10 +554,18 @@ public:
}
unordered_map<string, set<string>>
createConditionMap(const string &uri, const string &source_ip, const string &source_identifier)
createConditionMap(
const string &uri,
const string &source_ip,
const string &source_identifier,
const set<string> &country_codes,
const set<string> &country_names
)
{
unordered_map<string, set<string>> condition_map;
if (!source_ip.empty()) condition_map["sourceIP"].insert(source_ip);
if (!country_codes.empty()) condition_map["countryCode"].insert(country_codes.begin(), country_codes.end());
if (!country_names.empty()) condition_map["countryName"].insert(country_names.begin(), country_names.end());
condition_map["sourceIdentifier"].insert(source_identifier);
condition_map["url"].insert(uri);
@@ -616,6 +702,21 @@ public:
"Initialize rate limit component",
false
);
i_shell_cmd = Singleton::Consume<I_ShellCmd>::by<RateLimit>();
i_env_details = Singleton::Consume<I_EnvDetails>::by<RateLimit>();
env_type = i_env_details->getEnvType();
if (env_type == EnvType::K8S) {
kubernetes_token = i_env_details->getToken();
kubernetes_namespace = i_env_details->getNameSpace();
fetchReplicaCount();
Singleton::Consume<I_MainLoop>::by<RateLimit>()->addRecurringRoutine(
I_MainLoop::RoutineType::Offline,
chrono::seconds(120),
[this]() { fetchReplicaCount(); },
"Fetch current replica count from the Kubernetes cluster"
);
}
}
void
@@ -624,6 +725,9 @@ public:
disconnectRedis();
}
I_ShellCmd *i_shell_cmd = nullptr;
I_EnvDetails* i_env_details = nullptr;
private:
static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
@@ -634,6 +738,10 @@ private:
int burst;
float limit;
redisContext* redis = nullptr;
int replicas = 1;
EnvType env_type;
string kubernetes_namespace = "";
string kubernetes_token = "";
};
RateLimit::RateLimit() : Component("RateLimit"), pimpl(make_unique<Impl>()) {}

4
components/security_apps/waap/waap_clib/Serializator.cc
View File

@@ -194,6 +194,10 @@ void SerializeToFileBase::saveData()
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to gzip data";
} else {
ss.str(string((const char *)res.output, res.num_output_bytes));
// free the memory allocated by compressData
if (res.output) free(res.output);
res.output = nullptr;
res.num_output_bytes = 0;
}
if (res.output) free(res.output);
res.output = nullptr;

14
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@@ -112,20 +112,6 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
}
double res_score = getScoreFromPool(res, newKeywords, poolName);
std::string other_pool_name = Waap::Scores::getOtherScorePoolName();
Waap::Scores::ModelLoggingSettings modelLoggingSettings = Waap::Scores::getModelLoggingSettings();
if (applyLearning && poolName != other_pool_name &&
modelLoggingSettings.logLevel != Waap::Scores::ModelLogLevel::OFF) {
double other_score = getScoreFromPool(res, newKeywords, other_pool_name);
dbgDebug(D_WAAP_SCANNER) << "Comparing score from pool " << poolName << ": " << res_score
<< ", vs. pool " << other_pool_name << ": " << other_score
<< ", score difference: " << res_score - other_score
<< ", sample: " << res.unescaped_line;
res.other_model_score = other_score;
} else {
res.other_model_score = res_score;
}
return res_score;
}