github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-06-28 16:41:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #166 from openappsec/Jul_23_2024-Dev

Browse Source 
Jul 23rd update
This commit is contained in:
WrightNed 2024-07-24 16:01:45 +03:00 committed by GitHub
commit 705a5e6061
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 172 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc
View File

@ -155,6 +155,24 @@ getWaitingForVerdictThreadTimeout()
return conf_data.getNumericalValue("waiting_for_verdict_thread_timeout_msec");
}
unsigned int
getMinRetriesForVerdict()
{
return conf_data.getNumericalValue("min_retries_for_verdict");
}
unsigned int
getMaxRetriesForVerdict()
{
return conf_data.getNumericalValue("max_retries_for_verdict");
}
unsigned int
getReqBodySizeTrigger()
{
return conf_data.getNumericalValue("body_size_trigger");
}
int
isIPAddress(c_str ip_str)
{

8
attachments/nginx/nginx_attachment_util/nginx_attachment_util_ut/nginx_attachment_util_ut.cc
View File

@ -63,7 +63,10 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration)
"\"waiting_for_verdict_thread_timeout_msec\": 75,\n"
"\"req_header_thread_timeout_msec\": 10,\n"
"\"ip_ranges\": " + createIPRangesString(ip_ranges) + ",\n"
"\"static_resources_path\": \"" + static_resources_path + "\""
"\"static_resources_path\": \"" + static_resources_path + "\",\n"
"\"min_retries_for_verdict\": 1,\n"
"\"max_retries_for_verdict\": 3,\n"
"\"body_size_trigger\": 777\n"
"}\n";
ofstream valid_configuration_file(attachment_configuration_file_name);
valid_configuration_file << valid_configuration;
@ -87,6 +90,9 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration)
EXPECT_EQ(getReqBodyThreadTimeout(), 155);
EXPECT_EQ(getResHeaderThreadTimeout(), 1);
EXPECT_EQ(getResBodyThreadTimeout(), 0);
EXPECT_EQ(getMinRetriesForVerdict(), 1);
EXPECT_EQ(getMaxRetriesForVerdict(), 3);
EXPECT_EQ(getReqBodySizeTrigger(), 777);
EXPECT_EQ(getWaitingForVerdictThreadTimeout(), 75);
EXPECT_EQ(getInspectionMode(), ngx_http_inspection_mode::BLOCKING_THREAD);

26
components/attachment-intakers/nginx_attachment/nginx_attachment_config.cc
View File

@ -42,6 +42,7 @@ HttpAttachmentConfig::init()
setNumOfNginxIpcElements();
setDebugByContextValues();
setKeepAliveIntervalMsec();
setRetriesForVerdict();
}
bool
@ -215,6 +216,31 @@ HttpAttachmentConfig::setFailOpenTimeout()
conf_data.setNumericalValue("nginx_inspection_mode", inspection_mode);
}
void
HttpAttachmentConfig::setRetriesForVerdict()
{
conf_data.setNumericalValue("min_retries_for_verdict", getAttachmentConf<uint>(
3,
"agent.minRetriesForVerdict.nginxModule",
"HTTP manager",
"Min retries for verdict"
));
conf_data.setNumericalValue("max_retries_for_verdict", getAttachmentConf<uint>(
15,
"agent.maxRetriesForVerdict.nginxModule",
"HTTP manager",
"Max retries for verdict"
));
conf_data.setNumericalValue("body_size_trigger", getAttachmentConf<uint>(
200000,
"agent.reqBodySizeTrigger.nginxModule",
"HTTP manager",
"Request body size trigger"
));
}
void
HttpAttachmentConfig::setFailOpenWaitMode()
{

2
components/attachment-intakers/nginx_attachment/nginx_attachment_config.h
View File

@ -70,6 +70,8 @@ private:
void setDebugByContextValues();
void setRetriesForVerdict();
WebTriggerConf web_trigger_conf;
HttpAttachmentConfiguration conf_data;
};

4
components/security_apps/local_policy_mgmt_gen/triggers_section.cc
View File

@ -96,8 +96,8 @@ LogTriggerSection::save(cereal::JSONOutputArchive &out_ar) const
cereal::make_nvp("acDrop", acDrop),
cereal::make_nvp("complianceViolations", false),
cereal::make_nvp("complianceWarnings", false),
cereal::make_nvp("extendloggingMinSeverity", extendloggingMinSeverity),
cereal::make_nvp("extendlogging", extendlogging),
cereal::make_nvp("extendLoggingMinSeverity", extendloggingMinSeverity),
cereal::make_nvp("extendLogging", extendlogging),
cereal::make_nvp("logToAgent", logToAgent),
cereal::make_nvp("logToCef", logToCef),
cereal::make_nvp("logToCloud", logToCloud),

12
components/security_apps/orchestration/details_resolver/details_resolving_handler.cc
View File

@ -99,6 +99,7 @@ map<string, string>
DetailsResolvingHanlder::Impl::getResolvedDetails() const
{
I_ShellCmd *shell = Singleton::Consume<I_ShellCmd>::by<DetailsResolvingHanlder>();
I_AgentDetailsReporter *reporter = Singleton::Consume<I_AgentDetailsReporter>::by<DetailsResolvingHanlder>();
uint32_t timeout = getConfigurationWithDefault<uint32_t>(5000, "orchestration", "Details resolver time out");
for (auto &shell_pre_command : shell_pre_commands) {
@ -122,7 +123,15 @@ DetailsResolvingHanlder::Impl::getResolvedDetails() const
Maybe<string> shell_command_output = getCommandOutput(command);
if (!shell_command_output.ok()) continue;
Maybe<string> handler_ret = handler(*shell_command_output);
if (handler_ret.ok()) resolved_details[attr] = *handler_ret;
if (handler_ret.ok()) {
resolved_details[attr] = *handler_ret;
} else {
if (reporter->isPersistantAttr(attr)) {
dbgTrace(D_AGENT_DETAILS)<< "Persistent attribute changed, removing old value";
reporter->deleteAttr(attr);
}
}
}
for (auto file_handler : file_content_handlers) {
@ -157,7 +166,6 @@ DetailsResolvingHanlder::Impl::getResolvedDetails() const
}
}
I_AgentDetailsReporter *reporter = Singleton::Consume<I_AgentDetailsReporter>::by<DetailsResolvingHanlder>();
reporter->addAttr(resolved_details, true);
return resolved_details;

13
components/utils/generic_rulebase/evaluators/parameter_eval.cc
View File

@ -22,6 +22,8 @@
using namespace std;
USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
string ParameterMatcher::ctx_key = "parameters";
ParameterMatcher::ParameterMatcher(const vector<string> &params)
@ -33,6 +35,17 @@ ParameterMatcher::ParameterMatcher(const vector<string> &params)
Maybe<bool, Context::Error>
ParameterMatcher::evalVariable() const
{
I_Environment *env = Singleton::Consume<I_Environment>::by<ParameterMatcher>();
auto bc_param_id_ctx = env->get<set<GenericConfigId>>(ParameterMatcher::ctx_key);
dbgTrace(D_RULEBASE_CONFIG)
<< "Trying to match parameter. ID: "
<< parameter_id << ", Current set IDs: "
<< makeSeparatedStr(bc_param_id_ctx.ok() ? *bc_param_id_ctx : set<GenericConfigId>(), ", ");
if (bc_param_id_ctx.ok()) return bc_param_id_ctx.unpack().count(parameter_id) > 0;
dbgTrace(D_RULEBASE_CONFIG)
<< "Did not find current parameter in context."
<< " Match parameter from current rule";
auto rule = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
return rule.ok() && rule.unpack().isParameterActive(parameter_id);
}

12
core/agent_details_reporter/agent_details_reporter.cc
View File

@ -71,6 +71,7 @@ public:
bool addAttr(const string &key, const string &val, bool allow_override = false) override;
bool addAttr(const map<string, string> &attr, bool allow_override = false) override;
void deleteAttr(const string &key) override;
bool isPersistantAttr(const string &key) override;
bool sendAttributes() override;
@ -130,6 +131,7 @@ private:
map<string, string> persistant_attributes;
map<string, string> new_attributes;
map<string, string> attributes;
bool is_attr_deleted = false;
I_Messaging *messaging = nullptr;
bool is_server;
@ -207,6 +209,13 @@ AgentDetailsReporter::Impl::deleteAttr(const string &key)
attributes.erase(key);
new_attributes.erase(key);
persistant_attributes.erase(key);
is_attr_deleted = true;
}
bool
AgentDetailsReporter::Impl::isPersistantAttr(const std::string &key)
{
return persistant_attributes.count(key) > 0;
}
bool
@ -214,7 +223,7 @@ AgentDetailsReporter::Impl::sendAttributes()
{
dbgDebug(D_AGENT_DETAILS) << "Trying to send attributes";
if (new_attributes.empty()) {
if (new_attributes.empty() && !is_attr_deleted) {
dbgDebug(D_AGENT_DETAILS) << "Skipping current attempt since no new attributes were added";
return true;
}
@ -261,6 +270,7 @@ AgentDetailsReporter::Impl::sendAttributes()
if (add_agent_details_status.ok()) {
dbgDebug(D_AGENT_DETAILS) << "Successfully sent attributes to the Orchestrator";
new_attributes.clear();
is_attr_deleted = false;
return true;
}

1
core/agent_details_reporter/agent_details_reporter_ut/agent_details_reporter_ut.cc
View File

@ -213,6 +213,7 @@ TEST_F(AgentReporterTest, basicAttrTest)
EXPECT_TRUE(report->addAttr({{"c", "d"}, {"1", "2"}, {"delete", "me"}}));
EXPECT_FALSE(report->addAttr("a", "d"));
EXPECT_TRUE(report->addAttr("a", "1", true));
EXPECT_TRUE(report->isPersistantAttr("a"));
report->deleteAttr("delete");
{
AgentDataReport agent_data;

8
core/attachments/http_configuration/http_configuration.cc
View File

@ -108,7 +108,10 @@ HttpAttachmentConfiguration::save(cereal::JSONOutputArchive &archive) const
),
cereal::make_nvp("nginx_inspection_mode", getNumericalValue("inspection_mode")),
cereal::make_nvp("num_of_nginx_ipc_elements", getNumericalValue("num_of_nginx_ipc_elements")),
cereal::make_nvp("keep_alive_interval_msec", getNumericalValue("keep_alive_interval_msec"))
cereal::make_nvp("keep_alive_interval_msec", getNumericalValue("keep_alive_interval_msec")),
cereal::make_nvp("min_retries_for_verdict", getNumericalValue("min_retries_for_verdict")),
cereal::make_nvp("max_retries_for_verdict", getNumericalValue("max_retries_for_verdict")),
cereal::make_nvp("body_size_trigger", getNumericalValue("body_size_trigger"))
);
}
@ -161,6 +164,9 @@ HttpAttachmentConfiguration::load(cereal::JSONInputArchive &archive)
loadNumericalValue(archive, "nginx_inspection_mode", 0);
loadNumericalValue(archive, "num_of_nginx_ipc_elements", 200);
loadNumericalValue(archive, "keep_alive_interval_msec", DEFAULT_KEEP_ALIVE_INTERVAL_MSEC);
loadNumericalValue(archive, "min_retries_for_verdict", 3);
loadNumericalValue(archive, "max_retries_for_verdict", 15);
loadNumericalValue(archive, "body_size_trigger", 200000);
}
bool

3
core/include/attachments/nginx_attachment_util.h
View File

@ -54,6 +54,9 @@ unsigned int getReqBodyThreadTimeout();
unsigned int getResProccessingTimeout();
unsigned int getResHeaderThreadTimeout();
unsigned int getResBodyThreadTimeout();
unsigned int getMinRetriesForVerdict();
unsigned int getMaxRetriesForVerdict();
unsigned int getReqBodySizeTrigger();
unsigned int getWaitingForVerdictThreadTimeout();

1
core/include/services_sdk/interfaces/i_agent_details_reporter.h
View File

@ -48,6 +48,7 @@ public:
virtual bool addAttr(const std::map<std::string, std::string> &attr, bool allow_override = false) = 0;
virtual void deleteAttr(const std::string &key) = 0;
virtual bool sendAttributes() = 0;
virtual bool isPersistantAttr(const std::string &key) = 0;
protected:
~I_AgentDetailsReporter() = default;

1
core/include/services_sdk/interfaces/mock/mock_agent_details_reporter.h
View File

@ -26,6 +26,7 @@ public:
MOCK_METHOD3(addAttr, bool(const std::string &key, const std::string &val, bool allow_override));
MOCK_METHOD2(addAttr, bool(const std::map<std::string, std::string> &attr, bool allow_override));
MOCK_METHOD1(deleteAttr, void(const std::string &key));
MOCK_METHOD1(isPersistantAttr, bool(const std::string &key));
MOCK_METHOD0(sendAttributes, bool());
};

2
nodes/orchestration/package/k8s-check-update-listener.sh
View File

@ -69,7 +69,7 @@ while true; do
exception_pid=$!
saveRuningPids
fi
if [ ! -d /proc/${exception_pid} ]; then
if [ ! -d /proc/${policy_pid} ]; then
runGetResourceListener policies
policy_pid=$!
saveRuningPids

62
nodes/orchestration/package/open-appsec-cloud-mgmt-k8s
View File

@ -15,11 +15,13 @@ latest_policy_version=1
load_agent_details()
{
tenant_id=$(cat /etc/cp/conf/agent_details.json | sed "s|Tenant ID|TenantID|g" | /etc/cp/bin/yq -P '.TenantID')
agent_id=$(cat /etc/cp/conf/agent_details.json | sed "s|Agent ID|AgentID|g" | /etc/cp/bin/yq -P '.AgentID')
profile_id=$(cat /etc/cp/conf/agent_details.json | sed "s|Profile ID|ProfileID|g" | /etc/cp/bin/yq -P '.ProfileID')
tenant_id=$(awk -F\" '/Tenant ID/{print $4}' /etc/cp/conf/agent_details.json)
agent_id=$(awk -F\" '/Agent ID/{print $4}' /etc/cp/conf/agent_details.json)
profile_id=$(awk -F\" '/Profile ID/{print $4}' /etc/cp/conf/agent_details.json)
cluster_id=$(echo $(curl -s --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" -X GET ${APISERVER}/api/v1/namespaces/ ) \
| /etc/cp/bin/yq .items | /etc/cp/bin/yq '.[] | select(.metadata.name | contains("kube-system"))' | /etc/cp/bin/yq .metadata.uid)
| /etc/cp/bin/yq eval '.items' - \
| /etc/cp/bin/yq eval '.[] | select(.metadata.name | contains("kube-system"))' - \
| /etc/cp/bin/yq eval '.metadata.uid' -)
}
get_latest_policy_version()
@ -44,15 +46,16 @@ get_latest_policy_version()
concat_to_policy()
{
crd_to_concat="$1"
is_first=$2
api_version="$1"
crd_to_concat="$2"
is_first=$3
if [ ! -z $is_first ]; then
POLICY="$POLICY \"$1\": "
POLICY="$POLICY \"$crd_to_concat\": "
else
POLICY="$POLICY, \"$1\": "
POLICY="$POLICY, \"$crd_to_concat\": "
fi
CRD=$(curl -s --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" \
-X GET ${APISERVER}/apis/openappsec.io/v1beta1/$crd_to_concat)
-X GET ${APISERVER}/apis/openappsec.io/$api_version/$crd_to_concat)
CRD=$(echo $CRD|tr -d '\n')
if [ -z "$CRD" ]; then
CRD="{}"
@ -60,28 +63,49 @@ concat_to_policy()
POLICY="$POLICY $CRD"
}
get_api_version()
{
CRD=$(curl -s --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" \
-X GET ${APISERVER}/apis/openappsec.io/v1beta2/policies)
CRD=$(echo $CRD|tr -d '\n')
# if CRD is not empty and does not contain "page not found" then it is v1beta2
if [ ! -z "$CRD" ] && ! echo "$CRD" | grep -q "page not found"; then
echo "v1beta2"
else
echo "v1beta1"
fi
}
generate_policy()
{
POLICY="{ \"Policy\": {"
concat_to_policy policies true
concat_to_policy practices
concat_to_policy logtriggers
concat_to_policy customresponses
concat_to_policy exceptions
concat_to_policy sourcesidentifiers
concat_to_policy trustedsources
api_version=$(get_api_version)
concat_to_policy $api_version "policies" true
if [ "$api_version" = "v1beta2" ]; then
concat_to_policy $api_version "threatpreventionpractices"
concat_to_policy $api_version "accesscontrolpractices"
else
concat_to_policy $api_version "practices"
fi
concat_to_policy $api_version "logtriggers"
concat_to_policy $api_version "customresponses"
concat_to_policy $api_version "exceptions"
concat_to_policy $api_version "sourcesidentifiers"
concat_to_policy $api_version "trustedsources"
POLICY="$POLICY, \"assets\": { \"items\":[ "
FIRST="1"
all_ingresses=$(curl -s --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" \
-X GET ${APISERVER}/apis/networking.k8s.io/v1/ingresses)
namespaces=$(echo $all_ingresses | /etc/cp/bin/yq -P '.items[].metadata.namespace')
namespaces=$(echo $all_ingresses | /etc/cp/bin/yq eval '.items[].metadata.namespace' -)
for ns in ${namespaces}; do
ingress_in_ns=$(curl -s --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" \
-X GET ${APISERVER}/apis/networking.k8s.io/v1/namespaces/${ns}/ingresses)
ingress_list=$(echo $ingress_in_ns | /etc/cp/bin/yq -P '.items[].metadata.name')
ingress_list=$(echo $ingress_in_ns | /etc/cp/bin/yq eval '.items[].metadata.name' -)
for ingress_name in ${ingress_list}; do
ingress_crd=$(curl -s --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" \
-X GET ${APISERVER}/apis/networking.k8s.io/v1/namespaces/${ns}/ingresses/${ingress_name})
@ -273,7 +297,7 @@ while true; do
done
if [ -z "$var_fog" ]; then
var_fog=$(cat /etc/cp/conf/agent_details.json | sed "s|Fog domain|Fogdomain|g" | /etc/cp/bin/yq -P '.Fogdomain')
var_fog=$(awk -F\" '/Fog domain/{print $4}' /etc/cp/conf/agent_details.json)
var_fog="https://$var_fog"
fi

9
nodes/orchestration/package/open-appsec-ctl.sh
View File

@ -1363,9 +1363,12 @@ run_ai() # Initials - ra
exit 1
fi
if [ "$ra_upload_to_fog" = "true" ]; then
ra_token_data=$(curl_to_orchestration "show-access-token")
ra_token_hex=$(echo "$ra_token_data" | grep "token" | cut -d '"' -f4 | base64 -d | od -t x1 -An)
ra_token_hex_formatted=$(echo $ra_token_hex | tr -d ' ')
ra_token_data=$(curl_to_orchestration "show-access-token" | grep "token" | cut -d '"' -f4)
if [ -z "${ra_token_data}" ]; then
echo "Failed to get crediantials to upload the file to the cloud."
exit 1;
fi
ra_token_hex_formatted=$(echo $ra_token_data | base64 -d | od -t x1 -An | tr -d '[:space:]')
ra_token="$(xor_decrypt "${ra_token_hex_formatted}")"
ra_proxy_val=""

28
nodes/orchestration/package/orchestration_package.sh
View File

@ -302,13 +302,15 @@ while true; do
echo "Filesystem paths: ${FILESYSTEM_PATH}"
elif [ "$1" = "--vs_id" ]; then
shift
VS_ID=$1
export FILESYSTEM_PATH="/etc/cp/vs${VS_ID}"
NANO_AGENT_SERVICE_NAME="nano_agent_${VS_ID}"
NANO_AGENT_SERVICE_FILE="${NANO_AGENT_SERVICE_NAME}.service"
VS_LIB_SUB_FOLDER="/vs${VS_ID}"
LOG_FILE_PATH="${LOG_FILE_PATH}/vs${VS_ID}"
TMP_FOLDER="${TMP_FOLDER}/vs${VS_ID}"
if [ "$1" != "0" ]; then
VS_ID=$1
export FILESYSTEM_PATH="/etc/cp/vs${VS_ID}"
NANO_AGENT_SERVICE_NAME="nano_agent_${VS_ID}"
NANO_AGENT_SERVICE_FILE="${NANO_AGENT_SERVICE_NAME}.service"
VS_LIB_SUB_FOLDER="/vs${VS_ID}"
LOG_FILE_PATH="${LOG_FILE_PATH}/vs${VS_ID}"
TMP_FOLDER="${TMP_FOLDER}/vs${VS_ID}"
fi
elif [ "$1" = "--log_files_path" ]; then
shift
var=$1
@ -360,6 +362,16 @@ if [ -z "$VS_ID" ]; then
fi
fi
if [ -n "${VS_ID}" ]; then
if [ "$VS_ID" != "$INSTANCE_VSID" ]; then
echo "Error: Incorrect context, switch to VS${VS_ID} context first."
exit 1
fi
elif [ -n "$INSTANCE_VSID" ] && [ "$INSTANCE_VSID" != "0" ]; then
echo "Error: Incorrect context, exit vs${INSTANCE_VSID} first."
exit 1
fi
if [ "$RUN_MODE" = "install" ] && [ $var_offline_mode = false ]; then
if [ -n "$OTP_TOKEN" ] && [ -z "$var_token" ] && [ "$var_no_otp" = "false" ]; then
var_token=$OTP_TOKEN
@ -846,7 +858,7 @@ install_public_key()
fog_address=${var_fog_address}
if [ -n "${var_upgrade_mode}" ]; then
# Upgradde - look in policy.json
# Upgrade - look in policy.json
fog_address=$(cat ${FILESYSTEM_PATH}/${CONF_PATH}/${SERVICE_PATH}/orchestration.policy)
fi