github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feb 10 2025 dev (#255)

Browse Source 
* sync code

* sync code

* code sync

* code sync

---------

Co-authored-by: Ned Wright <nedwright@proton.me>
Co-authored-by: Daniel Eisenberg <danielei@checkpoint.com>
This commit is contained in:
Daniel-Eisenberg
2025-02-12 10:56:44 +02:00
committed by GitHub
parent 81433bac25
commit 4ddcd2462a
75 changed files with 1540 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc
View File

@@ -497,7 +497,8 @@ WebAppSection::WebAppSection(
const AppsecPracticeAntiBotSection &_anti_bots,
const LogTriggerSection &parsed_log_trigger,
const AppSecTrustedSources &parsed_trusted_sources,
const NewAppSecWebAttackProtections &protections)
const NewAppSecWebAttackProtections &protections,
const vector<InnerException> &exceptions)
:
application_urls(_application_urls),
asset_id(_asset_id),
@@ -541,6 +542,10 @@ WebAppSection::WebAppSection(
overrides.push_back(AppSecOverride(source_ident));
}
for (const auto &exception : exceptions) {
overrides.push_back(AppSecOverride(exception));
}
}
// LCOV_EXCL_STOP

3
components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h
View File

@@ -298,7 +298,8 @@ public:
const AppsecPracticeAntiBotSection &_anti_bots,
const LogTriggerSection &parsed_log_trigger,
const AppSecTrustedSources &parsed_trusted_sources,
const NewAppSecWebAttackProtections &protections);
const NewAppSecWebAttackProtections &protections,
const std::vector<InnerException> &exceptions);
void save(cereal::JSONOutputArchive &out_ar) const;

1
components/security_apps/local_policy_mgmt_gen/include/policy_activation_data.h
View File

@@ -45,7 +45,6 @@ public:
private:
std::string name;
std::string mode;
std::vector<std::string> hosts;
};

3
components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h
View File

@@ -206,7 +206,8 @@ private:
const RulesConfigRulebase& rule_config,
const std::string &practice_id, const std::string &full_url,
const std::string &default_mode,
std::map<AnnotationTypes, std::string> &rule_annotations
std::map<AnnotationTypes, std::string> &rule_annotations,
std::vector<InnerException>
);
void

8
components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc
View File

@@ -698,8 +698,12 @@ K8sPolicyUtils::createAppsecPolicies()
}
}
auto maybe_policy_activation =
getObjectFromCluster<PolicyActivationData>("/apis/openappsec.io/v1beta2/policyactivations");
string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : "";
string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : "";
auto maybe_policy_activation = getObjectFromCluster<PolicyActivationData>(
"/apis/openappsec.io/v1beta2/" + ns + agent_ns + "policyactivations" + ns_suffix
);
if (!maybe_policy_activation.ok()) {
dbgWarning(D_LOCAL_POLICY)

2
components/security_apps/local_policy_mgmt_gen/new_trusted_sources.cc
View File

@@ -69,7 +69,7 @@ Identifier::load(cereal::JSONInputArchive &archive_in)
dbgWarning(D_LOCAL_POLICY) << "AppSec identifier invalid: " << identifier;
identifier = "sourceip";
}
parseMandatoryAppsecJSONKey<vector<string>>("value", value, archive_in);
parseAppsecJSONKey<vector<string>>("value", value, archive_in);
}
const string &

13
components/security_apps/local_policy_mgmt_gen/policy_activation_data.cc
View File

@@ -18,14 +18,6 @@ using namespace std;
USE_DEBUG_FLAG(D_LOCAL_POLICY);
static const set<string> valid_modes = {
"prevent-learn",
"detect-learn",
"prevent",
"detect",
"inactive"
};
void
PolicyActivationMetadata::load(cereal::JSONInputArchive &archive_in)
{
@@ -39,11 +31,6 @@ EnabledPolicy::load(cereal::JSONInputArchive &archive_in)
dbgTrace(D_LOCAL_POLICY) << "Loading policyActivation enabled policy";
parseMandatoryAppsecJSONKey<vector<string>>("hosts", hosts, archive_in);
parseAppsecJSONKey<string>("name", name, archive_in);
parseAppsecJSONKey<string>("mode", mode, archive_in, "detect");
if (valid_modes.count(mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec policy activation mode invalid: " << mode;
mode = "detect";
}
}
const string &

10
components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc
View File

@@ -928,7 +928,6 @@ createMultiRulesSections(
PracticeSection practice = PracticeSection(practice_id, practice_type, practice_name);
vector<ParametersSection> exceptions_result;
for (auto exception : exceptions) {
const auto &exception_name = exception.first;
for (const auto &inner_exception : exception.second) {
exceptions_result.push_back(ParametersSection(inner_exception.getBehaviorId(), exception_name));
@@ -1220,7 +1219,8 @@ PolicyMakerUtils::createWebAppSection(
const string &practice_id,
const string &full_url,
const string &default_mode,
map<AnnotationTypes, string> &rule_annotations)
map<AnnotationTypes, string> &rule_annotations,
vector<InnerException> rule_inner_exceptions)
{
auto apssec_practice =
getAppsecPracticeSpec<V1beta2AppsecLinuxPolicy, NewAppSecPracticeSpec>(
@@ -1255,7 +1255,8 @@ PolicyMakerUtils::createWebAppSection(
apssec_practice.getAntiBot(),
log_triggers[rule_annotations[AnnotationTypes::TRIGGER]],
trusted_sources[rule_annotations[AnnotationTypes::TRUSTED_SOURCES]],
apssec_practice.getWebAttacks().getProtections()
apssec_practice.getWebAttacks().getProtections(),
rule_inner_exceptions
);
web_apps[rule_config.getAssetName()] = web_app;
}
@@ -1366,7 +1367,8 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
practice_id,
asset_name,
default_mode,
rule_annotations);
rule_annotations,
inner_exceptions[rule_annotations[AnnotationTypes::EXCEPTION]]);
}
}

18
components/security_apps/orchestration/manifest_controller/manifest_controller.cc
View File

@@ -100,6 +100,7 @@ private:
string packages_dir;
string orch_service_name;
set<string> ignore_packages;
Maybe<string> forbidden_versions = genError("Forbidden versions file does not exist");
};
void
@@ -135,7 +136,8 @@ ManifestController::Impl::init()
"Ignore packages list file path"
);
if (Singleton::Consume<I_OrchestrationTools>::by<ManifestController>()->doesFileExist(ignore_packages_path)) {
auto orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<ManifestController>();
if (orchestration_tools->doesFileExist(ignore_packages_path)) {
try {
ifstream input_stream(ignore_packages_path);
if (!input_stream) {
@@ -156,6 +158,9 @@ ManifestController::Impl::init()
<< " Error: " << f.what();
}
}
const string forbidden_versions_path = getFilesystemPathConfig() + "/revert/forbidden_versions";
forbidden_versions = orchestration_tools->readFile(forbidden_versions_path);
}
bool
@@ -271,6 +276,17 @@ ManifestController::Impl::updateManifest(const string &new_manifest_file)
}
map<string, Package> new_packages = parsed_manifest.unpack();
if (!new_packages.empty()) {
const Package &package = new_packages.begin()->second;
if (forbidden_versions.ok() &&
forbidden_versions.unpack().find(package.getVersion()) != string::npos
) {
dbgWarning(D_ORCHESTRATOR)
<< "Packages version is in the forbidden versions list. No upgrade will be performed.";
return true;
}
}
map<string, Package> all_packages = parsed_manifest.unpack();
map<string, Package> current_packages;
parsed_manifest = orchestration_tools->loadPackagesFromJson(manifest_file_path);

152
components/security_apps/orchestration/manifest_controller/manifest_controller_ut/manifest_controller_ut.cc
View File

@@ -58,6 +58,9 @@ public:
Debug::setUnitTestFlag(D_ORCHESTRATOR, Debug::DebugLevel::TRACE);
const string ignore_packages_file = "/etc/cp/conf/ignore-packages.txt";
EXPECT_CALL(mock_orchestration_tools, doesFileExist(ignore_packages_file)).WillOnce(Return(false));
Maybe<string> forbidden_versions(string("a1\na2"));
EXPECT_CALL(mock_orchestration_tools, readFile("/etc/cp/revert/forbidden_versions"))
.WillOnce(Return(forbidden_versions));
manifest_controller.init();
manifest_file_path = getConfigurationWithDefault<string>(
"/etc/cp/conf/manifest.json",
@@ -224,6 +227,10 @@ TEST_F(ManifestControllerTest, createNewManifest)
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, manifest_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillOnce(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -363,6 +370,11 @@ TEST_F(ManifestControllerTest, updateManifest)
EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).Times(2).WillRepeatedly(Return(true));
EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).Times(2).WillRepeatedly(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillOnce(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
manifest =
@@ -417,6 +429,9 @@ TEST_F(ManifestControllerTest, updateManifest)
EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(file_name)).WillOnce(Return(new_services));
EXPECT_CALL(mock_orchestration_tools,
loadPackagesFromJson(manifest_file_path)).WillOnce(Return(old_services));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillOnce(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillRepeatedly(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -478,6 +493,11 @@ TEST_F(ManifestControllerTest, selfUpdate)
EXPECT_CALL(mock_orchestration_tools, copyFile("/tmp/temp_file", path +
temp_ext)).WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillOnce(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -607,6 +627,10 @@ TEST_F(ManifestControllerTest, removeCurrentErrorPackage)
EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillOnce(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
corrupted_packages.clear();
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -666,6 +690,10 @@ TEST_F(ManifestControllerTest, selfUpdateWithOldCopy)
EXPECT_CALL(mock_orchestration_tools, copyFile("/tmp/temp_file", path +
temp_ext)).WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillOnce(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -722,6 +750,10 @@ TEST_F(ManifestControllerTest, selfUpdateWithOldCopyWithError)
EXPECT_CALL(mock_orchestration_tools, doesFileExist(path)).WillOnce(Return(false)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(path, path + backup_ext + temp_ext)).WillOnce(Return(false));
EXPECT_CALL(mock_details_resolver, getHostname()).WillOnce(Return(hostname));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillOnce(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_FALSE(i_manifest_controller->updateManifest(file_name));
}
@@ -798,6 +830,10 @@ TEST_F(ManifestControllerTest, installAndRemove)
EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).Times(2).WillRepeatedly(Return(true));
EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).Times(2).WillRepeatedly(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillOnce(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
string new_manifest =
@@ -858,6 +894,63 @@ TEST_F(ManifestControllerTest, installAndRemove)
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/packages/my1/my1")).Times(2)
.WillOnce(Return(false));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillOnce(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillRepeatedly(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
TEST_F(ManifestControllerTest, manifestWithForbiddenVersion)
{
new_services.clear();
old_services.clear();
string manifest =
"{"
" \"packages\": ["
" {"
" \"download-path\": \"http://172.23.92.135/my.sh\","
" \"relative-path\": \"\","
" \"name\": \"my\","
" \"version\": \"a1\","
" \"checksum-type\": \"sha1sum\","
" \"checksum\": \"a58bbab8020b0e6d08568714b5e582a3adf9c805\","
" \"package-type\": \"service\","
" \"require\": []"
" },"
" {"
" \"download-path\": \"http://172.23.92.135/my.sh\","
" \"relative-path\": \"\","
" \"name\": \"orchestration\","
" \"version\": \"a1\","
" \"checksum-type\": \"sha1sum\","
" \"checksum\": \"a58bbab8020b0e6d08568714b5e582a3adf9c805\","
" \"package-type\": \"service\","
" \"require\": []"
" },"
" {"
" \"download-path\": \"\","
" \"relative-path\": \"\","
" \"name\": \"waap\","
" \"version\": \"a1\","
" \"checksum-type\": \"sha1sum\","
" \"checksum\": \"\","
" \"package-type\": \"service\","
" \"status\": false,\n"
" \"message\": \"This security app isn't valid for this agent\"\n"
" }"
" ]"
"}";
map<string, Package> manifest_services;
load(manifest, manifest_services);
checkIfFileExistsCall(manifest_services.at("my"));
load(manifest, new_services);
load(old_manifest, old_services);
EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(file_name)).WillOnce(Return(new_services));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -947,6 +1040,10 @@ TEST_F(ManifestControllerTest, badInstall)
EXPECT_CALL(mock_orchestration_tools,
packagesToJsonFile(corrupted_packages, corrupted_file_list)).WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillOnce(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_FALSE(i_manifest_controller->updateManifest(file_name));
}
@@ -1112,6 +1209,12 @@ TEST_F(ManifestControllerTest, requireUpdate)
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, removeFile("new_manifest.json")).WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status"))
.WillOnce(Return(false))
.WillRepeatedly(Return(true));;
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -1212,6 +1315,10 @@ TEST_F(ManifestControllerTest, sharedObjectNotInstalled)
).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile("/tmp/temp_file1", path +
temp_ext)).WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillOnce(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -1313,6 +1420,12 @@ TEST_F(ManifestControllerTest, requireSharedObjectUpdate)
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, removeFile("new_manifest.json"))
.WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status"))
.WillOnce(Return(false))
.WillRepeatedly(Return(true));;
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -1389,6 +1502,7 @@ TEST_F(ManifestControllerTest, failureOnDownloadSharedObject)
EXPECT_CALL(mock_details_resolver, getHostname()).WillOnce(Return(string("hostname")));
EXPECT_CALL(mock_orchestration_tools, removeFile("/tmp/temp_file1")).WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("b"));
EXPECT_FALSE(i_manifest_controller->updateManifest(file_name));
}
@@ -1524,6 +1638,12 @@ TEST_F(ManifestControllerTest, multiRequireUpdate)
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, removeFile("new_manifest.json"))
.WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status"))
.WillOnce(Return(false))
.WillRepeatedly(Return(true));;
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -1610,6 +1730,12 @@ TEST_F(ManifestControllerTest, createNewManifestWithUninstallablePackage)
EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status"))
.WillOnce(Return(false))
.WillRepeatedly(Return(true));;
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -1624,7 +1750,7 @@ TEST_F(ManifestControllerTest, updateUninstallPackage)
" \"download-path\": \"\","
" \"relative-path\": \"\","
" \"name\": \"my\","
" \"version\": \"\","
" \"version\": \"c\","
" \"checksum-type\": \"sha1sum\","
" \"checksum\": \"\","
" \"package-type\": \"service\","
@@ -1721,6 +1847,11 @@ TEST_F(ManifestControllerTest, updateUninstallPackage)
EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(file_name)).WillOnce(Return(new_services));
EXPECT_CALL(mock_orchestration_tools,
loadPackagesFromJson(manifest_file_path)).WillOnce(Return(old_services));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillOnce(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -1744,6 +1875,9 @@ public:
setConfiguration<string>(ignore_packages_file, "orchestration", "Ignore packages list file path");
writeIgnoreList(ignore_packages_file, ignore_services);
EXPECT_CALL(mock_orchestration_tools, doesFileExist(ignore_packages_file)).WillOnce(Return(true));
Maybe<string> forbidden_versions(string("a1\na2"));
EXPECT_CALL(mock_orchestration_tools, readFile("/etc/cp/revert/forbidden_versions"))
.WillOnce(Return(forbidden_versions));
manifest_controller.init();
manifest_file_path = getConfigurationWithDefault<string>(
"/etc/cp/conf/manifest.json",
@@ -1839,6 +1973,7 @@ public:
StrictMock<MockOrchestrationStatus> mock_status;
StrictMock<MockDownloader> mock_downloader;
StrictMock<MockOrchestrationTools> mock_orchestration_tools;
StrictMock<MockDetailsResolver> mock_details_resolver;
NiceMock<MockShellCmd> mock_shell_cmd;
ManifestController manifest_controller;
@@ -2122,6 +2257,12 @@ TEST_F(ManifestControllerIgnorePakckgeTest, addIgnorePackageAndUpdateNormal)
EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status"))
.WillOnce(Return(false))
.WillRepeatedly(Return(true));;
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
}
@@ -2387,6 +2528,12 @@ TEST_F(ManifestControllerIgnorePakckgeTest, overrideIgnoredPackageFromProfileSet
EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("b"));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status"))
.WillOnce(Return(false))
.WillRepeatedly(Return(true));;
EXPECT_CALL(mock_orchestration_tools, writeFile(_, "/etc/cp/revert/upgrade_status", false))
.WillOnce(Return(true));
EXPECT_TRUE(i_manifest_controller->updateManifest(file_name));
EXPECT_THAT(capture_debug.str(), Not(HasSubstr("Ignoring a package from the manifest. Package name: my")));
@@ -2411,6 +2558,9 @@ public:
doesFileExist("/etc/cp/conf/ignore-packages.txt")
).WillOnce(Return(false));
Maybe<string> forbidden_versions(string("a1\na2"));
EXPECT_CALL(mock_orchestration_tools, readFile("/etc/cp/revert/forbidden_versions"))
.WillOnce(Return(forbidden_versions));
manifest_controller.init();
}

25
components/security_apps/orchestration/manifest_controller/manifest_handler.cc
View File

@@ -14,6 +14,7 @@
#include "manifest_handler.h"
#include <algorithm>
#include <ctime>
#include "debug.h"
#include "config.h"
@@ -201,18 +202,29 @@ ManifestHandler::installPackage(
auto span_scope = i_env->startNewSpanScope(Span::ContextType::CHILD_OF);
auto orchestration_status = Singleton::Consume<I_OrchestrationStatus>::by<ManifestHandler>();
auto details_resolver = Singleton::Consume<I_DetailsResolver>::by<ManifestHandler>();
auto orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<ManifestHandler>();
auto &package = package_downloaded_file.first;
auto &package_name = package.getName();
auto &package_handler_path = package_downloaded_file.second;
dbgInfo(D_ORCHESTRATOR) << "Handling package installation. Package: " << package_name;
string upgrade_info =
details_resolver->getAgentVersion() + " " + package.getVersion() + " " + getCurrentTimestamp();
if (!orchestration_tools->doesFileExist(getFilesystemPathConfig() + "/revert/upgrade_status") &&
!orchestration_tools->writeFile(upgrade_info, getFilesystemPathConfig() + "/revert/upgrade_status")
) {
dbgWarning(D_ORCHESTRATOR) << "Failed to write to " + getFilesystemPathConfig() + "/revert/upgrade_status";
}
if (package_name.compare(orch_service_name) == 0) {
orchestration_status->writeStatusToFile();
bool self_update_status = selfUpdate(package, current_packages, package_handler_path);
if (!self_update_status) {
auto details = Singleton::Consume<I_AgentDetails>::by<ManifestHandler>();
auto hostname = Singleton::Consume<I_DetailsResolver>::by<ManifestHandler>()->getHostname();
auto hostname = details_resolver->getHostname();
string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + details->getAgentId()) + "'";
string install_error =
"Warning: Agent/Gateway " +
@@ -246,7 +258,6 @@ ManifestHandler::installPackage(
return true;
}
string current_installation_file = packages_dir + "/" + package_name + "/" + package_name;
auto orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<ManifestHandler>();
bool is_clean_installation = !orchestration_tools->doesFileExist(current_installation_file);
@@ -368,3 +379,13 @@ ManifestHandler::selfUpdate(
package_handler->preInstallPackage(orch_service_name, current_installation_file) &&
package_handler->installPackage(orch_service_name, current_installation_file, false);
}
string
ManifestHandler::getCurrentTimestamp()
{
time_t now = time(nullptr);
tm* now_tm = localtime(&now);
char timestamp[20];
strftime(timestamp, sizeof(timestamp), "%Y-%m-%d %H:%M:%S", now_tm);
return string(timestamp);
}

137
components/security_apps/orchestration/orchestration_comp.cc
View File

@@ -55,6 +55,8 @@ USE_DEBUG_FLAG(D_ORCHESTRATOR);
static string fw_last_update_time = "";
#endif // gaia || smb
static const size_t MAX_SERVER_NAME_LENGTH = 253;
class SetAgentUninstall
:
public ServerRest,
@@ -103,6 +105,19 @@ public:
<< "Initializing Orchestration component, file system path prefix: "
<< filesystem_prefix;
int check_upgrade_success_interval = getSettingWithDefault<uint>(10, "successUpgradeInterval");
Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->addOneTimeRoutine(
I_MainLoop::RoutineType::Timer,
[this, check_upgrade_success_interval]()
{
Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->yield(
std::chrono::minutes(check_upgrade_success_interval)
);
processUpgradeCompletion();
},
"Orchestration successfully updated (One-Time After Interval)",
true
);
auto orch_policy = loadDefaultOrchestrationPolicy();
if (!orch_policy.ok()) {
dbgWarning(D_ORCHESTRATOR) << "Failed to load Orchestration Policy. Error: " << orch_policy.getErr();
@@ -141,6 +156,113 @@ public:
}
private:
void
saveLastKnownOrchInfo(string curr_agent_version)
{
static const string upgrades_dir = filesystem_prefix + "/revert";
static const string last_known_orchestrator = upgrades_dir + "/last_known_working_orchestrator";
static const string current_orchestration_package =
filesystem_prefix + "/packages/orchestration/orchestration";
static const string last_known_manifest = upgrades_dir + "/last_known_manifest";
static const string current_manifest_file = getConfigurationWithDefault<string>(
filesystem_prefix + "/conf/manifest.json",
"orchestration",
"Manifest file path"
);
if (!i_orchestration_tools->copyFile(current_orchestration_package, last_known_orchestrator)) {
dbgWarning(D_ORCHESTRATOR) << "Failed to copy the orchestration package to " << upgrades_dir;
} else {
dbgInfo(D_ORCHESTRATOR) << "last known orchestrator version updated to: " << curr_agent_version;
}
if (!i_orchestration_tools->copyFile(current_manifest_file, last_known_manifest)) {
dbgWarning(D_ORCHESTRATOR) << "Failed to copy " << current_manifest_file << " to " << upgrades_dir;
} else {
dbgInfo(D_ORCHESTRATOR) << "last known manifest updated";
}
return;
}
void
processUpgradeCompletion()
{
if (!is_first_check_update_success) {
int check_upgrade_success_interval = getSettingWithDefault<uint>(10, "successUpgradeInterval");
// LCOV_EXCL_START
Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->addOneTimeRoutine(
I_MainLoop::RoutineType::Timer,
[this, check_upgrade_success_interval]()
{
Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->yield(
std::chrono::minutes(check_upgrade_success_interval)
);
processUpgradeCompletion();
},
"Orchestration successfully updated",
true
);
// LCOV_EXCL_STOP
return;
}
static const string upgrades_dir = filesystem_prefix + "/revert";
static const string upgrade_status = upgrades_dir + "/upgrade_status";
static const string last_known_orchestrator = upgrades_dir + "/last_known_working_orchestrator";
static const string upgrade_failure_info_path = upgrades_dir + "/failed_upgrade_info";
I_DetailsResolver *i_details_resolver = Singleton::Consume<I_DetailsResolver>::by<OrchestrationComp>();
bool is_upgrade_status_exist = i_orchestration_tools->doesFileExist(upgrade_status);
bool is_last_known_orchestrator_exist = i_orchestration_tools->doesFileExist(last_known_orchestrator);
if (!is_upgrade_status_exist) {
if (!is_last_known_orchestrator_exist) {
saveLastKnownOrchInfo(i_details_resolver->getAgentVersion());
}
return;
}
auto maybe_upgrade_data = i_orchestration_tools->readFile(upgrade_status);
string upgrade_data, from_version, to_version;
if (maybe_upgrade_data.ok()) {
upgrade_data = maybe_upgrade_data.unpack();
istringstream stream(upgrade_data);
stream >> from_version >> to_version;
}
i_orchestration_tools->removeFile(upgrade_status);
if (i_orchestration_tools->doesFileExist(upgrade_failure_info_path)) {
string info = "Orchestration revert. ";
auto failure_info = i_orchestration_tools->readFile(upgrade_failure_info_path);
if (failure_info.ok()) info.append(failure_info.unpack());
LogGen(
info,
ReportIS::Level::ACTION,
ReportIS::Audience::INTERNAL,
ReportIS::Severity::CRITICAL,
ReportIS::Priority::URGENT,
ReportIS::Tags::ORCHESTRATOR
);
dbgError(D_ORCHESTRATOR) <<
"Error in orchestration version: " << to_version <<
". Orchestration reverted to version: " << i_details_resolver->getAgentVersion();
i_orchestration_tools->removeFile(upgrade_failure_info_path);
return;
}
saveLastKnownOrchInfo(i_details_resolver->getAgentVersion());
i_orchestration_tools->writeFile(
upgrade_data + "\n",
getLogFilesPathConfig() + "/nano_agent/prev_upgrades",
true
);
dbgWarning(D_ORCHESTRATOR) <<
"Upgrade process from version: " << from_version <<
" to version: " << to_version <<
" completed successfully";
}
Maybe<void>
registerToTheFog()
{
@@ -1022,6 +1144,7 @@ private:
UpdatesProcessResult::SUCCESS,
UpdatesConfigType::GENERAL
).notify();
if (!is_first_check_update_success) is_first_check_update_success = true;
return Maybe<void>();
}
@@ -1389,6 +1512,8 @@ private:
agent_data_report << AgentReportFieldWithLabel("userEdition", FogCommunication::getUserEdition());
agent_data_report << make_pair("registeredServer", i_agent_details->getRegisteredServer());
#if defined(gaia) || defined(smb)
if (i_details_resolver->compareCheckpointVersion(8100, greater_equal<int>())) {
agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionGER81", "true");
@@ -1403,6 +1528,7 @@ private:
} else {
curr_agent_data_report = agent_data_report;
curr_agent_data_report.disableReportSending();
agent_data_report << AgentReportFieldWithLabel("report_timestamp", i_time->getWalltimeStr());
}
}
@@ -1549,6 +1675,11 @@ private:
<< LogField("agentType", "Orchestration")
<< LogField("agentVersion", Version::get());
string registered_server = getAttribute("registered-server", "registered_server");
dbgTrace(D_ORCHESTRATOR) << "Registered server: " << registered_server;
if (!registered_server.empty()) {
i_agent_details->setRegisteredServer(registered_server.substr(0, MAX_SERVER_NAME_LENGTH));
}
auto mainloop = Singleton::Consume<I_MainLoop>::by<OrchestrationComp>();
mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::Offline,
@@ -1629,7 +1760,7 @@ private:
}
}
string server_name = getAttribute("registered-server", "registered_server");
string server_name = Singleton::Consume<I_AgentDetails>::by<OrchestrationComp>()->getRegisteredServer();
auto server = TagAndEnumManagement::convertStringToTag(server_name);
if (server_name == "'SWAG'" || server_name == "'SWAG Server'") server = Tags::WEB_SERVER_SWAG;
if (server.ok()) tags.insert(*server);
@@ -1653,7 +1784,7 @@ private:
tags
);
if (server_name != "") registration_report.addToOrigin(LogField("eventCategory", server_name));
registration_report.addToOrigin(LogField("eventCategory", server_name));
auto email = getAttribute("email-address", "user_email");
if (email != "") registration_report << LogField("userDefinedId", email);
@@ -2065,6 +2196,7 @@ private:
int failure_count = 0;
unsigned int sleep_interval = 0;
bool is_new_success = false;
bool is_first_check_update_success = false;
OrchestrationPolicy policy;
UpdatesProcessReporter updates_process_reporter_listener;
HybridModeMetric hybrid_mode_metric;
@@ -2130,6 +2262,7 @@ OrchestrationComp::preload()
registerExpectedSetting<vector<string>>("upgradeDay");
registerExpectedSetting<string>("email-address");
registerExpectedSetting<string>("registered-server");
registerExpectedSetting<uint>("successUpgradeInterval");
registerExpectedConfigFile("orchestration", Config::ConfigFileType::Policy);
registerExpectedConfigFile("registration-data", Config::ConfigFileType::Policy);
}

7
components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc
View File

@@ -89,6 +89,11 @@ public:
EXPECT_CALL(mock_service_controller, isServiceInstalled("Access Control")).WillRepeatedly(Return(false));
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(_, _, "Orchestration successfully updated (One-Time After Interval)", true)
).WillOnce(DoAll(SaveArg<1>(&upgrade_routine), Return(0)));
// This Holding the Main Routine of the Orchestration.
EXPECT_CALL(
mock_ml,
@@ -156,6 +161,7 @@ public:
runRoutine()
{
routine();
upgrade_routine();
}
void
@@ -235,6 +241,7 @@ private:
}
I_MainLoop::Routine routine;
I_MainLoop::Routine upgrade_routine;
I_MainLoop::Routine status_routine;
};

59
components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc
View File

@@ -83,6 +83,12 @@ public:
EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response));
EXPECT_CALL(mock_status, setFogAddress(host_url)).WillRepeatedly(Return());
EXPECT_CALL(mock_orchestration_tools, setClusterId());
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(_, _, "Orchestration successfully updated (One-Time After Interval)", true)
).WillOnce(DoAll(SaveArg<1>(&upgrade_routine), Return(0)));
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Orchestration runner", true)
@@ -281,6 +287,12 @@ public:
status_routine();
}
void
runUpgradeRoutine()
{
upgrade_routine();
}
void
preload()
{
@@ -359,6 +371,7 @@ private:
I_MainLoop::Routine routine;
I_MainLoop::Routine status_routine;
I_MainLoop::Routine upgrade_routine;
};
@@ -601,14 +614,6 @@ TEST_F(OrchestrationTest, check_sending_registration_data)
string version = "1";
EXPECT_CALL(mock_service_controller, getUpdatePolicyVersion()).WillOnce(ReturnRef(version));
EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>()))
.WillOnce(Return())
.WillOnce(Invoke([] (chrono::microseconds) { throw invalid_argument("stop while loop"); }));
try {
runRoutine();
} catch (const invalid_argument& e) {}
string config_json =
"{\n"
" \"email-address\": \"fake@example.com\",\n"
@@ -617,9 +622,19 @@ TEST_F(OrchestrationTest, check_sending_registration_data)
istringstream ss(config_json);
Singleton::Consume<Config::I_Config>::from(config_comp)->loadConfiguration(ss);
EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>()))
.WillOnce(Return())
.WillOnce(Invoke([] (chrono::microseconds) { throw invalid_argument("stop while loop"); }));
try {
runRoutine();
} catch (const invalid_argument& e) {}
sending_routine();
EXPECT_THAT(message_body, HasSubstr("\"userDefinedId\": \"fake@example.com\""));
EXPECT_THAT(message_body, HasSubstr("\"eventCategory\""));
EXPECT_THAT(message_body, AnyOf(HasSubstr("\"Embedded Deployment\""), HasSubstr("\"Kubernetes Deployment\"")));
EXPECT_THAT(message_body, HasSubstr("\"NGINX Server\""));
}
@@ -1004,6 +1019,11 @@ TEST_F(OrchestrationTest, loadOrchestrationPolicyFromBackup)
);
waitForRestCall();
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(_, _, "Orchestration successfully updated (One-Time After Interval)", true)
);
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Orchestration runner", true)
@@ -1170,6 +1190,29 @@ TEST_F(OrchestrationTest, manifestUpdate)
try {
runRoutine();
} catch (const invalid_argument& e) {}
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/upgrade_status")).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/last_known_working_orchestrator"))
.WillOnce(Return(true));
Maybe<string> upgrade_status(string("1.1.1 1.1.2 2025-01-28 07:53:23"));
EXPECT_CALL(mock_orchestration_tools, readFile("/etc/cp/revert/upgrade_status"))
.WillOnce(Return(upgrade_status));
EXPECT_CALL(mock_orchestration_tools, removeFile("/etc/cp/revert/upgrade_status")).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/revert/failed_upgrade_info"))
.WillOnce(Return(false));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("1.1.2"));
EXPECT_CALL(mock_orchestration_tools, copyFile(_, "/etc/cp/revert/last_known_working_orchestrator"))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(_, "/etc/cp/revert/last_known_manifest")).WillOnce(Return(true));
EXPECT_CALL(
mock_orchestration_tools,
writeFile("1.1.1 1.1.2 2025-01-28 07:53:23\n", "/var/log/nano_agent/prev_upgrades", true)
).WillOnce(Return(true));
EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>())).WillOnce(Return());
runUpgradeRoutine();
}
TEST_F(OrchestrationTest, getBadPolicyUpdate)

6
components/security_apps/orchestration/update_communication/fog_authenticator.cc
View File

@@ -377,9 +377,13 @@ FogAuthenticator::registerLocalAgentToFog()
{
auto local_reg_token = getRegistrationToken();
if (!local_reg_token.ok()) return;
string reg_token = local_reg_token.unpack().getData();
if (reg_token.empty()) return;
dbgInfo(D_ORCHESTRATOR) << "Start local agent registration to the fog";
string exec_command = "open-appsec-ctl --set-mode --online_mode --token " + local_reg_token.unpack().getData();
string exec_command = "open-appsec-ctl --set-mode --online_mode --token " + reg_token;
auto i_agent_details = Singleton::Consume<I_AgentDetails>::by<FogAuthenticator>();
auto fog_address = i_agent_details->getFogDomain();

8
components/security_apps/waap/include/i_serialize.h
View File

@@ -137,9 +137,13 @@ public:
void setRemoteSyncEnabled(bool enabled);
protected:
void mergeProcessedFromRemote();
std::string getWindowId();
void waitSync();
std::string getPostDataUrl();
std::string getUri();
size_t getIntervalsCount();
void incrementIntervalsCount();
bool isBase();
template<typename T>
bool sendObject(T &obj, HTTPMethod method, std::string uri)
@@ -252,14 +256,13 @@ protected:
const std::string m_remotePath; // Created from tenentId + / + assetId + / + class
std::chrono::seconds m_interval;
std::string m_owner;
const std::string m_assetId;
private:
bool localSyncAndProcess();
void updateStateFromRemoteService();
RemoteFilesList getProcessedFilesList();
RemoteFilesList getRemoteProcessedFilesList();
std::string getWindowId();
bool isBase();
std::string getLearningHost();
std::string getSharedStorageHost();
@@ -270,7 +273,6 @@ private:
size_t m_windowsCount;
size_t m_intervalsCounter;
bool m_remoteSyncEnabled;
const std::string m_assetId;
const bool m_isAssetIdUuid;
std::string m_type;
std::string m_lastProcessedModified;

6
components/security_apps/waap/include/i_waapConfig.h
View File

@@ -19,12 +19,14 @@
#include "../waap_clib/WaapParameters.h"
#include "../waap_clib/WaapOpenRedirectPolicy.h"
#include "../waap_clib/WaapErrorDisclosurePolicy.h"
#include "../waap_clib/DecisionType.h"
#include "../waap_clib/CsrfPolicy.h"
#include "../waap_clib/UserLimitsPolicy.h"
#include "../waap_clib/RateLimiting.h"
#include "../waap_clib/SecurityHeadersPolicy.h"
#include <memory>
enum class BlockingLevel {
NO_BLOCKING = 0,
LOW_BLOCKING_LEVEL,
@@ -44,8 +46,8 @@ public:
virtual const std::string& get_AssetId() const = 0;
virtual const std::string& get_AssetName() const = 0;
virtual const BlockingLevel& get_BlockingLevel() const = 0;
virtual const std::string& get_PracticeId() const = 0;
virtual const std::string& get_PracticeName() const = 0;
virtual const std::string& get_PracticeIdByPactice(DecisionType practiceType) const = 0;
virtual const std::string& get_PracticeNameByPactice(DecisionType practiceType) const = 0;
virtual const std::string& get_PracticeSubType() const = 0;
virtual const std::string& get_RuleId() const = 0;
virtual const std::string& get_RuleName() const = 0;

1
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@@ -91,6 +91,7 @@ add_library(waap_clib
ParserScreenedJson.cc
ParserBinaryFile.cc
RegexComparator.cc
RequestsMonitor.cc
)
add_definitions("-Wno-unused-function")

158
components/security_apps/waap/waap_clib/RequestsMonitor.cc Normal file
View File

@@ -0,0 +1,158 @@
#include "RequestsMonitor.h"
#include "waap.h"
#include "SyncLearningNotification.h"
#include "report_messaging.h"
#include "customized_cereal_map.h"
USE_DEBUG_FLAG(D_WAAP_CONFIDENCE_CALCULATOR);
using namespace std;
SourcesRequestMonitor::SourcesRequestMonitor(
const string& filePath,
const string& remotePath,
const string& assetId,
const string& owner) :
SerializeToLocalAndRemoteSyncBase(
chrono::minutes(10),
chrono::seconds(30),
filePath,
remotePath != "" ? remotePath + "/Monitor" : remotePath,
assetId,
owner
), m_sourcesRequests()
{
}
SourcesRequestMonitor::~SourcesRequestMonitor()
{
}
void SourcesRequestMonitor::syncWorker()
{
dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Running the sync worker for assetId='" << m_assetId << "', owner='" <<
m_owner << "'";
incrementIntervalsCount();
OrchestrationMode mode = Singleton::exists<I_AgentDetails>() ?
Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() : OrchestrationMode::ONLINE;
bool enabled = getProfileAgentSettingWithDefault<bool>(false, "appsec.sourceRequestsMonitor.enabled");
if (mode == OrchestrationMode::OFFLINE || !enabled || isBase() || !postData()) {
dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR)
<< "Did not report data. for asset: "
<< m_assetId
<< " Remote URL: "
<< m_remotePath
<< " is enabled: "
<< to_string(enabled)
<< ", mode: " << int(mode);
return;
}
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Waiting for all agents to post their data";
waitSync();
if (mode == OrchestrationMode::HYBRID) {
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "detected running in standalone mode. not sending sync notification";
} else {
SyncLearningNotificationObject syncNotification(m_assetId, "Monitor", getWindowId());
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "sending sync notification: " << syncNotification;
ReportMessaging(
"sync notification for '" + m_assetId + "'",
ReportIS::AudienceTeam::WAAP,
syncNotification,
MessageCategory::GENERIC,
ReportIS::Tags::WAF,
ReportIS::Notification::SYNC_LEARNING
);
}
}
void SourcesRequestMonitor::logSourceHit(const string& source)
{
m_sourcesRequests[chrono::duration_cast<chrono::minutes>(
Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime()
).count()][source]++;
}
// LCOV_EXCL_START Reason: internal functions not used
void SourcesRequestMonitor::pullData(const vector<string> &data)
{
// not used. report only
}
void SourcesRequestMonitor::processData()
{
// not used. report only
}
void SourcesRequestMonitor::postProcessedData()
{
// not used. report only
}
void SourcesRequestMonitor::pullProcessedData(const vector<string> &data)
{
// not used. report only
}
void SourcesRequestMonitor::updateState(const vector<string> &data)
{
// not used. report only
}
// LCOV_EXCL_STOP
typedef map<string, map<string, size_t>> MonitorJsonData;
class SourcesRequestsReport : public RestGetFile
{
public:
SourcesRequestsReport(MonitorData& _sourcesRequests, const string& _agentId)
: sourcesRequests(), agentId(_agentId)
{
MonitorJsonData montiorData;
for (const auto& window : _sourcesRequests) {
for (const auto& source : window.second) {
montiorData[to_string(window.first)][source.first] = source.second;
}
}
sourcesRequests = montiorData;
}
private:
C2S_PARAM(MonitorJsonData, sourcesRequests);
C2S_PARAM(string, agentId);
};
bool SourcesRequestMonitor::postData()
{
dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the data to remote";
// send collected data to remote and clear the local data
string url = getPostDataUrl();
string agentId = Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getAgentId();
SourcesRequestsReport currentWindow(m_sourcesRequests, agentId);
bool ok = sendNoReplyObjectWithRetry(currentWindow,
HTTPMethod::PUT,
url);
if (!ok) {
dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to post collected data to: " << url;
}
dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Data sent to remote: " << ok;
m_sourcesRequests.clear();
return ok;
}
void SourcesRequestMonitor::serialize(ostream& stream)
{
cereal::JSONOutputArchive archive(stream);
archive(m_sourcesRequests);
}
void SourcesRequestMonitor::deserialize(istream& stream)
{
cereal::JSONInputArchive archive(stream);
archive(m_sourcesRequests);
}

33
components/security_apps/waap/waap_clib/RequestsMonitor.h Normal file
View File

@@ -0,0 +1,33 @@
#ifndef __REQUESTS_MONITOR_H__
#define __REQUESTS_MONITOR_H__
#include "i_serialize.h"
typedef std::map<uint64_t, std::map<std::string, size_t>> MonitorData;
class SourcesRequestMonitor : public SerializeToLocalAndRemoteSyncBase
{
public:
SourcesRequestMonitor(
const std::string& filePath,
const std::string& remotePath,
const std::string& assetId,
const std::string& owner);
virtual ~SourcesRequestMonitor();
virtual void syncWorker() override;
void logSourceHit(const std::string& source);
protected:
virtual void pullData(const std::vector<std::string> &data) override;
virtual void processData() override;
virtual void postProcessedData() override;
virtual void pullProcessedData(const std::vector<std::string> &data) override;
virtual void updateState(const std::vector<std::string> &data) override;
virtual bool postData() override;
void serialize(std::ostream& stream);
void deserialize(std::istream& stream);
private:
// map of sources and their requests per minute (UNIX)
MonitorData m_sourcesRequests;
};
#endif // __REQUESTS_MONITOR_H__

20
components/security_apps/waap/waap_clib/Serializator.cc
View File

@@ -407,6 +407,7 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
m_remotePath(replaceAllCopy(remotePath, "//", "/")),
m_interval(0),
m_owner(owner),
m_assetId(replaceAllCopy(assetId, "/", "")),
m_pMainLoop(nullptr),
m_waitForSync(waitForSync),
m_workerRoutineId(0),
@@ -414,7 +415,6 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
m_windowsCount(0),
m_intervalsCounter(0),
m_remoteSyncEnabled(true),
m_assetId(replaceAllCopy(assetId, "/", "")),
m_isAssetIdUuid(Waap::Util::isUuid(assetId)),
m_shared_storage_host(genError("not set")),
m_learning_host(genError("not set"))
@@ -469,6 +469,15 @@ bool SerializeToLocalAndRemoteSyncBase::isBase()
return m_remotePath == "";
}
void SerializeToLocalAndRemoteSyncBase::waitSync()
{
if (m_pMainLoop == nullptr)
{
return;
}
m_pMainLoop->yield(m_waitForSync);
}
string SerializeToLocalAndRemoteSyncBase::getUri()
{
static const string hybridModeUri = "/api";
@@ -484,6 +493,11 @@ size_t SerializeToLocalAndRemoteSyncBase::getIntervalsCount()
return m_intervalsCounter;
}
void SerializeToLocalAndRemoteSyncBase::incrementIntervalsCount()
{
m_intervalsCounter++;
}
SerializeToLocalAndRemoteSyncBase::~SerializeToLocalAndRemoteSyncBase()
{
@@ -659,7 +673,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
{
dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Running the sync worker for assetId='" << m_assetId << "', owner='" <<
m_owner << "'" << " last modified state: " << m_lastProcessedModified;
m_intervalsCounter++;
incrementIntervalsCount();
OrchestrationMode mode = Singleton::exists<I_AgentDetails>() ?
Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() : OrchestrationMode::ONLINE;
@@ -678,7 +692,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
}
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Waiting for all agents to post their data";
m_pMainLoop->yield(m_waitForSync);
waitSync();
// check if learning service is operational
if (m_lastProcessedModified == "")
{

20
components/security_apps/waap/waap_clib/Telemetry.cc
View File

@@ -33,6 +33,7 @@ WaapTelemetryBase::sendLog(const LogRest &metric_client_rest) const
OrchestrationMode mode = Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getOrchestrationMode();
GenericMetric::sendLog(metric_client_rest);
dbgTrace(D_WAAP) << "Waap telemetry log sent: " << metric_client_rest.genJson().unpack();
if (mode == OrchestrationMode::ONLINE) {
return;
@@ -79,7 +80,16 @@ void
WaapTelemetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data)
{
initMetrics();
requests.report(1);
auto is_keep_alive_ctx = Singleton::Consume<I_Environment>::by<GenericMetric>()->get<bool>(
"keep_alive_request_ctx"
);
if (!is_keep_alive_ctx.ok() || !*is_keep_alive_ctx) {
requests.report(1);
} else {
dbgTrace(D_WAAP) << "Not increasing the number of requests due to keep alive";
}
if (sources_seen.find(data.source) == sources_seen.end()) {
if (sources.getCounter() == 0) sources_seen.clear();
sources_seen.insert(data.source);
@@ -274,7 +284,9 @@ WaapMetricWrapper::upon(const WaapTelemetryEvent &event)
ReportIS::IssuingEngine::AGENT_CORE,
chrono::minutes(LOGGING_INTERVAL_IN_MINUTES),
true,
ReportIS::Audience::INTERNAL
ReportIS::Audience::INTERNAL,
false,
asset_id
);
metrics[asset_id]->registerListener();
}
@@ -286,7 +298,9 @@ WaapMetricWrapper::upon(const WaapTelemetryEvent &event)
ReportIS::IssuingEngine::AGENT_CORE,
chrono::minutes(LOGGING_INTERVAL_IN_MINUTES),
true,
ReportIS::Audience::INTERNAL
ReportIS::Audience::INTERNAL,
false,
asset_id
);
attack_types[asset_id]->registerListener();
}

5
components/security_apps/waap/waap_clib/WaapAssetState.cc
View File

@@ -135,6 +135,7 @@ WaapAssetState::WaapAssetState(std::shared_ptr<Signatures> signatures,
m_Signatures(signatures),
m_waapDataFileName(waapDataFileName),
m_assetId(assetId),
m_requestsMonitor(nullptr),
scoreBuilder(this),
m_rateLimitingState(nullptr),
m_errorLimitingState(nullptr),
@@ -152,10 +153,14 @@ WaapAssetState::WaapAssetState(std::shared_ptr<Signatures> signatures,
I_AgentDetails* agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
std::string path = agentDetails->getTenantId() + "/" + assetId;
m_filtersMngr = std::make_shared<IndicatorsFiltersManager>(path, assetId, this);
m_requestsMonitor = std::make_shared<SourcesRequestMonitor>
(getWaapDataDir() + "/monitor.data", path, assetId, "State");
}
else
{
m_filtersMngr = std::make_shared<IndicatorsFiltersManager>("", "", this);
m_requestsMonitor = std::make_shared<SourcesRequestMonitor>
(getWaapDataDir() + "/monitor.data", "", assetId, "State");
}
// Load keyword scores - copy from ScoreBuilder
updateScores();

4
components/security_apps/waap/waap_clib/WaapAssetState.h
View File

@@ -33,6 +33,7 @@
#include "KeywordTypeValidator.h"
#include "ScanResult.h"
#include "WaapSampleValue.h"
#include "RequestsMonitor.h"
enum space_stage {SPACE_SYNBOL, BR_SYMBOL, BN_SYMBOL, BRN_SEQUENCE, BNR_SEQUENCE, NO_SPACES};
@@ -67,6 +68,8 @@ public:
const std::string m_assetId;
std::shared_ptr<SourcesRequestMonitor> m_requestsMonitor;
ScoreBuilder scoreBuilder;
std::shared_ptr<Waap::RateLimiting::State> m_rateLimitingState;
std::shared_ptr<Waap::RateLimiting::State> m_errorLimitingState;
@@ -90,6 +93,7 @@ public:
void logIndicatorsInFilters(const std::string &param, Waap::Keywords::KeywordsSet& keywords,
IWaf2Transaction* pTransaction);
void logParamHit(Waf2ScanResult& res, IWaf2Transaction* pTransaction);
void logSourceHit(const std::string& source);
void filterKeywords(const std::string &param, Waap::Keywords::KeywordsSet& keywords,
std::vector<std::string>& filteredKeywords);
void clearFilterVerbose();

31
components/security_apps/waap/waap_clib/WaapConfigBase.cc
View File

@@ -329,14 +329,37 @@ const std::string& WaapConfigBase::get_AssetName() const
return m_assetName;
}
const std::string& WaapConfigBase::get_PracticeId() const
const std::string& WaapConfigBase::get_PracticeIdByPactice(DecisionType practiceType) const
{
return m_practiceId;
switch (practiceType)
{
case DecisionType::AUTONOMOUS_SECURITY_DECISION:
return m_practiceId;
default:
dbgError(D_WAAP)
<< "Can't find practice type for practice ID by practice: "
<< practiceType
<< ", return web app practice ID";
return m_practiceId;
}
}
const std::string& WaapConfigBase::get_PracticeName() const
const std::string& WaapConfigBase::get_PracticeNameByPactice(DecisionType practiceType) const
{
return m_practiceName;
switch (practiceType)
{
case DecisionType::AUTONOMOUS_SECURITY_DECISION:
return m_practiceName;
default:
dbgError(D_WAAP)
<< "Can't find practice type for practice name by practice: "
<< practiceType
<< ", return web app practice name";
return m_practiceName;
}
}
const std::string& WaapConfigBase::get_RuleId() const

4
components/security_apps/waap/waap_clib/WaapConfigBase.h
View File

@@ -39,8 +39,8 @@ public:
virtual const std::string& get_AssetId() const;
virtual const std::string& get_AssetName() const;
virtual const BlockingLevel& get_BlockingLevel() const;
virtual const std::string& get_PracticeId() const;
virtual const std::string& get_PracticeName() const;
virtual const std::string& get_PracticeIdByPactice(DecisionType practiceType) const;
virtual const std::string& get_PracticeNameByPactice(DecisionType practiceType) const;
virtual const std::string& get_RuleId() const;
virtual const std::string& get_RuleName() const;
virtual const bool& get_WebAttackMitigation() const;

2
components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
View File

@@ -89,7 +89,7 @@ bool WaapOverrideFunctor::operator()(
}
else if (tagLower == "url") {
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getUriStr)) return true;
if (W2T_REGX_MATCH(getUri)) return true;
}
return false;
}

15
components/security_apps/waap/waap_clib/WaapResponseInjectReasons.cc
View File

@@ -23,6 +23,7 @@ ResponseInjectReasons::ResponseInjectReasons()
:
csrf(false),
antibot(false),
captcha(false),
securityHeaders(false)
{
}
@@ -53,6 +54,13 @@ ResponseInjectReasons::setAntibot(bool flag)
antibot = flag;
}
void
ResponseInjectReasons::setCaptcha(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInjectReasons(Captcha) " << captcha << " to " << flag;
captcha = flag;
}
void
ResponseInjectReasons::setCsrf(bool flag)
{
@@ -74,6 +82,13 @@ ResponseInjectReasons::shouldInjectAntibot() const
return antibot;
}
bool
ResponseInjectReasons::shouldInjectCaptcha() const
{
dbgTrace(D_WAAP) << "shouldInjectCaptcha():: " << captcha;
return captcha;
}
bool
ResponseInjectReasons::shouldInjectCsrf() const
{

3
components/security_apps/waap/waap_clib/WaapResponseInjectReasons.h
View File

@@ -21,14 +21,17 @@ public:
void clear();
bool shouldInject() const;
void setAntibot(bool flag);
void setCaptcha(bool flag);
void setCsrf(bool flag);
void setSecurityHeaders(bool flag);
bool shouldInjectAntibot() const;
bool shouldInjectCaptcha() const;
bool shouldInjectCsrf() const;
bool shouldInjectSecurityHeaders() const;
private:
bool csrf;
bool antibot;
bool captcha;
bool securityHeaders;
};

45
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -1098,6 +1098,7 @@ void Waf2Transaction::end_request_hdrs() {
// but the State itself is not needed now
Waap::Override::State overrideState = getOverrideState(m_siteConfig);
}
m_pWaapAssetState->m_requestsMonitor->logSourceHit(m_source_identifier);
IdentifiersEvent ids(m_source_identifier, m_pWaapAssetState->m_assetId);
ids.notify();
// Read relevant headers and extract meta information such as host name
@@ -1421,6 +1422,15 @@ Waf2Transaction::completeInjectionResponseBody(std::string& strInjection)
m_responseInjectReasons.setAntibot(false);
}
if(m_responseInjectReasons.shouldInjectCaptcha()) {
dbgTrace(D_WAAP_BOT_PROTECTION) <<
"Waf2Transaction::completeInjectionResponseBody(): Injecting data (captcha)";
//todo add captcha script
strInjection += "<script src=\"cp-cp.js\"></script>";
// No need to inject more than once
m_responseInjectReasons.setCaptcha(false);
}
if (m_responseInjectReasons.shouldInjectCsrf()) {
dbgTrace(D_WAAP) << "Waf2Transaction::completeInjectionResponseBody(): Injecting data (csrf)";
strInjection += "<script src=\"cp-csrf.js\"></script>";
@@ -1567,7 +1577,7 @@ Waf2Transaction::decideFinal(
dbgTrace(D_WAAP) << "Waf2Transaction::decideFinal(): got relevant API configuration from the I/S";
sitePolicy = &ngenAPIConfig;
m_overrideState = getOverrideState(sitePolicy);
shouldBlock = (getUserLimitVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP);
}
else if (WaapConfigApplication::getWaapSiteConfig(ngenSiteConfig)) {
dbgTrace(D_WAAP) << "Waf2Transaction::decideFinal(): got relevant Application configuration from the I/S";
@@ -1646,7 +1656,9 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
const std::shared_ptr<Waap::Trigger::Log> &triggerLog,
bool shouldBlock,
const std::string& logOverride,
const std::string& incidentType) const
const std::string& incidentType,
const std::string& practiceID,
const std::string& practiceName) const
{
auto env = Singleton::Consume<I_Environment>::by<WaapComponent>();
auto active_id = env->get<std::string>("ActiveTenantId");
@@ -1737,8 +1749,8 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
waapLog << LogField("practiceType", "Threat Prevention");
waapLog << LogField("practiceSubType", m_siteConfig->get_PracticeSubType());
waapLog << LogField("ruleName", m_siteConfig->get_RuleName());
waapLog << LogField("practiceId", m_siteConfig->get_PracticeId());
waapLog << LogField("practiceName", m_siteConfig->get_PracticeName());
waapLog << LogField("practiceId", practiceID);
waapLog << LogField("practiceName", practiceName);
waapLog << LogField("waapIncidentType", incidentType);
// Registering this value would append the list of matched override IDs to the unified log
@@ -1805,8 +1817,8 @@ Waf2Transaction::sendLog()
telemetryData.source = getSourceIdentifier();
telemetryData.assetName = m_siteConfig->get_AssetName();
telemetryData.practiceId = m_siteConfig->get_PracticeId();
telemetryData.practiceName = m_siteConfig->get_PracticeName();
telemetryData.practiceId = m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION);
telemetryData.practiceName = m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION);
if (m_scanResult) {
telemetryData.attackTypes = m_scanResult->attack_types;
}
@@ -1947,7 +1959,11 @@ Waf2Transaction::sendLog()
shouldBlock);
LogGen& waap_log = logGenWrapper.getLogGen();
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType);
appendCommonLogFields(
waap_log, triggerLog, shouldBlock, logOverride, incidentType,
m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION),
m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION)
);
waap_log << LogField("waapIncidentDetails", incidentDetails);
waap_log << LogField("eventConfidence", "High");
break;
@@ -1980,7 +1996,11 @@ Waf2Transaction::sendLog()
waap_log << LogField("waapFoundIndicators", getKeywordMatchesStr(), LogFieldOption::XORANDB64);
}
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType);
appendCommonLogFields(
waap_log, triggerLog, shouldBlock, logOverride, incidentType,
m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION),
m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION)
);
waap_log << LogField("waapIncidentDetails", incidentDetails);
break;
@@ -1996,7 +2016,11 @@ Waf2Transaction::sendLog()
shouldBlock);
LogGen& waap_log = logGenWrapper.getLogGen();
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, "Cross Site Request Forgery");
appendCommonLogFields(
waap_log, triggerLog, shouldBlock, logOverride, "Cross Site Request Forgery",
m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION),
m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION)
);
waap_log << LogField("waapIncidentDetails", "CSRF Attack discovered.");
break;
}
@@ -2177,14 +2201,13 @@ Waf2Transaction::decideAutonomousSecurity(
" effective overrides count: " << m_effectiveOverrideIds.size() <<
" learned overrides count: " << m_exceptionLearned.size();
bool log_all = false;
const std::shared_ptr<Waap::Trigger::Policy> triggerPolicy = sitePolicy.get_TriggerPolicy();
if (triggerPolicy) {
const std::shared_ptr<Waap::Trigger::Log> triggerLog = getTriggerLog(triggerPolicy);
if (triggerLog && triggerLog->webRequests) log_all = true;
}
if(decision->getThreatLevel() <= ThreatLevel::THREAT_INFO && !log_all) {
decision->setLog(false);
} else {

4
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@@ -247,7 +247,9 @@ private:
const std::shared_ptr<Waap::Trigger::Log> &triggerLog,
bool shouldBlock,
const std::string& logOverride,
const std::string& incidentType) const;
const std::string& incidentType,
const std::string& practiceID,
const std::string& practiceName) const;
std::string getUserReputationStr(double relativeReputation) const;
bool isTrustedSource() const;

6
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@@ -381,7 +381,11 @@ void Waf2Transaction::sendAutonomousSecurityLog(
waap_log << LogField("eventConfidence", confidence);
}
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, attackTypes);
appendCommonLogFields(
waap_log, triggerLog, shouldBlock, logOverride, attackTypes,
m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION),
m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION)
);
std::string sampleString = getSample();
if (sampleString.length() > MAX_LOG_FIELD_SIZE) {