From 1b7eafaa23e673f9951f563c02cef9bf84c41f48 Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Mon, 14 Oct 2024 16:32:23 +0000 Subject: [PATCH] code sync --- .../waap/include/i_waap_model_result_logger.h | 37 +++ .../waap/waap_clib/RegexComparator.cc | 36 +++ .../waap/waap_clib/RegexComparator.h | 32 +++ .../waap/waap_clib/WaapModelResultLogger.cc | 250 ++++++++++++++++++ .../waap/waap_clib/WaapModelResultLogger.h | 109 ++++++++ 5 files changed, 464 insertions(+) create mode 100644 components/security_apps/waap/include/i_waap_model_result_logger.h create mode 100644 components/security_apps/waap/waap_clib/RegexComparator.cc create mode 100644 components/security_apps/waap/waap_clib/RegexComparator.h create mode 100644 components/security_apps/waap/waap_clib/WaapModelResultLogger.cc create mode 100644 components/security_apps/waap/waap_clib/WaapModelResultLogger.h diff --git a/components/security_apps/waap/include/i_waap_model_result_logger.h b/components/security_apps/waap/include/i_waap_model_result_logger.h new file mode 100644 index 0000000..9d74a34 --- /dev/null +++ b/components/security_apps/waap/include/i_waap_model_result_logger.h @@ -0,0 +1,37 @@ +// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +class IWaf2Transaction; +struct Waf2ScanResult; +namespace Waap { +namespace Scores { +struct ModelLoggingSettings; +} +} + +class I_WaapModelResultLogger { +public: + virtual ~I_WaapModelResultLogger() {} + + virtual void + logModelResult( + Waap::Scores::ModelLoggingSettings &settings, + IWaf2Transaction* transaction, + Waf2ScanResult &res, + std::string modelName, + std::string otherModelName, + double newScore, + double baseScore) = 0; +}; diff --git a/components/security_apps/waap/waap_clib/RegexComparator.cc b/components/security_apps/waap/waap_clib/RegexComparator.cc new file mode 100644 index 0000000..63f4b9a --- /dev/null +++ b/components/security_apps/waap/waap_clib/RegexComparator.cc @@ -0,0 +1,36 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "RegexComparator.h" +#include + +namespace Waap { +namespace Util { + +std::string regexSetToString(const std::set, RegexComparator> ®exSet) { + std::stringstream ss; + ss << "["; + bool first = true; + for (const auto ®exPtr : regexSet) { + if (!first) ss << ", "; + if (regexPtr) { + first = false; + ss << regexPtr->str(); + } + } + ss << "]"; + return ss.str(); +} + +} +} diff --git a/components/security_apps/waap/waap_clib/RegexComparator.h b/components/security_apps/waap/waap_clib/RegexComparator.h new file mode 100644 index 0000000..ec43878 --- /dev/null +++ b/components/security_apps/waap/waap_clib/RegexComparator.h @@ -0,0 +1,32 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include +#include + +namespace Waap { +namespace Util { + +// Custom comparator for std::shared_ptr +struct RegexComparator { + bool operator()(const std::shared_ptr& lhs, const std::shared_ptr& rhs) const { + // Compare the actual regex patterns by string representation + return lhs->str() < rhs->str(); + } +}; + +std::string regexSetToString(const std::set, RegexComparator> ®exSet); + +} +} diff --git a/components/security_apps/waap/waap_clib/WaapModelResultLogger.cc b/components/security_apps/waap/waap_clib/WaapModelResultLogger.cc new file mode 100644 index 0000000..2a92f43 --- /dev/null +++ b/components/security_apps/waap/waap_clib/WaapModelResultLogger.cc @@ -0,0 +1,250 @@ +// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "WaapModelResultLogger.h" +#include "Waf2Engine.h" +#include "i_time_get.h" +#include "i_messaging.h" +#include "i_instance_awareness.h" +#include "http_manager.h" +#include "LogGenWrapper.h" +#include "rest.h" +#include "debug.h" + +USE_DEBUG_FLAG(D_WAAP_MODEL_LOGGER); + +using namespace std; + +static const unsigned int MAX_FILES_PER_WINDOW = 5; +static const unsigned int MAX_LOGS_PER_WINDOW = 1800; +static constexpr std::chrono::minutes RATE_LIMIT_WINDOW_MINUTES = std::chrono::minutes(30); + +class WaapModelReport : public RestGetFile +{ +public: + WaapModelReport(const vector &_data) : data(_data) {} + +private: + C2S_PARAM(vector, data); +}; + +class WaapModelResultLogger::Impl + : + Singleton::Provide::From +{ +public: + Impl(size_t maxLogs) : max_logs(maxLogs), sent_files_count(0), sent_logs_count(0), + last_sent_s3(std::chrono::minutes::zero()), + last_kusto_log_window(std::chrono::minutes::zero()) {} + virtual ~Impl(); + void + logModelResult( + Waap::Scores::ModelLoggingSettings &settings, + IWaf2Transaction* transaction, + Waf2ScanResult &res, + string modelName, + string otherModelName, + double score, + double otherScore) override; + +private: + void logToStream(WaapModelResult &result, chrono::minutes now); + void logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now); + bool shouldSendLogsToS3(chrono::minutes now); + void sendLogsToS3(); + size_t max_logs; + unsigned int sent_files_count; + unsigned int sent_logs_count; + std::chrono::minutes last_sent_s3; + std::chrono::minutes last_kusto_log_window; + std::map> logs; +}; + +WaapModelResultLogger::WaapModelResultLogger(size_t maxLogs) : pimpl(make_unique(maxLogs)) +{ +} + +WaapModelResultLogger::~WaapModelResultLogger() +{ +} + +void +WaapModelResultLogger::logModelResult( + Waap::Scores::ModelLoggingSettings &settings, + IWaf2Transaction* transaction, + Waf2ScanResult &res, + std::string modelName, + std::string otherModelName, + double score, + double otherScore +) +{ + pimpl->logModelResult(settings, transaction, res, modelName, otherModelName, score, otherScore); +} + +void +WaapModelResultLogger::Impl::logModelResult( + Waap::Scores::ModelLoggingSettings &settings, + IWaf2Transaction* transaction, + Waf2ScanResult &res, + string modelName, + string otherModelName, + double score, + double otherScore) +{ + if (transaction == NULL) return; + if (!Singleton::exists()) { + dbgError(D_WAAP_MODEL_LOGGER) << "Messaging service is not available, will not log"; + return; + } + + double score_diff = score - otherScore; + if (settings.logLevel == Waap::Scores::ModelLogLevel::DIFF && + ! ((score_diff > 0 && score >= 1.5f && otherScore < 4.0f) || + (score_diff < 0 && score < 4.0f && otherScore >= 1.5f))) { + return; + } + + auto current_time = Singleton::Consume::by()->getWalltime(); + auto now = chrono::duration_cast(current_time); + + WaapModelResult result = WaapModelResult( + *transaction, + res, + modelName, + otherModelName, + score, + otherScore, + now.count() + ); + + if (settings.logToStream) logToStream(result, now); + if (settings.logToS3) logToS3(result, transaction, now); +} + +void WaapModelResultLogger::Impl::logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now) +{ + auto asset_state = transaction->getAssetState(); + string asset_id = (asset_state != nullptr) ? asset_state->m_assetId : ""; + auto asset_logs = logs.find(asset_id); + if (asset_logs == logs.end()) { + logs.emplace(asset_id, vector()); + } + logs.at(asset_id).push_back(result); + if (shouldSendLogsToS3(now)) { + sendLogsToS3(); + } +} + +void WaapModelResultLogger::Impl::logToStream(WaapModelResult &result, chrono::minutes now) +{ + if (now - last_kusto_log_window > RATE_LIMIT_WINDOW_MINUTES) { + last_kusto_log_window = now; + sent_logs_count = 0; + } + else if (sent_logs_count > MAX_LOGS_PER_WINDOW) { + return; + } + sent_logs_count++; + dbgTrace(D_WAAP_MODEL_LOGGER) << "Logging WAAP model telemetry"; + + auto maybeLogTriggerConf = getConfiguration("rulebase", "log"); + LogGenWrapper logGenWrapper( + maybeLogTriggerConf, + "WAAP Model Telemetry", + ReportIS::Audience::SECURITY, + LogTriggerConf::SecurityType::ThreatPrevention, + ReportIS::Severity::CRITICAL, + ReportIS::Priority::HIGH, + false); + + LogGen& waap_log = logGenWrapper.getLogGen(); + waap_log.addMarkerSuffix(result.location); + waap_log << LogField("httpuripath", result.uri); + waap_log << LogField("matchedlocation", result.location); + waap_log << LogField("matchedparameter", result.param); + waap_log << LogField("matchedindicators", Waap::Util::vecToString(result.keywords), LogFieldOption::XORANDB64); + waap_log << LogField("matchedsample", result.sample, LogFieldOption::XORANDB64); + waap_log << LogField("waapkeywordsscore", (int)(result.otherScore * 100)); + waap_log << LogField("waapfinalscore", (int)(result.score * 100)); + waap_log << LogField("indicatorssource", result.modelName); + waap_log << LogField("indicatorsversion", result.otherModelName); +} + +bool WaapModelResultLogger::Impl::shouldSendLogsToS3(chrono::minutes now) +{ + if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) return true; + for (const auto &asset_logs : logs) { + if (asset_logs.second.size() >= max_logs) return true; + } + return false; +} + +void WaapModelResultLogger::Impl::sendLogsToS3() +{ + dbgFlow(D_WAAP_MODEL_LOGGER) << "Sending logs to fog"; + + I_Messaging *msg = Singleton::Consume::by(); + + for (auto &asset_logs : logs) { + if (asset_logs.second.empty()) { + continue; + } + if (sent_files_count >= MAX_FILES_PER_WINDOW) { + dbgInfo(D_WAAP_MODEL_LOGGER) << "Reached max files per window, will wait for next window"; + asset_logs.second.clear(); + continue; + } + I_AgentDetails *agentDetails = Singleton::Consume::by(); + string tenant_id = agentDetails->getTenantId(); + string agent_id = agentDetails->getAgentId(); + string asset_id = asset_logs.first; + if (Singleton::exists()) { + I_InstanceAwareness* instance = Singleton::Consume::by(); + Maybe uniqueId = instance->getUniqueID(); + if (uniqueId.ok()) + { + agent_id += "/" + uniqueId.unpack(); + } + } + string uri = "/storage/waap/" + + tenant_id + "/" + asset_id + "/waap_model_results/window_" + + to_string(last_sent_s3.count()) + "-" + to_string(sent_files_count) + + "/" + agent_id + "/data.data"; + WaapModelReport report = WaapModelReport(asset_logs.second); + + dbgInfo(D_WAAP_MODEL_LOGGER) << "Sending logs for asset " << asset_logs.first << + ", length " << asset_logs.second.size() << + ", uri " << uri; + msg->sendAsyncMessage( + HTTPMethod::PUT, + uri, + report, + MessageCategory::LOG + ); + + asset_logs.second.clear(); + } + + auto current_time = Singleton::Consume::by()->getWalltime(); + auto now = chrono::duration_cast(current_time); + if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) { + last_sent_s3 = now; + sent_files_count = 0; + } else { + sent_files_count++; + } +} + +WaapModelResultLogger::Impl::~Impl() +{} diff --git a/components/security_apps/waap/waap_clib/WaapModelResultLogger.h b/components/security_apps/waap/waap_clib/WaapModelResultLogger.h new file mode 100644 index 0000000..71a03e6 --- /dev/null +++ b/components/security_apps/waap/waap_clib/WaapModelResultLogger.h @@ -0,0 +1,109 @@ +// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include "cereal/archives/json.hpp" + +#include "i_waap_model_result_logger.h" +#include "DeepAnalyzer.h" +#include "i_transaction.h" +#include "ScanResult.h" +#include "WaapAssetState.h" +#include "WaapScores.h" + +class WaapModelResultLogger + : + Singleton::Provide +{ +public: + WaapModelResultLogger(size_t maxLogs = MAX_WAAP_MODEL_LOGS); + virtual ~WaapModelResultLogger(); + virtual void logModelResult( + Waap::Scores::ModelLoggingSettings &settings, + IWaf2Transaction* transaction, + Waf2ScanResult &res, + std::string modelName, + std::string otherModelName, + double score, + double otherScore + ); + class Impl; + +protected: + std::unique_ptr pimpl; + static const size_t MAX_WAAP_MODEL_LOGS = 20000; +}; + +class WaapModelResult +{ +public: + WaapModelResult( + IWaf2Transaction &transaction, + Waf2ScanResult &res, + const std::string &modelName, + const std::string &otherModelName, + double score, + double otherScore, + uint64_t time + ) : uri(transaction.getUri()), location(res.location), param(res.param_name), + modelName(modelName), otherModelName(otherModelName), + score(score), otherScore(otherScore), keywords(res.keywordsAfterFilter), + sample(res.unescaped_line.substr(0, 100)), id(transaction.getIndex()), time(time) + { + } + + template + void serialize(Archive &ar) const + { + ar(cereal::make_nvp("uri", uri)); + ar(cereal::make_nvp("location", location)); + ar(cereal::make_nvp("param", param)); + ar(cereal::make_nvp("modelName", modelName)); + ar(cereal::make_nvp("otherModelName", otherModelName)); + ar(cereal::make_nvp("score", score)); + ar(cereal::make_nvp("otherScore", otherScore)); + ar(cereal::make_nvp("keywords", keywords)); + ar(cereal::make_nvp("sample", sample)); + ar(cereal::make_nvp("id", id)); + ar(cereal::make_nvp("time", time)); + } + + std::string toString() const + { + std::stringstream message_stream; + { + cereal::JSONOutputArchive ar(message_stream); + serialize(ar); + } + return message_stream.str(); + } + + std::string uri; + std::string location; + std::string param; + std::string modelName; + std::string otherModelName; + double score; + double otherScore; + std::vector keywords; + std::string sample; + uint64_t id; + uint64_t time; +};