diff --git a/components/security_apps/waap/waap_clib/RequestsMonitor.cc b/components/security_apps/waap/waap_clib/RequestsMonitor.cc new file mode 100644 index 0000000..af156c1 --- /dev/null +++ b/components/security_apps/waap/waap_clib/RequestsMonitor.cc @@ -0,0 +1,158 @@ +#include "RequestsMonitor.h" +#include "waap.h" +#include "SyncLearningNotification.h" +#include "report_messaging.h" +#include "customized_cereal_map.h" + +USE_DEBUG_FLAG(D_WAAP_CONFIDENCE_CALCULATOR); +using namespace std; + +SourcesRequestMonitor::SourcesRequestMonitor( + const string& filePath, + const string& remotePath, + const string& assetId, + const string& owner) : + SerializeToLocalAndRemoteSyncBase( + chrono::minutes(10), + chrono::seconds(30), + filePath, + remotePath != "" ? remotePath + "/Monitor" : remotePath, + assetId, + owner + ), m_sourcesRequests() +{ +} + +SourcesRequestMonitor::~SourcesRequestMonitor() +{ +} + +void SourcesRequestMonitor::syncWorker() +{ + dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Running the sync worker for assetId='" << m_assetId << "', owner='" << + m_owner << "'"; + incrementIntervalsCount(); + OrchestrationMode mode = Singleton::exists() ? + Singleton::Consume::by()->getOrchestrationMode() : OrchestrationMode::ONLINE; + + bool enabled = getProfileAgentSettingWithDefault(false, "appsec.sourceRequestsMonitor.enabled"); + + if (mode == OrchestrationMode::OFFLINE || !enabled || isBase() || !postData()) { + dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) + << "Did not report data. for asset: " + << m_assetId + << " Remote URL: " + << m_remotePath + << " is enabled: " + << to_string(enabled) + << ", mode: " << int(mode); + return; + } + + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Waiting for all agents to post their data"; + waitSync(); + + if (mode == OrchestrationMode::HYBRID) { + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "detected running in standalone mode. not sending sync notification"; + } else { + SyncLearningNotificationObject syncNotification(m_assetId, "Monitor", getWindowId()); + + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "sending sync notification: " << syncNotification; + + ReportMessaging( + "sync notification for '" + m_assetId + "'", + ReportIS::AudienceTeam::WAAP, + syncNotification, + MessageCategory::GENERIC, + ReportIS::Tags::WAF, + ReportIS::Notification::SYNC_LEARNING + ); + } +} + +void SourcesRequestMonitor::logSourceHit(const string& source) +{ + m_sourcesRequests[chrono::duration_cast( + Singleton::Consume::by()->getWalltime() + ).count()][source]++; +} + +// LCOV_EXCL_START Reason: internal functions not used + +void SourcesRequestMonitor::pullData(const vector &data) +{ + // not used. report only +} + +void SourcesRequestMonitor::processData() +{ + // not used. report only +} + +void SourcesRequestMonitor::postProcessedData() +{ + // not used. report only +} + +void SourcesRequestMonitor::pullProcessedData(const vector &data) +{ + // not used. report only +} + +void SourcesRequestMonitor::updateState(const vector &data) +{ + // not used. report only +} + +// LCOV_EXCL_STOP + +typedef map> MonitorJsonData; + +class SourcesRequestsReport : public RestGetFile +{ +public: + SourcesRequestsReport(MonitorData& _sourcesRequests, const string& _agentId) + : sourcesRequests(), agentId(_agentId) + { + MonitorJsonData montiorData; + for (const auto& window : _sourcesRequests) { + for (const auto& source : window.second) { + montiorData[to_string(window.first)][source.first] = source.second; + } + } + sourcesRequests = montiorData; + } + private: + C2S_PARAM(MonitorJsonData, sourcesRequests); + C2S_PARAM(string, agentId); +}; + +bool SourcesRequestMonitor::postData() +{ + dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the data to remote"; + // send collected data to remote and clear the local data + string url = getPostDataUrl(); + string agentId = Singleton::Consume::by()->getAgentId(); + SourcesRequestsReport currentWindow(m_sourcesRequests, agentId); + bool ok = sendNoReplyObjectWithRetry(currentWindow, + HTTPMethod::PUT, + url); + if (!ok) { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to post collected data to: " << url; + } + dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Data sent to remote: " << ok; + m_sourcesRequests.clear(); + return ok; +} + +void SourcesRequestMonitor::serialize(ostream& stream) +{ + cereal::JSONOutputArchive archive(stream); + archive(m_sourcesRequests); +} + +void SourcesRequestMonitor::deserialize(istream& stream) +{ + cereal::JSONInputArchive archive(stream); + archive(m_sourcesRequests); +} diff --git a/components/security_apps/waap/waap_clib/RequestsMonitor.h b/components/security_apps/waap/waap_clib/RequestsMonitor.h new file mode 100644 index 0000000..f0298eb --- /dev/null +++ b/components/security_apps/waap/waap_clib/RequestsMonitor.h @@ -0,0 +1,33 @@ +#ifndef __REQUESTS_MONITOR_H__ +#define __REQUESTS_MONITOR_H__ +#include "i_serialize.h" + +typedef std::map> MonitorData; + +class SourcesRequestMonitor : public SerializeToLocalAndRemoteSyncBase +{ +public: + SourcesRequestMonitor( + const std::string& filePath, + const std::string& remotePath, + const std::string& assetId, + const std::string& owner); + virtual ~SourcesRequestMonitor(); + virtual void syncWorker() override; + void logSourceHit(const std::string& source); +protected: + virtual void pullData(const std::vector &data) override; + virtual void processData() override; + virtual void postProcessedData() override; + virtual void pullProcessedData(const std::vector &data) override; + virtual void updateState(const std::vector &data) override; + virtual bool postData() override; + + void serialize(std::ostream& stream); + void deserialize(std::istream& stream); +private: + // map of sources and their requests per minute (UNIX) + MonitorData m_sourcesRequests; +}; + +#endif // __REQUESTS_MONITOR_H__ diff --git a/components/security_apps/waap/waap_component_impl.h b/components/security_apps/waap/waap_component_impl.h index e791577..91ab2c3 100755 --- a/components/security_apps/waap/waap_component_impl.h +++ b/components/security_apps/waap/waap_component_impl.h @@ -19,7 +19,6 @@ #include "table_opaque.h" #include "i_transaction.h" #include "waap_clib/DeepAnalyzer.h" -#include "waap_clib/WaapModelResultLogger.h" #include "waap_clib/WaapAssetState.h" #include "waap_clib/WaapAssetStatesManager.h" #include "reputation_features_agg.h" diff --git a/core/include/services_sdk/resources/metric/metric_scraper.h b/core/include/services_sdk/resources/metric/metric_scraper.h new file mode 100644 index 0000000..1e020cd --- /dev/null +++ b/core/include/services_sdk/resources/metric/metric_scraper.h @@ -0,0 +1,45 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __METRIC_SCRAPER_H__ +#define __METRIC_SCRAPER_H__ + +#include +#include +#include +#include + +#include "singleton.h" +#include "debug.h" +#include "component.h" +#include "event.h" +#include "i_rest_api.h" +#include "generic_metric.h" + +class MetricScraper + : + public Component, + Singleton::Consume +{ +public: + MetricScraper(); + ~MetricScraper(); + + void init(); + +private: + class Impl; + std::unique_ptr pimpl; +}; + +#endif // __METRIC_SCRAPER_H__ diff --git a/core/metric/metric_scraper.cc b/core/metric/metric_scraper.cc new file mode 100644 index 0000000..9d90ff1 --- /dev/null +++ b/core/metric/metric_scraper.cc @@ -0,0 +1,50 @@ +#include "metric/metric_scraper.h" + +using namespace std; + +USE_DEBUG_FLAG(D_METRICS); + +class MetricScraper::Impl +{ +public: + void + init() + { + Singleton::Consume::by()->addGetCall( + "service-metrics", + [&] () { return getAllPrometheusMetrics(); } + ); + } + + string + getAllPrometheusMetrics() + { + auto all_metrics_events_res = MetricScrapeEvent().query(); + for (auto metric_vec : all_metrics_events_res) { + for (PrometheusData metric : metric_vec) { + metric.label = "{" + metric.label + "}"; + all_metrics.emplace_back(metric); + } + } + stringstream ss; + { + cereal::JSONOutputArchive archive(ss); + archive(cereal::make_nvp("metrics", all_metrics)); + } + all_metrics.clear(); + return ss.str(); + } + +private: + vector all_metrics; +}; + +MetricScraper::MetricScraper() : Component("MetricScraper"), pimpl(make_unique()) {} + +MetricScraper::~MetricScraper() {} + +void +MetricScraper::init() +{ + pimpl->init(); +}