sync code

components/security_apps/waap/waap_clib/RequestsMonitor.cc

@@ -0,0 +1,158 @@
+#include "RequestsMonitor.h"
+#include "waap.h"
+#include "SyncLearningNotification.h"
+#include "report_messaging.h"
+#include "customized_cereal_map.h"
+
+USE_DEBUG_FLAG(D_WAAP_CONFIDENCE_CALCULATOR);
+using namespace std;
+
+SourcesRequestMonitor::SourcesRequestMonitor(
+    const string& filePath,
+    const string& remotePath,
+    const string& assetId,
+    const string& owner) :
+    SerializeToLocalAndRemoteSyncBase(
+        chrono::minutes(10),
+        chrono::seconds(30),
+        filePath,
+        remotePath != "" ? remotePath + "/Monitor" : remotePath,
+        assetId,
+        owner
+    ), m_sourcesRequests()
+{
+}
+
+SourcesRequestMonitor::~SourcesRequestMonitor()
+{
+}
+
+void SourcesRequestMonitor::syncWorker()
+{
+    dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Running the sync worker for assetId='" << m_assetId << "', owner='" <<
+        m_owner << "'";
+    incrementIntervalsCount();
+    OrchestrationMode mode = Singleton::exists<I_AgentDetails>() ?
+        Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() : OrchestrationMode::ONLINE;
+
+    bool enabled = getProfileAgentSettingWithDefault<bool>(false, "appsec.sourceRequestsMonitor.enabled");
+
+    if (mode == OrchestrationMode::OFFLINE || !enabled || isBase() || !postData()) {
+        dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR)
+            << "Did not report data. for asset: "
+            << m_assetId
+            << " Remote URL: "
+            << m_remotePath
+            << " is enabled: "
+            << to_string(enabled)
+            << ", mode: " << int(mode);
+        return;
+    }
+
+    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Waiting for all agents to post their data";
+    waitSync();
+
+    if (mode == OrchestrationMode::HYBRID) {
+        dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "detected running in standalone mode. not sending sync notification";
+    } else {
+        SyncLearningNotificationObject syncNotification(m_assetId, "Monitor", getWindowId());
+
+        dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "sending sync notification: " << syncNotification;
+
+        ReportMessaging(
+            "sync notification for '" + m_assetId + "'",
+            ReportIS::AudienceTeam::WAAP,
+            syncNotification,
+            MessageCategory::GENERIC,
+            ReportIS::Tags::WAF,
+            ReportIS::Notification::SYNC_LEARNING
+        );
+    }
+}
+
+void SourcesRequestMonitor::logSourceHit(const string& source)
+{
+    m_sourcesRequests[chrono::duration_cast<chrono::minutes>(
+        Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime()
+    ).count()][source]++;
+}
+
+// LCOV_EXCL_START Reason: internal functions not used
+
+void SourcesRequestMonitor::pullData(const vector<string> &data)
+{
+    // not used. report only
+}
+
+void SourcesRequestMonitor::processData()
+{
+    // not used. report only
+}
+
+void SourcesRequestMonitor::postProcessedData()
+{
+    // not used. report only
+}
+
+void SourcesRequestMonitor::pullProcessedData(const vector<string> &data)
+{
+    // not used. report only
+}
+
+void SourcesRequestMonitor::updateState(const vector<string> &data)
+{
+    // not used. report only
+}
+
+// LCOV_EXCL_STOP
+
+typedef map<string, map<string, size_t>> MonitorJsonData;
+
+class SourcesRequestsReport : public RestGetFile
+{
+public:
+    SourcesRequestsReport(MonitorData& _sourcesRequests, const string& _agentId)
+        : sourcesRequests(), agentId(_agentId)
+    {
+        MonitorJsonData montiorData;
+        for (const auto& window : _sourcesRequests) {
+            for (const auto& source : window.second) {
+                montiorData[to_string(window.first)][source.first] = source.second;
+            }
+        }
+        sourcesRequests = montiorData;
+    }
+    private:
+    C2S_PARAM(MonitorJsonData, sourcesRequests);
+    C2S_PARAM(string, agentId);
+};
+
+bool SourcesRequestMonitor::postData()
+{
+    dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the data to remote";
+    // send collected data to remote and clear the local data
+    string url = getPostDataUrl();
+    string agentId = Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getAgentId();
+    SourcesRequestsReport currentWindow(m_sourcesRequests, agentId);
+    bool ok = sendNoReplyObjectWithRetry(currentWindow,
+        HTTPMethod::PUT,
+        url);
+    if (!ok) {
+        dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to post collected data to: " << url;
+    }
+    dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Data sent to remote: " << ok;
+    m_sourcesRequests.clear();
+    return ok;
+}
+
+void SourcesRequestMonitor::serialize(ostream& stream)
+{
+    cereal::JSONOutputArchive archive(stream);
+    archive(m_sourcesRequests);
+}
+
+void SourcesRequestMonitor::deserialize(istream& stream)
+{
+    cereal::JSONInputArchive archive(stream);
+    archive(m_sourcesRequests);
+}

components/security_apps/waap/waap_clib/RequestsMonitor.h

@@ -0,0 +1,33 @@
+#ifndef __REQUESTS_MONITOR_H__
+#define __REQUESTS_MONITOR_H__
+#include "i_serialize.h"
+
+typedef std::map<uint64_t, std::map<std::string, size_t>> MonitorData;
+
+class SourcesRequestMonitor : public SerializeToLocalAndRemoteSyncBase
+{
+public:
+    SourcesRequestMonitor(
+        const std::string& filePath,
+        const std::string& remotePath,
+        const std::string& assetId,
+        const std::string& owner);
+    virtual ~SourcesRequestMonitor();
+    virtual void syncWorker() override;
+    void logSourceHit(const std::string& source);
+protected:
+    virtual void pullData(const std::vector<std::string> &data) override;
+    virtual void processData() override;
+    virtual void postProcessedData() override;
+    virtual void pullProcessedData(const std::vector<std::string> &data) override;
+    virtual void updateState(const std::vector<std::string> &data) override;
+    virtual bool postData() override;
+
+    void serialize(std::ostream& stream);
+    void deserialize(std::istream& stream);
+private:
+    // map of sources and their requests per minute (UNIX)
+    MonitorData m_sourcesRequests;
+};
+
+#endif // __REQUESTS_MONITOR_H__

components/security_apps/waap/waap_component_impl.h

@@ -19,7 +19,6 @@
 #include "table_opaque.h"
 #include "i_transaction.h"
 #include "waap_clib/DeepAnalyzer.h"
-#include "waap_clib/WaapModelResultLogger.h"
 #include "waap_clib/WaapAssetState.h"
 #include "waap_clib/WaapAssetStatesManager.h"
 #include "reputation_features_agg.h"

core/include/services_sdk/resources/metric/metric_scraper.h

@@ -0,0 +1,45 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __METRIC_SCRAPER_H__
+#define __METRIC_SCRAPER_H__
+
+#include <string>
+#include <fstream>
+#include <vector>
+#include <streambuf>
+
+#include "singleton.h"
+#include "debug.h"
+#include "component.h"
+#include "event.h"
+#include "i_rest_api.h"
+#include "generic_metric.h"
+
+class MetricScraper
+        :
+    public Component,
+    Singleton::Consume<I_RestApi>
+{
+public:
+    MetricScraper();
+    ~MetricScraper();
+
+    void init();
+
+private:
+    class Impl;
+    std::unique_ptr<Impl> pimpl;
+};
+
+#endif // __METRIC_SCRAPER_H__

core/metric/metric_scraper.cc

@@ -0,0 +1,50 @@
+#include "metric/metric_scraper.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_METRICS);
+
+class MetricScraper::Impl
+{
+public:
+    void
+    init()
+    {
+        Singleton::Consume<I_RestApi>::by<MetricScraper>()->addGetCall(
+            "service-metrics",
+            [&] () { return getAllPrometheusMetrics(); }
+        );
+    }
+
+    string
+    getAllPrometheusMetrics()
+    {
+        auto all_metrics_events_res = MetricScrapeEvent().query();
+        for (auto metric_vec : all_metrics_events_res) {
+            for (PrometheusData metric : metric_vec) {
+                metric.label = "{" + metric.label + "}";
+                all_metrics.emplace_back(metric);
+            }
+        }
+        stringstream ss;
+        {
+            cereal::JSONOutputArchive archive(ss);
+            archive(cereal::make_nvp("metrics", all_metrics));
+        }
+        all_metrics.clear();
+        return ss.str();
+    }
+
+private:
+    vector<PrometheusData> all_metrics;
+};
+
+MetricScraper::MetricScraper() : Component("MetricScraper"), pimpl(make_unique<MetricScraper::Impl>()) {}
+
+MetricScraper::~MetricScraper() {}
+
+void
+MetricScraper::init()
+{
+    pimpl->init();
+}