sync code

core/include/services_sdk/interfaces/messaging/http_response.h

@@ -31,15 +31,26 @@ public:
     HTTPResponse() = default;
 // LCOV_EXCL_STOP
 
-    HTTPResponse(HTTPStatusCode _status_code, const std::string &_body) : status_code(_status_code), body(_body) {}
+    HTTPResponse(
+        HTTPStatusCode _status_code,
+        const std::string &_body,
+        std::unordered_map<std::string, std::string> _headers = std::unordered_map<std::string, std::string>()
+    )
+            :
+        status_code(_status_code),
+        body(_body),
+        headers(_headers)
+    {}
 
     HTTPStatusCode getHTTPStatusCode() const;
     const std::string & getBody() const;
     std::string toString() const;
+    Maybe<std::string> getHeaderVal(const std::string &header_key);
 
 private:
     HTTPStatusCode status_code;
     std::string body;
+    std::unordered_map<std::string, std::string> headers;
 };
 
 #endif // __HTTP_RESPONSE_H__

core/include/services_sdk/interfaces/messaging/messaging_enums.h

@@ -63,6 +63,7 @@ enum class HTTPStatusCode
     HTTP_PROXY_AUTHENTICATION_REQUIRED = 407,
     HTTP_REQUEST_TIME_OUT = 408,
     HTTP_PAYLOAD_TOO_LARGE = 413,
+    HTTP_TOO_MANY_REQUESTS = 429,
     // 5xx - Server error responses.
     HTTP_INTERNAL_SERVER_ERROR = 500,
     HTTP_NOT_IMPLEMENTED = 501,

core/include/services_sdk/interfaces/messaging/messaging_metadata.h

@@ -8,6 +8,7 @@
 #include "config.h"
 #include "singleton.h"
 #include "i_agent_details.h"
+#include "i_time_get.h"
 
 class MessageProxySettings
 {
@@ -54,7 +55,7 @@ private:
     uint16_t proxy_port = 0;
 };
 
-class MessageMetadata
+class MessageMetadata : Singleton::Consume<I_TimeGet>
 {
 public:
     inline MessageMetadata();
@@ -227,6 +228,26 @@ public:
         return sni_host_name;
     }
 
+    void
+    setRateLimitBlock(uint block_time)
+    {
+        is_rate_limit_block = true;
+        auto timer = Singleton::Consume<I_TimeGet>::by<MessageMetadata>();
+        auto current_timeout = timer->getMonotonicTime() + std::chrono::seconds(block_time);
+        rate_limit_block_time = current_timeout.count();
+    }
+
+    bool
+    isRateLimitBlock() const
+    {
+        if (is_rate_limit_block) {
+            auto timer = Singleton::Consume<I_TimeGet>::by<MessageMetadata>();
+            uint current_time = timer->getMonotonicTime().count();
+            if (current_time < rate_limit_block_time) return true;
+        }
+        return false;
+    }
+
     template <class Archive>
     void
     serialize(Archive &ar)
@@ -243,7 +264,9 @@ public:
             cereal::make_nvp("is_to_fog", is_to_fog),
             cereal::make_nvp("ca_path", ca_path),
             cereal::make_nvp("client_cert_path", client_cert_path),
-            cereal::make_nvp("client_key_path", client_key_path)
+            cereal::make_nvp("client_key_path", client_key_path),
+            cereal::make_nvp("is_rate_limit_block", is_rate_limit_block),
+            cereal::make_nvp("rate_limit_block_time", rate_limit_block_time)
         );
     }
 
@@ -262,6 +285,8 @@ private:
     std::string external_certificate = "";
     bool should_buffer = false;
     bool is_to_fog = false;
+    bool is_rate_limit_block = false;
+    uint rate_limit_block_time = 0;
 };
 
 #endif // __MESSAGING_METADATA_H__

core/include/services_sdk/resources/debug_flags.h

@@ -107,6 +107,8 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
             DEFINE_FLAG(D_WAAP_PARSER_PAIRS, D_WAAP_PARSER)
             DEFINE_FLAG(D_WAAP_PARSER_PDF, D_WAAP_PARSER)
             DEFINE_FLAG(D_WAAP_PARSER_BINARY_FILE, D_WAAP_PARSER)
+            DEFINE_FLAG(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER, D_WAAP_PARSER)
+            DEFINE_FLAG(D_WAAP_PARSER_SCREENED_JSON, D_WAAP_PARSER)
 
     DEFINE_FLAG(D_IPS, D_COMPONENT)
     DEFINE_FLAG(D_FILE_UPLOAD, D_COMPONENT)

core/include/services_sdk/resources/generic_metric.h

@@ -24,6 +24,7 @@
 #include "i_mainloop.h"
 #include "i_time_get.h"
 #include "i_agent_details.h"
+#include "i_encryptor.h"
 #include "i_instance_awareness.h"
 #include "i_environment.h"
 #include "i_messaging.h"
@@ -57,6 +58,7 @@ class GenericMetric
     Singleton::Consume<I_Environment>,
     Singleton::Consume<I_Messaging>,
     Singleton::Consume<I_RestApi>,
+    Singleton::Consume<I_Encryptor>,
     public Listener<AllMetricEvent>
 {
 public:

core/include/services_sdk/resources/metric/metric_calc.h

@@ -23,6 +23,8 @@
 
 #include "report/report.h"
 #include "customized_cereal_map.h"
+#include "compression_utils.h"
+#include "i_encryptor.h"
 
 class GenericMetric;
 
@@ -56,6 +58,11 @@ public:
         value(_value)
     {
         timestamp = Singleton::Consume<I_TimeGet>::by<GenericMetric>()->getWalltimeStr();
+        // convert timestamp to RFC 3339 format
+        std::size_t pos = timestamp.find('.');
+        if (pos != std::string::npos) {
+            timestamp = timestamp.substr(0, pos) + "Z";
+        }
         asset_id = Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId();
     }
 
@@ -118,7 +125,73 @@ public:
     }
 
 // LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
-    std::string
+    Maybe<std::string>
+    toString() const
+    {
+        std::stringstream ss;
+        {
+            cereal::JSONOutputArchive ar(ss);
+            serialize(ar);
+        }
+        auto res = compressAndEncodeData(ss.str());
+        if (!res.ok()) {
+            return genError("Failed to compress and encode the data");
+        }
+        return res.unpack();
+    }
+// LCOV_EXCL_STOP
+
+private:
+    Maybe<std::string>
+    compressAndEncodeData(const std::string &unhandled_data) const
+    {
+        std::string data_holder = unhandled_data;
+        auto compression_stream = initCompressionStream();
+        CompressionResult compression_response = compressData(
+            compression_stream,
+            CompressionType::GZIP,
+            data_holder.size(),
+            reinterpret_cast<const unsigned char *>(data_holder.c_str()),
+            true
+        );
+        finiCompressionStream(compression_stream);
+        if (!compression_response.ok) {
+            // send log to Kibana
+            return genError("Failed to compress(gzip) data");
+        }
+
+        std::string compressed_data =
+            std::string((const char *)compression_response.output, compression_response.num_output_bytes);
+
+        auto encryptor = Singleton::Consume<I_Encryptor>::by<GenericMetric>();
+        Maybe<std::string> handled_data = encryptor->base64Encode(compressed_data);
+
+        if (compression_response.output) free(compression_response.output);
+        compression_response.output = nullptr;
+        compression_response.num_output_bytes = 0;
+        return handled_data;
+    }
+
+    std::vector<AiopsMetricData> metrics;
+};
+
+class CompressAndEncodeAIOPSMetrics
+{
+public:
+    CompressAndEncodeAIOPSMetrics(const AiopsMetricList &_aiops_metrics) : aiops_metrics(_aiops_metrics) {}
+
+    void
+    serialize(cereal::JSONOutputArchive &ar) const
+    {
+        auto metric_str = aiops_metrics.toString();
+        if (!metric_str.ok()) {
+            return;
+        }
+        ar(cereal::make_nvp("records", metric_str.unpack()));
+    }
+
+// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
+    Maybe<std::string>
     toString() const
     {
         std::stringstream ss;
@@ -131,7 +204,7 @@ public:
 // LCOV_EXCL_STOP
 
 private:
-    std::vector<AiopsMetricData> metrics;
+    AiopsMetricList aiops_metrics;
 };
 
 class MetricCalc

core/include/services_sdk/resources/rest.h

@@ -33,6 +33,8 @@
 #include "maybe_res.h"
 #include "rest/schema_printer.h"
 
+static const std::string BULK_ARRAY_NAME = "bulkArray";
+
 /// @class JsonError
 /// @brief Class representing JSON parsing errors.
 ///