Jan 06 2026 dev (#387)

* sync code

* update code to support brotli

* update code to support brotli

* update code to support brotli

* sync code

* fix findBrotli

* sync code

* sync code

* sync code

* sync code

---------

Co-authored-by: Ned Wright <nedwright@proton.me>
Co-authored-by: Daniel Eisenberg <danielei@checkpoint.com>

components/security_apps/waap/include/buffered_compressed_stream.h

@@ -0,0 +1,127 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <iostream>
+#include <streambuf>
+#include <vector>
+#include <memory>
+#include "compression_utils.h"
+#include "debug.h"
+
+USE_DEBUG_FLAG(D_WAAP_SERIALIZE);
+
+// Forward declarations
+class WaapComponent;
+
+void yieldIfPossible(const std::string &func, int line);
+
+#define YIELD_IF_POSSIBLE() yieldIfPossible(__FUNCTION__, __LINE__)
+
+//
+// Buffered output stream that compresses and encrypts data when flushing
+//
+// Usage example:
+//   std::stringstream ss;
+//   BufferedCompressedOutputStream compressed_stream(ss);
+//   compressed_stream << "Hello, World!";
+//   compressed_stream.flush(); // Data is compressed, encrypted, and written to ss
+class BufferedCompressedOutputStream : public std::ostream
+{
+public:
+    explicit BufferedCompressedOutputStream(std::ostream &underlying_stream);
+    ~BufferedCompressedOutputStream();
+
+    // Manual flush to compress, encrypt and write data
+    void flush();
+    void close();
+
+private:
+    class CompressedBuffer : public std::streambuf, Singleton::Consume<I_Encryptor>
+    {
+    public:
+        explicit CompressedBuffer(std::ostream &underlying_stream);
+        ~CompressedBuffer();
+
+        // Public method to flush the buffer
+        void flushAndClose();
+        void flushBuffer();
+
+    protected:
+        virtual int overflow(int c) override;
+        virtual std::streamsize xsputn(const char* s, std::streamsize n) override;
+        virtual int sync() override;
+
+    private:
+        // Compress and encrypt buffer; is_last indicates final chunk
+        bool compressAndEncryptBuffer(bool is_last);
+        std::ostream &m_underlying_stream;
+        std::vector<char> m_buffer;
+        static const size_t BUFFER_SIZE = 16 * 1024; // 16KiB
+        CompressionStream* m_compression_stream;
+        bool m_closed;
+    };
+
+    std::unique_ptr<CompressedBuffer> m_buffer;
+};
+
+
+// Buffered input stream that decrypts and decompresses data when reading
+//
+// Usage example:
+//   std::stringstream ss("encrypted compressed data");
+//   BufferedCompressedInputStream decompressed_stream(ss);
+//   std::string line;
+//   std::getline(decompressed_stream, line); // Data is decrypted and decompressed
+
+class BufferedCompressedInputStream : public std::istream
+{
+public:
+    explicit BufferedCompressedInputStream(std::istream &underlying_stream);
+    ~BufferedCompressedInputStream();
+
+private:
+    class DecompressedBuffer : public std::streambuf
+    {
+    public:
+        explicit DecompressedBuffer(std::istream &underlying_stream);
+        ~DecompressedBuffer();
+
+    protected:
+        virtual int underflow() override;
+        virtual std::streamsize xsgetn(char* s, std::streamsize n) override;
+
+    private:
+        bool fillBuffer();
+        bool processNextChunk();
+        bool decryptChunk(const std::vector<char> &encrypted_chunk, std::vector<char> &decrypted_chunk);
+        bool decompressChunk(const std::vector<char> &compressed_chunk, std::vector<char> &decompressed_chunk);
+        
+        std::istream &m_underlying_stream;
+        std::vector<char> m_buffer; // Output buffer for decompressed data
+        std::vector<char> m_encrypted_buffer; // Buffer for encrypted data from stream
+        std::vector<char> m_compressed_buffer; // Buffer for decrypted but still compressed data
+        std::vector<char> m_decompressed_buffer; // Buffer for decompressed data chunks
+        size_t m_decompressed_pos; // Current position in decompressed buffer
+        
+        static const size_t OUTPUT_BUFFER_SIZE = 64 * 1024; // 64KiB output buffer
+        static const size_t CHUNK_SIZE = 16 * 1024; // 16KiB chunks for processing
+        
+        CompressionStream* m_compression_stream;
+        bool m_eof_reached;
+        bool m_stream_finished; // Whether we've finished processing the entire stream
+    };
+
+    std::unique_ptr<DecompressedBuffer> m_buffer;
+};

components/security_apps/waap/include/i_serialize.h

@@ -12,6 +12,7 @@
 // limitations under the License.
 
 #pragma once
+
 #include <chrono>
 #include <fstream>
 #include "i_time_get.h"
@@ -19,17 +20,23 @@
 #include "i_messaging.h"
 #include "i_mainloop.h"
 #include "i_agent_details.h"
+#include "compression_utils.h"
+#include <sstream>
+#include <vector>
 
 static const uint max_send_obj_retries = 3;
 static const std::chrono::microseconds wait_next_attempt(5000000);
 
 USE_DEBUG_FLAG(D_WAAP_SERIALIZE);
 
+// Forward declarations
+class WaapComponent;
+
 class RestGetFile : public ClientRest
 {
 public:
     // decrypts and load json
-    bool loadJson(const std::string& json);
+    bool loadJson(const std::string &json);
     // gen json and encrypt
     Maybe<std::string> genJson() const;
 };
@@ -47,10 +54,10 @@ public:
 
     // parses xml instead of json
     // extracts a file list in <Contents><Key>
-    bool loadJson(const std::string& xml);
+    bool loadJson(const std::string &xml);
 
-    const std::vector<FileMetaData>& getFilesMetadataList() const;
-    const std::vector<std::string>& getFilesList() const;
+    const std::vector<FileMetaData> &getFilesMetadataList() const;
+    const std::vector<std::string> &getFilesList() const;
 
 private:
     RestParam<std::vector<FileMetaData>> files;
@@ -59,18 +66,19 @@ private:
 
 class I_Serializable {
 public:
-    virtual void serialize(std::ostream& stream) = 0;
-    virtual void deserialize(std::istream& stream) = 0;
+    virtual void serialize(std::ostream &stream) = 0;
+    virtual void deserialize(std::istream &stream) = 0;
 };
 
 class I_RemoteSyncSerialize {
 public:
     virtual bool postData() = 0;
-    virtual void pullData(const std::vector<std::string>& files) = 0;
+    virtual void pullData(const std::vector<std::string> &files) = 0;
     virtual void processData() = 0;
     virtual void postProcessedData() = 0;
-    virtual void pullProcessedData(const std::vector<std::string>& files) = 0;
-    virtual void updateState(const std::vector<std::string>& files) = 0;
+    virtual void pullProcessedData(const std::vector<std::string> &files) = 0;
+    virtual void updateState(const std::vector<std::string> &files) = 0;
+    virtual Maybe<std::string> getRemoteStateFilePath() const { return genError("No remote state file path defined"); }
 };
 
 class I_Backup {
@@ -86,7 +94,7 @@ class SerializeToFileBase :
     public I_Serializable
 {
 public:
-    SerializeToFileBase(std::string filePath);
+    SerializeToFileBase(const std::string &filePath);
     virtual ~SerializeToFileBase();
 
     virtual void saveData();
@@ -96,13 +104,13 @@ protected:
     // saved file name for testing
     std::string m_filePath;
 private:
-    void loadFromFile(std::string filePath);
+    void loadFromFile(const std::string &filePath); // Updated to match implementation
 };
 
 class SerializeToFilePeriodically : public SerializeToFileBase
 {
 public:
-    SerializeToFilePeriodically(std::chrono::seconds pollingIntervals, std::string filePath);
+    SerializeToFilePeriodically(std::chrono::seconds pollingIntervals, const std::string &filePath);
     virtual ~SerializeToFilePeriodically();
 
     void setInterval(std::chrono::seconds newInterval);
@@ -122,16 +130,16 @@ class SerializeToLocalAndRemoteSyncBase : public I_RemoteSyncSerialize, public S
 public:
     SerializeToLocalAndRemoteSyncBase(std::chrono::minutes interval,
         std::chrono::seconds waitForSync,
-        const std::string& filePath,
-        const std::string& remotePath,
-        const std::string& assetId,
-        const std::string& owner);
+        const std::string &filePath,
+        const std::string &remotePath,
+        const std::string &assetId,
+        const std::string &owner);
     virtual ~SerializeToLocalAndRemoteSyncBase();
 
     virtual void restore();
 
     virtual void syncWorker();
-
+    bool shouldNotSync() const;
     void setInterval(std::chrono::seconds newInterval);
     std::chrono::seconds getIntervalDuration() const;
     void setRemoteSyncEnabled(bool enabled);
@@ -143,7 +151,7 @@ protected:
     std::string getUri();
     size_t getIntervalsCount();
     void incrementIntervalsCount();
-    bool isBase();
+    bool isBase() const;
 
     template<typename T>
     bool sendObject(T &obj, HTTPMethod method, std::string uri)
@@ -242,7 +250,7 @@ protected:
     }
 
     template<typename T>
-    bool sendNoReplyObjectWithRetry(T &obj, HTTPMethod method, std::string uri)
+    bool sendNoReplyObjectWithRetry(T &obj, HTTPMethod method, const std::string &uri)
     {
         I_MainLoop *mainloop= Singleton::Consume<I_MainLoop>::by<WaapComponent>();
         for (uint i = 0; i < max_send_obj_retries; i++)
@@ -270,10 +278,16 @@ protected:
 private:
     bool localSyncAndProcess();
     void updateStateFromRemoteService();
+    Maybe<std::string> getStateTimestampByListing();
+    bool checkAndUpdateStateTimestamp(const std::string& currentStateTimestamp);
     RemoteFilesList getProcessedFilesList();
     RemoteFilesList getRemoteProcessedFilesList();
     std::string getLearningHost();
     std::string getSharedStorageHost();
+    std::string getStateTimestampPath();
+    Maybe<std::string> getStateTimestamp();
+    Maybe<void> updateStateFromRemoteFile();
+    bool shouldSendSyncNotification() const;
 
     I_MainLoop* m_pMainLoop;
     std::chrono::microseconds m_waitForSync;
@@ -287,3 +301,4 @@ private:
     Maybe<std::string> m_shared_storage_host;
     Maybe<std::string> m_learning_host;
 };
+

components/security_apps/waap/include/i_transaction.h

@@ -89,7 +89,7 @@ public:
     virtual double getOtherModelScore() const = 0;
     virtual const std::vector<double> getScoreArray() const = 0;
     virtual Waap::CSRF::State& getCsrfState() = 0;
-    virtual ngx_http_cp_verdict_e getUserLimitVerdict() = 0;
+    virtual ServiceVerdict getUserLimitVerdict() = 0;
     virtual const std::string getUserLimitVerdictStr() const = 0;
     virtual const std::string getViolatedUserLimitTypeStr() const = 0;
     virtual void checkShouldInject() = 0;

components/security_apps/waap/include/i_waapConfig.h

@@ -40,6 +40,13 @@ enum class AttackMitigationMode
     PREVENT,
     UNKNOWN
 };
+
+enum class WaapConfigType {
+    Unknown = 0,
+    Application,
+    API
+};
+
 class IWaapConfig {
 public:
     virtual const std::string&   get_AssetId() const = 0;
@@ -65,6 +72,7 @@ public:
     virtual const std::shared_ptr<Waap::RateLimiting::Policy>& get_ErrorLimitingPolicy() const = 0;
     virtual const std::shared_ptr<Waap::SecurityHeaders::Policy>& get_SecurityHeadersPolicy() const = 0;
     virtual const std::shared_ptr<Waap::UserLimits::Policy>& get_UserLimitsPolicy() const = 0;
+    virtual WaapConfigType getType() const = 0;
 
     virtual void printMe(std::ostream& os) const = 0;
 };