Jan 06 2026 dev (#387)

* sync code * update code to support brotli * update code to support brotli * update code to support brotli * sync code * fix findBrotli * sync code * sync code * sync code * sync code --------- Co-authored-by: Ned Wright <nedwright@proton.me> Co-authored-by: Daniel Eisenberg <danielei@checkpoint.com>
2026-01-17 16:00:26 +03:00 · 2026-01-13 17:17:52 +02:00
parent c1058db57d
commit e7b6e51b31
216 changed files with 12601 additions and 2825 deletions
--- a/core/messaging/connection/connection.cc
+++ b/core/messaging/connection/connection.cc
@@ -39,7 +39,7 @@ using namespace smartBIO;
 USE_DEBUG_FLAG(D_CONNECTION);

 static const HTTPResponse sending_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to send all data in time");
-static const HTTPResponse receving_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to receive all data in time");
+static const HTTPResponse receiving_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to receive all data in time");
 static const HTTPResponse parsing_error(HTTPStatusCode::HTTP_UNKNOWN, "Failed to parse the HTTP response");
 static const HTTPResponse close_error(
    HTTPStatusCode::HTTP_UNKNOWN,
@@ -271,18 +271,11 @@ private:
            return *details_ssl_dir;
        }

-        // Use detail_resolver to determine platform-specific certificate directory
 #if defined(alpine)
-       string platform = "alpine";
+        return "/etc/ssl/certs/";
 #else
-       string platform = "linux";
-#endif
-
-        if (platform == "alpine") {
-            return "/etc/ssl/certs/";
-        }
-
        return "/usr/lib/ssl/certs/";
+#endif
    }

    Maybe<void>
@@ -741,20 +734,54 @@ private:
            }
        }

-        auto receiving_end_time = i_time->getMonotonicTime() + getConnectionTimeout();
+        auto base_timeout_config = getProfileAgentSettingWithDefault<uint>(
+            10,
+            "agent.config.message.chunk.connection.timeout"
+        );
+        auto base_timeout = chrono::seconds(base_timeout_config); // 10 seconds between data chunks
+        
+        auto global_timeout_config = getProfileAgentSettingWithDefault<uint>(
+            600,
+            "agent.config.message.global.connection.timeout"
+        );
+        auto global_timeout = chrono::seconds(global_timeout_config); // 600 seconds maximum for entire download
+        
+        auto receiving_end_time = i_time->getMonotonicTime() + base_timeout;
+        auto global_end_time = i_time->getMonotonicTime() + global_timeout;
        HTTPResponseParser http_parser;
-        dbgTrace(D_CONNECTION) << "Sent the message, now waiting for response";
+        dbgTrace(D_CONNECTION)
+            << "Sent the message, now waiting for response (global timeout: "
+            << global_timeout.count()
+            << " seconds)";
+
        while (!http_parser.hasReachedError()) {
+            // Check global timeout first
+            if (i_time->getMonotonicTime() > global_end_time) {
+                should_close_connection = true;
+                dbgWarning(D_CONNECTION)
+                    << "Global receive timeout reached after "
+                    << global_timeout.count() << " seconds";
+                return genError(receiving_timeout);
+            }
+            
+            // Check per-chunk timeout
            if (i_time->getMonotonicTime() > receiving_end_time) {
                should_close_connection = true;
-                return genError(receving_timeout);
-            };
+                dbgWarning(D_CONNECTION) << "No data received for " << base_timeout.count() << " seconds";
+                return genError(receiving_timeout);
+            }
+
            auto receieved = receiveData();
            if (!receieved.ok()) {
                should_close_connection = true;
                return receieved.passErr();
            }
+            // Reset timeout each time we receive data
+            if (!receieved.unpack().empty()) {
+                receiving_end_time = i_time->getMonotonicTime() + base_timeout;
+            }
            auto response = http_parser.parseData(*receieved, is_connect);
+
            i_mainloop->yield(receieved.unpack().empty());
            if (response.ok()) {
                dbgTrace(D_MESSAGING) << printOut(response.unpack().toString());
--- a/core/messaging/connection/connection_comp.cc
+++ b/core/messaging/connection/connection_comp.cc
@@ -48,6 +48,13 @@ public:
        return establishNewConnection(metadata, category);
    }

+    void
+    clearConnections() override
+    {
+        dbgTrace(D_CONNECTION) << "Clearing all persistent connections";
+        persistent_connections.clear();
+    }
+
    Maybe<Connection>
    getPersistentConnection(const string &host_name, uint16_t port, MessageCategory category) override
    {
--- a/core/messaging/connection/connection_ut/connection_comp_ut.cc
+++ b/core/messaging/connection/connection_ut/connection_comp_ut.cc
@@ -24,6 +24,7 @@
 #include "rest.h"
 #include "rest_server.h"
 #include "dummy_socket.h"
+#include <atomic>

 using namespace std;
 using namespace testing;
@@ -100,6 +101,11 @@ TEST_F(TestConnectionComp, testSetAndGetConnection)
    EXPECT_EQ(get_conn.getConnKey().getHostName(), "127.0.0.1");
    EXPECT_EQ(get_conn.getConnKey().getPort(), 8080);
    EXPECT_EQ(get_conn.getConnKey().getCategory(), MessageCategory::LOG);
+
+    i_conn->clearConnections();
+    maybe_get_connection = i_conn->getPersistentConnection("127.0.0.1", 8080, MessageCategory::LOG);
+    ASSERT_FALSE(maybe_get_connection.ok());
+
 }

 TEST_F(TestConnectionComp, testEstablishNewConnection)
@@ -279,19 +285,27 @@ TEST_F(TestConnectionComp, testSendRequestWithOneTimeFogConnection)
    auto req = HTTPRequest::prepareRequest(conn, HTTPMethod::POST, "/test", conn_metadata.getHeaders(), "test-body");
    ASSERT_TRUE(req.ok());

+    // Ensure we accept+respond exactly once regardless of yield overload order
+    std::atomic<bool> responded{false};
    EXPECT_CALL(mock_mainloop, yield(A<std::chrono::microseconds>()))
-        .WillOnce(
-            InvokeWithoutArgs(
-                [&]() {
-                    cerr << "accepting socket" << endl;
-                    dummy_socket.acceptSocket();
-                    dummy_socket.writeToSocket("HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\nmy-test");
-                }
-            )
-        ).WillRepeatedly(Return());
+        .WillRepeatedly(InvokeWithoutArgs([&]() {
+            if (!responded.exchange(true)) {
+                cerr << "accepting socket" << endl;
+                dummy_socket.acceptSocket();
+                dummy_socket.writeToSocket("HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\nmy-test");
+            }
+        }));
+    EXPECT_CALL(mock_mainloop, yield(A<bool>()))
+        .WillRepeatedly(InvokeWithoutArgs([&]() {
+            if (!responded.exchange(true)) {
+                cerr << "accepting socket while receiving" << endl;
+                dummy_socket.acceptSocket();
+                dummy_socket.writeToSocket("HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\nmy-test");
+            }
+        }));

    EXPECT_CALL(mock_timer, getMonotonicTime())
-        .WillRepeatedly(Invoke([]() { static int j = 0; return chrono::microseconds(++j * 10); }));
+        .WillRepeatedly(Invoke([]() { static int j = 0; return chrono::microseconds(++j * 1000 * 1000); }));

    auto maybe_response = i_conn->sendRequest(conn, *req);
    if (!maybe_response.ok()) {