Jan_31_2024-Dev

core/messaging/messaging_comp/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_library(messaging_comp
+    messaging_comp.cc
+    http_request.cc
+    http_response.cc)
+add_subdirectory(messaging_comp_ut)

core/messaging/messaging_comp/http_request.cc

@@ -0,0 +1,149 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "http_request.h"
+
+#include <sstream>
+#include <string>
+
+#include "messaging.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_MESSAGING);
+
+void
+HTTPRequest::insertHeader(const string &header_key, const string &header_val)
+{
+    headers[header_key] = header_val;
+}
+
+bool
+HTTPRequest::setConnectionHeaders(const Connection &conn)
+{
+    string host = conn.getConnKey().getHostName();
+    string uri_prefix = conn.isOverProxy() ? "http://" + host : "";
+
+    switch (method) {
+        case HTTPMethod::GET: {
+            method_statement = "GET " + uri_prefix + uri + " HTTP/1.1";
+            break;
+        }
+        case HTTPMethod::POST: {
+            method_statement = "POST " + uri_prefix + uri + " HTTP/1.1";
+            break;
+        }
+        case HTTPMethod::PATCH: {
+            method_statement = "PATCH " + uri_prefix + uri + " HTTP/1.1";
+            break;
+        }
+        case HTTPMethod::PUT: {
+            method_statement = "PUT " + uri_prefix + uri + " HTTP/1.1";
+            break;
+        }
+        case HTTPMethod::CONNECT: {
+            host = host + ":" + to_string(conn.getConnKey().getPort());
+            method_statement = "CONNECT " + host + " HTTP/1.1";
+            break;
+        }
+        default: {
+            return false;
+        }
+    }
+
+    insertHeader("Host", host);
+    insertHeader("Content-Length", to_string(body.size()));
+    insertHeader("Content-type", "application/json");
+    insertHeader("Accept-Encoding", "identity");
+    if (headers.find("Connection") == headers.end()) {
+        insertHeader("Connection", "keep-alive");
+    }
+    return true;
+}
+
+Maybe<HTTPRequest>
+HTTPRequest::prepareRequest(
+    const Connection &conn,
+    HTTPMethod method,
+    const string &uri,
+    const map<string, string> &headers,
+    const string &body
+)
+{
+    HTTPRequest req(method, uri, headers, body);
+
+    if (!req.setConnectionHeaders(conn)) return genError("Failed to identify the HTTP method");
+
+    string agent_registration_query = R"("authenticationMethod": "token")";
+    bool dont_add_access_token = false;
+    if (method == HTTPMethod::CONNECT || body.find(agent_registration_query) != string::npos) {
+        dont_add_access_token = true;
+        dbgTrace(D_MESSAGING) << "Request is for agent authentication";
+    }
+    auto res = req.addAccessToken(conn, dont_add_access_token);
+    if (!res.ok()) return res.passErr();
+
+    if (conn.isOverProxy()) {
+        auto res = req.addProxyAuthorization(conn);
+        if (!res.ok()) return res.passErr();
+    }
+
+    return req;
+}
+
+string
+HTTPRequest::toString() const
+{
+    stringstream res;
+    res << method_statement << "\r\n";
+    for (const auto &header : headers) {
+        res << header.first << ": " << header.second << "\r\n";
+    }
+    res << "\r\n" << body;
+    return res.str();
+}
+
+Maybe<void>
+HTTPRequest::addAccessToken(const Connection &conn, bool dont_add_access_token)
+{
+    if (headers.find("Authorization") != headers.end() || dont_add_access_token) return Maybe<void>();
+
+    if (!conn.getExternalCertificate().empty()) {
+        insertHeader("Authorization", conn.getExternalCertificate());
+        return Maybe<void>();
+    }
+
+    string access_token = Singleton::Consume<I_AgentDetails>::by<Messaging>()->getAccessToken();
+    if (access_token.empty()) return genError("Access token is missing.");
+    insertHeader("Authorization", "Bearer " + access_token);
+    return Maybe<void>();
+}
+
+Maybe<void>
+HTTPRequest::addProxyAuthorization(const Connection &conn)
+{
+    insertHeader("Accept", "*/*");
+    insertHeader("Proxy-Connection", "Keep-Alive");
+
+    if (!conn.isUnsecure()) return Maybe<void>();
+
+    MessageProxySettings proxy_settings = conn.getProxySettings();
+    if (proxy_settings.getProxyAuth().empty()) {
+        dbgTrace(D_MESSAGING) << "No proxy authentication was set";
+        return Maybe<void>();
+    }
+
+    I_Encryptor *encryptor = Singleton::Consume<I_Encryptor>::by<Messaging>();
+    insertHeader("Proxy-Authorization", "Basic " + encryptor->base64Encode(proxy_settings.getProxyAuth()));
+    return Maybe<void>();
+}

core/messaging/messaging_comp/http_response.cc

@@ -0,0 +1,286 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "response_parser.h"
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_MESSAGING);
+
+static const string CRLF = "\r\n";
+
+static const map<HTTPStatusCode, string> status_code_to_string = {
+    {HTTPStatusCode::NO_HTTP_RESPONSE,                      "0 - NO_HTTP_RESPONSE"                      },
+    { HTTPStatusCode::HTTP_OK,                              "200 - HTTP_OK"                             },
+    { HTTPStatusCode::HTTP_NO_CONTENT,                      "204 - HTTP_NO_CONTENT"                     },
+    { HTTPStatusCode::HTTP_MULTI_STATUS,                    "207 - HTTP_MULTI_STATUS"                   },
+    { HTTPStatusCode::HTTP_BAD_REQUEST,                     "400 - HTTP_BAD_REQUEST"                    },
+    { HTTPStatusCode::HTTP_UNAUTHORIZED,                    "401 - HTTP_UNAUTHORIZED"                   },
+    { HTTPStatusCode::HTTP_FORBIDDEN,                       "403 - HTTP_FORBIDDEN"                      },
+    { HTTPStatusCode::HTTP_NOT_FOUND,                       "404 - HTTP_NOT_FOUND"                      },
+    { HTTPStatusCode::HTTP_METHOD_NOT_ALLOWED,              "405 - HTTP_METHOD_NOT_ALLOWED"             },
+    { HTTPStatusCode::HTTP_PROXY_AUTHENTICATION_REQUIRED,   "407 - HTTP_PROXY_AUTHENTICATION_REQUIRED"  },
+    { HTTPStatusCode::HTTP_REQUEST_TIME_OUT,                "408 - HTTP_REQUEST_TIME_OUT"               },
+    { HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE,               "413 - HTTP_PAYLOAD_TOO_LARGE"              },
+    { HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR,           "500 - HTTP_INTERNAL_SERVER_ERROR"          },
+    { HTTPStatusCode::HTTP_NOT_IMPLEMENTED,                 "501 - HTTP_NOT_IMPLEMENTED"                },
+    { HTTPStatusCode::HTTP_BAD_GATEWAY,                     "502 - HTTP_BAD_GATEWAY"                    },
+    { HTTPStatusCode::HTTP_SERVICE_UNABAILABLE,             "503 - HTTP_SERVICE_UNABAILABLE"            },
+    { HTTPStatusCode::HTTP_GATEWAY_TIMEOUT,                 "504 - HTTP_GATEWAY_TIMEOUT"                },
+    { HTTPStatusCode::HTTP_VERSION_NOT_SUPPORTED,           "505 - HTTP_VERSION_NOT_SUPPORTED"          },
+    { HTTPStatusCode::HTTP_VARIANT_ALSO_NEGOTIATES,         "506 - HTTP_VARIANT_ALSO_NEGOTIATES"        },
+    { HTTPStatusCode::HTTP_INSUFFICIENT_STORAGE,            "507 - HTTP_INSUFFICIENT_STORAGE"           },
+    { HTTPStatusCode::HTTP_LOOP_DETECTED,                   "508 - HTTP_LOOP_DETECTED"                  },
+    { HTTPStatusCode::HTTP_NOT_EXTENDED,                    "510 - HTTP_NOT_EXTENDED"                   },
+    { HTTPStatusCode::HTTP_NETWORK_AUTHENTICATION_REQUIRED, "511 - HTTP_NETWORK_AUTHENTICATION_REQUIRED"},
+    { HTTPStatusCode::HTTP_UNKNOWN,                         "-1 - HTTP_UNKNOWN"                         },
+    { HTTPStatusCode::HTTP_SUSPEND,                         "-2 - HTTP_SUSPEND"                         }
+};
+
+static const map<int, HTTPStatusCode> num_to_status_code = {
+    {200,  HTTPStatusCode::HTTP_OK                             },
+    { 204, HTTPStatusCode::HTTP_NO_CONTENT                     },
+    { 207, HTTPStatusCode::HTTP_MULTI_STATUS                   },
+    { 400, HTTPStatusCode::HTTP_BAD_REQUEST                    },
+    { 401, HTTPStatusCode::HTTP_UNAUTHORIZED                   },
+    { 403, HTTPStatusCode::HTTP_FORBIDDEN                      },
+    { 404, HTTPStatusCode::HTTP_NOT_FOUND                      },
+    { 405, HTTPStatusCode::HTTP_METHOD_NOT_ALLOWED             },
+    { 407, HTTPStatusCode::HTTP_PROXY_AUTHENTICATION_REQUIRED  },
+    { 408, HTTPStatusCode::HTTP_REQUEST_TIME_OUT               },
+    { 413, HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE              },
+    { 500, HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR          },
+    { 501, HTTPStatusCode::HTTP_NOT_IMPLEMENTED                },
+    { 502, HTTPStatusCode::HTTP_BAD_GATEWAY                    },
+    { 503, HTTPStatusCode::HTTP_SERVICE_UNABAILABLE            },
+    { 504, HTTPStatusCode::HTTP_GATEWAY_TIMEOUT                },
+    { 505, HTTPStatusCode::HTTP_VERSION_NOT_SUPPORTED          },
+    { 506, HTTPStatusCode::HTTP_VARIANT_ALSO_NEGOTIATES        },
+    { 507, HTTPStatusCode::HTTP_INSUFFICIENT_STORAGE           },
+    { 508, HTTPStatusCode::HTTP_LOOP_DETECTED                  },
+    { 510, HTTPStatusCode::HTTP_NOT_EXTENDED                   },
+    { 511, HTTPStatusCode::HTTP_NETWORK_AUTHENTICATION_REQUIRED}
+};
+
+const string &
+HTTPResponse::getBody() const
+{
+    return body;
+}
+
+HTTPStatusCode
+HTTPResponse::getHTTPStatusCode() const
+{
+    return status_code;
+}
+
+string
+HTTPResponse::toString() const
+{
+    auto code = status_code_to_string.find(status_code);
+    dbgAssert(code != status_code_to_string.end()) << "Unknown status code " << int(status_code);
+
+    return "[Status-code]: " + code->second + ", [Body]: " + (body.empty() ? "{}" : body);
+}
+
+Maybe<HTTPResponse>
+HTTPResponseParser::parseData(const string &data, bool is_connect)
+{
+    if (data.empty()) return genError("Data is empty");
+    raw_response += data;
+
+    if (!status_code.ok()) {
+        if (!parseStatusLine()) return genError("Failed to parse the status line. Error: " + status_code.getErr());
+    }
+
+    if (!headers.ok()) {
+        if (!handleHeaders()) return genError("Failed to parse the headers. Error: " + headers.getErr());
+    }
+
+    if (!handleBody(is_connect)) return genError("Response not ready!");
+
+    return HTTPResponse(status_code.unpack(), body);
+}
+
+static string
+strip(const string &str)
+{
+    string res;
+    for (auto ch : str) {
+        if (!isspace(ch)) res += tolower(ch);
+    }
+    return res;
+}
+
+bool
+HTTPResponseParser::handleHeaders()
+{
+    stringstream ss(raw_response);
+    map<string, string> header_map;
+
+    while (true) {
+        string header;
+        getline(ss, header);
+
+        if (header.empty()) {
+            headers = genError("Headers not complete");
+            return false;
+        }
+
+        if (header == "\r") {
+            headers = header_map;
+            ss.sync();
+            raw_response = raw_response.substr(ss.tellg());
+            return true;
+        }
+
+        auto colon_index = header.find_first_of(":");
+        if (colon_index == string::npos) {
+            // The only case where not finding a colon isn't an error is if we run out of data
+            error = !ss.str().empty();
+            headers = genError(error ? "Invalid headers: " + header : "Did not reach end of headers");
+            return false;
+        }
+
+        auto header_key = header.substr(0, colon_index);
+        auto header_val = header.substr(colon_index + 2);
+        header_map[strip(header_key)] = strip(header_val);
+    }
+}
+
+Maybe<string>
+HTTPResponseParser::getHeaderVal(const string &header_key)
+{
+    auto headers_map = headers.unpack();
+    auto header = headers_map.find(header_key);
+    if (header == headers_map.end()) {
+        return genError("Header\'" + header_key + "\' not found.");
+    }
+    return header->second;
+}
+
+bool
+HTTPResponseParser::handleBody(bool is_connect)
+{
+    if (*status_code == HTTPStatusCode::HTTP_OK && is_connect) return true;
+
+    if (*status_code == HTTPStatusCode::HTTP_NO_CONTENT) return raw_response.empty();
+
+    auto content_length = getHeaderVal("content-length");
+    if (content_length.ok()) {
+        size_t body_length;
+        try {
+            body_length = stoi(content_length.unpack());
+        } catch (const exception &err) {
+            return false;
+        }
+
+        body += raw_response;
+        raw_response.clear();
+        return body.size() == body_length;
+    }
+
+    auto transfer_encoding = getHeaderVal("transfer-encoding");
+    if (transfer_encoding.ok() && *transfer_encoding == "chunked") return getChunkedResponse();
+
+    dbgError(D_MESSAGING) << "Response has neither content-lenght nor chunked encoded";
+    return false;
+}
+
+bool
+HTTPResponseParser::getChunkedResponse()
+{
+    if (!isLegalChunkedResponse(raw_response)) return false;
+
+    size_t chunk_size = 0;
+
+    for (auto line_end = raw_response.find(CRLF); line_end != string::npos; line_end = raw_response.find(CRLF)) {
+        string line = raw_response.substr(0, line_end);
+        try {
+            chunk_size = stoi(line, nullptr, 16);
+        } catch (const exception &) {
+            dbgWarning(D_MESSAGING) << "Failed to convert chunk length to a number. Line: " << line;
+            return false;
+        }
+
+        if (line_end + 2 + chunk_size > raw_response.length()) {
+            dbgWarning(D_MESSAGING) << "Invalid chunked data structure - chunk-size is bigger than chunk-data";
+            return false;
+        }
+        string chunk_body = raw_response.substr(line_end + 2, chunk_size);
+        raw_response = raw_response.substr(line_end + 2 + chunk_size);
+
+        if (raw_response.find(CRLF) != 0) {
+            dbgWarning(D_MESSAGING) << "Invalid chunked data structure - chunk-data missing final CRLF sequence";
+            return false;
+        }
+        raw_response = raw_response.substr(2);
+
+        body += chunk_body;
+    }
+
+    if (chunk_size != 0) {
+        dbgDebug(D_MESSAGING) << "Invalid chunked data structure - last-chunk of the body is not sized 0";
+        return false;
+    }
+    return true;
+}
+
+bool
+HTTPResponseParser::isLegalChunkedResponse(const string &res)
+{
+    auto end_of_data = res.find("0\r\n\r\n");
+    return end_of_data != string::npos && res.length() == end_of_data + 5;
+}
+
+bool
+HTTPResponseParser::parseStatusLine()
+{
+    auto end_of_first_line = raw_response.find(CRLF);
+    if (end_of_first_line == string::npos) {
+        status_code = genError("No Status Line was received.");
+        return false;
+    }
+
+    auto status_line = raw_response.substr(0, end_of_first_line);
+    raw_response = raw_response.substr(end_of_first_line + 2);
+
+    // Also status text can be supported at the future.
+    if (status_line.find("HTTP/1.") == string::npos) {
+        status_code = genError("Status code not found.");
+        error = true;
+        return false;
+    }
+
+    int status_num;
+    try {
+        status_num = stoi(status_line.substr(9, 3));
+    } catch (const exception &err) {
+        status_code = genError("Failed to convert status code to a number. Status code: " + status_line.substr(9, 3));
+        error = true;
+        return false;
+    }
+
+    auto status = num_to_status_code.find(status_num);
+    if (status != num_to_status_code.end()) {
+        status_code = status->second;
+    } else {
+        dbgWarning(D_MESSAGING) << "Unknown HTTP status code: " << status_num;
+        status_code = HTTPStatusCode::HTTP_UNKNOWN;
+    }
+    return true;
+}

core/messaging/messaging_comp/messaging_comp.cc

@@ -0,0 +1,320 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "messaging_comp.h"
+
+#include <fstream>
+#include <string>
+
+#include "agent_core_utilities.h"
+#include "connection_comp.h"
+#include "debug.h"
+#include "messaging_buffer.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_MESSAGING);
+
+void
+MessagingComp::init()
+{
+    i_conn = Singleton::Consume<I_MessagingConnection>::from<ConnectionComponent>();
+    i_messaging_buffer = Singleton::Consume<I_MessageBuffer>::from<MessagingBufferComponent>();
+    agent_details = Singleton::Consume<I_AgentDetails>::by<Messaging>();
+
+    auto i_mainloop = Singleton::Consume<I_MainLoop>::by<Messaging>();
+    auto i_time_get = Singleton::Consume<I_TimeGet>::by<Messaging>();
+    auto cache_timeout = getConfigurationWithDefault<int>(40, "message", "Cache timeout");
+    fog_get_requests_cache.startExpiration(chrono::seconds(cache_timeout), i_mainloop, i_time_get);
+
+    should_buffer_failed_messages = getConfigurationWithDefault<bool>(
+        getProfileAgentSettingWithDefault<bool>(true, "eventBuffer.bufferFailedRequests"),
+        "message",
+        "Buffer Failed Requests"
+    );
+}
+
+static bool
+isMessageToFog(const MessageMetadata message_metadata)
+{
+    return message_metadata.isToFog();
+}
+
+Maybe<Connection>
+MessagingComp::getConnection(MessageCategory category, const MessageMetadata &metadata)
+{
+    auto persistant_conn = getPersistentConnection(metadata, category);
+    if (persistant_conn.ok()) {
+        dbgTrace(D_MESSAGING) << "Found a persistant connection";
+        return persistant_conn;
+    }
+    dbgDebug(D_MESSAGING) << persistant_conn.getErr();
+
+    auto maybe_conn = i_conn->establishConnection(metadata, category);
+    if (!maybe_conn.ok()) {
+        dbgWarning(D_MESSAGING) << "Failed to establish connection: " << maybe_conn.getErr();
+    }
+    return maybe_conn;
+}
+
+Maybe<HTTPResponse, HTTPResponse>
+MessagingComp::sendMessage(
+    HTTPMethod method,
+    const string &uri,
+    const string &body,
+    MessageCategory category,
+    const MessageMetadata &message_metadata
+)
+{
+    auto maybe_conn = getConnection(category, message_metadata);
+    if (!maybe_conn.ok()) {
+        dbgWarning(D_MESSAGING) << "Failed to get connection. Error: " << maybe_conn.getErr();
+        return genError<HTTPResponse>(HTTPStatusCode::HTTP_UNKNOWN, maybe_conn.getErr());
+    }
+
+    Connection conn = maybe_conn.unpack();
+    if (conn.isSuspended()) return suspendMessage(body, method, uri, category, message_metadata);
+
+    bool is_to_fog = isMessageToFog(message_metadata);
+    auto metadata = message_metadata;
+    if (is_to_fog) {
+        if (method == HTTPMethod::GET && fog_get_requests_cache.doesKeyExists(uri)) {
+            HTTPResponse res = fog_get_requests_cache.getEntry(uri);
+            dbgTrace(D_MESSAGING) << "Response returned from Fog cache. res body: " << res.getBody();
+
+            return fog_get_requests_cache.getEntry(uri);
+        }
+
+        auto i_env = Singleton::Consume<I_Environment>::by<Messaging>();
+        metadata.insertHeader("User-Agent", "Infinity Next (a7030abf93a4c13)");
+        metadata.insertHeaders(i_env->getCurrentHeadersMap());
+    }
+
+    auto req = HTTPRequest::prepareRequest(conn, method, uri, metadata.getHeaders(), body);
+    if (!req.ok()) return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, req.getErr()));
+
+    auto response = i_conn->sendRequest(conn, *req);
+    if (!response.ok()) return response.passErr();
+
+    if (is_to_fog && method == HTTPMethod::GET) fog_get_requests_cache.emplaceEntry(uri, *response);
+    return response;
+}
+
+Maybe<HTTPResponse, HTTPResponse>
+MessagingComp::sendSyncMessage(
+    HTTPMethod method,
+    const string &uri,
+    const string &body,
+    MessageCategory category,
+    const MessageMetadata &message_metadata
+)
+{
+    Maybe<HTTPResponse, HTTPResponse> is_msg_send = sendMessage(method, uri, body, category, message_metadata);
+
+    if (is_msg_send.ok()) return *is_msg_send;
+
+    if (should_buffer_failed_messages && message_metadata.shouldBufferMessage()) {
+        dbgTrace(D_MESSAGING) << "After sending error, buffering the message";
+        i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false);
+    }
+    return is_msg_send.passErr();
+}
+
+void
+MessagingComp::sendAsyncMessage(
+    HTTPMethod method,
+    const string &uri,
+    const string &body,
+    MessageCategory category,
+    const MessageMetadata &message_metadata
+)
+{
+    i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false);
+}
+
+Maybe<HTTPStatusCode, HTTPResponse>
+MessagingComp::downloadFile(
+    HTTPMethod method,
+    const string &uri,
+    const string &download_file_path,
+    MessageCategory category,
+    const MessageMetadata &message_metadata
+)
+{
+    dbgTrace(D_MESSAGING) << "Send download file message";
+    string parent_directory = download_file_path.substr(0, download_file_path.find_last_of("/\\"));
+    if (!NGEN::Filesystem::exists(parent_directory)) {
+        if (!NGEN::Filesystem::makeDirRecursive(parent_directory)) {
+            string creation_err = "Failed to create the parent directory. Path: " + parent_directory;
+            dbgWarning(D_MESSAGING) << creation_err;
+            return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, creation_err));
+        }
+    }
+
+    auto response = sendSyncMessage(method, uri, "", category, message_metadata);
+    if (!response.ok()) return response.passErr();
+
+    ofstream file_stream(download_file_path);
+    if (!file_stream.is_open()) {
+        string open_err = "Failed to open the destination file. Path: " + download_file_path;
+        dbgWarning(D_MESSAGING) << open_err;
+        return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, open_err));
+    }
+    file_stream << response.unpack().getBody();
+    file_stream.close();
+
+    dbgTrace(D_MESSAGING) << "Successfully downloaded and save file to: " << download_file_path;
+    return HTTPStatusCode::HTTP_OK;
+}
+
+Maybe<HTTPStatusCode, HTTPResponse>
+MessagingComp::uploadFile(
+    const string &uri,
+    const string &upload_file_path,
+    MessageCategory category,
+    const MessageMetadata &message_metadata
+)
+{
+    dbgTrace(D_MESSAGING) << "Send upload file message";
+
+    ifstream file(upload_file_path);
+    if (!file.is_open()) {
+        string open_err = "Failed to open the file to upload. Path: " + upload_file_path;
+        dbgWarning(D_MESSAGING) << open_err;
+        return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, open_err));
+    }
+
+    stringstream buffer;
+    buffer << file.rdbuf();
+    file.close();
+
+    Maybe<HTTPResponse, HTTPResponse> response =
+        sendSyncMessage(HTTPMethod::PUT, uri, buffer.str(), category, message_metadata);
+
+    if (!response.ok()) return response.passErr();
+
+    dbgTrace(D_MESSAGING) << "Successfully upload file from: " << upload_file_path;
+    return HTTPStatusCode::HTTP_OK;
+}
+
+bool
+MessagingComp::setFogConnection(const string &host, uint16_t port, bool is_secure, MessageCategory category)
+{
+    dbgTrace(D_MESSAGING) << "Setting a fog connection to " << host << ":" << port;
+    MessageMetadata metadata(host, port, true);
+
+    I_ProxyConfiguration *proxy_configuration = Singleton::Consume<I_ProxyConfiguration>::by<Messaging>();
+    auto load_env_proxy = proxy_configuration->loadProxy();
+    if (!load_env_proxy.ok()) {
+        dbgDebug(D_MESSAGING)
+            << "Could not initialize load proxy from environment, Error: "
+            << load_env_proxy.getErr();
+    }
+
+    ProxyProtocol proxy_protocol = is_secure ? ProxyProtocol::HTTPS : ProxyProtocol::HTTP;
+    if (proxy_configuration->getProxyExists(proxy_protocol)) {
+        auto proxy_host = proxy_configuration->getProxyDomain(proxy_protocol);
+        auto proxy_port = proxy_configuration->getProxyPort(proxy_protocol);
+        auto maybe_proxy_auth = proxy_configuration->getProxyAuthentication(proxy_protocol);
+
+        if (proxy_host.ok() && proxy_port.ok()) {
+            string proxy_auth = maybe_proxy_auth.ok() ? *maybe_proxy_auth : "";
+            dbgDebug(D_MESSAGING) << "Setting proxy address: " << *proxy_host << ":" << *proxy_port;
+            MessageProxySettings proxy_settings(proxy_host.unpack(), proxy_auth, proxy_port.unpack());
+            metadata.setProxySettings(proxy_settings);
+        }
+    }
+
+    I_MessagingConnection *i_conn = Singleton::Consume<I_MessagingConnection>::from<ConnectionComponent>();
+    auto conn = i_conn->establishConnection(metadata, category);
+    if (!conn.ok()) {
+        dbgWarning(D_MESSAGING) << "Failed to establish connection to fog: " << conn.getErr();
+        return false;
+    }
+
+    dbgInfo(D_MESSAGING)
+        << "Successfully connected to the Fog: "
+        << host
+        << ":"
+        << port
+        << " via "
+        << (metadata.isProxySet() ? "proxy, using " : "")
+        << (is_secure ? "secure" : "clear")
+        << " connection";
+
+    return true;
+}
+
+bool
+MessagingComp::setFogConnection(MessageCategory category)
+{
+    I_AgentDetails *agent_details = Singleton::Consume<I_AgentDetails>::by<Messaging>();
+
+    if (agent_details->getOrchestrationMode() == OrchestrationMode::OFFLINE) {
+        dbgDebug(D_MESSAGING) << "Agent Is in offline mode and would not attempt connecting to the fog";
+        return true;
+    }
+
+    if (!agent_details->readAgentDetails()) {
+        dbgWarning(D_MESSAGING) << "Cannot establish connection to the Fog, failed to read agent details";
+        return false;
+    }
+
+    auto domain = agent_details->getFogDomain();
+    auto port = agent_details->getFogPort();
+    auto is_secure_connection = agent_details->getSSLFlag();
+
+    if (!domain.ok() || *domain == "" || !port.ok() || port == 0) {
+        dbgWarning(D_MESSAGING) << "Cannot establish connection to the Fog, failed to get host and port details";
+        return false;
+    }
+
+    return setFogConnection(*domain, *port, is_secure_connection, category);
+}
+
+Maybe<Connection>
+MessagingComp::getPersistentConnection(const MessageMetadata &metadata, MessageCategory category) const
+{
+    if (!metadata.isToFog()) {
+        auto maybe_conn = i_conn->getPersistentConnection(metadata.getHostName(), metadata.getPort(), category);
+        if (maybe_conn.ok()) return *maybe_conn;
+        return genError("Failed to get persistant connection based on host and port");
+    }
+
+    auto maybe_conn = i_conn->getFogConnectionByCategory(category);
+    if (maybe_conn.ok()) return maybe_conn;
+    return genError("Failed to get persistant connection to the fog");
+}
+
+Maybe<HTTPResponse, HTTPResponse>
+MessagingComp::suspendMessage(
+    const string &body,
+    HTTPMethod method,
+    const string &uri,
+    MessageCategory category,
+    const MessageMetadata &message_metadata
+) const
+{
+    if (message_metadata.shouldBufferMessage()) {
+        dbgWarning(D_MESSAGING) << "Buffering message due to connection suspended";
+        i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false);
+        return genError<HTTPResponse>(
+            HTTPStatusCode::HTTP_SUSPEND,
+            "The connection is suspended due to consecutive message sending errors, message is buffered."
+        );
+    }
+
+    return genError<HTTPResponse>(
+        HTTPStatusCode::HTTP_SUSPEND, "The connection is suspended due to consecutive message sending errors."
+    );
+}

core/messaging/messaging_comp/messaging_comp_ut/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_unit_test(
+    messaging_comp_ut
+    "messaging_comp_ut.cc"
+    "messaging_comp;connection;messaging_buffer_comp;ssl;connkey;singleton;boost_context;rest;metric;event_is;-lboost_regex;agent_details;-lcrypto;"
+)

core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc

@@ -0,0 +1,238 @@
+#include "messaging_comp.h"
+
+#include "agent_core_utilities.h"
+#include "config.h"
+#include "config_component.h"
+#include "connection.h"
+#include "cptest.h"
+#include "environment.h"
+#include "mainloop.h"
+#include "mock/mock_agent_details.h"
+#include "mock/mock_mainloop.h"
+#include "mock/mock_time_get.h"
+#include "mock/mock_proxy_configuration.h"
+#include "mocks/mock_messaging_buffer.h"
+#include "mocks/mock_messaging_connection.h"
+#include "rest.h"
+#include "rest_server.h"
+
+using namespace std;
+using namespace testing;
+
+static ostream &
+operator<<(ostream &os, const Maybe<BufferedMessage> &)
+{
+    return os;
+}
+
+static std::ostream &
+operator<<(std::ostream &os, const HTTPResponse &)
+{
+    return os;
+}
+
+static std::ostream &
+operator<<(std::ostream &os, const HTTPStatusCode &)
+{
+    return os;
+}
+
+static std::ostream &
+operator<<(std::ostream &os, const Connection &)
+{
+    return os;
+}
+
+class TestMessagingComp : public testing::Test
+{
+public:
+    TestMessagingComp()
+    {
+        EXPECT_CALL(mock_time_get, getMonotonicTime()).WillRepeatedly(Return(chrono::microseconds(0)));
+
+        ON_CALL(mock_agent_details, getFogDomain()).WillByDefault(Return(Maybe<string>(fog_addr)));
+        ON_CALL(mock_agent_details, getFogPort()).WillByDefault(Return(Maybe<uint16_t>(fog_port)));
+        messaging_comp.init();
+    }
+
+    void
+    setAgentDetails()
+    {
+        EXPECT_CALL(mock_agent_details, getFogDomain()).WillRepeatedly(Return(string(fog_addr)));
+        EXPECT_CALL(mock_agent_details, getFogPort()).WillRepeatedly(Return(fog_port));
+        EXPECT_CALL(mock_agent_details, getOpenSSLDir()).WillRepeatedly(Return(string("/usr/lib/ssl/certs/")));
+        EXPECT_CALL(mock_agent_details, getAccessToken()).WillRepeatedly(Return(string("accesstoken")));
+        EXPECT_CALL(mock_agent_details, readAgentDetails()).WillRepeatedly(Return(true));
+        EXPECT_CALL(mock_proxy_conf, loadProxy()).WillRepeatedly(Return(Maybe<void>()));
+        EXPECT_CALL(mock_proxy_conf, getProxyExists(_)).WillRepeatedly(Return(true));
+        EXPECT_CALL(mock_proxy_conf, getProxyDomain(_)).WillRepeatedly(Return(string("7.7.7.7")));
+        EXPECT_CALL(mock_proxy_conf, getProxyPort(_)).WillRepeatedly(Return(8080));
+        EXPECT_CALL(mock_proxy_conf, getProxyAuthentication(_)).WillRepeatedly(Return(string("cred")));
+    }
+
+    const string fog_addr = "1.2.3.4";
+    uint16_t fog_port = 80;
+    CPTestTempfile agent_details_file;
+    MessagingComp messaging_comp;
+    ::Environment env;
+    ConfigComponent config;
+    NiceMock<MockMessagingConnection> mock_messaging_connection;
+    NiceMock<MockMessagingBuffer> mock_messaging_buffer;
+    NiceMock<MockMainLoop> mock_mainloop;
+    NiceMock<MockTimeGet> mock_time_get;
+    NiceMock<MockAgentDetails> mock_agent_details;
+    NiceMock<MockProxyConfiguration> mock_proxy_conf;
+};
+
+TEST_F(TestMessagingComp, testInitComp)
+{
+    EXPECT_CALL(
+        mock_mainloop, addRecurringRoutine(I_MainLoop::RoutineType::Timer, _, _, "Delete expired cache entries", _)
+    )
+        .WillOnce(Return(0));
+    messaging_comp.init();
+}
+
+TEST_F(TestMessagingComp, testSendSyncMessage)
+{
+    setAgentDetails();
+    string body = "test body";
+    HTTPMethod method = HTTPMethod::POST;
+    string uri = "/test-uri";
+    MessageCategory category = MessageCategory::GENERIC;
+    MessageMetadata message_metadata;
+
+    MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC);
+    Connection conn(conn_key, message_metadata);
+    EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC))
+        .WillOnce(Return(conn));
+
+    HTTPResponse res(HTTPStatusCode::HTTP_OK, "response!!");
+    EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res));
+    auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, message_metadata);
+    ASSERT_TRUE(sending_res.ok());
+    HTTPResponse http_res = sending_res.unpack();
+    EXPECT_EQ(http_res.getBody(), "response!!");
+    EXPECT_EQ(http_res.getHTTPStatusCode(), HTTPStatusCode::HTTP_OK);
+}
+
+TEST_F(TestMessagingComp, testSendAsyncMessage)
+{
+    setAgentDetails();
+    string body = "test body";
+    HTTPMethod method = HTTPMethod::POST;
+    string uri = "/test-uri";
+    MessageCategory category = MessageCategory::GENERIC;
+    MessageMetadata message_metadata;
+
+    EXPECT_CALL(mock_messaging_buffer, pushNewBufferedMessage(body, method, uri, category, _, _)).Times(1);
+    messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata);
+}
+
+TEST_F(TestMessagingComp, testSendSyncMessageOnSuspendedConn)
+{
+    setAgentDetails();
+    string body = "test body";
+    HTTPMethod method = HTTPMethod::POST;
+    string uri = "/test-uri";
+    MessageCategory category = MessageCategory::GENERIC;
+    MessageMetadata message_metadata;
+
+    MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC);
+    Connection conn(conn_key, message_metadata);
+
+    EXPECT_CALL(mock_time_get, getMonotonicTime())
+        .WillRepeatedly(Invoke([] () { static int j = 0; return chrono::microseconds(++j * 1000 * 1000); }));
+    for (int i = 0; i < 20; i++) {
+        conn.sendRequest(".");
+    }
+    EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC))
+        .WillOnce(Return(conn));
+
+    auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, message_metadata);
+    ASSERT_FALSE(sending_res.ok());
+    HTTPResponse http_res = sending_res.getErr();
+    EXPECT_EQ(http_res.getBody(), "The connection is suspended due to consecutive message sending errors.");
+    EXPECT_EQ(http_res.getHTTPStatusCode(), HTTPStatusCode::HTTP_SUSPEND);
+}
+
+TEST_F(TestMessagingComp, testUploadFile)
+{
+    string path = cptestFnameInSrcDir("tests_files/file_to_send.txt");
+
+    setAgentDetails();
+    string uri = "/test-uri";
+    MessageCategory category = MessageCategory::GENERIC;
+    MessageMetadata message_metadata;
+
+    MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC);
+    Connection conn(conn_key, message_metadata);
+    EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC))
+        .WillOnce(Return(conn));
+
+    HTTPResponse res(HTTPStatusCode::HTTP_OK, "");
+    EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res));
+    auto upload_res = messaging_comp.uploadFile(uri, path, category, message_metadata);
+    ASSERT_TRUE(upload_res.ok());
+    EXPECT_EQ(upload_res.unpack(), HTTPStatusCode::HTTP_OK);
+}
+
+TEST_F(TestMessagingComp, testDownloadFile)
+{
+    string path = cptestFnameInSrcDir("tests_files/file_to_send.txt");
+
+    setAgentDetails();
+    string uri = "/test-uri";
+    HTTPMethod method = HTTPMethod::GET;
+    MessageCategory category = MessageCategory::GENERIC;
+    MessageMetadata message_metadata;
+
+    MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC);
+    Connection conn(conn_key, message_metadata);
+    EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC))
+        .WillOnce(Return(conn));
+
+    HTTPResponse res(HTTPStatusCode::HTTP_OK, "");
+    EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res));
+    auto upload_res = messaging_comp.downloadFile(method, uri, "/tmp/test.txt", category, message_metadata);
+    ASSERT_TRUE(upload_res.ok());
+    EXPECT_EQ(upload_res.unpack(), HTTPStatusCode::HTTP_OK);
+}
+
+bool
+operator==(const MessageProxySettings &one, const MessageProxySettings &two)
+{
+    return
+        one.getProxyHost() == two.getProxyHost() &&
+        one.getProxyAuth() == two.getProxyAuth() &&
+        one.getProxyPort() == two.getProxyPort();
+}
+
+bool
+operator==(const MessageMetadata &one, const MessageMetadata &two)
+{
+    return
+        one.getHostName() == two.getHostName() &&
+        one.getPort() == two.getPort() &&
+        one.getConnectionFlags() == two.getConnectionFlags() &&
+        one.getProxySettings() == two.getProxySettings() &&
+        one.getExternalCertificate() == two.getExternalCertificate() &&
+        one.getHeaders() == two.getHeaders() &&
+        one.shouldBufferMessage() == two.shouldBufferMessage() &&
+        one.isProxySet() == two.isProxySet();
+}
+
+TEST_F(TestMessagingComp, testSetFogConnection)
+{
+    setAgentDetails();
+
+    MessageCategory category = MessageCategory::GENERIC;
+    MessageConnectionKey conn_key(fog_addr, fog_port, category);
+    MessageMetadata metadata(fog_addr, fog_port, true);
+    MessageProxySettings proxy_settings("7.7.7.7", "cred", 8080);
+    metadata.setProxySettings(proxy_settings);
+    Connection conn(conn_key, metadata);
+
+    EXPECT_CALL(mock_messaging_connection, establishConnection(metadata, category)).WillOnce(Return(conn));
+    EXPECT_TRUE(messaging_comp.setFogConnection(category));
+}

core/messaging/messaging_comp/messaging_comp_ut/tests_files/file_to_send.txt

@@ -0,0 +1 @@
+file to send