From 6db87fc7fe4e1831873e705a9e30f140d2dc248e Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Mon, 13 Jan 2025 12:35:42 +0000 Subject: [PATCH 1/6] central nginx manager --- components/CMakeLists.txt | 1 + components/include/waap.h | 4 +- components/security_apps/CMakeLists.txt | 1 + .../health_check_manager.cc | 6 +- .../modules/orchestration_status.cc | 6 +- .../orchestration/orchestration_comp.cc | 37 +++++++---- .../orchestration_tools.cc | 2 +- .../package_handler/package_handler.cc | 4 +- .../update_communication/fog_authenticator.cc | 4 +- components/utils/CMakeLists.txt | 2 + components/utils/pm/debugpm.cc | 2 +- core/CMakeLists.txt | 2 +- .../agent_core_utilities.cc | 29 +++++++++ .../agent_core_utilities_ut.cc | 24 +++++++ core/agent_details/agent_details.cc | 18 ++++-- core/buffers/buffer.cc | 64 +++++++++++++------ core/compression/CMakeLists.txt | 3 + core/connkey/connkey.cc | 6 +- core/debug_is/debug.cc | 20 +++++- core/debug_is/debug_streams.cc | 8 ++- core/environment/context.cc | 7 +- core/environment/span.cc | 4 +- core/include/general/debug.h | 10 ++- .../services_sdk/interfaces/i_rest_api.h | 4 ++ .../interfaces/mock/mock_rest_api.h | 5 ++ .../services_sdk/resources/debug_flags.h | 1 + .../resources/report/base_field.h | 2 +- .../resources/report/report_bulks.h | 5 +- .../utilities/agent_core_utilities.h | 6 +- core/include/services_sdk/utilities/connkey.h | 18 ++++-- core/logging/cef_stream.cc | 9 ++- core/logging/log_streams.h | 2 + core/logging/logging_ut/logging_ut.cc | 20 ++++++ core/logging/syslog_stream.cc | 9 ++- core/mainloop/coroutine.cc | 10 +-- core/mainloop/mainloop.cc | 30 +++++++-- .../messaging/messaging_comp/http_response.cc | 11 ++-- core/report/tag_and_enum_management.cc | 36 +++++------ core/rest/rest_server.cc | 42 +++++++++--- core/rest/rest_ut/CMakeLists.txt | 2 +- core/rest/rest_ut/rest_config_ut.cc | 24 +++++++ core/socket_is/socket_is.cc | 8 ++- core/time_proxy/time_proxy.cc | 9 ++- nodes/CMakeLists.txt | 1 + unit_test.cmake | 2 +- 45 files changed, 390 insertions(+), 130 deletions(-) diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index 58aba03..8dcb528 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -7,3 +7,4 @@ add_subdirectory(pending_key) add_subdirectory(utils) add_subdirectory(attachment-intakers) add_subdirectory(security_apps) +add_subdirectory(nginx_message_reader) diff --git a/components/include/waap.h b/components/include/waap.h index 02f3983..4df31ba 100755 --- a/components/include/waap.h +++ b/components/include/waap.h @@ -33,7 +33,6 @@ class I_WaapAssetStatesManager; class I_Messaging; class I_AgentDetails; class I_Encryptor; -class I_WaapModelResultLogger; const std::string WAAP_APPLICATION_NAME = "waap application"; @@ -51,8 +50,7 @@ class WaapComponent Singleton::Consume, Singleton::Consume, Singleton::Consume, - Singleton::Consume, - Singleton::Consume + Singleton::Consume { public: WaapComponent(); diff --git a/components/security_apps/CMakeLists.txt b/components/security_apps/CMakeLists.txt index d9dd887..206262c 100644 --- a/components/security_apps/CMakeLists.txt +++ b/components/security_apps/CMakeLists.txt @@ -5,3 +5,4 @@ add_subdirectory(local_policy_mgmt_gen) add_subdirectory(orchestration) add_subdirectory(rate_limit) add_subdirectory(waap) +add_subdirectory(central_nginx_manager) diff --git a/components/security_apps/orchestration/health_check_manager/health_check_manager.cc b/components/security_apps/orchestration/health_check_manager/health_check_manager.cc index a683258..b53d87c 100644 --- a/components/security_apps/orchestration/health_check_manager/health_check_manager.cc +++ b/components/security_apps/orchestration/health_check_manager/health_check_manager.cc @@ -266,10 +266,10 @@ private: case OrchestrationStatusFieldType::COUNT : return "Count"; } - dbgAssert(false) + dbgAssertOpt(false) << AlertInfo(AlertTeam::CORE, "orchestration health") << "Trying to convert unknown orchestration status field to string."; - return ""; + return "Unknown Field"; } HealthCheckStatus @@ -282,7 +282,7 @@ private: case UpdatesProcessResult::DEGRADED : return HealthCheckStatus::DEGRADED; } - dbgAssert(false) + dbgAssertOpt(false) << AlertInfo(AlertTeam::CORE, "orchestration health") << "Trying to convert unknown update process result field to health check status."; return HealthCheckStatus::IGNORED; diff --git a/components/security_apps/orchestration/modules/orchestration_status.cc b/components/security_apps/orchestration/modules/orchestration_status.cc index cbcdbe0..f2fae7d 100755 --- a/components/security_apps/orchestration/modules/orchestration_status.cc +++ b/components/security_apps/orchestration/modules/orchestration_status.cc @@ -429,7 +429,7 @@ public: status.insertServiceSetting(service_name, path); return; case OrchestrationStatusConfigType::MANIFEST: - dbgAssert(false) + dbgAssertOpt(false) << AlertInfo(AlertTeam::CORE, "sesrvice configuration") << "Manifest is not a service configuration file type"; break; @@ -438,7 +438,9 @@ public: case OrchestrationStatusConfigType::COUNT: break; } - dbgAssert(false) << AlertInfo(AlertTeam::CORE, "sesrvice configuration") << "Unknown configuration file type"; + dbgAssertOpt(false) + << AlertInfo(AlertTeam::CORE, "service configuration") + << "Unknown configuration file type"; } void diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 5e3db9c..6774c4e 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -1587,6 +1587,7 @@ private: } setDelayedUpgradeTime(); + while (true) { Singleton::Consume::by()->startNewTrace(false); if (shouldReportAgentDetailsMetadata()) { @@ -1695,13 +1696,19 @@ private: auto backup_installation_file = current_installation_file + backup_ext; auto temp_ext = getConfigurationWithDefault("_temp", "orchestration", "Temp file extension"); - dbgAssert(i_orchestration_tools->doesFileExist(backup_installation_file)) - << AlertInfo(AlertTeam::CORE, "orchestration backup") - << "There is no backup installation package"; + if (!i_orchestration_tools->doesFileExist(backup_installation_file)) { + dbgAssertOpt(false) + << AlertInfo(AlertTeam::CORE, "orchestration backup") + << "There is no backup installation package"; + return; + } - dbgAssert(i_orchestration_tools->copyFile(backup_installation_file, current_installation_file)) - << AlertInfo(AlertTeam::CORE, "orchestration backup") - << "Failed to copy backup installation package"; + if (!i_orchestration_tools->copyFile(backup_installation_file, current_installation_file)) { + dbgAssertOpt(false) + << AlertInfo(AlertTeam::CORE, "orchestration backup") + << "Failed to copy backup installation package"; + return; + } // Copy the backup manifest file to the default manifest file path. auto manifest_file_path = getConfigurationWithDefault( @@ -1716,12 +1723,18 @@ private: auto package_handler = Singleton::Consume::by(); // Install the backup orchestration service installation package. - dbgAssert(package_handler->preInstallPackage(service_name, current_installation_file)) - << AlertInfo(AlertTeam::CORE, "orchestration backup") - << "Failed to restore from backup, pre install test failed"; - dbgAssert(package_handler->installPackage(service_name, current_installation_file, true)) - << AlertInfo(AlertTeam::CORE, "orchestration backup") - << "Failed to restore from backup, installation failed"; + if (!package_handler->preInstallPackage(service_name, current_installation_file)) { + dbgAssertOpt(false) + << AlertInfo(AlertTeam::CORE, "orchestration backup") + << "Failed to restore from backup, pre install test failed"; + return; + } + if (!package_handler->installPackage(service_name, current_installation_file, true)) { + dbgAssertOpt(false) + << AlertInfo(AlertTeam::CORE, "orchestration backup") + << "Failed to restore from backup, installation failed"; + return; + } } // LCOV_EXCL_STOP diff --git a/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc b/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc index 3efee4e..db58383 100755 --- a/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc +++ b/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc @@ -386,7 +386,7 @@ OrchestrationTools::Impl::calculateChecksum(Package::ChecksumTypes checksum_type return genError("Error while reading file " + path + ", " + e.what()); } - dbgAssert(false) + dbgAssertOpt(false) << AlertInfo(AlertTeam::CORE, "service configuration") << "Checksum type is not supported. Checksum type: " << static_cast(checksum_type); diff --git a/components/security_apps/orchestration/package_handler/package_handler.cc b/components/security_apps/orchestration/package_handler/package_handler.cc index 6071bd8..c397732 100755 --- a/components/security_apps/orchestration/package_handler/package_handler.cc +++ b/components/security_apps/orchestration/package_handler/package_handler.cc @@ -141,11 +141,11 @@ packageHandlerActionsToString(PackageHandlerActions action) } } - dbgAssert(false) + dbgAssertOpt(false) << AlertInfo(AlertTeam::CORE, "service configuration") << "Package handler action is not supported. Action: " << static_cast(action); - return string(); + return string("--UNSUPPORTED"); } void diff --git a/components/security_apps/orchestration/update_communication/fog_authenticator.cc b/components/security_apps/orchestration/update_communication/fog_authenticator.cc index e50601c..8ee56d4 100755 --- a/components/security_apps/orchestration/update_communication/fog_authenticator.cc +++ b/components/security_apps/orchestration/update_communication/fog_authenticator.cc @@ -467,9 +467,9 @@ getDeplymentType() case EnvType::COUNT: break; } - dbgAssert(false) + dbgAssertOpt(false) << AlertInfo(AlertTeam::CORE, "fog communication") - << "Failed to get a legitimate deplyment type: " + << "Failed to get a legitimate deployment type: " << static_cast(deplyment_type); return "Embedded"; } diff --git a/components/utils/CMakeLists.txt b/components/utils/CMakeLists.txt index 3e6baa7..999f821 100644 --- a/components/utils/CMakeLists.txt +++ b/components/utils/CMakeLists.txt @@ -5,3 +5,5 @@ add_subdirectory(ip_utilities) add_subdirectory(keywords) add_subdirectory(pm) add_subdirectory(service_health_status) +add_subdirectory(nginx_utils) +add_subdirectory(utilities) diff --git a/components/utils/pm/debugpm.cc b/components/utils/pm/debugpm.cc index 8d57da0..4bc15fd 100755 --- a/components/utils/pm/debugpm.cc +++ b/components/utils/pm/debugpm.cc @@ -46,7 +46,7 @@ panicCFmt(const string &func, uint line, const char *fmt, ...) { va_list va; va_start(va, fmt); - Debug("PM", func, line).getStreamAggr() << CFmtPrinter(fmt, va); + Debug("PM", func, line, true).getStreamAggr() << CFmtPrinter(fmt, va); va_end(va); } diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 0b3f0fe..38fb5f6 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -40,7 +40,7 @@ target_link_libraries( "table;debug_is;shell_cmd;metric;tenant_manager;messaging;encryptor;time_proxy;singleton;mainloop;environment;logging;report;rest" "compression_utils;-lz;config;intelligence_is_v2;event_is;memory_consumption;connkey" "instance_awareness;socket_is;agent_details;agent_details_reporter;buffers;cpu;agent_core_utilities" - "report_messaging;env_details" + "report_messaging;env_details;version" -Wl,-no-whole-archive ) diff --git a/core/agent_core_utilities/agent_core_utilities.cc b/core/agent_core_utilities/agent_core_utilities.cc index e863f9a..6c24616 100644 --- a/core/agent_core_utilities/agent_core_utilities.cc +++ b/core/agent_core_utilities/agent_core_utilities.cc @@ -203,6 +203,18 @@ deleteFile(const string &path) return true; } +string +resolveFullPath(const string &input_path) { + dbgTrace(D_INFRA_UTILS) << "Resolving absolute path: " << input_path; + char resolved_path[PATH_MAX]; + if (!realpath(input_path.c_str(), resolved_path)) { + dbgWarning(D_INFRA_UTILS) << "Error resolving path: " << input_path << ", errno: " << errno; + return ""; + } + + return string(resolved_path); +} + bool deleteDirectory(const string &path, bool delete_content) { @@ -510,6 +522,23 @@ removeTrailingWhitespaces(string str) return str; } +string +removeLeadingWhitespaces(string str) +{ + str.erase( + str.begin(), + find_if(str.begin(), str.end(), [] (char c) { return !isspace(c); }) + ); + + return str; +} + +string +trim(string str) +{ + return removeLeadingWhitespaces(removeTrailingWhitespaces(str)); +} + } // namespace Strings } // namespace NGEN diff --git a/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc b/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc index bf769b2..334cd56 100644 --- a/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc +++ b/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc @@ -184,3 +184,27 @@ TEST_F(AgentCoreUtilUT, removeTrailingWhitespacesTest) string str_with_trailing_whitespace = "str_with_trailing_whitespace\n\n\n\r \n\n\r"; EXPECT_EQ(NGEN::Strings::removeTrailingWhitespaces(str_with_trailing_whitespace), "str_with_trailing_whitespace"); } + +TEST_F(AgentCoreUtilUT, removeLeadingWhitespacesTest) +{ + string str_with_leading_whitespace = "\n\n\n\r \n\n\rstr_with_leading_whitespace"; + EXPECT_EQ(NGEN::Strings::removeLeadingWhitespaces(str_with_leading_whitespace), "str_with_leading_whitespace"); +} + +TEST_F(AgentCoreUtilUT, trimTest) +{ + string str_with_leading_and_trailing_whitespace = "\n\n \r \rstr_with_whitespace\n\r \n\n\r"; + EXPECT_EQ(NGEN::Strings::trim(str_with_leading_and_trailing_whitespace), "str_with_whitespace"); +} + +TEST_F(AgentCoreUtilUT, resolveFullPathTest) +{ + string working_dir = cptestFnameInExeDir(""); + ofstream file(working_dir + "test.txt"); + ASSERT_TRUE(file.is_open()); + file.close(); + string relative_path = "test.txt"; + string full_path = NGEN::Filesystem::resolveFullPath(relative_path); + EXPECT_EQ(full_path, working_dir + "test.txt"); + ASSERT_TRUE(NGEN::Filesystem::deleteFile(working_dir + "test.txt")); +} diff --git a/core/agent_details/agent_details.cc b/core/agent_details/agent_details.cc index 8d28ed2..e459644 100644 --- a/core/agent_details/agent_details.cc +++ b/core/agent_details/agent_details.cc @@ -388,8 +388,9 @@ AgentDetails::convertProxyProtocolToString(ProxyProtocol proto) const case ProxyProtocol::HTTP: return "http"; case ProxyProtocol::HTTPS: return "https"; } - dbgAssert(false) << alert << "Unsupported Proxy Protocol " << static_cast(proto); - return ""; + dbgAssertOpt(false) << alert << "Unsupported Proxy Protocol " << static_cast(proto); + dbgWarning(D_ORCHESTRATOR) << "Using https proxy as default"; + return "https"; } Maybe @@ -475,11 +476,14 @@ Maybe AgentDetails::loadProxyType(ProxyProtocol protocol) { dbgFlow(D_ORCHESTRATOR) << "Loading proxy type: " << convertProxyProtocolToString(protocol); - dbgAssert(protocol == ProxyProtocol::HTTP || protocol == ProxyProtocol::HTTPS) - << alert - << "Unsupported Proxy Protocol " - << static_cast(protocol); - + if (!(protocol == ProxyProtocol::HTTP || protocol == ProxyProtocol::HTTPS)) { + dbgAssertOpt(false) + << alert + << "Unsupported Proxy Protocol " + << static_cast(protocol); + protocol = ProxyProtocol::HTTPS; + dbgWarning(D_ORCHESTRATOR) << "Using https proxy as default"; + } static const map env_var_name = { {ProxyProtocol::HTTPS, "https_proxy"}, {ProxyProtocol::HTTP, "http_proxy"} diff --git a/core/buffers/buffer.cc b/core/buffers/buffer.cc index 081a7ee..37c16d1 100644 --- a/core/buffers/buffer.cc +++ b/core/buffers/buffer.cc @@ -144,8 +144,8 @@ Buffer::operator+(const Buffer &other) const Buffer Buffer::getSubBuffer(uint start, uint end) const { - dbgAssert(start<=end && end<=len) << alert << "Buffer::getSubBuffer() returned: Illegal scoping of buffer"; - if (start == end) return Buffer(); + dbgAssertOpt(start<=end && end<=len) << alert << "Buffer::getSubBuffer() returned: Illegal scoping of buffer"; + if (start >= end || end > len) return Buffer(); Buffer res; uint offset = 0; @@ -178,8 +178,12 @@ Buffer::getSubBuffer(uint start, uint end) const Maybe Buffer::findFirstOf(char ch, uint start) const { - dbgAssert(start <= len) << alert << "Buffer::findFirstOf() returned: Cannot set a start point after buffer's end"; - + if (start > len) { + dbgAssertOpt(start <= len) + << alert + << "Buffer::findFirstOf() returned: Cannot set a start point after buffer's end"; + return genError("Cannot set a start point after buffer's end"); + } for (; start < len; ++start) { if ((*this)[start] == ch) return start; } @@ -189,8 +193,12 @@ Buffer::findFirstOf(char ch, uint start) const Maybe Buffer::findFirstOf(const Buffer &buf, uint start) const { - dbgAssert(start <= len) << alert << "Buffer::findFirstOf() returned: Cannot set a start point after buffer's end"; - + if (start > len) { + dbgAssertOpt(start <= len) + << alert + << "Buffer::findFirstOf() returned: Cannot set a start point after buffer's end"; + return genError("Cannot set a start point after buffer's end"); + } for (; start + buf.size() <= len; ++start) { auto sub_buffer = getSubBuffer(start, start + buf.size()); if (sub_buffer == buf) return start; @@ -201,9 +209,13 @@ Buffer::findFirstOf(const Buffer &buf, uint start) const Maybe Buffer::findFirstNotOf(char ch, uint start) const { - dbgAssert(start <= len) - << alert - << "Buffer::findFirstNotOf() returned: Cannot set a start point after buffer's end"; + if (start > len) { + dbgAssertOpt(start <= len) + << alert + << "Buffer::findFirstNotOf() returned: Cannot set a start point after buffer's end"; + return genError("Cannot set a start point after buffer's end"); + } + for (; start < len; ++start) { if ((*this)[start] != ch) return start; } @@ -213,7 +225,12 @@ Buffer::findFirstNotOf(char ch, uint start) const Maybe Buffer::findLastOf(char ch, uint start) const { - dbgAssert(start <= len) << alert << "Buffer::findLastOf() returned: Cannot set a start point after buffer's end"; + if (start > len) { + dbgAssertOpt(start <= len) + << alert + << "Buffer::findLastOf() returned: Cannot set a start point after buffer's end"; + return genError("Cannot set a start point after buffer's end"); + } for (; 0 < start; --start) { if ((*this)[start - 1] == ch) return start - 1; } @@ -223,9 +240,12 @@ Buffer::findLastOf(char ch, uint start) const Maybe Buffer::findLastNotOf(char ch, uint start) const { - dbgAssert(start <= len) - << alert - << "Buffer::findLastNotOf() returned: Cannot set a start point after buffer's end"; + if (start > len) { + dbgAssertOpt(start <= len) + << alert + << "Buffer::findLastNotOf() returned: Cannot set a start point after buffer's end"; + return genError("Cannot set a start point after buffer's end"); + } for (; 0 < start; --start) { if ((*this)[start - 1] != ch) return start - 1; } @@ -235,8 +255,8 @@ Buffer::findLastNotOf(char ch, uint start) const void Buffer::truncateHead(uint size) { - dbgAssert(size <= len) << alert << "Cannot set a new start of buffer after the buffer's end"; - if (size == 0) return; + dbgAssertOpt(size <= len) << alert << "Cannot set a new start of buffer after the buffer's end"; + if (size == 0 || size > len) return; if (size == len) { clear(); return; @@ -261,8 +281,8 @@ Buffer::truncateHead(uint size) void Buffer::truncateTail(uint size) { - dbgAssert(size <= len) << alert << "Cannot set a new end of buffer after the buffer's end"; - if (size == 0) return; + dbgAssertOpt(size <= len) << alert << "Cannot set a new end of buffer after the buffer's end"; + if (size == 0 || size > len) return; if (size == len) { clear(); return; @@ -285,14 +305,20 @@ Buffer::truncateTail(uint size) void Buffer::keepHead(uint size) { - dbgAssert(size <= len) << alert << "Cannot set a new end of buffer before the buffer's start"; + if (size > len) { + dbgAssertOpt(size <= len) << alert << "Cannot set a new end of buffer before the buffer's start"; + return; + } truncateTail(len - size); } void Buffer::keepTail(uint size) { - dbgAssert(size <= len) << alert << "Cannot set a new start of buffer after the buffer's end"; + if (size > len) { + dbgAssertOpt(size <= len) << alert << "Cannot set a new start of buffer after the buffer's end"; + return; + } truncateHead(len - size); } diff --git a/core/compression/CMakeLists.txt b/core/compression/CMakeLists.txt index 490c86d..f9b5a18 100755 --- a/core/compression/CMakeLists.txt +++ b/core/compression/CMakeLists.txt @@ -2,8 +2,11 @@ include_directories(${ng_module_osrc_zlib_path}/include) add_definitions(-DZLIB_CONST) add_library(compression_utils SHARED compression_utils.cc) +add_library(static_compression_utils compression_utils.cc) add_subdirectory(compression_utils_ut) install(TARGETS compression_utils DESTINATION lib) install(TARGETS compression_utils DESTINATION http_transaction_handler_service/lib) + +install(TARGETS static_compression_utils DESTINATION lib) diff --git a/core/connkey/connkey.cc b/core/connkey/connkey.cc index 7788ca6..a70ee81 100644 --- a/core/connkey/connkey.cc +++ b/core/connkey/connkey.cc @@ -64,12 +64,12 @@ IPAddr::print(ostream &os) const switch (type) { case IPType::V4: { formatted_addr = inet_ntop(AF_INET, &v4, buf, sizeof(buf)); - dbgAssert(formatted_addr == buf) << alert("conversion error") << "Failed to convert an IPv4 address"; + dbgAssertOpt(formatted_addr == buf) << alert("conversion error") << "Failed to convert an IPv4 address"; break; } case IPType::V6: { formatted_addr = inet_ntop(AF_INET6, &v6, buf, sizeof(buf)); - dbgAssert(formatted_addr == buf) << alert("conversion error") << "Failed to convert an IPv6 address"; + dbgAssertOpt(formatted_addr == buf) << alert("conversion error") << "Failed to convert an IPv6 address"; break; } case IPType::UNINITIALIZED: { @@ -116,7 +116,7 @@ ConnKey::reverse() size_t ConnKey::hash() const { - dbgAssert(src.type != IPType::UNINITIALIZED) + dbgAssertOpt(src.type != IPType::UNINITIALIZED) << alert("hashing") << "ConnKey::hash was called on an uninitialized object"; size_t seed = 0; diff --git a/core/debug_is/debug.cc b/core/debug_is/debug.cc index 00dacd5..3588853 100644 --- a/core/debug_is/debug.cc +++ b/core/debug_is/debug.cc @@ -27,6 +27,7 @@ #include "i_instance_awareness.h" #include "i_signal_handler.h" #include "hash_combine.h" +#include "version.h" using namespace std; @@ -298,14 +299,19 @@ AlertInfo::evalParams() Debug::Debug( const string &file_name, const string &func_name, - const uint &line) + const uint &line, + bool force_assert) { - if (Singleton::exists()) { - do_assert = getConfigurationWithDefault(true, "Debug I/S", "Abort on assertion"); + if (!force_assert && !should_assert_optional) { + do_assert = false; } else { do_assert = true; } + if (Singleton::exists()) { + do_assert = getConfigurationWithDefault(do_assert, "Debug I/S", "Abort on assertion"); + } + auto current_configuration = Singleton::exists() ? getConfigurationWithDefault(default_config, "Debug") : default_config; @@ -519,6 +525,13 @@ Debug::preload() active_streams["STDOUT"] = make_shared(&cout); active_streams["FOG"] = make_shared(); + + string branch = Version::getBranch(); + if (branch == "master" || branch.substr(0, 6) == "hotfix") { + should_assert_optional = false; + } else { + should_assert_optional = true; + } } void @@ -844,3 +857,4 @@ bool Debug::is_fail_open_mode = false; bool Debug::debug_override_exist = false; string Debug::default_debug_file_stream_path = ""; vector Debug::streams_from_mgmt; +bool Debug::should_assert_optional = true; diff --git a/core/debug_is/debug_streams.cc b/core/debug_is/debug_streams.cc index b28c235..949efa3 100644 --- a/core/debug_is/debug_streams.cc +++ b/core/debug_is/debug_streams.cc @@ -396,14 +396,18 @@ LogLevel DebugFogStream::getLogLevel() const { switch (level) { - case Debug::DebugLevel::NOISE: dbgAssert(false) << alert << "Impossible LogLevel 'Noise'"; break; + case Debug::DebugLevel::NOISE: + dbgAssertOpt(false) << alert << "Impossible LogLevel 'Noise'"; + return LogLevel::TRACE; case Debug::DebugLevel::TRACE: return LogLevel::TRACE; case Debug::DebugLevel::DEBUG: return LogLevel::DEBUG; case Debug::DebugLevel::WARNING: return LogLevel::WARNING; case Debug::DebugLevel::INFO: return LogLevel::INFO; case Debug::DebugLevel::ERROR: return LogLevel::ERROR; case Debug::DebugLevel::ASSERTION: return LogLevel::ERROR; - case Debug::DebugLevel::NONE: dbgAssert(false) << alert << "Impossible LogLevel 'None'"; break; + case Debug::DebugLevel::NONE: + dbgAssertOpt(false) << alert << "Impossible LogLevel 'None'"; + return LogLevel::ERROR; } return LogLevel::INFO; diff --git a/core/environment/context.cc b/core/environment/context.cc index c34d44c..303ed7d 100644 --- a/core/environment/context.cc +++ b/core/environment/context.cc @@ -60,10 +60,11 @@ Context::convertToString(MetaDataType type) case MetaDataType::Direction: return "direction"; case MetaDataType::Email: return "email"; case MetaDataType::COUNT: - dbgAssert(false) << alert << "COUNT is not a valid meta data type"; + dbgAssertOpt(false) << alert << "COUNT is not a valid meta data type"; + return "invalid_count"; } - dbgAssert(false) << alert << "Reached impossible case with type=" << static_cast(type); - return ""; + dbgAssertOpt(false) << alert << "Reached impossible case with type=" << static_cast(type); + return "invalid_metadata_type"; } map diff --git a/core/environment/span.cc b/core/environment/span.cc index 92915be..180dbab 100644 --- a/core/environment/span.cc +++ b/core/environment/span.cc @@ -97,8 +97,8 @@ Span::convertSpanContextTypeToString(ContextType type) return "Follows from"; } } - dbgAssert(false) << AlertInfo(AlertTeam::CORE, "tracing") << "Span context not supported"; - return string(); + dbgAssertOpt(false) << AlertInfo(AlertTeam::CORE, "tracing") << "Span context not supported"; + return "Invalid context type"; } SpanWrapper::SpanWrapper(string _trace_id, Span::ContextType _type, string _prev_span) diff --git a/core/include/general/debug.h b/core/include/general/debug.h index a1c1153..141caf4 100644 --- a/core/include/general/debug.h +++ b/core/include/general/debug.h @@ -159,7 +159,8 @@ public: Debug( const std::string &file_name, const std::string &func_name, - const uint &line + const uint &line, + bool force_assert ); Debug( @@ -273,6 +274,7 @@ private: static bool debug_override_exist; static std::string default_debug_file_stream_path; static std::vector streams_from_mgmt; + static bool should_assert_optional; bool do_assert; bool is_communication = false; @@ -328,7 +330,11 @@ getBaseName(const char *iter, const char *base) #define dbgAssert(cond) \ if (CP_LIKELY(cond)) { \ - } else Debug::DebugAlert(__FILENAME__, __FUNCTION__, __LINE__).getStreamAggr() + } else Debug::DebugAlert(__FILENAME__, __FUNCTION__, __LINE__, true).getStreamAggr() + +#define dbgAssertOpt(cond) \ + if (CP_LIKELY(cond)) { \ + } else Debug::DebugAlert(__FILENAME__, __FUNCTION__, __LINE__, false).getStreamAggr() // Macros to allow simple debug messaging #define DBG_GENERIC(level, ...) \ diff --git a/core/include/services_sdk/interfaces/i_rest_api.h b/core/include/services_sdk/interfaces/i_rest_api.h index ca5e0af..b968057 100644 --- a/core/include/services_sdk/interfaces/i_rest_api.h +++ b/core/include/services_sdk/interfaces/i_rest_api.h @@ -49,6 +49,10 @@ public: } virtual bool addGetCall(const std::string &uri, const std::function &callback) = 0; + virtual bool addWildcardGetCall( + const std::string &uri, + const std::function &callback + ) = 0; virtual uint16_t getListeningPort() const = 0; diff --git a/core/include/services_sdk/interfaces/mock/mock_rest_api.h b/core/include/services_sdk/interfaces/mock/mock_rest_api.h index 5cd9887..7cb9f3f 100644 --- a/core/include/services_sdk/interfaces/mock/mock_rest_api.h +++ b/core/include/services_sdk/interfaces/mock/mock_rest_api.h @@ -10,6 +10,11 @@ class MockRestApi : public Singleton::Provide::From &)); + MOCK_METHOD2( + addWildcardGetCall, + bool(const std::string &, const std::function &) + ); + // You can't mock a function with an R-value reference. So mock a slightly different one MOCK_METHOD3(mockRestCall, bool(RestAction, const std::string &, const std::unique_ptr &)); diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index 4fb78fb..67d164c 100644 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -114,6 +114,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_FILE_UPLOAD, D_COMPONENT) DEFINE_FLAG(D_RATE_LIMIT, D_COMPONENT) DEFINE_FLAG(D_ROLLBACK_TESTING, D_COMPONENT) + DEFINE_FLAG(D_NGINX_MANAGER, D_COMPONENT) DEFINE_FLAG(D_PARSER, D_COMPONENT) DEFINE_FLAG(D_WS, D_COMPONENT) diff --git a/core/include/services_sdk/resources/report/base_field.h b/core/include/services_sdk/resources/report/base_field.h index bf33788..67ebfa1 100644 --- a/core/include/services_sdk/resources/report/base_field.h +++ b/core/include/services_sdk/resources/report/base_field.h @@ -162,7 +162,7 @@ class LogField : Singleton::Consume void addFields(const LogField &) { - dbgAssert(false) + dbgAssertOpt(false) << AlertInfo(AlertTeam::CORE, "report i/s") << "Trying to add a log field to a 'type'ed field"; } diff --git a/core/include/services_sdk/resources/report/report_bulks.h b/core/include/services_sdk/resources/report/report_bulks.h index 61515cb..b8274f7 100644 --- a/core/include/services_sdk/resources/report/report_bulks.h +++ b/core/include/services_sdk/resources/report/report_bulks.h @@ -26,7 +26,10 @@ public: void setBulkSize(uint size) { - dbgAssert(size > 0) << AlertInfo(AlertTeam::CORE, "report i/s") << "Bulk size must be larger than 0"; + if (size <= 0) { + dbgAssertOpt(size > 0) << AlertInfo(AlertTeam::CORE, "report i/s") << "Bulk size must be larger than 0"; + size = 100; + } dbgDebug(D_REPORT_BULK) << "Bulk size is set to " << size; bulk_size = size; } diff --git a/core/include/services_sdk/utilities/agent_core_utilities.h b/core/include/services_sdk/utilities/agent_core_utilities.h index 446230a..e6a8e5c 100644 --- a/core/include/services_sdk/utilities/agent_core_utilities.h +++ b/core/include/services_sdk/utilities/agent_core_utilities.h @@ -33,6 +33,7 @@ bool makeDir(const std::string &path, mode_t permission = S_IRWXU); bool makeDirRecursive(const std::string &path, mode_t permission = S_IRWXU); bool deleteDirectory(const std::string &path, bool delete_content = false); bool touchFile(const std::string &path); +std::string resolveFullPath(const std::string &input_path); bool copyFile( @@ -43,11 +44,8 @@ copyFile( ); bool deleteFile(const std::string &path); - std::string convertToHumanReadable(uint64_t size_in_bytes); - std::string getFileName(const std::string &path); - bool copyDirectory(const std::string &src_dir_path, const std::string &dst_dir_path); }// namespace Filesystem @@ -85,6 +83,8 @@ namespace Strings { std::string removeTrailingWhitespaces(std::string str); +std::string removeLeadingWhitespaces(std::string str); +std::string trim(std::string str); } // namespace Strings diff --git a/core/include/services_sdk/utilities/connkey.h b/core/include/services_sdk/utilities/connkey.h index e0d3532..2a10af3 100644 --- a/core/include/services_sdk/utilities/connkey.h +++ b/core/include/services_sdk/utilities/connkey.h @@ -87,9 +87,12 @@ public: bool operator==(const IPAddr &other) const { - dbgAssert(type!=IPType::UNINITIALIZED && other.type!=IPType::UNINITIALIZED) - << AlertInfo(AlertTeam::CORE, "connkey") - << "Called on an uninitialized IPType object"; + if (type == IPType::UNINITIALIZED || other.type == IPType::UNINITIALIZED) { + dbgAssertOpt(type!=IPType::UNINITIALIZED && other.type!=IPType::UNINITIALIZED) + << AlertInfo(AlertTeam::CORE, "connkey") + << "Called on an uninitialized IPType object"; + return false; + } // Always compairing as if IPv6, in case of Ipv4 the rest of the address is zeroed out. int ip_len = (other.type == IPType::V4) ? sizeof(v4.s_addr) : sizeof(v6.s6_addr); return (type == other.type) && (memcmp(v6.s6_addr, other.v6.s6_addr, ip_len) == 0); @@ -308,9 +311,12 @@ public: IPType getType() const { - dbgAssert(src.type == dst.type) - << AlertInfo(AlertTeam::CORE, "connkey") - << "Mismatch in connection types (Src and Dst types are not identical)"; + if (src.type != dst.type) { + dbgAssertOpt(src.type == dst.type) + << AlertInfo(AlertTeam::CORE, "connkey") + << "Mismatch in connection types (Src and Dst types are not identical)"; + return IPType::V6; + } return src.type; } diff --git a/core/logging/cef_stream.cc b/core/logging/cef_stream.cc index 1c81ec5..0cbfb3f 100644 --- a/core/logging/cef_stream.cc +++ b/core/logging/cef_stream.cc @@ -59,7 +59,14 @@ CefStream::sendLog(const Report &log) void CefStream::init() { updateSettings(); - maintainConnection(); + mainloop->addOneTimeRoutine( + I_MainLoop::RoutineType::Offline, + [this] () + { + dbgTrace(D_REPORT) << FIRST_CEF_CONNECT_NAME; + }, + FIRST_CEF_CONNECT_NAME + ); auto ceflog_retry_interval = getProfileAgentSettingWithDefault( RETRY_CONNECT_INTERVAL, diff --git a/core/logging/log_streams.h b/core/logging/log_streams.h index a0766b5..560309d 100644 --- a/core/logging/log_streams.h +++ b/core/logging/log_streams.h @@ -26,7 +26,9 @@ #include "logging_comp.h" static const int RETRY_CONNECT_INTERVAL = 120; +static const std::string FIRST_SYSLOG_CONNECT_NAME = "first connecting to Syslog server"; static const std::string SYSLOG_CONNECT_NAME = "connecting to Syslog server"; +static const std::string FIRST_CEF_CONNECT_NAME = "first connecting to CEF server"; static const std::string CEF_CONNECT_NAME = "connecting to CEF server"; static const int NUMBER_OF_LOGS_PER_SEND = 15; static size_t MAX_LOG_QUEUE = 1000; diff --git a/core/logging/logging_ut/logging_ut.cc b/core/logging/logging_ut/logging_ut.cc index 5861f48..b65c4f1 100644 --- a/core/logging/logging_ut/logging_ut.cc +++ b/core/logging/logging_ut/logging_ut.cc @@ -134,6 +134,16 @@ public: DoAll(SaveArg<1>(&sysog_routine), Return(0)) ); + EXPECT_CALL( + mock_mainloop, + addOneTimeRoutine(_, _, "first connecting to Syslog server", _) + ).WillRepeatedly(DoAll(SaveArg<1>(&first_connect_syslog_routine), Return(0))); + + EXPECT_CALL( + mock_mainloop, + addOneTimeRoutine(_, _, "first connecting to CEF server", _) + ).WillRepeatedly(DoAll(SaveArg<1>(&first_connect_cef_routine), Return(0))); + EXPECT_CALL( mock_mainloop, addRecurringRoutine(_, _, _, "connecting to Syslog server", _) @@ -303,6 +313,8 @@ public: ConfigComponent config; vector capture_syslog_cef_data; I_MainLoop::Routine sysog_routine = nullptr; + I_MainLoop::Routine first_connect_syslog_routine = nullptr; + I_MainLoop::Routine first_connect_cef_routine = nullptr; I_MainLoop::Routine connect_syslog_routine = nullptr; I_MainLoop::Routine connect_cef_routine = nullptr; StrictMock mock_shell_cmd; @@ -1517,6 +1529,8 @@ TEST_F(LogTest, ObfuscationCefSysLogTest) I_Socket::SocketType protocol = I_Socket::SocketType::TCP; // for cef CefStream cef_stream(address, port, protocol); + ASSERT_NE(first_connect_cef_routine, nullptr); + first_connect_cef_routine(); ASSERT_NE(connect_cef_routine, nullptr); connect_cef_routine(); cef_stream.sendLog(CreateReport(tag1, tag2)); @@ -1525,6 +1539,8 @@ TEST_F(LogTest, ObfuscationCefSysLogTest) SyslogStream syslog_stream(address, port, protocol); // connection to socket before send log + ASSERT_NE(first_connect_syslog_routine, nullptr); + first_connect_syslog_routine(); ASSERT_NE(connect_syslog_routine, nullptr); connect_syslog_routine(); @@ -1554,6 +1570,8 @@ TEST_F(LogTest, SysLogWriteFailTest) I_Socket::SocketType protocol = I_Socket::SocketType::TCP; SyslogStream syslog_stream(address, port, protocol); + ASSERT_NE(first_connect_syslog_routine, nullptr); + first_connect_syslog_routine(); ASSERT_NE(connect_syslog_routine, nullptr); connect_syslog_routine(); @@ -1599,6 +1617,8 @@ TEST_F(LogTest, CefWriteFailTest) I_Socket::SocketType protocol = I_Socket::SocketType::TCP; CefStream cef_stream(address, port, protocol); + ASSERT_NE(first_connect_cef_routine, nullptr); + first_connect_cef_routine(); ASSERT_NE(connect_cef_routine, nullptr); connect_cef_routine(); diff --git a/core/logging/syslog_stream.cc b/core/logging/syslog_stream.cc index 23128db..1fc675e 100644 --- a/core/logging/syslog_stream.cc +++ b/core/logging/syslog_stream.cc @@ -71,7 +71,14 @@ SyslogStream::sendLog(const vector &data) void SyslogStream::init() { updateSettings(); - maintainConnection(); + mainloop->addOneTimeRoutine( + I_MainLoop::RoutineType::Offline, + [this] () + { + dbgTrace(D_REPORT) << FIRST_SYSLOG_CONNECT_NAME; + }, + FIRST_SYSLOG_CONNECT_NAME + ); auto syslog_retry_interval = getProfileAgentSettingWithDefault( RETRY_CONNECT_INTERVAL, diff --git a/core/mainloop/coroutine.cc b/core/mainloop/coroutine.cc index c73dd6e..20ddcae 100644 --- a/core/mainloop/coroutine.cc +++ b/core/mainloop/coroutine.cc @@ -76,10 +76,12 @@ RoutineWrapper::resume() void RoutineWrapper::invoke(pull_type &pull, I_MainLoop::Routine func) { - dbgAssert(active != nullptr) - << AlertInfo(AlertTeam::CORE, "mainloop i/s") - << "Trying to invoke without an active routine"; - + if (!active) { + dbgAssertOpt(active != nullptr) + << AlertInfo(AlertTeam::CORE, "mainloop i/s") + << "Trying to invoke without an active routine"; + return; + } active->pull = move(pull); // First invokation (other invokaction will start inside `func`), set the `pull` object func(); } diff --git a/core/mainloop/mainloop.cc b/core/mainloop/mainloop.cc index 36630b7..4302db5 100644 --- a/core/mainloop/mainloop.cc +++ b/core/mainloop/mainloop.cc @@ -227,7 +227,10 @@ MainloopComponent::Impl::reportStartupEvent() void MainloopComponent::Impl::run() { - dbgAssert(!is_running) << alert << "MainloopComponent::Impl::run was called while it was already running"; + if (is_running) { + dbgAssertOpt(!is_running) << alert << "MainloopComponent::Impl::run was called while it was already running"; + return; + } is_running = true; bool has_primary_routines = true; @@ -467,7 +470,10 @@ MainloopComponent::Impl::getCurrentRoutineId() const void MainloopComponent::Impl::yield(bool force) { - dbgAssert(curr_iter != routines.end()) << alert << "Calling 'yield' without a running current routine"; + if (curr_iter == routines.end()) { + dbgAssertOpt(curr_iter != routines.end()) << alert << "Calling 'yield' without a running current routine"; + return; + } if (do_stop) throw MainloopStop(); if (!force && getTimer()->getMonotonicTime() < stop_time) return; @@ -508,7 +514,10 @@ MainloopComponent::Impl::stopAll() void MainloopComponent::Impl::stop() { - dbgAssert(curr_iter != routines.end()) << alert << "Attempting to stop a routine when none is running"; + if (curr_iter == routines.end()) { + dbgAssertOpt(curr_iter != routines.end()) << alert << "Attempting to stop a routine when none is running"; + return; + } stop(curr_iter); } @@ -526,7 +535,10 @@ MainloopComponent::Impl::stop(RoutineID id) void MainloopComponent::Impl::halt() { - dbgAssert(curr_iter != routines.end()) << alert << "Calling 'halt' without a running current routine"; + if (curr_iter == routines.end()) { + dbgAssertOpt(curr_iter != routines.end()) << alert << "Calling 'halt' without a running current routine"; + return; + } curr_iter->second.halt(); yield(true); } @@ -535,7 +547,10 @@ void MainloopComponent::Impl::halt(RoutineID id) { auto iter = routines.find(id); - dbgAssert(iter != routines.end()) << alert << "No routine " << id << " to halt"; + if (iter == routines.end()) { + dbgAssertOpt(iter != routines.end()) << alert << "No routine " << id << " to halt"; + return; + } iter->second.halt(); if (iter == curr_iter) yield(true); } @@ -544,7 +559,10 @@ void MainloopComponent::Impl::resume(RoutineID id) { auto iter = routines.find(id); - dbgAssert(iter != routines.end()) << alert << "No routine " << id << " to resume"; + if (iter == routines.end()) { + dbgAssertOpt(iter != routines.end()) << alert << "No routine " << id << " to resume"; + return; + } iter->second.resume(); } diff --git a/core/messaging/messaging_comp/http_response.cc b/core/messaging/messaging_comp/http_response.cc index babbc96..3907b29 100644 --- a/core/messaging/messaging_comp/http_response.cc +++ b/core/messaging/messaging_comp/http_response.cc @@ -94,10 +94,13 @@ string HTTPResponse::toString() const { auto code = status_code_to_string.find(status_code); - dbgAssert(code != status_code_to_string.end()) - << AlertInfo(AlertTeam::CORE, "messaging i/s") - << "Unknown status code " - << int(status_code); + if (code == status_code_to_string.end()) { + dbgAssertOpt(code != status_code_to_string.end()) + << AlertInfo(AlertTeam::CORE, "messaging i/s") + << "Unknown status code " + << int(status_code); + return "[Status-code]: 500 - HTTP_INTERNAL_SERVER_ERROR, [Body]: " + (body.empty() ? "{}" : body); + } return "[Status-code]: " + code->second + ", [Body]: " + (body.empty() ? "{}" : body); } diff --git a/core/report/tag_and_enum_management.cc b/core/report/tag_and_enum_management.cc index c6ee59f..1c711b0 100644 --- a/core/report/tag_and_enum_management.cc +++ b/core/report/tag_and_enum_management.cc @@ -160,8 +160,8 @@ TagAndEnumManagement::convertToString(const StreamType &stream_type) case StreamType::COUNT: break; } - dbgAssert(false) << alert << "Unknown log stream type. Type: " << static_cast(stream_type); - return ""; + dbgAssertOpt(false) << alert << "Unknown log stream type. Type: " << static_cast(stream_type); + return "Unknown stream"; } string @@ -175,8 +175,8 @@ TagAndEnumManagement::convertToString(const Severity &severity) case Severity::INFO: return "Info"; } - dbgAssert(false) << alert << "Reached an impossible severity value of: " << static_cast(severity); - return ""; + dbgAssertOpt(false) << alert << "Reached an impossible severity value of: " << static_cast(severity); + return "Unknown severity"; } string @@ -188,8 +188,8 @@ TagAndEnumManagement::convertToString(const Type &type) case Type::CODE: return "Code Related"; } - dbgAssert(false) << alert << "Reached an impossible type value of: " << static_cast(type); - return ""; + dbgAssertOpt(false) << alert << "Reached an impossible type value of: " << static_cast(type); + return "Unknown type"; } string @@ -203,8 +203,8 @@ TagAndEnumManagement::convertToString(const Level &level) case Level::CUSTOM: return "Custom"; } - dbgAssert(false) << alert << "Reached an impossible type value of: " << static_cast(level); - return ""; + dbgAssertOpt(false) << alert << "Reached an impossible type value of: " << static_cast(level); + return "Unknown Level"; } string @@ -218,8 +218,8 @@ TagAndEnumManagement::convertToString(const LogLevel &log_level) case LogLevel::ERROR: return "error"; } - dbgAssert(false) << alert << "Reached an impossible type value of: " << static_cast(log_level); - return ""; + dbgAssertOpt(false) << alert << "Reached an impossible type value of: " << static_cast(log_level); + return "Unknown log level"; } string @@ -230,8 +230,8 @@ TagAndEnumManagement::convertToString(const Audience &audience) case Audience::INTERNAL: return "Internal"; } - dbgAssert(false) << alert << "Reached an impossible audience value of: " << static_cast(audience); - return ""; + dbgAssertOpt(false) << alert << "Reached an impossible audience value of: " << static_cast(audience); + return "Unknown audience"; } string @@ -244,8 +244,8 @@ TagAndEnumManagement::convertToString(const Priority &priority) case Priority::LOW: return "Low"; } - dbgAssert(false) << alert << "Reached impossible priority value of: " << static_cast(priority); - return ""; + dbgAssertOpt(false) << alert << "Reached impossible priority value of: " << static_cast(priority); + return "Unknown priority"; } string @@ -263,8 +263,8 @@ TagAndEnumManagement::convertToString(const Notification ¬ification) case Notification::SDWAN_POLICY_WARNING_LOG: return "c58d490e-6aa0-43da-bfaa-7edad0a57b7a"; } - dbgAssert(false) << alert << "Reached impossible notification value of: " << static_cast(notification); - return ""; + dbgAssertOpt(false) << alert << "Reached impossible notification value of: " << static_cast(notification); + return "00000000-0000-0000-0000-000000000000"; } string @@ -281,8 +281,8 @@ TagAndEnumManagement::convertToString(const IssuingEngine &issuing_engine) case IssuingEngine::HORIZON_TELEMETRY_METRICS: return "horizonTelemetryMetrics"; } - dbgAssert(false) << alert << "Reached impossible engine value of: " << static_cast(issuing_engine); - return ""; + dbgAssertOpt(false) << alert << "Reached impossible engine value of: " << static_cast(issuing_engine); + return "Unknown Issuer"; } diff --git a/core/rest/rest_server.cc b/core/rest/rest_server.cc index 159953a..8f99d11 100644 --- a/core/rest/rest_server.cc +++ b/core/rest/rest_server.cc @@ -50,6 +50,7 @@ public: bool bindRestServerSocket(struct sockaddr_in6 &addr, vector port_range); bool addRestCall(RestAction oper, const string &uri, unique_ptr &&init) override; bool addGetCall(const string &uri, const function &cb) override; + bool addWildcardGetCall(const string &uri, const function &callback); uint16_t getListeningPort() const override { return listening_port; } Maybe getSchema(const string &uri) const override; Maybe invokeRest(const string &uri, istream &in) const override; @@ -67,6 +68,7 @@ private: I_MainLoop *mainloop; map> rest_calls; map> get_calls; + map> wildcard_get_calls; uint16_t listening_port = 0; vector port_range; }; @@ -128,11 +130,14 @@ RestServer::Impl::prepareConfiguration() } else { auto range_start = getPortConfig("Nano service API Port Range start"); auto range_end = getPortConfig("Nano service API Port Range end"); - dbgAssert(range_start.ok() && range_end.ok()) << alert << "Rest port configuration was not provided"; - dbgAssert(*range_start < *range_end) - << alert - << "Rest port range corrupted (lower bound higher then upper bound)"; - + if (!(range_start.ok() && range_end.ok()) || !(*range_start < *range_end)) { + dbgAssertOpt(range_start.ok() && range_end.ok()) << alert << "Rest port configuration was not provided"; + dbgAssertOpt(*range_start < *range_end) + << alert + << "Rest port range corrupted (lower bound higher then upper bound)"; + range_start = 0; + range_end = 1; + } port_range.resize(*range_end - *range_start); for (uint16_t i = 0, port = *range_start; i < port_range.size(); i++, port++) { port_range[i] = port; @@ -283,6 +288,13 @@ RestServer::Impl::addGetCall(const string &uri, const function &callba return get_calls.emplace(uri, callback).second; } +bool +RestServer::Impl::addWildcardGetCall(const string &uri, const function &callback) +{ + if (rest_calls.find(uri) != rest_calls.end()) return false; + return wildcard_get_calls.emplace(uri, callback).second; +} + Maybe RestServer::Impl::getSchema(const string &uri) const { @@ -307,14 +319,26 @@ RestServer::Impl::invokeRest(const string &uri, istream &in) const bool RestServer::Impl::isGetCall(const string &uri) const { - return get_calls.find(uri) != get_calls.end(); + if (get_calls.find(uri) != get_calls.end()) return true; + + for (const auto &wildcard : wildcard_get_calls) { + if (!uri.find(wildcard.first)) return true; + } + + return false; } string RestServer::Impl::invokeGet(const string &uri) const { auto instance = get_calls.find(uri); - return instance != get_calls.end() ? instance->second() : ""; + if (instance != get_calls.end()) return instance->second(); + + for (const auto &wildcard : wildcard_get_calls) { + if (!uri.find(wildcard.first)) return wildcard.second(uri); + } + + return ""; } string @@ -334,8 +358,8 @@ RestServer::Impl::changeActionToString(RestAction oper) return "delete-"; } default: { - dbgAssert(false) << alert << "Unknown REST action"; - return ""; + dbgAssertOpt(false) << alert << "Unknown REST action"; + return "unknown-"; } } } diff --git a/core/rest/rest_ut/CMakeLists.txt b/core/rest/rest_ut/CMakeLists.txt index b677797..d16ec83 100755 --- a/core/rest/rest_ut/CMakeLists.txt +++ b/core/rest/rest_ut/CMakeLists.txt @@ -4,5 +4,5 @@ link_directories(${ng_module_osrc_zlib_path}/lib) add_unit_test( rest_server_ut "rest_schema_ut.cc;rest_must_param_ut.cc;rest_config_ut.cc" - "singleton;messaging;tenant_manager;rest;environment;-lz;shell_cmd;-lboost_filesystem;instance_awareness;-lz;debug_is;time_proxy;mainloop;agent_details;encryptor;event_is;metric;-lboost_context;-lboost_regex;-lboost_system;-lssl;-lcrypto;connkey" + "singleton;messaging;tenant_manager;rest;environment;-lz;shell_cmd;-lboost_filesystem;instance_awareness;-lz;version;debug_is;time_proxy;mainloop;agent_details;encryptor;event_is;metric;-lboost_context;-lboost_regex;-lboost_system;-lssl;-lcrypto;connkey" ) diff --git a/core/rest/rest_ut/rest_config_ut.cc b/core/rest/rest_ut/rest_config_ut.cc index 4490696..1145a6e 100644 --- a/core/rest/rest_ut/rest_config_ut.cc +++ b/core/rest/rest_ut/rest_config_ut.cc @@ -171,11 +171,16 @@ TEST_F(RestConfigTest, basic_flow) auto i_rest = Singleton::Consume::from(rest_server); ASSERT_TRUE(i_rest->addRestCall(RestAction::ADD, "test")); ASSERT_TRUE(i_rest->addGetCall("stuff", [] () { return string("blabla"); })); + ASSERT_TRUE( + i_rest->addWildcardGetCall("api/", [] (const string &uri) { return uri.substr(uri.find_last_of('/') + 1); }) + ); int file_descriptor1 = socket(AF_INET, SOCK_STREAM, 0); EXPECT_NE(file_descriptor1, -1); int file_descriptor2 = socket(AF_INET, SOCK_STREAM, 0); EXPECT_NE(file_descriptor2, -1); + int file_descriptor3 = socket(AF_INET, SOCK_STREAM, 0); + EXPECT_NE(file_descriptor3, -1); auto primary_port = getConfiguration("connection", "Nano service API Port Alternative"); struct sockaddr_in sa; @@ -185,6 +190,7 @@ TEST_F(RestConfigTest, basic_flow) int socket_enable = 1; EXPECT_EQ(setsockopt(file_descriptor1, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0); EXPECT_EQ(setsockopt(file_descriptor2, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0); + EXPECT_EQ(setsockopt(file_descriptor3, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0); EXPECT_CALL(messaging, sendSyncMessage(_, _, _, _, _)) .WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); @@ -203,6 +209,11 @@ TEST_F(RestConfigTest, basic_flow) string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}"; EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast(msg2.size())); + EXPECT_EQ(connect(file_descriptor3, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0) + << "file_descriptor3 Error: " + << strerror(errno); + string msg3 = "GET /api/123 HTTP/1.1\r\n\r\n"; + EXPECT_EQ(write(file_descriptor3, msg3.data(), msg3.size()), static_cast(msg3.size())); while(!TestServer::g_num) { mainloop->yield(true); } @@ -215,6 +226,14 @@ TEST_F(RestConfigTest, basic_flow) mainloop->yield(true); } + struct pollfd s_poll3; + s_poll3.fd = file_descriptor3; + s_poll3.events = POLLIN; + s_poll3.revents = 0; + while(poll(&s_poll3, 1, 0) <= 0) { + mainloop->yield(true); + } + mainloop->stopAll(); }; mainloop->addOneTimeRoutine( @@ -233,6 +252,11 @@ TEST_F(RestConfigTest, basic_flow) string(respose, 76), "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 6\r\n\r\nblabla" ); + EXPECT_EQ(read(file_descriptor3, respose, 1000), 73); + EXPECT_EQ( + string(respose, 73), + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 3\r\n\r\n123" + ); } string diff --git a/core/socket_is/socket_is.cc b/core/socket_is/socket_is.cc index b8fe36b..415cc42 100644 --- a/core/socket_is/socket_is.cc +++ b/core/socket_is/socket_is.cc @@ -182,9 +182,11 @@ public: Maybe> acceptConn(bool is_blocking, const string &authorized_ip = "") { - dbgAssert(is_server_socket) << alert << "Failed to accept new connections from a client socket"; - dbgAssert(socket_int > 0) << alert << "Called with uninitialized server socket"; - + if (!(is_server_socket) || !(socket_int > 0)) { + dbgAssertOpt(is_server_socket) << alert << "Failed to accept new connections from a client socket"; + dbgAssertOpt(socket_int > 0) << alert << "Called with uninitialized server socket"; + return genError("Failed due to internal error"); + } dbgDebug(D_SOCKET) << "Attempt to accept new socket. Server Socket FD: " << socket_int; int client_socket; if (!authorized_ip.empty()) { diff --git a/core/time_proxy/time_proxy.cc b/core/time_proxy/time_proxy.cc index 42d18e6..1d67804 100644 --- a/core/time_proxy/time_proxy.cc +++ b/core/time_proxy/time_proxy.cc @@ -52,9 +52,12 @@ public: setMonotonicTime(microseconds new_time) override { if (is_monotomic_set) { - dbgAssert((new_time+monotonic_delta) >= monotonic_now) - << AlertInfo(AlertTeam::CORE, "time proxy") - << "Monotonic time must not go back!"; + if ((new_time+monotonic_delta) < monotonic_now) { + dbgAssertOpt((new_time+monotonic_delta) >= monotonic_now) + << AlertInfo(AlertTeam::CORE, "time proxy") + << "Monotonic time must not go back!"; + return; + } } else { // The first time that the monotonic time is been set, we take the current value to be the base line. // This is in order to avoid the clock going backwards. diff --git a/nodes/CMakeLists.txt b/nodes/CMakeLists.txt index 6937a7a..85ee401 100644 --- a/nodes/CMakeLists.txt +++ b/nodes/CMakeLists.txt @@ -14,3 +14,4 @@ add_subdirectory(orchestration) add_subdirectory(agent_cache) add_subdirectory(http_transaction_handler) add_subdirectory(attachment_registration_manager) +add_subdirectory(central_nginx_manager) diff --git a/unit_test.cmake b/unit_test.cmake index 032aa8f..6eacca4 100644 --- a/unit_test.cmake +++ b/unit_test.cmake @@ -2,7 +2,7 @@ enable_testing() function(add_unit_test ut_name ut_sources use_libs) add_executable(${ut_name} ${ut_sources}) - target_link_libraries(${ut_name} -Wl,--start-group ${use_libs} debug_is report cptest pthread packet singleton environment metric event_is buffers rest config compression_utils z ${GTEST_BOTH_LIBRARIES} gmock boost_regex pthread dl -Wl,--end-group) + target_link_libraries(${ut_name} -Wl,--start-group ${use_libs} version debug_is report cptest pthread packet singleton environment metric event_is buffers rest config compression_utils z ${GTEST_BOTH_LIBRARIES} gmock boost_regex pthread dl -Wl,--end-group) add_test(NAME ${ut_name} COMMAND ${ut_name} From 493d9a6627b928c857d1823d5ad98f413f69058f Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Mon, 13 Jan 2025 13:25:05 +0000 Subject: [PATCH 2/6] central nginx manager --- components/include/central_nginx_manager.h | 45 ++ components/include/nginx_message_reader.h | 28 + components/include/nginx_utils.h | 51 ++ .../nginx_message_reader/CMakeLists.txt | 3 + .../nginx_message_reader.cc | 735 ++++++++++++++++++ .../central_nginx_manager/CMakeLists.txt | 3 + .../central_nginx_manager.cc | 390 ++++++++++ .../include/lets_encrypt_listener.h | 30 + .../lets_encrypt_listener.cc | 76 ++ components/utils/nginx_utils/CMakeLists.txt | 1 + components/utils/nginx_utils/nginx_utils.cc | 281 +++++++ components/utils/utilities/CMakeLists.txt | 1 + .../nginx_conf_collector/CMakeLists.txt | 63 ++ .../nginx_conf_collector.cc | 148 ++++ nodes/central_nginx_manager/CMakeLists.txt | 34 + nodes/central_nginx_manager/main.cc | 31 + .../package/CMakeLists.txt | 4 + .../cp-nano-central-nginx-manager-conf.json | 1 + ...nano-central-nginx-manager-debug-conf.json | 11 + .../package/cp-nano-central-nginx-manager.cfg | 2 + .../install-cp-nano-central-nginx-manager.sh | 171 ++++ 21 files changed, 2109 insertions(+) create mode 100755 components/include/central_nginx_manager.h create mode 100755 components/include/nginx_message_reader.h create mode 100755 components/include/nginx_utils.h create mode 100755 components/nginx_message_reader/CMakeLists.txt create mode 100755 components/nginx_message_reader/nginx_message_reader.cc create mode 100755 components/security_apps/central_nginx_manager/CMakeLists.txt create mode 100755 components/security_apps/central_nginx_manager/central_nginx_manager.cc create mode 100755 components/security_apps/central_nginx_manager/include/lets_encrypt_listener.h create mode 100755 components/security_apps/central_nginx_manager/lets_encrypt_listener.cc create mode 100755 components/utils/nginx_utils/CMakeLists.txt create mode 100755 components/utils/nginx_utils/nginx_utils.cc create mode 100755 components/utils/utilities/CMakeLists.txt create mode 100755 components/utils/utilities/nginx_conf_collector/CMakeLists.txt create mode 100755 components/utils/utilities/nginx_conf_collector/nginx_conf_collector.cc create mode 100755 nodes/central_nginx_manager/CMakeLists.txt create mode 100755 nodes/central_nginx_manager/main.cc create mode 100755 nodes/central_nginx_manager/package/CMakeLists.txt create mode 100755 nodes/central_nginx_manager/package/cp-nano-central-nginx-manager-conf.json create mode 100755 nodes/central_nginx_manager/package/cp-nano-central-nginx-manager-debug-conf.json create mode 100755 nodes/central_nginx_manager/package/cp-nano-central-nginx-manager.cfg create mode 100755 nodes/central_nginx_manager/package/install-cp-nano-central-nginx-manager.sh diff --git a/components/include/central_nginx_manager.h b/components/include/central_nginx_manager.h new file mode 100755 index 0000000..5c3d134 --- /dev/null +++ b/components/include/central_nginx_manager.h @@ -0,0 +1,45 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __CENTRAL_NGINX_MANAGER_H__ +#define __CENTRAL_NGINX_MANAGER_H__ + +#include "component.h" +#include "singleton.h" +#include "i_messaging.h" +#include "i_rest_api.h" +#include "i_mainloop.h" +#include "i_agent_details.h" + +class CentralNginxManager + : + public Component, + Singleton::Consume, + Singleton::Consume, + Singleton::Consume, + Singleton::Consume +{ +public: + CentralNginxManager(); + ~CentralNginxManager(); + + void preload() override; + void init() override; + void fini() override; + +private: + class Impl; + std::unique_ptr pimpl; +}; + +#endif // __CENTRAL_NGINX_MANAGER_H__ diff --git a/components/include/nginx_message_reader.h b/components/include/nginx_message_reader.h new file mode 100755 index 0000000..19b7fb9 --- /dev/null +++ b/components/include/nginx_message_reader.h @@ -0,0 +1,28 @@ +#ifndef __NGINX_MESSAGE_READER_H__ +#define __NGINX_MESSAGE_READER_H__ + +#include "singleton.h" +#include "i_mainloop.h" +#include "i_socket_is.h" +#include "component.h" + +class NginxMessageReader + : + public Component, + Singleton::Consume, + Singleton::Consume +{ +public: + NginxMessageReader(); + ~NginxMessageReader(); + + void init() override; + void fini() override; + void preload() override; + +private: + class Impl; + std::unique_ptr pimpl; +}; + +#endif //__NGINX_MESSAGE_READER_H__ diff --git a/components/include/nginx_utils.h b/components/include/nginx_utils.h new file mode 100755 index 0000000..7f434e2 --- /dev/null +++ b/components/include/nginx_utils.h @@ -0,0 +1,51 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __NGINX_UTILS_H__ +#define __NGINX_UTILS_H__ + +#include + +#include "maybe_res.h" +#include "singleton.h" +#include "i_shell_cmd.h" + +class NginxConfCollector +{ +public: + NginxConfCollector(const std::string &nginx_conf_input_path, const std::string &nginx_conf_output_path); + Maybe generateFullNginxConf() const; + +private: + std::vector expandIncludes(const std::string &includePattern) const; + void processConfigFile( + const std::string &path, + std::ostringstream &conf_output, + std::vector &errors + ) const; + + std::string main_conf_input_path; + std::string main_conf_output_path; + std::string main_conf_directory_path; +}; + +class NginxUtils : Singleton::Consume +{ +public: + static std::string getModulesPath(); + static std::string getMainNginxConfPath(); + static Maybe validateNginxConf(const std::string &nginx_conf_path); + static Maybe reloadNginx(const std::string &nginx_conf_path); +}; + +#endif // __NGINX_UTILS_H__ diff --git a/components/nginx_message_reader/CMakeLists.txt b/components/nginx_message_reader/CMakeLists.txt new file mode 100755 index 0000000..ec26913 --- /dev/null +++ b/components/nginx_message_reader/CMakeLists.txt @@ -0,0 +1,3 @@ +link_directories(${BOOST_ROOT}/lib) + +add_library(nginx_message_reader nginx_message_reader.cc) diff --git a/components/nginx_message_reader/nginx_message_reader.cc b/components/nginx_message_reader/nginx_message_reader.cc new file mode 100755 index 0000000..a9c4f31 --- /dev/null +++ b/components/nginx_message_reader/nginx_message_reader.cc @@ -0,0 +1,735 @@ +#include "nginx_message_reader.h" + +#include +#include +#include +#include + +#include "config.h" +#include "singleton.h" +#include "i_mainloop.h" +#include "enum_array.h" +#include "log_generator.h" +#include "maybe_res.h" +#include "http_transaction_data.h" +#include "generic_rulebase/rulebase_config.h" +#include "generic_rulebase/evaluators/asset_eval.h" +#include "generic_rulebase/triggers_config.h" +#include "agent_core_utilities.h" +#include "rate_limit_config.h" + +USE_DEBUG_FLAG(D_NGINX_MESSAGE_READER); + +using namespace std; + +static const string syslog_regex_string = ( + "<[0-9]+>([A-Z][a-z][a-z]\\s{1,2}\\d{1,2}\\s\\d{2}" + "[:]\\d{2}[:]\\d{2})\\s([\\w][\\w\\d\\.@-]*)\\s(nginx:)" +); + +static const boost::regex socket_address_regex("(\\d+\\.\\d+\\.\\d+\\.\\d+):(\\d+)"); +static const boost::regex syslog_regex(syslog_regex_string); +static const boost::regex alert_log_regex( + "(" + + syslog_regex_string + ") " + + "(.+?\\[alert\\] )(.+?)" + ", (client: .+?)" + ", (server: .+?)" + ", (request: \".+?\")" + ", (upstream: \".+?\")" + ", (host: \".+?\")$" +); + +static const boost::regex error_log_regex( + "(" + + syslog_regex_string + ") " + + "(.+?\\[error\\] )(.+?)" + ", (client: .+?)" + ", (server: .+?)" + ", (request: \".+?\")" + ", (upstream: \".+?\")" + ", (host: \".+?\")$" +); + +static const boost::regex server_regex("(\\d+\\.\\d+\\.\\d+\\.\\d+)|(\\w+\\.\\w+)"); +static const boost::regex uri_regex("^/"); +static const boost::regex port_regex("\\d+"); +static const boost::regex response_code_regex("[0-9]{3}"); +static const boost::regex http_method_regex("[A-Za-z]+"); + +class NginxMessageReader::Impl +{ +public: + void + init() + { + dbgFlow(D_NGINX_MESSAGE_READER); + I_MainLoop *mainloop = Singleton::Consume::by(); + mainloop->addOneTimeRoutine( + I_MainLoop::RoutineType::System, + [this] () + { + initSyslogServerSocket(); + handleNginxLogs(); + }, + "Initialize nginx syslog", + true + ); + } + + void + preload() + { + registerConfigLoadCb([this]() { loadNginxMessageReaderConfig(); }); + } + + void + fini() + { + I_Socket *i_socket = Singleton::Consume::by(); + i_socket->closeSocket(syslog_server_socket); + } + + void + loadNginxMessageReaderConfig() + { + rate_limit_status_code = getProfileAgentSettingWithDefault( + "429", + "accessControl.rateLimit.returnCode" + ); + dbgTrace(D_NGINX_MESSAGE_READER) << "Selected rate-limit status code: " << rate_limit_status_code; + } + +private: + enum class LogInfo { + HTTP_METHOD, + URI, + RESPONSE_CODE, + HOST, + SOURCE, + DESTINATION_IP, + DESTINATION_PORT, + EVENT_MESSAGE, + ASSET_ID, + ASSET_NAME, + RULE_NAME, + RULE_ID, + COUNT + }; + + void + initSyslogServerSocket() + { + dbgFlow(D_NGINX_MESSAGE_READER); + I_MainLoop *mainloop = Singleton::Consume::by(); + I_Socket *i_socket = Singleton::Consume::by(); + string nginx_syslog_server_address = getProfileAgentSettingWithDefault( + "127.0.0.1:1514", + "reverseProxy.nginx.syslogAddress" + ); + dbgInfo(D_NGINX_MESSAGE_READER) << "Attempting to open a socket: " << nginx_syslog_server_address; + do { + Maybe new_socket = i_socket->genSocket( + I_Socket::SocketType::UDP, + false, + true, + nginx_syslog_server_address + ); + if (!new_socket.ok()) { + dbgError(D_NGINX_MESSAGE_READER) << "Failed to open a socket. Error: " << new_socket.getErr(); + mainloop->yield(chrono::milliseconds(500)); + continue; + } + + if (new_socket.unpack() < 0) { + dbgError(D_NGINX_MESSAGE_READER)<< "Generated socket is OK yet negative"; + mainloop->yield(chrono::milliseconds(500)); + continue; + } + syslog_server_socket = new_socket.unpack(); + dbgInfo(D_NGINX_MESSAGE_READER) + << "Opened socket for nginx logs over syslog. Socket: " + << syslog_server_socket; + } while (syslog_server_socket < 0); + } + + void + handleNginxLogs() + { + dbgFlow(D_NGINX_MESSAGE_READER); + I_MainLoop::Routine read_logs = + [this] () + { + Maybe logs = getLogsFromSocket(syslog_server_socket); + + if (!logs.ok()) { + dbgWarning(D_NGINX_MESSAGE_READER) + << "Failed to get NGINX logs from the socket. Error: " + << logs.getErr(); + return; + } + string raw_logs_to_parse = logs.unpackMove(); + vector logs_to_parse = separateLogs(raw_logs_to_parse); + + for (auto const &log: logs_to_parse) { + bool log_sent; + if (isAccessLog(log)) { + log_sent = sendAccessLog(log); + } else if (isAlertErrorLog(log) || isErrorLog(log)) { + log_sent = sendErrorLog(log); + } else { + dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format"; + continue; + } + if (!log_sent) { + dbgWarning(D_NGINX_MESSAGE_READER) << "Failed to send Log to Infinity Portal"; + } else { + dbgTrace(D_NGINX_MESSAGE_READER) << "Succesfully sent nginx log to Infinity Portal"; + } + } + }; + I_MainLoop *mainloop = Singleton::Consume::by(); + mainloop->addFileRoutine( + I_MainLoop::RoutineType::RealTime, + syslog_server_socket, + read_logs, + "Process nginx logs", + true + ); + } + + bool + sendAccessLog(const string &log) + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Access log" << log; + Maybe> log_info = parseAccessLog(log); + if (!log_info.ok()) { + dbgWarning(D_NGINX_MESSAGE_READER) + << "Failed parsing the NGINX logs. Error: " + << log_info.getErr(); + return false; + } + auto unpacked_log_info = log_info.unpack(); + + if (unpacked_log_info[LogInfo::RESPONSE_CODE] == rate_limit_status_code) { + return sendRateLimitLog(unpacked_log_info); + } + return sendLog(unpacked_log_info); + } + + bool + sendErrorLog(const string &log) + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Error log" << log; + Maybe> log_info = parseErrorLog(log); + if (!log_info.ok()) { + dbgWarning(D_NGINX_MESSAGE_READER) + << "Failed parsing the NGINX logs. Error: " + << log_info.getErr(); + return false; + } + return sendLog(log_info.unpack()); + } + + bool + isAccessLog(const string &log) const + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Chekck if string contains \"accessLog\"" << log; + return log.find("accessLog") != string::npos; + } + + bool + isAlertErrorLog(const string &log) const + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Check if log is of type 'error log'. Log: " << log; + return log.find("[alert]") != string::npos; + } + + bool + isErrorLog(const string &log) const + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Check if log is of type 'error log'. Log: " << log; + return log.find("[error]") != string::npos; + } + + bool + sendLog(const EnumArray &log_info) + { + dbgFlow(D_NGINX_MESSAGE_READER); + string event_name; + switch (log_info[LogInfo::RESPONSE_CODE][0]) { + case '4': { + event_name = "Invalid request or incorrect reverse proxy configuration - Request dropped." + " Please check the reverse proxy configuration of your relevant assets"; + break; + } + case '5': { + event_name = "AppSec Gateway reverse proxy error - Request dropped. " + "Please verify the reverse proxy configuration of your relevant assets. " + "If the issue persists please contact Check Point Support"; + break; + } + default: { + dbgError(D_NGINX_MESSAGE_READER) << "Irrelevant status code"; + return false; + } + } + + dbgTrace(D_NGINX_MESSAGE_READER) + << "Nginx log's event name and response code: " + << event_name + << ", " + << log_info[LogInfo::RESPONSE_CODE]; + LogGen log( + event_name, + ReportIS::Audience::SECURITY, + ReportIS::Severity::INFO, + ReportIS::Priority::LOW, + ReportIS::Tags::REVERSE_PROXY + ); + log << LogField("eventConfidence", "High"); + + for (LogInfo field : makeRange()) { + Maybe string_field = convertLogFieldToString(field); + if (!string_field.ok()) { + dbgDebug(D_NGINX_MESSAGE_READER) << "Enum field was not converted: " << string_field.getErr(); + return false; + } + + if (field != LogInfo::DESTINATION_PORT) { + log << LogField(string_field.unpack(), log_info[field]); + continue; + } + + try { + log << LogField(string_field.unpack(), stoi(log_info[field])); + } catch (const exception &e) { + dbgError(D_NGINX_MESSAGE_READER) + << "Unable to convert port to numeric value: " + << e.what(); + log << LogField(string_field.unpack(), 0); + } + } + return true; + } + + bool + sendRateLimitLog(const EnumArray &log_info) + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Getting rate-limit rules of asset ID: " << log_info[LogInfo::ASSET_ID]; + + ScopedContext rate_limit_ctx; + + rate_limit_ctx.registerValue(AssetMatcher::ctx_key, log_info[LogInfo::ASSET_ID]); + auto rate_limit_config = getConfiguration("rulebase", "rateLimit"); + if (!rate_limit_config.ok()) { + dbgTrace(D_NGINX_MESSAGE_READER) + << "Rate limit context does not match asset ID: " << log_info[LogInfo::ASSET_ID]; + return false; + } + RateLimitConfig unpacked_rate_limit_config = rate_limit_config.unpack(); + + string nginx_uri = log_info[LogInfo::URI]; + const LogTriggerConf &rate_limit_trigger = unpacked_rate_limit_config.getRateLimitTrigger(nginx_uri); + + dbgTrace(D_NGINX_MESSAGE_READER)<< "About to generate NGINX rate-limit log"; + + string event_name = "Rate limit"; + string security_action = "Drop"; + bool is_log_required = false; + + // Prevent events checkbox (in triggers) + if (rate_limit_trigger.isPreventLogActive(LogTriggerConf::SecurityType::AccessControl)) { + is_log_required = true; + } + + if (!is_log_required) { + dbgTrace(D_NGINX_MESSAGE_READER) << "Not sending NGINX rate-limit log as it is not required"; + return false; + } + + ostringstream src_ip; + ostringstream dst_ip; + src_ip << log_info[LogInfo::SOURCE]; + dst_ip << log_info[LogInfo::DESTINATION_IP]; + + ReportIS::Severity log_severity = ReportIS::Severity::MEDIUM; + ReportIS::Priority log_priority = ReportIS::Priority::MEDIUM; + + LogGen log = rate_limit_trigger( + event_name, + LogTriggerConf::SecurityType::AccessControl, + log_severity, + log_priority, + true, // is drop + LogField("practiceType", "Rate Limit"), + ReportIS::Tags::RATE_LIMIT + ); + + for (LogInfo field : makeRange()) { + Maybe string_field = convertLogFieldToString(field); + if (!string_field.ok()) { + dbgDebug(D_NGINX_MESSAGE_READER) << "Enum field was not converted: " << string_field.getErr(); + return false; + } + + if ( + field == LogInfo::HOST || + field == LogInfo::URI || + field == LogInfo::HTTP_METHOD || + field == LogInfo::SOURCE || + field == LogInfo::DESTINATION_IP || + field == LogInfo::ASSET_ID || + field == LogInfo::ASSET_NAME || + field == LogInfo::RESPONSE_CODE + ) { + if (!log_info[field].empty()) { + log << LogField(string_field.unpack(), log_info[field]); + continue; + } + } + + if (field == LogInfo::DESTINATION_PORT) { + try { + int numeric_dst_port = stoi(log_info[field]); + log << LogField(string_field.unpack(), numeric_dst_port); + } catch (const exception &e) { + dbgWarning(D_NGINX_MESSAGE_READER) + << "Unable to convert dst port: " + << log_info[field] + << " to numberic value. Error: " + << e.what(); + } + } + } + + return true; + } + + Maybe + convertLogFieldToString(LogInfo field) + { + dbgFlow(D_NGINX_MESSAGE_READER); + switch (field) { + case LogInfo::HTTP_METHOD: + return string("httpMethod"); + case LogInfo::URI: + return string("httpUriPath"); + case LogInfo::RESPONSE_CODE: + return string("httpResponseCode"); + case LogInfo::HOST: + return string("httpHostName"); + case LogInfo::SOURCE: + return string("httpSourceId"); + case LogInfo::DESTINATION_IP: + return string("destinationIp"); + case LogInfo::DESTINATION_PORT: + return string("destinationPort"); + case LogInfo::ASSET_ID: + return string("assetId"); + case LogInfo::ASSET_NAME: + return string("assetName"); + case LogInfo::EVENT_MESSAGE: + return string("httpResponseBody"); + case LogInfo::RULE_ID: + return string("ruleId"); + case LogInfo::RULE_NAME: + return string("ruleName"); + case LogInfo::COUNT: + dbgError(D_NGINX_MESSAGE_READER) << "LogInfo::COUNT is not allowed"; + return genError("LogInfo::COUNT is not allowed"); + } + dbgError(D_NGINX_MESSAGE_READER) << "No Enum found, int value: " << static_cast(field); + return genError("No Enum found"); + } + + static vector + separateLogs(const string &raw_logs_to_parse) + { + dbgFlow(D_NGINX_MESSAGE_READER) << "separating logs. logs: " << raw_logs_to_parse; + dbgTrace(D_NGINX_MESSAGE_READER) << "separateLogs start of function. Logs to parse: " << raw_logs_to_parse; + boost::smatch matcher; + vector logs; + + if (raw_logs_to_parse.empty()) return logs; + + size_t pos = 0; + while (NGEN::Regex::regexSearch(__FILE__, __LINE__, raw_logs_to_parse.substr(pos), matcher, syslog_regex)) { + if (pos == 0) { + dbgTrace(D_NGINX_MESSAGE_READER) << "separateLogs pos = 0"; + pos++; + continue; + } + auto log_length = matcher.position(); + logs.push_back(raw_logs_to_parse.substr(pos - 1, log_length)); + + pos += log_length + 1; + } + logs.push_back(raw_logs_to_parse.substr(pos - 1)); + dbgTrace(D_NGINX_MESSAGE_READER) << "separateLogs end of function"; + + return logs; + } + + static pair + parseErrorLogRequestField(const string &request) + { + dbgFlow(D_NGINX_MESSAGE_READER) << "parsing request field. request: " << request; + string formatted_request = request; + vector result; + boost::erase_all(formatted_request, "\""); + boost::erase_all(formatted_request, "\n"); + boost::split(result, formatted_request, boost::is_any_of(" "), boost::token_compress_on); + + const int http_method_index = 1; + const int uri_index = 2; + return pair(result[http_method_index], result[uri_index]); + } + + static string + parseErrorLogField(const string &field) + { + dbgFlow(D_NGINX_MESSAGE_READER) << "parsing error log field " << field; + string formatted_field = field; + vector result; + boost::erase_all(formatted_field, "\""); + boost::erase_all(formatted_field, "\n"); + boost::split(result, formatted_field, boost::is_any_of(" "), boost::token_compress_on); + + const int field_index = 1; + return result[field_index]; + } + + void + addContextFieldsToLogInfo(EnumArray &log_info) + { + dbgFlow(D_NGINX_MESSAGE_READER); + ScopedContext ctx; + + try { + ctx.registerValue( + HttpTransactionData::listening_port_ctx, + static_cast(stoi(log_info[LogInfo::DESTINATION_PORT])) + ); + } catch (const exception &e) { + dbgError(D_NGINX_MESSAGE_READER) << "Failed register values for context " << e.what(); + } + ctx.registerValue(HttpTransactionData::host_name_ctx, log_info[LogInfo::HOST]); + ctx.registerValue(HttpTransactionData::uri_ctx, log_info[LogInfo::URI]); + auto rule_by_ctx = getConfiguration("rulebase", "rulesConfig"); + if (!rule_by_ctx.ok()) { + dbgWarning(D_NGINX_MESSAGE_READER) + << "AssetId was not found by the given context. Reason: " + << rule_by_ctx.getErr(); + return; + } + + BasicRuleConfig context = rule_by_ctx.unpack(); + log_info[LogInfo::ASSET_ID] = context.getAssetId(); + log_info[LogInfo::ASSET_NAME] = context.getAssetName(); + log_info[LogInfo::RULE_ID] = context.getRuleId(); + log_info[LogInfo::RULE_NAME] = context.getRuleName(); + } + + Maybe> + parseErrorLog(const string &log_line) + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Handling log line:" << log_line; + string port; + EnumArray log_info(EnumArray::Fill(), string("")); + + boost::smatch matcher; + vector result; + if ( + !NGEN::Regex::regexSearch( + __FILE__, + __LINE__, + log_line, + matcher, + isAlertErrorLog(log_line) ? alert_log_regex : error_log_regex + ) + ) { + dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format"; + return genError("Unexpected nginx log format"); + } + + const int event_message_index = 6; + const int source_index = 7; + const int request_index = 9; + const int host_index = 11; + string host = string(matcher[host_index].first, matcher[host_index].second); + string source = string(matcher[source_index].first, matcher[source_index].second); + string event_message = string(matcher[event_message_index].first, matcher[event_message_index].second); + string request = string(matcher[request_index].first, matcher[request_index].second); + + host = parseErrorLogField(host); + source = parseErrorLogField(source); + pair parsed_request = parseErrorLogRequestField(request); + string http_method = parsed_request.first; + string uri = parsed_request.second; + + if (NGEN::Regex::regexSearch(__FILE__, __LINE__, host, matcher, socket_address_regex)) { + int host_index = 1; + int port_index = 2; + host = string(matcher[host_index].first, matcher[host_index].second); + port = string(matcher[port_index].first, matcher[port_index].second); + } else if (NGEN::Regex::regexSearch(__FILE__, __LINE__, host, matcher, boost::regex("https://"))) { + port = "443"; + } else { + port = "80"; + } + + log_info[LogInfo::HOST] = host; + log_info[LogInfo::URI] = uri; + log_info[LogInfo::RESPONSE_CODE] = "500"; + log_info[LogInfo::HTTP_METHOD] = http_method; + log_info[LogInfo::SOURCE] = source; + log_info[LogInfo::DESTINATION_IP] = host; + log_info[LogInfo::DESTINATION_PORT] = port; + log_info[LogInfo::EVENT_MESSAGE] = event_message; + + addContextFieldsToLogInfo(log_info); + + if (!validateLog(log_info)) { + dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format"; + return genError("Unexpected nginx log format"); + } + + return log_info; + } + + Maybe> + parseAccessLog(const string &log_line) + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Parsing log line: " << log_line; + string formatted_log = log_line; + EnumArray log_info(EnumArray::Fill(), string("")); + vector result; + boost::erase_all(formatted_log, "\""); + boost::erase_all(formatted_log, "\n"); + boost::split(result, formatted_log, boost::is_any_of(" "), boost::token_compress_on); + + const int valid_log_size = 20; + + if (result.size() < valid_log_size) { + dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format"; + return genError("Unexpected nginx log format"); + } + + const int host_index = 6; + const int host_port_index = 7; + const int http_method_index = 13; + const int uri_index = 14; + const int response_cod_index = 16; + const int source_index = 8; + + log_info[LogInfo::HOST] = result[host_index]; + log_info[LogInfo::URI] = result[uri_index]; + log_info[LogInfo::RESPONSE_CODE] = result[response_cod_index]; + log_info[LogInfo::HTTP_METHOD] = result[http_method_index]; + log_info[LogInfo::SOURCE] = result[source_index]; + log_info[LogInfo::DESTINATION_IP] = result[host_index]; + log_info[LogInfo::DESTINATION_PORT] = result[host_port_index]; + log_info[LogInfo::EVENT_MESSAGE] = "Invalid request or incorrect reverse proxy configuration - " + "Request dropped. Please check the reverse proxy configuration of your relevant assets"; + + addContextFieldsToLogInfo(log_info); + + if (!validateLog(log_info)) { + dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format"; + return genError("Unexpected nginx log format"); + } + return log_info; + } + + static bool + validateLog(const EnumArray &log_info) + { + dbgFlow(D_NGINX_MESSAGE_READER); + + boost::smatch matcher; + if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, log_info[LogInfo::HOST], matcher, server_regex)) { + dbgTrace(D_NGINX_MESSAGE_READER) << "Could not validate server (Host): " << log_info[LogInfo::HOST]; + return false; + } + if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, log_info[LogInfo::URI], matcher, uri_regex)) { + dbgTrace(D_NGINX_MESSAGE_READER) << "Could not validate Uri: " << log_info[LogInfo::URI]; + return false; + } + + if ( + !NGEN::Regex::regexSearch( + __FILE__, + __LINE__, + log_info[LogInfo::RESPONSE_CODE], + matcher, response_code_regex + ) + ) { + dbgTrace(D_NGINX_MESSAGE_READER) + << "Could not validate response code: " + << log_info[LogInfo::RESPONSE_CODE]; + return false; + } + + if ( + !NGEN::Regex::regexSearch(__FILE__, __LINE__, log_info[LogInfo::HTTP_METHOD], matcher, http_method_regex) + ) { + dbgTrace(D_NGINX_MESSAGE_READER) << "Could not validate HTTP method: " << log_info[LogInfo::HTTP_METHOD]; + return false; + } + + if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, log_info[LogInfo::DESTINATION_PORT], matcher, port_regex)) { + dbgTrace(D_NGINX_MESSAGE_READER) + << "Could not validate destination port : " + << log_info[LogInfo::DESTINATION_PORT]; + return false; + } + + if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, log_info[LogInfo::SOURCE], matcher, server_regex)) { + dbgTrace(D_NGINX_MESSAGE_READER) << "Could not validate source : " << log_info[LogInfo::SOURCE]; + return false; + } + + return true; + } + + Maybe + getLogsFromSocket(const I_Socket::socketFd &client_socket) const + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Reading logs from socket. fd: " << client_socket; + I_Socket *i_socket = Singleton::Consume::by(); + Maybe> raw_log_data = i_socket->receiveData(client_socket, 0, false); + if (!raw_log_data.ok()) { + dbgWarning(D_NGINX_MESSAGE_READER) << "Error receiving data from socket"; + return genError("Error receiving data from socket"); + } + + string raw_log(raw_log_data.unpack().begin(), raw_log_data.unpack().end()); + return move(raw_log); + } + + I_Socket::socketFd syslog_server_socket = -1; + string rate_limit_status_code = "429"; +}; + +NginxMessageReader::NginxMessageReader() : Component("NginxMessageReader"), pimpl(make_unique()) {} + +NginxMessageReader::~NginxMessageReader() {} + +void +NginxMessageReader::init() +{ + pimpl->init(); +} + +void +NginxMessageReader::preload() +{ + pimpl->preload(); +} + +void +NginxMessageReader::fini() +{ + pimpl->fini(); +} diff --git a/components/security_apps/central_nginx_manager/CMakeLists.txt b/components/security_apps/central_nginx_manager/CMakeLists.txt new file mode 100755 index 0000000..b58bb6b --- /dev/null +++ b/components/security_apps/central_nginx_manager/CMakeLists.txt @@ -0,0 +1,3 @@ +include_directories(include) + +add_library(central_nginx_manager central_nginx_manager.cc lets_encrypt_listener.cc) diff --git a/components/security_apps/central_nginx_manager/central_nginx_manager.cc b/components/security_apps/central_nginx_manager/central_nginx_manager.cc new file mode 100755 index 0000000..6848e95 --- /dev/null +++ b/components/security_apps/central_nginx_manager/central_nginx_manager.cc @@ -0,0 +1,390 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "central_nginx_manager.h" +#include "lets_encrypt_listener.h" + +#include +#include +#include + +#include "debug.h" +#include "config.h" +#include "rest.h" +#include "log_generator.h" +#include "nginx_utils.h" +#include "agent_core_utilities.h" + +using namespace std; + +USE_DEBUG_FLAG(D_NGINX_MANAGER); + +class CentralNginxConfig +{ +public: + void load(cereal::JSONInputArchive &ar) + { + try { + string nginx_conf_base64; + ar(cereal::make_nvp("id", file_id)); + ar(cereal::make_nvp("name", file_name)); + ar(cereal::make_nvp("data", nginx_conf_base64)); + nginx_conf_content = cereal::base64::decode(nginx_conf_base64); + central_nginx_conf_path = getCentralNginxConfPath(); + shared_config_path = getSharedConfigPath(); + if (!nginx_conf_content.empty()) configureCentralNginx(); + } catch (const cereal::Exception &e) { + dbgDebug(D_NGINX_MANAGER) << "Could not load Central Management Config JSON. Error: " << e.what(); + ar.setNextName(nullptr); + } + } + + const string & getFileId() const { return file_id; } + const string & getFileName() const { return file_name; } + const string & getFileContent() const { return nginx_conf_content; } + + static string + getCentralNginxConfPath() + { + string central_nginx_conf_path = getProfileAgentSettingWithDefault( + string("/tmp/central_nginx.conf"), + "centralNginxManagement.confDownloadPath" + ); + dbgInfo(D_NGINX_MANAGER) << "Central NGINX configuration path: " << central_nginx_conf_path; + + return central_nginx_conf_path; + } + + static string + getSharedConfigPath() + { + string central_shared_conf_path = getConfigurationWithDefault( + "/etc/cp/conf", + "Config Component", + "configuration path" + ); + central_shared_conf_path += "/centralNginxManager/shared/central_nginx_shared.conf"; + dbgInfo(D_NGINX_MANAGER) << "Shared NGINX configuration path: " << central_shared_conf_path; + + return central_shared_conf_path; + } + +private: + void + loadAttachmentModule() + { + string attachment_module_path = NginxUtils::getModulesPath() + "/ngx_cp_attachment_module.so"; + if (!NGEN::Filesystem::exists(attachment_module_path)) { + dbgTrace(D_NGINX_MANAGER) << "Attachment module " << attachment_module_path << " does not exist"; + return; + } + + string attachment_module_conf = "load_module " + attachment_module_path + ";"; + if (nginx_conf_content.find(attachment_module_conf) != string::npos) { + dbgTrace(D_NGINX_MANAGER) << "Attachment module " << attachment_module_path << " already loaded"; + return; + } + + nginx_conf_content = attachment_module_conf + "\n" + nginx_conf_content; + } + + Maybe + loadSharedDirective(const string &directive) + { + dbgFlow(D_NGINX_MANAGER) << "Loading shared directive into the servers " << directive; + + if (!NGEN::Filesystem::copyFile(shared_config_path, shared_config_path + ".bak", true)) { + return genError("Could not create a backup of the shared NGINX configuration file"); + } + + ifstream shared_config(shared_config_path); + if (!shared_config.is_open()) { + return genError("Could not open shared NGINX configuration file"); + } + + string shared_config_content((istreambuf_iterator(shared_config)), istreambuf_iterator()); + shared_config.close(); + + if (shared_config_content.find(directive) != string::npos) { + dbgTrace(D_NGINX_MANAGER) << "Shared directive " << directive << " already loaded"; + return {}; + } + + ofstream new_shared_config(shared_config_path, ios::app); + if (!new_shared_config.is_open()) { + return genError("Could not open shared NGINX configuration file"); + } + + dbgTrace(D_NGINX_MANAGER) << "Adding shared directive " << directive; + new_shared_config << directive << "\n"; + new_shared_config.close(); + + auto validation = NginxUtils::validateNginxConf(central_nginx_conf_path); + if (!validation.ok()) { + if (!NGEN::Filesystem::copyFile(shared_config_path + ".bak", shared_config_path, true)) { + return genError("Could not restore the shared NGINX configuration file"); + } + return genError("Could not validate shared NGINX configuration file. Error: " + validation.getErr()); + } + + return {}; + } + + Maybe + loadSharedConfig() + { + dbgFlow(D_NGINX_MANAGER) << "Loading shared configuration into the servers"; + + ofstream shared_config(shared_config_path); + if (!shared_config.is_open()) { + return genError("Could not create shared NGINX configuration file"); + } + shared_config.close(); + + string shared_config_directive = "include " + shared_config_path + ";\n"; + boost::regex server_regex("server\\s*\\{"); + nginx_conf_content = NGEN::Regex::regexReplace( + __FILE__, + __LINE__, + nginx_conf_content, + server_regex, + "server {\n" + shared_config_directive + ); + + ofstream nginx_conf_file(central_nginx_conf_path); + if (!nginx_conf_file.is_open()) { + return genError("Could not open a temporary central NGINX configuration file"); + } + nginx_conf_file << nginx_conf_content; + nginx_conf_file.close(); + + auto validation = NginxUtils::validateNginxConf(central_nginx_conf_path); + if (!validation.ok()) { + return genError("Could not validate central NGINX configuration file. Error: " + validation.getErr()); + } + + return {}; + } + + Maybe + configureSyslog() + { + if (!getProfileAgentSettingWithDefault(true, "centralNginxManagement.syslogEnabled")) { + dbgTrace(D_NGINX_MANAGER) << "Syslog is disabled via settings"; + return {}; + } + + string syslog_directive = "error_log syslog:server=127.0.0.1:1514 warn;"; + auto load_shared_directive_result = loadSharedDirective(syslog_directive); + if (!load_shared_directive_result.ok()) { + return genError("Could not configure syslog directive, error: " + load_shared_directive_result.getErr()); + } + + return {}; + } + + Maybe + saveBaseCentralNginxConf() + { + ofstream central_nginx_conf_base_file(central_nginx_conf_path + ".base"); + if (!central_nginx_conf_base_file.is_open()) { + return genError("Could not open a temporary central NGINX configuration file"); + } + central_nginx_conf_base_file << nginx_conf_content; + central_nginx_conf_base_file.close(); + + return {}; + } + + void + configureCentralNginx() + { + loadAttachmentModule(); + auto save_base_nginx_conf = saveBaseCentralNginxConf(); + if (!save_base_nginx_conf.ok()) { + dbgWarning(D_NGINX_MANAGER) + << "Could not save base NGINX configuration. Error: " + << save_base_nginx_conf.getErr(); + return; + } + + string nginx_conf_content_backup = nginx_conf_content; + auto shared_config_result = loadSharedConfig(); + if (!shared_config_result.ok()) { + dbgWarning(D_NGINX_MANAGER) + << "Could not load shared configuration. Error: " + << shared_config_result.getErr(); + nginx_conf_content = nginx_conf_content_backup; + return; + } + + auto syslog_result = configureSyslog(); + if (!syslog_result.ok()) { + dbgWarning(D_NGINX_MANAGER) << "Could not configure syslog. Error: " << syslog_result.getErr(); + } + } + + string file_id; + string file_name; + string nginx_conf_content; + string central_nginx_conf_path; + string shared_config_path; +}; + +class CentralNginxManager::Impl +{ +public: + void + init() + { + dbgInfo(D_NGINX_MANAGER) << "Starting Central NGINX Manager"; + + string main_nginx_conf_path = NginxUtils::getMainNginxConfPath(); + if ( + NGEN::Filesystem::exists(main_nginx_conf_path) + && !NGEN::Filesystem::exists(main_nginx_conf_path + ".orig") + ) { + dbgInfo(D_NGINX_MANAGER) << "Creating a backup of the original main NGINX configuration file"; + NGEN::Filesystem::copyFile(main_nginx_conf_path, main_nginx_conf_path + ".orig", true); + } + + i_mainloop = Singleton::Consume::by(); + if (!lets_encrypt_listener.init()) { + dbgWarning(D_NGINX_MANAGER) << "Could not start Lets Encrypt Listener, scheduling retry"; + i_mainloop->addOneTimeRoutine( + I_MainLoop::RoutineType::System, + [this] () + { + while(!lets_encrypt_listener.init()) { + dbgWarning(D_NGINX_MANAGER) << "Could not start Lets Encrypt Listener, will retry"; + i_mainloop->yield(chrono::seconds(5)); + } + }, + "Lets Encrypt Listener initializer", + false + ); + } + } + + void + loadPolicy() + { + auto central_nginx_config = getSetting>("centralNginxManagement"); + if (!central_nginx_config.ok() || central_nginx_config.unpack().empty()) { + dbgWarning(D_NGINX_MANAGER) + << "Could not load Central NGINX Management settings. Error: " + << central_nginx_config.getErr(); + return; + } + + auto &config = central_nginx_config.unpack().front(); + if (config.getFileContent().empty()) { + dbgWarning(D_NGINX_MANAGER) << "Empty NGINX configuration file"; + return; + } + + dbgTrace(D_NGINX_MANAGER) + << "Handling Central NGINX Management settings: " + << config.getFileId() + << ", " + << config.getFileName() + << ", " + << config.getFileContent(); + + string central_nginx_conf_path = config.getCentralNginxConfPath(); + ofstream central_nginx_conf_file(central_nginx_conf_path); + if (!central_nginx_conf_file.is_open()) { + dbgWarning(D_NGINX_MANAGER) + << "Could not open central NGINX configuration file: " + << central_nginx_conf_path; + return; + } + central_nginx_conf_file << config.getFileContent(); + central_nginx_conf_file.close(); + + auto validation_result = NginxUtils::validateNginxConf(central_nginx_conf_path); + if (!validation_result.ok()) { + dbgWarning(D_NGINX_MANAGER) + << "Could not validate central NGINX configuration file. Error: " + << validation_result.getErr(); + logError(validation_result.getErr()); + return; + } + + dbgTrace(D_NGINX_MANAGER) << "Validated central NGINX configuration file"; + + auto reload_result = NginxUtils::reloadNginx(central_nginx_conf_path); + if (!reload_result.ok()) { + dbgWarning(D_NGINX_MANAGER) + << "Could not reload central NGINX configuration. Error: " + << reload_result.getErr(); + logError("Could not reload central NGINX configuration. Error: " + reload_result.getErr()); + return; + } + } + + void + fini() + { + string central_nginx_base_path = CentralNginxConfig::getCentralNginxConfPath() + ".base"; + if (!NGEN::Filesystem::exists(central_nginx_base_path)) { + dbgWarning(D_NGINX_MANAGER) << "Could not find base NGINX configuration file: " << central_nginx_base_path; + return; + } + + NginxUtils::reloadNginx(central_nginx_base_path); + } + +private: + void + logError(const string &error) + { + LogGen log( + error, + ReportIS::Audience::SECURITY, + ReportIS::Severity::CRITICAL, + ReportIS::Priority::HIGH, + ReportIS::Tags::POLICY_INSTALLATION + ); + } + + I_MainLoop *i_mainloop = nullptr; + LetsEncryptListener lets_encrypt_listener; +}; + +CentralNginxManager::CentralNginxManager() + : + Component("Central NGINX Manager"), + pimpl(make_unique()) {} + +CentralNginxManager::~CentralNginxManager() {} + +void +CentralNginxManager::init() +{ + pimpl->init(); +} + +void +CentralNginxManager::fini() +{ + pimpl->fini(); +} + +void +CentralNginxManager::preload() +{ + registerExpectedSetting>("centralNginxManagement"); + registerExpectedConfiguration("Config Component", "configuration path"); + registerConfigLoadCb([this]() { pimpl->loadPolicy(); }); +} diff --git a/components/security_apps/central_nginx_manager/include/lets_encrypt_listener.h b/components/security_apps/central_nginx_manager/include/lets_encrypt_listener.h new file mode 100755 index 0000000..cad66cf --- /dev/null +++ b/components/security_apps/central_nginx_manager/include/lets_encrypt_listener.h @@ -0,0 +1,30 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __LETS_ENCRYPT_HANDLER_H__ +#define __LETS_ENCRYPT_HANDLER_H__ + +#include + +#include "maybe_res.h" + +class LetsEncryptListener +{ +public: + bool init(); + +private: + Maybe getChallengeValue(const std::string &uri) const; +}; + +#endif // __LETS_ENCRYPT_HANDLER_H__ diff --git a/components/security_apps/central_nginx_manager/lets_encrypt_listener.cc b/components/security_apps/central_nginx_manager/lets_encrypt_listener.cc new file mode 100755 index 0000000..93be82c --- /dev/null +++ b/components/security_apps/central_nginx_manager/lets_encrypt_listener.cc @@ -0,0 +1,76 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "lets_encrypt_listener.h" + +#include + +#include "central_nginx_manager.h" +#include "debug.h" + +using namespace std; + +USE_DEBUG_FLAG(D_NGINX_MANAGER); + +bool +LetsEncryptListener::init() +{ + dbgInfo(D_NGINX_MANAGER) << "Starting Lets Encrypt Listener"; + return Singleton::Consume::by()->addWildcardGetCall( + ".well-known/acme-challenge/", + [&] (const string &uri) -> string + { + Maybe maybe_challenge_value = getChallengeValue(uri); + if (!maybe_challenge_value.ok()) { + dbgWarning(D_NGINX_MANAGER) + << "Could not get challenge value for uri: " + << uri + << ", error: " + << maybe_challenge_value.getErr(); + return string{""}; + }; + + dbgTrace(D_NGINX_MANAGER) << "Got challenge value: " << maybe_challenge_value.unpack(); + return maybe_challenge_value.unpack(); + } + ); +} + +Maybe +LetsEncryptListener::getChallengeValue(const string &uri) const +{ + string challenge_key = uri.substr(uri.find_last_of('/') + 1); + string api_query = "/api/lets-encrypt-challenge?http_challenge_key=" + challenge_key; + + dbgInfo(D_NGINX_MANAGER) << "Getting challenge value via: " << api_query; + + MessageMetadata md; + md.insertHeader("X-Tenant-Id", Singleton::Consume::by()->getTenantId()); + Maybe maybe_http_challenge_value = + Singleton::Consume::by()->sendSyncMessage( + HTTPMethod::GET, + api_query, + string("{}"), + MessageCategory::GENERIC, + md + ); + + if (!maybe_http_challenge_value.ok()) return genError(maybe_http_challenge_value.getErr().getBody()); + + string challenge_value = maybe_http_challenge_value.unpack().getBody(); + if (!challenge_value.empty() && challenge_value.front() == '"' && challenge_value.back() == '"') { + challenge_value = challenge_value.substr(1, challenge_value.size() - 2); + } + + return challenge_value; +} diff --git a/components/utils/nginx_utils/CMakeLists.txt b/components/utils/nginx_utils/CMakeLists.txt new file mode 100755 index 0000000..000525a --- /dev/null +++ b/components/utils/nginx_utils/CMakeLists.txt @@ -0,0 +1 @@ +add_library(nginx_utils nginx_utils.cc) diff --git a/components/utils/nginx_utils/nginx_utils.cc b/components/utils/nginx_utils/nginx_utils.cc new file mode 100755 index 0000000..e0af8fa --- /dev/null +++ b/components/utils/nginx_utils/nginx_utils.cc @@ -0,0 +1,281 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "nginx_utils.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "maybe_res.h" +#include "config.h" +#include "agent_core_utilities.h" + +using namespace std; + +USE_DEBUG_FLAG(D_NGINX_MANAGER); + +NginxConfCollector::NginxConfCollector(const string &input_path, const string &output_path) + : + main_conf_input_path(input_path), + main_conf_output_path(output_path) +{ + main_conf_directory_path = main_conf_input_path.substr(0, main_conf_input_path.find_last_of('/')); +} + +vector +NginxConfCollector::expandIncludes(const string &include_pattern) const { + vector matching_files; + string absolute_include_pattern = include_pattern; + string maybe_directory = include_pattern.substr(0, include_pattern.find_last_of('/')); + if (!maybe_directory.empty() && maybe_directory.front() != '/') { + dbgTrace(D_NGINX_MANAGER) << "Include pattern is a relative path: " << include_pattern; + maybe_directory = main_conf_directory_path + '/' + maybe_directory; + absolute_include_pattern = main_conf_directory_path + '/' + include_pattern; + } + + if (!NGEN::Filesystem::exists(maybe_directory)) { + dbgTrace(D_NGINX_MANAGER) << "Include pattern directory/file does not exist: " << maybe_directory; + return matching_files; + } + + string filename_pattern = absolute_include_pattern.substr(absolute_include_pattern.find_last_of('/') + 1); + boost::regex wildcard_regex("\\*"); + boost::regex pattern( + NGEN::Regex::regexReplace(__FILE__, __LINE__, filename_pattern, wildcard_regex, string("[^/]*")) + ); + + if (!NGEN::Filesystem::isDirectory(maybe_directory)) { + dbgTrace(D_NGINX_MANAGER) << "Include pattern is a file: " << absolute_include_pattern; + matching_files.push_back(absolute_include_pattern); + return matching_files; + } + + DIR* dir = opendir(maybe_directory.c_str()); + if (!dir) { + dbgTrace(D_NGINX_MANAGER) << "Could not open directory: " << maybe_directory; + return matching_files; + } + + struct dirent *entry; + while ((entry = readdir(dir)) != nullptr) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) continue; + + if (NGEN::Regex::regexMatch(__FILE__, __LINE__, entry->d_name, pattern)) { + matching_files.push_back(maybe_directory + "/" + entry->d_name); + dbgTrace(D_NGINX_MANAGER) << "Matched file: " << maybe_directory << '/' << entry->d_name; + } + } + closedir(dir); + + return matching_files; +} + +void +NginxConfCollector::processConfigFile(const string &path, ostringstream &conf_output, vector &errors) const +{ + ifstream file(path); + if (!file.is_open()) return; + + string content((istreambuf_iterator(file)), istreambuf_iterator()); + file.close(); + + dbgTrace(D_NGINX_MANAGER) << "Processing file: " << path; + + if (content.empty()) return; + + try { + boost::regex include_regex(R"(^\s*include\s+([^;]+);)"); + boost::smatch match; + + while (NGEN::Regex::regexSearch(__FILE__, __LINE__, content, match, include_regex)) { + string include_pattern = match[1].str(); + include_pattern = NGEN::Strings::trim(include_pattern); + dbgTrace(D_NGINX_MANAGER) << "Include pattern: " << include_pattern; + + vector included_files = expandIncludes(include_pattern); + if (included_files.empty()) { + dbgTrace(D_NGINX_MANAGER) << "No files matched the include pattern: " << include_pattern; + content.replace(match.position(), match.length(), ""); + continue; + } + + ostringstream included_content; + for (const string &included_file : included_files) { + dbgTrace(D_NGINX_MANAGER) << "Processing included file: " << included_file; + processConfigFile(included_file, included_content, errors); + } + content.replace(match.position(), match.length(), included_content.str()); + } + } catch (const boost::regex_error &e) { + errors.emplace_back(e.what()); + return; + } catch (const exception &e) { + errors.emplace_back(e.what()); + return; + } + + conf_output << content; +} + +Maybe +NginxConfCollector::generateFullNginxConf() const +{ + if (!NGEN::Filesystem::exists(main_conf_input_path)) { + return genError("Input file does not exist: " + main_conf_input_path); + } + + ostringstream conf_output; + vector errors; + processConfigFile(main_conf_input_path, conf_output, errors); + + if (!errors.empty()) { + for (const string &error : errors) dbgWarning(D_NGINX_MANAGER) << error; + return genError("Errors occurred while processing configuration files"); + } + + ofstream single_nginx_conf_file(main_conf_output_path); + if (!single_nginx_conf_file.is_open()) return genError("Could not create output file: " + main_conf_output_path); + + single_nginx_conf_file << conf_output.str(); + single_nginx_conf_file.close(); + + return NGEN::Filesystem::resolveFullPath(main_conf_output_path); +} + +string +NginxUtils::getMainNginxConfPath() +{ + static string main_nginx_conf_path; + if (!main_nginx_conf_path.empty()) return main_nginx_conf_path; + + auto main_nginx_conf_path_setting = getProfileAgentSetting("centralNginxManagement.mainConfPath"); + if (main_nginx_conf_path_setting.ok()) { + main_nginx_conf_path = main_nginx_conf_path_setting.unpack(); + return main_nginx_conf_path; + } + + string default_main_nginx_conf_path = "/etc/nginx/nginx.conf"; + string command = "nginx -V 2>&1"; + auto result = Singleton::Consume::by()->getExecOutputAndCode(command); + if (!result.ok()) return default_main_nginx_conf_path; + + string output = result.unpack().first; + boost::regex conf_regex(R"(--conf-path=([^ ]+))"); + boost::smatch match; + if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, output, match, conf_regex)) { + main_nginx_conf_path = default_main_nginx_conf_path; + return main_nginx_conf_path; + } + + string conf_path = match[1].str(); + conf_path = NGEN::Strings::trim(conf_path); + if (conf_path.empty()) { + main_nginx_conf_path = default_main_nginx_conf_path; + return main_nginx_conf_path; + } + + main_nginx_conf_path = conf_path; + return main_nginx_conf_path; +} + +string +NginxUtils::getModulesPath() +{ + static string main_modules_path; + if (!main_modules_path.empty()) return main_modules_path; + + auto modules_path_setting = getProfileAgentSetting("centralNginxManagement.modulesPath"); + if (modules_path_setting.ok()) { + main_modules_path = modules_path_setting.unpack(); + return main_modules_path; + } + + string default_modules_path = "/usr/share/nginx/modules"; + string command = "nginx -V 2>&1"; + auto result = Singleton::Consume::by()->getExecOutputAndCode(command); + if (!result.ok()) return default_modules_path; + + string output = result.unpack().first; + boost::regex modules_regex(R"(--modules-path=([^ ]+))"); + boost::smatch match; + if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, output, match, modules_regex)) { + main_modules_path = default_modules_path; + return main_modules_path; + } + + string modules_path = match[1].str(); + modules_path = NGEN::Strings::trim(modules_path); + if (modules_path.empty()) { + main_modules_path = default_modules_path; + return main_modules_path; + } + + main_modules_path = modules_path; + return modules_path; +} + +Maybe +NginxUtils::validateNginxConf(const string &nginx_conf_path) +{ + dbgTrace(D_NGINX_MANAGER) << "Validating NGINX configuration file: " << nginx_conf_path; + if (!NGEN::Filesystem::exists(nginx_conf_path)) return genError("Nginx configuration file does not exist"); + + string command = "nginx -t -c " + nginx_conf_path + " 2>&1"; + auto result = Singleton::Consume::by()->getExecOutputAndCode(command); + if (!result.ok()) return genError(result.getErr()); + if (result.unpack().second != 0) return genError(result.unpack().first); + + dbgTrace(D_NGINX_MANAGER) << "NGINX configuration file is valid"; + + return {}; +} + +Maybe +NginxUtils::reloadNginx(const string &nginx_conf_path) +{ + dbgTrace(D_NGINX_MANAGER) << "Applying and reloading new NGINX configuration file: " << nginx_conf_path; + string main_nginx_conf_path = getMainNginxConfPath(); + + string backup_conf_path = main_nginx_conf_path + ".bak"; + if ( + NGEN::Filesystem::exists(main_nginx_conf_path) + && !NGEN::Filesystem::copyFile(main_nginx_conf_path, backup_conf_path, true) + ) { + return genError("Could not create backup of NGINX configuration file"); + } + + dbgTrace(D_NGINX_MANAGER) << "Copying new NGINX configuration file to: " << main_nginx_conf_path; + if (!NGEN::Filesystem::copyFile(nginx_conf_path, main_nginx_conf_path, true)) { + return genError("Could not copy new NGINX configuration file"); + } + + string command = "nginx -s reload 2>&1"; + auto result = Singleton::Consume::by()->getExecOutputAndCode(command); + if (!result.ok() || result.unpack().second != 0) { + if (!NGEN::Filesystem::copyFile(backup_conf_path, main_nginx_conf_path, true)) { + return genError("Could not restore backup of NGINX configuration file"); + } + dbgTrace(D_NGINX_MANAGER) << "Successfully restored backup of NGINX configuration file"; + return result.ok() ? genError(result.unpack().first) : genError(result.getErr()); + } + + dbgInfo(D_NGINX_MANAGER) << "Successfully reloaded NGINX configuration file"; + + return {}; +} diff --git a/components/utils/utilities/CMakeLists.txt b/components/utils/utilities/CMakeLists.txt new file mode 100755 index 0000000..f4655dd --- /dev/null +++ b/components/utils/utilities/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(nginx_conf_collector) \ No newline at end of file diff --git a/components/utils/utilities/nginx_conf_collector/CMakeLists.txt b/components/utils/utilities/nginx_conf_collector/CMakeLists.txt new file mode 100755 index 0000000..c18fc1e --- /dev/null +++ b/components/utils/utilities/nginx_conf_collector/CMakeLists.txt @@ -0,0 +1,63 @@ +include_directories(${PROJECT_SOURCE_DIR}/core/include/) + +link_directories(${Boost_LIBRARY_DIRS}) +link_directories(${ZLIB_ROOT}/lib) + +link_directories(${ZLIB_ROOT}/lib) +link_directories(${CMAKE_BINARY_DIR}/core) +link_directories(${CMAKE_BINARY_DIR}/core/compression) + +SET(EXECUTABLE_NAME "nginx_conf_collector_bin") +add_executable(${EXECUTABLE_NAME} nginx_conf_collector.cc) +target_compile_definitions(${EXECUTABLE_NAME} PRIVATE "NGINX_CONF_COLLECTOR_VERSION=\"$ENV{CI_PIPELINE_ID}\"") + +# if("${PLATFORM_TYPE}" STREQUAL "x86") +set_target_properties(${EXECUTABLE_NAME} PROPERTIES + LINK_FLAGS "-static -static-libgcc -static-libstdc++" +) +target_link_libraries(${EXECUTABLE_NAME} + shell_cmd + mainloop + messaging + event_is + metric + static_compression_utils + z + nginx_utils + time_proxy + debug_is + version + report + config + environment + singleton + rest + boost_context + boost_regex + pthread +) +# endif() + + +# target_link_libraries(${EXECUTABLE_NAME} +# shell_cmd +# mainloop +# messaging +# event_is +# metric +# compression_utils +# -lz +# nginx_utils +# time_proxy +# debug_is +# version +# report +# config +# environment +# singleton +# rest +# boost_context +# ) + +install(TARGETS ${EXECUTABLE_NAME} DESTINATION bin) +install(PROGRAMS ${EXECUTABLE_NAME} DESTINATION central_nginx_manager/bin RENAME cp-nano-nginx-conf-collector) diff --git a/components/utils/utilities/nginx_conf_collector/nginx_conf_collector.cc b/components/utils/utilities/nginx_conf_collector/nginx_conf_collector.cc new file mode 100755 index 0000000..092e15f --- /dev/null +++ b/components/utils/utilities/nginx_conf_collector/nginx_conf_collector.cc @@ -0,0 +1,148 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "agent_core_utilities.h" +#include "debug.h" +#include "internal/shell_cmd.h" +#include "mainloop.h" +#include "nginx_utils.h" +#include "time_proxy.h" + +using namespace std; + +USE_DEBUG_FLAG(D_NGINX_MANAGER); + +class MainComponent +{ +public: + MainComponent() + { + time_proxy.init(); + environment.init(); + mainloop.init(); + shell_cmd.init(); + } + + ~MainComponent() + { + shell_cmd.fini(); + mainloop.fini(); + environment.fini(); + time_proxy.fini(); + } +private: + ShellCmd shell_cmd; + MainloopComponent mainloop; + Environment environment; + TimeProxyComponent time_proxy; +}; + +void +printVersion() +{ +#ifdef NGINX_CONF_COLLECTOR_VERSION + cout << "Check Point NGINX configuration collector version: " << NGINX_CONF_COLLECTOR_VERSION << '\n'; +#else + cout << "Check Point NGINX configuration collector version: Private" << '\n'; +#endif +} + +void +printUsage(const char *prog_name) +{ + cout << "Usage: " << prog_name << " [-v] [-i /path/to/nginx.conf] [-o /path/to/output.conf]" << '\n'; + cout << " -V Print version" << '\n'; + cout << " -v Enable verbose output" << '\n'; + cout << " -i input_file Specify input file (default is /etc/nginx/nginx.conf)" << '\n'; + cout << " -o output_file Specify output file (default is ./full_nginx.conf)" << '\n'; + cout << " -h Print this help message" << '\n'; +} + +int +main(int argc, char *argv[]) +{ + string nginx_input_file = "/etc/nginx/nginx.conf"; + string nginx_output_file = "full_nginx.conf"; + + int opt; + while ((opt = getopt(argc, argv, "Vvhi:o:h")) != -1) { + switch (opt) { + case 'V': + printVersion(); + return 0; + case 'v': + Debug::setUnitTestFlag(D_NGINX_MANAGER, Debug::DebugLevel::TRACE); + break; + case 'i': + nginx_input_file = optarg; + break; + case 'o': + nginx_output_file = optarg; + break; + case 'h': + printUsage(argv[0]); + return 0; + default: + printUsage(argv[0]); + return 1; + } + } + + for (int i = optind; i < argc;) { + cerr << "Unknown argument: " << argv[i] << '\n'; + printUsage(argv[0]); + return 1; + } + + dbgTrace(D_NGINX_MANAGER) << "Starting nginx configuration collector"; + + MainComponent main_component; + auto validation_result = NginxUtils::validateNginxConf(nginx_input_file); + if (!validation_result.ok()) { + cerr + << "Could not validate nginx configuration file: " + << nginx_input_file + << '\n' + << validation_result.getErr(); + return 1; + } + + NginxConfCollector nginx_collector(nginx_input_file, nginx_output_file); + auto result = nginx_collector.generateFullNginxConf(); + if (!result.ok()) { + cerr << "Could not generate full nginx configuration file, error: " << result.getErr() << '\n'; + return 1; + } + + if (result.unpack().empty() || !NGEN::Filesystem::exists(result.unpack())) { + cerr << "Generated nginx configuration file does not exist: " << result.unpack() << '\n'; + return 1; + } + + validation_result = NginxUtils::validateNginxConf(result.unpack()); + if (!validation_result.ok()) { + cerr + << "Could not validate generated nginx configuration file: " + << nginx_output_file + << '\n' + << validation_result.getErr(); + return 1; + } + + cout << "Full nginx configuration file was successfully generated: " << result.unpack() << '\n'; + + return 0; +} diff --git a/nodes/central_nginx_manager/CMakeLists.txt b/nodes/central_nginx_manager/CMakeLists.txt new file mode 100755 index 0000000..783e929 --- /dev/null +++ b/nodes/central_nginx_manager/CMakeLists.txt @@ -0,0 +1,34 @@ +add_subdirectory(package) + +add_executable(cp-nano-central-nginx-manager main.cc) + +target_link_libraries(cp-nano-central-nginx-manager + -Wl,--start-group + ${COMMON_LIBRARIES} + + generic_rulebase + generic_rulebase_evaluators + ip_utilities + version + signal_handler + + central_nginx_manager + nginx_message_reader + rate_limit_comp + rate_limit_config + nginx_utils + http_transaction_data + -Wl,--end-group +) + +add_dependencies(cp-nano-central-nginx-manager ngen_core) + +install(TARGETS cp-nano-central-nginx-manager DESTINATION bin) +install(TARGETS cp-nano-central-nginx-manager DESTINATION central_nginx_manager/bin) + +gen_package( + install-cp-nano-central-nginx-manager.sh + central_nginx_manager + ./install-cp-nano-central-nginx-manager.sh + Check Point Central NGINX Manager Nano Service Version ${PACKAGE_VERSION} Install Package +) \ No newline at end of file diff --git a/nodes/central_nginx_manager/main.cc b/nodes/central_nginx_manager/main.cc new file mode 100755 index 0000000..bad8e39 --- /dev/null +++ b/nodes/central_nginx_manager/main.cc @@ -0,0 +1,31 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "central_nginx_manager.h" + +#include "components_list.h" +#include "nginx_message_reader.h" + +using namespace std; + +int +main(int argc, char **argv) +{ + NodeComponents comps; + + comps.registerGlobalValue("Is Rest primary routine", true); + comps.registerGlobalValue("Nano service API Port Primary", 7555); + comps.registerGlobalValue("Nano service API Port Alternative", 7556); + + return comps.run("Central NGINX Manager", argc, argv); +} diff --git a/nodes/central_nginx_manager/package/CMakeLists.txt b/nodes/central_nginx_manager/package/CMakeLists.txt new file mode 100755 index 0000000..fe82417 --- /dev/null +++ b/nodes/central_nginx_manager/package/CMakeLists.txt @@ -0,0 +1,4 @@ +install(FILES install-cp-nano-central-nginx-manager.sh DESTINATION central_nginx_manager PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) +install(FILES cp-nano-central-nginx-manager.cfg DESTINATION central_nginx_manager/conf PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) +install(FILES cp-nano-central-nginx-manager-conf.json DESTINATION central_nginx_manager/conf PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) +install(FILES cp-nano-central-nginx-manager-debug-conf.json DESTINATION central_nginx_manager/conf PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) diff --git a/nodes/central_nginx_manager/package/cp-nano-central-nginx-manager-conf.json b/nodes/central_nginx_manager/package/cp-nano-central-nginx-manager-conf.json new file mode 100755 index 0000000..9e26dfe --- /dev/null +++ b/nodes/central_nginx_manager/package/cp-nano-central-nginx-manager-conf.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/nodes/central_nginx_manager/package/cp-nano-central-nginx-manager-debug-conf.json b/nodes/central_nginx_manager/package/cp-nano-central-nginx-manager-debug-conf.json new file mode 100755 index 0000000..7b9773c --- /dev/null +++ b/nodes/central_nginx_manager/package/cp-nano-central-nginx-manager-debug-conf.json @@ -0,0 +1,11 @@ +{ + "Debug": [ + { + "Streams": [ + { + "Output": "/var/log/nano_agent/cp-nano-central-nginx-manager.dbg" + } + ] + } + ] +} \ No newline at end of file diff --git a/nodes/central_nginx_manager/package/cp-nano-central-nginx-manager.cfg b/nodes/central_nginx_manager/package/cp-nano-central-nginx-manager.cfg new file mode 100755 index 0000000..a4a5210 --- /dev/null +++ b/nodes/central_nginx_manager/package/cp-nano-central-nginx-manager.cfg @@ -0,0 +1,2 @@ +srv_debug_file=/var/log/nano_agent/cp-nano-central-nginx-manager.dbg +srv_log_file=/var/log/nano_agent/cp-nano-central-nginx-manager.log \ No newline at end of file diff --git a/nodes/central_nginx_manager/package/install-cp-nano-central-nginx-manager.sh b/nodes/central_nginx_manager/package/install-cp-nano-central-nginx-manager.sh new file mode 100755 index 0000000..5c00f04 --- /dev/null +++ b/nodes/central_nginx_manager/package/install-cp-nano-central-nginx-manager.sh @@ -0,0 +1,171 @@ +#!/bin/sh + +FORCE_STDOUT=true +INSTALLATION_LOG_FILE="/var/log/nano_agent/install-cp-nano-central-nginx-manager.log" +INSTALLATION_TIME=$(date) +CONF_PATH=/etc/cp/conf +SERVICE_PATH=/etc/cp/centralNginxManager +WATCHDOG_PATH=/etc/cp/watchdog/cp-nano-watchdog +NGINX_CONF_PATH="/etc/nginx/nginx.conf" +CENTRAL_NGINX_CONF_PATH="/tmp/central_nginx.conf" + +export INSTALL_COMMAND +is_install="$(command -v install)" +if [ -z ${is_install} ]; then + INSTALL_COMMAND="cp -f" + cp_print "[WARNING]: install command not found - using cp instead" ${FORCE_STDOUT} +else + INSTALL_COMMAND=install +fi + +mkdir -p /var/log/nano_agent +mkdir -p /tmp/ + +cp_print() +{ + var_text=${1} + var_std_out=${2} + touch ${INSTALLATION_LOG_FILE} + if [ -n "${var_std_out}" ]; then + if [ "${var_std_out}" = "true" ]; then + printf "%b\n" "${var_text}" + fi + fi + printf "%b\n" "${var_text}" >> ${INSTALLATION_LOG_FILE} +} + +cp_exec() +{ + var_cmd=${1} + var_std_out=${2} + # Send exec output to RES + RES=$(${var_cmd} 2>&1) + if [ -n "${RES}" ]; then + cp_print "${RES}" "${var_std_out}" + fi +} + +is_nginx_installed() +{ + if [ -x "$(command -v nginx)" ]; then + return 0 + fi + + return 1 +} + +get_nginx_conf_path() +{ + if ! is_nginx_installed; then + return + fi + + NGINX_CONF_PATH=$(nginx -V 2>&1 | grep -o '\--conf-path=[^ ]*' | cut -d= -f2) + if [ -z "${NGINX_CONF_PATH}" ]; then + NGINX_CONF_PATH="/etc/nginx/nginx.conf" + fi +} + +run_installation() +{ + cp_print "Starting installation of Check Point Central NGINX Manager [${INSTALLATION_TIME}]\n" ${FORCE_STDOUT} + cp_exec "${WATCHDOG_PATH} --un-register ${SERVICE_PATH}/cp-nano-central-nginx-manager" + cp_exec "mkdir -p ${SERVICE_PATH}" + cp_exec "mkdir -p ${CONF_PATH}/centralNginxManager/shared" + + cp_exec "touch ${CONF_PATH}/centralNginxManager/shared/central_nginx_shared.conf" + cp_exec "${INSTALL_COMMAND} bin/cp-nano-central-nginx-manager ${SERVICE_PATH}/cp-nano-central-nginx-manager" + cp_exec "${INSTALL_COMMAND} bin/cp-nano-nginx-conf-collector ${SERVICE_PATH}/cp-nano-nginx-conf-collector" + cp_exec "${INSTALL_COMMAND} conf/cp-nano-central-nginx-manager.cfg ${CONF_PATH}/cp-nano-central-nginx-manager.cfg" + cp_exec "${INSTALL_COMMAND} conf/cp-nano-central-nginx-manager-conf.json ${CONF_PATH}/cp-nano-central-nginx-manager-conf.json" + cp_exec "${INSTALL_COMMAND} conf/cp-nano-central-nginx-manager-debug-conf.json ${CONF_PATH}/cp-nano-central-nginx-manager-debug-conf.json" + cp_exec "chmod +x ${SERVICE_PATH}/cp-nano-central-nginx-manager" + cp_exec "chmod +x ${SERVICE_PATH}/cp-nano-nginx-conf-collector" + cp_exec "chmod 600 ${CONF_PATH}/cp-nano-central-nginx-manager.cfg" + cp_exec "chmod 600 ${CONF_PATH}/cp-nano-central-nginx-manager-conf.json" + + cp_exec "${WATCHDOG_PATH} --register ${SERVICE_PATH}/cp-nano-central-nginx-manager" + cp_print "Installation completed successfully." ${FORCE_STDOUT} +} + +usage() +{ + echo "Check Point: available flags are" + echo "--install : install central nginx manager" + echo "--uninstall : remove central nginx manager" + echo "--pre_install_test : run Pre-installation test for central nginx manager install package" + echo "--post_install_test : run Post-installation test for central nginx manager install package" + exit 255 +} + +run_uninstall() +{ + cp_print "Starting uninstall of Check Point Central NGINX Manager service [${INSTALLATION_TIME}]\n" ${FORCE_STDOUT} + + cp_exec "${WATCHDOG_PATH} --un-register ${SERVICE_PATH}/cp-nano-central-nginx-manager" + cp_exec "rm -rf ${SERVICE_PATH}/" + cp_exec "rm -f ${CONF_PATH}/cp-nano-central-nginx-manager.cfg" + cp_exec "rm -f ${CONF_PATH}/cp-nano-central-nginx-manager-conf.json" + + if [ -f "${CENTRAL_NGINX_CONF_PATH}.base" ]; then + cp_print "Restoring central NGINX configuration file" ${FORCE_STDOUT} + cp_exec "${INSTALL_COMMAND} ${CENTRAL_NGINX_CONF_PATH}.base ${NGINX_CONF_PATH}" + if is_nginx_installed; then + if nginx -t > /dev/null 2>&1; then + cp_exec "nginx -s reload" + else + cp_print "Could not reload central NGINX configuration, run 'nginx -t' for more details." ${FORCE_STDOUT} + fi + fi + fi + + if [ -f "${NGINX_CONF_PATH}.orig" ]; then + cp_print "Original (pre Check Point Nano Agent deployment) NGINX configuration file can be found at: ${NGINX_CONF_PATH}.orig" ${FORCE_STDOUT} + fi + cp_print "Check Point Central NGINX Manager service was removed successfully\n" ${FORCE_STDOUT} +} + +run_pre_install_test() +{ + cp_print "Successfully finished pre-installation test for Check Point Central NGINX Manager service installation package [${INSTALLATION_TIME}]\n" ${FORCE_STDOUT} + exit 0 +} + +run_post_install_test() +{ + if [ ! -d ${SERVICE_PATH} ]; then + cp_print "Failed post-installation test for Check Point Central NGINX Manager service installation package [${INSTALLATION_TIME}]\n" ${FORCE_STDOUT} + exit 1 + fi + + cp_print "Successfully finished post-installation test for Check Point Central NGINX Manager service installation package [${INSTALLATION_TIME}]\n" ${FORCE_STDOUT} + exit 0 +} + + +run() +{ + get_nginx_conf_path + if [ '--install' = "${1}" ]; then + run_installation "${@}" + elif [ '--uninstall' = "${1}" ]; then + run_uninstall + elif [ '--pre_install_test' = "${1}" ]; then + run_pre_install_test + elif [ '--post_install_test' = "${1}" ]; then + run_post_install_test + else + usage + exit 1 + fi +} + +if [ "$(id -u)" != "0" ]; then + echo "Administrative privileges required for this Package (use su or sudo)" + exit 1 +fi + +shift +run "${@}" + +exit 0 From 0db666ac4f11cac40734e273ae89b471b2f5ac4b Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Mon, 13 Jan 2025 14:29:58 +0000 Subject: [PATCH 3/6] central nginx manager - add new package to packages list --- nodes/orchestration/package/cp-nano-package-list | 1 + 1 file changed, 1 insertion(+) diff --git a/nodes/orchestration/package/cp-nano-package-list b/nodes/orchestration/package/cp-nano-package-list index 75b8e6c..170f1b3 100644 --- a/nodes/orchestration/package/cp-nano-package-list +++ b/nodes/orchestration/package/cp-nano-package-list @@ -28,4 +28,5 @@ sdwan_logger="sdwanLogger 2204 SD-WAN_Logger" cpview_metric_provider="cpviewMetricProvider 8282" hello_world="hello_world" crowdsec_aux="crowdsecAux 8081" +central_nginx_manager="centralNginxManager 7555" # ## Please do not remove this comment - newline at end of file required. From 4fd2aa6c6bd41544617dfe8c853f24bcd5e35c39 Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Tue, 14 Jan 2025 16:00:54 +0000 Subject: [PATCH 4/6] central nginx manager --- .../nginx_conf_collector/CMakeLists.txt | 29 +------------------ 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/components/utils/utilities/nginx_conf_collector/CMakeLists.txt b/components/utils/utilities/nginx_conf_collector/CMakeLists.txt index c18fc1e..7c745fb 100755 --- a/components/utils/utilities/nginx_conf_collector/CMakeLists.txt +++ b/components/utils/utilities/nginx_conf_collector/CMakeLists.txt @@ -11,17 +11,13 @@ SET(EXECUTABLE_NAME "nginx_conf_collector_bin") add_executable(${EXECUTABLE_NAME} nginx_conf_collector.cc) target_compile_definitions(${EXECUTABLE_NAME} PRIVATE "NGINX_CONF_COLLECTOR_VERSION=\"$ENV{CI_PIPELINE_ID}\"") -# if("${PLATFORM_TYPE}" STREQUAL "x86") -set_target_properties(${EXECUTABLE_NAME} PROPERTIES - LINK_FLAGS "-static -static-libgcc -static-libstdc++" -) target_link_libraries(${EXECUTABLE_NAME} shell_cmd mainloop messaging event_is metric - static_compression_utils + compression_utils z nginx_utils time_proxy @@ -33,31 +29,8 @@ target_link_libraries(${EXECUTABLE_NAME} singleton rest boost_context - boost_regex pthread ) -# endif() - - -# target_link_libraries(${EXECUTABLE_NAME} -# shell_cmd -# mainloop -# messaging -# event_is -# metric -# compression_utils -# -lz -# nginx_utils -# time_proxy -# debug_is -# version -# report -# config -# environment -# singleton -# rest -# boost_context -# ) install(TARGETS ${EXECUTABLE_NAME} DESTINATION bin) install(PROGRAMS ${EXECUTABLE_NAME} DESTINATION central_nginx_manager/bin RENAME cp-nano-nginx-conf-collector) From 9a516899e8094ea12b1219dc3b108fca54c5365d Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Tue, 14 Jan 2025 16:14:25 +0000 Subject: [PATCH 5/6] central nginx manager --- components/utils/utilities/nginx_conf_collector/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/components/utils/utilities/nginx_conf_collector/CMakeLists.txt b/components/utils/utilities/nginx_conf_collector/CMakeLists.txt index 7c745fb..56072c4 100755 --- a/components/utils/utilities/nginx_conf_collector/CMakeLists.txt +++ b/components/utils/utilities/nginx_conf_collector/CMakeLists.txt @@ -29,6 +29,7 @@ target_link_libraries(${EXECUTABLE_NAME} singleton rest boost_context + boost_regex pthread ) From abdee954bbb2e3cabf1f997968ecc72825e249e2 Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Wed, 15 Jan 2025 12:22:16 +0000 Subject: [PATCH 6/6] fix log-file-handler --- core/debug_is/debug.cc | 2 +- nodes/http_transaction_handler/package/k8s-log-file-handler.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/debug_is/debug.cc b/core/debug_is/debug.cc index 3588853..d7dd483 100644 --- a/core/debug_is/debug.cc +++ b/core/debug_is/debug.cc @@ -527,7 +527,7 @@ Debug::preload() active_streams["FOG"] = make_shared(); string branch = Version::getBranch(); - if (branch == "master" || branch.substr(0, 6) == "hotfix") { + if (branch == "open-source" || branch == "master" || branch.substr(0, 6) == "hotfix") { should_assert_optional = false; } else { should_assert_optional = true; diff --git a/nodes/http_transaction_handler/package/k8s-log-file-handler.sh b/nodes/http_transaction_handler/package/k8s-log-file-handler.sh index 97d3b44..1735c7c 100755 --- a/nodes/http_transaction_handler/package/k8s-log-file-handler.sh +++ b/nodes/http_transaction_handler/package/k8s-log-file-handler.sh @@ -7,5 +7,5 @@ while true; do sleep 5 continue fi - tail -q -f /var/log/nano_agent/cp-nano-http-transaction-handler.log? >> /proc/1/fd/1 + tail -q -f /var/log/nano_agent/cp-nano-http-transaction-handler.log* >> /proc/1/fd/1 done