From e7b6e51b3169147fabc61a5b5237dbffdcc40366 Mon Sep 17 00:00:00 2001 From: Daniel-Eisenberg <59121493+Daniel-Eisenberg@users.noreply.github.com> Date: Tue, 13 Jan 2026 17:17:52 +0200 Subject: [PATCH] Jan 06 2026 dev (#387) * sync code * update code to support brotli * update code to support brotli * update code to support brotli * sync code * fix findBrotli * sync code * sync code * sync code * sync code --------- Co-authored-by: Ned Wright Co-authored-by: Daniel Eisenberg --- CMakeLists.txt | 6 + .../nginx_attachment_util.cc | 34 +- .../nginx_attachment_util_ut.cc | 15 +- build_system/docker/Dockerfile | 1 + cmake/FindBrotli.cmake | 226 +++++ .../attachment_registrator.cc | 2 +- .../nginx_attachment/nginx_attachment.cc | 624 ++++++++++-- .../nginx_attachment_config.cc | 74 +- .../nginx_attachment_config.h | 8 + .../nginx_attachment_opaque.cc | 10 +- .../nginx_attachment/nginx_intaker_metric.cc | 298 +++--- .../nginx_attachment/nginx_parser.cc | 127 ++- .../nginx_attachment/nginx_parser.h | 2 +- components/http_manager/http_manager.cc | 97 +- .../http_manager/http_manager_opaque.cc | 50 +- components/http_manager/http_manager_opaque.h | 20 +- .../include/generic_rulebase/match_query.h | 11 +- .../generic_rulebase/parameters_config.h | 15 +- .../generic_rulebase/rulebase_config.h | 2 +- .../generic_rulebase/triggers_config.h | 26 +- .../include/http_event_impl/filter_verdict.h | 24 +- .../http_event_impl/i_http_event_impl.h | 74 +- components/include/http_inspection_events.h | 1 + components/include/hyperscan_hook.h | 40 + .../include/i_static_resources_handler.h | 11 +- components/include/ips_comp.h | 2 +- .../include/mock/mock_nginx_attachment.h | 12 +- components/include/nginx_intaker_metric.h | 8 +- components/include/rate_limit_config.h | 4 + components/include/reverse_proxy_defaults.h | 1 + components/include/user_identifiers_config.h | 1 - .../nginx_message_reader.cc | 627 +++++++++++- .../http_geo_filter/http_geo_filter.cc | 108 +- .../ips/include/ips_signatures.h | 14 +- components/security_apps/ips/ips_comp.cc | 12 +- .../security_apps/ips/ips_signatures.cc | 34 +- .../security_apps/ips/ips_ut/component_ut.cc | 6 +- .../security_apps/ips/ips_ut/signatures_ut.cc | 115 ++- .../layer_7_access_control.cc | 18 +- .../layer_7_access_control_ut.cc | 8 +- .../checkpoint_product_handlers.h | 170 +++- .../details_resolver_impl.h | 75 +- .../health_check/health_check.cc | 41 +- .../manifest_diff_calculator.cc | 6 + .../orchestration/orchestration_comp.cc | 7 +- .../prometheus/prometheus_comp.cc | 23 +- .../prometheus/prometheus_metric_names.h | 58 +- .../prometheus/prometheus_ut/prometheus_ut.cc | 212 ++-- .../security_apps/rate_limit/rate_limit.cc | 30 +- .../rate_limit/rate_limit_config.cc | 11 + .../waap/include/buffered_compressed_stream.h | 127 +++ .../security_apps/waap/include/i_serialize.h | 53 +- .../waap/include/i_transaction.h | 2 +- .../security_apps/waap/include/i_waapConfig.h | 8 + .../reputation/reputation_features_agg.cc | 4 +- .../waap/waap_clib/AssertionRegexes.h | 35 + .../waap/waap_clib/CMakeLists.txt | 8 + .../waap/waap_clib/ConfidenceCalculator.cc | 652 ++++++------ .../waap/waap_clib/ConfidenceCalculator.h | 22 +- .../waap/waap_clib/ConfidenceFile.cc | 10 +- .../waap/waap_clib/ConfidenceFile.h | 7 +- .../waap/waap_clib/DeepParser.cc | 6 +- .../waap/waap_clib/IndicatorsFilterBase.cc | 9 +- .../waap/waap_clib/IndicatorsFilterBase.h | 3 +- .../waap_clib/IndicatorsFiltersManager.cc | 279 +++++- .../waap/waap_clib/IndicatorsFiltersManager.h | 51 +- .../security_apps/waap/waap_clib/KeyStack.cc | 311 +++++- .../security_apps/waap/waap_clib/KeyStack.h | 77 +- .../waap/waap_clib/KeywordIndicatorFilter.cc | 16 +- .../waap/waap_clib/KeywordIndicatorFilter.h | 2 + .../waap/waap_clib/KeywordTypeValidator.cc | 9 +- .../waap/waap_clib/ParserGzip.cc | 230 ++--- .../security_apps/waap/waap_clib/ParserGzip.h | 92 +- .../waap/waap_clib/ParserUrlEncode.cc | 2 +- .../waap/waap_clib/ScannerDetector.cc | 256 +++-- .../waap/waap_clib/ScannersDetector.h | 69 +- .../waap/waap_clib/ScoreBuilder.cc | 1 + .../waap/waap_clib/Serializator.cc | 429 +++++--- .../waap/waap_clib/Signatures.cc | 468 ++++++++- .../security_apps/waap/waap_clib/Signatures.h | 134 ++- .../security_apps/waap/waap_clib/Telemetry.cc | 2 +- .../waap_clib/TrustedSourcesConfidence.cc | 209 +++- .../waap/waap_clib/TrustedSourcesConfidence.h | 30 +- .../waap/waap_clib/TuningDecision.cc | 19 +- .../waap/waap_clib/TypeIndicatorsFilter.cc | 35 +- .../waap/waap_clib/TypeIndicatorsFilter.h | 2 +- .../waap_clib/UnifiedIndicatorsContainer.cc | 339 +++++++ .../waap_clib/UnifiedIndicatorsContainer.h | 220 +++++ .../waap/waap_clib/WaapAssetState.cc | 675 +++++-------- .../waap/waap_clib/WaapAssetState.h | 22 +- .../waap/waap_clib/WaapAssetStatesManager.cc | 22 +- .../waap/waap_clib/WaapAssetStatesManager.h | 2 + .../waap/waap_clib/WaapConfigApi.cc | 2 +- .../waap/waap_clib/WaapConfigApi.h | 1 + .../waap/waap_clib/WaapConfigApplication.cc | 23 +- .../waap/waap_clib/WaapConfigApplication.h | 1 + .../waap/waap_clib/WaapHyperscanEngine.cc | 767 +++++++++++++++ .../waap/waap_clib/WaapHyperscanEngine.h | 54 + .../waap/waap_clib/WaapOverride.cc | 2 +- .../waap/waap_clib/WaapRegexPreconditions.cc | 18 +- .../waap/waap_clib/WaapRegexPreconditions.h | 45 +- .../waap/waap_clib/WaapSampleValue.cc | 35 +- .../waap/waap_clib/WaapSampleValue.h | 6 +- .../waap/waap_clib/WaapScanner.cc | 7 +- .../waap/waap_clib/WaapTrigger.cc | 2 +- .../waap/waap_clib/WaapTrigger.h | 4 +- .../waap/waap_clib/WaapValueStatsAnalyzer.cc | 66 +- .../waap/waap_clib/Waf2Engine.cc | 75 +- .../security_apps/waap/waap_clib/Waf2Engine.h | 4 +- .../waap/waap_clib/Waf2EngineGetters.cc | 140 ++- .../security_apps/waap/waap_clib/Waf2Regex.cc | 31 +- .../security_apps/waap/waap_clib/Waf2Regex.h | 10 +- .../security_apps/waap/waap_clib/Waf2Util.cc | 123 ++- .../waap_clib/buffered_compressed_stream.cc | 481 +++++++++ .../security_apps/waap/waap_component.cc | 10 +- .../security_apps/waap/waap_component_impl.cc | 83 +- components/signal_handler/signal_handler.cc | 4 +- .../evaluators/parameter_eval.cc | 2 +- .../evaluators/practice_eval.cc | 2 +- .../evaluators/trigger_eval.cc | 2 +- .../generic_rulebase/generic_rulebase.cc | 10 +- .../generic_rulebase_context.cc | 8 +- .../utils/generic_rulebase/match_query.cc | 42 +- .../generic_rulebase/parameters_config.cc | 77 +- .../utils/generic_rulebase/triggers_config.cc | 2 + components/utils/geo_location/geo_location.cc | 6 +- .../http_transaction_data.cc | 2 +- components/utils/pm/CMakeLists.txt | 2 +- components/utils/pm/hyperscan_hook.cc | 135 +++ .../nginx_conf_collector/CMakeLists.txt | 1 + .../agent_core_utilities.cc | 43 + .../agent_core_utilities_ut.cc | 223 +++++ core/agent_details/agent_details.cc | 37 +- .../agent_details_ut/agent_details_ut.cc | 2 + .../agent_details_reporter.cc | 6 + .../http_configuration/http_configuration.cc | 24 +- .../http_configuration_ut.cc | 12 +- core/compression/CMakeLists.txt | 4 + core/compression/compression_utils.cc | 241 ++++- .../compression_utils_ut.cc | 193 ++++ core/config/CMakeLists.txt | 2 +- core/config/config.cc | 142 ++- core/config/config_cache_stats.cc | 13 + core/cpu/cpu.cc | 4 +- core/cpu/cpu_ut/cpu_ut.cc | 20 +- core/curl_http_client/curl_http_client.cc | 28 + core/debug_is/debug.cc | 41 +- core/debug_is/debug_is_ut/debug_ut.cc | 8 + .../attachments/nano_attachment_common.h | 672 +++++++++++++ .../attachments/nginx_attachment_common.h | 282 +----- .../attachments/nginx_attachment_util.h | 9 +- core/include/general/common.h | 9 + core/include/general/debug.h | 2 + .../general/intell_registration_event.h | 28 + core/include/general/intelligence_comp_v2.h | 3 +- core/include/internal/curl_http_client.h | 13 + .../services_sdk/interfaces/i_encryptor.h | 3 + .../services_sdk/interfaces/i_mainloop.h | 10 + .../services_sdk/interfaces/i_messaging.h | 6 +- .../services_sdk/interfaces/i_rest_api.h | 5 + .../services_sdk/interfaces/i_socket_is.h | 1 + .../interfaces/messaging/interface_impl.h | 20 + .../interfaces/mock/mock_mainloop.h | 5 + .../interfaces/mock/mock_messaging.h | 1 + .../interfaces/mock/mock_rest_api.h | 5 + .../interfaces/mock/mock_socket_is.h | 1 + .../services_sdk/resources/agent_details.h | 9 +- .../resources/config/config_impl.h | 299 +++++- .../services_sdk/resources/config/i_config.h | 12 + core/include/services_sdk/resources/context.h | 13 + .../services_sdk/resources/debug_flags.h | 8 + .../resources/environment/context_impl.h | 41 +- .../resources/intelligence_invalidation.h | 6 + .../resources/metric/metric_calc.h | 2 + .../resources/report/report_enums.h | 8 +- core/include/services_sdk/resources/rest.h | 23 +- .../utilities/agent_core_utilities.h | 9 + .../intelligence_comp_v2.cc | 70 +- .../intelligence_is_v2_ut/invalidation_ut.cc | 25 +- core/intelligence_is_v2/invalidation.cc | 70 +- core/mainloop/mainloop.cc | 54 + core/mainloop/mainloop_ut/mainloop_ut.cc | 138 ++- core/messaging/connection/connection.cc | 55 +- core/messaging/connection/connection_comp.cc | 7 + .../connection_ut/connection_comp_ut.cc | 34 +- .../interfaces/i_messaging_connection.h | 1 + core/messaging/include/messaging_comp.h | 2 + .../include/mocks/mock_messaging_connection.h | 1 + core/messaging/messaging.cc | 8 +- .../messaging_buffer_comp.cc | 15 +- .../messaging_comp/messaging_comp.cc | 18 +- .../messaging_comp_ut/messaging_comp_ut.cc | 23 + core/metric/generic_metric.cc | 40 +- core/metric/metric_ut/metric_ut.cc | 18 + core/report/tag_and_enum_management.cc | 12 +- core/rest/i_rest_invoke.h | 12 +- core/rest/rest.cc | 5 +- core/rest/rest_conn.cc | 63 +- core/rest/rest_conn.h | 1 + core/rest/rest_server.cc | 48 +- core/rest/rest_ut/rest_config_ut.cc | 219 +++++ core/rest/rest_ut/rest_schema_ut.cc | 1 + core/shmem_ipc/shmem_ipc.c | 41 + core/socket_is/socket_is.cc | 124 +++ .../CMakeLists.txt | 1 + nodes/http_transaction_handler/CMakeLists.txt | 7 + nodes/orchestration/CMakeLists.txt | 2 + nodes/orchestration/package/CMakeLists.txt | 2 +- .../package/certificate/inext-ca-bundle.crt | 927 ++++++++++++++++++ nodes/orchestration/package/cp-nano-cli.sh | 11 +- .../package/cpnano_debug/cpnano_debug.cc | 7 + .../package/local-default-policy-v1beta2.yaml | 101 ++ .../orchestration/package/open-appsec-ctl.sh | 21 +- .../package/orchestration_package.sh | 53 +- nodes/prometheus/CMakeLists.txt | 4 +- unit_test.cmake | 2 +- 216 files changed, 12601 insertions(+), 2825 deletions(-) create mode 100644 cmake/FindBrotli.cmake create mode 100644 components/include/hyperscan_hook.h create mode 100644 components/security_apps/waap/include/buffered_compressed_stream.h create mode 100644 components/security_apps/waap/waap_clib/AssertionRegexes.h create mode 100644 components/security_apps/waap/waap_clib/UnifiedIndicatorsContainer.cc create mode 100644 components/security_apps/waap/waap_clib/UnifiedIndicatorsContainer.h create mode 100644 components/security_apps/waap/waap_clib/WaapHyperscanEngine.cc create mode 100644 components/security_apps/waap/waap_clib/WaapHyperscanEngine.h create mode 100644 components/security_apps/waap/waap_clib/buffered_compressed_stream.cc create mode 100644 components/utils/pm/hyperscan_hook.cc create mode 100644 core/config/config_cache_stats.cc create mode 100644 core/include/attachments/nano_attachment_common.h create mode 100644 core/include/general/intell_registration_event.h create mode 100755 nodes/orchestration/package/certificate/inext-ca-bundle.crt create mode 100755 nodes/orchestration/package/local-default-policy-v1beta2.yaml diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d46122..dbcd7cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,9 +8,15 @@ if(NOT IS_ALPINE EQUAL "0") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Dalpine") endif() +list(APPEND CMAKE_MODULE_PATH + "${CMAKE_CURRENT_SOURCE_DIR}/cmake" +) + find_package(Boost REQUIRED) find_package(ZLIB REQUIRED) find_package(GTest REQUIRED) +find_package(PkgConfig REQUIRED) +find_package(Brotli REQUIRED MODULE) include(cppcheck.cmake) diff --git a/attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc b/attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc index 85cd5b6..71313a9 100644 --- a/attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc +++ b/attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc @@ -28,10 +28,10 @@ initAttachmentConfig(c_str conf_file) return conf_data.init(conf_file); } -ngx_http_inspection_mode_e +NanoHttpInspectionMode getInspectionMode() { - return static_cast(conf_data.getNumericalValue("nginx_inspection_mode")); + return static_cast(conf_data.getNumericalValue("nginx_inspection_mode")); } unsigned int @@ -191,6 +191,24 @@ getRemoveResServerHeader() return conf_data.getNumericalValue("remove_server_header"); } +unsigned int +getDecompressionPoolSize() +{ + return conf_data.getNumericalValue("decompression_pool_size"); +} + +unsigned int +getRecompressionPoolSize() +{ + return conf_data.getNumericalValue("recompression_pool_size"); +} + +unsigned int +getIsBrotliInspectionEnabled() +{ + return conf_data.getNumericalValue("is_brotli_inspection_enabled"); +} + int isIPAddress(c_str ip_str) { @@ -285,3 +303,15 @@ isSkipSource(c_str ip_str) return 0; } + +unsigned int +isPairedAffinityEnabled() +{ + return conf_data.getNumericalValue("is_paired_affinity_enabled") != 0; +} + +unsigned int +isAsyncModeEnabled() +{ + return conf_data.getNumericalValue("is_async_mode_enabled") != 0; +} diff --git a/attachments/nginx/nginx_attachment_util/nginx_attachment_util_ut/nginx_attachment_util_ut.cc b/attachments/nginx/nginx_attachment_util/nginx_attachment_util_ut/nginx_attachment_util_ut.cc index 4cefb7c..70d7589 100644 --- a/attachments/nginx/nginx_attachment_util/nginx_attachment_util_ut/nginx_attachment_util_ut.cc +++ b/attachments/nginx/nginx_attachment_util/nginx_attachment_util_ut/nginx_attachment_util_ut.cc @@ -69,7 +69,12 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration) "\"hold_verdict_retries\": 3,\n" "\"hold_verdict_polling_time\": 1,\n" "\"body_size_trigger\": 777,\n" - "\"remove_server_header\": 1\n" + "\"remove_server_header\": 1,\n" + "\"decompression_pool_size\": 524288,\n" + "\"recompression_pool_size\": 32768,\n" + "\"is_paired_affinity_enabled\": 0,\n" + "\"is_async_mode_enabled\": 0,\n" + "\"is_brotli_inspection_enabled\": 1\n" "}\n"; ofstream valid_configuration_file(attachment_configuration_file_name); valid_configuration_file << valid_configuration; @@ -97,10 +102,13 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration) EXPECT_EQ(getMaxRetriesForVerdict(), 3u); EXPECT_EQ(getReqBodySizeTrigger(), 777u); EXPECT_EQ(getWaitingForVerdictThreadTimeout(), 75u); - EXPECT_EQ(getInspectionMode(), ngx_http_inspection_mode::BLOCKING_THREAD); + EXPECT_EQ(getInspectionMode(), NanoHttpInspectionMode::BLOCKING_THREAD); EXPECT_EQ(getRemoveResServerHeader(), 1u); + EXPECT_EQ(getDecompressionPoolSize(), 524288u); + EXPECT_EQ(getRecompressionPoolSize(), 32768u); EXPECT_EQ(getHoldVerdictRetries(), 3u); EXPECT_EQ(getHoldVerdictPollingTime(), 1u); + EXPECT_EQ(getIsBrotliInspectionEnabled(), 1u); EXPECT_EQ(isDebugContext("1.2.3.4", "5.6.7.8", 80, "GET", "test", "/abc"), 1); EXPECT_EQ(isDebugContext("1.2.3.9", "5.6.7.8", 80, "GET", "test", "/abc"), 0); @@ -125,6 +133,9 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration) EXPECT_EQ(isSkipSource("0:0:0:0:0:0:0:4"), 1); EXPECT_EQ(isSkipSource("0:0:0:0:0:0:0:5"), 1); EXPECT_EQ(isSkipSource("0:0:0:0:0:0:0:6"), 0); + + EXPECT_EQ(isPairedAffinityEnabled(), 0u); + EXPECT_EQ(isAsyncModeEnabled(), 0u); } TEST_F(HttpAttachmentUtilTest, CheckIPAddrValidity) diff --git a/build_system/docker/Dockerfile b/build_system/docker/Dockerfile index 3489bb0..e83c8b8 100644 --- a/build_system/docker/Dockerfile +++ b/build_system/docker/Dockerfile @@ -4,6 +4,7 @@ ENV OPENAPPSEC_NANO_AGENT=TRUE RUN apk add --no-cache -u busybox RUN apk add --no-cache -u zlib +RUN apk add --no-cache -u brotli brotli-dev RUN apk add --no-cache bash RUN apk add --no-cache libstdc++ RUN apk add --no-cache boost diff --git a/cmake/FindBrotli.cmake b/cmake/FindBrotli.cmake new file mode 100644 index 0000000..c4aedc9 --- /dev/null +++ b/cmake/FindBrotli.cmake @@ -0,0 +1,226 @@ +# FindBrotli.cmake +# +# Supports COMPONENTS: +# decoder, encoder, common +# +# Exports targets: +# Brotli::decoder +# Brotli::encoder +# Brotli::common +# +# Optional variables: +# BROTLI_ROOT_DIR +# BROTLI_USE_STATIC_LIBS + +# ------------------------------------------------------------ +# Version handling (not supported) +# ------------------------------------------------------------ +if(Brotli_FIND_VERSION) + set(_brotli_version_error_msg "FindBrotli.cmake does not support version checking.") + if(Brotli_FIND_REQUIRED) + message(FATAL_ERROR "${_brotli_version_error_msg}") + elseif(NOT Brotli_FIND_QUIETLY) + message(WARNING "${_brotli_version_error_msg}") + endif() +endif() + +# ------------------------------------------------------------ +# Component dependencies +# ------------------------------------------------------------ +if(Brotli_FIND_REQUIRED_decoder OR Brotli_FIND_REQUIRED_encoder) + set(Brotli_FIND_REQUIRED_common TRUE) +endif() + +# ------------------------------------------------------------ +# Static library preference +# ------------------------------------------------------------ +if(BROTLI_USE_STATIC_LIBS) + set(_brotli_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) + if(WIN32) + set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a) + else() + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + endif() +endif() + +# ------------------------------------------------------------ +# Optional pkg-config +# ------------------------------------------------------------ +find_package(PkgConfig QUIET) + +# ------------------------------------------------------------ +# Includes +# ------------------------------------------------------------ +find_path(Brotli_INCLUDE_DIR + NAMES + brotli/decode.h + brotli/encode.h + HINTS + ${BROTLI_ROOT_DIR} + PATH_SUFFIXES + include + includes +) +mark_as_advanced(Brotli_INCLUDE_DIR) + +# ------------------------------------------------------------ +# Internal state +# ------------------------------------------------------------ +set(_brotli_req_vars "") + +# For figuring out the real (non-ALIAS) targets when using pkg-config +set(_brotli_decoder_real_target "") +set(_brotli_encoder_real_target "") +set(_brotli_common_real_target "") + +if(BROTLI_USE_STATIC_LIBS) + set(_brotli_stat_str "_STATIC") +else() + set(_brotli_stat_str "") +endif() + +# ------------------------------------------------------------ +# Components loop +# ------------------------------------------------------------ +foreach(_listvar "common;common" "decoder;dec" "encoder;enc") + list(GET _listvar 0 _component) + list(GET _listvar 1 _libname) + + # ---- pkg-config path ---- + if(PKG_CONFIG_FOUND) + if(BROTLI_USE_STATIC_LIBS) + pkg_check_modules( + Brotli_${_component}_STATIC + QUIET + GLOBAL + IMPORTED_TARGET + libbrotli${_libname} + ) + else() + pkg_check_modules( + Brotli_${_component} + QUIET + GLOBAL + IMPORTED_TARGET + libbrotli${_libname} + ) + endif() + endif() + + # If pkg-config created an imported target, make our alias to it. + if(TARGET PkgConfig::Brotli_${_component}${_brotli_stat_str}) + add_library( + Brotli::${_component} + ALIAS + PkgConfig::Brotli_${_component}${_brotli_stat_str} + ) + + # Save the underlying real target name for later linkage fixes + set(_brotli_${_component}_real_target "PkgConfig::Brotli_${_component}${_brotli_stat_str}") + + set(Brotli_${_component}_FOUND TRUE) + + if(Brotli_FIND_REQUIRED_${_component}) + # For FindPackageHandleStandardArgs: ensure libraries are actually present + if(BROTLI_USE_STATIC_LIBS) + list(APPEND _brotli_req_vars Brotli_${_component}_STATIC_LIBRARIES) + else() + list(APPEND _brotli_req_vars Brotli_${_component}_LINK_LIBRARIES) + endif() + endif() + + continue() + endif() + + # ---- find_library path ---- + if(Brotli_FIND_REQUIRED_${_component}) + list(APPEND _brotli_req_vars Brotli_${_component}) + endif() + + if(BROTLI_USE_STATIC_LIBS) + set(_brotli_names + brotli${_libname}-static + libbrotli${_libname}-static + ) + else() + set(_brotli_names + brotli${_libname} + libbrotli${_libname} + ) + endif() + + find_library(Brotli_${_component} + NAMES ${_brotli_names} + HINTS ${BROTLI_ROOT_DIR} + PATH_SUFFIXES + lib + lib64 + libs + libs64 + lib/x86_64-linux-gnu + ) + mark_as_advanced(Brotli_${_component}) + + if(Brotli_${_component}) + set(Brotli_${_component}_FOUND TRUE) + + add_library(Brotli::${_component} UNKNOWN IMPORTED) + set_target_properties(Brotli::${_component} PROPERTIES + IMPORTED_LOCATION "${Brotli_${_component}}" + INTERFACE_INCLUDE_DIRECTORIES "${Brotli_INCLUDE_DIR}" + ) + + # In this branch, our target is real (not ALIAS), so it can be linked later. + set(_brotli_${_component}_real_target "Brotli::${_component}") + else() + set(Brotli_${_component}_FOUND FALSE) + endif() +endforeach() + +# ------------------------------------------------------------ +# Link decoder/encoder → common (but never on ALIAS targets or IMPORTED targets) +# ------------------------------------------------------------ +if(_brotli_common_real_target) + foreach(_comp decoder encoder) + if(_brotli_${_comp}_real_target) + # Only link if the target is NOT an ALIAS and NOT an IMPORTED target + get_target_property(_aliased ${_brotli_${_comp}_real_target} ALIASED_TARGET) + get_target_property(_imported ${_brotli_${_comp}_real_target} IMPORTED) + if(NOT _aliased AND NOT _imported) + target_link_libraries(${_brotli_${_comp}_real_target} INTERFACE ${_brotli_common_real_target}) + endif() + endif() + endforeach() +endif() + +# ------------------------------------------------------------ +# Aggregate convenience variables +# ------------------------------------------------------------ +set(Brotli_LIBRARIES "") +foreach(_comp decoder encoder common) + if(TARGET Brotli::${_comp}) + list(APPEND Brotli_LIBRARIES Brotli::${_comp}) + endif() +endforeach() + +# ------------------------------------------------------------ +# Final package check (FIXED: use _brotli_req_vars) +# ------------------------------------------------------------ +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Brotli + FOUND_VAR + Brotli_FOUND + REQUIRED_VARS + Brotli_INCLUDE_DIR + ${_brotli_req_vars} + HANDLE_COMPONENTS +) + +# ------------------------------------------------------------ +# Restore suffixes +# ------------------------------------------------------------ +if(BROTLI_USE_STATIC_LIBS) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${_brotli_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) +endif() + + diff --git a/components/attachment-intakers/attachment_registrator/attachment_registrator.cc b/components/attachment-intakers/attachment_registrator/attachment_registrator.cc index 98adf25..55f86c6 100755 --- a/components/attachment-intakers/attachment_registrator/attachment_registrator.cc +++ b/components/attachment-intakers/attachment_registrator/attachment_registrator.cc @@ -33,7 +33,7 @@ #include "i_mainloop.h" #include "buffer.h" #include "enum_array.h" -#include "nginx_attachment_common.h" +#include "nano_attachment_common.h" USE_DEBUG_FLAG(D_ATTACHMENT_REGISTRATION); diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc index 3410146..6ee47cb 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc @@ -15,6 +15,8 @@ #include #include +#include +#include #include #include #include @@ -29,6 +31,7 @@ #include #include #include +#include #include #include @@ -48,7 +51,7 @@ #include "shmem_ipc.h" #include "i_http_manager.h" #include "http_transaction_common.h" -#include "nginx_attachment_common.h" +#include "nano_attachment_common.h" #include "hash_combine.h" #include "cpu/failopen_mode_status.h" #include "attachment_registrator.h" @@ -75,7 +78,7 @@ USE_DEBUG_FLAG(D_METRICS_NGINX_ATTACHMENT); using namespace std; -using ChunkType = ngx_http_chunk_type_e; +using ChunkType = AttachmentDataType; static const uint32_t corrupted_session_id = CORRUPTED_SESSION_ID; static const AlertInfo alert(AlertTeam::CORE, "nginx attachment"); @@ -130,14 +133,15 @@ class NginxAttachment::Impl : Singleton::Provide::From { - static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; - static constexpr auto LIMIT_RESPONSE_HEADERS = ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS; - static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; - static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; - static constexpr auto INJECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT; - static constexpr auto IRRELEVANT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; - static constexpr auto RECONF = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_RECONF; - static constexpr auto WAIT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT; + static constexpr auto INSPECT = ServiceVerdict::TRAFFIC_VERDICT_INSPECT; + static constexpr auto LIMIT_RESPONSE_HEADERS = ServiceVerdict::LIMIT_RESPONSE_HEADERS; + static constexpr auto ACCEPT = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; + static constexpr auto DROP = ServiceVerdict::TRAFFIC_VERDICT_DROP; + static constexpr auto INJECT = ServiceVerdict::TRAFFIC_VERDICT_INJECT; + static constexpr auto IRRELEVANT = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT; + static constexpr auto RECONF = ServiceVerdict::TRAFFIC_VERDICT_RECONF; + static constexpr auto WAIT = ServiceVerdict::TRAFFIC_VERDICT_DELAYED; + static constexpr auto CUSTOM_RESPONSE = ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE; public: Impl() @@ -197,6 +201,8 @@ public: NUM_OF_NGINX_IPC_ELEMENTS, "nginxAttachment.numOfNginxIpcElements" ); + dbgInfo(D_NGINX_ATTACHMENT) << "Async mode configuration: enabled=" << attachment_config.isAsyncModeEnabled(); + nginx_attachment_metric.init( "Nginx Attachment data", ReportIS::AudienceTeam::AGENT_CORE, @@ -366,10 +372,13 @@ public: } bool - registerStaticResource(const string &resource_name, const string &resource_path) + registerStaticResource( + const string &resource_name, + const string &resource_path, + bool overwrite_if_exists) { string dest_path = static_resources_path + "/" + resource_name; - if (NGEN::Filesystem::exists(dest_path)) { + if (!overwrite_if_exists && NGEN::Filesystem::exists(dest_path)) { dbgDebug(D_NGINX_ATTACHMENT) << "Static resource already exist. path: " << dest_path; return true; } @@ -377,7 +386,7 @@ public: if (!NGEN::Filesystem::copyFile( resource_path, dest_path, - false, + overwrite_if_exists, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH )) { dbgWarning(D_NGINX_ATTACHMENT) @@ -397,6 +406,36 @@ public: return true; } + bool + registerStaticResourceByContent( + const string &resource_name, + const string &file_content + ) + { + string dest_path = static_resources_path + "/" + resource_name; + + if (!NGEN::Filesystem::createFileWithContent( + dest_path, + file_content, + true, + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH + )) { + dbgWarning(D_NGINX_ATTACHMENT) + << "Failed to create static resource file. Resource name: " + << resource_name + << ", destination path: " << dest_path + << ", content size: " << file_content.size(); + return false; + } + + dbgTrace(D_NGINX_ATTACHMENT) + << "Successfully created static resource. Resource name: " + << resource_name + << ", destination path: " << dest_path + << ", content size: " << file_content.size(); + return true; + } + void printMetrics() { @@ -443,6 +482,11 @@ private: attachment_routine_id = 0; } + if (async_attachment_routine_id > 0 && mainloop->doesRoutineExist(async_attachment_routine_id)) { + mainloop->stop(async_attachment_routine_id); + async_attachment_routine_id = 0; + } + string curr_instance_unique_id = inst_awareness->getUniqueID().unpack(); if (attachment_ipc != nullptr) { if (nginx_worker_user_id != nginx_user_id || nginx_worker_group_id != nginx_group_id) { @@ -548,7 +592,11 @@ private: ); while (isSignalPending()) { - if (!handleInspection()) break; + if (attachment_config.isAsyncModeEnabled()) { + if (!handleInspectionAsync()) break; + } else { + if (!handleInspection()) break; + } } }, "Nginx Attachment inspection handler", @@ -791,7 +839,7 @@ private: } void - sendMetricToKibana(const ngx_http_cp_metric_data_t *received_metric_data) + sendMetricToKibana(const NanoHttpMetricData *received_metric_data) { nginx_intaker_event.addPluginMetricCounter(received_metric_data); nginx_intaker_event.notify(); @@ -822,7 +870,7 @@ private: return "Request End"; case ChunkType::METRIC_DATA_FROM_PLUGIN: return "Metrics"; - case ChunkType::HOLD_DATA: + case ChunkType::REQUEST_DELAYED_VERDICT: return "HOLD_DATA"; case ChunkType::COUNT: dbgAssert(false) << alert << "Invalid 'COUNT' ChunkType"; @@ -842,7 +890,7 @@ private: return default_verdict; } - auto rule_by_ctx = getConfiguration("rulebase", "rulesConfig"); + auto rule_by_ctx = getConfigurationWithCache("rulebase", "rulesConfig"); if (rule_by_ctx.ok()) { BasicRuleConfig rule = rule_by_ctx.unpack(); opaque.setSavedData("assetId", rule.getAssetId(), EnvKeyAttr::LogSection::SOURCEANDDATA); @@ -917,6 +965,7 @@ private: FilterVerdict cur_verdict = http_manager->inspect(chunk, is_request); if (cur_verdict.getVerdict() == ACCEPT || cur_verdict.getVerdict() == DROP || + cur_verdict.getVerdict() == CUSTOM_RESPONSE || cur_verdict.getVerdict() == WAIT) { return cur_verdict; } @@ -1030,16 +1079,16 @@ private: handleChunkedData(ChunkType chunk_type, const Buffer &data, NginxAttachmentOpaque &opaque) { ScopedContext event_type; - event_type.registerValue("HTTP Chunk type", chunk_type); + event_type.registerValue("HTTP Chunk type", chunk_type); if (chunk_type > ChunkType::REQUEST_HEADER && opaque.getApplicationState() == ApplicationState::UNKOWN) { - auto rule_by_ctx = getConfiguration("rulebase", "rulesConfig"); + auto rule_by_ctx = getConfigurationWithCache("rulebase", "rulesConfig"); ApplicationState state = rule_by_ctx.ok() ? ApplicationState::DEFINED : ApplicationState::UNDEFINED; opaque.setApplicationState(state); } if (opaque.getApplicationState() == ApplicationState::UNDEFINED) { - ngx_http_cp_verdict_e verdict_action = + ServiceVerdict verdict_action = getSettingWithDefault(false, "allowOnlyDefinedApplications") ? DROP : ACCEPT; dbgDebug(D_NGINX_ATTACHMENT) @@ -1079,8 +1128,8 @@ private: case ChunkType::RESPONSE_END: return FilterVerdict(http_manager->inspectEndTransaction()); case ChunkType::METRIC_DATA_FROM_PLUGIN: - return FilterVerdict(ngx_http_cp_verdict::TRAFFIC_VERDICT_IRRELEVANT); - case ChunkType::HOLD_DATA: + return FilterVerdict(ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT); + case ChunkType::REQUEST_DELAYED_VERDICT: return FilterVerdict(http_manager->inspectDelayedVerdict()); case ChunkType::COUNT: break; @@ -1106,10 +1155,10 @@ private: << "Handling Injection of HTTP session modification data. Modifications amount: " << modifications_amount; - vector injection_data_persistency(modifications_amount); + vector injection_data_persistency(modifications_amount); for (const EventModifications &modifications : modifications_lists) { for (const ModificationBuffer &modification_buffer_list : modifications.second) { - ngx_http_cp_inject_data injection_data; + HttpInjectData injection_data; injection_data.orig_buff_index = modifications.first; injection_data.injection_pos = std::get<0>(modification_buffer_list); injection_data.mod_type = std::get<1>(modification_buffer_list); @@ -1149,22 +1198,9 @@ private: SharedMemoryIPC *ipc, vector &verdict_data, vector &verdict_data_sizes, - string web_user_response_id) + const WebTriggerConf &web_trigger_conf) { - ngx_http_cp_web_response_data_t web_response_data; - ScopedContext ctx; - if (web_user_response_id != "") { - dbgTrace(D_NGINX_ATTACHMENT) - << "web user response ID registered in contex: " - << web_user_response_id; - set triggers_set{web_user_response_id}; - ctx.registerValue>(TriggerMatcher::ctx_key, triggers_set); - } - WebTriggerConf web_trigger_conf = getConfigurationWithDefault( - WebTriggerConf::default_trigger_conf, - "rulebase", - "webUserResponse" - ); + HttpWebResponseData web_response_data; bool remove_event_id_param = getProfileAgentSettingWithDefault("false", "nginxAttachment.removeRedirectEventId") == "true"; @@ -1176,8 +1212,16 @@ private: uuid = opaque.getSessionUUID(); } web_response_data.uuid_size = uuid.size(); - - if (web_trigger_conf.getDetailsLevel() == "Redirect") { + string details_level = web_trigger_conf.getDetailsLevel(); + if (details_level == "Custom Block Page") { + auto body = web_trigger_conf.getResponseBody(); + auto content_type = web_trigger_conf.getContentType(); + auto response_code = web_trigger_conf.getResponseCode(); + string processed_body = process_html_content(body, uuid); + auto custom_response = CustomResponse(processed_body, response_code, content_type); + handleCustomResponse(ipc, verdict_data, verdict_data_sizes, custom_response); + return; + } else if (details_level == "Redirect") { web_response_data.response_data.redirect_data.redirect_location_size = web_trigger_conf.getRedirectURL().size(); bool add_event = web_trigger_conf.getAddEventId(); @@ -1186,19 +1230,19 @@ private: strlen("?event_id=") + uuid.size(); } web_response_data.response_data.redirect_data.add_event_id = add_event ? 1 : 0; - web_response_data.web_repsonse_type = static_cast(ngx_web_response_type_e::REDIRECT_WEB_RESPONSE); + web_response_data.web_response_type = static_cast(NanoWebResponseType::REDIRECT_WEB_RESPONSE); } else { web_response_data.response_data.custom_response_data.title_size = web_trigger_conf.getResponseTitle().size(); web_response_data.response_data.custom_response_data.body_size = web_trigger_conf.getResponseBody().size(); web_response_data.response_data.custom_response_data.response_code = web_trigger_conf.getResponseCode(); - web_response_data.web_repsonse_type = static_cast(ngx_web_response_type_e::CUSTOM_WEB_RESPONSE); + web_response_data.web_response_type = static_cast(NanoWebResponseType::CUSTOM_WEB_RESPONSE); } verdict_data.push_back(reinterpret_cast(&web_response_data)); - verdict_data_sizes.push_back(sizeof(ngx_http_cp_web_response_data_t)); + verdict_data_sizes.push_back(sizeof(HttpWebResponseData)); - if (web_trigger_conf.getDetailsLevel() == "Redirect") { + if (details_level == "Redirect") { redirectUrl = web_trigger_conf.getRedirectURL(); if (!remove_event_id_param && web_trigger_conf.getAddEventId()) { redirectUrl += "?event-id=" + uuid; @@ -1242,23 +1286,88 @@ private: << " (title size: " << static_cast(web_response_data.response_data.custom_response_data.title_size) << "), Body: " - << web_trigger_conf.getResponseBody() + << web_trigger_conf.getResponseBody() << " (body size: " << static_cast(web_response_data.response_data.custom_response_data.body_size) << "), UUID: " << uuid << " (UUID size: " << static_cast(web_response_data.uuid_size) + << "), Response Type: " + << static_cast(web_response_data.web_response_type) << ")"; } sendChunkedData(ipc, verdict_data_sizes.data(), verdict_data.data(), verdict_data.size()); } + string + process_html_content(const string &body, const string &uuid) + { + const string uuid_placeholder = ""; + size_t placeholder_pos = body.find(uuid_placeholder); + + if (placeholder_pos == string::npos) { + dbgTrace(D_NGINX_ATTACHMENT) << "No UUID placeholder found in body content"; + return body; + } + + dbgTrace(D_NGINX_ATTACHMENT) << "Found UUID placeholder at position: " << placeholder_pos; + + string incident_id_text = "Incident Id: " + uuid; + string processed_body = body; + processed_body.replace(placeholder_pos, uuid_placeholder.length(), incident_id_text); + + dbgTrace(D_NGINX_ATTACHMENT) + << "UUID replacement: original_len=" << body.length() + << ", processed_len=" << processed_body.length() + << ", uuid=" << uuid; + + return processed_body; + } + + void + handleCustomResponse( + SharedMemoryIPC *ipc, + vector &verdict_data, + vector &verdict_data_sizes, + const CustomResponse &custom_response_data) + { + HttpJsonResponseData json_response_data; + + json_response_data.response_code = custom_response_data.getStatusCode(); + string response_body = custom_response_data.getBody(); + string content_type = custom_response_data.getContentType(); + json_response_data.body_size = response_body.size(); + + if (content_type == "application/json") { + json_response_data.content_type = AttachmentContentType::CONTENT_TYPE_APPLICATION_JSON; + } else if (content_type == "text/html") { + json_response_data.content_type = AttachmentContentType::CONTENT_TYPE_TEXT_HTML; + } else { + json_response_data.content_type = AttachmentContentType::CONTENT_TYPE_OTHER; + } + + verdict_data.push_back(reinterpret_cast(&json_response_data)); + verdict_data_sizes.push_back(sizeof(HttpJsonResponseData)); + + verdict_data.push_back(reinterpret_cast(response_body.data())); + verdict_data_sizes.push_back(response_body.size()); + + dbgTrace(D_NGINX_ATTACHMENT) + << "Added Custom JSON Response to current session." + << " Response code: " + << static_cast(json_response_data.response_code) + << ", Body size: " + << static_cast(json_response_data.body_size); + + sendChunkedData(ipc, verdict_data_sizes.data(), verdict_data.data(), verdict_data.size()); + } + void handleVerdictResponse(const FilterVerdict &verdict, SharedMemoryIPC *ipc, SessionID session_id, bool is_header) { - ngx_http_cp_reply_from_service_t verdict_to_send; + HttpReplyFromService verdict_to_send; verdict_to_send.verdict = static_cast(verdict.getVerdict()); verdict_to_send.session_id = session_id; @@ -1281,7 +1390,31 @@ private: if (verdict.getVerdict() == DROP) { nginx_attachment_event.addTrafficVerdictCounter(nginxAttachmentEvent::trafficVerdict::DROP); verdict_to_send.modification_count = 1; - return handleCustomWebResponse(ipc, verdict_fragments, fragments_sizes, verdict.getWebUserResponseID()); + ScopedContext ctx; + set triggers_set{verdict.getWebUserResponseID()}; + ctx.registerValue>(TriggerMatcher::ctx_key, triggers_set); + auto web_trigger_config = getConfiguration("rulebase", "webUserResponse"); + WebTriggerConf web_trigger_conf = web_trigger_config.ok() ? + web_trigger_config.unpack() : WebTriggerConf::default_trigger_conf; + + if (web_trigger_conf.getDetailsLevel() == "Custom Block Page") { + dbgTrace(D_NGINX_ATTACHMENT) << "Changing verdict from DROP to CUSTOM_BLOCK_PAGE"; + verdict_to_send.verdict = static_cast(CUSTOM_RESPONSE); + } + return handleCustomWebResponse(ipc, verdict_fragments, fragments_sizes, web_trigger_conf); + } + + if (verdict.getVerdict() == CUSTOM_RESPONSE) { + verdict_to_send.modification_count = 1; + if (!verdict.getCustomResponse().ok()) { + dbgWarning(D_NGINX_ATTACHMENT) + << "Failed to get custom web response data. Returning default verdict: " + << ", Error: " + << verdict.getCustomResponse().getErr(); + return handleCustomResponse(ipc, verdict_fragments, fragments_sizes, + CustomResponse("{\"error\": \"Internal Server Error\"}", 500)); + } + return handleCustomResponse(ipc, verdict_fragments, fragments_sizes, verdict.getCustomResponse().unpack()); } if (verdict.getVerdict() == ACCEPT) { @@ -1329,7 +1462,7 @@ private: break; } - auto transaction_data = reinterpret_cast(incoming_data); + auto transaction_data = reinterpret_cast(incoming_data); if (transaction_data->session_id != cur_session_id) break; popData(attachment_ipc); @@ -1371,15 +1504,15 @@ private: } if (SHOULD_FAIL( - incoming_data_size >= sizeof(ngx_http_cp_request_data_t), + incoming_data_size >= sizeof(NanoHttpRequestData), IntentionalFailureHandler::FailureType::GetDataFromAttchment, &did_fail_on_purpose )) { dbgError(D_NGINX_ATTACHMENT) << "Corrupted transaction raw data received from NGINX attachment, size received: " << incoming_data_size - << " is lower than ngx_http_cp_request_data_t size=" - << sizeof(ngx_http_cp_request_data_t) + << " is lower than NanoHttpRequestData size=" + << sizeof(NanoHttpRequestData) << ". Resetting IPC" << dumpIpcWrapper(attachment_ipc) << (did_fail_on_purpose ? "[Intentional Failure]" : ""); @@ -1409,8 +1542,8 @@ private: return make_pair(corrupted_session_id, false); } - const ngx_http_cp_request_data_t *transaction_data = - reinterpret_cast(incoming_data); + const NanoHttpRequestData *transaction_data = + reinterpret_cast(incoming_data); Maybe chunked_data_type = convertToEnum(transaction_data->data_type); if (!chunked_data_type.ok()) { @@ -1426,8 +1559,8 @@ private: } if (chunked_data_type.unpack() == ChunkType::METRIC_DATA_FROM_PLUGIN) { - const ngx_http_cp_metric_data_t *recieved_metric_data = - reinterpret_cast(incoming_data); + const NanoHttpMetricData *recieved_metric_data = + reinterpret_cast(incoming_data); sendMetricToKibana(recieved_metric_data); popData(attachment_ipc); return pair(0, false); @@ -1478,7 +1611,7 @@ private: const Buffer inspection_data( transaction_data->data, - incoming_data_size - sizeof(ngx_http_cp_request_data_t), + incoming_data_size - sizeof(NanoHttpRequestData), Buffer::MemoryType::VOLATILE ); @@ -1522,6 +1655,7 @@ private: bool is_final_verdict = verdict.getVerdict() == ACCEPT || verdict.getVerdict() == DROP || + verdict.getVerdict() == CUSTOM_RESPONSE || verdict.getVerdict() == IRRELEVANT; dbgTrace(D_NGINX_ATTACHMENT) @@ -1638,6 +1772,8 @@ private: return "RECONF"; case WAIT: return "WAIT"; + case CUSTOM_RESPONSE: + return "CUSTOM_RESPONSE"; } dbgAssert(false) << alert << "Invalid EventVerdict enum: " << static_cast(verdict.getVerdict()); return string(); @@ -1716,7 +1852,16 @@ private: Maybe uid = getUidFromSocket(new_attachment_socket); Maybe nginx_user_id = readIdFromSocket(new_attachment_socket); Maybe nginx_group_id = readIdFromSocket(new_attachment_socket); + Maybe target_core = readSignedIdFromSocket(new_attachment_socket); DELAY_IF_NEEDED(IntentionalFailureHandler::FailureType::RegisterAttchment); + + // Target core failure should not block registration - maintain backward compatibility + bool target_core_available = target_core.ok(); + if (!target_core_available) { + dbgInfo(D_NGINX_ATTACHMENT) << "Failed to read target core (backward compatibility mode): " + << target_core.getErr() << ". Proceeding without affinity information."; + } + if (SHOULD_FAIL( nginx_user_id.ok() && nginx_group_id.ok() && uid.ok(), IntentionalFailureHandler::FailureType::RegisterAttchment, @@ -1743,6 +1888,7 @@ private: return; } + int32_t target_core_value = target_core_available ? *target_core : -1; if (!registerAttachmentProcess(*nginx_user_id, *nginx_group_id, new_attachment_socket)) { i_socket->closeSocket(new_attachment_socket); new_attachment_socket = -1; @@ -1753,6 +1899,48 @@ private: nginx_attachment_event.notify(); nginx_attachment_event.resetAllCounters(); dbgWarning(D_NGINX_ATTACHMENT) << "Failed to register attachment"; + } else { + // Set affinity to core based on received target core from NGINX + if (attachment_config.getIsPairedAffinityEnabled() + && target_core_available && target_core_value >= 0) { + auto uid_integer = getUidInteger(uid.unpack()); + if (!uid_integer.ok()) { + dbgWarning(D_NGINX_ATTACHMENT) + << "Failed to convert UID to integer. Error: " << uid_integer.getErr(); + } else { + dbgDebug(D_NGINX_ATTACHMENT) + << "Setting CPU affinity for NGINX attachment using target core from NGINX" + << " UID: " << uid_integer.unpack() + << ", unique_id: " << uid.unpack() + << ", received_target_core: " << target_core_value; + + // Use common affinity function + int result = set_affinity_to_core(target_core_value); + if (result == 0) { + dbgDebug(D_NGINX_ATTACHMENT) + << "Successfully set CPU affinity for NGINX attachment to core " + << target_core_value; + } else { + dbgWarning(D_NGINX_ATTACHMENT) + << "Failed to set CPU affinity for NGINX attachment to core " + << target_core_value + << ". Error code: " + << result; + } + } + } else if (!target_core_available) { + dbgTrace(D_NGINX_ATTACHMENT) + << "Target core not available from NGINX" + " (backward compatibility mode), skipping CPU affinity setting"; + } else if (target_core_value < 0) { + dbgTrace(D_NGINX_ATTACHMENT) + << "Paired affinity disabled by NGINX (target_core=" << target_core_value + << "), skipping CPU affinity setting for NGINX attachment"; + } else { + dbgTrace(D_NGINX_ATTACHMENT) + << "Paired affinity is not enabled in service config, " + "skipping setting CPU affinity for NGINX attachment"; + } } }; mainloop->addFileRoutine( @@ -1811,6 +1999,35 @@ private: return uid; } + Maybe + getUidInteger(string uid) + { + if (uid.empty()) { + return genError("UID is empty"); + } + + uint32_t uid_integer = 0; + try { + // Handle both container format "{container_id}_{worker_id}" and simple format "{worker_id}" + size_t underscore_pos = uid.find_last_of('_'); + string worker_id_str; + + if (underscore_pos != string::npos) { + // Container format: extract worker_id after the last underscore + worker_id_str = uid.substr(underscore_pos + 1); + } else { + // Simple format: entire string is the worker_id + worker_id_str = uid; + } + + uid_integer = stoul(worker_id_str); + } catch (const std::exception &e) { + return genError(string("Failed to convert UID to integer: ") + e.what()); + } + + return uid_integer; + } + Maybe readIdFromSocket(I_Socket::socketFd new_attachment_socket) { @@ -1833,6 +2050,36 @@ private: return attachment_id; } + Maybe + readSignedIdFromSocket(I_Socket::socketFd new_attachment_socket) + { + bool did_fail_on_purpose = false; + DELAY_IF_NEEDED(IntentionalFailureHandler::FailureType::ReceiveDataFromSocket); + + // Try to read target core - if this fails, it's likely backward compatibility + if (!i_socket->isDataAvailable(new_attachment_socket)) { + dbgTrace(D_NGINX_ATTACHMENT) + << "No target core data available on socket - likely backward compatibility mode"; + return genError("No target core data available on socket"); + } + + Maybe> id = i_socket->receiveData(new_attachment_socket, sizeof(int32_t)); + if (SHOULD_FAIL( + id.ok(), + IntentionalFailureHandler::FailureType::ReceiveDataFromSocket, + &did_fail_on_purpose + )) { + return genError( + string("Failed to read the signed attachment ID (likely backward compatibility issue): ") + + (did_fail_on_purpose ? "[Intentional Failure]" : id.getErr()) + ); + } + + int32_t attachment_id = *reinterpret_cast(id.unpack().data()); + dbgTrace(D_NGINX_ATTACHMENT) << "Signed Attachment ID: " << static_cast(attachment_id); + return attachment_id; + } + string static_resources_path; FilterVerdict default_verdict; FailopenModeListener fail_open_mode_listener; @@ -1852,6 +2099,7 @@ private: SharedMemoryIPC *attachment_ipc = nullptr; HttpAttachmentConfig attachment_config; I_MainLoop::RoutineID attachment_routine_id = 0; + I_MainLoop::RoutineID async_attachment_routine_id = 0; bool traffic_indicator = false; unordered_set ignored_headers; @@ -1882,7 +2130,267 @@ private: nginxIntakerMetric nginx_intaker_metric; TransactionTableEvent transaction_table_event; TransactionTableMetric transaction_table_metric; + + /// + /// @brief Async version of handleInspection - simplified for fastest response time + /// @return true on success, false on error + /// + // LCOV_EXCL_START Reason: Temporary INXT-49318 + bool + handleInspectionAsync() + { + Maybe> comm_trigger = genError("comm trigger uninitialized"); + + static map comm_status; + if (comm_status.find(attachment_sock) == comm_status.end()) { + comm_status[attachment_sock] = true; + } + + DELAY_IF_NEEDED(IntentionalFailureHandler::FailureType::ReceiveDataFromSocket); + + // Read session ID from socket (used as doorbell only) + uint32_t signaled_session_id = 0; + for (int retry = 0; retry < 3; retry++) { + comm_trigger = i_socket->receiveData(attachment_sock, sizeof(signaled_session_id)); + if (comm_trigger.ok()) break; + } + + bool did_fail_on_purpose = false; + if (SHOULD_FAIL( + comm_trigger.ok(), + IntentionalFailureHandler::FailureType::ReceiveDataFromSocket, + &did_fail_on_purpose + )) { + if (comm_status[attachment_sock] == true) { + dbgDebug(D_NGINX_ATTACHMENT) + << "Failed to get signal from attachment socket (async mode)" + << ", Socket: " << attachment_sock + << ", Error: " << (did_fail_on_purpose ? "Intentional Failure" : comm_trigger.getErr()); + comm_status[attachment_sock] = false; + } + return false; + } + + signaled_session_id = *reinterpret_cast(comm_trigger.unpack().data()); + comm_status.erase(attachment_sock); + traffic_indicator = true; + + while (isDataAvailable(attachment_ipc)) { + traffic_indicator = true; + + uint32_t handled_session_id = handleRequestFromQueueAsync(attachment_ipc); + + if (handled_session_id == 0 || handled_session_id == corrupted_session_id) { + continue; + } + + // Always signal back to nginx - never leave it waiting + dbgTrace(D_NGINX_ATTACHMENT) << "Signaling attachment to read verdict (async mode)"; + bool res = false; + vector session_id_data( + reinterpret_cast(&handled_session_id), + reinterpret_cast(&handled_session_id) + sizeof(handled_session_id) + ); + + DELAY_IF_NEEDED(IntentionalFailureHandler::FailureType::WriteDataToSocket); + + if (!SHOULD_FAIL( + true, + IntentionalFailureHandler::FailureType::WriteDataToSocket, + &did_fail_on_purpose + )) { + for (int retry = 0; retry < 3; retry++) { + if (i_socket->writeDataAsync(attachment_sock, session_id_data)) { + dbgTrace(D_NGINX_ATTACHMENT) + << "Successfully sent signal to attachment (async mode)."; + res = true; + break; + } + dbgDebug(D_NGINX_ATTACHMENT) + << "Failed to send ACK to attachment (async mode, try " << retry << ")"; + mainloop->yield(true); + } + } + + if (!res) { + dbgWarning(D_NGINX_ATTACHMENT) << "Failed to send ACK to attachment (async mode)" + << (did_fail_on_purpose ? "[Intentional Failure]" : ""); + if (!did_fail_on_purpose) { + dbgWarning(D_NGINX_ATTACHMENT) << "Resetting IPC and socket"; + resetIpc(attachment_ipc, num_of_nginx_ipc_elements); + } + return false; + } + } + + return true; + } +// LCOV_EXCL_STOP + + /// + /// @brief Async version of handleRequestFromQueue - always signals back to nginx + /// @param[in] attachment_ipc IPC channel + /// @param[in] signaled_session_id Session ID that triggered the signal (not used, doorbell only) + /// @return session_id of processed request + /// +// LCOV_EXCL_START Reason: Temporary INXT-49318 + uint32_t + handleRequestFromQueueAsync(SharedMemoryIPC *attachment_ipc) + { + Maybe> read_data = readData(attachment_ipc); + if (!read_data.ok()) { + dbgWarning(D_NGINX_ATTACHMENT) << "Failed to read data. Error: " << read_data.getErr(); + return corrupted_session_id; + } + + uint16_t incoming_data_size = read_data.unpack().first; + const char *incoming_data = read_data.unpack().second; + if (incoming_data_size == 0 || incoming_data == nullptr) { + dbgWarning(D_NGINX_ATTACHMENT) << "No data received from NGINX attachment"; + return corrupted_session_id; + } + + const NanoHttpRequestData *transaction_data = + reinterpret_cast(incoming_data); + + Maybe chunked_data_type = convertToEnum(transaction_data->data_type); + if (!chunked_data_type.ok()) { + dbgWarning(D_NGINX_ATTACHMENT) + << "Could not convert " + << static_cast(transaction_data->data_type) + << " to ChunkType enum. Resetting IPC" + << dumpIpcWrapper(attachment_ipc); + popData(attachment_ipc); + resetIpc(attachment_ipc, num_of_nginx_ipc_elements); + nginx_attachment_event.addNetworkingCounter(nginxAttachmentEvent::networkVerdict::CONNECTION_FAIL); + return corrupted_session_id; + } + + if (chunked_data_type.unpack() == ChunkType::METRIC_DATA_FROM_PLUGIN) { + const NanoHttpMetricData *recieved_metric_data = + reinterpret_cast(incoming_data); + sendMetricToKibana(recieved_metric_data); + popData(attachment_ipc); + return 0; // No signaling needed for metrics + } + + dbgTrace(D_NGINX_ATTACHMENT) + << "Reading " + << incoming_data_size + <<" bytes " + << convertChunkTypeToString(*chunked_data_type) + << "(type = " + << static_cast(*chunked_data_type) + << ") of data from NGINX attachment for session ID: " + << transaction_data->session_id; + + const uint32_t cur_session_id = transaction_data->session_id; + + if ( + isFailOpenTriggered() + && ( + chunked_data_type.unpack() == ChunkType::REQUEST_START + || chunked_data_type.unpack() == ChunkType::REQUEST_HEADER + ) + ) { + dbgTrace(D_NGINX_ATTACHMENT) + << "Agent is set to Fail Open Mode. Passing inspection and returning Accept." + << " Session ID: " + << cur_session_id + << ", Chunked data type: " + << static_cast(*chunked_data_type); + + if (i_transaction_table->hasEntry(cur_session_id)) { + i_transaction_table->deleteEntry(cur_session_id); + } + + popData(attachment_ipc); + handleVerdictResponse( + FilterVerdict(ACCEPT), + attachment_ipc, + cur_session_id, + false + ); + return cur_session_id; + } + + if (!setActiveTransactionEntry(cur_session_id, chunked_data_type.unpack())) { + popData(attachment_ipc); + return cur_session_id; + } + + const Buffer inspection_data( + transaction_data->data, + incoming_data_size - sizeof(NanoHttpRequestData), + Buffer::MemoryType::VOLATILE + ); + + if (*chunked_data_type == ChunkType::REQUEST_START && !createTransactionState(inspection_data)) { + dbgWarning(D_NGINX_ATTACHMENT) + << "Failed to handle new request. Returning default verdict: " + << verdictToString(default_verdict.getVerdict()); + + handleVerdictResponse( + default_verdict, + attachment_ipc, + cur_session_id, + false + ); + popData(attachment_ipc); + removeTransactionEntry(cur_session_id); + return cur_session_id; + } + + if (i_transaction_table != nullptr) { + transaction_table_event.setTransactionTableSize(i_transaction_table->count()); + transaction_table_event.notify(); + } + + NginxAttachmentOpaque &opaque = i_transaction_table->getState(); + opaque.activateContext(); + + FilterVerdict verdict = handleChunkedData(*chunked_data_type, inspection_data, opaque); + bool is_header = + *chunked_data_type == ChunkType::REQUEST_HEADER || + *chunked_data_type == ChunkType::RESPONSE_HEADER || + *chunked_data_type == ChunkType::CONTENT_LENGTH; + + if (verdict.getVerdict() == LIMIT_RESPONSE_HEADERS) { + handleVerdictResponse(verdict, attachment_ipc, transaction_data->session_id, is_header); + popData(attachment_ipc); + verdict = FilterVerdict(INSPECT); + } + + handleVerdictResponse(verdict, attachment_ipc, cur_session_id, is_header); + + bool is_final_verdict = verdict.getVerdict() == ACCEPT || + verdict.getVerdict() == DROP || + verdict.getVerdict() == CUSTOM_RESPONSE || + verdict.getVerdict() == IRRELEVANT; + + dbgTrace(D_NGINX_ATTACHMENT) + << "Request handled successfully - for" + << " NGINX attachment session ID: " + << transaction_data->session_id + << " verdict: " + << verdictToString(verdict.getVerdict()) + << " verdict_data_code=" + << static_cast(verdict.getVerdict()); + + popData(attachment_ipc); + + opaque.deactivateContext(); + if (is_final_verdict) { + removeTransactionEntry(cur_session_id); + } else { + i_transaction_table->unsetActiveKey(); + } + + // Always signal back in async mode - never leave nginx waiting + return cur_session_id; + } }; +// LCOV_EXCL_STOP NginxAttachment::NginxAttachment() : Component("NginxAttachment"), pimpl(make_unique()) {} @@ -1921,7 +2429,7 @@ NginxAttachment::preload() registerExpectedConfiguration("Nginx Attachment", "metric reporting interval"); registerExpectedSetting("allowOnlyDefinedApplications"); registerExpectedConfigFile("activeContextConfig", Config::ConfigFileType::Policy); - registerExpectedConfiguration("rulebase", "usersIdentifiers"); + registerExpectedConfigurationWithCache("assetId", "rulebase", "usersIdentifiers"); BasicRuleConfig::preload(); WebTriggerConf::preload(); } diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment_config.cc b/components/attachment-intakers/nginx_attachment/nginx_attachment_config.cc index ccbf5c4..8e65627 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment_config.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment_config.cc @@ -14,6 +14,7 @@ #include "nginx_attachment_config.h" #include +#include #include "nginx_attachment.h" #include "config.h" @@ -25,7 +26,7 @@ USE_DEBUG_FLAG(D_NGINX_ATTACHMENT); using namespace std; -using DebugLevel = ngx_http_cp_debug_level_e; +using DebugLevel = nano_http_cp_debug_level_e; void HttpAttachmentConfig::init() @@ -43,6 +44,8 @@ HttpAttachmentConfig::init() setDebugByContextValues(); setKeepAliveIntervalMsec(); setRetriesForVerdict(); + setPairedAffinityEnabled(); + setAsyncMode(); } bool @@ -210,15 +213,36 @@ HttpAttachmentConfig::setFailOpenTimeout() "Response server header removal" )); + conf_data.setNumericalValue("decompression_pool_size", getAttachmentConf( + 262144, + "agent.decompressionPoolSize.nginxModule", + "HTTP manager", + "Decompression pool size in bytes" + )); + + conf_data.setNumericalValue("recompression_pool_size", getAttachmentConf( + 16384, + "agent.recompressionPoolSize.nginxModule", + "HTTP manager", + "Recompression pool size in bytes" + )); + + conf_data.setNumericalValue("is_brotli_inspection_enabled", getAttachmentConf( + 0, + "agent.isBrotliInspectionEnabled.nginxModule", + "HTTP manager", + "Brotli inspection enabled" + )); + uint inspection_mode = getAttachmentConf( - static_cast(ngx_http_inspection_mode_e::NON_BLOCKING_THREAD), + static_cast(NanoHttpInspectionMode::NON_BLOCKING_THREAD), "agent.inspectionMode.nginxModule", "HTTP manager", "NGINX inspection mode" ); - if (inspection_mode >= ngx_http_inspection_mode_e::INSPECTION_MODE_COUNT) { - inspection_mode = ngx_http_inspection_mode_e::NON_BLOCKING_THREAD; + if (inspection_mode >= static_cast(NanoHttpInspectionMode::INSPECTION_MODE_COUNT)) { + inspection_mode = static_cast(NanoHttpInspectionMode::NON_BLOCKING_THREAD); } conf_data.setNumericalValue("nginx_inspection_mode", inspection_mode); } @@ -377,3 +401,45 @@ HttpAttachmentConfig::setDebugByContextValues() << ", listening_port: " << new_ctx_cfg.port; } + +void +HttpAttachmentConfig::setPairedAffinityEnabled() { + bool is_paired_affinity_enabled = getAttachmentConf( + false, + "agent.pairedAffinity.nginxModule", + "HTTP manager", + "Paired Affinity state" + ); + + // Check environment variable to allow runtime override + const char* env_enable_affinity = getenv("ENABLE_AFFINITY_PAIRING"); + if (env_enable_affinity != nullptr) { + string env_value(env_enable_affinity); + // Convert to lowercase for case insensitive comparison + transform(env_value.begin(), env_value.end(), env_value.begin(), ::tolower); + is_paired_affinity_enabled = (env_value == "true"); + dbgTrace(D_NGINX_ATTACHMENT) + << "Paired affinity overridden by environment variable ENABLE_AFFINITY_PAIRING=" + << env_enable_affinity; + } + + dbgTrace(D_NGINX_ATTACHMENT) + << "Attachment paired affinity is: " + << (is_paired_affinity_enabled ? "Enabled" : "Disabled"); + conf_data.setNumericalValue("is_paired_affinity_enabled", is_paired_affinity_enabled); +} + +void +HttpAttachmentConfig::setAsyncMode() { + bool is_async_mode_enabled = getAttachmentConf( + false, + "agent.asyncMode.nginxModule", + "HTTP manager", + "Async Mode state" + ); + + dbgTrace(D_NGINX_ATTACHMENT) + << "Attachment async mode is: " + << (is_async_mode_enabled ? "Enabled" : "Disabled"); + conf_data.setNumericalValue("is_async_mode_enabled", is_async_mode_enabled); +} diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment_config.h b/components/attachment-intakers/nginx_attachment/nginx_attachment_config.h index df7ae6e..28e6604 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment_config.h +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment_config.h @@ -44,6 +44,10 @@ public: unsigned int getMaxSessionsPerMinute() const { return conf_data.getNumericalValue("max_sessions_per_minute"); } unsigned int getNumOfNginxElements() const { return conf_data.getNumericalValue("num_of_nginx_ipc_elements"); } unsigned int getKeepAliveIntervalMsec() const { return conf_data.getNumericalValue("keep_alive_interval_msec"); } + bool getIsPairedAffinityEnabled() const { return conf_data.getNumericalValue("is_paired_affinity_enabled"); } +// LCOV_EXCL_START Reason: Temporary INXT-49318 + bool isAsyncModeEnabled() const { return conf_data.getNumericalValue("is_async_mode_enabled"); } +// LCOV_EXCL_STOP private: void setGradualDeploymentIPs(); @@ -72,6 +76,10 @@ private: void setRetriesForVerdict(); + void setPairedAffinityEnabled(); + + void setAsyncMode(); + WebTriggerConf web_trigger_conf; HttpAttachmentConfiguration conf_data; }; diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc b/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc index 8f6263c..5c407b0 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc @@ -51,18 +51,19 @@ NginxAttachmentOpaque::NginxAttachmentOpaque(HttpTransactionData _transaction_da std::stringstream client_ip_str; client_ip_str << client_ip; setSourceIdentifier("sourceip", client_ip_str.str()); - ctx.registerValue("eventReferenceId", uuid, EnvKeyAttr::LogSection::DATA); ctx.registerValue(HttpTransactionData::http_proto_ctx, transaction_data.getHttpProtocol()); ctx.registerValue(HttpTransactionData::method_ctx, transaction_data.getHttpMethod()); - ctx.registerValue(HttpTransactionData::host_name_ctx, transaction_data.getParsedHost()); - ctx.registerValue(HttpTransactionData::listening_port_ctx, transaction_data.getListeningPort()); + ctx.registerQuickAccessValue(HttpTransactionData::host_name_ctx, transaction_data.getParsedHost()); + ctx.registerQuickAccessValue( + HttpTransactionData::listening_port_ctx, + transaction_data.getListeningPort()); ctx.registerValue(HttpTransactionData::listening_ip_ctx, transaction_data.getListeningIP()); ctx.registerValue(HttpTransactionData::client_ip_ctx, transaction_data.getSourceIP()); ctx.registerValue(HttpTransactionData::client_port_ctx, transaction_data.getSourcePort()); ctx.registerFunc(HttpTransactionData::source_identifier, [this](){ return source_identifier; }); - ctx.registerValue(HttpTransactionData::uri_ctx, transaction_data.getParsedURI()); + ctx.registerQuickAccessValue(HttpTransactionData::uri_ctx, transaction_data.getParsedURI()); auto decoder = makeVirtualContainer>(transaction_data.getURI()); string decoded_url(decoder.begin(), decoder.end()); auto question_mark_location = decoded_url.find('?'); @@ -143,6 +144,7 @@ NginxAttachmentOpaque::setKeepAliveCtx(const string &hdr_key, const string &hdr_ ctx.registerValue("keep_alive_request_ctx", true); return true; } + dbgTrace(D_HTTP_MANAGER) << "Not a keep alive header"; return false; } diff --git a/components/attachment-intakers/nginx_attachment/nginx_intaker_metric.cc b/components/attachment-intakers/nginx_attachment/nginx_intaker_metric.cc index fd2cf93..ea769e9 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_intaker_metric.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_intaker_metric.cc @@ -54,204 +54,204 @@ nginxIntakerEvent::resetAllCounters() cpu_event.setCPU(0); } -ngx_http_plugin_metric_type_e +AttachmentMetricType nginxIntakerEvent::EnumOfIndex(int i) { - return static_cast(i); + return static_cast(i); } void -nginxIntakerEvent::addPluginMetricCounter(const ngx_http_cp_metric_data_t *recieved_metric_data) +nginxIntakerEvent::addPluginMetricCounter(const NanoHttpMetricData *recieved_metric_data) { - for (int i = 0; i < static_cast(ngx_http_plugin_metric_type_e::METRIC_TYPES_COUNT); i++) { - ngx_http_plugin_metric_type_e metric_type = EnumOfIndex(i); + for (int i = 0; i < static_cast(AttachmentMetricType::METRIC_TYPES_COUNT); i++) { + AttachmentMetricType metric_type = EnumOfIndex(i); uint64_t amount = recieved_metric_data->data[i]; switch (metric_type) { - case ngx_http_plugin_metric_type_e::INSPECTION_SUCCESSES_COUNT: { + case AttachmentMetricType::INSPECTION_SUCCESSES_COUNT: { successfull_inspection_counter += amount; break; } - case ngx_http_plugin_metric_type_e::INSPECTION_OPEN_FAILURES_COUNT: { + case AttachmentMetricType::INSPECTION_OPEN_FAILURES_COUNT: { open_failure_inspection_counter += amount; break; } - case ngx_http_plugin_metric_type_e::INSPECTION_CLOSE_FAILURES_COUNT: { + case AttachmentMetricType::INSPECTION_CLOSE_FAILURES_COUNT: { close_failure_inspection_counter += amount; break; } - case ngx_http_plugin_metric_type_e::TRANSPARENTS_COUNT: { + case AttachmentMetricType::TRANSPARENTS_COUNT: { transparent_mode_counter += amount; break; } - case ngx_http_plugin_metric_type_e::TOTAL_TRANSPARENTS_TIME: { + case AttachmentMetricType::TOTAL_TRANSPARENTS_TIME: { total_transparent_time += amount; break; } - case ngx_http_plugin_metric_type_e::INSPECT_VERDICTS_COUNT: { + case AttachmentMetricType::INSPECT_VERDICTS_COUNT: { inspect_verdict_counter += amount; break; } - case ngx_http_plugin_metric_type_e::ACCEPT_VERDICTS_COUNT: { + case AttachmentMetricType::ACCEPT_VERDICTS_COUNT: { accept_verdict_counter += amount; break; } - case ngx_http_plugin_metric_type_e::DROP_VERDICTS_COUNT: { + case AttachmentMetricType::DROP_VERDICTS_COUNT: { drop_verdict_counter += amount; break; } - case ngx_http_plugin_metric_type_e::INJECT_VERDICTS_COUNT: { + case AttachmentMetricType::INJECT_VERDICTS_COUNT: { inject_verdict_counter += amount; break; } - case ngx_http_plugin_metric_type_e::IRRELEVANT_VERDICTS_COUNT: { + case AttachmentMetricType::IRRELEVANT_VERDICTS_COUNT: { irrelevant_verdict_counter += amount; break; } - case ngx_http_plugin_metric_type_e::RECONF_VERDICTS_COUNT: { + case AttachmentMetricType::RECONF_VERDICTS_COUNT: { reconf_verdict_counter += amount; break; } - case ngx_http_plugin_metric_type_e::AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: { + case AttachmentMetricType::AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: { if (amount > 0) average_overall_processing_time_until_verdict = amount; break; } - case ngx_http_plugin_metric_type_e::MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: { + case AttachmentMetricType::MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: { if (amount > 0) max_overall_processing_time_until_verdict = amount; break; } - case ngx_http_plugin_metric_type_e::MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: { + case AttachmentMetricType::MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: { if (amount > 0) min_overall_processing_time_until_verdict = amount; break; } - case ngx_http_plugin_metric_type_e::AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT: { + case AttachmentMetricType::AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT: { if (amount > 0) average_req_processing_time_until_verdict = amount; break; } - case ngx_http_plugin_metric_type_e::MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT: { + case AttachmentMetricType::MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT: { if (amount > 0) max_req_processing_time_until_verdict = amount; break; } - case ngx_http_plugin_metric_type_e::MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT: { + case AttachmentMetricType::MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT: { if (amount > 0) min_req_processing_time_until_verdict = amount; break; } - case ngx_http_plugin_metric_type_e::AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT: { + case AttachmentMetricType::AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT: { if (amount > 0) average_res_processing_time_until_verdict = amount; break; } - case ngx_http_plugin_metric_type_e::MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT: { + case AttachmentMetricType::MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT: { if (amount > 0) max_res_processing_time_until_verdict = amount; break; } - case ngx_http_plugin_metric_type_e::MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT: { + case AttachmentMetricType::MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT: { if (amount > 0) min_res_processing_time_until_verdict = amount; break; } - case ngx_http_plugin_metric_type_e::REQ_FAILED_COMPRESSION_COUNT: { + case AttachmentMetricType::REQ_FAILED_COMPRESSION_COUNT: { req_failed_compression_counter += amount; break; } - case ngx_http_plugin_metric_type_e::RES_FAILED_COMPRESSION_COUNT: { + case AttachmentMetricType::RES_FAILED_COMPRESSION_COUNT: { res_failed_compression_counter += amount; break; } - case ngx_http_plugin_metric_type_e::REQ_FAILED_DECOMPRESSION_COUNT: { + case AttachmentMetricType::REQ_FAILED_DECOMPRESSION_COUNT: { req_failed_decompression_counter += amount; break; } - case ngx_http_plugin_metric_type_e::RES_FAILED_DECOMPRESSION_COUNT: { + case AttachmentMetricType::RES_FAILED_DECOMPRESSION_COUNT: { res_failed_decompression_counter += amount; break; } - case ngx_http_plugin_metric_type_e::REQ_SUCCESSFUL_COMPRESSION_COUNT: { + case AttachmentMetricType::REQ_SUCCESSFUL_COMPRESSION_COUNT: { req_successful_compression_counter += amount; break; } - case ngx_http_plugin_metric_type_e::RES_SUCCESSFUL_COMPRESSION_COUNT: { + case AttachmentMetricType::RES_SUCCESSFUL_COMPRESSION_COUNT: { res_successful_compression_counter += amount; break; } - case ngx_http_plugin_metric_type_e::REQ_SUCCESSFUL_DECOMPRESSION_COUNT: { + case AttachmentMetricType::REQ_SUCCESSFUL_DECOMPRESSION_COUNT: { req_successful_decompression_counter += amount; break; } - case ngx_http_plugin_metric_type_e::RES_SUCCESSFUL_DECOMPRESSION_COUNT: { + case AttachmentMetricType::RES_SUCCESSFUL_DECOMPRESSION_COUNT: { res_successful_decompression_counter += amount; break; } - case ngx_http_plugin_metric_type_e::CORRUPTED_ZIP_SKIPPED_SESSION_COUNT: { + case AttachmentMetricType::CORRUPTED_ZIP_SKIPPED_SESSION_COUNT: { corrupted_zip_skipped_session_counter += amount; break; } - case ngx_http_plugin_metric_type_e::THREAD_TIMEOUT: { + case AttachmentMetricType::THREAD_TIMEOUT: { thread_timeout += amount; break; } - case ngx_http_plugin_metric_type_e::REG_THREAD_TIMEOUT: { + case AttachmentMetricType::REG_THREAD_TIMEOUT: { reg_thread_timeout += amount; break; } - case ngx_http_plugin_metric_type_e::REQ_HEADER_THREAD_TIMEOUT: { + case AttachmentMetricType::REQ_HEADER_THREAD_TIMEOUT: { req_header_thread_timeout += amount; break; } - case ngx_http_plugin_metric_type_e::REQ_BODY_THREAD_TIMEOUT: { + case AttachmentMetricType::REQ_BODY_THREAD_TIMEOUT: { req_body_thread_timeout += amount; break; } - case ngx_http_plugin_metric_type_e::AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT: { + case AttachmentMetricType::AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT: { if (amount > 0) average_req_body_size_upon_timeout = amount; break; } - case ngx_http_plugin_metric_type_e::MAX_REQ_BODY_SIZE_UPON_TIMEOUT: { + case AttachmentMetricType::MAX_REQ_BODY_SIZE_UPON_TIMEOUT: { if (amount > 0) max_req_body_size_upon_timeout = amount; break; } - case ngx_http_plugin_metric_type_e::MIN_REQ_BODY_SIZE_UPON_TIMEOUT: { + case AttachmentMetricType::MIN_REQ_BODY_SIZE_UPON_TIMEOUT: { if (amount > 0) min_req_body_size_upon_timeout = amount; break; } - case ngx_http_plugin_metric_type_e::RES_HEADER_THREAD_TIMEOUT: { + case AttachmentMetricType::RES_HEADER_THREAD_TIMEOUT: { res_header_thread_timeout += amount; break; } - case ngx_http_plugin_metric_type_e::RES_BODY_THREAD_TIMEOUT: { + case AttachmentMetricType::RES_BODY_THREAD_TIMEOUT: { res_body_thread_timeout += amount; break; } - case ngx_http_plugin_metric_type_e::AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT: { + case AttachmentMetricType::AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT: { if (amount > 0) average_res_body_size_upon_timeout = amount; break; } - case ngx_http_plugin_metric_type_e::MAX_RES_BODY_SIZE_UPON_TIMEOUT: { + case AttachmentMetricType::MAX_RES_BODY_SIZE_UPON_TIMEOUT: { if (amount > 0) max_res_body_size_upon_timeout = amount; break; } - case ngx_http_plugin_metric_type_e::MIN_RES_BODY_SIZE_UPON_TIMEOUT: { + case AttachmentMetricType::MIN_RES_BODY_SIZE_UPON_TIMEOUT: { if (amount > 0) min_res_body_size_upon_timeout = amount; break; } - case ngx_http_plugin_metric_type_e::THREAD_FAILURE: { + case AttachmentMetricType::THREAD_FAILURE: { thread_failure += amount; break; } - case ngx_http_plugin_metric_type_e::REQ_PROCCESSING_TIMEOUT: { + case AttachmentMetricType::REQ_PROCCESSING_TIMEOUT: { req_proccessing_timeout += amount; break; } - case ngx_http_plugin_metric_type_e::RES_PROCCESSING_TIMEOUT: { + case AttachmentMetricType::RES_PROCCESSING_TIMEOUT: { res_proccessing_timeout += amount; break; } - case ngx_http_plugin_metric_type_e::REQ_FAILED_TO_REACH_UPSTREAM: { + case AttachmentMetricType::REQ_FAILED_TO_REACH_UPSTREAM: { req_failed_to_reach_upstream += amount; break; } - case ngx_http_plugin_metric_type_e::CPU_USAGE: { + case AttachmentMetricType::CPU_USAGE: { cpu_event.setCPU(amount); break; } - case ngx_http_plugin_metric_type_e::REQUEST_OVERALL_SIZE_COUNT: { + case AttachmentMetricType::REQUEST_OVERALL_SIZE_COUNT: { req_overall_size += amount; static const uint64_t max_expected_res_size = 100ULL * 1024 * 1024 * 1024; if (amount > max_expected_res_size) { @@ -259,7 +259,7 @@ nginxIntakerEvent::addPluginMetricCounter(const ngx_http_cp_metric_data_t *recie } break; } - case ngx_http_plugin_metric_type_e::RESPONSE_OVERALL_SIZE_COUNT: { + case AttachmentMetricType::RESPONSE_OVERALL_SIZE_COUNT: { res_overall_size += amount; break; } @@ -272,104 +272,104 @@ nginxIntakerEvent::addPluginMetricCounter(const ngx_http_cp_metric_data_t *recie } uint64_t -nginxIntakerEvent::getPluginMetricCounter(ngx_http_plugin_metric_type_e metric_type) const +nginxIntakerEvent::getPluginMetricCounter(AttachmentMetricType metric_type) const { switch (metric_type) { - case ngx_http_plugin_metric_type_e::INSPECTION_SUCCESSES_COUNT: + case AttachmentMetricType::INSPECTION_SUCCESSES_COUNT: return successfull_inspection_counter; - case ngx_http_plugin_metric_type_e::INSPECTION_OPEN_FAILURES_COUNT: + case AttachmentMetricType::INSPECTION_OPEN_FAILURES_COUNT: return open_failure_inspection_counter; - case ngx_http_plugin_metric_type_e::INSPECTION_CLOSE_FAILURES_COUNT: + case AttachmentMetricType::INSPECTION_CLOSE_FAILURES_COUNT: return close_failure_inspection_counter; - case ngx_http_plugin_metric_type_e::TRANSPARENTS_COUNT: + case AttachmentMetricType::TRANSPARENTS_COUNT: return transparent_mode_counter; - case ngx_http_plugin_metric_type_e::TOTAL_TRANSPARENTS_TIME: + case AttachmentMetricType::TOTAL_TRANSPARENTS_TIME: return total_transparent_time; - case ngx_http_plugin_metric_type_e::INSPECT_VERDICTS_COUNT: + case AttachmentMetricType::INSPECT_VERDICTS_COUNT: return inspect_verdict_counter; - case ngx_http_plugin_metric_type_e::ACCEPT_VERDICTS_COUNT: + case AttachmentMetricType::ACCEPT_VERDICTS_COUNT: return accept_verdict_counter; - case ngx_http_plugin_metric_type_e::DROP_VERDICTS_COUNT: + case AttachmentMetricType::DROP_VERDICTS_COUNT: return drop_verdict_counter; - case ngx_http_plugin_metric_type_e::INJECT_VERDICTS_COUNT: + case AttachmentMetricType::INJECT_VERDICTS_COUNT: return inject_verdict_counter; - case ngx_http_plugin_metric_type_e::IRRELEVANT_VERDICTS_COUNT: + case AttachmentMetricType::IRRELEVANT_VERDICTS_COUNT: return irrelevant_verdict_counter; - case ngx_http_plugin_metric_type_e::RECONF_VERDICTS_COUNT: + case AttachmentMetricType::RECONF_VERDICTS_COUNT: return reconf_verdict_counter; - case ngx_http_plugin_metric_type_e::AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: + case AttachmentMetricType::AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: return average_overall_processing_time_until_verdict; - case ngx_http_plugin_metric_type_e::MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: + case AttachmentMetricType::MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: return max_overall_processing_time_until_verdict; - case ngx_http_plugin_metric_type_e::MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: + case AttachmentMetricType::MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: return min_overall_processing_time_until_verdict; - case ngx_http_plugin_metric_type_e::AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT: + case AttachmentMetricType::AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT: return average_req_processing_time_until_verdict; - case ngx_http_plugin_metric_type_e::MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT: + case AttachmentMetricType::MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT: return max_req_processing_time_until_verdict; - case ngx_http_plugin_metric_type_e::MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT: + case AttachmentMetricType::MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT: return min_req_processing_time_until_verdict; - case ngx_http_plugin_metric_type_e::AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT: + case AttachmentMetricType::AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT: return average_res_processing_time_until_verdict; - case ngx_http_plugin_metric_type_e::MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT: + case AttachmentMetricType::MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT: return max_res_processing_time_until_verdict; - case ngx_http_plugin_metric_type_e::MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT: + case AttachmentMetricType::MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT: return min_res_processing_time_until_verdict; - case ngx_http_plugin_metric_type_e::REQ_FAILED_COMPRESSION_COUNT: + case AttachmentMetricType::REQ_FAILED_COMPRESSION_COUNT: return req_failed_compression_counter; - case ngx_http_plugin_metric_type_e::RES_FAILED_COMPRESSION_COUNT: + case AttachmentMetricType::RES_FAILED_COMPRESSION_COUNT: return res_failed_compression_counter; - case ngx_http_plugin_metric_type_e::REQ_FAILED_DECOMPRESSION_COUNT: + case AttachmentMetricType::REQ_FAILED_DECOMPRESSION_COUNT: return req_failed_decompression_counter; - case ngx_http_plugin_metric_type_e::RES_FAILED_DECOMPRESSION_COUNT: + case AttachmentMetricType::RES_FAILED_DECOMPRESSION_COUNT: return res_failed_decompression_counter; - case ngx_http_plugin_metric_type_e::REQ_SUCCESSFUL_COMPRESSION_COUNT: + case AttachmentMetricType::REQ_SUCCESSFUL_COMPRESSION_COUNT: return req_successful_compression_counter; - case ngx_http_plugin_metric_type_e::RES_SUCCESSFUL_COMPRESSION_COUNT: + case AttachmentMetricType::RES_SUCCESSFUL_COMPRESSION_COUNT: return res_successful_compression_counter; - case ngx_http_plugin_metric_type_e::REQ_SUCCESSFUL_DECOMPRESSION_COUNT: + case AttachmentMetricType::REQ_SUCCESSFUL_DECOMPRESSION_COUNT: return req_successful_decompression_counter; - case ngx_http_plugin_metric_type_e::RES_SUCCESSFUL_DECOMPRESSION_COUNT: + case AttachmentMetricType::RES_SUCCESSFUL_DECOMPRESSION_COUNT: return res_successful_decompression_counter; - case ngx_http_plugin_metric_type_e::CORRUPTED_ZIP_SKIPPED_SESSION_COUNT: + case AttachmentMetricType::CORRUPTED_ZIP_SKIPPED_SESSION_COUNT: return corrupted_zip_skipped_session_counter; - case ngx_http_plugin_metric_type_e::THREAD_TIMEOUT: + case AttachmentMetricType::THREAD_TIMEOUT: return thread_timeout; - case ngx_http_plugin_metric_type_e::REG_THREAD_TIMEOUT: + case AttachmentMetricType::REG_THREAD_TIMEOUT: return reg_thread_timeout; - case ngx_http_plugin_metric_type_e::REQ_HEADER_THREAD_TIMEOUT: + case AttachmentMetricType::REQ_HEADER_THREAD_TIMEOUT: return req_header_thread_timeout; - case ngx_http_plugin_metric_type_e::REQ_BODY_THREAD_TIMEOUT: + case AttachmentMetricType::REQ_BODY_THREAD_TIMEOUT: return req_body_thread_timeout; - case ngx_http_plugin_metric_type_e::AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT: + case AttachmentMetricType::AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT: return average_req_body_size_upon_timeout; - case ngx_http_plugin_metric_type_e::MAX_REQ_BODY_SIZE_UPON_TIMEOUT: + case AttachmentMetricType::MAX_REQ_BODY_SIZE_UPON_TIMEOUT: return max_req_body_size_upon_timeout; - case ngx_http_plugin_metric_type_e::MIN_REQ_BODY_SIZE_UPON_TIMEOUT: + case AttachmentMetricType::MIN_REQ_BODY_SIZE_UPON_TIMEOUT: return min_req_body_size_upon_timeout; - case ngx_http_plugin_metric_type_e::RES_HEADER_THREAD_TIMEOUT: + case AttachmentMetricType::RES_HEADER_THREAD_TIMEOUT: return res_header_thread_timeout; - case ngx_http_plugin_metric_type_e::RES_BODY_THREAD_TIMEOUT: + case AttachmentMetricType::RES_BODY_THREAD_TIMEOUT: return res_body_thread_timeout; - case ngx_http_plugin_metric_type_e::AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT: + case AttachmentMetricType::AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT: return average_res_body_size_upon_timeout; - case ngx_http_plugin_metric_type_e::MAX_RES_BODY_SIZE_UPON_TIMEOUT: + case AttachmentMetricType::MAX_RES_BODY_SIZE_UPON_TIMEOUT: return max_res_body_size_upon_timeout; - case ngx_http_plugin_metric_type_e::MIN_RES_BODY_SIZE_UPON_TIMEOUT: + case AttachmentMetricType::MIN_RES_BODY_SIZE_UPON_TIMEOUT: return min_res_body_size_upon_timeout; - case ngx_http_plugin_metric_type_e::THREAD_FAILURE: + case AttachmentMetricType::THREAD_FAILURE: return thread_failure; - case ngx_http_plugin_metric_type_e::REQ_PROCCESSING_TIMEOUT: + case AttachmentMetricType::REQ_PROCCESSING_TIMEOUT: return req_proccessing_timeout; - case ngx_http_plugin_metric_type_e::RES_PROCCESSING_TIMEOUT: + case AttachmentMetricType::RES_PROCCESSING_TIMEOUT: return res_proccessing_timeout; - case ngx_http_plugin_metric_type_e::REQ_FAILED_TO_REACH_UPSTREAM: + case AttachmentMetricType::REQ_FAILED_TO_REACH_UPSTREAM: return req_failed_to_reach_upstream; - case ngx_http_plugin_metric_type_e::CPU_USAGE: + case AttachmentMetricType::CPU_USAGE: return static_cast(cpu_event.getCPU()); - case ngx_http_plugin_metric_type_e::REQUEST_OVERALL_SIZE_COUNT: + case AttachmentMetricType::REQUEST_OVERALL_SIZE_COUNT: return req_overall_size; - case ngx_http_plugin_metric_type_e::RESPONSE_OVERALL_SIZE_COUNT: + case AttachmentMetricType::RESPONSE_OVERALL_SIZE_COUNT: return res_overall_size; default: dbgWarning(D_METRICS_NGINX_ATTACHMENT) @@ -382,145 +382,145 @@ void nginxIntakerMetric::upon(const nginxIntakerEvent &event) { successfull_inspection_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::INSPECTION_SUCCESSES_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::INSPECTION_SUCCESSES_COUNT) ); transparent_mode_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::TRANSPARENTS_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::TRANSPARENTS_COUNT) ); total_transparent_time.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::TOTAL_TRANSPARENTS_TIME) + event.getPluginMetricCounter(AttachmentMetricType::TOTAL_TRANSPARENTS_TIME) ); open_failure_inspection_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::INSPECTION_OPEN_FAILURES_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::INSPECTION_OPEN_FAILURES_COUNT) ); close_failure_inspection_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::INSPECTION_CLOSE_FAILURES_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::INSPECTION_CLOSE_FAILURES_COUNT) ); inject_verdict_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::INJECT_VERDICTS_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::INJECT_VERDICTS_COUNT) ); inspect_verdict_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::INSPECT_VERDICTS_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::INSPECT_VERDICTS_COUNT) ); accept_verdict_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::ACCEPT_VERDICTS_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::ACCEPT_VERDICTS_COUNT) ); drop_verdict_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::DROP_VERDICTS_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::DROP_VERDICTS_COUNT) ); irrelevant_verdict_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::IRRELEVANT_VERDICTS_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::IRRELEVANT_VERDICTS_COUNT) ); reconf_verdict_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RECONF_VERDICTS_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::RECONF_VERDICTS_COUNT) ); average_overall_processing_time_until_verdict.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT) + event.getPluginMetricCounter(AttachmentMetricType::AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT) ); max_overall_processing_time_until_verdict.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT) + event.getPluginMetricCounter(AttachmentMetricType::MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT) ); min_overall_processing_time_until_verdict.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT) + event.getPluginMetricCounter(AttachmentMetricType::MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT) ); average_req_processing_time_until_verdict.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT) + event.getPluginMetricCounter(AttachmentMetricType::AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT) ); max_req_processing_time_until_verdict.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT) + event.getPluginMetricCounter(AttachmentMetricType::MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT) ); min_req_processing_time_until_verdict.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT) + event.getPluginMetricCounter(AttachmentMetricType::MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT) ); average_res_processing_time_until_verdict.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT) + event.getPluginMetricCounter(AttachmentMetricType::AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT) ); max_res_processing_time_until_verdict.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT) + event.getPluginMetricCounter(AttachmentMetricType::MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT) ); min_res_processing_time_until_verdict.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT) + event.getPluginMetricCounter(AttachmentMetricType::MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT) ); req_failed_compression_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_FAILED_COMPRESSION_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::REQ_FAILED_COMPRESSION_COUNT) ); res_failed_compression_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_FAILED_COMPRESSION_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::RES_FAILED_COMPRESSION_COUNT) ); req_failed_decompression_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_FAILED_DECOMPRESSION_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::REQ_FAILED_DECOMPRESSION_COUNT) ); res_failed_decompression_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_FAILED_DECOMPRESSION_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::RES_FAILED_DECOMPRESSION_COUNT) ); req_successful_compression_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_SUCCESSFUL_COMPRESSION_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::REQ_SUCCESSFUL_COMPRESSION_COUNT) ); res_successful_compression_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_SUCCESSFUL_COMPRESSION_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::RES_SUCCESSFUL_COMPRESSION_COUNT) ); req_successful_decompression_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_SUCCESSFUL_DECOMPRESSION_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::REQ_SUCCESSFUL_DECOMPRESSION_COUNT) ); res_successful_decompression_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_SUCCESSFUL_DECOMPRESSION_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::RES_SUCCESSFUL_DECOMPRESSION_COUNT) ); corrupted_zip_skipped_session_counter.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::CORRUPTED_ZIP_SKIPPED_SESSION_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::CORRUPTED_ZIP_SKIPPED_SESSION_COUNT) ); thread_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::THREAD_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::THREAD_TIMEOUT) ); reg_thread_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REG_THREAD_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::REG_THREAD_TIMEOUT) ); req_header_thread_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_HEADER_THREAD_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::REQ_HEADER_THREAD_TIMEOUT) ); req_body_thread_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_BODY_THREAD_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::REQ_BODY_THREAD_TIMEOUT) ); average_req_body_size_upon_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT) ); max_req_body_size_upon_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MAX_REQ_BODY_SIZE_UPON_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::MAX_REQ_BODY_SIZE_UPON_TIMEOUT) ); min_req_body_size_upon_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MIN_REQ_BODY_SIZE_UPON_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::MIN_REQ_BODY_SIZE_UPON_TIMEOUT) ); res_header_thread_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_HEADER_THREAD_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::RES_HEADER_THREAD_TIMEOUT) ); res_body_thread_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_BODY_THREAD_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::RES_BODY_THREAD_TIMEOUT) ); average_res_body_size_upon_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT) ); max_res_body_size_upon_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MAX_RES_BODY_SIZE_UPON_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::MAX_RES_BODY_SIZE_UPON_TIMEOUT) ); min_res_body_size_upon_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MIN_RES_BODY_SIZE_UPON_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::MIN_RES_BODY_SIZE_UPON_TIMEOUT) ); thread_failure.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::THREAD_FAILURE) + event.getPluginMetricCounter(AttachmentMetricType::THREAD_FAILURE) ); req_proccessing_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_PROCCESSING_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::REQ_PROCCESSING_TIMEOUT) ); res_proccessing_timeout.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_PROCCESSING_TIMEOUT) + event.getPluginMetricCounter(AttachmentMetricType::RES_PROCCESSING_TIMEOUT) ); req_failed_to_reach_upstream.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_FAILED_TO_REACH_UPSTREAM) + event.getPluginMetricCounter(AttachmentMetricType::REQ_FAILED_TO_REACH_UPSTREAM) ); req_overall_size.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQUEST_OVERALL_SIZE_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::REQUEST_OVERALL_SIZE_COUNT) ); res_overall_size.report( - event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RESPONSE_OVERALL_SIZE_COUNT) + event.getPluginMetricCounter(AttachmentMetricType::RESPONSE_OVERALL_SIZE_COUNT) ); event.notifyCPU(); } diff --git a/components/attachment-intakers/nginx_attachment/nginx_parser.cc b/components/attachment-intakers/nginx_attachment/nginx_parser.cc index f3d7eb9..2a9acf7 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_parser.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_parser.cc @@ -34,7 +34,8 @@ bool is_keep_alive_ctx = getenv("SAAS_KEEP_ALIVE_HDR_NAME") != nullptr; map NginxParser::content_encodings = { {Buffer("identity"), CompressionType::NO_COMPRESSION}, {Buffer("gzip"), CompressionType::GZIP}, - {Buffer("deflate"), CompressionType::ZLIB} + {Buffer("deflate"), CompressionType::ZLIB}, + {Buffer("br"), CompressionType::BROTLI} }; Maybe @@ -180,6 +181,7 @@ getActivetenantAndProfile(const string &str, const string &deli = ",") Maybe> NginxParser::parseRequestHeaders(const Buffer &data, const unordered_set &ignored_headers) { + dbgFlow(D_NGINX_ATTACHMENT_PARSER) << "Parsing request headers"; auto maybe_parsed_headers = genHeaders(data); if (!maybe_parsed_headers.ok()) return maybe_parsed_headers.passErr(); @@ -188,44 +190,72 @@ NginxParser::parseRequestHeaders(const Buffer &data, const unordered_set NginxAttachmentOpaque &opaque = i_transaction_table->getState(); if (is_keep_alive_ctx || !ignored_headers.empty()) { - bool is_last_header_removed = false; - parsed_headers.erase( - remove_if( - parsed_headers.begin(), - parsed_headers.end(), - [&opaque, &is_last_header_removed, &ignored_headers](const HttpHeader &header) - { - string hdr_key = static_cast(header.getKey()); - string hdr_val = static_cast(header.getValue()); - if ( - opaque.setKeepAliveCtx(hdr_key, hdr_val) - || ignored_headers.find(hdr_key) != ignored_headers.end() - ) { - dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Header was removed from headers list: " << hdr_key; - if (header.isLastHeader()) { - dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Last header was removed from headers list"; - is_last_header_removed = true; - } - return true; - } - return false; + dbgTrace(D_NGINX_ATTACHMENT_PARSER) + << "is_keep_alive_ctx: " << is_keep_alive_ctx + << ", ignored_headers size: " << ignored_headers.size() + << ", headers count before removal: " << parsed_headers.size(); + + // First, determine which headers would be removed + std::vector remove_indices; + for (size_t i = 0; i < parsed_headers.size(); ++i) { + const auto &header = parsed_headers[i]; + string hdr_key = static_cast(header.getKey()); + string hdr_val = static_cast(header.getValue()); + bool should_remove = + opaque.setKeepAliveCtx(hdr_key, hdr_val) || + ignored_headers.find(hdr_key) != ignored_headers.end(); + if (should_remove) { + dbgTrace(D_NGINX_ATTACHMENT_PARSER) + << "Header should be removed from headers list: " + << dumpHex(header.getKey()); + remove_indices.push_back(i); + } + } + if (remove_indices.size() == parsed_headers.size() && !parsed_headers.empty()) { + // All headers would be removed: keep the last, mark as shouldNotLog + parsed_headers.back().setShouldNotLog(); + // Remove all except the last + parsed_headers.erase(parsed_headers.begin(), parsed_headers.end() - 1); + // Ensure the last header is marked as last + parsed_headers.back().setIsLastHeader(); + dbgTrace(D_NGINX_ATTACHMENT_PARSER) + << "All headers were marked for removal. Keeping last header and marking it as shouldNotLog: " + << dumpHex(parsed_headers.back().getKey()); + } else { + // Remove the marked headers as usual + // (remove from end to start to keep indices valid) + bool is_last_header_removed = false; + for (auto it = remove_indices.rbegin(); it != remove_indices.rend(); ++it) { + if (parsed_headers[*it].isLastHeader()) { + is_last_header_removed = true; } - ), - parsed_headers.end() - ); - if (is_last_header_removed) { - dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Adjusting last header flag"; - if (!parsed_headers.empty()) parsed_headers.back().setIsLastHeader(); + parsed_headers.erase(parsed_headers.begin() + *it); + } + dbgTrace(D_NGINX_ATTACHMENT_PARSER) + << "Headers removal completed. Remaining headers count: " + << parsed_headers.size(); + if (is_last_header_removed) { + dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Adjusting last header flag"; + if (!parsed_headers.empty()) { + parsed_headers.back().setIsLastHeader(); + dbgTrace(D_NGINX_ATTACHMENT_PARSER) + << "New last header after removal: " + << dumpHex(parsed_headers.back().getKey()); + } + } } } + static auto default_identifiers = UsersAllIdentifiersConfig(); + auto maybe_source_identifiers = + getConfigurationWithCache("rulebase", "usersIdentifiers"); + auto source_identifiers = maybe_source_identifiers.ok() ? maybe_source_identifiers.unpack() : default_identifiers; + for (const HttpHeader &header : parsed_headers) { - auto source_identifiers = getConfigurationWithDefault( - UsersAllIdentifiersConfig(), - "rulebase", - "usersIdentifiers" - ); source_identifiers.parseRequestHeaders(header); + if (!header.shouldLog()) { + continue; + } opaque.addToSavedData( HttpTransactionData::req_headers, static_cast(header.getKey()) + ": " + static_cast(header.getValue()) + "\r\n" @@ -419,11 +449,38 @@ NginxParser::convertToContentEncoding(const Buffer &content_encoding_header_valu return genError("Multiple content encodings for a specific HTTP request/response body are not supported"); } - if (content_encodings.find(content_encoding_header_value) == content_encodings.end()) { + auto it = find_if( + content_encodings.begin(), + content_encodings.end(), + [&content_encoding_header_value](const pair &encoding_pair) { + bool is_equal = content_encoding_header_value.isEqualLowerCase(encoding_pair.first); + + dbgTrace(D_NGINX_ATTACHMENT_PARSER) + << "Comparing '" + << static_cast(content_encoding_header_value) + << "' with '" + << static_cast(encoding_pair.first) + << ". Comparison result: " + << (is_equal ? "MATCH" : "NO MATCH"); + return is_equal; + } + ); + + if (it == content_encodings.end()) { + dbgDebug(D_NGINX_ATTACHMENT_PARSER) + << "No matching compression type found for: '" + << static_cast(content_encoding_header_value) + << "'"; return genError( "Unsupported or undefined \"Content-Encoding\" value: " + static_cast(content_encoding_header_value) ); } - return content_encodings[content_encoding_header_value]; + + dbgDebug(D_NGINX_ATTACHMENT_PARSER) + << "Successfully matched '" + << static_cast(content_encoding_header_value) + << "' to compression type"; + + return it->second; } diff --git a/components/attachment-intakers/nginx_attachment/nginx_parser.h b/components/attachment-intakers/nginx_attachment/nginx_parser.h index 499fb4a..9969731 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_parser.h +++ b/components/attachment-intakers/nginx_attachment/nginx_parser.h @@ -17,7 +17,7 @@ #include #include "compression_utils.h" -#include "nginx_attachment_common.h" +#include "nano_attachment_common.h" #include "http_transaction_common.h" #include "http_inspection_events.h" #include "i_encryptor.h" diff --git a/components/http_manager/http_manager.cc b/components/http_manager/http_manager.cc index 9fb2fba..e80182e 100755 --- a/components/http_manager/http_manager.cc +++ b/components/http_manager/http_manager.cc @@ -36,14 +36,15 @@ static ostream & operator<<(ostream &os, const EventVerdict &event) { switch (event.getVerdict()) { - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT: return os << "Inspect"; - case ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS: return os << "Limit Response Headers"; - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT: return os << "Accept"; - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP: return os << "Drop"; - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT: return os << "Inject"; - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT: return os << "Irrelevant"; - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_RECONF: return os << "Reconf"; - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT: return os << "Wait"; + case ServiceVerdict::TRAFFIC_VERDICT_INSPECT: return os << "Inspect"; + case ServiceVerdict::LIMIT_RESPONSE_HEADERS: return os << "Limit Response Headers"; + case ServiceVerdict::TRAFFIC_VERDICT_ACCEPT: return os << "Accept"; + case ServiceVerdict::TRAFFIC_VERDICT_DROP: return os << "Drop"; + case ServiceVerdict::TRAFFIC_VERDICT_INJECT: return os << "Inject"; + case ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT: return os << "Irrelevant"; + case ServiceVerdict::TRAFFIC_VERDICT_RECONF: return os << "Reconf"; + case ServiceVerdict::TRAFFIC_VERDICT_DELAYED: return os << "Wait"; + case ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE: return os << "Force 200"; } dbgAssert(false) @@ -66,6 +67,13 @@ public: i_transaction_table = Singleton::Consume::by(); Singleton::Consume::by()->addGeneralModifier(compressAppSecLogs); + custom_header = getProfileAgentSettingWithDefault("", "agent.customHeaderValueLogging"); + + registerConfigLoadCb( + [this]() { + custom_header = getProfileAgentSettingWithDefault("", "agent.customHeaderValueLogging"); + } + ); } FilterVerdict @@ -94,9 +102,7 @@ public: ctx.registerValue(app_sec_marker_key, i_transaction_table->keyToString(), EnvKeyAttr::LogSection::MARKER); HttpManagerOpaque &state = i_transaction_table->getState(); - - const auto &custom_header = getProfileAgentSettingWithDefault("", "agent.customHeaderValueLogging"); - + if (event.getKey().isEqualLowerCase(custom_header)) { string event_value = static_cast(event.getValue()); dbgTrace(D_HTTP_MANAGER) @@ -117,7 +123,7 @@ public: HttpRequestHeaderEvent(event).performNamedQuery() : HttpResponseHeaderEvent(event).performNamedQuery(); FilterVerdict verdict = handleEvent(event_responds); - if (verdict.getVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT) { + if (verdict.getVerdict() == ServiceVerdict::TRAFFIC_VERDICT_INJECT) { applyInjectionModifications(verdict, event_responds, event.getHeaderIndex()); } return verdict; @@ -131,8 +137,8 @@ public: return FilterVerdict(default_verdict); } - ngx_http_cp_verdict_e body_size_limit_verdict = handleBodySizeLimit(is_request, event); - if (body_size_limit_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT) { + ServiceVerdict body_size_limit_verdict = handleBodySizeLimit(is_request, event); + if (body_size_limit_verdict != ServiceVerdict::TRAFFIC_VERDICT_INSPECT) { return FilterVerdict(body_size_limit_verdict); } @@ -144,7 +150,7 @@ public: ctx.registerValue("UserDefined", state.getUserDefinedValue().unpack(), EnvKeyAttr::LogSection::DATA); } - FilterVerdict verdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT); + FilterVerdict verdict(ServiceVerdict::TRAFFIC_VERDICT_INSPECT); if (!is_request && event.getData().size() == 0 && !event.isLastChunk()) { dbgDebug(D_HTTP_MANAGER) << "Skipping inspection of first empty chunk for respond body"; return verdict; @@ -156,7 +162,7 @@ public: HttpResponseBodyEvent(event, state.getPreviousDataCache()).performNamedQuery(); verdict = handleEvent(event_responds); state.saveCurrentDataToCache(event.getData()); - if (verdict.getVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT) { + if (verdict.getVerdict() == ServiceVerdict::TRAFFIC_VERDICT_INJECT) { applyInjectionModifications(verdict, event_responds, event.getBodyChunkIndex()); } return verdict; @@ -252,13 +258,13 @@ public: } private: - ngx_http_cp_verdict_e + ServiceVerdict handleBodySizeLimit(bool is_request_body_type, const HttpBody &event) { HttpManagerOpaque &state = i_transaction_table->getState(); state.updatePayloadSize(event.getData().size()); - auto size_limit = getConfiguration( + auto size_limit = getConfigurationWithCache( "HTTP manager", is_request_body_type ? "Max Request Body Size" : "Max Response Body Size" ); @@ -270,12 +276,12 @@ private: ); if (!size_limit.ok() || state.getAggeregatedPayloadSize() < size_limit.unpack()) { - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + return ServiceVerdict::TRAFFIC_VERDICT_INSPECT; } - ngx_http_cp_verdict_e verdict = size_limit_verdict == "Drop" ? - ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP : - ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + ServiceVerdict verdict = size_limit_verdict == "Drop" ? + ServiceVerdict::TRAFFIC_VERDICT_DROP : + ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; dbgDebug(D_HTTP_MANAGER) << "Transaction body size is over the limit. Max body size: " @@ -295,7 +301,7 @@ private: ModifiedChunkIndex event_idx) { for (const pair &respond : event_responds) { - if (respond.second.getVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT) { + if (respond.second.getVerdict() == ServiceVerdict::TRAFFIC_VERDICT_INJECT) { dbgTrace(D_HTTP_MANAGER) << "Applying inject verdict modifications for security App: " << respond.first; @@ -310,7 +316,7 @@ private: HttpManagerOpaque &state = i_transaction_table->getState(); for (const pair &respond : event_responds) { - if (state.getApplicationsVerdict(respond.first) == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT) { + if (state.getApplicationsVerdict(respond.first) == ServiceVerdict::TRAFFIC_VERDICT_ACCEPT) { dbgTrace(D_HTTP_MANAGER) << "Skipping event verdict for app that already accepted traffic. App: " << respond.first; @@ -325,12 +331,32 @@ private: state.setApplicationVerdict(respond.first, respond.second.getVerdict()); state.setApplicationWebResponse(respond.first, respond.second.getWebUserResponseByPractice()); + if (respond.second.getVerdict() == ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE) { + if (!respond.second.getCustomResponse().ok()) { + dbgWarning(D_HTTP_MANAGER) + << "Security app: " + << respond.first + << ", returned verdict CUSTOM_RESPONSE, but no custom response was found."; + continue; + } + state.setCustomResponse(respond.first, respond.second.getCustomResponse().unpack()); + } } - FilterVerdict aggregated_verdict(state.getCurrVerdict(), state.getCurrWebUserResponse()); - if (aggregated_verdict.getVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) { - SecurityAppsDropEvent(state.getCurrentDropVerdictCausers()).notify(); + auto ver = state.getCurrVerdict(); + dbgTrace(D_HTTP_MANAGER) << "Aggregated verdict is: " << ver; + if (ver == ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE) { + if (!state.getCurrentCustomResponse().ok()) { + dbgWarning(D_HTTP_MANAGER) << "No custom response found for verdict CUSTOM_RESPONSE"; + return FilterVerdict(ServiceVerdict::TRAFFIC_VERDICT_ACCEPT); + } + return FilterVerdict(ver, state.getCurrentCustomResponse().unpack()); + } else { + FilterVerdict aggregated_verdict(ver, state.getCurrWebUserResponse()); + if (aggregated_verdict.getVerdict() == ServiceVerdict::TRAFFIC_VERDICT_DROP) { + SecurityAppsDropEvent(state.getCurrentDropVerdictCausers()).notify(); + } + return aggregated_verdict; } - return aggregated_verdict; } static void @@ -394,11 +420,12 @@ private: } I_Table *i_transaction_table; - static const ngx_http_cp_verdict_e default_verdict; + string custom_header = ""; + static const ServiceVerdict default_verdict; static const string app_sec_marker_key; }; -const ngx_http_cp_verdict_e HttpManager::Impl::default_verdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP); +const ServiceVerdict HttpManager::Impl::default_verdict(ServiceVerdict::TRAFFIC_VERDICT_DROP); const string HttpManager::Impl::app_sec_marker_key = "app_sec_marker"; HttpManager::HttpManager() : Component("HttpManager"), pimpl(make_unique()) {} @@ -409,10 +436,10 @@ void HttpManager::init() { pimpl->init(); } void HttpManager::preload() { - registerExpectedConfiguration("HTTP manager", "Previous Buffer Cache size"); - registerExpectedConfiguration("HTTP manager", "Max Request Body Size"); - registerExpectedConfiguration("HTTP manager", "Max Response Body Size"); - registerExpectedConfiguration("HTTP manager", "Request Size Limit Verdict"); - registerExpectedConfiguration("HTTP manager", "Response Size Limit Verdict"); + registerExpectedConfigurationWithCache("assetId", "HTTP manager", "Previous Buffer Cache size"); + registerExpectedConfigurationWithCache("assetId", "HTTP manager", "Max Request Body Size"); + registerExpectedConfigurationWithCache("assetId", "HTTP manager", "Max Response Body Size"); + registerExpectedConfigurationWithCache("assetId", "HTTP manager", "Request Size Limit Verdict"); + registerExpectedConfigurationWithCache("assetId", "HTTP manager", "Response Size Limit Verdict"); registerConfigLoadCb([this] () { pimpl->sendPolicyLog(); }); } diff --git a/components/http_manager/http_manager_opaque.cc b/components/http_manager/http_manager_opaque.cc index 06e0719..a8f2241 100644 --- a/components/http_manager/http_manager_opaque.cc +++ b/components/http_manager/http_manager_opaque.cc @@ -27,7 +27,7 @@ HttpManagerOpaque::HttpManagerOpaque() } void -HttpManagerOpaque::setApplicationVerdict(const string &app_name, ngx_http_cp_verdict_e verdict) +HttpManagerOpaque::setApplicationVerdict(const string &app_name, ServiceVerdict verdict) { applications_verdicts[app_name] = verdict; } @@ -39,49 +39,61 @@ HttpManagerOpaque::setApplicationWebResponse(const string &app_name, string web_ applications_web_user_response[app_name] = web_user_response_id; } -ngx_http_cp_verdict_e +void +HttpManagerOpaque::setCustomResponse(const std::string &app_name, const CustomResponse &custom_response) +{ + dbgTrace(D_HTTP_MANAGER) << "Security app: " << app_name + << ", has custom response: " << custom_response.getBody() + << ", with code: " << custom_response.getStatusCode(); + current_custom_response = custom_response; +} + +ServiceVerdict HttpManagerOpaque::getApplicationsVerdict(const string &app_name) const { auto verdict = applications_verdicts.find(app_name); - return verdict == applications_verdicts.end() ? ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT : verdict->second; + return verdict == applications_verdicts.end() ? ServiceVerdict::TRAFFIC_VERDICT_INSPECT : verdict->second; } -ngx_http_cp_verdict_e +ServiceVerdict HttpManagerOpaque::getCurrVerdict() const { - if (manager_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) { + if (manager_verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP) { return manager_verdict; } uint accepted_apps = 0; - ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + ServiceVerdict verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT; for (const auto &app_verdic_pair : applications_verdicts) { switch (app_verdic_pair.second) { - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP: + case ServiceVerdict::TRAFFIC_VERDICT_DROP: dbgTrace(D_HTTP_MANAGER) << "Verdict DROP for app: " << app_verdic_pair.first; current_web_user_response = applications_web_user_response.at(app_verdic_pair.first); dbgTrace(D_HTTP_MANAGER) << "current_web_user_response=" << current_web_user_response; return app_verdic_pair.second; - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT: + case ServiceVerdict::TRAFFIC_VERDICT_INJECT: // Sent in ResponseHeaders and ResponseBody. - verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT; + verdict = ServiceVerdict::TRAFFIC_VERDICT_INJECT; break; - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT: + case ServiceVerdict::TRAFFIC_VERDICT_ACCEPT: accepted_apps++; break; - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT: + case ServiceVerdict::TRAFFIC_VERDICT_INSPECT: break; - case ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS: + case ServiceVerdict::LIMIT_RESPONSE_HEADERS: // Sent in End Request. - verdict = ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS; + verdict = ServiceVerdict::LIMIT_RESPONSE_HEADERS; break; - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT: + case ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT: dbgTrace(D_HTTP_MANAGER) << "Verdict 'Irrelevant' is not yet supported. Returning Accept"; accepted_apps++; break; - case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT: + case ServiceVerdict::TRAFFIC_VERDICT_DELAYED: // Sent in Request Headers and Request Body. - verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT; + verdict = ServiceVerdict::TRAFFIC_VERDICT_DELAYED; + break; + case ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE: + verdict = ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE; break; default: dbgAssert(false) @@ -91,18 +103,18 @@ HttpManagerOpaque::getCurrVerdict() const } } - return accepted_apps == applications_verdicts.size() ? ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT : verdict; + return accepted_apps == applications_verdicts.size() ? ServiceVerdict::TRAFFIC_VERDICT_ACCEPT : verdict; } std::set HttpManagerOpaque::getCurrentDropVerdictCausers() const { std::set causers; - if (manager_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) { + if (manager_verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP) { causers.insert(HTTP_MANAGER_NAME); } for (const auto &app_verdic_pair : applications_verdicts) { - bool was_dropped = app_verdic_pair.second == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + bool was_dropped = app_verdic_pair.second == ServiceVerdict::TRAFFIC_VERDICT_DROP; dbgTrace(D_HTTP_MANAGER) << "The verdict from: " << app_verdic_pair.first << (was_dropped ? " is \"drop\"" : " is not \"drop\" "); diff --git a/components/http_manager/http_manager_opaque.h b/components/http_manager/http_manager_opaque.h index 53e733e..e37941b 100644 --- a/components/http_manager/http_manager_opaque.h +++ b/components/http_manager/http_manager_opaque.h @@ -18,7 +18,8 @@ #include "buffer.h" #include "table_opaque.h" -#include "nginx_attachment_common.h" +#include "nano_attachment_common.h" +#include "http_inspection_events.h" static const std::string HTTP_MANAGER_NAME = "HTTP Manager"; @@ -27,17 +28,19 @@ class HttpManagerOpaque : public TableOpaqueSerialize public: HttpManagerOpaque(); - void setApplicationVerdict(const std::string &app_name, ngx_http_cp_verdict_e verdict); + void setApplicationVerdict(const std::string &app_name, ServiceVerdict verdict); void setApplicationWebResponse(const std::string &app_name, std::string web_user_response_id); - ngx_http_cp_verdict_e getApplicationsVerdict(const std::string &app_name) const; - void setManagerVerdict(ngx_http_cp_verdict_e verdict) { manager_verdict = verdict; } - ngx_http_cp_verdict_e getManagerVerdict() const { return manager_verdict; } - ngx_http_cp_verdict_e getCurrVerdict() const; + void setCustomResponse(const std::string &app_name, const CustomResponse &custom_response); + ServiceVerdict getApplicationsVerdict(const std::string &app_name) const; + void setManagerVerdict(ServiceVerdict verdict) { manager_verdict = verdict; } + ServiceVerdict getManagerVerdict() const { return manager_verdict; } + ServiceVerdict getCurrVerdict() const; const std::string & getCurrWebUserResponse() const { return current_web_user_response; }; std::set getCurrentDropVerdictCausers() const; void saveCurrentDataToCache(const Buffer &full_data); void setUserDefinedValue(const std::string &value) { user_defined_value = value; } Maybe getUserDefinedValue() const { return user_defined_value; } + Maybe getCurrentCustomResponse() const { return current_custom_response; } const Buffer & getPreviousDataCache() const { return prev_data_cache; } uint getAggeregatedPayloadSize() const { return aggregated_payload_size; } void updatePayloadSize(const uint curr_payload); @@ -53,12 +56,13 @@ public: static uint minVer() { return 0; } private: - std::unordered_map applications_verdicts; + std::unordered_map applications_verdicts; std::unordered_map applications_web_user_response; mutable std::string current_web_user_response; - ngx_http_cp_verdict_e manager_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + ServiceVerdict manager_verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT; Buffer prev_data_cache; uint aggregated_payload_size = 0; + Maybe current_custom_response = genError("uninitialized"); Maybe user_defined_value = genError("uninitialized"); }; diff --git a/components/include/generic_rulebase/match_query.h b/components/include/generic_rulebase/match_query.h index ebf9824..b59ad63 100755 --- a/components/include/generic_rulebase/match_query.h +++ b/components/include/generic_rulebase/match_query.h @@ -68,8 +68,12 @@ public: const std::vector & getProtoValue() const { return ip_proto_value; } const std::vector & getItems() const { return items; } std::string getFirstValue() const { return first_value; } - MatchResult getMatch(const std::unordered_map> &key_value_pairs) const; - bool matchAttributes(const std::unordered_map> &key_value_pairs) const; + MatchResult getMatch( + const std::unordered_map> &key_value_pairs, + bool skip_irrelevant_key = false) const; + bool matchAttributes( + const std::unordered_map> &key_value_pairs, + bool skip_irrelevant_key = false) const; bool matchException(const std::string &behaviorKey, const std::string &behaviorValue) const; bool isKeyTypeIp() const; bool isKeyTypePort() const; @@ -82,7 +86,8 @@ public: private: bool matchAttributes( const std::unordered_map> &key_value_pairs, - std::set &matched_override_keywords) const; + std::set &matched_override_keywords, + bool skip_irrelevant_key = false) const; StaticKeys getKeyByName(const std::string &key_type_name); bool matchAttributes(const std::set &values, std::set &matched_override_keywords) const; diff --git a/components/include/generic_rulebase/parameters_config.h b/components/include/generic_rulebase/parameters_config.h index 4f4e52a..b3a245c 100755 --- a/components/include/generic_rulebase/parameters_config.h +++ b/components/include/generic_rulebase/parameters_config.h @@ -190,7 +190,7 @@ public: static void preload() { - registerExpectedConfiguration("rulebase", "exception"); + registerExpectedConfigurationWithCache("assetId", "rulebase", "exception"); registerConfigLoadCb([](){ is_geo_location_exception_exists = is_geo_location_exception_being_loaded; }); registerConfigPrepareCb([](){ is_geo_location_exception_being_loaded = false; }); } @@ -198,14 +198,20 @@ public: void load(cereal::JSONInputArchive &archive_in); std::set - getBehavior(const std::unordered_map> &key_value_pairs) const; + getBehavior( + const std::unordered_map> &key_value_pairs, + bool skip_irrelevant_key = false) const; std::set getBehavior( - const std::unordered_map> &key_value_pairs, - std::set &matched_override_keywords) const; + const std::unordered_map> &key_value_pairs, + std::set &matched_override_keywords, + bool skip_irrelevant_key = false) const; static bool isGeoLocationExceptionExists() { return is_geo_location_exception_exists; } + const MatchQuery& getMatch() const { return match; } + bool isContainingKVPair() const { return is_containing_kv_pair; } + bool checkKVPair() const; private: class MatchBehaviorPair @@ -221,6 +227,7 @@ private: ParameterBehavior behavior; static bool is_geo_location_exception_exists; static bool is_geo_location_exception_being_loaded; + bool is_containing_kv_pair; }; static const ParameterBehavior action_ignore(BehaviorKey::ACTION, BehaviorValue::IGNORE); diff --git a/components/include/generic_rulebase/rulebase_config.h b/components/include/generic_rulebase/rulebase_config.h index a2f6a0d..4aad018 100755 --- a/components/include/generic_rulebase/rulebase_config.h +++ b/components/include/generic_rulebase/rulebase_config.h @@ -112,7 +112,7 @@ public: static void preload() { - registerExpectedConfiguration("rulebase", "rulesConfig"); + registerExpectedConfigurationWithCache("assetId", "rulebase", "rulesConfig"); registerExpectedSetting>("rulebase", "rulesConfig"); registerConfigLoadCb(BasicRuleConfig::updateCountMetric); registerConfigPrepareCb([](){ BasicRuleConfig::assets_ids_aggregation.clear(); }); diff --git a/components/include/generic_rulebase/triggers_config.h b/components/include/generic_rulebase/triggers_config.h index 4dfb8b2..3d7a9ef 100755 --- a/components/include/generic_rulebase/triggers_config.h +++ b/components/include/generic_rulebase/triggers_config.h @@ -52,7 +52,7 @@ public: static void preload() { - registerExpectedConfiguration("rulebase", "webUserResponse"); + registerExpectedConfigurationWithCache("triggerId", "rulebase", "webUserResponse"); } /// \brief Load function to deserialize configuration from JSONInputArchive. @@ -104,6 +104,14 @@ public: return redirect_url; } + /// \brief Get the content type for the trigger. + /// \return The content type for the trigger. + const std::string & + getContentType() const + { + return content_type; + } + /// \brief Check if the trigger should add an event ID to the header. /// \return True if the trigger should add an event ID, otherwise false. bool @@ -120,6 +128,7 @@ private: std::string details_level; std::string response_body; std::string redirect_url; + std::string content_type; uint response_code; bool add_event_id_to_header = false; }; @@ -175,7 +184,8 @@ public: static void preload() { - registerExpectedConfiguration("rulebase", "log"); + //registerExpectedConfiguration("rulebase", "log"); + registerExpectedConfigurationWithCache("triggerId", "rulebase", "log"); } /// \brief LogGen operator for LogTriggerConf. @@ -260,6 +270,15 @@ public: return should_log_on_prevent.isSet(security_type); } + /// \brief Check if should ignore exception log for the given security type. + /// \param security_type The security type to check. + /// \return True if should ignore exception, otherwise false. + bool + shouldIgnoreExceptionLog(SecurityType security_type) const + { + return should_log_exception.isSet(security_type); + } + /// \brief Check if the log is active on detect for the given security type. /// \param security_type The security type to check. /// \return True if the log is active on detect, otherwise false. @@ -333,6 +352,7 @@ private: Flags active_streams; Flags should_log_on_detect; Flags should_log_on_prevent; + Flags should_log_exception; Flags log_geo_location; Flags log_web_fields; extendLoggingSeverity extend_logging_severity = extendLoggingSeverity::None; @@ -349,7 +369,7 @@ public: static void preload() { - registerExpectedConfiguration("rulebase", "report"); + registerExpectedConfigurationWithCache("triggerId", "rulebase", "report"); } /// \brief Load function to deserialize configuration from JSONInputArchive. diff --git a/components/include/http_event_impl/filter_verdict.h b/components/include/http_event_impl/filter_verdict.h index af81c64..d34309e 100755 --- a/components/include/http_event_impl/filter_verdict.h +++ b/components/include/http_event_impl/filter_verdict.h @@ -22,13 +22,13 @@ class FilterVerdict { public: - FilterVerdict(ngx_http_cp_verdict_e _verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT) + FilterVerdict(ServiceVerdict _verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT) : verdict(_verdict) {} FilterVerdict( - ngx_http_cp_verdict_e _verdict, + ServiceVerdict _verdict, const std::string &_web_reponse_id) : verdict(_verdict), @@ -40,15 +40,21 @@ public: verdict(_verdict.getVerdict()), web_user_response_id(_verdict.getWebUserResponseByPractice()) { - if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT) { + if (verdict == ServiceVerdict::TRAFFIC_VERDICT_INJECT) { addModifications(_verdict.getModifications(), _event_idx); } } + FilterVerdict(ServiceVerdict _verdict, const CustomResponse &_custom_response) + : + verdict(_verdict), + custom_response(_custom_response) + {} + void addModifications(const FilterVerdict &other) { - if (other.verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT) return; + if (other.verdict != ServiceVerdict::TRAFFIC_VERDICT_INJECT) return; modifications.insert(modifications.end(), other.modifications.begin(), other.modifications.end()); total_modifications += other.total_modifications; @@ -58,22 +64,24 @@ public: addModifications( const ModificationList &mods, ModifiedChunkIndex _event_idx, - ngx_http_cp_verdict_e alt_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) + ServiceVerdict alt_verdict = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT) { total_modifications += mods.size(); modifications.push_back(EventModifications(_event_idx, mods)); - if (alt_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) verdict = alt_verdict; + if (alt_verdict != ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT) verdict = alt_verdict; } uint getModificationsAmount() const { return total_modifications; } - ngx_http_cp_verdict_e getVerdict() const { return verdict; } + ServiceVerdict getVerdict() const { return verdict; } const std::vector & getModifications() const { return modifications; } const std::string getWebUserResponseID() const { return web_user_response_id; } + Maybe getCustomResponse() const { return custom_response; } private: - ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + ServiceVerdict verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT; std::vector modifications; std::string web_user_response_id; + Maybe custom_response = genError("uninitialized"); uint total_modifications = 0; }; diff --git a/components/include/http_event_impl/i_http_event_impl.h b/components/include/http_event_impl/i_http_event_impl.h index 729e306..0e175a0 100755 --- a/components/include/http_event_impl/i_http_event_impl.h +++ b/components/include/http_event_impl/i_http_event_impl.h @@ -25,12 +25,12 @@ #include "debug.h" #include "buffer.h" #include "http_transaction_data.h" -#include "nginx_attachment_common.h" +#include "nano_attachment_common.h" USE_DEBUG_FLAG(D_HTTP_MANAGER); -using ModificationType = ngx_http_modification_type_e; -using ModificationPosition = ngx_http_cp_inject_pos_t; +using ModificationType = HttpModificationType; +using ModificationPosition = NanoHttpCpInjectPos; static const ModificationPosition injection_pos_irrelevant = INJECT_POS_IRRELEVANT; @@ -185,12 +185,18 @@ class HttpHeader { public: HttpHeader() = default; - HttpHeader(const Buffer &_key, const Buffer &_value, uint8_t _header_index, bool _is_last_header = false) + HttpHeader( + const Buffer &_key, + const Buffer &_value, + uint8_t _header_index, + bool _is_last_header = false, + bool _should_log = true) : key(_key), value(_value), is_last_header(_is_last_header), - header_index(_header_index) + header_index(_header_index), + should_log(_should_log) { } @@ -203,7 +209,8 @@ public: key, value, is_last_header, - header_index + header_index, + should_log ); } @@ -215,7 +222,8 @@ public: key, value, is_last_header, - header_index + header_index, + should_log ); } // LCOV_EXCL_STOP @@ -232,6 +240,8 @@ public: << std::to_string(header_index) << ", Is last header: " << (is_last_header ? "True" : "False") + << ", Should log: " + << (should_log ? "True" : "False") << ")"; } @@ -241,12 +251,18 @@ public: bool isLastHeader() const { return is_last_header; } void setIsLastHeader() { is_last_header = true; } uint8_t getHeaderIndex() const { return header_index; } + bool shouldLog() const { return should_log; } + void setShouldNotLog() { + dbgTrace(D_HTTP_MANAGER) << "Header '" << std::dumpHex(key) << "' marked as should not log"; + should_log = false; + } private: Buffer key; Buffer value; bool is_last_header = false; uint8_t header_index = 0; + bool should_log = true; }; using BodyModification = Buffer; @@ -362,23 +378,54 @@ private: uint8_t body_chunk_index; }; +class CustomResponse +{ +public: + CustomResponse( + const std::string& body, + uint16_t status_code, + const std::string& content_type = "application/json" + ) : + body(body), + status_code(status_code), + content_type(content_type) + {} + + std::string getBody() const { return body; } + uint16_t getStatusCode() const { return status_code; } + std::string getContentType() const { return content_type; } + +private: + std::string body; + uint16_t status_code; + std::string content_type; +}; + class EventVerdict { public: EventVerdict() = default; - EventVerdict(ngx_http_cp_verdict_e event_verdict) : modifications(), verdict(event_verdict) {} + EventVerdict(ServiceVerdict event_verdict) : modifications(), verdict(event_verdict) {} EventVerdict(const ModificationList &mods) : modifications(mods) {} - EventVerdict(const ModificationList &mods, ngx_http_cp_verdict_e event_verdict) : + EventVerdict(const ModificationList &mods, ServiceVerdict event_verdict) : modifications(mods), verdict(event_verdict) {} + + EventVerdict( + const CustomResponse &custom_response + ) : + modifications(), + verdict(ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE), + custom_response(custom_response) + {} EventVerdict( const ModificationList &mods, - ngx_http_cp_verdict_e event_verdict, + ServiceVerdict event_verdict, std::string response_id) : modifications(mods), verdict(event_verdict), @@ -390,17 +437,20 @@ public: // LCOV_EXCL_STOP const ModificationList & getModifications() const { return modifications; } - ngx_http_cp_verdict_e getVerdict() const { return verdict; } + ServiceVerdict getVerdict() const { return verdict; } const std::string getWebUserResponseByPractice() const { return webUserResponseByPractice; } void setWebUserResponseByPractice(const std::string id) { dbgTrace(D_HTTP_MANAGER) << "current verdict web user response set to: " << id; webUserResponseByPractice = id; } + Maybe getCustomResponse() const { return custom_response; } + private: ModificationList modifications; - ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + ServiceVerdict verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT; std::string webUserResponseByPractice; + Maybe custom_response = genError("uninitialized"); }; #endif // __I_HTTP_EVENT_IMPL_H__ diff --git a/components/include/http_inspection_events.h b/components/include/http_inspection_events.h index c96e76f..55ece98 100755 --- a/components/include/http_inspection_events.h +++ b/components/include/http_inspection_events.h @@ -31,6 +31,7 @@ public: const Buffer & getValue() const { return req_header.getValue(); } bool isLastHeader() const { return req_header.isLastHeader(); } uint8_t getHeaderIndex() const { return req_header.getHeaderIndex(); } + bool shouldLog() const { return req_header.shouldLog(); } template void diff --git a/components/include/hyperscan_hook.h b/components/include/hyperscan_hook.h new file mode 100644 index 0000000..955e8fc --- /dev/null +++ b/components/include/hyperscan_hook.h @@ -0,0 +1,40 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __HYPERSCAN_HOOK_H__ +#define __HYPERSCAN_HOOK_H__ + +#ifdef USE_HYPERSCAN +#include +#include + +#include +#include "hs.h" +#include "i_pm_scan.h" + +class HyperscanHook : public I_PMScan { +public: + HyperscanHook(); + ~HyperscanHook(); + Maybe prepare(const std::set &patterns); + std::set scanBuf(const Buffer &buf) const override; + std::set> scanBufWithOffset(const Buffer &buf) const override; + void scanBufWithOffsetLambda(const Buffer &buf, I_PMScan::CBFunction cb) const override; + bool ok() const { return m_hsReady; } +private: + hs_database_t *m_hsDatabase; + hs_scratch_t *m_hsScratch; + std::vector m_hsPatterns; + std::vector m_idToPattern; + bool m_hsReady; +}; +#endif // USE_HYPERSCAN + +#endif // __HYPERSCAN_HOOK_H__ diff --git a/components/include/i_static_resources_handler.h b/components/include/i_static_resources_handler.h index 0cf6e4e..dcc5aaf 100755 --- a/components/include/i_static_resources_handler.h +++ b/components/include/i_static_resources_handler.h @@ -21,7 +21,16 @@ class I_StaticResourcesHandler { public: - virtual bool registerStaticResource(const std::string &resource_name, const std::string &resource_full_path) = 0; + virtual bool registerStaticResource( + const std::string &resource_name, + const std::string &resource_full_path, + bool overwrite_if_exists = false + ) = 0; + + virtual bool registerStaticResourceByContent( + const std::string &resource_name, + const std::string &file_content + ) = 0; protected: virtual ~I_StaticResourcesHandler() {} diff --git a/components/include/ips_comp.h b/components/include/ips_comp.h index 996b863..a66ee7b 100644 --- a/components/include/ips_comp.h +++ b/components/include/ips_comp.h @@ -2,13 +2,13 @@ #define __IPS_COMP_H__ #include "singleton.h" +#include "i_generic_rulebase.h" #include "i_keywords_rule.h" #include "i_table.h" #include "i_mainloop.h" #include "i_http_manager.h" #include "i_environment.h" #include "http_inspection_events.h" -#include "i_generic_rulebase.h" #include "component.h" class IPSComp diff --git a/components/include/mock/mock_nginx_attachment.h b/components/include/mock/mock_nginx_attachment.h index 8d51642..c5dbb7f 100755 --- a/components/include/mock/mock_nginx_attachment.h +++ b/components/include/mock/mock_nginx_attachment.h @@ -7,9 +7,17 @@ class MockNginxAttachment: public Singleton::Provide::From> { public: - MOCK_METHOD2( + MOCK_METHOD3( registerStaticResource, - bool(const std::string &static_resource_name, const std::string &static_resource_path) + bool(const std::string &static_resource_name, + const std::string &static_resource_path, + bool overwrite_if_exists + ) + ); + + MOCK_METHOD2( + registerStaticResourceByContent, + bool(const std::string &resource_name, const std::string &file_content) ); }; diff --git a/components/include/nginx_intaker_metric.h b/components/include/nginx_intaker_metric.h index decb4da..1102e11 100755 --- a/components/include/nginx_intaker_metric.h +++ b/components/include/nginx_intaker_metric.h @@ -14,7 +14,7 @@ #ifndef __NGINX_INTAKER_METRIC_H__ #define __NGINX_INTAKER_METRIC_H__ -#include "nginx_attachment_common.h" +#include "nano_attachment_common.h" #include "generic_metric.h" #include "cpu/cpu_metric.h" @@ -26,11 +26,11 @@ public: void resetAllCounters(); - ngx_http_plugin_metric_type_e EnumOfIndex(int i); + AttachmentMetricType EnumOfIndex(int i); - void addPluginMetricCounter(const ngx_http_cp_metric_data_t *recieved_metric_data); + void addPluginMetricCounter(const NanoHttpMetricData *recieved_metric_data); - uint64_t getPluginMetricCounter(ngx_http_plugin_metric_type_e _verdict) const; + uint64_t getPluginMetricCounter(AttachmentMetricType _verdict) const; void notifyCPU() const { cpu_event.notify(); } diff --git a/components/include/rate_limit_config.h b/components/include/rate_limit_config.h index 186eec2..bc751cb 100755 --- a/components/include/rate_limit_config.h +++ b/components/include/rate_limit_config.h @@ -120,6 +120,7 @@ public: const std::vector & getRateLimitRules() const { return rate_limit_rules; } const RateLimitAction & getRateLimitMode() const { return mode; } + const std::string & getWebUserResponse() const { return web_user_response; } RateLimitRule generateSiblingRateLimitRule(const RateLimitRule &rule); @@ -150,6 +151,8 @@ public: static bool isActive() { return is_active; } + void setWebUserResponse(const std::string& response_id) { web_user_response = response_id; } + static const std::map rate_limit_action_to_string; static const std::map rate_limit_string_to_action; @@ -160,6 +163,7 @@ private: static bool is_active; RateLimitAction mode; std::vector rate_limit_rules; + std::string web_user_response; }; #endif // __RATE_LIMIT_CONFIG_H__ diff --git a/components/include/reverse_proxy_defaults.h b/components/include/reverse_proxy_defaults.h index 6c07ad2..4876f45 100644 --- a/components/include/reverse_proxy_defaults.h +++ b/components/include/reverse_proxy_defaults.h @@ -44,6 +44,7 @@ static const std::string default_syslog_socket_address = "127.0.0.1:1514"; static const std::string rpm_full_load_path = "/tmp/rpm_full_load"; static const std::string rpm_partial_load_path = "/tmp/rpm_partial_load"; static const std::string first_rpm_policy_load_path = "/tmp/first_rpm_policy_load"; +static const std::string readiness_file_path = "/tmp/readiness"; static const int default_port = 5555; diff --git a/components/include/user_identifiers_config.h b/components/include/user_identifiers_config.h index 61ba320..ad29e27 100755 --- a/components/include/user_identifiers_config.h +++ b/components/include/user_identifiers_config.h @@ -31,7 +31,6 @@ public: std::vector getHeaderValuesFromConfig(const std::string &header_key) const; void setXFFValuesToOpaqueCtx(const HttpHeader &header, ExtractType type) const; void setWafTagValuesToOpaqueCtx(const HttpHeader &header) const; - private: class UsersIdentifiersConfig { diff --git a/components/nginx_message_reader/nginx_message_reader.cc b/components/nginx_message_reader/nginx_message_reader.cc index a9c4f31..1e93b22 100755 --- a/components/nginx_message_reader/nginx_message_reader.cc +++ b/components/nginx_message_reader/nginx_message_reader.cc @@ -51,12 +51,56 @@ static const boost::regex error_log_regex( ", (host: \".+?\")$" ); +static const boost::regex incident_log_regex( + "(" + + syslog_regex_string + ") " + + "incidentLog\\s+" + "host=([^\\s]+)\\s+" + "host_port=(\\d+)\\s+" + "client_addr=([\\d\\.]+)\\s+" + "client_port=(\\d+)\\s+" + "time_local=\\[([^\\]]+)\\]\\s+" + "request=\"([^\"]+)\"\\s+" + "status=(\\d{3})\\s+" + "uri=\"([^\"]+)\"\\s+" + "request_id=([^\\s]+)\\s+" + "upstream_status=([^,\\s]+)(?:,\\s*[^\\s]+)?\\s+" + "upstream_response_time=([^,\\s]+)(?:,\\s*[^\\s]+)?\\s+" + "body_bytes_sent=(\\d+)\\s+" + "referer=\"([^\"]*)\"\\s+" + "user_agent=\"([^\"]*)\"\\s+" + "pf=([^\\s]*)\\s+" + "x_event_id=([^\\s]*)" +); + +static const boost::regex generic_crit_log_regex( + "(" + + syslog_regex_string + ") " + + "(?:\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2} )?" // Optional nginx timestamp + + "\\[crit\\] (.+)$" +); + +static const boost::regex generic_emerg_log_regex( + "(" + + syslog_regex_string + ") " + + "(?:\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2} )?" // Optional nginx timestamp + + "\\[emerg\\] (.+)$" +); + +static const boost::regex generic_fallback_log_regex( + "(" + + syslog_regex_string + ") " + + "(?:\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2} )?" // Optional nginx timestamp + + "\\[(\\w+)\\] (.+)$" +); + static const boost::regex server_regex("(\\d+\\.\\d+\\.\\d+\\.\\d+)|(\\w+\\.\\w+)"); static const boost::regex uri_regex("^/"); static const boost::regex port_regex("\\d+"); static const boost::regex response_code_regex("[0-9]{3}"); static const boost::regex http_method_regex("[A-Za-z]+"); +static const string central_nginx_manager = "Central NGINX Manager"; class NginxMessageReader::Impl { public: @@ -64,6 +108,15 @@ public: init() { dbgFlow(D_NGINX_MESSAGE_READER); + + if (Singleton::exists()) { + auto name = Singleton::Consume::by()->get("Service Name"); + if (name.ok()) { + dbgInfo(D_NGINX_MESSAGE_READER) << "Service name: " << *name; + service_name = *name; + } + } + I_MainLoop *mainloop = Singleton::Consume::by(); mainloop->addOneTimeRoutine( I_MainLoop::RoutineType::System, @@ -97,6 +150,7 @@ public: "429", "accessControl.rateLimit.returnCode" ); + dbgTrace(D_NGINX_MESSAGE_READER) << "Selected rate-limit status code: " << rate_limit_status_code; } @@ -107,6 +161,7 @@ private: RESPONSE_CODE, HOST, SOURCE, + SOURCE_PORT, DESTINATION_IP, DESTINATION_PORT, EVENT_MESSAGE, @@ -114,9 +169,21 @@ private: ASSET_NAME, RULE_NAME, RULE_ID, + REFERENCE_ID, + LOG_TYPE, + PROXY_FAULT, + X_EVENT_ID, + MATCHED_REASON, + REMEDIATION, COUNT }; + struct GenericLogInfo { + string timestamp; + string severity; + string message; + }; + void initSyslogServerSocket() { @@ -175,10 +242,12 @@ private: bool log_sent; if (isAccessLog(log)) { log_sent = sendAccessLog(log); - } else if (isAlertErrorLog(log) || isErrorLog(log)) { + } else if (isAlertErrorLog(log) || isErrorLog(log) || isCritErrorLog(log) || isEmergErrorLog(log)) { log_sent = sendErrorLog(log); + } else if (isIncidentLog(log)) { + log_sent = sendIncidentLog(log); } else { - dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format"; + dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format for message: "<< log; continue; } if (!log_sent) { @@ -222,13 +291,74 @@ private: { dbgFlow(D_NGINX_MESSAGE_READER) << "Error log" << log; Maybe> log_info = parseErrorLog(log); - if (!log_info.ok()) { - dbgWarning(D_NGINX_MESSAGE_READER) - << "Failed parsing the NGINX logs. Error: " - << log_info.getErr(); + if (log_info.ok()) { + return sendLog(log_info.unpack()); + } + + if (service_name == central_nginx_manager) { + dbgDebug(D_NGINX_MESSAGE_READER) << "Detailed parsing failed, trying generic parsing"; + Maybe generic_log = parseGenericErrorLog(log); + if (generic_log.ok()) { + return sendGenericLog(generic_log.unpack()); + } + } + + dbgWarning(D_NGINX_MESSAGE_READER) + << "Failed parsing the NGINX logs. Error: " + << log_info.getErr() + << service_name; + return false; + } + + static bool + isValidUuid(const string &uuid_str) + { + if (uuid_str.empty() || uuid_str == "-") { return false; } - return sendLog(log_info.unpack()); + + if (uuid_str.length() != 36) { + return false; + } + + if (uuid_str[8] != '-' || uuid_str[13] != '-' || uuid_str[18] != '-' || uuid_str[23] != '-') { + return false; + } + + for (size_t i = 0; i < uuid_str.length(); ++i) { + if (i == 8 || i == 13 || i == 18 || i == 23) { + continue; + } + char c = uuid_str[i]; + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) { + return false; + } + } + + return true; + } + + bool + sendIncidentLog(const string &log) + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Incident log" << log; + Maybe> log_info = parseIncidentLog(log); + if (!log_info.ok()) { + dbgTrace(D_NGINX_MESSAGE_READER) << log_info.getErr(); + return false; + } + auto unpacked_log_info = log_info.unpack(); + + if (isValidUuid(unpacked_log_info[LogInfo::X_EVENT_ID])) { + dbgTrace(D_NGINX_MESSAGE_READER) + << "Skipping incident log - valid x_event_id present: " + << unpacked_log_info[LogInfo::X_EVENT_ID]; + return true; + } + + unpacked_log_info[LogInfo::LOG_TYPE] = "incidentLog"; + + return sendNotification(unpacked_log_info); } bool @@ -253,7 +383,111 @@ private: } bool - sendLog(const EnumArray &log_info) + isIncidentLog(const string &log) const + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Check if log is of type 'incident log'. Log: " << log; + return log.find("incidentLog") != string::npos; + } + + bool + sendNotification(const EnumArray &log_info) + { + dbgFlow(D_NGINX_MESSAGE_READER); + string event_name; + + if ( + log_info[LogInfo::LOG_TYPE] == "incidentLog" + && (log_info[LogInfo::RESPONSE_CODE][0] == '4' || log_info[LogInfo::RESPONSE_CODE][0] == '5') + ) { + bool is_proxy_fault = (log_info[LogInfo::PROXY_FAULT] == "yes"); + event_name = getEventName(is_proxy_fault); + } else { + return false; + } + + LogGen log( + event_name, + ReportIS::Level::LOG, + ReportIS::Audience::SECURITY, + ReportIS::Severity::HIGH, + ReportIS::Priority::HIGH, + ReportIS::Tags::REVERSE_PROXY + ); + log << LogField("eventConfidence", "High"); + + for (LogInfo field : makeRange()) { + Maybe string_field = convertLogFieldToString(field); + if (!string_field.ok()) { + dbgDebug(D_NGINX_MESSAGE_READER) << "Enum field was not converted: " << string_field.getErr(); + return false; + } + + if (string_field.unpack().empty() || log_info[field].empty()) { + continue; + } + + if (field == LogInfo::ASSET_ID || field == LogInfo::ASSET_NAME) { + log.addToOrigin(LogField(string_field.unpack(), log_info[field])); + continue; + } + + if (field != LogInfo::DESTINATION_PORT && field != LogInfo::SOURCE_PORT) { + log << LogField(string_field.unpack(), log_info[field]); + continue; + } + + try { + log << LogField(string_field.unpack(), stoi(log_info[field])); + } catch (const exception &e) { + dbgError(D_NGINX_MESSAGE_READER) + << "Unable to convert port to numeric value: " + << e.what(); + log << LogField(string_field.unpack(), 0); + } + } + return true; + } + + bool + isCritErrorLog(const string &log) const + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Check if log is of type 'crit log'. Log: " << log; + return log.find("[crit]") != string::npos; + } + + bool + isEmergErrorLog(const string &log) const + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Check if log is of type 'emerg log'. Log: " << log; + return log.find("[emerg]") != string::npos; + } + + Maybe + getCNMEventName(const EnumArray &log_info) const + { + dbgFlow(D_NGINX_MESSAGE_READER); + string event_name; + switch (log_info[LogInfo::RESPONSE_CODE][0]) { + case '4': { + event_name = "NGINX Proxy Error: Invalid request or incorrect NGINX configuration - Request dropped." + " Please check the reverse proxy configuration of your relevant assets"; + break; + } + case '5': { + event_name = "NGINX Proxy Error: Request failed! Please verify your proxy configuration." + "If the issue persists please contact open-appsec support"; + break; + } + default: { + dbgError(D_NGINX_MESSAGE_READER) << "Irrelevant status code"; + return genError("Irrelevant status code"); + } + } + return event_name; + } + + Maybe + getRPMEventName(const EnumArray &log_info) const { dbgFlow(D_NGINX_MESSAGE_READER); string event_name; @@ -265,27 +499,55 @@ private: } case '5': { event_name = "AppSec Gateway reverse proxy error - Request dropped. " - "Please verify the reverse proxy configuration of your relevant assets. " - "If the issue persists please contact Check Point Support"; + "Please verify the reverse proxy configuration of your relevant assets. " + "If the issue persists please contact Check Point Support"; break; } default: { dbgError(D_NGINX_MESSAGE_READER) << "Irrelevant status code"; - return false; + return genError("Irrelevant status code"); } } + return event_name; + } + + Maybe getEventName(const EnumArray &log_info) + { + if (service_name != central_nginx_manager) { + dbgWarning(D_NGINX_MESSAGE_READER) + << "Unknown service name: " + << service_name + << " Response will be sent as CNM"; + } + + return getCNMEventName(log_info); + } + + bool + sendLog(const EnumArray &log_info) + { + dbgFlow(D_NGINX_MESSAGE_READER); + Maybe event_name = getEventName(log_info); + + if (!event_name.ok()) { + dbgError(D_NGINX_MESSAGE_READER) << event_name.getErr(); + return false; + } dbgTrace(D_NGINX_MESSAGE_READER) << "Nginx log's event name and response code: " - << event_name + << event_name.unpack() << ", " << log_info[LogInfo::RESPONSE_CODE]; + LogGen log( - event_name, + event_name.unpack(), ReportIS::Audience::SECURITY, - ReportIS::Severity::INFO, + ReportIS::Severity::HIGH, ReportIS::Priority::LOW, - ReportIS::Tags::REVERSE_PROXY + service_name == central_nginx_manager ? + ReportIS::Tags::CENTRAL_NGINX_MANAGER : + ReportIS::Tags::REVERSE_PROXY ); log << LogField("eventConfidence", "High"); @@ -296,6 +558,10 @@ private: return false; } + if (string_field.unpack().empty() || log_info[field].empty()) { + continue; + } + if (field != LogInfo::DESTINATION_PORT) { log << LogField(string_field.unpack(), log_info[field]); continue; @@ -313,6 +579,45 @@ private: return true; } + bool + sendGenericLog(const GenericLogInfo &log_info) + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Sending generic log"; + + string event_name = "NGINX Proxy Error: Request failed! Please verify your proxy configuration." + "If the issue persists please contact open-appsec support"; + + ReportIS::Severity severity = ReportIS::Severity::MEDIUM; + ReportIS::Priority priority = ReportIS::Priority::MEDIUM; + if (log_info.severity == "emerg" || log_info.severity == "crit") { + severity = ReportIS::Severity::CRITICAL; + priority = ReportIS::Priority::URGENT; + } else if (log_info.severity == "error" || log_info.severity == "alert") { + severity = ReportIS::Severity::HIGH; + priority = ReportIS::Priority::HIGH; + } + + LogGen log( + event_name, + ReportIS::Audience::SECURITY, + severity, + priority, + ReportIS::Tags::CENTRAL_NGINX_MANAGER + ); + + log << LogField("eventConfidence", "High"); + log << LogField("timestamp", log_info.timestamp); + log << LogField("httpResponseBody", formatGenericLogMessage(log_info)); + + return true; + } + + string + formatGenericLogMessage(const GenericLogInfo &log_info) + { + return "[" + log_info.severity + "] " + log_info.message; + } + bool sendRateLimitLog(const EnumArray &log_info) { @@ -338,7 +643,6 @@ private: string security_action = "Drop"; bool is_log_required = false; - // Prevent events checkbox (in triggers) if (rate_limit_trigger.isPreventLogActive(LogTriggerConf::SecurityType::AccessControl)) { is_log_required = true; } @@ -372,7 +676,11 @@ private: dbgDebug(D_NGINX_MESSAGE_READER) << "Enum field was not converted: " << string_field.getErr(); return false; } - + + if (string_field.unpack().empty() || log_info[field].empty()) { + continue; + } + if ( field == LogInfo::HOST || field == LogInfo::URI || @@ -406,6 +714,55 @@ private: return true; } + static string + getStatusCodeMessage(const string &status_code) + { + static map status_messages = { + // 4xx Client Error codes + {"400", "Bad Request - malformed syntax"}, + {"401", "Unauthorized - authentication required"}, + {"403", "Forbidden - access denied"}, + {"404", "Not Found - resource does not exist"}, + {"405", "Method Not Allowed - HTTP verb not permitted"}, + {"408", "Request Timeout - client too slow"}, + {"411", "Length Required - missing Content-Length"}, + {"413", "Payload Too Large - body exceeds limit"}, + {"414", "URI Too Long - request target exceeds limit"}, + {"416", "Range Not Satisfiable - invalid byte range"}, + {"429", "Too Many Requests - rate limit exceeded"}, + {"431", "Header Fields Too Large - headers exceed limit"}, + {"451", "Unavailable For Legal Reasons"}, + + // NGINX specific 4xx codes + {"494", "Request Header Too Large - NGINX internal"}, + {"495", "SSL Certificate Error - invalid client cert"}, + {"496", "SSL Certificate Required - none provided"}, + {"497", "Plain HTTP sent to HTTPS port"}, + + // 5xx Server Error codes + {"500", "Internal Server Error"}, + {"501", "Not Implemented - feature unsupported"}, + {"502", "Bad Gateway - upstream connection failed"}, + {"503", "Service Unavailable - server overloaded"}, + {"504", "Gateway Timeout - upstream too slow"}, + {"505", "HTTP Version Not Supported"}, + {"507", "Insufficient Storage - WebDAV quota exceeded"} + }; + + auto it = status_messages.find(status_code); + return (it != status_messages.end()) ? it->second : "HTTP Error " + status_code; + } + + static string + getEventName(bool is_proxy_fault) + { + if (!is_proxy_fault) { + return "Upstream Application Error"; + } + + return "Reverse Proxy Error"; + } + Maybe convertLogFieldToString(LogInfo field) { @@ -420,7 +777,9 @@ private: case LogInfo::HOST: return string("httpHostName"); case LogInfo::SOURCE: - return string("httpSourceId"); + return string("sourceip"); + case LogInfo::SOURCE_PORT: + return string("sourcePort"); case LogInfo::DESTINATION_IP: return string("destinationIp"); case LogInfo::DESTINATION_PORT: @@ -435,6 +794,18 @@ private: return string("ruleId"); case LogInfo::RULE_NAME: return string("ruleName"); + case LogInfo::REFERENCE_ID: + return string("eventReferenceId"); + case LogInfo::MATCHED_REASON: + return string("matchreason"); + case LogInfo::REMEDIATION: + return string("eventRemediation"); + case LogInfo::LOG_TYPE: + return string(""); + case LogInfo::PROXY_FAULT: + return string(""); + case LogInfo::X_EVENT_ID: + return string(""); case LogInfo::COUNT: dbgError(D_NGINX_MESSAGE_READER) << "LogInfo::COUNT is not allowed"; return genError("LogInfo::COUNT is not allowed"); @@ -512,7 +883,7 @@ private: static_cast(stoi(log_info[LogInfo::DESTINATION_PORT])) ); } catch (const exception &e) { - dbgError(D_NGINX_MESSAGE_READER) << "Failed register values for context " << e.what(); + dbgWarning(D_NGINX_MESSAGE_READER) << "Failed register values for context " << e.what(); } ctx.registerValue(HttpTransactionData::host_name_ctx, log_info[LogInfo::HOST]); ctx.registerValue(HttpTransactionData::uri_ctx, log_info[LogInfo::URI]); @@ -525,12 +896,65 @@ private: } BasicRuleConfig context = rule_by_ctx.unpack(); + + dbgTrace(D_NGINX_MESSAGE_READER) + << "Adding context fields to log info. Asset ID: " + << context.getAssetId() + << ", Asset Name: " + << context.getAssetName() + << ", Rule ID: " + << context.getRuleId() + << ", Rule Name: " + << context.getRuleName(); + log_info[LogInfo::ASSET_ID] = context.getAssetId(); log_info[LogInfo::ASSET_NAME] = context.getAssetName(); log_info[LogInfo::RULE_ID] = context.getRuleId(); log_info[LogInfo::RULE_NAME] = context.getRuleName(); } + Maybe + parseGenericErrorLog(const string &log_line) + { + dbgFlow(D_NGINX_MESSAGE_READER) << "Parsing generic error log: " << log_line; + + boost::smatch matcher; + GenericLogInfo generic_log; + + if (isCritErrorLog(log_line)) { + if (NGEN::Regex::regexSearch(__FILE__, __LINE__, log_line, matcher, generic_crit_log_regex)) { + const int timestamp_index = 2; + const int message_index = 5; + generic_log.timestamp = string(matcher[timestamp_index].first, matcher[timestamp_index].second); + generic_log.severity = "crit"; + generic_log.message = string(matcher[message_index].first, matcher[message_index].second); + return generic_log; + } + } else if (isEmergErrorLog(log_line)) { + if (NGEN::Regex::regexSearch(__FILE__, __LINE__, log_line, matcher, generic_emerg_log_regex)) { + const int timestamp_index = 2; + const int message_index = 5; + generic_log.timestamp = string(matcher[timestamp_index].first, matcher[timestamp_index].second); + generic_log.severity = "emerg"; + generic_log.message = string(matcher[message_index].first, matcher[message_index].second); + return generic_log; + } + } + + if (NGEN::Regex::regexSearch(__FILE__, __LINE__, log_line, matcher, generic_fallback_log_regex)) { + const int timestamp_index = 2; + const int severity_index = 5; + const int message_index = 6; + generic_log.timestamp = string(matcher[timestamp_index].first, matcher[timestamp_index].second); + generic_log.severity = string(matcher[severity_index].first, matcher[severity_index].second); + generic_log.message = string(matcher[message_index].first, matcher[message_index].second); + return generic_log; + } + + dbgWarning(D_NGINX_MESSAGE_READER) << "Could not parse log with generic method: " << log_line; + return genError("Could not parse log with generic method"); + } + Maybe> parseErrorLog(const string &log_line) { @@ -540,17 +964,28 @@ private: boost::smatch matcher; vector result; + boost::regex selected_regex; + + if (isAlertErrorLog(log_line)) { + selected_regex = alert_log_regex; + } else if (isErrorLog(log_line)) { + selected_regex = error_log_regex; + } else { + dbgWarning(D_NGINX_MESSAGE_READER) << "No matching log type found for log: " << log_line; + return genError("No matching log type found"); + } + if ( !NGEN::Regex::regexSearch( __FILE__, __LINE__, log_line, matcher, - isAlertErrorLog(log_line) ? alert_log_regex : error_log_regex + selected_regex ) ) { - dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format"; - return genError("Unexpected nginx log format"); + dbgWarning(D_NGINX_MESSAGE_READER) << "Detailed regex parsing failed for log: " << log_line; + return genError("Detailed regex parsing failed"); } const int event_message_index = 6; @@ -591,8 +1026,8 @@ private: addContextFieldsToLogInfo(log_info); if (!validateLog(log_info)) { - dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format"; - return genError("Unexpected nginx log format"); + dbgWarning(D_NGINX_MESSAGE_READER) << "Log validation failed for detailed parsing"; + return genError("Log validation failed for detailed parsing"); } return log_info; @@ -642,6 +1077,104 @@ private: return log_info; } + + Maybe> + parseIncidentLog(const string &log_line) + { + dbgTrace(D_NGINX_MESSAGE_READER) << "Parsing incident log line: " << log_line; + EnumArray log_info(EnumArray::Fill(), string("")); + + boost::smatch matcher; + if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, log_line, matcher, incident_log_regex)) { + dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx incident log format"; + return genError("Unexpected nginx incident log format"); + } + + const int host_index = 5; // host=server_name + const int host_port_index = 6; // host_port=port + const int client_addr_index = 7; // client_addr=IP + const int client_port_index = 8; // client_port=port + const int request_index = 10; // request="METHOD URI HTTP/1.1" + const int status_index = 11; // status=CODE + const int uri_index = 12; // uri="PATH" + const int request_id_index = 13; // request_id=ID + const int proxy_fault_index = 19; // pf=yes or empty + const int x_event_id_index = 20; // x_event_id=value or empty + + string host = string(matcher[host_index].first, matcher[host_index].second); + string host_port = string(matcher[host_port_index].first, matcher[host_port_index].second); + string uri = string(matcher[uri_index].first, matcher[uri_index].second); + string proxy_fault = string(matcher[proxy_fault_index].first, matcher[proxy_fault_index].second); + + log_info[LogInfo::HOST] = host; + log_info[LogInfo::URI] = uri; + log_info[LogInfo::DESTINATION_PORT] = host_port; + log_info[LogInfo::PROXY_FAULT] = proxy_fault; + + dbgTrace(D_NGINX_MESSAGE_READER) + << "Parsed host: " + << host + << ", host_port: " + << host_port + << ", uri: " + << uri + << ", proxy_fault: " + << proxy_fault; + + addContextFieldsToLogInfo(log_info); + if (log_info[LogInfo::ASSET_ID].empty() || log_info[LogInfo::ASSET_NAME].empty()) { + return genError("Asset ID or Asset Name is empty"); + } + + string client_addr = string(matcher[client_addr_index].first, matcher[client_addr_index].second); + string client_port = string(matcher[client_port_index].first, matcher[client_port_index].second); + string request = string(matcher[request_index].first, matcher[request_index].second); + string status = string(matcher[status_index].first, matcher[status_index].second); + string request_id = string(matcher[request_id_index].first, matcher[request_id_index].second); + string x_event_id = string(matcher[x_event_id_index].first, matcher[x_event_id_index].second); + + dbgTrace(D_NGINX_MESSAGE_READER) + << "Parsed client_addr: " + << client_addr + << ", client_port: " + << client_port + << ", request: " + << request + << ", status: " + << status + << ", request_id: " + << request_id + << ", proxy_fault: " + << proxy_fault + << ", x_event_id: " + << x_event_id; + + vector request_parts; + boost::split(request_parts, request, boost::is_any_of(" "), boost::token_compress_on); + string http_method = request_parts.size() > 0 ? request_parts[0] : ""; + + log_info[LogInfo::REFERENCE_ID] = request_id; + log_info[LogInfo::RESPONSE_CODE] = status; + log_info[LogInfo::HTTP_METHOD] = http_method; + log_info[LogInfo::SOURCE] = client_addr; + log_info[LogInfo::SOURCE_PORT] = client_port; + log_info[LogInfo::DESTINATION_IP] = host; + log_info[LogInfo::X_EVENT_ID] = x_event_id; + + + if (!validateLog(log_info)) { + dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx incident log format after validation"; + return genError("Unexpected nginx incident log format after validation"); + } + + if (proxy_fault == "yes") { + log_info[LogInfo::MATCHED_REASON] = getStatusCodeMessage(status); + log_info[LogInfo::REMEDIATION] = getIncidentLogRemediation(status); + } + + return log_info; + } + static bool validateLog(const EnumArray &log_info) { @@ -708,8 +1241,54 @@ private: return move(raw_log); } + static string + getIncidentLogRemediation(const string &status_code) + { + static map status_remediations = { + {"400", "Check request syntax and format; validate client input and HTTP headers"}, + {"401", "Verify authentication credentials and configuration"}, + {"403", "If this is a valid WAF block, no action is required; otherwise," + " contact Check Point Support"}, + {"404", "Validate the requested asset URI"}, + {"405", "Ensure allowed HTTP methods are configured correctly"}, + {"408", "Adjust `client_header_timeout` and `client_body_timeout`" + " in the assets advanced nginx server block"}, + {"411", "Require a Content-Length header on requests with a body"}, + {"413", "Increase `client_max_body_size` in the asset advanced nginx server block"}, + {"414", "Increase `large_client_header_buffers` (e.g. `large_client_header_buffers 4 32k;`)" + " in the asset advanced nginx server block"}, + {"416", "Validate Range request headers against actual file sizes"}, + {"429", "Review rate limiting settings and implement proper backoff strategies"}, + {"431", "Increase `large_client_header_buffers` (e.g. `large_client_header_buffers 4 32k;`)" + " in the asset advanced nginx server block"}, + {"451", "Review content filtering policies and legal compliance requirements"}, + {"494", "Increase `large_client_header_buffers` (e.g. `large_client_header_buffers 4 32k;`)" + " in the asset advanced nginx server block"}, + {"495", "Confirm client SSL certificates are valid and properly configured"}, + {"496", "Configure SSL client certificate requirements correctly"}, + {"497", "Enforce HTTPS for secure endpoints"}, + {"500", "Contact Check Point Support"}, + {"501", "Ensure requested features are supported and configured correctly"}, + {"502", "Verify upstream server connectivity and health; check proxy configuration"}, + {"503", "Assess server capacity and load balancing; verify upstream availability"}, + {"504", "Increase `proxy_connect_timeout` and `proxy_read_timeout`" + " in the asset advanced nginx configuration"}, + {"505", "Ensure HTTP version compatibility between client and server"}, + {"507", "Contact Check Point Support"} + }; + + auto it = status_remediations.find(status_code); + if (it != status_remediations.end()) { + return it->second + ". If this suggestion does not resolve the issue, please contact support."; + } + + return "Please verify the nginx configuration of your relevant assets. " + "Please contact support if the issue persists."; + } + I_Socket::socketFd syslog_server_socket = -1; string rate_limit_status_code = "429"; + string service_name = "Unnamed Nano Service"; }; NginxMessageReader::NginxMessageReader() : Component("NginxMessageReader"), pimpl(make_unique()) {} diff --git a/components/security_apps/http_geo_filter/http_geo_filter.cc b/components/security_apps/http_geo_filter/http_geo_filter.cc index fb0f026..7ec6a79 100644 --- a/components/security_apps/http_geo_filter/http_geo_filter.cc +++ b/components/security_apps/http_geo_filter/http_geo_filter.cc @@ -35,7 +35,8 @@ public: init() { dbgTrace(D_GEO_FILTER) << "Init Http Geo filter component"; - registerListener(); + handleNewPolicy(); + registerConfigLoadCb([this]() { handleNewPolicy(); }); } void @@ -44,6 +45,19 @@ public: unregisterListener(); } + void + handleNewPolicy() + { + if (ParameterException::isGeoLocationExceptionExists()) { + registerListener(); + return; + } + + if (!ParameterException::isGeoLocationExceptionExists()) { + unregisterListener(); + } + } + string getListenerName() const override { return "HTTP geo filter"; } void @@ -56,7 +70,7 @@ public: << "Load http geo filter default action. Action: " << default_action_maybe.unpack(); } else { - default_action = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; + default_action = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT; dbgTrace(D_GEO_FILTER) << "No http geo filter default action. Action: Irrelevant"; } } @@ -66,7 +80,7 @@ public: { dbgTrace(D_GEO_FILTER) << getListenerName() << " new transaction event"; - if (!event.isLastHeader()) return EventVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT); + if (!event.isLastHeader()) return EventVerdict(ServiceVerdict::TRAFFIC_VERDICT_INSPECT); std::set ip_set; auto env = Singleton::Consume::by(); auto maybe_xff = env->get(HttpTransactionData::xff_vals_ctx); @@ -101,15 +115,17 @@ public: } - ngx_http_cp_verdict_e exception_verdict = getExceptionVerdict(ip_set); - if (exception_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) { + ServiceVerdict exception_verdict = getExceptionVerdict(ip_set); + if (exception_verdict != ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT) { return EventVerdict(exception_verdict); } - ngx_http_cp_verdict_e geo_lookup_verdict = getGeoLookupVerdict(ip_set); - if (geo_lookup_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) { - return EventVerdict(geo_lookup_verdict); - } + // deprecated for now + // ServiceVerdict geo_lookup_verdict = getGeoLookupVerdict(ip_set); + // if (geo_lookup_verdict != ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT) { + // return EventVerdict(geo_lookup_verdict); + // } + return EventVerdict(default_action); } @@ -146,7 +162,7 @@ private: void removeTrustedIpsFromXff(std::set &xff_set) { - auto identify_config = getConfiguration( + auto identify_config = getConfigurationWithCache( "rulebase", "usersIdentifiers" ); @@ -189,33 +205,34 @@ private: return os.str(); } - ngx_http_cp_verdict_e - convertActionToVerdict(const string &action) const - { - if (action == "accept") return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; - if (action == "drop") return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; - } - - ngx_http_cp_verdict_e + ServiceVerdict convertBehaviorValueToVerdict(const BehaviorValue &behavior_value) const { if (behavior_value == BehaviorValue::ACCEPT || behavior_value == BehaviorValue::IGNORE) { - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; } if (behavior_value == BehaviorValue::DROP || behavior_value == BehaviorValue::REJECT) { - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + return ServiceVerdict::TRAFFIC_VERDICT_DROP; } - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; + return ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT; } - ngx_http_cp_verdict_e + // LCOV_EXCL_START Reason: deprecated for now + ServiceVerdict + convertActionToVerdict(const string &action) const + { + if (action == "accept") return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; + if (action == "drop") return ServiceVerdict::TRAFFIC_VERDICT_DROP; + return ServiceVerdict::TRAFFIC_VERDICT_INSPECT; + } + + ServiceVerdict getGeoLookupVerdict(const std::set &sources) { auto maybe_geo_config = getConfiguration("rulebase", "httpGeoFilter"); if (!maybe_geo_config.ok()) { dbgTrace(D_GEO_FILTER) << "Failed to load HTTP Geo Filter config. Error:" << maybe_geo_config.getErr(); - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; + return ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT; } GeoConfig geo_config = maybe_geo_config.unpack(); EnumArray geo_location_data; @@ -252,12 +269,12 @@ private: << ", country code: " << country_code; generateVerdictLog( - ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT, + ServiceVerdict::TRAFFIC_VERDICT_ACCEPT, geo_config.getId(), true, geo_location_data ); - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; } if (geo_config.isBlockedCountry(country_code)) { dbgTrace(D_GEO_FILTER) @@ -266,12 +283,12 @@ private: << ", country code: " << country_code; generateVerdictLog( - ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP, + ServiceVerdict::TRAFFIC_VERDICT_DROP, geo_config.getId(), true, geo_location_data ); - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + return ServiceVerdict::TRAFFIC_VERDICT_DROP; } } dbgTrace(D_GEO_FILTER) @@ -286,22 +303,23 @@ private: ); return convertActionToVerdict(geo_config.getDefaultAction()); } + // LCOV_EXCL_STOP - Maybe> + Maybe> getBehaviorsVerdict( const unordered_map> &behaviors_map_to_search, EnumArray geo_location_data) { bool is_matched = false; ParameterBehavior matched_behavior; - ngx_http_cp_verdict_e matched_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; + ServiceVerdict matched_verdict = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT; I_GenericRulebase *i_rulebase = Singleton::Consume::by(); set behaviors_set = i_rulebase->getBehavior(behaviors_map_to_search); dbgTrace(D_GEO_FILTER) << "get verdict from: " << behaviors_set.size() << " behaviors"; for (const ParameterBehavior &behavior : behaviors_set) { matched_verdict = convertBehaviorValueToVerdict(behavior.getValue()); if ( - matched_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP + matched_verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP ){ dbgTrace(D_GEO_FILTER) << "behavior verdict: DROP, exception id: " << behavior.getId(); generateVerdictLog( @@ -310,10 +328,10 @@ private: false, geo_location_data ); - return pair(matched_verdict, behavior.getId()); + return pair(matched_verdict, behavior.getId()); } else if ( - matched_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT + matched_verdict == ServiceVerdict::TRAFFIC_VERDICT_ACCEPT ){ dbgTrace(D_GEO_FILTER) << "behavior verdict: ACCEPT, exception id: " << behavior.getId(); matched_behavior = behavior; @@ -321,19 +339,19 @@ private: } } if (is_matched) { - return pair( - ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT, + return pair( + ServiceVerdict::TRAFFIC_VERDICT_ACCEPT, matched_behavior.getId() ); } return genError("No exception matched to HTTP geo filter rule"); } - ngx_http_cp_verdict_e + ServiceVerdict getExceptionVerdict(const std::set &sources) { - pair curr_matched_behavior; - ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; + pair curr_matched_behavior; + ServiceVerdict verdict = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT; I_GeoLocation *i_geo_location = Singleton::Consume::by(); EnumArray geo_location_data; auto env = Singleton::Consume::by(); @@ -389,7 +407,7 @@ private: if (matched_behavior_maybe.ok()) { curr_matched_behavior = matched_behavior_maybe.unpack(); verdict = curr_matched_behavior.first; - if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) { + if (verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP) { return verdict; } } @@ -402,13 +420,13 @@ private: if (matched_behavior_maybe.ok()) { curr_matched_behavior = matched_behavior_maybe.unpack(); verdict = curr_matched_behavior.first; - if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) { + if (verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP) { return verdict; } } } - if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT) { + if (verdict == ServiceVerdict::TRAFFIC_VERDICT_ACCEPT) { generateVerdictLog( verdict, curr_matched_behavior.second, @@ -421,7 +439,7 @@ private: void generateVerdictLog( - const ngx_http_cp_verdict_e &verdict, + const ServiceVerdict &verdict, const string &matched_id, bool is_geo_filter, const EnumArray geo_location_data, @@ -430,7 +448,7 @@ private: { dbgTrace(D_GEO_FILTER) << "Generate Log for verdict - HTTP geo filter"; auto &trigger = getConfigurationWithDefault(default_triger, "rulebase", "log"); - bool is_prevent = verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + bool is_prevent = verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP; string matched_on = is_geo_filter ? "geoFilterPracticeId" : "exceptionId"; LogGen log = trigger( "Web Request - HTTP Geo Filter", @@ -469,7 +487,7 @@ private: << LogField("sourceCountryName", geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_NAME]); } - ngx_http_cp_verdict_e default_action = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; + ServiceVerdict default_action = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT; }; HttpGeoFilter::HttpGeoFilter() : Component("HttpGeoFilter"), pimpl(make_unique()) {} @@ -491,6 +509,6 @@ void HttpGeoFilter::preload() { registerExpectedConfiguration("rulebase", "httpGeoFilter"); - registerExpectedConfiguration("rulebase", "usersIdentifiers"); + registerExpectedConfigurationWithCache("assetId", "rulebase", "usersIdentifiers"); registerConfigLoadCb([this]() { pimpl->loadDefaultAction(); }); } diff --git a/components/security_apps/ips/include/ips_signatures.h b/components/security_apps/ips/include/ips_signatures.h index 0ecee73..aee5c90 100644 --- a/components/security_apps/ips/include/ips_signatures.h +++ b/components/security_apps/ips/include/ips_signatures.h @@ -21,13 +21,13 @@ #include #include "config.h" +#include "i_generic_rulebase.h" #include "i_first_tier_agg.h" #include "ips_entry.h" #include "ips_enums.h" #include "log_generator.h" #include "parsed_context.h" #include "pm_hook.h" -#include "i_generic_rulebase.h" #define DEFAULT_IPS_YIELD_COUNT 500 @@ -402,6 +402,7 @@ public: LogTriggerConf getTrigger() const; + std::set getBehavior(const std::unordered_map> &exceptions_dict) const; @@ -410,10 +411,21 @@ private: /// \param ips_state The IPS entry. ActionResults getAction(const IPSEntry &ips_state) const; + void sendLog( + const Buffer &context_buffer, + const IPSEntry &ips_state, + const std::tuple< + IPSSignatureSubTypes::SignatureAction, + std::string, std::vector + > &override_action, + bool is_prevent + ) const; + std::shared_ptr signature; SignatureAction action; std::string trigger_id; std::string exception_id; + mutable bool bSupressLog = false; }; } // namespace IPSSignatureSubTypes diff --git a/components/security_apps/ips/ips_comp.cc b/components/security_apps/ips/ips_comp.cc index 2f44a2f..0040f34 100644 --- a/components/security_apps/ips/ips_comp.cc +++ b/components/security_apps/ips/ips_comp.cc @@ -13,7 +13,7 @@ #include "virtual_modifiers.h" #include "helper.h" #include "ips_common_types.h" -#include "nginx_attachment_common.h" +#include "nano_attachment_common.h" using namespace std; @@ -46,9 +46,9 @@ class IPSComp::Impl public Listener, public Listener { - static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; - static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; - static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + static constexpr auto DROP = ServiceVerdict::TRAFFIC_VERDICT_DROP; + static constexpr auto ACCEPT = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; + static constexpr auto INSPECT = ServiceVerdict::TRAFFIC_VERDICT_INSPECT; class SigsFirstTierAgg { @@ -383,8 +383,8 @@ IPSComp::preload() registerExpectedResource("IPSSnortSigs", "protections"); registerExpectedConfiguration("IPS", "IpsConfigurations"); registerExpectedConfiguration("IPS", "Max Field Size"); - registerExpectedConfiguration("IPS", "IpsProtections"); - registerExpectedConfiguration("IPSSnortSigs", "SnortProtections"); + registerExpectedConfigurationWithCache("assetId", "IPS", "IpsProtections"); + registerExpectedConfigurationWithCache("assetId", "IPSSnortSigs", "SnortProtections"); registerExpectedConfigFile("ips", Config::ConfigFileType::Policy); registerExpectedConfigFile("ips", Config::ConfigFileType::Data); registerExpectedConfigFile("snort", Config::ConfigFileType::Policy); diff --git a/components/security_apps/ips/ips_signatures.cc b/components/security_apps/ips/ips_signatures.cc index b4f40bd..2da3fde 100644 --- a/components/security_apps/ips/ips_signatures.cc +++ b/components/security_apps/ips/ips_signatures.cc @@ -321,6 +321,9 @@ SignatureAndAction::getAction(const IPSEntry &ips_state) const override_actions.insert(behavior.getValue()); const string &override_id = behavior.getId(); if (!override_id.empty()) override_ids.push_back(override_id); + } else if(behavior.getKey() == BehaviorKey::LOG && behavior.getValue() == BehaviorValue::IGNORE) { + dbgTrace(D_IPS) << "setting bSupressLog due to override behavior: " << behavior.getId(); + bSupressLog = true; } } @@ -358,8 +361,10 @@ set SignatureAndAction::getBehavior(const unordered_map> &exceptions_dict) const { I_GenericRulebase *i_rulebase = Singleton::Consume::by(); - if (exception_id.empty()) return i_rulebase->getBehavior(exceptions_dict); - + if (exception_id.empty()) { + dbgTrace(D_RULEBASE_CONFIG) << "No exception id provided, using default behavior"; + return i_rulebase->getBehavior(exceptions_dict); + } return i_rulebase->getParameterException(exception_id).getBehavior(exceptions_dict); } @@ -453,10 +458,25 @@ SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set(override_action) == IPSSignatureSubTypes::SignatureAction::PREVENT; + if(bSupressLog) { + dbgTrace(D_IPS) << "Signature matched - not sending log due to exception behavior"; + } else { + sendLog(context_buffer, ips_state, override_action, is_prevent); + } + return is_prevent; +} + +void SignatureAndAction::sendLog( + const Buffer &context_buffer, + const IPSEntry &ips_state, + const tuple> &override_action, + bool is_prevent +) const +{ + dbgFlow(D_IPS) << "Signature matched - sending log"; auto trigger = getTrigger(); - bool is_prevent = get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::PREVENT; auto severity = signature->getSeverity() < IPSLevel::HIGH ? Severity::HIGH : Severity::CRITICAL; if (get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::DETECT) severity = Severity::INFO; @@ -560,7 +580,7 @@ SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set max_size) { if (req_size + 500 > max_size) { - res_size = std::min(500u, res_size); + res_size = min(500u, res_size); req_size = max_size - res_size; } else { res_size = max_size - req_size; @@ -572,10 +592,8 @@ SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set(override_action)); if (!get<2>(override_action).empty()) log.addToOrigin(LogField("exceptionIdList", get<2>(override_action))); - - log << LogField("securityAction", is_prevent ? "Prevent" : "Detect"); - return is_prevent; + log << LogField("securityAction", is_prevent ? "Prevent" : "Detect"); } void diff --git a/components/security_apps/ips/ips_ut/component_ut.cc b/components/security_apps/ips/ips_ut/component_ut.cc index 3df1e43..812bcbf 100644 --- a/components/security_apps/ips/ips_ut/component_ut.cc +++ b/components/security_apps/ips/ips_ut/component_ut.cc @@ -109,11 +109,11 @@ operator ==(const EventVerdict &first, const EventVerdict &second) return first.getVerdict() == second.getVerdict(); } -const EventVerdict ComponentTest::inspect(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT); +const EventVerdict ComponentTest::inspect(ServiceVerdict::TRAFFIC_VERDICT_INSPECT); -const EventVerdict ComponentTest::accept(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT); +const EventVerdict ComponentTest::accept(ServiceVerdict::TRAFFIC_VERDICT_ACCEPT); -const EventVerdict ComponentTest::drop(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP); +const EventVerdict ComponentTest::drop(ServiceVerdict::TRAFFIC_VERDICT_DROP); TEST_F(ComponentTest, check_init_fini_do_not_crush) { diff --git a/components/security_apps/ips/ips_ut/signatures_ut.cc b/components/security_apps/ips/ips_ut/signatures_ut.cc index b52eaba..4a78e35 100644 --- a/components/security_apps/ips/ips_ut/signatures_ut.cc +++ b/components/security_apps/ips/ips_ut/signatures_ut.cc @@ -19,6 +19,7 @@ #include "generic_rulebase/generic_rulebase_context.h" #include "encryptor.h" #include "mock/mock_table.h" +USE_DEBUG_FLAG(D_IPS); using namespace testing; using namespace std; @@ -194,6 +195,101 @@ public: gen_ctx->activate(); } + // Loads an exception with log: ignore behavior, as in the provided JSON + void loadExceptionsIgnoreLog() + { + BasicRuleConfig::preload(); + registerExpectedConfiguration("rulebase", "exception"); + + string test_config( + "{" + " \"rulebase\": {" + " \"rulesConfig\": [" + " {" + " \"context\": \"All()\"," + " \"priority\": 1," + " \"ruleId\": \"5eaef0726765c30010bae8bb\"," + " \"ruleName\": \"Acme web API\"," + " \"assetId\": \"5e243effd858007660b758ad\"," + " \"assetName\": \"Acme Power API\"," + " \"parameters\": [" + " {" + " \"parameterId\": \"6c3867be-4da5-42c2-93dc-8f509a764003\"," + " \"parameterType\": \"exceptions\"," + " \"parameterName\": \"exception\"" + " }" + " ]," + " \"zoneId\": \"\"," + " \"zoneName\": \"\"" + " }" + " ]," + " \"exception\": [" + " {" + " \"context\": \"Any(parameterId(6c3867be-4da5-42c2-93dc-8f509a764003))\"," + " \"exceptions\": [" + " {" + " \"match\": {" + " \"type\": \"operator\"," + " \"op\": \"and\"," + " \"items\": [" + " {" + " \"type\": \"condition\"," + " \"op\": \"equals\"," + " \"key\": \"sourceIdentifier\"," + " \"value\": [\"1.1.1.1\"]" + " }" + " ]" + " }," + " \"behavior\": {" + " \"key\": \"log\"," + " \"value\": \"ignore\"" + " }" + " }," + " {" + " \"match\": {" + " \"type\": \"operator\"," + " \"op\": \"or\"," + " \"items\": [" + " {" + " \"type\": \"condition\"," + " \"op\": \"equals\"," + " \"key\": \"protectionName\"," + " \"value\": [\"Test1\"]" + " }," + " {" + " \"type\": \"condition\"," + " \"op\": \"equals\"," + " \"key\": \"protectionName\"," + " \"value\": [\"Test2\"]" + " }," + " {" + " \"type\": \"condition\"," + " \"op\": \"equals\"," + " \"key\": \"sourceIdentifier\"," + " \"value\": [\"1.1.1.1\"]" + " }" + " ]" + " }," + " \"behavior\": {" + " \"key\": \"action\"," + " \"value\": \"accept\"" + " }" + " }" + " ]" + " }" + " ]" + " }" + "}" + ); + + istringstream ss(test_config); + auto i_config = Singleton::Consume::from(config); + i_config->loadConfiguration(ss); + + gen_ctx = make_unique(); + gen_ctx->activate(); + } + void load(const IPSSignaturesResource &policy, const string &severity, const string &confidence) { @@ -261,6 +357,7 @@ public: IPSSignaturesResource performance_signatures3; IPSSignaturesResource single_broken_signature; NiceMock table; + NiceMock logs; MockAgg mock_agg; private: @@ -273,7 +370,6 @@ private: ConfigComponent config; Encryptor encryptor; AgentDetails details; - StrictMock logs; IPSEntry ips_state; string signature1 = @@ -524,6 +620,23 @@ TEST_F(SignatureTest, basic_load_of_signatures) EXPECT_FALSE(sigs.isEmpty("HTTP_REQUEST_BODY")); } +TEST_F(SignatureTest, ignore_exception_suppresses_log) +{ + load(single_signature2, "Low or above", "Low"); + loadExceptionsIgnoreLog(); + + expectLog("\"protectionId\": \"Test3\"", "\"eventSeverity\": \"Critical\""); + + EXPECT_TRUE(checkData("gggddd")); + + ScopedContext ctx; + ctx.registerValue("sourceIdentifiers", "1.1.1.1"); + + // No log should be sent when the exception matches + EXPECT_CALL(logs, sendLog(_)).Times(0); + EXPECT_FALSE(checkData("gggddd")); +} + TEST_F(SignatureTest, single_signature_matching_override) { load(single_signature, "Low or above", "Low"); diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control.cc b/components/security_apps/layer_7_access_control/layer_7_access_control.cc index b17057f..5e6ac5e 100644 --- a/components/security_apps/layer_7_access_control/layer_7_access_control.cc +++ b/components/security_apps/layer_7_access_control/layer_7_access_control.cc @@ -7,7 +7,7 @@ #include "config.h" #include "cache.h" #include "http_inspection_events.h" -#include "nginx_attachment_common.h" +#include "nano_attachment_common.h" #include "intelligence_comp_v2.h" #include "intelligence_is_v2/query_request_v2.h" #include "log_generator.h" @@ -117,12 +117,12 @@ public: if (!isAppEnabled()) { dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Accept verdict as the Layer-7 Access Control app is disabled"; - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; } if (!event.isLastHeader()) { dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Inspect verdict"; - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + return ServiceVerdict::TRAFFIC_VERDICT_INSPECT; } return handleEvent(); @@ -133,7 +133,7 @@ public: { if (!isAppEnabled()) { dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Accept verdict as the Layer-7 Access Control app is disabled"; - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; } dbgTrace(D_L7_ACCESS_CONTROL) << "Handling wait verdict"; @@ -217,13 +217,13 @@ Layer7AccessControl::Impl::queryIpReputation(const string &source_ip) if (!ip_reputation.ok()) { dbgTrace(D_L7_ACCESS_CONTROL) << "Scheduling Intelligence query - returning Wait verdict"; scheduleIntelligenceQuery(source_ip); - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT; + return ServiceVerdict::TRAFFIC_VERDICT_DELAYED; } if (!ip_reputation.unpack().isMalicious()) { dbgTrace(D_L7_ACCESS_CONTROL) << "Accepting IP: " << source_ip; ip_reputation_cache.deleteEntry(source_ip); - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; } return generateLog(source_ip, ip_reputation.unpack()); @@ -246,7 +246,7 @@ Layer7AccessControl::Impl::handleEvent() } dbgWarning(D_L7_ACCESS_CONTROL) << "Could not extract the Client IP address from context"; - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; } void @@ -354,11 +354,11 @@ Layer7AccessControl::Impl::generateLog(const string &source_ip, const Intelligen if (isPrevent()) { dbgTrace(D_L7_ACCESS_CONTROL) << "Dropping IP: " << source_ip; - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + return ServiceVerdict::TRAFFIC_VERDICT_DROP; } dbgTrace(D_L7_ACCESS_CONTROL) << "Detecting IP: " << source_ip; - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; } Maybe diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc index eee99a0..209dab9 100644 --- a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc +++ b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc @@ -49,10 +49,10 @@ public: void registerTransactionData(); void verifyReport(const Report &report, const string &source_identifier, const string &security_action); - const EventVerdict drop_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; - const EventVerdict accept_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; - const EventVerdict inspect_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; - const EventVerdict wait_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT; + const EventVerdict drop_verdict = ServiceVerdict::TRAFFIC_VERDICT_DROP; + const EventVerdict accept_verdict = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; + const EventVerdict inspect_verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT; + const EventVerdict wait_verdict = ServiceVerdict::TRAFFIC_VERDICT_DELAYED; Layer7AccessControl l7_access_control; ::Environment env; ConfigComponent config; diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h index 0560360..1f95f4a 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h @@ -68,6 +68,15 @@ checkSAMLPortal(const string &command_output) return string("false"); } +Maybe +checkIdaPDP(const string &command_output) +{ + if (command_output.find("is_collecting_identities (true)") != string::npos) { + return string("true"); + } + return string("false"); +} + Maybe checkInfinityIdentityEnabled(const string &command_output) { @@ -139,6 +148,14 @@ checkIsInstallHorizonTelemetrySucceeded(const string &command_output) return command_output; } +Maybe +checkIsCME(const string &command_output) +{ + if (command_output == "" ) return string("false"); + + return command_output; +} + Maybe getOtlpAgentGaiaOsRole(const string &command_output) { @@ -147,6 +164,129 @@ getOtlpAgentGaiaOsRole(const string &command_output) return command_output; } + +// Helper function for case-insensitive substring search +inline bool +containsIgnoreCase(const string& text, const string& pattern) { + string lowerText = text; + string lowerPattern = pattern; + transform(lowerText.begin(), lowerText.end(), lowerText.begin(), ::tolower); + transform(lowerPattern.begin(), lowerPattern.end(), lowerPattern.begin(), ::tolower); + return lowerText.find(lowerPattern) != string::npos; +} + +inline Maybe +extractValue(const string& line, const string& field) { + size_t colonPos = line.find(':'); + if (colonPos == string::npos) { + return Maybe(Error("no match")); + } + string key = line.substr(0, colonPos); + string value = line.substr(colonPos + 1); + key.erase(0, key.find_first_not_of(" \t")); + key.erase(key.find_last_not_of(" \t") + 1); + value.erase(0, value.find_first_not_of(" \t")); + value.erase(value.find_last_not_of(" \t") + 1); + if (containsIgnoreCase(key, field)) { + return Maybe(value); + } + return Maybe(Error("no match")); +} + +inline std::pair +parseDmidecodeOutput(const std::string& dmidecodeOutput) { + string manufacturer; + string product; + istringstream stream(dmidecodeOutput); + string line; + while (getline(stream, line)) { + if (manufacturer.empty()) { + auto extractedManufacturer = extractValue(line, "Manufacturer"); + if (extractedManufacturer.ok() && !extractedManufacturer->empty()) { + manufacturer = *extractedManufacturer; + } + } + if (product.empty()) { + auto extractedProduct = extractValue(line, "Product Name"); + if (extractedProduct.ok() && !extractedProduct->empty()) { + product = *extractedProduct; + } + } + if (!manufacturer.empty() && !product.empty()) { + break; + } + } + + return make_pair(manufacturer, product); +} + +Maybe +getAiopCgnsHardwareType(const string &command_output) +{ + if (command_output == "" ) return string("NA"); + + auto pair = parseDmidecodeOutput(command_output); + + if (containsIgnoreCase(pair.first, "Amazon")) { + return string("AWS"); + } + if (containsIgnoreCase(pair.first, "Microsoft")) { + return string("Azure"); + } + if (containsIgnoreCase(pair.first, "Google")) { + return string("Google Cloud"); + } + if (containsIgnoreCase(pair.first, "Oracle")) { + return string("OCI"); + } + if (containsIgnoreCase(pair.first, "Alibaba")) { + return string("Alibaba"); + } + if (containsIgnoreCase(pair.second, "VMware")) { + return string("VMware"); + } + if (containsIgnoreCase(pair.first, "OpenStack")) { + return string("OpenStack"); + } + + // Check for KVM (manufacturer OR product) + if (containsIgnoreCase(pair.first, "QEMU") || containsIgnoreCase(pair.second, "KVM")) { + return string("KVM"); + } + + if (containsIgnoreCase(pair.first, "Xen")) { + return string("Xen"); + } + if (containsIgnoreCase(pair.first, "Nutanix")) { + return string("Nutanix"); + } + + return string("NA"); +} + +Maybe +getAiopsCgnsCloudVendor(const string &command_output) +{ + if (command_output == "" ) return string("NA"); + + string platform = "NA"; + istringstream stream(command_output); + string line; + while (getline(stream, line)) { + if (line.find("platform") != string::npos) { + size_t colonPos = line.find(' '); + if (colonPos != string::npos) { + platform = line.substr(colonPos + 1); + platform.erase(0, platform.find_first_not_of(" \t")); + platform.erase(platform.find_last_not_of(" \t") + 1); + break; + } + } + } + + return platform; +} + Maybe getQUID(const string &command_output) { @@ -158,11 +298,24 @@ getQUID(const string &command_output) return command_output; } +// Handler for a comma-separated list of QUIDs Maybe -getIsAiopsRunning(const string &command_output) +getQUIDList(const string &command_output) { - if (command_output == "" ) return string("false"); - + if (command_output.empty()) { + return string("false"); + } + + std::istringstream ss(command_output); + std::string quid; + + while (std::getline(ss, quid, ',')) { + const auto res = getQUID(quid); + if (!res.ok()) { + return res; // Return the error directly with context from getQUID + } + } + return command_output; } @@ -349,6 +502,15 @@ getGWIPAddress(const string &command_output) return getAttr(command_output, "IP Address was not found"); } +Maybe +getGWIPv6Address(const string &command_output) +{ + if (command_output.empty() || command_output == "null") { + return genError("IPv6 Address was not found"); + } + return string(command_output); +} + Maybe getGWVersion(const string &command_output) { @@ -366,7 +528,7 @@ checkIfSdwanRunning(const string &command_output) Maybe getClusterObjectIP(const string &command_output) { - return getAttr(command_output, "Cluster object IP was not found"); + return command_output; } Maybe diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index 1a541a7..40b1eba 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -46,27 +46,33 @@ SHELL_CMD_HANDLER("prerequisitesForHorizonTelemetry", "FS_PATH=; [ -f ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log ] " "&& head -1 ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log || echo ''", checkIsInstallHorizonTelemetrySucceeded) -SHELL_CMD_HANDLER( - "IS_AIOPS_RUNNING", - "FS_PATH=; " - "PID=$(ps auxf | grep -v grep | grep -E ${FS_PATH}.*cp-nano-horizon-telemetry | awk -F' ' '{printf $2}'); " - "[ -z \"${PID}\" ] && echo 'false' || echo 'true'", - getIsAiopsRunning) +SHELL_CMD_HANDLER("isCME", "[ -d /opt/CPcme ] && echo 'true' || echo 'false'", checkIsCME) #endif #if defined(gaia) SHELL_CMD_HANDLER("GLOBAL_QUID", "[ -d /opt/CPquid ] " "&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo ''", getQUID) SHELL_CMD_HANDLER("QUID", "FS_PATH=;" - "VS_ID=$(echo \"${FS_PATH}\" | grep -o -E \"vs[0-9]+\" | grep -o -E \"[0-9]+\");" - "[ -z \"${VS_ID}\" ] && " - "(python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo '');" - "[ -n \"${VS_ID}\" ] && " - "(sed \"s|###VS_ID###|${VS_ID}|g\" /opt/CPotelcol/quid_api/get_vs_quid.json" - " > /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID}); " - "[ -n \"${VS_ID}\" ] && [ -f /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} ] && " - "(python3 /opt/CPquid/Quid_Api.py -i " - "/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');", + "IS_MDS=$(cpprod_util CPPROD_IsConfigured PROVIDER-1 2>/dev/null | tr -d ' ');" + "if [ \"${IS_MDS}\" = \"1\" ]; then " + "DOMAIN_NAME=$(echo \"${FS_PATH}\" | grep -o -E \"domain-[^/]+\" | sed 's|domain-||');" + "[ -z \"${DOMAIN_NAME}\" ] && echo '' && exit 0;" + "sed \"s|###DOMAIN_NAME###|${DOMAIN_NAME}|g\" /opt/CPotelcol/quid_api/get_mds_quid.json" + " > /opt/CPotelcol/quid_api/get_mds_quid.json.${DOMAIN_NAME};" + "[ -f /opt/CPotelcol/quid_api/get_mds_quid.json.${DOMAIN_NAME} ] && " + "python3 /opt/CPquid/Quid_Api.py -i " + "/opt/CPotelcol/quid_api/get_mds_quid.json.${DOMAIN_NAME} 2>/dev/null | jq -r .message[0].MDS_QUID || echo '';" + "else " + "VS_ID=$(echo \"${FS_PATH}\" | grep -o -E \"vs[0-9]+\" | grep -o -E \"[0-9]+\");" + "[ -z \"${VS_ID}\" ] && " + "(python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo '');" + "[ -n \"${VS_ID}\" ] && " + "(sed \"s|###VS_ID###|${VS_ID}|g\" /opt/CPotelcol/quid_api/get_vs_quid.json" + " > /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID}); " + "[ -n \"${VS_ID}\" ] && [ -f /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} ] && " + "(python3 /opt/CPquid/Quid_Api.py -i " + "/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');" + "fi", getQUID) SHELL_CMD_HANDLER("SMO_QUID", "[ -d /opt/CPquid ] " "&& python3 /opt/CPquid/Quid_Api.py -i " @@ -76,9 +82,21 @@ SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] " "&& python3 /opt/CPquid/Quid_Api.py -i " "/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''", getQUID) +SHELL_CMD_HANDLER("MHO_QUID", + "[ -d /opt/CPquid ] && " + "python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_mho_quid.json 2>/dev/null | " + "jq -r '[.message[]? | select(.MHO_QUID != \"\") | .MHO_QUID] | join(\",\")' 2>/dev/null || " + "echo ''", + getQUIDList) SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "[ -d /opt/CPOtlpAgent/custom_scripts ] " "&& ENV_NO_FORMAT=1 /opt/CPOtlpAgent/custom_scripts/agent_role.sh", getOtlpAgentGaiaOsRole) +SHELL_CMD_HANDLER("AIOPS_CGNS_HW_TYPE", "" + "command -v dmidecode &>/dev/null && dmidecode -t 1 2>/dev/null", + getAiopCgnsHardwareType) +SHELL_CMD_HANDLER("AIOPS_CGNS_CLOUD_VENDOR", + "cat /etc/cloud-version 2>/dev/null", + getAiopsCgnsCloudVendor) SHELL_CMD_HANDLER("ETH_MGMT_IP", "FS_PATH=;" "VS_ID=$(echo \"${FS_PATH}\" | grep -o -E \"vs[0-9]+\" | grep -o -E \"[0-9]+\");" @@ -104,7 +122,12 @@ SHELL_CMD_HANDLER("MGMT_QUID", "echo ''", getQUID) SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "echo 'SMB'", getOtlpAgentGaiaOsRole) #endif #if defined(gaia) || defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1) -SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan) +SHELL_CMD_HANDLER( + "hasSDWan", + "[ $(cpprod_util CPPROD_IsMgmtMachine) -eq 1 ] && echo '0' ||" + "([ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0')", + checkHasSDWan +) SHELL_CMD_HANDLER( "canUpdateSDWanData", "jq -r .can_update_sdwan_data /tmp/cpsdwan_getdata_orch.json", @@ -131,10 +154,11 @@ SHELL_CMD_HANDLER( ) SHELL_CMD_HANDLER( "cpProductIntegrationMgmtParentObjectIP", - "obj=\"$(jq -r .cluster_name /tmp/cpsdwan_getdata_orch.json)\";" + "[ $(cpprod_util FwIsHighAvail) -eq 1 ] && " + "(obj=\"$(jq -r .cluster_name /tmp/cpsdwan_getdata_orch.json)\";" " awk -v obj=\"$obj\" '$1 == \":\" && $2 == \"(\" obj, $1 == \":ip_address\" { if ($1 == \":ip_address\")" " { gsub(/[()]/, \"\", $2); print $2; exit; } }'" - " $FWDIR/state/local/FW1/local.gateway_cluster", + " $FWDIR/state/local/FW1/local.gateway_cluster) || echo \"\"", getClusterObjectIP ) SHELL_CMD_HANDLER( @@ -146,7 +170,16 @@ SHELL_CMD_HANDLER("is_legacy_qos_blade_enabled", "cpprod_util CPPROD_GetValue FG1 ProdActive 1 | grep -q '^1$' " "&& (cpprod_util CPPROD_GetValue FG1 FgSDWAN 1 | grep -q '^1$' && echo false || echo true) || " "echo false", - checkQosLegacyBlade) + checkQosLegacyBlade +) +SHELL_CMD_HANDLER( + "IPv6 Address", + "( [ $(cpprod_util FwIsHighAvail) -eq 1 ] && [ $(cpprod_util FwIsVSX) -eq 1 ]" + "&& (jq -r .cluster_main_ipv6 /tmp/cpsdwan_getdata_orch.json) )" + "|| ( [ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" )" + "|| (jq -r .main_ipv6 /tmp/cpsdwan_getdata_orch.json)", + getGWIPv6Address +) #endif //gaia || smb #if defined(gaia) @@ -154,6 +187,10 @@ SHELL_CMD_HANDLER("hasSAMLSupportedBlade", "enabled_blades", checkSAMLSupportedB SHELL_CMD_HANDLER("hasIDABlade", "enabled_blades", checkIDABlade) SHELL_CMD_HANDLER("hasVPNBlade", "enabled_blades", checkVPNBlade) SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status nac", checkSAMLPortal) +SHELL_CMD_HANDLER("hasIdaPdpEnabled", + "cat $FWDIR/database/myself_objects.C | grep is_collecting_identities", + checkIdaPDP +) SHELL_CMD_HANDLER("hasInfinityIdentityEnabled", "cat $FWDIR/database/myself_objects.C | grep get_identities_from_infinity_identity", checkInfinityIdentityEnabled diff --git a/components/security_apps/orchestration/health_check/health_check.cc b/components/security_apps/orchestration/health_check/health_check.cc index a7e55fa..a822e8b 100755 --- a/components/security_apps/orchestration/health_check/health_check.cc +++ b/components/security_apps/orchestration/health_check/health_check.cc @@ -154,7 +154,8 @@ private: static const map> ip_port_defaults_map = { {"Azure", make_pair(getenv("DOCKER_RPM_ENABLED") ? "" : "168.63.129.16", 8117)}, {"Aws", make_pair("", 8117)}, - {"Local", make_pair("", 8117)} + {"Local", make_pair("", 8117)}, + {"VMware", make_pair("", 8117)} }; auto cloud_vendor_maybe = getSetting("reverseProxy", "cloudVendorName"); @@ -271,6 +272,12 @@ private: return HealthCheckStatus::UNHEALTHY; } + if (checkReadinessFilesExist()) { + dbgTrace(D_HEALTH_CHECK) + << "Readiness file exists, instance not ready for traffic, returning unhealthy status"; + return HealthCheckStatus::UNHEALTHY; + } + if (NGEN::Filesystem::exists(rpm_full_load_path)) { dbgTrace(D_HEALTH_CHECK) << "RPM is fully loaded"; return i_service_controller->getServicesPolicyStatus() @@ -289,6 +296,24 @@ private: return HealthCheckStatus::UNHEALTHY; } + bool + checkReadinessFilesExist() + { + string readiness_dir = readiness_file_path.substr(0, readiness_file_path.find_last_of('/')); + string readiness_filename = NGEN::Filesystem::getFileName(readiness_file_path); + + auto directory_files = NGEN::Filesystem::getDirectoryFiles(readiness_dir); + if (!directory_files.ok()) return false; + + for (const string& filename : directory_files.unpack()) { + if (NGEN::Strings::startsWith(filename, readiness_filename)) { + return true; + } + } + + return false; + } + bool nginxContainerIsRunning() { @@ -304,7 +329,19 @@ private: return false; } - return (*maybe_result).find(nginx_container_name) != string::npos; + bool container_running = (*maybe_result).find(nginx_container_name) != string::npos; + if (!container_running) { + dbgTrace(D_HEALTH_CHECK) << "Nginx container is not running"; + return false; + } + + if (checkReadinessFilesExist()) { + dbgTrace(D_HEALTH_CHECK) << "Readiness file exists on host machine, not ready for traffic"; + return false; + } + + dbgTrace(D_HEALTH_CHECK) << "Nginx container is running and no readiness files found - ready for traffic"; + return true; } void diff --git a/components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc b/components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc index bcc13e6..0fc6806 100755 --- a/components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc +++ b/components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc @@ -158,6 +158,12 @@ ManifestDiffCalculator::buildInstallationQueue( installation_queue.push_back(orchestration_it->second); } + auto shared_libs_it = new_packages.find("sharedLibs"); + if (shared_libs_it != new_packages.end()) { + installation_queue.push_back(shared_libs_it->second); + } + + auto wlp_standalone_it = new_packages.find("wlpStandalone"); if (wlp_standalone_it != new_packages.end()){ installation_queue.push_back(wlp_standalone_it->second); diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 08bed76..6349126 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -1535,6 +1535,11 @@ private: if (i_details_resolver->compareCheckpointVersion(8200, greater_equal())) { agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionGER82", "true"); } + if (i_details_resolver->compareCheckpointVersion(8200, equal_to())) { + agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionR82", "true"); + } else { + agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionR82", "false"); + } #endif // gaia || smb if (agent_data_report == curr_agent_data_report) { @@ -2278,4 +2283,4 @@ OrchestrationComp::preload() registerExpectedSetting("successUpgradeInterval"); registerExpectedConfigFile("orchestration", Config::ConfigFileType::Policy); registerExpectedConfigFile("registration-data", Config::ConfigFileType::Policy); -} \ No newline at end of file +} diff --git a/components/security_apps/prometheus/prometheus_comp.cc b/components/security_apps/prometheus/prometheus_comp.cc index 5b06f2e..1f67d68 100755 --- a/components/security_apps/prometheus/prometheus_comp.cc +++ b/components/security_apps/prometheus/prometheus_comp.cc @@ -40,7 +40,13 @@ struct ServiceData class PrometheusMetricData { public: - PrometheusMetricData(const string &n, const string &t, const string &d) : name(n), type(t), description(d) {} + PrometheusMetricData(const string &n, const string &u, const string &t, const string &d) + : + name(n), + unique_name(u), + type(t), + description(d) + {} void addElement(const string &labels, const string &value) @@ -55,7 +61,9 @@ public: string representative_name = ""; if (!name.empty()) { - auto metric_name = convertMetricName(name); + string metric_name; + if (!unique_name.empty()) metric_name = convertMetricName(unique_name); + if (metric_name.empty()) metric_name = convertMetricName(name); !metric_name.empty() ? representative_name = metric_name : representative_name = name; } @@ -73,6 +81,7 @@ public: private: string name; + string unique_name; string type; string description; map metric_labels_to_values; @@ -98,6 +107,7 @@ public: for(auto &metric : metrics) { auto &metric_object = getDataObject( metric.name, + metric.unique_name, metric.type, metric.description ); @@ -107,11 +117,14 @@ public: private: PrometheusMetricData & - getDataObject(const string &name, const string &type, const string &description) + getDataObject(const string &name, const string &unique_name, const string &type, const string &description) { - auto elem = prometheus_metrics.find(name); + auto elem = prometheus_metrics.find(unique_name); if (elem == prometheus_metrics.end()) { - elem = prometheus_metrics.emplace(name, PrometheusMetricData(name, type, description)).first; + elem = prometheus_metrics.emplace( + unique_name, + PrometheusMetricData(name, unique_name, type, description) + ).first; } return elem->second; diff --git a/components/security_apps/prometheus/prometheus_metric_names.h b/components/security_apps/prometheus/prometheus_metric_names.h index 1a5fdf0..7443e22 100755 --- a/components/security_apps/prometheus/prometheus_metric_names.h +++ b/components/security_apps/prometheus/prometheus_metric_names.h @@ -71,37 +71,37 @@ convertMetricName(const std::string &original_metric_name) {"responseBodySizeUponTimeoutMaxSample", "response_body_size_max"}, {"responseBodySizeUponTimeoutMinSample", "response_body_size_min"}, // WaapTelemetrics - {"reservedNgenA", "total_requests_counter"}, - {"reservedNgenB", "unique_sources_counter"}, - {"reservedNgenC", "requests_blocked_by_force_and_exception_counter"}, - {"reservedNgenD", "requests_blocked_by_waf_counter"}, - {"reservedNgenE", "requests_blocked_by_open_api_counter"}, - {"reservedNgenF", "requests_blocked_by_bot_protection_counter"}, - {"reservedNgenG", "requests_threat_level_info_and_no_threat_counter"}, - {"reservedNgenH", "requests_threat_level_low_counter"}, - {"reservedNgenI", "requests_threat_level_medium_counter"}, - {"reservedNgenJ", "requests_threat_level_high_counter"}, + {"reservedNgenA_WAAP telemetry", "total_requests_counter"}, + {"reservedNgenB_WAAP telemetry", "unique_sources_counter"}, + {"reservedNgenC_WAAP telemetry", "requests_blocked_by_force_and_exception_counter"}, + {"reservedNgenD_WAAP telemetry", "requests_blocked_by_waf_counter"}, + {"reservedNgenE_WAAP telemetry", "requests_blocked_by_open_api_counter"}, + {"reservedNgenF_WAAP telemetry", "requests_blocked_by_bot_protection_counter"}, + {"reservedNgenG_WAAP telemetry", "requests_threat_level_info_and_no_threat_counter"}, + {"reservedNgenH_WAAP telemetry", "requests_threat_level_low_counter"}, + {"reservedNgenI_WAAP telemetry", "requests_threat_level_medium_counter"}, + {"reservedNgenJ_WAAP telemetry", "requests_threat_level_high_counter"}, // WaapTrafficTelemetrics - {"reservedNgenA", "post_requests_counter"}, - {"reservedNgenB", "get_requests_counter"}, - {"reservedNgenC", "put_requests_counter"}, - {"reservedNgenD", "patch_requests_counter"}, - {"reservedNgenE", "delete_requests_counter"}, - {"reservedNgenF", "other_requests_counter"}, - {"reservedNgenG", "2xx_status_code_responses_counter"}, - {"reservedNgenH", "4xx_status_code_responses_counter"}, - {"reservedNgenI", "5xx_status_code_responses_counter"}, - {"reservedNgenJ", "requests_time_latency_average"}, + {"reservedNgenA_WAAP traffic telemetry", "post_requests_counter"}, + {"reservedNgenB_WAAP traffic telemetry", "get_requests_counter"}, + {"reservedNgenC_WAAP traffic telemetry", "put_requests_counter"}, + {"reservedNgenD_WAAP traffic telemetry", "patch_requests_counter"}, + {"reservedNgenE_WAAP traffic telemetry", "delete_requests_counter"}, + {"reservedNgenF_WAAP traffic telemetry", "other_requests_counter"}, + {"reservedNgenG_WAAP traffic telemetry", "status_code_2xx_responses_counter"}, + {"reservedNgenH_WAAP traffic telemetry", "status_code_4xx_responses_counter"}, + {"reservedNgenI_WAAP traffic telemetry", "status_code_5xx_responses_counter"}, + {"reservedNgenJ_WAAP traffic telemetry", "requests_time_latency_average"}, // WaapAttackTypesMetrics - {"reservedNgenA", "sql_injection_attacks_type_counter"}, - {"reservedNgenB", "vulnerability_scanning_attacks_type_counter"}, - {"reservedNgenC", "path_traversal_attacks_type_counter"}, - {"reservedNgenD", "ldap_injection_attacks_type_counter"}, - {"reservedNgenE", "evasion_techniques_attacks_type_counter"}, - {"reservedNgenF", "remote_code_execution_attacks_type_counter"}, - {"reservedNgenG", "xml_extern_entity_attacks_type_counter"}, - {"reservedNgenH", "cross_site_scripting_attacks_type_counter"}, - {"reservedNgenI", "general_attacks_type_counter"}, + {"reservedNgenA_WAAP attack type telemetry", "sql_injection_attacks_type_counter"}, + {"reservedNgenB_WAAP attack type telemetry", "vulnerability_scanning_attacks_type_counter"}, + {"reservedNgenC_WAAP attack type telemetry", "path_traversal_attacks_type_counter"}, + {"reservedNgenD_WAAP attack type telemetry", "ldap_injection_attacks_type_counter"}, + {"reservedNgenE_WAAP attack type telemetry", "evasion_techniques_attacks_type_counter"}, + {"reservedNgenF_WAAP attack type telemetry", "remote_code_execution_attacks_type_counter"}, + {"reservedNgenG_WAAP attack type telemetry", "xml_extern_entity_attacks_type_counter"}, + {"reservedNgenH_WAAP attack type telemetry", "cross_site_scripting_attacks_type_counter"}, + {"reservedNgenI_WAAP attack type telemetry", "general_attacks_type_counter"}, // AssetsMetric {"numberOfProtectedApiAssetsSample", "api_assets_counter"}, {"numberOfProtectedWebAppAssetsSample", "web_api_assets_counter"}, diff --git a/components/security_apps/prometheus/prometheus_ut/prometheus_ut.cc b/components/security_apps/prometheus/prometheus_ut/prometheus_ut.cc index f021450..49329d0 100755 --- a/components/security_apps/prometheus/prometheus_ut/prometheus_ut.cc +++ b/components/security_apps/prometheus/prometheus_ut/prometheus_ut.cc @@ -1,79 +1,133 @@ -#include "prometheus_comp.h" - -#include -#include -#include - -#include "cmock.h" -#include "cptest.h" -#include "maybe_res.h" -#include "debug.h" -#include "config.h" -#include "environment.h" -#include "config_component.h" -#include "agent_details.h" -#include "time_proxy.h" -#include "mock/mock_mainloop.h" -#include "mock/mock_rest_api.h" -#include "mock/mock_messaging.h" - -using namespace std; -using namespace testing; - -USE_DEBUG_FLAG(D_PROMETHEUS); - -class PrometheusCompTest : public Test -{ -public: - PrometheusCompTest() - { - EXPECT_CALL(mock_rest, mockRestCall(_, "declare-boolean-variable", _)).WillOnce(Return(false)); - env.preload(); - config.preload(); - env.init(); - - EXPECT_CALL( - mock_rest, - addGetCall("metrics", _) - ).WillOnce(DoAll(SaveArg<1>(&get_metrics_func), Return(true))); - - prometheus_comp.init(); - } - - ::Environment env; - ConfigComponent config; - PrometheusComp prometheus_comp; - StrictMock mock_rest; - StrictMock mock_ml; - NiceMock mock_messaging; - unique_ptr agent_uninstall; - function get_metrics_func; - CPTestTempfile status_file; - string registered_services_file_path; - -}; - -TEST_F(PrometheusCompTest, checkAddingMetric) -{ - registered_services_file_path = cptestFnameInSrcDir(string("registered_services.json")); - setConfiguration(registered_services_file_path, "orchestration", "Orchestration registered services"); - string metric_body = "{\n" - " \"metrics\": [\n" - " {\n" - " \"metric_name\": \"watchdogProcessStartupEventsSum\",\n" - " \"metric_type\": \"counter\",\n" - " \"metric_description\": \"\",\n" - " \"labels\": \"{method=\\\"post\\\",code=\\\"200\\\"}\",\n" - " \"value\": \"1534\"\n" - " }\n" - " ]\n" - "}"; - - string message_body; - EXPECT_CALL(mock_messaging, sendSyncMessage(_, "/service-metrics", _, _, _)) - .Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, metric_body))); - - string metric_str = "# TYPE nano_service_restarts_counter counter\n" - "nano_service_restarts_counter{method=\"post\",code=\"200\"} 1534\n\n"; - EXPECT_EQ(metric_str, get_metrics_func()); -} +#include "prometheus_comp.h" + +#include +#include +#include + +#include "cmock.h" +#include "cptest.h" +#include "maybe_res.h" +#include "debug.h" +#include "config.h" +#include "environment.h" +#include "config_component.h" +#include "agent_details.h" +#include "time_proxy.h" +#include "mock/mock_mainloop.h" +#include "mock/mock_rest_api.h" +#include "mock/mock_messaging.h" + +using namespace std; +using namespace testing; + +USE_DEBUG_FLAG(D_PROMETHEUS); + +class PrometheusCompTest : public Test +{ +public: + PrometheusCompTest() + { + EXPECT_CALL(mock_rest, mockRestCall(_, "declare-boolean-variable", _)).WillOnce(Return(false)); + env.preload(); + config.preload(); + env.init(); + + EXPECT_CALL( + mock_rest, + addGetCall("metrics", _) + ).WillOnce(DoAll(SaveArg<1>(&get_metrics_func), Return(true))); + + prometheus_comp.init(); + } + + ::Environment env; + ConfigComponent config; + PrometheusComp prometheus_comp; + StrictMock mock_rest; + StrictMock mock_ml; + NiceMock mock_messaging; + unique_ptr agent_uninstall; + function get_metrics_func; + CPTestTempfile status_file; + string registered_services_file_path; + +}; + +TEST_F(PrometheusCompTest, checkAddingMetricWithEmptyUniqueName) +{ + registered_services_file_path = cptestFnameInSrcDir(string("registered_services.json")); + setConfiguration(registered_services_file_path, "orchestration", "Orchestration registered services"); + string metric_body = "{\n" + " \"metrics\": [\n" + " {\n" + " \"metric_name\": \"watchdogProcessStartupEventsSum\",\n" + " \"unique_name\": \"\",\n" + " \"metric_type\": \"counter\",\n" + " \"metric_description\": \"\",\n" + " \"labels\": \"{method=\\\"post\\\",code=\\\"200\\\"}\",\n" + " \"value\": \"1534\"\n" + " }\n" + " ]\n" + "}"; + + string message_body; + EXPECT_CALL(mock_messaging, sendSyncMessage(_, "/service-metrics", _, _, _)) + .Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, metric_body))); + + string metric_str = "# TYPE nano_service_restarts_counter counter\n" + "nano_service_restarts_counter{method=\"post\",code=\"200\"} 1534\n\n"; + EXPECT_EQ(metric_str, get_metrics_func()); +} + +TEST_F(PrometheusCompTest, checkAddingMetricWithoutUniqueName) +{ + registered_services_file_path = cptestFnameInSrcDir(string("registered_services.json")); + setConfiguration(registered_services_file_path, "orchestration", "Orchestration registered services"); + string metric_body = "{\n" + " \"metrics\": [\n" + " {\n" + " \"metric_name\": \"watchdogProcessStartupEventsSum\",\n" + " \"unique_name\": \"watchdogProcessStartupEventsSum_Bla bla\",\n" + " \"metric_type\": \"counter\",\n" + " \"metric_description\": \"\",\n" + " \"labels\": \"{method=\\\"post\\\",code=\\\"200\\\"}\",\n" + " \"value\": \"1534\"\n" + " }\n" + " ]\n" + "}"; + + string message_body; + EXPECT_CALL(mock_messaging, sendSyncMessage(_, "/service-metrics", _, _, _)) + .Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, metric_body))); + + string metric_str = "# TYPE nano_service_restarts_counter counter\n" + "nano_service_restarts_counter{method=\"post\",code=\"200\"} 1534\n\n"; + EXPECT_EQ(metric_str, get_metrics_func()); +} + +TEST_F(PrometheusCompTest, checkAddingMetricWithUniqueName) +{ + registered_services_file_path = cptestFnameInSrcDir(string("registered_services.json")); + setConfiguration(registered_services_file_path, "orchestration", "Orchestration registered services"); + string metric_body = "{\n" + " \"metrics\": [\n" + " {\n" + " \"metric_name\": \"reservedNgenA\",\n" + " \"unique_name\": \"reservedNgenA_WAAP telemetry\",\n" + " \"metric_type\": \"counter\",\n" + " \"metric_description\": \"\",\n" + " \"labels\": \"{method=\\\"post\\\",code=\\\"200\\\"}\",\n" + " \"value\": \"1534\"\n" + " }\n" + " ]\n" + "}"; + + string message_body; + EXPECT_CALL(mock_messaging, sendSyncMessage(_, "/service-metrics", _, _, _)) + .Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, metric_body))); + + string metric_str = "# TYPE total_requests_counter counter\n" + "total_requests_counter{method=\"post\",code=\"200\"} 1534\n\n"; + EXPECT_EQ(metric_str, get_metrics_func()); +} + diff --git a/components/security_apps/rate_limit/rate_limit.cc b/components/security_apps/rate_limit/rate_limit.cc index dc410cb..a12f62f 100755 --- a/components/security_apps/rate_limit/rate_limit.cc +++ b/components/security_apps/rate_limit/rate_limit.cc @@ -9,7 +9,7 @@ #include "i_mainloop.h" #include "i_time_get.h" #include "rate_limit_config.h" -#include "nginx_attachment_common.h" +#include "nano_attachment_common.h" #include "http_inspection_events.h" #include "Waf2Util.h" #include "generic_rulebase/evaluators/asset_eval.h" @@ -329,7 +329,7 @@ public: auto asset_location = Singleton::Consume::by()->lookupLocation(maybe_source_ip.unpack()); if (!asset_location.ok()) { - dbgWarning(D_RATE_LIMIT) + dbgDebug(D_RATE_LIMIT) << "Rate limit lookup location failed for source: " << source_ip << ", Error: " @@ -400,7 +400,19 @@ public: if (calcRuleAction(rule) == RateLimitAction::PREVENT) { dbgTrace(D_RATE_LIMIT) << "Received DROP verdict, this request will be blocked by rate limit"; - return DROP; + + EventVerdict verdict = DROP; + ScopedContext rate_limit_ctx; + rate_limit_ctx.registerValue(AssetMatcher::ctx_key, asset_id); + auto maybe_rate_limit_config = getConfiguration("rulebase", "rateLimit"); + if (maybe_rate_limit_config.ok()) { + const string &web_user_response_id = maybe_rate_limit_config.unpack().getWebUserResponse(); + if (!web_user_response_id.empty()) { + verdict.setWebUserResponseByPractice(web_user_response_id); + dbgTrace(D_RATE_LIMIT) << "Set web user response: " << web_user_response_id; + } + } + return verdict; } dbgTrace(D_RATE_LIMIT) << "Received DROP in detect mode, will not block."; @@ -490,7 +502,7 @@ public: return; } - auto maybe_rule_by_ctx = getConfiguration("rulebase", "rulesConfig"); + auto maybe_rule_by_ctx = getConfigurationWithCache("rulebase", "rulesConfig"); if (!maybe_rule_by_ctx.ok()) { dbgWarning(D_RATE_LIMIT) << "rule was not found by the given context. Reason: " @@ -732,9 +744,9 @@ public: I_EnvDetails* i_env_details = nullptr; private: - static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; - static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; - static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + static constexpr auto DROP = ServiceVerdict::TRAFFIC_VERDICT_DROP; + static constexpr auto ACCEPT = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; + static constexpr auto INSPECT = ServiceVerdict::TRAFFIC_VERDICT_INSPECT; RateLimitAction practice_action; string rate_limit_lua_script_hash; @@ -761,8 +773,8 @@ RateLimit::~RateLimit() = default; void RateLimit::preload() { - registerExpectedConfiguration("WAAP", "WebApplicationSecurity"); - registerExpectedConfiguration("WAAP", "WebAPISecurity"); + registerExpectedConfigurationWithCache("assetId", "WAAP", "WebApplicationSecurity"); + registerExpectedConfigurationWithCache("assetId", "WAAP", "WebAPISecurity"); registerExpectedConfigFile("waap", Config::ConfigFileType::Policy); registerExpectedConfiguration("rulebase", "rateLimit"); registerExpectedConfigFile("accessControlV2", Config::ConfigFileType::Policy); diff --git a/components/security_apps/rate_limit/rate_limit_config.cc b/components/security_apps/rate_limit/rate_limit_config.cc index b5f5dff..7dd5602 100755 --- a/components/security_apps/rate_limit/rate_limit_config.cc +++ b/components/security_apps/rate_limit/rate_limit_config.cc @@ -120,6 +120,17 @@ RateLimitConfig::load(cereal::JSONInputArchive &ar) dbgWarning(D_RATE_LIMIT) << "Failed to load single Rate Limit JSON config. Error: " << e.what(); ar.setNextName(nullptr); } + + try { + string rate_limit_response; + ar(cereal::make_nvp("webUserResponseId", rate_limit_response)); + setWebUserResponse(rate_limit_response); + } catch(const std::exception& e) { + dbgDebug(D_RATE_LIMIT) + << "No webUserResponseId defined for Rate Limit config. Using default. Error: " + << e.what(); + ar.setNextName(nullptr); + } } RateLimitRule diff --git a/components/security_apps/waap/include/buffered_compressed_stream.h b/components/security_apps/waap/include/buffered_compressed_stream.h new file mode 100644 index 0000000..8cf1b2a --- /dev/null +++ b/components/security_apps/waap/include/buffered_compressed_stream.h @@ -0,0 +1,127 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include "compression_utils.h" +#include "debug.h" + +USE_DEBUG_FLAG(D_WAAP_SERIALIZE); + +// Forward declarations +class WaapComponent; + +void yieldIfPossible(const std::string &func, int line); + +#define YIELD_IF_POSSIBLE() yieldIfPossible(__FUNCTION__, __LINE__) + +// +// Buffered output stream that compresses and encrypts data when flushing +// +// Usage example: +// std::stringstream ss; +// BufferedCompressedOutputStream compressed_stream(ss); +// compressed_stream << "Hello, World!"; +// compressed_stream.flush(); // Data is compressed, encrypted, and written to ss +class BufferedCompressedOutputStream : public std::ostream +{ +public: + explicit BufferedCompressedOutputStream(std::ostream &underlying_stream); + ~BufferedCompressedOutputStream(); + + // Manual flush to compress, encrypt and write data + void flush(); + void close(); + +private: + class CompressedBuffer : public std::streambuf, Singleton::Consume + { + public: + explicit CompressedBuffer(std::ostream &underlying_stream); + ~CompressedBuffer(); + + // Public method to flush the buffer + void flushAndClose(); + void flushBuffer(); + + protected: + virtual int overflow(int c) override; + virtual std::streamsize xsputn(const char* s, std::streamsize n) override; + virtual int sync() override; + + private: + // Compress and encrypt buffer; is_last indicates final chunk + bool compressAndEncryptBuffer(bool is_last); + std::ostream &m_underlying_stream; + std::vector m_buffer; + static const size_t BUFFER_SIZE = 16 * 1024; // 16KiB + CompressionStream* m_compression_stream; + bool m_closed; + }; + + std::unique_ptr m_buffer; +}; + + +// Buffered input stream that decrypts and decompresses data when reading +// +// Usage example: +// std::stringstream ss("encrypted compressed data"); +// BufferedCompressedInputStream decompressed_stream(ss); +// std::string line; +// std::getline(decompressed_stream, line); // Data is decrypted and decompressed + +class BufferedCompressedInputStream : public std::istream +{ +public: + explicit BufferedCompressedInputStream(std::istream &underlying_stream); + ~BufferedCompressedInputStream(); + +private: + class DecompressedBuffer : public std::streambuf + { + public: + explicit DecompressedBuffer(std::istream &underlying_stream); + ~DecompressedBuffer(); + + protected: + virtual int underflow() override; + virtual std::streamsize xsgetn(char* s, std::streamsize n) override; + + private: + bool fillBuffer(); + bool processNextChunk(); + bool decryptChunk(const std::vector &encrypted_chunk, std::vector &decrypted_chunk); + bool decompressChunk(const std::vector &compressed_chunk, std::vector &decompressed_chunk); + + std::istream &m_underlying_stream; + std::vector m_buffer; // Output buffer for decompressed data + std::vector m_encrypted_buffer; // Buffer for encrypted data from stream + std::vector m_compressed_buffer; // Buffer for decrypted but still compressed data + std::vector m_decompressed_buffer; // Buffer for decompressed data chunks + size_t m_decompressed_pos; // Current position in decompressed buffer + + static const size_t OUTPUT_BUFFER_SIZE = 64 * 1024; // 64KiB output buffer + static const size_t CHUNK_SIZE = 16 * 1024; // 16KiB chunks for processing + + CompressionStream* m_compression_stream; + bool m_eof_reached; + bool m_stream_finished; // Whether we've finished processing the entire stream + }; + + std::unique_ptr m_buffer; +}; diff --git a/components/security_apps/waap/include/i_serialize.h b/components/security_apps/waap/include/i_serialize.h index 9d19353..908ea79 100755 --- a/components/security_apps/waap/include/i_serialize.h +++ b/components/security_apps/waap/include/i_serialize.h @@ -12,6 +12,7 @@ // limitations under the License. #pragma once + #include #include #include "i_time_get.h" @@ -19,17 +20,23 @@ #include "i_messaging.h" #include "i_mainloop.h" #include "i_agent_details.h" +#include "compression_utils.h" +#include +#include static const uint max_send_obj_retries = 3; static const std::chrono::microseconds wait_next_attempt(5000000); USE_DEBUG_FLAG(D_WAAP_SERIALIZE); +// Forward declarations +class WaapComponent; + class RestGetFile : public ClientRest { public: // decrypts and load json - bool loadJson(const std::string& json); + bool loadJson(const std::string &json); // gen json and encrypt Maybe genJson() const; }; @@ -47,10 +54,10 @@ public: // parses xml instead of json // extracts a file list in - bool loadJson(const std::string& xml); + bool loadJson(const std::string &xml); - const std::vector& getFilesMetadataList() const; - const std::vector& getFilesList() const; + const std::vector &getFilesMetadataList() const; + const std::vector &getFilesList() const; private: RestParam> files; @@ -59,18 +66,19 @@ private: class I_Serializable { public: - virtual void serialize(std::ostream& stream) = 0; - virtual void deserialize(std::istream& stream) = 0; + virtual void serialize(std::ostream &stream) = 0; + virtual void deserialize(std::istream &stream) = 0; }; class I_RemoteSyncSerialize { public: virtual bool postData() = 0; - virtual void pullData(const std::vector& files) = 0; + virtual void pullData(const std::vector &files) = 0; virtual void processData() = 0; virtual void postProcessedData() = 0; - virtual void pullProcessedData(const std::vector& files) = 0; - virtual void updateState(const std::vector& files) = 0; + virtual void pullProcessedData(const std::vector &files) = 0; + virtual void updateState(const std::vector &files) = 0; + virtual Maybe getRemoteStateFilePath() const { return genError("No remote state file path defined"); } }; class I_Backup { @@ -86,7 +94,7 @@ class SerializeToFileBase : public I_Serializable { public: - SerializeToFileBase(std::string filePath); + SerializeToFileBase(const std::string &filePath); virtual ~SerializeToFileBase(); virtual void saveData(); @@ -96,13 +104,13 @@ protected: // saved file name for testing std::string m_filePath; private: - void loadFromFile(std::string filePath); + void loadFromFile(const std::string &filePath); // Updated to match implementation }; class SerializeToFilePeriodically : public SerializeToFileBase { public: - SerializeToFilePeriodically(std::chrono::seconds pollingIntervals, std::string filePath); + SerializeToFilePeriodically(std::chrono::seconds pollingIntervals, const std::string &filePath); virtual ~SerializeToFilePeriodically(); void setInterval(std::chrono::seconds newInterval); @@ -122,16 +130,16 @@ class SerializeToLocalAndRemoteSyncBase : public I_RemoteSyncSerialize, public S public: SerializeToLocalAndRemoteSyncBase(std::chrono::minutes interval, std::chrono::seconds waitForSync, - const std::string& filePath, - const std::string& remotePath, - const std::string& assetId, - const std::string& owner); + const std::string &filePath, + const std::string &remotePath, + const std::string &assetId, + const std::string &owner); virtual ~SerializeToLocalAndRemoteSyncBase(); virtual void restore(); virtual void syncWorker(); - + bool shouldNotSync() const; void setInterval(std::chrono::seconds newInterval); std::chrono::seconds getIntervalDuration() const; void setRemoteSyncEnabled(bool enabled); @@ -143,7 +151,7 @@ protected: std::string getUri(); size_t getIntervalsCount(); void incrementIntervalsCount(); - bool isBase(); + bool isBase() const; template bool sendObject(T &obj, HTTPMethod method, std::string uri) @@ -242,7 +250,7 @@ protected: } template - bool sendNoReplyObjectWithRetry(T &obj, HTTPMethod method, std::string uri) + bool sendNoReplyObjectWithRetry(T &obj, HTTPMethod method, const std::string &uri) { I_MainLoop *mainloop= Singleton::Consume::by(); for (uint i = 0; i < max_send_obj_retries; i++) @@ -270,10 +278,16 @@ protected: private: bool localSyncAndProcess(); void updateStateFromRemoteService(); + Maybe getStateTimestampByListing(); + bool checkAndUpdateStateTimestamp(const std::string& currentStateTimestamp); RemoteFilesList getProcessedFilesList(); RemoteFilesList getRemoteProcessedFilesList(); std::string getLearningHost(); std::string getSharedStorageHost(); + std::string getStateTimestampPath(); + Maybe getStateTimestamp(); + Maybe updateStateFromRemoteFile(); + bool shouldSendSyncNotification() const; I_MainLoop* m_pMainLoop; std::chrono::microseconds m_waitForSync; @@ -287,3 +301,4 @@ private: Maybe m_shared_storage_host; Maybe m_learning_host; }; + diff --git a/components/security_apps/waap/include/i_transaction.h b/components/security_apps/waap/include/i_transaction.h index b1bb897..e674504 100755 --- a/components/security_apps/waap/include/i_transaction.h +++ b/components/security_apps/waap/include/i_transaction.h @@ -89,7 +89,7 @@ public: virtual double getOtherModelScore() const = 0; virtual const std::vector getScoreArray() const = 0; virtual Waap::CSRF::State& getCsrfState() = 0; - virtual ngx_http_cp_verdict_e getUserLimitVerdict() = 0; + virtual ServiceVerdict getUserLimitVerdict() = 0; virtual const std::string getUserLimitVerdictStr() const = 0; virtual const std::string getViolatedUserLimitTypeStr() const = 0; virtual void checkShouldInject() = 0; diff --git a/components/security_apps/waap/include/i_waapConfig.h b/components/security_apps/waap/include/i_waapConfig.h index 5acbe9e..97dc465 100755 --- a/components/security_apps/waap/include/i_waapConfig.h +++ b/components/security_apps/waap/include/i_waapConfig.h @@ -40,6 +40,13 @@ enum class AttackMitigationMode PREVENT, UNKNOWN }; + +enum class WaapConfigType { + Unknown = 0, + Application, + API +}; + class IWaapConfig { public: virtual const std::string& get_AssetId() const = 0; @@ -65,6 +72,7 @@ public: virtual const std::shared_ptr& get_ErrorLimitingPolicy() const = 0; virtual const std::shared_ptr& get_SecurityHeadersPolicy() const = 0; virtual const std::shared_ptr& get_UserLimitsPolicy() const = 0; + virtual WaapConfigType getType() const = 0; virtual void printMe(std::ostream& os) const = 0; }; diff --git a/components/security_apps/waap/reputation/reputation_features_agg.cc b/components/security_apps/waap/reputation/reputation_features_agg.cc index 7d44cca..722b0c8 100755 --- a/components/security_apps/waap/reputation/reputation_features_agg.cc +++ b/components/security_apps/waap/reputation/reputation_features_agg.cc @@ -30,7 +30,7 @@ template class DefaultListener : public Listener { public: - DefaultListener(EventVerdict defaultVerdict = EventVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT)) + DefaultListener(EventVerdict defaultVerdict = EventVerdict(ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT)) : m_default_verdict(defaultVerdict) {} @@ -58,7 +58,7 @@ class ReputationFeaturesAgg::Impl public: Impl() : - DefaultListener(EventVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT)), + DefaultListener(EventVerdict(ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT)), m_agg_entries() { } diff --git a/components/security_apps/waap/waap_clib/AssertionRegexes.h b/components/security_apps/waap/waap_clib/AssertionRegexes.h new file mode 100644 index 0000000..000aa45 --- /dev/null +++ b/components/security_apps/waap/waap_clib/AssertionRegexes.h @@ -0,0 +1,35 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __ASSERTION_REGEXES_H__ +#define __ASSERTION_REGEXES_H__ + +#include + +namespace Waap { +namespace AssertionRegexes { + +// Static const boost regexes used in processAssertions() function +// These regexes detect various assertion patterns in regex strings +// The patterns are in a separate file to avoid this codestyle checker issue: +// "error T009: comma should be followed by whitespace" +static const boost::regex reStartNonWordBehind(R"(\(\?:=\]\|\$\))"); // (?=[^\w?<>:=]|$) +static const boost::regex rePathTraversalStart(R"(\(\? #include #include // For DIR, opendir, readdir, closedir @@ -32,13 +33,15 @@ USE_DEBUG_FLAG(D_WAAP); #define BUSY_WAIT_TIME chrono::microseconds(100000) // 0.1 seconds #define WAIT_LIMIT 10 #define BENIGN_PARAM_FACTOR 2 +#define MAX_TRACKING_KEYS 1000 // Maximum number of keys to track double logn(double x, double n) { return log(x) / log(n); } -ConfidenceCalculator::ConfidenceCalculator(size_t minSources, +ConfidenceCalculator::ConfidenceCalculator( + size_t minSources, size_t minIntervals, chrono::minutes intervalDuration, double ratioThreshold, @@ -65,12 +68,17 @@ ConfidenceCalculator::ConfidenceCalculator(size_t minSources, m_ignoreSources(ignoreSrc), m_tuning(tuning), m_estimated_memory_usage(0), + m_post_index(0), m_mainLoop(Singleton::Consume::by()), m_routineId(0), - m_filesToRemove() + m_filesToRemove(), + m_indicator_tracking_keys(), + m_tracking_keys_received(false) { restore(); + extractLowConfidenceKeys(m_confidence_level); + // Start asynchronous deletion of existing carry-on data files garbageCollector(); } @@ -103,10 +111,11 @@ void ConfidenceCalculator::hardReset() m_estimated_memory_usage = 0; m_confidence_level.clear(); m_confident_sets.clear(); + m_indicator_tracking_keys.clear(); + m_tracking_keys_received = false; remove(m_filePath.c_str()); } - void ConfidenceCalculator::reset() { if (m_time_window_logger) { @@ -120,7 +129,7 @@ void ConfidenceCalculator::reset() } } -bool ConfidenceCalculator::reset(ConfidenceCalculatorParams& params) +bool ConfidenceCalculator::reset(ConfidenceCalculatorParams ¶ms) { if (params == m_params) { @@ -138,15 +147,16 @@ bool ConfidenceCalculator::reset(ConfidenceCalculatorParams& params) class WindowLogPost : public RestGetFile { public: - WindowLogPost(ConfidenceCalculator::KeyValSourcesLogger& _window_logger) - : window_logger(_window_logger) + WindowLogPost(std::shared_ptr _window_logger_ptr) { + // Initialize the RestParam with a reference to the shared data + // do not copy the shared pointer, but move it to avoid copying - do not use the ptr after it + window_logger = std::move(*_window_logger_ptr); } ~WindowLogPost() { - window_logger.get().clear(); - window_logger.get().rehash(0); + // Container will be automatically cleaned up when RestParam goes out of scope } private: @@ -169,73 +179,9 @@ private: S2C_PARAM(ConfidenceCalculator::KeyValSourcesLogger, window_logger) }; - -// Function to handle compression -void compressDataWrapper(const string& uncompressed_data, size_t chunk_size, vector& compressed_data) { - auto compression_stream = initCompressionStream(); - size_t offset = 0; - - while (offset < uncompressed_data.size()) { - size_t current_chunk_size = std::min(chunk_size, uncompressed_data.size() - offset); - bool is_last = (offset + current_chunk_size >= uncompressed_data.size()); - CompressionResult chunk_res = compressData( - compression_stream, - CompressionType::GZIP, - static_cast(current_chunk_size), - reinterpret_cast(uncompressed_data.c_str() + offset), - is_last ? 1 : 0 - ); - - if (!chunk_res.ok) { - finiCompressionStream(compression_stream); - throw runtime_error("Compression failed"); - } - - if (chunk_res.output && chunk_res.num_output_bytes > 0) { - compressed_data.insert( - compressed_data.end(), - chunk_res.output, - chunk_res.output + chunk_res.num_output_bytes - ); - free(chunk_res.output); - } - - offset += current_chunk_size; - } - - finiCompressionStream(compression_stream); -} - - -Maybe ConfidenceCalculator::writeToFile(const string& path, const vector& data) -{ - ofstream file(path, ios::binary); - if (!file.is_open()) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open file: " << path - << ", errno: " << errno << ", strerror: " << strerror(errno); - return genError("Failed to open file"); - } - - // Write compressed data to file in chunks to avoid large memory usage - const uint CHUNK_SIZE = getProfileAgentSettingWithDefault( - 64 * 1024, // 64 KiB - "appsecLearningSettings.writeChunkSize" - ); - size_t offset = 0; - while (offset < data.size()) { - size_t current_chunk_size = min(static_cast(CHUNK_SIZE), data.size() - offset); - file.write(reinterpret_cast(data.data()) + offset, current_chunk_size); - offset += current_chunk_size; - m_mainLoop->yield(false); - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Write progress: " << offset << "/" << data.size() - << " bytes (" << (offset * 100 / data.size()) << "%) - yielded"; - } - file.close(); - return Maybe(); -} - void ConfidenceCalculator::saveTimeWindowLogger() { + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Saving the time window logger to: " << m_path_to_backup; if (m_path_to_backup != "") // remove old file from exceed memory cap flow { remove(m_path_to_backup.c_str()); @@ -257,69 +203,34 @@ void ConfidenceCalculator::saveTimeWindowLogger() m_path_to_backup = temp_filename; - stringstream ss; - { - cereal::JSONOutputArchive archive(ss); - archive(cereal::make_nvp("logger", *m_time_window_logger)); - } - - m_mainLoop->yield(false); - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "JSON serialized, size: " << ss.str().size() << " bytes"; - - string data = ss.str(); - - const uint COMPRESSED_CHUNK_SIZE = getProfileAgentSettingWithDefault( - 16 * 1024, // 16KB - "appsecLearningSettings.compressionChunkSize" - ); - auto compression_stream = initCompressionStream(); - size_t offset = 0; - vector compressed_data; - bool ok = true; - - while (offset < data.size()) { - size_t chunk_size = min(static_cast(COMPRESSED_CHUNK_SIZE), data.size() - offset); - bool is_last = (offset + chunk_size >= data.size()); - CompressionResult chunk_res = compressData( - compression_stream, - CompressionType::GZIP, - static_cast(chunk_size), - reinterpret_cast(data.c_str() + offset), - is_last ? 1 : 0 - ); - if (!chunk_res.ok) { - ok = false; - break; + try { + ofstream file(m_path_to_backup, ios::binary); + if (!file.is_open()) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open file: " << m_path_to_backup + << ", errno: " << errno << ", strerror: " << strerror(errno); + m_time_window_logger_backup = m_time_window_logger; + return; } - if (chunk_res.output && chunk_res.num_output_bytes > 0) { - compressed_data.insert( - compressed_data.end(), - chunk_res.output, - chunk_res.output + chunk_res.num_output_bytes - ); - free(chunk_res.output); + + BufferedCompressedOutputStream compressed_out(file); + { + cereal::JSONOutputArchive archive(compressed_out); + archive(cereal::make_nvp("logger", *m_time_window_logger)); } - offset += chunk_size; + compressed_out.close(); + file.close(); + m_mainLoop->yield(false); - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Compression progress: " << offset << "/" << data.size() - << " bytes processed (" << (offset * 100 / data.size()) << "%) - yielded"; + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "JSON serialized and compressed to file: " << m_path_to_backup; } - finiCompressionStream(compression_stream); - - if (!ok) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to compress data"; + catch (const std::exception &e) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to serialize and compress data: " << e.what(); m_time_window_logger_backup = m_time_window_logger; + m_path_to_backup = ""; return; } - auto maybeError = writeToFile(m_path_to_backup, compressed_data); - if (!maybeError.ok()) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to write the backup file: " << m_path_to_backup; - m_time_window_logger_backup = m_time_window_logger; - m_path_to_backup = ""; - } else { - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Finished writing the backup file: " << m_path_to_backup; - } + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Finished writing the backup file: " << m_path_to_backup; } shared_ptr ConfidenceCalculator::loadTimeWindowLogger() @@ -330,107 +241,41 @@ shared_ptr ConfidenceCalculator::load return nullptr; } - ifstream file(m_path_to_backup, ios::binary); + ifstream file(m_path_to_backup); if (!file.is_open()) { dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open file: " << m_path_to_backup << ", errno: " << errno << ", strerror: " << strerror(errno); return nullptr; } - stringstream buffer; - // Read the file in chunks to avoid large memory usage - const uint READ_CHUNK_SIZE = getProfileAgentSettingWithDefault( - 16 * 1024, - "appsecLearningSettings.readChunkSize"); // 16 KiB - vector chunk(READ_CHUNK_SIZE); - size_t chunk_size = static_cast(READ_CHUNK_SIZE); - size_t total_bytes_read = 0; - size_t chunks_read = 0; - - while (file.peek() != EOF) { - file.read(chunk.data(), chunk_size); - streamsize bytesRead = file.gcount(); - if (bytesRead > 0) { - buffer.write(chunk.data(), bytesRead); - total_bytes_read += bytesRead; - chunks_read++; - } - m_mainLoop->yield(false); - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Read chunk " << chunks_read - << " (" << total_bytes_read << " bytes total) - yielded"; - } - file.close(); - - remove(m_path_to_backup.c_str()); - m_path_to_backup = ""; - - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Completed reading " << total_bytes_read - << " bytes in " << chunks_read << " chunks"; - - string compressed_data = buffer.str(); - auto compression_stream = initCompressionStream(); - - chunk_size = static_cast( - getProfileAgentSettingWithDefault( - 32 * 1024, // 32KiB - "appsecLearningSettings.compressionChunkSize" - ) - ); - size_t offset = 0; - string decompressed_data; - - while (offset < compressed_data.size()) { - size_t current_chunk_size = min(chunk_size, compressed_data.size() - offset); - DecompressionResult res = decompressData( - compression_stream, - current_chunk_size, - reinterpret_cast(compressed_data.c_str() + offset) - ); - - if (!res.ok) { - dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to decompress data from file: " << m_path_to_backup; - finiCompressionStream(compression_stream); - return nullptr; - } - - decompressed_data.append(reinterpret_cast(res.output), res.num_output_bytes); - free(res.output); - res.output = nullptr; - res.num_output_bytes = 0; - - offset += current_chunk_size; - - // Yield control after processing each chunk - m_mainLoop->yield(false); - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Decompression progress: " << offset << "/" << compressed_data.size() - << " bytes (" << (offset * 100 / compressed_data.size()) << "%) - yielded"; - } - - finiCompressionStream(compression_stream); - - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Decompressed data size: " << decompressed_data.size() - << " bytes (compression ratio: " << (float)decompressed_data.size() / compressed_data.size() << "x)"; - - stringstream decompressed_stream(decompressed_data); auto window_logger = make_shared(); try { - cereal::JSONInputArchive archive(decompressed_stream); + BufferedCompressedInputStream compressed_in(file); + cereal::JSONInputArchive archive(compressed_in); archive(cereal::make_nvp("logger", *window_logger)); dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Successfully deserialized logger from JSON"; - } catch (cereal::Exception& e) { + } catch (cereal::Exception &e) { dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to load the time window logger: " << e.what(); + file.close(); return nullptr; } + file.close(); + return window_logger; } bool ConfidenceCalculator::postData() { + if (m_time_window_logger->empty()) + { + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "No data to post, skipping"; + return true; // Nothing to post + } saveTimeWindowLogger(); m_mainLoop->yield(false); - WindowLogPost currentWindow(*m_time_window_logger); + WindowLogPost currentWindow(m_time_window_logger); m_mainLoop->yield(false); m_time_window_logger = make_shared(); string url = getPostDataUrl() + to_string(m_post_index++); @@ -447,7 +292,7 @@ bool ConfidenceCalculator::postData() return ok; } -void ConfidenceCalculator::pullData(const vector& files) +void ConfidenceCalculator::pullData(const vector &files) { if (getIntervalsCount() == m_params.minIntervals) { @@ -465,7 +310,7 @@ void ConfidenceCalculator::pullData(const vector& files) string url = getPostDataUrl(); string sentFile = url.erase(0, strlen("/storage/waap/")); dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "pulling files, skipping: " << sentFile; - for (auto file : files) + for (const auto &file : files) // Use const reference to avoid copying { if (file == sentFile) { @@ -483,13 +328,13 @@ void ConfidenceCalculator::pullData(const vector& files) } KeyValSourcesLogger remoteLogger = getWindow.getWindowLogger().unpack(); - for (auto& log : remoteLogger) + for (const auto &log : remoteLogger) // Use const reference { - const string & key = log.first; - for (auto& entry : log.second) + const string &key = log.first; + for (const auto &entry : log.second) // Use const reference { - const string & value = entry.first; - for (auto & source : entry.second) + const string &value = entry.first; + for (const auto &source : entry.second) // Use const reference { (*m_time_window_logger_backup)[key][value].insert(source); } @@ -504,7 +349,7 @@ void ConfidenceCalculator::processData() m_post_index = 0; if (m_time_window_logger_backup == nullptr || m_time_window_logger_backup->empty()) { - if (m_path_to_backup != "") + if (!m_path_to_backup.empty()) { m_time_window_logger_backup = loadTimeWindowLogger(); m_mainLoop->yield(false); @@ -523,14 +368,14 @@ void ConfidenceCalculator::processData() // clear temp data m_time_window_logger_backup->clear(); m_time_window_logger_backup.reset(); - if (m_path_to_backup != "") + if (!m_path_to_backup.empty()) { remove(m_path_to_backup.c_str()); - m_path_to_backup = ""; + m_path_to_backup.clear(); } } -void ConfidenceCalculator::updateState(const vector& files) +void ConfidenceCalculator::updateState(const vector &files) { pullProcessedData(files); // clear temp data @@ -539,23 +384,23 @@ void ConfidenceCalculator::updateState(const vector& files) m_time_window_logger_backup->clear(); m_time_window_logger_backup.reset(); } - if (m_path_to_backup != "") - { - remove(m_path_to_backup.c_str()); - m_path_to_backup = ""; - } } -void ConfidenceCalculator::pullProcessedData(const vector& files) +Maybe ConfidenceCalculator::getRemoteStateFilePath() const +{ + return m_remotePath + "/remote/confidence.data"; +} + +void ConfidenceCalculator::pullProcessedData(const vector &files) { dbgTrace(D_WAAP) << "Fetching the confidence set object"; m_post_index = 0; bool is_first_pull = true; bool is_ok = false; - for (auto file : files) + for (const auto &file : files) // Use const reference to avoid copying { ConfidenceFileDecryptor getConfFile; - bool res = sendObjectWithRetry(getConfFile, + bool res = sendObject(getConfFile, HTTPMethod::GET, getUri() + "/" + file); is_ok |= res; @@ -564,15 +409,32 @@ void ConfidenceCalculator::pullProcessedData(const vector& files) mergeFromRemote(getConfFile.getConfidenceSet().unpackMove(), is_first_pull); is_first_pull = false; } + if (res && getConfFile.getTrackingKeys().ok()) + { + auto trackingKeys = getConfFile.getTrackingKeys().unpackMove(); + m_indicator_tracking_keys = unordered_set(trackingKeys.begin(), trackingKeys.end()); + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Received tracking keys: " << m_indicator_tracking_keys.size(); + m_tracking_keys_received = true; + } if (res && getConfFile.getConfidenceLevels().ok()) { // write to disk the confidence levels - saveConfidenceLevels(getConfFile.getConfidenceLevels()); + saveConfidenceLevels(getConfFile.getConfidenceLevels().unpackMove()); + } + else + { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get tracking keys from file: " << file; } } // is_ok = false -> no file was downloaded and merged if (!is_ok) { dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the remote state"; + return; + } + if (m_path_to_backup != "") + { + remove(m_path_to_backup.c_str()); + m_path_to_backup = ""; } } @@ -591,7 +453,7 @@ void ConfidenceCalculator::postProcessedData() postUrl); } -void ConfidenceCalculator::serialize(ostream& stream) +void ConfidenceCalculator::serialize(ostream &stream) { cereal::JSONOutputArchive archive(stream); @@ -604,7 +466,7 @@ void ConfidenceCalculator::serialize(ostream& stream) cereal::make_nvp("latest_index", m_latest_index + getIntervalsCount()) ); } -void ConfidenceCalculator::deserialize(istream& stream) +void ConfidenceCalculator::deserialize(istream &stream) { size_t version; cereal::JSONInputArchive archive(stream); @@ -639,7 +501,7 @@ void ConfidenceCalculator::deserialize(istream& stream) } } -void ConfidenceCalculator::loadVer0(cereal::JSONInputArchive& archive) +void ConfidenceCalculator::loadVer0(cereal::JSONInputArchive &archive) { if (!tryParseVersionBasedOnNames( archive, @@ -660,13 +522,13 @@ void ConfidenceCalculator::loadVer0(cereal::JSONInputArchive& archive) } -void ConfidenceCalculator::convertWindowSummaryToConfidenceLevel(const WindowsConfidentValuesList& windows) +void ConfidenceCalculator::convertWindowSummaryToConfidenceLevel(const WindowsConfidentValuesList &windows) { - for (const auto& windowKey : windows) + for (const auto &windowKey : windows) { - for (const auto& window : windowKey.second) + for (const auto &window : windowKey.second) { - for (const auto& value : window) + for (const auto &value : window) { m_confidence_level[windowKey.first][value] += ceil(SCORE_THRESHOLD / m_params.minIntervals); } @@ -674,7 +536,7 @@ void ConfidenceCalculator::convertWindowSummaryToConfidenceLevel(const WindowsCo } } -void ConfidenceCalculator::loadVer2(cereal::JSONInputArchive& archive) +void ConfidenceCalculator::loadVer2(cereal::JSONInputArchive &archive) { ConfidenceCalculatorParams params; ConfidenceSet confidenceSets; @@ -687,11 +549,11 @@ void ConfidenceCalculator::loadVer2(cereal::JSONInputArchive& archive) ); params.maxMemoryUsage = defaultConfidenceMemUsage; reset(params); - for (auto& confidentSet : confidenceSets) + for (auto &confidentSet : confidenceSets) { m_confident_sets[normalize_param(confidentSet.first)] = confidentSet.second; } - for (auto& confidenceLevel : confidenceLevels) + for (auto &confidenceLevel : confidenceLevels) { string normParam = normalize_param(confidenceLevel.first); if (m_confidence_level.find(normParam) == m_confidence_level.end()) @@ -700,7 +562,7 @@ void ConfidenceCalculator::loadVer2(cereal::JSONInputArchive& archive) } else { - for (auto& valueLevelItr : confidenceLevel.second) + for (auto &valueLevelItr : confidenceLevel.second) { if (m_confidence_level[normParam].find(valueLevelItr.first) == m_confidence_level[normParam].end()) { @@ -717,7 +579,7 @@ void ConfidenceCalculator::loadVer2(cereal::JSONInputArchive& archive) } } -void ConfidenceCalculator::loadVer3(cereal::JSONInputArchive& archive) +void ConfidenceCalculator::loadVer3(cereal::JSONInputArchive &archive) { ConfidenceCalculatorParams params; archive( @@ -745,7 +607,7 @@ void ConfidenceCalculator::loadVer3(cereal::JSONInputArchive& archive) } -void ConfidenceCalculator::loadVer1(cereal::JSONInputArchive& archive) +void ConfidenceCalculator::loadVer1(cereal::JSONInputArchive &archive) { WindowsConfidentValuesList windows_summary_list; ConfidenceCalculatorParams params; @@ -764,7 +626,7 @@ void ConfidenceCalculator::loadVer1(cereal::JSONInputArchive& archive) } bool ConfidenceCalculator::tryParseVersionBasedOnNames( - cereal::JSONInputArchive& archive, + cereal::JSONInputArchive &archive, const string ¶ms_field_name, const string &indicators_update_field_name, const string &windows_summary_field_name, @@ -837,17 +699,17 @@ bool ConfidenceCalculator::tryParseVersionBasedOnNames( } void ConfidenceCalculator::mergeConfidenceSets( - ConfidenceSet& confidence_set, - const ConfidenceSet& confidence_set_to_merge, - size_t& last_indicators_update + ConfidenceSet &confidence_set, + const ConfidenceSet &confidence_set_to_merge, + size_t &last_indicators_update ) { - for (auto& set : confidence_set_to_merge) + for (auto &set : confidence_set_to_merge) { size_t num_of_values = confidence_set[set.first].first.size(); dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Merging the set for the key: " << set.first << ". Number of present values: " << num_of_values; - for (auto& value : set.second.first) + for (auto &value : set.second.first) { confidence_set[normalize_param(set.first)].first.insert(value); } @@ -857,7 +719,7 @@ void ConfidenceCalculator::mergeConfidenceSets( } }; -void ConfidenceCalculator::mergeFromRemote(const ConfidenceSet& remote_confidence_set, bool is_first_pull) +void ConfidenceCalculator::mergeFromRemote(const ConfidenceSet &remote_confidence_set, bool is_first_pull) { if (is_first_pull) { m_confident_sets.clear(); @@ -875,7 +737,7 @@ bool ConfidenceCalculator::is_confident(const Key &key, const Val &value) const return false; } - const ValuesSet& confidentValues = confidentSetItr->second.first; + const ValuesSet &confidentValues = confidentSetItr->second.first; if (confidentValues.find(value) != confidentValues.end()) { dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " -" << @@ -895,10 +757,10 @@ void ConfidenceCalculator::calcConfidentValues() m_confident_sets.clear(); } - for (auto& confidenceLevels : m_confidence_level) + for (auto &confidenceLevels : m_confidence_level) { Key key = confidenceLevels.first; - for (auto& valConfidenceLevel : confidenceLevels.second) + for (auto &valConfidenceLevel : confidenceLevels.second) { Val value = valConfidenceLevel.first; dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "key: " << key << ", value: " << value @@ -939,25 +801,28 @@ size_t ConfidenceCalculator::getLastConfidenceUpdate() void ConfidenceCalculator::log(const Key &key, const Val &value, const string &source) { - auto& sources_set = (*m_time_window_logger)[key][value]; - auto result = sources_set.insert(source); - if (result.second) { - // New entry added, update memory usage - if ((*m_time_window_logger)[key][value].size() == 1) { - // first source for this value - means new value - m_estimated_memory_usage += sizeof(value) + value.capacity(); - m_estimated_memory_usage += sizeof(SourcesSet); + // Only track in time window logger if we should track this parameter + if (shouldTrackParameter(key, value)) { + auto &sources_set = (*m_time_window_logger)[key][value]; + auto result = sources_set.insert(source); + if (result.second) { + // New entry added, update memory usage + if ((*m_time_window_logger)[key][value].size() == 1) { + // first source for this value - means new value + m_estimated_memory_usage += sizeof(value) + value.capacity(); + m_estimated_memory_usage += sizeof(SourcesSet); - if ((*m_time_window_logger)[key].size() == 1) { - // first value for this key - means new key - m_estimated_memory_usage += sizeof(key) + key.capacity(); - m_estimated_memory_usage += sizeof(SourcesCounters); + if ((*m_time_window_logger)[key].size() == 1) { + // first value for this key - means new key + m_estimated_memory_usage += sizeof(key) + key.capacity(); + m_estimated_memory_usage += sizeof(SourcesCounters); + } } + m_estimated_memory_usage += sizeof(source) + source.capacity(); } - m_estimated_memory_usage += sizeof(source) + source.capacity(); + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "memory usage: " << m_estimated_memory_usage << + "/" << m_params.maxMemoryUsage; } - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "memory usage: " << m_estimated_memory_usage << - "/" << m_params.maxMemoryUsage; if (value != m_null_obj) { @@ -987,7 +852,65 @@ void ConfidenceCalculator::logSourceHit(const Key &key, const string &source) log(key, m_null_obj, source); } -void ConfidenceCalculator::removeBadSources(SourcesSet& sources, const vector* badSources) +void ConfidenceCalculator::setIndicatorTrackingKeys(const std::vector& keys) +{ + m_indicator_tracking_keys.clear(); + for (const auto& key : keys) { + m_indicator_tracking_keys.insert(key); + } + m_tracking_keys_received = true; + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - received " << keys.size() << " indicator tracking keys from service"; +} + +void ConfidenceCalculator::markKeyAsConfident(const Key &key) +{ + // This method is kept for API compatibility but doesn't affect conditional tracking + // The confidence set is managed independently and not altered by conditional tracking feature + // Add the key to confident sets if not already present + if (m_confident_sets.find(key) == m_confident_sets.end()) { + size_t current_time = chrono::duration_cast( + Singleton::Consume::by()->getWalltime() + ).count(); + m_confident_sets[key] = std::make_pair(ValuesSet(), current_time); + } +} + +bool ConfidenceCalculator::shouldTrackParameter(const Key &key, const Val &value) +{ + // For backward compatibility: if tracking list hasn't been received from service yet, track all + if (!m_tracking_keys_received) { + return true; + } + + // Should NOT track if key->value combination is already in confidence set + if (is_confident(key, value)) { + return !m_params.learnPermanently; // If learnPermanently is true, we don't track confident values + } + if (!m_params.learnPermanently && m_confident_sets.find(key) != m_confident_sets.end()) { + m_indicator_tracking_keys.insert(key); // Ensure the key is in the tracking list + return true; + } + + // Should NOT track if key is not in tracking list AND value is null obj + bool keyInTrackingList = (m_indicator_tracking_keys.find(key) != m_indicator_tracking_keys.end()); + if (!keyInTrackingList && value == m_null_obj) { + return false; + } + + if (!keyInTrackingList) { + // If tracking list is full, do not track this key + m_indicator_tracking_keys.insert(key); // Ensure the key is in the tracking list + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - tracking key: " << key << ", value: " << value; + } + // Should track if: + // 1. Key is in tracking list, OR + // 2. New value (not null obj) AND key->value not in confidence set (already checked above) + return keyInTrackingList || (value != m_null_obj); +} + +void ConfidenceCalculator::removeBadSources(SourcesSet &sources, const vector* badSources) { if (badSources == nullptr) { @@ -999,14 +922,14 @@ void ConfidenceCalculator::removeBadSources(SourcesSet& sources, const vectorgetDecision(source, SOURCE) == BENIGN) { @@ -1026,86 +949,28 @@ void ConfidenceCalculator::loadConfidenceLevels() string file_path = m_filePath + ".levels." + to_string((m_latest_index + getIntervalsCount() - 1) % 2) + ".gz"; ifstream file(file_path, ios::binary); if (!file.is_open()) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open the file: " << file_path; + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open the file: " << file_path << + ", errno: " << errno << ", strerror: " << strerror(errno); return; } - - stringstream buffer; - // Read the file in chunks to avoid large memory usage - const uint READ_CHUNK_SIZE = getProfileAgentSettingWithDefault( - 16 * 1024, - "appsecLearningSettings.readChunkSize"); // 16 KiB - vector chunk(READ_CHUNK_SIZE); - size_t chunk_size = static_cast(READ_CHUNK_SIZE); - while (file.peek() != EOF) { - file.read(chunk.data(), chunk_size); - streamsize bytesRead = file.gcount(); - if (bytesRead > 0) { - buffer.write(chunk.data(), bytesRead); - } - m_mainLoop->yield(false); - } - file.close(); - - string compressed_data = buffer.str(); - - auto compression_stream = initCompressionStream(); - DecompressionResult res; - res.ok = true; - res.output = nullptr; - res.num_output_bytes = 0; - size_t offset = 0; - const size_t CHUNK_SIZE = static_cast( - getProfileAgentSettingWithDefault( - 16 * 1024, // 16KiB - "appsecLearningSettings.compressionChunkSize" - ) - ); - vector decompressed_data_vec; - while (offset < compressed_data.size()) { - size_t current_chunk_size = min(CHUNK_SIZE, compressed_data.size() - offset); - DecompressionResult chunk_res = decompressData( - compression_stream, - current_chunk_size, - reinterpret_cast(compressed_data.c_str() + offset) - ); - if (!chunk_res.ok) { - res.ok = false; - break; - } - if (chunk_res.output && chunk_res.num_output_bytes > 0) { - // Append directly to the vector to avoid extra copies - size_t old_size = decompressed_data_vec.size(); - decompressed_data_vec.resize(old_size + chunk_res.num_output_bytes); - memcpy(decompressed_data_vec.data() + old_size, chunk_res.output, chunk_res.num_output_bytes); - free(chunk_res.output); - } - offset += current_chunk_size; - m_mainLoop->yield(false); - } - finiCompressionStream(compression_stream); - - if (!res.ok) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to decompress the confidence levels data."; - return; - } - - string decompressed_data(decompressed_data_vec.begin(), decompressed_data_vec.end()); - stringstream decompressed_stream(decompressed_data); - try { + BufferedCompressedInputStream decompressed_stream(file); cereal::JSONInputArchive archive(decompressed_stream); archive(cereal::make_nvp("confidence_levels", m_confidence_level)); } catch (runtime_error &e) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to load the confidence levels from disk: " << e.what(); + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) + << "Failed to load the confidence levels, owner: " + << m_owner << ", error: " << e.what(); } + file.close(); dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " - loaded the confidence levels from disk, latest index: " << m_latest_index << ", intervals count: " << getIntervalsCount(); m_mainLoop->yield(false); if (m_confidence_level.empty()) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to load the confidence levels from disk"; + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "confidence levels are empty, owner: " << m_owner << + ", file: " << file_path; } } @@ -1117,34 +982,42 @@ void ConfidenceCalculator::saveConfidenceLevels() void ConfidenceCalculator::saveConfidenceLevels(Maybe confidenceLevels) { + if (!confidenceLevels.ok()) + { + // if confidence levels are not available, use the current confidence level + extractLowConfidenceKeys(m_confidence_level); + } + else + { + // if confidence levels are empty, use the current confidence level + extractLowConfidenceKeys(confidenceLevels.unpackMove()); + } string file_path = m_filePath + ".levels." + to_string((m_latest_index + getIntervalsCount()) % 2) + ".gz"; - stringstream serialized_data; - - try { - { - cereal::JSONOutputArchive archive(serialized_data); - if (confidenceLevels.ok()) { - archive(cereal::make_nvp("confidence_levels", confidenceLevels.unpackMove())); - } else { - archive(cereal::make_nvp("confidence_levels", m_confidence_level)); - } - } - } catch (runtime_error &e) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to serialize the confidence levels: " << e.what(); + ofstream file(file_path, ios::binary); + if (!file.is_open()) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open file: " << file_path + << ", errno: " << errno << ", strerror: " << strerror(errno); return; } - m_mainLoop->yield(false); - - string uncompressed_data = serialized_data.str(); - const size_t CHUNK_SIZE = 16 * 1024; // 16 KiB - vector compressed_data; - - try { - compressDataWrapper(uncompressed_data, CHUNK_SIZE, compressed_data); - writeToFile(file_path, compressed_data); - } catch (const runtime_error& e) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to compress the confidence levels data: " << e.what(); + { + try { + BufferedCompressedOutputStream compressed_out(file); + cereal::JSONOutputArchive archive(compressed_out); + if (confidenceLevels.ok()) { + archive(cereal::make_nvp("confidence_levels", confidenceLevels.unpackMove())); + } + else + { + archive(cereal::make_nvp("confidence_levels", m_confidence_level)); + } + } catch (runtime_error &e) { + file.close(); + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to serialize the confidence levels: " << e.what(); + return; + } } + file.close(); + m_mainLoop->yield(false); m_confidence_level.clear(); @@ -1176,14 +1049,13 @@ void ConfidenceCalculator::calculateInterval() } int itr = 0; - for (auto sourcesCtrItr : *m_time_window_logger_backup) { if (++itr % 20 == 0) { // yield every 20 iterations m_mainLoop->yield(false); } - SourcesCounters& srcCtrs = sourcesCtrItr.second; + SourcesCounters &srcCtrs = sourcesCtrItr.second; Key key = sourcesCtrItr.first; ValuesSet summary; double factor = 1.0; @@ -1203,7 +1075,7 @@ void ConfidenceCalculator::calculateInterval() dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " -" << " calculate window summary for the parameter: " << key; // get all unique sources from the null object counter - SourcesSet& sourcesUnion = srcCtrs[m_null_obj]; + SourcesSet &sourcesUnion = srcCtrs[m_null_obj]; removeBadSources(sourcesUnion, sourcesToIgnore); size_t numOfSources = sumSourcesWeight(sourcesUnion); @@ -1219,7 +1091,7 @@ void ConfidenceCalculator::calculateInterval() for (auto srcSets : srcCtrs) { // log the ratio of unique sources from all sources for each value - SourcesSet& currentSourcesSet = srcSets.second; + SourcesSet ¤tSourcesSet = srcSets.second; Val value = srcSets.first; if (value == m_null_obj) { @@ -1227,7 +1099,7 @@ void ConfidenceCalculator::calculateInterval() } removeBadSources(currentSourcesSet, sourcesToIgnore); size_t currentSourcesCount = sumSourcesWeight(currentSourcesSet); - auto& confidenceLevel = m_confidence_level[key][value]; + auto &confidenceLevel = m_confidence_level[key][value]; if (currentSourcesCount == 0) { confidenceLevel -= ceil(SCORE_THRESHOLD / m_params.minIntervals); @@ -1244,9 +1116,9 @@ void ConfidenceCalculator::calculateInterval() } } - for (auto& keyMap : m_confidence_level) + for (auto &keyMap : m_confidence_level) { - for (auto& valMap : keyMap.second) + for (auto &valMap : keyMap.second) { if (m_time_window_logger_backup->find(keyMap.first) != m_time_window_logger_backup->end() && (*m_time_window_logger_backup)[keyMap.first].find(valMap.first) == @@ -1263,12 +1135,12 @@ void ConfidenceCalculator::calculateInterval() saveConfidenceLevels(); } -void ConfidenceCalculator::setOwner(const string& owner) +void ConfidenceCalculator::setOwner(const string &owner) { m_owner = owner + "/ConfidenceCalculator"; } -bool ConfidenceCalculatorParams::operator==(const ConfidenceCalculatorParams& other) +bool ConfidenceCalculatorParams::operator==(const ConfidenceCalculatorParams &other) { return (minSources == other.minSources && minIntervals == other.minIntervals && @@ -1278,7 +1150,7 @@ bool ConfidenceCalculatorParams::operator==(const ConfidenceCalculatorParams& ot maxMemoryUsage == other.maxMemoryUsage); } -ostream& operator<<(ostream& os, const ConfidenceCalculatorParams& ccp) +ostream &operator<<(ostream &os, const ConfidenceCalculatorParams &ccp) { os << "min sources: " << ccp.minSources << " min intervals: " << ccp.minIntervals << @@ -1296,6 +1168,7 @@ void ConfidenceCalculator::garbageCollector() I_MainLoop *mainLoop = Singleton::Consume::by(); mainLoop->addOneTimeRoutine(I_MainLoop::RoutineType::Offline, + // LCOV_EXCL_START [this, mainLoop]() { dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " - running garbage collection of carry-on data files"; @@ -1406,6 +1279,47 @@ void ConfidenceCalculator::garbageCollector() dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " - finished garbage collection."; }, + // LCOV_EXCL_STOP "ConfidenceCalculator garbage collection" ); } + +void ConfidenceCalculator::extractLowConfidenceKeys(const ConfidenceLevels& confidence_levels) +{ + const double LOW_CONFIDENCE_THRESHOLD = 100.0; + size_t keys_added = 0; + m_tracking_keys_received = true; // Ensure tracking keys are considered received + + for (const auto& keyEntry : confidence_levels) { + const std::string& key = keyEntry.first; + const auto& valueConfidenceMap = keyEntry.second; + + if (!m_params.learnPermanently) { + m_indicator_tracking_keys.insert(key); // Ensure the key is in the tracking list + continue; + } + + // Check if any value for this key has confidence level below threshold + bool hasLowConfidence = false; + for (const auto& valueEntry : valueConfidenceMap) { + if (valueEntry.second < LOW_CONFIDENCE_THRESHOLD) { + hasLowConfidence = true; + break; + } + } + + // If key has low confidence values, add it to tracking keys + if (hasLowConfidence) { + auto result = m_indicator_tracking_keys.insert(key); + if (result.second) { // Key was newly inserted + keys_added++; + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - added key '" << key << "' to tracking list (has confidence < " << + LOW_CONFIDENCE_THRESHOLD << ")"; + } + } + } + + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - added " << keys_added << " keys with low confidence values to tracking list"; +} diff --git a/components/security_apps/waap/waap_clib/ConfidenceCalculator.h b/components/security_apps/waap/waap_clib/ConfidenceCalculator.h index bb591d3..2fcf8e5 100755 --- a/components/security_apps/waap/waap_clib/ConfidenceCalculator.h +++ b/components/security_apps/waap/waap_clib/ConfidenceCalculator.h @@ -66,6 +66,8 @@ struct ConfidenceCalculatorParams friend std::ostream & operator<<(std::ostream &os, const ConfidenceCalculatorParams &ccp); }; +// TODO PHASE3: remove inheritance from SerializeToLocalAndRemoteSyncBase + class ConfidenceCalculator : public SerializeToLocalAndRemoteSyncBase { public: @@ -106,17 +108,16 @@ public: bool reset(ConfidenceCalculatorParams ¶ms); virtual bool postData(); - virtual void pullData(const std::vector& files); + virtual void pullData(const std::vector &files); virtual void processData(); virtual void postProcessedData(); - virtual void pullProcessedData(const std::vector& files); - virtual void updateState(const std::vector& files); + virtual void pullProcessedData(const std::vector &files); + virtual void updateState(const std::vector &files); + virtual Maybe getRemoteStateFilePath() const override; virtual void serialize(std::ostream &stream); virtual void deserialize(std::istream &stream); - Maybe writeToFile(const std::string& path, const std::vector& data); - void mergeFromRemote(const ConfidenceSet &remote_confidence_set, bool is_first_pull); bool is_confident(const Key &key, const Val &value) const; @@ -135,6 +136,13 @@ public: static void mergeConfidenceSets(ConfidenceSet &confidence_set, const ConfidenceSet &confidence_set_to_merge, size_t &last_indicators_update); + + // Methods for conditional parameter tracking (INXT-46771) + void setIndicatorTrackingKeys(const std::vector& keys); + void markKeyAsConfident(const Key &key); + bool shouldTrackParameter(const Key &key, const Val &value); + void extractLowConfidenceKeys(const ConfidenceLevels& confidence_levels); + private: void loadVer0(cereal::JSONInputArchive &archive); void loadVer1(cereal::JSONInputArchive &archive); @@ -178,4 +186,8 @@ private: I_MainLoop *m_mainLoop; I_MainLoop::RoutineID m_routineId; std::vector m_filesToRemove; + + // Additional member variables for conditional tracking (INXT-46771) + std::unordered_set m_indicator_tracking_keys; + bool m_tracking_keys_received; }; diff --git a/components/security_apps/waap/waap_clib/ConfidenceFile.cc b/components/security_apps/waap/waap_clib/ConfidenceFile.cc index 35a74ce..c6b42d4 100755 --- a/components/security_apps/waap/waap_clib/ConfidenceFile.cc +++ b/components/security_apps/waap/waap_clib/ConfidenceFile.cc @@ -29,9 +29,15 @@ Maybe ConfidenceFileDecryptor::getConfid return genError("failed to get confidence levels"); } +Maybe> ConfidenceFileDecryptor::getTrackingKeys() const +{ + if (!tracking_keys.get().empty()) return tracking_keys.get(); + return genError("failed to get tracking keys"); +} -ConfidenceFileEncryptor::ConfidenceFileEncryptor(const ConfidenceCalculator::ConfidenceSet& _confidence_set, - const ConfidenceCalculator::ConfidenceLevels& _confidence_levels) : + +ConfidenceFileEncryptor::ConfidenceFileEncryptor(const ConfidenceCalculator::ConfidenceSet &_confidence_set, + const ConfidenceCalculator::ConfidenceLevels &_confidence_levels) : confidence_set(_confidence_set), confidence_levels(_confidence_levels) { } diff --git a/components/security_apps/waap/waap_clib/ConfidenceFile.h b/components/security_apps/waap/waap_clib/ConfidenceFile.h index 1cd56d6..056cfd5 100755 --- a/components/security_apps/waap/waap_clib/ConfidenceFile.h +++ b/components/security_apps/waap/waap_clib/ConfidenceFile.h @@ -23,17 +23,20 @@ public: getConfidenceSet() const; Maybe getConfidenceLevels() const; + Maybe> + getTrackingKeys() const; private: S2C_PARAM(ConfidenceCalculator::ConfidenceSet, confidence_set); S2C_OPTIONAL_PARAM(ConfidenceCalculator::ConfidenceLevels, confidence_levels); + S2C_OPTIONAL_PARAM(std::vector, tracking_keys); }; class ConfidenceFileEncryptor : public RestGetFile { public: - ConfidenceFileEncryptor(const ConfidenceCalculator::ConfidenceSet& _confidence_set, - const ConfidenceCalculator::ConfidenceLevels& _confidence_levels); + ConfidenceFileEncryptor(const ConfidenceCalculator::ConfidenceSet &_confidence_set, + const ConfidenceCalculator::ConfidenceLevels &_confidence_levels); private: C2S_PARAM(ConfidenceCalculator::ConfidenceSet, confidence_set); diff --git a/components/security_apps/waap/waap_clib/DeepParser.cc b/components/security_apps/waap/waap_clib/DeepParser.cc index af97ab8..b873d74 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.cc +++ b/components/security_apps/waap/waap_clib/DeepParser.cc @@ -414,7 +414,9 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f if (valueStats.canSplitPipe || valueStats.canSplitSemicolon) { std::string key = IndicatorsFiltersManager::generateKey(m_key.first(), m_key.str(), m_pTransaction); - m_pWaapAssetState->m_filtersMngr->pushSample(key, cur_val, m_pTransaction); + if (m_pWaapAssetState->m_filtersMngr != nullptr) { + m_pWaapAssetState->m_filtersMngr->pushSample(key, cur_val, m_pTransaction); + } } // Detect and decode UTF-16 data @@ -1090,7 +1092,7 @@ DeepParser::createInternalParser( int offset = -1; auto pWaapAssetState = m_pTransaction->getAssetState(); std::shared_ptr signatures = m_pWaapAssetState->getSignatures(); - if (pWaapAssetState != nullptr) { + if (pWaapAssetState != nullptr && pWaapAssetState->m_filtersMngr != nullptr) { // Find out learned type std::set paramTypes = pWaapAssetState->m_filtersMngr->getParameterTypes( IndicatorsFiltersManager::generateKey(m_key.first(), m_key.str(), m_pTransaction) diff --git a/components/security_apps/waap/waap_clib/IndicatorsFilterBase.cc b/components/security_apps/waap/waap_clib/IndicatorsFilterBase.cc index babdb3b..b99b44f 100755 --- a/components/security_apps/waap/waap_clib/IndicatorsFilterBase.cc +++ b/components/security_apps/waap/waap_clib/IndicatorsFilterBase.cc @@ -14,6 +14,7 @@ #include "i_indicatorsFilter.h" #include "IndicatorsFilterBase.h" #include "Waf2Engine.h" +#include "maybe_res.h" IndicatorFilterBase::IndicatorFilterBase(const std::string& confidence_path, const std::string& trusted_path, @@ -113,12 +114,12 @@ bool IndicatorFilterBase::isTrustedSourceOfType(const std::string& source, } -std::string IndicatorFilterBase::getTrustedSource(IWaf2Transaction* pTransaction) +Maybe IndicatorFilterBase::getTrustedSource(IWaf2Transaction* pTransaction) { if (m_policy == nullptr) { dbgTrace(D_WAAP) << "Policy for trusted sources is not set"; - return ""; + return genError("Policy for trusted sources is not set"); } auto trustedTypes = m_policy->getTrustedTypes(); std::string xFwdVal; @@ -171,7 +172,7 @@ std::string IndicatorFilterBase::getTrustedSource(IWaf2Transaction* pTransaction } } - return ""; + return genError("No trusted source found"); } void IndicatorFilterBase::registerKeyword(const std::string& key, @@ -191,3 +192,5 @@ void IndicatorFilterBase::registerKeyword(const std::string& key, m_trusted_confidence_calc.log(key, keyword, trusted_src); } } + +//TODO: update states function (use getRemoteStateFilePath) diff --git a/components/security_apps/waap/waap_clib/IndicatorsFilterBase.h b/components/security_apps/waap/waap_clib/IndicatorsFilterBase.h index 183b3e0..81c34cf 100755 --- a/components/security_apps/waap/waap_clib/IndicatorsFilterBase.h +++ b/components/security_apps/waap/waap_clib/IndicatorsFilterBase.h @@ -19,6 +19,7 @@ #include "TrustedSourcesConfidence.h" #include "ConfidenceCalculator.h" #include "TuningDecisions.h" +#include "maybe_res.h" class IndicatorFilterBase : public I_IndicatorsFilter { @@ -39,8 +40,8 @@ public: bool setTrustedSrcParameter(std::shared_ptr policy); void reset(); + Maybe getTrustedSource(IWaf2Transaction* pTransaction); protected: - std::string getTrustedSource(IWaf2Transaction* pTransaction); void registerKeyword(const std::string& key, const std::string& keyword, const std::string& source, diff --git a/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc b/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc index f93a4ae..91c12c3 100755 --- a/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc +++ b/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc @@ -19,44 +19,168 @@ #include "FpMitigation.h" #include "Waf2Engine.h" #include "WaapKeywords.h" +#include "config.h" -IndicatorsFiltersManager::IndicatorsFiltersManager(const std::string& remotePath, const std::string &assetId, +USE_DEBUG_FLAG(D_WAAP_LEARN); +static constexpr int DEFAULT_SOURCES_LIMIT = 1000; + + +using namespace std; + +// // Helper class for posting unified indicators +// class UnifiedIndicatorsLogPost : public RestGetFile { +// public: +// UnifiedIndicatorsLogPost(shared_ptr container_ptr) +// { +// window_logger = move(*container_ptr); +// } +// private: +// C2S_PARAM(UnifiedIndicatorsContainer, window_logger); +// }; + +IndicatorsFiltersManager::IndicatorsFiltersManager(const string& remotePath, const string &assetId, I_WaapAssetState* pWaapAssetState) : - SerializeToFileBase(pWaapAssetState->getWaapDataDir() + "/6.data"), + SerializeToLocalAndRemoteSyncBase( + chrono::minutes(120), + chrono::seconds(300), + pWaapAssetState->getWaapDataDir() + "/6.data", + (remotePath == "") ? remotePath : remotePath + "/CentralizedData", + assetId, + "IndicatorsFiltersManager" + ), + m_pWaapAssetState(pWaapAssetState), m_ignoreSources(pWaapAssetState->getWaapDataDir(), remotePath, assetId), m_tuning(remotePath), - m_matchedOverrideKeywords() + m_matchedOverrideKeywords(), + m_isLeading(getSettingWithDefault(true, "features", "learningLeader")), + m_sources_limit(DEFAULT_SOURCES_LIMIT), + m_uniqueSources(), + m_unifiedIndicators(make_shared()) { restore(); - m_keywordsFreqFilter = std::make_unique( + m_keywordsFreqFilter = make_unique( pWaapAssetState->getWaapDataDir(), remotePath, assetId, &m_ignoreSources, &m_tuning); - m_typeFilter = std::make_unique(pWaapAssetState, remotePath, assetId, &m_tuning); + m_typeFilter = make_unique(pWaapAssetState, remotePath, assetId, &m_tuning); + m_uniqueSources.reserve(m_sources_limit); + + registerConfigLoadCb([this](){ + updateSourcesLimit(); + updateLearningLeaderFlag(); + }); } IndicatorsFiltersManager::~IndicatorsFiltersManager() { } -void IndicatorsFiltersManager::registerKeywords(const std::string& key, +bool IndicatorsFiltersManager::shouldRegister( + const string& key, + const Waap::Keywords::KeywordsSet& keywords, + const IWaf2Transaction* pTransaction) +{ + // Check if the learning leader flag is true + if (!m_isLeading) { + dbgDebug(D_WAAP_LEARN) << "Learning leader flag is false. Skipping source ID assertion."; + return false; + } + + // if key needs tracking + if (m_keywordsFreqFilter->shouldTrack(key, keywords) || + m_typeFilter->shouldTrack(key, pTransaction)) { + dbgDebug(D_WAAP_LEARN) << "Key '" << key << "' needs tracking."; + } else { + dbgTrace(D_WAAP_LEARN) << "Key '" << key << "' does not need tracking."; + return false; + } + + auto sourceId = pTransaction->getSourceIdentifier(); + + // Check if the database has reached its limit and source is unknown + if (m_uniqueSources.size() >= static_cast(m_sources_limit) && + m_uniqueSources.find(sourceId) == m_uniqueSources.end() ) { + dbgDebug(D_WAAP_LEARN) << "Database limit reached. Cannot insert new source ID '" << sourceId << "'."; + return false; + } + + // Insert the sourceId into the database when did not reached the limit + // If limit is reached, we know that sourceId is already in the database + auto insertResult = m_uniqueSources.insert(sourceId); + if (insertResult.second) { + dbgDebug(D_WAAP_LEARN) << "Inserted new source ID '" << sourceId << "' into the database."; + } else { + dbgTrace(D_WAAP_LEARN) << "source ID '" << sourceId << "' exists in database."; + } + return true; +} + +void IndicatorsFiltersManager::registerKeywords( + const string& key, Waap::Keywords::KeywordsSet& keywords, IWaf2Transaction* pWaapTransaction) { + // Check should register - if false, do not collect data + if (!shouldRegister(key, keywords, pWaapTransaction)) { + return; + } + + const std::string& sourceId = pWaapTransaction->getSourceIdentifier(); + if (m_tuning.getDecision(pWaapTransaction->getLastScanParamName(), PARAM_NAME) == MALICIOUS || m_tuning.getDecision(pWaapTransaction->getLastScanSample(), PARAM_VALUE) == MALICIOUS || m_tuning.getDecision(pWaapTransaction->getUri(), URL) == MALICIOUS || - m_tuning.getDecision(pWaapTransaction->getSourceIdentifier(), SOURCE) == MALICIOUS) + m_tuning.getDecision(sourceId, SOURCE) == MALICIOUS) { + dbgDebug(D_WAAP_LEARN) << "Skipping registration due to tuning decision (malicious)"; return; } + + // TODO: add configuration to choose central logging and return, else do legacy + if(getProfileAgentSettingWithDefault(false, "agent.learning.centralLogging")) { + dbgDebug(D_WAAP_LEARN) << "Central logging is enabled."; + // Build unified entry + UnifiedIndicatorsContainer::Entry entry; + entry.key = key; + entry.sourceId = pWaapTransaction->getSourceIdentifier(); + + // Use the new getTrustedSource method for proper trusted source checking + if (m_keywordsFreqFilter) { + auto trustedSourceResult = m_keywordsFreqFilter->getTrustedSource(pWaapTransaction); + entry.isTrusted = trustedSourceResult.ok(); + if (entry.isTrusted) { + dbgDebug(D_WAAP_LEARN) << "Entry is from trusted source: " << trustedSourceResult.unpack(); + } + } else { + entry.isTrusted = false; + } + + for (const auto &kw : keywords) { + entry.indicators.push_back(kw); + } + + // Add parameter types as TYPE indicators if applicable (skip url# keys) + if (key.rfind("url#", 0) != 0) { + string sample = pWaapTransaction->getLastScanSample(); + auto sampleTypes = m_pWaapAssetState->getSampleType(sample); + entry.types.insert(entry.types.end(), sampleTypes.begin(), sampleTypes.end()); + } + + // Push to unified container + m_unifiedIndicators->addEntry(entry); + return; + } + dbgTrace(D_WAAP_LEARN) << "Central logging is disabled. Using legacy filters."; + + // Legacy behavior (optional): keep existing filters updates for backward compatibility if (!keywords.empty()) { - m_ignoreSources.log(pWaapTransaction->getSourceIdentifier(), key, keywords); + m_ignoreSources.log(sourceId, key, keywords); } + m_keywordsFreqFilter->registerKeywords(key, keywords, pWaapTransaction); if (key.rfind("url#", 0) == 0) { @@ -73,7 +197,7 @@ void IndicatorsFiltersManager::registerKeywords(const std::string& key, } } -bool IndicatorsFiltersManager::shouldFilterKeyword(const std::string &key, const std::string &keyword) const +bool IndicatorsFiltersManager::shouldFilterKeyword(const string &key, const string &keyword) const { bool shouldFilter = false; if (m_keywordsFreqFilter != nullptr) @@ -99,14 +223,14 @@ bool IndicatorsFiltersManager::shouldFilterKeyword(const std::string &key, const return shouldFilter; } -void IndicatorsFiltersManager::serialize(std::ostream& stream) +void IndicatorsFiltersManager::serialize(ostream& stream) { cereal::JSONOutputArchive archive(stream); archive(cereal::make_nvp("version", 1), cereal::make_nvp("trustedSrcParams", m_trustedSrcParams)); } -void IndicatorsFiltersManager::deserialize(std::istream& stream) +void IndicatorsFiltersManager::deserialize(istream& stream) { cereal::JSONInputArchive archive(stream); @@ -116,10 +240,10 @@ void IndicatorsFiltersManager::deserialize(std::istream& stream) { archive(cereal::make_nvp("version", version)); } - catch (std::runtime_error & e) { + catch (runtime_error & e) { archive.setNextName(nullptr); version = 0; - dbgDebug(D_WAAP) << "Can't load file version: " << e.what(); + dbgDebug(D_WAAP_LEARN) << "Can't load file version: " << e.what(); } switch (version) @@ -131,12 +255,12 @@ void IndicatorsFiltersManager::deserialize(std::istream& stream) archive(cereal::make_nvp("trustedSrcParams", m_trustedSrcParams)); break; default: - dbgWarning(D_WAAP) << "unknown file format version: " << version; + dbgWarning(D_WAAP_LEARN) << "unknown file format version: " << version; break; } } -std::set IndicatorsFiltersManager::getParameterTypes(const std::string& canonicParam) const +set IndicatorsFiltersManager::getParameterTypes(const string& canonicParam) const { return m_typeFilter->getParamTypes(canonicParam); } @@ -166,18 +290,18 @@ bool IndicatorsFiltersManager::loadPolicy(IWaapConfig* pConfig) } else { - dbgWarning(D_WAAP) << "Failed to get configuration"; + dbgWarning(D_WAAP_LEARN) << "Failed to get configuration"; } return pConfig != NULL; } -void IndicatorsFiltersManager::filterVerbose(const std::string ¶m, - std::vector& filteredKeywords, - std::map>& filteredKeywordsVerbose) +void IndicatorsFiltersManager::filterVerbose(const string ¶m, + vector& filteredKeywords, + map>& filteredKeywordsVerbose) { - static std::string typeFilterName = "type indicators filter"; - static std::string keywordsFilterName = "keywords frequency indicators filter"; + static string typeFilterName = "type indicators filter"; + static string keywordsFilterName = "keywords frequency indicators filter"; filteredKeywordsVerbose[typeFilterName]; filteredKeywordsVerbose[keywordsFilterName]; auto types = getParameterTypes(param); @@ -209,12 +333,12 @@ void IndicatorsFiltersManager::reset() } -std::string IndicatorsFiltersManager::extractUri(const std::string& referer, const IWaf2Transaction* pTransaction) +string IndicatorsFiltersManager::extractUri(const string& referer, const IWaf2Transaction* pTransaction) { - std::string url; + string url; size_t pos = referer.find("://"); - if (pos == std::string::npos || (pos + 3) > referer.size()) + if (pos == string::npos || (pos + 3) > referer.size()) { url = referer; } @@ -223,11 +347,11 @@ std::string IndicatorsFiltersManager::extractUri(const std::string& referer, con url = referer.substr(pos + 3); } pos = url.find('/'); - if (pos == std::string::npos) + if (pos == string::npos) { return url; } - std::string host = url.substr(0, pos); + string host = url.substr(0, pos); if (host == pTransaction->getHdrContent("host")) { return url.substr(pos); @@ -235,13 +359,13 @@ std::string IndicatorsFiltersManager::extractUri(const std::string& referer, con return url; } -std::string IndicatorsFiltersManager::generateKey(const std::string& location, - const std::string& param_name, +string IndicatorsFiltersManager::generateKey(const string& location, + const string& param_name, const IWaf2Transaction* pTransaction) { - std::string key = location; - static const std::string delim = "#"; - std::string param = normalize_param(param_name); + string key = location; + static const string delim = "#"; + string param = normalize_param(param_name); if (location == "header" || location == "cookie" || location == "url_param") { @@ -268,8 +392,8 @@ std::string IndicatorsFiltersManager::generateKey(const std::string& location, } else if (location == "referer") { - std::string referer = pTransaction->getHdrContent("referer"); - std::string uri = extractUri(referer, pTransaction); + string referer = pTransaction->getHdrContent("referer"); + string uri = extractUri(referer, pTransaction); key = "url" + delim + normalize_uri(uri); } else @@ -279,10 +403,10 @@ std::string IndicatorsFiltersManager::generateKey(const std::string& location, return key; } -std::string IndicatorsFiltersManager::getLocationFromKey(const std::string& canonicKey, IWaf2Transaction* pTransaction) +string IndicatorsFiltersManager::getLocationFromKey(const string& canonicKey, IWaf2Transaction* pTransaction) { - std::vector known_locations = { "header", "cookie", "url", "body", "referer", "url_param" }; - std::string delim = "#"; + vector known_locations = { "header", "cookie", "url", "body", "referer", "url_param" }; + string delim = "#"; for (auto location : known_locations) { if (canonicKey.find(location + delim) == 0) @@ -294,9 +418,9 @@ std::string IndicatorsFiltersManager::getLocationFromKey(const std::string& cano } void IndicatorsFiltersManager::filterKeywords( - const std::string &key, + const string &key, Waap::Keywords::KeywordsSet& keywords, - std::vector& filteredKeywords) + vector& filteredKeywords) { for (auto keyword = keywords.begin(); keyword != keywords.end(); ) { @@ -313,10 +437,15 @@ void IndicatorsFiltersManager::filterKeywords( } void IndicatorsFiltersManager::pushSample( - const std::string& key, - const std::string& sample, + const string& key, + const string& sample, IWaf2Transaction* pTransaction) { + // Check learning leader flag - if false, do not collect data + if (!m_isLeading) { + return; + } + if (key.rfind("url#", 0) == 0) { return; @@ -324,7 +453,75 @@ void IndicatorsFiltersManager::pushSample( m_typeFilter->registerKeywords(key, sample, pTransaction); } -std::set & IndicatorsFiltersManager::getMatchedOverrideKeywords(void) +set & IndicatorsFiltersManager::getMatchedOverrideKeywords(void) { return m_matchedOverrideKeywords; } + +void IndicatorsFiltersManager::updateLearningLeaderFlag() { + m_isLeading = getSettingWithDefault(true, "features", "learningLeader"); + dbgDebug(D_WAAP_LEARN) << "Updating learning leader flag from configuration: " << (m_isLeading ? "true" : "false"); +} + +void IndicatorsFiltersManager::updateSourcesLimit() +{ + int new_limit = getProfileAgentSettingWithDefault(DEFAULT_SOURCES_LIMIT, "agent.learning.sourcesLimit"); + if (new_limit != m_sources_limit) { + m_sources_limit = new_limit; + m_uniqueSources.reserve(m_sources_limit); + } +} + +bool IndicatorsFiltersManager::postData() +{ + dbgDebug(D_WAAP_LEARN) << "Posting indicators data"; + // Example: post unified indicators data if present + if (m_unifiedIndicators->getKeyCount() == 0) { + dbgDebug(D_WAAP_LEARN) << "No unified indicators to post, skipping"; + return true; // Nothing to post + } + + // Post unified indicators using REST client with C2S_PARAM + UnifiedIndicatorsLogPost logPost(m_unifiedIndicators); + string postUrl = getPostDataUrl(); + dbgTrace(D_WAAP_LEARN) << "Posting unified indicators to: " << postUrl; + bool ok = sendNoReplyObjectWithRetry(logPost, HTTPMethod::PUT, postUrl); + if (!ok) { + dbgError(D_WAAP_LEARN) << "Failed to post unified indicators to: " << postUrl; + } + m_unifiedIndicators = make_shared(); + m_uniqueSources.clear(); + return ok; +} + + +void IndicatorsFiltersManager::pullData(const vector& files) +{ + // Phase 2 : backup sync flow + // Add logic for pulling data from a remote service +} + +void IndicatorsFiltersManager::processData() +{ + // Phase 2 : backup sync flow + // Add logic for processing pulled data + // call filters with ptr to unified data to process data from m_unifiedIndicators +} + +void IndicatorsFiltersManager::postProcessedData() +{ + // Add logic for posting processed data to a remote service +} + +void IndicatorsFiltersManager::pullProcessedData(const vector& files) +{ + // Add logic for pulling processed data from a remote service +} + +// TODO: Phase 3 implement getRemoteStateFilePath to return the base dir + +void IndicatorsFiltersManager::updateState(const vector& files) +{ + // files is a list of single file base dir + // TODO phase 3: call each filter to update internal states +} diff --git a/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.h b/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.h index 852a175..be278a9 100755 --- a/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.h +++ b/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.h @@ -24,48 +24,73 @@ #include #include #include +#include "UnifiedIndicatorsContainer.h" using namespace Waap::Parameters; class IWaf2Transaction; struct Waf2ScanResult; -class IndicatorsFiltersManager : public I_IndicatorsFilter, public SerializeToFileBase +class IndicatorsFiltersManager : public I_IndicatorsFilter, public SerializeToLocalAndRemoteSyncBase { public: IndicatorsFiltersManager(const std::string &remotePath, const std::string &assetId, I_WaapAssetState* pWaapAssetState); + ~IndicatorsFiltersManager(); - virtual void registerKeywords(const std::string& key, Waap::Keywords::KeywordsSet& keywords, + virtual void registerKeywords(const std::string &key, Waap::Keywords::KeywordsSet &keywords, IWaf2Transaction* pWaapTransaction); virtual bool shouldFilterKeyword(const std::string &key, const std::string &keyword) const; - virtual void filterKeywords(const std::string &key, Waap::Keywords::KeywordsSet& keywords, - std::vector& filteredKeywords); + virtual void filterKeywords(const std::string &key, Waap::Keywords::KeywordsSet &keywords, + std::vector &filteredKeywords); std::set &getMatchedOverrideKeywords(void); - void pushSample(const std::string& key, const std::string& sample, IWaf2Transaction* pTransaction); + void pushSample(const std::string &key, const std::string &sample, IWaf2Transaction* pTransaction); bool loadPolicy(IWaapConfig* pConfig); void reset(); void filterVerbose(const std::string ¶m, - std::vector& filteredKeywords, + std::vector &filteredKeywords, std::map>& filteredKeywordsVerbose); - static std::string getLocationFromKey(const std::string& canonicKey, IWaf2Transaction* pTransaction); - static std::string generateKey(const std::string& location, - const std::string& param, + static std::string getLocationFromKey(const std::string &canonicKey, IWaf2Transaction* pTransaction); + static std::string generateKey(const std::string &location, + const std::string ¶m, const IWaf2Transaction* pTransaction); - virtual void serialize(std::ostream& stream); - virtual void deserialize(std::istream& stream); + virtual void serialize(std::ostream &stream); + virtual void deserialize(std::istream &stream); - virtual std::set getParameterTypes(const std::string& canonicParam) const; + virtual std::set getParameterTypes(const std::string &canonicParam) const; + + // New required functions from SerializeToLocalAndRemoteSyncBase + virtual bool postData() override; + virtual void pullData(const std::vector& files) override; + virtual void processData() override; + virtual void postProcessedData() override; + virtual void pullProcessedData(const std::vector& files) override; + virtual void updateState(const std::vector& files) override; + + // Getter for unified indicators (for testing) + const UnifiedIndicatorsContainer& getUnifiedIndicators() const { return *m_unifiedIndicators; } private: - static std::string extractUri(const std::string& referer, const IWaf2Transaction* pTransaction); + static std::string extractUri(const std::string &referer, const IWaf2Transaction* pTransaction); + void updateLearningLeaderFlag(); + bool shouldRegister( + const std::string& key, + const Waap::Keywords::KeywordsSet& keywords, + const IWaf2Transaction* pTransaction + ); + void updateSourcesLimit(); std::unique_ptr m_keywordsFreqFilter; std::unique_ptr m_typeFilter; + I_WaapAssetState* m_pWaapAssetState; std::shared_ptr m_trustedSrcParams; ScannerDetector m_ignoreSources; TuningDecision m_tuning; std::set m_matchedOverrideKeywords; + bool m_isLeading; + int m_sources_limit = 0; + std::unordered_set m_uniqueSources; + std::shared_ptr m_unifiedIndicators; }; diff --git a/components/security_apps/waap/waap_clib/KeyStack.cc b/components/security_apps/waap/waap_clib/KeyStack.cc index 4662c58..face2e8 100755 --- a/components/security_apps/waap/waap_clib/KeyStack.cc +++ b/components/security_apps/waap/waap_clib/KeyStack.cc @@ -20,56 +20,131 @@ USE_DEBUG_FLAG(D_WAAP); KeyStack::KeyStack(const char* name) - :m_name(name), m_nameDepth(0) { + :m_name(name), m_nameDepth(0), m_total_length(0), m_using_buffer(true), + m_str_cache_valid(false), m_first_cache_valid(false) { + m_buffer[0] = '\0'; + m_positions.reserve(16); // Reserve reasonable capacity + m_lengths.reserve(16); + m_fallback_stack.reserve(16); } -void KeyStack::push(const char* subkey, size_t subkeySize, bool countDepth) { - m_stack.push_back(m_key.size()); +void +KeyStack::push(const char *subkey, size_t subkeySize, bool countDepth) +{ + bool dot_needed = !m_positions.empty() && subkey != nullptr && subkeySize > 0; + if (m_using_buffer) { + // Calculate space needed: subkey + dot (if not first) + null terminator + size_t dot_size = dot_needed ? 1 : 0; + size_t needed_space = subkeySize + dot_size + 1; // +1 for null terminator - // Prefix all subkeys (except the first) with '.' - if (!m_key.empty()) { - m_key += '.'; + if (can_fit_in_buffer(needed_space)) { + // Fast path: use fixed buffer + if (dot_needed) { + m_buffer[m_total_length] = '.'; + m_total_length++; + } + + m_positions.push_back(m_total_length); + m_lengths.push_back(subkeySize); + + memcpy(m_buffer + m_total_length, subkey, subkeySize); + m_total_length += subkeySize; + m_buffer[m_total_length] = '\0'; + } else { + // Switch to fallback mode + switch_to_fallback(); + // Continue with fallback logic below + } } - m_key += std::string(subkey, subkeySize); + if (!m_using_buffer) { + // Slow path: use dynamic string + m_fallback_stack.push_back(m_fallback_key.size()); + + if (dot_needed) { + m_fallback_key.append(1, '.'); + } + m_fallback_key.append(subkey, subkeySize); + } if (countDepth) { m_nameDepth++; } + // Invalidate cache since key structure changed + invalidate_cache(); + dbgTrace(D_WAAP) << "KeyStack(" << m_name << ")::push(): '" << std::string(subkey, subkeySize) << "' => full_key='" - << std::string(m_key.data(), m_key.size()) + << c_str() << "'"; } void KeyStack::pop(const char* log, bool countDepth) { - // Keep depth balanced even if m_key[] buffer is full - if (m_key.empty() || m_stack.empty()) { - dbgDebug(D_WAAP) - << "KeyStack(" - << m_name - << ")::pop(): [ERROR] ATTEMPT TO POP FROM EMPTY KEY STACK! " - << log; - return; + if (m_using_buffer) { + if (m_positions.empty()) { + dbgDebug(D_WAAP) + << "KeyStack(" + << m_name + << ")::pop(): [ERROR] ATTEMPT TO POP FROM EMPTY KEY STACK! " + << log; + return; + } + + // Remove last subkey from buffer + m_total_length = m_positions.back(); + // Only remove dot if: + // 1. There are multiple elements (not the first) + // 2. The element being popped had content (length > 0, meaning a dot was added) + // 3. The character before current position is actually a dot (safety check) + if (m_positions.size() > 1 && m_lengths.back() > 0) { + if (m_total_length > 0 && m_buffer[m_total_length - 1] == '.') { + m_total_length -= 1; // Remove the dot + } + } else if (m_positions.size() == 1) { + m_total_length = 0; // First element, no dot to remove + } + + m_positions.pop_back(); + m_lengths.pop_back(); + m_buffer[m_total_length] = '\0'; + } else { + // Fallback mode + if (m_fallback_key.empty() || m_fallback_stack.empty()) { + dbgDebug(D_WAAP) + << "KeyStack(" + << m_name + << ")::pop(): [ERROR] ATTEMPT TO POP FROM EMPTY KEY STACK! " + << log; + return; + } + + // Remove last subkey + m_fallback_key.erase(m_fallback_stack.back()); + m_fallback_stack.pop_back(); + + // Try to switch back to buffer if possible + if (m_fallback_key.size() + 1 < MAX_KEY_SIZE) { + rebuild_buffer_from_fallback(); + } } if (countDepth) { m_nameDepth--; } - // Remove last subkey. - m_key.erase(m_stack.back()); - m_stack.pop_back(); + // Invalidate cache since key structure changed + invalidate_cache(); + dbgTrace(D_WAAP) << "KeyStack(" << m_name << ")::pop(): full_key='" - << std::string(m_key.data(), (int)m_key.size()) + << c_str() << "': pop_key=" << log << "'"; @@ -81,6 +156,200 @@ void KeyStack::print(std::ostream &os) const << "KeyStack(" << m_name << ")::show(): full_key='" - << std::string(m_key.data(), (int)m_key.size()) + << c_str() << "'"; } + +void KeyStack::clear() { + if (m_using_buffer) { + m_positions.clear(); + m_lengths.clear(); + m_total_length = 0; + m_buffer[0] = '\0'; + } else { + m_fallback_key.clear(); + m_fallback_stack.clear(); + m_using_buffer = true; + m_total_length = 0; + m_buffer[0] = '\0'; + } + m_nameDepth = 0; + invalidate_cache(); +} + +size_t KeyStack::size() const { + if (m_using_buffer) { + if (m_positions.size() <= 1 || m_positions[1] >= m_total_length) { + // No second element or second element has no content + return 0; + } + // Return size from second subkey onwards + return m_total_length - m_positions[1]; + } else { + // Fallback mode + if (m_fallback_stack.size() <= 1) { + return 0; + } + // m_fallback_stack[1] points to the dot preceding the 2nd subkey. + // Exclude the dot itself from the reported size. + if (m_fallback_stack[1] + 1 >= m_fallback_key.size()) { + return 0; // Defensive: nothing after the dot + } + return m_fallback_key.size() - (m_fallback_stack[1] + 1); + } +} + +const char* KeyStack::c_str() const { + if (m_using_buffer) { + if (m_positions.size() <= 1 || m_positions[1] >= m_total_length) { + // No second element or second element has no content + return ""; + } + // Return pointer to second subkey (skip first + dot) + return m_buffer + m_positions[1]; + } else { + // Fallback mode + if (m_fallback_stack.size() <= 1) { + return ""; + } + // m_fallback_stack[1] points to the dot. Skip it for consistency with buffer mode. + static thread_local std::string temp_result; + size_t start = m_fallback_stack[1] + 1; + if (start >= m_fallback_key.size()) { + temp_result.clear(); + } else { + temp_result = m_fallback_key.substr(start); + } + return temp_result.c_str(); + } +} + +const std::string KeyStack::str() const { + if (m_str_cache_valid) { + return m_cached_str; + } + + if (m_using_buffer) { + if (m_positions.size() <= 1 || m_positions[1] >= m_total_length) { + // No second element or second element has no content + m_cached_str = std::string(); + } else { + // Return string from second subkey onwards + m_cached_str = std::string(m_buffer + m_positions[1], m_total_length - m_positions[1]); + } + } else { + // Fallback mode + if (m_fallback_stack.size() <= 1) { + m_cached_str = std::string(); + } else { + size_t start = m_fallback_stack[1] + 1; // Skip the dot + if (start >= m_fallback_key.size()) { + m_cached_str.clear(); + } else { + m_cached_str = m_fallback_key.substr(start); + } + } + } + + m_str_cache_valid = true; + return m_cached_str; +} + +const std::string KeyStack::first() const { + if (m_first_cache_valid) { + return m_cached_first; + } + + if (m_using_buffer) { + if (m_positions.empty()) { + m_cached_first = std::string(); + } else if (m_positions.size() == 1) { + // Only one subkey, return the whole buffer content + m_cached_first = std::string(m_buffer, m_lengths[0]); + } else { + // Multiple subkeys, return first one + m_cached_first = std::string(m_buffer + m_positions[0], m_lengths[0]); + } + } else { + // Fallback mode + if (m_fallback_stack.empty()) { + m_cached_first = std::string(); + } else if (m_fallback_stack.size() == 1) { + m_cached_first = m_fallback_key; + } else { + // m_fallback_stack[1] points to the dot; substring up to dot (exclude it) + size_t dot_pos = m_fallback_stack[1]; + if (dot_pos == 0 || dot_pos > m_fallback_key.size()) { + m_cached_first.clear(); + } else { + m_cached_first = m_fallback_key.substr(0, dot_pos); + } + } + } + + m_first_cache_valid = true; + return m_cached_first; +} + +bool KeyStack::can_fit_in_buffer(size_t additional_size) const { + return (m_total_length + additional_size) < MAX_KEY_SIZE; +} + +void KeyStack::switch_to_fallback() { + // Copy buffer content to fallback string + m_fallback_key.assign(m_buffer, m_total_length); + + // Convert positions to stack format used by fallback + m_fallback_stack.clear(); + for (size_t i = 0; i < m_positions.size(); ++i) { + if (i == 0) { + m_fallback_stack.push_back(0); + } else { + // Position after dot + m_fallback_stack.push_back(m_positions[i] - 1); + } + } + + m_using_buffer = false; + invalidate_cache(); +}void KeyStack::rebuild_buffer_from_fallback() { + if (m_fallback_key.size() + 1 >= MAX_KEY_SIZE) { + return; // Still too big for buffer + } + + // Copy fallback content back to buffer + memcpy(m_buffer, m_fallback_key.c_str(), m_fallback_key.size()); + m_total_length = m_fallback_key.size(); + m_buffer[m_total_length] = '\0'; + + // Rebuild positions and lengths by parsing the buffer + m_positions.clear(); + m_lengths.clear(); + + size_t pos = 0; + while (pos < m_total_length) { + m_positions.push_back(pos); + + // Find length of current subkey + size_t start = pos; + while (pos < m_total_length && m_buffer[pos] != '.') { + pos++; + } + m_lengths.push_back(pos - start); + + if (pos < m_total_length) { + pos++; // Skip the dot + } + } + + // Clear fallback data + m_fallback_key.clear(); + m_fallback_stack.clear(); + m_using_buffer = true; + invalidate_cache(); +} + +void KeyStack::invalidate_cache() { + m_str_cache_valid = false; + m_first_cache_valid = false; +} diff --git a/components/security_apps/waap/waap_clib/KeyStack.h b/components/security_apps/waap/waap_clib/KeyStack.h index bac022d..32956dc 100755 --- a/components/security_apps/waap/waap_clib/KeyStack.h +++ b/components/security_apps/waap/waap_clib/KeyStack.h @@ -21,62 +21,49 @@ // Represent string (key) that is concatenation of substrings (subkeys) separated by '.' character. // Mostly emulates API of C++ std::string class, with addition of push() and pop() methods // that append individual subkey and delete last subkey from the string efficiently. +// Uses fixed buffer for performance with fallback to dynamic string for long keys. class KeyStack { public: KeyStack(const char *name); void push(const char *subkey, size_t subkeySize, bool countDepth=true); void pop(const char* log, bool countDepth=true); - bool empty() const { return m_key.empty(); } - void clear() { m_key.clear(); m_stack.clear(); } + bool empty() const { return m_using_buffer ? m_positions.empty() : m_fallback_key.empty(); } + void clear(); void print(std::ostream &os) const; size_t depth() const { return m_nameDepth; } - size_t size() const { - return str().size(); - } - const char *c_str() const { - // If pushed none - return empty string. - // If pushed once - still return empty string (the once-pushed subkey will only be returned - // by the first() method. - // If pushed twice or more - return all subkeys starting from the second one. - // Also, even if pushed 2 or more times, but pushed empty strings as subkeys, - // then it could happen that m_key is still empty, in which case we should still return empty string. - if (m_stack.size() <= 1 || m_stack[1] + 1 >= m_key.size()) { - return ""; - } + size_t size() const; + const char *c_str() const; + const std::string str() const; + const std::string first() const; - return m_key.c_str() + m_stack[1] + 1; - } - const std::string str() const { - // If pushed none - return empty string. - // If pushed once - still return empty string (the once-pushed subkey will only be returned - // by the first() method. - // If pushed twice or more - return all subkeys starting from the second one. - // Also, even if pushed 2 or more times, but pushed empty strings as subkeys, - // then it could happen that m_key is still empty, in which case we should still return empty string. - if (m_stack.size() <= 1 || m_stack[1] + 1 >= m_key.size()) { - return ""; - } - - return m_key.substr(m_stack[1] + 1); - } - const std::string first() const { - if (m_stack.size() == 0) { - return ""; - } - else if (m_stack.size() == 1) { - return m_key; - } - else { - // m_stack.size() > 1, so m_stack[1] is valid - return m_key.substr(0, m_stack[1]); - } - } private: + static const size_t MAX_KEY_SIZE = 1024; + const char *m_name; - std::string m_key; - std::vector m_stack; // position of individual key name starts in m_key, - // used to backtrack 1 key at a time. int m_nameDepth; + + // Fixed buffer approach for common case (fast path) + char m_buffer[MAX_KEY_SIZE]; + std::vector m_positions; // Start positions of each subkey in buffer + std::vector m_lengths; // Length of each subkey + size_t m_total_length; + bool m_using_buffer; + + // Fallback to dynamic approach for long keys (slow path) + std::string m_fallback_key; + std::vector m_fallback_stack; + + // Caching for frequently accessed methods + mutable std::string m_cached_str; + mutable std::string m_cached_first; + mutable bool m_str_cache_valid; + mutable bool m_first_cache_valid; + + // Helper methods + void switch_to_fallback(); + void rebuild_buffer_from_fallback(); + bool can_fit_in_buffer(size_t additional_size) const; + void invalidate_cache(); }; #endif // __KEYSTACK_H__0a8039e6 diff --git a/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.cc b/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.cc index daed6f4..a795ad6 100755 --- a/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.cc +++ b/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.cc @@ -12,11 +12,11 @@ // limitations under the License. #include "KeywordIndicatorFilter.h" +#include "i_transaction.h" #include "waap.h" #include "WaapConfigApi.h" #include "WaapConfigApplication.h" #include "FpMitigation.h" -#include "i_transaction.h" #include #include @@ -98,12 +98,24 @@ bool KeywordIndicatorFilter::loadParams(std::shared_ptrgetSourceIdentifier()); - std::string trusted_source = getTrustedSource(pTransaction); + auto trusted_source_maybe = getTrustedSource(pTransaction); + std::string trusted_source = trusted_source_maybe.ok() ? trusted_source_maybe.unpack() : ""; if (keywords.empty()) { registerSource(key, source); diff --git a/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.h b/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.h index 1fc7177..67634e7 100755 --- a/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.h +++ b/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.h @@ -36,10 +36,12 @@ public: size_t minIntervals = CONFIDENCE_MIN_INTERVALS, std::chrono::minutes intervalDuration = CONFIDENCE_WINDOW_INTERVAL, double ratioThreshold = CONFIDENCE_THRESHOLD); + ~KeywordIndicatorFilter(); virtual void registerKeywords(const std::string& key, Waap::Keywords::KeywordsSet& keywords, IWaf2Transaction* pTransaction); + bool shouldTrack(const std::string& key, const Waap::Keywords::KeywordsSet& keywords); virtual bool shouldFilterKeyword(const std::string &key, const std::string &keyword) const; diff --git a/components/security_apps/waap/waap_clib/KeywordTypeValidator.cc b/components/security_apps/waap/waap_clib/KeywordTypeValidator.cc index 8078a9c..553da8c 100755 --- a/components/security_apps/waap/waap_clib/KeywordTypeValidator.cc +++ b/components/security_apps/waap/waap_clib/KeywordTypeValidator.cc @@ -19,7 +19,7 @@ USE_DEBUG_FLAG(D_WAAP); -KeywordTypeValidator::KeywordTypeValidator(const std::string& mapFilePath) : +KeywordTypeValidator::KeywordTypeValidator(const std::string &mapFilePath) : SerializeToFileBase(mapFilePath), m_serializedData(), m_keywordTypeMap(m_serializedData.m_keywordTypeMap) @@ -32,7 +32,7 @@ KeywordTypeValidator::~KeywordTypeValidator() } -void KeywordTypeValidator::serialize(std::ostream& stream) +void KeywordTypeValidator::serialize(std::ostream &stream) { (void)stream; } @@ -42,7 +42,7 @@ void KeywordTypeValidator::saveData() // do not override existing file } -void KeywordTypeValidator::deserialize(std::istream& stream) +void KeywordTypeValidator::deserialize(std::istream &stream) { try { @@ -54,6 +54,7 @@ void KeywordTypeValidator::deserialize(std::istream& stream) } catch (std::runtime_error & e) { dbgWarning(D_WAAP) << "failed to deserialize keyword types validator file. Error: " << e.what(); + throw e; } } @@ -62,7 +63,7 @@ void KeywordTypeValidator::operator=(const KeywordTypeValidator &other) { m_serializedData.m_keywordTypeMap = other.m_serializedData.m_keywordTypeMap; } -bool KeywordTypeValidator::isKeywordOfType(const std::string& keyword, ParamType type) const +bool KeywordTypeValidator::isKeywordOfType(const std::string &keyword, ParamType type) const { auto keywordEntry = m_keywordTypeMap.find(keyword); if (keywordEntry != m_keywordTypeMap.end()) diff --git a/components/security_apps/waap/waap_clib/ParserGzip.cc b/components/security_apps/waap/waap_clib/ParserGzip.cc index 893ba8e..bb85afd 100755 --- a/components/security_apps/waap/waap_clib/ParserGzip.cc +++ b/components/security_apps/waap/waap_clib/ParserGzip.cc @@ -1,115 +1,115 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ParserGzip.h" -#include "debug.h" - -USE_DEBUG_FLAG(D_WAAP_PARSER_GZIP); - -const std::string ParserGzip::m_parserName = "ParserGzip"; - -ParserGzip::ParserGzip(IParserStreamReceiver &receiver, size_t parser_depth) -:m_receiver(receiver), m_key("gzip"), m_state(s_start), m_stream(nullptr) { -} - -ParserGzip::~ParserGzip() { - if (m_stream != nullptr) { - finiCompressionStream(m_stream); - m_stream = nullptr; - } -} - -size_t ParserGzip::push(const char *buf, size_t len) { - dbgTrace(D_WAAP_PARSER_GZIP) << "len=" << (unsigned long int)len << ")"; - - if (len == 0) { - dbgTrace(D_WAAP_PARSER_GZIP) << "end of data signal! m_state=" << m_state; - - // flush - if (m_state != s_start) { // only emit if at least something was pushed - if (m_receiver.onKvDone() != 0) { - m_state = s_error; - } - } - - return 0; - } - DecompressionResult res; - switch (m_state) { - case s_start: - dbgTrace(D_WAAP_PARSER_GZIP) << "s_start"; - if (m_receiver.onKey(m_key.data(), m_key.size()) != 0) { - m_state = s_error; - return 0; - } - m_stream = initCompressionStream(); - m_state = s_forward; - // fallthrough // - CP_FALL_THROUGH; - case s_forward: - dbgTrace(D_WAAP_PARSER_GZIP) << "s_forward"; - res = decompressData( - m_stream, - len, - reinterpret_cast(buf)); - dbgTrace(D_WAAP_PARSER_GZIP) << "res: " << res.ok - << ", size: " << res.num_output_bytes - << ", is last: " << res.is_last_chunk; - - if (!res.ok) { - m_state = s_error; - break; - } - - if (res.num_output_bytes != 0 && - m_receiver.onValue(reinterpret_cast(res.output), res.num_output_bytes) != 0) { - m_state = s_error; - break; - } - - if (res.is_last_chunk) { - m_state = s_done; - break; - } - break; - case s_done: - if (len > 0) { - dbgTrace(D_WAAP_PARSER_GZIP) << " unexpected data after completion, len=" << len; - m_state = s_error; - return 0; // Return 0 to indicate error - } - break; - case s_error: - dbgTrace(D_WAAP_PARSER_GZIP) << "s_error"; - return 0; - } - - return len; -} - -void ParserGzip::finish() { - push(NULL, 0); - if (m_state != s_done) { - m_state = s_error; - return; - } -} - -const std::string & -ParserGzip::name() const { - return m_parserName; -} - -bool ParserGzip::error() const { - return m_state == s_error; -} +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ParserGzip.h" +#include "debug.h" + +USE_DEBUG_FLAG(D_WAAP_PARSER_GZIP); + +const std::string ParserGzip::m_parserName = "ParserGzip"; + +ParserGzip::ParserGzip(IParserStreamReceiver &receiver, size_t parser_depth) +:m_receiver(receiver), m_key("gzip"), m_state(s_start), m_stream(nullptr) { +} + +ParserGzip::~ParserGzip() { + if (m_stream != nullptr) { + finiCompressionStream(m_stream); + m_stream = nullptr; + } +} + +size_t ParserGzip::push(const char *buf, size_t len) { + dbgTrace(D_WAAP_PARSER_GZIP) << "len=" << (unsigned long int)len << ")"; + + if (len == 0) { + dbgTrace(D_WAAP_PARSER_GZIP) << "end of data signal! m_state=" << m_state; + + // flush + if (m_state != s_start) { // only emit if at least something was pushed + if (m_receiver.onKvDone() != 0) { + m_state = s_error; + } + } + + return 0; + } + DecompressionResult res; + switch (m_state) { + case s_start: + dbgTrace(D_WAAP_PARSER_GZIP) << "s_start"; + if (m_receiver.onKey(m_key.data(), m_key.size()) != 0) { + m_state = s_error; + return 0; + } + m_stream = initCompressionStream(); + m_state = s_forward; + // fallthrough // + CP_FALL_THROUGH; + case s_forward: + dbgTrace(D_WAAP_PARSER_GZIP) << "s_forward"; + res = decompressData( + m_stream, + len, + reinterpret_cast(buf)); + dbgTrace(D_WAAP_PARSER_GZIP) << "res: " << res.ok + << ", size: " << res.num_output_bytes + << ", is last: " << res.is_last_chunk; + + if (!res.ok) { + m_state = s_error; + break; + } + + if (res.num_output_bytes != 0 && + m_receiver.onValue(reinterpret_cast(res.output), res.num_output_bytes) != 0) { + m_state = s_error; + break; + } + + if (res.is_last_chunk) { + m_state = s_done; + break; + } + break; + case s_done: + if (len > 0) { + dbgTrace(D_WAAP_PARSER_GZIP) << " unexpected data after completion, len=" << len; + m_state = s_error; + return 0; // Return 0 to indicate error + } + break; + case s_error: + dbgTrace(D_WAAP_PARSER_GZIP) << "s_error"; + return 0; + } + + return len; +} + +void ParserGzip::finish() { + push(NULL, 0); + if (m_state != s_done) { + m_state = s_error; + return; + } +} + +const std::string & +ParserGzip::name() const { + return m_parserName; +} + +bool ParserGzip::error() const { + return m_state == s_error; +} diff --git a/components/security_apps/waap/waap_clib/ParserGzip.h b/components/security_apps/waap/waap_clib/ParserGzip.h index 224e9d0..a35eb68 100755 --- a/components/security_apps/waap/waap_clib/ParserGzip.h +++ b/components/security_apps/waap/waap_clib/ParserGzip.h @@ -1,46 +1,46 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __PARSER_GZIP_H_ -#define __PARSER_GZIP_H_ - -#include "ParserBase.h" -#include -#include "compression_utils.h" - -class ParserGzip : public ParserBase { -public: - ParserGzip(IParserStreamReceiver &receiver, size_t parser_depth); - virtual ~ParserGzip(); - size_t push(const char *data, size_t data_len); - void finish(); - virtual const std::string &name() const; - bool error() const; - virtual size_t depth() { return 1; } -private: - enum state { - s_start, - s_forward, - s_done, - s_error - }; - - IParserStreamReceiver &m_receiver; - std::string m_key; - state m_state; - CompressionStream * m_stream; - - static const std::string m_parserName; -}; - -#endif // __PARSER_GZIP_H_ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __PARSER_GZIP_H_ +#define __PARSER_GZIP_H_ + +#include "ParserBase.h" +#include +#include "compression_utils.h" + +class ParserGzip : public ParserBase { +public: + ParserGzip(IParserStreamReceiver &receiver, size_t parser_depth); + virtual ~ParserGzip(); + size_t push(const char *data, size_t data_len); + void finish(); + virtual const std::string &name() const; + bool error() const; + virtual size_t depth() { return 1; } +private: + enum state { + s_start, + s_forward, + s_done, + s_error + }; + + IParserStreamReceiver &m_receiver; + std::string m_key; + state m_state; + CompressionStream * m_stream; + + static const std::string m_parserName; +}; + +#endif // __PARSER_GZIP_H_ diff --git a/components/security_apps/waap/waap_clib/ParserUrlEncode.cc b/components/security_apps/waap/waap_clib/ParserUrlEncode.cc index 3c7d59d..4903992 100755 --- a/components/security_apps/waap/waap_clib/ParserUrlEncode.cc +++ b/components/security_apps/waap/waap_clib/ParserUrlEncode.cc @@ -92,7 +92,7 @@ ParserUrlEncode::push(const char *buf, size_t len) is_last = (i == (len - 1)); // Checking valid char urlencode - if (c < 32) { + if (static_cast(c) < 32u || static_cast(c) > 126u) { dbgDebug(D_WAAP_PARSER_URLENCODE) << "invalid URL encoding character: " << c; m_state = s_error; return i; diff --git a/components/security_apps/waap/waap_clib/ScannerDetector.cc b/components/security_apps/waap/waap_clib/ScannerDetector.cc index 19d0bf2..1fc6a7e 100755 --- a/components/security_apps/waap/waap_clib/ScannerDetector.cc +++ b/components/security_apps/waap/waap_clib/ScannerDetector.cc @@ -19,17 +19,20 @@ USE_DEBUG_FLAG(D_WAAP); #define SYNC_WAIT_TIME std::chrono::seconds(300) // 5 minutes in seconds #define INTERVAL std::chrono::minutes(120) #define EQUAL_VALUES_COUNT_THRESHOLD 2 -#define MAX_RETENTION 5 +#define MAX_RETENTION 2 +#define DEFAULT_MAX_SOURCES 256 -ScannerDetector::ScannerDetector(const std::string& localPath, const std::string& remotePath, +ScannerDetector::ScannerDetector(const std::string &localPath, const std::string &remotePath, const std::string &assetId) : SerializeToLocalAndRemoteSyncBase(INTERVAL, SYNC_WAIT_TIME, localPath + "/11.data", (remotePath == "") ? remotePath : remotePath + "/ScannersDetector", assetId, - "ScannerDetector") + "ScannerDetector"), + m_current_accumulator(std::make_shared()), + m_maxSources(getProfileAgentSettingWithDefault(-1, "scannerDetector.maxSources")) { - m_sources_monitor.push_front(SourceKeyValsMap()); + dbgTrace(D_WAAP) << "ScannerDetector constructor: m_maxSources set to " << m_maxSources; } bool ScannerDetector::ready() @@ -47,25 +50,76 @@ std::vector* ScannerDetector::getSourcesToIgnore() return &m_sources; } -void ScannerDetector::log(const std::string& source, const std::string& key, Waap::Keywords::KeywordsSet& keywords) +void ScannerDetector::log( + const std::string &source, + const std::string &key, + Waap::Keywords::KeywordsSet &keywords) { - m_sources_monitor.front()[source][key].insert(keywords.begin(), keywords.end()); + if (m_maxSources == uint(-1)) { + m_maxSources = getProfileAgentSettingWithDefault(DEFAULT_MAX_SOURCES, "scannerDetector.maxSources"); + dbgTrace(D_WAAP) << "log: m_maxSources set to " << m_maxSources; + } + + // Add to accumulator for processing - same as original + (*m_current_accumulator)[source][key].insert(keywords.begin(), keywords.end()); + + // Optimized O(1) cache update - just add the key directly to the source cache + auto currentTime = Singleton::Consume::by()->getWalltime(); + + auto cache_it = m_sourceCache.find(source); + if (cache_it != m_sourceCache.end()) { + // Source exists, just add the key - O(1) operation + cache_it->second.keys.insert(key); + cache_it->second.lastUpdate = currentTime; + cache_it->second.accessCount++; + + // Move to front of LRU - O(1) operation + auto lru_it = m_lruMap.find(source); + if (lru_it != m_lruMap.end()) { + m_lruOrder.erase(lru_it->second); + m_lruOrder.push_front(source); + m_lruMap[source] = m_lruOrder.begin(); + } + + dbgTrace(D_WAAP) << "log: Updated existing source " << source << " with key " << key; + return; + } + + // New source - check if cache is full + if (m_sourceCache.size() >= m_maxSources) { + evictLeastImportantSource(); + } + + // Add new source - O(1) operations + SourceInfo newSource(source, currentTime); + newSource.keys.insert(key); + m_sourceCache[source] = newSource; + + // Add to front of LRU list - O(1) operation + m_lruOrder.push_front(source); + m_lruMap[source] = m_lruOrder.begin(); + + dbgTrace(D_WAAP) << "log: Added new source " << source << " with key " << key + << " (cache size: " << m_sourceCache.size() << ")"; } void ScannerDetector::loadParams(std::shared_ptr pParams) { std::string interval = pParams->getParamVal("learnIndicators.intervalDuration", - std::to_string(INTERVAL.count())); + std::to_string(INTERVAL.count())); setInterval(std::chrono::minutes(std::stoul(interval))); std::string remoteSyncStr = pParams->getParamVal("remoteSync", "true"); setRemoteSyncEnabled(!boost::iequals(remoteSyncStr, "false")); + + m_maxSources = getProfileAgentSettingWithDefault(DEFAULT_MAX_SOURCES, "scannerDetector.maxSources"); + dbgTrace(D_WAAP) << "loadParams: m_maxSources set to " << m_maxSources; } class SourcesMonitorPost : public RestGetFile { public: - SourcesMonitorPost(ScannerDetector::SourceKeyValsMap& _monitor) - : monitor(_monitor) + SourcesMonitorPost(ScannerDetector::SourceKeyValsMap &_monitor) + : monitor(std::move(_monitor)) { } @@ -92,31 +146,31 @@ private: bool ScannerDetector::postData() { - m_sources_monitor_backup = m_sources_monitor.front(); - m_sources_monitor.push_front(SourceKeyValsMap()); - std::string url = getPostDataUrl(); - - dbgTrace(D_WAAP) << "Sending the data to: " << url; - - SourcesMonitorPost currentWindow(m_sources_monitor_backup); - bool ok = sendNoReplyObjectWithRetry(currentWindow, - HTTPMethod::PUT, - url); - if (!ok) { - dbgError(D_WAAP) << "Failed to post collected data to: " << url; + if (m_current_accumulator->empty()) { + dbgDebug(D_WAAP) << "No data to post, skipping"; + return true; } + SourcesMonitorPost postMonitor(*m_current_accumulator); + bool ok = sendNoReplyObjectWithRetry(postMonitor, + HTTPMethod::PUT, + getPostDataUrl()); + + if (ok) { + m_current_accumulator = std::make_shared(); + } + return ok; } -void ScannerDetector::pullData(const std::vector& files) +void ScannerDetector::pullData(const std::vector &files) { std::string url = getPostDataUrl(); std::string sentFile = url.erase(0, url.find_first_of('/') + 1); dbgTrace(D_WAAP) << "pulling files, skipping: " << sentFile; - for (auto file : files) + + for (const auto &file : files) // Use const reference { - if (file == sentFile) - { + if (file == sentFile) { continue; } dbgTrace(D_WAAP) << "Pulling the file: " << file; @@ -131,113 +185,125 @@ void ScannerDetector::pullData(const std::vector& files) } SourceKeyValsMap remoteMonitor = getMonitor.getSourcesMonitor().unpack(); - for (const auto& srcData : remoteMonitor) - { - for (const auto& keyData : srcData.second) - { - m_sources_monitor_backup[srcData.first][keyData.first].insert( - keyData.second.begin(), - keyData.second.end()); - } - } - // update the sources monitor in previous "time window" - auto temp = m_sources_monitor.front(); - m_sources_monitor.pop_front(); - m_sources_monitor.pop_front(); - m_sources_monitor.push_front(m_sources_monitor_backup); - m_sources_monitor.push_front(temp); + mergeMonitors(*m_current_accumulator, remoteMonitor); } } void ScannerDetector::postProcessedData() { - + // Empty implementation as in original } void ScannerDetector::updateState(const std::vector&) { + // Empty implementation as in original } -void ScannerDetector::pullProcessedData(const std::vector& files) +void ScannerDetector::pullProcessedData(const std::vector &files) { (void)files; } -void ScannerDetector::mergeMonitors(SourceKeyValsMap& mergeTo, SourceKeyValsMap& mergeFrom) +void ScannerDetector::evictLeastImportantSource() { - for (const auto& srcData : mergeFrom) - { - for (const auto& keyData : srcData.second) - { - dbgTrace(D_WAAP) << "merging src: " << srcData.first << ", key: " << keyData.first << - ", keywords: " << Waap::Util::setToString(keyData.second); - mergeTo[srcData.first][keyData.first].insert(keyData.second.begin(), keyData.second.end()); + if (m_lruOrder.empty()) { + return; + } + // Enhanced eviction: scan last N sources in LRU and evict the one with the smallest key count + constexpr size_t N = 10; // Number of candidates to consider + auto it = m_lruOrder.rbegin(); + auto it_end = m_lruOrder.rend(); + size_t checked = 0; + std::string evictCandidate; + size_t minKeyCount = std::numeric_limits::max(); + auto candidateIt = m_lruOrder.rbegin(); + for (; it != it_end && checked < N; ++it, ++checked) { + const std::string &source = *it; + auto cacheIt = m_sourceCache.find(source); + size_t keyCount = (cacheIt != m_sourceCache.end()) ? cacheIt->second.keys.size() : 0; + if (keyCount < minKeyCount) { + minKeyCount = keyCount; + evictCandidate = source; + candidateIt = it; + } + } + if (evictCandidate.empty()) { + // fallback to classic LRU + evictCandidate = m_lruOrder.back(); + candidateIt = m_lruOrder.rbegin(); + } + // Remove from all data structures - O(1) operations + m_sourceCache.erase(evictCandidate); + m_lruMap.erase(evictCandidate); + // Erase from m_lruOrder using base iterator + m_lruOrder.erase(std::next(candidateIt).base()); + // Remove evicted source from current accumulator + m_current_accumulator->erase(evictCandidate); + dbgTrace(D_WAAP) << "evictLeastImportantSource: Evicted " << evictCandidate + << " (key count: " << minKeyCount << ", cache size: " << m_sourceCache.size() << ")"; +} + +void ScannerDetector::mergeMonitors(SourceKeyValsMap &mergeTo, const SourceKeyValsMap &mergeFrom) +{ + for (const auto &sourceEntry : mergeFrom) { // Use const reference + const std::string &source = sourceEntry.first; + for (const auto &keyEntry : sourceEntry.second) { // Use const reference + const std::string &key = keyEntry.first; + for (const auto &value : keyEntry.second) { // Use const reference + mergeTo[source][key].insert(value); + } } } } void ScannerDetector::processData() { - if (m_sources_monitor_backup.empty()) - { - m_sources_monitor_backup = m_sources_monitor.front(); - m_sources_monitor.push_front(SourceKeyValsMap()); + dbgTrace(D_WAAP) << "processData: Processing accumulated sources"; + + // Move current data to monitor deque for analysis + if (!m_current_accumulator->empty()) { + m_sources_monitor.push_front(m_current_accumulator); + m_current_accumulator = std::make_shared(); } - if (m_sources_monitor.size() > 2) - { - auto monitorItr = m_sources_monitor.begin()++; - for (monitorItr++; monitorItr != m_sources_monitor.end(); monitorItr++) - { - mergeMonitors(m_sources_monitor_backup, *monitorItr); - } + // Merge all monitors into a single monitor, but only include cached sources + SourceKeyValsMap mergedMonitor; + for (const auto &monitor : m_sources_monitor) { + mergeMonitors(mergedMonitor, *monitor); } + if (m_sources_monitor.size() == MAX_RETENTION) { + m_sources_monitor.pop_back(); // Keep only the latest MAX_RETENTION cycles + } + + // Analyze cached sources to identify scanners m_sources.clear(); - for (auto source : m_sources_monitor_backup) - { - if (source.second.size() <= 2) - { - continue; - } - std::map>& keyVals = source.second; - for (auto key = keyVals.begin(); key != keyVals.end(); key++) - { - auto otherKey = key; - int counter = 0; - for (++otherKey; otherKey != keyVals.end(); otherKey++) - { - if (key->second != otherKey->second) - { - continue; - } - dbgTrace(D_WAAP) << "source monitor: src: " << source.first << ", key_1: " << key->first << ", key_2: " - << otherKey->first << ", vals: " << Waap::Util::setToString(key->second); - counter++; - } - if (counter >= EQUAL_VALUES_COUNT_THRESHOLD) - { - dbgDebug(D_WAAP) << "source: " << source.first << " will be ignored"; - m_sources.push_back(source.first); - break; - } + + // Simple threshold-based scanner detection + const uint SCANNER_KEY_THRESHOLD = 3; + + for (const auto &sourceInfo : mergedMonitor) { + const std::string &source = sourceInfo.first; + const auto &keys = sourceInfo.second; + if (keys.size() >= SCANNER_KEY_THRESHOLD) { + dbgTrace(D_WAAP) << "processData: Source " << source + << " flagged as scanner (keyCount=" << keys.size() << ")"; + m_sources.push_back(source); } } - if (m_sources_monitor.size() > MAX_RETENTION) - { - m_sources_monitor.pop_back(); - } - m_sources_monitor_backup.clear(); + dbgTrace(D_WAAP) << "processData: Found " << m_sources.size() << " scanners out of " + << m_sourceCache.size() << " sources in cache"; + m_lastSync = Singleton::Consume::by()->getWalltime(); } -void ScannerDetector::serialize(std::ostream& stream) +void ScannerDetector::serialize(std::ostream &stream) { (void)stream; } -void ScannerDetector::deserialize(std::istream& stream) +void ScannerDetector::deserialize(std::istream &stream) { (void)stream; } diff --git a/components/security_apps/waap/waap_clib/ScannersDetector.h b/components/security_apps/waap/waap_clib/ScannersDetector.h index e668e5c..c0f02bb 100755 --- a/components/security_apps/waap/waap_clib/ScannersDetector.h +++ b/components/security_apps/waap/waap_clib/ScannersDetector.h @@ -11,45 +11,84 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef __SCANNERS_DETECTOR_H__ -#define __SCANNERS_DETECTOR_H__ +#ifndef __OPTIMIZED_SCANNERS_DETECTOR_H__ +#define __OPTIMIZED_SCANNERS_DETECTOR_H__ #include "WaapKeywords.h" #include "i_serialize.h" #include "i_ignoreSources.h" #include "WaapParameters.h" +#include +#include +#include +#include + +// TODO PHASE3: remove inheritance from SerializeToLocalAndRemoteSyncBase class ScannerDetector : public SerializeToLocalAndRemoteSyncBase, public I_IgnoreSources { public: typedef std::map>> SourceKeyValsMap; - ScannerDetector(const std::string& localPath, const std::string& remotePath = "", const std::string &assetId = ""); + struct SourceInfo { + std::string source; + std::unordered_set keys; + std::chrono::microseconds lastUpdate; + uint32_t accessCount; // Track access frequency for LFU eviction + + // Default constructor for container requirements + SourceInfo() : lastUpdate(std::chrono::microseconds(0)), accessCount(0) {} + + SourceInfo(const std::string &src, std::chrono::microseconds time) + : source(src), lastUpdate(time), accessCount(1) {} + + uint getKeyCount() const { return keys.size(); } + + void updateKeys(const std::set &newKeys) { + keys.clear(); + keys.insert(newKeys.begin(), newKeys.end()); + lastUpdate = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()); + accessCount++; + } + }; + + ScannerDetector( + const std::string &localPath, + const std::string &remotePath = "", + const std::string &assetId = ""); virtual bool ready(); virtual std::vector* getSourcesToIgnore(); - void log(const std::string& source, const std::string& key, Waap::Keywords::KeywordsSet& keywords); + void log(const std::string &source, const std::string &key, Waap::Keywords::KeywordsSet &keywords); void loadParams(std::shared_ptr pParams); virtual bool postData(); - virtual void pullData(const std::vector& files); + virtual void pullData(const std::vector &files); virtual void processData(); virtual void postProcessedData(); - virtual void pullProcessedData(const std::vector& files); - virtual void updateState(const std::vector& files); + virtual void pullProcessedData(const std::vector &files); + virtual void updateState(const std::vector &files); - virtual void serialize(std::ostream& stream); - virtual void deserialize(std::istream& stream); + virtual void serialize(std::ostream &stream); + virtual void deserialize(std::istream &stream); private: - void mergeMonitors(SourceKeyValsMap& mergeTo, SourceKeyValsMap& mergeFrom); - - std::list m_sources_monitor; // list of map source -> key -> set of indicators - SourceKeyValsMap m_sources_monitor_backup; // stores data of the last window to process - + void evictLeastImportantSource(); + void mergeMonitors(SourceKeyValsMap &mergeTo, const SourceKeyValsMap &mergeFrom); + + // Optimized data structures + std::unordered_map m_sourceCache; + std::list m_lruOrder; + std::unordered_map::iterator> m_lruMap; + + // Original data structures for compatibility + std::shared_ptr m_current_accumulator; + std::deque> m_sources_monitor; + SourceKeyValsMap m_sources_monitor_backup; std::vector m_sources; std::chrono::microseconds m_lastSync; + uint m_maxSources; }; - #endif diff --git a/components/security_apps/waap/waap_clib/ScoreBuilder.cc b/components/security_apps/waap/waap_clib/ScoreBuilder.cc index 3112608..6367376 100755 --- a/components/security_apps/waap/waap_clib/ScoreBuilder.cc +++ b/components/security_apps/waap/waap_clib/ScoreBuilder.cc @@ -113,6 +113,7 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState) : restore(); } + ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& baseScores) : m_scoreTrigger(0), m_fpStore(), diff --git a/components/security_apps/waap/waap_clib/Serializator.cc b/components/security_apps/waap/waap_clib/Serializator.cc index 362c704..8e7e8e3 100755 --- a/components/security_apps/waap/waap_clib/Serializator.cc +++ b/components/security_apps/waap/waap_clib/Serializator.cc @@ -16,6 +16,7 @@ #include "Waf2Util.h" #include "WaapAssetState.h" #include "i_instance_awareness.h" +#include "buffered_compressed_stream.h" #include #include #include @@ -33,7 +34,7 @@ USE_DEBUG_FLAG(D_WAAP_SERIALIZE); namespace ch = std::chrono; using namespace std; -typedef ch::duration> days; +typedef ch::duration> days; // Define interval between successful sync times static const ch::minutes assetSyncTimeSliceLength(10); @@ -44,26 +45,40 @@ static const string defaultSharedStorageHost = "appsec-shared-storage-svc"; #define SHARED_STORAGE_HOST_ENV_NAME "SHARED_STORAGE_HOST" #define LEARNING_HOST_ENV_NAME "LEARNING_HOST" -void yieldIfPossible(const string& func, int line) +bool RestGetFile::loadJson(const string &json) { - // Check if we are in the main loop - if (Singleton::exists() && - Singleton::Consume::by()->getCurrentRoutineId().ok()) - { - // If we are not in the main loop, yield to allow other routines to run - // This is important for the main loop to be able to process other events - // and avoid blocking the entire system. - dbgTrace(D_WAAP_SERIALIZE) << "Yielding to main loop from: " << func << ":" << line; - Singleton::Consume::by()->yield(false); + // Try streaming approach first - handles both encryption and compression + try { + dbgTrace(D_WAAP_SERIALIZE) << "Attempting to use streaming approach for JSON loading, data size: " + << json.size() << " bytes"; + stringstream json_stream(json); + // if input json is big then yield to allow other routines to run + if (json.size() > 1000000) { + dbgTrace(D_WAAP_SERIALIZE) << "Input JSON is large, yielding to allow other routines to run"; + YIELD_IF_POSSIBLE(); + } + BufferedCompressedInputStream decompressed_stream(json_stream); + { + cereal::JSONInputArchive json_archive(decompressed_stream); + load(json_archive); + } + YIELD_IF_POSSIBLE(); + dbgTrace(D_WAAP_SERIALIZE) << "Successfully loaded JSON using streaming approach"; + return true; + } + catch (const exception &e) { + dbgDebug(D_WAAP_SERIALIZE) << "Failed to load JSON using streaming approach: " << e.what() + << ". Falling back to legacy approach."; + // Fall back to the legacy approach for backward compatibility + } + catch (...) { + dbgDebug(D_WAAP_SERIALIZE) << "Failed to load JSON using streaming approach" + << ". Falling back to legacy approach."; + // Fall back to the legacy approach for backward compatibility } -} -#define YIELD_IF_POSSIBLE() yieldIfPossible(__FUNCTION__, __LINE__) - -bool RestGetFile::loadJson(const string& json) -{ + // Legacy approach: manual decryption and decompression string json_str; - json_str = json; if (!Waap::Util::isGzipped(json_str)) { @@ -87,7 +102,7 @@ bool RestGetFile::loadJson(const string& json) finiCompressionStream(compression_stream); YIELD_IF_POSSIBLE(); - dbgTrace(D_WAAP_SERIALIZE) << "Yielded after decompression in loadJson, decompressed size: " + dbgTrace(D_WAAP_SERIALIZE) << "Yielded after legacy decompression in loadJson, decompressed size: " << json_str.size() << " bytes"; return ClientRest::loadJson(json_str); @@ -95,76 +110,42 @@ bool RestGetFile::loadJson(const string& json) Maybe RestGetFile::genJson() const { - Maybe json = ClientRest::genJson(); - YIELD_IF_POSSIBLE(); - - if (json.ok()) + stringstream output_stream; + try { - string data = json.unpack(); - - // Get chunk size from profile settings for compression chunks - const size_t COMPRESSED_CHUNK_SIZE = static_cast( - getProfileAgentSettingWithDefault(64 * 1024, "appsecLearningSettings.compressionChunkSize")); - - auto compression_stream = initCompressionStream(); - size_t offset = 0; - std::vector compressed_data; - bool ok = true; - size_t chunk_count = 0; - - // Process data in chunks for compression - while (offset < data.size()) { - size_t chunk_size = std::min(COMPRESSED_CHUNK_SIZE, data.size() - offset); - bool is_last = (offset + chunk_size >= data.size()); - CompressionResult chunk_res = compressData( - compression_stream, - CompressionType::GZIP, - static_cast(chunk_size), - reinterpret_cast(data.c_str() + offset), - is_last ? 1 : 0 - ); - - if (!chunk_res.ok) { - ok = false; - break; - } - - if (chunk_res.output && chunk_res.num_output_bytes > 0) { - compressed_data.insert( - compressed_data.end(), - chunk_res.output, - chunk_res.output + chunk_res.num_output_bytes - ); - free(chunk_res.output); - chunk_res.output = nullptr; - } - - offset += chunk_size; - chunk_count++; - YIELD_IF_POSSIBLE(); - dbgTrace(D_WAAP_SERIALIZE) << "Processed compression chunk " << chunk_count - << ", progress: " << offset << "/" << data.size() << " bytes (" - << (offset * 100 / data.size()) << "%) - yielded"; + BufferedCompressedOutputStream compressed_out(output_stream); + { + cereal::JSONOutputArchive json_archive(compressed_out, cereal::JSONOutputArchive::Options::NoIndent()); + save(json_archive); } - - finiCompressionStream(compression_stream); - dbgDebug(D_WAAP_SERIALIZE) << "Yielded after finalizing compression stream. " - << "Total chunks: " << chunk_count << ", Compression ratio: " - << (data.size() > 0 ? (float)compressed_data.size() / data.size() : 0) << "x"; - - if (!ok) { - dbgWarning(D_WAAP_SERIALIZE) << "Failed to gzip data"; - return genError("Failed to compress data"); - } - - // Create string from compressed data - string compressed_str(reinterpret_cast(compressed_data.data()), compressed_data.size()); - - json = compressed_str; + compressed_out.close(); } - return json; + catch (const exception &e) + { + dbgWarning(D_WAAP_SERIALIZE) << "Failed to generate JSON: " << e.what(); + return genError("Failed to generate JSON: " + string(e.what())); + } + return output_stream.str(); } -SerializeToFilePeriodically::SerializeToFilePeriodically(ch::seconds pollingIntervals, string filePath) : + +// Class to handle retrieving the state timestamp file from learning service +class StateTimestampRetriever : public ClientRest +{ +public: + StateTimestampRetriever() {} + + Maybe getStateTimestamp() const + { + if (timestamp.get().empty()) { + return genError("State timestamp is empty"); + } + return timestamp.get(); + } +private: + S2C_PARAM(string, timestamp); +}; + +SerializeToFilePeriodically::SerializeToFilePeriodically(ch::seconds pollingIntervals, const string &filePath) : SerializeToFileBase(filePath), m_lastSerialization(0), m_interval(pollingIntervals) @@ -210,7 +191,7 @@ void SerializeToFilePeriodically::setInterval(ch::seconds newInterval) } } -SerializeToFileBase::SerializeToFileBase(string fileName) : m_filePath(fileName) +SerializeToFileBase::SerializeToFileBase(const string &fileName) : m_filePath(fileName) { dbgTrace(D_WAAP_SERIALIZE) << "SerializeToFileBase::SerializeToFileBase() fname='" << m_filePath << "'"; @@ -243,7 +224,7 @@ void SerializeToFileBase::saveData() if (maybe_routine.ok()) { Singleton::Consume::by()->yield(false); } - string data = ss.str(); + const string &data = ss.str(); // Use const reference to avoid copying dbgDebug(D_WAAP_SERIALIZE) << "Serialized data size: " << data.size() << " bytes"; // Get chunk size from profile settings, with default of 16 MiB for compression chunks @@ -255,13 +236,13 @@ void SerializeToFileBase::saveData() auto compression_stream = initCompressionStream(); size_t offset = 0; - std::vector compressed_data; + vector compressed_data; bool ok = true; size_t chunk_count = 0; // Process data in chunks for compression while (offset < data.size()) { - size_t chunk_size = std::min(COMPRESSED_CHUNK_SIZE, data.size() - offset); + size_t chunk_size = min(COMPRESSED_CHUNK_SIZE, data.size() - offset); bool is_last = (offset + chunk_size >= data.size()); CompressionResult chunk_res = compressData( compression_stream, @@ -319,7 +300,7 @@ void SerializeToFileBase::saveData() size_t write_chunks = 0; while (offset < data_to_write.size()) { - size_t current_chunk_size = std::min(CHUNK_SIZE, data_to_write.size() - offset); + size_t current_chunk_size = min(CHUNK_SIZE, data_to_write.size() - offset); filestream.write(data_to_write.c_str() + offset, current_chunk_size); offset += current_chunk_size; write_chunks++; @@ -334,7 +315,7 @@ void SerializeToFileBase::saveData() << " (" << data_to_write.size() << " bytes in " << write_chunks << " chunks)"; } -string decompress(string fileContent) { +string decompress(const string &fileContent) { if (!Waap::Util::isGzipped(fileContent)) { dbgTrace(D_WAAP_SERIALIZE) << "file note zipped"; return fileContent; @@ -360,7 +341,7 @@ string decompress(string fileContent) { return fileContent; } -void SerializeToFileBase::loadFromFile(string filePath) +void SerializeToFileBase::loadFromFile(const string &filePath) { dbgTrace(D_WAAP_SERIALIZE) << "loadFromFile() file: " << filePath; fstream filestream; @@ -368,7 +349,7 @@ void SerializeToFileBase::loadFromFile(string filePath) filestream.open(filePath, fstream::in); if (filestream.is_open() == false) { - dbgTrace(D_WAAP_SERIALIZE) << "failed to open file: " << filePath << " Error: " << + dbgWarning(D_WAAP_SERIALIZE) << "failed to open file: " << filePath << " Error: " << strerror(errno); if (!Singleton::exists() || errno != ENOENT) { @@ -387,34 +368,48 @@ void SerializeToFileBase::loadFromFile(string filePath) size_t idPosition = filePath.find(idStr); if (idPosition != string::npos) { - filePath.erase(idPosition, idStr.length() - 1); - dbgDebug(D_WAAP_SERIALIZE) << "retry to load file from : " << filePath; - loadFromFile(filePath); + string modifiedFilePath = filePath; // Create a mutable copy + modifiedFilePath.erase(idPosition, idStr.length() - 1); + dbgDebug(D_WAAP_SERIALIZE) << "retry to load file from : " << modifiedFilePath; + loadFromFile(modifiedFilePath); } return; } dbgTrace(D_WAAP_SERIALIZE) << "loading from file: " << filePath; - int length; - filestream.seekg(0, ios::end); // go to the end - length = filestream.tellg(); // report location (this is the length) + try { + dbgTrace(D_WAAP_SERIALIZE) << "Attempting to load file using streaming approach"; + BufferedCompressedInputStream decompressed_stream(filestream); + deserialize(decompressed_stream); + filestream.close(); + dbgTrace(D_WAAP_SERIALIZE) << "Successfully loaded file using streaming approach"; + return; + } + catch (const exception &e) { + dbgDebug(D_WAAP_SERIALIZE) << "Failed to load file using streaming approach: " << e.what() + << ". Falling back to legacy approach."; + // Fall back to the legacy approach for backward compatibility + filestream.clear(); + filestream.seekg(0, ios::beg); + } + + // Legacy approach: manual file reading, decryption, and decompression + filestream.seekg(0, ios::end); + int length = filestream.tellg(); dbgTrace(D_WAAP_SERIALIZE) << "file length: " << length; assert(length >= 0); // length -1 really happens if filePath is a directory (!) - char* buffer = new char[length]; // allocate memory for a buffer of appropriate dimension + vector buffer(length); // Use vector instead of raw pointer for safety filestream.seekg(0, ios::beg); // go back to the beginning - if (!filestream.read(buffer, length)) // read the whole file into the buffer + if (!filestream.read(buffer.data(), length)) // read the whole file into the buffer { filestream.close(); - delete[] buffer; dbgWarning(D_WAAP_SERIALIZE) << "Failed to read file, file: " << filePath; return; } filestream.close(); - string dataObfuscated(buffer, length); - - delete[] buffer; + string dataObfuscated(buffer.begin(), buffer.end()); stringstream ss; ss << decompress(dataObfuscated); @@ -422,6 +417,7 @@ void SerializeToFileBase::loadFromFile(string filePath) try { deserialize(ss); + dbgTrace(D_WAAP_SERIALIZE) << "Successfully loaded file using legacy approach"; } catch (runtime_error & e) { dbgWarning(D_WAAP_SERIALIZE) << "failed to deserialize file: " << m_filePath << ", error: " << @@ -441,7 +437,7 @@ RemoteFilesList::RemoteFilesList() : files(), filesPathsList() // parses xml instead of json // extracts a file list in -bool RemoteFilesList::loadJson(const string& xml) +bool RemoteFilesList::loadJson(const string &xml) { xmlDocPtr doc; // the resulting document tree dbgTrace(D_WAAP_SERIALIZE) << "XML input: " << xml; @@ -497,8 +493,8 @@ bool RemoteFilesList::loadJson(const string& xml) } contents_node = contents_node->next; } - files.get().push_back(FileMetaData{ file, lastModified }); - filesPathsList.push_back(file); + files.get().push_back(FileMetaData{ move(file), move(lastModified) }); + filesPathsList.push_back(files.get().back().filename); // Use the moved string to avoid extra copy } node = node->next; } @@ -511,12 +507,12 @@ bool RemoteFilesList::loadJson(const string& xml) return true; } -const vector& RemoteFilesList::getFilesList() const +const vector &RemoteFilesList::getFilesList() const { return filesPathsList; } -const vector& RemoteFilesList::getFilesMetadataList() const +const vector &RemoteFilesList::getFilesMetadataList() const { return files.get(); } @@ -547,6 +543,7 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase( { dbgInfo(D_WAAP_SERIALIZE) << "Create SerializeToLocalAndRemoteSyncBase. assetId='" << assetId << "', owner='" << m_owner << "'"; + m_pMainLoop = Singleton::Consume::by(); if (Singleton::exists() && Singleton::Consume::by()->getOrchestrationMode() == @@ -586,11 +583,10 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase( m_type = type; } } - m_pMainLoop = Singleton::Consume::by(); setInterval(interval); } -bool SerializeToLocalAndRemoteSyncBase::isBase() +bool SerializeToLocalAndRemoteSyncBase::isBase() const { return m_remotePath == ""; } @@ -656,13 +652,12 @@ void SerializeToLocalAndRemoteSyncBase::setRemoteSyncEnabled(bool enabled) void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval) { - dbgDebug(D_WAAP_SERIALIZE) << "setInterval: from " << m_interval.count() << " to " << - newInterval.count() << " seconds. assetId='" << m_assetId << "', owner='" << m_owner << "'"; - if (newInterval == m_interval) { return; } + dbgDebug(D_WAAP_SERIALIZE) << "setInterval: from " << m_interval.count() << " to " << + newInterval.count() << " seconds. assetId='" << m_assetId << "', owner='" << m_owner << "'"; m_interval = newInterval; @@ -730,7 +725,7 @@ void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval) m_pMainLoop->yield(remainingTime); timeBeforeSyncWorker = timer->getWalltime(); - m_pMainLoop->addOneTimeRoutine(I_MainLoop::RoutineType::System, [this]() {syncWorker();}, "Sync worker"); + syncWorker(); timeAfterSyncWorker = timer->getWalltime(); } }; @@ -780,32 +775,117 @@ ch::seconds SerializeToLocalAndRemoteSyncBase::getIntervalDuration() const return m_interval; } +Maybe SerializeToLocalAndRemoteSyncBase::getStateTimestampByListing() +{ + RemoteFilesList remoteFiles = getRemoteProcessedFilesList(); + if (remoteFiles.getFilesMetadataList().empty()) + { + return genError("No remote processed files available"); + } + dbgDebug(D_WAAP_SERIALIZE) << "State timestamp by listing: " + << remoteFiles.getFilesMetadataList()[0].modified; + + return remoteFiles.getFilesMetadataList()[0].modified; +} + +bool SerializeToLocalAndRemoteSyncBase::checkAndUpdateStateTimestamp(const string& currentStateTimestamp) +{ + // Check if the state has been updated since last check + if (currentStateTimestamp != m_lastProcessedModified) + { + m_lastProcessedModified = currentStateTimestamp; + dbgDebug(D_WAAP_SERIALIZE) << "State timestamp updated: " << m_lastProcessedModified; + return true; // State was updated + } + return false; // State unchanged +} + void SerializeToLocalAndRemoteSyncBase::updateStateFromRemoteService() { + bool useFallbackMethod = false; for (int i = 0; i < remoteSyncMaxPollingAttempts; i++) { m_pMainLoop->yield(ch::seconds(60)); - RemoteFilesList remoteFiles = getRemoteProcessedFilesList(); - if (remoteFiles.getFilesMetadataList().empty()) + + // Try the dedicated timestamp file first + Maybe timestampResult(genError("Failed to get state timestamp")); + if (!useFallbackMethod) { + timestampResult = getStateTimestamp(); + if (!timestampResult.ok()) { + dbgDebug(D_WAAP_SERIALIZE) << "Failed to get state timestamp from file: " + << timestampResult.getErr() << ", trying listing method"; + useFallbackMethod = true; // Switch to listing method on first failure + } + } + else { - dbgWarning(D_WAAP_SERIALIZE) << "no files generated by the remote service were found"; + dbgDebug(D_WAAP_SERIALIZE) << "trying listing method"; + timestampResult = getStateTimestampByListing(); + } + + if (!timestampResult.ok()) + { + dbgWarning(D_WAAP_SERIALIZE) << "Failed to get state timestamp using any method: " + << timestampResult.getErr(); continue; } - string lastModified = remoteFiles.getFilesMetadataList().begin()->modified; - if (lastModified != m_lastProcessedModified) + + string currentStateTimestamp = timestampResult.unpack(); + + if (checkAndUpdateStateTimestamp(currentStateTimestamp)) { - m_lastProcessedModified = lastModified; - updateState(remoteFiles.getFilesList()); - dbgInfo(D_WAAP_SERIALIZE) << "Owner: " << m_owner << - ". updated state generated by remote at " << m_lastProcessedModified; + // Update state directly from the known remote file path + updateStateFromRemoteFile(); + dbgInfo(D_WAAP_SERIALIZE) << "Owner: " << m_owner + << ". updated state using " << (useFallbackMethod ? "file listing (fallback)" : "timestamp file") + << ": " << m_lastProcessedModified; return; } + else + { + dbgWarning(D_WAAP_SERIALIZE) << "State timestamp unchanged (" + << (useFallbackMethod ? "file listing (fallback)" : "timestamp file") << "): " + << currentStateTimestamp; + } } - dbgWarning(D_WAAP_SERIALIZE) << "polling for update state timeout. for assetId='" + + // All polling attempts failed - fall back to local sync + dbgWarning(D_WAAP_SERIALIZE) << "Polling for update state timeout, falling back to local sync. for assetId='" << m_assetId << "', owner='" << m_owner; localSyncAndProcess(); } +Maybe SerializeToLocalAndRemoteSyncBase::updateStateFromRemoteFile() +{ + auto maybeRemoteFilePath = getRemoteStateFilePath(); + if (!maybeRemoteFilePath.ok()) + { + string error = "Owner: " + m_owner + ", no remote state file path defined: " + maybeRemoteFilePath.getErr(); + dbgWarning(D_WAAP_SERIALIZE) << error; + return genError(error); + } + + string remoteFilePath = maybeRemoteFilePath.unpack(); + vector files = {remoteFilePath}; + updateState(files); + dbgDebug(D_WAAP_SERIALIZE) << "updated state from remote file: " << remoteFilePath; + return Maybe(); +} + +bool SerializeToLocalAndRemoteSyncBase::shouldNotSync() const +{ + OrchestrationMode mode = Singleton::exists() ? + Singleton::Consume::by()->getOrchestrationMode() : OrchestrationMode::ONLINE; + return mode == OrchestrationMode::OFFLINE || !m_remoteSyncEnabled || isBase(); +} + +bool SerializeToLocalAndRemoteSyncBase::shouldSendSyncNotification() const +{ + return getSettingWithDefault(true, "features", "learningLeader") && + ((m_type == "CentralizedData") == + (getProfileAgentSettingWithDefault(false, "agent.learning.centralLogging"))); +} + void SerializeToLocalAndRemoteSyncBase::syncWorker() { dbgInfo(D_WAAP_SERIALIZE) << "Running the sync worker for assetId='" << m_assetId << "', owner='" << @@ -814,7 +894,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() OrchestrationMode mode = Singleton::exists() ? Singleton::Consume::by()->getOrchestrationMode() : OrchestrationMode::ONLINE; - if (mode == OrchestrationMode::OFFLINE || !m_remoteSyncEnabled || isBase() || !postData()) { + if (shouldNotSync() || !postData()) { dbgDebug(D_WAAP_SERIALIZE) << "Did not synchronize the data. for asset: " << m_assetId @@ -834,12 +914,31 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() if (m_lastProcessedModified == "") { dbgTrace(D_WAAP_SERIALIZE) << "check if remote service is operational"; - RemoteFilesList remoteFiles = getRemoteProcessedFilesList(); - if (!remoteFiles.getFilesMetadataList().empty()) + Maybe maybeTimestamp = getStateTimestamp(); + if (maybeTimestamp.ok() && !maybeTimestamp.unpack().empty()) { - m_lastProcessedModified = remoteFiles.getFilesMetadataList()[0].modified; + m_lastProcessedModified = maybeTimestamp.unpack(); dbgInfo(D_WAAP_SERIALIZE) << "First sync by remote service: " << m_lastProcessedModified; } + else + { + dbgWarning(D_WAAP_SERIALIZE) << "Failed to get state timestamp from remote service: " + << maybeTimestamp.getErr(); + maybeTimestamp = getStateTimestampByListing(); + if (maybeTimestamp.ok() && !maybeTimestamp.unpack().empty()) + { + m_lastProcessedModified = maybeTimestamp.unpack(); + dbgInfo(D_WAAP_SERIALIZE) << "First sync by remote service using listing: " << m_lastProcessedModified; + } + else + { + dbgWarning(D_WAAP_SERIALIZE) + << "Failed to get state timestamp from remote service by listing: " + << maybeTimestamp.getErr() + << " skipping syncWorker for assetId='" + << m_assetId << "', owner='" << m_owner << "'"; + } + } } // check if learning service is enabled @@ -854,6 +953,8 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() return; } + // TODO: add should send sync notification function (e.g. do not sync if not leader) + if (mode == OrchestrationMode::HYBRID) { dbgDebug(D_WAAP_SERIALIZE) << "detected running in standalone mode"; I_AgentDetails *agentDetails = Singleton::Consume::by(); @@ -876,7 +977,8 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() if (!ok) { dbgWarning(D_WAAP_SERIALIZE) << "failed to send learning notification"; } - } else { + } else if (shouldSendSyncNotification()) + { SyncLearningNotificationObject syncNotification(m_assetId, m_type, getWindowId()); dbgDebug(D_WAAP_SERIALIZE) << "sending sync notification: " << syncNotification; @@ -941,7 +1043,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList() if (!processedFilesList.getFilesList().empty()) { - const vector& filesMD = processedFilesList.getFilesMetadataList(); + const vector &filesMD = processedFilesList.getFilesMetadataList(); if (filesMD.size() > 1) { dbgWarning(D_WAAP_SERIALIZE) << "got more than 1 expected processed file"; } @@ -1014,8 +1116,40 @@ void SerializeToLocalAndRemoteSyncBase::mergeProcessedFromRemote() I_MainLoop::RoutineType::Offline, [&]() { - RemoteFilesList processedFiles = getProcessedFilesList(); - pullProcessedData(processedFiles.getFilesList()); + // Instrumentation breadcrumbs to help diagnose startup crash inside this routine + dbgTrace(D_WAAP_SERIALIZE) << "start routine for assetId='" << m_assetId + << "', owner='" << m_owner << "'"; + try { + auto success = updateStateFromRemoteFile(); + if (!success.ok()) { + dbgInfo(D_WAAP_SERIALIZE) << "direct state file unavailable: " + << success.getErr() << ". Falling back to listing."; + RemoteFilesList remoteFiles = getProcessedFilesList(); + if (remoteFiles.getFilesList().empty()) { + dbgWarning(D_WAAP_SERIALIZE) << "no remote processed files"; + return; + } + const auto &md_list = remoteFiles.getFilesMetadataList(); + if (!md_list.empty()) { + m_lastProcessedModified = md_list[0].modified; + } else { + dbgWarning(D_WAAP_SERIALIZE) << "metadata list empty while files list not empty"; + } + updateState(remoteFiles.getFilesList()); + dbgInfo(D_WAAP_SERIALIZE) << "updated state from remote files. Last modified: " + << m_lastProcessedModified; + } else { + dbgTrace(D_WAAP_SERIALIZE) << "updated state via direct remote file"; + } + } catch (const JsonError &j) { + dbgError(D_WAAP_SERIALIZE) << "JsonError caught: '" << j.getMsg() + << "' assetId='" << m_assetId << "' owner='" << m_owner << "'"; + throw std::runtime_error(std::string("mergeProcessedFromRemote JsonError: ") + j.getMsg()); + } catch (const std::exception &e) { + dbgError(D_WAAP_SERIALIZE) << "std::exception caught: " << e.what() + << " assetId='" << m_assetId << "' owner='" << m_owner << "'"; + throw; // Let mainloop handle termination with detailed message + } }, "Merge processed data from remote for asset Id: " + m_assetId + ", owner:" + m_owner ); @@ -1052,3 +1186,32 @@ SerializeToLocalAndRemoteSyncBase::getSharedStorageHost() } return defaultSharedStorageHost; } + +string SerializeToLocalAndRemoteSyncBase::getStateTimestampPath() +{ + return m_remotePath + "/internal/lastModified.data"; +} + +Maybe SerializeToLocalAndRemoteSyncBase::getStateTimestamp() +{ + string timestampPath = getStateTimestampPath(); + if (timestampPath.empty()) { + dbgWarning(D_WAAP_SERIALIZE) << "Cannot get state timestamp - invalid path"; + return genError("Invalid timestamp path"); + } + + StateTimestampRetriever timestampRetriever; + bool isSuccessful = sendObject( + timestampRetriever, + HTTPMethod::GET, + getUri() + "/" + timestampPath); + + if (!isSuccessful) { + dbgDebug(D_WAAP_SERIALIZE) << "Failed to get state timestamp file from: " << timestampPath; + return genError("Failed to retrieve timestamp file from: " + timestampPath); + } + + dbgDebug(D_WAAP_SERIALIZE) << "Retrieved state timestamp: " << timestampRetriever.getStateTimestamp().unpack() + << " from path: " << timestampPath; + return timestampRetriever.getStateTimestamp().unpack(); +} diff --git a/components/security_apps/waap/waap_clib/Signatures.cc b/components/security_apps/waap/waap_clib/Signatures.cc index 030b50e..5b74f2e 100755 --- a/components/security_apps/waap/waap_clib/Signatures.cc +++ b/components/security_apps/waap/waap_clib/Signatures.cc @@ -12,18 +12,24 @@ // limitations under the License. #include "Signatures.h" +#include "AssertionRegexes.h" +#include "agent_core_utilities.h" +#include "debug.h" #include "waap.h" +#include // for getenv +#include // for strcmp #include USE_DEBUG_FLAG(D_WAAP); +USE_DEBUG_FLAG(D_WAAP_SAMPLE_SCAN); +USE_DEBUG_FLAG(D_WAAP_HYPERSCAN); typedef picojson::value::object JsObj; typedef picojson::value JsVal; typedef picojson::value::array JsArr; typedef std::map> filtered_parameters_t; - -static std::vector to_strvec(const picojson::value::array& jsV) +static std::vector to_strvec(const picojson::value::array &jsV) { std::vector r; @@ -34,7 +40,7 @@ static std::vector to_strvec(const picojson::value::array& jsV) return r; } -static std::set to_strset(const picojson::value::array& jsA) +static std::set to_strset(const picojson::value::array &jsA) { std::set r; @@ -45,18 +51,18 @@ static std::set to_strset(const picojson::value::array& jsA) return r; } -static std::map to_regexmap(const picojson::value::object& jsO, bool& error) +static std::map to_regexmap(const picojson::value::object &jsO, bool &error) { - std::map r; + std::map r; for (auto it = jsO.begin(); it != jsO.end(); ++it) { - const std::string& n = it->first; + const std::string &n = it->first; // convert name to lowercase now (so we don't need to do it at runtime every time). std::string n_lower; for (std::string::const_iterator pCh = n.begin(); pCh != n.end(); ++pCh) { n_lower += std::tolower(*pCh); } - const picojson::value& v = it->second; + const picojson::value &v = it->second; if (error) { // stop loading regexes if there's previous error... @@ -73,13 +79,12 @@ static std::map to_regexmap(const picojson::value::object& return r; } -static filtered_parameters_t to_filtermap(const picojson::value::object& JsObj) +static filtered_parameters_t to_filtermap(const picojson::value::object &JsObj) { filtered_parameters_t result; - for (auto it = JsObj.begin(); it != JsObj.end(); ++it) - { + for (auto it = JsObj.begin(); it != JsObj.end(); ++it) { const std::string parameter = it->first; - const picojson::value::array& arr = it->second.get(); + const picojson::value::array &arr = it->second.get(); result[parameter] = to_strvec(arr); } return result; @@ -215,9 +220,14 @@ Signatures::Signatures(const std::string& filepath) : to_strset(sigsSource["remove_keywords_always"].get())), user_agent_prefix_re(sigsSource["user_agent_prefix_re"].get()), binary_data_kw_filter(sigsSource["binary_data_kw_filter"].get()), - wbxml_data_kw_filter(sigsSource["wbxml_data_kw_filter"].get()) + wbxml_data_kw_filter(sigsSource["wbxml_data_kw_filter"].get()), + m_hyperscanInitialized(false) { - + // Only preprocess hyperscan patterns if hyperscan is enabled + bool should_use_hyperscan = Signatures::shouldUseHyperscan(); + if (should_use_hyperscan) { + preprocessHyperscanPatterns(); + } } Signatures::~Signatures() @@ -229,21 +239,299 @@ bool Signatures::fail() return error; } -picojson::value::object Signatures::loadSource(const std::string& waapDataFileName) +// Static helper to process assertion flags for a pattern (for testing and internal use) +std::string +Signatures::processAssertions(const std::string &groupName, const std::string &pattern, AssertionFlags &flags) +{ + std::string processed = pattern; + + // Use regexes from AssertionRegexes namespace to detect assertions at start/end of the pattern string + using namespace Waap::AssertionRegexes; + boost::smatch match; + + // Start assertions - only a single '(' can precede + if (boost::regex_search(processed, match, reStartNonWordBehind) && match.position() >= 0) { + flags.setFlag(AssertionFlag::START_NON_WORD_BEHIND); + processed = boost::regex_replace(processed, reStartNonWordBehind, std::string("")); + } + + // Path traversal start assertion + if (boost::regex_search(processed, match, rePathTraversalStart) && match.position() >= 0) { + flags.setFlag(AssertionFlag::PATH_TRAVERSAL_START); + processed = boost::regex_replace(processed, rePathTraversalStart, std::string("")); + } + + // End assertions - only a single ')' can follow + if (boost::regex_search(processed, match, reEndNonWordAhead) && match.position() >= 0) { + flags.setFlag(AssertionFlag::END_NON_WORD_AHEAD); + processed = boost::regex_replace(processed, reEndNonWordAhead, std::string("")); + } else if (boost::regex_search(processed, match, reEndNonWordSpecial) && match.position() >= 0) { + flags.setFlag(AssertionFlag::END_NON_WORD_SPECIAL); + processed = boost::regex_replace(processed, reEndNonWordSpecial, std::string("")); + } + + // Path traversal end assertion + if (boost::regex_search(processed, match, rePathTraversalEnd) && match.position() >= 0) { + flags.setFlag(AssertionFlag::PATH_TRAVERSAL_END); + processed = boost::regex_replace(processed, rePathTraversalEnd, std::string("")); + } + + // wildcard evasion regex group name starts with evasion_wildcard_regex + if (groupName.find("evasion_wildcard_regex") == 0) { + flags.setFlag(AssertionFlag::WILDCARD_EVASION); + } + + return processed; +} + +// Extracts the group name from a regex pattern string (e.g., (?P...)) +std::string Signatures::extractGroupName(const std::string &pattern) { + boost::regex namedGroupRegex(R"(\(\?P<([^>]+)>)"); + boost::smatch match; + if (boost::regex_search(pattern, match, namedGroupRegex)) { + return match[1].str(); + } + return ""; +} + +void Signatures::preprocessHyperscanPatterns() +{ + std::map categoryCount; + + // Helper function to check if a pattern is hyperscan compatible + auto isHyperscanCompatible = [&categoryCount](const std::string &pattern) -> bool { + // Hyperscan doesn't support certain regex features that we can't easily convert + static const std::vector incompatibleFeatures = { + R"((?!\w)", R"((?)", R"((?&)", R"((?|)", R"((?P<)", // Atomic groups, named groups, and branching + R"((?R)" // Recursion + }; + + for (const auto &feature : incompatibleFeatures) { + if (pattern.find(feature) != std::string::npos) { + dbgInfo(D_WAAP_HYPERSCAN) << "Incompatible feature found: " << feature << " in pattern: " << pattern; + categoryCount[feature]++; + return false; + } + } + + boost::regex backrefRegex(R"(\(\\\d+\))"); + if (boost::regex_search(pattern, backrefRegex)) { + dbgInfo(D_WAAP_HYPERSCAN) << "Incompatible backreference found: " << pattern; + categoryCount["backreference"]++; + return false; + } + return true; + }; + + // Helper function to convert regex pattern to hyperscan-compatible format + auto convertToHyperscanPattern = [](const std::string &originalPattern) -> std::string { + std::string converted = originalPattern; + + // Remove named group syntax - convert (?P...) to ... + boost::regex namedGroupRegex(R"(\(\?P<[^>]+>)"); + if (boost::regex_search(converted, namedGroupRegex)) { + std::string end_str = ")"; + if (converted.back() == ')') { + converted.pop_back(); // Remove the trailing ')' + end_str = ""; + } + converted = boost::regex_replace(converted, namedGroupRegex, end_str); + } + + // Handle atomic groups first (before removing word boundaries) + // Hyperscan doesn't support atomic groups, so we need to convert them + + // Convert (?>\b) to nothing (remove word boundary atomic groups) + converted = boost::regex_replace(converted, boost::regex(R"(\(\?\>\\b\))"), std::string("")); + // Convert (?>\B) to nothing (remove non-word boundary atomic groups) + converted = boost::regex_replace(converted, boost::regex(R"(\(\?\>\\B\))"), std::string("")); + // Convert empty atomic groups (?>) to nothing + converted = boost::regex_replace(converted, boost::regex(R"(\(\?\>\))"), std::string("")); + + // // Now remove remaining word boundaries (not supported by Hyperscan) + // // At this point, any \b or \B that was inside atomic groups has been handled above + // converted = boost::regex_replace(converted, boost::regex(R"(\\b)"), std::string("")); + // converted = boost::regex_replace(converted, boost::regex(R"(\\B)"), std::string("")); + + return converted; + }; + + // Helper function to get patterns from sigsSource for each category + auto getCommonPatternsForCategory = [this](const std::string &category, + const std::string ®exSource) -> std::vector { + std::vector patterns; + + // Map regexSource/category to the JSON key in sigsSource + std::string key; + if (regexSource == "specific_acuracy_keywords_regex" || category == "specific_accuracy") { + key = "specific_acuracy_keywords_regex_list"; + } else if (regexSource == "words_regex" || category == "keywords") { + key = "words_regex_list"; + } else if (regexSource == "pattern_regex" || category == "patterns") { + key = "pattern_regex_list"; + } else { + // Fallback: allow passing the exact key name + key = regexSource; + dbgDebug(D_WAAP_HYPERSCAN) << "Unknown category/regexSource: " << category << "/" << regexSource + << ". Using regexSource as key."; + } + + // Fetch patterns directly from sigsSource if available + auto it = sigsSource.find(key); + if (it != sigsSource.end()) { + try { + patterns = to_strvec(it->second.get()); + } catch (...) { + // If the type is unexpected, return empty and continue gracefully + patterns.clear(); + dbgWarning(D_WAAP_HYPERSCAN) << "Unexpected type for key: " << key; + } + } + + return patterns; + }; + + // Process specific_acuracy_keywords_regex patterns + std::vector incompatiblePatterns; + { + auto patterns = getCommonPatternsForCategory("specific_accuracy", "specific_acuracy_keywords_regex"); + for (const auto &pattern : patterns) { + AssertionFlags flags; + std::string groupName = extractGroupName(pattern); + std::string processedPattern = convertToHyperscanPattern(pattern); + std::string hyperscanPattern = processAssertions(groupName, processedPattern, flags); + + if (hyperscanPattern != pattern) { + dbgTrace(D_WAAP_HYPERSCAN) << pattern << " -> " << hyperscanPattern; + } + + if (isHyperscanCompatible(hyperscanPattern)) { + HyperscanPattern hsPattern; + hsPattern.originalPattern = pattern; + hsPattern.hyperscanPattern = hyperscanPattern; + hsPattern.category = "specific_accuracy"; + hsPattern.regexSource = "specific_acuracy_keywords_regex"; + hsPattern.groupName = groupName; + if (hsPattern.groupName.empty()) { + hsPattern.groupName = "specific_accuracy_match"; + } + hsPattern.isFastReg = (hsPattern.groupName.find("fast_reg") != std::string::npos); + hsPattern.isEvasion = (hsPattern.groupName.find("evasion") != std::string::npos); + + m_keywordHyperscanPatterns.push_back(hsPattern); + m_keywordAssertionFlags.push_back(flags); + } else { + incompatiblePatterns.push_back(pattern); + } + } + } + + // Process words_regex patterns + { + auto patterns = getCommonPatternsForCategory("keywords", "words_regex"); + for (const auto &pattern : patterns) { + AssertionFlags flags; + std::string groupName = extractGroupName(pattern); + std::string processedPattern = convertToHyperscanPattern(pattern); + std::string hyperscanPattern = processAssertions(groupName, processedPattern, flags); + + if (hyperscanPattern != pattern) { + dbgTrace(D_WAAP_HYPERSCAN) << pattern << " -> " << hyperscanPattern; + } + + if (isHyperscanCompatible(hyperscanPattern)) { + HyperscanPattern hsPattern; + hsPattern.originalPattern = pattern; + hsPattern.hyperscanPattern = hyperscanPattern; + hsPattern.category = "keywords"; + hsPattern.regexSource = "words_regex"; + hsPattern.groupName = groupName; + if (hsPattern.groupName.empty()) { + hsPattern.groupName = "keywords_match"; + } + hsPattern.isFastReg = (hsPattern.groupName.find("fast_reg") != std::string::npos); + hsPattern.isEvasion = (hsPattern.groupName.find("evasion") != std::string::npos); + + m_keywordHyperscanPatterns.push_back(hsPattern); + m_keywordAssertionFlags.push_back(flags); + } else { + incompatiblePatterns.push_back(pattern); + } + } + } + + // Process pattern_regex patterns + { + auto patterns = getCommonPatternsForCategory("patterns", "pattern_regex"); + for (const auto &pattern : patterns) { + AssertionFlags flags; + std::string groupName = extractGroupName(pattern); + std::string processedPattern = convertToHyperscanPattern(pattern); + std::string hyperscanPattern = processAssertions(groupName, processedPattern, flags); + + if (hyperscanPattern != pattern) { + dbgTrace(D_WAAP_HYPERSCAN) << pattern << " -> " << hyperscanPattern; + } + + if (isHyperscanCompatible(hyperscanPattern)) { + HyperscanPattern hsPattern; + hsPattern.originalPattern = pattern; + hsPattern.hyperscanPattern = hyperscanPattern; + hsPattern.category = "patterns"; + hsPattern.regexSource = "pattern_regex"; + hsPattern.groupName = groupName; + if (hsPattern.groupName.empty()) { + hsPattern.groupName = "patterns_match"; + } + hsPattern.isFastReg = (hsPattern.groupName.find("fast_reg") != std::string::npos); + hsPattern.isEvasion = (hsPattern.groupName.find("evasion") != std::string::npos); + + m_patternHyperscanPatterns.push_back(hsPattern); + m_patternAssertionFlags.push_back(flags); + } else { + incompatiblePatterns.push_back(pattern); + } + } + } + + dbgInfo(D_WAAP_HYPERSCAN) << "Preprocessed Hyperscan patterns: " + << "keywords=" << m_keywordHyperscanPatterns.size() + << ", patterns=" << m_patternHyperscanPatterns.size() + << ", incompatible=" << incompatiblePatterns.size(); + for (const auto &it : categoryCount) { + dbgInfo(D_WAAP_HYPERSCAN) << "Feature: " << it.first << ", Count: " << it.second; + } + + // Convert incompatible patterns to PmWordSet for traditional regex processing + if (m_regexPreconditions && !incompatiblePatterns.empty()) { + for (const auto &pattern : incompatiblePatterns) { + Waap::RegexPreconditions::WordIndex wordIndex = m_regexPreconditions->getWordByRegex(pattern); + if (wordIndex != Waap::RegexPreconditions::emptyWordIndex) { + m_incompatiblePatternsPmWordSet.insert(wordIndex); + } + } + dbgInfo(D_WAAP_HYPERSCAN) << "Created PmWordSet for " << m_incompatiblePatternsPmWordSet.size() + << " incompatible patterns (from " << incompatiblePatterns.size() << " total)"; + } +} + +picojson::value::object Signatures::loadSource(const std::string &waapDataFileName) { picojson::value doc; std::ifstream f(waapDataFileName); if (f.fail()) { dbgError(D_WAAP) << "Failed to open json data file '" << waapDataFileName << "'!"; - error = true; // flag an error + error = true; // flag an error return picojson::value::object(); } int length; f.seekg(0, std::ios::end); // go to the end length = f.tellg(); // report location (this is the length) - char* buffer = new char[length]; // allocate memory for a buffer of appropriate dimension + char *buffer = new char[length]; // allocate memory for a buffer of appropriate dimension f.seekg(0, std::ios::beg); // go back to the beginning f.read(buffer, length); // read the whole file into the buffer f.close(); @@ -258,11 +546,153 @@ picojson::value::object Signatures::loadSource(const std::string& waapDataFileNa ss >> doc; if (!picojson::get_last_error().empty()) { - dbgError(D_WAAP) << "WaapAssetState::loadSource('" << waapDataFileName << "') failed (parse error: '" << - picojson::get_last_error() << "')."; - error = true; // flag an error + dbgError(D_WAAP) << "WaapAssetState::loadSource('" << waapDataFileName << "') failed (parse error: '" + << picojson::get_last_error() << "')."; + error = true; // flag an error return picojson::value::object(); } return doc.get()["waap_signatures"].get(); } + +const std::vector &Signatures::getKeywordHyperscanPatterns() const +{ + return m_keywordHyperscanPatterns; +} + +const std::vector &Signatures::getPatternHyperscanPatterns() const +{ + return m_patternHyperscanPatterns; +} + +const std::vector &Signatures::getKeywordAssertionFlags() const +{ + return m_keywordAssertionFlags; +} + +const std::vector &Signatures::getPatternAssertionFlags() const +{ + return m_patternAssertionFlags; +} + +const Waap::RegexPreconditions::PmWordSet &Signatures::getIncompatiblePatternsPmWordSet() const +{ + return m_incompatiblePatternsPmWordSet; +} + +void Signatures::processRegexMatch(const std::string &groupName, const std::string &groupValue, std::string &word, + std::vector &keyword_matches, + Waap::Util::map_of_stringlists_t &found_patterns, bool longTextFound, + bool binaryDataFound) const +{ + std::string group = groupName; + + if (group == "") { + return; // skip unnamed group + } + + const std::string &value = groupValue; + dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: group name='" << group << "' value='" << value << "', word='" << word + << "':"; + + if (group.find("fast_reg") != std::string::npos) { + dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: found '*fast_reg*' in group name"; + if (group.find("evasion") != std::string::npos) { + dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: found both 'fast_reg' and 'evasion' in group name."; + word = "encoded_" + repr_uniq(value); + if (word == "encoded_") { + dbgTrace(D_WAAP_SAMPLE_SCAN) + << "checkRegex: empty word after repr_uniq: resetting word to 'character_encoding'" + " and group to 'evasion'."; + word = "character_encoding"; + } else if (Waap::Util::str_isalnum(word)) { + dbgTrace(D_WAAP_SAMPLE_SCAN) + << "checkRegex: isalnum word after repr_uniq: resetting group to 'evasion'."; + // If the found match is alphanumeric (we've seen strings like "640x480" match) + // we still should assume evasion but it doesn't need to include "fast_reg", + // which would cause unconditional report to stage2 and hit performance... + // This is why we remove the word "fast_reg" from the group name. + group = "evasion"; + } + + if (longTextFound) { + dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: longTextFound so resetting group name to 'longtext'"; + group = "longtext"; + } + } else { + word = group; + } + } + // In sequences detected as "longTextFound" or "longBinaryFound", do not add words in the + // "keyword_matches" list that: + // - starts with "encoded_" + // - or startswith("\") + // - or equal to "character_encoding" + if ((longTextFound || binaryDataFound) && + (word == "character_encoding" || word.substr(0, 1) == "\\" || word.substr(0, 8) == "encoded_")) { + // For now, do not skip + // TODO - check if skipping improves detection + dbgTrace(D_WAAP_SAMPLE_SCAN) << "longText/binaryData found with character_encoding"; + } else if (binaryDataFound && (isShortWord(word) || isShortHtmlTag(word) || + NGEN::Regex::regexMatch(__FILE__, __LINE__, group, binary_data_kw_filter))) { + dbgTrace(D_WAAP_SAMPLE_SCAN) << "Not adding group='" << group << "', word='" << word + << "' - due to binary data"; + return; + } else if ((std::find(keyword_matches.begin(), keyword_matches.end(), word) == keyword_matches.end())) { + // python: if (word not in current_matches): current_matches.append(word) + keyword_matches.push_back(word); + dbgTrace(D_WAAP_SAMPLE_SCAN) << "added keyword match for group='" << group << "', value='" << value + << "', word='" << word << "'"; + } + + // python: + // if group not in found_patterns: + // found_patterns[group]=[] + if (found_patterns.find(group) == found_patterns.end()) { + found_patterns[group] = std::vector(); + } + + // python: + // if value not in found_patterns[group]: + // found_patterns[group].append(value) + if (std::find(found_patterns[group].begin(), found_patterns[group].end(), value) == found_patterns[group].end()) { + found_patterns[group].push_back(value); + dbgTrace(D_WAAP_SAMPLE_SCAN) << "added pattern match for group='" << group << "', value='" << value + << "', word='" << word << "'"; + } +} + +bool Signatures::isHyperscanInitialized() const +{ + return m_hyperscanInitialized; +} + +void Signatures::setHyperscanInitialized(bool initialized) +{ + m_hyperscanInitialized = initialized; +} + +bool Signatures::shouldUseHyperscan(bool force) +{ + // This can be controlled by environment variable or configuration + static bool useHyperscan = false; + +#ifdef USE_HYPERSCAN + static bool checked = false; + if (!checked || force) { + // Check environment variable first + const char *env = getenv("WAAP_USE_HYPERSCAN"); + if (env) { + useHyperscan = (strcmp(env, "1") == 0 || strcasecmp(env, "true") == 0); + dbgDebug(D_WAAP_SAMPLE_SCAN) << "Hyperscan usage set by environment: " << useHyperscan; + } else { + // Default to false to maintain backward compatibility - Hyperscan is opt-in + useHyperscan = false; + dbgDebug(D_WAAP_SAMPLE_SCAN) << "Hyperscan usage default (disabled): " << useHyperscan; + } + checked = true; + } +#endif // USE_HYPERSCAN + + return useHyperscan; +} diff --git a/components/security_apps/waap/waap_clib/Signatures.h b/components/security_apps/waap/waap_clib/Signatures.h index ab7bffd..68daa54 100755 --- a/components/security_apps/waap/waap_clib/Signatures.h +++ b/components/security_apps/waap/waap_clib/Signatures.h @@ -16,19 +16,40 @@ #include "Waf2Regex.h" #include "picojson.h" +#include "flags.h" #include class Signatures { -private: - // json parsed sources (not really needed once data is loaded) - picojson::value::object sigsSource; - bool error; public: + // Enum for zero-length assertion flags + enum class AssertionFlag { + START_WORD_BEHIND = 0, // (?<=\w) + START_NON_WORD_BEHIND, // (?:=]|$) + PATH_TRAVERSAL_START, // (?; + + static std::string extractGroupName(const std::string &pattern); + + static std::string processAssertions(const std::string &groupName, + const std::string &pattern, + AssertionFlags &flags); + Signatures(const std::string& filepath); ~Signatures(); bool fail(); + picojson::value::object sigsSource; + bool error; std::shared_ptr m_regexPreconditions; // Regexes loaded from compiled signatures @@ -81,8 +102,113 @@ public: const boost::regex binary_data_kw_filter; const boost::regex wbxml_data_kw_filter; + // Pre-compiled Hyperscan patterns and metadata for performance optimization + struct HyperscanPattern { + std::string originalPattern; + std::string hyperscanPattern; + std::string groupName; + std::string category; + std::string regexSource; + bool isFastReg; + bool isEvasion; + + HyperscanPattern() : isFastReg(false), isEvasion(false) {} + }; + + // Pre-processed hyperscan patterns for each regex category + std::vector m_keywordHyperscanPatterns; + std::vector m_patternHyperscanPatterns; + + // Assertion flags corresponding to each pattern (same indices as above vectors) + std::vector m_keywordAssertionFlags; + std::vector m_patternAssertionFlags; + + // Getter methods for precompiled patterns + const std::vector& getKeywordHyperscanPatterns() const; + const std::vector& getPatternHyperscanPatterns() const; + + // Getter methods for assertion flags + const std::vector& getKeywordAssertionFlags() const; + const std::vector& getPatternAssertionFlags() const; + + // PmWordSet for incompatible patterns that need to use traditional regex scanning + Waap::RegexPreconditions::PmWordSet m_incompatiblePatternsPmWordSet; + + // Getter method for incompatible patterns PmWordSet + const Waap::RegexPreconditions::PmWordSet& getIncompatiblePatternsPmWordSet() const; + + // Hyperscan initialization state management + bool isHyperscanInitialized() const; + void setHyperscanInitialized(bool initialized); + + // Check if Hyperscan should be used (based on configuration) + static bool shouldUseHyperscan(bool force = false); + + void processRegexMatch( + const std::string &groupName, + const std::string &groupValue, + std::string &word, + std::vector &keyword_matches, + Waap::Util::map_of_stringlists_t &found_patterns, + bool longTextFound, + bool binaryDataFound + ) const; + private: picojson::value::object loadSource(const std::string& waapDataFileName); + void preprocessHyperscanPatterns(); + bool m_hyperscanInitialized; }; +inline std::string repr_uniq(const std::string & value) { + std::string result; + char hist[256]; + memset(&hist, 0, sizeof(hist)); + + for (std::string::const_iterator pC = value.begin(); pC != value.end(); ++pC) { + unsigned char ch = (unsigned char)(*pC); + + // Only take ASCII characters that are not alphanumeric, and each character only once + if (ch <= 127 && !isalnum(ch) && hist[ch] == 0) { + // Convert low ASCII characters to their C/C++ printable equivalent + // (used for easier viewing. Also, binary data causes issues with ElasticSearch) + switch (ch) { + case 0x07: result += "\\a"; break; + case 0x08: result += "\\b"; break; + case 0x09: result += "\\t"; break; + case 0x0A: result += "\\n"; break; + case 0x0B: result += "\\v"; break; + case 0x0C: result += "\\f"; break; + case 0x0D: result += "\\r"; break; + case 0x5C: result += "\\\\"; break; + case 0x27: result += "\\\'"; break; + case 0x22: result += "\\\""; break; + case 0x3F: result += "\\\?"; break; + default: { + if (ch >= 32) { + result += (char)ch; + } + else { + char buf[16]; + sprintf(buf, "\\" "x%02X", ch); + result += buf; + } + } + } + + hist[ch] = 1; + } + } + + return result; +} + +inline bool isShortWord(const std::string& word) { + return word.size() <= 2; +} + +inline bool isShortHtmlTag(const std::string& word) { + return !word.empty() && word.size() <= 4 && word[0] == '<' && word[word.size() - 1] == '>'; +} + #endif diff --git a/components/security_apps/waap/waap_clib/Telemetry.cc b/components/security_apps/waap/waap_clib/Telemetry.cc index dc6c3e5..ebe2d0d 100755 --- a/components/security_apps/waap/waap_clib/Telemetry.cc +++ b/components/security_apps/waap/waap_clib/Telemetry.cc @@ -225,7 +225,7 @@ void WaapAttackTypesMetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data) { if (data.blockType == FORCE_EXCEPTION) { - dbgInfo(D_WAAP) << "Data block type is FORCE_EXCEPTION, no update needed"; + dbgDebug(D_WAAP) << "Data block type is FORCE_EXCEPTION, no update needed"; return; } diff --git a/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc b/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc index 3947532..29e554b 100755 --- a/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc +++ b/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc @@ -20,30 +20,60 @@ USE_DEBUG_FLAG(D_WAAP_CONFIDENCE_CALCULATOR); TrustedSourcesConfidenceCalculator::TrustedSourcesConfidenceCalculator( std::string path, - const std::string& remotePath, - const std::string& assetId) + const std::string &remotePath, + const std::string &assetId) : SerializeToLocalAndRemoteSyncBase(std::chrono::minutes(120), SYNC_WAIT_TIME, path, (remotePath == "") ? remotePath : remotePath + "/Trust", assetId, - "TrustedSourcesConfidenceCalculator") + "TrustedSourcesConfidenceCalculator"), + m_persistent_state(), + m_incremental_logger(std::make_shared()) { restore(); } bool TrustedSourcesConfidenceCalculator::is_confident(Key key, Val value, size_t minSources) const { - auto sourceCtrItr = m_logger.find(key); - if (sourceCtrItr != m_logger.end()) + // Check persistent state first (accumulated data from previous syncs) + auto sourceCtrItr = m_persistent_state.find(key); + if (sourceCtrItr != m_persistent_state.end()) { auto sourceSetItr = sourceCtrItr->second.find(value); if (sourceSetItr != sourceCtrItr->second.end()) { + size_t persistent_sources = sourceSetItr->second.size(); + + if (persistent_sources >= minSources) { + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "The number of trusted sources for " << key + << " : " << value << " is " << persistent_sources << " (persistent only)"; + return true; + } + + // Also check incremental logger for recent data + size_t incremental_sources = 0; + if (m_incremental_logger) { + auto incr_ctr_itr = m_incremental_logger->find(key); + if (incr_ctr_itr != m_incremental_logger->end()) { + auto incr_set_itr = incr_ctr_itr->second.find(value); + if (incr_set_itr != incr_ctr_itr->second.end()) { + // Count unique sources (avoid double counting) + for (const auto &src : incr_set_itr->second) { + if (sourceSetItr->second.find(src) == sourceSetItr->second.end()) { + incremental_sources++; + } + } + } + } + } + + size_t total_sources = persistent_sources + incremental_sources; dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "The number of trusted sources for " << key - << " : " << value << " is " << sourceSetItr->second.size(); - return sourceSetItr->second.size() >= minSources; + << " : " << value << " is " << total_sources << " (persistent: " << persistent_sources + << ", incremental: " << incremental_sources << ")"; + return total_sources >= minSources; } else { @@ -52,6 +82,18 @@ bool TrustedSourcesConfidenceCalculator::is_confident(Key key, Val value, size_t } else { + // Check if data exists only in incremental logger + if (m_incremental_logger) { + auto incr_ctr_itr = m_incremental_logger->find(key); + if (incr_ctr_itr != m_incremental_logger->end()) { + auto incr_set_itr = incr_ctr_itr->second.find(value); + if (incr_set_itr != incr_ctr_itr->second.end()) { + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "The number of trusted sources for " << key + << " : " << value << " is " << incr_set_itr->second.size() << " (incremental only)"; + return incr_set_itr->second.size() >= minSources; + } + } + } dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to find the key(" << key << ")"; } return false; @@ -76,25 +118,36 @@ private: S2C_PARAM(TrustedSourcesConfidenceCalculator::KeyValSourceLogger, logger) }; -class TrsutedSourcesLogger : public RestGetFile +class TrustedSourcesLogger : public RestGetFile { public: - TrsutedSourcesLogger(const TrustedSourcesConfidenceCalculator::KeyValSourceLogger& _logger) - : logger(_logger) + TrustedSourcesLogger(std::shared_ptr _logger_ptr) + : logger_ptr(_logger_ptr) { - + logger = move(*logger_ptr); } private: + std::shared_ptr logger_ptr; C2S_PARAM(TrustedSourcesConfidenceCalculator::KeyValSourceLogger, logger); }; bool TrustedSourcesConfidenceCalculator::postData() { + if (m_incremental_logger->empty()) + { + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "No data to post, skipping"; + return true; // Nothing to post + } std::string url = getPostDataUrl(); dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the data to: " << url; + mergeIncrementalToPersistent(); + + TrustedSourcesLogger logger(m_incremental_logger); + + // Clear and reset incremental logger for next cycle + m_incremental_logger = std::make_shared(); - TrsutedSourcesLogger logger(m_logger); bool ok = sendNoReplyObjectWithRetry(logger, HTTPMethod::PUT, url); @@ -104,12 +157,12 @@ bool TrustedSourcesConfidenceCalculator::postData() return ok; } -void TrustedSourcesConfidenceCalculator::pullData(const std::vector& files) +void TrustedSourcesConfidenceCalculator::pullData(const std::vector &files) { dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Fetching the window data for trusted sources"; std::string url = getPostDataUrl(); std::string sentFile = url.erase(0, url.find_first_of('/') + 1); - for (auto file : files) + for (const auto &file : files) // Use const reference to avoid copying { if (file == sentFile) { @@ -135,19 +188,22 @@ void TrustedSourcesConfidenceCalculator::processData() { } - -void TrustedSourcesConfidenceCalculator::updateState(const std::vector& files) +void TrustedSourcesConfidenceCalculator::updateState(const std::vector &files) { - m_logger.clear(); pullProcessedData(files); } -void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector& files) { +Maybe TrustedSourcesConfidenceCalculator::getRemoteStateFilePath() const +{ + return m_remotePath + "/remote/data.data"; +} + +void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector &files) { dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Fetching the logger object for trusted sources"; bool pull_ok = false; - for (auto file: files) { + for (const auto &file : files) { // Use const reference GetTrustedFile getTrustFile; - bool res = sendObjectWithRetry(getTrustFile, + bool res = sendObject(getTrustFile, HTTPMethod::GET, getUri() + "/" + file); pull_ok |= res; @@ -165,43 +221,89 @@ void TrustedSourcesConfidenceCalculator::postProcessedData() std::string url = getUri() + "/" + m_remotePath + "/processed/data.data"; dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the processed data to: " << url; - TrsutedSourcesLogger logger(m_logger); + // Send persistent state as processed data + auto logger_ptr = std::make_shared(m_persistent_state); + TrustedSourcesLogger logger(logger_ptr); sendNoReplyObjectWithRetry(logger, HTTPMethod::PUT, url); } TrustedSourcesConfidenceCalculator::ValuesSet TrustedSourcesConfidenceCalculator::getConfidenceValues( - const Key& key, + const Key &key, size_t minSources) const { ValuesSet values; - auto sourceCtrItr = m_logger.find(key); - if (sourceCtrItr != m_logger.end()) + + // Check persistent state + auto sourceCtrItr = m_persistent_state.find(key); + if (sourceCtrItr != m_persistent_state.end()) { for (auto sourceSetItr : sourceCtrItr->second) { - if (sourceSetItr.second.size() >= minSources) + size_t persistent_sources = sourceSetItr.second.size(); + if (persistent_sources >= minSources) + { + values.insert(sourceSetItr.first); + continue; // No need to check incremental logger if we already have enough sources + } + + // Also check incremental logger for recent data + size_t incremental_sources = 0; + if (m_incremental_logger) { + auto incr_ctr_itr = m_incremental_logger->find(key); + if (incr_ctr_itr != m_incremental_logger->end()) { + auto incr_set_itr = incr_ctr_itr->second.find(sourceSetItr.first); + if (incr_set_itr != incr_ctr_itr->second.end()) { + // Count unique sources (avoid double counting) + for (const auto &src : incr_set_itr->second) { + if (sourceSetItr.second.find(src) == sourceSetItr.second.end()) { + incremental_sources++; + } + } + } + } + } + + if (persistent_sources + incremental_sources >= minSources) { values.insert(sourceSetItr.first); } } } - else - { + + // Also check values that exist only in incremental logger + if (m_incremental_logger) { + auto incr_ctr_itr = m_incremental_logger->find(key); + if (incr_ctr_itr != m_incremental_logger->end()) { + for (auto incr_set_itr : incr_ctr_itr->second) { + // Skip if already processed in persistent state + if (sourceCtrItr != m_persistent_state.end() && + sourceCtrItr->second.find(incr_set_itr.first) != sourceCtrItr->second.end()) { + continue; + } + + if (incr_set_itr.second.size() >= minSources) { + values.insert(incr_set_itr.first); + } + } + } + } + + if (values.empty()) { dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to find the key(" << key << ")"; } return values; } -void TrustedSourcesConfidenceCalculator::serialize(std::ostream& stream) +void TrustedSourcesConfidenceCalculator::serialize(std::ostream &stream) { cereal::JSONOutputArchive archive(stream); - archive(cereal::make_nvp("version", 2), cereal::make_nvp("logger", m_logger)); + archive(cereal::make_nvp("version", 3), cereal::make_nvp("persistent_state", m_persistent_state)); } -void TrustedSourcesConfidenceCalculator::deserialize(std::istream& stream) +void TrustedSourcesConfidenceCalculator::deserialize(std::istream &stream) { cereal::JSONInputArchive archive(stream); size_t version = 0; @@ -218,24 +320,31 @@ void TrustedSourcesConfidenceCalculator::deserialize(std::istream& stream) switch (version) { + case 3: + { + archive(cereal::make_nvp("persistent_state", m_persistent_state)); + break; + } case 2: { - archive(cereal::make_nvp("logger", m_logger)); + // Legacy: load into persistent state + archive(cereal::make_nvp("logger", m_persistent_state)); break; } case 1: { KeyValSourceLogger logger; archive(cereal::make_nvp("logger", logger)); - for (auto& log : logger) + for (auto &log : logger) { - m_logger[normalize_param(log.first)] = log.second; + m_persistent_state[normalize_param(log.first)] = log.second; } break; } case 0: { - archive(cereal::make_nvp("m_logger", m_logger)); + // Legacy: load into persistent state + archive(cereal::make_nvp("m_logger", m_persistent_state)); break; } default: @@ -244,17 +353,17 @@ void TrustedSourcesConfidenceCalculator::deserialize(std::istream& stream) } } -void TrustedSourcesConfidenceCalculator::mergeFromRemote(const KeyValSourceLogger& logs) +void TrustedSourcesConfidenceCalculator::mergeFromRemote(const KeyValSourceLogger &logs) { - for (auto& srcCounterItr : logs) + for (auto &srcCounterItr : logs) { - for (auto& sourcesItr : srcCounterItr.second) + for (auto &sourcesItr : srcCounterItr.second) { - for (auto& src : sourcesItr.second) + for (auto &src : sourcesItr.second) { dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Registering the source: " << src << " for the value: " << sourcesItr.first << " and the key: " << srcCounterItr.first; - m_logger[normalize_param(srcCounterItr.first)][sourcesItr.first].insert(src); + m_persistent_state[normalize_param(srcCounterItr.first)][sourcesItr.first].insert(src); } } } @@ -269,10 +378,28 @@ void TrustedSourcesConfidenceCalculator::log(Key key, Val value, Source source) << key << " from the source: " << source; - m_logger[key][value].insert(source); + (*m_incremental_logger)[key][value].insert(source); } void TrustedSourcesConfidenceCalculator::reset() { - m_logger.clear(); + m_persistent_state.clear(); + m_incremental_logger->clear(); +} + +void TrustedSourcesConfidenceCalculator::mergeIncrementalToPersistent() +{ + // Merge incremental data into persistent state (same logic as in postData but without network operations) + for (const auto &keyEntry : *m_incremental_logger) { + const std::string &key = keyEntry.first; + for (const auto &valueEntry : keyEntry.second) { + const std::string &value = valueEntry.first; + for (const std::string &source : valueEntry.second) { + m_persistent_state[key][value].insert(source); + } + } + } + + // Clear incremental logger after merging + m_incremental_logger->clear(); } diff --git a/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.h b/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.h index db31bb8..4a044af 100755 --- a/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.h +++ b/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.h @@ -19,6 +19,8 @@ USE_DEBUG_FLAG(D_WAAP); +// TODO PHASE3: remove inheritance from SerializeToLocalAndRemoteSyncBase + // this class is responsible for logging trusted sources indicators matches (without validation) class TrustedSourcesConfidenceCalculator : public SerializeToLocalAndRemoteSyncBase { @@ -31,27 +33,35 @@ public: typedef std::unordered_map SourcesCounter; typedef std::unordered_map KeyValSourceLogger; - TrustedSourcesConfidenceCalculator(std::string path, const std::string& remotePath, - const std::string& assetId); + TrustedSourcesConfidenceCalculator(std::string path, const std::string &remotePath, + const std::string &assetId); bool is_confident(Key key, Val value, size_t minSources) const; virtual bool postData(); - virtual void pullData(const std::vector& files); + virtual void pullData(const std::vector &files); virtual void processData(); virtual void postProcessedData(); - virtual void pullProcessedData(const std::vector& files); - virtual void updateState(const std::vector& files); + virtual void pullProcessedData(const std::vector &files); + virtual void updateState(const std::vector &files); + virtual Maybe getRemoteStateFilePath() const override; - ValuesSet getConfidenceValues(const Key& key, size_t minSources) const; + ValuesSet getConfidenceValues(const Key &key, size_t minSources) const; - virtual void serialize(std::ostream& stream); - virtual void deserialize(std::istream& stream); + virtual void serialize(std::ostream &stream); + virtual void deserialize(std::istream &stream); - void mergeFromRemote(const KeyValSourceLogger& logs); + void mergeFromRemote(const KeyValSourceLogger &logs); void log(Key key, Val value, Source source); void reset(); + + // Helper method for testing - merges incremental data to persistent state without network operations + void mergeIncrementalToPersistent(); private: - KeyValSourceLogger m_logger; + // Persistent state - accumulated trusted sources data across sync cycles + KeyValSourceLogger m_persistent_state; + + // Incremental logger - only new data since last sync (gets moved during postData) + std::shared_ptr m_incremental_logger; }; diff --git a/components/security_apps/waap/waap_clib/TuningDecision.cc b/components/security_apps/waap/waap_clib/TuningDecision.cc index 6ff5a1e..4a0a194 100755 --- a/components/security_apps/waap/waap_clib/TuningDecision.cc +++ b/components/security_apps/waap/waap_clib/TuningDecision.cc @@ -26,7 +26,7 @@ USE_DEBUG_FLAG(D_WAAP); TuningDecision::TuningDecision(const string& remotePath) : - m_remotePath(replaceAllCopy(remotePath + "/tuning", "//", "/")), + m_remotePath(replaceAllCopy(remotePath + "/tuning/decisions.data", "//", "/")), m_baseUri() { if (remotePath == "") @@ -35,7 +35,7 @@ TuningDecision::TuningDecision(const string& remotePath) } Singleton::Consume::by()->addRecurringRoutine( I_MainLoop::RoutineType::System, - chrono::minutes(10), + chrono::minutes(30), [&]() { updateDecisions(); }, "Get tuning updates" ); @@ -118,27 +118,16 @@ TuningDecisionType TuningDecision::convertDecisionType(string decisionTypeStr) void TuningDecision::updateDecisions() { TuningEvents tuningEvents; - RemoteFilesList tuningDecisionFiles; I_AgentDetails *agentDetails = Singleton::Consume::by(); if (agentDetails->getOrchestrationMode() != OrchestrationMode::ONLINE) { m_baseUri = "/api/"; } else { m_baseUri = "/storage/waap/"; } - dbgTrace(D_WAAP) << "URI prefix: " << m_baseUri; - bool isSuccessful = sendObject(tuningDecisionFiles, - HTTPMethod::GET, - m_baseUri + "?list-type=2&prefix=" + m_remotePath); - - if (!isSuccessful || tuningDecisionFiles.getFilesList().empty()) - { - dbgDebug(D_WAAP) << "Failed to get the list of files"; - return; - } if (!sendObject(tuningEvents, HTTPMethod::GET, - m_baseUri + tuningDecisionFiles.getFilesList()[0])) + m_baseUri + m_remotePath)) { return; } @@ -181,7 +170,7 @@ TuningDecision::getSharedStorageHost() char* sharedStorageHost = getenv(SHARED_STORAGE_HOST_ENV_NAME); if (sharedStorageHost != NULL) { shared_storage_host = string(sharedStorageHost); - dbgInfo(D_WAAP) << "shared storage host is set to " << shared_storage_host; + dbgDebug(D_WAAP) << "shared storage host is set to " << shared_storage_host; return shared_storage_host; } dbgWarning(D_WAAP) << "shared storage host is not set. using default: " << defaultSharedStorageHost; diff --git a/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc b/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc index 8c4372b..fa98b6b 100755 --- a/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc +++ b/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc @@ -29,8 +29,8 @@ USE_DEBUG_FLAG(D_WAAP); #define TYPES_FILTER_TRUST_PATH(dirPath) dirPath + "/9.data" TypeIndicatorFilter::TypeIndicatorFilter(I_WaapAssetState* pWaapAssetState, - const std::string& remotePath, - const std::string& assetId, + const std::string &remotePath, + const std::string &assetId, TuningDecision* tuning, size_t minSources, size_t minIntervals, @@ -78,7 +78,7 @@ bool TypeIndicatorFilter::shouldFilterKeyword(const std::string &key, const std: return false; } -void TypeIndicatorFilter::registerKeywords(const std::string& key, Waap::Keywords::KeywordsSet& keywords, +void TypeIndicatorFilter::registerKeywords(const std::string &key, Waap::Keywords::KeywordsSet &keywords, IWaf2Transaction* pTransaction) { (void)keywords; @@ -86,15 +86,21 @@ void TypeIndicatorFilter::registerKeywords(const std::string& key, Waap::Keyword registerKeywords(key, sample, pTransaction); } -void TypeIndicatorFilter::registerKeywords(const std::string& key, const std::string& sample, +void TypeIndicatorFilter::registerKeywords(const std::string &key, const std::string &sample, IWaf2Transaction* pTransaction) { std::set types = m_pWaapAssetState->getSampleType(sample); std::string source = pTransaction->getSourceIdentifier(); - std::string trusted_source = getTrustedSource(pTransaction); + auto trusted_source_maybe = getTrustedSource(pTransaction); + std::string trusted_source = trusted_source_maybe.ok() ? trusted_source_maybe.unpack() : ""; for (const std::string &type : types) { + if (type == "long_random_text") + { + // Skip long random text types, as they are not useful for filtering + continue; + } if (type == "local_file_path") { std::string location = IndicatorsFiltersManager::getLocationFromKey(key, pTransaction); @@ -142,7 +148,7 @@ void TypeIndicatorFilter::loadParams(std::shared_ptr TypeIndicatorFilter::getParamTypes(const std::string& canonicParam) const +std::set TypeIndicatorFilter::getParamTypes(const std::string &canonicParam) const { std::set types = m_confidence_calc.getConfidenceValues(canonicParam); if (m_policy != nullptr) @@ -153,3 +159,20 @@ std::set TypeIndicatorFilter::getParamTypes(const std::string& cano } return types; } + +bool TypeIndicatorFilter::shouldTrack(const std::string& key, const IWaf2Transaction* pTransaction) { + // Retrieve the sample from the transaction + std::string sample = pTransaction->getLastScanSample(); + + // Get the types associated with the sample + std::set sampleTypes = m_pWaapAssetState->getSampleType(sample); + + // Check if any of the sample types should be tracked + for (const auto& type : sampleTypes) { + if (m_confidence_calc.shouldTrackParameter(key, type)) { + return true; + } + } + + return false; +} diff --git a/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.h b/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.h index 1a8676b..3133b22 100755 --- a/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.h +++ b/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.h @@ -41,7 +41,7 @@ public: virtual void registerKeywords(const std::string& key, Waap::Keywords::KeywordsSet& keyword, IWaf2Transaction* pTransaction); - + bool shouldTrack(const std::string& key, const IWaf2Transaction* pTransaction); void registerKeywords(const std::string& key, const std::string& sample, IWaf2Transaction* pTransaction); void loadParams(std::shared_ptr pParams); diff --git a/components/security_apps/waap/waap_clib/UnifiedIndicatorsContainer.cc b/components/security_apps/waap/waap_clib/UnifiedIndicatorsContainer.cc new file mode 100644 index 0000000..48c6b3e --- /dev/null +++ b/components/security_apps/waap/waap_clib/UnifiedIndicatorsContainer.cc @@ -0,0 +1,339 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "UnifiedIndicatorsContainer.h" + +#include +#include +#include +#include + +using std::string; +using std::unordered_map; +using std::unordered_set; +using std::ostream; +using std::istream; + +// ------------------------------- +// Interning helpers +// ------------------------------- +const std::string* +UnifiedIndicatorsContainer::internValue(const std::string &value) +{ + auto it = valuePool.find(value); + if (it == valuePool.end()) it = valuePool.insert(value).first; + return &(*it); +} + +const std::string* +UnifiedIndicatorsContainer::internSource(const std::string &source) +{ + auto it = sourcesPool.find(source); + if (it == sourcesPool.end()) it = sourcesPool.insert(source).first; + return &(*it); +} + +// ------------------------------- +// Public API +// ------------------------------- +void +UnifiedIndicatorsContainer::addIndicator( + const std::string &key, + const std::string &value, + IndicatorType type, + const std::string &source) +{ + auto &filters = filtersDataPerKey[key]; + + const std::string *valPtr = internValue(value); + const std::string *srcPtr = internSource(source); + + FilterData &bucket = (type == IndicatorType::KEYWORD) + ? filters.getIndicators() + : filters.getTypes(); + + auto &srcSet = bucket[const_cast(valPtr)]; + srcSet.insert(const_cast(srcPtr)); + + // Update per-key total sources union + filters.getTotalSources().insert(const_cast(srcPtr)); +} + +void UnifiedIndicatorsContainer::addEntry(const Entry &entry) +{ + const std::string *srcPtr = internSource(entry.sourceId); + if (entry.isTrusted && srcPtr) { + trustedSources.insert(srcPtr); + } + for (const auto &val : entry.indicators) { + addIndicator(entry.key, val, IndicatorType::KEYWORD, entry.sourceId); + } + for (const auto &val : entry.types) { + addIndicator(entry.key, val, IndicatorType::TYPE, entry.sourceId); + } +} + +bool +UnifiedIndicatorsContainer::hasIndicator( + const std::string &key, + const std::string &value, + IndicatorType type) const +{ + auto keyIt = filtersDataPerKey.find(key); + if (keyIt == filtersDataPerKey.end()) return false; + + const Filters &filters = keyIt->second; + const FilterData &bucket = (type == IndicatorType::KEYWORD) + ? filters.getIndicators() + : filters.getTypes(); + + auto valIt = valuePool.find(value); + if (valIt == valuePool.end()) return false; + + auto it = bucket.find(const_cast(&(*valIt))); + return it != bucket.end(); +} + +std::unordered_set +UnifiedIndicatorsContainer::getSources( + const std::string &key, + const std::string &value, + IndicatorType type) const +{ + std::unordered_set out; + + auto keyIt = filtersDataPerKey.find(key); + if (keyIt == filtersDataPerKey.end()) return out; + + const Filters &filters = keyIt->second; + const FilterData &bucket = (type == IndicatorType::KEYWORD) + ? filters.getIndicators() + : filters.getTypes(); + + auto valIt = valuePool.find(value); + if (valIt == valuePool.end()) return out; + + auto it = bucket.find(const_cast(&(*valIt))); + if (it == bucket.end()) return out; + + for (auto p : it->second) if (p) out.insert(*p); + return out; +} + +size_t +UnifiedIndicatorsContainer::getIndicatorCount() const +{ + size_t count = 0; + for (const auto &k : filtersDataPerKey) { + count += k.second.getIndicators().size(); + count += k.second.getTypes().size(); + } + return count; +} + +size_t +UnifiedIndicatorsContainer::getKeyCount() const +{ + return filtersDataPerKey.size(); +} + +size_t +UnifiedIndicatorsContainer::getValuePoolSize() const +{ + return valuePool.size(); +} + +void +UnifiedIndicatorsContainer::clear() +{ + filtersDataPerKey.clear(); + valuePool.clear(); + sourcesPool.clear(); + trustedSources.clear(); +} + +// ------------------------------- +// Serialization +// ------------------------------- +void +UnifiedIndicatorsContainer::serialize(std::ostream &stream) const +{ + cereal::JSONOutputArchive ar(stream); + + // Write trustedSources as a named array under the root object (global trusted only) + ar.setNextName("trustedSources"); + ar.startNode(); + cereal::size_type n_trusted = static_cast(trustedSources.size()); + ar(cereal::make_size_tag(n_trusted)); + for (auto p : trustedSources) ar(p ? *p : std::string()); + ar.finishNode(); + + // logger: object of keys -> { totalSources: [...], indicators: {...}, types: {...} } + ar.setNextName("logger"); + ar.startNode(); + for (const auto &k : filtersDataPerKey) { + ar.setNextName(k.first.c_str()); + ar.startNode(); + + // totalSources section (union per key) + ar.setNextName("totalSources"); + ar.startNode(); + const auto &ts = k.second.getTotalSources(); + cereal::size_type ts_sz = static_cast(ts.size()); + ar(cereal::make_size_tag(ts_sz)); + for (auto p : ts) ar(p ? *p : std::string()); + ar.finishNode(); + + // indicators section + ar.setNextName("indicators"); + ar.startNode(); + for (const auto &kv : k.second.getIndicators()) { + const std::string *val = kv.first; + ar.setNextName(val ? val->c_str() : ""); + ar.startNode(); + cereal::size_type sz = static_cast(kv.second.size()); + ar(cereal::make_size_tag(sz)); + for (auto p : kv.second) ar(p ? *p : std::string()); + ar.finishNode(); // end value array + } + ar.finishNode(); // end indicators + + // types section + ar.setNextName("types"); + ar.startNode(); + for (const auto &kv : k.second.getTypes()) { + const std::string *val = kv.first; + ar.setNextName(val ? val->c_str() : ""); + ar.startNode(); + cereal::size_type sz = static_cast(kv.second.size()); + ar(cereal::make_size_tag(sz)); + for (auto p : kv.second) ar(p ? *p : std::string()); + ar.finishNode(); // end value array + } + ar.finishNode(); // end types + + ar.finishNode(); // end key object + } + ar.finishNode(); // end logger +} + +void +UnifiedIndicatorsContainer::deserialize(std::istream &stream) +{ + cereal::JSONInputArchive ar(stream); + clear(); + + // trustedSources (optional) as a named array + try { + ar.setNextName("trustedSources"); + ar.startNode(); + cereal::size_type n = 0; + ar(cereal::make_size_tag(n)); + for (cereal::size_type i = 0; i < n; ++i) { + std::string s; ar(s); + const std::string *p = internSource(s); + trustedSources.insert(p); + } + ar.finishNode(); + } catch (...) { + // Field may be absent + } + + // logger + try { + ar.setNextName("logger"); + ar.startNode(); + while (true) { + const auto node_name = ar.getNodeName(); + if (!node_name) break; + + std::string key = node_name; + ar.startNode(); // enter key object + + // totalSources (optional) + try { + ar.setNextName("totalSources"); + ar.startNode(); + cereal::size_type ts_sz = 0; + ar(cereal::make_size_tag(ts_sz)); + auto &ts = filtersDataPerKey[key].getTotalSources(); + for (cereal::size_type i = 0; i < ts_sz; ++i) { + std::string s; ar(s); + const std::string *p = internSource(s); + ts.insert(const_cast(p)); + } + ar.finishNode(); + } catch (...) { + // no totalSources + } + + // indicators + try { + ar.setNextName("indicators"); + ar.startNode(); + while (true) { + const auto val_name = ar.getNodeName(); + if (!val_name) break; + std::string value = val_name; + ar.startNode(); + cereal::size_type sz = 0; + ar(cereal::make_size_tag(sz)); + for (cereal::size_type i = 0; i < sz; ++i) { + std::string src; ar(src); + addIndicator(key, value, IndicatorType::KEYWORD, src); + } + ar.finishNode(); // end value array + } + ar.finishNode(); + } catch (...) { + // no indicators + } + + // types + try { + ar.setNextName("types"); + ar.startNode(); + while (true) { + const auto val_name = ar.getNodeName(); + if (!val_name) break; + std::string value = val_name; + ar.startNode(); + cereal::size_type sz = 0; + ar(cereal::make_size_tag(sz)); + for (cereal::size_type i = 0; i < sz; ++i) { + std::string src; ar(src); + addIndicator(key, value, IndicatorType::TYPE, src); + } + ar.finishNode(); // end value array + } + ar.finishNode(); + } catch (...) { + // no types + } + + ar.finishNode(); // finish key object + } + ar.finishNode(); // finish logger + } catch (...) { + // Field may be absent + } +} + +bool UnifiedIndicatorsContainer::isTrustedSource(const std::string &source) const { + // Linear check via interning: attempt to find an interned pointer matching source + // We maintain sourcesPool mapping actual std::string storage, so compare by value. + for (const auto &p : trustedSources) { + if (p && *p == source) return true; + } + return false; +} diff --git a/components/security_apps/waap/waap_clib/UnifiedIndicatorsContainer.h b/components/security_apps/waap/waap_clib/UnifiedIndicatorsContainer.h new file mode 100644 index 0000000..cef7535 --- /dev/null +++ b/components/security_apps/waap/waap_clib/UnifiedIndicatorsContainer.h @@ -0,0 +1,220 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "i_serialize.h" +// #include "custom_serialization.h" + +// Indicator type enumeration for type safety and compactness +enum class IndicatorType : uint8_t { + KEYWORD = 0, + TYPE = 1 +}; + +typedef std::unordered_set SourcesSet; +typedef std::unordered_map FilterData; + +// Proposed name for `Filters`: KeyLog (represents the per-key section under "logger") +// Keeping class name as Filters to minimize changes; can be renamed in a follow-up. +class Filters { +public: + Filters() = default; + ~Filters() = default; + + // Const overload for cereal serialization + template + void serialize(Archive& ar) const { + std::vector totalSourcesVec; + std::unordered_map> indicatorsMap, typesMap; + + for (auto p : totalSources) { + if (p) totalSourcesVec.push_back(*p); + } + for (const auto& kv : indicators) { + std::string key = kv.first ? *kv.first : std::string(); + std::vector sources; + for (auto p : kv.second) { + if (p) sources.push_back(*p); + } + indicatorsMap[key] = sources; + } + for (const auto& kv : types) { + std::string key = kv.first ? *kv.first : std::string(); + std::vector sources; + for (auto p : kv.second) { + if (p) sources.push_back(*p); + } + typesMap[key] = sources; + } + + ar( + cereal::make_nvp("totalSources", totalSourcesVec), + cereal::make_nvp("indicators", indicatorsMap), + cereal::make_nvp("types", typesMap) + ); + } + + // Accessors for container implementation + FilterData & getIndicators() { return indicators; } + FilterData & getTypes() { return types; } + const FilterData & getIndicators() const { return indicators; } + const FilterData & getTypes() const { return types; } + + // Per-key total sources (union of sources from indicators and types) + SourcesSet & getTotalSources() { return totalSources; } + const SourcesSet & getTotalSources() const { return totalSources; } + +private: + FilterData indicators; + FilterData types; + SourcesSet totalSources; +}; + +// Unified indicators container with string interning and memory optimization +class UnifiedIndicatorsContainer { +public: + // Batch entry input + struct Entry { + std::string key; + std::string sourceId; + bool isTrusted = false; + std::vector indicators; // values treated as KEYWORD + std::vector types; // values treated as TYPE + }; + void addEntry(const Entry& entry); + + // Check if an indicator exists + bool hasIndicator(const std::string& key, const std::string& value, IndicatorType type) const; + + // Get all sources for a specific indicator + std::unordered_set getSources(const std::string& key, + const std::string& value, + IndicatorType type) const; + + // Statistics and metrics + size_t getIndicatorCount() const; + size_t getKeyCount() const; + size_t getValuePoolSize() const; + // Returns true if the given source string is marked as trusted (appears in the global trustedSources set) + bool isTrustedSource(const std::string &source) const; + + // Container management + void clear(); + + // Serialization for cross-agent compatibility + // void serialize(std::ostream& stream) const; + template + void serialize(Archive& ar) const { + // trustedSources as array + std::vector trusted_srcs; + for (auto p : trustedSources) { + if (p) trusted_srcs.push_back(*p); + } + ar.setNextName("trustedSources"); + ar.startNode(); + cereal::size_type n_trusted = static_cast(trusted_srcs.size()); + ar(cereal::make_size_tag(n_trusted)); + for (const auto &s : trusted_srcs) ar(s); + ar.finishNode(); + + // logger: object of keys -> { totalSources: [...], indicators: {...}, types: {...} } + ar.setNextName("logger"); + ar.startNode(); + for (const auto &k : filtersDataPerKey) { + ar.setNextName(k.first.c_str()); + ar.startNode(); + + // totalSources section (union per key) + ar.setNextName("totalSources"); + ar.startNode(); + const auto &ts = k.second.getTotalSources(); + cereal::size_type ts_sz = static_cast(ts.size()); + ar(cereal::make_size_tag(ts_sz)); + for (auto p : ts) ar(p ? *p : std::string()); + ar.finishNode(); + + // indicators section + ar.setNextName("indicators"); + ar.startNode(); + for (const auto &kv : k.second.getIndicators()) { + const std::string *val = kv.first; + ar.setNextName(val ? val->c_str() : ""); + ar.startNode(); + cereal::size_type sz = static_cast(kv.second.size()); + ar(cereal::make_size_tag(sz)); + for (auto p : kv.second) ar(p ? *p : std::string()); + ar.finishNode(); // end value array + } + ar.finishNode(); // end indicators + + // types section + ar.setNextName("types"); + ar.startNode(); + for (const auto &kv : k.second.getTypes()) { + const std::string *val = kv.first; + ar.setNextName(val ? val->c_str() : ""); + ar.startNode(); + cereal::size_type sz = static_cast(kv.second.size()); + ar(cereal::make_size_tag(sz)); + for (auto p : kv.second) ar(p ? *p : std::string()); + ar.finishNode(); // end value array + } + ar.finishNode(); // end types + + ar.finishNode(); // end key object + } + ar.finishNode(); // end logger + } + void serialize(std::ostream &stream) const; + void deserialize(std::istream& stream); + +private: + // Single indicator add + void addIndicator(const std::string& key, const std::string& value, + IndicatorType type, const std::string& source); + + // String interning pool for values + std::unordered_set valuePool; + + // String interning pool for sources + std::unordered_set sourcesPool; + + // Main storage: key -> Filters + std::unordered_map filtersDataPerKey; + + // Global set of trusted sources + std::unordered_set trustedSources; + + // Helper methods + const std::string* internValue(const std::string& value); + const std::string* internSource(const std::string& source); +}; +// UnifiedIndicatorsLogPost for REST, compatible with cereal and messaging +class UnifiedIndicatorsLogPost : public RestGetFile { +public: + UnifiedIndicatorsLogPost(std::shared_ptr container_ptr) + { + unifiedIndicators = std::move(*container_ptr); + } + +private: + C2S_PARAM(UnifiedIndicatorsContainer, unifiedIndicators); +}; diff --git a/components/security_apps/waap/waap_clib/WaapAssetState.cc b/components/security_apps/waap/waap_clib/WaapAssetState.cc index f360b3f..e4a0075 100755 --- a/components/security_apps/waap/waap_clib/WaapAssetState.cc +++ b/components/security_apps/waap/waap_clib/WaapAssetState.cc @@ -13,6 +13,8 @@ // #define WAF2_LOGGING_ENABLE (does performance impact) #include "WaapAssetState.h" +#include "WaapHyperscanEngine.h" +#include "Signatures.h" #include "Waf2Regex.h" #include "debug.h" #include "Waf2Util.h" @@ -53,8 +55,7 @@ print_filtered(std::string title, const std::set& ignored_set, cons if (ignored_set.find(word) == ignored_set.end()) { // not in ignored_set dbgTrace(D_WAAP_SAMPLE_SCAN) << "+'" << word << "'"; - } - else { + } else { // in ignored set dbgTrace(D_WAAP_SAMPLE_SCAN) << "-'" << word << "'"; } @@ -102,11 +103,13 @@ static const boost::regex utf_evasion_for_dot_regex(utf_evasion_for_dot_helper); static const std::string sqli_comma_evasion_regex_helper = "\"\\s*,\\s*\""; static const boost::regex sqli_comma_evasion_regex(sqli_comma_evasion_regex_helper); -WaapAssetState::WaapAssetState(const std::shared_ptr& pWaapAssetState, - const std::string& waapDataFileName, - const std::string& id) : - WaapAssetState(pWaapAssetState->m_Signatures, +WaapAssetState::WaapAssetState( + const std::shared_ptr &pWaapAssetState, const std::string &waapDataFileName, const std::string &id +) : + WaapAssetState( + pWaapAssetState->m_Signatures, waapDataFileName, + pWaapAssetState->m_hyperscanEngine, pWaapAssetState->m_cleanValuesCache.capacity(), pWaapAssetState->m_suspiciousValuesCache.capacity(), pWaapAssetState->m_sampleTypeCache.capacity(), @@ -122,17 +125,19 @@ WaapAssetState::WaapAssetState(const std::shared_ptr& pWaapAsset clearRateLimitingState(); clearSecurityHeadersState(); clearErrorLimitingState(); - } - ); + }); } -WaapAssetState::WaapAssetState(std::shared_ptr signatures, - const std::string& waapDataFileName, +WaapAssetState::WaapAssetState( + std::shared_ptr signatures, + const std::string &waapDataFileName, + std::shared_ptr hyperscanEngine, size_t cleanValuesCacheCapacity, size_t suspiciousValuesCacheCapacity, size_t sampleTypeCacheCapacity, const std::string& assetId) : m_Signatures(signatures), + m_hyperscanEngine(hyperscanEngine), m_waapDataFileName(waapDataFileName), m_assetId(assetId), m_requestsMonitor(nullptr), @@ -536,194 +541,105 @@ WaapAssetState::WaapAssetState(std::shared_ptr signatures, return text; } - inline std::string repr_uniq(const std::string & value) { - std::string result; - char hist[256]; - memset(&hist, 0, sizeof(hist)); +void +WaapAssetState::checkRegex( + const SampleValue &sample, + const Regex & pattern, + std::vector& keyword_matches, + Waap::Util::map_of_stringlists_t & found_patterns, + bool longTextFound, + bool binaryDataFound) const +{ + dbgFlow(D_WAAP_SAMPLE_SCAN) << "checkRegex: line='" << sample.getSampleString() << "' patt='" << + pattern.getName() << "' longTextFound=" << longTextFound << " binaryDataFound=" << binaryDataFound; - for (std::string::const_iterator pC = value.begin(); pC != value.end(); ++pC) { - unsigned char ch = (unsigned char)(*pC); + std::vector matches; + sample.findMatches(pattern, matches); - // Only take ASCII characters that are not alphanumeric, and each character only once - if (ch <= 127 && !isalnum(ch) && hist[ch] == 0) { - // Convert low ASCII characters to their C/C++ printable equivalent - // (used for easier viewing. Also, binary data causes issues with ElasticSearch) - switch (ch) { - case 0x07: result += "\\a"; break; - case 0x08: result += "\\b"; break; - case 0x09: result += "\\t"; break; - case 0x0A: result += "\\n"; break; - case 0x0B: result += "\\v"; break; - case 0x0C: result += "\\f"; break; - case 0x0D: result += "\\r"; break; - case 0x5C: result += "\\\\"; break; - case 0x27: result += "\\\'"; break; - case 0x22: result += "\\\""; break; - case 0x3F: result += "\\\?"; break; - default: { - if (ch >= 32) { - result += (char)ch; - } - else { - char buf[16]; - sprintf(buf, "\\" "x%02X", ch); - result += buf; - } - } - } + for (std::vector::const_iterator pMatch = matches.begin(); pMatch != matches.end(); ++pMatch) { + const RegexMatch& match = *pMatch; - hist[ch] = 1; - } + // Get whole match (group[0], which is always present in any match) + std::string word = match.groups.front().value; + + dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: match='" << word << "':"; + + // Short words matched by regexes wont be detected in some cases like + // if enough binary data is present in the value. + if (binaryDataFound && word.size() <= 2) { + dbgTrace(D_WAAP_SAMPLE_SCAN) << "Will not add a short keyword '" << word << + "' because binaryData was found"; + continue; } - return result; + for (std::vector::const_iterator pGroup = match.groups.begin() + 1; + pGroup != match.groups.end(); ++pGroup) { + m_Signatures->processRegexMatch(pGroup->name, pGroup->value, word, keyword_matches, found_patterns, + longTextFound, binaryDataFound); + } } +} - static bool isShortWord(const std::string &word) - { - return word.size() <= 2; - } - - static bool isShortHtmlTag(const std::string &word) - { - return !word.empty() && word.size() <= 3 && word[0] == '<'; - } - - void - WaapAssetState::checkRegex( +void WaapAssetState::performStandardRegexChecks( const SampleValue &sample, - const Regex & pattern, - std::vector& keyword_matches, - Waap::Util::map_of_stringlists_t & found_patterns, + Waf2ScanResult &res, + bool longTextFound, + bool binaryDataFound, + bool includePatternRegex) const +{ + // Check if Hyperscan should be used based on configuration and availability + if (m_hyperscanEngine && m_hyperscanEngine->isInitialized()) { + // Use Hyperscan implementation for compatible patterns + dbgTrace(D_WAAP_SAMPLE_SCAN) << "using Hyperscan engine, includePatternRegex=" << includePatternRegex; + m_hyperscanEngine->scanSample(sample, res, longTextFound, binaryDataFound, true, includePatternRegex); + } + + // Always run regular regex checks for patterns incompatible with Hyperscan + // When Hyperscan is used, pmSet contains only the regex patterns that couldn't be compiled with Hyperscan + // When Hyperscan is not used, pmSet contains all patterns found by Aho-Corasick precondition scan + checkRegex(sample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, res.found_patterns, + longTextFound, binaryDataFound); + checkRegex(sample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, longTextFound, + binaryDataFound); + if (includePatternRegex) { + checkRegex(sample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, longTextFound, + binaryDataFound); + } +} + +void WaapAssetState::performResponseRegexChecks( + const SampleValue &sample, + Waf2ScanResult &res, bool longTextFound, bool binaryDataFound) const - { - dbgFlow(D_WAAP_SAMPLE_SCAN) << "checkRegex: line='" << sample.getSampleString() << "' patt='" << - pattern.getName() << "' longTextFound=" << longTextFound << " binaryDataFound=" << binaryDataFound; +{ + checkRegex(sample, m_Signatures->resp_body_words_regex_list, res.keyword_matches, res.found_patterns, + longTextFound, binaryDataFound); + checkRegex(sample, m_Signatures->resp_body_pattern_regex_list, res.keyword_matches, res.found_patterns, + longTextFound, binaryDataFound); +} - std::vector matches; - sample.findMatches(pattern, matches); - - for (std::vector::const_iterator pMatch = matches.begin(); pMatch != matches.end(); ++pMatch) { - const RegexMatch& match = *pMatch; - - // Get whole match (group[0], which is always present in any match) - std::string word = match.groups.front().value; - - dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: match='" << word << "':"; - - // Short words matched by regexes wont be detected in some cases like - // if enough binary data is present in the value. - if (binaryDataFound && word.size() <= 2) { - dbgTrace(D_WAAP_SAMPLE_SCAN) << "Will not add a short keyword '" << word << - "' because binaryData was found"; - continue; - } - - for (std::vector::const_iterator pGroup = match.groups.begin() + 1; - pGroup != match.groups.end(); - ++pGroup) { - std::string group = pGroup->name; - - if (group == "") { - continue; // skip unnamed group - } - - const std::string& value = pGroup->value; - dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: group name='" << group << - "' value='" << value << "', word='" << word << "':"; - - // python: - // if 'fast_reg' in group: - // if 'evasion' in group: - // word = repr(str(''.join(set(value)))) - // else: - // word =group - if (group.find("fast_reg") != std::string::npos) { - dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: found '*fast_reg*' in group name"; - if (group.find("evasion") != std::string::npos) { - dbgTrace(D_WAAP_SAMPLE_SCAN) << - "checkRegex: found both 'fast_reg' and 'evasion' in group name."; - - word = "encoded_" + repr_uniq(value); - - // Normally, the word added to the keyword_matches list contain the character sequence. - // However, sometimes (for example in case the sequence contained only unicode characters), - // after running repr_uniq() the word will remain empty string. In this case leave - // something meaningful/readable there. - if (word == "encoded_") { - dbgTrace(D_WAAP_SAMPLE_SCAN) << - "checkRegex: empty word after repr_uniq: resetting word to 'character_encoding'" - " and group to 'evasion'."; - word = "character_encoding"; - } - else if (Waap::Util::str_isalnum(word)) { - dbgTrace(D_WAAP_SAMPLE_SCAN) << - "checkRegex: isalnum word after repr_uniq: resetting group to 'evasion'."; - // If the found match is alphanumeric (we've seen strings like "640x480" match) - // we still should assume evasion but it doesn't need to include "fast_reg", - // which would cause unconditional report to stage2 and hit performance... - // This is why we remove the word "fast_reg" from the group name. - group = "evasion"; - } - - if (longTextFound) { - dbgTrace(D_WAAP_SAMPLE_SCAN) << - "checkRegex: longTextFound so resetting group name to 'longtext'"; - group = "longtext"; - } - } - else { - word = group; - } - } - - // In sequences detected as "longTextFound" or "longBinaryFound", do not add words in the - // "keyword_matches" list that: - // - starts with "encoded_" - // - or startswith("\") - // - or equal to "character_encoding" - if ((longTextFound || binaryDataFound) && - (word == "character_encoding" || word.substr(0, 1) == "\\" || word.substr(0, 8) == "encoded_")) { - dbgTrace(D_WAAP_SAMPLE_SCAN) << "Not adding keyword '" << word << "' because longtext was found"; - } - else if (binaryDataFound && (isShortWord(word) || isShortHtmlTag(word) || - NGEN::Regex::regexMatch(__FILE__, __LINE__, group, m_Signatures->binary_data_kw_filter))) { - dbgTrace(D_WAAP_SAMPLE_SCAN) << "Not adding group='" << group << "', word='" << word << - "' - due to binary data"; - continue; - } - else if ((std::find( - keyword_matches.begin(), - keyword_matches.end(), - word) == keyword_matches.end())) { - // python: if (word not in current_matches): current_matches.append(word) - keyword_matches.push_back(word); - } - - // python: - // if group not in found_patterns: - // found_patterns[group]=[] - if (found_patterns.find(group) == found_patterns.end()) { - found_patterns[group] = std::vector(); - } - - // python: - // if value not in found_patterns[group]: - // found_patterns[group].append(value) - if (std::find( - found_patterns[group].begin(), - found_patterns[group].end(), - value - ) == found_patterns[group].end()) { - found_patterns[group].push_back(value); - } - } - } +void WaapAssetState::performPatternRegexChecks( + const SampleValue &sample, + Waf2ScanResult &res, + bool longTextFound, + bool binaryDataFound) const +{ + // Check if Hyperscan should be used based on configuration and availability + if (m_hyperscanEngine && m_hyperscanEngine->isInitialized()) { + // Use Hyperscan implementation for compatible patterns + dbgTrace(D_WAAP_SAMPLE_SCAN) << "using Hyperscan engine"; + m_hyperscanEngine->scanSample(sample, res, longTextFound, binaryDataFound, false, true); } + // Always run regular regex checks for patterns incompatible with Hyperscan + // When Hyperscan is used, pmSet contains only the regex patterns that couldn't be compiled with Hyperscan + // When Hyperscan is not used, pmSet contains all patterns found by Aho-Corasick precondition scan + checkRegex(sample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, longTextFound, + binaryDataFound); +} - // TODO:: implement onload mechanism. - static bool isOnLoad = 0; +// TODO:: implement onload mechanism. +static bool isOnLoad = 0; static void calcRepeatAndWordsCount(const std::string &line, unsigned int &repeat, unsigned int &wordsCount) { @@ -747,13 +663,14 @@ static void calcRepeatAndWordsCount(const std::string &line, unsigned int &repea } static void calcRepetitionAndProbing(Waf2ScanResult &res, const std::set *ignored_keywords, - const std::string &line, bool &detectedRepetition, bool &detectedProbing, unsigned int &wordsCount) + const std::string &line, bool &detectedRepetition, bool &detectedProbing, + unsigned int &wordsCount) { unsigned int repeat; calcRepeatAndWordsCount(line, repeat, wordsCount); - if (!detectedRepetition && repeat>100) { // detect potential buffer overflow attacks - dbgTrace(D_WAAP_SAMPLE_SCAN) << "repetition detected: repeat=" << repeat; + if (!detectedRepetition && repeat > 100) { // detect potential buffer overflow attacks + dbgTrace(D_WAAP_SAMPLE_SCAN) << "repetition detected: repeat=" << repeat; detectedRepetition = true; res.keyword_matches.push_back("repetition"); } @@ -763,22 +680,21 @@ static void calcRepetitionAndProbing(Waf2ScanResult &res, const std::set wordsCount // res.keyword_matches.size() - && keywords_num != 0) - { - dbgTrace(D_WAAP_SAMPLE_SCAN) << "probing detected: keywords_num=" << keywords_num << - ", wordsCount=" << wordsCount; + && keywords_num != 0) { + dbgTrace(D_WAAP_SAMPLE_SCAN) << "probing detected: keywords_num=" << keywords_num + << ", wordsCount=" << wordsCount; detectedProbing = true; res.keyword_matches.push_back("probing"); } } -void -WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const +void WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const { // Test for long value without spaces (these can often cause false alarms) if (m_Signatures->nospaces_long_value_re.hasMatch(res.unescaped_line)) { @@ -790,12 +706,9 @@ WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const if (m_Signatures->ignored_for_nospace_long_value.find(word) != m_Signatures->ignored_for_nospace_long_value.end()) { dbgTrace(D_WAAP_SAMPLE_SCAN) - << "Removing keyword '" - << word - << "' because nospaces_long_value was found"; + << "Removing keyword '" << word << "' because nospaces_long_value was found"; it = v.erase(it); - } - else { + } else { ++it; } } @@ -813,8 +726,7 @@ WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const // collected inside Waf2ScanResult object. This function is used for debugging purposes. it should make deep-dive // into the object easier. -std::string -WaapAssetState::nicePrint(Waf2ScanResult &res) const +std::string WaapAssetState::nicePrint(Waf2ScanResult &res) const { std::string result = "Waf2ScanResult:\n"; result += "keyword_matches:\n"; @@ -907,18 +819,7 @@ WaapAssetState::apply( if (scanStage == "resp_body") { res.clear(); SampleValue sample(line, nullptr); - checkRegex(sample, - m_Signatures->resp_body_words_regex_list, - res.keyword_matches, - res.found_patterns, - false, - false); - checkRegex(sample, - m_Signatures->resp_body_pattern_regex_list, - res.keyword_matches, - res.found_patterns, - false, - false); + performResponseRegexChecks(sample, res, false, false); dbgTrace(D_WAAP_SAMPLE_SCAN) << "WaapAssetState::apply(): response body " << (res.keyword_matches.empty() ? "is not" : "is") << " suspicious"; return !res.keyword_matches.empty(); @@ -927,18 +828,7 @@ WaapAssetState::apply( if (scanStage == "resp_header") { res.clear(); SampleValue sample(line, nullptr); - checkRegex(sample, - m_Signatures->resp_body_words_regex_list, - res.keyword_matches, - res.found_patterns, - false, - false); - checkRegex(sample, - m_Signatures->resp_body_pattern_regex_list, - res.keyword_matches, - res.found_patterns, - false, - false); + performResponseRegexChecks(sample, res, false, false); dbgTrace(D_WAAP_SAMPLE_SCAN) << "WaapAssetState::apply(): response header " << (res.keyword_matches.empty() ? "is not" : "is") << " suspicious"; return !res.keyword_matches.empty(); @@ -992,9 +882,7 @@ WaapAssetState::apply( ignored_keywords = &m_Signatures->url_ignored_keywords; ignored_patterns = &m_Signatures->url_ignored_patterns; isUrlScanStage = true; - } - else if ((scanStage.size() == 6 && scanStage == "header") || - (scanStage.size() == 6 && scanStage == "cookie")) { + } else if ((scanStage.size() == 6 && scanStage == "header") || (scanStage.size() == 6 && scanStage == "cookie")) { if (m_Signatures->header_ignored_re.hasMatch(line)) { dbgTrace(D_WAAP_SAMPLE_SCAN) << "WaapAssetState::apply('" << line << "'): ignored for header."; @@ -1029,12 +917,12 @@ WaapAssetState::apply( //} #endif -// Only perform these checks under load + // Only perform these checks under load if (isOnLoad) { // Skip values that are too short if (line.length() < 3) { - dbgTrace(D_WAAP_SAMPLE_SCAN) << "WaapAssetState::apply('" << line << - "'): skipping: did not pass the length check."; + dbgTrace(D_WAAP_SAMPLE_SCAN) + << "WaapAssetState::apply('" << line << "'): skipping: did not pass the length check."; if (shouldCache) { m_cleanValuesCache.insert(CacheKey(line, scanStage, isBinaryData, splitType.ok() ? *splitType : "")); @@ -1068,8 +956,8 @@ WaapAssetState::apply( // Skip values that are longer than 10 characters, and match allowed_text_re regex if (line.length() > 10) { if (m_Signatures->allowed_text_re.hasMatch(line) > 0) { - dbgTrace(D_WAAP_SAMPLE_SCAN) << "WaapAssetState::apply('" << line << - "'): matched on allowed_text - ignoring."; + dbgTrace(D_WAAP_SAMPLE_SCAN) + << "WaapAssetState::apply('" << line << "'): matched on allowed_text - ignoring."; if (shouldCache) { m_cleanValuesCache.insert( @@ -1110,18 +998,10 @@ WaapAssetState::apply( // Scan unescaped_line with aho-corasick once, and reuse it in multiple calls to checkRegex below // This is done to improve performance of regex matching. - SampleValue unescapedLineSample(res.unescaped_line, m_Signatures->m_regexPreconditions); + SampleValue unescapedLineSample(res.unescaped_line, m_Signatures.get()); dbgTrace(D_WAAP_SAMPLE_SCAN) << "after doing second set of checkRegex calls..." << nicePrint(res); - checkRegex( - unescapedLineSample, - m_Signatures->specific_acuracy_keywords_regex, - res.keyword_matches, - res.found_patterns, - longTextFound, - binaryDataFound - ); - checkRegex(unescapedLineSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, longTextFound, - binaryDataFound); + + performStandardRegexChecks(unescapedLineSample, res, longTextFound, binaryDataFound, false); filterKeywordsDueToLongText(res); @@ -1132,7 +1012,7 @@ WaapAssetState::apply( // Calculate repetition and/or probing indicators if (!binaryDataFound) { calcRepetitionAndProbing(res, ignored_keywords, res.unescaped_line, detectedRepetition, detectedProbing, - wordsCount); + wordsCount); } // List of keywords to remove @@ -1160,13 +1040,8 @@ WaapAssetState::apply( unescaped = "|" + res.unescaped_line; } - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); filterKeywordsDueToLongText(res); @@ -1220,21 +1095,15 @@ WaapAssetState::apply( unescaped += res.unescaped_line.substr(pos); // add tail if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount == res.keyword_matches.size()) { // Remove the evasion keyword if no real evasion found keywordsToRemove.push_back("os_cmd_ev"); os_cmd_ev = false; - } - else if (!binaryDataFound) { + } else if (!binaryDataFound) { // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1256,21 +1125,15 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount == res.keyword_matches.size()) { // Remove the evasion keyword if no real evasion found keywordsToRemove.push_back("quotes_ev"); quotes_ev = false; - } - else if (!binaryDataFound) { + } else if (!binaryDataFound) { // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1291,13 +1154,8 @@ WaapAssetState::apply( unescaped = unescape(unescaped); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { @@ -1305,7 +1163,7 @@ WaapAssetState::apply( // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, - newWordsCount); + newWordsCount); // Take minimal words count because empirically it means evasion was probably succesfully decoded wordsCount = std::min(wordsCount, newWordsCount); } @@ -1321,13 +1179,8 @@ WaapAssetState::apply( unescaped = unescape(unescaped); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { @@ -1335,7 +1188,7 @@ WaapAssetState::apply( // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, - newWordsCount); + newWordsCount); // Take minimal words count because empirically it means evasion was probably succesfully decoded wordsCount = std::min(wordsCount, newWordsCount); } @@ -1351,21 +1204,15 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount == res.keyword_matches.size()) { // Remove the evasion keyword if no real evasion found keywordsToRemove.push_back("comment_ev"); comment_ev = false; - } - else if (!binaryDataFound) { + } else if (!binaryDataFound) { // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1390,21 +1237,15 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount == res.keyword_matches.size()) { // Remove the evasion keyword if no real evasion found keywordsToRemove.push_back("path_traversal"); path_traversal_ev = false; - } - else if (!binaryDataFound) { + } else if (!binaryDataFound) { // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1430,13 +1271,8 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { @@ -1446,7 +1282,7 @@ WaapAssetState::apply( // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, - newWordsCount); + newWordsCount); // Take minimal words count because empirically it means evasion was probably successfully decoded wordsCount = std::min(wordsCount, newWordsCount); } @@ -1464,21 +1300,15 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount == res.keyword_matches.size()) { // Remove the evasion keyword if no real evasion found keywordsToRemove.push_back("quotes_space_evasion"); quoutes_space_evasion = false; - } - else if (!binaryDataFound) { + } else if (!binaryDataFound) { // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1503,13 +1333,8 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { @@ -1517,7 +1342,7 @@ WaapAssetState::apply( // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, - newWordsCount); + newWordsCount); // Take minimal words count because empirically it means evasion was probably succesfully decoded wordsCount = std::min(wordsCount, newWordsCount); } @@ -1533,13 +1358,8 @@ WaapAssetState::apply( unescaped = unescape(unescaped); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } evasion_detected = true; @@ -1549,7 +1369,6 @@ WaapAssetState::apply( newWordsCount); // Take minimal words count because empirically it means evasion was probably succesfully decoded wordsCount = std::min(wordsCount, newWordsCount); - } if ((res.unescaped_line.find("0x") != std::string::npos) && evasion_hex_regex.hasMatch(res.unescaped_line)) { @@ -1563,13 +1382,8 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, false, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - false, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - false, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, false, binaryDataFound); } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { @@ -1587,11 +1401,10 @@ WaapAssetState::apply( // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, - newWordsCount); + newWordsCount); // Take minimal words count because empirically it means evasion was probably succesfully decoded wordsCount = std::min(wordsCount, newWordsCount); } - } if ((line.find("0x") != std::string::npos) && evasion_hex_regex.hasMatch(line)) { @@ -1604,13 +1417,8 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, false, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - false, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - false, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, false, binaryDataFound); } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { @@ -1646,13 +1454,8 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { @@ -1660,11 +1463,10 @@ WaapAssetState::apply( // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, - newWordsCount); + newWordsCount); // Take minimal words count because empirically it means evasion was probably succesfully decoded wordsCount = std::min(wordsCount, newWordsCount); } - } if ((line.find("%") != std::string::npos) && evasion_bad_hex_regex.hasMatch(line)) { @@ -1678,13 +1480,8 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { @@ -1710,13 +1507,8 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { @@ -1730,7 +1522,6 @@ WaapAssetState::apply( } } - if ((line.find("%") != std::string::npos) && utf_evasion_for_dot.hasMatch(line)) { dbgTrace(D_WAAP_EVASIONS) << "UTF evasion for dot found (%c0%*e) in raw line"; std::string unescaped = line; @@ -1742,13 +1533,8 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { @@ -1785,13 +1571,20 @@ WaapAssetState::apply( bool evBinaryDataFound = binaryDataFound; if (line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, evKeywordMatches, - res.found_patterns, evLongTextFound, evBinaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, evKeywordMatches, evFoundPatterns, - evLongTextFound, evBinaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, evRegexMatches, evFoundPatterns, - evLongTextFound, evBinaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + + // Create a temporary result structure to use with our helper function + Waf2ScanResult evRes; + evRes.keyword_matches = evKeywordMatches; + evRes.regex_matches = evRegexMatches; + evRes.found_patterns = evFoundPatterns; + + performStandardRegexChecks(unescapedSample, evRes, evLongTextFound, evBinaryDataFound); + + // Extract results back to individual variables + evKeywordMatches = evRes.keyword_matches; + evRegexMatches = evRes.regex_matches; + evFoundPatterns = evRes.found_patterns; } if (evKeywordMatches.size() != res.keyword_matches.size()) { @@ -1857,21 +1650,15 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount == res.keyword_matches.size()) { // Remove the evasion keyword if no real evasion found keywordsToRemove.push_back("evasion"); escape = false; - } - else if (!binaryDataFound) { + } else if (!binaryDataFound) { evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; @@ -1928,27 +1715,21 @@ WaapAssetState::apply( unescaped += ch; } - prev_ch = ch; + prev_ch = ch; } size_t kwCount = res.keyword_matches.size(); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount == res.keyword_matches.size()) { // Remove the evasion keyword if no real evasion found keywordsToRemove.push_back("evasion"); escape = false; - } - else if (!binaryDataFound) { + } else if (!binaryDataFound) { evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; @@ -1970,20 +1751,14 @@ WaapAssetState::apply( size_t kwCount = res.keyword_matches.size(); if (res.unescaped_line != unescaped) { - SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); - checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, - res.found_patterns, longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, - longTextFound, binaryDataFound); - checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + SampleValue unescapedSample(unescaped, m_Signatures.get()); + performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound); } if (kwCount == res.keyword_matches.size()) { // Remove the evasion keyword if no real evasion found keywordsToRemove.push_back("evasion"); - } - else if (!binaryDataFound) { + } else if (!binaryDataFound) { evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; @@ -2043,8 +1818,7 @@ WaapAssetState::apply( detectedProbing) { dbgTrace(D_WAAP_SAMPLE_SCAN) << "pre-suspicion found."; // apply regex signatures - checkRegex(unescapedLineSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, - longTextFound, binaryDataFound); + performPatternRegexChecks(unescapedLineSample, res, longTextFound, binaryDataFound); // python: // if len(regex_matches) and 'probing' not in keyword_matches: @@ -2069,13 +1843,11 @@ WaapAssetState::apply( // python: // if 'acuracy' in patterns and not url: - if (Waap::Util::find_in_map_of_stringlists_keys("acur", res.found_patterns)) - { + if (Waap::Util::find_in_map_of_stringlists_keys("acur", res.found_patterns)) { acuracy = 1; // search for "high_acuracy" or "hi_acur" signature names if (Waap::Util::find_in_map_of_stringlists_keys("high", res.found_patterns) || - Waap::Util::find_in_map_of_stringlists_keys("hi_acur", res.found_patterns)) - { + Waap::Util::find_in_map_of_stringlists_keys("hi_acur", res.found_patterns)) { acuracy = 2; } } @@ -2090,20 +1862,19 @@ WaapAssetState::apply( print_filtered("patterns", *ignored_patterns, res.regex_matches); print_found_patterns(res.found_patterns); - dbgTrace(D_WAAP_SAMPLE_SCAN) << "before decision: keywords(num=" << keywords_num << ", size=" << - res.keyword_matches.size() << "); regex(num=" << regex_num << ", size=" << res.regex_matches.size() << - "; acuracy=" << acuracy << "; score=" << score << "; forceReport=" << forceReport << "; probing=" << - detectedProbing << "; repetition=" << detectedRepetition << "; 'fast_reg' in found_patterns: " << - Waap::Util::find_in_map_of_stringlists_keys("fast_reg", res.found_patterns); + dbgTrace(D_WAAP_SAMPLE_SCAN) << "before decision: keywords(num=" << keywords_num + << ", size=" << res.keyword_matches.size() << "); regex(num=" << regex_num + << ", size=" << res.regex_matches.size() << "; acuracy=" << acuracy + << "; score=" << score << "; forceReport=" << forceReport + << "; probing=" << detectedProbing << "; repetition=" << detectedRepetition + << "; 'fast_reg' in found_patterns: " + << Waap::Util::find_in_map_of_stringlists_keys("fast_reg", res.found_patterns); #endif // python: // if (keywords_num+acuracy+2*regex_num)>2 or special_patten in patterns or // 'fast_reg' in patterns or 'probing' in keyword_matches or 'repetition' in keyword_matches: - if (score > 2 || - forceReport || - detectedProbing || - detectedRepetition || + if (score > 2 || forceReport || detectedProbing || detectedRepetition || Waap::Util::find_in_map_of_stringlists_keys("fast_reg", res.found_patterns)) { dbgTrace(D_WAAP_SAMPLE_SCAN) << "apply(): suspicion found (score=" << score << ")."; @@ -2134,7 +1905,8 @@ void WaapAssetState::updateScores() scoreBuilder.snap(); } -std::string WaapAssetState::getWaapDataFileName() const { +std::string WaapAssetState::getWaapDataFileName() const +{ return m_waapDataFileName; } @@ -2143,7 +1915,8 @@ std::map>& WaapAssetState::getFilterVerbos return m_filtered_keywords_verbose; } -std::string WaapAssetState::getWaapDataDir() const { +std::string WaapAssetState::getWaapDataDir() const +{ size_t lastSlash = m_waapDataFileName.find_last_of('/'); std::string sigsFilterDir = ((lastSlash == std::string::npos) ? m_waapDataFileName : m_waapDataFileName.substr(0, lastSlash)); @@ -2266,8 +2039,7 @@ std::set WaapAssetState::getSampleType(const std::string & sample) types.insert("binary_input"); } - if (types.empty()) - { + if (types.empty()) { types.insert("unknown"); m_sampleTypeCache.insert(sample); } @@ -2313,33 +2085,26 @@ void WaapAssetState::filterKeywordsByParameters( { dbgTrace(D_WAAP_ASSET_STATE) << "filter keywords based on parameter name: " << parameter_name; auto filter_parameters_itr = m_Signatures->filter_parameters.find(parameter_name); - if (filter_parameters_itr != m_Signatures->filter_parameters.end()) - { + if (filter_parameters_itr != m_Signatures->filter_parameters.end()) { dbgTrace(D_WAAP_ASSET_STATE) << "Found keywords to filter based on parameter name"; const auto &vec = filter_parameters_itr->second; - for (auto keyword_to_filter : vec) - { + for (auto keyword_to_filter : vec) { auto keywords_set_itr = keywords_set.find(keyword_to_filter); - if (keywords_set_itr != keywords_set.end()) - { + if (keywords_set_itr != keywords_set.end()) { dbgTrace(D_WAAP_ASSET_STATE) << "Filtering keyword: " << keyword_to_filter; keywords_set.erase(keyword_to_filter); } } - } - else - { + } else { dbgTrace(D_WAAP_ASSET_STATE) << "No keywords need to be filtered for this parameter"; } } void WaapAssetState::removeKeywords(Waap::Keywords::KeywordsSet &keywords_set) { - for (auto &keyword_to_remove : m_Signatures->remove_keywords_always) - { + for (auto &keyword_to_remove : m_Signatures->remove_keywords_always) { auto keyword_set_itr = keywords_set.find(keyword_to_remove); - if (keyword_set_itr != keywords_set.end()) - { + if (keyword_set_itr != keywords_set.end()) { dbgTrace(D_WAAP_ASSET_STATE) << "Removing keyword: " << keyword_to_remove << " from keyword set"; keywords_set.erase(keyword_set_itr); } @@ -2354,8 +2119,7 @@ void WaapAssetState::removeWBXMLKeywords(Waap::Keywords::KeywordsSet &keywords_s dbgTrace(D_WAAP_ASSET_STATE) << "Filtering keyword due to wbxml: '" << *it << "'"; filtered_keywords.push_back(*it); it = keywords_set.erase(it); - } - else { + } else { ++it; } } @@ -2407,4 +2171,3 @@ void WaapAssetState::clearSecurityHeadersState() { m_securityHeadersState.reset(); } - diff --git a/components/security_apps/waap/waap_clib/WaapAssetState.h b/components/security_apps/waap/waap_clib/WaapAssetState.h index 8a12f9f..8cddbdf 100755 --- a/components/security_apps/waap/waap_clib/WaapAssetState.h +++ b/components/security_apps/waap/waap_clib/WaapAssetState.h @@ -34,6 +34,7 @@ #include "ScanResult.h" #include "WaapSampleValue.h" #include "RequestsMonitor.h" +#include "WaapHyperscanEngine.h" enum space_stage {SPACE_SYNBOL, BR_SYMBOL, BN_SYMBOL, BRN_SEQUENCE, BNR_SEQUENCE, NO_SPACES}; @@ -43,18 +44,34 @@ class WaapAssetState : public boost::noncopyable, public I_WaapAssetState { private: //ugly but needed for build std::shared_ptr m_Signatures; + std::shared_ptr m_hyperscanEngine; std::string m_waapDataFileName; std::map> m_filtered_keywords_verbose; void checkRegex(const SampleValue &sample, const Regex & pattern, std::vector& keyword_matches, Waap::Util::map_of_stringlists_t & found_patterns, bool longTextFound, bool binaryDataFound) const; + // Helper function to perform the common pattern of checkRegex calls (2 or 3 calls based on parameter) + void performStandardRegexChecks(const SampleValue &sample, Waf2ScanResult &res, + bool longTextFound, bool binaryDataFound, bool includePatternRegex = true) const; + + // Helper function to perform response body/header pattern of two checkRegex calls + void performResponseRegexChecks(const SampleValue &sample, Waf2ScanResult &res, + bool longTextFound, bool binaryDataFound) const; + + // Helper function to perform pattern subset match only + void performPatternRegexChecks(const SampleValue &sample, Waf2ScanResult &res, + bool longTextFound, bool binaryDataFound) const; + void filterKeywordsDueToLongText(Waf2ScanResult &res) const; std::string nicePrint(Waf2ScanResult &res) const; public: - // Load and compile signatures from file - explicit WaapAssetState(std::shared_ptr signatures, const std::string& waapDataFileName, + + explicit WaapAssetState( + std::shared_ptr signatures, + const std::string &waapDataFileName, + std::shared_ptr hyperscanEngine = nullptr, size_t cleanCacheCapacity = SIGS_APPLY_CLEAN_CACHE_CAPACITY, size_t suspiciousCacheCapacity = SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY, size_t sampleTypeCacheCapacity = SIGS_SAMPLE_TYPE_CACHE_CAPACITY, @@ -111,7 +128,6 @@ public: void clearErrorLimitingState(); void clearSecurityHeadersState(); - std::shared_ptr& getRateLimitingState(); std::shared_ptr& getErrorLimitingState(); std::shared_ptr& getSecurityHeadersState(); diff --git a/components/security_apps/waap/waap_clib/WaapAssetStatesManager.cc b/components/security_apps/waap/waap_clib/WaapAssetStatesManager.cc index 48ea36d..a5141b9 100755 --- a/components/security_apps/waap/waap_clib/WaapAssetStatesManager.cc +++ b/components/security_apps/waap/waap_clib/WaapAssetStatesManager.cc @@ -55,6 +55,7 @@ void WaapAssetStatesManager::setAssetDirectoryPath(const std::string &assetDirec WaapAssetStatesManager::Impl::Impl() : m_signatures(nullptr), + m_hyperscanEngine(nullptr), m_basicWaapSigs(nullptr), m_AssetBasedWaapSigs(), m_assetDirectoryPath(BACKUP_DIRECTORY_PATH) @@ -67,18 +68,33 @@ WaapAssetStatesManager::Impl::~Impl() bool WaapAssetStatesManager::Impl::initBasicWaapSigs(const std::string& waapDataFileName) { - if (m_signatures && !m_signatures->fail() && m_basicWaapSigs) + if (m_signatures && !m_signatures->fail() && m_hyperscanEngine && m_basicWaapSigs) { // already initialized successfully. return true; } try { m_signatures = std::make_shared(waapDataFileName); + + // Initialize Hyperscan engine + m_hyperscanEngine = std::make_shared(); + if (!Signatures::shouldUseHyperscan()) { + dbgTrace(D_WAAP) << "Hyperscan disabled by configuration, will use PCRE2"; + } else if (!m_hyperscanEngine->initialize(m_signatures)) { + dbgTrace(D_WAAP) << "Hyperscan initialization failed, will use PCRE2"; + } else { + m_signatures->setHyperscanInitialized(true); + dbgTrace(D_WAAP) << "Hyperscan initialized successfully"; + } + m_basicWaapSigs = std::make_shared( m_signatures, waapDataFileName, + m_hyperscanEngine, SIGS_APPLY_CLEAN_CACHE_CAPACITY, - SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY); + SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY, + SIGS_SAMPLE_TYPE_CACHE_CAPACITY, + ""); } catch (std::runtime_error & e) { // TODO:: properly handle component initialization failure @@ -89,7 +105,7 @@ bool WaapAssetStatesManager::Impl::initBasicWaapSigs(const std::string& waapData return false; } - return m_signatures && !m_signatures->fail() && m_basicWaapSigs; + return m_signatures && !m_signatures->fail() && m_hyperscanEngine && m_basicWaapSigs; } std::shared_ptr WaapAssetStatesManager::Impl::getWaapAssetStateGlobal() diff --git a/components/security_apps/waap/waap_clib/WaapAssetStatesManager.h b/components/security_apps/waap/waap_clib/WaapAssetStatesManager.h index 8812433..267cbac 100755 --- a/components/security_apps/waap/waap_clib/WaapAssetStatesManager.h +++ b/components/security_apps/waap/waap_clib/WaapAssetStatesManager.h @@ -15,6 +15,7 @@ #include "singleton.h" #include "Signatures.h" +#include "WaapHyperscanEngine.h" #include #include #include @@ -65,6 +66,7 @@ private: const std::string& instanceId); std::shared_ptr m_signatures; + std::shared_ptr m_hyperscanEngine; std::shared_ptr m_basicWaapSigs; std::unordered_map> m_AssetBasedWaapSigs; std::string m_assetDirectoryPath; diff --git a/components/security_apps/waap/waap_clib/WaapConfigApi.cc b/components/security_apps/waap/waap_clib/WaapConfigApi.cc index 6b94e4e..d8a2b27 100755 --- a/components/security_apps/waap/waap_clib/WaapConfigApi.cc +++ b/components/security_apps/waap/waap_clib/WaapConfigApi.cc @@ -26,7 +26,7 @@ set WaapConfigAPI::assets_ids_aggregation{}; bool WaapConfigAPI::getWaapAPIConfig(WaapConfigAPI& ngenAPIConfig) { - auto &maybe_ngen_config = getConfiguration( + auto &maybe_ngen_config = getConfigurationWithCache( "WAAP", "WebAPISecurity" ); diff --git a/components/security_apps/waap/waap_clib/WaapConfigApi.h b/components/security_apps/waap/waap_clib/WaapConfigApi.h index 7c16d07..33b6bc5 100755 --- a/components/security_apps/waap/waap_clib/WaapConfigApi.h +++ b/components/security_apps/waap/waap_clib/WaapConfigApi.h @@ -34,6 +34,7 @@ public: static bool getWaapAPIConfig(WaapConfigAPI &ngenAPIConfig); static void notifyAssetsCount(); static void clearAssetsCount(); + virtual WaapConfigType getType() const override { return WaapConfigType::API; } private: diff --git a/components/security_apps/waap/waap_clib/WaapConfigApplication.cc b/components/security_apps/waap/waap_clib/WaapConfigApplication.cc index 2b5167d..6c4cd15 100755 --- a/components/security_apps/waap/waap_clib/WaapConfigApplication.cc +++ b/components/security_apps/waap/waap_clib/WaapConfigApplication.cc @@ -23,16 +23,19 @@ set WaapConfigApplication::assets_ids{}; set WaapConfigApplication::assets_ids_aggregation{}; bool WaapConfigApplication::getWaapSiteConfig(WaapConfigApplication& ngenSiteConfig) { - auto maybe_tenant_id = Singleton::Consume::by()->get( - "ActiveTenantId" - ); - auto maybe_profile_id = Singleton::Consume::by()->get( - "ActiveProfileId" - ); - string tenant_id = (maybe_tenant_id.ok() ? *maybe_tenant_id : "not found"); - string profile_id = (maybe_profile_id.ok() ? *maybe_profile_id : "not found"); - dbgTrace(D_WAAP) << "Tenant ID: " << tenant_id << ", Profile ID: " << profile_id; - auto &maybe_ngen_config = getConfiguration( + // Only get tenant/profile id if debug is active + if (isDebugRequired(TRACE, D_WAAP)) { + auto maybe_tenant_id = Singleton::Consume::by()->get( + "ActiveTenantId" + ); + auto maybe_profile_id = Singleton::Consume::by()->get( + "ActiveProfileId" + ); + string tenant_id = (maybe_tenant_id.ok() ? *maybe_tenant_id : "not found"); + string profile_id = (maybe_profile_id.ok() ? *maybe_profile_id : "not found"); + dbgTrace(D_WAAP) << "Tenant ID: " << tenant_id << ", Profile ID: " << profile_id; + } + auto &maybe_ngen_config = getConfigurationWithCache( "WAAP", "WebApplicationSecurity" ); diff --git a/components/security_apps/waap/waap_clib/WaapConfigApplication.h b/components/security_apps/waap/waap_clib/WaapConfigApplication.h index b22f3d4..314c572 100755 --- a/components/security_apps/waap/waap_clib/WaapConfigApplication.h +++ b/components/security_apps/waap/waap_clib/WaapConfigApplication.h @@ -39,6 +39,7 @@ public: static bool getWaapSiteConfig(WaapConfigApplication& ngenSiteConfig); static void notifyAssetsCount(); static void clearAssetsCount(); + virtual WaapConfigType getType() const override { return WaapConfigType::Application; } private: static const std::string s_PracticeSubType; diff --git a/components/security_apps/waap/waap_clib/WaapHyperscanEngine.cc b/components/security_apps/waap/waap_clib/WaapHyperscanEngine.cc new file mode 100644 index 0000000..b0b191b --- /dev/null +++ b/components/security_apps/waap/waap_clib/WaapHyperscanEngine.cc @@ -0,0 +1,767 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "WaapHyperscanEngine.h" +#include "Signatures.h" +#include "ScanResult.h" +#include "WaapSampleValue.h" +#include "Waf2Regex.h" +#include "Waf2Util.h" +#include "debug.h" +#include +#include +#include +#include +#include + +#ifdef USE_HYPERSCAN +#include "hs.h" +#endif + +USE_DEBUG_FLAG(D_WAAP_SAMPLE_SCAN); +USE_DEBUG_FLAG(D_WAAP_HYPERSCAN); + +#ifdef USE_HYPERSCAN +static const unsigned int HS_STANDARD_FLAGS = HS_FLAG_CASELESS | HS_FLAG_SOM_LEFTMOST; +#endif // USE_HYPERSCAN +static const bool matchOriginalPattern = true; +static const size_t maxRegexValidationMatches = 10; + +class WaapHyperscanEngine::Impl { +public: + struct PatternInfo { + std::string originalPattern; + std::string hyperscanPattern; + std::string groupName; + std::string category; // "keywords", "specific_accuracy", "patterns" + bool isFastReg; + bool isEvasion; + std::string regexSource; // "specific_acuracy_keywords_regex", "words_regex", "pattern_regex" + Signatures::AssertionFlags assertionFlags; // Zero-length assertion flags + std::unique_ptr originalRegex; // Precompiled original pattern for validation + + PatternInfo() : isFastReg(false), isEvasion(false) {} + }; + + struct MatchContext { + const WaapHyperscanEngine::Impl* engine; + const std::string* sampleText; + std::vector* keyword_matches; + std::vector* regex_matches; + Waap::Util::map_of_stringlists_t* found_patterns; + bool longTextFound; + bool binaryDataFound; + bool includePatternRegex; + bool includeKeywordRegex; + + // Per-signature tracking of last match end (pattern id => last end offset) + std::unordered_map lastMatchEndPerSignature; + }; + + Impl(); + ~Impl(); + + bool initialize(const std::shared_ptr& signatures); + void scanSample(const SampleValue& sample, + Waf2ScanResult& res, + bool longTextFound, + bool binaryDataFound, + bool includeKeywordRegex, + bool includePatternRegex) const; + bool isInitialized() const { return m_isInitialized; } + size_t getPatternCount() const { return m_patternInfos.size(); } + size_t getCompiledPatternCount() const { return m_compiledPatternCount; } + size_t getFailedPatternCount() const { return m_failedPatternCount; } + +private: +#ifdef USE_HYPERSCAN + hs_database_t* m_keywordDatabase; + hs_database_t* m_patternDatabase; + hs_scratch_t* m_keywordScratch; + hs_scratch_t* m_patternScratch; +#endif + + std::shared_ptr m_Signatures; + std::vector m_patternInfos; + bool m_isInitialized; + size_t m_compiledPatternCount; + size_t m_failedPatternCount; + + // Helper methods + bool compileHyperscanDatabases(const std::shared_ptr& signatures); + void loadPrecompiledPatterns(const std::shared_ptr& signatures); + + // use an ordered set to keep PCRE2-validated matches sorted and unique in input order + // LCOV_EXCL_START Reason: Trivial + struct Match { + size_t from; + size_t to; + Match(size_t from, size_t to) : from(from), to(to) {} + bool operator<(const Match& other) const { + return (from < other.from) || (from == other.from && to < other.to); + } + }; + + // LCOV_EXCL_STOP + + // Assertion validation helpers + bool validateAssertions(const std::string& sampleText, + size_t matchStart, + size_t matchEnd, + const PatternInfo& patternInfo, + std::set &foundMatches, + size_t maxMatches) const; + static bool isWordChar(char c); + static bool isNonWordSpecialChar(char c); + +#ifdef USE_HYPERSCAN + // Hyperscan callback function + static int onMatch(unsigned int id, + unsigned long long from, + unsigned long long to, + unsigned int flags, + void* context); + + void processMatch(unsigned int id, + unsigned long long from, + unsigned long long to, + MatchContext* context) const; + + void identifyFailingPatterns(const std::vector& patterns, + const std::vector& hsPatterns, + const std::string& logPrefix) { + for (size_t i = 0; i < patterns.size(); ++i) { + const char *single_pattern = patterns[i].c_str(); + unsigned int single_flag = HS_STANDARD_FLAGS; + unsigned int single_id = static_cast(i); + hs_database_t *test_db = nullptr; + hs_compile_error_t *single_err = nullptr; + hs_error_t single_result = hs_compile_multi(&single_pattern, + &single_flag, + &single_id, + 1, + HS_MODE_BLOCK, + nullptr, + &test_db, + &single_err); + if (single_result != HS_SUCCESS) { + std::string additional_info = ""; + if (i < hsPatterns.size()) { + const auto &hsPattern = hsPatterns[i]; + additional_info = " | Category: " + hsPattern.category + + " | Group: " + hsPattern.groupName + + " | Source: " + hsPattern.regexSource; + if (!hsPattern.originalPattern.empty() && + hsPattern.originalPattern != hsPattern.hyperscanPattern) { + additional_info += " | Original: '" + hsPattern.originalPattern + "'"; + } + } + dbgWarning(D_WAAP_HYPERSCAN) + << logPrefix << " [" << i << "]: '" << patterns[i] + << "' - Error: " << (single_err ? single_err->message : "unknown") << additional_info; + if (single_err) { + hs_free_compile_error(single_err); + single_err = nullptr; + } + } else { + if (test_db) { + hs_free_database(test_db); + test_db = nullptr; + } + } + if (single_err) { + hs_free_compile_error(single_err); + single_err = nullptr; + } + } + } +#endif // USE_HYPERSCAN +}; + +WaapHyperscanEngine::Impl::Impl() + : +#ifdef USE_HYPERSCAN + m_keywordDatabase(nullptr), m_patternDatabase(nullptr), m_keywordScratch(nullptr), m_patternScratch(nullptr), +#endif // USE_HYPERSCAN + m_isInitialized(false), m_compiledPatternCount(0), m_failedPatternCount(0) +{ +} + +WaapHyperscanEngine::Impl::~Impl() +{ +#ifdef USE_HYPERSCAN + if (m_keywordScratch) hs_free_scratch(m_keywordScratch); + if (m_patternScratch) hs_free_scratch(m_patternScratch); + if (m_keywordDatabase) hs_free_database(m_keywordDatabase); + if (m_patternDatabase) hs_free_database(m_patternDatabase); +#endif +} + +bool WaapHyperscanEngine::Impl::initialize(const std::shared_ptr &signatures) +{ + if (!signatures) { + dbgWarning(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine::initialize: null signatures"; + return false; + } + m_Signatures = signatures; + +#ifdef USE_HYPERSCAN + m_isInitialized = compileHyperscanDatabases(signatures); + if (m_isInitialized) { + dbgInfo(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine initialized successfully. " + << "Compiled: " << m_compiledPatternCount << ", Failed: " << m_failedPatternCount; + } else { + dbgWarning(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine initialization failed"; + } + return m_isInitialized; +#else + dbgInfo(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine: Hyperscan not available on this platform"; + return false; +#endif +} + +bool WaapHyperscanEngine::Impl::compileHyperscanDatabases(const std::shared_ptr &signatures) +{ +#ifdef USE_HYPERSCAN + // Load precompiled patterns from signatures instead of extracting at runtime + loadPrecompiledPatterns(signatures); + + std::vector keywordPatterns; + std::vector patternRegexPatterns; + + // Collect keyword patterns (from specific_accuracy and keywords categories) + auto keywordAssertionFlags = signatures->getKeywordAssertionFlags(); + for (size_t i = 0; i < signatures->getKeywordHyperscanPatterns().size(); ++i) { + const auto &hsPattern = signatures->getKeywordHyperscanPatterns()[i]; + keywordPatterns.push_back(hsPattern.hyperscanPattern); + + PatternInfo info; + info.originalPattern = hsPattern.originalPattern; + info.hyperscanPattern = hsPattern.hyperscanPattern; + info.category = hsPattern.category; + info.regexSource = hsPattern.regexSource; + info.groupName = hsPattern.groupName; + info.isFastReg = hsPattern.isFastReg; + info.isEvasion = hsPattern.isEvasion; + + // Set assertion flags if available + if (i < keywordAssertionFlags.size()) { + info.assertionFlags = keywordAssertionFlags[i]; + } + + // Compile original regex pattern for validation only when matchOriginal flag is set + if (!info.originalPattern.empty() && matchOriginalPattern) { + bool regexError = false; + info.originalRegex = std::make_unique( + info.originalPattern, regexError, "ValidationRegex_" + info.groupName + "_" + std::to_string(i)); + if (regexError) { + dbgWarning(D_WAAP_HYPERSCAN) + << "Failed to compile original regex for pattern: " << info.originalPattern + << " (group: " << info.groupName << ")"; + info.originalRegex.reset(); // Clear failed regex + } + } + + m_patternInfos.push_back(std::move(info)); + } + + // Collect pattern regex patterns (from patterns category) + auto patternAssertionFlags = signatures->getPatternAssertionFlags(); + for (size_t i = 0; i < signatures->getPatternHyperscanPatterns().size(); ++i) { + const auto &hsPattern = signatures->getPatternHyperscanPatterns()[i]; + patternRegexPatterns.push_back(hsPattern.hyperscanPattern); + + PatternInfo info; + info.originalPattern = hsPattern.originalPattern; + info.hyperscanPattern = hsPattern.hyperscanPattern; + info.category = hsPattern.category; + info.regexSource = hsPattern.regexSource; + info.groupName = hsPattern.groupName; + info.isFastReg = hsPattern.isFastReg; + info.isEvasion = hsPattern.isEvasion; + + // Set assertion flags if available + if (i < patternAssertionFlags.size()) { + info.assertionFlags = patternAssertionFlags[i]; + } + + // Compile original regex pattern for validation only when matchOriginal flag is set + if (!info.originalPattern.empty() && matchOriginalPattern) { + bool regexError = false; + size_t patternIndex = keywordPatterns.size() + i; // Offset by keyword patterns count + info.originalRegex = std::make_unique(info.originalPattern, regexError, + "ValidationRegex_" + info.groupName + "_" + std::to_string(patternIndex)); + if (regexError) { + dbgWarning(D_WAAP_HYPERSCAN) + << "Failed to compile original regex for pattern: " << info.originalPattern + << " (group: " << info.groupName << ")"; + info.originalRegex.reset(); // Clear failed regex + } + } + + m_patternInfos.push_back(std::move(info)); + } + + dbgInfo(D_WAAP_HYPERSCAN) << "Using precompiled patterns: " + << "keywords=" << keywordPatterns.size() + << ", patterns=" << patternRegexPatterns.size(); + + // Compile keyword database (specific_acuracy_keywords_regex + words_regex) + size_t total_ids = 0; + if (!keywordPatterns.empty()) { + std::vector c_patterns; + std::vector flags; + std::vector ids; + + for (size_t i = 0; i < keywordPatterns.size(); ++i) { + c_patterns.push_back(keywordPatterns[i].c_str()); + flags.push_back(HS_STANDARD_FLAGS); + ids.push_back(static_cast(total_ids++)); + } + + // Defensive checks before calling hs_compile_multi + if (c_patterns.size() != flags.size() || c_patterns.size() != ids.size()) { + dbgWarning(D_WAAP_HYPERSCAN) << "Pattern, flag, and id arrays are not the same size!"; + return false; + } + if (c_patterns.empty()) { + dbgWarning(D_WAAP_HYPERSCAN) << "No patterns to compile!"; + return false; + } + dbgInfo(D_WAAP_HYPERSCAN) << "Compiling " << c_patterns.size() + << " keyword patterns with hs_compile_multi. First pattern: '" + << keywordPatterns[0] << "'"; + + hs_compile_error_t *compile_err = nullptr; + hs_error_t result = + hs_compile_multi(c_patterns.data(), + flags.data(), + ids.data(), + static_cast(c_patterns.size()), + HS_MODE_BLOCK, + nullptr, + &m_keywordDatabase, + &compile_err); + + if (result != HS_SUCCESS) { + std::string error_msg = compile_err ? compile_err->message : "unknown error"; + dbgWarning(D_WAAP_HYPERSCAN) << "Failed to compile keyword database: " << error_msg; + + // Try to identify the specific failing pattern(s) + if (compile_err) { + dbgWarning(D_WAAP_HYPERSCAN) << "Attempting to identify failing keyword pattern(s)..."; + auto keywordHsPatterns = signatures->getKeywordHyperscanPatterns(); + std::vector keywordPatternInfos; + keywordPatternInfos.reserve(keywordHsPatterns.size()); + for (const auto& hsPattern : keywordHsPatterns) { + keywordPatternInfos.emplace_back(); + PatternInfo& info = keywordPatternInfos.back(); + info.originalPattern = hsPattern.originalPattern; + info.hyperscanPattern = hsPattern.hyperscanPattern; + info.category = hsPattern.category; + info.regexSource = hsPattern.regexSource; + info.groupName = hsPattern.groupName; + info.isFastReg = hsPattern.isFastReg; + info.isEvasion = hsPattern.isEvasion; + } + identifyFailingPatterns(keywordPatterns, keywordPatternInfos, "Failing keyword pattern"); + } + if (compile_err) { + hs_free_compile_error(compile_err); + compile_err = nullptr; + } + return false; + } + + if (hs_alloc_scratch(m_keywordDatabase, &m_keywordScratch) != HS_SUCCESS) { + dbgWarning(D_WAAP_HYPERSCAN) << "Failed to allocate keyword scratch space"; + return false; + } + + m_compiledPatternCount += keywordPatterns.size(); + } + + // Compile pattern database (pattern_regex) + if (!patternRegexPatterns.empty()) { + std::vector c_patterns; + std::vector flags; + std::vector ids; + + for (size_t i = 0; i < patternRegexPatterns.size(); ++i) { + c_patterns.push_back(patternRegexPatterns[i].c_str()); + flags.push_back(HS_STANDARD_FLAGS); + ids.push_back(static_cast(total_ids++)); + } + + // Defensive checks before calling hs_compile_multi + if (c_patterns.size() != flags.size() || c_patterns.size() != ids.size()) { + dbgWarning(D_WAAP_HYPERSCAN) + << "Pattern, flag, and id arrays are not the same size! (patternRegexPatterns)"; + return false; + } + if (c_patterns.empty()) { + dbgWarning(D_WAAP_HYPERSCAN) << "No pattern regex patterns to compile!"; + return false; + } + dbgInfo(D_WAAP_HYPERSCAN) << "Compiling " << c_patterns.size() + << " pattern regex patterns with hs_compile_multi. First pattern: '" + << patternRegexPatterns[0] << "'"; + + hs_compile_error_t *compile_err = nullptr; + hs_error_t result = + hs_compile_multi(c_patterns.data(), + flags.data(), + ids.data(), + static_cast(c_patterns.size()), + HS_MODE_BLOCK, + nullptr, + &m_patternDatabase, + &compile_err); + + if (result != HS_SUCCESS) { + std::string error_msg = compile_err ? compile_err->message : "unknown error"; + dbgWarning(D_WAAP_HYPERSCAN) << "Failed to compile pattern database: " << error_msg; + + // Try to identify the specific failing pattern(s) + if (compile_err) { + dbgWarning(D_WAAP_HYPERSCAN) << "Attempting to identify failing pattern regex pattern(s)..."; + auto patternHsPatterns = signatures->getPatternHyperscanPatterns(); + std::vector patternPatternInfos; + patternPatternInfos.reserve(patternHsPatterns.size()); + for (const auto& hsPattern : patternHsPatterns) { + patternPatternInfos.emplace_back(); + PatternInfo& info = patternPatternInfos.back(); + info.originalPattern = hsPattern.originalPattern; + info.hyperscanPattern = hsPattern.hyperscanPattern; + info.category = hsPattern.category; + info.regexSource = hsPattern.regexSource; + info.groupName = hsPattern.groupName; + info.isFastReg = hsPattern.isFastReg; + info.isEvasion = hsPattern.isEvasion; + } + identifyFailingPatterns(patternRegexPatterns, patternPatternInfos, "Failing pattern regex"); + } + if (compile_err) { + hs_free_compile_error(compile_err); + compile_err = nullptr; + } + return false; + } + + if (hs_alloc_scratch(m_patternDatabase, &m_patternScratch) != HS_SUCCESS) { + dbgWarning(D_WAAP_HYPERSCAN) << "Failed to allocate pattern scratch space"; + return false; + } + + m_compiledPatternCount += patternRegexPatterns.size(); + } + + return true; +#else // USE_HYPERSCAN + return false; +#endif // USE_HYPERSCAN +} + +void WaapHyperscanEngine::Impl::loadPrecompiledPatterns(const std::shared_ptr &signatures) +{ + // This method is called to initialize any additional pattern processing if needed + // For now, the patterns are directly accessed from the signatures object + dbgTrace(D_WAAP_HYPERSCAN) << "Loading precompiled patterns from Signatures"; + m_Signatures = signatures; +} + +#ifdef USE_HYPERSCAN +int WaapHyperscanEngine::Impl::onMatch(unsigned int id, + unsigned long long from, + unsigned long long to, + unsigned int flags, + void *context) +{ + MatchContext *ctx = static_cast(context); + ctx->engine->processMatch(id, from, to, ctx); + return 0; // Continue scanning +} + +void WaapHyperscanEngine::Impl::processMatch(unsigned int id, + unsigned long long from, + unsigned long long to, + MatchContext *context) const +{ + if (id >= m_patternInfos.size()) { + dbgWarning(D_WAAP_HYPERSCAN) << "Invalid pattern ID: " << id; + return; + } + + const PatternInfo &info = m_patternInfos[id]; + const std::string &sampleText = *context->sampleText; + size_t start = static_cast(from); + size_t end = static_cast(to); + + if (end > sampleText.length()) end = sampleText.length(); + if (start >= end) return; + + // skip overlaps for this pattern + size_t &lastEnd = context->lastMatchEndPerSignature[id]; + if (start < lastEnd) { + dbgTrace(D_WAAP_HYPERSCAN) << "Skipping overlapping match for pattern id=" << id << " start=" << start + << " lastEnd=" << lastEnd << ", match: '" << sampleText.substr(start, end - start) + << "'"; + return; + } + + std::set foundMatches; + if (!validateAssertions(sampleText, start, end, info, foundMatches, maxRegexValidationMatches)) return; + + for (const auto &match : foundMatches) { + std::string matchedText = sampleText.substr(match.from, match.to - match.from); + std::string word = matchedText; + + dbgTrace(D_WAAP_HYPERSCAN) << " match='" << word << "' id='" << id << "' group='" << info.groupName + << "' category=" << info.category; + + if (context->binaryDataFound && word.size() <= 2) { + dbgTrace(D_WAAP_HYPERSCAN) + << "Will not add a short keyword '" << word << "' because binaryData was found"; + continue; + } + + if (context->includeKeywordRegex && (info.category == "keywords" || info.category == "specific_accuracy")) { + m_Signatures->processRegexMatch(info.groupName, matchedText, word, *context->keyword_matches, + *context->found_patterns, context->longTextFound, + context->binaryDataFound); + } else if (context->includePatternRegex && info.category == "patterns") { + m_Signatures->processRegexMatch(info.groupName, matchedText, word, *context->regex_matches, + *context->found_patterns, context->longTextFound, + context->binaryDataFound); + } + lastEnd = std::max(lastEnd, match.to); + } +} +#endif // USE_HYPERSCAN + +void WaapHyperscanEngine::Impl::scanSample(const SampleValue &sample, Waf2ScanResult &res, bool longTextFound, + bool binaryDataFound, bool includeKeywordRegex, bool includePatternRegex) const +{ +#ifdef USE_HYPERSCAN + if (!m_isInitialized) { + dbgTrace(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine: not initialized, skipping scan"; + return; + } + + const std::string &sampleText = sample.getSampleString(); + + MatchContext context; + context.engine = this; + context.sampleText = &sampleText; + context.keyword_matches = &res.keyword_matches; + context.regex_matches = &res.regex_matches; + context.found_patterns = &res.found_patterns; + context.longTextFound = longTextFound; + context.binaryDataFound = binaryDataFound; + context.includePatternRegex = includePatternRegex; + context.includeKeywordRegex = includeKeywordRegex; + + context.lastMatchEndPerSignature.clear(); + dbgTrace(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine::scanSample: scanning '" << sampleText + << "' longTextFound=" << longTextFound << " binaryDataFound=" << binaryDataFound + << " includeKeywordRegex=" << includeKeywordRegex + << " includePatternRegex=" << includePatternRegex; + + if (includeKeywordRegex && m_keywordDatabase && m_keywordScratch) { + hs_error_t result = + hs_scan(m_keywordDatabase, sampleText.c_str(), static_cast(sampleText.length()), 0, + m_keywordScratch, onMatch, &context); + + if (result != HS_SUCCESS) { + dbgWarning(D_WAAP_HYPERSCAN) << "Keyword database scan failed: " << result; + } + } + + if (includePatternRegex && m_patternDatabase && m_patternScratch) { + hs_error_t result = + hs_scan(m_patternDatabase, sampleText.c_str(), static_cast(sampleText.length()), 0, + m_patternScratch, onMatch, &context); + + if (result != HS_SUCCESS) { + dbgWarning(D_WAAP_HYPERSCAN) << "Pattern database scan failed: " << result; + } + } + + dbgTrace(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine::scanSample: found " << res.keyword_matches.size() + << " keyword matches, " << res.regex_matches.size() << " regex matches"; +#else + dbgWarning(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine::scanSample called but Hyperscan not available"; +#endif +} + +bool WaapHyperscanEngine::Impl::validateAssertions(const std::string &sampleText, size_t matchStart, size_t matchEnd, + const PatternInfo &patternInfo, std::set &foundMatches, + size_t maxMatches) const +{ + foundMatches.clear(); + + // If we don't have an original regex compiled, fall back to the assertion flags validation + if (!patternInfo.originalRegex) { + dbgTrace(D_WAAP_HYPERSCAN) << "No original regex available for validation, " + << "falling back to assertion flags check"; + foundMatches.emplace(matchStart, matchEnd); + // If no assertion flags are set, the match is valid + if (patternInfo.assertionFlags.empty()) { + return true; + } + + if ( + patternInfo.assertionFlags.isSet(Signatures::AssertionFlag::END_NON_WORD_AHEAD) && + matchEnd < sampleText.length() && + isWordChar(sampleText[matchEnd])) { + // (?!\w) - requires NO word character after the match + return false; + } + + if (patternInfo.assertionFlags.isSet(Signatures::AssertionFlag::START_NON_WORD_BEHIND) && matchStart > 0 && + isWordChar(sampleText[matchStart - 1])) { + // (?= sampleText.length() || !isWordChar(sampleText[matchEnd]))) { + // (?=\w) - requires a word character after the match + return false; + } + + if (patternInfo.assertionFlags.isSet(Signatures::AssertionFlag::END_NON_WORD_SPECIAL)) { + // (?=[^\w?<>:=]|$) - requires a non-word character (excluding ?<>:=) or end of string after the match + if (matchEnd < sampleText.length()) { + char nextChar = sampleText[matchEnd]; + if (isWordChar(nextChar) || nextChar == '?' || nextChar == '<' || nextChar == '>' || nextChar == ':' || + nextChar == '=') { + return false; + } + } + // If we're at the end of string, this condition is satisfied + } + + return true; + } + + if (patternInfo.assertionFlags.isSet(Signatures::AssertionFlag::WILDCARD_EVASION)) { + // skip if the match does not contain either type of slash, and not a question mark + bool hasSlash = false; + bool hasQuestionMark = false; + + for (size_t i = matchStart; i < matchEnd && !(hasSlash && hasQuestionMark); ++i) { + if (sampleText[i] == '\\' || sampleText[i] == '/') { + hasSlash = true; + } + if (sampleText[i] == '?') { + hasQuestionMark = true; + } + } + dbgTrace(D_WAAP_HYPERSCAN) << "Testing for wildcard evasion: '" + << " hasSlash=" << hasSlash << " hasQuestionMark=" << hasQuestionMark; + if (!hasSlash || !hasQuestionMark) { + return false; + } + } + + // Use the original compiled regex to find matches within the specified range + std::vector matchRanges; + + // look behind to cover possible assertions, look ahead much further to cover lazy hyperscan match end + static const size_t lookbehind_range = 4, lookahead_range = 32; + size_t searchStart = (matchStart > lookbehind_range) ? (matchStart - lookbehind_range) : 0UL; + size_t searchEnd = ((matchEnd + lookahead_range) < matchEnd || (matchEnd + lookahead_range) > sampleText.length()) + ? sampleText.length() // overflow + : (matchEnd + lookahead_range); // within bounds + + std::vector regex_matches; + patternInfo.originalRegex->findMatchRanges(sampleText, regex_matches, maxMatches, searchStart, searchEnd); + + for (const auto &match : regex_matches) { + foundMatches.emplace(match.start, match.end); + if (isDebugRequired(TRACE, D_WAAP_HYPERSCAN)) { + dbgTrace(D_WAAP_HYPERSCAN) << "Match for: '" << patternInfo.originalPattern << "' matched in range [" + << match.start << "," << match.end << "] " + << "matched text: '" + << sampleText.substr(match.start, match.end - match.start) + << "'"; + } + } + + if (foundMatches.empty()) { + if (isDebugRequired(TRACE, D_WAAP_HYPERSCAN)) { + dbgTrace(D_WAAP_HYPERSCAN) << "No match for: '" << patternInfo.originalPattern + << "' did not match in range [" << matchStart << "," << matchEnd << "] " + << "matched text: '" << sampleText.substr(matchStart, matchEnd - matchStart) + << "'"; + } + return false; + } + return true; +} + +// LCOV_EXCL_START Reason: Not in use currently, but kept for future reference +bool WaapHyperscanEngine::Impl::isWordChar(char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +bool WaapHyperscanEngine::Impl::isNonWordSpecialChar(char c) +{ + return c == '?' || c == '<' || c == '>' || c == ':' || c == '='; +} +// LCOV_EXCL_STOP + +// WaapHyperscanEngine public interface - delegates to Impl +WaapHyperscanEngine::WaapHyperscanEngine() : pimpl(std::make_unique()) +{ +} + +WaapHyperscanEngine::~WaapHyperscanEngine() = default; + +bool WaapHyperscanEngine::initialize(const std::shared_ptr& signatures) +{ + return pimpl->initialize(signatures); +} + +void WaapHyperscanEngine::scanSample(const SampleValue& sample, Waf2ScanResult& res, bool longTextFound, + bool binaryDataFound, bool includeKeywordRegex, bool includePatternRegex) const +{ + pimpl->scanSample(sample, res, longTextFound, binaryDataFound, includeKeywordRegex, includePatternRegex); +} + +bool WaapHyperscanEngine::isInitialized() const +{ + return pimpl->isInitialized(); +} + +size_t WaapHyperscanEngine::getPatternCount() const +{ + return pimpl->getPatternCount(); +} + +size_t WaapHyperscanEngine::getCompiledPatternCount() const +{ + return pimpl->getCompiledPatternCount(); +} + +size_t WaapHyperscanEngine::getFailedPatternCount() const +{ + return pimpl->getFailedPatternCount(); +} diff --git a/components/security_apps/waap/waap_clib/WaapHyperscanEngine.h b/components/security_apps/waap/waap_clib/WaapHyperscanEngine.h new file mode 100644 index 0000000..7979a23 --- /dev/null +++ b/components/security_apps/waap/waap_clib/WaapHyperscanEngine.h @@ -0,0 +1,54 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __WAAP_HYPERSCAN_ENGINE_H__ +#define __WAAP_HYPERSCAN_ENGINE_H__ + +#include +#include +#include +#include +#include + +class Signatures; +class SampleValue; +struct Waf2ScanResult; + +class WaapHyperscanEngine { +public: + + WaapHyperscanEngine(); + ~WaapHyperscanEngine(); + + // Initialize with patterns from Signatures + bool initialize(const std::shared_ptr& signatures); + + // Main scanning function - same interface as performStandardRegexChecks + void scanSample(const SampleValue& sample, + Waf2ScanResult& res, + bool longTextFound, + bool binaryDataFound, + bool includeKeywordRegex, + bool includePatternRegex) const; + + // Check if the engine is ready to use + bool isInitialized() const; + + // Get statistics + size_t getPatternCount() const; + size_t getCompiledPatternCount() const; + size_t getFailedPatternCount() const; + +private: + class Impl; + std::unique_ptr pimpl; +}; + +#endif // __WAAP_HYPERSCAN_ENGINE_H__ diff --git a/components/security_apps/waap/waap_clib/WaapOverride.cc b/components/security_apps/waap/waap_clib/WaapOverride.cc index 7111c3d..610357f 100755 --- a/components/security_apps/waap/waap_clib/WaapOverride.cc +++ b/components/security_apps/waap/waap_clib/WaapOverride.cc @@ -103,7 +103,7 @@ const std::vector& ExceptionsByPractice::getExceptionsOfPractice(De case DecisionType::AUTONOMOUS_SECURITY_DECISION: return m_web_app_ids; default: - dbgError(D_WAAP) << + dbgDebug(D_WAAP) << "Can't find practice type for exceptions by practice: " << practiceType << ", return web app exceptions"; diff --git a/components/security_apps/waap/waap_clib/WaapRegexPreconditions.cc b/components/security_apps/waap/waap_clib/WaapRegexPreconditions.cc index f3b49f8..af05ba6 100755 --- a/components/security_apps/waap/waap_clib/WaapRegexPreconditions.cc +++ b/components/security_apps/waap/waap_clib/WaapRegexPreconditions.cc @@ -26,21 +26,24 @@ namespace Waap { // Register empty string work under known index registerWord(""); + std::string pm_list_name = "preconditions"; + std::string pm_keys_name = "precondition_keys"; + // The key should always be there unless data file is corrupted (but there's a unit test that tests exactly // that!) - if (jsObj.find("preconditions") == jsObj.end()) { + if (jsObj.find(pm_list_name) == jsObj.end()) { dbgError(D_WAAP_REGEX) << "Error loading regex preconditions (signatures data file corrupt?)..."; error = true; return; } - if (jsObj.find("precondition_keys") == jsObj.end()) { + if (jsObj.find(pm_keys_name) == jsObj.end()) { dbgError(D_WAAP_REGEX) << "Error loading regex precondition sets (signatures data file corrupt?)..."; error = true; return; } - auto preconditions = jsObj.at("preconditions").get(); + auto preconditions = jsObj.at(pm_list_name).get(); // Loop over pre-conditions (rules) and load them dbgTrace(D_WAAP_REGEX) << "Loading regex preconditions..."; @@ -142,7 +145,7 @@ namespace Waap { // Build full list of words to load into aho-corasick pattern matcher dbgTrace(D_WAAP_REGEX) << "Loading regex precondition_keys into Aho-Corasick pattern matcher..."; - auto preconditionKeys = jsObj.at("precondition_keys").get(); + auto preconditionKeys = jsObj.at(pm_keys_name).get(); std::set pmPatterns; for (const auto &preconditionKey : preconditionKeys) { @@ -167,18 +170,19 @@ namespace Waap { // Initialize the aho-corasick pattern matcher with the patterns Maybe pmHookStatus = m_pmHook.prepare(pmPatterns); - if (!pmHookStatus.ok()) { dbgError(D_WAAP_REGEX) << "Aho-Corasick engine failed to load!"; error = true; return; } - dbgTrace(D_WAAP_REGEX) << "Aho-Corasick engine loaded."; - dbgTrace(D_WAAP_REGEX) << "Aho-corasick pattern matching engine initialized!"; } + RegexPreconditions::~RegexPreconditions() { + // No Hyperscan resource management here; handled by HyperscanHook + } + bool Waap::RegexPreconditions::isNoRegexPattern(const std::string &pattern) const { return m_noRegexPatterns.find(pattern) != m_noRegexPatterns.end(); diff --git a/components/security_apps/waap/waap_clib/WaapRegexPreconditions.h b/components/security_apps/waap/waap_clib/WaapRegexPreconditions.h index e84f725..9ede28c 100755 --- a/components/security_apps/waap/waap_clib/WaapRegexPreconditions.h +++ b/components/security_apps/waap/waap_clib/WaapRegexPreconditions.h @@ -22,9 +22,13 @@ #include #include +// Forward declaration for test friend class +class TestableRegexPreconditions; + namespace Waap { class RegexPreconditions { + friend class TestableRegexPreconditions; // Allow test access to private methods public: typedef size_t WordIndex; static const WordIndex emptyWordIndex; // special word index used to index the "impossible" empty word @@ -37,8 +41,29 @@ namespace Waap { public: typedef std::unordered_set PmWordSet; + struct WordInfo { + WordIndex napostNapreWordIndex; + WordIndex napostWordIndex; + WordIndex napreWordIndex; + WordIndex baseWordIndex; + std::string wordStr; + bool noRegex; + + WordInfo() + : + napostNapreWordIndex(emptyWordIndex), + napostWordIndex(emptyWordIndex), + napreWordIndex(emptyWordIndex), + baseWordIndex(0), + wordStr(), + noRegex(false) + { + } + }; + // The constructor builds internal data from Json object. Once built - the object becomes read-only. RegexPreconditions(const picojson::value::object &jsObj, bool &error); + ~RegexPreconditions(); bool isNoRegexPattern(const std::string &pattern) const; const std::string &getWordStrByWordIndex(WordIndex wordIndex) const; Waap::RegexPreconditions::WordIndex getWordByRegex(const std::string &pattern) const; @@ -61,26 +86,6 @@ namespace Waap { // Aho-Corasick pattern matcher object PMHook m_pmHook; - struct WordInfo { - WordIndex napostNapreWordIndex; - WordIndex napostWordIndex; - WordIndex napreWordIndex; - WordIndex baseWordIndex; - std::string wordStr; - bool noRegex; - - WordInfo() - : - napostNapreWordIndex(emptyWordIndex), - napostWordIndex(emptyWordIndex), - napreWordIndex(emptyWordIndex), - baseWordIndex(0), - wordStr(), - noRegex(false) - { - } - }; - WordIndex registerWord(const std::string &wordStr); std::vector m_pmWordInfo; std::map m_wordStrToIndex; // TODO:: remove this into throwaway object, no need to keep diff --git a/components/security_apps/waap/waap_clib/WaapSampleValue.cc b/components/security_apps/waap/waap_clib/WaapSampleValue.cc index c6ac241..f7909a5 100644 --- a/components/security_apps/waap/waap_clib/WaapSampleValue.cc +++ b/components/security_apps/waap/waap_clib/WaapSampleValue.cc @@ -12,19 +12,31 @@ // limitations under the License. #include "WaapSampleValue.h" +#include "Signatures.h" +#include "debug.h" -SampleValue::SampleValue(const std::string &sample, - const std::shared_ptr ®exPreconditions) +USE_DEBUG_FLAG(D_WAAP_SAMPLE_SCAN); + +SampleValue::SampleValue(const std::string &sample, const Signatures* signatures) : m_sample(sample), - m_regexPreconditions(regexPreconditions), + m_signatures(signatures), m_pmWordSet() { - if (m_regexPreconditions) { - // Run aho-corasick and related rules once the sample value is known. - // The result pmWordSet is reused later for multiple calls to findMatches on the same sample. - regexPreconditions->pmScan( - Buffer(m_sample.data(), m_sample.size(), Buffer::MemoryType::STATIC), m_pmWordSet); + if (m_signatures) { + if (m_signatures->m_regexPreconditions) { + if (!m_signatures->isHyperscanInitialized()) { + // Run aho-corasick and related rules once the sample value is known. + // The result pmWordSet is reused later for multiple calls to findMatches on the same sample. + m_signatures->m_regexPreconditions->pmScan( + Buffer(m_sample.data(), m_sample.size(), Buffer::MemoryType::STATIC), m_pmWordSet); + } else { + // Add incompatible patterns from Signatures to the PmWordSet so they will be processed + // by traditional regex matching in Regex::findAllMatches + const auto& incompatiblePmWordSet = m_signatures->getIncompatiblePatternsPmWordSet(); + m_pmWordSet.insert(incompatiblePmWordSet.begin(), incompatiblePmWordSet.end()); + } + } } } @@ -38,5 +50,10 @@ void SampleValue::findMatches(const Regex &pattern, std::vector &matches) const { static const size_t maxMatchesPerSignature = 5; - pattern.findAllMatches(m_sample, matches, m_regexPreconditions ? &m_pmWordSet : nullptr, maxMatchesPerSignature); + pattern.findAllMatches( + m_sample, + matches, + (m_signatures && m_signatures->m_regexPreconditions) ? &m_pmWordSet : nullptr, + maxMatchesPerSignature + ); } diff --git a/components/security_apps/waap/waap_clib/WaapSampleValue.h b/components/security_apps/waap/waap_clib/WaapSampleValue.h index 022f533..757dfcc 100644 --- a/components/security_apps/waap/waap_clib/WaapSampleValue.h +++ b/components/security_apps/waap/waap_clib/WaapSampleValue.h @@ -21,16 +21,18 @@ #include "WaapRegexPreconditions.h" #include "buffer.h" +class Signatures; + class SampleValue { public: - SampleValue(const std::string &sample, const std::shared_ptr ®exPreconditions); + SampleValue(const std::string &sample, const Signatures* signatures); const std::string &getSampleString() const; void findMatches(const Regex &pattern, std::vector &matches) const; private: std::string m_sample; - const std::shared_ptr m_regexPreconditions; + const Signatures* m_signatures; Waap::RegexPreconditions::PmWordSet m_pmWordSet; }; diff --git a/components/security_apps/waap/waap_clib/WaapScanner.cc b/components/security_apps/waap/waap_clib/WaapScanner.cc index 0b50158..0894371 100755 --- a/components/security_apps/waap/waap_clib/WaapScanner.cc +++ b/components/security_apps/waap/waap_clib/WaapScanner.cc @@ -287,8 +287,11 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len m_transaction->getAssetState()->logParamHit(res, m_transaction); - std::set paramTypes = m_transaction->getAssetState()->m_filtersMngr->getParameterTypes( - IndicatorsFiltersManager::generateKey(res.location, res.param_name, m_transaction)); + std::set paramTypes; + if (m_transaction->getAssetState()->m_filtersMngr != nullptr) { + paramTypes = m_transaction->getAssetState()->m_filtersMngr->getParameterTypes( + IndicatorsFiltersManager::generateKey(res.location, res.param_name, m_transaction)); + } if (paramTypes.size() == 1 && paramTypes.find("local_file_path") != paramTypes.end()) { diff --git a/components/security_apps/waap/waap_clib/WaapTrigger.cc b/components/security_apps/waap/waap_clib/WaapTrigger.cc index a253ed4..55904a9 100755 --- a/components/security_apps/waap/waap_clib/WaapTrigger.cc +++ b/components/security_apps/waap/waap_clib/WaapTrigger.cc @@ -89,7 +89,7 @@ const std::vector& TriggersByPractice::getTriggersByPractice(Decisi case DecisionType::AUTONOMOUS_SECURITY_DECISION: return m_web_app_ids; default: - dbgError(D_WAAP) << + dbgDebug(D_WAAP) << "Can't find practice type for triggers by practice: " << practiceType << ", return web app triggers"; diff --git a/components/security_apps/waap/waap_clib/WaapTrigger.h b/components/security_apps/waap/waap_clib/WaapTrigger.h index 01cbf55..3639728 100755 --- a/components/security_apps/waap/waap_clib/WaapTrigger.h +++ b/components/security_apps/waap/waap_clib/WaapTrigger.h @@ -201,7 +201,7 @@ struct Policy { } catch (std::runtime_error &e) { ar.setNextName(nullptr); - dbgInfo(D_WAAP) << "Failed to load triggers per practice, error: " << e.what(); + dbgDebug(D_WAAP) << "Failed to load triggers per practice, error: " << e.what(); triggersByPractice = TriggersByPractice(); } try { @@ -211,7 +211,7 @@ struct Policy { } catch (std::runtime_error &e) { ar.setNextName(nullptr); - dbgInfo(D_WAAP) << "Failed to load web user response per practice, error: " << e.what(); + dbgDebug(D_WAAP) << "Failed to load web user response per practice, error: " << e.what(); responseByPractice = WebUserResponseByPractice(); } ar( diff --git a/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc b/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc index f89d9f5..21c63b8 100755 --- a/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc +++ b/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc @@ -246,36 +246,38 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val) } // Detect URLEncode value isUrlEncoded = checkUrlEncoded(cur_val.data(), cur_val.size()); - - textual.clear(); - textual.append("hasCharSlash = "); - textual +=(hasCharSlash ? "true" : "false"); - textual.append("\nhasCharColon = "); - textual +=(hasCharColon ? "true" : "false"); - textual.append("\nhasCharAmpersand = "); - textual +=(hasCharAmpersand ? "true" : "false"); - textual.append("\nhasCharEqual = "); - textual +=(hasCharEqual ? "true" : "false"); - textual.append("\nhasTwoCharsEqual = "); - textual +=(hasTwoCharsEqual ? "true" : "false"); - textual.append("\nhasCharSemicolon = "); - textual +=(hasCharSemicolon ? "true" : "false"); - textual.append("\nhasCharPipe = "); - textual +=(hasCharPipe ? "true" : "false"); - textual.append("\nisUTF16 = "); - textual +=(isUTF16 ? "true" : "false"); - textual.append("\ncanSplitSemicolon = "); - textual +=(canSplitSemicolon ? "true" : "false"); - textual.append("\ncanSplitPipe = "); - textual +=(canSplitPipe ? "true" : "false"); - textual.append("\nhasSpace = "); - textual +=(hasSpace ? "true" : "false"); - textual.append("\nisUrlEncoded = "); - textual +=(isUrlEncoded ? "true" : "false"); - textual.append("\nhasCharLess = "); - textual +=(hasCharLess ? "true" : "false"); - textual.append("\nhasDoubleQuote = "); - textual +=(hasDoubleQuote ? "true" : "false"); - textual.append("\nhasPercent = "); - textual +=(hasPercent ? "true" : "false"); + if (isDebugRequired(TRACE, D_WAAP)) + { + textual.clear(); + textual.append("hasCharSlash = "); + textual +=(hasCharSlash ? "true" : "false"); + textual.append("\nhasCharColon = "); + textual +=(hasCharColon ? "true" : "false"); + textual.append("\nhasCharAmpersand = "); + textual +=(hasCharAmpersand ? "true" : "false"); + textual.append("\nhasCharEqual = "); + textual +=(hasCharEqual ? "true" : "false"); + textual.append("\nhasTwoCharsEqual = "); + textual +=(hasTwoCharsEqual ? "true" : "false"); + textual.append("\nhasCharSemicolon = "); + textual +=(hasCharSemicolon ? "true" : "false"); + textual.append("\nhasCharPipe = "); + textual +=(hasCharPipe ? "true" : "false"); + textual.append("\nisUTF16 = "); + textual +=(isUTF16 ? "true" : "false"); + textual.append("\ncanSplitSemicolon = "); + textual +=(canSplitSemicolon ? "true" : "false"); + textual.append("\ncanSplitPipe = "); + textual +=(canSplitPipe ? "true" : "false"); + textual.append("\nhasSpace = "); + textual +=(hasSpace ? "true" : "false"); + textual.append("\nisUrlEncoded = "); + textual +=(isUrlEncoded ? "true" : "false"); + textual.append("\nhasCharLess = "); + textual +=(hasCharLess ? "true" : "false"); + textual.append("\nhasDoubleQuote = "); + textual +=(hasDoubleQuote ? "true" : "false"); + textual.append("\nhasPercent = "); + textual +=(hasPercent ? "true" : "false"); + } } diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.cc b/components/security_apps/waap/waap_clib/Waf2Engine.cc index 5370567..f1c07f7 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.cc +++ b/components/security_apps/waap/waap_clib/Waf2Engine.cc @@ -371,6 +371,7 @@ Waf2Transaction::Waf2Transaction(std::shared_ptr pWaapAssetState m_entry_time = chrono::duration_cast(timeGet->getMonotonicTime()); } + Waf2Transaction::~Waf2Transaction() { dbgTrace(D_WAAP) << "Waf2Transaction::~Waf2Transaction: deleting m_requestBodyParser"; delete m_requestBodyParser; @@ -606,6 +607,7 @@ void Waf2Transaction::set_method(const char* method) { bool Waf2Transaction::checkIsScanningRequired() { bool result = false; + if (WaapConfigAPI::getWaapAPIConfig(m_ngenAPIConfig)) { m_siteConfig = &m_ngenAPIConfig; auto rateLimitingPolicy = m_siteConfig ? m_siteConfig->get_RateLimitingPolicy() : NULL; @@ -639,7 +641,6 @@ bool Waf2Transaction::checkIsScanningRequired() result = true; } } - return result; } @@ -652,7 +653,7 @@ bool Waf2Transaction::setCurrentAssetContext() result |= checkIsScanningRequired(); if (!m_siteConfig) { - dbgWarning(D_WAAP) << "[transaction:" << this << "] " + dbgDebug(D_WAAP) << "[transaction:" << this << "] " "Failed to set sitePolicy for asset... using the original signatures"; return result; } @@ -1572,8 +1573,7 @@ Waf2Transaction::decideFinal( poolName; // decision of (either) API or Application module - bool shouldBlock = false; - + bool shouldBlock = false; // TODO:: base class for both, with common inteface WaapConfigAPI ngenAPIConfig; WaapConfigApplication ngenSiteConfig; @@ -1586,7 +1586,7 @@ Waf2Transaction::decideFinal( m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(sitePolicy); // User limits - shouldBlock = (getUserLimitVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP); + shouldBlock = (getUserLimitVerdict() == ServiceVerdict::TRAFFIC_VERDICT_DROP); } else if (WaapConfigApplication::getWaapSiteConfig(ngenSiteConfig)) { dbgTrace(D_WAAP) << "Waf2Transaction::decideFinal(): got relevant Application configuration from the I/S"; @@ -1608,7 +1608,7 @@ Waf2Transaction::decideFinal( shouldBlock |= m_csrfState.decide(m_methodStr, m_waapDecision, csrfPolicy); } // User limits - shouldBlock |= (getUserLimitVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP); + shouldBlock |= (getUserLimitVerdict() == ServiceVerdict::TRAFFIC_VERDICT_DROP); } if (mode == 2) { @@ -1641,7 +1641,11 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog, } waapLog << LogField("sourceIP", m_remote_addr); waapLog << LogField("httpSourceId", m_source_identifier); - waapLog << LogField("sourcePort", m_remote_port); + if (getProfileAgentSettingWithDefault(false, "agent.saasProfile.ignoreSourceIP")){ + dbgTrace(D_WAAP) << "ignoring remote port in nexus log"; + } else { + waapLog << LogField("sourcePort", m_remote_port); + } waapLog << LogField("httpHostName", m_hostStr); waapLog << LogField("httpMethod", m_methodStr); if (!m_siteConfig->get_AssetId().empty()) waapLog << LogField("assetId", m_siteConfig->get_AssetId()); @@ -1757,7 +1761,9 @@ Waf2Transaction::sendLog() dbgTrace(D_WAAP) << "send log got decision type: " << decision_type; auto final_decision = m_waapDecision.getDecision(decision_type); if (!final_decision) { + dbgTrace(D_WAAP) << "send log no decision found, using AUTONOMOUS_SECURITY_DECISION"; final_decision = m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION); + decision_type = AUTONOMOUS_SECURITY_DECISION; } std::string attackTypes = buildAttackTypes(); @@ -1853,6 +1859,14 @@ Waf2Transaction::sendLog() return; } auto triggerLog = maybeTriggerLog.unpack(); + + if(final_decision->shouldForceLog() && + triggerLog.shouldIgnoreExceptionLog(LogTriggerConf::SecurityType::ThreatPrevention)) { + // If we should ignore exception log, we need to handle it + dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: ignoring exception log"; + return; + } + bool send_extended_log = shouldSendExtendedLog(triggerLog, decision_type); shouldBlock |= m_waapDecision.getShouldBlockFromHighestPriorityDecision(); // Do not send Detect log if trigger disallows it @@ -1898,6 +1912,16 @@ Waf2Transaction::sendLog() return; } + ReportIS::Severity severity = + decision_type == USER_LIMITS_DECISION ? ReportIS::Severity::HIGH : ReportIS::Severity::CRITICAL; + if (final_decision->shouldForceLog()) { + if (logOverride == OVERRIDE_DROP) { + severity = ReportIS::Severity::MEDIUM; + } else if (logOverride == OVERRIDE_ACCEPT) { + severity = ReportIS::Severity::INFO; + } + } + switch (decision_type) { case USER_LIMITS_DECISION: { @@ -1923,7 +1947,7 @@ Waf2Transaction::sendLog() "Web Request", ReportIS::Audience::SECURITY, LogTriggerConf::SecurityType::ThreatPrevention, - Severity::HIGH, + severity, Priority::HIGH, shouldBlock); @@ -1942,7 +1966,7 @@ Waf2Transaction::sendLog() "API Request", ReportIS::Audience::SECURITY, LogTriggerConf::SecurityType::ThreatPrevention, - Severity::CRITICAL, + severity, Priority::HIGH, shouldBlock); @@ -1972,7 +1996,7 @@ Waf2Transaction::sendLog() "CSRF Protection", ReportIS::Audience::SECURITY, LogTriggerConf::SecurityType::ThreatPrevention, - Severity::CRITICAL, + severity, Priority::HIGH, shouldBlock); @@ -2277,18 +2301,27 @@ bool Waf2Transaction::decideResponse() } auto triggerLog = maybeTriggerLog.unpack(); auto env = Singleton::Consume::by(); - auto http_chunk_type = env->get("HTTP Chunk type"); + auto http_chunk_type = env->get("HTTP Chunk type"); bool should_send_extended_log = shouldSendExtendedLog(triggerLog) && http_chunk_type.ok(); if (should_send_extended_log && - *http_chunk_type == ngx_http_chunk_type_e::RESPONSE_CODE && + *http_chunk_type == AttachmentDataType::RESPONSE_CODE && !triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody) ) { + dbgTrace(D_WAAP) << "response body is not active. Disabling extended logging"; should_send_extended_log = false; } else if (should_send_extended_log && - *http_chunk_type == ngx_http_chunk_type_e::REQUEST_END && + *http_chunk_type == AttachmentDataType::REQUEST_END && !triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseCode) && !triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody) ) { + dbgTrace(D_WAAP) << "response code is not active. Disabling extended logging"; + should_send_extended_log = false; + } else if (should_send_extended_log && + *http_chunk_type == AttachmentDataType::RESPONSE_BODY && + triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody) && + m_response_body.length() >= MAX_RESPONSE_BODY_SIZE) + { + dbgTrace(D_WAAP) << "response body collected (" << m_response_body.length() << " bytes). Disabling extended logging"; should_send_extended_log = false; } @@ -2347,7 +2380,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) { // collect sourceip, sourceIdentifier, url exceptions_dict["sourceIP"].insert(m_remote_addr); exceptions_dict["sourceIdentifier"].insert(m_source_identifier); - exceptions_dict["url"].insert(getUriStr()); + exceptions_dict["url"].insert(m_uriPath); exceptions_dict["hostName"].insert(m_hostStr); exceptions_dict["method"].insert(m_methodStr); @@ -2359,15 +2392,21 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) { } bool isConfigExist = false; - if (WaapConfigAPI::getWaapAPIConfig(m_ngenAPIConfig)) { - dbgTrace(D_WAAP_OVERRIDE) << "waap api config found"; - m_siteConfig = &m_ngenAPIConfig; + if (m_siteConfig && + (m_siteConfig->getType() == WaapConfigType::API || m_siteConfig->getType() == WaapConfigType::Application)) { + dbgTrace(D_WAAP_OVERRIDE) << "waap config already exists"; isConfigExist = true; } else if (WaapConfigApplication::getWaapSiteConfig(m_ngenSiteConfig)) { dbgTrace(D_WAAP_OVERRIDE) << "waap web application config found"; m_siteConfig = &m_ngenSiteConfig; isConfigExist = true; } + else if (WaapConfigAPI::getWaapAPIConfig(m_ngenAPIConfig)) { + dbgTrace(D_WAAP_OVERRIDE) << "waap api config found"; + m_siteConfig = &m_ngenAPIConfig; + isConfigExist = true; + } + std::vector site_exceptions; if (isConfigExist) { dbgTrace(D_WAAP_OVERRIDE) << "config exists, get override policy"; @@ -2387,7 +2426,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) { behaviors.insert(params.begin(), params.end()); } } else { - auto exceptions = getConfiguration("rulebase", "exception"); + auto exceptions = getConfigurationWithCache("rulebase", "exception"); if (!exceptions.ok()) { dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error: " << exceptions.getErr(); return false; diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.h b/components/security_apps/waap/waap_clib/Waf2Engine.h index ebf0a5f..1c1ea31 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.h +++ b/components/security_apps/waap/waap_clib/Waf2Engine.h @@ -63,6 +63,8 @@ typedef void(*subtransaction_cb_t)(Waf2Transaction* subTransaction, void *ctx); #define OVERRIDE_ACCEPT "Accept" #define OVERRIDE_DROP "Drop" +#define NEXUS_PORT 99999 + class Waf2Transaction : public IWaf2Transaction, public TableOpaqueSerialize, @@ -134,7 +136,7 @@ public: virtual std::shared_ptr getAssetState(); virtual const std::string getLocation() const; - ngx_http_cp_verdict_e getUserLimitVerdict(); + ServiceVerdict getUserLimitVerdict(); const std::string getUserLimitVerdictStr() const; const std::string getViolatedUserLimitTypeStr() const; const std::string getCurrentWebUserResponse(); diff --git a/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc b/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc index 0cf658d..9869dfd 100755 --- a/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc +++ b/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc @@ -22,6 +22,8 @@ #include USE_DEBUG_FLAG(D_WAAP_ULIMITS); +USE_DEBUG_FLAG(D_WAAP); +USE_DEBUG_FLAG(D_WAAP_OVERRIDE); #define LOW_REPUTATION_THRESHOLD 4 #define NORMAL_REPUTATION_THRESHOLD 6 @@ -363,7 +365,7 @@ void Waf2Transaction::sendAutonomousSecurityLog( const ReportIS::Priority priority = Waap::Util::computePriorityFromThreatLevel(autonomousSecurityDecision->getThreatLevel()); - auto maybeLogTriggerConf = getConfiguration("rulebase", "log"); + auto maybeLogTriggerConf = getConfigurationWithCache("rulebase", "log"); LogGenWrapper logGenWrapper( maybeLogTriggerConf, "Web Request", @@ -445,12 +447,12 @@ void Waf2Transaction::createUserLimitsState() } } -ngx_http_cp_verdict_e +ServiceVerdict Waf2Transaction::getUserLimitVerdict() { if (!isUserLimitReached()) { // Either limit not reached or attack mitigation mode is DISABLED - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + return ServiceVerdict::TRAFFIC_VERDICT_INSPECT; } std::string msg; @@ -460,7 +462,7 @@ Waf2Transaction::getUserLimitVerdict() std::string reason; reason = " reason: " + getViolatedUserLimitTypeStr(); - ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + ServiceVerdict verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT; const AttackMitigationMode mode = WaapConfigBase::get_WebAttackMitigationMode(*m_siteConfig); auto decision = m_waapDecision.getDecision(USER_LIMITS_DECISION); if (mode == AttackMitigationMode::LEARNING) { @@ -469,19 +471,19 @@ Waf2Transaction::getUserLimitVerdict() // detect mode and no active drop exception decision->setBlock(false); if (isIllegalMethodViolation()) { - dbgInfo(D_WAAP_ULIMITS) << msg << "INSPECT" << reason << " in detect mode"; - verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + dbgDebug(D_WAAP_ULIMITS) << msg << "INSPECT" << reason << " in detect mode"; + verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT; } else { - dbgInfo(D_WAAP_ULIMITS) << msg << "PASS" << reason; - verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + dbgDebug(D_WAAP_ULIMITS) << msg << "PASS" << reason; + verdict = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; } } else { // detect mode and active drop exception decision->setBlock(true); decision->setForceBlock(true); - dbgInfo(D_WAAP_ULIMITS) << msg << "Override Block" << reason; - verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + dbgDebug(D_WAAP_ULIMITS) << msg << "Override Block" << reason; + verdict = ServiceVerdict::TRAFFIC_VERDICT_DROP; } } else if (mode == AttackMitigationMode::PREVENT) { @@ -489,14 +491,14 @@ Waf2Transaction::getUserLimitVerdict() if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException) { // prevent mode and no active accept exception decision->setBlock(true); - dbgInfo(D_WAAP_ULIMITS) << msg << "BLOCK" << reason; - verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + dbgDebug(D_WAAP_ULIMITS) << msg << "BLOCK" << reason; + verdict = ServiceVerdict::TRAFFIC_VERDICT_DROP; } else { // prevent mode and active accept exception decision->setBlock(false); decision->setForceAllow(true); - dbgInfo(D_WAAP_ULIMITS) << msg << "Override Accept" << reason; - verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + dbgDebug(D_WAAP_ULIMITS) << msg << "Override Accept" << reason; + verdict = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT; } } @@ -662,25 +664,79 @@ Waf2Transaction::getBehaviors( std::set all_params; I_GenericRulebase *i_rulebase = Singleton::Consume::by(); for (const auto &id : exceptions) { - dbgTrace(D_WAAP) << "get parameter exception for: " << id; - auto params = i_rulebase->getParameterException(id).getBehavior(exceptions_dict); + dbgTrace(D_WAAP_OVERRIDE) << "get parameter exception for: " << id; - if (checkResponse && !getResponseBody().empty()) { - std::unordered_map> response_dict = { - {"responseBody", {getResponseBody()}} - }; - auto params = i_rulebase->getParameterException(id).getBehavior(response_dict); + const auto ¶mException = i_rulebase->getParameterException(id); + auto params = paramException.getBehavior(exceptions_dict); + bool hasKVPair = paramException.isContainingKVPair(); + + // If isContainingKVPair() is false, continue as usual + if (!hasKVPair) { + dbgTrace(D_WAAP_OVERRIDE) << "Using traditional matching for exception " << id << " (no KV pairs)"; + if (checkResponse && !getResponseBody().empty()) { + std::unordered_map> response_dict = { + {"responseBody", {getResponseBody()}} + }; + auto responseParams = paramException.getBehavior(response_dict); + if (responseParams.size() > 0) { + dbgTrace(D_WAAP_OVERRIDE) << "got responseBody behavior, setApplyOverride(true)"; + m_responseInspectReasons.setApplyOverride(true); + all_params.insert(responseParams.begin(), responseParams.end()); + // once found, no need to check again + checkResponse = false; + } + } + + dbgTrace(D_WAAP_OVERRIDE) << "got "<< params.size() << " behaviors (non-KV pair)"; + all_params.insert(params.begin(), params.end()); + } else { if (params.size() > 0) { - dbgTrace(D_WAAP) << "got responseBody behavior, setApplyOverride(true)"; - m_responseInspectReasons.setApplyOverride(true); - all_params.insert(params.begin(), params.end()); - // once found, no need to check again - checkResponse = false; + bool anyKVMatched = false; + std::set kvParams; + + dbgTrace(D_WAAP_OVERRIDE) << "Using KV pair matching for exception " << id; + // Check parameter name/value pairs + for (const DeepParser::KeywordInfo& keywordInfo : getKeywordInfo()) { + std::unordered_map> kv_dict = { + {"paramName", {keywordInfo.getName()}}, + {"paramValue", {keywordInfo.getValue()}} + }; + auto kvBehaviors = i_rulebase->getParameterException(id).getBehavior(kv_dict, true); + if (kvBehaviors.size() > 0) { + anyKVMatched = true; + kvParams.insert(kvBehaviors.begin(), kvBehaviors.end()); + } + } + + // Check header name/value pairs + for (const auto& hdr_pair : getHdrPairs()) { + std::string name = hdr_pair.first; + std::transform(name.begin(), name.end(), name.begin(), ::tolower); + std::string value = hdr_pair.second; + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + + std::unordered_map> kv_dict = { + {"headerName", {name}}, + {"headerValue", {value}} + }; + auto kvBehaviors = i_rulebase->getParameterException(id).getBehavior(kv_dict, true); + if (kvBehaviors.size() > 0) { + anyKVMatched = true; + kvParams.insert(kvBehaviors.begin(), kvBehaviors.end()); + } + } + + // Consider matches only if at least one of the key/value pairs have been successfully matched + if (anyKVMatched) { + dbgTrace(D_WAAP_OVERRIDE) << "got "<< kvParams.size() << " behaviors (KV pair matched)"; + all_params.insert(kvParams.begin(), kvParams.end()); + } else { + dbgTrace(D_WAAP_OVERRIDE) << "KV pair exception found but no specific pairs matched"; + } + } else { + dbgTrace(D_WAAP_OVERRIDE) << "KV pair exception found but no initial match"; } } - - dbgTrace(D_WAAP) << "got "<< params.size() << " behaviors"; - all_params.insert(params.begin(), params.end()); } return all_params; } @@ -696,26 +752,26 @@ bool Waf2Transaction::shouldEnforceByPracticeExceptions(DecisionType practiceTyp auto exceptions = overridePolicy->getExceptionsByPractice().getExceptionsOfPractice(practiceType); if (!exceptions.empty()) { - dbgTrace(D_WAAP) << "get behaviors for practice: " << practiceType; + dbgTrace(D_WAAP_OVERRIDE) << "get behaviors for practice: " << practiceType; auto behaviors = getBehaviors(getExceptionsDict(practiceType), exceptions); if (behaviors.size() > 0) { auto it = m_overrideStateByPractice.find(practiceType); if (it == m_overrideStateByPractice.end()) { - dbgWarning(D_WAAP) << "no override state for practice: " << practiceType; + dbgDebug(D_WAAP_OVERRIDE) << "no override state for practice: " << practiceType; return false; } setOverrideState(behaviors, it->second); if (it->second.bForceBlock) { - dbgTrace(D_WAAP) + dbgTrace(D_WAAP_OVERRIDE) << "should block by exceptions for practice: " << practiceType; decision->setBlock(true); decision->setForceBlock(true); shouldEnforce = true; } if (it->second.bForceException) { - dbgTrace(D_WAAP) + dbgTrace(D_WAAP_OVERRIDE) << "should not block by exceptions for practice: " << practiceType; decision->setBlock(false); decision->setForceAllow(true); @@ -748,16 +804,16 @@ void Waf2Transaction::setOverrideState(const std::set& behavi m_matchedOverrideIds.insert(behavior.getId()); if (behavior.getKey() == BehaviorKey::ACTION) { if (behavior.getValue() == BehaviorValue::ACCEPT) { - dbgTrace(D_WAAP) << "setting bForceException due to override behavior: " << behavior.getId(); + dbgTrace(D_WAAP_OVERRIDE) << "setting bForceException due to override behavior: " << behavior.getId(); state.bForceException = true; state.forceExceptionIds.insert(behavior.getId()); } else if (behavior.getValue() == BehaviorValue::REJECT) { - dbgTrace(D_WAAP) << "setting bForceBlock due to override behavior: " << behavior.getId(); + dbgTrace(D_WAAP_OVERRIDE) << "setting bForceBlock due to override behavior: " << behavior.getId(); state.bForceBlock = true; state.forceBlockIds.insert(behavior.getId()); } } else if(behavior.getKey() == BehaviorKey::LOG && behavior.getValue() == BehaviorValue::IGNORE) { - dbgTrace(D_WAAP) << "setting bSupressLog due to override behavior: " << behavior.getId(); + dbgTrace(D_WAAP_OVERRIDE) << "setting bSupressLog due to override behavior: " << behavior.getId(); state.bSupressLog = true; } } @@ -771,11 +827,11 @@ Waap::Override::State Waf2Transaction::getOverrideState(IWaapConfig* sitePolicy) auto exceptions = overridePolicy->getExceptionsByPractice(). getExceptionsOfPractice(AUTONOMOUS_SECURITY_DECISION); if (!exceptions.empty()) { - dbgTrace(D_WAAP) << "get behaviors for override state"; + dbgTrace(D_WAAP_OVERRIDE) << "get behaviors for override state"; m_responseInspectReasons.setApplyOverride(false); auto behaviors = getBehaviors(getExceptionsDict(AUTONOMOUS_SECURITY_DECISION), exceptions, true); if (behaviors.size() > 0) { - dbgTrace(D_WAAP) << "set override state by practice found behaviors"; + dbgTrace(D_WAAP_OVERRIDE) << "set override state by practice found behaviors"; setOverrideState(behaviors, m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION]); } m_isHeaderOverrideScanRequired = false; @@ -819,10 +875,10 @@ const Maybe Waf2Transaction::getTriggerLog(const ScopedContext ctx; ctx.registerValue>(TriggerMatcher::ctx_key, triggers_set); - auto trigger_config = getConfiguration("rulebase", "log"); + auto trigger_config = getConfigurationWithCache("rulebase", "log"); if (!trigger_config.ok()) { - dbgError(D_WAAP) << "Failed to get log trigger configuration, err: " << trigger_config.getErr(); + dbgDebug(D_WAAP) << "Failed to get log trigger configuration, err: " << trigger_config.getErr(); } return trigger_config; } @@ -851,9 +907,9 @@ bool Waf2Transaction::isTriggerReportExists(const std::shared_ptr< ScopedContext ctx; ctx.registerValue>(TriggerMatcher::ctx_key, triggers_set); - auto trigger_config = getConfiguration("rulebase", "report"); + auto trigger_config = getConfigurationWithCache("rulebase", "report"); if (!trigger_config.ok()) { - dbgWarning(D_WAAP) << "Failed to get report trigger configuration, err: " << trigger_config.getErr(); + dbgDebug(D_WAAP) << "Failed to get report trigger configuration, err: " << trigger_config.getErr(); m_triggerReport = false; return false; } diff --git a/components/security_apps/waap/waap_clib/Waf2Regex.cc b/components/security_apps/waap/waap_clib/Waf2Regex.cc index c6b1ad4..d5078bd 100755 --- a/components/security_apps/waap/waap_clib/Waf2Regex.cc +++ b/components/security_apps/waap/waap_clib/Waf2Regex.cc @@ -117,11 +117,14 @@ SingleRegex::~SingleRegex() { } } -bool SingleRegex::hasMatch(const std::string& s) const { +bool SingleRegex::hasMatch(const std::string& s, size_t start, size_t end) const { + PCRE2_SIZE startOffset = static_cast(start); + size_t data_size = std::min(s.size(), end); + int rc = pcre2_match( m_re, // code - reinterpret_cast(s.data()), s.size(), // subject/subject length - 0, // start offset + reinterpret_cast(s.data()), data_size, // subject/subject length + startOffset, // start offset 0, // options m_matchData, NULL // match_context @@ -244,13 +247,23 @@ const std::string &SingleRegex::getName() const return m_regexName; } -size_t SingleRegex::findMatchRanges(const std::string& s, std::vector& matchRanges) const { - PCRE2_SIZE startOffset = 0; +size_t +SingleRegex::findMatchRanges( + const std::string &s, + std::vector &matchRanges, + size_t maxMatches, + size_t start, + size_t end +) const +{ + PCRE2_SIZE startOffset = static_cast(start); + size_t matchCount = 0; + size_t data_size = std::min(s.size(), end); do { int rc = pcre2_match( m_re, // code - reinterpret_cast(s.data()), s.size(), // subject/subject length + reinterpret_cast(s.data()), data_size, // subject/subject length startOffset, // start offset 0, // options m_matchData, @@ -277,6 +290,12 @@ size_t SingleRegex::findMatchRanges(const std::string& s, std::vector 0 && matchCount >= maxMatches) { + break; + } } while (true); return matchRanges.size(); diff --git a/components/security_apps/waap/waap_clib/Waf2Regex.h b/components/security_apps/waap/waap_clib/Waf2Regex.h index 2a15f41..8c64465 100755 --- a/components/security_apps/waap/waap_clib/Waf2Regex.h +++ b/components/security_apps/waap/waap_clib/Waf2Regex.h @@ -54,10 +54,16 @@ public: SingleRegex(const std::string &pattern, bool &error, const std::string ®exName, bool bNoRegex=false, const std::string ®exMatchName="", const std::string ®exMatchValue=""); ~SingleRegex(); - bool hasMatch(const std::string &s) const; + bool hasMatch(const std::string &s, size_t start = 0, size_t end = SIZE_MAX) const; size_t findAllMatches(const std::string &s, std::vector &matches, size_t max_matches = std::string::npos) const; - size_t findMatchRanges(const std::string &s, std::vector &matchRanges) const; + size_t findMatchRanges( + const std::string &s, + std::vector &matchRanges, + size_t maxMatches = 0, + size_t start = 0, + size_t end = SIZE_MAX + ) const; const std::string &getName() const; private: pcre2_code *m_re; diff --git a/components/security_apps/waap/waap_clib/Waf2Util.cc b/components/security_apps/waap/waap_clib/Waf2Util.cc index 11d6684..acf7fd8 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.cc +++ b/components/security_apps/waap/waap_clib/Waf2Util.cc @@ -48,6 +48,49 @@ USE_DEBUG_FLAG(D_WAAP_JSON); #define MIN_HEX_LENGTH 6 #define charToDigit(c) (c - '0') +// Base64 decoding lookup table constants +#define BASE64_INVALID -1 +#define BASE64_PADDING -2 + +// Base64 lookup table for optimized decoding +static int base64_table[256]; +static bool base64_table_initialized = false; + +// Initialize the base64 lookup table for optimized decoding +static void initialize_base64_table() +{ + if (base64_table_initialized) { + return; + } + + // Initialize all entries to invalid + for (int i = 0; i < 256; i++) { + base64_table[i] = BASE64_INVALID; + } + + // Set valid base64 characters (A-Z = 0-25) + for (int i = 0; i < 26; i++) { + base64_table['A' + i] = i; + } + + // Set valid base64 characters (a-z = 26-51) + for (int i = 0; i < 26; i++) { + base64_table['a' + i] = 26 + i; + } + + // Set valid base64 characters (0-9 = 52-61) + for (int i = 0; i < 10; i++) { + base64_table['0' + i] = 52 + i; + } + + // Set special base64 characters + base64_table['+'] = 62; + base64_table['/'] = 63; + base64_table['='] = BASE64_PADDING; + + base64_table_initialized = true; +} + // See https://dev.w3.org/html5/html-author/charref const struct HtmlEntity g_htmlEntities[] = { @@ -1086,6 +1129,19 @@ base64_decode_status decideStatusBase64Decoded( return B64_DECODE_INVALID; } +// +// Base64 decoding with direct array lookup +// Performance optimization notes: +// - Replaced unordered_map lookups with direct array indexing +// - Improved cache locality with a simple 256-element array +// - Eliminated hash computation and collision handling overhead +// - One-time initialization of the lookup table +// +// The table maps each ASCII character to its base64 value: +// - For valid base64 characters (A-Z, a-z, 0-9, +, /): returns 0-63 value +// - For padding character ('='): returns BASE64_PADDING (-2) +// - For invalid characters: returns BASE64_INVALID (-1) +// // Attempts to validate and decode base64-encoded chunk. // Value is the full value inside which potential base64-encoded chunk was found, @@ -1107,6 +1163,9 @@ base64_decode_status decodeBase64Chunk( bool clear_on_error, bool called_with_prefix) { + // Initialize the base64 lookup table on first call + initialize_base64_table(); + decoded.clear(); uint32_t acc = 0; int acc_bits = 0; // how many bits are filled in acc @@ -1135,8 +1194,9 @@ base64_decode_status decodeBase64Chunk( return B64_DECODE_INVALID; } - std::unordered_map original_occurences_counter; - std::unordered_map decoded_occurences_counter; + // Fixed arrays for character counting - Proposal 1 optimization + uint32_t original_counts[256] = {0}; // Count for each ASCII character in original + uint32_t decoded_counts[256] = {0}; // Count for each byte value in decoded while (it != end) { unsigned char c = *it; @@ -1159,35 +1219,21 @@ base64_decode_status decodeBase64Chunk( // allow for more terminator characters it++; - original_occurences_counter[c]++; + original_counts[static_cast(c)]++; continue; } - unsigned char val = 0; + // Use lookup table for faster decoding + int lookup_value = base64_table[static_cast(c)]; - if (c >= 'A' && c <= 'Z') { - val = c - 'A'; - } - else if (c >= 'a' && c <= 'z') { - val = c - 'a' + 26; - } - else if (isdigit(c)) { - val = c - '0' + 52; - } - else if (c == '+') { - val = 62; - } - else if (c == '/') { - val = 63; - } - else if (c == '=') { - // Start tracking terminator characters + if (lookup_value == BASE64_PADDING) { + // Start tracking terminator characters ('=') terminatorCharsSeen++; it++; - original_occurences_counter[c]++; + original_counts[static_cast(c)]++; continue; } - else { + else if (lookup_value == BASE64_INVALID) { dbgTrace(D_WAAP_BASE64) << "(leave as-is) because of non-base64 character ('" << c @@ -1199,6 +1245,8 @@ base64_decode_status decodeBase64Chunk( return B64_DECODE_INVALID; // non-base64 character } + unsigned char val = static_cast(lookup_value); + acc = (acc << 6) | val; acc_bits += 6; @@ -1218,11 +1266,11 @@ base64_decode_status decodeBase64Chunk( } decoded += (char)code; - decoded_occurences_counter[(char)code]++; + decoded_counts[static_cast(code)]++; } it++; - original_occurences_counter[c]++; + original_counts[static_cast(c)]++; } // end of encoded sequence decoded. @@ -1242,13 +1290,21 @@ base64_decode_status decodeBase64Chunk( double entropy = 0; double p = 0; double decoded_entropy = 0; - for (const auto& pair : original_occurences_counter) { - p = pair.second / length; - entropy -= p * std::log2(p); + + // Calculate entropy from original character counts + for (int i = 0; i < 256; i++) { + if (original_counts[i] > 0) { + p = static_cast(original_counts[i]) / length; + entropy -= p * std::log2(p); + } } - for (const auto &pair : decoded_occurences_counter) { - p = pair.second / decoded.size(); - decoded_entropy -= p * std::log2(p); + + // Calculate entropy from decoded character counts + for (int i = 0; i < 256; i++) { + if (decoded_counts[i] > 0) { + p = static_cast(decoded_counts[i]) / decoded.size(); + decoded_entropy -= p * std::log2(p); + } } dbgTrace(D_WAAP_BASE64) << "Base entropy = " @@ -1917,9 +1973,6 @@ isGzipped(const string &stream) { if (stream.size() < 2) return false; auto unsinged_stream = reinterpret_cast(stream.data()); - dbgTrace(D_WAAP) << "isGzipped: first two bytes: " - << std::hex << static_cast(unsinged_stream[0]) << " " - << std::hex << static_cast(unsinged_stream[1]); return unsinged_stream[0] == 0x1f && unsinged_stream[1] == 0x8b; } @@ -2310,7 +2363,7 @@ string extractForwardedIp(const string &x_forwarded_hdr_val) vector trusted_ips; string forward_ip; - auto identify_config = getConfiguration( + auto identify_config = getConfigurationWithCache( "rulebase", "usersIdentifiers" ); diff --git a/components/security_apps/waap/waap_clib/buffered_compressed_stream.cc b/components/security_apps/waap/waap_clib/buffered_compressed_stream.cc new file mode 100644 index 0000000..1f43e2c --- /dev/null +++ b/components/security_apps/waap/waap_clib/buffered_compressed_stream.cc @@ -0,0 +1,481 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "buffered_compressed_stream.h" +#include "waap.h" +#include "compression_utils.h" +#include + +USE_DEBUG_FLAG(D_WAAP_SERIALIZE); + +using namespace std; + +void yieldIfPossible(const string &func, int line) +{ + // check mainloop exists and current routine is not the main routine + if (Singleton::exists() && + Singleton::Consume::by()->getCurrentRoutineId().ok()) + { + dbgDebug(D_WAAP_SERIALIZE) << "Yielding to main loop from: " << func << ":" << line; + Singleton::Consume::by()->yield(false); + } +} + +// Static member definitions +const size_t BufferedCompressedOutputStream::CompressedBuffer::BUFFER_SIZE; +const size_t BufferedCompressedInputStream::DecompressedBuffer::OUTPUT_BUFFER_SIZE; +const size_t BufferedCompressedInputStream::DecompressedBuffer::CHUNK_SIZE; + +BufferedCompressedOutputStream::BufferedCompressedOutputStream(ostream &underlying_stream) + : + ostream(nullptr), + m_buffer(make_unique(underlying_stream)) +{ + rdbuf(m_buffer.get()); +} + +BufferedCompressedOutputStream::~BufferedCompressedOutputStream() +{ + try { + close(); + } catch (exception &e) { + // Destructor should not throw + dbgWarning(D_WAAP_SERIALIZE) << "Exception in BufferedCompressedOutputStream destructor: " << e.what(); + } +} + +void BufferedCompressedOutputStream::flush() +{ + if (m_buffer) { + dbgTrace(D_WAAP_SERIALIZE) << "Flushing internal buffer..."; + m_buffer->flushBuffer(); // This will compress and encrypt the current buffer with is_last=false + // and flush the underlying stream. + } + // Do NOT call ostream::flush() here, as it would call sync() on our m_buffer, + // which calls compressAndEncryptBuffer(true) and finalizes the GZIP stream prematurely. + // The m_underlying_stream within m_buffer is flushed by compressAndEncryptBuffer itself. +} + +void BufferedCompressedOutputStream::close() +{ + if (m_buffer) { + dbgTrace(D_WAAP_SERIALIZE) << "Closing stream and flushing buffer..."; + m_buffer->flushAndClose(); + } +} + +BufferedCompressedOutputStream::CompressedBuffer::CompressedBuffer(ostream &underlying_stream) + : + m_underlying_stream(underlying_stream), + m_buffer(), + m_compression_stream(nullptr), + m_closed(false) +{ + m_buffer.reserve(BUFFER_SIZE); + m_compression_stream = initCompressionStream(); + +} + +BufferedCompressedOutputStream::CompressedBuffer::~CompressedBuffer() +{ + try { + if (!m_closed) { + sync(); + } + if (m_compression_stream) { + finiCompressionStream(m_compression_stream); + m_compression_stream = nullptr; + } + } catch (exception &e) { + // Destructor should not throw + dbgWarning(D_WAAP_SERIALIZE) << "Exception in CompressedBuffer destructor: " << e.what(); + } +} + +void BufferedCompressedOutputStream::CompressedBuffer::flushAndClose() +{ + sync(); +} + +int BufferedCompressedOutputStream::CompressedBuffer::overflow(int c) +{ + if (m_closed) { + dbgTrace(D_WAAP_SERIALIZE) << "Stream is closed, returning EOF"; + return traits_type::eof(); + } + + if (c != traits_type::eof()) { + m_buffer.push_back(static_cast(c)); + dbgTrace(D_WAAP_SERIALIZE) << "Added char, buffer size now: " << m_buffer.size(); + } + + if (m_buffer.size() >= BUFFER_SIZE) { + dbgTrace(D_WAAP_SERIALIZE) << "Buffer full, flushing..."; + compressAndEncryptBuffer(false); + } + + return c; +} + +streamsize BufferedCompressedOutputStream::CompressedBuffer::xsputn(const char* s, streamsize n) +{ + if (m_closed) { + dbgDebug(D_WAAP_SERIALIZE) << "Stream is closed, returning 0"; + return 0; + } + + dbgTrace(D_WAAP_SERIALIZE) << "Writing " << n << " bytes"; + streamsize written = 0; + while (written < n) { + size_t space_available = BUFFER_SIZE - m_buffer.size(); + size_t to_write = min(static_cast(n - written), space_available); + + m_buffer.insert(m_buffer.end(), s + written, s + written + to_write); + written += to_write; + + dbgTrace(D_WAAP_SERIALIZE) << "Wrote " << to_write << " bytes, total written: " << written + << ", buffer size: " << m_buffer.size(); + + if (m_buffer.size() >= BUFFER_SIZE) { + dbgTrace(D_WAAP_SERIALIZE) << "Buffer full, flushing..."; + compressAndEncryptBuffer(false); + } + } + + dbgTrace(D_WAAP_SERIALIZE) << "Completed, total written: " << written; + return written; +} + +int BufferedCompressedOutputStream::CompressedBuffer::sync() +{ + dbgTrace(D_WAAP_SERIALIZE) << "Called, closed=" << m_closed << ", buffer size=" << m_buffer.size(); + if (!m_closed) { + bool success = compressAndEncryptBuffer(true); // Attempt final compression/encryption + // Mark as closed REGARDLESS of the success of the attempt to ensure finalization logic + // for this context isn't re-attempted if this call failed. + m_closed = true; + if (!success) { + dbgWarning(D_WAAP_SERIALIZE) << "Final compression/encryption failed"; + return -1; + } + dbgTrace(D_WAAP_SERIALIZE) << "Stream closed successfully"; + } else { + dbgDebug(D_WAAP_SERIALIZE) << "Stream already closed, skipping"; + } + return 0; +} + +void BufferedCompressedOutputStream::CompressedBuffer::flushBuffer() +{ + if (m_buffer.empty() || m_closed) { + return; + } + + dbgTrace(D_WAAP_SERIALIZE) << "Flushing buffer with " << m_buffer.size() << " bytes"; + compressAndEncryptBuffer(false); +} + +bool BufferedCompressedOutputStream::CompressedBuffer::compressAndEncryptBuffer(bool is_last) +{ + // If the stream is already marked as closed at this buffer's level, + // it means sync() has run, and everything, including encryption, has been finalized. + if (m_closed) { + dbgTrace(D_WAAP_SERIALIZE) << "Stream is already closed, skipping."; + return true; + } + + // Skip if there's nothing to compress and this is not the final flush + if (m_buffer.empty() && !is_last) { + dbgTrace(D_WAAP_SERIALIZE) << "Buffer empty and not last call, skipping."; + return true; + } + + dbgTrace(D_WAAP_SERIALIZE) << "Compressing and encrypting " << m_buffer.size() << " bytes, is_last: " << is_last; + + // Compress the buffer + CompressionResult result = compressData( + m_compression_stream, + CompressionType::GZIP, + static_cast(m_buffer.size()), + reinterpret_cast(m_buffer.data()), + is_last ? 1 : 0 + ); + + if (!result.ok) { + dbgWarning(D_WAAP_SERIALIZE) << "Failed to compress data"; + return false; + } + + string compressed_data; + if (result.output && result.num_output_bytes > 0) { + compressed_data = string(reinterpret_cast(result.output), result.num_output_bytes); + free(result.output); + } + + dbgDebug(D_WAAP_SERIALIZE) << "Compression complete: " << m_buffer.size() + << " bytes -> " << compressed_data.size() << " bytes"; + + // Yield after compression to allow other routines to run + YIELD_IF_POSSIBLE(); + + string final_data = compressed_data; + + // Write to underlying stream only if we have data to write + if (!final_data.empty()) { + m_underlying_stream.write(final_data.c_str(), final_data.size()); + m_underlying_stream.flush(); + } + + m_buffer.clear(); + + // Yield after writing chunk to allow other routines to run + YIELD_IF_POSSIBLE(); + + return true; +} + +BufferedCompressedInputStream::BufferedCompressedInputStream(istream &underlying_stream) + : + istream(nullptr), + m_buffer(make_unique(underlying_stream)) +{ + rdbuf(m_buffer.get()); +} + +BufferedCompressedInputStream::~BufferedCompressedInputStream() +{ + // DecompressedBuffer destructor will handle cleanup +} + +BufferedCompressedInputStream::DecompressedBuffer::DecompressedBuffer(istream &underlying_stream) + : +m_underlying_stream(underlying_stream), + m_buffer(), + m_encrypted_buffer(), + m_compressed_buffer(), + m_decompressed_buffer(), + m_decompressed_pos(0), + m_compression_stream(nullptr), + m_eof_reached(false), + m_stream_finished(false) +{ + m_buffer.resize(OUTPUT_BUFFER_SIZE); + m_encrypted_buffer.reserve(CHUNK_SIZE); + m_compressed_buffer.reserve(CHUNK_SIZE); + m_decompressed_buffer.reserve(OUTPUT_BUFFER_SIZE); + m_compression_stream = initCompressionStream(); + + + // Set buffer pointers to indicate empty buffer + setg(m_buffer.data(), m_buffer.data(), m_buffer.data()); +} + +BufferedCompressedInputStream::DecompressedBuffer::~DecompressedBuffer() +{ + try { + if (m_compression_stream) { + finiCompressionStream(m_compression_stream); + m_compression_stream = nullptr; + } + } catch (exception &e) { + // Destructor should not throw + dbgWarning(D_WAAP_SERIALIZE) << "Exception in DecompressedBuffer destructor: " << e.what(); + } +} + +int BufferedCompressedInputStream::DecompressedBuffer::underflow() +{ + if (gptr() < egptr()) { + return traits_type::to_int_type(*gptr()); + } + + if (m_eof_reached) { + return traits_type::eof(); + } + + if (!fillBuffer()) { + m_eof_reached = true; + return traits_type::eof(); + } + + return traits_type::to_int_type(*gptr()); +} + +streamsize BufferedCompressedInputStream::DecompressedBuffer::xsgetn(char* s, streamsize n) +{ + streamsize total_read = 0; + + while (total_read < n) { + if (gptr() >= egptr()) { + if (!fillBuffer()) { + m_eof_reached = true; + break; + } + } + + streamsize available = egptr() - gptr(); + streamsize to_copy = min(n - total_read, available); + + memcpy(s + total_read, gptr(), to_copy); + gbump(static_cast(to_copy)); + total_read += to_copy; + } + + return total_read; +} + +bool BufferedCompressedInputStream::DecompressedBuffer::fillBuffer() +{ + if (m_eof_reached) { + return false; + } + + // If we have remaining data in the decompressed buffer, use it first + if (m_decompressed_pos < m_decompressed_buffer.size()) { + size_t remaining = m_decompressed_buffer.size() - m_decompressed_pos; + size_t to_copy = min(remaining, OUTPUT_BUFFER_SIZE); + + memcpy(m_buffer.data(), m_decompressed_buffer.data() + m_decompressed_pos, to_copy); + m_decompressed_pos += to_copy; + + // Set up the buffer pointers for streambuf: + // eback() = m_buffer.data() (start of buffer) + // gptr() = m_buffer.data() (current position) + // egptr() = m_buffer.data() + to_copy (end of valid data) + setg(m_buffer.data(), m_buffer.data(), m_buffer.data() + to_copy); + + dbgTrace(D_WAAP_SERIALIZE) << "Serving " << to_copy << " bytes from existing decompressed buffer"; + + // Yield after serving data from buffer to allow other routines to run + YIELD_IF_POSSIBLE(); + return true; + } + + // Need to process the next chunk + if (!processNextChunk()) { + m_eof_reached = true; + return false; + } + + // Now try again with the new data + return fillBuffer(); +} + +bool BufferedCompressedInputStream::DecompressedBuffer::processNextChunk() +{ + while (true) { + if (m_stream_finished) { + return false; + } + + // Read a chunk of encrypted data from the underlying stream + if (m_encrypted_buffer.size() < CHUNK_SIZE) { + m_encrypted_buffer.resize(CHUNK_SIZE); + } + m_underlying_stream.read(m_encrypted_buffer.data(), CHUNK_SIZE); + streamsize bytes_read = m_underlying_stream.gcount(); + + if (bytes_read <= 0) { + m_stream_finished = true; + + // End of stream - no more data to process + dbgTrace(D_WAAP_SERIALIZE) << "Reached end of input stream"; + return false; + } + + m_encrypted_buffer.resize(bytes_read); + + dbgTrace(D_WAAP_SERIALIZE) << "Read " << bytes_read << " encrypted bytes from stream"; + + // Decrypt the chunk + std::vector decrypted_chunk; + if (!decryptChunk(m_encrypted_buffer, decrypted_chunk)) { + dbgWarning(D_WAAP_SERIALIZE) << "Failed to decrypt chunk"; + break; + } + + // Decompress the chunk + std::vector decompressed_chunk; + if (!decompressChunk(decrypted_chunk, decompressed_chunk)) { + dbgWarning(D_WAAP_SERIALIZE) << "Failed to decompress chunk"; + break; + } + + if (decompressed_chunk.empty()) { + dbgTrace(D_WAAP_SERIALIZE) << "Decompressed chunk is empty, skipping"; + continue; // Nothing to add to the buffer + } + // Replace the decompressed buffer with new data using swap to avoid unnecessary allocations + m_decompressed_buffer.swap(decompressed_chunk); + m_decompressed_pos = 0; + + dbgTrace(D_WAAP_SERIALIZE) << "Processed chunk: " << bytes_read + << " encrypted -> " << decrypted_chunk.size() + << " compressed -> " << m_decompressed_buffer.size() << " decompressed"; + + // Yield after processing chunk to allow other routines to run + YIELD_IF_POSSIBLE(); + return true; + } + return false; +} + +bool BufferedCompressedInputStream::DecompressedBuffer::decryptChunk( + const std::vector &encrypted_chunk, + std::vector &decrypted_chunk) +{ + + // No encryption - just copy the data + decrypted_chunk = encrypted_chunk; + return true; +} + +bool BufferedCompressedInputStream::DecompressedBuffer::decompressChunk( + const std::vector &compressed_chunk, + std::vector &decompressed_chunk) +{ + if (compressed_chunk.empty()) { + return true; // Nothing to decompress + } + + // Use the streaming decompression + DecompressionResult result = decompressData( + m_compression_stream, + compressed_chunk.size(), + reinterpret_cast(compressed_chunk.data()) + ); + + if (!result.ok) { + dbgWarning(D_WAAP_SERIALIZE) << "Failed to decompress chunk"; + return false; + } + + if (result.output && result.num_output_bytes > 0) { + decompressed_chunk.assign( + reinterpret_cast(result.output), + reinterpret_cast(result.output) + result.num_output_bytes + ); + free(result.output); + + dbgTrace(D_WAAP_SERIALIZE) << "Decompressed chunk: " << compressed_chunk.size() + << " -> " << decompressed_chunk.size() << " bytes"; + + // Yield after decompression to allow other routines to run + YIELD_IF_POSSIBLE(); + return true; + } + + // No output data yet (might need more input for compression algorithm) + decompressed_chunk.clear(); + return true; +} diff --git a/components/security_apps/waap/waap_component.cc b/components/security_apps/waap/waap_component.cc index 5288e16..4dccd78 100755 --- a/components/security_apps/waap/waap_component.cc +++ b/components/security_apps/waap/waap_component.cc @@ -50,10 +50,16 @@ void WaapComponent::preload() { // TODO:: call stuff like registerExpectedCofiguration here.. - registerExpectedConfiguration("WAAP", "WebApplicationSecurity"); - registerExpectedConfiguration("WAAP", "WebAPISecurity"); + // registerExpectedConfiguration("WAAP", "WebApplicationSecurity"); + // registerExpectedConfiguration("WAAP", "WebAPISecurity"); + + registerExpectedConfigurationWithCache( + "assetId", "WAAP", "WebApplicationSecurity"); + registerExpectedConfigurationWithCache("assetId", "WAAP", "WebAPISecurity"); + registerExpectedConfiguration("WAAP", "Sigs file path"); registerExpectedConfigFile("waap", Config::ConfigFileType::Policy); + registerExpectedSetting("features", "learningLeader"); registerConfigLoadCb( [this]() { diff --git a/components/security_apps/waap/waap_component_impl.cc b/components/security_apps/waap/waap_component_impl.cc index 741c584..1eb5aab 100755 --- a/components/security_apps/waap/waap_component_impl.cc +++ b/components/security_apps/waap/waap_component_impl.cc @@ -24,6 +24,7 @@ #include #include "debug.h" +#include "user_identifiers_config.h" #include "waap_clib/WaapAssetStatesManager.h" #include "waap_clib/Waf2Engine.h" #include "waap_clib/WaapConfigApi.h" @@ -45,10 +46,10 @@ USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); USE_DEBUG_FLAG(D_NGINX_EVENTS); WaapComponent::Impl::Impl() : - pending_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT), - accept_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT), - drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP), - limit_response_headers(ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS), + pending_response(ServiceVerdict::TRAFFIC_VERDICT_INSPECT), + accept_response(ServiceVerdict::TRAFFIC_VERDICT_ACCEPT), + drop_response(ServiceVerdict::TRAFFIC_VERDICT_DROP), + limit_response_headers(ServiceVerdict::LIMIT_RESPONSE_HEADERS), waapStateTable(NULL), transactionsCount(0), deepAnalyzer() @@ -221,17 +222,47 @@ WaapComponent::Impl::respond(const HttpRequestHeaderEvent &event) IWaf2Transaction& waf2Transaction = waapStateTable->getState(); // Tell waf2 API that another request header arrived - waf2Transaction.add_request_hdr( - reinterpret_cast(header_name.data()), //const char * name // - header_name.size(), //int name_len // - reinterpret_cast(header_value.data()), //const char * value // - header_value.size() //int value_len // - ); + if (event.shouldLog()) { + waf2Transaction.add_request_hdr( + reinterpret_cast(header_name.data()), //const char * name // + header_name.size(), //int name_len // + reinterpret_cast(header_value.data()), //const char * value // + header_value.size() //int value_len // + ); + } else { + dbgTrace(D_WAAP) + << "Header '" + << std::dumpHex(header_name) + << "' marked as should not log, skipping adding to WAF2"; + } EventVerdict verdict = pending_response; // Last header handled if (event.isLastHeader()) { + + // NEXUS env should take real client ip from X-Forwarded-For header + if (getProfileAgentSettingWithDefault(false, "agent.saasProfile.ignoreSourceIP")) { + + auto env = Singleton::Consume::by(); + auto maybe_xff = env->get(HttpTransactionData::xff_vals_ctx); + if (!maybe_xff.ok()) { + dbgTrace(D_WAAP) << "failed to get xff vals from env"; + } else { + // Extract the last IP from XFF header (actual client IP) + std::string xff_header = maybe_xff.unpack(); + size_t last_comma_pos = xff_header.find_last_of(','); + std::string sourceIpStr = (last_comma_pos != std::string::npos) ? + xff_header.substr(last_comma_pos + 1) : xff_header; + // Trim whitespace + sourceIpStr.erase(0, sourceIpStr.find_first_not_of(" \t")); + sourceIpStr.erase(sourceIpStr.find_last_not_of(" \t") + 1); + + dbgTrace(D_WAAP) << "taking source IP for nexus from last xff header: " << sourceIpStr; + waf2Transaction.set_transaction_remote(sourceIpStr.c_str(), NEXUS_PORT); + } + } + waf2Transaction.end_request_hdrs(); verdict = waf2Transaction.getUserLimitVerdict(); @@ -283,9 +314,9 @@ WaapComponent::Impl::respond(const HttpRequestBodyEvent &event) waf2Transaction.add_request_body_chunk(dataBuf, dataBufLen); - ngx_http_cp_verdict_e verdict = waf2Transaction.getUserLimitVerdict(); + ServiceVerdict verdict = waf2Transaction.getUserLimitVerdict(); EventVerdict eventVedict(verdict); - if (verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT) { + if (verdict != ServiceVerdict::TRAFFIC_VERDICT_INSPECT) { finishTransaction(waf2Transaction, eventVedict); } @@ -324,8 +355,8 @@ WaapComponent::Impl::respond(const EndRequestEvent &) EventVerdict verdict = waapDecision(waf2Transaction); // Delete state before returning any verdict which is not pending - if (verdict.getVerdict() != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT && - verdict.getVerdict() != ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS && + if (verdict.getVerdict() != ServiceVerdict::TRAFFIC_VERDICT_INSPECT && + verdict.getVerdict() != ServiceVerdict::LIMIT_RESPONSE_HEADERS && waapStateTable->hasState() ) { finishTransaction(waf2Transaction, verdict); @@ -373,8 +404,8 @@ WaapComponent::Impl::respond(const ResponseCodeEvent &event) } // Delete state before returning any verdict which is not pending - if (verdict.getVerdict() != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT && - verdict.getVerdict() != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT && + if (verdict.getVerdict() != ServiceVerdict::TRAFFIC_VERDICT_INSPECT && + verdict.getVerdict() != ServiceVerdict::TRAFFIC_VERDICT_INJECT && waapStateTable->hasState() ) { finishTransaction(waf2Transaction, verdict); @@ -413,7 +444,7 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event) header_value.size() ); - ngx_http_cp_verdict_e verdict = pending_response.getVerdict(); + ServiceVerdict verdict = pending_response.getVerdict(); HttpHeaderModification modifications; std::string webUserResponseByPractice; bool isSecurityHeadersInjected = false; @@ -441,7 +472,7 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event) } } isSecurityHeadersInjected = true; - verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT; + verdict = ServiceVerdict::TRAFFIC_VERDICT_INJECT; } } @@ -461,7 +492,7 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event) << ". Error: " << result.getErr(); } - verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT; + verdict = ServiceVerdict::TRAFFIC_VERDICT_INJECT; } } @@ -476,15 +507,15 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event) } if (waf2Transaction.shouldInjectSecurityHeaders() && isSecurityHeadersInjected && - verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT + verdict == ServiceVerdict::TRAFFIC_VERDICT_INJECT ) { // disable should inject security headers after injection to avoid response body scanning when it's unnecessary waf2Transaction.disableShouldInjectSecurityHeaders(); } EventVerdict eventVedict(move(modifications.getModificationList()), verdict); // Delete state before returning any verdict which is not pending - if (verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT && - verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT && + if (verdict != ServiceVerdict::TRAFFIC_VERDICT_INSPECT && + verdict != ServiceVerdict::TRAFFIC_VERDICT_INJECT && waapStateTable->hasState() ) { finishTransaction(waf2Transaction, eventVedict); @@ -530,7 +561,7 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event) waf2Transaction.add_response_body_chunk(dataBuf, dataBufLen); - ngx_http_cp_verdict_e verdict = pending_response.getVerdict(); + ServiceVerdict verdict = pending_response.getVerdict(); HttpBodyModification modifications; std::string webUserResponseByPractice; @@ -571,7 +602,7 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event) if(!result.ok()) { dbgWarning(D_WAAP) << "HttpBodyResponse(): Scripts injection failed!"; } - verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT; + verdict = ServiceVerdict::TRAFFIC_VERDICT_INJECT; } else { // This response body is not considered "HTML" - disable injection dbgTrace(D_WAAP) << "HttpBodyResponse(): the response body is not HTML - disabling injection"; @@ -589,8 +620,8 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event) } EventVerdict eventVedict(modifications.getModificationList(), verdict); // Delete state before returning any verdict which is not pending or inject - if (verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT && - verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT && + if (verdict != ServiceVerdict::TRAFFIC_VERDICT_INSPECT && + verdict != ServiceVerdict::TRAFFIC_VERDICT_INJECT && waapStateTable->hasState() ) { finishTransaction(waf2Transaction, eventVedict); diff --git a/components/signal_handler/signal_handler.cc b/components/signal_handler/signal_handler.cc index 37e8792..b53d2c4 100755 --- a/components/signal_handler/signal_handler.cc +++ b/components/signal_handler/signal_handler.cc @@ -339,7 +339,7 @@ private: char signal_num[3]; snprintf(signal_num, sizeof(signal_num), "%d", _signal); - if (out_trace_file_fd == -1) exit(_signal); + if (out_trace_file_fd == -1) _exit(_signal); reset_signal_handler = true; @@ -383,7 +383,7 @@ private: close(out_trace_file_fd); out_trace_file_fd = -1; - exit(_signal); + _exit(_signal); } static void diff --git a/components/utils/generic_rulebase/evaluators/parameter_eval.cc b/components/utils/generic_rulebase/evaluators/parameter_eval.cc index 2649133..e4881e6 100644 --- a/components/utils/generic_rulebase/evaluators/parameter_eval.cc +++ b/components/utils/generic_rulebase/evaluators/parameter_eval.cc @@ -46,6 +46,6 @@ ParameterMatcher::evalVariable() const dbgTrace(D_RULEBASE_CONFIG) << "Did not find current parameter in context." << " Match parameter from current rule"; - auto rule = getConfiguration("rulebase", "rulesConfig"); + auto rule = getConfigurationWithCache("rulebase", "rulesConfig"); return rule.ok() && rule.unpack().isParameterActive(parameter_id); } diff --git a/components/utils/generic_rulebase/evaluators/practice_eval.cc b/components/utils/generic_rulebase/evaluators/practice_eval.cc index fc570e1..186ab97 100644 --- a/components/utils/generic_rulebase/evaluators/practice_eval.cc +++ b/components/utils/generic_rulebase/evaluators/practice_eval.cc @@ -45,6 +45,6 @@ PracticeMatcher::evalVariable() const return bc_practice_id_ctx.unpack().count(practice_id) > 0; } - auto rule = getConfiguration("rulebase", "rulesConfig"); + auto rule = getConfigurationWithCache("rulebase", "rulesConfig"); return rule.ok() && rule.unpack().isPracticeActive(practice_id); } diff --git a/components/utils/generic_rulebase/evaluators/trigger_eval.cc b/components/utils/generic_rulebase/evaluators/trigger_eval.cc index 7e15b47..89242b2 100644 --- a/components/utils/generic_rulebase/evaluators/trigger_eval.cc +++ b/components/utils/generic_rulebase/evaluators/trigger_eval.cc @@ -52,6 +52,6 @@ TriggerMatcher::evalVariable() const << makeSeparatedStr(bc_trigger_id_ctx.ok() ? *bc_trigger_id_ctx : set(), ", "); if (bc_trigger_id_ctx.ok()) return bc_trigger_id_ctx.unpack().count(trigger_id) > 0; - auto rule = getConfiguration("rulebase", "rulesConfig"); + auto rule = getConfigurationWithCache("rulebase", "rulesConfig"); return rule.ok() && rule.unpack().isTriggerActive(trigger_id); } diff --git a/components/utils/generic_rulebase/generic_rulebase.cc b/components/utils/generic_rulebase/generic_rulebase.cc index ed13b7c..85889fe 100644 --- a/components/utils/generic_rulebase/generic_rulebase.cc +++ b/components/utils/generic_rulebase/generic_rulebase.cc @@ -104,7 +104,8 @@ GenericRulebase::Impl::getLogTriggerConf(const string& trigger_Id) const ScopedContext ctx; set triggers = {trigger_Id}; ctx.registerValue>(TriggerMatcher::ctx_key, triggers); - return getConfigurationWithDefault(LogTriggerConf(), "rulebase", "log"); + auto config = getConfigurationWithCache("rulebase", "log"); + return config.ok() ? config.unpack() : LogTriggerConf(); } ParameterException @@ -113,19 +114,20 @@ GenericRulebase::Impl::getParameterException(const string& parameter_Id) const ScopedContext ctx; set exceptions = {parameter_Id}; ctx.registerValue>(ParameterMatcher::ctx_key, exceptions); - return getConfigurationWithDefault(ParameterException(), "rulebase", "exception"); + auto config = getConfigurationWithCache("rulebase", "exception"); + return config.ok() ? config.unpack() : ParameterException(); } set GenericRulebase::Impl::getBehavior(const ParameterKeyValues &key_value_pairs) const { - auto &exceptions = getConfiguration("rulebase", "exception"); + auto exceptions = getConfigurationWithCache("rulebase", "exception"); if (!exceptions.ok()) { dbgTrace(D_RULEBASE_CONFIG) << "Could not find any exception with the current rule's context"; return {}; } - return (*exceptions).getBehavior(key_value_pairs); + return exceptions.unpack().getBehavior(key_value_pairs); } GenericRulebase::GenericRulebase() : Component("GenericRulebase"), pimpl(make_unique()) {} diff --git a/components/utils/generic_rulebase/generic_rulebase_context.cc b/components/utils/generic_rulebase/generic_rulebase_context.cc index 7dd98cb..a267138 100644 --- a/components/utils/generic_rulebase/generic_rulebase_context.cc +++ b/components/utils/generic_rulebase/generic_rulebase_context.cc @@ -64,10 +64,14 @@ GenericRulebaseContext::activate(const BasicRuleConfig &rule) ZoneMatcher::ctx_key, rule.getZoneId() ); - ctx.registerValue( + ctx.registerQuickAccessValue( AssetMatcher::ctx_key, rule.getAssetId() ); + ctx.registerQuickAccessValue( + TriggerMatcher::ctx_key, + rule.getAssetId() + ); ctx.activate(); break; } @@ -87,7 +91,7 @@ GenericRulebaseContext::activate() { switch(registration_state) { case RuleRegistrationState::UNINITIALIZED: { - auto maybe_rule = getConfiguration("rulebase", "rulesConfig"); + auto maybe_rule = setConfigurationInCache("rulebase", "rulesConfig"); if (!maybe_rule.ok()) { registration_state = RuleRegistrationState::UNREGISTERED; return; diff --git a/components/utils/generic_rulebase/match_query.cc b/components/utils/generic_rulebase/match_query.cc index 17584e4..a187e44 100644 --- a/components/utils/generic_rulebase/match_query.cc +++ b/components/utils/generic_rulebase/match_query.cc @@ -244,7 +244,8 @@ MatchQuery::getAllKeys() const bool MatchQuery::matchAttributes( const unordered_map> &key_value_pairs, - set &matched_override_keywords) const + set &matched_override_keywords, + bool skip_irrelevant_key) const { dbgTrace(D_RULEBASE_CONFIG) << "Start matching attributes"; @@ -256,25 +257,47 @@ MatchQuery::matchAttributes( } return matchAttributes(key_value_pair->second, matched_override_keywords); } else if (type == MatchType::Operator && operator_type == Operators::And) { - for (const MatchQuery &inner_match: items) { - if (!inner_match.matchAttributes(key_value_pairs, matched_override_keywords)) { + bool has_relevant_keys = false; + for (const MatchQuery &inner_match : items) { + // Skip irrelevant keys when skip_irrelevant_key is false + if (skip_irrelevant_key && key_value_pairs.find(inner_match.getKey()) == key_value_pairs.end()) { + dbgTrace(D_RULEBASE_CONFIG) << "Skipping irrelevant key in AND operator"; + continue; + } + has_relevant_keys = true; + if (!inner_match.matchAttributes(key_value_pairs, matched_override_keywords, skip_irrelevant_key)) { dbgTrace(D_RULEBASE_CONFIG) << "Failed to match attributes for AND operator"; return false; } } + if (!has_relevant_keys) { + dbgTrace(D_RULEBASE_CONFIG) << "No relevant keys found for AND operator"; + return false; + } dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched all inner matches for AND operator"; return true; } else if (type == MatchType::Operator && operator_type == Operators::Or) { // With 'or' condition, evaluate matched override keywords first and add the ones that were fully matched set inner_override_keywords; bool res = false; + bool has_relevant_keys = false; for (const MatchQuery &inner_match: items) { + // Skip irrelevant keys when skip_irrelevant_key is false + if (skip_irrelevant_key && key_value_pairs.find(inner_match.getKey()) == key_value_pairs.end()) { + dbgTrace(D_RULEBASE_CONFIG) << "Skipping irrelevant key in OR operator"; + continue; + } + has_relevant_keys = true; inner_override_keywords.clear(); - if (inner_match.matchAttributes(key_value_pairs, inner_override_keywords)) { + if (inner_match.matchAttributes(key_value_pairs, inner_override_keywords, skip_irrelevant_key)) { matched_override_keywords.insert(inner_override_keywords.begin(), inner_override_keywords.end()); res = true; } } + if (!has_relevant_keys) { + dbgTrace(D_RULEBASE_CONFIG) << "No relevant keys found for OR operator"; + return false; + } dbgTrace(D_RULEBASE_CONFIG) << "Match result for OR operator is: " << res; return res; } else { @@ -284,20 +307,23 @@ MatchQuery::matchAttributes( } MatchQuery::MatchResult -MatchQuery::getMatch( const unordered_map> &key_value_pairs) const +MatchQuery::getMatch( + const unordered_map> &key_value_pairs, + bool skip_irrelevant_key) const { MatchQuery::MatchResult matches; matches.matched_keywords = make_shared>(); - matches.is_match = matchAttributes(key_value_pairs, *matches.matched_keywords); + matches.is_match = matchAttributes(key_value_pairs, *matches.matched_keywords, skip_irrelevant_key); dbgTrace(D_RULEBASE_CONFIG) << "Match result: " << matches.is_match; return matches; } bool MatchQuery::matchAttributes( - const unordered_map> &key_value_pairs) const + const unordered_map> &key_value_pairs, + bool skip_irrelevant_key) const { - return getMatch(key_value_pairs).is_match; + return getMatch(key_value_pairs, skip_irrelevant_key).is_match; } bool diff --git a/components/utils/generic_rulebase/parameters_config.cc b/components/utils/generic_rulebase/parameters_config.cc index 8c0d953..986c94d 100644 --- a/components/utils/generic_rulebase/parameters_config.cc +++ b/components/utils/generic_rulebase/parameters_config.cc @@ -105,19 +105,22 @@ ParameterException::load(cereal::JSONInputArchive &archive_in) for (const MatchBehaviorPair &match_query : match_queries) { if (isGeoLocationExists(match_query.match)) return; } + + is_containing_kv_pair = checkKVPair(); } set ParameterException::getBehavior( - const unordered_map> &key_value_pairs, - set &matched_override_keywords) const + const unordered_map> &key_value_pairs, + set &matched_override_keywords, + bool skip_irrelevant_key) const { set matched_behaviors; matched_override_keywords.clear(); dbgTrace(D_RULEBASE_CONFIG) << "Matching exception"; for (const MatchBehaviorPair &match_behavior_pair: match_queries) { - MatchQuery::MatchResult match_res = match_behavior_pair.match.getMatch(key_value_pairs); + MatchQuery::MatchResult match_res = match_behavior_pair.match.getMatch(key_value_pairs, skip_irrelevant_key); if (match_res.is_match) { dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception from a list of matches, behavior: " @@ -136,7 +139,7 @@ ParameterException::getBehavior( } if (match_queries.empty()) { - MatchQuery::MatchResult match_res = match.getMatch(key_value_pairs); + MatchQuery::MatchResult match_res = match.getMatch(key_value_pairs, skip_irrelevant_key); if (match_res.is_match) { dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception."; // When matching indicators with action=ignore, we expect no behavior override. @@ -156,8 +159,70 @@ ParameterException::getBehavior( } set -ParameterException::getBehavior(const unordered_map> &key_value_pairs) const +ParameterException::getBehavior( + const unordered_map> &key_value_pairs, + bool skip_irrelevant_key) const { set keywords; // placeholder only, this function will be used where there's no need for ignored keywords - return getBehavior(key_value_pairs, keywords); + return getBehavior(key_value_pairs, keywords, skip_irrelevant_key); +} + +static bool +checkMatchQueryForKVPair(const MatchQuery &query) +{ + if (query.getType() == MatchQuery::MatchType::Condition) { + return false; + } + + if (query.getType() == MatchQuery::MatchType::Operator) { + if (query.getOperatorType() == MatchQuery::Operators::And) { + set found_keys; + + for (const MatchQuery &item : query.getItems()) { + if (item.getType() == MatchQuery::MatchType::Condition) { + found_keys.insert(item.getKey()); + } + } + + bool hasParamName = found_keys.find("paramName") != found_keys.end(); + bool hasParamValue = found_keys.find("paramValue") != found_keys.end(); + + if (hasParamName && hasParamValue) { + return true; + } + + bool hasHeaderName = found_keys.find("headerName") != found_keys.end(); + bool hasHeaderValue = found_keys.find("headerValue") != found_keys.end(); + + if (hasHeaderName && hasHeaderValue) { + return true; + } + } + + for (const MatchQuery &item : query.getItems()) { + if (item.getType() == MatchQuery::MatchType::Operator) { + if (checkMatchQueryForKVPair(item)) { + return true; + } + } + } + } + + return false; +} + +bool +ParameterException::checkKVPair() const +{ + if (checkMatchQueryForKVPair(match)) { + return true; + } + + for (const MatchBehaviorPair &match_behavior_pair : match_queries) { + if (checkMatchQueryForKVPair(match_behavior_pair.match)) { + return true; + } + } + + return false; } diff --git a/components/utils/generic_rulebase/triggers_config.cc b/components/utils/generic_rulebase/triggers_config.cc index 930dc15..f86ac1a 100644 --- a/components/utils/generic_rulebase/triggers_config.cc +++ b/components/utils/generic_rulebase/triggers_config.cc @@ -59,6 +59,7 @@ WebTriggerConf::load(cereal::JSONInputArchive &archive_in) parseJSONKey("response body", response_body, archive_in); parseJSONKey("response title", response_title, archive_in); + parseJSONKey("content type", content_type, archive_in); } catch (const exception &e) { dbgWarning(D_RULEBASE_CONFIG) << "Failed to parse the web trigger configuration: '" << e.what() << "'"; archive_in.setNextName(nullptr); @@ -185,6 +186,7 @@ LogTriggerConf::load(cereal::JSONInputArchive& archive_in) setTriggersFlag("acLogGeoLocation", archive_in, SecurityType::AccessControl, log_geo_location); setTriggersFlag("tpLogGeoLocation", archive_in, SecurityType::ThreatPrevention, log_geo_location); setTriggersFlag("complianceLogGeoLocation", archive_in, SecurityType::Compliance, log_geo_location); + setTriggersFlag("shouldIgnoreExceptionLog", archive_in, SecurityType::ThreatPrevention, should_log_exception); bool extend_logging = false; parseJSONKey("extendLogging", extend_logging, archive_in); diff --git a/components/utils/geo_location/geo_location.cc b/components/utils/geo_location/geo_location.cc index 1fab700..88ead34 100644 --- a/components/utils/geo_location/geo_location.cc +++ b/components/utils/geo_location/geo_location.cc @@ -78,7 +78,7 @@ public: MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&max_mind_db_obj, &sockaddr_to_search, &maxminddb_error); if (maxminddb_error != MMDB_SUCCESS) { - dbgWarning(D_GEO_DB) << "maxMindDB error: " << MMDB_strerror(maxminddb_error); + dbgDebug(D_GEO_DB) << "maxMindDB error: " << MMDB_strerror(maxminddb_error); return genError("maxMindDB error: " + string(MMDB_strerror(maxminddb_error))); } if (result.found_entry) { @@ -153,9 +153,9 @@ private: } } if (status != MMDB_SUCCESS) { - dbgWarning(D_GEO_DB) << "maxMindDB error: " << MMDB_strerror(status); + dbgDebug(D_GEO_DB) << "maxMindDB error: " << MMDB_strerror(status); } else if (!entry_data.has_data) { - dbgWarning(D_GEO_DB) << "maxMindDB Entry has no data"; + dbgDebug(D_GEO_DB) << "maxMindDB Entry has no data"; } else { string search_result(entry_data.utf8_string, entry_data.data_size); return search_result; diff --git a/components/utils/http_transaction_data/http_transaction_data.cc b/components/utils/http_transaction_data/http_transaction_data.cc index ec9c5ee..a54ad99 100644 --- a/components/utils/http_transaction_data/http_transaction_data.cc +++ b/components/utils/http_transaction_data/http_transaction_data.cc @@ -19,7 +19,7 @@ #include "enum_array.h" #include "buffer.h" -#include "nginx_attachment_common.h" +#include "nano_attachment_common.h" using namespace std; diff --git a/components/utils/pm/CMakeLists.txt b/components/utils/pm/CMakeLists.txt index a953975..3e6e6de 100644 --- a/components/utils/pm/CMakeLists.txt +++ b/components/utils/pm/CMakeLists.txt @@ -1,3 +1,3 @@ -add_library(pm general_adaptor.cc kiss_hash.cc kiss_patterns.cc kiss_pm_stats.cc kiss_thin_nfa.cc kiss_thin_nfa_analyze.cc kiss_thin_nfa_build.cc kiss_thin_nfa_compile.cc pm_adaptor.cc pm_hook.cc debugpm.cc) +add_library(pm general_adaptor.cc kiss_hash.cc kiss_patterns.cc kiss_pm_stats.cc kiss_thin_nfa.cc kiss_thin_nfa_analyze.cc kiss_thin_nfa_build.cc kiss_thin_nfa_compile.cc pm_adaptor.cc pm_hook.cc debugpm.cc hyperscan_hook.cc) add_subdirectory(pm_ut) diff --git a/components/utils/pm/hyperscan_hook.cc b/components/utils/pm/hyperscan_hook.cc new file mode 100644 index 0000000..9f00073 --- /dev/null +++ b/components/utils/pm/hyperscan_hook.cc @@ -0,0 +1,135 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifdef USE_HYPERSCAN +#include "hyperscan_hook.h" +#include +#include + +// Helper function to escape regex special characters for literal matching +static std::string escapeRegexChars(const std::string& input) { + std::string escaped; + for (char c : input) { + switch (c) { + case '.': + case '^': + case '$': + case '*': + case '+': + case '?': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '\\': + case '|': + escaped += '\\'; + escaped += c; + break; + default: + escaped += c; + break; + } + } + return escaped; +} + +HyperscanHook::HyperscanHook() : m_hsDatabase(nullptr), m_hsScratch(nullptr), m_hsReady(false) {} + +HyperscanHook::~HyperscanHook() { + if (m_hsScratch) hs_free_scratch(m_hsScratch); + if (m_hsDatabase) hs_free_database(m_hsDatabase); +} + +Maybe HyperscanHook::prepare(const std::set &patterns) { + m_hsPatterns.clear(); + m_idToPattern.clear(); for (const auto &pat : patterns) { + if (pat.empty()) continue; // Use pat.empty() instead of pat.pattern().empty() + + // Convert pattern data to string using the public interface + std::string pattern_str(reinterpret_cast(pat.data()), pat.size()); + + // Escape regex special characters for literal matching + std::string escaped_pattern = escapeRegexChars(pattern_str); + + m_hsPatterns.push_back(escaped_pattern); + m_idToPattern.push_back(pat); + } + + std::vector c_patterns; + std::vector flags; + std::vector ids; + + for (size_t i = 0; i < m_hsPatterns.size(); ++i) { + c_patterns.push_back(m_hsPatterns[i].c_str()); + flags.push_back(HS_FLAG_CASELESS); // adjust as needed + ids.push_back((unsigned int)i); + } hs_compile_error_t *compile_err = nullptr; + hs_error_t result = hs_compile_multi(c_patterns.data(), flags.data(), ids.data(), + (unsigned int)c_patterns.size(), HS_MODE_BLOCK, nullptr, + &m_hsDatabase, &compile_err); + + if (result != HS_SUCCESS) { + std::string error_msg = "Failed to compile Hyperscan database"; + if (compile_err) { + error_msg += ": "; + error_msg += compile_err->message; + hs_free_compile_error(compile_err); + } + return genError(error_msg); + }if (hs_alloc_scratch(m_hsDatabase, &m_hsScratch) != HS_SUCCESS) { + return genError("Failed to allocate Hyperscan scratch space"); + } + + m_hsReady = true; + return Maybe(); +} + +// TODO - No need for HS hook, scanning is done by WaapHyperscanEngine::scanSample() +std::set HyperscanHook::scanBuf(const Buffer &buf) const { + std::set results; + scanBufWithOffsetLambda(buf, [&results](uint, const PMPattern &pattern, bool) { + results.insert(pattern); + }); + return results; +} + +std::set> HyperscanHook::scanBufWithOffset(const Buffer &buf) const { + std::set> results; + scanBufWithOffsetLambda(buf, [&results](uint endMatchOffset, const PMPattern &pattern, bool) { + uint startOffset = endMatchOffset + 1 - pattern.size(); + results.insert(std::make_pair(startOffset, endMatchOffset)); + }); + return results; +} + +void HyperscanHook::scanBufWithOffsetLambda(const Buffer &buf, I_PMScan::CBFunction cb) const { + if (!m_hsReady) return; + struct HyperScanContext { + const HyperscanHook *self; + const Buffer *buffer; + I_PMScan::CBFunction cb; + }; + auto onMatch = [](unsigned int id, unsigned long long, unsigned long long to, unsigned int, + void *ctx) -> int { + HyperScanContext *hctx = (HyperScanContext*)ctx; + const HyperscanHook *self = hctx->self; + const PMPattern &pat = self->m_idToPattern[id]; + uint endMatchOffset = (uint)to - 1; + hctx->cb(endMatchOffset, pat, false); // matchAll logic can be extended if needed + return 0; + }; + HyperScanContext ctx{this, &buf, cb}; + hs_scan(m_hsDatabase, (const char*)buf.data(), (unsigned int)buf.size(), 0, m_hsScratch, onMatch, &ctx); +} + +#endif // USE_HYPERSCAN diff --git a/components/utils/utilities/nginx_conf_collector/CMakeLists.txt b/components/utils/utilities/nginx_conf_collector/CMakeLists.txt index 6e56c58..e362854 100755 --- a/components/utils/utilities/nginx_conf_collector/CMakeLists.txt +++ b/components/utils/utilities/nginx_conf_collector/CMakeLists.txt @@ -32,6 +32,7 @@ target_link_libraries(${EXECUTABLE_NAME} boost_context boost_regex pthread + ${Brotli_LIBRARIES} ) install(TARGETS ${EXECUTABLE_NAME} DESTINATION bin) diff --git a/core/agent_core_utilities/agent_core_utilities.cc b/core/agent_core_utilities/agent_core_utilities.cc index 4e6b63f..bd6c735 100644 --- a/core/agent_core_utilities/agent_core_utilities.cc +++ b/core/agent_core_utilities/agent_core_utilities.cc @@ -150,6 +150,42 @@ makeDirRecursive(const string &path, mode_t permission) return true; } +bool +createFileWithContent(const string &dest, const string &content, bool overide_if_exists, mode_t permission) +{ + dbgFlow(D_INFRA_UTILS) + << "Trying to create file with content. Destination: " + << dest + << ", Content size: " + << content.size() + << ", Should override: " + << (overide_if_exists? "true" : "false") + << ", permission: " + << to_string(permission); + + if (exists(dest) && !overide_if_exists) { + dbgDebug(D_INFRA_UTILS) << "Failed to create file. Error: destination file already exists"; + return false; + } + + ofstream file(dest, ios::out | ios::trunc); + if (!file.is_open()) { + dbgDebug(D_INFRA_UTILS) << "Failed to create file. Error: could not open destination file"; + return false; + } + + file << content; + file.close(); + + if (chmod(dest.c_str(), permission) != 0) { + dbgWarning(D_INFRA_UTILS) << "Failed to set file permissions. Path: " << dest; + // Don't return false here as the file was created successfully + } + + dbgTrace(D_INFRA_UTILS) << "Successfully created file with content. Path: " << dest; + return true; +} + bool copyFile(const string &src, const string &dest, bool overide_if_exists, mode_t permission) { @@ -546,6 +582,13 @@ toLower(string str) return str; } +bool startsWith(const std::string& str, const std::string& prefix) { + if (prefix.size() > str.size()) { + return false; + } + return std::equal(prefix.begin(), prefix.end(), str.begin()); +} + } // namespace Strings diff --git a/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc b/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc index 89d90d2..abe252a 100644 --- a/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc +++ b/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc @@ -254,3 +254,226 @@ TEST_F(AgentCoreUtilUT, regexReplaceTest) EXPECT_EQ(replaced, testCase.expected); } } + +TEST_F(AgentCoreUtilUT, createFileWithContentTest) +{ + // Test basic file creation with content + string test_file_path = cptestFnameInExeDir("test_create_file.txt"); + string content = "Hello, World!\nThis is test content."; + unsigned int permissions = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH; + + EXPECT_TRUE(NGEN::Filesystem::createFileWithContent(test_file_path, content, false, permissions)); + EXPECT_TRUE(NGEN::Filesystem::exists(test_file_path)); + + // Verify content was written correctly + ifstream file_stream(test_file_path); + ASSERT_TRUE(file_stream.good()); + stringstream buffer; + buffer << file_stream.rdbuf(); + string read_content = buffer.str(); + file_stream.close(); + + EXPECT_EQ(read_content, content); + EXPECT_TRUE(NGEN::Filesystem::deleteFile(test_file_path)); +} + +TEST_F(AgentCoreUtilUT, createFileWithContentEmptyContentTest) +{ + // Test creating file with empty content + string test_file_path = cptestFnameInExeDir("test_empty_file.txt"); + string empty_content = ""; + unsigned int permissions = S_IRUSR | S_IWUSR; + + EXPECT_TRUE(NGEN::Filesystem::createFileWithContent(test_file_path, empty_content, false, permissions)); + EXPECT_TRUE(NGEN::Filesystem::exists(test_file_path)); + + // Verify file is empty + ifstream file_stream(test_file_path); + ASSERT_TRUE(file_stream.good()); + stringstream buffer; + buffer << file_stream.rdbuf(); + string read_content = buffer.str(); + file_stream.close(); + + EXPECT_EQ(read_content, ""); + EXPECT_TRUE(NGEN::Filesystem::deleteFile(test_file_path)); +} + +TEST_F(AgentCoreUtilUT, createFileWithContentLargeContentTest) +{ + // Test creating file with large content + string test_file_path = cptestFnameInExeDir("test_large_file.txt"); + string large_content; + + // Create large content (10KB) + for (int i = 0; i < 1000; ++i) { + large_content += "0123456789"; + } + + unsigned int permissions = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; + + EXPECT_TRUE(NGEN::Filesystem::createFileWithContent(test_file_path, large_content, false, permissions)); + EXPECT_TRUE(NGEN::Filesystem::exists(test_file_path)); + + // Verify content was written correctly + ifstream file_stream(test_file_path); + ASSERT_TRUE(file_stream.good()); + stringstream buffer; + buffer << file_stream.rdbuf(); + string read_content = buffer.str(); + file_stream.close(); + + EXPECT_EQ(read_content.size(), large_content.size()); + EXPECT_EQ(read_content, large_content); + EXPECT_TRUE(NGEN::Filesystem::deleteFile(test_file_path)); +} + +TEST_F(AgentCoreUtilUT, createFileWithContentOverwriteFalseTest) +{ + // Test that overwrite=false prevents overwriting existing file + string test_file_path = cptestFnameInExeDir("test_no_overwrite.txt"); + string original_content = "Original content"; + string new_content = "New content that should not be written"; + unsigned int permissions = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP; + + // Create initial file + EXPECT_TRUE(NGEN::Filesystem::createFileWithContent(test_file_path, original_content, false, permissions)); + EXPECT_TRUE(NGEN::Filesystem::exists(test_file_path)); + + // Try to overwrite with overwrite=false (should fail) + EXPECT_FALSE(NGEN::Filesystem::createFileWithContent(test_file_path, new_content, false, permissions)); + + // Verify original content is preserved + ifstream file_stream(test_file_path); + ASSERT_TRUE(file_stream.good()); + stringstream buffer; + buffer << file_stream.rdbuf(); + string read_content = buffer.str(); + file_stream.close(); + + EXPECT_EQ(read_content, original_content); + EXPECT_TRUE(NGEN::Filesystem::deleteFile(test_file_path)); +} + +TEST_F(AgentCoreUtilUT, createFileWithContentOverwriteTrueTest) +{ + // Test that overwrite=true allows overwriting existing file + string test_file_path = cptestFnameInExeDir("test_with_overwrite.txt"); + string original_content = "Original content"; + string new_content = "New content that should overwrite"; + unsigned int permissions = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP; + + // Create initial file + EXPECT_TRUE(NGEN::Filesystem::createFileWithContent(test_file_path, original_content, false, permissions)); + EXPECT_TRUE(NGEN::Filesystem::exists(test_file_path)); + + // Overwrite with overwrite=true (should succeed) + EXPECT_TRUE(NGEN::Filesystem::createFileWithContent(test_file_path, new_content, true, permissions)); + + // Verify new content was written + ifstream file_stream(test_file_path); + ASSERT_TRUE(file_stream.good()); + stringstream buffer; + buffer << file_stream.rdbuf(); + string read_content = buffer.str(); + file_stream.close(); + + EXPECT_EQ(read_content, new_content); + EXPECT_TRUE(NGEN::Filesystem::deleteFile(test_file_path)); +} + +TEST_F(AgentCoreUtilUT, createFileWithContentDifferentPermissionsTest) +{ + // Test creating files with different permission sets + string test_file_path = cptestFnameInExeDir("test_permissions.txt"); + string content = "Content for permission test"; + + // Test read-only permissions + unsigned int read_only_permissions = S_IRUSR | S_IRGRP | S_IROTH; + EXPECT_TRUE(NGEN::Filesystem::createFileWithContent(test_file_path, content, false, read_only_permissions)); + EXPECT_TRUE(NGEN::Filesystem::exists(test_file_path)); + EXPECT_TRUE(NGEN::Filesystem::deleteFile(test_file_path)); + + // Test read-write permissions + unsigned int read_write_permissions = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP; + EXPECT_TRUE(NGEN::Filesystem::createFileWithContent(test_file_path, content, false, read_write_permissions)); + EXPECT_TRUE(NGEN::Filesystem::exists(test_file_path)); + EXPECT_TRUE(NGEN::Filesystem::deleteFile(test_file_path)); + + // Test full permissions + unsigned int full_permissions; + full_permissions = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | S_IWOTH | S_IXOTH; + EXPECT_TRUE(NGEN::Filesystem::createFileWithContent(test_file_path, content, false, full_permissions)); + EXPECT_TRUE(NGEN::Filesystem::exists(test_file_path)); + EXPECT_TRUE(NGEN::Filesystem::deleteFile(test_file_path)); +} + +TEST_F(AgentCoreUtilUT, createFileWithContentSpecialCharactersTest) +{ + // Test creating file with special characters in content + string test_file_path = cptestFnameInExeDir("test_special_chars.txt"); + string content = "Special chars: !@#$%^&*()[]{}|\\:;\"'<>?,./\n\t\r\n"; + content += "Unicode: \u00A9 \u00AE \u2122\n"; + content += "Null byte in middle: before\0after\n"; + unsigned int permissions = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP; + + EXPECT_TRUE(NGEN::Filesystem::createFileWithContent(test_file_path, content, false, permissions)); + EXPECT_TRUE(NGEN::Filesystem::exists(test_file_path)); + + // Verify content (note: null byte will terminate string reading early) + ifstream file_stream(test_file_path, ios::binary); + ASSERT_TRUE(file_stream.good()); + stringstream buffer; + buffer << file_stream.rdbuf(); + string read_content = buffer.str(); + file_stream.close(); + + // Content should match up to the special characters + EXPECT_THAT(read_content, HasSubstr("Special chars: !@#$%^&*()[]{}|\\:;\"'<>?,./")); + EXPECT_TRUE(NGEN::Filesystem::deleteFile(test_file_path)); +} + +TEST_F(AgentCoreUtilUT, createFileWithContentInvalidPathTest) +{ + // Test creating file with invalid/inaccessible path + string invalid_path = "/root/inaccessible/path/test_file.txt"; + string content = "This should fail"; + unsigned int permissions = S_IRUSR | S_IWUSR; + + // This should fail due to invalid/inaccessible path + EXPECT_FALSE(NGEN::Filesystem::createFileWithContent(invalid_path, content, false, permissions)); + EXPECT_FALSE(NGEN::Filesystem::exists(invalid_path)); +} + +TEST_F(AgentCoreUtilUT, createFileWithContentCreateDirectoryTest) +{ + // Test creating file in a subdirectory (should create intermediate directories) + string test_dir = cptestFnameInExeDir("test_subdir"); + string test_file_path = test_dir + "/nested_file.txt"; + string content = "Content in nested directory"; + unsigned int permissions = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP; + + // Ensure directory doesn't exist initially + EXPECT_FALSE(NGEN::Filesystem::exists(test_dir)); + + // Create directory first + EXPECT_TRUE(NGEN::Filesystem::makeDir(test_dir)); + + // Now create file in the directory + EXPECT_TRUE(NGEN::Filesystem::createFileWithContent(test_file_path, content, false, permissions)); + EXPECT_TRUE(NGEN::Filesystem::exists(test_file_path)); + + // Verify content + ifstream file_stream(test_file_path); + ASSERT_TRUE(file_stream.good()); + stringstream buffer; + buffer << file_stream.rdbuf(); + string read_content = buffer.str(); + file_stream.close(); + + EXPECT_EQ(read_content, content); + + // Cleanup + EXPECT_TRUE(NGEN::Filesystem::deleteFile(test_file_path)); + EXPECT_TRUE(NGEN::Filesystem::deleteDirectory(test_dir)); +} diff --git a/core/agent_details/agent_details.cc b/core/agent_details/agent_details.cc index b817b12..56df2f1 100644 --- a/core/agent_details/agent_details.cc +++ b/core/agent_details/agent_details.cc @@ -19,6 +19,7 @@ #include #include #include +#include "i_messaging.h" #include "config.h" #include "debug.h" @@ -64,19 +65,37 @@ AgentDetails::init() writeAgentDetails(); } + previous_proxy = proxy; + registerConfigLoadCb( [&]() { - auto proxy_config = getProfileAgentSetting("agent.config.message.proxy"); - if (proxy_config.ok()) { - is_proxy_configured_via_settings = true; - setProxy(*proxy_config); - writeAgentDetails(); - } else if (is_proxy_configured_via_settings) { - is_proxy_configured_via_settings = false; - setProxy(string("")); - writeAgentDetails(); + auto load_env_proxy = loadProxy(); + auto proxy_config = getProxy(); + if (proxy != previous_proxy) { + dbgInfo(D_ORCHESTRATOR) + << "Proxy configuration changed from '" + << previous_proxy + << "' to '" + << proxy + << "'"; + auto messaging = Singleton::Consume::by(); + messaging->clearConnections(); } + if (!proxy_config.ok() || proxy_config.unpack() == "none") { + auto proxy_config = getProfileAgentSetting("agent.config.message.proxy"); + if (proxy_config.ok()) { + is_proxy_configured_via_settings = true; + setProxy(*proxy_config); + writeAgentDetails(); + } else if (is_proxy_configured_via_settings) { + is_proxy_configured_via_settings = false; + setProxy(string("")); + writeAgentDetails(); + } + } + + previous_proxy = proxy; } ); diff --git a/core/agent_details/agent_details_ut/agent_details_ut.cc b/core/agent_details/agent_details_ut/agent_details_ut.cc index 47f04ce..a0971d4 100644 --- a/core/agent_details/agent_details_ut/agent_details_ut.cc +++ b/core/agent_details/agent_details_ut/agent_details_ut.cc @@ -5,6 +5,7 @@ #include "mock/mock_encryptor.h" #include "mock/mock_shell_cmd.h" +#include "mock/mock_messaging.h" #include "mock/mock_mainloop.h" #include "cptest.h" #include "config.h" @@ -25,6 +26,7 @@ public: ::Environment env; ConfigComponent conf; + StrictMock mock_messaging; StrictMock mock_encryptor; StrictMock mock_shell_cmd; Config::I_Config *config = nullptr; diff --git a/core/agent_details_reporter/agent_details_reporter.cc b/core/agent_details_reporter/agent_details_reporter.cc index fe5b110..88f402b 100644 --- a/core/agent_details_reporter/agent_details_reporter.cc +++ b/core/agent_details_reporter/agent_details_reporter.cc @@ -179,6 +179,12 @@ AgentDetailsReporter::Impl::addAttr(const string &key, const string &val, bool a } } + if (val.empty()) { + deleteAttr(key); + dbgDebug(D_AGENT_DETAILS) << "Attribute " << key << " was empty, deleting"; + return true; + } + if (persistant_attributes[key] == val) { dbgDebug(D_AGENT_DETAILS) << "Attribute " << key << " did not change. Value: " << val; return true; diff --git a/core/attachments/http_configuration/http_configuration.cc b/core/attachments/http_configuration/http_configuration.cc index 18639ea..34f963c 100644 --- a/core/attachments/http_configuration/http_configuration.cc +++ b/core/attachments/http_configuration/http_configuration.cc @@ -106,7 +106,7 @@ HttpAttachmentConfiguration::save(cereal::JSONOutputArchive &archive) const "waiting_for_verdict_thread_timeout_msec", getNumericalValue("waiting_for_verdict_thread_timeout_msec") ), - cereal::make_nvp("nginx_inspection_mode", getNumericalValue("inspection_mode")), + cereal::make_nvp("nginx_inspection_mode", getNumericalValue("nginx_inspection_mode")), cereal::make_nvp("num_of_nginx_ipc_elements", getNumericalValue("num_of_nginx_ipc_elements")), cereal::make_nvp("keep_alive_interval_msec", getNumericalValue("keep_alive_interval_msec")), cereal::make_nvp("min_retries_for_verdict", getNumericalValue("min_retries_for_verdict")), @@ -114,7 +114,12 @@ HttpAttachmentConfiguration::save(cereal::JSONOutputArchive &archive) const cereal::make_nvp("hold_verdict_retries", getNumericalValue("hold_verdict_retries")), cereal::make_nvp("hold_verdict_polling_time", getNumericalValue("hold_verdict_polling_time")), cereal::make_nvp("body_size_trigger", getNumericalValue("body_size_trigger")), - cereal::make_nvp("remove_server_header", getNumericalValue("remove_server_header")) + cereal::make_nvp("remove_server_header", getNumericalValue("remove_server_header")), + cereal::make_nvp("decompression_pool_size", getNumericalValue("decompression_pool_size")), + cereal::make_nvp("recompression_pool_size", getNumericalValue("recompression_pool_size")), + cereal::make_nvp("is_paired_affinity_enabled", getNumericalValue("is_paired_affinity_enabled")), + cereal::make_nvp("is_async_mode_enabled", getNumericalValue("is_async_mode_enabled")), + cereal::make_nvp("is_brotli_inspection_enabled", getNumericalValue("is_brotli_inspection_enabled")) ); } @@ -173,6 +178,21 @@ HttpAttachmentConfiguration::load(cereal::JSONInputArchive &archive) loadNumericalValue(archive, "hold_verdict_polling_time", 1); loadNumericalValue(archive, "body_size_trigger", 200000); loadNumericalValue(archive, "remove_server_header", 0); + loadNumericalValue(archive, "decompression_pool_size", 262144); + loadNumericalValue(archive, "recompression_pool_size", 16384); + loadNumericalValue(archive, "is_paired_affinity_enabled", 0); + loadNumericalValue(archive, "is_brotli_inspection_enabled", 0); + + int g_env_async_mode = 1; + char *env_async_mode = getenv("CP_ASYNC_MODE"); + if (env_async_mode != NULL) { + if (strcmp(env_async_mode, "true") == 0 || strcmp(env_async_mode, "1") == 0) { + g_env_async_mode = 1; + } else { + g_env_async_mode = 0; + } + } + loadNumericalValue(archive, "is_async_mode_enabled", g_env_async_mode); } bool diff --git a/core/attachments/http_configuration/http_configuration_ut/http_configuration_ut.cc b/core/attachments/http_configuration/http_configuration_ut/http_configuration_ut.cc index dc415f9..b75d979 100644 --- a/core/attachments/http_configuration/http_configuration_ut/http_configuration_ut.cc +++ b/core/attachments/http_configuration/http_configuration_ut/http_configuration_ut.cc @@ -64,7 +64,12 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration) "\"req_header_thread_timeout_msec\": 10,\n" "\"ip_ranges\": " + createIPRangesString(ip_ranges) + ",\n" "\"static_resources_path\": \"" + static_resources_path + "\",\n" - "\"remove_server_header\": 0" + "\"remove_server_header\": 0,\n" + "\"decompression_pool_size\": 524288,\n" + "\"recompression_pool_size\": 32768,\n" + "\"is_paired_affinity_enabled\": 0,\n" + "\"is_async_mode_enabled\": 0,\n" + "\"is_brotli_inspection_enabled\": 1\n" "}\n"; ofstream valid_configuration_file(attachment_configuration_file_name); valid_configuration_file << valid_configuration; @@ -91,6 +96,11 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration) EXPECT_EQ(conf_data_out.getNumericalValue("waiting_for_verdict_thread_timeout_msec"), 60u); EXPECT_EQ(conf_data_out.getNumericalValue("nginx_inspection_mode"), 1u); EXPECT_EQ(conf_data_out.getNumericalValue("remove_server_header"), 0u); + EXPECT_EQ(conf_data_out.getNumericalValue("decompression_pool_size"), 524288u); + EXPECT_EQ(conf_data_out.getNumericalValue("recompression_pool_size"), 32768u); + EXPECT_EQ(conf_data_out.getNumericalValue("is_paired_affinity_enabled"), 0u); + EXPECT_EQ(conf_data_out.getNumericalValue("is_async_mode_enabled"), 0u); + EXPECT_EQ(conf_data_out.getNumericalValue("is_brotli_inspection_enabled"), 1u); } TEST_F(HttpAttachmentUtilTest, GetMalformedAttachmentConfiguration) diff --git a/core/compression/CMakeLists.txt b/core/compression/CMakeLists.txt index f9b5a18..1a30be1 100755 --- a/core/compression/CMakeLists.txt +++ b/core/compression/CMakeLists.txt @@ -4,6 +4,10 @@ add_definitions(-DZLIB_CONST) add_library(compression_utils SHARED compression_utils.cc) add_library(static_compression_utils compression_utils.cc) +target_link_libraries(compression_utils + ${Brotli_LIBRARIES} +) + add_subdirectory(compression_utils_ut) install(TARGETS compression_utils DESTINATION lib) diff --git a/core/compression/compression_utils.cc b/core/compression/compression_utils.cc index 0aaa656..c7e8922 100644 --- a/core/compression/compression_utils.cc +++ b/core/compression/compression_utils.cc @@ -22,6 +22,8 @@ #include #include #include +#include +#include using namespace std; @@ -29,6 +31,10 @@ using DebugFunction = void(*)(const char *); static const int max_debug_level = static_cast(CompressionUtilsDebugLevel::COMPRESSION_DBG_LEVEL_ASSERTION); +static const int max_retries = 3; +static const size_t default_brotli_buffer_size = 16384; +static const size_t brotli_decompression_probe_size = 64; + static void defaultPrint(const char *debug_message) { @@ -104,12 +110,23 @@ static const int zlib_no_flush = Z_NO_FLUSH; struct CompressionStream { - CompressionStream() { bzero(&stream, sizeof(z_stream)); } + CompressionStream() + : + br_encoder_state(nullptr), + br_decoder_state(nullptr) + { + bzero(&stream, sizeof(z_stream)); + } + ~CompressionStream() { fini(); } tuple, bool> decompress(const unsigned char *data, uint32_t size) { + if (state == TYPE::UNINITIALIZED && size > 0 && isBrotli(data, size)) return decompressBrotli(data, size); + + if (state == TYPE::DECOMPRESS_BROTLI) return decompressBrotli(data, size); + initInflate(); if (state != TYPE::DECOMPRESS) throw runtime_error("Could not start decompression"); @@ -138,7 +155,7 @@ struct CompressionStream res.append(work_space.data(), stream.total_out - old_total_out); } else { ++retries; - if (retries > 3) { + if (retries > max_retries) { fini(); throw runtime_error("No results from inflate more than three times"); } @@ -156,6 +173,7 @@ struct CompressionStream basic_string compress(CompressionType type, const unsigned char *data, uint32_t size, int is_last_chunk) { + if (type == CompressionType::BROTLI) return compressBrotli(data, size, is_last_chunk); initDeflate(type); if (state != TYPE::COMPRESS) throw runtime_error("Could not start compression"); @@ -183,7 +201,7 @@ struct CompressionStream res.append(work_space.data(), stream.total_out - old_total_out); } else { ++retries; - if (retries > 3) { + if (retries > max_retries) { fini(); throw runtime_error("No results from deflate more than three times"); } @@ -201,7 +219,7 @@ private: void initInflate() { - if (state != TYPE::UNINITIALIZAED) return; + if (state != TYPE::UNINITIALIZED) return; auto init_status = inflateInit2(&stream, default_num_window_bits + 32); if (init_status != zlib_ok_return_value) { @@ -216,7 +234,7 @@ private: void initDeflate(CompressionType type) { - if (state != TYPE::UNINITIALIZAED) return; + if (state != TYPE::UNINITIALIZED) return; int num_history_window_bits; switch (type) { @@ -228,6 +246,10 @@ private: num_history_window_bits = default_num_window_bits; break; } + case CompressionType::BROTLI: { + zlibDbgAssertion << "Brotli compression should use compressBrotli()"; + return; + } default: { zlibDbgAssertion << "Invalid compression type value: " @@ -253,6 +275,190 @@ private: state = TYPE::COMPRESS; } + basic_string + compressBrotli(const unsigned char *data, uint32_t size, int is_last_chunk) + { + if (state == TYPE::UNINITIALIZED) { + br_encoder_state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); + if (!br_encoder_state) throw runtime_error("Failed to create Brotli encoder state"); + + BrotliEncoderSetParameter(br_encoder_state, BROTLI_PARAM_QUALITY, BROTLI_DEFAULT_QUALITY); + BrotliEncoderSetParameter(br_encoder_state, BROTLI_PARAM_LGWIN, BROTLI_DEFAULT_WINDOW); + state = TYPE::COMPRESS_BROTLI; + } else if (state != TYPE::COMPRESS_BROTLI) { + throw runtime_error("Compression stream in inconsistent state for Brotli compression"); + } + + basic_string output; + vector buffer(16384); + int retries = 0; + const uint8_t* next_in = data; + size_t available_in = size; + + while (available_in > 0 || is_last_chunk) { + size_t available_out = buffer.size(); + uint8_t* next_out = buffer.data(); + + + BrotliEncoderOperation op = is_last_chunk ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS; + auto brotli_success = BrotliEncoderCompressStream( + br_encoder_state, + op, + &available_in, + &next_in, + &available_out, + &next_out, + nullptr + ); + + if (brotli_success == BROTLI_FALSE) { + fini(); + throw runtime_error("Brotli compression error"); + } + + size_t bytes_written = buffer.size() - available_out; + if (bytes_written > 0) { + output.append(buffer.data(), bytes_written); + retries = 0; + } else { + retries++; + if (retries > max_retries) { + fini(); + throw runtime_error("Brotli compression error: Exceeded retry limit."); + } + } + + if (BrotliEncoderIsFinished(br_encoder_state)) break; + + if (available_in == 0 && !is_last_chunk) break; + } + + if (is_last_chunk) fini(); + + return output; + } + + tuple, bool> + decompressBrotli(const unsigned char *data, uint32_t size) + { + if (state != TYPE::DECOMPRESS_BROTLI) { + br_decoder_state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + + if (!br_decoder_state) throw runtime_error("Failed to create Brotli decoder state"); + + BrotliDecoderSetParameter(br_decoder_state, BROTLI_DECODER_PARAM_LARGE_WINDOW, 1u); + state = TYPE::DECOMPRESS_BROTLI; + } + + basic_string output; + const uint8_t* next_in = data; + size_t available_in = size; + + size_t buffer_size = max(size * 4, default_brotli_buffer_size); + vector buffer(buffer_size); + + // Use a constant ratio for max buffer size relative to input size + const size_t max_buffer_size = 256 * 1024 * 1024; // 256 MB max buffer size + + while (true) { + size_t available_out = buffer.size(); + uint8_t* next_out = buffer.data(); + + BrotliDecoderResult result = BrotliDecoderDecompressStream( + br_decoder_state, + &available_in, + &next_in, + &available_out, + &next_out, + nullptr + ); + + if (result == BROTLI_DECODER_RESULT_ERROR) { + fini(); + auto error_msg = string(BrotliDecoderErrorString(BrotliDecoderGetErrorCode(br_decoder_state))); + throw runtime_error("Brotli decompression error: " + error_msg); + } + + // Handle any produced output + size_t bytes_produced = buffer.size() - available_out; + if (bytes_produced > 0) { + output.append(buffer.data(), bytes_produced); + } + + if (result == BROTLI_DECODER_RESULT_SUCCESS) { + bool is_finished = BrotliDecoderIsFinished(br_decoder_state); + if (is_finished) fini(); + return make_tuple(output, is_finished); + } + + if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + // Check if we've exceeded the maximum buffer size limit + if (buffer.size() >= max_buffer_size) { + fini(); + throw runtime_error("Brotli decompression buffer size limit exceeded - possibly corrupted data"); + } + + // Resize buffer to accommodate more output + size_t new_size = min(buffer.size() * 2, max_buffer_size); + buffer.resize(new_size); + continue; // Continue with the same input, new buffer + } + + // If we reach here, we need more input but have no more to provide + if (available_in == 0) { + // No more input data available, return what we have so far + return make_tuple(output, false); + } + } + + return make_tuple(output, false); + } + + bool + isBrotli(const unsigned char *data, uint32_t size) + { + if (size < 4) return false; + + BrotliDecoderState* test_decoder = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + if (!test_decoder) return false; + + const uint8_t* next_in = data; + size_t available_in = min(size, brotli_decompression_probe_size); + uint8_t output[brotli_decompression_probe_size]; + size_t available_out = sizeof(output); + uint8_t* next_out = output; + + BrotliDecoderResult result = BrotliDecoderDecompressStream( + test_decoder, + &available_in, + &next_in, + &available_out, + &next_out, + nullptr + ); + + bool is_brotli = false; + + if ( + result != BROTLI_DECODER_RESULT_ERROR && + ( + available_out < sizeof(output) || + available_in < min(size, brotli_decompression_probe_size) + ) + ) { + is_brotli = true; + } + + BrotliDecoderDestroyInstance(test_decoder); + if (is_brotli) { + br_decoder_state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + BrotliDecoderSetParameter(br_decoder_state, BROTLI_DECODER_PARAM_LARGE_WINDOW, 1u); + state = TYPE::DECOMPRESS_BROTLI; + return true; + } + return false; + } + void fini() { @@ -261,11 +467,21 @@ private: if (state == TYPE::DECOMPRESS) end_stream_res = inflateEnd(&stream); if (state == TYPE::COMPRESS) end_stream_res = deflateEnd(&stream); - if (end_stream_res != zlib_ok_return_value) { + if (br_encoder_state) { + BrotliEncoderDestroyInstance(br_encoder_state); + br_encoder_state = nullptr; + } + + if (br_decoder_state) { + BrotliDecoderDestroyInstance(br_decoder_state); + br_decoder_state = nullptr; + } + + if (end_stream_res != zlib_ok_return_value && end_stream_res != Z_DATA_ERROR) { zlibDbgError << "Failed to clean state: " << getZlibError(end_stream_res); } - state = TYPE::UNINITIALIZAED; + state = TYPE::UNINITIALIZED; } string @@ -288,7 +504,16 @@ private: } z_stream stream; - enum class TYPE { UNINITIALIZAED, COMPRESS, DECOMPRESS } state = TYPE::UNINITIALIZAED; + enum class TYPE { + UNINITIALIZED, + COMPRESS, + DECOMPRESS, + COMPRESS_BROTLI, + DECOMPRESS_BROTLI + } state = TYPE::UNINITIALIZED; + + BrotliEncoderState* br_encoder_state = nullptr; + BrotliDecoderState* br_decoder_state = nullptr; }; void diff --git a/core/compression/compression_utils_ut/compression_utils_ut.cc b/core/compression/compression_utils_ut/compression_utils_ut.cc index 2f5782b..bcebdfe 100644 --- a/core/compression/compression_utils_ut/compression_utils_ut.cc +++ b/core/compression/compression_utils_ut/compression_utils_ut.cc @@ -284,6 +284,25 @@ private: const string test_files_dir_name = "test_files"; }; +TEST_F(CompressionUtilsTest, BrotliBufferLimitTest) +{ + // Create a large string with highly compressible data that will expand significantly + // This should trigger the buffer size limit if max_ratio is too restrictive + string large_compressible_string = string(10000, 'A') + string(10000, 'B') + "CCCCCC"; // 10KB of 'A' characters + + Maybe compressed = compressString(CompressionType::BROTLI, large_compressible_string); + EXPECT_TRUE(compressed.ok()); + + cout << "Compressed size: " << compressed.unpack().size() << ". compression ratio: " + << static_cast(large_compressible_string.size()) / compressed.unpack().size() << endl; + + // This decompression should fail if max_ratio is too restrictive (like 1) + // because decompressed data (100KB) will be much larger than compressed data + Maybe decompressed = decompressString(compressed.unpack()); + ASSERT_TRUE(decompressed.ok()); + EXPECT_EQ(large_compressible_string, decompressed.unpack()); +} + TEST_F(CompressionUtilsTest, CompressAndDecompressSimpleString) { for (auto single_compression_type : compression_types) { @@ -460,3 +479,177 @@ TEST_F(CompressionUtilsTest, DecompressPlainText) HasSubstr("error in 'inflate': Invalid or corrupted stream data") ); } + +TEST_F(CompressionUtilsTest, BrotliCompressAndDecompressSimpleString) +{ + Maybe compressed_string_maybe = compressString( + CompressionType::BROTLI, + simple_test_string + ); + EXPECT_TRUE(compressed_string_maybe.ok()); + + Maybe decompressed_string_maybe = decompressString(compressed_string_maybe.unpack()); + EXPECT_TRUE(decompressed_string_maybe.ok()); + + EXPECT_EQ(simple_test_string, decompressed_string_maybe.unpack()); +} + +TEST_F(CompressionUtilsTest, BrotliCompressAndDecompressChunkSizedString) +{ + string test_string = readTestFileContents(chunk_sized_string_file_name); + + Maybe compressed_string_maybe = compressString( + CompressionType::BROTLI, + test_string + ); + EXPECT_TRUE(compressed_string_maybe.ok()); + + Maybe decompressed_string_maybe = decompressString(compressed_string_maybe.unpack()); + EXPECT_TRUE(decompressed_string_maybe.ok()); + + EXPECT_EQ(test_string, decompressed_string_maybe.unpack()); +} + +TEST_F(CompressionUtilsTest, BrotliCompressMultipleChunkSizedStringAndDecompress) +{ + string test_string = readTestFileContents(multi_chunk_sized_string_file_name); + Maybe chunked_compress_result = chunkedCompressString(CompressionType::BROTLI, test_string); + EXPECT_TRUE(chunked_compress_result.ok()); + + Maybe chunked_decompress_result = chunkedDecompressString(chunked_compress_result.unpack()); + EXPECT_TRUE(chunked_decompress_result.ok()); + + EXPECT_EQ(chunked_decompress_result.unpack(), test_string); +} + +TEST_F(CompressionUtilsTest, BrotliEmptyBuffer) +{ + auto compression_stream = initCompressionStream(); + stringstream compressed_stream; + + Maybe compressed_string = compressString( + CompressionType::BROTLI, + simple_test_string, + false, + compression_stream + ); + EXPECT_TRUE(compressed_string.ok()); + compressed_stream << compressed_string.unpack(); + + compressed_string = compressString( + CompressionType::BROTLI, + "", + true, + compression_stream + ); + finiCompressionStream(compression_stream); + EXPECT_TRUE(compressed_string.ok()); + compressed_stream << compressed_string.unpack(); + + int is_last_chunk; + auto decompression_stream = initCompressionStream(); + + Maybe decompressed_string = decompressString( + compressed_stream.str(), + &is_last_chunk, + decompression_stream + ); + + EXPECT_TRUE(decompressed_string.ok()); + EXPECT_EQ(decompressed_string.unpack(), simple_test_string); + finiCompressionStream(decompression_stream); +} + +TEST_F(CompressionUtilsTest, BrotliCompressionRatio) +{ + // Test if Brotli provides reasonable compression ratio for highly compressible content + string test_string = string(10000, 'A'); + + Maybe gzip_compressed = compressString(CompressionType::GZIP, test_string); + Maybe brotli_compressed = compressString(CompressionType::BROTLI, test_string); + + EXPECT_TRUE(gzip_compressed.ok()); + EXPECT_TRUE(brotli_compressed.ok()); + + // Both should compress well + EXPECT_LT(gzip_compressed.unpack().size(), test_string.size() / 10); + EXPECT_LT(brotli_compressed.unpack().size(), test_string.size() / 10); +} + +TEST_F(CompressionUtilsTest, BrotliVariousSizedPayloads) +{ + const vector test_strings = { + "", // Empty string + "a", // Single character + "Hello, Brotli compression!", // Short string + string(1024, 'A'), // 1KB of repeating data + readTestFileContents(chunk_sized_string_file_name) // Test file + }; + + for (const auto& test_string : test_strings) { + Maybe compressed_string_maybe = compressString( + CompressionType::BROTLI, + test_string + ); + EXPECT_TRUE(compressed_string_maybe.ok()); + + Maybe decompressed_string_maybe = decompressString(compressed_string_maybe.unpack()); + EXPECT_TRUE(decompressed_string_maybe.ok()); + + EXPECT_EQ(test_string, decompressed_string_maybe.unpack()); + } +} + +TEST_F(CompressionUtilsTest, ExceptionHandling_NullDataPointer) +{ + auto compression_stream = initCompressionStream(); + + CompressionResult result = compressData( + compression_stream, + CompressionType::GZIP, + 100, + nullptr, // Null data pointer + 1 + ); + EXPECT_EQ(result.ok, 0); + EXPECT_THAT( + capture_debug.str(), + HasSubstr("Compression failed Data pointer is NULL") + ); + + finiCompressionStream(compression_stream); +} + +TEST_F(CompressionUtilsTest, ExceptionHandling_InvalidDecompressionData) +{ + auto compression_stream = initCompressionStream(); + + unsigned char invalid_data[] = "This is not compressed data"; + DecompressionResult result = decompressData( + compression_stream, + sizeof(invalid_data), + invalid_data + ); + EXPECT_EQ(result.ok, 0); + EXPECT_THAT( + capture_debug.str(), + AnyOf(HasSubstr("Decompression failed"), HasSubstr("error in 'inflate'")) + ); + + finiCompressionStream(compression_stream); +} + +TEST_F(CompressionUtilsTest, ExceptionHandling_ResourceCleanup) +{ + // Verify no memory leaks by creating and destroying multiple streams + // with failing operations + unsigned char invalid_data[] = "This is not compressed data"; + + for (int i = 0; i < 10; i++) { + auto temp_stream = initCompressionStream(); + decompressData(temp_stream, 5, invalid_data); // This should fail + finiCompressionStream(temp_stream); + } + + // No crashes = success +} diff --git a/core/config/CMakeLists.txt b/core/config/CMakeLists.txt index f1b5811..21c1d3d 100644 --- a/core/config/CMakeLists.txt +++ b/core/config/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(config config.cc config_specific.cc config_globals.cc) +add_library(config config.cc config_specific.cc config_globals.cc config_cache_stats.cc) target_link_libraries(config agent_core_utilities) link_directories(${BOOST_ROOT}/lib) diff --git a/core/config/config.cc b/core/config/config.cc index 2443b17..1a1ac64 100644 --- a/core/config/config.cc +++ b/core/config/config.cc @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include "agent_core_utilities.h" #include "cereal/archives/json.hpp" @@ -109,7 +111,7 @@ public: void init(); const TypeWrapper & getConfiguration(const vector &paths) const override; - PerContextValue getAllConfiguration(const std::vector &paths) const; + PerContextValue getAllConfiguration(const vector &paths) const; const TypeWrapper & getResource(const vector &paths) const override; const TypeWrapper & getSetting(const vector &paths) const override; string getProfileAgentSetting(const string &setting_name) const override; @@ -120,15 +122,22 @@ public: const string & getFilesystemPathConfig() const override; const string & getLogFilesPathConfig() const override; + bool + isConfigCacheEnabled() const override + { + return is_cache_enabled && !policy_load_id.empty(); + } + const string & getPolicyLoadId() const override { return policy_load_id; } + string getPolicyConfigPath( const string &name, ConfigFileType type, const string &tenant = "", const string &profile = "") const override; - bool setConfiguration(TypeWrapper &&value, const std::vector &paths) override; - bool setResource(TypeWrapper &&value, const std::vector &paths) override; - bool setSetting(TypeWrapper &&value, const std::vector &paths) override; + bool setConfiguration(TypeWrapper &&value, const vector &paths) override; + bool setResource(TypeWrapper &&value, const vector &paths) override; + bool setSetting(TypeWrapper &&value, const vector &paths) override; void registerExpectedConfigFile(const string &file_name, ConfigFileType type) override; void registerExpectedConfiguration(unique_ptr> &&config) override; @@ -145,6 +154,15 @@ public: void registerConfigLoadCb(ConfigCb) override; void registerConfigAbortCb(ConfigCb) override; void clearOldTenants() override; + void resetConfigCache() override; + + // Cache statistics interface implementation + uint64_t getCacheHits() const override; + uint64_t getCacheMisses() const override; + void resetCacheStats() override; + void enableCacheTracking() override; + void disableCacheTracking() override; + bool isCacheTrackingEnabled() const override; private: bool areTenantAndProfileActive(const TenantProfilePair &tenant_profile) const; @@ -157,7 +175,8 @@ private: void reloadConfigurationContinuesWrapper(const string &version, uint id); vector fillMultiTenantConfigFiles(const map> &tenants); vector fillMultiTenantExpectedConfigFiles(const map> &tenants); - map getProfileAgentSetting() const; + map & getProfileAgentSetting() const; + void resolveVsId() const; string @@ -234,7 +253,7 @@ private: secondary_port_req_md ); } - + if (!service_config_status.ok()) { dbgWarning(D_CONFIG) << "Could not send configuration to orchestrator 7778, error: " @@ -343,6 +362,12 @@ private: string default_config_directory_path = "/conf/"; string config_directory_path = ""; string error_to_report = ""; + string current_policy_version = ""; + mutable size_t cached_last_policy_count = 0; + mutable map cached_agent_settings; + size_t policy_load_count = 0; + string policy_load_id = ""; + bool is_cache_enabled = false; TypeWrapper empty; }; @@ -350,6 +375,7 @@ private: void ConfigComponent::Impl::preload() { + resetConfigCache(); I_Environment *environment = Singleton::Consume::by(); auto executable = environment->get("Base Executable Name"); if (!executable.ok() || *executable == "") { @@ -733,6 +759,7 @@ ConfigComponent::Impl::periodicRegistrationRefresh() { I_Environment *environment = Singleton::Consume::by(); I_MainLoop *mainloop = Singleton::Consume::by(); + I_Environment *env = Singleton::Consume::by(); while (true) { auto env_listening_port = environment->get("Listening Port"); @@ -742,15 +769,17 @@ ConfigComponent::Impl::periodicRegistrationRefresh() << "Internal rest server listening port is not yet set." << " Setting retry attempt to 500 milliseconds from now"; mainloop->yield(chrono::milliseconds(500)); - } else if (!sendOrchestatorConfMsg(env_listening_port.unpack())) { - mainloop->yield(chrono::milliseconds(500)); - } else { + } else if (sendOrchestatorConfMsg(env_listening_port.unpack())) { + dbgInfo(D_CONFIG) << "Configuration update registration with orchestrator succeeded."; + env->registerValue("isRegisteredWithOrchestrator", true); uint next_iteration_in_sec = getConfigurationWithDefault( 600, "Config Component", "Refresh config update registration time interval" ); mainloop->yield(chrono::seconds(next_iteration_in_sec)); + } else { + mainloop->yield(chrono::milliseconds(500)); } } } @@ -758,6 +787,7 @@ ConfigComponent::Impl::periodicRegistrationRefresh() bool ConfigComponent::Impl::loadConfiguration(vector> &file_archives, bool is_async) { + is_cache_enabled = false; auto mainloop = is_async ? Singleton::Consume::by() : nullptr; for (auto &cb : configuration_prepare_cbs) { @@ -828,6 +858,15 @@ ConfigComponent::Impl::commitSuccess() for (auto &cb : configuration_commit_cbs) { cb(); } + policy_load_count++; + try { + policy_load_id = to_string(boost::uuids::random_generator()()); + } catch (const boost::uuids::entropy_error &e) { + dbgWarning(D_CONFIG) << "Failed to create random id for policy_load_id"; + policy_load_id = ""; + } + is_cache_enabled = true; + initializeCacheTracking(); return true; } @@ -842,6 +881,7 @@ ConfigComponent::Impl::commitFailure(const string &error) for (auto &cb : configuration_abort_cbs) { cb(); } + policy_load_id = ""; return false; } @@ -933,24 +973,38 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as env->registerValue("Is Async Config Load", is_async); bool res = loadConfiguration(archives, is_async); env->unregisterKey("Is Async Config Load"); - if (res) env->registerValue("Current Policy Version", version); + + if (res) { + env->registerValue("Current Policy Version", version); + current_policy_version = version; + dbgTrace(D_CONFIG) << "Successfully loaded configuration. Version: " << version + << ", Policy Load Count: " << policy_load_count + << ", Policy Load ID: " << policy_load_id; + } return res; } -map +map & ConfigComponent::Impl::getProfileAgentSetting() const { - auto general_sets = getSettingWithDefault(AgentProfileSettings::default_profile_settings, "generalAgentSettings"); + if (!is_cache_enabled || + policy_load_count == 0 || + cached_last_policy_count != policy_load_count) { + cached_agent_settings.clear(); + auto general_sets = getSettingWithDefault( + AgentProfileSettings::default_profile_settings, + "generalAgentSettings"); + cached_agent_settings = general_sets.getSettings(); - auto settings = general_sets.getSettings(); - - auto profile_sets = getSettingWithDefault(AgentProfileSettings::default_profile_settings, "agentSettings"); - auto profile_settings = profile_sets.getSettings(); - for (const auto &profile_setting : profile_settings) { - settings.insert(profile_setting); + auto profile_sets = getSettingWithDefault(AgentProfileSettings::default_profile_settings, "agentSettings"); + auto profile_settings = profile_sets.getSettings(); + for (const auto &profile_setting : profile_settings) { + cached_agent_settings.insert(profile_setting); + } + cached_last_policy_count = policy_load_count; } - return settings; + return cached_agent_settings; } void @@ -963,7 +1017,7 @@ ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version LoadNewConfigurationStatus in_progress(id, service_name, false, false); auto routine_id = mainloop->addRecurringRoutine( I_MainLoop::RoutineType::Timer, - std::chrono::seconds(30), + chrono::seconds(30), [=] () { sendOrchestatorReloadStatusMsg(in_progress); }, "A-Synchronize reload configuraion monitoring" ); @@ -1006,13 +1060,59 @@ ConfigComponent::Impl::resolveVsId() const return; } +void +ConfigComponent::Impl::resetConfigCache() +{ + cached_last_policy_count = 0; + policy_load_count = 0; + policy_load_id = ""; + is_cache_enabled = false; + cached_agent_settings.clear(); +} + +uint64_t +ConfigComponent::Impl::getCacheHits() const +{ + return CacheStats::getHits(); +} + +uint64_t +ConfigComponent::Impl::getCacheMisses() const +{ + return CacheStats::getMisses(); +} + +void +ConfigComponent::Impl::resetCacheStats() +{ + CacheStats::reset(); +} + +void +ConfigComponent::Impl::enableCacheTracking() +{ + CacheStats::enableTracking(); +} + +void +ConfigComponent::Impl::disableCacheTracking() +{ + CacheStats::disableTracking(); +} + +bool +ConfigComponent::Impl::isCacheTrackingEnabled() const +{ + return CacheStats::isTrackingEnabled(); +} + ConfigComponent::ConfigComponent() : Component("ConfigComponent"), pimpl(make_unique()) {} ConfigComponent::~ConfigComponent() {} void ConfigComponent::preload() { - registerExpectedConfiguration("Config Component", "configuration path"); + registerExpectedConfigurationWithCache("assetId", "Config Component", "configuration path"); registerExpectedConfiguration("Config Component", "Refresh config update registration time interval"); registerExpectedConfiguration("Config Component", "Periodic Registration Refresh"); registerExpectedResource("Config Component", "Config Load Test"); diff --git a/core/config/config_cache_stats.cc b/core/config/config_cache_stats.cc new file mode 100644 index 0000000..c3b8ffe --- /dev/null +++ b/core/config/config_cache_stats.cc @@ -0,0 +1,13 @@ +#include + +// Forward declaration of CacheStats struct +struct CacheStats { + static std::atomic hits; + static std::atomic misses; + static bool tracking_enabled; +}; + +// Define static members for cache statistics +std::atomic CacheStats::hits{0}; +std::atomic CacheStats::misses{0}; +bool CacheStats::tracking_enabled = false; diff --git a/core/cpu/cpu.cc b/core/cpu/cpu.cc index 59d2461..4cdc73f 100644 --- a/core/cpu/cpu.cc +++ b/core/cpu/cpu.cc @@ -211,6 +211,8 @@ CPUManager::init() } } + registerConfigLoadCb([this]() { loadCPUConfig(); }); + i_mainloop->addOneTimeRoutine( I_MainLoop::RoutineType::Timer, [this]() { checkCPUStatus(); }, @@ -265,8 +267,6 @@ void CPUManager::checkCPUStatus() { while (true) { - loadCPUConfig(); - auto is_orchestrator = Singleton::Consume::by()->get("Is Orchestrator"); if (is_orchestrator.ok() && is_orchestrator.unpack()) { Maybe current_general_cpu = i_cpu->getCurrentGeneralCPUUsage(); diff --git a/core/cpu/cpu_ut/cpu_ut.cc b/core/cpu/cpu_ut/cpu_ut.cc index 5cfd93f..1a5a621 100644 --- a/core/cpu/cpu_ut/cpu_ut.cc +++ b/core/cpu/cpu_ut/cpu_ut.cc @@ -47,8 +47,6 @@ public: StrictMock mock_ml; StrictMock mock_time; I_Environment *i_env; - -private: ConfigComponent conf; ::Environment env; }; @@ -261,8 +259,24 @@ TEST_F(CPUTest, noDebugTest) StrictMock mock_cpu; CPUManager cpu; cpu.preload(); - setConfiguration(0, string("CPU"), string("debug period")); cpu.init(); + // Test loadConfiguration functionality like in compression_ut + string config_json = + "{" + " \"CPU\": {" + " \"debug period\": [" + " {" + " \"value\": 0" + " }" + " ]" + " }" + "}"; + istringstream ss(config_json); + Singleton::Consume::from(conf)->loadConfiguration(ss); + + auto loaded_debug_period = getConfiguration("CPU", "debug period"); + EXPECT_TRUE(loaded_debug_period.ok()); + EXPECT_EQ((int)loaded_debug_period.unpack(), 0); doFWError(); EXPECT_THAT(debug_output.str(), HasSubstr("!!!] FW error message\n")); diff --git a/core/curl_http_client/curl_http_client.cc b/core/curl_http_client/curl_http_client.cc index db00b8b..ae4626b 100644 --- a/core/curl_http_client/curl_http_client.cc +++ b/core/curl_http_client/curl_http_client.cc @@ -50,6 +50,7 @@ CurlHttpClient::~CurlHttpClient() curl_global_cleanup(); } +// LCOV_EXCL_START void CurlHttpClient::setProxy(const string& hosts) { @@ -68,7 +69,15 @@ CurlHttpClient::authEnabled(bool enabled) { auth_enabled = enabled; } +// LCOV_EXCL_STOP +void +CurlHttpClient::setConfigs(const CurlHttpClientConfig& config) +{ + this->config = config; +} + +// LCOV_EXCL_START HTTPResponse CurlHttpClient::get(const string& url, const map& headers) { @@ -106,6 +115,7 @@ CurlHttpClient::WriteCallback(void *contents, size_t size, size_t nmemb, string userp->append(static_cast(contents), totalSize); return totalSize; } +// LCOV_EXCL_STOP HTTPResponse CurlHttpClient::perform_request( @@ -128,6 +138,24 @@ CurlHttpClient::perform_request( curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_body); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, config.timeout_seconds); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, config.connect_timeout_seconds); + + if (config.verbose_enabled) { + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + } + + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, config.ssl_verify_peer ? 1L : 0L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, config.ssl_verify_host ? 2L : 0L); + + if (config.http_version != CURL_HTTP_VERSION_NONE) { + curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, config.http_version); + } + + if (!config.user_agent.empty()) { + curl_easy_setopt(curl, CURLOPT_USERAGENT, config.user_agent.c_str()); + } + if (!no_proxy_hosts.empty()) { dbgTrace(D_NGINX_MANAGER) << "Using proxy url: " << no_proxy_hosts; curl_easy_setopt(curl, CURLOPT_PROXY, no_proxy_hosts.c_str()); diff --git a/core/debug_is/debug.cc b/core/debug_is/debug.cc index d7dd483..363087d 100644 --- a/core/debug_is/debug.cc +++ b/core/debug_is/debug.cc @@ -84,6 +84,15 @@ static map flags_to_setting_name = { #undef DEFINE_FLAG }; +// Reverse mapping: flag name string to enum (using the same source as flags_to_setting_name) +static map string_to_flag_map = { + {"D_ALL", Debug::DebugFlags::D_ALL}, +#define DEFINE_FLAG(flag_name, parent_name) \ + {#flag_name, Debug::DebugFlags::flag_name}, +#include "debug_flags.h" +#undef DEFINE_FLAG +}; + static map> preparing_streams; static FlagsArray global_flags_levels(FlagsArray::Fill(), default_level); @@ -510,7 +519,7 @@ Debug::~Debug() void Debug::preload() { - registerExpectedConfiguration("Debug"); + registerExpectedConfigurationWithCache("assetId", "Debug"); registerExpectedConfiguration("Debug I/S", "Fog Debug URI"); registerExpectedConfiguration("Debug I/S", "Debug conf file path"); registerExpectedConfiguration("Debug I/S", "Enable bulk of debugs"); @@ -750,6 +759,25 @@ Debug::isFlagAtleastLevel(Debug::DebugFlags flag, Debug::DebugLevel level) return global_flags_levels[flag] <= level; } +void +Debug::setDebugFlag(Debug::DebugFlags flag, Debug::DebugLevel level) +{ + global_flags_levels[flag] = level; + default_config.streams_in_context[0].flag_values[flag] = level; + + //if the new level is lower than the current lowest, update it + if (lowest_global_level >= level) { + lowest_global_level = level; + return; + } + + // if the new level is higher, recalculate lowest_global_level by scanning all flag levels + lowest_global_level = global_flags_levels[Debug::DebugFlags::D_ALL]; + for (const auto ¤t_level : global_flags_levels) { + if (current_level < lowest_global_level) lowest_global_level = current_level; + } +} + void Debug::setUnitTestFlag(Debug::DebugFlags flag, Debug::DebugLevel level) { @@ -783,6 +811,17 @@ Debug::getExecutableName() return executable.ok() ? *executable : ""; } +bool +Debug::getDebugFlagFromString(const string &flag_name, DebugFlags &flag) +{ + auto flag_it = string_to_flag_map.find(flag_name); + if (flag_it != string_to_flag_map.end()) { + flag = flag_it->second; + return true; + } + return false; +} + void Debug::addActiveStream(const string &name) { diff --git a/core/debug_is/debug_is_ut/debug_ut.cc b/core/debug_is/debug_is_ut/debug_ut.cc index 106a028..c8faf5f 100644 --- a/core/debug_is/debug_is_ut/debug_ut.cc +++ b/core/debug_is/debug_is_ut/debug_ut.cc @@ -587,6 +587,13 @@ public: return msg; } + void + clearDebugMessage() + { + capture_debug.str(""); + capture_debug.clear(); + } + bool loadConfiguration(const string &conf_str) { @@ -667,6 +674,7 @@ TEST_F(DebugConfigTest, debug_all) CPTestTempfile debug_file; loadConfiguration("{\"Output\": \"STDOUT\", \"D_PM\": \"Error\", \"D_ALL\": \"Trace\"}"); + clearDebugMessage(); doFWWarning(); EXPECT_EQ(getDebugMessage(), diff --git a/core/include/attachments/nano_attachment_common.h b/core/include/attachments/nano_attachment_common.h new file mode 100644 index 0000000..85aa1ed --- /dev/null +++ b/core/include/attachments/nano_attachment_common.h @@ -0,0 +1,672 @@ +#ifndef __NANO_ATTACHMENT_COMMON_H__ +#define __NANO_ATTACHMENT_COMMON_H__ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "compression_utils.h" + +typedef uint32_t SessionID; +typedef void* DataBuffer; +typedef int64_t NanoHttpCpInjectPos; + +#define MAX_NGINX_UID_LEN 32 +#define MAX_SHARED_MEM_PATH_LEN 128 +#define NUM_OF_NGINX_IPC_ELEMENTS 200 +#define DEFAULT_KEEP_ALIVE_INTERVAL_MSEC 300000u +#define SHARED_MEM_PATH "/dev/shm/" +#define SHARED_REGISTRATION_SIGNAL_PATH SHARED_MEM_PATH "check-point/cp-nano-attachment-registration" +#define SHARED_KEEP_ALIVE_PATH SHARED_MEM_PATH "check-point/cp-nano-attachment-registration-expiration-socket" +#define SHARED_VERDICT_SIGNAL_PATH SHARED_MEM_PATH "check-point/cp-nano-http-transaction-handler" +#define SHARED_ATTACHMENT_CONF_PATH SHARED_MEM_PATH "cp_nano_http_attachment_conf" +#define DEFAULT_STATIC_RESOURCES_PATH SHARED_MEM_PATH "static_resources" +#define INJECT_POS_IRRELEVANT -1 +#define CORRUPTED_SESSION_ID 0 +#define METRIC_PERIODIC_TIMEOUT 600 +#define MAX_CONTAINER_ID_LEN 12 +#define CONTAINER_ID_FILE_PATH "/proc/self/cgroup" +#define RESPONSE_PAGE_PARTS 4 +#define UUID_SIZE 64 +#define CUSTOM_RESPONSE_TITLE_SIZE 64 +#define CUSTOM_RESPONSE_BODY_SIZE 128 +#define REDIRECT_RESPONSE_LOCATION_SIZE 512 + +#ifdef __cplusplus +typedef enum class NanoWebResponseType +#else +typedef enum NanoWebResponseType +#endif +{ + CUSTOM_WEB_RESPONSE, + CUSTOM_WEB_BLOCK_PAGE_RESPONSE, + RESPONSE_CODE_ONLY, + REDIRECT_WEB_RESPONSE, + + NO_WEB_RESPONSE +} NanoWebResponseType; + +#ifdef __cplusplus +typedef enum class NanoHttpInspectionMode +#else +typedef enum NanoHttpInspectionMode +#endif +{ + NON_BLOCKING_THREAD, + BLOCKING_THREAD, + NO_THREAD, + + INSPECTION_MODE_COUNT +} NanoHttpInspectionMode; + +#ifdef __cplusplus +typedef enum class NanoCommunicationResult +#else +typedef enum NanoCommunicationResult +#endif +{ + NANO_OK, + NANO_ERROR, + NANO_ABORT, + NANO_AGAIN, + NANO_HTTP_FORBIDDEN, + NANO_DECLINED, + NANO_TIMEOUT +} NanoCommunicationResult; + +#ifdef __cplusplus +typedef enum class nano_http_cp_debug_level +#else +typedef enum nano_http_cp_debug_level +#endif +{ + DBG_LEVEL_TRACE, + DBG_LEVEL_DEBUG, + DBG_LEVEL_INFO, + DBG_LEVEL_WARNING, + DBG_LEVEL_ERROR, +#ifndef __cplusplus + DBG_LEVEL_ASSERT, +#endif + DBG_LEVEL_COUNT +} nano_http_cp_debug_level_e; + +#ifdef __cplusplus +typedef enum class AttachmentMetricType +#else +typedef enum AttachmentMetricType +#endif +{ + TRANSPARENTS_COUNT, + TOTAL_TRANSPARENTS_TIME, + INSPECTION_OPEN_FAILURES_COUNT, + INSPECTION_CLOSE_FAILURES_COUNT, + INSPECTION_SUCCESSES_COUNT, + INJECT_VERDICTS_COUNT, + DROP_VERDICTS_COUNT, + ACCEPT_VERDICTS_COUNT, + IRRELEVANT_VERDICTS_COUNT, + RECONF_VERDICTS_COUNT, + INSPECT_VERDICTS_COUNT, + HOLD_VERDICTS_COUNT, + AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT, + MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT, + MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT, + AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT, + MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT, + MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT, + AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT, + MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT, + MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT, + THREAD_TIMEOUT, + REG_THREAD_TIMEOUT, + REQ_METADATA_THREAD_TIMEOUT, + REQ_HEADER_THREAD_TIMEOUT, + REQ_BODY_THREAD_TIMEOUT, + REQ_END_THREAD_TIMEOUT, + AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT, + MAX_REQ_BODY_SIZE_UPON_TIMEOUT, + MIN_REQ_BODY_SIZE_UPON_TIMEOUT, + RES_HEADER_THREAD_TIMEOUT, + RES_BODY_THREAD_TIMEOUT, + RES_END_THREAD_TIMEOUT, + HOLD_THREAD_TIMEOUT, + AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT, + MAX_RES_BODY_SIZE_UPON_TIMEOUT, + MIN_RES_BODY_SIZE_UPON_TIMEOUT, + THREAD_FAILURE, + REQ_PROCCESSING_TIMEOUT, + RES_PROCCESSING_TIMEOUT, + REQ_FAILED_TO_REACH_UPSTREAM, + REQ_FAILED_COMPRESSION_COUNT, + RES_FAILED_COMPRESSION_COUNT, + REQ_FAILED_DECOMPRESSION_COUNT, + RES_FAILED_DECOMPRESSION_COUNT, + REQ_SUCCESSFUL_COMPRESSION_COUNT, + RES_SUCCESSFUL_COMPRESSION_COUNT, + REQ_SUCCESSFUL_DECOMPRESSION_COUNT, + RES_SUCCESSFUL_DECOMPRESSION_COUNT, + CORRUPTED_ZIP_SKIPPED_SESSION_COUNT, + CPU_USAGE, + AVERAGE_VM_MEMORY_USAGE, + AVERAGE_RSS_MEMORY_USAGE, + MAX_VM_MEMORY_USAGE, + MAX_RSS_MEMORY_USAGE, + REQUEST_OVERALL_SIZE_COUNT, + RESPONSE_OVERALL_SIZE_COUNT, + + METRIC_TYPES_COUNT +} AttachmentMetricType; + +#ifdef __cplusplus +typedef enum class AttachmentDataType +#else +typedef enum AttachmentDataType +#endif +{ + REQUEST_START, + REQUEST_HEADER, + REQUEST_BODY, + REQUEST_END, + RESPONSE_CODE, + RESPONSE_HEADER, + RESPONSE_BODY, + RESPONSE_END, + CONTENT_LENGTH, + METRIC_DATA_FROM_PLUGIN, + REQUEST_DELAYED_VERDICT, + + COUNT +} AttachmentDataType; + +#ifdef __cplusplus +typedef enum class HttpChunkType +#else +typedef enum HttpChunkType +#endif +{ + HTTP_REQUEST_FILTER, + HTTP_REQUEST_METADATA, + HTTP_REQUEST_HEADER, + HTTP_REQUEST_BODY, + HTTP_REQUEST_END, + HTTP_RESPONSE_HEADER, + HTTP_RESPONSE_BODY, + HTTP_RESPONSE_END, + HOLD_DATA +} HttpChunkType; + +#ifdef __cplusplus +typedef enum class ServiceVerdict +#else +typedef enum ServiceVerdict +#endif +{ + TRAFFIC_VERDICT_INSPECT, + TRAFFIC_VERDICT_ACCEPT, + TRAFFIC_VERDICT_DROP, + TRAFFIC_VERDICT_INJECT, + TRAFFIC_VERDICT_IRRELEVANT, + TRAFFIC_VERDICT_RECONF, + TRAFFIC_VERDICT_DELAYED, + LIMIT_RESPONSE_HEADERS, + TRAFFIC_VERDICT_CUSTOM_RESPONSE +} ServiceVerdict; + +#ifdef __cplusplus +typedef enum class AttachmentContentType +#else +typedef enum AttachmentContentType +#endif +{ + CONTENT_TYPE_APPLICATION_JSON, + CONTENT_TYPE_TEXT_HTML, + CONTENT_TYPE_TEXT_PLAIN, + CONTENT_TYPE_OTHER +} AttachmentContentType; + +#ifdef __cplusplus +typedef enum class AttachmentVerdict +#else +typedef enum AttachmentVerdict +#endif +{ + ATTACHMENT_VERDICT_INSPECT, + ATTACHMENT_VERDICT_ACCEPT, + ATTACHMENT_VERDICT_DROP, + ATTACHMENT_VERDICT_INJECT +} AttachmentVerdict; + +#ifdef __cplusplus +typedef enum class HttpModificationType +#else +typedef enum HttpModificationType +#endif +{ + APPEND, + INJECT, + REPLACE +} HttpModificationType; + +typedef struct __attribute__((__packed__)) HttpInjectData { + NanoHttpCpInjectPos injection_pos; + HttpModificationType mod_type; + uint16_t injection_size; + uint8_t is_header; + uint8_t orig_buff_index; + char data[0]; +} HttpInjectData; + +typedef struct __attribute__((__packed__)) HttpWebResponseData { + uint8_t web_response_type; + uint8_t uuid_size; + + union { + struct __attribute__((__packed__)) NanoHttpCpCustomWebResponseData { + uint16_t response_code; + uint8_t title_size; + uint8_t body_size; + char data[0]; + } custom_response_data; + + struct __attribute__((__packed__)) NanoHttpCpRedirectData { + uint8_t unused_dummy; + uint8_t add_event_id; + uint16_t redirect_location_size; + char redirect_location[0]; + } redirect_data; + } response_data; +} HttpWebResponseData; + +typedef struct __attribute__((__packed__)) HttpJsonResponseData { + uint16_t response_code; + uint16_t body_size; + AttachmentContentType content_type; + char body[0]; +} HttpJsonResponseData; + +typedef struct { + size_t len; + unsigned char *data; +} nano_str_t; + +typedef struct CustomResponseData { + uint16_t response_code; + unsigned char title[CUSTOM_RESPONSE_TITLE_SIZE]; + unsigned char body[CUSTOM_RESPONSE_BODY_SIZE]; +} CustomResponseData; + +typedef struct RedirectData { + unsigned char redirect_location[REDIRECT_RESPONSE_LOCATION_SIZE]; +} RedirectData; + +typedef struct WebResponseData { + NanoWebResponseType web_response_type; + unsigned char uuid[UUID_SIZE]; + DataBuffer data; +} WebResponseData; + +#ifdef __cplusplus +typedef enum class HttpMetaDataType +#else +typedef enum HttpMetaDataType +#endif +{ + HTTP_PROTOCOL_SIZE, + HTTP_PROTOCOL_DATA, + HTTP_METHOD_SIZE, + HTTP_METHOD_DATA, + HOST_NAME_SIZE, + HOST_NAME_DATA, + LISTENING_ADDR_SIZE, + LISTENING_ADDR_DATA, + LISTENING_PORT, + URI_SIZE, + URI_DATA, + CLIENT_ADDR_SIZE, + CLIENT_ADDR_DATA, + CLIENT_PORT, + PARSED_HOST_SIZE, + PARSED_HOST_DATA, + PARSED_URI_SIZE, + PARSED_URI_DATA, + WAF_TAG_SIZE, + WAF_TAG_DATA, + + META_DATA_COUNT +} HttpMetaDataType; + +#ifdef __cplusplus +typedef enum class HttpHeaderDataType +#else +typedef enum HttpHeaderDataType +#endif +{ + HEADER_KEY_SIZE, + HEADER_KEY_DATA, + HEADER_VAL_SIZE, + HEADER_VAL_DATA, + + HEADER_DATA_COUNT +} HttpHeaderDataType; + +/// @struct NanoHttpModificationList +/// @brief A node that holds all the information regarding modifications. +typedef struct NanoHttpModificationList { + struct NanoHttpModificationList *next; ///< Next node. + HttpInjectData modification; ///< Modification data. + char *modification_buffer; ///< Modification buffer used to store extra needed data. +} NanoHttpModificationList; + +/// @struct NanoHttpResponseData +/// Holds all the data for Compression in a session. +typedef struct { + + /// Original compression type, can hold the following values: + /// - #GZIP + /// - #ZLIB + CompressionType compression_type; + + /// Compression stream + CompressionStream *compression_stream; + + /// Decompression stream + CompressionStream *decompression_stream; +} NanoHttpResponseData; + +/// @struct HttpSessionData +/// @brief Holds all the session's information needed to communicate with the nano service. +/// @details Such as to save verdict and session ID between the request and the response +typedef struct HttpSessionData { + int was_request_fully_inspected; ///< Holds if the request fully inspected. + ServiceVerdict verdict; ///< Holds the session's verdict from the Nano Service. + uint32_t session_id; ///< Current session's Id. + unsigned int remaining_messages_to_reply; ///< Remaining messages left for the agent to respond to. + + NanoHttpResponseData response_data; ///< Holds session's response data. + + double req_proccesing_time; ///< Holds session's request processing time. + double res_proccesing_time; ///< Holds session's response processing time. + uint64_t processed_req_body_size; ///< Holds session's request body's size. + uint64_t processed_res_body_size; ///< Holds session's response body's size'. +} HttpSessionData; + +typedef struct HttpMetaData { + nano_str_t http_protocol; + nano_str_t method_name; + nano_str_t host; + nano_str_t listening_ip; + uint16_t listening_port; + nano_str_t uri; + nano_str_t client_ip; + uint16_t client_port; + nano_str_t parsed_host; + nano_str_t parsed_uri; +} HttpMetaData; + +typedef struct HttpHeaderData { + nano_str_t key; + nano_str_t value; +} HttpHeaderData; + +typedef struct HttpHeaders { + HttpHeaderData *data; + size_t headers_count; +} HttpHeaders; + +typedef struct HttpRequestFilterData { + HttpMetaData *meta_data; + HttpHeaders *req_headers; + bool contains_body; +} HttpRequestFilterData; + +typedef struct ResHttpHeaders { + HttpHeaders *headers; + uint16_t response_code; + uint64_t content_length; +} ResHttpHeaders; + +typedef struct NanoHttpBody { + nano_str_t *data; + size_t bodies_count; +} NanoHttpBody; + +typedef struct AttachmentData { + SessionID session_id; + HttpChunkType chunk_type; + HttpSessionData *session_data; + DataBuffer data; +} AttachmentData; + +typedef union __attribute__((__packed__)) HttpModifyData { + HttpInjectData inject_data[0]; + HttpWebResponseData web_response_data[0]; + HttpJsonResponseData json_response_data[0]; +} HttpModifyData; + +typedef struct __attribute__((__packed__)) HttpReplyFromService { + uint16_t verdict; + SessionID session_id; + uint8_t modification_count; + HttpModifyData modify_data[0]; +} HttpReplyFromService; + +typedef struct AttachmentVerdictResponse { + AttachmentVerdict verdict; + SessionID session_id; + WebResponseData *web_response_data; + NanoHttpModificationList *modifications; +} AttachmentVerdictResponse; + +typedef struct __attribute__((__packed__)) AttachmentRequest { + struct __attribute__((__packed__)) connection { + int sockaddr; + int local_sockaddr; + } connection; + + struct __attribute__((__packed__)) http_protocol { + int len; + int data; + } http_protocol; + + struct __attribute__((__packed__)) method { + int name; + int data; + } method; + + struct __attribute__((__packed__)) uri { + int len; + int data; + } uri; + + struct __attribute__((__packed__)) unparsed_uri { + int len; + int data; + } unparsed_uri; +} AttachmentRequest; + +typedef struct BlockPageData { + uint16_t response_code; + nano_str_t title_prefix; + nano_str_t title; + nano_str_t body_prefix; + nano_str_t body; + nano_str_t uuid_prefix; + nano_str_t uuid; + nano_str_t uuid_suffix; +} BlockPageData; + +typedef struct RedirectPageData { + nano_str_t redirect_location; +} RedirectPageData; + +typedef struct NanoResponseModifications { + NanoHttpModificationList *modifications; +} NanoResponseModifications; + +typedef struct __attribute__((__packed__)) NanoHttpRequestData { + uint16_t data_type; + uint32_t session_id; + unsigned char data[0]; +} NanoHttpRequestData; + +typedef struct __attribute__((__packed__)) NanoHttpMetricData { + uint16_t data_type; +#ifdef __cplusplus + uint64_t data[static_cast(AttachmentMetricType::METRIC_TYPES_COUNT)]; +#else + uint64_t data[METRIC_TYPES_COUNT]; +#endif +} NanoHttpMetricData; + +// Simple but reliable hash function for generating consistent, well-distributed offsets +// Uses a basic polynomial hash that avoids large intermediate values +static inline uint32_t hash_string(const char *str) { + uint32_t hash = 0; + while (*str) { + hash = (hash * 31 + (unsigned char)*str++) % 10000; // Keep values under 10000 + } + return hash; // Return bounded hash - modulo will be applied by caller +} + +static inline int set_affinity_by_uid(uint32_t uid) { + int num_cores = sysconf(_SC_NPROCESSORS_CONF); + // Debug print for troubleshooting + fprintf(stderr, "[DEBUG] set_affinity_by_uid: num_cores=%d, uid=%u\n", num_cores, uid); + uint32_t core_num = (uid - 1) % num_cores; // Ensure core_num is within bounds + cpu_set_t mask, mask_check; + CPU_ZERO(&mask); + CPU_ZERO(&mask_check); + CPU_SET(core_num, &mask); + pid_t pid = getpid(); // Use process PID, not thread ID + + if (sched_setaffinity(pid, sizeof(mask), &mask) != 0) { + return -1; // Error setting affinity + } + if (sched_getaffinity(pid, sizeof(mask_check), &mask_check) != 0) { + return -2; // Error getting affinity + } + // Compare mask and mask_check + int i; + for (i = 0; i < num_cores; ++i) { + if (CPU_ISSET(i, &mask) != CPU_ISSET(i, &mask_check)) { + return -3; // Affinity not set as expected + } + } + return 0; // Success +} + +static inline int set_affinity_by_uid_with_offset(uint32_t uid, uint32_t offset) { + int num_cores = sysconf(_SC_NPROCESSORS_CONF); + // Debug print for troubleshooting + fprintf( + stderr, "[DEBUG] set_affinity_by_uid_with_offset: num_cores=%d, uid=%u, offset=%u\n", num_cores, uid, offset); + // Prevent integer overflow by applying modulo to offset first + uint32_t safe_offset = offset % num_cores; + uint32_t core_num = ((uid - 1) + safe_offset) % num_cores; + cpu_set_t mask, mask_check; + CPU_ZERO(&mask); + CPU_ZERO(&mask_check); + CPU_SET(core_num, &mask); + pid_t pid = getpid(); // Use process PID, not thread ID + + if (sched_setaffinity(pid, sizeof(mask), &mask) != 0) { + return -1; // Error setting affinity + } + if (sched_getaffinity(pid, sizeof(mask_check), &mask_check) != 0) { + return -2; // Error getting affinity + } + // Compare mask and mask_check + int i; + for (i = 0; i < num_cores; ++i) { + if (CPU_ISSET(i, &mask) != CPU_ISSET(i, &mask_check)) { + return -3; // Affinity not set as expected + } + } + return 0; // Success +} + +static inline int set_affinity_by_uid_with_offset_fixed_cores(uint32_t uid, uint32_t offset, int num_cores) { + // Debug print for troubleshooting + fprintf( + stderr, + "[DEBUG] set_affinity_by_uid_with_offset_fixed_cores: num_cores=%d, uid=%u, offset=%u\n", + num_cores, + uid, + offset + ); + // Prevent integer overflow by applying modulo to offset first + + uint32_t safe_offset = offset % num_cores; + uint32_t core_num = ((uid - 1) + safe_offset) % num_cores; + cpu_set_t mask, mask_check; + CPU_ZERO(&mask); + CPU_ZERO(&mask_check); + CPU_SET(core_num, &mask); + pid_t pid = getpid(); // Use process PID, not thread ID + + if (sched_setaffinity(pid, sizeof(mask), &mask) != 0) { + return -1; // Error setting affinity + } + if (sched_getaffinity(pid, sizeof(mask_check), &mask_check) != 0) { + return -2; // Error getting affinity + } + // Compare mask and mask_check + int i; + for (i = 0; i < num_cores; ++i) { + if (CPU_ISSET(i, &mask) != CPU_ISSET(i, &mask_check)) { + return -3; // Affinity not set as expected + } + } + return 0; // Success +} + +static inline int set_affinity_to_core(int target_core) { + // Debug print for troubleshooting + fprintf(stderr, "[DEBUG] set_affinity_to_core: target_core=%d\n", target_core); + cpu_set_t mask, mask_check; + CPU_ZERO(&mask); + CPU_ZERO(&mask_check); + CPU_SET(target_core, &mask); + pid_t pid = getpid(); // Use process PID, not thread ID + + if (sched_setaffinity(pid, sizeof(mask), &mask) != 0) { + return -1; // Error setting affinity + } + if (sched_getaffinity(pid, sizeof(mask_check), &mask_check) != 0) { + return -2; // Error getting affinity + } + // Compare mask and mask_check + int num_cores = sysconf(_SC_NPROCESSORS_CONF); + int i; + for (i = 0; i < num_cores; ++i) { + if (CPU_ISSET(i, &mask) != CPU_ISSET(i, &mask_check)) { + return -3; // Affinity not set as expected + } + } + return 0; // Success +} + +static inline int reset_affinity() { + int num_cores = sysconf(_SC_NPROCESSORS_CONF); + // Debug print for troubleshooting + fprintf(stderr, "[DEBUG] reset_affinity: num_cores=%d\n", num_cores); + cpu_set_t mask; + CPU_ZERO(&mask); + int i; + for (i = 0; i < num_cores; ++i) CPU_SET(i, &mask); + pid_t pid = getpid(); // Use process PID, not thread ID + if (sched_setaffinity(pid, sizeof(mask), &mask) != 0) { + return -1; // Error setting affinity + } + return 0; // Success +} + +#endif // __NANO_ATTACHMENT_COMMON_H__ diff --git a/core/include/attachments/nginx_attachment_common.h b/core/include/attachments/nginx_attachment_common.h index 3929dc2..6f8030f 100644 --- a/core/include/attachments/nginx_attachment_common.h +++ b/core/include/attachments/nginx_attachment_common.h @@ -15,286 +15,10 @@ #ifndef __NGINX_ATTACHMENT_COMMON_H__ #define __NGINX_ATTACHMENT_COMMON_H__ -#include -#include -#include -#include +// This file has been deprecated. Do not add anything here. +// Any future additions should be added to nano_attachment_common.h +// For any inquiries please contact Daniel Yashin. -#define MAX_NGINX_UID_LEN 32 -#define NUM_OF_NGINX_IPC_ELEMENTS 200 -#define DEFAULT_KEEP_ALIVE_INTERVAL_MSEC 300000 -#define SHARED_MEM_PATH "/dev/shm/" -#define SHARED_REGISTRATION_SIGNAL_PATH SHARED_MEM_PATH "check-point/cp-nano-attachment-registration" -#define SHARED_KEEP_ALIVE_PATH SHARED_MEM_PATH "check-point/cp-nano-attachment-registration-expiration-socket" -#define SHARED_VERDICT_SIGNAL_PATH SHARED_MEM_PATH "check-point/cp-nano-http-transaction-handler" -#define SHARED_ATTACHMENT_CONF_PATH SHARED_MEM_PATH "cp_nano_http_attachment_conf" -#define DEFAULT_STATIC_RESOURCES_PATH SHARED_MEM_PATH "static_resources" -#define INJECT_POS_IRRELEVANT -1 -#define CORRUPTED_SESSION_ID 0 -#define METRIC_PERIODIC_TIMEOUT 600 - -extern char shared_verdict_signal_path[]; extern int workers_amount_to_send; -typedef int64_t ngx_http_cp_inject_pos_t; - -#ifdef __cplusplus -typedef enum class ngx_http_modification_type -#else -typedef enum ngx_http_modification_type -#endif -{ - APPEND, - INJECT, - REPLACE -} ngx_http_modification_type_e; - -#ifdef __cplusplus -typedef enum class ngx_http_chunk_type -#else -typedef enum ngx_http_chunk_type -#endif -{ - REQUEST_START, - REQUEST_HEADER, - REQUEST_BODY, - REQUEST_END, - RESPONSE_CODE, - RESPONSE_HEADER, - RESPONSE_BODY, - RESPONSE_END, - CONTENT_LENGTH, - METRIC_DATA_FROM_PLUGIN, - HOLD_DATA, - - COUNT -} ngx_http_chunk_type_e; - -#ifdef __cplusplus -typedef enum class ngx_http_plugin_metric_type -#else -typedef enum ngx_http_plugin_metric_type -#endif -{ - TRANSPARENTS_COUNT, - TOTAL_TRANSPARENTS_TIME, - INSPECTION_OPEN_FAILURES_COUNT, - INSPECTION_CLOSE_FAILURES_COUNT, - INSPECTION_SUCCESSES_COUNT, - INJECT_VERDICTS_COUNT, - DROP_VERDICTS_COUNT, - ACCEPT_VERDICTS_COUNT, - IRRELEVANT_VERDICTS_COUNT, - RECONF_VERDICTS_COUNT, - INSPECT_VERDICTS_COUNT, - HOLD_VERDICTS_COUNT, - AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT, - MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT, - MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT, - AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT, - MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT, - MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT, - AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT, - MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT, - MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT, - THREAD_TIMEOUT, - REG_THREAD_TIMEOUT, - REQ_HEADER_THREAD_TIMEOUT, - REQ_BODY_THREAD_TIMEOUT, - AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT, - MAX_REQ_BODY_SIZE_UPON_TIMEOUT, - MIN_REQ_BODY_SIZE_UPON_TIMEOUT, - RES_HEADER_THREAD_TIMEOUT, - RES_BODY_THREAD_TIMEOUT, - HOLD_THREAD_TIMEOUT, - AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT, - MAX_RES_BODY_SIZE_UPON_TIMEOUT, - MIN_RES_BODY_SIZE_UPON_TIMEOUT, - THREAD_FAILURE, - REQ_PROCCESSING_TIMEOUT, - RES_PROCCESSING_TIMEOUT, - REQ_FAILED_TO_REACH_UPSTREAM, - REQ_FAILED_COMPRESSION_COUNT, - RES_FAILED_COMPRESSION_COUNT, - REQ_FAILED_DECOMPRESSION_COUNT, - RES_FAILED_DECOMPRESSION_COUNT, - REQ_SUCCESSFUL_COMPRESSION_COUNT, - RES_SUCCESSFUL_COMPRESSION_COUNT, - REQ_SUCCESSFUL_DECOMPRESSION_COUNT, - RES_SUCCESSFUL_DECOMPRESSION_COUNT, - CORRUPTED_ZIP_SKIPPED_SESSION_COUNT, - CPU_USAGE, - AVERAGE_VM_MEMORY_USAGE, - AVERAGE_RSS_MEMORY_USAGE, - MAX_VM_MEMORY_USAGE, - MAX_RSS_MEMORY_USAGE, - REQUEST_OVERALL_SIZE_COUNT, - RESPONSE_OVERALL_SIZE_COUNT, - - METRIC_TYPES_COUNT -} ngx_http_plugin_metric_type_e; - -#ifdef __cplusplus -typedef enum class ngx_http_cp_verdict -#else -typedef enum ngx_http_cp_verdict -#endif -{ - TRAFFIC_VERDICT_INSPECT, - TRAFFIC_VERDICT_ACCEPT, - TRAFFIC_VERDICT_DROP, - TRAFFIC_VERDICT_INJECT, - TRAFFIC_VERDICT_IRRELEVANT, - TRAFFIC_VERDICT_RECONF, - TRAFFIC_VERDICT_WAIT, - LIMIT_RESPONSE_HEADERS -} ngx_http_cp_verdict_e; - -#ifdef __cplusplus -typedef enum class ngx_http_cp_debug_level -#else -typedef enum ngx_http_cp_debug_level -#endif -{ - DBG_LEVEL_TRACE, - DBG_LEVEL_DEBUG, - DBG_LEVEL_INFO, - DBG_LEVEL_WARNING, - DBG_LEVEL_ERROR, -#ifndef __cplusplus - DBG_LEVEL_ASSERT, -#endif - DBG_LEVEL_COUNT -} ngx_http_cp_debug_level_e; - -#ifdef __cplusplus -typedef enum class ngx_http_meta_data -#else -typedef enum ngx_http_meta_data -#endif -{ - HTTP_PROTOCOL_SIZE, - HTTP_PROTOCOL_DATA, - HTTP_METHOD_SIZE, - HTTP_METHOD_DATA, - HOST_NAME_SIZE, - HOST_NAME_DATA, - LISTENING_ADDR_SIZE, - LISTENING_ADDR_DATA, - LISTENING_PORT, - URI_SIZE, - URI_DATA, - CLIENT_ADDR_SIZE, - CLIENT_ADDR_DATA, - CLIENT_PORT, - PARSED_HOST_SIZE, - PARSED_HOST_DATA, - PARSED_URI_SIZE, - PARSED_URI_DATA, - WAF_TAG_SIZE, - WAF_TAG_DATA, - - META_DATA_COUNT -} ngx_http_meta_data_e; - -#ifdef __cplusplus -typedef enum class ngx_http_header_data -#else -typedef enum ngx_http_header_data -#endif -{ - HEADER_KEY_SIZE, - HEADER_KEY_DATA, - HEADER_VAL_SIZE, - HEADER_VAL_DATA, - - HEADER_DATA_COUNT -} ngx_http_header_data_e; - -typedef enum ngx_http_inspection_mode -{ - NON_BLOCKING_THREAD, - BLOCKING_THREAD, - NO_THREAD, - - INSPECTION_MODE_COUNT -} ngx_http_inspection_mode_e; - -#ifdef __cplusplus -typedef enum class ngx_web_response_type -#else -typedef enum ngx_web_response_type -#endif -{ - CUSTOM_WEB_RESPONSE, - CUSTOM_WEB_BLOCK_PAGE_RESPONSE, - RESPONSE_CODE_ONLY, - REDIRECT_WEB_RESPONSE, - - NO_WEB_RESPONSE -} ngx_web_response_type_e; - -typedef struct __attribute__((__packed__)) ngx_http_cp_inject_data { - ngx_http_cp_inject_pos_t injection_pos; - ngx_http_modification_type_e mod_type; - uint16_t injection_size; - uint8_t is_header; - uint8_t orig_buff_index; - char data[0]; -} ngx_http_cp_inject_data_t; - -typedef struct __attribute__((__packed__)) ngx_http_cp_web_response_data { - uint8_t web_repsonse_type; - uint8_t uuid_size; - - union { - struct __attribute__((__packed__)) ngx_http_cp_custom_web_response_data { - uint16_t response_code; - uint8_t title_size; - uint8_t body_size; - char data[0]; - } custom_response_data; - - struct __attribute__((__packed__)) ngx_http_cp_redirect_data { - uint8_t unused_dummy; - uint8_t add_event_id; - uint16_t redirect_location_size; - char redirect_location[0]; - } redirect_data; - } response_data; -} ngx_http_cp_web_response_data_t; - -static_assert( - sizeof(((ngx_http_cp_web_response_data_t*)0)->response_data.custom_response_data) == - sizeof(((ngx_http_cp_web_response_data_t*)0)->response_data.redirect_data), - "custom_response_data must be equal to redirect_data in size" -); - -typedef union __attribute__((__packed__)) ngx_http_cp_modify_data { - ngx_http_cp_inject_data_t inject_data[0]; - ngx_http_cp_web_response_data_t web_response_data[0]; -} ngx_http_cp_modify_data_t; - -typedef struct __attribute__((__packed__)) ngx_http_cp_reply_from_service { - uint16_t verdict; - uint32_t session_id; - uint8_t modification_count; - ngx_http_cp_modify_data_t modify_data[0]; -} ngx_http_cp_reply_from_service_t; - -typedef struct __attribute__((__packed__)) ngx_http_cp_request_data { - uint16_t data_type; - uint32_t session_id; - unsigned char data[0]; -} ngx_http_cp_request_data_t; - -typedef struct __attribute__((__packed__)) ngx_http_cp_metric_data { - uint16_t data_type; -#ifdef __cplusplus - uint64_t data[static_cast(ngx_http_plugin_metric_type::METRIC_TYPES_COUNT)]; -#else - uint64_t data[METRIC_TYPES_COUNT]; -#endif -} ngx_http_cp_metric_data_t; - #endif // __NGINX_ATTACHMENT_COMMON_H__ diff --git a/core/include/attachments/nginx_attachment_util.h b/core/include/attachments/nginx_attachment_util.h index 6a2b5e9..0a4e16b 100644 --- a/core/include/attachments/nginx_attachment_util.h +++ b/core/include/attachments/nginx_attachment_util.h @@ -17,7 +17,7 @@ #include -#include "nginx_attachment_common.h" +#include "nano_attachment_common.h" #ifdef __cplusplus extern "C" { @@ -29,7 +29,7 @@ typedef const char * c_str; int initAttachmentConfig(c_str conf_file); -ngx_http_inspection_mode_e getInspectionMode(); +NanoHttpInspectionMode getInspectionMode(); unsigned int getNumOfNginxIpcElements(); unsigned int getKeepAliveIntervalMsec(); unsigned int getDbgLevel(); @@ -61,11 +61,16 @@ unsigned int getMinRetriesForVerdict(); unsigned int getMaxRetriesForVerdict(); unsigned int getReqBodySizeTrigger(); unsigned int getRemoveResServerHeader(); +unsigned int getDecompressionPoolSize(); +unsigned int getRecompressionPoolSize(); +unsigned int getIsBrotliInspectionEnabled(); unsigned int getWaitingForVerdictThreadTimeout(); int isIPAddress(c_str ip_str); int isSkipSource(c_str ip_str); +unsigned int isPairedAffinityEnabled(); +unsigned int isAsyncModeEnabled(); #ifdef __cplusplus } diff --git a/core/include/general/common.h b/core/include/general/common.h index 7029922..e7935fe 100644 --- a/core/include/general/common.h +++ b/core/include/general/common.h @@ -32,6 +32,7 @@ #include #include #include +#include namespace std { @@ -118,6 +119,14 @@ struct IsPrintable(declval() << declval { }; +template +std::ostream& +operator<<(std::ostream& os, const std::chrono::duration& d) +{ + os << d.count(); + return os; +} + template ostream & operator<<(ostream &os, const decltype(declval().print(declval()), declval()) &obj) diff --git a/core/include/general/debug.h b/core/include/general/debug.h index 141caf4..d597e40 100644 --- a/core/include/general/debug.h +++ b/core/include/general/debug.h @@ -233,9 +233,11 @@ public: static void setNewDefaultStdout(std::ostream *new_stream); static void setUnitTestFlag(DebugFlags flag, DebugLevel level); + static void setDebugFlag(DebugFlags flag, DebugLevel level); static std::string findDebugFilePrefix(const std::string &file_name); static std::string getExecutableName(); + static bool getDebugFlagFromString(const std::string &flag_name, DebugFlags &flag); private: template diff --git a/core/include/general/intell_registration_event.h b/core/include/general/intell_registration_event.h new file mode 100644 index 0000000..3ea0d69 --- /dev/null +++ b/core/include/general/intell_registration_event.h @@ -0,0 +1,28 @@ +#ifndef __INTELL_REGISTRATION_EVENT_H__ +#define __INTELL_REGISTRATION_EVENT_H__ + +#include "event.h" + +class IntelligenceRegistrationEvent : public Event +{ +public: + IntelligenceRegistrationEvent(bool registration_successful, std::string registration_response) + : + registration_successful(registration_successful), + registration_response(registration_response) + {} + + IntelligenceRegistrationEvent(bool registration_successful) + : + IntelligenceRegistrationEvent(registration_successful, "") + {} + + bool isRegistrationSuccessful() const { return registration_successful; } + std::string getRegistrationResponse() const { return registration_response; } + +private: + bool registration_successful; + std::string registration_response; +}; + +#endif // __INTELL_REGISTRATION_EVENT_H__ diff --git a/core/include/general/intelligence_comp_v2.h b/core/include/general/intelligence_comp_v2.h index 796207a..193a9bd 100644 --- a/core/include/general/intelligence_comp_v2.h +++ b/core/include/general/intelligence_comp_v2.h @@ -32,7 +32,8 @@ class IntelligenceComponentV2 Singleton::Consume, Singleton::Consume, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: IntelligenceComponentV2(); diff --git a/core/include/internal/curl_http_client.h b/core/include/internal/curl_http_client.h index ed4efd0..1aade99 100644 --- a/core/include/internal/curl_http_client.h +++ b/core/include/internal/curl_http_client.h @@ -8,6 +8,16 @@ #include "messaging/http_response.h" #include "i_http_client.h" +struct CurlHttpClientConfig { + int timeout_seconds = 30; + int connect_timeout_seconds = 10; + bool verbose_enabled = false; + bool ssl_verify_peer = true; + bool ssl_verify_host = true; + long http_version = CURL_HTTP_VERSION_NONE; + std::string user_agent = ""; +}; + class CurlHttpClient : public I_HttpClient { public: @@ -17,6 +27,7 @@ public: void setProxy(const std::string& hosts) override; void setBasicAuth(const std::string& username, const std::string& password) override; void authEnabled(bool enabled) override; + void setConfigs(const CurlHttpClientConfig& config); HTTPResponse get( @@ -70,6 +81,8 @@ private: bool auth_enabled; std::string username; std::string password; + CurlHttpClientConfig config; + }; #endif // __CURL_HTTP_CLIENT_H__ diff --git a/core/include/services_sdk/interfaces/i_encryptor.h b/core/include/services_sdk/interfaces/i_encryptor.h index a2c83f5..7989401 100644 --- a/core/include/services_sdk/interfaces/i_encryptor.h +++ b/core/include/services_sdk/interfaces/i_encryptor.h @@ -17,6 +17,8 @@ #include "maybe_res.h" #include +#include + static const std::string data1_file_name = "data1.a"; static const std::string data4_file_name = "data4.a"; @@ -29,6 +31,7 @@ static const std::string session_token_file_name = "data3.a"; class I_Encryptor { public: + // Base64 virtual std::string base64Encode(const std::string &input) = 0; virtual std::string base64Decode(const std::string &input) = 0; diff --git a/core/include/services_sdk/interfaces/i_mainloop.h b/core/include/services_sdk/interfaces/i_mainloop.h index 4f6006f..99c1c3b 100644 --- a/core/include/services_sdk/interfaces/i_mainloop.h +++ b/core/include/services_sdk/interfaces/i_mainloop.h @@ -47,6 +47,16 @@ public: const std::string &routine_name, bool is_primary = false ) = 0; + + virtual RoutineID + addBalancedIntervalRoutine( + RoutineType priority, + std::chrono::microseconds interval, + Routine func, + const std::string &routine_name, + std::chrono::microseconds offset = std::chrono::microseconds(0), + bool is_primary = false + ) = 0; virtual RoutineID addFileRoutine( diff --git a/core/include/services_sdk/interfaces/i_messaging.h b/core/include/services_sdk/interfaces/i_messaging.h index e8ba7e6..b9e704b 100644 --- a/core/include/services_sdk/interfaces/i_messaging.h +++ b/core/include/services_sdk/interfaces/i_messaging.h @@ -85,8 +85,8 @@ public: ) = 0; virtual Maybe uploadFile( - const std::string & uri, - const std::string & upload_file_path, + const std::string &uri, + const std::string &upload_file_path, const MessageCategory category = MessageCategory::GENERIC, MessageMetadata message_metadata = MessageMetadata() ) = 0; @@ -100,6 +100,8 @@ public: virtual bool setFogConnection(MessageCategory category = MessageCategory::GENERIC) = 0; + virtual void clearConnections() = 0; + protected: virtual ~I_Messaging() {} }; diff --git a/core/include/services_sdk/interfaces/i_rest_api.h b/core/include/services_sdk/interfaces/i_rest_api.h index b968057..7c919ac 100644 --- a/core/include/services_sdk/interfaces/i_rest_api.h +++ b/core/include/services_sdk/interfaces/i_rest_api.h @@ -53,8 +53,13 @@ public: const std::string &uri, const std::function &callback ) = 0; + virtual bool addPostCall( + const std::string &uri, + const std::function(const std::string &)> &callback + ) = 0; virtual uint16_t getListeningPort() const = 0; + virtual uint16_t getStartingPortRange() const = 0; protected: ~I_RestApi() {} diff --git a/core/include/services_sdk/interfaces/i_socket_is.h b/core/include/services_sdk/interfaces/i_socket_is.h index 202bc90..b900145 100644 --- a/core/include/services_sdk/interfaces/i_socket_is.h +++ b/core/include/services_sdk/interfaces/i_socket_is.h @@ -36,6 +36,7 @@ public: virtual void closeSocket(socketFd &socket) = 0; virtual bool writeData(socketFd socket, const std::vector &data) = 0; + virtual bool writeDataAsync(socketFd socket, const std::vector &data) = 0; virtual Maybe> receiveData(socketFd socket, uint data_size, bool is_blocking = true) = 0; virtual bool isDataAvailable(socketFd socket) = 0; diff --git a/core/include/services_sdk/interfaces/messaging/interface_impl.h b/core/include/services_sdk/interfaces/messaging/interface_impl.h index ea6ea6d..f26decf 100644 --- a/core/include/services_sdk/interfaces/messaging/interface_impl.h +++ b/core/include/services_sdk/interfaces/messaging/interface_impl.h @@ -80,6 +80,15 @@ I_Messaging::sendSyncMessage( ); if (!response_data.ok()) return response_data.passErr(); + if (response_data.unpack().getHTTPStatusCode() != HTTPStatusCode::HTTP_OK) { + return genError( + HTTPResponse( + response_data.unpack().getHTTPStatusCode(), + response_data.unpack().getBody() + ) + ); + } + auto res_obj = req_obj.loadJson(response_data.unpack().getBody()); if (!res_obj) { return genError( @@ -114,6 +123,7 @@ I_Messaging::sendSyncMessageWithoutResponse( category, message_metadata ); + if (!response_data.ok()) { dbgWarning(D_MESSAGING) << "Received error from server. Status code: " @@ -122,6 +132,16 @@ I_Messaging::sendSyncMessageWithoutResponse( << response_data.getErr().getBody(); return false; } + + if (response_data.unpack().getHTTPStatusCode() != HTTPStatusCode::HTTP_OK) { + dbgWarning(D_MESSAGING) + << "Unexpected status code from server. Status code: " + << int(response_data.unpack().getHTTPStatusCode()) + << ", response body: " + << response_data.unpack().getBody(); + return false; + } + return true; } diff --git a/core/include/services_sdk/interfaces/mock/mock_mainloop.h b/core/include/services_sdk/interfaces/mock/mock_mainloop.h index a439995..dd06432 100644 --- a/core/include/services_sdk/interfaces/mock/mock_mainloop.h +++ b/core/include/services_sdk/interfaces/mock/mock_mainloop.h @@ -15,6 +15,11 @@ public: uint (RoutineType, std::chrono::microseconds, Routine, const std::string &, bool) ); + MOCK_METHOD6( + addBalancedIntervalRoutine, + uint (RoutineType, std::chrono::microseconds, Routine, const std::string &, std::chrono::microseconds, bool) + ); + MOCK_METHOD5( addFileRoutine, uint (RoutineType, int, Routine, const std::string &, bool) diff --git a/core/include/services_sdk/interfaces/mock/mock_messaging.h b/core/include/services_sdk/interfaces/mock/mock_messaging.h index 8cb38af..b300c3c 100644 --- a/core/include/services_sdk/interfaces/mock/mock_messaging.h +++ b/core/include/services_sdk/interfaces/mock/mock_messaging.h @@ -54,6 +54,7 @@ public: MOCK_METHOD4(setFogConnection, bool(const string &, uint16_t, bool, MessageCategory)); MOCK_METHOD0(setFogConnection, bool()); MOCK_METHOD1(setFogConnection, bool(MessageCategory)); + MOCK_METHOD0(clearConnections, void()); }; static std::ostream & diff --git a/core/include/services_sdk/interfaces/mock/mock_rest_api.h b/core/include/services_sdk/interfaces/mock/mock_rest_api.h index 7cb9f3f..69368bf 100644 --- a/core/include/services_sdk/interfaces/mock/mock_rest_api.h +++ b/core/include/services_sdk/interfaces/mock/mock_rest_api.h @@ -9,7 +9,12 @@ class MockRestApi : public Singleton::Provide::From &)); + MOCK_METHOD2( + addPostCall, + bool(const std::string &, const std::function(const std::string &)> &) + ); MOCK_METHOD2( addWildcardGetCall, bool(const std::string &, const std::function &) diff --git a/core/include/services_sdk/interfaces/mock/mock_socket_is.h b/core/include/services_sdk/interfaces/mock/mock_socket_is.h index bcda2b4..9161503 100644 --- a/core/include/services_sdk/interfaces/mock/mock_socket_is.h +++ b/core/include/services_sdk/interfaces/mock/mock_socket_is.h @@ -15,6 +15,7 @@ public: MOCK_METHOD1(closeSocket, void (socketFd &)); MOCK_METHOD1(isDataAvailable, bool (socketFd)); MOCK_METHOD2(writeData, bool (socketFd, const std::vector &)); + MOCK_METHOD2(writeDataAsync, bool (socketFd, const std::vector &)); MOCK_METHOD3(receiveData, Maybe> (socketFd, uint, bool is_blocking)); }; diff --git a/core/include/services_sdk/resources/agent_details.h b/core/include/services_sdk/resources/agent_details.h index 6b554c7..81e6b83 100644 --- a/core/include/services_sdk/resources/agent_details.h +++ b/core/include/services_sdk/resources/agent_details.h @@ -55,7 +55,8 @@ class AgentDetails Singleton::Consume, Singleton::Consume, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: AgentDetails() : Component("AgentDetails") {} @@ -80,7 +81,10 @@ public: void setFogDomain(const std::string &_fog_domain) { fog_domain = _fog_domain; } void setFogPort(const uint16_t _fog_port) { fog_port = _fog_port; } - void setProxy(const std::string &_proxy) { proxy = _proxy; } + void setProxy(const std::string &_proxy) { + previous_proxy = proxy; + proxy = _proxy; + } void setAgentId(const std::string &_agent_id) { agent_id = _agent_id; } void setProfileId(const std::string &_profile_id) { profile_id = _profile_id; } void setTenantId(const std::string &_tenant_id) { tenant_id = _tenant_id; } @@ -121,6 +125,7 @@ private: OrchestrationMode orchestration_mode = OrchestrationMode::ONLINE; std::string server = "Unknown"; bool is_proxy_configured_via_settings = false; + std::string previous_proxy = ""; std::map proxies; static const std::map machineTypes; diff --git a/core/include/services_sdk/resources/config/config_impl.h b/core/include/services_sdk/resources/config/config_impl.h index 9f318f8..81505c0 100644 --- a/core/include/services_sdk/resources/config/config_impl.h +++ b/core/include/services_sdk/resources/config/config_impl.h @@ -18,10 +18,20 @@ #error "config_impl.h should not be included directly" #endif // __CONFIG_H__ +#include +#include +#include +#include +#include +#include + namespace Config { - -class MockConfigProvider : Singleton::Provide {}; +class MockConfigProvider + : + public Singleton::Provide, + public Singleton::Consume +{}; template std::size_t @@ -62,22 +72,290 @@ getVector(const Strings & ... strs) return res; } +// Utility function to create a separated string from a vector +inline std::string +makeSeparatedStr(const std::vector &vec, const std::string &separator = ", ") +{ + if (vec.empty()) return ""; + if (vec.size() == 1) return vec[0]; + + std::string result = vec[0]; + for (size_t i = 1; i < vec.size(); ++i) { + result += separator + vec[i]; + } + return result; +} + } // namespace Config +// Efficient service type checking for caching +inline bool isHttpTransactionHandler() { + static bool is_http_transaction_handler = false; + static bool service_checked = false; + + if (!service_checked) { + auto i_environment = Singleton::Consume::by(); + if (i_environment != nullptr) { + auto maybe_service_name = i_environment->get("Service Name"); + if (maybe_service_name.ok()) { + is_http_transaction_handler = (maybe_service_name.unpack() == "HTTP Transaction Handler"); + service_checked = true; + } + } + } + return is_http_transaction_handler; +} + +// Context registration for cache-enabled configurations +template +struct ContextRegistration { + static std::map, std::string> path_to_context_map; + + static void registerContext(const std::vector& paths, const std::string& context_type) { + path_to_context_map[paths] = context_type; + } + + static std::string getContext(const std::vector& paths) { + auto it = path_to_context_map.find(paths); + return (it != path_to_context_map.end()) ? it->second : ""; + } +}; + +// Static member definition +template +std::map, std::string> ContextRegistration::path_to_context_map; + +template +struct ConfigCacheKey { + std::vector paths; + std::string context_value; + std::string policy_load_id; + + bool operator==(const ConfigCacheKey &other) const + { + return paths == other.paths && + context_value == other.context_value && + policy_load_id == other.policy_load_id; + } + + bool match( + const std::vector& other_paths, + const std::string& other_context_value, + const std::string& other_policy_load_id + ) const + { + return paths == other_paths && + context_value == other_context_value && + policy_load_id == other_policy_load_id; + } +}; + +template +struct ConfigCacheEntry { + ConfigCacheKey key; + Maybe value; + ConfigCacheEntry() + : key(), value(genError(Config::Errors::MISSING_TAG)) {} + + bool isValid() const { return !key.context_value.empty(); } + void invalidate() + { + key.context_value.clear(); + value = genError(Config::Errors::MISSING_TAG); + } +}; + + template const Maybe & getConfiguration(const Strings & ... strs) { auto i_config = Singleton::Consume::from(); - return i_config->getConfiguration(Config::getVector(strs ...)).template getValue(); + const auto &paths = Config::getVector(strs ...); + + return i_config->getConfiguration(paths).template getValue(); }; +// LCOV_EXCL_START - Helper function to isolate static variables from lcov function data mismatch +// Helper function to get cache array - isolates static variables +template +ConfigCacheEntry* getCacheArray() { + static ConfigCacheEntry config_cache[3]; + return config_cache; +} + +// Cache statistics tracking +struct CacheStats { + static std::atomic hits; + static std::atomic misses; + static bool tracking_enabled; + + static void recordHit() { + if (tracking_enabled) hits.fetch_add(1, std::memory_order_relaxed); + } + + static void recordMiss() { + if (tracking_enabled) misses.fetch_add(1, std::memory_order_relaxed); + } + + static uint64_t getHits() { return hits.load(std::memory_order_relaxed); } + static uint64_t getMisses() { return misses.load(std::memory_order_relaxed); } + + static void reset() { + hits.store(0, std::memory_order_relaxed); + misses.store(0, std::memory_order_relaxed); + } + + static void enableTracking() { tracking_enabled = true; } + static void disableTracking() { tracking_enabled = false; } + static bool isTrackingEnabled() { return tracking_enabled; } +}; + +// Initialize cache tracking from environment variable +inline void initializeCacheTracking() { + const char* enable_tracking = std::getenv("ENABLE_CONFIG_CACHE_TRACKING"); + if (enable_tracking != nullptr) { + // Check for various "true" values + std::string tracking_value(enable_tracking); + std::transform(tracking_value.begin(), tracking_value.end(), tracking_value.begin(), ::tolower); + if (tracking_value == "true") { + CacheStats::enableTracking(); + CacheStats::reset(); // Start with clean counters when enabling tracking + } + } +} +// LCOV_EXCL_STOP + +template +const Maybe & +getConfigurationWithCache(const Strings & ... strs) +{ + // Step 1: Check if current service is HTTP Transaction Handler + if (!isHttpTransactionHandler()) { + return getConfiguration(strs...); + } + + // Step 2: Fast checks - get basic info + auto i_config = Singleton::Consume::from(); + const auto &paths = Config::getVector(strs ...); + size_t idx = paths.size(); + + // Step 3: Quick validation checks (fastest) + bool idx_valid = (idx >= 1 && idx <= 3); // max_cache_key_size = 3 + if (!idx_valid || !i_config->isConfigCacheEnabled()) { + return getConfiguration(strs...); + } + + // Step 4: Single map lookup - get context if registered, empty string if not + std::string context_type = ContextRegistration::getContext(paths); + if (context_type.empty()) { + return getConfiguration(strs...); + } + + // Step 5: Now we know it's registered - get environment value using the context + std::string context_value; + auto i_environment = Singleton::Consume::by(); + if (i_environment != nullptr) { + auto maybe_context_value = i_environment->get( + (context_type == "triggerId") ? "triggers" : "asset_id"); + if (maybe_context_value.ok()) { + context_value = maybe_context_value.unpack(); + } + } + + // Step 6: Final cache enablement check + if (context_value.empty()) { + return getConfiguration(strs...); + } + + // Step 7: Cache operations + auto* config_cache = getCacheArray(); + std::string policy_load_id = i_config->getPolicyLoadId(); + + // Check cache first + ConfigCacheEntry &entry = config_cache[idx - 1]; + if (entry.key.match(paths, context_value, policy_load_id)) { + // Cache hit + CacheStats::recordHit(); + return entry.value; + } + + // Cache miss - get configuration and update cache + CacheStats::recordMiss(); + const auto &maybe_val = i_config->getConfiguration(paths).template getValue(); + + // Update cache + config_cache[idx - 1].key = ConfigCacheKey{paths, context_value, policy_load_id}; + config_cache[idx - 1].value = maybe_val; + + return maybe_val; +} + +template +const Maybe & +setConfigurationInCache(const Strings & ... strs) +{ + // Step 1: Check if current service is HTTP Transaction Handler + if (!isHttpTransactionHandler()) { + return getConfiguration(strs...); + } + + // Step 2: Fast checks - get basic info + auto i_config = Singleton::Consume::from(); + const auto &paths = Config::getVector(strs ...); + size_t idx = paths.size(); + + // Step 3: Quick validation checks (fastest) + bool idx_valid = (idx >= 1 && idx <= 3); // max_cache_key_size = 3 + if (!idx_valid || !i_config->isConfigCacheEnabled()) { + // Early exit - no caching possible, just fetch and return + return getConfiguration(strs...); + } + + // Step 4: Single map lookup - get context if registered, empty string if not + std::string context_type = ContextRegistration::getContext(paths); + if (context_type.empty()) { + // Not registered for caching - just fetch and return + return getConfiguration(strs...); + } + + // Step 5: Now we know it's registered - get environment value using the context + std::string context_value; + auto i_environment = Singleton::Consume::by(); + if (i_environment != nullptr) { + auto maybe_context_value = i_environment->get( + (context_type == "triggerId") ? "triggers" : "asset_id"); + if (maybe_context_value.ok()) { + context_value = maybe_context_value.unpack(); + } + } + + // Step 6: Final cache enablement check + if (context_value.empty()) { + // No valid context value - just fetch and return + return getConfiguration(strs...); + } + + // Step 7: Always fetch configuration and update cache (no cache check first) + auto* config_cache = getCacheArray(); + std::string policy_load_id = i_config->getPolicyLoadId(); + + // Fetch configuration directly - no cache hit check + const auto &maybe_val = i_config->getConfiguration(paths).template getValue(); + + // Update cache with fresh value + config_cache[idx - 1].key = ConfigCacheKey{paths, context_value, policy_load_id}; + config_cache[idx - 1].value = maybe_val; + + return maybe_val; +} + template const ConfigurationType & getConfigurationWithDefault(const ConfigurationType &deafult_val, const Strings & ... tags) { if (!Singleton::exists()) return deafult_val; - auto &res = getConfiguration(tags ...); + auto &res = getConfigurationWithCache(tags ...); return res.ok() ? res.unpack() : deafult_val; } @@ -235,6 +513,18 @@ registerExpectedConfiguration(const Strings & ... tags) i_config->registerExpectedConfiguration(std::move(conf)); } +template +void +registerExpectedConfigurationWithCache(const std::string& context_type, const Strings & ... tags) +{ + // Register with the original system using existing function + registerExpectedConfiguration(tags...); + + // Register the context mapping + const auto &paths = Config::getVector(tags ...); + ContextRegistration::registerContext(paths, context_type); +} + template void registerExpectedResource(const Strings & ... tags) @@ -254,3 +544,4 @@ registerExpectedSetting(const Strings & ... tags) } #endif // __CONFIG_IMPL_H__ + diff --git a/core/include/services_sdk/resources/config/i_config.h b/core/include/services_sdk/resources/config/i_config.h index 70e5ab3..7b7e0a8 100644 --- a/core/include/services_sdk/resources/config/i_config.h +++ b/core/include/services_sdk/resources/config/i_config.h @@ -106,6 +106,18 @@ public: virtual void clearOldTenants() = 0; + virtual bool isConfigCacheEnabled() const = 0; + virtual void resetConfigCache() = 0; + virtual const std::string & getPolicyLoadId() const = 0; + + // Cache statistics access functions + virtual uint64_t getCacheHits() const = 0; + virtual uint64_t getCacheMisses() const = 0; + virtual void resetCacheStats() = 0; + virtual void enableCacheTracking() = 0; + virtual void disableCacheTracking() = 0; + virtual bool isCacheTrackingEnabled() const = 0; + protected: virtual ~I_Config() {} }; diff --git a/core/include/services_sdk/resources/context.h b/core/include/services_sdk/resources/context.h index 58cf922..93ed499 100644 --- a/core/include/services_sdk/resources/context.h +++ b/core/include/services_sdk/resources/context.h @@ -76,15 +76,27 @@ public: template void registerValue(const std::string &name, const T &value, Attr ... attr); + template + void registerQuickAccessValue(const std::string &name, const T &value, Attr ... attr); + template void registerValue(MetaDataType name, Params ... params); + template + void registerQuickAccessValue(MetaDataType name, Params ... params); + template void registerFunc(const std::string &name, std::function &&func, Attr ... attr); + template + void registerQuickAccessFunc(const std::string &name, std::function &&func, Attr ... attr); + template void registerFunc(const std::string &name, std::function()> &&func, Attr ... attr); + template + void registerQuickAccessFunc(const std::string &name, std::function()> &&func, Attr ... attr); + template void unregisterKey(const std::string &name); @@ -105,6 +117,7 @@ public: private: std::map> values; + std::map> quick_access_values; // Common values for all contexts }; class ScopedContext; diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index c15587a..3842b91 100644 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -22,6 +22,7 @@ DEFINE_FLAG(D_INFRA, D_ALL) DEFINE_FLAG(D_COMPRESSION, D_INFRA) DEFINE_FLAG(D_SHMEM, D_INFRA) DEFINE_FLAG(D_CONFIG, D_INFRA) + DEFINE_FLAG(D_CONFIG_CACHE, D_INFRA) DEFINE_FLAG(D_ENVIRONMENT, D_INFRA) DEFINE_FLAG(D_INTELLIGENCE, D_INFRA) DEFINE_FLAG(D_RULEBASE_CONFIG, D_INFRA) @@ -74,6 +75,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_WAAP_AUTOMATION, D_WAAP) DEFINE_FLAG(D_WAAP_REGEX, D_WAAP) DEFINE_FLAG(D_WAAP_SAMPLE_SCAN, D_WAAP) + DEFINE_FLAG(D_WAAP_HYPERSCAN, D_WAAP) DEFINE_FLAG(D_WAAP_ASSET_STATE, D_WAAP) DEFINE_FLAG(D_WAAP_CONFIDENCE_CALCULATOR, D_WAAP) DEFINE_FLAG(D_WAAP_SERIALIZE, D_WAAP) @@ -89,6 +91,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_WAAP_STREAMING_PARSING, D_WAAP) DEFINE_FLAG(D_WAAP_HEADERS, D_WAAP) DEFINE_FLAG(D_WAAP_OVERRIDE, D_WAAP) + DEFINE_FLAG(D_WAAP_LEARN, D_WAAP) DEFINE_FLAG(D_WAAP_SAMPLE_HANDLING, D_WAAP_GLOBAL) DEFINE_FLAG(D_WAAP_SAMPLE_PREPROCESS, D_WAAP_SAMPLE_HANDLING) @@ -119,8 +122,11 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_IPS, D_COMPONENT) DEFINE_FLAG(D_FILE_UPLOAD, D_COMPONENT) DEFINE_FLAG(D_RATE_LIMIT, D_COMPONENT) + DEFINE_FLAG(D_AUTH_ENFORCE, D_COMPONENT) + DEFINE_FLAG(D_ANOMALY_DETECTION, D_COMPONENT) DEFINE_FLAG(D_ROLLBACK_TESTING, D_COMPONENT) DEFINE_FLAG(D_NGINX_MANAGER, D_COMPONENT) + DEFINE_FLAG(D_BROWSER_AGENT, D_COMPONENT) DEFINE_FLAG(D_PARSER, D_COMPONENT) DEFINE_FLAG(D_WS, D_COMPONENT) @@ -168,6 +174,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_NGINX_MESSAGE_READER, D_REVERSE_PROXY) DEFINE_FLAG(D_ERROR_REPORTER, D_REVERSE_PROXY) DEFINE_FLAG(D_UPSTREAM_KEEPALIVE, D_REVERSE_PROXY) + DEFINE_FLAG(D_UPSTREAM_HEALTH_CHECKER, D_REVERSE_PROXY) DEFINE_FLAG(D_FORWARD_PROXY, D_REVERSE_PROXY) DEFINE_FLAG(D_IDA, D_COMPONENT) @@ -204,6 +211,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_HORIZON_TELEMETRY, D_COMPONENT) DEFINE_FLAG(D_PROMETHEUS, D_COMPONENT) DEFINE_FLAG(D_AIGUARD, D_COMPONENT) + DEFINE_FLAG(D_ERM, D_COMPONENT) DEFINE_FLAG(D_FLOW, D_ALL) DEFINE_FLAG(D_DROP, D_FLOW) diff --git a/core/include/services_sdk/resources/environment/context_impl.h b/core/include/services_sdk/resources/environment/context_impl.h index 8e9397c..b31738d 100644 --- a/core/include/services_sdk/resources/environment/context_impl.h +++ b/core/include/services_sdk/resources/environment/context_impl.h @@ -109,6 +109,14 @@ Context::registerValue(const std::string &name, const T &value, Attr ... attr) registerFunc(name, std::move(new_func), attr ...); } +template +void +Context::registerQuickAccessValue(const std::string &name, const T &value, Attr ... attr) +{ + std::function()> new_func = [value] () { return Return(value); }; + registerQuickAccessFunc(name, std::move(new_func), attr ...); +} + template void Context::registerValue(MetaDataType name, Params ... params) @@ -116,6 +124,13 @@ Context::registerValue(MetaDataType name, Params ... params) return registerValue(convertToString(name), params ...); } +template +void +Context::registerQuickAccessValue(MetaDataType name, Params ... params) +{ + return registerQuickAccessValue(convertToString(name), params ...); +} + template void Context::registerFunc(const std::string &name, std::function &&func, Attr ... attr) @@ -124,6 +139,14 @@ Context::registerFunc(const std::string &name, std::function &&func, Attr . registerFunc(name, std::move(new_func), attr ...); } +template +void +Context::registerQuickAccessFunc(const std::string &name, std::function &&func, Attr ... attr) +{ + std::function()> new_func = [func] () { return Return(func()); }; + registerQuickAccessFunc(name, std::move(new_func), attr ...); +} + template void Context::registerFunc(const std::string &name, std::function()> &&func, Attr ... attr) @@ -133,6 +156,15 @@ Context::registerFunc(const std::string &name, std::function()> &&func values[key] = std::make_unique>(std::move(func)); } +template +void +Context::registerQuickAccessFunc(const std::string &name, std::function()> &&func, Attr ... attr) +{ + dbgTrace(D_ENVIRONMENT) << "Registering key : " << name; + Key key(name, typeid(T), EnvKeyAttr::ParamAttr(attr ...)); + quick_access_values[key] = std::make_unique>(std::move(func)); +} + template void Context::unregisterKey(const std::string &name) @@ -140,6 +172,7 @@ Context::unregisterKey(const std::string &name) dbgTrace(D_ENVIRONMENT) << "Unregistering key : " << name; Key key(name, typeid(T)); values.erase(key); + quick_access_values.erase(key); } template @@ -154,8 +187,12 @@ Context::Return Context::get(const std::string &name) const { Key key(name, typeid(T)); - auto iter = values.find(key); - if (iter == values.end()) return genError(Error::NO_VALUE); + auto iter = quick_access_values.find(key); + if (iter == quick_access_values.end()) { + // If not found in quick access, search in the main values map + iter = values.find(key); + if (iter == values.end()) return genError(Error::NO_VALUE); + } Value *val = dynamic_cast *>(iter->second.get()); return val->get(); } diff --git a/core/include/services_sdk/resources/intelligence_invalidation.h b/core/include/services_sdk/resources/intelligence_invalidation.h index e051a13..34e8376 100644 --- a/core/include/services_sdk/resources/intelligence_invalidation.h +++ b/core/include/services_sdk/resources/intelligence_invalidation.h @@ -188,6 +188,11 @@ public: bool matches(const Invalidation &other) const; void serialize(cereal::JSONInputArchive &ar); + Invalidation & addHeader(const std::string &key, const std::string &value); + Maybe getHeader(const std::string &key) const; + const std::map & getHeaders() const; + bool hasHeader(const std::string &key) const; + private: bool attr_matches(const std::vector ¤t, const std::vector &other) const; bool attr_matches(const std::vector ¤t, const std::vector &other) const; @@ -200,6 +205,7 @@ private: Maybe invalidation_type; Maybe listening_id; Maybe registration_id; + std::map headers; }; } // namespace Intelligence diff --git a/core/include/services_sdk/resources/metric/metric_calc.h b/core/include/services_sdk/resources/metric/metric_calc.h index eda3025..60a11d8 100644 --- a/core/include/services_sdk/resources/metric/metric_calc.h +++ b/core/include/services_sdk/resources/metric/metric_calc.h @@ -41,6 +41,7 @@ struct PrometheusData { try { ar(cereal::make_nvp("metric_name", name)); + ar(cereal::make_nvp("unique_name", unique_name)); ar(cereal::make_nvp("metric_type", type)); ar(cereal::make_nvp("metric_description", description)); ar(cereal::make_nvp("labels", label)); @@ -51,6 +52,7 @@ struct PrometheusData } std::string name; + std::string unique_name; std::string type; std::string description; std::string label; diff --git a/core/include/services_sdk/resources/report/report_enums.h b/core/include/services_sdk/resources/report/report_enums.h index 3c3cfd3..f4e08bc 100644 --- a/core/include/services_sdk/resources/report/report_enums.h +++ b/core/include/services_sdk/resources/report/report_enums.h @@ -66,13 +66,15 @@ enum class Tags { CROWDSEC, PLAYGROUND, API_DISCOVERY, + LB_HEALTH_STATUS, NGINX_PROXY_MANAGER, WEB_SERVER_APISIX, DEPLOYMENT_DOCKER, WEB_SERVER_SWAG, WEB_SERVER_NGINX_UNIFIED, AIGUARD, - + CENTRAL_NGINX_MANAGER, + BROWSER_AGENT, COUNT }; @@ -162,7 +164,9 @@ enum class IssuingEngine { IDA_SAML_IDN_BLADE_REGISTRATION, IDA_SAML_IDN_CLIENT_IP_NOTIFY, HORIZON_TELEMETRY_METRICS, - API_DISCOVERY + API_DISCOVERY, + LB_HEALTH_STATUS, + BROWSER_AGENT }; } // namespace ReportIS diff --git a/core/include/services_sdk/resources/rest.h b/core/include/services_sdk/resources/rest.h index c9a34fb..f246302 100644 --- a/core/include/services_sdk/resources/rest.h +++ b/core/include/services_sdk/resources/rest.h @@ -180,10 +180,31 @@ public: /// @brief Performs the REST call using the input stream. /// @param in The input stream containing the JSON data for the REST call. + /// @param headers The HTTP headers from the request. /// @return A Maybe object containing the result of the REST call (either the JSON data or an error message). - Maybe performRestCall(std::istream &in); + Maybe performRestCall(std::istream &in, const std::map &headers); + + /// @brief Performs the REST call using the input stream (backwards compatibility overload). + /// @param in The input stream containing the JSON data for the REST call. + /// @return A Maybe object containing the result of the REST call (either the JSON data or an error message). + Maybe performRestCall(std::istream &in) { + return performRestCall(in, std::map()); + } + + /// @brief Indicates whether this handler wants to receive HTTP headers. + /// @return True if the handler wants headers, false otherwise. Default is false. + virtual bool wantsHeaders() const { return false; } + + /// @brief Sets the HTTP headers for this handler (used by bulk handlers to propagate headers). + /// @param headers The HTTP headers to set. + void setRequestHeaders(const std::map &headers) { + request_headers = headers; + } protected: + /// @brief HTTP headers from the current request (only populated if wantsHeaders() returns true). + std::map request_headers; + /// @brief Determines if the direction is for input. /// @param dir The direction of the communication. /// @return True if the direction is for input, false otherwise. diff --git a/core/include/services_sdk/utilities/agent_core_utilities.h b/core/include/services_sdk/utilities/agent_core_utilities.h index ea0a821..8fab25d 100644 --- a/core/include/services_sdk/utilities/agent_core_utilities.h +++ b/core/include/services_sdk/utilities/agent_core_utilities.h @@ -43,6 +43,14 @@ copyFile( mode_t permission = (S_IWUSR | S_IRUSR) ); +bool +createFileWithContent( + const std::string &dest, + const std::string &content, + bool overide_if_exists, + mode_t permission = (S_IWUSR | S_IRUSR) +); + bool deleteFile(const std::string &path); std::string convertToHumanReadable(uint64_t size_in_bytes); std::string getFileName(const std::string &path); @@ -86,6 +94,7 @@ std::string removeTrailingWhitespaces(std::string str); std::string removeLeadingWhitespaces(std::string str); std::string trim(std::string str); std::string toLower(std::string str); +bool startsWith(const std::string &str, const std::string &prefix); } // namespace Strings diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc index 193e11b..52f08f8 100644 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -17,9 +17,11 @@ #include "cache.h" #include "config.h" +#include "i_environment.h" #include "intelligence_invalidation.h" #include "intelligence_is_v2/intelligence_response.h" #include "intelligence_request.h" +#include "intell_registration_event.h" using namespace std; using namespace chrono; @@ -37,6 +39,8 @@ static const string queries_uri = "/api/v2/intelligence/assets/queries"; static const string fog_health_uri = "/access-manager/health/live"; static const string intelligence_health_uri = "/show-health"; static const string time_range_invalidation_uri = "/api/v2/intelligence/invalidation/get"; +static const uint default_registration_interval_seconds = 720; // 12 minutes +static const uint min_registration_interval_seconds = 30; class I_InvalidationCallBack { @@ -100,7 +104,7 @@ public: res << "\"name\": \"" << (agent_id.empty() ? details->getAgentId() : agent_id) << "\", "; auto rest = Singleton::Consume::by(); res << "\"url\": \"http://127.0.0.1:" << rest->getListeningPort() <<"/set-new-invalidation\", "; - res << "\"capabilities\": { \"getBulkCallback\": " << "true" << " }, "; + res << "\"capabilities\": { \"getBulkCallback\": true, \"returnRegistrationTTL\": true }, "; res << "\"dataMap\": ["; res << stream.str(); res << " ] }"; @@ -200,11 +204,17 @@ private: class SingleReceivedInvalidation : public ServerRest { public: + void doCall() override { Invalidation invalidation(class_name); + for (const auto& header : request_headers) { + dbgTrace(D_INTELLIGENCE) << "Adding header: " << header.first << " = " << header.second; + invalidation.addHeader(header.first, header.second); + } + if (category.isActive()) invalidation.setClassifier(ClassifierType::CATEGORY, category.get()); if (family.isActive()) invalidation.setClassifier(ClassifierType::FAMILY, family.get()); if (group.isActive()) invalidation.setClassifier(ClassifierType::GROUP, group.get()); @@ -268,10 +278,10 @@ private: C2S_OPTIONAL_PARAM(string, invalidationType); }; - class ReceiveInvalidation : public ServerRest { public: + bool wantsHeaders() const override { return true; } void doCall() override @@ -282,6 +292,8 @@ public: : "error in format, expected bulk invalidations, not single"); for (SingleReceivedInvalidation &r : bulkArray.get()) { + // Copy headers from the bulk request to each individual invalidation + r.setRequestHeaders(request_headers); r.doCall(); } return; @@ -360,7 +372,7 @@ public: mainloop->addRecurringRoutine( I_MainLoop::RoutineType::System, - chrono::minutes(12), + chrono::seconds(getRegistrationIntervalSec()), [this] () { sendRecurringInvalidationRegistration(); }, "Sending intelligence invalidation" ); @@ -467,6 +479,27 @@ public: } private: + uint + getRegistrationIntervalSec() const + { + uint interval_in_seconds = getConfigurationWithDefault( + default_registration_interval_seconds, + "intelligence", + "registration interval seconds" + ); + if (interval_in_seconds < min_registration_interval_seconds) { + dbgWarning(D_INTELLIGENCE) + << "Registration interval is too low, setting to minimum: " + << min_registration_interval_seconds; + interval_in_seconds = min_registration_interval_seconds; + } + dbgInfo(D_INTELLIGENCE) + << "Using registration interval: " + << interval_in_seconds + << " seconds"; + return interval_in_seconds; + } + bool hasLocalIntelligenceSupport() const { @@ -585,6 +618,7 @@ private: sendIntelligenceRequestImpl(const Invalidation &invalidation, const MessageMetadata &local_req_md) const { dbgFlow(D_INTELLIGENCE) << "Sending intelligence invalidation"; + auto res = message->sendSyncMessageWithoutResponse( HTTPMethod::POST, invalidation_uri, @@ -634,15 +668,29 @@ private: ) const { dbgFlow(D_INTELLIGENCE) << "Sending intelligence invalidation registration"; - auto res = message->sendSyncMessageWithoutResponse( + Maybe registration_body = registration.genJson(); + if (!registration_body.ok()) { + return genError("Could not generate intelligence invalidation registration body. Error: " + + registration_body.getErr()); + } + + auto res = message->sendSyncMessage( HTTPMethod::POST, registration_uri, - registration, + registration_body.unpack(), MessageCategory::INTELLIGENCE, registration_req_md ); - if (res) return Response(); - dbgWarning(D_INTELLIGENCE) << "Could not send intelligence invalidation registration."; + if (res.ok()){ + string registration_response = res.unpack().getBody(); + dbgInfo(D_INTELLIGENCE) + << "Intelligence invalidation registration sent successfully"; + IntelligenceRegistrationEvent(true, registration_response).notify(); + return Response(); + } + IntelligenceRegistrationEvent(false).notify(); + dbgWarning(D_INTELLIGENCE) << "Could not send intelligence invalidation registration. Error: " + << res.getErr().toString(); return genError("Could not send intelligence invalidation registration"); } @@ -719,6 +767,13 @@ private: auto rest = Singleton::Consume::by(); auto agent = (agent_id.empty() ? details->getAgentId() : agent_id) + ":" + to_string(rest->getListeningPort()); headers["X-Source-Id"] = agent; + auto env = Singleton::Consume::by(); + auto exec_name = env->get("Base Executable Name"); + if (exec_name.ok() && *exec_name != "") { + headers["X-Calling-Service"] = *exec_name; + } else { + dbgTrace(D_INTELLIGENCE) << "getHTTPHeaders: X-Calling-Service NOT added - exec_name not available"; + } return headers; } @@ -762,6 +817,7 @@ IntelligenceComponentV2::preload() { registerExpectedConfiguration("intelligence", "maximum request overall time"); registerExpectedConfiguration("intelligence", "maximum request lap time"); + registerExpectedConfiguration("intelligence", "registration interval seconds"); registerExpectedConfiguration("intelligence", "support Invalidation"); registerExpectedSetting("intelligence", "local intelligence server ip"); registerExpectedSetting("intelligence", primary_port_setting); diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc index 752c61d..d2d8528 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc @@ -344,6 +344,7 @@ public: ON_CALL(mock_details, getFogDomain()).WillByDefault(Return(Maybe(string("fog_domain.com")))); ON_CALL(mock_details, getFogPort()).WillByDefault(Return(Maybe(443))); + env.preload(); conf.preload(); intelligence.preload(); intelligence.init(); @@ -389,6 +390,8 @@ TEST_F(IntelligenceInvalidation, sending_incomplete_invalidation) TEST_F(IntelligenceInvalidation, sending_public_invalidation) { + Singleton::Consume::from(env)->registerValue("Base Executable Name", "idn"); + auto invalidation = Invalidation("aaa") .addMainAttr(main_attr) .addAttr(attr) @@ -422,10 +425,16 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation) " } ] }"; EXPECT_EQ(invalidation_json, expected_json); EXPECT_FALSE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN)); + + auto headers = md.getHeaders(); + EXPECT_NE(headers.find("X-Calling-Service"), headers.end()) << "X-Calling-Service header should be present"; + EXPECT_EQ(headers.at("X-Calling-Service"), "idn"); } TEST_F(IntelligenceInvalidation, multiple_assets_invalidation) { + Singleton::Consume::from(env)->registerValue("Base Executable Name", "orchestration"); + auto main_attr_2 = StrAttributes() .addStringAttr("attr2", "22") .addStringSetAttr("attr3", {"33", "44"}); @@ -440,11 +449,13 @@ TEST_F(IntelligenceInvalidation, multiple_assets_invalidation) .setObjectType(Intelligence::ObjectType::ASSET); string invalidation_json; + MessageMetadata md; EXPECT_CALL( messaging_mock, sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _) ).WillOnce(DoAll( SaveArg<2>(&invalidation_json), + SaveArg<4>(&md), Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")) )); @@ -461,10 +472,16 @@ TEST_F(IntelligenceInvalidation, multiple_assets_invalidation) "\"attributes\": [ { \"ipv4Addresses\": [ \"1.1.1.1\" ] } ]" " } ] }"; EXPECT_EQ(invalidation_json, expected_json); + + auto headers = md.getHeaders(); + EXPECT_NE(headers.find("X-Calling-Service"), headers.end()); + EXPECT_EQ(headers.at("X-Calling-Service"), "orchestration"); } TEST_F(IntelligenceInvalidation, sending_private_invalidation) { + Singleton::Consume::from(env)->registerValue("Base Executable Name", "idn"); + auto invalidation = Invalidation("aaa") .addMainAttr(main_attr) .addAttr(attr) @@ -511,6 +528,10 @@ TEST_F(IntelligenceInvalidation, sending_private_invalidation) " } ] }"; EXPECT_EQ(invalidation_json, expected_json); EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN)); + + auto headers = md.getHeaders(); + EXPECT_NE(headers.find("X-Calling-Service"), headers.end()); + EXPECT_EQ(headers.at("X-Calling-Service"), "idn"); } TEST_F(IntelligenceInvalidation, register_for_invalidation) @@ -554,7 +575,7 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation) EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]")); EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"ipv4Addresses\": [ \"1.1.1.1\" ] } ]")); EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN)); - EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true }")); + EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true, \"returnRegistrationTTL\": true }")); } @@ -599,7 +620,7 @@ TEST_F(IntelligenceInvalidation, register_for_multiple_assets_invalidation) )); EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0); - EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true }")); + EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true, \"returnRegistrationTTL\": true }")); EXPECT_THAT( body, diff --git a/core/intelligence_is_v2/invalidation.cc b/core/intelligence_is_v2/invalidation.cc index 4dbfb97..f4818d5 100644 --- a/core/intelligence_is_v2/invalidation.cc +++ b/core/intelligence_is_v2/invalidation.cc @@ -274,44 +274,44 @@ Invalidation::serialize(cereal::JSONInputArchive &ar) try { ar(cereal::make_nvp("class", class_)); } catch (const cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgWarning(D_INTELLIGENCE) << e.what(); } try { ar(cereal::make_nvp("category", category)); } catch (const cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgTrace(D_INTELLIGENCE) << e.what(); } try { ar(cereal::make_nvp("family", family)); } catch (const cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgTrace(D_INTELLIGENCE) << e.what(); } try { ar(cereal::make_nvp("group", group)); } catch (const cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgTrace(D_INTELLIGENCE) << e.what(); } try { ar(cereal::make_nvp("order", order)); } catch (const cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgTrace(D_INTELLIGENCE) << e.what(); } try { ar(cereal::make_nvp("kind", kind)); } catch (const cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgTrace(D_INTELLIGENCE) << e.what(); } try { ar(cereal::make_nvp("mainAttributes", main_attributes)); ar(cereal::make_nvp("attributes", attributes)); } catch (const cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgTrace(D_INTELLIGENCE) << e.what(); } try { @@ -323,21 +323,21 @@ Invalidation::serialize(cereal::JSONInputArchive &ar) throw std::invalid_argument("Invalid string for ObjectType: " + object_type_); } } catch (const cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgTrace(D_INTELLIGENCE) << e.what(); } try { ar(cereal::make_nvp("sourceId", source_id_)); source_id = source_id_; } catch (const cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgTrace(D_INTELLIGENCE) << e.what(); } try { ar(cereal::make_nvp("invalidationRegistrationId", registration_id_)); registration_id = registration_id_; } catch (const cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgTrace(D_INTELLIGENCE) << e.what(); } try { @@ -349,14 +349,14 @@ Invalidation::serialize(cereal::JSONInputArchive &ar) throw std::invalid_argument("Invalid string for InvalidationType: " + invalidation_type_); } } catch (const cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgWarning(D_INTELLIGENCE) << e.what(); } try { ar(cereal::make_nvp("listeningId", listening_id_)); listening_id = listening_id_; } catch (const cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgTrace(D_INTELLIGENCE) << e.what(); } classifiers[ClassifierType::CLASS] = class_; @@ -381,6 +381,35 @@ Invalidation::addMainAttr(const StrAttributes &attr) return *this; } +Invalidation & +Invalidation::addHeader(const string &name, const string &value) +{ + headers[name] = value; + return *this; +} + +Maybe +Invalidation::getHeader(const string &name) const +{ + auto it = headers.find(name); + if (it != headers.end()) { + return it->second; + } + return genError("Header not found: " + name); +} + +const map & +Invalidation::getHeaders() const +{ + return headers; +} + +bool +Invalidation::hasHeader(const string &name) const +{ + return headers.find(name) != headers.end(); +} + Maybe Invalidation::getRegistrationID() const{ return registration_id; @@ -660,11 +689,26 @@ IpAttributes::serialize(cereal::JSONInputArchive &ar) { try { ar(cereal::make_nvp("ipv4Addresses", ipv4_addresses)); + } catch (cereal::Exception &e) { + dbgTrace(D_INTELLIGENCE) << e.what(); + } + + try { ar(cereal::make_nvp("ipv4AddressesRange", ipv4_address_ranges)); + } catch (cereal::Exception &e) { + dbgTrace(D_INTELLIGENCE) << e.what(); + } + + try { ar(cereal::make_nvp("ipv6Addresses", ipv6_addresses)); + } catch (cereal::Exception &e) { + dbgTrace(D_INTELLIGENCE) << e.what(); + } + + try { ar(cereal::make_nvp("ipv6AddressesRange", ipv6_address_ranges)); } catch (cereal::Exception &e) { - dbgError(D_INTELLIGENCE) << e.what(); + dbgTrace(D_INTELLIGENCE) << e.what(); } } diff --git a/core/mainloop/mainloop.cc b/core/mainloop/mainloop.cc index b025cc5..7dea47a 100644 --- a/core/mainloop/mainloop.cc +++ b/core/mainloop/mainloop.cc @@ -13,12 +13,14 @@ #include "mainloop.h" +#include #include #include #include #include #include #include +#include #include "config.h" #include "coroutine.h" @@ -62,6 +64,16 @@ public: bool is_primary ) override; + RoutineID + addBalancedIntervalRoutine( + RoutineType priority, + std::chrono::microseconds interval, + Routine func, + const std::string &routine_name, + chrono::microseconds offset, + bool is_primary + ) override; + RoutineID addFileRoutine( RoutineType priority, @@ -430,6 +442,48 @@ MainloopComponent::Impl::addRecurringRoutine( return addOneTimeRoutine(priority, func_wrapper, routine_name, is_primary); } +I_MainLoop::RoutineID +MainloopComponent::Impl::addBalancedIntervalRoutine( + RoutineType priority, + chrono::microseconds interval, + Routine func, + const string &routine_name, + chrono::microseconds offset, + bool is_primary +) +{ + Routine func_wrapper = [this, interval, offset, func, routine_name]() + { + using namespace std::chrono; + I_TimeGet *timer = Singleton::Consume::by(); + typedef duration> days; + static const microseconds one_day_in_microseconds = duration_cast(days(1)); + + while (true) { + microseconds now = timer->getWalltime(); + + size_t whole_days = now.count() / one_day_in_microseconds.count(); + microseconds time_since_midnight = now - microseconds(whole_days * one_day_in_microseconds.count()); + // Calculate next aligned execution time from midnight + size_t intervals_from_midnight = time_since_midnight / interval; + microseconds next_aligned_time = (intervals_from_midnight + 1) * interval; + // Calculate wait time until next execution + microseconds target_time = next_aligned_time + offset; + microseconds wait_time = target_time - time_since_midnight; + + if (wait_time > interval) { + wait_time -= interval; + } + dbgTrace(D_MAINLOOP) << "Balanced interval routine waiting " << wait_time << " microseconds"; + yield(wait_time); + + func(); + } + }; + + return addOneTimeRoutine(priority, func_wrapper, routine_name, is_primary); +} + I_MainLoop::RoutineID MainloopComponent::Impl::addFileRoutine( RoutineType priority, diff --git a/core/mainloop/mainloop_ut/mainloop_ut.cc b/core/mainloop/mainloop_ut/mainloop_ut.cc index f6f67be..0706d3a 100644 --- a/core/mainloop/mainloop_ut/mainloop_ut.cc +++ b/core/mainloop/mainloop_ut/mainloop_ut.cc @@ -24,7 +24,7 @@ USE_DEBUG_FLAG(D_MAINLOOP); class EndTest { }; - +typedef std::chrono::duration> days; class MainloopTest : public Test { public: @@ -74,6 +74,45 @@ public: ); } + void + setupBalancedIntervalTest( + chrono::microseconds start_time, + chrono::microseconds interval, + const string &routine_name, + chrono::microseconds offset = chrono::microseconds(0), + chrono::microseconds time_advance = chrono::hours(24) + ) + { + chrono::microseconds time = start_time; + EXPECT_CALL(mock_time, getWalltime()).WillRepeatedly(InvokeWithoutArgs([&]() { return time; })); + + EXPECT_CALL(mock_time, getMonotonicTime()) + .WillRepeatedly(InvokeWithoutArgs( + [&]() + { + auto old_time = time; + time += time_advance; + return old_time; + } + )); + auto callback = [this]() { mainloop->stop(); }; + + mainloop->addBalancedIntervalRoutine( + I_MainLoop::RoutineType::RealTime, interval, callback, routine_name, offset, true + ); + + // Run the mainloop + mainloop->run(); + } + + void + expectWaitTimeInDebug(chrono::microseconds expected_wait_time) + { + string expected_debug = + "Balanced interval routine waiting " + to_string(expected_wait_time.count()) + " microseconds"; + EXPECT_THAT(capture_debug.str(), HasSubstr(expected_debug)); + } + I_Environment::ActiveContexts active_context; NiceMock mock_time; @@ -532,3 +571,100 @@ TEST_F(MainloopTest, check_routine_name) HasSubstr("Starting execution of corutine. Routine named: check routine name test") ); } + +TEST_F(MainloopTest, balanced_interval_empty_routine_name_hour_start) +{ + Debug::setUnitTestFlag(D_MAINLOOP, Debug::DebugLevel::TRACE); + + // Start on day 1 at 4:00 (at the start of the hour) + // Interval is 2 hours + // Because routine_name is empty, no shifting will be done so remaining time will be exactly equal to interval + const std::string routine_name = ""; + const std::chrono::milliseconds time(days(1) + std::chrono::hours(4)); + const std::chrono::minutes interval(std::chrono::hours(2)); + + setupBalancedIntervalTest(time, interval, routine_name); + expectWaitTimeInDebug(interval); +} + +TEST_F(MainloopTest, balanced_interval_empty_routine_name_middle_of_the_hour) +{ + Debug::setUnitTestFlag(D_MAINLOOP, Debug::DebugLevel::TRACE); + // Start 1 day and 5:54 (in the middle of the second hour) + // Interval is 2 hours + // Because routine_name is empty, no shifting will be done so remaining time will be 6 minutes + const std::string routine_name = ""; + const std::chrono::milliseconds time(days(1) + std::chrono::hours(5) + std::chrono::minutes(54)); + const std::chrono::minutes interval(std::chrono::hours(2)); + + setupBalancedIntervalTest(time, interval, routine_name); + expectWaitTimeInDebug(std::chrono::minutes(6)); +} + +TEST_F(MainloopTest, balanced_interval_non_empty_routine_name_hour_start) +{ + Debug::setUnitTestFlag(D_MAINLOOP, Debug::DebugLevel::TRACE); + // Start on day 1 at 4:00 (at the start of the hour) + // Interval is 2 hours + // The routine_name is chosen so that it results in hashed slot #2 which shifts the remaining time by 2 * 10 + // minutes So, the remaining time to wait would be original interval + 20 minutes + const std::string routine_name = "5e9dac5d204a8f35b264a932"; + const std::chrono::milliseconds time(days(1) + std::chrono::hours(4)); + const std::chrono::minutes interval(std::chrono::hours(2)); + const std::chrono::microseconds offset = std::chrono::minutes(2*10); + + setupBalancedIntervalTest(time, interval, routine_name, offset); + + expectWaitTimeInDebug(offset); +} + +TEST_F(MainloopTest, balanced_interval_non_empty_routine_name_hour_start_post_offset) +{ + Debug::setUnitTestFlag(D_MAINLOOP, Debug::DebugLevel::TRACE); + // Start on day 1 at 4:40 (at the start of the hour) + // Interval is 2 hours + // The routine_name is chosen so that it results in hashed slot #2 which shifts the remaining time by 2 * 10 + // minutes So, the remaining time to wait would be original interval - 20 minutes + const std::string routine_name = "5e9dac5d204a8f35b264a932"; + const std::chrono::milliseconds time(days(1) + std::chrono::hours(4) + std::chrono::minutes(40)); + const std::chrono::minutes interval(std::chrono::hours(2)); + const std::chrono::microseconds offset = std::chrono::minutes(2*10); + + setupBalancedIntervalTest(time, interval, routine_name, offset); + + expectWaitTimeInDebug(interval - offset); +} + +TEST_F(MainloopTest, balanced_interval_non_empty_routine_name_middle_of_the_hour) +{ + Debug::setUnitTestFlag(D_MAINLOOP, Debug::DebugLevel::TRACE); + // Start 1 day and 5:54 (in the middle of the second hour) + // Interval is 2 hours + // The routine_name is chosen so that it results in hashed slot #2 which shifts the remaining time by 2 * 10 + // minutes So, the remaining time to wait would be 6 minutes + 20 minutes shift + const std::string routine_name = "5e9dac5d204a8f35b264a932"; + const std::chrono::milliseconds time(days(1) + std::chrono::hours(5) + std::chrono::minutes(54)); + const std::chrono::minutes interval(std::chrono::hours(2)); + const std::chrono::microseconds offset = std::chrono::minutes(2*10); + + setupBalancedIntervalTest(time, interval, routine_name, offset); + + expectWaitTimeInDebug(std::chrono::minutes(6) + offset); +} + +TEST_F(MainloopTest, balanced_interval_another_asset_hour_start) +{ + Debug::setUnitTestFlag(D_MAINLOOP, Debug::DebugLevel::TRACE); + // Start on day 1 at 4:00 (at the start of the hour) + // Interval is 2 hours + // The routine_name is chosen so that it results in hashed slot #7 which shifts the remaining time by 7 * 10 + // minutes So, the remaining time to wait would be 70 minutes + const std::string routine_name = "5e9da89572f6f9af9bebc0da"; + const std::chrono::milliseconds time(days(1) + std::chrono::hours(4)); + const std::chrono::minutes interval(std::chrono::hours(2)); + const std::chrono::microseconds offset = std::chrono::minutes(7*10); + + setupBalancedIntervalTest(time, interval, routine_name, offset); + + expectWaitTimeInDebug(offset); +} diff --git a/core/messaging/connection/connection.cc b/core/messaging/connection/connection.cc index b6796e9..c95c37f 100644 --- a/core/messaging/connection/connection.cc +++ b/core/messaging/connection/connection.cc @@ -39,7 +39,7 @@ using namespace smartBIO; USE_DEBUG_FLAG(D_CONNECTION); static const HTTPResponse sending_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to send all data in time"); -static const HTTPResponse receving_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to receive all data in time"); +static const HTTPResponse receiving_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to receive all data in time"); static const HTTPResponse parsing_error(HTTPStatusCode::HTTP_UNKNOWN, "Failed to parse the HTTP response"); static const HTTPResponse close_error( HTTPStatusCode::HTTP_UNKNOWN, @@ -271,18 +271,11 @@ private: return *details_ssl_dir; } - // Use detail_resolver to determine platform-specific certificate directory #if defined(alpine) - string platform = "alpine"; + return "/etc/ssl/certs/"; #else - string platform = "linux"; -#endif - - if (platform == "alpine") { - return "/etc/ssl/certs/"; - } - return "/usr/lib/ssl/certs/"; +#endif } Maybe @@ -741,20 +734,54 @@ private: } } - auto receiving_end_time = i_time->getMonotonicTime() + getConnectionTimeout(); + auto base_timeout_config = getProfileAgentSettingWithDefault( + 10, + "agent.config.message.chunk.connection.timeout" + ); + auto base_timeout = chrono::seconds(base_timeout_config); // 10 seconds between data chunks + + auto global_timeout_config = getProfileAgentSettingWithDefault( + 600, + "agent.config.message.global.connection.timeout" + ); + auto global_timeout = chrono::seconds(global_timeout_config); // 600 seconds maximum for entire download + + auto receiving_end_time = i_time->getMonotonicTime() + base_timeout; + auto global_end_time = i_time->getMonotonicTime() + global_timeout; HTTPResponseParser http_parser; - dbgTrace(D_CONNECTION) << "Sent the message, now waiting for response"; + dbgTrace(D_CONNECTION) + << "Sent the message, now waiting for response (global timeout: " + << global_timeout.count() + << " seconds)"; + while (!http_parser.hasReachedError()) { + // Check global timeout first + if (i_time->getMonotonicTime() > global_end_time) { + should_close_connection = true; + dbgWarning(D_CONNECTION) + << "Global receive timeout reached after " + << global_timeout.count() << " seconds"; + return genError(receiving_timeout); + } + + // Check per-chunk timeout if (i_time->getMonotonicTime() > receiving_end_time) { should_close_connection = true; - return genError(receving_timeout); - }; + dbgWarning(D_CONNECTION) << "No data received for " << base_timeout.count() << " seconds"; + return genError(receiving_timeout); + } + auto receieved = receiveData(); if (!receieved.ok()) { should_close_connection = true; return receieved.passErr(); } + // Reset timeout each time we receive data + if (!receieved.unpack().empty()) { + receiving_end_time = i_time->getMonotonicTime() + base_timeout; + } auto response = http_parser.parseData(*receieved, is_connect); + i_mainloop->yield(receieved.unpack().empty()); if (response.ok()) { dbgTrace(D_MESSAGING) << printOut(response.unpack().toString()); diff --git a/core/messaging/connection/connection_comp.cc b/core/messaging/connection/connection_comp.cc index f75ce55..ca3bd11 100644 --- a/core/messaging/connection/connection_comp.cc +++ b/core/messaging/connection/connection_comp.cc @@ -48,6 +48,13 @@ public: return establishNewConnection(metadata, category); } + void + clearConnections() override + { + dbgTrace(D_CONNECTION) << "Clearing all persistent connections"; + persistent_connections.clear(); + } + Maybe getPersistentConnection(const string &host_name, uint16_t port, MessageCategory category) override { diff --git a/core/messaging/connection/connection_ut/connection_comp_ut.cc b/core/messaging/connection/connection_ut/connection_comp_ut.cc index 869160b..1cc6c30 100644 --- a/core/messaging/connection/connection_ut/connection_comp_ut.cc +++ b/core/messaging/connection/connection_ut/connection_comp_ut.cc @@ -24,6 +24,7 @@ #include "rest.h" #include "rest_server.h" #include "dummy_socket.h" +#include using namespace std; using namespace testing; @@ -100,6 +101,11 @@ TEST_F(TestConnectionComp, testSetAndGetConnection) EXPECT_EQ(get_conn.getConnKey().getHostName(), "127.0.0.1"); EXPECT_EQ(get_conn.getConnKey().getPort(), 8080); EXPECT_EQ(get_conn.getConnKey().getCategory(), MessageCategory::LOG); + + i_conn->clearConnections(); + maybe_get_connection = i_conn->getPersistentConnection("127.0.0.1", 8080, MessageCategory::LOG); + ASSERT_FALSE(maybe_get_connection.ok()); + } TEST_F(TestConnectionComp, testEstablishNewConnection) @@ -279,19 +285,27 @@ TEST_F(TestConnectionComp, testSendRequestWithOneTimeFogConnection) auto req = HTTPRequest::prepareRequest(conn, HTTPMethod::POST, "/test", conn_metadata.getHeaders(), "test-body"); ASSERT_TRUE(req.ok()); + // Ensure we accept+respond exactly once regardless of yield overload order + std::atomic responded{false}; EXPECT_CALL(mock_mainloop, yield(A())) - .WillOnce( - InvokeWithoutArgs( - [&]() { - cerr << "accepting socket" << endl; - dummy_socket.acceptSocket(); - dummy_socket.writeToSocket("HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\nmy-test"); - } - ) - ).WillRepeatedly(Return()); + .WillRepeatedly(InvokeWithoutArgs([&]() { + if (!responded.exchange(true)) { + cerr << "accepting socket" << endl; + dummy_socket.acceptSocket(); + dummy_socket.writeToSocket("HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\nmy-test"); + } + })); + EXPECT_CALL(mock_mainloop, yield(A())) + .WillRepeatedly(InvokeWithoutArgs([&]() { + if (!responded.exchange(true)) { + cerr << "accepting socket while receiving" << endl; + dummy_socket.acceptSocket(); + dummy_socket.writeToSocket("HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\nmy-test"); + } + })); EXPECT_CALL(mock_timer, getMonotonicTime()) - .WillRepeatedly(Invoke([]() { static int j = 0; return chrono::microseconds(++j * 10); })); + .WillRepeatedly(Invoke([]() { static int j = 0; return chrono::microseconds(++j * 1000 * 1000); })); auto maybe_response = i_conn->sendRequest(conn, *req); if (!maybe_response.ok()) { diff --git a/core/messaging/include/interfaces/i_messaging_connection.h b/core/messaging/include/interfaces/i_messaging_connection.h index 5fa050a..9c7e057 100644 --- a/core/messaging/include/interfaces/i_messaging_connection.h +++ b/core/messaging/include/interfaces/i_messaging_connection.h @@ -27,6 +27,7 @@ class I_MessagingConnection { public: virtual Maybe establishConnection(const MessageMetadata &metadata, MessageCategory category) = 0; + virtual void clearConnections() = 0; virtual Maybe getPersistentConnection( const std::string &host_name, uint16_t port, MessageCategory category diff --git a/core/messaging/include/messaging_comp.h b/core/messaging/include/messaging_comp.h index 5d68834..80f71da 100644 --- a/core/messaging/include/messaging_comp.h +++ b/core/messaging/include/messaging_comp.h @@ -71,6 +71,7 @@ public: bool setFogConnection(const std::string &host, uint16_t port, bool is_secure, MessageCategory category); bool setFogConnection(MessageCategory category); + void clearConnections(); private: Maybe getConnection(MessageCategory category, const MessageMetadata &message_metadata); @@ -96,6 +97,7 @@ private: I_MessageBuffer *i_messaging_buffer; I_AgentDetails *agent_details; bool should_buffer_failed_messages; + std::string proxy_addr; TemporaryCache fog_get_requests_cache; }; diff --git a/core/messaging/include/mocks/mock_messaging_connection.h b/core/messaging/include/mocks/mock_messaging_connection.h index 5572cc1..95f3760 100644 --- a/core/messaging/include/mocks/mock_messaging_connection.h +++ b/core/messaging/include/mocks/mock_messaging_connection.h @@ -29,6 +29,7 @@ public: MOCK_METHOD3(mockSendRequest, Maybe(Connection &, HTTPRequest, bool)); + MOCK_METHOD0(clearConnections, void()); MOCK_METHOD3(getPersistentConnection, Maybe(const string &, uint16_t, MessageCategory)); MOCK_METHOD1(getFogConnectionByCategory, Maybe(MessageCategory)); }; diff --git a/core/messaging/messaging.cc b/core/messaging/messaging.cc index 0a8398b..f5bdc70 100644 --- a/core/messaging/messaging.cc +++ b/core/messaging/messaging.cc @@ -97,6 +97,12 @@ public: return messaging_comp.setFogConnection(category); } + void + clearConnections() override + { + messaging_comp.clearConnections(); + } + private: MessagingComp messaging_comp; ConnectionComponent connection_comp; @@ -119,7 +125,7 @@ void Messaging::preload() { registerExpectedConfiguration("message", "Cache timeout"); - registerExpectedConfiguration("message", "Connection timeout"); + registerExpectedConfigurationWithCache("assetId", "message", "Connection timeout"); registerExpectedConfiguration("message", "Connection handshake timeout"); registerExpectedConfiguration("message", "Verify SSL pinning"); registerExpectedConfiguration("message", "Buffer Failed Requests"); diff --git a/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc b/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc index 8b0e668..dca7625 100644 --- a/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc +++ b/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc @@ -204,12 +204,15 @@ MessagingBufferComponent::Impl::pushNewBufferedMessage( Maybe MessagingBufferComponent::Impl::peekMessage() { - auto move_cmd = - "if [ -s " + buffer_input + " ] && [ ! -s " + buffer_output + " ];" - "then mv " + buffer_input + " " + buffer_output + ";" - "fi"; - - shell_cmd->getExecOutput(move_cmd); + // Native replacement for shell mv command + struct stat stat_input, stat_output; + bool input_exists = (stat(buffer_input.c_str(), &stat_input) == 0 && stat_input.st_size > 0); + bool output_exists = (stat(buffer_output.c_str(), &stat_output) == 0 && stat_output.st_size > 0); + if (input_exists && !output_exists) { + if (rename(buffer_input.c_str(), buffer_output.c_str()) != 0) { + dbgWarning(D_MESSAGING_BUFFER) << "Failed to move buffer input to output: " << strerror(errno); + } + } if (!checkExistence(buffer_output)) return genError(buffer_output + " does not exist"); diff --git a/core/messaging/messaging_comp/messaging_comp.cc b/core/messaging/messaging_comp/messaging_comp.cc index fb80d06..d65bf52 100644 --- a/core/messaging/messaging_comp/messaging_comp.cc +++ b/core/messaging/messaging_comp/messaging_comp.cc @@ -72,6 +72,11 @@ MessagingComp::init() auto i_time_get = Singleton::Consume::by(); auto cache_timeout = getConfigurationWithDefault(40, "message", "Cache timeout"); fog_get_requests_cache.startExpiration(chrono::seconds(cache_timeout), i_mainloop, i_time_get); + proxy_addr = getConfigurationWithDefault( + getProfileAgentSettingWithDefault("", "proxy.address"), + "message", + "Proxy Address" + ); should_buffer_failed_messages = getConfigurationWithDefault( getProfileAgentSettingWithDefault(true, "eventBuffer.bufferFailedRequests"), @@ -125,7 +130,7 @@ MessagingComp::sendMessage( dbgWarning(D_MESSAGING) << "Failed to get connection. Error: " << maybe_conn.getErr(); return genError(HTTPStatusCode::HTTP_UNKNOWN, maybe_conn.getErr()); } - + Connection conn = maybe_conn.unpack(); if (message_metadata.shouldSuspend() && conn.isSuspended()) { return suspendMessage(body, method, uri, category, message_metadata); @@ -133,12 +138,11 @@ MessagingComp::sendMessage( bool is_to_fog = isMessageToFog(message_metadata); auto metadata = message_metadata; - + if (is_to_fog) { if (method == HTTPMethod::GET && fog_get_requests_cache.doesKeyExists(uri)) { HTTPResponse res = fog_get_requests_cache.getEntry(uri); dbgTrace(D_MESSAGING) << "Response returned from Fog cache. res body: " << res.getBody(); - return fog_get_requests_cache.getEntry(uri); } @@ -197,7 +201,6 @@ MessagingComp::sendSyncMessage( ) { Maybe is_msg_send = sendMessage(method, uri, body, category, message_metadata); - if (is_msg_send.ok()) return *is_msg_send; if (should_buffer_failed_messages && message_metadata.shouldBufferMessage()) { @@ -412,3 +415,10 @@ MessagingComp::suspendMessage( HTTPStatusCode::HTTP_SUSPEND, "The connection is suspended due to consecutive message sending errors." ); } + +void +MessagingComp::clearConnections() +{ + dbgTrace(D_MESSAGING) << "Clearing all connections (called from AgentDetails)"; + i_conn->clearConnections(); +} diff --git a/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc index dae991f..86a1bf2 100644 --- a/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc +++ b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc @@ -62,6 +62,7 @@ public: { EXPECT_CALL(mock_agent_details, getFogDomain()).WillRepeatedly(Return(string(fog_addr))); EXPECT_CALL(mock_agent_details, getFogPort()).WillRepeatedly(Return(fog_port)); + EXPECT_CALL(mock_agent_details, getProxy()).WillRepeatedly(Return(string(""))); EXPECT_CALL(mock_agent_details, getOpenSSLDir()).WillRepeatedly(Return(string("/usr/lib/ssl/certs/"))); EXPECT_CALL(mock_agent_details, getAccessToken()).WillRepeatedly(Return(string("accesstoken"))); EXPECT_CALL(mock_agent_details, readAgentDetails()).WillRepeatedly(Return(true)); @@ -262,6 +263,28 @@ operator==(const MessageMetadata &one, const MessageMetadata &two) one.isDualAuth() == two.isDualAuth(); } +TEST_F(TestMessagingComp, testClearConnections) +{ + setAgentDetails(); + + MessageCategory category = MessageCategory::GENERIC; + MessageConnectionKey conn_key(fog_addr, fog_port, category); + MessageMetadata metadata(fog_addr, fog_port, true); + MessageProxySettings proxy_settings("7.7.7.7", "cred", 8080); + metadata.setProxySettings(proxy_settings); + Connection conn(conn_key, metadata); + + EXPECT_CALL(mock_messaging_connection, establishConnection(metadata, category)).WillOnce(Return(conn)); + EXPECT_TRUE(messaging_comp.setFogConnection(category)); + + EXPECT_CALL(mock_messaging_connection, clearConnections()).Times(1); + messaging_comp.clearConnections(); + + EXPECT_CALL(mock_messaging_connection, establishConnection(metadata, category)).WillOnce(Return(conn)); + EXPECT_TRUE(messaging_comp.setFogConnection(category)); +} + + TEST_F(TestMessagingComp, testSetFogConnection) { setAgentDetails(); diff --git a/core/metric/generic_metric.cc b/core/metric/generic_metric.cc index 48caa01..46f80ef 100644 --- a/core/metric/generic_metric.cc +++ b/core/metric/generic_metric.cc @@ -33,16 +33,35 @@ MetricMetadata::Description operator"" _desc(const char *str, size_t) { return M static const set default_metrics = { "watchdogProcessStartupEventsSum", - "reservedNgenA", - "reservedNgenB", - "reservedNgenC" - "reservedNgenD" - "reservedNgenE", - "reservedNgenF", - "reservedNgenG" - "reservedNgenH", - "reservedNgenI", - "reservedNgenJ", + "reservedNgenA_WAAP telemetry", + "reservedNgenB_WAAP telemetry", + "reservedNgenC_WAAP telemetry", + "reservedNgenD_WAAP telemetry", + "reservedNgenE_WAAP telemetry", + "reservedNgenF_WAAP telemetry", + "reservedNgenG_WAAP telemetry", + "reservedNgenH_WAAP telemetry", + "reservedNgenI_WAAP telemetry", + "reservedNgenJ_WAAP telemetry", + "reservedNgenA_WAAP traffic telemetry", + "reservedNgenB_WAAP traffic telemetry", + "reservedNgenC_WAAP traffic telemetry", + "reservedNgenD_WAAP traffic telemetry", + "reservedNgenE_WAAP traffic telemetry", + "reservedNgenF_WAAP traffic telemetry", + "reservedNgenG_WAAP traffic telemetry", + "reservedNgenH_WAAP traffic telemetry", + "reservedNgenI_WAAP traffic telemetry", + "reservedNgenJ_WAAP traffic telemetry", + "reservedNgenA_WAAP attack type telemetry", + "reservedNgenB_WAAP attack type telemetry", + "reservedNgenC_WAAP attack type telemetry", + "reservedNgenD_WAAP attack type telemetry", + "reservedNgenE_WAAP attack type telemetry", + "reservedNgenF_WAAP attack type telemetry", + "reservedNgenG_WAAP attack type telemetry", + "reservedNgenH_WAAP attack type telemetry", + "reservedNgenI_WAAP attack type telemetry", "numberOfProtectedAssetsSample", "preventEngineMatchesSample", "detectEngineMatchesSample", @@ -115,6 +134,7 @@ MetricCalc::getPrometheusMetrics(const std::string &metric_name, const string &a PrometheusData res; res.name = getMetricDotName() != "" ? getMetricDotName() : getMetricName(); + res.unique_name = res.name + "_" + metric_name; res.type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter"; res.description = getMetircDescription(); diff --git a/core/metric/metric_ut/metric_ut.cc b/core/metric/metric_ut/metric_ut.cc index f32d73a..2abf270 100644 --- a/core/metric/metric_ut/metric_ut.cc +++ b/core/metric/metric_ut/metric_ut.cc @@ -575,6 +575,7 @@ TEST_F(MetricTest, getPromeathusMetric) " \"metrics\": [\n" " {\n" " \"metric_name\": \"cpuMax\",\n" + " \"unique_name\": \"cpuMax_CPU usage\",\n" " \"metric_type\": \"gauge\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -583,6 +584,7 @@ TEST_F(MetricTest, getPromeathusMetric) " },\n" " {\n" " \"metric_name\": \"cpuMin\",\n" + " \"unique_name\": \"cpuMin_CPU usage\",\n" " \"metric_type\": \"gauge\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -591,6 +593,7 @@ TEST_F(MetricTest, getPromeathusMetric) " },\n" " {\n" " \"metric_name\": \"cpuAvg\",\n" + " \"unique_name\": \"cpuAvg_CPU usage\",\n" " \"metric_type\": \"gauge\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -599,6 +602,7 @@ TEST_F(MetricTest, getPromeathusMetric) " },\n" " {\n" " \"metric_name\": \"cpuCurrent\",\n" + " \"unique_name\": \"cpuCurrent_CPU usage\",\n" " \"metric_type\": \"gauge\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -607,6 +611,7 @@ TEST_F(MetricTest, getPromeathusMetric) " },\n" " {\n" " \"metric_name\": \"cpuCounter\",\n" + " \"unique_name\": \"cpuCounter_CPU usage\",\n" " \"metric_type\": \"gauge\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -615,6 +620,7 @@ TEST_F(MetricTest, getPromeathusMetric) " },\n" " {\n" " \"metric_name\": \"cpuTotalCounter\",\n" + " \"unique_name\": \"cpuTotalCounter_CPU usage\",\n" " \"metric_type\": \"counter\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -665,6 +671,7 @@ TEST_F(MetricTest, getPromeathusMultiMap) " \"metrics\": [\n" " {\n" " \"metric_name\": \"request.total\",\n" + " \"unique_name\": \"GET_Bytes per URL\",\n" " \"metric_type\": \"counter\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -673,6 +680,7 @@ TEST_F(MetricTest, getPromeathusMultiMap) " },\n" " {\n" " \"metric_name\": \"request.total\",\n" + " \"unique_name\": \"POST_Bytes per URL\",\n" " \"metric_type\": \"counter\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -681,6 +689,7 @@ TEST_F(MetricTest, getPromeathusMultiMap) " },\n" " {\n" " \"metric_name\": \"request.total\",\n" + " \"unique_name\": \"GET_Bytes per URL\",\n" " \"metric_type\": \"counter\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -750,6 +759,7 @@ TEST_F(MetricTest, getPromeathusTwoMetrics) " \"metrics\": [\n" " {\n" " \"metric_name\": \"request.total\",\n" + " \"unique_name\": \"GET_Bytes per URL\",\n" " \"metric_type\": \"counter\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -758,6 +768,7 @@ TEST_F(MetricTest, getPromeathusTwoMetrics) " },\n" " {\n" " \"metric_name\": \"request.total\",\n" + " \"unique_name\": \"POST_Bytes per URL\",\n" " \"metric_type\": \"counter\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -766,6 +777,7 @@ TEST_F(MetricTest, getPromeathusTwoMetrics) " },\n" " {\n" " \"metric_name\": \"request.total\",\n" + " \"unique_name\": \"GET_Bytes per URL\",\n" " \"metric_type\": \"counter\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -774,6 +786,7 @@ TEST_F(MetricTest, getPromeathusTwoMetrics) " },\n" " {\n" " \"metric_name\": \"cpuMax\",\n" + " \"unique_name\": \"cpuMax_CPU usage\",\n" " \"metric_type\": \"gauge\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -782,6 +795,7 @@ TEST_F(MetricTest, getPromeathusTwoMetrics) " },\n" " {\n" " \"metric_name\": \"cpuMin\",\n" + " \"unique_name\": \"cpuMin_CPU usage\",\n" " \"metric_type\": \"gauge\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -790,6 +804,7 @@ TEST_F(MetricTest, getPromeathusTwoMetrics) " },\n" " {\n" " \"metric_name\": \"cpuAvg\",\n" + " \"unique_name\": \"cpuAvg_CPU usage\",\n" " \"metric_type\": \"gauge\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -798,6 +813,7 @@ TEST_F(MetricTest, getPromeathusTwoMetrics) " },\n" " {\n" " \"metric_name\": \"cpuCurrent\",\n" + " \"unique_name\": \"cpuCurrent_CPU usage\",\n" " \"metric_type\": \"gauge\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -806,6 +822,7 @@ TEST_F(MetricTest, getPromeathusTwoMetrics) " },\n" " {\n" " \"metric_name\": \"cpuCounter\",\n" + " \"unique_name\": \"cpuCounter_CPU usage\",\n" " \"metric_type\": \"gauge\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," @@ -814,6 +831,7 @@ TEST_F(MetricTest, getPromeathusTwoMetrics) " },\n" " {\n" " \"metric_name\": \"cpuTotalCounter\",\n" + " \"unique_name\": \"cpuTotalCounter_CPU usage\",\n" " \"metric_type\": \"counter\",\n" " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",assetId=\\\"asset id\\\",id=\\\"87\\\"," diff --git a/core/report/tag_and_enum_management.cc b/core/report/tag_and_enum_management.cc index 2a69307..793aa5b 100644 --- a/core/report/tag_and_enum_management.cc +++ b/core/report/tag_and_enum_management.cc @@ -110,13 +110,16 @@ TagAndEnumManagement::convertStringToTag(const string &tag) {"Horizon Telemetry Metrics", ReportIS::Tags::HORIZON_TELEMETRY_METRICS}, {"Crowdsec", ReportIS::Tags::CROWDSEC}, {"apiDiscoveryCloudMessaging", ReportIS::Tags::API_DISCOVERY}, + {"lbHealthStatusEngine", ReportIS::Tags::LB_HEALTH_STATUS}, {"Playground", ReportIS::Tags::PLAYGROUND}, {"Nginx Proxy Manager", ReportIS::Tags::NGINX_PROXY_MANAGER}, {"APISIX Server", ReportIS::Tags::WEB_SERVER_APISIX}, {"Docker Deployment", ReportIS::Tags::DEPLOYMENT_DOCKER}, {"SWAG Server", ReportIS::Tags::WEB_SERVER_SWAG}, {"NGINX Unified Server", ReportIS::Tags::WEB_SERVER_NGINX_UNIFIED}, - {"AI Guard", ReportIS::Tags::AIGUARD} + {"AI Guard", ReportIS::Tags::AIGUARD}, + {"Central NGINX Manager", ReportIS::Tags::CENTRAL_NGINX_MANAGER}, + {"Browser Agent", ReportIS::Tags::BROWSER_AGENT} }; auto report_is_tag = strings_to_tags.find(tag); @@ -280,6 +283,8 @@ TagAndEnumManagement::convertToString(const IssuingEngine &issuing_engine) case IssuingEngine::IDA_SAML_IDN_CLIENT_IP_NOTIFY: return "quantumIPNotifyIdn"; case IssuingEngine::API_DISCOVERY: return "apiDiscoveryCloudMessaging"; case IssuingEngine::HORIZON_TELEMETRY_METRICS: return "horizonTelemetryMetrics"; + case IssuingEngine::LB_HEALTH_STATUS: return "lbHealthStatusEngine"; + case IssuingEngine::BROWSER_AGENT: return "browserAgentEngine"; } dbgAssertOpt(false) << alert << "Reached impossible engine value of: " << static_cast(issuing_engine); @@ -323,12 +328,15 @@ EnumArray TagAndEnumManagement::tags_translation_arr { "Crowdsec", "Playground", "apiDiscoveryCloudMessaging", + "lbHealthStatusEngine", "Nginx Proxy Manager", "APISIX Server", "Docker Deployment", "SWAG Server", "NGINX Unified Server", - "AI Guard" + "AI Guard", + "Central NGINX Manager", + "Browser Agent" }; EnumArray TagAndEnumManagement::audience_team_translation { diff --git a/core/rest/i_rest_invoke.h b/core/rest/i_rest_invoke.h index 4ae053b..6a34e82 100644 --- a/core/rest/i_rest_invoke.h +++ b/core/rest/i_rest_invoke.h @@ -16,6 +16,7 @@ #include #include +#include #include "maybe_res.h" @@ -23,11 +24,20 @@ class I_RestInvoke { public: virtual Maybe getSchema(const std::string &uri) const = 0; - virtual Maybe invokeRest(const std::string &uri, std::istream &in) const = 0; + virtual Maybe invokeRest( + const std::string &uri, + std::istream &in, + const std::map &headers + ) const = 0; virtual bool isGetCall(const std::string &uri) const = 0; virtual std::string invokeGet(const std::string &uri) const = 0; + virtual bool isPostCall(const std::string &uri) const = 0; + virtual Maybe invokePost(const std::string &uri, const std::string &body) const = 0; + + virtual bool shouldCaptureHeaders(const std::string &uri) const = 0; + protected: ~I_RestInvoke() {} }; diff --git a/core/rest/rest.cc b/core/rest/rest.cc index 3dcbfe8..69f4cd8 100644 --- a/core/rest/rest.cc +++ b/core/rest/rest.cc @@ -29,8 +29,11 @@ RestHelper::reportError(std::string const &err) } Maybe -ServerRest::performRestCall(istream &in) +ServerRest::performRestCall(istream &in, const map &headers) { + if (wantsHeaders()) { + request_headers = headers; + } try { try { int firstChar = in.peek(); diff --git a/core/rest/rest_conn.cc b/core/rest/rest_conn.cc index 5e2d843..59965fb 100644 --- a/core/rest/rest_conn.cc +++ b/core/rest/rest_conn.cc @@ -75,17 +75,52 @@ RestConn::parseConn() const dbgDebug(D_API) << "Call identifier: " << identifier; uint len = 0; + map headers; + bool should_capture_headers = invoke->shouldCaptureHeaders(identifier); + while (true) { line = readLine(); if (line.size() < 3) break; - os.str(line); - string head, data; - os >> head >> data; - if (compareStringCaseInsensitive(head, "Content-Length:")) { - try { - len = stoi(data, nullptr); - } catch (...) { + if (should_capture_headers) { + size_t colon_pos = line.find(':'); + if (colon_pos == string::npos) continue; + + string head = line.substr(0, colon_pos); + string data = line.substr(colon_pos + 1); + + size_t data_start = data.find_first_not_of(" \t\r\n"); + if (data_start != string::npos) { + data = data.substr(data_start); + } else { + data = ""; + } + + size_t data_end = data.find_last_not_of(" \t\r\n"); + if (data_end != string::npos) { + data = data.substr(0, data_end + 1); + } + + if (!head.empty()) { + headers[head] = data; + dbgTrace(D_API) << "Captured header: " << head << " = " << data; + } + + if (compareStringCaseInsensitive(head, "Content-Length")) { + try { + len = stoi(data, nullptr); + } catch (...) { + } + } + } else { + os.str(line); + string head, data; + os >> head >> data; + if (compareStringCaseInsensitive(head, "Content-Length:")) { + try { + len = stoi(data, nullptr); + } catch (...) { + } } } } @@ -113,7 +148,19 @@ RestConn::parseConn() const dbgTrace(D_API) << "Message content: " << body.str(); - Maybe res = (method == "POST") ? invoke->invokeRest(identifier, body) : invoke->getSchema(identifier); + if (method == "POST" && invoke->isPostCall(identifier)) { + Maybe result = invoke->invokePost(identifier, body.str()); + if (!result.ok()) { + dbgWarning(D_API) << "Failed to invoke POST call: " << result.getErr(); + sendResponse("500 Internal Server Error", result.getErr()); + return; + } + sendResponse("200 OK", result.unpack()); + return; + } + + Maybe res = (method == "POST") ? + invoke->invokeRest(identifier, body, headers) : invoke->getSchema(identifier); if (res.ok()) { sendResponse("200 OK", res.unpack()); diff --git a/core/rest/rest_conn.h b/core/rest/rest_conn.h index c7489ef..aa1af70 100644 --- a/core/rest/rest_conn.h +++ b/core/rest/rest_conn.h @@ -15,6 +15,7 @@ #define __REST_CONN_H__ #include +#include #include "i_mainloop.h" #include "i_rest_invoke.h" diff --git a/core/rest/rest_server.cc b/core/rest/rest_server.cc index a92a4fa..f10627b 100644 --- a/core/rest/rest_server.cc +++ b/core/rest/rest_server.cc @@ -53,11 +53,16 @@ public: bool addRestCall(RestAction oper, const string &uri, unique_ptr &&init) override; bool addGetCall(const string &uri, const function &cb) override; bool addWildcardGetCall(const string &uri, const function &callback); + bool addPostCall(const string &uri, const function(const string &)> &callback) override; uint16_t getListeningPort() const override { return listening_port; } + uint16_t getStartingPortRange() const override { return starting_port_range; } Maybe getSchema(const string &uri) const override; - Maybe invokeRest(const string &uri, istream &in) const override; + Maybe invokeRest(const string &uri, istream &in, const map &headers) const override; bool isGetCall(const string &uri) const override; string invokeGet(const string &uri) const override; + bool isPostCall(const string &uri) const override; + Maybe invokePost(const string &uri, const string &body) const override; + bool shouldCaptureHeaders(const string &uri) const override; private: void prepareConfiguration(); @@ -71,7 +76,9 @@ private: map> rest_calls; map> get_calls; map> wildcard_get_calls; + map(const string &)>> post_calls; uint16_t listening_port = 0; + uint16_t starting_port_range = 0; vector port_range; }; @@ -241,6 +248,9 @@ RestServer::Impl::prepareConfiguration() range_start = 0; range_end = 1; } + starting_port_range = range_start.unpack(); + dbgInfo(D_API) << "Rest port range start: " << *range_start << ", end: " << *range_end; + // starting_port_range = *range_start; port_range.resize(*range_end - *range_start); for (uint16_t i = 0, port = *range_start; i < port_range.size(); i++, port++) { port_range[i] = port; @@ -379,6 +389,14 @@ RestServer::Impl::addWildcardGetCall(const string &uri, const function(const string&)> &callback) +{ + if (rest_calls.find(uri) != rest_calls.end()) return false; + if (get_calls.find(uri) != get_calls.end()) return false; + return post_calls.emplace(uri, callback).second; +} + Maybe RestServer::Impl::getSchema(const string &uri) const { @@ -392,12 +410,12 @@ RestServer::Impl::getSchema(const string &uri) const } Maybe -RestServer::Impl::invokeRest(const string &uri, istream &in) const +RestServer::Impl::invokeRest(const string &uri, istream &in, const map &headers) const { auto iter = rest_calls.find(uri); if (iter == rest_calls.end()) return genError("No matching REST call was found"); auto instance = iter->second->getRest(); - return instance->performRestCall(in); + return instance->performRestCall(in, headers); } bool @@ -412,6 +430,13 @@ RestServer::Impl::isGetCall(const string &uri) const return false; } +bool +RestServer::Impl::isPostCall(const string &uri) const +{ + if (post_calls.find(uri) != post_calls.end()) return true; + return false; +} + string RestServer::Impl::invokeGet(const string &uri) const { @@ -425,6 +450,23 @@ RestServer::Impl::invokeGet(const string &uri) const return ""; } +Maybe +RestServer::Impl::invokePost(const string &uri, const string &body) const +{ + auto instance = post_calls.find(uri); + if (instance != post_calls.end()) return instance->second(body); + return genError("No matching POST call was found for URI: " + uri); +} + +bool +RestServer::Impl::shouldCaptureHeaders(const string &uri) const +{ + auto iter = rest_calls.find(uri); + if (iter == rest_calls.end()) return false; + auto instance = iter->second->getRest(); + return instance->wantsHeaders(); +} + string RestServer::Impl::changeActionToString(RestAction oper) { diff --git a/core/rest/rest_ut/rest_config_ut.cc b/core/rest/rest_ut/rest_config_ut.cc index 1145a6e..7e2cd9f 100644 --- a/core/rest/rest_ut/rest_config_ut.cc +++ b/core/rest/rest_ut/rest_config_ut.cc @@ -414,3 +414,222 @@ TEST_F(RestConfigTest, not_loopback_flow) "HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\nContent-Length: 0\r\n\r\n" ); } + +TEST_F(RestConfigTest, getStartingPortRange) +{ + // Use a configuration with port range instead of primary/alternative ports + string config_json_with_range = + "{\n" + " \"connection\": {\n" + " \"Nano service API Port Range start\": [\n" + " {\n" + " \"value\": 8000\n" + " }\n" + " ],\n" + " \"Nano service API Port Range end\": [\n" + " {\n" + " \"value\": 8010\n" + " }\n" + " ]\n" + " }\n" + "}\n"; + + istringstream ss(config_json_with_range); + Singleton::Consume::from(config)->loadConfiguration(ss); + + rest_server.init(); + + auto i_rest = Singleton::Consume::from(rest_server); + EXPECT_EQ(i_rest->getStartingPortRange(), 8000); + + auto mainloop = Singleton::Consume::from(mainloop_comp); + I_MainLoop::Routine stop_routine = [mainloop] () { mainloop->stopAll(); }; + mainloop->addOneTimeRoutine( + I_MainLoop::RoutineType::RealTime, + stop_routine, + "RestConfigTest-getStartingPortRange stop routine", + false + ); + mainloop->run(); +} + +TEST_F(RestConfigTest, addPostCall) +{ + rest_server.init(); + time_proxy.init(); + mainloop_comp.init(); + + auto i_rest = Singleton::Consume::from(rest_server); + + // Test addPostCall + ASSERT_TRUE(i_rest->addPostCall("test-post", [](const string &body) { + return "Received: " + body; + })); + + // Test that adding the same POST call twice fails + ASSERT_FALSE(i_rest->addPostCall("test-post", [](const string &) -> Maybe { + return string("Different handler"); + })); + + // Test that adding POST call with existing GET call fails + ASSERT_TRUE(i_rest->addGetCall("test-get", []() { return "get response"; })); + ASSERT_FALSE(i_rest->addPostCall("test-get", [](const string &) -> Maybe { + return string("post response"); + })); + + auto mainloop = Singleton::Consume::from(mainloop_comp); + I_MainLoop::Routine stop_routine = [mainloop] () { mainloop->stopAll(); }; + mainloop->addOneTimeRoutine( + I_MainLoop::RoutineType::RealTime, + stop_routine, + "RestConfigTest-addPostCall stop routine", + false + ); + mainloop->run(); +} + +TEST_F(RestConfigTest, post_call_integration_test) +{ + env.preload(); + Singleton::Consume::from(env)->registerValue("Base Executable Name", "tmp_test_file"); + + config.preload(); + config.init(); + + rest_server.init(); + time_proxy.init(); + mainloop_comp.init(); + + auto i_rest = Singleton::Consume::from(rest_server); + + // Add a POST endpoint that echoes back the request body with prefix + ASSERT_TRUE(i_rest->addPostCall("echo", [](const string &body) -> Maybe { + return string("Echo: ") + body; + })); + + int file_descriptor = socket(AF_INET, SOCK_STREAM, 0); + EXPECT_NE(file_descriptor, -1); + + auto primary_port = getConfiguration("connection", "Nano service API Port Alternative"); + struct sockaddr_in sa; + sa.sin_family = AF_INET; + sa.sin_port = htons(primary_port.unpack()); + sa.sin_addr.s_addr = inet_addr("127.0.0.1"); + int socket_enable = 1; + EXPECT_EQ(setsockopt(file_descriptor, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0); + + EXPECT_CALL(messaging, sendSyncMessage(_, _, _, _, _)) + .WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); + + auto mainloop = Singleton::Consume::from(mainloop_comp); + I_MainLoop::Routine stop_routine = [&] () { + EXPECT_EQ(connect(file_descriptor, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0) + << "file_descriptor Error: " << strerror(errno); + + string test_body = "Hello World"; + string msg = "POST /echo HTTP/1.1\r\nContent-Length: " + + to_string(test_body.length()) + + "\r\n\r\n" + test_body; + EXPECT_EQ(write(file_descriptor, msg.data(), msg.size()), static_cast(msg.size())); + + struct pollfd s_poll; + s_poll.fd = file_descriptor; + s_poll.events = POLLIN; + s_poll.revents = 0; + while(poll(&s_poll, 1, 0) <= 0) { + mainloop->yield(true); + } + + mainloop->stopAll(); + }; + mainloop->addOneTimeRoutine( + I_MainLoop::RoutineType::RealTime, + stop_routine, + "RestConfigTest-post_call_integration_test stop routine", + true + ); + mainloop->run(); + + char response[1000]; + int bytes_read = read(file_descriptor, response, 1000); + EXPECT_GT(bytes_read, 0); + + string response_str(response, bytes_read); + EXPECT_THAT(response_str, HasSubstr("HTTP/1.1 200 OK")); + EXPECT_THAT(response_str, HasSubstr("Echo: Hello World")); + + close(file_descriptor); +} + +TEST_F(RestConfigTest, post_call_generic_error_test) +{ + env.preload(); + Singleton::Consume::from(env)->registerValue("Base Executable Name", "tmp_test_file"); + + config.preload(); + config.init(); + + rest_server.init(); + time_proxy.init(); + mainloop_comp.init(); + + auto i_rest = Singleton::Consume::from(rest_server); + + // Add a POST endpoint that returns a generic error + ASSERT_TRUE(i_rest->addPostCall("error-test", [](const string &) -> Maybe { + return genError("Test error message"); + })); + + int file_descriptor = socket(AF_INET, SOCK_STREAM, 0); + EXPECT_NE(file_descriptor, -1); + + auto primary_port = getConfiguration("connection", "Nano service API Port Alternative"); + struct sockaddr_in sa; + sa.sin_family = AF_INET; + sa.sin_port = htons(primary_port.unpack()); + sa.sin_addr.s_addr = inet_addr("127.0.0.1"); + int socket_enable = 1; + EXPECT_EQ(setsockopt(file_descriptor, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0); + + EXPECT_CALL(messaging, sendSyncMessage(_, _, _, _, _)) + .WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); + + auto mainloop = Singleton::Consume::from(mainloop_comp); + I_MainLoop::Routine stop_routine = [&] () { + EXPECT_EQ(connect(file_descriptor, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0) + << "file_descriptor Error: " << strerror(errno); + + string test_body = "Test request body"; + string msg = "POST /error-test HTTP/1.1\r\nContent-Length: " + + to_string(test_body.length()) + + "\r\n\r\n" + test_body; + EXPECT_EQ(write(file_descriptor, msg.data(), msg.size()), static_cast(msg.size())); + + struct pollfd s_poll; + s_poll.fd = file_descriptor; + s_poll.events = POLLIN; + s_poll.revents = 0; + while(poll(&s_poll, 1, 0) <= 0) { + mainloop->yield(true); + } + + mainloop->stopAll(); + }; + mainloop->addOneTimeRoutine( + I_MainLoop::RoutineType::RealTime, + stop_routine, + "RestConfigTest-post_call_generic_error_test stop routine", + true + ); + mainloop->run(); + + char response[1000]; + int bytes_read = read(file_descriptor, response, 1000); + EXPECT_GT(bytes_read, 0); + + string response_str(response, bytes_read); + EXPECT_THAT(response_str, HasSubstr("HTTP/1.1 500 Internal Server Error")); + EXPECT_THAT(response_str, HasSubstr("Test error message")); + + close(file_descriptor); +} diff --git a/core/rest/rest_ut/rest_schema_ut.cc b/core/rest/rest_ut/rest_schema_ut.cc index 1d63cc6..fccc371 100644 --- a/core/rest/rest_ut/rest_schema_ut.cc +++ b/core/rest/rest_ut/rest_schema_ut.cc @@ -444,6 +444,7 @@ TEST(RestSchema, server_schema) EXPECT_CALL(mock_agent_details, getAccessToken()).WillRepeatedly(testing::Return(string("accesstoken"))); EXPECT_CALL(mock_agent_details, getFogDomain()).WillRepeatedly(testing::Return(string("127.0.0.1"))); EXPECT_CALL(mock_agent_details, getFogPort()).WillRepeatedly(testing::Return(9777)); + EXPECT_CALL(mock_agent_details, getProxy()).WillRepeatedly(testing::Return(string(""))); string config_json = "{" diff --git a/core/shmem_ipc/shmem_ipc.c b/core/shmem_ipc/shmem_ipc.c index d004da6..c303990 100755 --- a/core/shmem_ipc/shmem_ipc.c +++ b/core/shmem_ipc/shmem_ipc.c @@ -199,6 +199,11 @@ void resetIpc(SharedMemoryIPC *ipc, uint16_t num_of_data_segments) { writeDebug(TraceLevel, "Reseting IPC queues\n"); + if (!ipc || !ipc->rx_queue || !ipc->tx_queue) { + writeDebug(WarningLevel, "resetIpc called with NULL ipc pointer\n"); + return; + } + resetRingQueue(ipc->rx_queue, num_of_data_segments); resetRingQueue(ipc->tx_queue, num_of_data_segments); } @@ -208,6 +213,11 @@ destroyIpc(SharedMemoryIPC *shmem, int is_owner) { writeDebug(TraceLevel, "Destroying IPC queues\n"); + if (!shmem) { + writeDebug(WarningLevel, "Destroying IPC queues called with NULL shmem pointer\n"); + return; + } + if (shmem->rx_queue != NULL) { destroySharedRingQueue(shmem->rx_queue, is_owner, isTowardsOwner(is_owner, 0)); shmem->rx_queue = NULL; @@ -225,6 +235,10 @@ dumpIpcMemory(SharedMemoryIPC *ipc) { writeDebug(WarningLevel, "Ipc memory dump:\n"); writeDebug(WarningLevel, "RX queue:\n"); + if (!ipc || !ipc->rx_queue) { + writeDebug(WarningLevel, "RX queue is NULL\n"); + return; + } dumpRingQueueShmem(ipc->rx_queue); writeDebug(WarningLevel, "TX queue:\n"); dumpRingQueueShmem(ipc->tx_queue); @@ -234,6 +248,10 @@ int sendData(SharedMemoryIPC *ipc, const uint16_t data_to_send_size, const char *data_to_send) { writeDebug(TraceLevel, "Sending data of size %u\n", data_to_send_size); + if (!ipc || !ipc->tx_queue) { + writeDebug(WarningLevel, "sendData called with NULL ipc pointer\n"); + return -1; + } return pushToQueue(ipc->tx_queue, data_to_send, data_to_send_size); } @@ -247,12 +265,22 @@ sendChunkedData( { writeDebug(TraceLevel, "Sending %u chunks of data\n", num_of_data_elem); + if (!ipc) { + writeDebug(WarningLevel, "sendChunkedData called with NULL ipc pointer\n"); + return -1; + } + return pushBuffersToQueue(ipc->tx_queue, data_elem_to_send, data_to_send_sizes, num_of_data_elem); } int receiveData(SharedMemoryIPC *ipc, uint16_t *received_data_size, const char **received_data) { + if (!ipc) { + writeDebug(WarningLevel, "receiveData called with NULL ipc pointer\n"); + return -1; + } + int res = peekToQueue(ipc->rx_queue, received_data, received_data_size); writeDebug(TraceLevel, "Received data from queue. Res: %d, data size: %u\n", res, *received_data_size); return res; @@ -261,6 +289,10 @@ receiveData(SharedMemoryIPC *ipc, uint16_t *received_data_size, const char **rec int popData(SharedMemoryIPC *ipc) { + if (!ipc) { + writeDebug(WarningLevel, "popData called with NULL ipc pointer\n"); + return -1; + } int res = popFromQueue(ipc->rx_queue); writeDebug(TraceLevel, "Popped data from queue. Res: %d\n", res); return res; @@ -269,6 +301,10 @@ popData(SharedMemoryIPC *ipc) int isDataAvailable(SharedMemoryIPC *ipc) { + if (!ipc) { + writeDebug(WarningLevel, "isDataAvailable called with NULL ipc pointer\n"); + return 0; + } int res = !isQueueEmpty(ipc->rx_queue); writeDebug(TraceLevel, "Checking if there is data pending to be read. Res: %d\n", res); return res; @@ -277,6 +313,11 @@ isDataAvailable(SharedMemoryIPC *ipc) int isCorruptedShmem(SharedMemoryIPC *ipc, int is_owner) { + if (!ipc) { + writeDebug(WarningLevel, "isCorruptedShmem called with NULL ipc pointer\n"); + return 1; + } + if (isCorruptedQueue(ipc->rx_queue, isTowardsOwner(is_owner, 0)) || isCorruptedQueue(ipc->tx_queue, isTowardsOwner(is_owner, 1)) ) { diff --git a/core/socket_is/socket_is.cc b/core/socket_is/socket_is.cc index 415cc42..be86f8e 100644 --- a/core/socket_is/socket_is.cc +++ b/core/socket_is/socket_is.cc @@ -54,6 +54,7 @@ public: is_server_socket = from.is_server_socket; socket_int = from.socket_int; from.socket_int = -1; + i_mainloop = Singleton::Consume::by(); } virtual ~SocketInternal() @@ -112,6 +113,115 @@ public: return true; } + bool + writeDataAsync(const vector &data) + { + uint32_t bytes_sent = 0; + bool is_first_iter = true; + uint32_t max_retries = 10; + uint32_t retry_count = 0; + + while (bytes_sent < data.size() && retry_count < max_retries) { + if (!is_first_iter && !is_blocking) { + dbgTrace(D_SOCKET) + << "Trying to yield before writing to socket again. Bytes written: " + << bytes_sent + << ", Total bytes: " + << data.size(); + + if (!i_mainloop) { + i_mainloop = Singleton::Consume::by(); + } + i_mainloop->yield(false); + } + is_first_iter = false; + + int res = send(socket_int, data.data() + bytes_sent, data.size() - bytes_sent, MSG_NOSIGNAL); + if (res <= 0) { + int err = errno; + + // Check if it's a temporary error that can be retried + if (res == -1 && (err == EAGAIN || err == EWOULDBLOCK)) { + dbgTrace(D_SOCKET) + << "Send would block (EAGAIN/EWOULDBLOCK), waiting for socket to become writable. " + << "Bytes sent so far: " + << bytes_sent; + + // Use poll to wait for socket to become writable with 10ms timeout + struct pollfd pfd; + pfd.fd = socket_int; + pfd.events = POLLOUT; + pfd.revents = 0; + + int poll_result = poll(&pfd, 1, 10); + + if (poll_result > 0 && (pfd.revents & POLLOUT)) { + dbgTrace(D_SOCKET) << "Socket became writable, retrying send"; + retry_count++; + continue; + } else if (poll_result == 0) { + dbgWarning(D_SOCKET) + << "Timeout waiting for socket to become writable after 100ms. " + << "Bytes sent: " << bytes_sent << "/" << data.size(); + retry_count++; + continue; + } else { + dbgWarning(D_SOCKET) + << "Poll failed while waiting for writable socket. Error: " + << strerror(errno); + return false; + } + } + + if ( + res == 0 + || err == EPIPE + || err == ECONNRESET + || err == ENOTCONN + || err == ESHUTDOWN + || err == EBADF + || err == EINVAL + ) { + dbgWarning(D_SOCKET) + << "Fatal error sending data. Error: " + << strerror(err) + << ", bytes sent: " + << bytes_sent + << "/" + << data.size(); + return false; + } + + if (err == EINTR) { + dbgTrace(D_SOCKET) << "Send interrupted (EINTR), retrying immediately"; + retry_count++; + continue; + } + + dbgWarning(D_SOCKET) + << "Unexpected error sending data. Error: " + << strerror(err) + << ", errno: " + << err + << ", bytes sent: " + << bytes_sent + << "/" + << data.size(); + return false; + } + + bytes_sent += res; + retry_count = 0; + } + + if (retry_count >= max_retries) { + dbgWarning(D_SOCKET) << "Reached max retries (" << max_retries << ") for socket write"; + return false; + } + + return true; + } + bool isDataAvailable() { @@ -223,6 +333,7 @@ protected: bool is_blocking = false; bool is_server_socket = true; int socket_int = -1; + I_MainLoop *i_mainloop = nullptr; private: Maybe @@ -718,6 +829,7 @@ public: void closeSocket(socketFd &socket_fd) override; bool writeData(socketFd socket_fd, const vector &data) override; + bool writeDataAsync(socketFd socket_fd, const vector &data) override; Maybe> receiveData(socketFd socket_fd, uint data_size, bool is_blocking = true) override; bool isDataAvailable(socketFd socket) override; @@ -820,6 +932,18 @@ SocketIS::Impl::writeData(socketFd socket_fd, const vector &data) return sock->second->writeData(data); } +bool +SocketIS::Impl::writeDataAsync(socketFd socket_fd, const vector &data) +{ + auto sock = active_sockets.find(socket_fd); + if (sock == active_sockets.end()) { + dbgWarning(D_SOCKET) << "The provided socket file descriptor does not exist. Socket FD: " << socket_fd; + return false; + } + + return sock->second->writeDataAsync(data); +} + Maybe> SocketIS::Impl::receiveData(socketFd socket_fd, uint data_size, bool is_blocking) { diff --git a/nodes/attachment_registration_manager/CMakeLists.txt b/nodes/attachment_registration_manager/CMakeLists.txt index 02db893..d93a6d8 100755 --- a/nodes/attachment_registration_manager/CMakeLists.txt +++ b/nodes/attachment_registration_manager/CMakeLists.txt @@ -14,6 +14,7 @@ target_link_libraries(attachment_registration_manager attachment_registrator http_transaction_data + ${Brotli_LIBRARIES} -Wl,--end-group ) diff --git a/nodes/http_transaction_handler/CMakeLists.txt b/nodes/http_transaction_handler/CMakeLists.txt index f8f8f88..d5b44ea 100755 --- a/nodes/http_transaction_handler/CMakeLists.txt +++ b/nodes/http_transaction_handler/CMakeLists.txt @@ -41,6 +41,7 @@ target_link_libraries(cp-nano-http-transaction-handler l7_access_control geo_location http_geo_filter + ${Brotli_LIBRARIES} -Wl,--end-group ) @@ -79,6 +80,12 @@ execute_process ( ) install(FILES ${maxminddb} DESTINATION http_transaction_handler_service/lib) +execute_process ( + COMMAND sh -c "find /usr/lib* -name \"libbrotli*.so*\" | awk '{printf \$0\";\"}'" + OUTPUT_VARIABLE brotli +) +install(FILES ${brotli} DESTINATION http_transaction_handler_service/lib) + gen_help( "--certs-dir " "Path to the hosts trusted ca directory" ) diff --git a/nodes/orchestration/CMakeLists.txt b/nodes/orchestration/CMakeLists.txt index 229208b..92008cc 100755 --- a/nodes/orchestration/CMakeLists.txt +++ b/nodes/orchestration/CMakeLists.txt @@ -32,6 +32,7 @@ target_link_libraries( curl external_sdk_server service_health_status + ${Brotli_LIBRARIES} -Wl,--end-group ) @@ -41,6 +42,7 @@ install(TARGETS orchestration_comp DESTINATION bin) install(TARGETS orchestration_comp DESTINATION orchestration/bin) install(FILES package/certificate/ngen.body.crt DESTINATION orchestration/certificate/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) +install(FILES package/certificate/inext-ca-bundle.crt DESTINATION orchestration/certificate/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) install(FILES package/certificate/public-keys/cloud-ngen.pem DESTINATION orchestration/certificate/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) install(FILES package/certificate/public-keys/dev-i2.pem DESTINATION orchestration/certificate/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) install(FILES package/certificate/public-keys/i2.pem DESTINATION orchestration/certificate/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) diff --git a/nodes/orchestration/package/CMakeLists.txt b/nodes/orchestration/package/CMakeLists.txt index f1558dd..0ac350c 100755 --- a/nodes/orchestration/package/CMakeLists.txt +++ b/nodes/orchestration/package/CMakeLists.txt @@ -8,6 +8,7 @@ install(FILES orchestration_package.sh DESTINATION ./orchestration/ PERMISSIONS install(FILES cp-agent-info.sh DESTINATION ./orchestration/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) install(FILES k8s-check-update-listener.sh DESTINATION ./orchestration/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) install(FILES k8s-check-update-trigger.sh DESTINATION ./orchestration/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) +install(FILES local-default-policy-v1beta2.yaml DESTINATION ./orchestration/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) install(FILES get-cloud-metadata.sh DESTINATION ./orchestration/scripts/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) install(FILES cp-agent-uninstall.sh DESTINATION ./orchestration/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) @@ -28,7 +29,6 @@ install(FILES watchdog/wait-for-networking-inspection-modules.sh DESTINATION ./o install(FILES watchdog/access_pre_init DESTINATION ./orchestration/watchdog/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) install(FILES watchdog/revert_orchestrator_version.sh DESTINATION ./orchestration/watchdog/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) -install(FILES local-default-policy.yaml DESTINATION ./orchestration/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) install(FILES open-appsec-cloud-mgmt DESTINATION ./orchestration/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) install(FILES open-appsec-cloud-mgmt-k8s DESTINATION ./orchestration/ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ) diff --git a/nodes/orchestration/package/certificate/inext-ca-bundle.crt b/nodes/orchestration/package/certificate/inext-ca-bundle.crt new file mode 100755 index 0000000..2a0d3b9 --- /dev/null +++ b/nodes/orchestration/package/certificate/inext-ca-bundle.crt @@ -0,0 +1,927 @@ +GlobalSign Root CA +================== +-----BEGIN CERTIFICATE----- +MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkGA1UEBhMCQkUx +GTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jvb3QgQ0ExGzAZBgNVBAMTEkds +b2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAwMDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNV +BAYTAkJFMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYD +VQQDExJHbG9iYWxTaWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDa +DuaZjc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavpxy0Sy6sc +THAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp1Wrjsok6Vjk4bwY8iGlb +Kk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdGsnUOhugZitVtbNV4FpWi6cgKOOvyJBNP +c1STE4U6G7weNLWLBYy5d4ux2x8gkasJU26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrX +gzT/LCrBbBlDSgeF59N89iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0BAQUF +AAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOzyj1hTdNGCbM+w6Dj +Y1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE38NflNUVyRRBnMRddWQVDf9VMOyG +j/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymPAbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhH +hm4qxFYxldBniYUr+WymXUadDKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveC +X4XSQRjbgbMEHMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A== +-----END CERTIFICATE----- + +Entrust.net Premium 2048 Secure Server CA +========================================= +-----BEGIN CERTIFICATE----- +MIIEKjCCAxKgAwIBAgIEOGPe+DANBgkqhkiG9w0BAQUFADCBtDEUMBIGA1UEChMLRW50cnVzdC5u +ZXQxQDA+BgNVBAsUN3d3dy5lbnRydXN0Lm5ldC9DUFNfMjA0OCBpbmNvcnAuIGJ5IHJlZi4gKGxp +bWl0cyBsaWFiLikxJTAjBgNVBAsTHChjKSAxOTk5IEVudHJ1c3QubmV0IExpbWl0ZWQxMzAxBgNV +BAMTKkVudHJ1c3QubmV0IENlcnRpZmljYXRpb24gQXV0aG9yaXR5ICgyMDQ4KTAeFw05OTEyMjQx +NzUwNTFaFw0yOTA3MjQxNDE1MTJaMIG0MRQwEgYDVQQKEwtFbnRydXN0Lm5ldDFAMD4GA1UECxQ3 +d3d3LmVudHJ1c3QubmV0L0NQU18yMDQ4IGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxpYWIuKTEl +MCMGA1UECxMcKGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDEzMDEGA1UEAxMqRW50cnVzdC5u +ZXQgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgKDIwNDgpMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEArU1LqRKGsuqjIAcVFmQqK0vRvwtKTY7tgHalZ7d4QMBzQshowNtTK91euHaYNZOL +Gp18EzoOH1u3Hs/lJBQesYGpjX24zGtLA/ECDNyrpUAkAH90lKGdCCmziAv1h3edVc3kw37XamSr +hRSGlVuXMlBvPci6Zgzj/L24ScF2iUkZ/cCovYmjZy/Gn7xxGWC4LeksyZB2ZnuU4q941mVTXTzW +nLLPKQP5L6RQstRIzgUyVYr9smRMDuSYB3Xbf9+5CFVghTAp+XtIpGmG4zU/HoZdenoVve8AjhUi +VBcAkCaTvA5JaJG/+EfTnZVCwQ5N328mz8MYIWJmQ3DW1cAH4QIDAQABo0IwQDAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUVeSB0RGAvtiJuQijMfmhJAkWuXAwDQYJ +KoZIhvcNAQEFBQADggEBADubj1abMOdTmXx6eadNl9cZlZD7Bh/KM3xGY4+WZiT6QBshJ8rmcnPy +T/4xmf3IDExoU8aAghOY+rat2l098c5u9hURlIIM7j+VrxGrD9cv3h8Dj1csHsm7mhpElesYT6Yf +zX1XEC+bBAlahLVu2B064dae0Wx5XnkcFMXj0EyTO2U87d89vqbllRrDtRnDvV5bu/8j72gZyxKT +J1wDLW8w0B62GqzeWvfRqqgnpv55gcR5mTNXuhKwqeBCbJPKVt7+bYQLCIt+jerXmCHG8+c8eS9e +nNFMFY3h7CI3zJpDC5fcgJCNs2ebb0gIFVbPv/ErfF6adulZkMV8gzURZVE= +-----END CERTIFICATE----- + +Entrust Root Certification Authority +==================================== +-----BEGIN CERTIFICATE----- +MIIEkTCCA3mgAwIBAgIERWtQVDANBgkqhkiG9w0BAQUFADCBsDELMAkGA1UEBhMCVVMxFjAUBgNV +BAoTDUVudHJ1c3QsIEluYy4xOTA3BgNVBAsTMHd3dy5lbnRydXN0Lm5ldC9DUFMgaXMgaW5jb3Jw +b3JhdGVkIGJ5IHJlZmVyZW5jZTEfMB0GA1UECxMWKGMpIDIwMDYgRW50cnVzdCwgSW5jLjEtMCsG +A1UEAxMkRW50cnVzdCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA2MTEyNzIwMjM0 +MloXDTI2MTEyNzIwNTM0MlowgbAxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMu +MTkwNwYDVQQLEzB3d3cuZW50cnVzdC5uZXQvQ1BTIGlzIGluY29ycG9yYXRlZCBieSByZWZlcmVu +Y2UxHzAdBgNVBAsTFihjKSAyMDA2IEVudHJ1c3QsIEluYy4xLTArBgNVBAMTJEVudHJ1c3QgUm9v +dCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALaVtkNC+sZtKm9I35RMOVcF7sN5EUFoNu3s/poBj6E4KPz3EEZmLk0eGrEaTsbRwJWIsMn/MYsz +A9u3g3s+IIRe7bJWKKf44LlAcTfFy0cOlypowCKVYhXbR9n10Cv/gkvJrT7eTNuQgFA/CYqEAOww +Cj0Yzfv9KlmaI5UXLEWeH25DeW0MXJj+SKfFI0dcXv1u5x609mhF0YaDW6KKjbHjKYD+JXGIrb68 +j6xSlkuqUY3kEzEZ6E5Nn9uss2rVvDlUccp6en+Q3X0dgNmBu1kmwhH+5pPi94DkZfs0Nw4pgHBN +rziGLp5/V6+eF67rHMsoIV+2HNjnogQi+dPa2MsCAwEAAaOBsDCBrTAOBgNVHQ8BAf8EBAMCAQYw +DwYDVR0TAQH/BAUwAwEB/zArBgNVHRAEJDAigA8yMDA2MTEyNzIwMjM0MlqBDzIwMjYxMTI3MjA1 +MzQyWjAfBgNVHSMEGDAWgBRokORnpKZTgMeGZqTx90tD+4S9bTAdBgNVHQ4EFgQUaJDkZ6SmU4DH +hmak8fdLQ/uEvW0wHQYJKoZIhvZ9B0EABBAwDhsIVjcuMTo0LjADAgSQMA0GCSqGSIb3DQEBBQUA +A4IBAQCT1DCw1wMgKtD5Y+iRDAUgqV8ZyntyTtSx29CW+1RaGSwMCPeyvIWonX9tO1KzKtvn1ISM +Y/YPyyYBkVBs9F8U4pN0wBOeMDpQ47RgxRzwIkSNcUesyBrJ6ZuaAGAT/3B+XxFNSRuzFVJ7yVTa +v52Vr2ua2J7p8eRDjeIRRDq/r72DQnNSi6q7pynP9WQcCk3RvKqsnyrQ/39/2n3qse0wJcGE2jTS +W3iDVuycNsMm4hH2Z0kdkquM++v/eu6FSqdQgPCnXEqULl8FmTxSQeDNtGPPAUO6nIPcj2A781q0 +tHuu2guQOHXvgR1m0vdXcDazv/wor3ElhVsT/h5/WrQ8 +-----END CERTIFICATE----- + +Comodo AAA Services root +======================== +-----BEGIN CERTIFICATE----- +MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEbMBkGA1UECAwS +R3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRowGAYDVQQKDBFDb21vZG8gQ0Eg +TGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmljYXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAw +MFoXDTI4MTIzMTIzNTk1OVowezELMAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hl +c3RlcjEQMA4GA1UEBwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNV +BAMMGEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQuaBtDFcCLNSS1UY8y2bmhG +C1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe3M/vg4aijJRPn2jymJBGhCfHdr/jzDUs +i14HZGWCwEiwqJH5YZ92IFCokcdmtet4YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszW +Y19zjNoFmag4qMsXeDZRrOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjH +Ypy+g8cmez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQUoBEK +Iz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wewYDVR0f +BHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20vQUFBQ2VydGlmaWNhdGVTZXJ2aWNl +cy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29tb2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2Vz +LmNybDANBgkqhkiG9w0BAQUFAAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm +7l3sAg9g1o1QGE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz +Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2G9w84FoVxp7Z +8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsil2D4kF501KKaU73yqWjgom7C +12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== +-----END CERTIFICATE----- + +Go Daddy Class 2 CA +=================== +-----BEGIN CERTIFICATE----- +MIIEADCCAuigAwIBAgIBADANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMY +VGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRp +ZmljYXRpb24gQXV0aG9yaXR5MB4XDTA0MDYyOTE3MDYyMFoXDTM0MDYyOTE3MDYyMFowYzELMAkG +A1UEBhMCVVMxITAfBgNVBAoTGFRoZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28g +RGFkZHkgQ2xhc3MgMiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQAD +ggENADCCAQgCggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCAPVYYYwhv +2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6wwdhFJ2+qN1j3hybX2C32 +qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXiEqITLdiOr18SPaAIBQi2XKVlOARFmR6j +YGB0xUGlcmIbYsUfb18aQr4CUWWoriMYavx4A6lNf4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmY +vLEHZ6IVDd2gWMZEewo+YihfukEHU1jPEX44dMX4/7VpkI+EdOqXG68CAQOjgcAwgb0wHQYDVR0O +BBYEFNLEsNKR1EwRcbNhyz2h/t2oatTjMIGNBgNVHSMEgYUwgYKAFNLEsNKR1EwRcbNhyz2h/t2o +atTjoWekZTBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMu +MTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5ggEAMAwG +A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBADJL87LKPpH8EsahB4yOd6AzBhRckB4Y9wim +PQoZ+YeAEW5p5JYXMP80kWNyOO7MHAGjHZQopDH2esRU1/blMVgDoszOYtuURXO1v0XJJLXVggKt +I3lpjbi2Tc7PTMozI+gciKqdi0FuFskg5YmezTvacPd+mSYgFFQlq25zheabIZ0KbIIOqPjCDPoQ +HmyW74cNxA9hi63ugyuV+I6ShHI56yDqg+2DzZduCLzrTia2cyvk0/ZM/iZx4mERdEr/VxqHD3VI +Ls9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5CufReYNnyicsbkqWletNw+vHX/b +vZ8= +-----END CERTIFICATE----- + +DigiCert Assured ID Root CA +=========================== +-----BEGIN CERTIFICATE----- +MIIDtzCCAp+gAwIBAgIQDOfg5RfYRv6P5WD8G/AwOTANBgkqhkiG9w0BAQUFADBlMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSQw +IgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgQ0EwHhcNMDYxMTEwMDAwMDAwWhcNMzEx +MTEwMDAwMDAwWjBlMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQL +ExB3d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgQ0Ew +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtDhXO5EOAXLGH87dg+XESpa7cJpSIqvTO +9SA5KFhgDPiA2qkVlTJhPLWxKISKityfCgyDF3qPkKyK53lTXDGEKvYPmDI2dsze3Tyoou9q+yHy +UmHfnyDXH+Kx2f4YZNISW1/5WBg1vEfNoTb5a3/UsDg+wRvDjDPZ2C8Y/igPs6eD1sNuRMBhNZYW +/lmci3Zt1/GiSw0r/wty2p5g0I6QNcZ4VYcgoc/lbQrISXwxmDNsIumH0DJaoroTghHtORedmTpy +oeb6pNnVFzF1roV9Iq4/AUaG9ih5yLHa5FcXxH4cDrC0kqZWs72yl+2qp/C3xag/lRbQ/6GW6whf +GHdPAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRF +66Kv9JLLgjEtUYunpyGd823IDzAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYunpyGd823IDzANBgkq +hkiG9w0BAQUFAAOCAQEAog683+Lt8ONyc3pklL/3cmbYMuRCdWKuh+vy1dneVrOfzM4UKLkNl2Bc +EkxY5NM9g0lFWJc1aRqoR+pWxnmrEthngYTffwk8lOa4JiwgvT2zKIn3X/8i4peEH+ll74fg38Fn +SbNd67IJKusm7Xi+fT8r87cmNW1fiQG2SVufAQWbqz0lwcy2f8Lxb4bG+mRo64EtlOtCt/qMHt1i +8b5QZ7dsvfPxH2sMNgcWfzd8qVttevESRmCD1ycEvkvOl77DZypoEd+A5wwzZr8TDRRu838fYxAe ++o0bJW1sj6W3YQGx0qMmoRBxna3iw/nDmVG3KwcIzi7mULKn+gpFL6Lw8g== +-----END CERTIFICATE----- + +DigiCert Global Root CA +======================= +-----BEGIN CERTIFICATE----- +MIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBhMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSAw +HgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBDQTAeFw0wNjExMTAwMDAwMDBaFw0zMTExMTAw +MDAwMDBaMGExCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3 +dy5kaWdpY2VydC5jb20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IENBMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4jvhEXLeqKTTo1eqUKKPC3eQyaKl7hLOllsBCSDMAZOn +TjC3U/dDxGkAV53ijSLdhwZAAIEJzs4bg7/fzTtxRuLWZscFs3YnFo97nh6Vfe63SKMI2tavegw5 +BmV/Sl0fvBf4q77uKNd0f3p4mVmFaG5cIzJLv07A6Fpt43C/dxC//AH2hdmoRBBYMql1GNXRor5H +4idq9Joz+EkIYIvUX7Q6hL+hqkpMfT7PT19sdl6gSzeRntwi5m3OFBqOasv+zbMUZBfHWymeMr/y +7vrTC0LUq7dBMtoM1O/4gdW7jVg/tRvoSSiicNoxBN33shbyTApOB6jtSj1etX+jkMOvJwIDAQAB +o2MwYTAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUA95QNVbRTLtm +8KPiGxvDl7I90VUwHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUwDQYJKoZIhvcNAQEF +BQADggEBAMucN6pIExIK+t1EnE9SsPTfrgT1eXkIoyQY/EsrhMAtudXH/vTBH1jLuG2cenTnmCmr +EbXjcKChzUyImZOMkXDiqw8cvpOp/2PV5Adg06O/nVsJ8dWO41P0jmP6P6fbtGbfYmbW0W5BjfIt +tep3Sp+dWOIrWcBAI+0tKIJFPnlUkiaY4IBIqDfv8NZ5YBberOgOzW6sRBc4L0na4UU+Krk2U886 +UAb3LujEV0lsYSEY1QSteDwsOoBrp+uvFRTp2InBuThs4pFsiv9kuXclVzDAGySj4dzp30d8tbQk +CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4= +-----END CERTIFICATE----- + +DigiCert High Assurance EV Root CA +================================== +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBsMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSsw +KQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5jZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAw +MFoXDTMxMTExMDAwMDAwMFowbDELMAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZ +MBcGA1UECxMQd3d3LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFu +Y2UgRVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm+9S75S0t +Mqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTWPNt0OKRKzE0lgvdKpVMS +OO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEMxChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3 +MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFBIk5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQ +NAQTXKFx01p8VdteZOE3hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUe +h10aUAsgEsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMB +Af8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaAFLE+w2kD+L9HAdSY +JhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3NecnzyIZgYIVyHbIUf4KmeqvxgydkAQ +V8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6zeM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFp +myPInngiK3BD41VHMWEZ71jFhS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkK +mNEVX58Svnw2Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe +vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep+OkuE6N36B9K +-----END CERTIFICATE----- + +COMODO Certification Authority +============================== +-----BEGIN CERTIFICATE----- +MIIEHTCCAwWgAwIBAgIQToEtioJl4AsC7j41AkblPTANBgkqhkiG9w0BAQUFADCBgTELMAkGA1UE +BhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgG +A1UEChMRQ09NT0RPIENBIExpbWl0ZWQxJzAlBgNVBAMTHkNPTU9ETyBDZXJ0aWZpY2F0aW9uIEF1 +dGhvcml0eTAeFw0wNjEyMDEwMDAwMDBaFw0yOTEyMzEyMzU5NTlaMIGBMQswCQYDVQQGEwJHQjEb +MBkGA1UECBMSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRowGAYDVQQKExFD +T01PRE8gQ0EgTGltaXRlZDEnMCUGA1UEAxMeQ09NT0RPIENlcnRpZmljYXRpb24gQXV0aG9yaXR5 +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0ECLi3LjkRv3UcEbVASY06m/weaKXTuH ++7uIzg3jLz8GlvCiKVCZrts7oVewdFFxze1CkU1B/qnI2GqGd0S7WWaXUF601CxwRM/aN5VCaTww +xHGzUvAhTaHYujl8HJ6jJJ3ygxaYqhZ8Q5sVW7euNJH+1GImGEaaP+vB+fGQV+useg2L23IwambV +4EajcNxo2f8ESIl33rXp+2dtQem8Ob0y2WIC8bGoPW43nOIv4tOiJovGuFVDiOEjPqXSJDlqR6sA +1KGzqSX+DT+nHbrTUcELpNqsOO9VUCQFZUaTNE8tja3G1CEZ0o7KBWFxB3NH5YoZEr0ETc5OnKVI +rLsm9wIDAQABo4GOMIGLMB0GA1UdDgQWBBQLWOWLxkwVN6RAqTCpIb5HNlpW/zAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zBJBgNVHR8EQjBAMD6gPKA6hjhodHRwOi8vY3JsLmNvbW9k +b2NhLmNvbS9DT01PRE9DZXJ0aWZpY2F0aW9uQXV0aG9yaXR5LmNybDANBgkqhkiG9w0BAQUFAAOC +AQEAPpiem/Yb6dc5t3iuHXIYSdOH5EOC6z/JqvWote9VfCFSZfnVDeFs9D6Mk3ORLgLETgdxb8CP +OGEIqB6BCsAvIC9Bi5HcSEW88cbeunZrM8gALTFGTO3nnc+IlP8zwFboJIYmuNg4ON8qa90SzMc/ +RxdMosIGlgnW2/4/PEZB31jiVg88O8EckzXZOFKs7sjsLjBOlDW0JB9LeGna8gI4zJVSk/BwJVmc +IGfE7vmLV2H0knZ9P4SNVbfo5azV8fUZVqZa+5Acr5Pr5RzUZ5ddBA6+C4OmF4O5MBKgxTMVBbkN ++8cFduPYSo38NBejxiEovjBFMR7HeL5YYTisO+IBZQ== +-----END CERTIFICATE----- + +COMODO ECC Certification Authority +================================== +-----BEGIN CERTIFICATE----- +MIICiTCCAg+gAwIBAgIQH0evqmIAcFBUTAGem2OZKjAKBggqhkjOPQQDAzCBhTELMAkGA1UEBhMC +R0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UE +ChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlvbiBB +dXRob3JpdHkwHhcNMDgwMzA2MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0Ix +GzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR +Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRo +b3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQDR3svdcmCFYX7deSRFtSrYpn1PlILBs5BAH+X +4QokPB0BBO490o0JlwzgdeT6+3eKKvUDYEs2ixYjFq0JcfRK9ChQtP6IHG4/bC8vCVlbpVsLM5ni +wz2J+Wos77LTBumjQjBAMB0GA1UdDgQWBBR1cacZSBm8nZ3qQUfflMRId5nTeTAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjEA7wNbeqy3eApyt4jf/7VG +FAkK+qDmfQjGGoe9GKhzvSbKYAydzpmfz1wPMOG+FDHqAjAU9JM8SaczepBGR7NjfRObTrdvGDeA +U/7dIOA1mjbRxwG55tzd8/8dLDoWV9mSOdY= +-----END CERTIFICATE----- + +GlobalSign Root CA - R3 +======================= +-----BEGIN CERTIFICATE----- +MIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4GA1UECxMXR2xv +YmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNpZ24xEzARBgNVBAMTCkdsb2Jh +bFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4MTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxT +aWduIFJvb3QgQ0EgLSBSMzETMBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2ln +bjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWt +iHL8RgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsTgHeMCOFJ +0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmmKPZpO/bLyCiR5Z2KYVc3 +rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zdQQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjl +OCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZXriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2 +xmmFghcCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE +FI/wS3+oLkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZURUm7 +lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMpjjM5RcOO5LlXbKr8 +EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK6fBdRoyV3XpYKBovHd7NADdBj+1E +bddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQXmcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18 +YIvDQVETI53O9zJrlAGomecsMx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7r +kpeDMdmztcpHWD9f +-----END CERTIFICATE----- + +Go Daddy Root Certificate Authority - G2 +======================================== +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIBADANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMxEDAOBgNVBAgT +B0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoTEUdvRGFkZHkuY29tLCBJbmMu +MTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5 +MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgYMxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6 +b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjExMC8G +A1UEAxMoR28gRGFkZHkgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAL9xYgjx+lk09xvJGKP3gElY6SKDE6bFIEMBO4Tx5oVJnyfq +9oQbTqC023CYxzIBsQU+B07u9PpPL1kwIuerGVZr4oAH/PMWdYA5UXvl+TW2dE6pjYIT5LY/qQOD ++qK+ihVqf94Lw7YZFAXK6sOoBJQ7RnwyDfMAZiLIjWltNowRGLfTshxgtDj6AozO091GB94KPutd +fMh8+7ArU6SSYmlRJQVhGkSBjCypQ5Yj36w6gZoOKcUcqeldHraenjAKOc7xiID7S13MMuyFYkMl +NAJWJwGRtDtwKj9useiciAF9n9T521NtYJ2/LOdYq7hfRvzOxBsDPAnrSTFcaUaz4EcCAwEAAaNC +MEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFDqahQcQZyi27/a9 +BUFuIMGU2g/eMA0GCSqGSIb3DQEBCwUAA4IBAQCZ21151fmXWWcDYfF+OwYxdS2hII5PZYe096ac +vNjpL9DbWu7PdIxztDhC2gV7+AJ1uP2lsdeu9tfeE8tTEH6KRtGX+rcuKxGrkLAngPnon1rpN5+r +5N9ss4UXnT3ZJE95kTXWXwTrgIOrmgIttRD02JDHBHNA7XIloKmf7J6raBKZV8aPEjoJpL1E/QYV +N8Gb5DKj7Tjo2GTzLH4U/ALqn83/B2gX2yKQOC16jdFU8WnjXzPKej17CuPKf1855eJ1usV2GDPO +LPAvTK33sefOT6jEm0pUBsV/fdUID+Ic/n4XuKxe9tQWskMJDE32p2u0mYRlynqI4uJEvlz36hz1 +-----END CERTIFICATE----- + +DigiCert Assured ID Root G2 +=========================== +-----BEGIN CERTIFICATE----- +MIIDljCCAn6gAwIBAgIQC5McOtY5Z+pnI7/Dr5r0SzANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSQw +IgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzIwHhcNMTMwODAxMTIwMDAwWhcNMzgw +MTE1MTIwMDAwWjBlMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQL +ExB3d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzIw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDZ5ygvUj82ckmIkzTz+GoeMVSAn61UQbVH +35ao1K+ALbkKz3X9iaV9JPrjIgwrvJUXCzO/GU1BBpAAvQxNEP4HteccbiJVMWWXvdMX0h5i89vq +bFCMP4QMls+3ywPgym2hFEwbid3tALBSfK+RbLE4E9HpEgjAALAcKxHad3A2m67OeYfcgnDmCXRw +VWmvo2ifv922ebPynXApVfSr/5Vh88lAbx3RvpO704gqu52/clpWcTs/1PPRCv4o76Pu2ZmvA9OP +YLfykqGxvYmJHzDNw6YuYjOuFgJ3RFrngQo8p0Quebg/BLxcoIfhG69Rjs3sLPr4/m3wOnyqi+Rn +lTGNAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBTO +w0q5mVXyuNtgv6l+vVa1lzan1jANBgkqhkiG9w0BAQsFAAOCAQEAyqVVjOPIQW5pJ6d1Ee88hjZv +0p3GeDgdaZaikmkuOGybfQTUiaWxMTeKySHMq2zNixya1r9I0jJmwYrA8y8678Dj1JGG0VDjA9tz +d29KOVPt3ibHtX2vK0LRdWLjSisCx1BL4GnilmwORGYQRI+tBev4eaymG+g3NJ1TyWGqolKvSnAW +hsI6yLETcDbYz+70CjTVW0z9B5yiutkBclzzTcHdDrEcDcRjvq30FPuJ7KJBDkzMyFdA0G4Dqs0M +jomZmWzwPDCvON9vvKO+KSAnq3T/EyJ43pdSVR6DtVQgA+6uwE9W3jfMw3+qBCe703e4YtsXfJwo +IhNzbM8m9Yop5w== +-----END CERTIFICATE----- + +DigiCert Assured ID Root G3 +=========================== +-----BEGIN CERTIFICATE----- +MIICRjCCAc2gAwIBAgIQC6Fa+h3foLVJRK/NJKBs7DAKBggqhkjOPQQDAzBlMQswCQYDVQQGEwJV +UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSQwIgYD +VQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzMwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1 +MTIwMDAwWjBlMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzMwdjAQ +BgcqhkjOPQIBBgUrgQQAIgNiAAQZ57ysRGXtzbg/WPuNsVepRC0FFfLvC/8QdJ+1YlJfZn4f5dwb +RXkLzMZTCp2NXQLZqVneAlr2lSoOjThKiknGvMYDOAdfVdp+CW7if17QRSAPWXYQ1qAk8C3eNvJs +KTmjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBTL0L2p4ZgF +UaFNN6KDec6NHSrkhDAKBggqhkjOPQQDAwNnADBkAjAlpIFFAmsSS3V0T8gj43DydXLefInwz5Fy +YZ5eEJJZVrmDxxDnOOlYJjZ91eQ0hjkCMHw2U/Aw5WJjOpnitqM7mzT6HtoQknFekROn3aRukswy +1vUhZscv6pZjamVFkpUBtA== +-----END CERTIFICATE----- + +DigiCert Global Root G2 +======================= +-----BEGIN CERTIFICATE----- +MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBhMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSAw +HgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBHMjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUx +MjAwMDBaMGExCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3 +dy5kaWdpY2VydC5jb20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI2/Ou8jqJ +kTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx1x7e/dfgy5SDN67sH0NO +3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQq2EGnI/yuum06ZIya7XzV+hdG82MHauV +BJVJ8zUtluNJbd134/tJS7SsVQepj5WztCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyM +UNGPHgm+F6HmIcr9g+UQvIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQAB +o0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV5uNu +5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY1Yl9PMWLSn/pvtsr +F9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4NeF22d+mQrvHRAiGfzZ0JFrabA0U +WTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NGFdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBH +QRFXGU7Aj64GxJUTFy8bJZ918rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/ +iyK5S9kJRaTepLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl +MrY= +-----END CERTIFICATE----- + +DigiCert Global Root G3 +======================= +-----BEGIN CERTIFICATE----- +MIICPzCCAcWgAwIBAgIQBVVWvPJepDU1w6QP1atFcjAKBggqhkjOPQQDAzBhMQswCQYDVQQGEwJV +UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSAwHgYD +VQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBHMzAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAw +MDBaMGExCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5k +aWdpY2VydC5jb20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEczMHYwEAYHKoZIzj0C +AQYFK4EEACIDYgAE3afZu4q4C/sLfyHS8L6+c/MzXRq8NOrexpu80JX28MzQC7phW1FGfp4tn+6O +YwwX7Adw9c+ELkCDnOg/QW07rdOkFFk2eJ0DQ+4QE2xy3q6Ip6FrtUPOZ9wj/wMco+I+o0IwQDAP +BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUs9tIpPmhxdiuNkHMEWNp +Yim8S8YwCgYIKoZIzj0EAwMDaAAwZQIxAK288mw/EkrRLTnDCgmXc/SINoyIJ7vmiI1Qhadj+Z4y +3maTD/HMsQmP3Wyr+mt/oAIwOWZbwmSNuJ5Q3KjVSaLtx9zRSX8XAbjIho9OjIgrqJqpisXRAL34 +VOKa5Vt8sycX +-----END CERTIFICATE----- + +DigiCert Trusted Root G4 +======================== +-----BEGIN CERTIFICATE----- +MIIFkDCCA3igAwIBAgIQBZsbV56OITLiOQe9p3d1XDANBgkqhkiG9w0BAQwFADBiMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSEw +HwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1 +MTIwMDAwWjBiMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwggIiMA0G +CSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC/5pBzaN675F1KPDAiMGkz7MKnJS7JIT3yithZwuEp +pz1Yq3aaza57G4QNxDAf8xukOBbrVsaXbR2rsnnyyhHS5F/WBTxSD1Ifxp4VpX6+n6lXFllVcq9o +k3DCsrp1mWpzMpTREEQQLt+C8weE5nQ7bXHiLQwb7iDVySAdYyktzuxeTsiT+CFhmzTrBcZe7Fsa +vOvJz82sNEBfsXpm7nfISKhmV1efVFiODCu3T6cw2Vbuyntd463JT17lNecxy9qTXtyOj4DatpGY +QJB5w3jHtrHEtWoYOAMQjdjUN6QuBX2I9YI+EJFwq1WCQTLX2wRzKm6RAXwhTNS8rhsDdV14Ztk6 +MUSaM0C/CNdaSaTC5qmgZ92kJ7yhTzm1EVgX9yRcRo9k98FpiHaYdj1ZXUJ2h4mXaXpI8OCiEhtm +mnTK3kse5w5jrubU75KSOp493ADkRSWJtppEGSt+wJS00mFt6zPZxd9LBADMfRyVw4/3IbKyEbe7 +f/LVjHAsQWCqsWMYRJUadmJ+9oCw++hkpjPRiQfhvbfmQ6QYuKZ3AeEPlAwhHbJUKSWJbOUOUlFH +dL4mrLZBdd56rF+NP8m800ERElvlEFDrMcXKchYiCd98THU/Y+whX8QgUWtvsauGi0/C1kVfnSD8 +oR7FwI+isX4KJpn15GkvmB0t9dmpsh3lGwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1Ud +DwEB/wQEAwIBhjAdBgNVHQ4EFgQU7NfjgtJxXWRM3y5nP+e6mK4cD08wDQYJKoZIhvcNAQEMBQAD +ggIBALth2X2pbL4XxJEbw6GiAI3jZGgPVs93rnD5/ZpKmbnJeFwMDF/k5hQpVgs2SV1EY+CtnJYY +ZhsjDT156W1r1lT40jzBQ0CuHVD1UvyQO7uYmWlrx8GnqGikJ9yd+SeuMIW59mdNOj6PWTkiU0Tr +yF0Dyu1Qen1iIQqAyHNm0aAFYF/opbSnr6j3bTWcfFqK1qI4mfN4i/RN0iAL3gTujJtHgXINwBQy +7zBZLq7gcfJW5GqXb5JQbZaNaHqasjYUegbyJLkJEVDXCLG4iXqEI2FCKeWjzaIgQdfRnGTZ6iah +ixTXTBmyUEFxPT9NcCOGDErcgdLMMpSEDQgJlxxPwO5rIHQw0uA5NBCFIRUBCOhVMt5xSdkoF1BN +5r5N0XWs0Mr7QbhDparTwwVETyw2m+L64kW4I1NsBm9nVX9GtUw/bihaeSbSpKhil9Ie4u1Ki7wb +/UdKDd9nZn6yW0HQO+T0O/QEY+nvwlQAUaCKKsnOeMzV6ocEGLPOr0mIr/OSmbaz5mEP0oUA51Aa +5BuVnRmhuZyxm7EAHu/QD09CbMkKvO5D+jpxpchNJqU1/YldvIViHTLSoCtU7ZpXwdv6EM8Zt4tK +G48BtieVU+i2iW1bvGjUI+iLUaJW+fCmgKDWHrO8Dw9TdSmq6hN35N6MgSGtBxBHEa2HPQfRdbzP +82Z+ +-----END CERTIFICATE----- + +COMODO RSA Certification Authority +================================== +-----BEGIN CERTIFICATE----- +MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCBhTELMAkGA1UE +BhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgG +A1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlv +biBBdXRob3JpdHkwHhcNMTAwMTE5MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMC +R0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UE +ChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBB +dXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR6FSS0gpWsawNJN3Fz0Rn +dJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8Xpz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZ +FGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+ +5eNu/Nio5JIk2kNrYrhV/erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pG +x8cgoLEfZd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z+pUX +2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7wqP/0uK3pN/u6uPQL +OvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZahSL0896+1DSJMwBGB7FY79tOi4lu3 +sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVICu9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+C +GCe01a60y1Dma/RMhnEw6abfFobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5 +WdYgGq/yapiqcrxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E +FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8w +DQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvlwFTPoCWOAvn9sKIN9SCYPBMt +rFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+ +nq6PK7o9mfjYcwlYRm6mnPTXJ9OV2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSg +tZx8jb8uk2IntznaFxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwW +sRqZCuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiKboHGhfKp +pC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmckejkk9u+UJueBPSZI9FoJA +zMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yLS0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHq +ZJx64SIDqZxubw5lT2yHh17zbqD5daWbQOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk52 +7RH89elWsn2/x20Kk4yl0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7I +LaZRfyHBNVOFBkpdn627G190 +-----END CERTIFICATE----- + +USERTrust RSA Certification Authority +===================================== +-----BEGIN CERTIFICATE----- +MIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCBiDELMAkGA1UE +BhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQK +ExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwHhcNMTAwMjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UE +BhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQK +ExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCAEmUXNg7D2wiz +0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2j +Y0K2dvKpOyuR+OJv0OwWIJAJPuLodMkYtJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFn +RghRy4YUVD+8M/5+bJz/Fp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O ++T23LLb2VN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT79uq +/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6c0Plfg6lZrEpfDKE +Y1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmTYo61Zs8liM2EuLE/pDkP2QKe6xJM +lXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97lc6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8 +yexDJtC/QV9AqURE9JnnV4eeUB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+ +eLf8ZxXhyVeEHg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd +BgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF +MAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPFUp/L+M+ZBn8b2kMVn54CVVeW +FPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KOVWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ +7l8wXEskEVX/JJpuXior7gtNn3/3ATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQ +Eg9zKC7F4iRO/Fjs8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM +8WcRiQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYzeSf7dNXGi +FSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZXHlKYC6SQK5MNyosycdi +yA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/qS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9c +J2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRBVXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGw +sAvgnEzDHNb842m1R0aBL6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gx +Q+6IHdfGjjxDah2nGN59PRbxYvnKkKj9 +-----END CERTIFICATE----- + +USERTrust ECC Certification Authority +===================================== +-----BEGIN CERTIFICATE----- +MIICjzCCAhWgAwIBAgIQXIuZxVqUxdJxVt7NiYDMJjAKBggqhkjOPQQDAzCBiDELMAkGA1UEBhMC +VVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU +aGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBFQ0MgQ2VydGlmaWNhdGlv +biBBdXRob3JpdHkwHhcNMTAwMjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMC +VVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU +aGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBFQ0MgQ2VydGlmaWNhdGlv +biBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQarFRaqfloI+d61SRvU8Za2EurxtW2 +0eZzca7dnNYMYf3boIkDuAUU7FfO7l0/4iGzzvfUinngo4N+LZfQYcTxmdwlkWOrfzCjtHDix6Ez +nPO/LlxTsV+zfTJ/ijTjeXmjQjBAMB0GA1UdDgQWBBQ64QmG1M8ZwpZ2dEl23OA1xmNjmjAOBgNV +HQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjA2Z6EWCNzklwBB +HU6+4WMBzzuqQhFkoJ2UOQIReVx7Hfpkue4WQrO/isIJxOzksU0CMQDpKmFHjFJKS04YcPbWRNZu +9YO6bVi9JNlWSOrvxKJGgYhqOkbRqZtNyWHa0V1Xahg= +-----END CERTIFICATE----- + +GlobalSign ECC Root CA - R4 +=========================== +-----BEGIN CERTIFICATE----- +MIIB4TCCAYegAwIBAgIRKjikHJYKBN5CsiilC+g0mAIwCgYIKoZIzj0EAwIwUDEkMCIGA1UECxMb +R2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI0MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQD +EwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoXDTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMb +R2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI0MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQD +EwpHbG9iYWxTaWduMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuMZ5049sJQ6fLjkZHAOkrprl +OQcJFspjsbmG+IpXwVfOQvpzofdlQv8ewQCybnMO/8ch5RikqtlxP6jUuc6MHaNCMEAwDgYDVR0P +AQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFFSwe61FuOJAf/sKbvu+M8k8o4TV +MAoGCCqGSM49BAMCA0gAMEUCIQDckqGgE6bPA7DmxCGXkPoUVy0D7O48027KqGx2vKLeuwIgJ6iF +JzWbVsaj8kfSt24bAgAXqmemFZHe+pTsewv4n4Q= +-----END CERTIFICATE----- + +GlobalSign ECC Root CA - R5 +=========================== +-----BEGIN CERTIFICATE----- +MIICHjCCAaSgAwIBAgIRYFlJ4CYuu1X5CneKcflK2GwwCgYIKoZIzj0EAwMwUDEkMCIGA1UECxMb +R2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI1MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQD +EwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoXDTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMb +R2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI1MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQD +EwpHbG9iYWxTaWduMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAER0UOlvt9Xb/pOdEh+J8LttV7HpI6 +SFkc8GIxLcB6KP4ap1yztsyX50XUWPrRd21DosCHZTQKH3rd6zwzocWdTaRvQZU4f8kehOvRnkmS +h5SHDDqFSmafnVmTTZdhBoZKo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAd +BgNVHQ4EFgQUPeYpSJvqB8ohREom3m7e0oPQn1kwCgYIKoZIzj0EAwMDaAAwZQIxAOVpEslu28Yx +uglB4Zf4+/2a4n0Sye18ZNPLBSWLVtmg515dTguDnFt2KaAJJiFqYgIwcdK1j1zqO+F4CYWodZI7 +yFz9SO8NdCKoCOJuxUnOxwy8p2Fp8fc74SrL+SvzZpA3 +-----END CERTIFICATE----- + +IdenTrust Commercial Root CA 1 +============================== +-----BEGIN CERTIFICATE----- +MIIFYDCCA0igAwIBAgIQCgFCgAAAAUUjyES1AAAAAjANBgkqhkiG9w0BAQsFADBKMQswCQYDVQQG +EwJVUzESMBAGA1UEChMJSWRlblRydXN0MScwJQYDVQQDEx5JZGVuVHJ1c3QgQ29tbWVyY2lhbCBS +b290IENBIDEwHhcNMTQwMTE2MTgxMjIzWhcNMzQwMTE2MTgxMjIzWjBKMQswCQYDVQQGEwJVUzES +MBAGA1UEChMJSWRlblRydXN0MScwJQYDVQQDEx5JZGVuVHJ1c3QgQ29tbWVyY2lhbCBSb290IENB +IDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCnUBneP5k91DNG8W9RYYKyqU+PZ4ld +hNlT3Qwo2dfw/66VQ3KZ+bVdfIrBQuExUHTRgQ18zZshq0PirK1ehm7zCYofWjK9ouuU+ehcCuz/ +mNKvcbO0U59Oh++SvL3sTzIwiEsXXlfEU8L2ApeN2WIrvyQfYo3fw7gpS0l4PJNgiCL8mdo2yMKi +1CxUAGc1bnO/AljwpN3lsKImesrgNqUZFvX9t++uP0D1bVoE/c40yiTcdCMbXTMTEl3EASX2MN0C +XZ/g1Ue9tOsbobtJSdifWwLziuQkkORiT0/Br4sOdBeo0XKIanoBScy0RnnGF7HamB4HWfp1IYVl +3ZBWzvurpWCdxJ35UrCLvYf5jysjCiN2O/cz4ckA82n5S6LgTrx+kzmEB/dEcH7+B1rlsazRGMzy +NeVJSQjKVsk9+w8YfYs7wRPCTY/JTw436R+hDmrfYi7LNQZReSzIJTj0+kuniVyc0uMNOYZKdHzV +WYfCP04MXFL0PfdSgvHqo6z9STQaKPNBiDoT7uje/5kdX7rL6B7yuVBgwDHTc+XvvqDtMwt0viAg +xGds8AgDelWAf0ZOlqf0Hj7h9tgJ4TNkK2PXMl6f+cB7D3hvl7yTmvmcEpB4eoCHFddydJxVdHix +uuFucAS6T6C6aMN7/zHwcz09lCqxC0EOoP5NiGVreTO01wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMC +AQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU7UQZwNPwBovupHu+QucmVMiONnYwDQYJKoZI +hvcNAQELBQADggIBAA2ukDL2pkt8RHYZYR4nKM1eVO8lvOMIkPkp165oCOGUAFjvLi5+U1KMtlwH +6oi6mYtQlNeCgN9hCQCTrQ0U5s7B8jeUeLBfnLOic7iPBZM4zY0+sLj7wM+x8uwtLRvM7Kqas6pg +ghstO8OEPVeKlh6cdbjTMM1gCIOQ045U8U1mwF10A0Cj7oV+wh93nAbowacYXVKV7cndJZ5t+qnt +ozo00Fl72u1Q8zW/7esUTTHHYPTa8Yec4kjixsU3+wYQ+nVZZjFHKdp2mhzpgq7vmrlR94gjmmmV +YjzlVYA211QC//G5Xc7UI2/YRYRKW2XviQzdFKcgyxilJbQN+QHwotL0AMh0jqEqSI5l2xPE4iUX +feu+h1sXIFRRk0pTAwvsXcoz7WL9RccvW9xYoIA55vrX/hMUpu09lEpCdNTDd1lzzY9GvlU47/ro +kTLql1gEIt44w8y8bckzOmoKaT+gyOpyj4xjhiO9bTyWnpXgSUyqorkqG5w2gXjtw+hG4iZZRHUe +2XWJUc0QhJ1hYMtd+ZciTY6Y5uN/9lu7rs3KSoFrXgvzUeF0K+l+J6fZmUlO+KWA2yUPHGNiiskz +Z2s8EIPGrd6ozRaOjfAHN3Gf8qv8QfXBi+wAN10J5U6A7/qxXDgGpRtK4dw4LTzcqx+QGtVKnO7R +cGzM7vRX+Bi6hG6H +-----END CERTIFICATE----- + +IdenTrust Public Sector Root CA 1 +================================= +-----BEGIN CERTIFICATE----- +MIIFZjCCA06gAwIBAgIQCgFCgAAAAUUjz0Z8AAAAAjANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQG +EwJVUzESMBAGA1UEChMJSWRlblRydXN0MSowKAYDVQQDEyFJZGVuVHJ1c3QgUHVibGljIFNlY3Rv +ciBSb290IENBIDEwHhcNMTQwMTE2MTc1MzMyWhcNMzQwMTE2MTc1MzMyWjBNMQswCQYDVQQGEwJV +UzESMBAGA1UEChMJSWRlblRydXN0MSowKAYDVQQDEyFJZGVuVHJ1c3QgUHVibGljIFNlY3RvciBS +b290IENBIDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC2IpT8pEiv6EdrCvsnduTy +P4o7ekosMSqMjbCpwzFrqHd2hCa2rIFCDQjrVVi7evi8ZX3yoG2LqEfpYnYeEe4IFNGyRBb06tD6 +Hi9e28tzQa68ALBKK0CyrOE7S8ItneShm+waOh7wCLPQ5CQ1B5+ctMlSbdsHyo+1W/CD80/HLaXI +rcuVIKQxKFdYWuSNG5qrng0M8gozOSI5Cpcu81N3uURF/YTLNiCBWS2ab21ISGHKTN9T0a9SvESf +qy9rg3LvdYDaBjMbXcjaY8ZNzaxmMc3R3j6HEDbhuaR672BQssvKplbgN6+rNBM5Jeg5ZuSYeqoS +mJxZZoY+rfGwyj4GD3vwEUs3oERte8uojHH01bWRNszwFcYr3lEXsZdMUD2xlVl8BX0tIdUAvwFn +ol57plzy9yLxkA2T26pEUWbMfXYD62qoKjgZl3YNa4ph+bz27nb9cCvdKTz4Ch5bQhyLVi9VGxyh +LrXHFub4qjySjmm2AcG1hp2JDws4lFTo6tyePSW8Uybt1as5qsVATFSrsrTZ2fjXctscvG29ZV/v +iDUqZi/u9rNl8DONfJhBaUYPQxxp+pu10GFqzcpL2UyQRqsVWaFHVCkugyhfHMKiq3IXAAaOReyL +4jM9f9oZRORicsPfIsbyVtTdX5Vy7W1f90gDW/3FKqD2cyOEEBsB5wIDAQABo0IwQDAOBgNVHQ8B +Af8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU43HgntinQtnbcZFrlJPrw6PRFKMw +DQYJKoZIhvcNAQELBQADggIBAEf63QqwEZE4rU1d9+UOl1QZgkiHVIyqZJnYWv6IAcVYpZmxI1Qj +t2odIFflAWJBF9MJ23XLblSQdf4an4EKwt3X9wnQW3IV5B4Jaj0z8yGa5hV+rVHVDRDtfULAj+7A +mgjVQdZcDiFpboBhDhXAuM/FSRJSzL46zNQuOAXeNf0fb7iAaJg9TaDKQGXSc3z1i9kKlT/YPyNt +GtEqJBnZhbMX73huqVjRI9PHE+1yJX9dsXNw0H8GlwmEKYBhHfpe/3OsoOOJuBxxFcbeMX8S3OFt +m6/n6J91eEyrRjuazr8FGF1NFTwWmhlQBJqymm9li1JfPFgEKCXAZmExfrngdbkaqIHWchezxQMx +NRF4eKLg6TCMf4DfWN88uieW4oA0beOY02QnrEh+KHdcxiVhJfiFDGX6xDIvpZgF5PgLZxYWxoK4 +Mhn5+bl53B/N66+rDt0b20XkeucC4pVd/GnwU2lhlXV5C15V5jgclKlZM57IcXR5f1GJtshquDDI +ajjDbp7hNxbqBWJMWxJH7ae0s1hWx0nzfxJoCTFx8G34Tkf71oXuxVhAGaQdp/lLQzfcaFpPz+vC +ZHTetBXZ9FRUGi8c15dxVJCO2SCdUyt/q4/i6jC8UDfv8Ue1fXwsBOxonbRJRBD0ckscZOf85muQ +3Wl9af0AVqW3rLatt8o+Ae+c +-----END CERTIFICATE----- + +Entrust Root Certification Authority - G2 +========================================= +-----BEGIN CERTIFICATE----- +MIIEPjCCAyagAwIBAgIESlOMKDANBgkqhkiG9w0BAQsFADCBvjELMAkGA1UEBhMCVVMxFjAUBgNV +BAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50cnVzdC5uZXQvbGVnYWwtdGVy +bXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3QsIEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ug +b25seTEyMDAGA1UEAxMpRW50cnVzdCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRzIw +HhcNMDkwNzA3MTcyNTU0WhcNMzAxMjA3MTc1NTU0WjCBvjELMAkGA1UEBhMCVVMxFjAUBgNVBAoT +DUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50cnVzdC5uZXQvbGVnYWwtdGVybXMx +OTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3QsIEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25s +eTEyMDAGA1UEAxMpRW50cnVzdCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRzIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6hLZy254Ma+KZ6TABp3bqMriVQRrJ2mFOWHLP +/vaCeb9zYQYKpSfYs1/TRU4cctZOMvJyig/3gxnQaoCAAEUesMfnmr8SVycco2gvCoe9amsOXmXz +HHfV1IWNcCG0szLni6LVhjkCsbjSR87kyUnEO6fe+1R9V77w6G7CebI6C1XiUJgWMhNcL3hWwcKU +s/Ja5CeanyTXxuzQmyWC48zCxEXFjJd6BmsqEZ+pCm5IO2/b1BEZQvePB7/1U1+cPvQXLOZprE4y +TGJ36rfo5bs0vBmLrpxR57d+tVOxMyLlbc9wPBr64ptntoP0jaWvYkxN4FisZDQSA/i2jZRjJKRx +AgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqciZ6 +0B7vfec7aVHUbI2fkBJmqzANBgkqhkiG9w0BAQsFAAOCAQEAeZ8dlsa2eT8ijYfThwMEYGprmi5Z +iXMRrEPR9RP/jTkrwPK9T3CMqS/qF8QLVJ7UG5aYMzyorWKiAHarWWluBh1+xLlEjZivEtRh2woZ +Rkfz6/djwUAFQKXSt/S1mja/qYh2iARVBCuch38aNzx+LaUa2NSJXsq9rD1s2G2v1fN2D807iDgi +nWyTmsQ9v4IbZT+mD12q/OWyFcq1rca8PdCE6OoGcrBNOTJ4vz4RnAuknZoh8/CbCzB428Hch0P+ +vGOaysXCHMnHjf87ElgI5rY97HosTvuDls4MPGmHVHOkc8KT/1EQrBVUAdj8BbGJoX90g5pJ19xO +e4pIb4tF9g== +-----END CERTIFICATE----- + +Entrust Root Certification Authority - EC1 +========================================== +-----BEGIN CERTIFICATE----- +MIIC+TCCAoCgAwIBAgINAKaLeSkAAAAAUNCR+TAKBggqhkjOPQQDAzCBvzELMAkGA1UEBhMCVVMx +FjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50cnVzdC5uZXQvbGVn +YWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDEyIEVudHJ1c3QsIEluYy4gLSBmb3IgYXV0aG9yaXpl +ZCB1c2Ugb25seTEzMDEGA1UEAxMqRW50cnVzdCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5 +IC0gRUMxMB4XDTEyMTIxODE1MjUzNloXDTM3MTIxODE1NTUzNlowgb8xCzAJBgNVBAYTAlVTMRYw +FAYDVQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9TZWUgd3d3LmVudHJ1c3QubmV0L2xlZ2Fs +LXRlcm1zMTkwNwYDVQQLEzAoYykgMjAxMiBFbnRydXN0LCBJbmMuIC0gZm9yIGF1dGhvcml6ZWQg +dXNlIG9ubHkxMzAxBgNVBAMTKkVudHJ1c3QgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAt +IEVDMTB2MBAGByqGSM49AgEGBSuBBAAiA2IABIQTydC6bUF74mzQ61VfZgIaJPRbiWlH47jCffHy +AsWfoPZb1YsGGYZPUxBtByQnoaD41UcZYUx9ypMn6nQM72+WCf5j7HBdNq1nd67JnXxVRDqiY1Ef +9eNi1KlHBz7MIKNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE +FLdj5xrdjekIplWDpOBqUEFlEUJJMAoGCCqGSM49BAMDA2cAMGQCMGF52OVCR98crlOZF7ZvHH3h +vxGU0QOIdeSNiaSKd0bebWHvAvX7td/M/k7//qnmpwIwW5nXhTcGtXsI/esni0qU+eH6p44mCOh8 +kmhtc9hvJqwhAriZtyZBWyVgrtBIGu4G +-----END CERTIFICATE----- + +ISRG Root X1 +============ +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAwTzELMAkGA1UE +BhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2VhcmNoIEdyb3VwMRUwEwYDVQQD +EwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQG +EwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMT +DElTUkcgUm9vdCBYMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54r +Vygch77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+0TM8ukj1 +3Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6UA5/TR5d8mUgjU+g4rk8K +b4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sWT8KOEUt+zwvo/7V3LvSye0rgTBIlDHCN +Aymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyHB5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ +4Q7e2RCOFvu396j3x+UCB5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf +1b0SHzUvKBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWnOlFu +hjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTnjh8BCNAw1FtxNrQH +usEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbwqHyGO0aoSCqI3Haadr8faqU9GY/r +OPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CIrU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4G +A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY +9umbbjANBgkqhkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL +ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ3BebYhtF8GaV +0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KKNFtY2PwByVS5uCbMiogziUwt +hDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJw +TdwJx4nLCgdNbOhdjsnvzqvHu7UrTkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nx +e5AW0wdeRlN8NwdCjNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZA +JzVcoyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq4RgqsahD +YVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPAmRGunUHBcnWEvgJBQl9n +JEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57demyPxgcYxn/eR44/KJ4EBs+lVDR3veyJ +m+kXQ99b21/+jh5Xos1AnX5iItreGCc= +-----END CERTIFICATE----- + +Amazon Root CA 1 +================ +-----BEGIN CERTIFICATE----- +MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsFADA5MQswCQYD +VQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24gUm9vdCBDQSAxMB4XDTE1 +MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTELMAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpv +bjEZMBcGA1UEAxMQQW1hem9uIFJvb3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBALJ4gHHKeNXjca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgH +FzZM9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qwIFAGbHrQ +gLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6VOujw5H5SNz/0egwLX0t +dHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L93FcXmn/6pUCyziKrlA4b9v7LWIbxcce +VOF34GfID5yHI9Y/QCB/IIDEgEw+OyQmjgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB +/zAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3 +DQEBCwUAA4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDIU5PM +CCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUsN+gDS63pYaACbvXy +8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vvo/ufQJVtMVT8QtPHRh8jrdkPSHCa +2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2 +xJNDd2ZhwLnoQdeXeGADbkpyrqXRfboQnoZsG4q5WTP468SQvvG5 +-----END CERTIFICATE----- + +Amazon Root CA 2 +================ +-----BEGIN CERTIFICATE----- +MIIFQTCCAymgAwIBAgITBmyf0pY1hp8KD+WGePhbJruKNzANBgkqhkiG9w0BAQwFADA5MQswCQYD +VQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24gUm9vdCBDQSAyMB4XDTE1 +MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpv +bjEZMBcGA1UEAxMQQW1hem9uIFJvb3QgQ0EgMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC +ggIBAK2Wny2cSkxKgXlRmeyKy2tgURO8TW0G/LAIjd0ZEGrHJgw12MBvIITplLGbhQPDW9tK6Mj4 +kHbZW0/jTOgGNk3Mmqw9DJArktQGGWCsN0R5hYGCrVo34A3MnaZMUnbqQ523BNFQ9lXg1dKmSYXp +N+nKfq5clU1Imj+uIFptiJXZNLhSGkOQsL9sBbm2eLfq0OQ6PBJTYv9K8nu+NQWpEjTj82R0Yiw9 +AElaKP4yRLuH3WUnAnE72kr3H9rN9yFVkE8P7K6C4Z9r2UXTu/Bfh+08LDmG2j/e7HJV63mjrdvd +fLC6HM783k81ds8P+HgfajZRRidhW+mez/CiVX18JYpvL7TFz4QuK/0NURBs+18bvBt+xa47mAEx +kv8LV/SasrlX6avvDXbR8O70zoan4G7ptGmh32n2M8ZpLpcTnqWHsFcQgTfJU7O7f/aS0ZzQGPSS +btqDT6ZjmUyl+17vIWR6IF9sZIUVyzfpYgwLKhbcAS4y2j5L9Z469hdAlO+ekQiG+r5jqFoz7Mt0 +Q5X5bGlSNscpb/xVA1wf+5+9R+vnSUeVC06JIglJ4PVhHvG/LopyboBZ/1c6+XUyo05f7O0oYtlN +c/LMgRdg7c3r3NunysV+Ar3yVAhU/bQtCSwXVEqY0VThUWcI0u1ufm8/0i2BWSlmy5A5lREedCf+ +3euvAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSw +DPBMMPQFWAJI/TPlUq9LhONmUjANBgkqhkiG9w0BAQwFAAOCAgEAqqiAjw54o+Ci1M3m9Zh6O+oA +A7CXDpO8Wqj2LIxyh6mx/H9z/WNxeKWHWc8w4Q0QshNabYL1auaAn6AFC2jkR2vHat+2/XcycuUY ++gn0oJMsXdKMdYV2ZZAMA3m3MSNjrXiDCYZohMr/+c8mmpJ5581LxedhpxfL86kSk5Nrp+gvU5LE +YFiwzAJRGFuFjWJZY7attN6a+yb3ACfAXVU3dJnJUH/jWS5E4ywl7uxMMne0nxrpS10gxdr9HIcW +xkPo1LsmmkVwXqkLN1PiRnsn/eBG8om3zEK2yygmbtmlyTrIQRNg91CMFa6ybRoVGld45pIq2WWQ +gj9sAq+uEjonljYE1x2igGOpm/HlurR8FLBOybEfdF849lHqm/osohHUqS0nGkWxr7JOcQ3AWEbW +aQbLU8uz/mtBzUF+fUwPfHJ5elnNXkoOrJupmHN5fLT0zLm4BwyydFy4x2+IoZCn9Kr5v2c69BoV +Yh63n749sSmvZ6ES8lgQGVMDMBu4Gon2nL2XA46jCfMdiyHxtN/kHNGfZQIG6lzWE7OE76KlXIx3 +KadowGuuQNKotOrN8I1LOJwZmhsoVLiJkO/KdYE+HvJkJMcYr07/R54H9jVlpNMKVv/1F2Rs76gi +JUmTtt8AF9pYfl3uxRuw0dFfIRDH+fO6AgonB8Xx1sfT4PsJYGw= +-----END CERTIFICATE----- + +Amazon Root CA 3 +================ +-----BEGIN CERTIFICATE----- +MIIBtjCCAVugAwIBAgITBmyf1XSXNmY/Owua2eiedgPySjAKBggqhkjOPQQDAjA5MQswCQYDVQQG +EwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24gUm9vdCBDQSAzMB4XDTE1MDUy +NjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZ +MBcGA1UEAxMQQW1hem9uIFJvb3QgQ0EgMzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCmXp8ZB +f8ANm+gBG1bG8lKlui2yEujSLtf6ycXYqm0fc4E7O5hrOXwzpcVOho6AF2hiRVd9RFgdszflZwjr +Zt6jQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSrttvXBp43 +rDCGB5Fwx5zEGbF4wDAKBggqhkjOPQQDAgNJADBGAiEA4IWSoxe3jfkrBqWTrBqYaGFy+uGh0Psc +eGCmQ5nFuMQCIQCcAu/xlJyzlvnrxir4tiz+OpAUFteMYyRIHN8wfdVoOw== +-----END CERTIFICATE----- + +Amazon Root CA 4 +================ +-----BEGIN CERTIFICATE----- +MIIB8jCCAXigAwIBAgITBmyf18G7EEwpQ+Vxe3ssyBrBDjAKBggqhkjOPQQDAzA5MQswCQYDVQQG +EwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24gUm9vdCBDQSA0MB4XDTE1MDUy +NjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZ +MBcGA1UEAxMQQW1hem9uIFJvb3QgQ0EgNDB2MBAGByqGSM49AgEGBSuBBAAiA2IABNKrijdPo1MN +/sGKe0uoe0ZLY7Bi9i0b2whxIdIA6GO9mif78DluXeo9pcmBqqNbIJhFXRbb/egQbeOc4OO9X4Ri +83BkM6DLJC9wuoihKqB1+IGuYgbEgds5bimwHvouXKNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNV +HQ8BAf8EBAMCAYYwHQYDVR0OBBYEFNPsxzplbszh2naaVvuc84ZtV+WBMAoGCCqGSM49BAMDA2gA +MGUCMDqLIfG9fhGt0O9Yli/W651+kI0rz2ZVwyzjKKlwCkcO8DdZEv8tmZQoTipPNU0zWgIxAOp1 +AE47xDqUEpHJWEadIRNyp4iciuRMStuW1KyLa2tJElMzrdfkviT8tQp21KW8EA== +-----END CERTIFICATE----- + +GlobalSign Root CA - R6 +======================= +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEgMB4GA1UECxMX +R2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2JhbFNpZ24xEzARBgNVBAMTCkds +b2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQxMjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9i +YWxTaWduIFJvb3QgQ0EgLSBSNjETMBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFs +U2lnbjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQss +grRIxutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1kZguSgMpE +3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxDaNc9PIrFsmbVkJq3MQbF +vuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJwLnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqM +PKq0pPbzlUoSB239jLKJz9CgYXfIWHSw1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+ +azayOeSsJDa38O+2HBNXk7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05O +WgtH8wY2SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/hbguy +CLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4nWUx2OVvq+aWh2IMP +0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpYrZxCRXluDocZXFSxZba/jJvcE+kN +b7gu3GduyYsRtYQUigAZcIN5kZeR1BonvzceMgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQE +AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNV +HSMEGDAWgBSubAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN +nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGtIxg93eFyRJa0 +lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr6155wsTLxDKZmOMNOsIeDjHfrY +BzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLjvUYAGm0CuiVdjaExUd1URhxN25mW7xocBFym +Fe944Hn+Xds+qkxV/ZoVqW/hpvvfcDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr +3TsTjxKM4kEaSHpzoHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB1 +0jZpnOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfspA9MRf/T +uTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+vJJUEeKgDu+6B5dpffItK +oZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+t +JDfLRVpOoERIyNiwmcUVhAn21klJwGW45hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= +-----END CERTIFICATE----- + +Entrust Root Certification Authority - G4 +========================================= +-----BEGIN CERTIFICATE----- +MIIGSzCCBDOgAwIBAgIRANm1Q3+vqTkPAAAAAFVlrVgwDQYJKoZIhvcNAQELBQAwgb4xCzAJBgNV +BAYTAlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9TZWUgd3d3LmVudHJ1c3Qu +bmV0L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAxNSBFbnRydXN0LCBJbmMuIC0gZm9yIGF1 +dGhvcml6ZWQgdXNlIG9ubHkxMjAwBgNVBAMTKUVudHJ1c3QgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1 +dGhvcml0eSAtIEc0MB4XDTE1MDUyNzExMTExNloXDTM3MTIyNzExNDExNlowgb4xCzAJBgNVBAYT +AlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9TZWUgd3d3LmVudHJ1c3QubmV0 +L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAxNSBFbnRydXN0LCBJbmMuIC0gZm9yIGF1dGhv +cml6ZWQgdXNlIG9ubHkxMjAwBgNVBAMTKUVudHJ1c3QgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhv +cml0eSAtIEc0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAsewsQu7i0TD/pZJH4i3D +umSXbcr3DbVZwbPLqGgZ2K+EbTBwXX7zLtJTmeH+H17ZSK9dE43b/2MzTdMAArzE+NEGCJR5WIoV +3imz/f3ET+iq4qA7ec2/a0My3dl0ELn39GjUu9CH1apLiipvKgS1sqbHoHrmSKvS0VnM1n4j5pds +8ELl3FFLFUHtSUrJ3hCX1nbB76W1NhSXNdh4IjVS70O92yfbYVaCNNzLiGAMC1rlLAHGVK/XqsEQ +e9IFWrhAnoanw5CGAlZSCXqc0ieCU0plUmr1POeo8pyvi73TDtTUXm6Hnmo9RR3RXRv06QqsYJn7 +ibT/mCzPfB3pAqoEmh643IhuJbNsZvc8kPNXwbMv9W3y+8qh+CmdRouzavbmZwe+LGcKKh9asj5X +xNMhIWNlUpEbsZmOeX7m640A2Vqq6nPopIICR5b+W45UYaPrL0swsIsjdXJ8ITzI9vF01Bx7owVV +7rtNOzK+mndmnqxpkCIHH2E6lr7lmk/MBTwoWdPBDFSoWWG9yHJM6Nyfh3+9nEg2XpWjDrk4JFX8 +dWbrAuMINClKxuMrLzOg2qOGpRKX/YAr2hRC45K9PvJdXmd0LhyIRyk0X+IyqJwlN4y6mACXi0mW +Hv0liqzc2thddG5msP9E36EYxr5ILzeUePiVSj9/E15dWf10hkNjc0kCAwEAAaNCMEAwDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJ84xFYjwznooHFs6FRM5Og6sb9n +MA0GCSqGSIb3DQEBCwUAA4ICAQAS5UKme4sPDORGpbZgQIeMJX6tuGguW8ZAdjwD+MlZ9POrYs4Q +jbRaZIxowLByQzTSGwv2LFPSypBLhmb8qoMi9IsabyZIrHZ3CL/FmFz0Jomee8O5ZDIBf9PD3Vht +7LGrhFV0d4QEJ1JrhkzO3bll/9bGXp+aEJlLdWr+aumXIOTkdnrG0CSqkM0gkLpHZPt/B7NTeLUK +YvJzQ85BK4FqLoUWlFPUa19yIqtRLULVAJyZv967lDtX/Zr1hstWO1uIAeV8KEsD+UmDfLJ/fOPt +jqF/YFOOVZ1QNBIPt5d7bIdKROf1beyAN/BYGW5KaHbwH5Lk6rWS02FREAutp9lfx1/cH6NcjKF+ +m7ee01ZvZl4HliDtC3T7Zk6LERXpgUl+b7DUUH8i119lAg2m9IUe2K4GS0qn0jFmwvjO5QimpAKW +RGhXxNUzzxkvFMSUHHuk2fCfDrGA4tGeEWSpiBE6doLlYsKA2KSD7ZPvfC+QsDJMlhVoSFLUmQjA +JOgc47OlIQ6SwJAfzyBfyjs4x7dtOvPmRLgOMWuIjnDrnBdSqEGULoe256YSxXXfW8AKbnuk5F6G ++TaU33fD6Q3AOfF5u0aOq0NZJ7cguyPpVkAh7DE9ZapD8j3fcEThuk0mEDuYn/PIjhs4ViFqUZPT +kcpG2om3PVODLAgfi49T3f+sHw== +-----END CERTIFICATE----- + +GlobalSign Root R46 +=================== +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgISEdK7udcjGJ5AXwqdLdDfJWfRMA0GCSqGSIb3DQEBDAUAMEYxCzAJBgNV +BAYTAkJFMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMRwwGgYDVQQDExNHbG9iYWxTaWduIFJv +b3QgUjQ2MB4XDTE5MDMyMDAwMDAwMFoXDTQ2MDMyMDAwMDAwMFowRjELMAkGA1UEBhMCQkUxGTAX +BgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExHDAaBgNVBAMTE0dsb2JhbFNpZ24gUm9vdCBSNDYwggIi +MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCsrHQy6LNl5brtQyYdpokNRbopiLKkHWPd08Es +CVeJOaFV6Wc0dwxu5FUdUiXSE2te4R2pt32JMl8Nnp8semNgQB+msLZ4j5lUlghYruQGvGIFAha/ +r6gjA7aUD7xubMLL1aa7DOn2wQL7Id5m3RerdELv8HQvJfTqa1VbkNud316HCkD7rRlr+/fKYIje +2sGP1q7Vf9Q8g+7XFkyDRTNrJ9CG0Bwta/OrffGFqfUo0q3v84RLHIf8E6M6cqJaESvWJ3En7YEt +bWaBkoe0G1h6zD8K+kZPTXhc+CtI4wSEy132tGqzZfxCnlEmIyDLPRT5ge1lFgBPGmSXZgjPjHvj +K8Cd+RTyG/FWaha/LIWFzXg4mutCagI0GIMXTpRW+LaCtfOW3T3zvn8gdz57GSNrLNRyc0NXfeD4 +12lPFzYE+cCQYDdF3uYM2HSNrpyibXRdQr4G9dlkbgIQrImwTDsHTUB+JMWKmIJ5jqSngiCNI/on +ccnfxkF0oE32kRbcRoxfKWMxWXEM2G/CtjJ9++ZdU6Z+Ffy7dXxd7Pj2Fxzsx2sZy/N78CsHpdls +eVR2bJ0cpm4O6XkMqCNqo98bMDGfsVR7/mrLZqrcZdCinkqaByFrgY/bxFn63iLABJzjqls2k+g9 +vXqhnQt2sQvHnf3PmKgGwvgqo6GDoLclcqUC4wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYD +VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUA1yrc4GHqMywptWU4jaWSf8FmSwwDQYJKoZIhvcNAQEM +BQADggIBAHx47PYCLLtbfpIrXTncvtgdokIzTfnvpCo7RGkerNlFo048p9gkUbJUHJNOxO97k4Vg +JuoJSOD1u8fpaNK7ajFxzHmuEajwmf3lH7wvqMxX63bEIaZHU1VNaL8FpO7XJqti2kM3S+LGteWy +gxk6x9PbTZ4IevPuzz5i+6zoYMzRx6Fcg0XERczzF2sUyQQCPtIkpnnpHs6i58FZFZ8d4kuaPp92 +CC1r2LpXFNqD6v6MVenQTqnMdzGxRBF6XLE+0xRFFRhiJBPSy03OXIPBNvIQtQ6IbbjhVp+J3pZm +OUdkLG5NrmJ7v2B0GbhWrJKsFjLtrWhV/pi60zTe9Mlhww6G9kuEYO4Ne7UyWHmRVSyBQ7N0H3qq +JZ4d16GLuc1CLgSkZoNNiTW2bKg2SnkheCLQQrzRQDGQob4Ez8pn7fXwgNNgyYMqIgXQBztSvwye +qiv5u+YfjyW6hY0XHgL+XVAEV8/+LbzvXMAaq7afJMbfc2hIkCwU9D9SGuTSyxTDYWnP4vkYxboz +nxSjBF25cfe1lNj2M8FawTSLfJvdkzrnE6JwYZ+vj+vYxXX4M2bUdGc6N3ec592kD3ZDZopD8p/7 +DEJ4Y9HiD2971KE9dJeFt0g5QdYg/NA6s/rob8SKunE3vouXsXgxT7PntgMTzlSdriVZzH81Xwj3 +QEUxeCp6 +-----END CERTIFICATE----- + +GlobalSign Root E46 +=================== +-----BEGIN CERTIFICATE----- +MIICCzCCAZGgAwIBAgISEdK7ujNu1LzmJGjFDYQdmOhDMAoGCCqGSM49BAMDMEYxCzAJBgNVBAYT +AkJFMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMRwwGgYDVQQDExNHbG9iYWxTaWduIFJvb3Qg +RTQ2MB4XDTE5MDMyMDAwMDAwMFoXDTQ2MDMyMDAwMDAwMFowRjELMAkGA1UEBhMCQkUxGTAXBgNV +BAoTEEdsb2JhbFNpZ24gbnYtc2ExHDAaBgNVBAMTE0dsb2JhbFNpZ24gUm9vdCBFNDYwdjAQBgcq +hkjOPQIBBgUrgQQAIgNiAAScDrHPt+ieUnd1NPqlRqetMhkytAepJ8qUuwzSChDH2omwlwxwEwkB +jtjqR+q+soArzfwoDdusvKSGN+1wCAB16pMLey5SnCNoIwZD7JIvU4Tb+0cUB+hflGddyXqBPCCj +QjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQxCpCPtsad0kRL +gLWi5h+xEk8blTAKBggqhkjOPQQDAwNoADBlAjEA31SQ7Zvvi5QCkxeCmb6zniz2C5GMn0oUsfZk +vLtoURMMA/cVi4RguYv/Uo7njLwcAjA8+RHUjE7AwWHCFUyqqx0LMV87HOIAl0Qx5v5zli/altP+ +CAezNIm8BZ/3Hobui3A= +-----END CERTIFICATE----- +# Used by internal servers and TLS inspection +CheckPoint-RootCA +================= +-----BEGIN CERTIFICATE----- +MIIHEzCCBPugAwIBAgIQbXDyWvMpA5NGmWs/Roy/6DANBgkqhkiG9w0BAQwFADBC +MQswCQYDVQQGEwJJTDEXMBUGA1UEChMOQ2hlY2tQb2ludCBMVEQxGjAYBgNVBAMT +EUNoZWNrUG9pbnQtUm9vdENBMB4XDTIyMDYwMjExMTYzM1oXDTM0MDYwMjExMjYz +MVowQjELMAkGA1UEBhMCSUwxFzAVBgNVBAoTDkNoZWNrUG9pbnQgTFREMRowGAYD +VQQDExFDaGVja1BvaW50LVJvb3RDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC +AgoCggIBANaxBAFxmZ8QkD9G/5PcNlxJMIy7nnhzA3Q7CEt4KNFM2TAKJKGpyuTb +R2EU7er/44A1skXvkBco50F3hLscK9yzg5IPK2zBEfUp9FVC5FPmUEhKRAv+PDV1 +mSaWEo4BfdctiYz0qA6WIqRfdebDKKpclfpvuq8K27GQmQUGbXBCVA5alZ11ovbP +PyVbSgGm7jWHyGW/arMsMkNY0wHsoT36hDkHob8CcEKhwYckpBgbob4VsivO+VCm +CbdXuizeb0HzDGyI1bgXqk51Ae17KpeEu0PcbxGI6myC9smOBfWboogQycOvYgHB +QrkbQ5wgv1COiQZSwrhlwCtipBvlSP1AhhWcXGJe8SRlkFEgvlCe9jqnuP4OwDv6 +f9EtBHVFJJzMs0jXEgpBXihysUGhoySk783RyXpjsDPCbMlF9cOM3Zj7C0DIzpoh +i4F7ikNLZezHyLvm7v2Iv+5QGhokOnCk4MH1I9UY6WOZLM3NfXZidlrlo3tKQq91 +b/8dTvnKFr9X2Nvn0TsFJlqaXNOEdA7gfKNHO5/NC3QyhMsmpgjO4Nj5FWgZoTZ9 +CsEhG9A7L33kGucvQThO+TPq+Fj1Ee01GnXqGXXpYduG82GmzAfVHN+S2cidSGpv +jnSkAgvPx4aRbsFSfy/bZe/guzOlIWBdHz52uHT7rAMx3bqgekL/AgMBAAGjggID +MIIB/zALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUZOiT +q6L4EpWsFnbvha23XfUSY5swEAYJKwYBBAGCNxUBBAMCAQAwggGsBgNVHSAEggGj +MIIBnzCCAZsGCCoDBIsvQ1kFMIIBjTCCAVQGCCsGAQUFBwICMIIBRh6CAUIATABl +AGcAYQBsACAAUABvAGwAaQBjAHkAIABTAHQAYQB0AGUAbQBlAG4AdAAgAC0AIABU +AGgAaQBzACAAUABLAEkAIABpAHMAIABmAG8AcgAgAEMAaABlAGMAawBQAG8AaQBu +AHQAIABMAFQARAAgAHAAcgBvAGQAdQBjAHQAaQBvAG4AIABlAG4AdgBpAHIAbwBu +AG0AZQBuAHQALgAgAFIAZQBmAGUAcgAgAHQAbwAgAHQAaABlACAAYQBzAHMAbwBj +AGkAYQB0AGUAZAAgAEMAZQByAHQAaQBmAGkAYwBhAHQAZQAgAFAAcgBhAGMAdABp +AGMAZQAgAFMAdABhAHQAZQBtAGUAbgB0ACAAKABDAFAAUwApACAAZgBvAHIAIABt +AG8AcgBlACAAaQBuAGYAbwByAG0AYQB0AGkAbwBuMDMGCCsGAQUFBwIBFidodHRw +Oi8vY2VydHMucGtpLmNoZWNrcG9pbnQuY29tL0NQUy5wZGYwDQYJKoZIhvcNAQEM +BQADggIBAEuZDSw0PZEtml/JiXuBshfixutDDFvilTO9+4vA7Zwgxi00yA+t/TgA +35V49EKkrqAaFKkaBGcx5kildSbtZdox9Je+rbfKbQQoEAhOt1b5CkiaUGBsxsXr +fS3/NwZVQQRm3XbTCBlIFdH3VzNIyCi7/MWKW2VMIP9BmUjyO6WZZ/Ta+99oa56n +8zxKfGrksdLXDSYkL8OWj5F02FO82R56h6SYTR2fjiUoZi3C4PYpZHDqH6AlJFle +1i0ZfWpqgQW+PSDLG5yIvA2KTlrD5UAtpXijQLFXL4pnLxwDdCARlD/6d+5gLH3k +MCe2iY10HSq7zDlS1J9nkFhG1KeoOs6ay5u+LU5PkCbacFOiD8TD0/B7IOKbpge7 +5GL7u7LORRXfomqoQkSep+t8OTE1OPTvcJx2WUFsAcoLVyB2WOKr3a3hvI51KAAd +Cu0nw3knU/xxYJtCFt2R4+VcxloBRaZxkfwK4lCp67ViBvBaQKgjciJGqA516r9w ++E6QucuzEm3Pe0Yw3Xf8xtmJH7RcY6hqcWi73gQdVs/vAHemSfbfHsYwVMtqgHhh +nElKz91Ba2folEH3GHB6EPkcdBaVe0X5mlH3RaFwo+zVf2zThghp1yM5Wob2/2D+ +kMyfu0kwRqZEo7Bcu91zTw1NZPvmbZ3Idy+tEvWIUmvJFOcXA2J6 +-----END CERTIFICATE----- + +# Used by quay.io +DigiCert High Assurance EV Root CA +================================== +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j +ZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAwMFoXDTMxMTExMDAwMDAwMFowbDEL +MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3 +LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug +RVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm ++9S75S0tMqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTW +PNt0OKRKzE0lgvdKpVMSOO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEM +xChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFB +Ik5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQNAQTXKFx01p8VdteZOE3 +hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUeh10aUAsg +EsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQF +MAMBAf8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaA +FLE+w2kD+L9HAdSYJhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3Nec +nzyIZgYIVyHbIUf4KmeqvxgydkAQV8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6z +eM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFpmyPInngiK3BD41VHMWEZ71jF +hS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkKmNEVX58Svnw2 +Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe +vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep ++OkuE6N36B9K +-----END CERTIFICATE----- + +# Used by docker.io and docker.com +Starfield Class 2 Certification Authority +========================================= +-----BEGIN CERTIFICATE----- +MIIEDzCCAvegAwIBAgIBADANBgkqhkiG9w0BAQUFADBoMQswCQYDVQQGEwJVUzEl +MCMGA1UEChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMp +U3RhcmZpZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDQw +NjI5MTczOTE2WhcNMzQwNjI5MTczOTE2WjBoMQswCQYDVQQGEwJVUzElMCMGA1UE +ChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMpU3RhcmZp +ZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggEgMA0GCSqGSIb3 +DQEBAQUAA4IBDQAwggEIAoIBAQC3Msj+6XGmBIWtDBFk385N78gDGIc/oav7PKaf +8MOh2tTYbitTkPskpD6E8J7oX+zlJ0T1KKY/e97gKvDIr1MvnsoFAZMej2YcOadN ++lq2cwQlZut3f+dZxkqZJRRU6ybH838Z1TBwj6+wRir/resp7defqgSHo9T5iaU0 +X9tDkYI22WY8sbi5gv2cOj4QyDvvBmVmepsZGD3/cVE8MC5fvj13c7JdBmzDI1aa +K4UmkhynArPkPw2vCHmCuDY96pzTNbO8acr1zJ3o/WSNF4Azbl5KXZnJHoe0nRrA +1W4TNSNe35tfPe/W93bC6j67eA0cQmdrBNj41tpvi/JEoAGrAgEDo4HFMIHCMB0G +A1UdDgQWBBS/X7fRzt0fhvRbVazc1xDCDqmI5zCBkgYDVR0jBIGKMIGHgBS/X7fR +zt0fhvRbVazc1xDCDqmI56FspGowaDELMAkGA1UEBhMCVVMxJTAjBgNVBAoTHFN0 +YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAsTKVN0YXJmaWVsZCBD +bGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQEFBQADggEBAAWdP4id0ckaVaGsafPzWdqbAYcaT1epoXkJKtv3 +L7IezMdeatiDh6GX70k1PncGQVhiv45YuApnP+yz3SFmH8lU+nLMPUxA2IGvd56D +eruix/U0F47ZEUD0/CwqTRV/p2JdLiXTAAsgGh1o+Re49L2L7ShZ3U0WixeDyLJl +xy16paq8U4Zt3VekyvggQQto8PT7dL5WXXp59fkdheMtlb71cZBDzI0fmgAKhynp +VSJYACPq4xJDKVtHCN2MQWplBqjlIapBtJUhlbl90TSrE9atvNziPTnNvT51cKEY +WQPJIrSPnNVeKtelttQKbfi3QBFGmh95DmK/D5fs4C8fF5Q= +-----END CERTIFICATE----- + +# used by cdn03.quay.io (CA cert expires in 2025) +Baltimore CyberTrust Root +========================= +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIEAgAAuTANBgkqhkiG9w0BAQUFADBaMQswCQYDVQQGEwJJ +RTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJlclRydXN0MSIwIAYD +VQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTAwMDUxMjE4NDYwMFoX +DTI1MDUxMjIzNTkwMFowWjELMAkGA1UEBhMCSUUxEjAQBgNVBAoTCUJhbHRpbW9y +ZTETMBEGA1UECxMKQ3liZXJUcnVzdDEiMCAGA1UEAxMZQmFsdGltb3JlIEN5YmVy +VHJ1c3QgUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKMEuyKr +mD1X6CZymrV51Cni4eiVgLGw41uOKymaZN+hXe2wCQVt2yguzmKiYv60iNoS6zjr +IZ3AQSsBUnuId9Mcj8e6uYi1agnnc+gRQKfRzMpijS3ljwumUNKoUMMo6vWrJYeK +mpYcqWe4PwzV9/lSEy/CG9VwcPCPwBLKBsua4dnKM3p31vjsufFoREJIE9LAwqSu +XmD+tqYF/LTdB1kC1FkYmGP1pWPgkAx9XbIGevOF6uvUA65ehD5f/xXtabz5OTZy +dc93Uk3zyZAsuT3lySNTPx8kmCFcB5kpvcY67Oduhjprl3RjM71oGDHweI12v/ye +jl0qhqdNkNwnGjkCAwEAAaNFMEMwHQYDVR0OBBYEFOWdWTCCR1jMrPoIVDaGezq1 +BE3wMBIGA1UdEwEB/wQIMAYBAf8CAQMwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3 +DQEBBQUAA4IBAQCFDF2O5G9RaEIFoN27TyclhAO992T9Ldcw46QQF+vaKSm2eT92 +9hkTI7gQCvlYpNRhcL0EYWoSihfVCr3FvDB81ukMJY2GQE/szKN+OMY3EU/t3Wgx +jkzSswF07r51XgdIGn9w/xZchMB5hbgF/X++ZRGjD8ACtPhSNzkE1akxehi/oCr0 +Epn3o0WC4zxe9Z2etciefC7IpJ5OCBRLbf1wbWsaY71k5h+3zvDyny67G7fyUIhz +ksLi4xaNmjICq44Y3ekQEe5+NauQrz4wlHrQMz2nZQ/1/I6eYs9HRCwBXbsdtTLS +R9I4LtD+gdwyah617jzV/OeBHRnDJELqYzmp +-----END CERTIFICATE----- diff --git a/nodes/orchestration/package/cp-nano-cli.sh b/nodes/orchestration/package/cp-nano-cli.sh index 5173a0a..b98650e 100755 --- a/nodes/orchestration/package/cp-nano-cli.sh +++ b/nodes/orchestration/package/cp-nano-cli.sh @@ -256,7 +256,7 @@ usage() uninstall_option="-u, --uninstall" load_config_option="-lc, --load-config <$(get_installed_services '|')>" display_config_option="-dc, --display-config [$(get_installed_services '|')]" - cp_agent_info_option="--info [-wd|--with_dump|-u|--upload|-fms|--file_max_size|-an|--additional_name]" + cp_agent_info_option="--info [-wd|--with_dump|-fms|--file_max_size|-an|--additional_name]" display_policy_option="-dp, --display-policy" set_gradual_policy_option="-gp, --set-gradual-policy [access-control|http-manager] " delete_gradual_policy_option="-dg, --delete-gradual-policy [access-control|http-manager]" @@ -1198,7 +1198,6 @@ run_ai() # Initials - ra for arg; do if [ "$arg" = "--upload" ] || [ "$arg" = "-u" ]; then - ra_upload_to_fog=true shift continue elif [ "$arg" = "--verbose" ] || [ "$arg" = "-v" ]; then @@ -1210,14 +1209,6 @@ run_ai() # Initials - ra shift done - if [ "$ra_upload_to_fog" = "false" ]; then - printf "Would you like to upload the file to be inspected by the product support team? [y/n] " && read -r ra_should_upload - case $ra_should_upload in - [Yy] | [Yy][Ee][Ss]) ra_upload_to_fog=true ;; - *) ;; - esac - fi - ra_https_prefix="https://" ra_agent_details=$(cat ${FILESYSTEM_PATH}/$cp_nano_conf_location/agent_details.json) if echo "$ra_agent_details" | grep -q "Fog domain"; then diff --git a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc index 3d43480..9a756e9 100755 --- a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc +++ b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc @@ -98,6 +98,7 @@ enum class Service { IDA_IDN_BG, IOT_ACCESS_CONTROL, HORIZON_TELEMETRY, + ERM, COUNT }; @@ -177,6 +178,7 @@ getServiceString(const Service service) case (Service::IDA_IDN_BG): return "ida-idn-bg"; case (Service::IOT_ACCESS_CONTROL): return "iot-access-control"; case (Service::HORIZON_TELEMETRY): return "horizon-telemetry"; + case (Service::ERM): return "erm"; default: cerr << "Internal Error: the provided service (" @@ -380,6 +382,11 @@ getServiceConfig (const Service service) filesystem_path + "/conf/cp-nano-horizon-telemetry-debug-conf.json", log_files_path + "/nano_agent/cp-nano-horizon-telemetry.dbg" ); + case (Service::ERM): + return ServiceConfig( + filesystem_path + "/conf/cp-nano-erm-conf.json", + log_files_path + "/nano_agent/cp-nano-erm.dbg" + ); default: cerr << "Internal Error: the provided service (" diff --git a/nodes/orchestration/package/local-default-policy-v1beta2.yaml b/nodes/orchestration/package/local-default-policy-v1beta2.yaml new file mode 100755 index 0000000..04f95a5 --- /dev/null +++ b/nodes/orchestration/package/local-default-policy-v1beta2.yaml @@ -0,0 +1,101 @@ +# open-appsec default declarative configuration file +# based on schema version: "v1beta2" +# more information on declarative configuration: https://docs.openappsec.io + +apiVersion: v1beta2 + +policies: + default: + # start in detect-learn and move to prevent-learn based on learning progress + mode: detect-learn + threatPreventionPractices: [default-threat-prevention-practice] + accessControlPractices: [default-access-control-practice] + customResponses: default-web-user-response + triggers: [default-log-trigger] + sourceIdentifiers: "" + trustedSources: "" + exceptions: [] + specificRules: [] + +threatPreventionPractices: + - name: default-threat-prevention-practice + practiceMode: inherited + webAttacks: + overrideMode: inherited + minimumConfidence: high + intrusionPrevention: + # intrusion prevention (IPS) requires "Premium Edition" + overrideMode: inherited + maxPerformanceImpact: medium + minSeverityLevel: medium + minCveYear: 2016 + highConfidenceEventAction: inherited + mediumConfidenceEventAction: inherited + lowConfidenceEventAction: detect + fileSecurity: + # file security requires "Premium Edition" + overrideMode: inherited + minSeverityLevel: medium + highConfidenceEventAction: inherited + mediumConfidenceEventAction: inherited + lowConfidenceEventAction: detect + snortSignatures: + # you must specify snort signatures in configmap or file to activate snort inspection + overrideMode: inherited + configmap: [] + # relevant for deployments on kubernetes + # 0 or 1 configmaps supported in array + files: [] + # relevant for docker and linux embedded deployments + # 0 or 1 files supported in array + schemaValidation: # schema validation requires "Premium Edition" + overrideMode: inherited + configmap: [] + # relevant for deployments on kubernetes + # 0 or 1 configmaps supported in array + files: [] + # relevant for docker and linux embedded deployments + # 0 or 1 files supported in array + antiBot: # antibot requires "Premium Edition" + overrideMode: inherited + injectedUris: [] + validatedUris: [] + +accessControlPractices: + - name: default-access-control-practice + practiceMode: inherited + rateLimit: + # specify one or more rules below to use rate limiting + overrideMode: inherited + rules: [] + +logTriggers: + - name: default-log-trigger + accessControlLogging: + allowEvents: false + dropEvents: true + appsecLogging: + detectEvents: true + preventEvents: true + allWebRequests: false + extendedLogging: + urlPath: true + urlQuery: true + httpHeaders: false + requestBody: false + additionalSuspiciousEventsLogging: + enabled: true + minSeverity: high + responseBody: false + responseCode: true + + logDestination: + cloud: true + logToAgent: false + stdout: + format: json + +customResponses: + - name: default-web-user-response + mode: response-code-only + httpResponseCode: 403 diff --git a/nodes/orchestration/package/open-appsec-ctl.sh b/nodes/orchestration/package/open-appsec-ctl.sh index 4869e78..3449613 100644 --- a/nodes/orchestration/package/open-appsec-ctl.sh +++ b/nodes/orchestration/package/open-appsec-ctl.sh @@ -27,6 +27,7 @@ var_default_us_fog_address="inext-agents-us.cloud.ngen.checkpoint.com" var_default_au_fog_address="inext-agents-aus1.cloud.ngen.checkpoint.com" var_default_in_fog_address="inext-agents-ind1.cloud.ngen.checkpoint.com" var_default_ae_fog_address="inext-agents-ae.cloud.ngen.checkpoint.com" +var_default_ca_fog_address="inext-agents-ca.cloud.ngen.checkpoint.com" #NOTE: open-appsec-ctl only supports nano services with name of the format cp-nano- cp_nano_service_name_prefix="cp-nano" @@ -280,7 +281,7 @@ usage() uninstall_option="-u, --uninstall" load_config_option="-lc, --load-config <$(get_installed_services '|')>" display_config_option="-dc, --display-config [$(get_installed_services '|')]" - cp_agent_info_option="--info [-wd|--with_dump|-u|--upload|-fms|--file_max_size|-an|--additional_name]" + cp_agent_info_option="--info [-wd|--with_dump|-fms|--file_max_size|-an|--additional_name]" display_policy_option="-dp, --display-policy" set_gradual_policy_option="-gp, --set-gradual-policy [access-control|http-manager] " delete_gradual_policy_option="-dg, --delete-gradual-policy [access-control|http-manager]" @@ -1110,6 +1111,11 @@ set_proxy() # Initials - sp echo "Failed to set proxy. Error: ${sp_curl_output}" exit 1 fi + + for service in $(get_installed_services); do + run_load_settings "$service" + done + echo "Proxy successfully changed to $sp_proxy" } @@ -1312,7 +1318,6 @@ run_ai() # Initials - ra for arg; do if [ "$arg" = "--upload" ] || [ "$arg" = "-u" ]; then - ra_upload_to_fog=true shift continue elif [ "$arg" = "--verbose" ] || [ "$arg" = "-v" ]; then @@ -1324,14 +1329,6 @@ run_ai() # Initials - ra shift done - if [ "$ra_upload_to_fog" = "false" ]; then - printf "Would you like to upload the file to be inspected by the product support team? [y/n] " && read -r ra_should_upload - case $ra_should_upload in - [Yy] | [Yy][Ee][Ss]) ra_upload_to_fog=true ;; - *) ;; - esac - fi - ra_https_prefix="https://" ra_agent_details=$(cat ${FILESYSTEM_PATH}/$cp_nano_conf_location/agent_details.json) if echo "$ra_agent_details" | grep -q "Fog domain"; then @@ -1563,11 +1560,15 @@ set_mode() in_prefix_uppercase="CP-IN-" ae_prefix="cp-ae-" ae_prefix_uppercase="CP-AE-" + ca_prefix="cp-ca-" + ca_prefix_uppercase="CP-CA-" if [ "${var_token#"$us_prefix"}" != "${var_token}" ] || [ "${var_token#"$us_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_us_fog_address" elif [ "${var_token#"$ae_prefix"}" != "${var_token}" ] || [ "${var_token#"$ae_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_ae_fog_address" + elif [ "${var_token#"$ca_prefix"}" != "${var_token}" ] || [ "${var_token#"$ca_prefix_uppercase"}" != "${var_token}" ]; then + var_fog_address="$var_default_ca_fog_address" elif [ "${var_token#$au_prefix}" != "${var_token}" ] || [ "${var_token#"$au_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_au_fog_address" elif [ "${var_token#$in_prefix}" != "${var_token}" ] || [ "${var_token#"$in_prefix_uppercase"}" != "${var_token}" ]; then diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index 1dee032..487a2a3 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -26,6 +26,7 @@ VS_ID="" VS_LIB_SUB_FOLDER= is_wlp_orchestration="false" +is_static_orchestration="false" ORCHESTRATION_EXE_SOURCE_PATH="./bin/orchestration_comp" NGINX_METADAT_EXTRACTOR_PATH="./scripts/cp-nano-makefile-generator.sh" @@ -54,6 +55,8 @@ var_default_us_fog_address="https://inext-agents-us.cloud.ngen.checkpoint.com" var_default_au_fog_address="https://inext-agents-aus1.cloud.ngen.checkpoint.com" var_default_in_fog_address="https://inext-agents-ind1.cloud.ngen.checkpoint.com" var_default_ae_fog_address="https://inext-agents-ae.cloud.ngen.checkpoint.com" +var_default_ca_fog_address="https://inext-agents-ca.cloud.ngen.checkpoint.com" + var_fog_address= var_certs_dir= var_public_key= @@ -206,6 +209,9 @@ save_local_policy_config() } [ -f /etc/environment ] && . "/etc/environment" +if [ -n "${CP_ENV_WAAP_USE_HYPERSCAN}" ]; then + export WAAP_USE_HYPERSCAN=${CP_ENV_WAAP_USE_HYPERSCAN} +fi if [ -n "${CP_ENV_FILESYSTEM}" ] ; then export FILESYSTEM_PATH=$CP_ENV_FILESYSTEM fi @@ -237,6 +243,9 @@ while true; do elif [ "$1" = "--wlpOrchestration" ]; then is_wlp_orchestration="true" ORCHESTRATION_EXE_SOURCE_PATH="./bin/wlpStandalone" + elif [ "$1" = "--staticOrchestration" ]; then + is_static_orchestration="true" + ORCHESTRATION_EXE_SOURCE_PATH="./bin/orchestration_comp_static" elif [ "$1" = "--arm32_rpi" ]; then var_arch="arm" var_arch_flag="--arm32_rpi" @@ -402,11 +411,15 @@ if [ "$RUN_MODE" = "install" ] && [ $var_offline_mode = false ]; then in_prefix_uppercase="CP-IN-" ae_prefix="cp-ae-" ae_prefix_uppercase="CP-AE-" + ca_prefix="cp-ca-" + ca_prefix_uppercase="CP-CA-" if [ "${var_token#"$us_prefix"}" != "${var_token}" ] || [ "${var_token#"$us_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_us_fog_address" elif [ "${var_token#"$ae_prefix"}" != "${var_token}" ] || [ "${var_token#"$ae_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_ae_fog_address" + elif [ "${var_token#"$ca_prefix"}" != "${var_token}" ] || [ "${var_token#"$ca_prefix_uppercase"}" != "${var_token}" ]; then + var_fog_address="$var_default_ca_fog_address" elif [ "${var_token#$au_prefix}" != "${var_token}" ] || [ "${var_token#"$au_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_au_fog_address" elif [ "${var_token#$in_prefix}" != "${var_token}" ] || [ "${var_token#"$in_prefix_uppercase"}" != "${var_token}" ]; then @@ -602,7 +615,7 @@ install_watchdog() if [ "$old_cp_nano_watchdog_md5" = "$new_cp_nano_watchdog_md5" ]; then # Watchdog did not changed cp_print "There is no update in watchdog. Everything is up to date. Reregistering services to be on the same side." - cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register $is_upgrade ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration $var_arch_flag" + cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register $is_upgrade ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME} $var_arch_flag" if [ "$IS_K8S_ENV" = "true" ]; then cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh" fi @@ -619,7 +632,7 @@ install_watchdog() cp_exec "chmod 700 ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/revert_orchestrator_version.sh" cp_exec "touch ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/wd.services" - cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register $is_upgrade ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration $var_arch_flag" + cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register $is_upgrade ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME} $var_arch_flag" if [ "$IS_K8S_ENV" = "true" ]; then cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh" fi @@ -852,7 +865,7 @@ copy_orchestration_executable() cp_exec "ln -s /ext/appsec/local_policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml" else if [ ! -f ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml ]; then - cp_copy local-default-policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml + cp_copy local-default-policy-v1beta2.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml fi fi } @@ -991,8 +1004,13 @@ install_orchestration() cp_copy smb_egg/nano-egg-internal /opt/fw1/bin/nano-egg-internal chmod +x /opt/fw1/bin/nano-egg-internal fi - ${INSTALL_COMMAND} lib/*.so* ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/ - ${INSTALL_COMMAND} lib/boost/*.so* ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/ + + if [ "$is_static_orchestration" != "true" ]; then + ${INSTALL_COMMAND} lib/*.so* ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/ + ${INSTALL_COMMAND} lib/boost/*.so* ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/ + else + cp_print "Skipping shared library installation for static orchestration" ${FORCE_STDOUT} + fi if [ $var_compact_mode = true ]; then [ -f /etc/environment ] && . "/etc/environment" @@ -1008,7 +1026,7 @@ install_orchestration() cp_exec "rm -f $FILESYSTEM_PATH/$CONF_PATH/custom_policy.cfg" fi - if command -v ldconfig &>/dev/null; then + if [ "$is_static_orchestration" != "true" ] && command -v ldconfig &>/dev/null; then cp_exec "ldconfig" ${FORCE_STDOUT} fi cp_print "Copy cp-agent-info tool" @@ -1032,7 +1050,7 @@ install_orchestration() cp_print "\nStarting upgrading of open-appsec Nano Agent [$INSTALLATION_TIME]" ${FORCE_STDOUT} install_cp_nano_ctl add_uninstall_script - cp_exec "cp -f certificate/ngen.body.crt ${FILESYSTEM_PATH}/${CERTS_PATH}/fog.pem" + cp_exec "cp -f certificate/inext-ca-bundle.crt ${FILESYSTEM_PATH}/${CERTS_PATH}/fog.pem" if [ -n "${OTP_TOKEN}" ]; then cp_print "Saving authentication token to file" @@ -1079,7 +1097,7 @@ install_orchestration() update_cloudguard_appsec_manifest upgrade_conf_if_needed - cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration $var_arch_flag" + cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME} $var_arch_flag" if [ "$IS_K8S_ENV" = "true" ]; then cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh" fi @@ -1104,19 +1122,6 @@ install_orchestration() cp_print "Upgrade completed successfully" ${FORCE_STDOUT} - if [ -f /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} ]; then - cat "/etc/systemd/system/${NANO_AGENT_SERVICE_FILE}" | grep -q "EnvironmentFile=/etc/environment" - result=$? - - if [ $var_container_mode = false ] && [ $result -eq 0 ]; then - sed -i "$ d" /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} - echo "EnvironmentFile=/etc/environment" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} - echo >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} - check_and_run_restorecon "/etc/systemd/system/${NANO_AGENT_SERVICE_FILE}" - cp_exec "systemctl daemon-reload" - cp_exec "systemctl restart nano_agent" - fi - fi exit 0 fi @@ -1214,11 +1219,11 @@ install_orchestration() cp_print "Run Orchestration nano service in offline mode" ${FORCE_STDOUT} elif [ $var_hybrid_mode = true ]; then cp_print "Run Orchestration nano service in hybrid mode" ${FORCE_STDOUT} - cp_copy certificate/ngen.body.crt ${FILESYSTEM_PATH}/${CERTS_PATH}/fog.pem + cp_copy certificate/inext-ca-bundle.crt ${FILESYSTEM_PATH}/${CERTS_PATH}/fog.pem save_local_policy_config else - cp_copy certificate/ngen.body.crt ${FILESYSTEM_PATH}/${CERTS_PATH}/fog.pem + cp_copy certificate/inext-ca-bundle.crt ${FILESYSTEM_PATH}/${CERTS_PATH}/fog.pem fi cp_exec "chmod 600 ${FILESYSTEM_PATH}/${ORCHESTRATION_CONF_FILE}" @@ -1312,7 +1317,7 @@ run_pre_install_test() run_post_install_test() { - if [ $var_is_alpine = false ]; then + if [ "$var_is_alpine" = "false" ] && [ "$is_static_orchestration" = "false" ]; then if [ ! -f ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/libboost_chrono.so* ]; then cp_print "Error, libboost_chrono .so file is missing" ${FORCE_STDOUT} exit 1 diff --git a/nodes/prometheus/CMakeLists.txt b/nodes/prometheus/CMakeLists.txt index 7f0ca0e..33e637a 100755 --- a/nodes/prometheus/CMakeLists.txt +++ b/nodes/prometheus/CMakeLists.txt @@ -5,7 +5,6 @@ add_executable(prometheus main.cc) target_link_libraries(prometheus -Wl,--start-group ${COMMON_LIBRARIES} - generic_rulebase generic_rulebase_evaluators ip_utilities @@ -15,6 +14,9 @@ target_link_libraries(prometheus prometheus_comp http_transaction_data -Wl,--end-group + -Wl,--no-as-needed + ${Brotli_LIBRARIES} + -Wl,--as-needed ) add_dependencies(prometheus ngen_core) diff --git a/unit_test.cmake b/unit_test.cmake index 6eacca4..996a1bb 100644 --- a/unit_test.cmake +++ b/unit_test.cmake @@ -2,7 +2,7 @@ enable_testing() function(add_unit_test ut_name ut_sources use_libs) add_executable(${ut_name} ${ut_sources}) - target_link_libraries(${ut_name} -Wl,--start-group ${use_libs} version debug_is report cptest pthread packet singleton environment metric event_is buffers rest config compression_utils z ${GTEST_BOTH_LIBRARIES} gmock boost_regex pthread dl -Wl,--end-group) + target_link_libraries(${ut_name} -Wl,--start-group ${use_libs} version debug_is report cptest pthread packet singleton environment metric event_is buffers rest config compression_utils z ${GTEST_BOTH_LIBRARIES} gmock boost_regex pthread dl ${Brotli_LIBRARIES} -Wl,--end-group) add_test(NAME ${ut_name} COMMAND ${ut_name}