From 6d67818a94db2aec42a936708da1fd0f2d9f7fcb Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Wed, 31 Jan 2024 17:34:53 +0000 Subject: [PATCH] Jan_31_2024-Dev --- .../core/include/common_is/kdebug_flags.h | 1 + .../nginx_attachment/nginx_attachment.cc | 32 +- .../nginx_attachment_opaque.h | 5 + components/generic_rulebase/match_query.cc | 16 + components/generic_rulebase/zones_config.cc | 4 +- .../health_check_manager.cc | 9 +- .../health_check_manager_ut/CMakeLists.txt | 2 +- .../health_check_manager_ut.cc | 23 +- components/include/WaapEnums.h | 9 + .../include/generic_rulebase/match_query.h | 2 + components/include/geo_location.h | 24 + components/include/http_geo_filter.h | 35 + components/include/i_details_resolver.h | 2 +- components/include/i_downloader.h | 4 + components/include/i_geo_location.h | 22 + components/include/i_oa_schema_updater.h | 37 + components/include/i_update_communication.h | 5 +- components/include/i_waap_telemetry.h | 4 + components/include/manifest_diff_calculator.h | 13 +- components/include/manifest_handler.h | 17 +- components/include/oas_updater_entry_saver.h | 4 + .../orchestrator/rest_api/get_resource_file.h | 4 +- components/include/package.h | 6 +- components/include/rate_limit_config.h | 39 +- components/include/report_messaging.h | 12 +- components/include/service_controller.h | 1 + components/include/telemetry.h | 73 + components/packet/packet_ut/CMakeLists.txt | 2 +- .../report_messaging/report_messaging.cc | 14 +- .../report_messaging_ut/CMakeLists.txt | 2 +- .../report_messaging_ut.cc | 42 +- components/security_apps/CMakeLists.txt | 1 + .../http_geo_filter/CMakeLists.txt | 1 + .../http_geo_filter/geo_config.h | 90 + .../http_geo_filter/http_geo_filter.cc | 347 ++++ .../security_apps/ips/ips_ut/CMakeLists.txt | 2 +- .../layer_7_access_control_ut/CMakeLists.txt | 2 +- .../layer_7_access_control_ut.cc | 22 +- .../access_control_practice.cc | 9 +- .../include/k8s_policy_utils.h | 1 - .../include/local_policy_common.h | 43 +- .../local_policy_mgmt_gen/k8s_policy_utils.cc | 88 +- .../new_appsec_policy_crd_parser.cc | 11 +- .../new_custom_response.cc | 19 +- .../local_policy_mgmt_gen/new_exceptions.cc | 15 +- .../local_policy_mgmt_gen/new_log_trigger.cc | 16 +- .../local_policy_mgmt_gen/new_practice.cc | 87 +- .../new_trusted_sources.cc | 15 +- .../policy_maker_utils.cc | 55 +- .../rules_config_section.cc | 8 +- .../local_policy_mgmt_gen/settings_section.cc | 5 +- .../details_resolver/details_resolver.cc | 10 +- .../checkpoint_product_handlers.h | 19 +- .../details_resolver_impl.h | 37 +- .../details_resolving_handler.cc | 22 +- .../orchestration/downloader/curl_client.cc | 16 +- .../orchestration/downloader/curl_client.h | 2 +- .../orchestration/downloader/downloader.cc | 39 +- .../downloader/downloader_ut/downloader_ut.cc | 81 +- .../orchestration/downloader/http_client.cc | 4 +- .../orchestration/downloader/http_client.h | 2 - .../orchestration/downloader/https_client.cc | 4 +- .../health_check_ut/CMakeLists.txt | 2 +- .../orchestration/include/fog_communication.h | 5 +- .../orchestration/include/get_status_rest.h | 1 - .../include/hybrid_communication.h | 5 +- .../include/local_communication.h | 5 +- .../include/mock/mock_details_resolver.h | 2 +- .../include/mock/mock_downloader.h | 10 + .../manifest_controller.cc | 125 +- .../manifest_controller_ut.cc | 395 +++- .../manifest_diff_calculator.cc | 79 +- .../manifest_controller/manifest_handler.cc | 234 +-- .../modules/modules_ut/package_ut.cc | 10 +- .../orchestration/modules/package.cc | 13 +- .../orchestration/orchestration_comp.cc | 88 +- .../orchestration_tools.cc | 22 +- .../orchestration_tools_ut.cc | 14 +- .../orchestration_multitenant_ut.cc | 5 +- .../orchestration_ut/orchestration_ut.cc | 45 +- .../service_controller/service_controller.cc | 65 +- .../update_communication/fog_authenticator.cc | 51 +- .../update_communication/fog_communication.cc | 44 +- .../hybrid_communication.cc | 68 +- .../local_communication.cc | 7 +- .../update_communication.cc | 4 +- .../local_communication_ut.cc | 6 +- .../security_apps/rate_limit/rate_limit.cc | 237 ++- .../rate_limit/rate_limit_config.cc | 123 +- .../security_apps/waap/include/i_serialize.h | 68 +- .../reputation/reputation_features_agg.cc | 10 +- .../waap/reputation/reputation_features_agg.h | 1 + .../waap/waap_clib/ConfidenceCalculator.cc | 9 +- .../waap/waap_clib/DeepParser.cc | 29 +- .../waap/waap_clib/IndicatorsFilterBase.h | 1 - .../waap/waap_clib/IndicatorsFiltersManager.h | 1 - .../security_apps/waap/waap_clib/ParserGql.cc | 18 +- .../security_apps/waap/waap_clib/ParserGql.h | 17 +- .../waap/waap_clib/ParserJson.cc | 9 +- .../security_apps/waap/waap_clib/ParserJson.h | 12 +- .../waap/waap_clib/ScannerDetector.cc | 5 +- .../waap/waap_clib/Serializator.cc | 54 +- .../security_apps/waap/waap_clib/Telemetry.cc | 163 +- .../waap_clib/TrustedSourcesConfidence.cc | 9 +- .../waap/waap_clib/TuningDecision.cc | 4 +- .../waap/waap_clib/TuningDecisions.h | 33 +- .../waap/waap_clib/WaapConfigBase.cc | 2 + .../waap/waap_clib/WaapOverrideFunctor.cc | 2 +- .../waap/waap_clib/WaapScanner.cc | 1 - .../waap/waap_clib/WaapScanner.h | 1 - .../waap/waap_clib/Waf2Engine.cc | 103 +- .../security_apps/waap/waap_clib/Waf2Engine.h | 4 - .../waap/waap_clib/Waf2EngineGetters.cc | 1 + .../security_apps/waap/waap_clib/Waf2Util.cc | 28 +- .../security_apps/waap/waap_clib/Waf2Util.h | 1 + .../waap/waap_clib/events_for_oa_schema.h | 0 .../waap/waap_clib/oasu_key_types.h | 20 + .../security_apps/waap/waap_component_impl.cc | 9 +- .../security_apps/waap/waap_component_impl.h | 1 - components/signal_handler/signal_handler.cc | 10 +- components/utils/CMakeLists.txt | 1 + components/utils/geo_location/CMakeLists.txt | 1 + components/utils/geo_location/geo_location.cc | 194 ++ .../utils/keywords/keywords_ut/CMakeLists.txt | 2 +- core/CMakeLists.txt | 7 +- .../agent_core_utilities.cc | 0 .../agent_core_utilities_ut/CMakeLists.txt | 2 +- core/agent_details/agent_details.cc | 4 +- .../agent_details_reporter.cc | 38 +- .../agent_details_reporter_ut/CMakeLists.txt | 2 +- .../agent_details_reporter_ut.cc | 337 ++-- core/buffers/buffer.cc | 0 core/buffers/buffers_ut/CMakeLists.txt | 2 +- core/buffers/char_iterator.cc | 0 core/buffers/data_container.cc | 0 core/buffers/segment.cc | 0 core/compression/compression_utils.cc | 2 +- .../compression_utils_ut.cc | 0 core/config/config.cc | 67 +- core/config/include/profile_settings.h | 0 core/connkey/connkey.cc | 0 core/core_ut/maybe_res_ut.cc | 9 + core/cptest/cptest.cc | 0 core/cptest/cptest_data_buf.cc | 0 core/cptest/cptest_tcppacket.cc | 0 core/cptest/cptest_ut/cptest_packet_ut.cc | 0 core/cpu/cpu.cc | 0 core/cpu/cpu_ut/CMakeLists.txt | 8 +- core/cpu/cpu_ut/cpu_ut.cc | 0 core/debug_is/debug.cc | 8 +- core/debug_is/debug_is_ut/CMakeLists.txt | 2 +- core/debug_is/debug_is_ut/debug_ut.cc | 26 +- core/debug_is/debug_streams.cc | 5 +- core/encryptor/cpnano_base64/base64.cc | 0 core/encryptor/cpnano_base64/base64.h | 0 core/encryptor/cpnano_base64/cpnano_base64.cc | 0 core/encryptor/encryptor_ut/CMakeLists.txt | 2 +- core/environment/environment.cc | 29 + .../environment/environment_ut/CMakeLists.txt | 2 +- .../environment_ut/base_evaluators_ut.cc | 0 core/environment/environment_ut/context_ut.cc | 0 core/environment/environment_ut/parsing_ut.cc | 0 core/environment/environment_ut/span_ut.cc | 0 core/environment/environment_ut/trace_ut.cc | 0 core/environment/environment_ut/tracing_ut.cc | 2 - core/environment/span.cc | 0 core/environment/trace.cc | 0 core/include/attachments/compression_utils.h | 0 .../attachments/nginx_attachment_common.h | 0 core/include/attachments/shmem_ipc.h | 0 core/include/general/buffer.h | 0 core/include/general/buffer/char_iterator.h | 0 core/include/general/buffer/data_container.h | 0 .../include/general/buffer/helper_functions.h | 0 core/include/general/buffer/internal_ptr.h | 0 core/include/general/buffer/segment.h | 0 core/include/general/c_common/network_defs.h | 0 .../general/c_common/networking_headers.h | 0 core/include/general/cptest.h | 0 .../include/general/cptest/cptest_tcppacket.h | 0 core/include/general/debug.h | 0 core/include/general/intelligence_comp_v2.h | 0 core/include/general/scope_exit.h | 4 +- core/include/general/time_proxy.h | 0 core/include/internal/http_encoder.h | 95 - core/include/internal/ioctl_is.h | 0 .../internal/mainloop/mainloop_metric.h | 0 .../{proto_message_comp.h => messaging.h} | 53 +- .../messaging_buffer/bucket_manager.h | 58 - .../internal/messaging_buffer/event_queue.h | 125 -- core/include/internal/shell_cmd.h | 0 core/include/internal/trap_handler.h | 0 .../services_sdk/interfaces/i_agent_details.h | 0 core/include/services_sdk/interfaces/i_cpu.h | 0 .../services_sdk/interfaces/i_environment.h | 1 + .../services_sdk/interfaces/i_failopen.h | 0 .../interfaces/i_health_check_manager.h | 0 .../interfaces/i_intelligence_is_v2.h | 39 +- .../include/services_sdk/interfaces/i_ioctl.h | 0 .../services_sdk/interfaces/i_messaging.h | 220 +- .../interfaces/i_proxy_configuration.h | 12 +- .../services_sdk/interfaces/i_shell_cmd.h | 0 .../interfaces/i_signal_handler.h | 0 .../services_sdk/interfaces/i_socket_is.h | 0 .../services_sdk/interfaces/i_trap_handler.h | 0 .../intelligence_is_v2/asset_source_v2.h | 0 .../intelligence_is_v2/asset_source_v2_impl.h | 0 .../intelligence_query_v2.h | 0 .../intelligence_query_v2_impl.h | 0 .../intelligence_types_v2.h | 5 +- .../intelligence_is_v2/query_filter_v2.h | 11 +- .../intelligence_is_v2/query_request_v2.h | 15 + .../intelligence_is_v2/query_response_v2.h | 0 .../query_response_v2_impl.h | 0 .../requested_attributes_v2.h | 0 .../interfaces/messaging/http_response.h | 45 + .../interfaces/messaging/interface_impl.h | 148 ++ .../{http_core.h => messaging_enums.h} | 90 +- .../interfaces/messaging/messaging_metadata.h | 198 ++ .../interfaces/mock/mock_agent_details.h | 0 .../services_sdk/interfaces/mock/mock_cpu.h | 0 .../interfaces/mock/mock_environment.h | 1 + .../interfaces/mock/mock_logging.h | 0 .../interfaces/mock/mock_messaging.h | 92 +- .../mock/mock_proxy_configuration.h | 23 + .../interfaces/mock/mock_shell_cmd.h | 0 .../interfaces/mock/mock_socket_is.h | 0 .../services_sdk/interfaces/mock/mock_table.h | 0 .../interfaces/mock/mock_time_get.h | 0 .../services_sdk/resources/agent_details.h | 2 +- .../services_sdk/resources/component.h | 0 .../component_is/components_list_impl.h | 6 +- .../component_is/node_components_impl.h | 0 .../services_sdk/resources/components_list.h | 0 .../resources/config/i_config_iterator.h | 0 core/include/services_sdk/resources/cpu.h | 0 .../services_sdk/resources/cpu/cpu_metric.h | 0 .../resources/cpu/failopen_mode_status.h | 0 .../services_sdk/resources/debug_flags.h | 14 +- .../services_sdk/resources/environment/span.h | 0 .../resources/environment/trace.h | 0 .../resources/environment/tracing_metric.h | 0 .../services_sdk/resources/generic_metric.h | 0 .../health_check_status/health_check_status.h | 0 .../resources/intelligence_filter.h | 0 .../resources/intelligence_is/data_string.h | 0 .../intelligence_is/read_attribute_impl.h | 0 .../intelligence_is_v2/data_string_v2.h | 0 .../intelligence_is_v2/data_vector_v2.h | 0 .../read_attribute_v2_impl.h | 0 .../services_sdk/resources/log_generator.h | 0 .../resources/memory_consumption.h | 0 .../resources/metric/all_metric_event.h | 0 .../services_sdk/resources/metric/average.h | 0 .../services_sdk/resources/metric/counter.h | 0 .../resources/metric/last_reported_value.h | 0 .../services_sdk/resources/metric/max.h | 0 .../resources/metric/metric_calc.h | 0 .../resources/metric/metric_map.h | 18 +- .../services_sdk/resources/metric/min.h | 0 .../resources/metric/top_values.h | 0 .../services_sdk/resources/read_attribute.h | 0 .../resources/read_attribute_v2.h | 0 .../resources/report/base_field.h | 0 .../services_sdk/resources/report/log_rest.h | 0 .../services_sdk/resources/report/report.h | 0 .../resources/report/report_bulks.h | 0 .../resources/report/report_enums.h | 4 +- .../utilities/agent_core_utilities.h | 0 core/include/services_sdk/utilities/connkey.h | 0 .../utilities/customized_cereal_map.h | 0 .../utilities/customized_cereal_multimap.h | 0 core/include/services_sdk/utilities/flags.h | 4 +- core/include/services_sdk/utilities/sasal.h | 28 - .../services_sdk/utilities/socket_is.h | 0 .../instance_awareness_ut/CMakeLists.txt | 2 +- .../intelligence_comp_v2.cc | 58 +- .../intelligence_is_v2_ut/CMakeLists.txt | 2 +- .../intelligence_comp_v2_ut.cc | 55 +- .../intelligence_is_v2_ut/invalidation_ut.cc | 48 +- .../query_request_v2_ut.cc | 46 + .../query_response_v2_ut.cc | 0 .../intelligence_types_v2.cc | 1 + core/intelligence_is_v2/query_filter_v2.cc | 21 + core/intelligence_is_v2/query_request_v2.cc | 24 +- .../requested_attributes_v2.cc | 0 core/logging/cef_stream.cc | 0 core/logging/debug_stream.cc | 0 core/logging/file_stream.cc | 0 core/logging/fog_stream.cc | 15 +- core/logging/k8s_svc_stream.cc | 42 +- core/logging/log_generator.cc | 0 core/logging/log_streams.h | 1 - core/logging/logging.cc | 5 +- core/logging/logging_metric.h | 0 core/logging/logging_ut/CMakeLists.txt | 2 +- core/logging/logging_ut/logging_ut.cc | 100 +- core/logging/syslog_stream.cc | 0 core/mainloop/mainloop.cc | 10 +- core/mainloop/mainloop_ut/CMakeLists.txt | 2 +- core/mainloop/mainloop_ut/mainloop_ut.cc | 15 +- core/memory_consumption/memory_consumption.cc | 0 .../memory_consumption_ut/CMakeLists.txt | 14 +- .../memory_consumption_ut.cc | 0 core/memory_consumption/memory_metric.h | 0 core/message/CMakeLists.txt | 4 - core/message/http_core.cc | 107 - core/message/http_decoder.cc | 294 --- core/message/http_decoder.h | 54 - core/message/http_encoder.cc | 171 -- core/message/message.cc | 1793 ----------------- core/message/message_metric.h | 58 - core/messaging/CMakeLists.txt | 12 + core/messaging/connection/CMakeLists.txt | 2 + core/messaging/connection/connection.cc | 694 +++++++ core/messaging/connection/connection_comp.cc | 157 ++ .../connection/connection_ut/CMakeLists.txt | 5 + .../connection_ut/connection_comp_ut.cc | 298 +++ core/messaging/include/buffered_message.h | 63 + core/messaging/include/connection.h | 75 + .../include/connection_comp.h} | 38 +- core/messaging/include/http_request.h | 68 + .../include}/http_request_event.h | 4 + .../include}/interfaces/i_messaging_buffer.h | 30 +- .../interfaces/i_messaging_connection.h | 42 + .../include/messaging_buffer.h} | 23 +- core/messaging/include/messaging_comp.h | 101 + .../include/mocks/mock_messaging_buffer.h | 25 + .../include/mocks/mock_messaging_connection.h | 38 + core/messaging/include/response_parser.h | 34 + .../include}/smart_bio.h | 19 +- core/messaging/messaging.cc | 132 ++ .../messaging_buffer_comp/CMakeLists.txt | 2 + .../messaging_buffer_comp/buffered_message.cc | 128 ++ .../messaging_buffer_comp.cc | 544 +++++ .../messaging_buffer_comp_ut/CMakeLists.txt | 7 + .../messaging_buffer_comp_ut.cc | 264 +++ .../test_data/CMakeLists.txt | 2 + core/messaging/messaging_comp/CMakeLists.txt | 5 + core/messaging/messaging_comp/http_request.cc | 149 ++ .../messaging/messaging_comp/http_response.cc | 286 +++ .../messaging_comp/messaging_comp.cc | 320 +++ .../messaging_comp_ut/CMakeLists.txt | 5 + .../messaging_comp_ut/messaging_comp_ut.cc | 238 +++ .../tests_files/file_to_send.txt | 1 + core/messaging_buffer/CMakeLists.txt | 6 - core/messaging_buffer/bucket_manager.cc | 217 -- core/messaging_buffer/event_queue.cc | 846 -------- core/messaging_buffer/messaging_buffer.cc | 275 --- .../messaging_buffer_ut/CMakeLists.txt | 6 - .../messaging_buffer_ut.cc | 755 ------- core/metric/generic_metric.cc | 10 +- core/metric/metric_ut/CMakeLists.txt | 8 +- core/metric/metric_ut/metric_ut.cc | 124 +- core/report/report_ut/CMakeLists.txt | 2 +- core/report/tag_and_enum_management.cc | 4 +- core/rest/rest_server.cc | 1 + core/rest/rest_ut/CMakeLists.txt | 2 +- core/rest/rest_ut/rest_config_ut.cc | 71 + core/rest/rest_ut/rest_must_param_ut.cc | 0 core/rest/rest_ut/rest_schema_ut.cc | 65 +- core/shell_cmd/shell_cmd.cc | 0 core/shmem_ipc/shared_ipc_debug.h | 0 core/shmem_ipc/shared_ring_queue.h | 0 .../shmem_ipc_ut/shared_ring_queue_ut.cc | 0 core/shmem_ipc/shmem_ipc_ut/shmem_ipc_ut.cc | 0 core/singleton/singleton_ut/CMakeLists.txt | 2 +- core/socket_is/socket_is.cc | 0 core/tenant_manager/tenant_manager.cc | 71 +- .../time_proxy/time_proxy_ut/time_print_ut.cc | 0 core/version/version.cc | 0 core/version/version_ut/version_ut.cc | 0 nodes/orchestration/main.cc | 2 - .../package/cpnano_debug/cpnano_debug.cc | 9 + .../orchestration/package/open-appsec-ctl.sh | 32 +- .../package/orchestration_package.sh | 10 +- 376 files changed, 8101 insertions(+), 7064 deletions(-) create mode 100644 components/include/geo_location.h create mode 100644 components/include/http_geo_filter.h create mode 100644 components/include/i_geo_location.h create mode 100644 components/include/i_oa_schema_updater.h create mode 100644 components/include/oas_updater_entry_saver.h create mode 100644 components/security_apps/http_geo_filter/CMakeLists.txt create mode 100644 components/security_apps/http_geo_filter/geo_config.h create mode 100644 components/security_apps/http_geo_filter/http_geo_filter.cc create mode 100644 components/security_apps/waap/waap_clib/events_for_oa_schema.h create mode 100644 components/security_apps/waap/waap_clib/oasu_key_types.h create mode 100644 components/utils/geo_location/CMakeLists.txt create mode 100644 components/utils/geo_location/geo_location.cc mode change 100755 => 100644 core/agent_core_utilities/agent_core_utilities.cc mode change 100755 => 100644 core/agent_details/agent_details.cc mode change 100755 => 100644 core/buffers/buffer.cc mode change 100755 => 100644 core/buffers/char_iterator.cc mode change 100755 => 100644 core/buffers/data_container.cc mode change 100755 => 100644 core/buffers/segment.cc mode change 100755 => 100644 core/compression/compression_utils.cc mode change 100755 => 100644 core/compression/compression_utils_ut/compression_utils_ut.cc mode change 100755 => 100644 core/config/include/profile_settings.h mode change 100755 => 100644 core/connkey/connkey.cc mode change 100755 => 100644 core/core_ut/maybe_res_ut.cc mode change 100755 => 100644 core/cptest/cptest.cc mode change 100755 => 100644 core/cptest/cptest_data_buf.cc mode change 100755 => 100644 core/cptest/cptest_tcppacket.cc mode change 100755 => 100644 core/cptest/cptest_ut/cptest_packet_ut.cc mode change 100755 => 100644 core/cpu/cpu.cc mode change 100755 => 100644 core/cpu/cpu_ut/cpu_ut.cc mode change 100755 => 100644 core/debug_is/debug.cc mode change 100755 => 100644 core/debug_is/debug_is_ut/debug_ut.cc mode change 100755 => 100644 core/encryptor/cpnano_base64/base64.cc mode change 100755 => 100644 core/encryptor/cpnano_base64/base64.h mode change 100755 => 100644 core/encryptor/cpnano_base64/cpnano_base64.cc mode change 100755 => 100644 core/environment/environment_ut/base_evaluators_ut.cc mode change 100755 => 100644 core/environment/environment_ut/context_ut.cc mode change 100755 => 100644 core/environment/environment_ut/parsing_ut.cc mode change 100755 => 100644 core/environment/environment_ut/span_ut.cc mode change 100755 => 100644 core/environment/environment_ut/trace_ut.cc mode change 100755 => 100644 core/environment/environment_ut/tracing_ut.cc mode change 100755 => 100644 core/environment/span.cc mode change 100755 => 100644 core/environment/trace.cc mode change 100755 => 100644 core/include/attachments/compression_utils.h mode change 100755 => 100644 core/include/attachments/nginx_attachment_common.h mode change 100755 => 100644 core/include/attachments/shmem_ipc.h mode change 100755 => 100644 core/include/general/buffer.h mode change 100755 => 100644 core/include/general/buffer/char_iterator.h mode change 100755 => 100644 core/include/general/buffer/data_container.h mode change 100755 => 100644 core/include/general/buffer/helper_functions.h mode change 100755 => 100644 core/include/general/buffer/internal_ptr.h mode change 100755 => 100644 core/include/general/buffer/segment.h mode change 100755 => 100644 core/include/general/c_common/network_defs.h mode change 100755 => 100644 core/include/general/c_common/networking_headers.h mode change 100755 => 100644 core/include/general/cptest.h mode change 100755 => 100644 core/include/general/cptest/cptest_tcppacket.h mode change 100755 => 100644 core/include/general/debug.h mode change 100755 => 100644 core/include/general/intelligence_comp_v2.h mode change 100755 => 100644 core/include/general/time_proxy.h delete mode 100755 core/include/internal/http_encoder.h mode change 100755 => 100644 core/include/internal/ioctl_is.h mode change 100755 => 100644 core/include/internal/mainloop/mainloop_metric.h rename core/include/internal/{proto_message_comp.h => messaging.h} (76%) delete mode 100644 core/include/internal/messaging_buffer/bucket_manager.h delete mode 100644 core/include/internal/messaging_buffer/event_queue.h mode change 100755 => 100644 core/include/internal/shell_cmd.h mode change 100755 => 100644 core/include/internal/trap_handler.h mode change 100755 => 100644 core/include/services_sdk/interfaces/i_agent_details.h mode change 100755 => 100644 core/include/services_sdk/interfaces/i_cpu.h mode change 100755 => 100644 core/include/services_sdk/interfaces/i_failopen.h mode change 100755 => 100644 core/include/services_sdk/interfaces/i_health_check_manager.h mode change 100755 => 100644 core/include/services_sdk/interfaces/i_intelligence_is_v2.h mode change 100755 => 100644 core/include/services_sdk/interfaces/i_ioctl.h mode change 100755 => 100644 core/include/services_sdk/interfaces/i_messaging.h mode change 100755 => 100644 core/include/services_sdk/interfaces/i_shell_cmd.h mode change 100755 => 100644 core/include/services_sdk/interfaces/i_signal_handler.h mode change 100755 => 100644 core/include/services_sdk/interfaces/i_socket_is.h mode change 100755 => 100644 core/include/services_sdk/interfaces/i_trap_handler.h mode change 100755 => 100644 core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2.h mode change 100755 => 100644 core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2_impl.h mode change 100755 => 100644 core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h mode change 100755 => 100644 core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h mode change 100755 => 100644 core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h mode change 100755 => 100644 core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h mode change 100755 => 100644 core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h mode change 100755 => 100644 core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2.h mode change 100755 => 100644 core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2_impl.h mode change 100755 => 100644 core/include/services_sdk/interfaces/intelligence_is_v2/requested_attributes_v2.h create mode 100644 core/include/services_sdk/interfaces/messaging/http_response.h create mode 100644 core/include/services_sdk/interfaces/messaging/interface_impl.h rename core/include/services_sdk/interfaces/messaging/{http_core.h => messaging_enums.h} (53%) create mode 100644 core/include/services_sdk/interfaces/messaging/messaging_metadata.h mode change 100755 => 100644 core/include/services_sdk/interfaces/mock/mock_agent_details.h mode change 100755 => 100644 core/include/services_sdk/interfaces/mock/mock_cpu.h mode change 100755 => 100644 core/include/services_sdk/interfaces/mock/mock_logging.h mode change 100755 => 100644 core/include/services_sdk/interfaces/mock/mock_messaging.h create mode 100644 core/include/services_sdk/interfaces/mock/mock_proxy_configuration.h mode change 100755 => 100644 core/include/services_sdk/interfaces/mock/mock_shell_cmd.h mode change 100755 => 100644 core/include/services_sdk/interfaces/mock/mock_socket_is.h mode change 100755 => 100644 core/include/services_sdk/interfaces/mock/mock_table.h mode change 100755 => 100644 core/include/services_sdk/interfaces/mock/mock_time_get.h mode change 100755 => 100644 core/include/services_sdk/resources/agent_details.h mode change 100755 => 100644 core/include/services_sdk/resources/component.h mode change 100755 => 100644 core/include/services_sdk/resources/component_is/components_list_impl.h mode change 100755 => 100644 core/include/services_sdk/resources/component_is/node_components_impl.h mode change 100755 => 100644 core/include/services_sdk/resources/components_list.h mode change 100755 => 100644 core/include/services_sdk/resources/config/i_config_iterator.h mode change 100755 => 100644 core/include/services_sdk/resources/cpu.h mode change 100755 => 100644 core/include/services_sdk/resources/cpu/cpu_metric.h mode change 100755 => 100644 core/include/services_sdk/resources/cpu/failopen_mode_status.h mode change 100755 => 100644 core/include/services_sdk/resources/debug_flags.h mode change 100755 => 100644 core/include/services_sdk/resources/environment/span.h mode change 100755 => 100644 core/include/services_sdk/resources/environment/trace.h mode change 100755 => 100644 core/include/services_sdk/resources/environment/tracing_metric.h mode change 100755 => 100644 core/include/services_sdk/resources/generic_metric.h mode change 100755 => 100644 core/include/services_sdk/resources/health_check_status/health_check_status.h mode change 100755 => 100644 core/include/services_sdk/resources/intelligence_filter.h mode change 100755 => 100644 core/include/services_sdk/resources/intelligence_is/data_string.h mode change 100755 => 100644 core/include/services_sdk/resources/intelligence_is/read_attribute_impl.h mode change 100755 => 100644 core/include/services_sdk/resources/intelligence_is_v2/data_string_v2.h mode change 100755 => 100644 core/include/services_sdk/resources/intelligence_is_v2/data_vector_v2.h mode change 100755 => 100644 core/include/services_sdk/resources/intelligence_is_v2/read_attribute_v2_impl.h mode change 100755 => 100644 core/include/services_sdk/resources/log_generator.h mode change 100755 => 100644 core/include/services_sdk/resources/memory_consumption.h mode change 100755 => 100644 core/include/services_sdk/resources/metric/all_metric_event.h mode change 100755 => 100644 core/include/services_sdk/resources/metric/average.h mode change 100755 => 100644 core/include/services_sdk/resources/metric/counter.h mode change 100755 => 100644 core/include/services_sdk/resources/metric/last_reported_value.h mode change 100755 => 100644 core/include/services_sdk/resources/metric/max.h mode change 100755 => 100644 core/include/services_sdk/resources/metric/metric_calc.h mode change 100755 => 100644 core/include/services_sdk/resources/metric/min.h mode change 100755 => 100644 core/include/services_sdk/resources/metric/top_values.h mode change 100755 => 100644 core/include/services_sdk/resources/read_attribute.h mode change 100755 => 100644 core/include/services_sdk/resources/read_attribute_v2.h mode change 100755 => 100644 core/include/services_sdk/resources/report/base_field.h mode change 100755 => 100644 core/include/services_sdk/resources/report/log_rest.h mode change 100755 => 100644 core/include/services_sdk/resources/report/report.h mode change 100755 => 100644 core/include/services_sdk/resources/report/report_bulks.h mode change 100755 => 100644 core/include/services_sdk/resources/report/report_enums.h mode change 100755 => 100644 core/include/services_sdk/utilities/agent_core_utilities.h mode change 100755 => 100644 core/include/services_sdk/utilities/connkey.h mode change 100755 => 100644 core/include/services_sdk/utilities/customized_cereal_map.h mode change 100755 => 100644 core/include/services_sdk/utilities/customized_cereal_multimap.h delete mode 100755 core/include/services_sdk/utilities/sasal.h mode change 100755 => 100644 core/include/services_sdk/utilities/socket_is.h mode change 100755 => 100644 core/intelligence_is_v2/intelligence_comp_v2.cc mode change 100755 => 100644 core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc mode change 100755 => 100644 core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc mode change 100755 => 100644 core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc mode change 100755 => 100644 core/intelligence_is_v2/intelligence_types_v2.cc mode change 100755 => 100644 core/intelligence_is_v2/query_filter_v2.cc mode change 100755 => 100644 core/intelligence_is_v2/query_request_v2.cc mode change 100755 => 100644 core/intelligence_is_v2/requested_attributes_v2.cc mode change 100755 => 100644 core/logging/cef_stream.cc mode change 100755 => 100644 core/logging/debug_stream.cc mode change 100755 => 100644 core/logging/file_stream.cc mode change 100755 => 100644 core/logging/fog_stream.cc mode change 100755 => 100644 core/logging/log_generator.cc mode change 100755 => 100644 core/logging/log_streams.h mode change 100755 => 100644 core/logging/logging.cc mode change 100755 => 100644 core/logging/logging_metric.h mode change 100755 => 100644 core/logging/logging_ut/logging_ut.cc mode change 100755 => 100644 core/logging/syslog_stream.cc mode change 100755 => 100644 core/memory_consumption/memory_consumption.cc mode change 100755 => 100644 core/memory_consumption/memory_consumption_ut/memory_consumption_ut.cc mode change 100755 => 100644 core/memory_consumption/memory_metric.h delete mode 100644 core/message/CMakeLists.txt delete mode 100644 core/message/http_core.cc delete mode 100755 core/message/http_decoder.cc delete mode 100755 core/message/http_decoder.h delete mode 100755 core/message/http_encoder.cc delete mode 100755 core/message/message.cc delete mode 100755 core/message/message_metric.h create mode 100644 core/messaging/CMakeLists.txt create mode 100644 core/messaging/connection/CMakeLists.txt create mode 100644 core/messaging/connection/connection.cc create mode 100644 core/messaging/connection/connection_comp.cc create mode 100644 core/messaging/connection/connection_ut/CMakeLists.txt create mode 100644 core/messaging/connection/connection_ut/connection_comp_ut.cc create mode 100644 core/messaging/include/buffered_message.h create mode 100644 core/messaging/include/connection.h rename core/{include/internal/messaging_buffer.h => messaging/include/connection_comp.h} (56%) create mode 100644 core/messaging/include/http_request.h rename core/{include/internal/messaging_buffer => messaging/include}/http_request_event.h (98%) rename core/{include/services_sdk => messaging/include}/interfaces/i_messaging_buffer.h (57%) create mode 100644 core/messaging/include/interfaces/i_messaging_connection.h rename core/{message/i_message_decoder.h => messaging/include/messaging_buffer.h} (59%) create mode 100644 core/messaging/include/messaging_comp.h create mode 100644 core/messaging/include/mocks/mock_messaging_buffer.h create mode 100644 core/messaging/include/mocks/mock_messaging_connection.h create mode 100644 core/messaging/include/response_parser.h rename core/{message => messaging/include}/smart_bio.h (91%) create mode 100644 core/messaging/messaging.cc create mode 100644 core/messaging/messaging_buffer_comp/CMakeLists.txt create mode 100644 core/messaging/messaging_buffer_comp/buffered_message.cc create mode 100644 core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc create mode 100644 core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/CMakeLists.txt create mode 100644 core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc create mode 100644 core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/test_data/CMakeLists.txt create mode 100644 core/messaging/messaging_comp/CMakeLists.txt create mode 100644 core/messaging/messaging_comp/http_request.cc create mode 100644 core/messaging/messaging_comp/http_response.cc create mode 100644 core/messaging/messaging_comp/messaging_comp.cc create mode 100644 core/messaging/messaging_comp/messaging_comp_ut/CMakeLists.txt create mode 100644 core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc create mode 100644 core/messaging/messaging_comp/messaging_comp_ut/tests_files/file_to_send.txt delete mode 100644 core/messaging_buffer/CMakeLists.txt delete mode 100644 core/messaging_buffer/bucket_manager.cc delete mode 100644 core/messaging_buffer/event_queue.cc delete mode 100644 core/messaging_buffer/messaging_buffer.cc delete mode 100644 core/messaging_buffer/messaging_buffer_ut/CMakeLists.txt delete mode 100644 core/messaging_buffer/messaging_buffer_ut/messaging_buffer_ut.cc mode change 100755 => 100644 core/metric/generic_metric.cc mode change 100755 => 100644 core/metric/metric_ut/metric_ut.cc mode change 100755 => 100644 core/report/tag_and_enum_management.cc mode change 100755 => 100644 core/rest/rest_ut/rest_config_ut.cc mode change 100755 => 100644 core/rest/rest_ut/rest_must_param_ut.cc mode change 100755 => 100644 core/shell_cmd/shell_cmd.cc mode change 100755 => 100644 core/shmem_ipc/shared_ipc_debug.h mode change 100755 => 100644 core/shmem_ipc/shared_ring_queue.h mode change 100755 => 100644 core/shmem_ipc/shmem_ipc_ut/shared_ring_queue_ut.cc mode change 100755 => 100644 core/shmem_ipc/shmem_ipc_ut/shmem_ipc_ut.cc mode change 100755 => 100644 core/socket_is/socket_is.cc mode change 100755 => 100644 core/time_proxy/time_proxy_ut/time_print_ut.cc mode change 100755 => 100644 core/version/version.cc mode change 100755 => 100644 core/version/version_ut/version_ut.cc diff --git a/attachments/kernel_modules/core/include/common_is/kdebug_flags.h b/attachments/kernel_modules/core/include/common_is/kdebug_flags.h index 80691ce..c93a2de 100755 --- a/attachments/kernel_modules/core/include/common_is/kdebug_flags.h +++ b/attachments/kernel_modules/core/include/common_is/kdebug_flags.h @@ -33,5 +33,6 @@ DEFINE_KDEBUG_FLAG(kernelMetric) DEFINE_KDEBUG_FLAG(tproxy) DEFINE_KDEBUG_FLAG(tenantStats) DEFINE_KDEBUG_FLAG(uuidTranslation) +DEFINE_KDEBUG_FLAG(antibotResolver) #endif // DEFINE_KDEBUG_FLAG diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc index 438970c..6174c65 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc @@ -810,7 +810,7 @@ private: } FilterVerdict - handleStartTransaction(const Buffer &data) + handleStartTransaction(const Buffer &data, NginxAttachmentOpaque &opaque) { if (data.size() == 0) { dbgWarning(D_NGINX_ATTACHMENT) @@ -819,7 +819,6 @@ private: return default_verdict; } - NginxAttachmentOpaque &opaque = i_transaction_table->getState(); auto rule_by_ctx = getConfiguration("rulebase", "rulesConfig"); if (rule_by_ctx.ok()) { BasicRuleConfig rule = rule_by_ctx.unpack(); @@ -928,13 +927,12 @@ private: } void - setResponseContentEncoding(const CompressionType content_encoding) + setResponseContentEncoding(const CompressionType content_encoding, NginxAttachmentOpaque &opaque) { if (content_encoding == HttpTransactionData::default_response_content_encoding) { dbgDebug(D_NGINX_ATTACHMENT) << "New content encoding is the default. Skipping change of currect state"; return; } - auto &opaque = i_transaction_table->getState(); auto &transaction_data = opaque.getTransactionData(); transaction_data.setResponseContentEncoding(content_encoding); @@ -951,7 +949,7 @@ private: } FilterVerdict - handleResponseHeaders(const Buffer &headers_data) + handleResponseHeaders(const Buffer &headers_data, NginxAttachmentOpaque &opaque) { dbgFlow(D_NGINX_ATTACHMENT) << "Handling response headers"; bool did_fail_on_purpose = false; @@ -985,16 +983,15 @@ private: dbgTrace(D_NGINX_ATTACHMENT) << "Successfully parsed response's content encoding"; auto parsed_content_encoding = parsed_content_encoding_maybe.unpack(); - setResponseContentEncoding(parsed_content_encoding); + setResponseContentEncoding(parsed_content_encoding, opaque); updateMetrics(parsed_content_encoding); return handleMultiModifiableChunks(response_headers, false); } FilterVerdict - handleResponseBody(const Buffer &data) + handleResponseBody(const Buffer &data, NginxAttachmentOpaque &opaque) { - auto &opaque = i_transaction_table->getState(); auto &transaction_data = opaque.getTransactionData(); CompressionType content_encoding = transaction_data.getResponseContentEncoding(); @@ -1007,13 +1004,18 @@ private: } FilterVerdict - handleChunkedData(ChunkType chunk_type, const Buffer &data) + handleChunkedData(ChunkType chunk_type, const Buffer &data, NginxAttachmentOpaque &opaque) { ScopedContext event_type; event_type.registerValue("HTTP Chunk type", chunk_type); - auto rule_by_ctx = getConfiguration("rulebase", "rulesConfig"); - if (!rule_by_ctx.ok() && chunk_type > ChunkType::REQUEST_HEADER) { + if (chunk_type > ChunkType::REQUEST_HEADER && opaque.getApplicationState() == ApplicationState::UNKOWN) { + auto rule_by_ctx = getConfiguration("rulebase", "rulesConfig"); + ApplicationState state = rule_by_ctx.ok() ? ApplicationState::DEFINED : ApplicationState::UNDEFINED; + opaque.setApplicationState(state); + } + + if (opaque.getApplicationState() == ApplicationState::UNDEFINED) { ngx_http_cp_verdict_e verdict_action = getSettingWithDefault(false, "allowOnlyDefinedApplications") ? DROP : ACCEPT; @@ -1026,7 +1028,7 @@ private: switch (chunk_type) { case ChunkType::REQUEST_START: - return handleStartTransaction(data); + return handleStartTransaction(data, opaque); case ChunkType::REQUEST_HEADER: return handleMultiModifiableChunks(NginxParser::parseRequestHeaders(data), "request header", true); case ChunkType::REQUEST_BODY: @@ -1043,10 +1045,10 @@ private: return handleContentLength(data); } case ChunkType::RESPONSE_HEADER: - return handleResponseHeaders(data); + return handleResponseHeaders(data, opaque); case ChunkType::RESPONSE_BODY: nginx_attachment_event.addResponseInspectionCounter(1); - return handleResponseBody(data); + return handleResponseBody(data, opaque); case ChunkType::RESPONSE_END: return FilterVerdict(http_manager->inspectEndTransaction()); case ChunkType::METRIC_DATA_FROM_PLUGIN: @@ -1456,7 +1458,7 @@ private: NginxAttachmentOpaque &opaque = i_transaction_table->getState(); opaque.activateContext(); - FilterVerdict verdict = handleChunkedData(*chunked_data_type, inspection_data); + FilterVerdict verdict = handleChunkedData(*chunked_data_type, inspection_data, opaque); bool is_header = *chunked_data_type == ChunkType::REQUEST_HEADER || diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.h b/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.h index 19262ca..17c4f5f 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.h +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.h @@ -26,6 +26,8 @@ #include "i_environment.h" #include "buffer.h" +enum class ApplicationState { UNDEFINED, DEFINED, UNKOWN }; + class NginxAttachmentOpaque : public TableOpaqueSerialize, Singleton::Consume { public: @@ -58,6 +60,7 @@ public: CompressionStream * getResponseCompressionStream() { return response_compression_stream; } HttpTransactionData & getTransactionData() { return transaction_data; } + const ApplicationState & getApplicationState() const { return application_state; } // LCOV_EXCL_START - sync functions, can only be tested once the sync module exists template void serialize(T &, uint) {} @@ -81,6 +84,7 @@ public: const std::string &data, EnvKeyAttr::LogSection log_ctx = EnvKeyAttr::LogSection::NONE ); + void setApplicationState(const ApplicationState &app_state) { application_state = app_state; } private: CompressionStream *response_compression_stream; @@ -93,6 +97,7 @@ private: std::string source_identifier; std::string identifier_type; std::map saved_data; + ApplicationState application_state = ApplicationState::UNKOWN; }; #endif // __NGINX_ATTACHMENT_OPAQUE_H__ diff --git a/components/generic_rulebase/match_query.cc b/components/generic_rulebase/match_query.cc index f472ae7..ef985dc 100755 --- a/components/generic_rulebase/match_query.cc +++ b/components/generic_rulebase/match_query.cc @@ -60,6 +60,22 @@ static const unordered_map string_to_key = { { "domain", MatchQuery::StaticKeys::Domain } }; +MatchQuery::MatchQuery(const string &match) : is_specific_label(false), is_ignore_keyword(false) +{ + try { + stringstream ss; + ss.str(match); + cereal::JSONInputArchive archive_in(ss); + load(archive_in); + } catch (const exception &e) { + dbgWarning(D_RULEBASE_CONFIG) + << "Unable to load match query JSON. JSON content: " + << match + << ", Error: " + << e.what(); + } +} + void MatchQuery::load(cereal::JSONInputArchive &archive_in) { diff --git a/components/generic_rulebase/zones_config.cc b/components/generic_rulebase/zones_config.cc index 7a4df79..97bd88e 100755 --- a/components/generic_rulebase/zones_config.cc +++ b/components/generic_rulebase/zones_config.cc @@ -49,7 +49,7 @@ ZonesConfig::load(cereal::JSONInputArchive &archive_in) return; } - dbgWarning(D_RULEBASE_CONFIG) + dbgDebug(D_RULEBASE_CONFIG) << "Adding specific zone to cache. Zone ID: " << single_zone.getId() << ", name: " @@ -93,7 +93,7 @@ ZonesConfig::load(cereal::JSONInputArchive &archive_in) } for (GenericConfigId &implied_id: implied_zones) { if (all_zones.find(implied_id) != all_zones.end()) { - dbgWarning(D_RULEBASE_CONFIG) << "Adding implied zone to cache. Zone ID: " << implied_id; + dbgDebug(D_RULEBASE_CONFIG) << "Adding implied zone to cache. Zone ID: " << implied_id; active_zones_set.emplace(implied_id, all_zones[implied_id]); if (any_zone_id != "" && active_zones_set.count(any_zone_id) == 0) { active_zones_set.emplace(any_zone_id, all_zones[any_zone_id]); diff --git a/components/health_check_manager/health_check_manager.cc b/components/health_check_manager/health_check_manager.cc index 41d852a..ea73d5d 100755 --- a/components/health_check_manager/health_check_manager.cc +++ b/components/health_check_manager/health_check_manager.cc @@ -166,11 +166,16 @@ private: bool sendHealthCheckPatch() { - dbgFlow(D_HEALTH_CHECK_MANAGER); + dbgFlow(D_HEALTH_CHECK_MANAGER) << "Sending a health check patch"; HealthCheckPatch patch_to_send(general_health_aggregated_status, all_comps_health_status); auto messaging = Singleton::Consume::by(); - return messaging->sendNoReplyObject(patch_to_send, I_Messaging::Method::PATCH, "/agents"); + return messaging->sendSyncMessageWithoutResponse( + HTTPMethod::PATCH, + "/agents", + patch_to_send, + MessageCategory::GENERIC + ); } void diff --git a/components/health_check_manager/health_check_manager_ut/CMakeLists.txt b/components/health_check_manager/health_check_manager_ut/CMakeLists.txt index 98cf82a..99d8148 100755 --- a/components/health_check_manager/health_check_manager_ut/CMakeLists.txt +++ b/components/health_check_manager/health_check_manager_ut/CMakeLists.txt @@ -4,5 +4,5 @@ link_directories(${BOOST_ROOT}/lib) add_unit_test( health_check_manager_ut "health_check_manager_ut.cc" - "singleton;mainloop;health_check_manager;event_is;metric;-lboost_regex" + "singleton;messaging;mainloop;health_check_manager;event_is;metric;-lboost_regex" ) diff --git a/components/health_check_manager/health_check_manager_ut/health_check_manager_ut.cc b/components/health_check_manager/health_check_manager_ut/health_check_manager_ut.cc index a1fb42e..8a69986 100755 --- a/components/health_check_manager/health_check_manager_ut/health_check_manager_ut.cc +++ b/components/health_check_manager/health_check_manager_ut/health_check_manager_ut.cc @@ -111,19 +111,18 @@ public: TEST_F(HealthCheckManagerTest, runPeriodicHealthCheckTest) { string actual_body; - EXPECT_CALL( - mock_message, - sendMessage( - false, - _, - I_Messaging::Method::PATCH, - "/agents", - "", - _, - _, - MessageTypeTag::GENERIC + EXPECT_CALL(mock_message, sendSyncMessage( + HTTPMethod::PATCH, + "/agents", + _, + _, + _ + )).Times(4).WillRepeatedly( + DoAll( + SaveArg<2>(&actual_body), + Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")) ) - ).Times(4).WillRepeatedly(DoAll(SaveArg<1>(&actual_body), Return(string()))); + ); try { health_check_periodic_routine(); diff --git a/components/include/WaapEnums.h b/components/include/WaapEnums.h index 0b69ae0..457361a 100755 --- a/components/include/WaapEnums.h +++ b/components/include/WaapEnums.h @@ -24,6 +24,15 @@ #define MAX_FINAL_SCORE 10.0 #define ATTACK_IN_PARAM "attack_in_param" +enum TrafficMethod { + POST, + GET, + PUT, + PATCH, + DELETE, + OTHER +}; + enum ThreatLevel { NO_THREAT = 0, THREAT_INFO, diff --git a/components/include/generic_rulebase/match_query.h b/components/include/generic_rulebase/match_query.h index 658f4a2..787a99f 100755 --- a/components/include/generic_rulebase/match_query.h +++ b/components/include/generic_rulebase/match_query.h @@ -54,6 +54,8 @@ public: MatchQuery(): is_specific_label(false), is_ignore_keyword(false) {} + MatchQuery(const std::string &match); + void load(cereal::JSONInputArchive &archive_in); MatchType getType() const { return type; } diff --git a/components/include/geo_location.h b/components/include/geo_location.h new file mode 100644 index 0000000..3438f85 --- /dev/null +++ b/components/include/geo_location.h @@ -0,0 +1,24 @@ +#ifndef __GEO_LOCATION_H__ +#define __GEO_LOCATION_H__ + +#include "i_geo_location.h" +#include "singleton.h" +#include "component.h" + +class GeoLocation : public Component, Singleton::Provide +{ +public: + GeoLocation(); + ~GeoLocation(); + + void preload(); + + void init(); + void fini(); + +private: + class Impl; + std::unique_ptr pimpl; +}; + +#endif // __GEO_LOCATION_H__ diff --git a/components/include/http_geo_filter.h b/components/include/http_geo_filter.h new file mode 100644 index 0000000..6868e1c --- /dev/null +++ b/components/include/http_geo_filter.h @@ -0,0 +1,35 @@ +#ifndef __HTTP_GEO_FILTER_H__ +#define __HTTP_GEO_FILTER_H__ + +#include + +#include "singleton.h" +#include "i_mainloop.h" +#include "component.h" +#include "http_inspection_events.h" +#include "i_geo_location.h" +#include "i_generic_rulebase.h" + +class HttpGeoFilter + : + public Component, + Singleton::Consume, + Singleton::Consume, + Singleton::Consume +{ +public: + HttpGeoFilter(); + ~HttpGeoFilter(); + + void preload() override; + + void init() override; + void fini() override; + + +private: + class Impl; + std::unique_ptr pimpl; +}; + +#endif // __HTTP_GEO_FILTER_H__ diff --git a/components/include/i_details_resolver.h b/components/include/i_details_resolver.h index 7bf45db..02d9c65 100644 --- a/components/include/i_details_resolver.h +++ b/components/include/i_details_resolver.h @@ -27,7 +27,7 @@ public: virtual std::string getAgentVersion() = 0; virtual bool isKernelVersion3OrHigher() = 0; virtual bool isGwNotVsx() = 0; - virtual bool isVersionEqualOrAboveR8110() = 0; + virtual bool isVersionAboveR8110() = 0; virtual bool isReverseProxy() = 0; virtual Maybe> parseNginxMetadata() = 0; virtual std::map getResolvedDetails() = 0; diff --git a/components/include/i_downloader.h b/components/include/i_downloader.h index 15116af..f85ec19 100755 --- a/components/include/i_downloader.h +++ b/components/include/i_downloader.h @@ -40,6 +40,10 @@ public: const std::string &service_name ) const = 0; + virtual Maybe checkIfFileExists(const Package &package) const = 0; + + virtual void removeDownloadFile(const std::string &file_name) const = 0; + virtual std::string getProfileFromMap(const std::string &tenant_id) const = 0; }; diff --git a/components/include/i_geo_location.h b/components/include/i_geo_location.h new file mode 100644 index 0000000..2a30aa3 --- /dev/null +++ b/components/include/i_geo_location.h @@ -0,0 +1,22 @@ +#ifndef __I_GEO_LOCATION_H__ +#define __I_GEO_LOCATION_H__ + +#include + +#include "connkey.h" +#include "enum_array.h" + + +class I_GeoLocation +{ +public: + enum class GeoLocationField { COUNTRY_NAME, COUNTRY_CODE, CONTINENT_NAME, CONTINENT_CODE, COUNT }; + + virtual Maybe> lookupLocation(const std::string &ip) = 0; + virtual Maybe> lookupLocation(const IPAddr &ip) = 0; + +protected: + virtual ~I_GeoLocation() {} +}; + +#endif // __I_GEO_LOCATION_H__ diff --git a/components/include/i_oa_schema_updater.h b/components/include/i_oa_schema_updater.h new file mode 100644 index 0000000..bc50e73 --- /dev/null +++ b/components/include/i_oa_schema_updater.h @@ -0,0 +1,37 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __I_OA_SCHEMA_UPDATER_H__ +#define __I_OA_SCHEMA_UPDATER_H__ + +#include +#include "i_transaction.h" +#include "../security_apps/waap/waap_clib/oasu_key_types.h" +#include "../security_apps/waap/waap_clib/events_for_oa_schema.h" + +class I_OASUpdater +{ +public: + virtual void onKvt(const std::string &value, SchemaKeyType type, IWaf2Transaction &waf2Transaction) = 0; + virtual void addOperationField( + const std::string &operation_name, + const std::string &operation_type, + const std::string &field_name, + IWaf2Transaction &waf2Transaction) = 0; + virtual void removeGraphQLData(IWaf2Transaction &waf2Transaction) = 0; + virtual void addActiveOperationName( + const std::string &operation_name, + IWaf2Transaction &waf2Transaction) = 0; +}; + +#endif // __I_OA_SCHEMA_UPDATER_H__ diff --git a/components/include/i_update_communication.h b/components/include/i_update_communication.h index 0d282b0..7193ff8 100755 --- a/components/include/i_update_communication.h +++ b/components/include/i_update_communication.h @@ -33,7 +33,10 @@ public: ) const = 0; virtual Maybe authenticateAgent() = 0; virtual Maybe getUpdate(CheckUpdateRequest &request) = 0; - virtual Maybe downloadAttributeFile(const GetResourceFile &resourse_file) = 0; + virtual Maybe downloadAttributeFile( + const GetResourceFile &resourse_file, + const std::string &file_path + ) = 0; virtual void setAddressExtenesion(const std::string &extension) = 0; }; diff --git a/components/include/i_waap_telemetry.h b/components/include/i_waap_telemetry.h index fe3ff06..c47c229 100755 --- a/components/include/i_waap_telemetry.h +++ b/components/include/i_waap_telemetry.h @@ -23,6 +23,8 @@ struct DecisionTelemetryData std::string practiceId; std::string practiceName; std::string source; + TrafficMethod method; + int responseCode; std::set attackTypes; DecisionTelemetryData() : @@ -32,6 +34,8 @@ struct DecisionTelemetryData practiceId(), practiceName(), source(), + method(POST), + responseCode(0), attackTypes() { } diff --git a/components/include/manifest_diff_calculator.h b/components/include/manifest_diff_calculator.h index 9494fa0..9d5f913 100755 --- a/components/include/manifest_diff_calculator.h +++ b/components/include/manifest_diff_calculator.h @@ -36,15 +36,22 @@ public: std::map &corrupted_packages ); - bool + Maybe> buildInstallationQueue( - const Package &updated_package, + const std::map ¤t_packages, + const std::map &new_packages + ); + + +private: + Maybe + buildRecInstallationQueue( + const Package &package, std::vector &installation_queue, const std::map ¤t_packages, const std::map &new_packages ); -private: std::string corrupted_file_path; }; #endif // __MANIFEST_DIFF_CALCULATOR_H__ diff --git a/components/include/manifest_handler.h b/components/include/manifest_handler.h index 0a2f382..5cfef3e 100755 --- a/components/include/manifest_handler.h +++ b/components/include/manifest_handler.h @@ -24,6 +24,8 @@ #include "i_details_resolver.h" #include "i_time_get.h" +using packageFilePath = std::string; + class ManifestHandler : Singleton::Consume, @@ -36,28 +38,23 @@ class ManifestHandler Singleton::Consume { public: - using packageFilePath = std::string; ManifestHandler() = default; void init(); - bool - downloadPackages( - const std::vector &updated_packages, - std::vector> &downloaded_packages + Maybe>> downloadPackages( + const std::map &new_packages_to_download ); - bool - installPackages( - const std::vector> &downloaded_packages_files, + bool installPackage( + const std::pair &package_downloaded_file, std::map ¤t_packages, std::map &corrupted_packages ); bool uninstallPackage(Package &removed_package); - bool - selfUpdate( + bool selfUpdate( const Package &updated_package, std::map ¤t_packages, const packageFilePath &installation_file diff --git a/components/include/oas_updater_entry_saver.h b/components/include/oas_updater_entry_saver.h new file mode 100644 index 0000000..b23e24e --- /dev/null +++ b/components/include/oas_updater_entry_saver.h @@ -0,0 +1,4 @@ +#ifndef __SCHEMA_UPDATER_DATA_STRUCTURE_H__ +#define __SCHEMA_UPDATER_DATA_STRUCTURE_H__ + +#endif // __SCHEMA_UPDATER_DATA_STRUCTURE_H__ diff --git a/components/include/orchestrator/rest_api/get_resource_file.h b/components/include/orchestrator/rest_api/get_resource_file.h index 90349db..6045b3b 100644 --- a/components/include/orchestrator/rest_api/get_resource_file.h +++ b/components/include/orchestrator/rest_api/get_resource_file.h @@ -120,10 +120,10 @@ public: return std::string(); } - I_Messaging::Method + HTTPMethod getRequestMethod() const { - return isVirtual() ? I_Messaging::Method::POST : I_Messaging::Method::GET; + return isVirtual() ? HTTPMethod::POST : HTTPMethod::GET; } private: diff --git a/components/include/package.h b/components/include/package.h index 568879c..7b479cc 100755 --- a/components/include/package.h +++ b/components/include/package.h @@ -35,10 +35,11 @@ public: const std::string & getName() const { return name; } const std::string & getVersion() const { return version; } const std::string & getChecksum() const { return checksum_value; } + const std::string & getErrorMessage() const { return error_message; } const PackageType & getType() const { return package_type; } const std::vector & getRequire() const { return require_packages; } const ChecksumTypes & getChecksumType() const { return checksum_type; } - const Maybe & isInstallable() const { return installable; } + const bool & isInstallable() const { return installable; } bool operator==(const Package &other) const; bool operator!=(const Package &other) const; @@ -60,7 +61,8 @@ private: return std::string(); } - Maybe installable = Maybe(); + bool installable = true; + std::string error_message; std::string mirror; std::string name; std::string version; diff --git a/components/include/rate_limit_config.h b/components/include/rate_limit_config.h index b464d3e..186eec2 100755 --- a/components/include/rate_limit_config.h +++ b/components/include/rate_limit_config.h @@ -9,9 +9,20 @@ #include "debug.h" #include "generic_rulebase/rulebase_config.h" #include "generic_rulebase/triggers_config.h" +#include "generic_rulebase/match_query.h" #include "generic_rulebase/evaluators/trigger_eval.h" -USE_DEBUG_FLAG(D_REVERSE_PROXY); +USE_DEBUG_FLAG(D_RATE_LIMIT); + +enum class RateLimitAction +{ + INACTIVE, + ACCORDING_TO_PRACTICE, + DETECT, + PREVENT, + + UNKNOWN +}; class RateLimitTrigger { @@ -34,12 +45,12 @@ public: operator bool() const { if (uri.empty()) { - dbgTrace(D_REVERSE_PROXY) << "Recived empty URI in rate-limit rule"; + dbgTrace(D_RATE_LIMIT) << "Recived empty URI in rate-limit rule"; return false; } if (uri.at(0) != '/') { - dbgWarning(D_REVERSE_PROXY) + dbgWarning(D_RATE_LIMIT) << "Recived invalid rate-limit URI in rate-limit rule: " << uri << " rate-limit URI must start with /"; @@ -47,7 +58,7 @@ public: } if (limit <= 0) { - dbgWarning(D_REVERSE_PROXY) + dbgWarning(D_RATE_LIMIT) << "Recived invalid rate-limit limit in rate-limit rule: " << limit << " rate-limit rule limit must be positive"; @@ -70,10 +81,13 @@ public: const std::string & getRateLimitReq() const { return limit_req_template_value; } const std::string & getRateLimitUri() const { return uri; } const std::string & getRateLimitScope() const { return scope; } + const RateLimitAction & getRateLimitAction() const { return action; } + const MatchQuery & getRateLimitMatch() const { return match; } const LogTriggerConf & getRateLimitTrigger() const { return trigger; } const std::vector & getRateLimitTriggers() const { return rate_limit_triggers; } bool isRootLocation() const; + bool isMatchAny() const; bool operator==(const RateLimitRule &rhs) { return uri == rhs.uri; } bool operator<(const RateLimitRule &rhs) { return uri < rhs.uri; } @@ -87,21 +101,27 @@ private: std::string limit_req_template_value; std::string limit_req_zone_template_value; std::string cache_size = "5m"; + RateLimitAction action = RateLimitAction::ACCORDING_TO_PRACTICE; + MatchQuery match = MatchQuery(default_match); std::vector rate_limit_triggers; LogTriggerConf trigger; int limit; bool exact_match = false; + + static const std::string default_match; }; class RateLimitConfig { public: void load(cereal::JSONInputArchive &ar); - void addSiblingRateLimitRule(RateLimitRule &rule); + void addSiblingRateLimitRules(); void prepare(); const std::vector & getRateLimitRules() const { return rate_limit_rules; } - const std::string & getRateLimitMode() const { return mode; } + const RateLimitAction & getRateLimitMode() const { return mode; } + + RateLimitRule generateSiblingRateLimitRule(const RateLimitRule &rule); const LogTriggerConf getRateLimitTrigger(const std::string &nginx_uri) const @@ -110,7 +130,7 @@ public: std::set rate_limit_triggers_set; for (const RateLimitTrigger &rate_limit_trigger : rule.getRateLimitTriggers()) { - dbgTrace(D_REVERSE_PROXY) + dbgTrace(D_RATE_LIMIT) << "Adding trigger ID: " << rate_limit_trigger.getTriggerId() << " of rule URI: " @@ -130,12 +150,15 @@ public: static bool isActive() { return is_active; } + static const std::map rate_limit_action_to_string; + static const std::map rate_limit_string_to_action; + private: const RateLimitRule findLongestMatchingRule(const std::string &nginx_uri) const; static bool is_active; - std::string mode; + RateLimitAction mode; std::vector rate_limit_rules; }; diff --git a/components/include/report_messaging.h b/components/include/report_messaging.h index fb863d1..3d6d479 100755 --- a/components/include/report_messaging.h +++ b/components/include/report_messaging.h @@ -37,7 +37,7 @@ public: audience_team, obj, false, - MessageTypeTag::GENERIC, + MessageCategory::GENERIC, std::forward(args)... ) { @@ -56,7 +56,7 @@ public: audience_team, obj, is_async_message, - MessageTypeTag::GENERIC, + MessageCategory::GENERIC, std::forward(args)... ) { @@ -68,7 +68,7 @@ public: const ReportIS::AudienceTeam &audience_team, const T &obj, bool is_async_message, - const MessageTypeTag &message_type, + const MessageCategory &message_type, Args ...args) : ReportMessaging( @@ -100,7 +100,7 @@ public: priority, obj, false, - MessageTypeTag::GENERIC, + MessageCategory::GENERIC, std::forward(args)... ) { @@ -115,7 +115,7 @@ public: const ReportIS::Priority &priority, const T &obj, bool _is_async_message, - const MessageTypeTag &message_type, + const MessageCategory &message_type, Args ...args) : report( @@ -144,7 +144,7 @@ public: private: Report report; bool is_async_message; - MessageTypeTag message_type_tag; + MessageCategory message_type_tag; }; #endif // __REPORT_MESSAGING_H__ diff --git a/components/include/service_controller.h b/components/include/service_controller.h index a7648f0..1ad10af 100755 --- a/components/include/service_controller.h +++ b/components/include/service_controller.h @@ -24,6 +24,7 @@ #include "i_shell_cmd.h" #include "i_rest_api.h" #include "i_tenant_manager.h" +#include "i_messaging.h" #include "service_details.h" #include "i_mainloop.h" #include "component.h" diff --git a/components/include/telemetry.h b/components/include/telemetry.h index e664414..ddbeebc 100755 --- a/components/include/telemetry.h +++ b/components/include/telemetry.h @@ -29,6 +29,7 @@ #include "waap.h" #include "generic_metric.h" +#define LOGGING_INTERVAL_IN_MINUTES 10 enum class AssetType { API, WEB, ALL, COUNT }; class WaapTelemetryEvent : public Event @@ -74,6 +75,24 @@ private: std::unordered_set sources_seen; }; +class WaapTrafficTelemetrics : public WaapTelemetryBase +{ +public: + void updateMetrics(const std::string &asset_id, const DecisionTelemetryData &data); + void initMetrics(); + +private: + MetricCalculations::Counter post_requests{this, "reservedNgenA"}; + MetricCalculations::Counter get_requests{this, "reservedNgenB"}; + MetricCalculations::Counter put_requests{this, "reservedNgenC"}; + MetricCalculations::Counter patch_requests{this, "reservedNgenD"}; + MetricCalculations::Counter delete_requests{this, "reservedNgenE"}; + MetricCalculations::Counter other_requests{this, "reservedNgenF"}; + MetricCalculations::Counter response_2xx{this, "reservedNgenG"}; + MetricCalculations::Counter response_4xx{this, "reservedNgenH"}; + MetricCalculations::Counter response_5xx{this, "reservedNgenI"}; +}; + class WaapAttackTypesMetrics : public WaapTelemetryBase { public: @@ -100,8 +119,62 @@ public: private: std::map> metrics; std::map> telemetries; + std::map> traffic_telemetries; std::map> attack_types; std::map> attack_types_telemetries; + + template + void initializeTelemetryData( + const std::string& asset_id, + const DecisionTelemetryData& data, + const std::string& telemetryName, + std::map>& telemetryMap + ) { + if (!telemetryMap.count(asset_id)) { + telemetryMap.emplace(asset_id, std::make_shared()); + telemetryMap[asset_id]->init( + telemetryName, + ReportIS::AudienceTeam::WAAP, + ReportIS::IssuingEngine::AGENT_CORE, + std::chrono::minutes(LOGGING_INTERVAL_IN_MINUTES), + true, + ReportIS::Audience::SECURITY + ); + + telemetryMap[asset_id]->template registerContext( + "pracitceType", + std::string("Threat Prevention"), + EnvKeyAttr::LogSection::SOURCE + ); + telemetryMap[asset_id]->template registerContext( + "practiceSubType", + std::string("Web Application"), + EnvKeyAttr::LogSection::SOURCE + ); + telemetryMap[asset_id]->template registerContext( + "assetId", + asset_id, + EnvKeyAttr::LogSection::SOURCE + ); + telemetryMap[asset_id]->template registerContext( + "assetName", + data.assetName, + EnvKeyAttr::LogSection::SOURCE + ); + telemetryMap[asset_id]->template registerContext( + "practiceId", + data.practiceId, + EnvKeyAttr::LogSection::SOURCE + ); + telemetryMap[asset_id]->template registerContext( + "practiceName", + data.practiceName, + EnvKeyAttr::LogSection::SOURCE + ); + + telemetryMap[asset_id]->registerListener(); + } + } }; class AssetCountEvent : public Event diff --git a/components/packet/packet_ut/CMakeLists.txt b/components/packet/packet_ut/CMakeLists.txt index b3e8001..9ad2107 100644 --- a/components/packet/packet_ut/CMakeLists.txt +++ b/components/packet/packet_ut/CMakeLists.txt @@ -3,5 +3,5 @@ link_directories(${BOOST_ROOT}/lib) add_unit_test( packet_ut "packet_ut.cc" - "packet;connkey;buffers;environment;metric;event_is;-lboost_regex" + "packet;messaging;connkey;buffers;environment;metric;event_is;-lboost_regex" ) diff --git a/components/report_messaging/report_messaging.cc b/components/report_messaging/report_messaging.cc index 528b5c2..b72cecf 100755 --- a/components/report_messaging/report_messaging.cc +++ b/components/report_messaging/report_messaging.cc @@ -29,17 +29,13 @@ ReportMessaging::~ReportMessaging() auto messaging = Singleton::Consume::by(); try { - messaging->sendObjectWithPersistence( - log_rest, - I_Messaging::Method::POST, + messaging->sendAsyncMessage( + HTTPMethod::POST, url, - "", - true, - message_type_tag, - is_async_message + log_rest, + message_type_tag ); - } catch (...) { - } + } catch (...) {} } ReportMessaging & diff --git a/components/report_messaging/report_messaging_ut/CMakeLists.txt b/components/report_messaging/report_messaging_ut/CMakeLists.txt index 028a5dd..8f263c7 100644 --- a/components/report_messaging/report_messaging_ut/CMakeLists.txt +++ b/components/report_messaging/report_messaging_ut/CMakeLists.txt @@ -1,3 +1,3 @@ link_directories(${BOOST_ROOT}/lib) -add_unit_test(report_messaging_ut "report_messaging_ut.cc" "report_messaging;report;singleton;-lboost_regex") \ No newline at end of file +add_unit_test(report_messaging_ut "report_messaging_ut.cc" "report_messaging;report;messaging;singleton;-lboost_regex") \ No newline at end of file diff --git a/components/report_messaging/report_messaging_ut/report_messaging_ut.cc b/components/report_messaging/report_messaging_ut/report_messaging_ut.cc index b37089d..b3c594f 100644 --- a/components/report_messaging/report_messaging_ut/report_messaging_ut.cc +++ b/components/report_messaging/report_messaging_ut/report_messaging_ut.cc @@ -69,7 +69,8 @@ TEST_F(ReportMessagingTest, title_only) { EXPECT_CALL( mock_messaging, - mockSendPersistentMessage( + sendAsyncMessage( + _, _, "{\n" " \"log\": {\n" @@ -98,20 +99,18 @@ TEST_F(ReportMessagingTest, title_only) " }\n" "}", _, - _, - _, - _, _ ) - ).WillOnce(Return(string())); - ReportMessaging("test", ReportIS::AudienceTeam::AGENT_CORE, 1, ReportIS::Tags::ACCESS_CONTROL); + ).Times(1); + ReportMessaging("test", ReportIS::AudienceTeam::AGENT_CORE, 1, true, ReportIS::Tags::ACCESS_CONTROL); } TEST_F(ReportMessagingTest, with_dynamic_fields) { EXPECT_CALL( mock_messaging, - mockSendPersistentMessage( + sendAsyncMessage( + _, _, "{\n" " \"log\": {\n" @@ -141,12 +140,9 @@ TEST_F(ReportMessagingTest, with_dynamic_fields) " }\n" "}", _, - _, - _, - _, _ ) - ).WillOnce(Return(string())); + ).Times(1); ReportMessaging("test", ReportIS::AudienceTeam::AGENT_CORE, 1, ReportIS::Tags::ACCESS_CONTROL) << LogField("ASD", "QWE"); } @@ -155,7 +151,8 @@ TEST_F(ReportMessagingTest, custom_event_object) { EXPECT_CALL( mock_messaging, - mockSendPersistentMessage( + sendAsyncMessage( + _, _, "{\n" " \"log\": {\n" @@ -192,12 +189,9 @@ TEST_F(ReportMessagingTest, custom_event_object) " }\n" "}", _, - _, - _, - _, _ ) - ).WillOnce(Return(string())); + ).Times(1); ReportMessaging( "test", @@ -211,7 +205,8 @@ TEST_F(ReportMessagingTest, custom_priority) { EXPECT_CALL( mock_messaging, - mockSendPersistentMessage( + sendAsyncMessage( + _, _, "{\n" " \"log\": {\n" @@ -248,12 +243,9 @@ TEST_F(ReportMessagingTest, custom_priority) " }\n" "}", _, - _, - _, - _, _ ) - ).WillOnce(Return(string())); + ).Times(1); ReportMessaging( "test", @@ -279,7 +271,8 @@ TEST_F(ReportMessagingTest, with_env_details) EXPECT_CALL( mock_messaging, - mockSendPersistentMessage( + sendAsyncMessage( + _, _, "{\n" " \"log\": {\n" @@ -316,12 +309,9 @@ TEST_F(ReportMessagingTest, with_env_details) " }\n" "}", _, - _, - _, - _, _ ) - ).WillOnce(Return(string())); + ).Times(1); ReportMessaging( "test", diff --git a/components/security_apps/CMakeLists.txt b/components/security_apps/CMakeLists.txt index bfde75c..ee0e943 100644 --- a/components/security_apps/CMakeLists.txt +++ b/components/security_apps/CMakeLists.txt @@ -1,3 +1,4 @@ +#add_subdirectory(http_geo_filter) add_subdirectory(ips) add_subdirectory(layer_7_access_control) add_subdirectory(local_policy_mgmt_gen) diff --git a/components/security_apps/http_geo_filter/CMakeLists.txt b/components/security_apps/http_geo_filter/CMakeLists.txt new file mode 100644 index 0000000..942023b --- /dev/null +++ b/components/security_apps/http_geo_filter/CMakeLists.txt @@ -0,0 +1 @@ +add_library(http_geo_filter http_geo_filter.cc) diff --git a/components/security_apps/http_geo_filter/geo_config.h b/components/security_apps/http_geo_filter/geo_config.h new file mode 100644 index 0000000..2a47fc4 --- /dev/null +++ b/components/security_apps/http_geo_filter/geo_config.h @@ -0,0 +1,90 @@ +#ifndef __GEO_CONFIG_H__ +#define __GEO_CONFIG_H__ + +#include + +#include "cereal/archives/json.hpp" +#include "debug.h" + +USE_DEBUG_FLAG(D_GEO_FILTER); + +class GeoFilterCountry +{ +public: + void + load(cereal::JSONInputArchive &ar) + { + try { + ar(cereal::make_nvp("countryName", country_name)); + ar(cereal::make_nvp("countryCode", country_code)); + ar(cereal::make_nvp("id", id)); + } catch (const cereal::Exception &e) { + dbgDebug(D_GEO_FILTER) << "Failed to load http geo country config, error: " << e.what(); + } + } + + const std::string & getCountryCode() const { return country_code; } + +private: + std::string country_name; + std::string country_code; + std::string id; +}; + +class GeoConfig +{ +public: + void + load(cereal::JSONInputArchive &ar) + { + try { + ar(cereal::make_nvp("name", name)); + ar(cereal::make_nvp("defaultAction", default_action)); + ar(cereal::make_nvp("practiceId", id)); + ar(cereal::make_nvp("allowedCountries", allowed_countries)); + ar(cereal::make_nvp("blockedCountries", blocked_countries)); + } catch (const cereal::Exception &e) { + dbgDebug(D_GEO_FILTER) << "Failed to load http geo config, error: " << e.what(); + } + } + + const std::string & getId() const { return id; } + const std::string & getDefaultAction() const { return default_action; } + + bool + isAllowedCountry(const std::string &_country_code) const + { + dbgTrace(D_GEO_FILTER) << "Check if country code: " << _country_code << " is allowed"; + for (const GeoFilterCountry &country : allowed_countries) { + if (country.getCountryCode() == _country_code) { + dbgTrace(D_GEO_FILTER) << "County code: " << _country_code << " is allowed"; + return true; + } + } + dbgTrace(D_GEO_FILTER) << "County code: " << _country_code << " not in allowed countries list"; + return false; + } + + bool + isBlockedCountry(const std::string &_country_code) const + { + dbgTrace(D_GEO_FILTER) << "Check if country code: " << _country_code << " is blocked"; + for (const GeoFilterCountry &country : blocked_countries) { + if (country.getCountryCode() == _country_code) { + dbgTrace(D_GEO_FILTER) << "County code: " << _country_code << " is blocked"; + return true; + } + } + dbgTrace(D_GEO_FILTER) << "County code: " << _country_code << " not in blocked countries list"; + return false; + } + +private: + std::string name; + std::string default_action; + std::string id; + std::vector allowed_countries; + std::vector blocked_countries; +}; + +#endif //__GEO_CONFIG_H__ diff --git a/components/security_apps/http_geo_filter/http_geo_filter.cc b/components/security_apps/http_geo_filter/http_geo_filter.cc new file mode 100644 index 0000000..bb9b8b7 --- /dev/null +++ b/components/security_apps/http_geo_filter/http_geo_filter.cc @@ -0,0 +1,347 @@ +#include "http_geo_filter.h" + +#include +#include +#include +#include + +#include "generic_rulebase/generic_rulebase.h" +#include "generic_rulebase/parameters_config.h" +#include "generic_rulebase/triggers_config.h" +#include "debug.h" +#include "config.h" +#include "rest.h" +#include "geo_config.h" +#include "ip_utilities.h" +#include "log_generator.h" + +using namespace std; + +USE_DEBUG_FLAG(D_GEO_FILTER); + +static const LogTriggerConf default_triger; + +class HttpGeoFilter::Impl : public Listener +{ +public: + void + init() + { + dbgTrace(D_GEO_FILTER) << "Init Http Geo filter component"; + registerListener(); + } + + void + fini() + { + unregisterListener(); + } + + string getListenerName() const override { return "HTTP geo filter"; } + + void + loadDefaultAction() + { + auto default_action_maybe = getProfileAgentSetting("httpGeoFilter.defaultAction"); + if(default_action_maybe.ok()) { + default_action = convertActionToVerdict(default_action_maybe.unpack()); + dbgTrace(D_GEO_FILTER) + << "Load http geo filter default action. Action: " + << default_action_maybe.unpack(); + } else { + default_action = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; + dbgTrace(D_GEO_FILTER) << "No http geo filter default action. Action: Irrelevant"; + } + } + + EventVerdict + respond(const NewHttpTransactionEvent &event) override + { + dbgTrace(D_GEO_FILTER) << getListenerName() << " new transaction event"; + + if (!ParameterException::isGeoLocationExceptionExists() && + !getConfiguration("rulebase", "httpGeoFilter").ok() + ) { + dbgTrace(D_GEO_FILTER) << "No geo location practice nor exception was found. Returning default verdict"; + return EventVerdict(default_action); + } + + I_GeoLocation *i_geo_location = Singleton::Consume::by(); + auto asset_location = i_geo_location->lookupLocation(event.getSourceIP()); + if (!asset_location.ok()) { + dbgTrace(D_GEO_FILTER) << "Lookup location failed, Error: " << asset_location.getErr(); + return EventVerdict(default_action); + } + + EnumArray geo_location_data = asset_location.unpack(); + + ngx_http_cp_verdict_e exception_verdict = getExceptionVerdict(event, geo_location_data); + if (exception_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) { + return EventVerdict(exception_verdict); + } + + ngx_http_cp_verdict_e geo_lookup_verdict = getGeoLookupVerdict(event, geo_location_data); + if (geo_lookup_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) { + return EventVerdict(geo_lookup_verdict); + } + return EventVerdict(default_action); + } + +private: + string + convertIpAddrToString(const IPAddr &ip_to_convert) + { + ostringstream os; + os << ip_to_convert; + return os.str(); + } + + ngx_http_cp_verdict_e + convertActionToVerdict(const string &action) const + { + if (action == "accept") return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + if (action == "drop") return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + } + + ngx_http_cp_verdict_e + convertBehaviorValueToVerdict(const BehaviorValue &behavior_value) const + { + if (behavior_value == BehaviorValue::ACCEPT || behavior_value == BehaviorValue::IGNORE) { + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + } + if (behavior_value == BehaviorValue::DROP || behavior_value == BehaviorValue::REJECT) { + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + } + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; + } + + ngx_http_cp_verdict_e + getGeoLookupVerdict( + const NewHttpTransactionEvent &event, + const EnumArray &geo_location_data) + { + auto maybe_geo_config = getConfiguration("rulebase", "httpGeoFilter"); + if (!maybe_geo_config.ok()) { + dbgWarning(D_GEO_FILTER) << "Failed to load HTTP Geo Filter config. Error:" << maybe_geo_config.getErr(); + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; + } + GeoConfig geo_config = maybe_geo_config.unpack(); + string country_code = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE]; + + if (geo_config.isAllowedCountry(country_code)) { + dbgTrace(D_GEO_FILTER) + << "geo verdict ACCEPT, practice id: " + << geo_config.getId() + << ", country code: " + << country_code; + generateVerdictLog( + ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT, + event, + geo_config.getId(), + true, + geo_location_data + ); + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + } + if (geo_config.isBlockedCountry(country_code)) { + dbgTrace(D_GEO_FILTER) + << "geo verdict DROP, practice id: " + << geo_config.getId() + << ", country code: " + << country_code; + generateVerdictLog( + ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP, + event, + geo_config.getId(), + true, + geo_location_data + ); + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + } + dbgTrace(D_GEO_FILTER) + << "No matched practice. Returned default action: " + << geo_config.getDefaultAction(); + generateVerdictLog( + convertActionToVerdict(geo_config.getDefaultAction()), + event, + geo_config.getId(), + true, + geo_location_data, + true + ); + return convertActionToVerdict(geo_config.getDefaultAction()); + } + + Maybe> + getBehaviorsVerdict( + const unordered_map> &behaviors_map_to_search, + const NewHttpTransactionEvent &event, + EnumArray geo_location_data) + { + bool is_matched = false; + ParameterBehavior matched_behavior; + ngx_http_cp_verdict_e matched_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; + I_GenericRulebase *i_rulebase = Singleton::Consume::by(); + set behaviors_set = i_rulebase->getBehavior(behaviors_map_to_search); + dbgTrace(D_GEO_FILTER) << "get verdict from: " << behaviors_set.size() << " behaviors"; + for (const ParameterBehavior &behavior : behaviors_set) { + matched_verdict = convertBehaviorValueToVerdict(behavior.getValue()); + if ( + matched_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP + ){ + dbgTrace(D_GEO_FILTER) << "behavior verdict: DROP, exception id: " << behavior.getId(); + generateVerdictLog( + matched_verdict, + event, + behavior.getId(), + false, + geo_location_data + ); + return pair(matched_verdict, behavior.getId()); + } + else if ( + matched_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT + ){ + dbgTrace(D_GEO_FILTER) << "behavior verdict: ACCEPT, exception id: " << behavior.getId(); + matched_behavior = behavior; + is_matched = true; + } + } + if (is_matched) { + return pair( + ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT, + matched_behavior.getId() + ); + } + return genError("No exception matched to HTTP geo filter rule"); + } + + ngx_http_cp_verdict_e + getExceptionVerdict( + const NewHttpTransactionEvent &event, + EnumArray geo_location_data + ){ + string country_code = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE]; + string country_name = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_NAME]; + string source_ip = convertIpAddrToString(event.getSourceIP()); + + pair curr_matched_behavior; + ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; + + dbgTrace(D_GEO_FILTER) + << "Get exception verdict. " + << "country code: " + << country_code + << ", country name: " + << country_name + << ", source ip address: " + << source_ip; + + unordered_map> exception_value_source_ip = {{"sourceIP", {source_ip}}}; + auto matched_behavior_maybe = getBehaviorsVerdict(exception_value_source_ip, event, geo_location_data); + if (matched_behavior_maybe.ok()) { + curr_matched_behavior = matched_behavior_maybe.unpack(); + verdict = curr_matched_behavior.first; + if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) { + return verdict; + } + } + + unordered_map> exception_value_country_code = { + {"countryCode", {country_code}} + }; + matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_code, event, geo_location_data); + if (matched_behavior_maybe.ok()) { + curr_matched_behavior = matched_behavior_maybe.unpack(); + verdict = curr_matched_behavior.first; + if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) { + return verdict; + } + } + + unordered_map> exception_value_country_name = { + {"countryName", {country_name}} + }; + matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_name, event, geo_location_data); + if (matched_behavior_maybe.ok()) { + curr_matched_behavior = matched_behavior_maybe.unpack(); + verdict = curr_matched_behavior.first; + if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) { + return verdict; + } + } + if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT) { + generateVerdictLog( + verdict, + event, + curr_matched_behavior.second, + false, + geo_location_data + ); + } + return verdict; + } + + void + generateVerdictLog( + const ngx_http_cp_verdict_e &verdict, + const NewHttpTransactionEvent &event, + const string &matched_id, + bool is_geo_filter, + const EnumArray geo_location_data, + bool is_default_action = false + ) + { + dbgTrace(D_GEO_FILTER) << "Generate Log for verdict - HTTP geo filter"; + auto &trigger = getConfigurationWithDefault(default_triger, "rulebase", "log"); + bool is_prevent = verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + string matched_on = is_geo_filter ? "geoFilterPracticeId" : "exceptionId"; + LogGen log = trigger( + "Web Request - HTTP Geo Filter", + LogTriggerConf::SecurityType::ThreatPrevention, + ReportIS::Severity::MEDIUM, + ReportIS::Priority::HIGH, + is_prevent, + LogField("practiceType", "HTTP Geo Filter"), + LogField(matched_on, matched_id), + ReportIS::Tags::HTTP_GEO_FILTER + ); + log + << LogField("sourceIP", convertIpAddrToString(event.getSourceIP())) + << LogField("sourcePort", event.getSourcePort()) + << LogField("hostName", event.getDestinationHost()) + << LogField("httpMethod", event.getHttpMethod()) + << LogField("securityAction", is_prevent ? "Prevent" : "Detect"); + + if (is_default_action) log << LogField("isDefaultSecurityAction", true); + + log + << LogField("sourceCountryCode", geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE]) + << LogField("sourceCountryName", geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_NAME]); + } + + ngx_http_cp_verdict_e default_action = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; +}; + +HttpGeoFilter::HttpGeoFilter() : Component("HttpGeoFilter"), pimpl(make_unique()) {} +HttpGeoFilter::~HttpGeoFilter() {} + +void +HttpGeoFilter::init() +{ + pimpl->init(); +} + +void +HttpGeoFilter::fini() +{ + pimpl->fini(); +} + +void +HttpGeoFilter::preload() +{ + registerExpectedConfiguration("rulebase", "httpGeoFilter"); + registerConfigLoadCb([this]() { pimpl->loadDefaultAction(); }); +} diff --git a/components/security_apps/ips/ips_ut/CMakeLists.txt b/components/security_apps/ips/ips_ut/CMakeLists.txt index 2a36d27..f0d33fa 100644 --- a/components/security_apps/ips/ips_ut/CMakeLists.txt +++ b/components/security_apps/ips/ips_ut/CMakeLists.txt @@ -3,5 +3,5 @@ link_directories(${CMAKE_BINARY_DIR}/core/shmem_ipc) add_unit_test( ips_ut "signatures_ut.cc;entry_ut.cc;component_ut.cc;configuration.cc;rule_selector_ut.cc;compound_ut.cc;resource_ut.cc" - "ips;keywords;pcre2-8;intelligence_is_v2;logging;compression_utils;agent_details;time_proxy;event_is;table;http_transaction_data;nginx_attachment;connkey;pm;metric;encryptor;generic_rulebase;generic_rulebase_evaluators;compression_utils;ip_utilities;-lboost_regex;-lcrypto;-lz" + "ips;messaging;keywords;pcre2-8;intelligence_is_v2;logging;compression_utils;agent_details;time_proxy;event_is;table;http_transaction_data;nginx_attachment;connkey;pm;metric;encryptor;generic_rulebase;generic_rulebase_evaluators;compression_utils;ip_utilities;-lboost_regex;-lcrypto;-lz" ) diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/CMakeLists.txt b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/CMakeLists.txt index 7c2cdc9..42adc28 100644 --- a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/CMakeLists.txt +++ b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/CMakeLists.txt @@ -3,5 +3,5 @@ file(COPY data DESTINATION .) add_unit_test( layer_7_access_control_ut "layer_7_access_control_ut.cc" - "l7_access_control;logging;agent_details;table;singleton;time_proxy;metric;event_is;connkey;http_transaction_data;generic_rulebase;generic_rulebase_evaluators;ip_utilities;intelligence_is_v2" + "l7_access_control;messaging;logging;agent_details;table;singleton;time_proxy;metric;event_is;connkey;http_transaction_data;generic_rulebase;generic_rulebase_evaluators;ip_utilities;intelligence_is_v2" ) diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc index 48bc1a9..3ab7117 100644 --- a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc +++ b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc @@ -261,13 +261,13 @@ TEST_F(Layer7AccessControlTest, ReturnAcceptVerdict) { stringstream ss_conf(prevent_settings + policy); Singleton::Consume::from(config)->loadConfiguration(ss_conf); - + string intelligence_response_ok = loadIntelligenceResponse("data/ok_intelligence_response.json"); EXPECT_CALL( messaging_mock, - sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) - ).WillOnce(Return(intelligence_response_ok)); + sendSyncMessage(_, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, intelligence_response_ok))); registerTransactionData(); ctx.registerValue(HttpTransactionData::source_identifier, "1.2.3.4"); @@ -312,8 +312,8 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictOnMaliciousReputation) EXPECT_CALL( messaging_mock, - sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) - ).WillOnce(Return(malicious_intelligence_response)); + sendSyncMessage(_, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, malicious_intelligence_response))); EXPECT_CALL( mock_ml, @@ -358,8 +358,8 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictCacheBased) EXPECT_CALL( messaging_mock, - sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) - ).WillOnce(Return(malicious_intelligence_response)); + sendSyncMessage(_, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, malicious_intelligence_response))); EXPECT_CALL( mock_ml, @@ -410,8 +410,8 @@ TEST_F(Layer7AccessControlTest, AcceptOnDetect) EXPECT_CALL( messaging_mock, - sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) - ).WillOnce(Return(malicious_intelligence_response)); + sendSyncMessage(_, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, malicious_intelligence_response))); EXPECT_CALL( mock_ml, @@ -456,8 +456,8 @@ TEST_F(Layer7AccessControlTest, FallbackToSourceIPAndDrop) EXPECT_CALL( messaging_mock, - sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) - ).WillOnce(Return(malicious_intelligence_response)); + sendSyncMessage(_, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, malicious_intelligence_response))); EXPECT_CALL( mock_ml, diff --git a/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc b/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc index 09aaab1..18b447c 100755 --- a/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc +++ b/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc @@ -133,11 +133,12 @@ AccessControlRateLimiteRules::load(cereal::JSONInputArchive &archive_in) dbgTrace(D_LOCAL_POLICY) << "Loading Access control rate limite rules"; parseAppsecJSONKey("limit", limit, archive_in); parseAppsecJSONKey("uri", uri, archive_in); - parseAppsecJSONKey("unit", unit, archive_in); + parseAppsecJSONKey("unit", unit, archive_in, "minute"); if (valid_units.count(unit) == 0) { dbgWarning(D_LOCAL_POLICY) << "Access control rate limite rules units invalid: " << unit; + throw PolicyGenException("Access control rate limite rules units invalid: " + unit); } parseAppsecJSONKey("comment", comment, archive_in); parseAppsecJSONKey>("triggers", triggers, archive_in); @@ -177,10 +178,10 @@ AccessControlRateLimit::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading Access control rate limit"; string in_mode; - parseAppsecJSONKey("overrideMode", in_mode, archive_in, "inactive"); + parseAppsecJSONKey("overrideMode", in_mode, archive_in, "detect"); if (valid_modes_to_key.find(in_mode) == valid_modes_to_key.end()) { dbgWarning(D_LOCAL_POLICY) << "AppSec access control rate limit override mode invalid: " << in_mode; - mode = "Inactive"; + throw PolicyGenException("AppSec access control rate limit override mode invalid: " + in_mode); } else { mode = valid_modes_to_key.at(in_mode); } @@ -216,7 +217,7 @@ AccessControlPracticeSpec::load(cereal::JSONInputArchive &archive_in) parseAppsecJSONKey("name", practice_name, archive_in); parseAppsecJSONKey("appsecClassName", appsec_class_name, archive_in); - parseAppsecJSONKey("rateLimit", rate_limit, archive_in); + parseMandatoryAppsecJSONKey("rateLimit", rate_limit, archive_in); } void diff --git a/components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h b/components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h index 99cefc2..32a579f 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h +++ b/components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h @@ -111,7 +111,6 @@ private: I_EnvDetails* env_details = nullptr; I_Messaging* messaging = nullptr; EnvType env_type; - Flags conn_flags; std::string token; }; diff --git a/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h b/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h index 1c7583b..6da97b0 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h +++ b/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include "config.h" @@ -66,27 +67,57 @@ static const std::unordered_map key_to_practices_val2 static const std::string default_appsec_url = "http://*:*"; +class PolicyGenException : public std::exception +{ +public: + PolicyGenException(const std::string& msg="") noexcept : m_msg(msg) {} + + const char* what() const noexcept override + { + return m_msg.c_str(); + } + +private: + std::string m_msg; +}; + template void parseAppsecJSONKey( const std::string &key_name, T &value, cereal::JSONInputArchive &archive_in, - const T &default_value = T()) + const T &default_value = T(), + bool mandatory = false) { try { archive_in(cereal::make_nvp(key_name, value)); } catch (const cereal::Exception &e) { archive_in.setNextName(nullptr); value = default_value; - dbgDebug(D_LOCAL_POLICY) - << "Could not parse the required key. Key: \"" - << key_name - << "\", Error: " - << e.what(); + if (!mandatory) { + dbgDebug(D_LOCAL_POLICY) + << "Could not parse the required key. Key: \""<< key_name + << "\", Error: " << e.what(); + } else { + throw PolicyGenException( + "Could not parse a mandatory key: \"" + key_name + "\", Error: " + std::string(e.what()) + ); + } } } +template +void +parseMandatoryAppsecJSONKey( + const std::string &key_name, + T &value, + cereal::JSONInputArchive &archive_in, + const T &default_value = T()) +{ + parseAppsecJSONKey(key_name, value, archive_in, default_value, true); +} + class AppsecSpecParserMetaData { public: diff --git a/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc b/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc index a036d17..450f6b7 100644 --- a/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc @@ -43,8 +43,6 @@ K8sPolicyUtils::init() if (env_type == EnvType::K8S) { token = env_details->getToken(); messaging = Singleton::Consume::by(); - conn_flags.setFlag(MessageConnConfig::SECURE_CONN); - conn_flags.setFlag(MessageConnConfig::IGNORE_SSL_VALIDATION); } } @@ -79,17 +77,19 @@ Maybe K8sPolicyUtils::getObjectFromCluster(const string &path) const { T object; - bool res = messaging->sendObject( - object, - I_Messaging::Method::GET, - "kubernetes.default.svc", - 443, - conn_flags, + MessageMetadata k8s_md("kubernetes.default.svc", 443); + k8s_md.insertHeader("Authorization", "Bearer " + token); + k8s_md.insertHeader("Connection", "close"); + k8s_md.setConnectioFlag(MessageConnectionConfig::IGNORE_SSL_VALIDATION); + auto res = messaging->sendSyncMessage( + HTTPMethod::GET, path, - "Authorization: Bearer " + token + "\nConnection: close" + object, + MessageCategory::GENERIC, + k8s_md ); - if (res) return object; + if (res.ok()) return object; return genError(string("Was not able to get object form k8s cluser in path: " + path)); } @@ -488,23 +488,33 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i if (!maybe_appsec_policy_spec.ok() || !doesVersionExist(maybe_appsec_policy_spec.unpack().getMetaData().getAnnotations(), "v1beta1") ) { - dbgWarning(D_LOCAL_POLICY) - << "Failed to retrieve Appsec policy with crds version: v1beta1, Trying version: v1beta2"; - auto maybe_v1beta2_appsec_policy_spec = getObjectFromCluster>( - "/apis/openappsec.io/v1beta2/policies/" + policy_name - ); - if(!maybe_v1beta2_appsec_policy_spec.ok()) { - dbgWarning(D_LOCAL_POLICY) - << "Failed to retrieve AppSec policy. Error: " - << maybe_v1beta2_appsec_policy_spec.getErr(); + try { + dbgWarning(D_LOCAL_POLICY + ) << "Failed to retrieve Appsec policy with crds version: v1beta1, Trying version: v1beta2"; + auto maybe_v1beta2_appsec_policy_spec = getObjectFromCluster>( + "/apis/openappsec.io/v1beta2/policies/" + policy_name + ); + if (!maybe_v1beta2_appsec_policy_spec.ok()) { + dbgWarning(D_LOCAL_POLICY) + << "Failed to retrieve AppSec policy. Error: " << maybe_v1beta2_appsec_policy_spec.getErr(); + return std::make_tuple( + genError("Failed to retrieve AppSec v1beta1 policy. Error: " + maybe_appsec_policy_spec.getErr()), + genError( + "Failed to retrieve AppSec v1beta2 policy. Error: " + maybe_v1beta2_appsec_policy_spec.getErr() + ) + ); + } return std::make_tuple( - genError("Failed to retrieve AppSec v1beta1 policy. Error: " + maybe_appsec_policy_spec.getErr()), - genError( - "Failed to retrieve AppSec v1beta2 policy. Error: " + maybe_v1beta2_appsec_policy_spec.getErr())); + genError("There is no v1beta1 policy"), + createAppsecPolicyK8sFromV1beta2Crds(maybe_v1beta2_appsec_policy_spec.unpack(), ingress_mode) + ); + } catch (const PolicyGenException &e) { + dbgDebug(D_LOCAL_POLICY) << "Failed in policy generation. Error: " << e.what(); + return std::make_tuple( + genError("There is no v1beta1 policy"), + genError("Failed to retrieve AppSec v1beta2 policy. Error: " + string(e.what())) + ); } - return std::make_tuple( - genError("There is no v1beta1 policy"), - createAppsecPolicyK8sFromV1beta2Crds(maybe_v1beta2_appsec_policy_spec.unpack(), ingress_mode)); } return std::make_tuple( @@ -521,22 +531,22 @@ K8sPolicyUtils::createPolicy( const SingleIngressData &item) const { for (const IngressDefinedRule &rule : item.getSpec().getRules()) { - string url = rule.getHost(); - for (const IngressRulePath &uri : rule.getPathsWrapper().getRulePaths()) { - if (!appsec_policy.getAppsecPolicySpec().isAssetHostExist(url + uri.getPath())) { - dbgTrace(D_LOCAL_POLICY) - << "Inserting Host data to the specific asset set:" - << "URL: '" - << url - << "' uri: '" - << uri.getPath() - << "'"; - K ingress_rule = K(url + uri.getPath()); - appsec_policy.addSpecificRule(ingress_rule); - } + string url = rule.getHost(); + for (const IngressRulePath &uri : rule.getPathsWrapper().getRulePaths()) { + if (!appsec_policy.getAppsecPolicySpec().isAssetHostExist(url + uri.getPath())) { + dbgTrace(D_LOCAL_POLICY) + << "Inserting Host data to the specific asset set:" + << "URL: '" + << url + << "' uri: '" + << uri.getPath() + << "'"; + K ingress_rule = K(url + uri.getPath()); + appsec_policy.addSpecificRule(ingress_rule); } } - policies[annotations_values[AnnotationKeys::PolicyKey]] = appsec_policy; + } + policies[annotations_values[AnnotationKeys::PolicyKey]] = appsec_policy; } diff --git a/components/security_apps/local_policy_mgmt_gen/new_appsec_policy_crd_parser.cc b/components/security_apps/local_policy_mgmt_gen/new_appsec_policy_crd_parser.cc index 42b1151..a040f65 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_appsec_policy_crd_parser.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_appsec_policy_crd_parser.cc @@ -26,13 +26,16 @@ NewParsedRule::load(cereal::JSONInputArchive &archive_in) dbgTrace(D_LOCAL_POLICY) << "Loading AppSec NewParsedRule"; parseAppsecJSONKey>("exceptions", exceptions, archive_in); parseAppsecJSONKey>("triggers", log_triggers, archive_in); - parseAppsecJSONKey>("threatPreventionPractices", threat_prevention_practices, archive_in); - parseAppsecJSONKey>("accessControlPractices", access_control_practices, archive_in); + parseMandatoryAppsecJSONKey>( + "threatPreventionPractices", + threat_prevention_practices, + archive_in); + parseMandatoryAppsecJSONKey>("accessControlPractices", access_control_practices, archive_in); parseAppsecJSONKey("mode", mode, archive_in); if (valid_modes.count(mode) == 0) { - dbgWarning(D_LOCAL_POLICY) << "AppSec New Parsed Rule mode invalid: " << mode; + throw PolicyGenException("AppSec New Parsed Rule mode invalid: " + mode); } - parseAppsecJSONKey("customResponse", custom_response, archive_in); + parseAppsecJSONKey("customResponse", custom_response, archive_in, "403"); parseAppsecJSONKey("sourceIdentifiers", source_identifiers, archive_in); parseAppsecJSONKey("trustedSources", trusted_sources, archive_in); parseAppsecJSONKey("autoUpgrade", upgrade_settings, archive_in); diff --git a/components/security_apps/local_policy_mgmt_gen/new_custom_response.cc b/components/security_apps/local_policy_mgmt_gen/new_custom_response.cc index d8f815b..61cbafc 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_custom_response.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_custom_response.cc @@ -21,7 +21,11 @@ using namespace std; USE_DEBUG_FLAG(D_LOCAL_POLICY); // LCOV_EXCL_START Reason: no test exist -static const set valid_modes = {"block-page", "response-code-only", "redirect"}; +static const map mode_to_appsec_mode_val = { + {"block-page", "Redirect"}, + {"response-code-only", "Response Code"}, + {"redirect", "Redirect"} +}; void NewAppSecCustomResponse::load(cereal::JSONInputArchive &archive_in) @@ -32,13 +36,10 @@ NewAppSecCustomResponse::load(cereal::JSONInputArchive &archive_in) if (http_response_code < MIN_RESPONSE_CODE || http_response_code > MAX_RESPOMSE_CODE) { dbgWarning(D_LOCAL_POLICY) << "AppSec web user response code invalid: " << http_response_code; } - parseAppsecJSONKey("mode", mode, archive_in, "block-page"); - if (valid_modes.count(mode) == 0) { - dbgWarning(D_LOCAL_POLICY) << "AppSec web user response mode invalid: " << mode; - } + parseMandatoryAppsecJSONKey("mode", mode, archive_in, "response-code-only"); parseAppsecJSONKey("name", name, archive_in); parseAppsecJSONKey("redirectUrl", redirect_url, archive_in); - parseAppsecJSONKey("redirectAddXEventId", redirect_add_x_event_id, archive_in); + parseAppsecJSONKey("redirectAddXEventId", redirect_add_x_event_id, archive_in, false); if (mode == "block-page") { parseAppsecJSONKey( "messageBody", @@ -53,6 +54,12 @@ NewAppSecCustomResponse::load(cereal::JSONInputArchive &archive_in) "Attack blocked by web application protection" ); } + if (mode_to_appsec_mode_val.find(mode) == mode_to_appsec_mode_val.end()) { + dbgWarning(D_LOCAL_POLICY) << "AppSec web user response mode invalid: " << mode; + mode = "Response Code"; + } else { + mode = mode_to_appsec_mode_val.at(mode); + } } void diff --git a/components/security_apps/local_policy_mgmt_gen/new_exceptions.cc b/components/security_apps/local_policy_mgmt_gen/new_exceptions.cc index b3b19a4..8a2ae01 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_exceptions.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_exceptions.cc @@ -23,9 +23,9 @@ static const set valid_actions = {"skip", "accept", "drop", "suppressLog void NewAppsecExceptionCondition::load(cereal::JSONInputArchive &archive_in) { - parseAppsecJSONKey("key", key, archive_in); - parseAppsecJSONKey("value", value, archive_in); - dbgTrace(D_LOCAL_POLICY) << "Key: " << key << " Value: " << value; + parseMandatoryAppsecJSONKey("key", key, archive_in); + parseMandatoryAppsecJSONKey("value", value, archive_in); + dbgTrace(D_LOCAL_POLICY) << "Parsed exception condition: Key: " << key << " Value: " << value; } const string & @@ -45,12 +45,17 @@ NewAppsecException::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading New AppSec exception"; parseAppsecJSONKey("name", name, archive_in, "exception"); - parseAppsecJSONKey("action", action, archive_in); + parseMandatoryAppsecJSONKey("action", action, archive_in, "accept"); parseAppsecJSONKey("appsecClassName", appsec_class_name, archive_in); if (valid_actions.count(action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec exception action invalid: " << action; + action = "accept"; + } + parseMandatoryAppsecJSONKey>("condition", conditions, archive_in); + if (conditions.empty()) { + dbgWarning(D_LOCAL_POLICY) << "AppSec exception conditions empty"; + throw PolicyGenException("AppSec exception conditions empty"); } - parseAppsecJSONKey>("condition", conditions, archive_in); } void diff --git a/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc b/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc index 1e1dc8d..5727c80 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc @@ -43,6 +43,10 @@ NewAppsecTriggerAdditionalSuspiciousEventsLogging::load(cereal::JSONInputArchive dbgWarning(D_LOCAL_POLICY) << "AppSec AppSec Trigger - Additional Suspicious Events Logging minimum severity invalid: " << minimum_severity; + throw PolicyGenException( + "AppSec AppSec Trigger - Additional Suspicious Events Logging minimum severity invalid: " + + minimum_severity + ); } } @@ -132,6 +136,7 @@ NewLoggingService::load(cereal::JSONInputArchive &archive_in) parseAppsecJSONKey("proto", proto, archive_in); if (valid_protocols.count(proto) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Logging Service - proto invalid: " << proto; + throw PolicyGenException("AppSec Logging Service - proto invalid: " + proto); } parseAppsecJSONKey("port", port, archive_in, 514); @@ -156,6 +161,7 @@ NewStdoutLogging::load(cereal::JSONInputArchive &archive_in) parseAppsecJSONKey("format", format, archive_in, "json"); if (valid_formats.count(format) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Stdout Logging - format invalid: " << format; + throw PolicyGenException("AppSec Stdout Logging - format invalid: " + format); } } @@ -261,19 +267,19 @@ NewAppsecLogTrigger::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec log trigger"; parseAppsecJSONKey("appsecClassName", appsec_class_name, archive_in); - parseAppsecJSONKey( + parseMandatoryAppsecJSONKey( "accessControlLogging", access_control_logging, archive_in ); - parseAppsecJSONKey( + parseMandatoryAppsecJSONKey( "additionalSuspiciousEventsLogging", additional_suspicious_events_logging, archive_in ); - parseAppsecJSONKey("appsecLogging", appsec_logging, archive_in); - parseAppsecJSONKey("extendedLogging", extended_logging, archive_in); - parseAppsecJSONKey("logDestination", log_destination, archive_in); + parseMandatoryAppsecJSONKey("appsecLogging", appsec_logging, archive_in); + parseMandatoryAppsecJSONKey("extendedLogging", extended_logging, archive_in); + parseMandatoryAppsecJSONKey("logDestination", log_destination, archive_in); parseAppsecJSONKey("name", name, archive_in); } diff --git a/components/security_apps/local_policy_mgmt_gen/new_practice.cc b/components/security_apps/local_policy_mgmt_gen/new_practice.cc index c612319..36e74d1 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_practice.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_practice.cc @@ -162,11 +162,12 @@ NewAppSecPracticeWebAttacks::load(cereal::JSONInputArchive &archive_in) } if (getMode() == "Prevent") { - parseAppsecJSONKey("minimumConfidence", minimum_confidence, archive_in, "critical"); + parseMandatoryAppsecJSONKey("minimumConfidence", minimum_confidence, archive_in, "critical"); if (valid_confidences.count(minimum_confidence) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec practice override minimum confidence invalid: " << minimum_confidence; + throw PolicyGenException("AppSec practice override minimum confidence invalid: " + minimum_confidence); } } else { minimum_confidence = "Transparent"; @@ -440,11 +441,12 @@ NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Snort Signatures practice"; parseAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); - parseAppsecJSONKey>("configmap", config_map, archive_in); + parseMandatoryAppsecJSONKey>("configmap", config_map, archive_in); parseAppsecJSONKey>("files", files, archive_in); is_temporary = false; if (valid_modes.count(override_mode) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode; + throw PolicyGenException("AppSec Snort Signatures override mode invalid: " + override_mode); } } @@ -567,12 +569,16 @@ NewIntrusionPrevention::load(cereal::JSONInputArchive &archive_in) parseAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); if (valid_modes.count(override_mode) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Intrusion Prevention override mode invalid: " << override_mode; + throw PolicyGenException("AppSec Intrusion Prevention override mode invalid: " + override_mode); } - parseAppsecJSONKey("maxPerformanceImpact", max_performance_impact, archive_in, "low"); + parseAppsecJSONKey("maxPerformanceImpact", max_performance_impact, archive_in, "medium"); if (performance_impacts.count(max_performance_impact) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Intrusion Prevention max performance impact invalid: " << max_performance_impact; + throw PolicyGenException( + "AppSec Intrusion Prevention max performance impact invalid: " + max_performance_impact + ); } parseAppsecJSONKey("minSeverityLevel", min_severity_level, archive_in, "low"); if (severity_levels.count(min_severity_level) == 0) { @@ -580,23 +586,32 @@ NewIntrusionPrevention::load(cereal::JSONInputArchive &archive_in) << "AppSec Intrusion Prevention min severity level invalid: " << min_severity_level; } - parseAppsecJSONKey("highConfidenceEventAction", high_confidence_event_action, archive_in, "inactive"); + parseAppsecJSONKey("highConfidenceEventAction", high_confidence_event_action, archive_in, "prevent"); if (confidences_actions.count(high_confidence_event_action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Intrusion Prevention high confidence event invalid: " << high_confidence_event_action; + throw PolicyGenException( + "AppSec Intrusion Prevention high confidence event invalid: " + high_confidence_event_action + ); } - parseAppsecJSONKey("mediumConfidenceEventAction", medium_confidence_event_action, archive_in, "inactive"); + parseAppsecJSONKey("mediumConfidenceEventAction", medium_confidence_event_action, archive_in, "prevent"); if (confidences_actions.count(medium_confidence_event_action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Intrusion Prevention medium confidence event invalid: " << medium_confidence_event_action; + throw PolicyGenException( + "AppSec Intrusion Prevention medium confidence event invalid: " + medium_confidence_event_action + ); } - parseAppsecJSONKey("lowConfidenceEventAction", low_confidence_event_action, archive_in, "inactive"); + parseAppsecJSONKey("lowConfidenceEventAction", low_confidence_event_action, archive_in, "detect"); if (confidences_actions.count(low_confidence_event_action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Intrusion Prevention low confidence event action invalid: " << low_confidence_event_action; + throw PolicyGenException( + "AppSec Intrusion Prevention low confidence event action invalid: " + low_confidence_event_action + ); } parseAppsecJSONKey("minCveYear", min_cve_Year, archive_in); } @@ -733,29 +748,36 @@ void NewFileSecurityArchiveInspection::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec File Security Archive Inspection practice"; - parseAppsecJSONKey("extractArchiveFiles", extract_archive_files, archive_in); - parseAppsecJSONKey("scanMaxFileSize", scan_max_file_size, archive_in, 0); - parseAppsecJSONKey("scanMaxFileSizeUnit", scan_max_file_size_unit, archive_in, "bytes"); + parseAppsecJSONKey("extractArchiveFiles", extract_archive_files, archive_in, true); + parseAppsecJSONKey("scanMaxFileSize", scan_max_file_size, archive_in, 10); + parseAppsecJSONKey("scanMaxFileSizeUnit", scan_max_file_size_unit, archive_in, "MB"); if (size_unit.count(scan_max_file_size_unit) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security Archive Inspection scan max file size unit invalid: " << scan_max_file_size_unit; + throw PolicyGenException( + "AppSec File Security Archive Inspection scan max file size unit invalid: " + scan_max_file_size_unit + ); } parseAppsecJSONKey( "archivedFilesWithinArchivedFiles", archived_files_within_archived_files, archive_in, - "inactive"); + "prevent"); if (confidences_actions.count(archived_files_within_archived_files) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security Archive Inspection archived files within archived files invalid: " << archived_files_within_archived_files; + throw PolicyGenException( + "AppSec File Security Archive Inspection archived files within archived files invalid: " + + archived_files_within_archived_files + ); } parseAppsecJSONKey( "archivedFilesWhereContentExtractionFailed", archived_files_where_content_extraction_failed, archive_in, - "inactive"); + "prevent"); if (confidences_actions.count(archived_files_where_content_extraction_failed) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security Archive Inspection archived files within archived file invalid: " @@ -798,22 +820,29 @@ void NewFileSecurityLargeFileInspection::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec File Security large File Inspection practice"; - parseAppsecJSONKey("fileSizeLimit", file_size_limit, archive_in); - parseAppsecJSONKey("fileSizeLimitUnit", file_size_limit_unit, archive_in, "bytes"); + parseAppsecJSONKey("fileSizeLimit", file_size_limit, archive_in, 10); + parseAppsecJSONKey("fileSizeLimitUnit", file_size_limit_unit, archive_in, "MB"); if (size_unit.count(file_size_limit_unit) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security large File Inspection file size limit unit invalid: " << file_size_limit_unit; + throw PolicyGenException( + "AppSec File Security large File Inspection file size limit unit invalid: " + file_size_limit_unit + ); } parseAppsecJSONKey( "filesExceedingSizeLimitAction", files_exceeding_size_limit_action, archive_in, - "inactive"); + "prevent"); if (confidences_actions.count(files_exceeding_size_limit_action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security Archive Inspection archived files within archived files invalid: " << files_exceeding_size_limit_action; + throw PolicyGenException( + "AppSec File Security Archive Inspection archived files within archived files invalid: " + + files_exceeding_size_limit_action + ); } } @@ -843,38 +872,52 @@ NewFileSecurity::load(cereal::JSONInputArchive &archive_in) parseAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); if (valid_modes.count(override_mode) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security override mode invalid: " << override_mode; + throw PolicyGenException("AppSec File Security override mode invalid: " + override_mode); } - parseAppsecJSONKey("minSeverityLevel", min_severity_level, archive_in, "low"); + parseMandatoryAppsecJSONKey("minSeverityLevel", min_severity_level, archive_in, "low"); if (severity_levels.count(min_severity_level) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security min severity level invalid: " << min_severity_level; + min_severity_level = "low"; } - parseAppsecJSONKey("highConfidenceEventAction", high_confidence_event_action, archive_in, "inactive"); + parseMandatoryAppsecJSONKey( + "highConfidenceEventAction", high_confidence_event_action, archive_in, "inactive" + ); if (confidences_actions.count(high_confidence_event_action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security high confidence event invalid: " << high_confidence_event_action; + high_confidence_event_action = "inactive"; } - parseAppsecJSONKey("mediumConfidenceEventAction", medium_confidence_event_action, archive_in, "inactive"); + parseMandatoryAppsecJSONKey( + "mediumConfidenceEventAction", medium_confidence_event_action, archive_in, "inactive" + ); if (confidences_actions.count(medium_confidence_event_action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security medium confidence event invalid: " << medium_confidence_event_action; + medium_confidence_event_action = "inactive"; } - parseAppsecJSONKey("lowConfidenceEventAction", low_confidence_event_action, archive_in, "inactive"); + parseMandatoryAppsecJSONKey( + "lowConfidenceEventAction", low_confidence_event_action, archive_in, "inactive" + ); if (confidences_actions.count(low_confidence_event_action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security low confidence event action invalid: " << low_confidence_event_action; + low_confidence_event_action = "inactive"; } - parseAppsecJSONKey("unnamedFilesAction", unnamed_files_action, archive_in, "inactive"); + parseMandatoryAppsecJSONKey("unnamedFilesAction", unnamed_files_action, archive_in, "inactive"); if (confidences_actions.count(unnamed_files_action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security low unnamed files action invalid: " << unnamed_files_action; + unnamed_files_action = "inactive"; } parseAppsecJSONKey("threatEmulationEnabled", threat_emulation_enabled, archive_in); - parseAppsecJSONKey("archiveInspection", archive_inspection, archive_in); - parseAppsecJSONKey("largeFileInspection", large_file_inspection, archive_in); + parseMandatoryAppsecJSONKey("archiveInspection", archive_inspection, archive_in); + parseMandatoryAppsecJSONKey( + "largeFileInspection", large_file_inspection, archive_in + ); } const string & @@ -939,7 +982,7 @@ NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in) parseAppsecJSONKey("fileSecurity", file_security, archive_in); parseAppsecJSONKey("intrusionPrevention", intrusion_prevention, archive_in); parseAppsecJSONKey("snortSignatures", snort_signatures, archive_in); - parseAppsecJSONKey("webAttacks", web_attacks, archive_in); + parseMandatoryAppsecJSONKey("webAttacks", web_attacks, archive_in); parseAppsecJSONKey("antiBot", anti_bot, archive_in); parseAppsecJSONKey("name", practice_name, archive_in); } diff --git a/components/security_apps/local_policy_mgmt_gen/new_trusted_sources.cc b/components/security_apps/local_policy_mgmt_gen/new_trusted_sources.cc index ef29d66..d9d2866 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_trusted_sources.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_trusted_sources.cc @@ -25,8 +25,8 @@ NewTrustedSourcesSpec::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading trusted sources spec"; parseAppsecJSONKey("appsecClassName", appsec_class_name, archive_in); - parseAppsecJSONKey("minNumOfSources", min_num_of_sources, archive_in, 3); - parseAppsecJSONKey>("sourcesIdentifiers", sources_identifiers, archive_in); + parseMandatoryAppsecJSONKey("minNumOfSources", min_num_of_sources, archive_in, 3); + parseMandatoryAppsecJSONKey>("sourcesIdentifiers", sources_identifiers, archive_in); parseAppsecJSONKey("name", name, archive_in); } @@ -64,11 +64,12 @@ void Identifier::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading source identifiers spec"; - parseAppsecJSONKey("identifier", identifier, archive_in); + parseMandatoryAppsecJSONKey("identifier", identifier, archive_in, "sourceip"); if (valid_identifiers.count(identifier) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec identifier invalid: " << identifier; + identifier = "sourceip"; } - parseAppsecJSONKey>("value", value, archive_in); + parseMandatoryAppsecJSONKey>("value", value, archive_in); } const string & @@ -88,7 +89,11 @@ NewSourcesIdentifiers::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading Sources Identifiers"; parseAppsecJSONKey("appsecClassName", appsec_class_name, archive_in); - parseAppsecJSONKey>("sourcesIdentifiers", sources_identifiers, archive_in); + parseMandatoryAppsecJSONKey>("sourcesIdentifiers", sources_identifiers, archive_in); + if (sources_identifiers.empty()) { + dbgWarning(D_LOCAL_POLICY) << "AppSec sources identifiers empty"; + throw PolicyGenException("AppSec sources identifiers empty"); + } parseAppsecJSONKey("name", name, archive_in); } diff --git a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc index d94a38a..442a5ac 100755 --- a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc @@ -1649,36 +1649,39 @@ PolicyMakerUtils::proccesSingleAppsecPolicy( const string &local_appsec_policy_path) { - Maybe maybe_policy_v1beta2 = openFileAsJson(policy_path); - if (maybe_policy_v1beta2.ok()) { - policy_version_name = "v1beta2"; - createAgentPolicyFromAppsecPolicy( - getPolicyName(policy_path), - maybe_policy_v1beta2.unpack() - ); - } else { - policy_version_name = "v1beta1"; - dbgInfo(D_LOCAL_POLICY) - << "Failed to retrieve AppSec local policy with version: v1beta2, Trying version: v1beta1"; + try { + Maybe maybe_policy_v1beta2 = openFileAsJson(policy_path); + if (maybe_policy_v1beta2.ok()) { + policy_version_name = "v1beta2"; + createAgentPolicyFromAppsecPolicy( + getPolicyName(policy_path), maybe_policy_v1beta2.unpack() + ); + } else { + policy_version_name = "v1beta1"; + dbgInfo(D_LOCAL_POLICY + ) << "Failed to retrieve AppSec local policy with version: v1beta2, Trying version: v1beta1"; - Maybe maybe_policy_v1beta1 = openFileAsJson(policy_path); - if (!maybe_policy_v1beta1.ok()){ - dbgWarning(D_LOCAL_POLICY) << maybe_policy_v1beta1.getErr(); - return ""; + Maybe maybe_policy_v1beta1 = openFileAsJson(policy_path); + if (!maybe_policy_v1beta1.ok()) { + dbgWarning(D_LOCAL_POLICY) << maybe_policy_v1beta1.getErr(); + return ""; + } + createAgentPolicyFromAppsecPolicy( + getPolicyName(policy_path), maybe_policy_v1beta1.unpack() + ); + + if (getenv("OPENAPPSEC_STANDALONE")) rpmBuildNginxServers(maybe_policy_v1beta1.unpack()); } - createAgentPolicyFromAppsecPolicy( - getPolicyName(policy_path), - maybe_policy_v1beta1.unpack() + + PolicyWrapper policy_wrapper = combineElementsToPolicy(policy_version); + return dumpPolicyToFile( + policy_wrapper, + local_appsec_policy_path ); - - if (getenv("OPENAPPSEC_STANDALONE")) rpmBuildNginxServers(maybe_policy_v1beta1.unpack()); + } catch (const PolicyGenException &e) { + dbgDebug(D_LOCAL_POLICY) << "Policy generation failed. Error: " << e.what(); + return ""; } - - PolicyWrapper policy_wrapper = combineElementsToPolicy(policy_version); - return dumpPolicyToFile( - policy_wrapper, - local_appsec_policy_path - ); } void diff --git a/components/security_apps/local_policy_mgmt_gen/rules_config_section.cc b/components/security_apps/local_policy_mgmt_gen/rules_config_section.cc index 699bca8..c001620 100755 --- a/components/security_apps/local_policy_mgmt_gen/rules_config_section.cc +++ b/components/security_apps/local_policy_mgmt_gen/rules_config_section.cc @@ -174,8 +174,12 @@ RulesConfigRulebase::RulesConfigRulebase( context ="All()"; return; } - string host_check = "Any(EqualHost(" + _url + ")),"; - string uri_check = (_uri.empty() || _uri == "/" ) ? "" : ",BeginWithUri(" + _uri + ")"; + bool uri_regex = false; + if (std::find(_uri.begin(), _uri.end(), '*') != _uri.end()) { + uri_regex = true; + } + string host_check = (_url.empty() || _url == "/") ? "" : "Any(EqualHost(" + _url + ")),"; + string uri_check = (_uri.empty() || _uri == "/" || uri_regex ) ? "" : ",BeginWithUri(" + _uri + ")"; auto ports = _port.empty() ? vector({"80", "443"}) : vector({_port}); context = "Any("; for (auto &port : ports) { diff --git a/components/security_apps/local_policy_mgmt_gen/settings_section.cc b/components/security_apps/local_policy_mgmt_gen/settings_section.cc index f066004..2dcba43 100755 --- a/components/security_apps/local_policy_mgmt_gen/settings_section.cc +++ b/components/security_apps/local_policy_mgmt_gen/settings_section.cc @@ -52,15 +52,14 @@ void SettingsRulebase::save(cereal::JSONOutputArchive &out_ar) const { string profile_type = "Kubernetes"; - string upgrade_mode = "automatic"; out_ar( cereal::make_nvp("agentSettings", agentSettings), cereal::make_nvp("agentType", profile_type), cereal::make_nvp("allowOnlyDefinedApplications", false), cereal::make_nvp("anyFog", true), - cereal::make_nvp("maxNumberOfAgents", 10), - cereal::make_nvp("upgradeMode", upgrade_mode) + cereal::make_nvp("maxNumberOfAgents", 10) ); + upgrade_settings.save(out_ar); } SettingsWrapper::SettingsWrapper(SettingsRulebase _agent) : agent(_agent) diff --git a/components/security_apps/orchestration/details_resolver/details_resolver.cc b/components/security_apps/orchestration/details_resolver/details_resolver.cc index d1debfa..8ef831a 100644 --- a/components/security_apps/orchestration/details_resolver/details_resolver.cc +++ b/components/security_apps/orchestration/details_resolver/details_resolver.cc @@ -41,7 +41,7 @@ public: string getAgentVersion() override; bool isKernelVersion3OrHigher() override; bool isGwNotVsx() override; - bool isVersionEqualOrAboveR8110() override; + bool isVersionAboveR8110() override; bool isReverseProxy() override; Maybe> parseNginxMetadata() override; #if defined(gaia) || defined(smb) @@ -193,10 +193,12 @@ DetailsResolver::Impl::getCheckpointVersion() const #endif // gaia || smb bool -DetailsResolver::Impl::isVersionEqualOrAboveR8110() +DetailsResolver::Impl::isVersionAboveR8110() { -#if defined(gaia) || defined(smb) - return compareCheckpointVersion(8110, std::greater_equal()); +#if defined(gaia) + return compareCheckpointVersion(8110, std::greater()); +#elif defined(smb) + return true; #endif return false; } diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h index bb1b700..e655ca4 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h @@ -20,7 +20,7 @@ #if defined(gaia) Maybe -checkHasSupportedBlade(const string &command_output) +checkSAMLSupportedBlade(const string &command_output) { string supportedBlades[3] = {"identityServer", "vpn", "cvpn"}; for(const string &blade : supportedBlades) { @@ -29,11 +29,11 @@ checkHasSupportedBlade(const string &command_output) } } - return genError("Current host does not have IDA capability"); + return genError("Current host does not have SAML capability"); } Maybe -checkSamlPortal(const string &command_output) +checkSAMLPortal(const string &command_output) { if (command_output.find("Portal is running") != string::npos) { return string("true"); @@ -43,9 +43,9 @@ checkSamlPortal(const string &command_output) } Maybe -getIDAGaia(const string &command_output) +getIDASSamlGaia(const string &command_output) { - return string("ida_gaia"); + return string("idaSaml_gaia"); } Maybe @@ -211,6 +211,15 @@ getClusterObjectIP(const string &command_output) return getAttr(command_output, "Cluster object IP was not found"); } +Maybe +getFecApplicable(const string &command_output) +{ + if (command_output == "0") return string("true"); + if (command_output == "1") return string("false"); + + return genError("Could not determine if fec applicable"); +} + Maybe getSmbObjectName(const string &command_output) { diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index f38dac1..5494e35 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -27,6 +27,15 @@ // use SHELL_CMD_HANDLER(key as string, shell command as string, ptr to Maybe handler(const string&)) // to return a string value for an attribute key based on a logic executed in a handler that receives // shell command execution output as its input + +#ifdef SHELL_PRE_CMD +#if defined(gaia) || defined(smb) +SHELL_PRE_CMD("read sdwan data", + "(cpsdwan get_data > /tmp/cpsdwan_getdata_orch.json~) " + "&& (mv /tmp/cpsdwan_getdata_orch.json~ /tmp/cpsdwan_getdata_orch.json)") +#endif +#endif + #ifdef SHELL_CMD_HANDLER #if defined(gaia) || defined(smb) SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType) @@ -41,7 +50,7 @@ SHELL_CMD_HANDLER("isCPotelcolGRET64", SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan) SHELL_CMD_HANDLER( "canUpdateSDWanData", - "CPSDWAN_NOLOGS=1 cpsdwan get_data -f can_update_sdwan_data | jq -r .can_update_sdwan_data", + "jq -r .can_update_sdwan_data /tmp/cpsdwan_getdata_orch.json", checkCanUpdateSDWanData ) SHELL_CMD_HANDLER( @@ -50,7 +59,8 @@ SHELL_CMD_HANDLER( checkIfSdwanRunning) SHELL_CMD_HANDLER( "IP Address", - "cpsdwan get_data | jq -r .main_ip", + "[ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" " + "|| (jq -r .main_ip /tmp/cpsdwan_getdata_orch.json)", getGWIPAddress ) SHELL_CMD_HANDLER( @@ -60,18 +70,23 @@ SHELL_CMD_HANDLER( ) SHELL_CMD_HANDLER( "cpProductIntegrationMgmtParentObjectIP", - "obj=\"$(cpsdwan get_data | jq -r .cluster_name)\";" + "obj=\"$(jq -r .cluster_name /tmp/cpsdwan_getdata_orch.json)\";" " awk -v obj=\"$obj\" '$1 == \":\" && $2 == \"(\" obj, $1 == \":ip_address\" { if ($1 == \":ip_address\")" " { gsub(/[()]/, \"\", $2); print $2; exit; } }'" " $FWDIR/state/local/FW1/local.gateway_cluster", getClusterObjectIP ) +SHELL_CMD_HANDLER( + "isFecApplicable", + "fw ctl get int support_fec |& grep -sq \"support_fec =\";echo $?", + getFecApplicable +) #endif //gaia || smb #if defined(gaia) -SHELL_CMD_HANDLER("hasSupportedBlade", "enabled_blades", checkHasSupportedBlade) -SHELL_CMD_HANDLER("hasSamlPortal", "mpclient status saml-vpn", checkSamlPortal) -SHELL_CMD_HANDLER("requiredNanoServices", "ida_gaia", getIDAGaia) +SHELL_CMD_HANDLER("hasSAMLSupportedBlade", "enabled_blades", checkSAMLSupportedBlade) +SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status saml-vpn", checkSAMLPortal) +SHELL_CMD_HANDLER("requiredNanoServices", "ida_saml_gaia", getIDASSamlGaia) SHELL_CMD_HANDLER( "cpProductIntegrationMgmtParentObjectName", "cat $FWDIR/database/myself_objects.C " @@ -109,12 +124,12 @@ SHELL_CMD_HANDLER( #if defined(smb) SHELL_CMD_HANDLER( "cpProductIntegrationMgmtParentObjectName", - "cpsdwan get_data | jq -r .cluster_name", + "jq -r .cluster_name /tmp/cpsdwan_getdata_orch.json", getSmbMgmtParentObjName ) SHELL_CMD_HANDLER( "cpProductIntegrationMgmtParentObjectUid", - "cpsdwan get_data | jq -r .cluster_uuid", + "jq -r .cluster_uuid /tmp/cpsdwan_getdata_orch.json", getSmbMgmtParentObjUid ) SHELL_CMD_HANDLER( @@ -150,7 +165,11 @@ SHELL_CMD_OUTPUT("helloWorld", "cat /tmp/agentHelloWorld 2>/dev/null") #if defined(gaia) -FILE_CONTENT_HANDLER("hasIdpConfigured", "/opt/CPSamlPortal/phpincs/spPortal/idpPolicy.xml", checkIDP) +FILE_CONTENT_HANDLER( + "hasIdpConfigured", + (getenv("SAMLPORTAL_HOME") ? string(getenv("SAMLPORTAL_HOME")) : "") + "/phpincs/spPortal/idpPolicy.xml", + checkIDP +) FILE_CONTENT_HANDLER( "cpProductIntegrationMgmtObjectName", (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C", diff --git a/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc b/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc index c68e9a7..97830e1 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc +++ b/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc @@ -43,6 +43,12 @@ public: static Maybe getCommandOutput(const string &cmd); private: +#define SHELL_PRE_CMD(NAME, COMMAND) {NAME, COMMAND}, + map shell_pre_commands = { + #include "details_resolver_impl.h" + }; +#undef SHELL_PRE_CMD + #define SHELL_CMD_OUTPUT(ATTRIBUTE, COMMAND) SHELL_CMD_HANDLER(ATTRIBUTE, COMMAND, [](const string &s) { return s; }) #define SHELL_CMD_HANDLER(ATTRIBUTE, COMMAND, HANDLER) {ATTRIBUTE, {COMMAND, ShellCommandHandler(HANDLER)}}, map> shell_command_handlers = { @@ -61,6 +67,21 @@ private: map DetailsResolvingHanlder::Impl::getResolvedDetails() const { + I_ShellCmd *shell = Singleton::Consume::by(); + uint32_t timeout = getConfigurationWithDefault(5000, "orchestration", "Details resolver time out"); + + for (auto &shell_pre_command : shell_pre_commands) { + const string &name = shell_pre_command.first; + const string &command = shell_pre_command.second; + Maybe command_ret = shell->getExecReturnCode(command, timeout); + + if (!command_ret.ok()) { + dbgWarning(D_AGENT_DETAILS) << "Failed to run pre-command " << name; + } else if (*command_ret) { + dbgWarning(D_AGENT_DETAILS) << "Pre-command " << name << " failed (rc: " << *command_ret << ")"; + } + } + map resolved_details; for (auto shell_handler : shell_command_handlers) { const string &attr = shell_handler.first; @@ -116,7 +137,6 @@ DetailsResolvingHanlder::Impl::getCommandOutput(const string &cmd) DetailsResolvingHanlder::DetailsResolvingHanlder() : pimpl(make_unique()) {} DetailsResolvingHanlder::~DetailsResolvingHanlder() {} - map DetailsResolvingHanlder::getResolvedDetails() const { diff --git a/components/security_apps/orchestration/downloader/curl_client.cc b/components/security_apps/orchestration/downloader/curl_client.cc index 01ec1f6..8ef78e4 100755 --- a/components/security_apps/orchestration/downloader/curl_client.cc +++ b/components/security_apps/orchestration/downloader/curl_client.cc @@ -55,7 +55,7 @@ HttpCurl::HttpCurl( const string &_bearer, const Maybe &proxy_url, const Maybe &proxy_port, - const Maybe &proxy_auth) + const Maybe &_proxy_auth) : url(_url), out_file(_out_file), @@ -85,10 +85,10 @@ HttpCurl::HttpCurl( proxy = proxy_url.unpack() + ":" + to_string(proxy_port.unpack()); } } - if (proxy_auth.ok()) + if (_proxy_auth.ok()) { I_Encryptor *encryptor = Singleton::Consume::by(); - proxy_credentials = "Proxy-Authorization: Basic " + encryptor->base64Encode(proxy_auth.unpack()); + proxy_auth = "Proxy-Authorization: Basic " + encryptor->base64Encode(_proxy_auth.unpack()); } } @@ -98,7 +98,7 @@ HttpCurl::HttpCurl(const HttpCurl &other) out_file(other.out_file), bearer(other.bearer), proxy(other.proxy), - proxy_credentials(other.proxy_credentials), + proxy_auth(other.proxy_auth), curl(unique_ptr>(curl_easy_init(), curl_easy_cleanup)) { } @@ -133,9 +133,9 @@ HttpCurl::setCurlOpts(long timeout, HTTP_VERSION http_version) if (!proxy.empty()) { curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.c_str()); - if (!proxy_credentials.empty()) + if (!proxy_auth.empty()) { - proxy_headers = curl_slist_append(proxy_headers, proxy_credentials.c_str()); + proxy_headers = curl_slist_append(proxy_headers, proxy_auth.c_str()); //Apply proxy headers curl_easy_setopt(curl_handle, CURLOPT_PROXYHEADER, proxy_headers); } @@ -330,9 +330,9 @@ HttpsCurl::setCurlOpts(long timeout, HTTP_VERSION http_version) if (!proxy.empty()) { curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.c_str()); - if (!proxy_credentials.empty()) + if (!proxy_auth.empty()) { - proxy_headers = curl_slist_append(proxy_headers, proxy_credentials.c_str()); + proxy_headers = curl_slist_append(proxy_headers, proxy_auth.c_str()); //Apply proxy headers curl_easy_setopt(curl_handle, CURLOPT_PROXYHEADER, proxy_headers); } diff --git a/components/security_apps/orchestration/downloader/curl_client.h b/components/security_apps/orchestration/downloader/curl_client.h index 9104f29..e01efd4 100755 --- a/components/security_apps/orchestration/downloader/curl_client.h +++ b/components/security_apps/orchestration/downloader/curl_client.h @@ -83,7 +83,7 @@ protected: std::ofstream &out_file; std::string bearer; std::string proxy; - std::string proxy_credentials; + std::string proxy_auth; std::unique_ptr> curl; std::string curl_url; }; diff --git a/components/security_apps/orchestration/downloader/downloader.cc b/components/security_apps/orchestration/downloader/downloader.cc index d0e66de..71da685 100755 --- a/components/security_apps/orchestration/downloader/downloader.cc +++ b/components/security_apps/orchestration/downloader/downloader.cc @@ -81,6 +81,8 @@ public: const string &service_name ) const override; + Maybe checkIfFileExists(const Package &package) const override; + void removeDownloadFile(const string &file_name) const override; void createTenantProfileMap(); string getProfileFromMap(const string &tenant_id) const override; @@ -194,12 +196,18 @@ Downloader::Impl::downloadVirtualFileFromFog( static const string error_text = "error"; map, string> res; + + string general_file_path = dir_path + "/" + resourse_file.getFileName() + "_general.download"; I_UpdateCommunication *update_communication = Singleton::Consume::by(); - auto downloaded_data = update_communication->downloadAttributeFile(resourse_file); + auto downloaded_data = update_communication->downloadAttributeFile(resourse_file, general_file_path); if (!downloaded_data.ok()) return downloaded_data.passErr(); + I_OrchestrationTools *orchestration_tools = Singleton::Consume::by(); + Maybe file_content = orchestration_tools->readFile(general_file_path); + if (!file_content.ok()) return file_content.passErr(); + Document document; - document.Parse(downloaded_data.unpack().c_str()); + document.Parse(file_content.unpack().c_str()); if (document.HasParseError()) { dbgWarning(D_ORCHESTRATOR) << "JSON file is not valid"; return genError("JSON file is not valid."); @@ -241,7 +249,6 @@ Downloader::Impl::downloadVirtualFileFromFog( rapidjson::Writer writer(buffer); artifact_data->value.Accept(writer); - I_OrchestrationTools *orchestration_tools = Singleton::Consume::by(); if (orchestration_tools->writeFile(buffer.GetString(), file_path)) { res.insert({{tenant_id, profile_id}, file_path}); } @@ -324,6 +331,24 @@ Downloader::Impl::downloadFileFromURL( return file_path; } +Maybe +Downloader::Impl::checkIfFileExists(const Package &package) const +{ + string file_name = package.getName() + ".download"; + Maybe maybe_path = dir_path + "/" + file_name; + + return validateChecksum(package.getChecksum(), package.getChecksumType(), maybe_path); +} + +void +Downloader::Impl::removeDownloadFile(const string &file_name) const +{ + string file_path = dir_path + "/" + file_name + ".download"; + dbgInfo(D_ORCHESTRATOR) << "Removing download file " << file_path; + I_OrchestrationTools *orchestration_tools = Singleton::Consume::by(); + orchestration_tools->removeFile(file_path); +} + Maybe Downloader::Impl::validateChecksum( const string &checksum, @@ -355,13 +380,11 @@ Downloader::Impl::downloadFileFromFogByHTTP(const GetResourceFile &resourse_file dbgInfo(D_ORCHESTRATOR) << "Downloading file from fog. File: " << resourse_file.getFileName(); I_UpdateCommunication *update_communication = Singleton::Consume::by(); - auto downloaded_file = update_communication->downloadAttributeFile(resourse_file); + auto downloaded_file = update_communication->downloadAttributeFile(resourse_file, file_path); if (!downloaded_file.ok()) return genError(downloaded_file.getErr()); - dbgInfo(D_ORCHESTRATOR) << "Download completed. File: " << resourse_file.getFileName(); - I_OrchestrationTools *orchestration_tools = Singleton::Consume::by(); - if (orchestration_tools->writeFile(downloaded_file.unpack(), file_path)) return file_path; - return genError("Failed to write the attribute file. File: " + file_name); + dbgInfo(D_ORCHESTRATOR) << "Download completed. File: " << resourse_file.getFileName(); + return file_path; } Maybe diff --git a/components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc b/components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc index df7fa0b..e29d9ee 100755 --- a/components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc +++ b/components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc @@ -18,6 +18,7 @@ class DownloaderTest : public Test public: DownloaderTest() { + Debug::setUnitTestFlag(D_ORCHESTRATOR, Debug::DebugLevel::TRACE); setConfiguration("/tmp", "orchestration", "Default file download path"); EXPECT_CALL(mock_orchestration_tools, createDirectory("/tmp")).WillOnce(Return(true)); downloader.init(); @@ -44,15 +45,14 @@ TEST_F(DownloaderTest, downloadFileFromFog) GetResourceFile resourse_file(GetResourceFile::ResourceFileType::VIRTUAL_SETTINGS); - EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file)).WillOnce(Return(fog_response)); + EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file, "/tmp/virtualSettings.download")) + .WillOnce(Return(fog_response)); EXPECT_CALL( mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, "/tmp/virtualSettings.download") ).WillOnce(Return(string("123"))); - EXPECT_CALL(mock_orchestration_tools, writeFile(fog_response, "/tmp/virtualSettings.download", false)) - .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile("/tmp/virtualSettings.download")).WillOnce(Return(true)); Maybe downloaded_file = i_downloader->downloadFileFromFog( @@ -71,7 +71,10 @@ TEST_F(DownloaderTest, downloadFileFromFogFailure) Maybe fog_response(genError("Failed to download")); GetResourceFile resourse_file(GetResourceFile::ResourceFileType::SETTINGS); - EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file)).WillOnce(Return(fog_response)); + EXPECT_CALL( + mock_communication, + downloadAttributeFile(resourse_file, "/tmp/settings.download") + ).WillOnce(Return(fog_response)); Maybe downloaded_file = i_downloader->downloadFileFromFog( checksum, @@ -124,6 +127,53 @@ TEST_F(DownloaderTest, registerConfig) env.fini(); } +TEST_F(DownloaderTest, checkIfFileExists) +{ + string local_file_path = "/tmp/test_file.sh"; + string url = "file://" + local_file_path; + string dir_path = getConfigurationWithDefault( + "/tmp/orchestration_downloads", + "orchestration", + "Default file download path" + ); + string manifest = + "{" + " \"packages\": [" + " {" + " \"name\": \"test\"," + " \"version\": \"c\"," + " \"download-path\": \"http://172.23.92.135/my.sh\"," + " \"relative-path\": \"\"," + " \"checksum-type\": \"sha1sum\"," + " \"checksum\": \"1234\"," + " \"package-type\": \"service\"," + " \"require\": []" + " }" + " ]" + "}"; + + vector manifest_services; + std::stringstream os(manifest); + cereal::JSONInputArchive archive_in(os); + archive_in(manifest_services); + + string service_name = "test"; + string file_name = service_name + ".download"; + string file_path = dir_path + "/" + file_name; + string checksum = "1234"; + Package::ChecksumTypes checksum_type = Package::ChecksumTypes::SHA1; + + EXPECT_CALL(mock_orchestration_tools, calculateChecksum(checksum_type, file_path)).WillOnce(Return(checksum)); + i_downloader->checkIfFileExists(manifest_services[0]); +} + +TEST_F(DownloaderTest, removeDownloadFile) +{ + string file_path = "/tmp/package.download"; + EXPECT_CALL(mock_orchestration_tools, removeFile(file_path)).WillOnce(Return(true)); + i_downloader->removeDownloadFile("package"); +} + TEST_F(DownloaderTest, downloadWithBadChecksum) { string local_file_path = "/tmp/test_file.sh"; @@ -181,10 +231,9 @@ TEST_F(DownloaderTest, downloadEmptyFileFromFog) GetResourceFile resourse_file(GetResourceFile::ResourceFileType::MANIFEST); - EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file)).WillOnce(Return(fog_response)); + EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file, "/tmp/manifest.download")) + .WillOnce(Return(fog_response)); - EXPECT_CALL(mock_orchestration_tools, writeFile(fog_response, "/tmp/manifest.download", false)) - .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile("/tmp/manifest.download")).WillOnce(Return(false)); EXPECT_CALL( @@ -340,7 +389,13 @@ TEST_F(DownloaderTest, download_virtual_policy) " ]\n" "}"; - EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file)).WillOnce(Return(fog_response)); + EXPECT_CALL( + mock_communication, + downloadAttributeFile(resourse_file, "/tmp/virtualPolicy_general.download")) + .WillOnce(Return(fog_response)); + + EXPECT_CALL(mock_orchestration_tools, readFile("/tmp/virtualPolicy_general.download")) + .WillOnce(Return(fog_response)); EXPECT_CALL( mock_orchestration_tools, @@ -428,7 +483,15 @@ TEST_F(DownloaderTest, download_virtual_settings) " ]\n" "}"; - EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file)).WillOnce(Return(fog_response)); + EXPECT_CALL( + mock_communication, + downloadAttributeFile(resourse_file, "/tmp/virtualSettings_general.download")) + .WillOnce(Return(fog_response)); + + EXPECT_CALL( + mock_orchestration_tools, + readFile("/tmp/virtualSettings_general.download") + ).WillOnce(Return(fog_response)); stringstream tenant_0000_path; tenant_0000_path << "/tmp/virtualSettings_4c721b40-85df-4364-be3d-303a10ee9789" diff --git a/components/security_apps/orchestration/downloader/http_client.cc b/components/security_apps/orchestration/downloader/http_client.cc index 7447515..49a5475 100755 --- a/components/security_apps/orchestration/downloader/http_client.cc +++ b/components/security_apps/orchestration/downloader/http_client.cc @@ -221,7 +221,7 @@ HTTPClient::curlGetFileOverHttp(const URLParser &url, ofstream &out_file, const token, proxy_config->getProxyDomain(ProxyProtocol::HTTPS), proxy_config->getProxyPort(ProxyProtocol::HTTPS), - proxy_config->getProxyCredentials(ProxyProtocol::HTTPS)); + proxy_config->getProxyAuthentication(ProxyProtocol::HTTPS)); http_curl_client.setCurlOpts(); bool connection_ok = http_curl_client.connect(); @@ -251,7 +251,7 @@ HTTPClient::getFileHttp(const URLParser &url, ofstream &out_file, const string & url, proxy_config->getProxyDomain(ProxyProtocol::HTTP), proxy_config->getProxyPort(ProxyProtocol::HTTP), - proxy_config->getProxyCredentials(ProxyProtocol::HTTP), + proxy_config->getProxyAuthentication(ProxyProtocol::HTTP), token ); auto handle_connect_res = client_connection.handleConnect(); diff --git a/components/security_apps/orchestration/downloader/http_client.h b/components/security_apps/orchestration/downloader/http_client.h index 68a4bb0..ed01606 100755 --- a/components/security_apps/orchestration/downloader/http_client.h +++ b/components/security_apps/orchestration/downloader/http_client.h @@ -17,14 +17,12 @@ #include #include "maybe_res.h" #include "url_parser.h" -#include "i_messaging.h" #include "i_agent_details.h" #include "i_proxy_configuration.h" // LCOV_EXCL_START Reason: Depends on real download server. class HTTPClient : - public Singleton::Consume, public Singleton::Consume, public Singleton::Consume { diff --git a/components/security_apps/orchestration/downloader/https_client.cc b/components/security_apps/orchestration/downloader/https_client.cc index 33d9031..fb96a80 100755 --- a/components/security_apps/orchestration/downloader/https_client.cc +++ b/components/security_apps/orchestration/downloader/https_client.cc @@ -544,7 +544,7 @@ HTTPClient::getFileSSL(const URLParser &url, ofstream &out_file, const string &t url, proxy_config->getProxyDomain(ProxyProtocol::HTTPS), proxy_config->getProxyPort(ProxyProtocol::HTTPS), - proxy_config->getProxyCredentials(ProxyProtocol::HTTPS), + proxy_config->getProxyAuthentication(ProxyProtocol::HTTPS), token ); @@ -589,7 +589,7 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s token, proxy_config->getProxyDomain(ProxyProtocol::HTTPS), proxy_config->getProxyPort(ProxyProtocol::HTTPS), - proxy_config->getProxyCredentials(ProxyProtocol::HTTPS), + proxy_config->getProxyAuthentication(ProxyProtocol::HTTPS), cert_file_path); ssl_curl_client.setCurlOpts(); diff --git a/components/security_apps/orchestration/health_check/health_check_ut/CMakeLists.txt b/components/security_apps/orchestration/health_check/health_check_ut/CMakeLists.txt index f6ea8e9..8553708 100755 --- a/components/security_apps/orchestration/health_check/health_check_ut/CMakeLists.txt +++ b/components/security_apps/orchestration/health_check/health_check_ut/CMakeLists.txt @@ -3,5 +3,5 @@ link_directories(${BOOST_ROOT}/lib) add_unit_test( health_check_ut "health_check_ut.cc" - "health_check;mainloop;singleton;agent_details;config;logging;metric;event_is;health_check_manager;-lboost_regex;-lboost_system" + "health_check;messaging;mainloop;singleton;agent_details;config;logging;metric;event_is;health_check_manager;-lboost_regex;-lboost_system" ) diff --git a/components/security_apps/orchestration/include/fog_communication.h b/components/security_apps/orchestration/include/fog_communication.h index 8406077..e803d79 100755 --- a/components/security_apps/orchestration/include/fog_communication.h +++ b/components/security_apps/orchestration/include/fog_communication.h @@ -40,7 +40,10 @@ class FogCommunication : public FogAuthenticator public: void init() override; Maybe getUpdate(CheckUpdateRequest &request) override; - Maybe downloadAttributeFile(const GetResourceFile &resourse_file) override; + Maybe downloadAttributeFile( + const GetResourceFile &resourse_file, + const std::string &file_path + ) override; Maybe sendPolicyVersion( const std::string &policy_version, const std::string &policy_versions diff --git a/components/security_apps/orchestration/include/get_status_rest.h b/components/security_apps/orchestration/include/get_status_rest.h index 655389f..6084532 100755 --- a/components/security_apps/orchestration/include/get_status_rest.h +++ b/components/security_apps/orchestration/include/get_status_rest.h @@ -14,7 +14,6 @@ #ifndef __GET_STATUS_RES_H__ #define __GET_STATUS_RES_H__ -#include "i_messaging.h" #include "i_mainloop.h" #include "i_shell_cmd.h" #include "i_encryptor.h" diff --git a/components/security_apps/orchestration/include/hybrid_communication.h b/components/security_apps/orchestration/include/hybrid_communication.h index b74e1ae..9c383f6 100755 --- a/components/security_apps/orchestration/include/hybrid_communication.h +++ b/components/security_apps/orchestration/include/hybrid_communication.h @@ -45,7 +45,10 @@ class HybridCommunication public: void init() override; Maybe getUpdate(CheckUpdateRequest &request) override; - Maybe downloadAttributeFile(const GetResourceFile &resourse_file) override; + Maybe downloadAttributeFile( + const GetResourceFile &resourse_file, + const std::string &file_path + ) override; Maybe sendPolicyVersion( const std::string &policy_version, const std::string &policy_versions diff --git a/components/security_apps/orchestration/include/local_communication.h b/components/security_apps/orchestration/include/local_communication.h index f6dcb9f..189a90f 100755 --- a/components/security_apps/orchestration/include/local_communication.h +++ b/components/security_apps/orchestration/include/local_communication.h @@ -31,7 +31,10 @@ public: Maybe authenticateAgent() override; Maybe getUpdate(CheckUpdateRequest &request) override; - Maybe downloadAttributeFile(const GetResourceFile &resourse_file) override; + Maybe downloadAttributeFile( + const GetResourceFile &resourse_file, + const std::string &file_path + ) override; void setAddressExtenesion(const std::string &extension) override; Maybe sendPolicyVersion( const std::string &policy_version, diff --git a/components/security_apps/orchestration/include/mock/mock_details_resolver.h b/components/security_apps/orchestration/include/mock/mock_details_resolver.h index cc0cfd5..d08304d 100644 --- a/components/security_apps/orchestration/include/mock/mock_details_resolver.h +++ b/components/security_apps/orchestration/include/mock/mock_details_resolver.h @@ -39,7 +39,7 @@ public: MOCK_METHOD0(isKernelVersion3OrHigher, bool()); MOCK_METHOD0(isGwNotVsx, bool()); MOCK_METHOD0(getResolvedDetails, std::map()); - MOCK_METHOD0(isVersionEqualOrAboveR8110, bool()); + MOCK_METHOD0(isVersionAboveR8110, bool()); MOCK_METHOD0(parseNginxMetadata, Maybe>()); }; diff --git a/components/security_apps/orchestration/include/mock/mock_downloader.h b/components/security_apps/orchestration/include/mock/mock_downloader.h index 8b357bd..7b3fa94 100755 --- a/components/security_apps/orchestration/include/mock/mock_downloader.h +++ b/components/security_apps/orchestration/include/mock/mock_downloader.h @@ -41,6 +41,16 @@ public: Maybe(const std::string &, const std::string &, Package::ChecksumTypes, const std::string &) ); + MOCK_CONST_METHOD1( + checkIfFileExists, + Maybe(const Package &) + ); + + MOCK_CONST_METHOD1( + removeDownloadFile, + void(const std::string &) + ); + MOCK_CONST_METHOD1( getProfileFromMap, std::string(const std::string &) diff --git a/components/security_apps/orchestration/manifest_controller/manifest_controller.cc b/components/security_apps/orchestration/manifest_controller/manifest_controller.cc index d2336c8..a0662c8 100755 --- a/components/security_apps/orchestration/manifest_controller/manifest_controller.cc +++ b/components/security_apps/orchestration/manifest_controller/manifest_controller.cc @@ -13,6 +13,8 @@ #include "manifest_controller.h" +#include + #include "config.h" #include "debug.h" #include "environment.h" @@ -80,9 +82,8 @@ private: bool handlePackage( - const Package &updated_package, + const pair &package_downloaded_file, map ¤t_packages, - const map &new_packages, map &corrupted_packages ); @@ -179,6 +180,34 @@ ManifestController::Impl::updateIgnoreListForNSaaS() return true; } +static vector>::const_iterator +findPackage(const vector> &packages, const string &name) +{ + using Pair = pair; + return find_if(packages.begin(), packages.end(), [&] (const Pair &pair) { return pair.first.getName() == name; }); +} + +static vector> +sortByInstallationQueue( + const vector> &downloaded_files, + const vector &installation_queue) +{ + vector> sorted_queue; + for (auto &package_file : installation_queue) { + if (package_file.getName() == "accessControlApp" || package_file.getName() == "accessControlKernel") continue; + + auto package_it = findPackage(downloaded_files, package_file.getName()); + if (package_it != downloaded_files.end()) sorted_queue.push_back(*package_it); + } + + auto ac_app_it = findPackage(downloaded_files, "accessControlApp"); + auto ac_kernel_it = findPackage(downloaded_files, "accessControlKernel"); + if (ac_app_it != downloaded_files.end()) sorted_queue.push_back(*ac_app_it); + if (ac_kernel_it != downloaded_files.end()) sorted_queue.push_back(*ac_kernel_it); + + return sorted_queue; +} + bool ManifestController::Impl::updateManifest(const string &new_manifest_file) { @@ -220,6 +249,7 @@ ManifestController::Impl::updateManifest(const string &new_manifest_file) } map new_packages = parsed_manifest.unpack(); + map all_packages = parsed_manifest.unpack(); map current_packages; parsed_manifest = orchestration_tools->loadPackagesFromJson(manifest_file_path); @@ -256,13 +286,14 @@ ManifestController::Impl::updateManifest(const string &new_manifest_file) auto packages_to_remove = manifest_diff_calc.filterUntrackedPackages(current_packages, new_packages); for (auto remove_package = packages_to_remove.begin(); remove_package != packages_to_remove.end();) { bool uninstall_response = true; - if (remove_package->second.isInstallable().ok()) { + if (remove_package->second.isInstallable()) { uninstall_response = manifest_handler.uninstallPackage(remove_package->second); } if (!uninstall_response) { dbgWarning(D_ORCHESTRATOR) - << "Failed to uninstall package. Package: " << remove_package->second.getName(); + << "Failed to uninstall package. Package: " + << remove_package->second.getName(); all_cleaned = false; remove_package++; } else { @@ -284,42 +315,40 @@ ManifestController::Impl::updateManifest(const string &new_manifest_file) bool no_change = new_packages.size() == 0; // Both new_packages & corrupted_packages will be updated based on updated manifest - bool no_corrupted_package = manifest_diff_calc.filterCorruptedPackages(new_packages, corrupted_packages); - auto orchestration_service = new_packages.find("orchestration"); - if (orchestration_service != new_packages.end()) { - // Orchestration needs special handling as manifest should be backup differently - return handlePackage( - orchestration_service->second, - current_packages, - new_packages, - corrupted_packages - ); + const auto &download_packages_res = manifest_handler.downloadPackages(new_packages); + if (!download_packages_res.ok()) { + dbgWarning(D_ORCHESTRATOR) + << "Failed to download required packages. Error: " + << download_packages_res.getErr(); + return false; } - auto wlp_standalone_service = new_packages.find("wlpStandalone"); - if (wlp_standalone_service != new_packages.end()) { - // wlpStandalone needs special handling as manifest should be backup differently - return handlePackage( - wlp_standalone_service->second, + const vector> &downloaded_files = download_packages_res.unpack(); + const auto &installation_queue_res = manifest_diff_calc.buildInstallationQueue( current_packages, - new_packages, - corrupted_packages - ); + new_packages + ); + if (!installation_queue_res.ok()) { + dbgWarning(D_ORCHESTRATOR) + << "Failed building installation queue. Error: " + << installation_queue_res.getErr(); + return false; } + const vector &installation_queue = installation_queue_res.unpack(); + + const auto &sortd_downloaded_files = sortByInstallationQueue(downloaded_files, installation_queue); bool all_installed = true; bool any_installed = false; - dbgDebug(D_ORCHESTRATOR) << "Starting to handle " << new_packages.size() <<" new packages"; - for (auto &new_package : new_packages) { - - if (new_package.second.getType() != Package::PackageType::Service) continue; - + dbgDebug(D_ORCHESTRATOR) << "Starting to handle " << downloaded_files.size() << " new packages"; + for (auto &package : sortd_downloaded_files) { + if (package.first.getType() != Package::PackageType::Service) continue; size_t prev_size = corrupted_packages.size(); + bool handling_response = handlePackage( - new_package.second, + package, current_packages, - new_packages, corrupted_packages ); @@ -331,7 +360,10 @@ ManifestController::Impl::updateManifest(const string &new_manifest_file) } // Orchestration needs special handling as manifest should be backup differently - if (new_package.first.compare(orch_service_name) == 0) { + if (package.first.getName().compare(orch_service_name) == 0) { + return handling_response; + } + if (package.first.getName().compare("wlpStandalone") == 0) { return handling_response; } @@ -341,14 +373,22 @@ ManifestController::Impl::updateManifest(const string &new_manifest_file) bool manifest_file_update = true; - if (all_installed && (any_installed || no_change) && no_corrupted_package) { + if (all_installed && (any_installed || no_change)) { manifest_file_update = changeManifestFile(new_manifest_file); // In NSaaS - set ignore packages to any ignore_packages_update = updateIgnoreListForNSaaS(); } else if (any_installed) { manifest_file_update = orchestration_tools->packagesToJsonFile(current_packages, manifest_file_path); } - return all_installed && manifest_file_update && no_corrupted_package && all_cleaned; + if (all_installed) { + auto orchestration_downloader = Singleton::Consume::by(); + for (auto &package : all_packages) { + dbgDebug(D_ORCHESTRATOR) + << "Removing temp Download file after successfull installation : " << package.second.getName(); + orchestration_downloader->removeDownloadFile(package.second.getName()); + } + } + return all_installed && manifest_file_update && all_cleaned; } // Orchestration package needs a special handling. Old service will die during the upgrade @@ -425,35 +465,26 @@ ManifestController::Impl::changeManifestFile(const string &new_manifest_file) bool ManifestController::Impl::handlePackage( - const Package &package, + const pair &package_downloaded_file, map ¤t_packages, - const map &new_packages, map &corrupted_packages) { + auto &package = package_downloaded_file.first; + auto i_env = Singleton::Consume::by(); auto span_scope = i_env->startNewSpanScope(Span::ContextType::CHILD_OF); dbgDebug(D_ORCHESTRATOR) << "Handling package. Package: " << package.getName(); - if (!package.isInstallable().ok()) { + if (!package.isInstallable()) { string report_msg = - "Skipping installation of package: " + package.getName() + ". Reason: " + package.isInstallable().getErr(); + "Skipping installation of package: " + package.getName() + ". Reason: " + package.getErrorMessage(); dbgWarning(D_ORCHESTRATOR) << report_msg; LogGen(report_msg, Audience::SECURITY, Severity::CRITICAL, Priority::HIGH, Tags::ORCHESTRATOR); current_packages.insert(make_pair(package.getName(), package)); return true; } - vector installation_queue; - - if (!manifest_diff_calc.buildInstallationQueue(package, installation_queue, current_packages, new_packages)) { - dbgWarning(D_ORCHESTRATOR) << "Failed building installation queue. Package: " << package.getName(); - return false; - } - - vector> downloaded_files; - - if (!manifest_handler.downloadPackages(installation_queue, downloaded_files)) return false; - if (!manifest_handler.installPackages(downloaded_files, current_packages, corrupted_packages)) { + if (!manifest_handler.installPackage(package_downloaded_file, current_packages, corrupted_packages)) { LogGen( "Failed to install package: " + package.getName(), Audience::SECURITY, diff --git a/components/security_apps/orchestration/manifest_controller/manifest_controller_ut/manifest_controller_ut.cc b/components/security_apps/orchestration/manifest_controller/manifest_controller_ut/manifest_controller_ut.cc index 83aeb8a..1cb2f88 100755 --- a/components/security_apps/orchestration/manifest_controller/manifest_controller_ut/manifest_controller_ut.cc +++ b/components/security_apps/orchestration/manifest_controller/manifest_controller_ut/manifest_controller_ut.cc @@ -91,6 +91,16 @@ public: archive_in(ret); } + void checkIfFileExistsCall(const Package &package) + { + Maybe checksum_validation( + genError("File /tmp/orchestration_downloads/" + package.getName() + ".download does not exist.") + ); + EXPECT_CALL( + mock_downloader, + checkIfFileExists(package)).WillRepeatedly(Return(checksum_validation)); + } + string manifest_file_path; string corrupted_file_list; string temp_ext; @@ -171,6 +181,10 @@ TEST_F(ManifestControllerTest, createNewManifest) " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("my")); + //mock_downloader EXPECT_CALL( mock_downloader, @@ -187,6 +201,8 @@ TEST_F(ManifestControllerTest, createNewManifest) EXPECT_CALL(mock_package_handler, preInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, installPackage("my", "/tmp/temp_file", _)).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, postInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); EXPECT_CALL(mock_package_handler, updateSavedPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); load(manifest, new_services); @@ -237,6 +253,10 @@ TEST_F(ManifestControllerTest, badChecksum) " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("my")); + //mock_downloader Maybe err(genError("Empty")); EXPECT_CALL( @@ -300,6 +320,11 @@ TEST_F(ManifestControllerTest, updateManifest) " }" " ]" "}"; + + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("my")); + //mock_downloader EXPECT_CALL( mock_downloader, @@ -316,6 +341,8 @@ TEST_F(ManifestControllerTest, updateManifest) EXPECT_CALL(mock_package_handler, preInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, installPackage("my", "/tmp/temp_file", _)).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, postInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")).Times(2); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")).Times(2); EXPECT_CALL(mock_package_handler, updateSavedPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); load(manifest, new_services); @@ -366,6 +393,9 @@ TEST_F(ManifestControllerTest, updateManifest) " ]" "}"; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("my")); + EXPECT_CALL( mock_downloader, downloadFileFromURL( @@ -412,6 +442,10 @@ TEST_F(ManifestControllerTest, selfUpdate) " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("orchestration")); + EXPECT_CALL( mock_downloader, downloadFileFromURL( @@ -491,63 +525,6 @@ TEST_F(ManifestControllerTest, successLoadAfteSelfUpdate) EXPECT_TRUE(i_manifest_controller->loadAfterSelfUpdate()); } -TEST_F(ManifestControllerTest, updateWhileErrorPackageExist) -{ - new_services.clear(); - old_services.clear(); - string manifest = - "{" - " \"packages\": [" - " {" - " \"name\": \"my\"," - " \"version\": \"c\"," - " \"download-path\": \"http://172.23.92.135/my.sh\"," - " \"relative-path\": \"\"," - " \"checksum-type\": \"sha1sum\"," - " \"checksum\": \"a58bbab8020b0e6d08568714b5e582a3adf9c805\"," - " \"package-type\": \"service\"," - " \"require\": []" - " }," - " {" - " \"name\": \"orchestration\"," - " \"version\": \"c\"," - " \"download-path\": \"http://172.23.92.135/my.sh\"," - " \"relative-path\": \"\"," - " \"checksum-type\": \"sha1sum\"," - " \"checksum\": \"a58bbab8020b0e6d08568714b5e582a3adf9c805\"," - " \"package-type\": \"service\"," - " \"require\": []" - " }" - " ]" - "}"; - - string corrupted_packages_manifest = - "{" - " \"packages\": [" - " {" - " \"name\": \"my\"," - " \"version\": \"c\"," - " \"download-path\": \"http://172.23.92.135/my.sh\"," - " \"relative-path\": \"\"," - " \"checksum-type\": \"sha1sum\"," - " \"checksum\": \"a58bbab8020b0e6d08568714b5e582a3adf9c805\"," - " \"package-type\": \"service\"," - " \"require\": []" - " }" - " ]" - "}"; - - load(manifest, new_services); - load(old_manifest, old_services); - load(corrupted_packages_manifest, corrupted_packages); - - EXPECT_CALL(mock_orchestration_tools, - loadPackagesFromJson(corrupted_file_list)).WillOnce(Return(corrupted_packages)); - EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(file_name)).WillOnce(Return(new_services)); - EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(manifest_file_path)).WillOnce(Return(old_services)); - EXPECT_FALSE(i_manifest_controller->updateManifest(file_name)); -} - TEST_F(ManifestControllerTest, removeCurrentErrorPackage) { new_services.clear(); @@ -598,6 +575,8 @@ TEST_F(ManifestControllerTest, removeCurrentErrorPackage) load(old_manifest, old_services); load(corrupted_packages_manifest, corrupted_packages); + checkIfFileExistsCall(new_services.at("my")); + //mock_downloader EXPECT_CALL( mock_downloader, @@ -613,6 +592,8 @@ TEST_F(ManifestControllerTest, removeCurrentErrorPackage) EXPECT_CALL(mock_package_handler, preInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, installPackage("my", "/tmp/temp_file", _)).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, postInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); EXPECT_CALL(mock_package_handler, updateSavedPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(file_name)).WillOnce(Return(new_services)); @@ -629,8 +610,6 @@ TEST_F(ManifestControllerTest, removeCurrentErrorPackage) EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true)); corrupted_packages.clear(); - EXPECT_CALL(mock_orchestration_tools, packagesToJsonFile(corrupted_packages, - corrupted_file_list)).WillOnce(Return(true)); EXPECT_TRUE(i_manifest_controller->updateManifest(file_name)); } @@ -656,6 +635,10 @@ TEST_F(ManifestControllerTest, selfUpdateWithOldCopy) " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("orchestration")); + EXPECT_CALL( mock_downloader, downloadFileFromURL( @@ -710,6 +693,10 @@ TEST_F(ManifestControllerTest, selfUpdateWithOldCopyWithError) " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("orchestration")); + EXPECT_CALL( mock_downloader, downloadFileFromURL( @@ -775,6 +762,10 @@ TEST_F(ManifestControllerTest, installAndRemove) " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("my")); + //mock_downloader EXPECT_CALL( mock_downloader, @@ -791,6 +782,8 @@ TEST_F(ManifestControllerTest, installAndRemove) EXPECT_CALL(mock_package_handler, preInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, installPackage("my", "/tmp/temp_file", _)).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, postInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); EXPECT_CALL(mock_package_handler, updateSavedPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); load(manifest, new_services); @@ -839,6 +832,9 @@ TEST_F(ManifestControllerTest, installAndRemove) " ]" "}"; + load(new_manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("my1")); + EXPECT_CALL( mock_downloader, downloadFileFromURL( @@ -854,6 +850,8 @@ TEST_F(ManifestControllerTest, installAndRemove) EXPECT_CALL(mock_package_handler, preInstallPackage("my1", "/tmp/temp_file")).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, installPackage("my1", "/tmp/temp_file", _)).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, postInstallPackage("my1", "/tmp/temp_file")).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("my1")); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); EXPECT_CALL(mock_package_handler, updateSavedPackage("my1", "/tmp/temp_file")).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, packagesToJsonFile(old_services, manifest_file_path)).WillOnce(Return(true)); @@ -900,6 +898,10 @@ TEST_F(ManifestControllerTest, badInstall) " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("my")); + //mock_downloader EXPECT_CALL( mock_downloader, @@ -979,6 +981,10 @@ TEST_F(ManifestControllerTest, failToDownloadWithselfUpdate) " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("orchestration")); + Maybe err(genError("Empty")); EXPECT_CALL( mock_downloader, @@ -1019,7 +1025,7 @@ TEST_F(ManifestControllerTest, requireUpdate) "{" " \"packages\": [" " {" - " \"name\": \"orchestration\"," + " \"name\": \"orchestration1\"," " \"version\": \"c\"," " \"download-path\": \"http://172.23.92.135/my.sh\"," " \"relative-path\": \"\"," @@ -1040,16 +1046,22 @@ TEST_F(ManifestControllerTest, requireUpdate) " }" " ]" "}"; - EXPECT_CALL(mock_status, writeStatusToFile()); + + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("orchestration1")); + checkIfFileExistsCall(manifest_services.at("pre_orchestration")); + EXPECT_CALL( mock_downloader, downloadFileFromURL( "http://172.23.92.135/my.sh", "a58bbab8020b0e6d08568714b5e582a3adf9c805", Package::ChecksumTypes::SHA1, - "orchestration" + "orchestration1" ) ).WillOnce(Return(string("/tmp/temp_file1"))); + EXPECT_CALL( mock_downloader, downloadFileFromURL( @@ -1059,10 +1071,16 @@ TEST_F(ManifestControllerTest, requireUpdate) "pre_orchestration" ) ).WillOnce(Return(string("/tmp/temp_file2"))); - string temp_orc_file = "/etc/cp/packages/orchestration/orchestration_temp"; - EXPECT_CALL(mock_package_handler, preInstallPackage(orch_service_name, temp_orc_file)) + EXPECT_CALL(mock_package_handler, preInstallPackage("orchestration1", "/tmp/temp_file1")) .WillOnce(Return(true)); - EXPECT_CALL(mock_package_handler, installPackage(orch_service_name, temp_orc_file, _)) + EXPECT_CALL(mock_package_handler, installPackage("orchestration1", "/tmp/temp_file1", _)) + .WillOnce(Return(true)); + + EXPECT_CALL( + mock_package_handler, + shouldInstallPackage("orchestration1", "/tmp/temp_file1") + ).WillOnce(Return(true)); + EXPECT_CALL(mock_package_handler, postInstallPackage("orchestration1", "/tmp/temp_file1")) .WillOnce(Return(true)); EXPECT_CALL( @@ -1075,8 +1093,12 @@ TEST_F(ManifestControllerTest, requireUpdate) .WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, postInstallPackage("pre_orchestration", "/tmp/temp_file2")) .WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("pre_orchestration")); EXPECT_CALL(mock_package_handler, updateSavedPackage("pre_orchestration", "/tmp/temp_file2")) .WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration1")); + EXPECT_CALL(mock_package_handler, updateSavedPackage("orchestration1", "/tmp/temp_file1")) + .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(corrupted_file_list)) @@ -1088,20 +1110,24 @@ TEST_F(ManifestControllerTest, requireUpdate) .WillOnce(Return(new_services)); EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(manifest_file_path)) .WillOnce(Return(old_services)); - string temp_manifest_path = manifest_file_path + temp_ext; - EXPECT_CALL(mock_orchestration_tools, packagesToJsonFile(new_services, temp_manifest_path)) + EXPECT_CALL(mock_orchestration_tools, doesFileExist(manifest_file_path)) .WillOnce(Return(true)); - string path = packages_dir + "/" + orch_service_name + "/" + - orch_service_name; + string temp_manifest_path = manifest_file_path + temp_ext; + + string path = packages_dir + "/orchestration1/" + "orchestration1"; EXPECT_CALL(mock_orchestration_tools, doesFileExist(path)).Times(2).WillOnce(Return(false)); EXPECT_CALL( mock_orchestration_tools, doesFileExist("/etc/cp/packages/pre_orchestration/pre_orchestration") ).Times(2).WillOnce(Return(true)); - EXPECT_CALL(mock_orchestration_tools, copyFile("/tmp/temp_file1", path + temp_ext)) + EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, manifest_file_path)) + .WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, copyFile(manifest_file_path, "/etc/cp/conf/manifest.json.bk")) .WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, removeFile("new_manifest.json")).WillOnce(Return(true)); EXPECT_TRUE(i_manifest_controller->updateManifest(file_name)); } @@ -1141,6 +1167,11 @@ TEST_F(ManifestControllerTest, sharedObjectNotInstalled) EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(corrupted_file_list)).WillOnce(Return(corrupted_packages)); + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("orchestration")); + checkIfFileExistsCall(manifest_services.at("pre_orchestration")); + EXPECT_CALL( mock_downloader, downloadFileFromURL( @@ -1151,6 +1182,16 @@ TEST_F(ManifestControllerTest, sharedObjectNotInstalled) ) ).WillOnce(Return(string("/tmp/temp_file1"))); + EXPECT_CALL( + mock_downloader, + downloadFileFromURL( + "http://172.23.92.135/my.sh", + "a58bbab8020b0e6d08568714b5e582a3adf9c806", + Package::ChecksumTypes::SHA1, + "pre_orchestration" + ) + ).WillOnce(Return(string("/tmp/temp_file2"))); + string temp_manifest_path = manifest_file_path + temp_ext; string writen_manifest = "{" @@ -1181,7 +1222,10 @@ TEST_F(ManifestControllerTest, sharedObjectNotInstalled) string path = packages_dir + "/" + orch_service_name + "/" + orch_service_name; EXPECT_CALL(mock_orchestration_tools, doesFileExist(path)).Times(2).WillOnce(Return(false)); - + EXPECT_CALL( + mock_orchestration_tools, + doesFileExist("/etc/cp/packages/pre_orchestration/pre_orchestration") + ).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, copyFile("/tmp/temp_file1", path + temp_ext)).WillOnce(Return(true)); EXPECT_TRUE(i_manifest_controller->updateManifest(file_name)); @@ -1195,7 +1239,7 @@ TEST_F(ManifestControllerTest, requireSharedObjectUpdate) "{" " \"packages\": [" " {" - " \"name\": \"orchestration\"," + " \"name\": \"orchestration1\"," " \"version\": \"c\"," " \"download-path\": \"http://172.23.92.135/my.sh\"," " \"relative-path\": \"\"," @@ -1211,21 +1255,27 @@ TEST_F(ManifestControllerTest, requireSharedObjectUpdate) " \"relative-path\": \"\"," " \"checksum-type\": \"sha1sum\"," " \"checksum\": \"a58bbab8020b0e6d08568714b5e582a3adf9c806\"," - " \"package-type\": \"shared objects\"," + " \"package-type\": \"service\"," " \"require\": []" " }" " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("orchestration1")); + checkIfFileExistsCall(manifest_services.at("pre_orchestration")); + EXPECT_CALL( mock_downloader, downloadFileFromURL( "http://172.23.92.135/my.sh", "a58bbab8020b0e6d08568714b5e582a3adf9c805", Package::ChecksumTypes::SHA1, - "orchestration" + "orchestration1" ) ).WillOnce(Return(string("/tmp/temp_file1"))); + EXPECT_CALL( mock_downloader, downloadFileFromURL( @@ -1235,15 +1285,26 @@ TEST_F(ManifestControllerTest, requireSharedObjectUpdate) "pre_orchestration" ) ).WillOnce(Return(string("/tmp/temp_file2"))); - EXPECT_CALL(mock_status, writeStatusToFile()); - string temp_orc_file = "/etc/cp/packages/orchestration/orchestration_temp"; + string temp_orc_file = "/etc/cp/packages/orchestration1/orchestration_temp"; EXPECT_CALL(mock_package_handler, shouldInstallPackage(_, _)).WillRepeatedly(Return(true)); - EXPECT_CALL(mock_package_handler, preInstallPackage(orch_service_name, - temp_orc_file)).WillOnce(Return(true)); - EXPECT_CALL(mock_package_handler, installPackage(orch_service_name, - temp_orc_file, _)).WillOnce(Return(true)); + EXPECT_CALL(mock_package_handler, installPackage("orchestration1", "/tmp/temp_file1", _)) + .WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, installPackage("pre_orchestration", "/tmp/temp_file2", _)).WillOnce(Return(true)); + EXPECT_CALL(mock_package_handler, preInstallPackage("orchestration1", "/tmp/temp_file1")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_package_handler, postInstallPackage("orchestration1", "/tmp/temp_file1")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration1")); + EXPECT_CALL(mock_package_handler, updateSavedPackage("orchestration1", "/tmp/temp_file1")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_package_handler, preInstallPackage("pre_orchestration", "/tmp/temp_file2")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_package_handler, postInstallPackage("pre_orchestration", "/tmp/temp_file2")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("pre_orchestration")); + EXPECT_CALL(mock_package_handler, updateSavedPackage("pre_orchestration", "/tmp/temp_file2")) + .WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(corrupted_file_list)).WillOnce(Return(corrupted_packages)); @@ -1252,18 +1313,22 @@ TEST_F(ManifestControllerTest, requireSharedObjectUpdate) EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(file_name)).WillOnce(Return(new_services)); EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(manifest_file_path)).WillOnce(Return(old_services)); string temp_manifest_path = manifest_file_path + temp_ext; - EXPECT_CALL(mock_orchestration_tools, packagesToJsonFile(new_services, temp_manifest_path)).WillOnce(Return(true)); - string path = packages_dir + "/" + orch_service_name + "/" + - orch_service_name; + string path = packages_dir + "/" + "orchestration1" + "/" + "orchestration1"; EXPECT_CALL(mock_orchestration_tools, doesFileExist(path)).Times(2).WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/conf/manifest.json")) + .WillOnce(Return(false)); EXPECT_CALL( mock_orchestration_tools, doesFileExist("/etc/cp/packages/pre_orchestration/pre_orchestration") ).Times(2).WillOnce(Return(false)); - EXPECT_CALL(mock_orchestration_tools, copyFile("/tmp/temp_file1", path + - temp_ext)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, copyFile("new_manifest.json", "/etc/cp/conf/manifest.json")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile("/etc/cp/conf/manifest.json")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, removeFile("new_manifest.json")) + .WillOnce(Return(true)); EXPECT_TRUE(i_manifest_controller->updateManifest(file_name)); } @@ -1297,6 +1362,21 @@ TEST_F(ManifestControllerTest, failureOnDownloadSharedObject) " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("orchestration")); + checkIfFileExistsCall(manifest_services.at("pre_orchestration")); + + EXPECT_CALL( + mock_downloader, + downloadFileFromURL( + "http://172.23.92.135/my.sh", + "a58bbab8020b0e6d08568714b5e582a3adf9c805", + Package::ChecksumTypes::SHA1, + "orchestration" + ) + ).WillOnce(Return(string("/tmp/temp_file1"))); + Maybe err = genError("error"); EXPECT_CALL( mock_downloader, @@ -1314,11 +1394,16 @@ TEST_F(ManifestControllerTest, failureOnDownloadSharedObject) EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(file_name)).WillOnce(Return(new_services)); EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(manifest_file_path)).WillOnce(Return(old_services)); + EXPECT_CALL( + mock_orchestration_tools, + doesFileExist("/etc/cp/packages/orchestration/orchestration") + ).WillOnce(Return(false)); EXPECT_CALL( mock_orchestration_tools, doesFileExist("/etc/cp/packages/pre_orchestration/pre_orchestration") ).WillOnce(Return(false)); EXPECT_CALL(mock_details_resolver, getHostname()).WillOnce(Return(string("hostname"))); + EXPECT_CALL(mock_orchestration_tools, removeFile("/tmp/temp_file1")).WillOnce(Return(true)); EXPECT_CALL( mock_status, setFieldStatus(OrchestrationStatusFieldType::MANIFEST, OrchestrationStatusResult::FAILED, _) @@ -1337,7 +1422,7 @@ TEST_F(ManifestControllerTest, multiRequireUpdate) "{" " \"packages\": [" " {" - " \"name\": \"orchestration\"," + " \"name\": \"orchestration1\"," " \"version\": \"c\"," " \"download-path\": \"http://172.23.92.135/my.sh\"," " \"relative-path\": \"\"," @@ -1353,7 +1438,7 @@ TEST_F(ManifestControllerTest, multiRequireUpdate) " \"relative-path\": \"\"," " \"checksum-type\": \"sha1sum\"," " \"checksum\": \"a58bbab8020b0e6d08568714b5e582a3adf9c806\"," - " \"package-type\": \"shared objects\"," + " \"package-type\": \"service\"," " \"require\": []" " }," " {" @@ -1363,19 +1448,25 @@ TEST_F(ManifestControllerTest, multiRequireUpdate) " \"relative-path\": \"\"," " \"checksum-type\": \"sha1sum\"," " \"checksum\": \"a58bbab8020b0e6d08568714b5e582a3adf9c807\"," - " \"package-type\": \"shared objects\"," + " \"package-type\": \"service\"," " \"require\": [ \"pre_orchestration001\" ]" " }" " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("orchestration1")); + checkIfFileExistsCall(manifest_services.at("pre_orchestration001")); + checkIfFileExistsCall(manifest_services.at("pre_orchestration002")); + EXPECT_CALL( mock_downloader, downloadFileFromURL( "http://172.23.92.135/my.sh", "a58bbab8020b0e6d08568714b5e582a3adf9c805", Package::ChecksumTypes::SHA1, - "orchestration" + "orchestration1" ) ).WillOnce(Return(string("/tmp/temp_file1"))); EXPECT_CALL( @@ -1396,13 +1487,9 @@ TEST_F(ManifestControllerTest, multiRequireUpdate) "pre_orchestration002" ) ).WillOnce(Return(string("/tmp/temp_file3"))); - EXPECT_CALL(mock_status, writeStatusToFile()); - string temp_orc_file = "/etc/cp/packages/orchestration/orchestration_temp"; EXPECT_CALL(mock_package_handler, shouldInstallPackage(_, _)).WillRepeatedly(Return(true)); - EXPECT_CALL(mock_package_handler, preInstallPackage(orch_service_name, - temp_orc_file)).WillOnce(Return(true)); - EXPECT_CALL(mock_package_handler, installPackage(orch_service_name, - temp_orc_file, _)).WillOnce(Return(true)); + EXPECT_CALL(mock_package_handler, installPackage("orchestration1", + "/tmp/temp_file1", _)).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, installPackage("pre_orchestration001", "/tmp/temp_file2", _)).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, installPackage("pre_orchestration002", @@ -1410,15 +1497,37 @@ TEST_F(ManifestControllerTest, multiRequireUpdate) EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(corrupted_file_list)).WillOnce(Return(corrupted_packages)); + EXPECT_CALL(mock_package_handler, preInstallPackage("orchestration1", "/tmp/temp_file1")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_package_handler, preInstallPackage("pre_orchestration001", "/tmp/temp_file2")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_package_handler, preInstallPackage("pre_orchestration002", "/tmp/temp_file3")) + .WillOnce(Return(true)); + + EXPECT_CALL(mock_package_handler, postInstallPackage("orchestration1", "/tmp/temp_file1")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_package_handler, postInstallPackage("pre_orchestration001", "/tmp/temp_file2")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_package_handler, postInstallPackage("pre_orchestration002", "/tmp/temp_file3")) + .WillOnce(Return(true)); + + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration1")); + EXPECT_CALL(mock_package_handler, updateSavedPackage("orchestration1", "/tmp/temp_file1")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("pre_orchestration001")); + EXPECT_CALL(mock_package_handler, updateSavedPackage("pre_orchestration001", "/tmp/temp_file2")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("pre_orchestration002")); + EXPECT_CALL(mock_package_handler, updateSavedPackage("pre_orchestration002", "/tmp/temp_file3")) + .WillOnce(Return(true)); + load(manifest, new_services); EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(file_name)).WillOnce(Return(new_services)); EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(manifest_file_path)).WillOnce(Return(old_services)); string temp_manifest_path = manifest_file_path + temp_ext; - EXPECT_CALL(mock_orchestration_tools, packagesToJsonFile(new_services, temp_manifest_path)).WillOnce(Return(true)); - string path = packages_dir + "/" + orch_service_name + "/" + - orch_service_name; + string path = packages_dir + "/" + "orchestration1" + "/" + "orchestration1"; EXPECT_CALL(mock_orchestration_tools, doesFileExist(path)).Times(2).WillOnce(Return(false)); EXPECT_CALL( mock_orchestration_tools, @@ -1429,8 +1538,14 @@ TEST_F(ManifestControllerTest, multiRequireUpdate) doesFileExist("/etc/cp/packages/pre_orchestration002/pre_orchestration002") ).Times(2).WillOnce(Return(false)); - EXPECT_CALL(mock_orchestration_tools, copyFile("/tmp/temp_file1", path + - temp_ext)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/conf/manifest.json")) + .WillOnce(Return(false)); + EXPECT_CALL(mock_orchestration_tools, copyFile("new_manifest.json", "/etc/cp/conf/manifest.json")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile("/etc/cp/conf/manifest.json")) + .WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, removeFile("new_manifest.json")) + .WillOnce(Return(true)); EXPECT_TRUE(i_manifest_controller->updateManifest(file_name)); } @@ -1476,6 +1591,10 @@ TEST_F(ManifestControllerTest, createNewManifestWithUninstallablePackage) " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("my")); + //mock_downloader EXPECT_CALL( mock_downloader, @@ -1492,6 +1611,9 @@ TEST_F(ManifestControllerTest, createNewManifestWithUninstallablePackage) EXPECT_CALL(mock_package_handler, preInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, installPackage("my", "/tmp/temp_file", _)).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, postInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); + EXPECT_CALL(mock_downloader, removeDownloadFile("waap")); EXPECT_CALL(mock_package_handler, updateSavedPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); load(manifest, new_services); @@ -1552,15 +1674,17 @@ TEST_F(ManifestControllerTest, updateUninstallPackage) EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson(corrupted_file_list)).Times(2).WillRepeatedly(Return(corrupted_packages)); - EXPECT_CALL(mock_orchestration_tools, doesFileExist(manifest_file_path)).Times(2).WillRepeatedly(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist(manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, - copyFile(manifest_file_path, "/etc/cp/conf/manifest.json.bk")).Times(2).WillRepeatedly(Return(true)); - EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, manifest_file_path)) + copyFile(manifest_file_path, "/etc/cp/conf/manifest.json.bk")).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, manifest_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/packages/my/my")) .Times(2).WillRepeatedly(Return(true)); - EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).Times(2).WillRepeatedly(Return(true)); - EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).Times(2).WillRepeatedly(Return(true)); - EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/packages/my/my")).Times(2).WillOnce(Return(true)); string hostname = "hostname"; + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); EXPECT_TRUE(i_manifest_controller->updateManifest(file_name)); @@ -1590,6 +1714,10 @@ TEST_F(ManifestControllerTest, updateUninstallPackage) " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("my")); + EXPECT_CALL( mock_downloader, downloadFileFromURL( @@ -1605,6 +1733,8 @@ TEST_F(ManifestControllerTest, updateUninstallPackage) EXPECT_CALL(mock_package_handler, preInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, installPackage("my", "/tmp/temp_file", _)).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, postInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); EXPECT_CALL(mock_package_handler, updateSavedPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); //mock_orchestration_tools @@ -1673,6 +1803,16 @@ public: archive_in(ret); } + void checkIfFileExistsCall(const Package &package) + { + Maybe checksum_validation( + genError("File /tmp/orchestration_downloads/" + package.getName() + ".download does not exist.") + ); + EXPECT_CALL( + mock_downloader, + checkIfFileExists(package)).WillRepeatedly(Return(checksum_validation)); + } + string manifest_file_path; string corrupted_file_list; string temp_ext; @@ -1801,6 +1941,9 @@ TEST_F(ManifestControllerIgnorePakckgeTest, addAndUpdateIgnorePackage) EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); + EXPECT_CALL(mock_downloader, removeDownloadFile("dummy_service")); EXPECT_TRUE(i_manifest_controller->updateManifest(file_name)); @@ -1855,11 +1998,13 @@ TEST_F(ManifestControllerIgnorePakckgeTest, addAndUpdateIgnorePackage) EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); + EXPECT_CALL(mock_downloader, removeDownloadFile("dummy_service")); EXPECT_TRUE(i_manifest_controller->updateManifest(file_name)); } - TEST_F(ManifestControllerIgnorePakckgeTest, addIgnorePackageAndUpdateNormal) { init(); @@ -1916,6 +2061,9 @@ TEST_F(ManifestControllerIgnorePakckgeTest, addIgnorePackageAndUpdateNormal) EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); + EXPECT_CALL(mock_downloader, removeDownloadFile("dummy_service")); EXPECT_TRUE(i_manifest_controller->updateManifest(file_name)); @@ -1958,6 +2106,7 @@ TEST_F(ManifestControllerIgnorePakckgeTest, addIgnorePackageAndUpdateNormal) //mock_orchestration_tools load(manifest, new_services); + checkIfFileExistsCall(new_services.at("my")); //mock_downloader EXPECT_CALL( @@ -1975,6 +2124,9 @@ TEST_F(ManifestControllerIgnorePakckgeTest, addIgnorePackageAndUpdateNormal) EXPECT_CALL(mock_package_handler, preInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, installPackage("my", "/tmp/temp_file", _)).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, postInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); + EXPECT_CALL(mock_downloader, removeDownloadFile("dummy_service")); EXPECT_CALL(mock_package_handler, updateSavedPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); load(manifest, new_services); @@ -2050,6 +2202,9 @@ TEST_F(ManifestControllerIgnorePakckgeTest, removeIgnoredPackage) EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); + EXPECT_CALL(mock_downloader, removeDownloadFile("dummy_service")); EXPECT_TRUE(i_manifest_controller->updateManifest(file_name)); @@ -2094,6 +2249,8 @@ TEST_F(ManifestControllerIgnorePakckgeTest, removeIgnoredPackage) EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); EXPECT_TRUE(i_manifest_controller->updateManifest(file_name)); } @@ -2147,6 +2304,8 @@ TEST_F(ManifestControllerIgnorePakckgeTest, freezeIgnoredPackage) EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile(manifest_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, removeFile(file_name)).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); EXPECT_TRUE(i_manifest_controller->updateManifest(file_name)); @@ -2210,6 +2369,10 @@ TEST_F(ManifestControllerIgnorePakckgeTest, overrideIgnoredPackageFromProfileSet " ]" "}"; + map manifest_services; + load(manifest, manifest_services); + checkIfFileExistsCall(manifest_services.at("my")); + //mock_downloader EXPECT_CALL( mock_downloader, @@ -2226,6 +2389,8 @@ TEST_F(ManifestControllerIgnorePakckgeTest, overrideIgnoredPackageFromProfileSet EXPECT_CALL(mock_package_handler, preInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, installPackage("my", "/tmp/temp_file", _)).WillOnce(Return(true)); EXPECT_CALL(mock_package_handler, postInstallPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); + EXPECT_CALL(mock_downloader, removeDownloadFile("my")); + EXPECT_CALL(mock_downloader, removeDownloadFile("orchestration")); EXPECT_CALL(mock_package_handler, updateSavedPackage("my", "/tmp/temp_file")).WillOnce(Return(true)); load(manifest, new_services); @@ -2270,6 +2435,17 @@ public: manifest_controller.init(); } + + void checkIfFileExistsCall(const Package &package) + { + Maybe checksum_validation( + genError("File /tmp/orchestration_downloads/" + package.getName() + ".download does not exist.") + ); + EXPECT_CALL( + mock_downloader, + checkIfFileExists(package)).WillRepeatedly(Return(checksum_validation)); + } + ::Environment env; ConfigComponent config; @@ -2335,6 +2511,9 @@ TEST_F(ManifestDownloadTest, download_relative_path) EXPECT_CALL(mock_orchestration_tools, loadPackagesFromJson("/etc/cp/conf/corrupted_packages.json")) .WillOnce(Return(corrupted_packages)); EXPECT_CALL(agent_details, getFogDomain()).WillOnce(Return(fog_domain)); + + checkIfFileExistsCall(new_packages.at("orchestration")); + EXPECT_CALL( mock_downloader, downloadFileFromURL( @@ -2413,6 +2592,8 @@ TEST_F(ManifestDownloadTest, download_relative_path_no_fog_domain) string not_error; EXPECT_CALL(mock_status, getManifestError()).WillOnce(ReturnRef(not_error)); + checkIfFileExistsCall(new_packages.at("orchestration")); + EXPECT_CALL( mock_downloader, downloadFileFromURL( diff --git a/components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc b/components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc index d96b1d0..dd5b996 100755 --- a/components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc +++ b/components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc @@ -13,6 +13,8 @@ #include "manifest_diff_calculator.h" +#include + #include "debug.h" #include "config.h" @@ -59,6 +61,8 @@ ManifestDiffCalculator::filterUntrackedPackages( return packages_to_remove; } +// LCOV_EXCL_START Reason: temp disabling corrupted packages mechanism + // If one of the new packages is already known as corrupted, new_packages map is // updated accordingly. // Otherwise, corrupted_packages is updated and old corrupted package is deleted. @@ -102,38 +106,71 @@ ManifestDiffCalculator::filterCorruptedPackages( } return no_corrupted_package_exist; } +// LCOV_EXCL_STOP -// This function build the installation queue recursively and return true if succeeded, false otherwise -// At the beginning, installation_queue is empty and will be filled according package dependences -bool -ManifestDiffCalculator::buildInstallationQueue( - const Package &updated_package, +Maybe +ManifestDiffCalculator::buildRecInstallationQueue( + const Package &package, vector &installation_queue, const map ¤t_packages, const map &new_packages) { - vector requires = updated_package.getRequire(); + const vector &requires = package.getRequire(); - for (size_t i = 0; i < requires.size(); i++) { - auto installed_package = current_packages.find(requires[i]); - auto new_package = new_packages.find(requires[i]); + for (const auto &require : requires) { + auto installed_package = current_packages.find(require); + auto new_package = new_packages.find(require); if (installed_package == current_packages.end() || (new_package != new_packages.end() && *installed_package != *new_package)) { - if(!buildInstallationQueue(new_package->second, - installation_queue, - current_packages, - new_packages)) { - return false; - } + auto rec_res = buildRecInstallationQueue( + new_package->second, + installation_queue, + current_packages, + new_packages + ); + if (!rec_res.ok()) return rec_res.passErr(); } else if (installed_package != current_packages.end()) { - dbgDebug(D_ORCHESTRATOR) << "Package is already installed. Package: " << installed_package->first; + dbgDebug(D_ORCHESTRATOR) << "Package is already in the queue. Package: " << installed_package->first; } else if (new_package == new_packages.end()) { - dbgWarning(D_ORCHESTRATOR) << "One of the requested dependencies is corrupted or doesn't exist." - << " Package: "<< requires[i]; - return false; + return genError( + "One of the requested dependencies is corrupted or doesn't exist. Package: " + require + ); } } - installation_queue.push_back(updated_package); - return true; + if (find(installation_queue.begin(), installation_queue.end(), package) == installation_queue.end()) { + installation_queue.push_back(package); + } + return Maybe(); +} + +// This function build the installation queue recursively and return true if succeeded, false otherwise +// At the beginning, installation_queue is empty and will be filled according package dependences +Maybe> +ManifestDiffCalculator::buildInstallationQueue( + const map ¤t_packages, + const map &new_packages) +{ + vector installation_queue; + installation_queue.reserve(new_packages.size()); + auto orchestration_it = new_packages.find("orchestration"); + if (orchestration_it != new_packages.end()) { + installation_queue.push_back(orchestration_it->second); + } + + auto wlp_standalone_it = new_packages.find("wlpStandalone"); + if (wlp_standalone_it != new_packages.end()){ + installation_queue.push_back(wlp_standalone_it->second); + } + + for (auto &package_pair : new_packages) { + auto build_queue_res = buildRecInstallationQueue( + package_pair.second, + installation_queue, + current_packages, + new_packages + ); + if (!build_queue_res.ok()) return build_queue_res.passErr(); + } + return installation_queue; } diff --git a/components/security_apps/orchestration/manifest_controller/manifest_handler.cc b/components/security_apps/orchestration/manifest_controller/manifest_handler.cc index 02f9a48..c2cbd7e 100755 --- a/components/security_apps/orchestration/manifest_controller/manifest_handler.cc +++ b/components/security_apps/orchestration/manifest_controller/manifest_handler.cc @@ -13,6 +13,8 @@ #include "manifest_handler.h" +#include + #include "debug.h" #include "config.h" #include "agent_details.h" @@ -57,6 +59,10 @@ ManifestHandler::downloadPackage(const Package &package, bool is_clean_installat fog_domain = Singleton::Consume::by()->getFogDomain(); } + auto orchestration_downloader = Singleton::Consume::by(); + auto maybe_package_exists = orchestration_downloader->checkIfFileExists(package); + if (maybe_package_exists.ok()) return maybe_package_exists; + if (!is_clean_installation) { I_MainLoop *i_mainloop = Singleton::Consume::by(); auto pending_time_frame_seconds = getConfigurationWithDefault( @@ -76,11 +82,10 @@ ManifestHandler::downloadPackage(const Package &package, bool is_clean_installat dbgTrace(D_ORCHESTRATOR) << "Proceeding to package downloading. Package name " << package.getName(); } - auto orchestration_downloader = Singleton::Consume::by(); if (!package.getRelativeDownloadPath().empty() && fog_domain.ok()) { string download_path = "https://" + fog_domain.unpack() + "/download" + package.getRelativeDownloadPath(); - package_download_file= orchestration_downloader->downloadFileFromURL( + package_download_file = orchestration_downloader->downloadFileFromURL( download_path, package.getChecksum(), package.getChecksumType(), @@ -99,15 +104,22 @@ ManifestHandler::downloadPackage(const Package &package, bool is_clean_installat return package_download_file; } -bool -ManifestHandler::downloadPackages( - const vector &packages_to_download, - vector> &downloaded_packages) +Maybe>> +ManifestHandler::downloadPackages(const map &new_packages_to_download) { auto i_env = Singleton::Consume::by(); auto i_orch_tools = Singleton::Consume::by(); auto span_scope = i_env->startNewSpanScope(Span::ContextType::CHILD_OF); - for (auto &package : packages_to_download) { + + vector> downloaded_packages; + for (auto &package_pair : new_packages_to_download) { + const Package &package = package_pair.second; + if (!package.isInstallable()) { + dbgTrace(D_ORCHESTRATOR) + << "Skipping package download, package isn't installable. Package: " + << package.getName() << ". Reason: " << package.getErrorMessage(); + continue; + } dbgInfo(D_ORCHESTRATOR) << "Downloading package file." << " Package: " << package.getName(); string packages_dir = getConfigurationWithDefault( @@ -170,133 +182,42 @@ ManifestHandler::downloadPackages( install_error ); } - return false; + return genError( + "Failed to download installation package. Package: " + + package.getName() + + ", Error: " + package_download_file.getErr()); } } - return true; + return downloaded_packages; } bool -ManifestHandler::installPackages( - const vector> &downloaded_package_files, +ManifestHandler::installPackage( + const pair &package_downloaded_file, map ¤t_packages, map &corrupted_packages) { auto i_env = Singleton::Consume::by(); auto span_scope = i_env->startNewSpanScope(Span::ContextType::CHILD_OF); - // Patch - reorder packages so that accessControlApp is installed before accessControlKernel - vector> patched_downloaded_package_files; - patched_downloaded_package_files.reserve(downloaded_package_files.size()); - int ac_kernel_package_idx = -1; - int ac_app_package_idx = -1; - int i = 0; - for (auto &downloaded_package : downloaded_package_files) { - if (downloaded_package.first.getName() == "accessControlApp") { - ac_app_package_idx = i; - } else if (downloaded_package.first.getName() == "accessControlKernel") { - ac_kernel_package_idx = i; - } else { - patched_downloaded_package_files.push_back(downloaded_package); - } - i++; - } - if (ac_app_package_idx != -1) { - patched_downloaded_package_files.push_back(downloaded_package_files.at(ac_app_package_idx)); - } - if (ac_kernel_package_idx != -1) { - patched_downloaded_package_files.push_back(downloaded_package_files.at(ac_kernel_package_idx)); - } - auto orchestration_status = Singleton::Consume::by(); - for (auto &downloaded_package : patched_downloaded_package_files) { - auto package = downloaded_package.first; - auto package_name = package.getName(); - auto package_handler_path = downloaded_package.second; - dbgInfo(D_ORCHESTRATOR) << "Handling package installation. Package: " << package_name; + auto &package = package_downloaded_file.first; + auto &package_name = package.getName(); + auto &package_handler_path = package_downloaded_file.second; - if (package_name.compare(orch_service_name) == 0) { - orchestration_status->writeStatusToFile(); - bool self_update_status = selfUpdate(package, current_packages, package_handler_path); - if (!self_update_status) { - auto details = Singleton::Consume::by(); - auto hostname = Singleton::Consume::by()->getHostname(); - string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + details->getAgentId()) + "'"; - string install_error = - "Warning: Agent/Gateway " + - err_hostname + - " software update failed. Agent is running previous software. Contact Check Point support."; - if (orchestration_status->getManifestError().find("Gateway was not fully deployed") == string::npos) { - orchestration_status->setFieldStatus( - OrchestrationStatusFieldType::MANIFEST, - OrchestrationStatusResult::FAILED, - install_error - ); - } - } + dbgInfo(D_ORCHESTRATOR) << "Handling package installation. Package: " << package_name; - return self_update_status; - } - - string packages_dir = getConfigurationWithDefault( - "/etc/cp/packages", - "orchestration", - "Packages directory" - ); - - string current_installation_file = packages_dir + "/" + package_name + "/" + package_name; - auto orchestration_tools = Singleton::Consume::by(); - bool is_clean_installation = !orchestration_tools->doesFileExist(current_installation_file); - - auto package_handler = Singleton::Consume::by(); - if (!package_handler->shouldInstallPackage(package_name, package_handler_path)) { - current_packages.insert(make_pair(package_name, package)); - dbgInfo(D_ORCHESTRATOR) - << "Skipping installation of new package with the same version as current. Package: " - << package_name; - continue; - } - - bool current_result = true; - bool is_service = package.getType() == Package::PackageType::Service; - if (is_service) { - current_result = package_handler->preInstallPackage(package_name, package_handler_path); - } - - current_result = current_result && package_handler->installPackage( - package_name, - package_handler_path, - false - ); - - if (current_result && is_service) { - current_result = package_handler->postInstallPackage(package_name, package_handler_path); - } - - if (current_result && is_service) { - current_result = package_handler->updateSavedPackage(package_name, package_handler_path); - } - - if (!current_result) { - auto agent_details = Singleton::Consume::by(); + if (package_name.compare(orch_service_name) == 0) { + orchestration_status->writeStatusToFile(); + bool self_update_status = selfUpdate(package, current_packages, package_handler_path); + if (!self_update_status) { + auto details = Singleton::Consume::by(); auto hostname = Singleton::Consume::by()->getHostname(); - string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" +agent_details->getAgentId()) + "'"; - string install_error; - if (is_clean_installation) { - install_error = - "Critical Error: Agent/Gateway was not fully deployed " + - err_hostname + - " and is not enforcing a security policy. Retry installation or contact Check Point support."; - } else { - install_error = - "Warning: Agent/Gateway " + - err_hostname + - " software update failed. Agent is running previous software. Contact Check Point support."; - } - corrupted_packages.insert(make_pair(package_name, package)); - dbgWarning(D_ORCHESTRATOR) << "Failed to install package. Package: " << package_name; - - auto orchestration_status = Singleton::Consume::by(); + string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + details->getAgentId()) + "'"; + string install_error = + "Warning: Agent/Gateway " + + err_hostname + + " software update failed. Agent is running previous software. Contact Check Point support."; if (orchestration_status->getManifestError().find("Gateway was not fully deployed") == string::npos) { orchestration_status->setFieldStatus( OrchestrationStatusFieldType::MANIFEST, @@ -304,11 +225,80 @@ ManifestHandler::installPackages( install_error ); } - return false; } - - current_packages.insert(make_pair(package_name, package)); + return self_update_status; } + + string packages_dir = getConfigurationWithDefault( + "/etc/cp/packages", + "orchestration", + "Packages directory" + ); + + auto package_handler = Singleton::Consume::by(); + if (!package_handler->shouldInstallPackage(package_name, package_handler_path)) { + current_packages.insert(make_pair(package_name, package)); + dbgInfo(D_ORCHESTRATOR) + << "Skipping installation of new package with the same version as current. Package: " + << package_name; + return true; + } + string current_installation_file = packages_dir + "/" + package_name + "/" + package_name; + auto orchestration_tools = Singleton::Consume::by(); + bool is_clean_installation = !orchestration_tools->doesFileExist(current_installation_file); + + + bool current_result = true; + bool is_service = package.getType() == Package::PackageType::Service; + if (is_service) { + current_result = package_handler->preInstallPackage(package_name, package_handler_path); + } + + current_result = current_result && package_handler->installPackage( + package_name, + package_handler_path, + false + ); + + if (current_result && is_service) { + current_result = package_handler->postInstallPackage(package_name, package_handler_path); + } + + if (current_result && is_service) { + current_result = package_handler->updateSavedPackage(package_name, package_handler_path); + } + + if (!current_result) { + auto agent_details = Singleton::Consume::by(); + auto hostname = Singleton::Consume::by()->getHostname(); + string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" +agent_details->getAgentId()) + "'"; + string install_error; + if (is_clean_installation) { + install_error = + "Critical Error: Agent/Gateway was not fully deployed " + + err_hostname + + " and is not enforcing a security policy. Retry installation or contact Check Point support."; + } else { + install_error = + "Warning: Agent/Gateway " + + err_hostname + + " software update failed. Agent is running previous software. Contact Check Point support."; + } + corrupted_packages.insert(make_pair(package_name, package)); + dbgWarning(D_ORCHESTRATOR) << "Failed to install package. Package: " << package_name; + + auto orchestration_status = Singleton::Consume::by(); + if (orchestration_status->getManifestError().find("Gateway was not fully deployed") == string::npos) { + orchestration_status->setFieldStatus( + OrchestrationStatusFieldType::MANIFEST, + OrchestrationStatusResult::FAILED, + install_error + ); + } + return false; + } + + current_packages.insert(make_pair(package_name, package)); return true; } diff --git a/components/security_apps/orchestration/modules/modules_ut/package_ut.cc b/components/security_apps/orchestration/modules/modules_ut/package_ut.cc index c4cc089..7d8f4f9 100755 --- a/components/security_apps/orchestration/modules/modules_ut/package_ut.cc +++ b/components/security_apps/orchestration/modules/modules_ut/package_ut.cc @@ -73,7 +73,7 @@ TEST_F(PackageTest, serializationFromString) EXPECT_EQ("orchestration", package.getName()); EXPECT_EQ("c", package.getVersion()); EXPECT_EQ(Package::PackageType::Service, package.getType()); - EXPECT_TRUE(package.isInstallable().ok()); + EXPECT_TRUE(package.isInstallable()); } TEST_F(PackageTest, writeAsJson) @@ -86,7 +86,8 @@ TEST_F(PackageTest, writeAsJson) " \"name\": \"orchestration\",\n" " \"checksum-type\": \"sha1sum\",\n" " \"checksum\": \"8d4a5709673a05b380ba7d6567e28910019118f5\",\n" - " \"package-type\": \"service\"\n" + " \"package-type\": \"service\",\n" + " \"status\": true\n" "}"; Package package; EXPECT_EQ(true, load(string_stream, package)); @@ -99,7 +100,7 @@ TEST_F(PackageTest, writeAsJson) EXPECT_EQ("orchestration", package.getName()); EXPECT_EQ("c", package.getVersion()); EXPECT_EQ(Package::PackageType::Service, package.getType()); - EXPECT_TRUE(package.isInstallable().ok()); + EXPECT_TRUE(package.isInstallable()); write("service.json", package); string data = readFile("service.json"); @@ -232,5 +233,6 @@ TEST_F(PackageTest, uninstallablePackage) "}"; Package package; EXPECT_TRUE(load(string_stream, package)); - EXPECT_THAT(package.isInstallable(), IsError("This security app isn't valid for this agent")); + EXPECT_FALSE(package.isInstallable()); + EXPECT_EQ(package.getErrorMessage(), "This security app isn't valid for this agent"); } diff --git a/components/security_apps/orchestration/modules/package.cc b/components/security_apps/orchestration/modules/package.cc index 637149d..70cc6ab 100755 --- a/components/security_apps/orchestration/modules/package.cc +++ b/components/security_apps/orchestration/modules/package.cc @@ -59,9 +59,9 @@ Package::serialize(JSONOutputArchive & out_archive) const out_archive(make_nvp("require", require_packages)); } - if (!installable.ok()) { - out_archive(make_nvp("status", installable.ok())); - out_archive(make_nvp("message", installable.getErr())); + out_archive(make_nvp("status", installable)); + if (!installable) { + out_archive(make_nvp("message", error_message)); } } @@ -89,21 +89,18 @@ Package::serialize(JSONInputArchive & in_archive) in_archive.setNextName(nullptr); } - bool is_installable = true; try { - in_archive(make_nvp("status", is_installable)); + in_archive(make_nvp("status", installable)); } catch (...) { in_archive.setNextName(nullptr); } - if (!is_installable) { - string error_message; + if (!installable) { try { in_archive(make_nvp("message", error_message)); } catch (...) { in_archive.setNextName(nullptr); } - installable = genError(error_message); } for (auto &character : name) { diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 33af4f5..44d3179 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -29,7 +29,6 @@ #include "service_controller.h" #include "manifest_controller.h" #include "url_parser.h" -#include "i_messaging.h" #include "agent_details_report.h" #include "maybe_res.h" #include "customized_cereal_map.h" @@ -227,7 +226,6 @@ private: Maybe maybe_policy = genError("Empty policy"); string policy_version = ""; auto orchestration_policy_file = getPolicyConfigPath("orchestration", Config::ConfigFileType::Policy); - auto orchestration_tools = Singleton::Consume::by(); if (orchestration_tools->doesFileExist(orchestration_policy_file)) { maybe_policy = loadOrchestrationPolicy(); @@ -296,7 +294,10 @@ private: } } - if (declarative) Singleton::Consume::from()->turnOnApplyPolicyFlag(); + if (declarative) { + Singleton::Consume::from()->turnOnApplyPolicyFlag(); + } + return authentication_res; } @@ -769,13 +770,11 @@ private: LogRest policy_update_message_client_rest(policy_update_message); - Singleton::Consume::by()->sendObjectWithPersistence( - policy_update_message_client_rest, - I_Messaging::Method::POST, + Singleton::Consume::by()->sendAsyncMessage( + HTTPMethod::POST, "/api/v1/agents/events", - "", - true, - MessageTypeTag::REPORT + policy_update_message_client_rest, + MessageCategory::LOG ); }, "Send policy update report" @@ -1471,8 +1470,8 @@ private: agent_data_report << AgentReportFieldWithLabel("isGwNotVsx", "true"); } - if (i_details_resolver->isVersionEqualOrAboveR8110()) { - agent_data_report << AgentReportFieldWithLabel("isVersionEqualOrAboveR8110", "true"); + if (i_details_resolver->isVersionAboveR8110()) { + agent_data_report << AgentReportFieldWithLabel("isVersionAboveR8110", "true"); } auto i_agent_details = Singleton::Consume::by(); @@ -1528,25 +1527,45 @@ private: encryptToFile(data3, data_path + data6_file_name); } + int + calcSleepInterval(int sleep_interval) + { + failure_count++; + int failure_multiplier = 1; + if (failure_count >= 10) { + failure_multiplier = 10; + } else if (failure_count >= 3) { + failure_multiplier = 2; + } + return sleep_interval * failure_multiplier; + } + void run() { int sleep_interval = policy.getErrorSleepInterval(); Maybe start_state(genError("Not running yet.")); while (!(start_state = start()).ok()) { - dbgDebug(D_ORCHESTRATOR) << "Orchestration not started yet. Status: " << start_state.getErr(); health_check_status_listener.setStatus( HealthCheckStatus::UNHEALTHY, OrchestrationStatusFieldType::REGISTRATION, start_state.getErr() ); sleep_interval = getConfigurationWithDefault( - 20, + 30, "orchestration", "Default sleep interval" ); + sleep_interval = calcSleepInterval(sleep_interval); + dbgWarning(D_ORCHESTRATOR) + << "Orchestration not started yet. Status: " + << start_state.getErr() + << " Next attempt to start the orchestration will be in: " + << sleep_interval + << " seconds"; Singleton::Consume::by()->yield(seconds(sleep_interval)); } + failure_count = 0; Singleton::Consume::by()->yield(chrono::seconds(1)); @@ -1589,24 +1608,14 @@ private: bool is_new_success = false; while (true) { - static int failure_count = 0; Singleton::Consume::by()->startNewTrace(false); if (shouldReportAgentDetailsMetadata()) { reportAgentDetailsMetaData(); } auto check_update_result = checkUpdate(); if (!check_update_result.ok()) { - failure_count++; is_new_success = false; - sleep_interval = policy.getErrorSleepInterval(); - int failure_multiplier = 1; - if (failure_count >= 10) { - failure_count = 10; - failure_multiplier = 10; - } else if (failure_count >= 3) { - failure_multiplier = 2; - } - sleep_interval *= failure_multiplier; + sleep_interval = calcSleepInterval(policy.getErrorSleepInterval()); dbgWarning(D_ORCHESTRATOR) << "Failed during check update from Fog. Error: " << check_update_result.getErr() @@ -1690,13 +1699,11 @@ private: if (email != "") registration_report << LogField("userDefinedId", email); LogRest registration_report_rest(registration_report); - Singleton::Consume::by()->sendObjectWithPersistence( - registration_report_rest, - I_Messaging::Method::POST, + Singleton::Consume::by()->sendAsyncMessage( + HTTPMethod::POST, "/api/v1/agents/events", - "", - true, - MessageTypeTag::REPORT + registration_report_rest, + MessageCategory::LOG ); } @@ -1764,6 +1771,7 @@ private: { auto agent_details = Singleton::Consume::by(); return + agent_details->getAccessToken().empty() || agent_details->getSSLFlag() != is_secure || !agent_details->getFogPort().ok() || agent_details->getFogPort().unpack() != port || !agent_details->getFogDomain().ok() || agent_details->getFogDomain().unpack() != fog; @@ -1772,6 +1780,7 @@ private: bool updateFogAddress(const string &fog_addr) { + dbgFlow(D_ORCHESTRATOR) << "Setting a fog address: " << fog_addr; auto orch_status = Singleton::Consume::by(); auto agent_details = Singleton::Consume::by(); auto orchestration_mode = getOrchestrationMode(); @@ -1783,7 +1792,7 @@ private: if (agent_details->writeAgentDetails()) { dbgDebug(D_ORCHESTRATOR) << "Agent details was successfully saved"; } else { - dbgWarning(D_COMMUNICATION) << "Failed to save agent details to a file"; + dbgWarning(D_ORCHESTRATOR) << "Failed to save agent details to a file"; } return true; } @@ -1803,16 +1812,12 @@ private: auto message = Singleton::Consume::by(); - if (!shouldReconnectToFog( - fog_domain, - fog_port, - encrypted_fog_connection - )) { + if (!shouldReconnectToFog(fog_domain, fog_port, encrypted_fog_connection)) { dbgDebug(D_ORCHESTRATOR) << "Skipping reconnection to the Fog - Fog details did not change"; return true; } - if (message->setActiveFog(fog_domain, fog_port, encrypted_fog_connection, MessageTypeTag::GENERIC)) { + if (message->setFogConnection(fog_domain, fog_port, encrypted_fog_connection, MessageCategory::GENERIC)) { agent_details->setFogPort(fog_port); agent_details->setFogDomain(fog_domain); agent_details->setSSLFlag(encrypted_fog_connection); @@ -1820,7 +1825,7 @@ private: if (agent_details->writeAgentDetails()) { dbgDebug(D_ORCHESTRATOR) << "Agent details was successfully saved"; } else { - dbgWarning(D_COMMUNICATION) << "Failed to save agent details to a file"; + dbgWarning(D_ORCHESTRATOR) << "Failed to save agent details to a file"; } auto update_communication = Singleton::Consume::by(); @@ -1894,7 +1899,11 @@ private: auto result = i_shell_cmd->getExecOutput(openssl_dir_cmd); if (result.ok()) { string val_openssl_dir = result.unpack(); - if (val_openssl_dir.empty()) return; + if (val_openssl_dir.empty()) { + dbgWarning(D_ORCHESTRATOR) + << "Failed to load OpenSSL default certificate authority. Error: no OpenSSL directory found"; + return; + } if (val_openssl_dir.back() == '\n') val_openssl_dir.pop_back(); dbgTrace(D_ORCHESTRATOR) << "Adding OpenSSL default directory to agent details. Directory: " @@ -1953,6 +1962,7 @@ private: }; const uint16_t default_fog_dport = 443; + int failure_count = 0; OrchestrationPolicy policy; HealthCheckStatusListener health_check_status_listener; HybridModeMetric hybrid_mode_metric; diff --git a/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc b/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc index 952dac6..b0bbd8f 100755 --- a/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc +++ b/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc @@ -147,21 +147,21 @@ getNamespaceDataFromCluster(const string &path) { NamespaceData name_space; string token = Singleton::Consume::by()->getToken(); - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::SECURE_CONN); - conn_flags.setFlag(MessageConnConfig::IGNORE_SSL_VALIDATION); auto messaging = Singleton::Consume::by(); - bool res = messaging->sendObject( - name_space, - I_Messaging::Method::GET, - "kubernetes.default.svc", - 443, - conn_flags, + + MessageMetadata get_ns_md("kubernetes.default.svc", 443); + get_ns_md.insertHeader("Authorization", "Bearer " + token); + get_ns_md.insertHeader("Connection", "close"); + get_ns_md.setConnectioFlag(MessageConnectionConfig::IGNORE_SSL_VALIDATION); + auto res = messaging->sendSyncMessage( + HTTPMethod::GET, path, - "Authorization: Bearer " + token + "\nConnection: close" + name_space, + MessageCategory::GENERIC, + get_ns_md ); - if (res) return name_space; + if (res.ok()) return name_space; return genError(string("Was not able to get object form k8s cluser in path: " + path)); } diff --git a/components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc b/components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc index 02581ed..cbb5e85 100755 --- a/components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc +++ b/components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc @@ -90,19 +90,15 @@ TEST_F(OrchestrationToolsTest, getClusterId) string namespaces = getResource("k8s_namespaces.json"); EXPECT_CALL( mock_messaging, - sendMessage( - true, - "", - I_Messaging::Method::GET, - "kubernetes.default.svc", - 443, - _, + sendSyncMessage( + HTTPMethod::GET, "/api/v1/namespaces/", - "Authorization: Bearer 123\nConnection: close", + _, _, _ ) - ).WillRepeatedly(Return(Maybe(namespaces))); + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, namespaces))); + i_orchestration_tools->getClusterId(); routine(); } diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc index 3d9c802..94de360 100644 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc @@ -110,7 +110,7 @@ public: EXPECT_CALL(mock_details_resolver, isReverseProxy()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isKernelVersion3OrHigher()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isGwNotVsx()).WillRepeatedly(Return(false)); - EXPECT_CALL(mock_details_resolver, isVersionEqualOrAboveR8110()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, isVersionAboveR8110()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, parseNginxMetadata()).WillRepeatedly(Return(no_nginx)); EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("1.1.1")); @@ -250,7 +250,8 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource) EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response)); - EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)).WillOnce(Return(true)); + EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) + .WillOnce(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc index 9f073a6..38a8b7a 100755 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc @@ -85,15 +85,13 @@ public: mockRestCall(RestAction::SET, "agent-uninstall", _) ).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandlerAgentUninstall))); - EXPECT_CALL(mock_message, mockSendPersistentMessage( - false, - _, - I_Messaging::Method::POST, + EXPECT_CALL(mock_message, sendAsyncMessage( + HTTPMethod::POST, "/api/v1/agents/events", _, - _, - MessageTypeTag::REPORT - )).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); + MessageCategory::LOG, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); doEncrypt(); EXPECT_CALL(mock_orchestration_tools, loadTenantsFromDir(_)).Times(1); @@ -137,7 +135,7 @@ public: EXPECT_CALL(mock_details_resolver, isReverseProxy()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isKernelVersion3OrHigher()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isGwNotVsx()).WillRepeatedly(Return(false)); - EXPECT_CALL(mock_details_resolver, isVersionEqualOrAboveR8110()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, isVersionAboveR8110()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, parseNginxMetadata()).WillRepeatedly(Return(no_nginx)); EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("1.1.1")); EXPECT_CALL(mock_details_resolver, getHostname()).WillRepeatedly(Return(string("hostname"))); @@ -505,7 +503,7 @@ TEST_F(OrchestrationTest, check_sending_registration_data) EXPECT_CALL(mock_orchestration_tools, readFile(_)).WillOnce(Return(response)); EXPECT_CALL(mock_service_controller, updateServiceConfiguration(_, _, _, _, _, _)) .WillOnce(Return(Maybe())); - EXPECT_CALL(mock_message, setActiveFog(_, _, _, _)).WillOnce(Return(true)); + EXPECT_CALL(mock_message, setFogConnection(_, _, _, _)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, _)).WillRepeatedly(Return(string())); EXPECT_CALL(mock_service_controller, getPolicyVersion()).WillRepeatedly(ReturnRef(first_policy_version)); EXPECT_CALL(mock_shell_cmd, getExecOutput(_, _, _)).WillRepeatedly(Return(string())); @@ -630,7 +628,7 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdatRollback) .WillOnce(Return(policy_response)); EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_path, policy_file_path + ".last")) .WillOnce(Return(true)); - EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)) + EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) .Times(2).WillRepeatedly(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("")).Times(2); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); @@ -699,7 +697,7 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdatRollback) EXPECT_CALL( mock_message, - setActiveFog(new_host_address, 443, true, MessageTypeTag::GENERIC) + setFogConnection(new_host_address, 443, true, MessageCategory::GENERIC) ).WillOnce(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("/test")); EXPECT_CALL(mock_status, setLastUpdateAttempt()); @@ -829,7 +827,8 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate) .WillOnce(Return(new_policy_response)); EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_path, policy_file_path + ".last")) .WillOnce(Return(true)); - EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)).WillOnce(Return(true)); + EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) + .WillOnce(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); expectDetailsResolver(); @@ -892,7 +891,7 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate) EXPECT_CALL( mock_message, - setActiveFog(new_host_address, 443, true, MessageTypeTag::GENERIC) + setFogConnection(new_host_address, 443, true, MessageCategory::GENERIC) ).WillOnce(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("/test")); EXPECT_CALL(mock_status, setLastUpdateAttempt()); @@ -989,7 +988,8 @@ TEST_F(OrchestrationTest, startOrchestrationPoliceWithFailures) updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "", _) ).Times(2).WillRepeatedly(Return(Maybe())); - EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)).WillOnce(Return(true)); + EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) + .WillOnce(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); expectDetailsResolver(); @@ -1117,7 +1117,8 @@ TEST_F(OrchestrationTest, loadOrchestrationPolicyFromBackup) mock_orchestration_tools, copyFile(orchestration_policy_file_path_bk, orchestration_policy_file_path) ).WillOnce(Return(true)); - EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)).WillOnce(Return(true)); + EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) + .WillOnce(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); expectDetailsResolver(); @@ -1245,7 +1246,8 @@ TEST_F(OrchestrationTest, manifestUpdate) EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response)); - EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)).WillOnce(Return(true)); + EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) + .WillOnce(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); expectDetailsResolver(); @@ -1381,7 +1383,8 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate) EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response)); EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_path, policy_file_path + ".last")) .WillOnce(Return(true)); - EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)).WillOnce(Return(true)); + EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) + .WillOnce(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); @@ -1527,7 +1530,8 @@ TEST_F(OrchestrationTest, failedDownloadSettings) EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response)); - EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)).WillOnce(Return(true)); + EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) + .WillOnce(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); @@ -1679,7 +1683,7 @@ TEST_P(OrchestrationTest, orchestrationFirstRun) EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(false)); EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)) .WillOnce(Return(response)); - EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)). + EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)). Times(1). WillRepeatedly(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); @@ -1929,7 +1933,8 @@ TEST_F(OrchestrationTest, dataUpdate) EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true)); - EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC)).WillOnce(Return(true)); + EXPECT_CALL(mock_message, setFogConnection(host_address, 443, true, MessageCategory::GENERIC)) + .WillOnce(Return(true)); EXPECT_CALL(mock_update_communication, setAddressExtenesion("")); EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe())); EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false)); diff --git a/components/security_apps/orchestration/service_controller/service_controller.cc b/components/security_apps/orchestration/service_controller/service_controller.cc index 7c73690..b09e9b7 100755 --- a/components/security_apps/orchestration/service_controller/service_controller.cc +++ b/components/security_apps/orchestration/service_controller/service_controller.cc @@ -17,12 +17,13 @@ #include #include #include +#include +#include #include "config.h" #include "debug.h" #include "rest.h" #include "connkey.h" -#include "i_messaging.h" #include "common.h" #include "log_generator.h" #include "i_orchestration_tools.h" @@ -165,7 +166,7 @@ ServiceDetails::isServiceActive() const bool is_registered = status.find("not-registered") == string::npos && status.find("registered") != string::npos; bool is_running = status.find("not-running") == string::npos && status.find("running") != string::npos; - dbgInfo(D_ORCHESTRATOR) + dbgTrace(D_ORCHESTRATOR) << "Successfully set service status. Service name: " << service_name << ", Status: " @@ -195,19 +196,25 @@ ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy SendConfigurations new_config(configuration_id, policy_version); I_Messaging *messaging = Singleton::Consume::by(); - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN); - bool res = messaging->sendObject( + + MessageMetadata new_config_req_md("127.0.0.1", service_port); + new_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); + new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + auto res = messaging->sendSyncMessage( + HTTPMethod::POST, + "/set-new-configuration", new_config, - I_Messaging::Method::POST, - "127.0.0.1", - service_port, - conn_flags, - "/set-new-configuration" + MessageCategory::GENERIC, + new_config_req_md ); - if (!res) { - dbgDebug(D_ORCHESTRATOR) << "Service " << service_name << " didn't respond to new configuration request"; + if (!res.ok()) { + auto err = res.getErr(); + dbgDebug(D_ORCHESTRATOR) + << "Service: " + << service_name + << " didn't get new configuration. Error: " + << err.getBody(); return ReconfStatus::FAILED; } @@ -322,7 +329,7 @@ private: Maybe updateServiceConfigurationFile( const string &configuration_name, const string &configuration_file_path, - const string &new_configuration_path); + const string &new_configuration); ReconfStatus getUpdatedReconfStatus(); Maybe getServiceDetails(const string &service_name); @@ -694,6 +701,26 @@ ServiceController::Impl::createDirectoryForChildTenant( return true; } +static string +getChecksum(const string &file_path) +{ + auto orchestration_tools = Singleton::Consume::by(); + Maybe file_checksum = orchestration_tools->calculateChecksum( + Package::ChecksumTypes::MD5, + file_path + ); + + if (file_checksum.ok()) return file_checksum.unpack(); + + string checksum = "unknown version"; + try { + checksum = to_string(boost::uuids::random_generator()()); + } catch (const boost::uuids::entropy_error &e) { + dbgDebug(D_ORCHESTRATOR) << "Couldn't generate random checksum"; + } + return checksum; +} + Maybe ServiceController::Impl::updateServiceConfiguration( const string &new_policy_path, @@ -869,7 +896,8 @@ ServiceController::Impl::updateServiceConfiguration( // In a multi-tenant env, we send the signal to the services only on the last iteration if (!is_multi_tenant_env || last_iteration) { - auto is_send_signal_for_services = sendSignalForServices(nano_services_to_update, version_value); + auto is_send_signal_for_services = + sendSignalForServices(nano_services_to_update, version_value + ',' + getChecksum(new_policy_path)); was_policy_updated &= is_send_signal_for_services.ok(); if (!is_send_signal_for_services.ok()) send_signal_for_services_err = is_send_signal_for_services.getErr(); } @@ -1003,21 +1031,20 @@ Maybe ServiceController::Impl::updateServiceConfigurationFile( const string &configuration_name, const string &configuration_file_path, - const string &new_configuration_path) + const string &new_configuration) { - dbgFlow(D_ORCHESTRATOR) << "Updating configuration. Config Name: " << configuration_name; if (orchestration_tools->doesFileExist(configuration_file_path)) { Maybe old_configuration = orchestration_tools->readFile(configuration_file_path); if (old_configuration.ok()) { - bool service_changed = old_configuration.unpack().compare(new_configuration_path) != 0; + bool service_changed = old_configuration.unpack().compare(new_configuration) != 0; if (service_changed == false) { dbgDebug(D_ORCHESTRATOR) << "There is no update for policy file: " << configuration_file_path; return Maybe(); } dbgDebug(D_ORCHESTRATOR) - << "Starting to update " << configuration_file_path << " to " << new_configuration_path; + << "Starting to update " << configuration_file_path << " to " << new_configuration; string old_configuration_backup_path = configuration_file_path + getConfigurationWithDefault( ".bk", "orchestration", @@ -1045,7 +1072,7 @@ ServiceController::Impl::updateServiceConfigurationFile( } } - if (orchestration_tools->writeFile(new_configuration_path, configuration_file_path)) { + if (orchestration_tools->writeFile(new_configuration, configuration_file_path)) { dbgDebug(D_ORCHESTRATOR) << "New policy file has been saved in: " << configuration_file_path; } else { dbgWarning(D_ORCHESTRATOR) << "Failed to save new policy file"; diff --git a/components/security_apps/orchestration/update_communication/fog_authenticator.cc b/components/security_apps/orchestration/update_communication/fog_authenticator.cc index 55ddfe8..a827bc2 100755 --- a/components/security_apps/orchestration/update_communication/fog_authenticator.cc +++ b/components/security_apps/orchestration/update_communication/fog_authenticator.cc @@ -17,6 +17,7 @@ #include "log_generator.h" #include "agent_details.h" #include "version.h" +#include "i_messaging.h" #include #include @@ -24,7 +25,6 @@ using namespace std; using namespace cereal; -using HTTPMethod = I_Messaging::Method; USE_DEBUG_FLAG(D_ORCHESTRATOR); @@ -141,7 +141,7 @@ FogAuthenticator::registerAgent( const string &platform, const string &architecture) const { - dbgInfo(D_ORCHESTRATOR) << "Starting agent registration to fog"; + dbgFlow(D_ORCHESTRATOR) << "Starting agent registration to fog"; auto details_resolver = Singleton::Consume::by(); RegistrationRequest request( @@ -201,8 +201,8 @@ FogAuthenticator::registerAgent( request << make_pair("isGwNotVsx", "true"); } - if (details_resolver->isVersionEqualOrAboveR8110()) { - request << make_pair("isVersionEqualOrAboveR8110", "true"); + if (details_resolver->isVersionAboveR8110()) { + request << make_pair("isVersionAboveR8110", "true"); } #if defined(gaia) || defined(smb) @@ -214,8 +214,13 @@ FogAuthenticator::registerAgent( } #endif // gaia || smb - auto fog_messaging = Singleton::Consume::by(); - if (fog_messaging->sendObject(request, HTTPMethod::POST, fog_address_ex + "/agents")) { + dbgDebug(D_ORCHESTRATOR) << "Sending registration request to fog"; + auto request_status = Singleton::Consume::by()->sendSyncMessage( + HTTPMethod::POST, + "/agents", + request + ); + if (request_status.ok()) { dbgDebug(D_ORCHESTRATOR) << "Agent has registered successfully."; i_agent_details->setAgentId(request.getAgentId()); i_agent_details->setProfileId(request.getProfileId()); @@ -236,7 +241,12 @@ FogAuthenticator::registerAgent( ReportIS::Tags::ORCHESTRATOR ); - return genError("Failed to register agent with the Fog"); + return genError( + "Failed to register agent with the Fog. " + + request_status.getErr().getBody() + + " " + + request_status.getErr().toString() + ); } Maybe @@ -246,15 +256,20 @@ FogAuthenticator::getAccessToken(const UserCredentials &user_credentials) const static const string grant_type_string = "/oauth/token?grant_type=client_credentials"; TokenRequest request = TokenRequest(); - auto fog_messaging = Singleton::Consume::by(); - auto sending_result = fog_messaging->sendObject( - request, - HTTPMethod::POST, - fog_address_ex + grant_type_string, + MessageMetadata request_token_md; + request_token_md.insertHeader( + "Authorization", buildBasicAuthHeader(user_credentials.getClientId(), user_credentials.getSharedSecret()) ); + auto request_token_status = Singleton::Consume::by()->sendSyncMessage( + HTTPMethod::POST, + grant_type_string, + request, + MessageCategory::GENERIC, + request_token_md + ); - if (sending_result) { + if (request_token_status.ok()) { auto data_path = getConfigurationWithDefault( filesystem_prefix + "/data/", "encryptor", @@ -371,6 +386,7 @@ FogAuthenticator::getCredentials() return maybe_credentials; } + dbgTrace(D_ORCHESTRATOR) << "Credentials were not not receoived from the file. Getting registration data."; auto reg_data = getRegistrationData(); if (!reg_data.ok()) { return genError("Failed to load a valid registration token, Error: " + reg_data.getErr()); @@ -436,13 +452,7 @@ FogAuthenticator::buildBasicAuthHeader(const string &username, const string &pas { auto orchestration_tools = Singleton::Consume::by(); auto auth_encode = orchestration_tools->base64Encode(username + ":" + pass); - return "Authorization: Basic " + auth_encode + "\r\n"; -} - -string -FogAuthenticator::buildOAuth2Header(const string &token) const -{ - return "Authorization: Bearer " + token + "\r\n"; + return "Basic " + auth_encode; } void @@ -455,6 +465,7 @@ FogAuthenticator::setAddressExtenesion(const std::string &extension) Maybe FogAuthenticator::authenticateAgent() { + dbgFlow(D_ORCHESTRATOR) << "Authenticating the agent"; const int min_expiration_time = 10; if (!credentials.ok()) { dbgDebug(D_ORCHESTRATOR) << "Getting Agent credentials."; diff --git a/components/security_apps/orchestration/update_communication/fog_communication.cc b/components/security_apps/orchestration/update_communication/fog_communication.cc index c70ea54..1772749 100755 --- a/components/security_apps/orchestration/update_communication/fog_communication.cc +++ b/components/security_apps/orchestration/update_communication/fog_communication.cc @@ -24,7 +24,6 @@ using namespace std; using namespace cereal; -using HTTPMethod = I_Messaging::Method; USE_DEBUG_FLAG(D_ORCHESTRATOR); @@ -43,16 +42,16 @@ FogCommunication::getUpdate(CheckUpdateRequest &request) auto unpacked_access_token = access_token.unpack().getToken(); static const string check_update_str = "/api/v2/agents/resources"; - auto request_status = Singleton::Consume::by()->sendObject( - request, + auto response = Singleton::Consume::by()->sendSyncMessage( HTTPMethod::POST, - fog_address_ex + check_update_str, - buildOAuth2Header(unpacked_access_token) + check_update_str, + request ); - if (!request_status) { - dbgDebug(D_ORCHESTRATOR) << "Failed to get response after check update request."; - return genError("Failed to request updates"); + if (!response.ok()) { + const auto &fog_err = response.getErr(); + dbgDebug(D_ORCHESTRATOR) << "Check update request fail. Error: " << fog_err.getBody(); + return genError(fog_err.getBody()); } string policy_mgmt_mode = getSettingWithDefault("management", "profileManagedMode"); @@ -93,7 +92,7 @@ FogCommunication::getUpdate(CheckUpdateRequest &request) } Maybe -FogCommunication::downloadAttributeFile(const GetResourceFile &resourse_file) +FogCommunication::downloadAttributeFile(const GetResourceFile &resourse_file, const string &file_path) { if (!access_token.ok()) return genError("Acccess Token not available."); @@ -105,27 +104,34 @@ FogCommunication::downloadAttributeFile(const GetResourceFile &resourse_file) return i_declarative_policy->getCurrPolicy(); } static const string file_attribute_str = "/api/v2/agents/resources/"; - Maybe attribute_file = Singleton::Consume::by()->downloadFile( - resourse_file, - resourse_file.getRequestMethod(), - fog_address_ex + file_attribute_str + resourse_file.getFileName(), - buildOAuth2Header(unpacked_access_token) // Header - ); - return attribute_file; + auto attribute_file = Singleton::Consume::by()->downloadFile( + resourse_file.getRequestMethod(), + file_attribute_str + resourse_file.getFileName(), + file_path + ); + if (!attribute_file.ok()) { + const auto &fog_err = attribute_file.getErr(); + return genError(fog_err.getBody()); + } + return file_path; } Maybe FogCommunication::sendPolicyVersion(const string &policy_version, const string &policy_versions) const { - PolicyVersionPatchRequest request(policy_version, policy_versions); - auto fog_messaging = Singleton::Consume::by(); dbgTrace(D_ORCHESTRATOR) << "Sending patch request to the fog. Policy version: " << policy_version << " , Policy versions: " << policy_versions; - if (fog_messaging->sendNoReplyObject(request, HTTPMethod::PATCH, fog_address_ex + "/agents")) { + PolicyVersionPatchRequest request(policy_version, policy_versions); + auto request_status = Singleton::Consume::by()->sendSyncMessageWithoutResponse( + HTTPMethod::PATCH, + "/agents", + request + ); + if (request_status) { dbgTrace(D_ORCHESTRATOR) << "Patch request was sent successfully to the fog." << " Policy versions: " diff --git a/components/security_apps/orchestration/update_communication/hybrid_communication.cc b/components/security_apps/orchestration/update_communication/hybrid_communication.cc index 4d594fe..6bf54aa 100755 --- a/components/security_apps/orchestration/update_communication/hybrid_communication.cc +++ b/components/security_apps/orchestration/update_communication/hybrid_communication.cc @@ -24,7 +24,6 @@ #include using namespace std; -using HTTPMethod = I_Messaging::Method; USE_DEBUG_FLAG(D_ORCHESTRATOR); @@ -51,16 +50,17 @@ HybridCommunication::getUpdate(CheckUpdateRequest &request) dbgTrace(D_ORCHESTRATOR) << "Getting updates in Hybrid Communication"; if (access_token.ok()) { static const string check_update_str = "/api/v2/agents/resources"; - auto request_status = Singleton::Consume::by()->sendObject( - request, + auto request_status = Singleton::Consume::by()->sendSyncMessage( HTTPMethod::POST, - fog_address_ex + check_update_str, - buildOAuth2Header((*access_token).getToken()) + check_update_str, + request ); - if (!request_status) { - dbgWarning(D_ORCHESTRATOR) << "Failed to get response after check update request."; - return genError("Failed to request updates"); + + if (!request_status.ok()) { + auto fog_err = request_status.getErr(); + dbgDebug(D_ORCHESTRATOR) << "Check update request fail. Error: " << fog_err.getBody(); + return genError(fog_err.getBody()); } Maybe maybe_new_manifest = request.getManifest(); @@ -82,14 +82,6 @@ HybridCommunication::getUpdate(CheckUpdateRequest &request) if (env == EnvType::K8S && !policy_response.empty()) { dbgDebug(D_ORCHESTRATOR) << "Policy has changes, sending notification to tuning host"; I_AgentDetails *agentDetails = Singleton::Consume::by(); - I_Messaging *messaging = Singleton::Consume::by(); - - UpdatePolicyCrdObject policy_change_object(policy_response); - - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::EXTERNAL); - - string tenant_header = "X-Tenant-Id: " + agentDetails->getTenantId(); auto get_tuning_host = []() { @@ -107,18 +99,22 @@ HybridCommunication::getUpdate(CheckUpdateRequest &request) return tuning_host; }; - bool ok = messaging->sendNoReplyObject( - policy_change_object, - I_Messaging::Method::POST, - get_tuning_host(), - 80, - conn_flags, + MessageMetadata update_policy_crd_md(get_tuning_host(), 80); + update_policy_crd_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); + UpdatePolicyCrdObject policy_change_object(policy_response); + + auto i_messaging = Singleton::Consume::by(); + bool tuning_req_status = i_messaging->sendSyncMessageWithoutResponse( + HTTPMethod::POST, "/api/update-policy-crd", - tenant_header + policy_change_object, + MessageCategory::GENERIC, + update_policy_crd_md ); - dbgDebug(D_ORCHESTRATOR) << "sent tuning policy update notification ok: " << ok; - if (!ok) { - dbgWarning(D_ORCHESTRATOR) << "failed to send tuning notification"; + if (!tuning_req_status) { + dbgWarning(D_ORCHESTRATOR) << "Failed to send tuning notification"; + } else { + dbgDebug(D_ORCHESTRATOR) << "Successfully sent tuning policy update notification"; } } @@ -128,14 +124,17 @@ HybridCommunication::getUpdate(CheckUpdateRequest &request) } Maybe -HybridCommunication::downloadAttributeFile(const GetResourceFile &resourse_file) +HybridCommunication::downloadAttributeFile(const GetResourceFile &resourse_file, const string &file_path) { dbgTrace(D_ORCHESTRATOR) << "Downloading attribute file on hybrid mode, file name: " << resourse_file.getFileName(); if (resourse_file.getFileName() =="policy") { - return i_declarative_policy->getCurrPolicy(); + string downloaded_file = i_declarative_policy->getCurrPolicy(); + auto *orchestration_tools = Singleton::Consume::by(); + if (orchestration_tools->writeFile(downloaded_file, file_path)) return downloaded_file; + return genError("Failed to write the attribute file in hybrid mode. File: " + downloaded_file); } if (resourse_file.getFileName() == "manifest") { if (!access_token.ok()) return genError("Acccess Token not available."); @@ -143,13 +142,16 @@ HybridCommunication::downloadAttributeFile(const GetResourceFile &resourse_file) auto unpacked_access_token = access_token.unpack().getToken(); static const string file_attribute_str = "/api/v2/agents/resources/"; - Maybe attribute_file = Singleton::Consume::by()->downloadFile( - resourse_file, + auto attribute_file = Singleton::Consume::by()->downloadFile( resourse_file.getRequestMethod(), - fog_address_ex + file_attribute_str + resourse_file.getFileName(), - buildOAuth2Header((*access_token).getToken()) // Header + file_attribute_str + resourse_file.getFileName(), + file_path ); - return attribute_file; + if (!attribute_file.ok()) { + auto fog_err = attribute_file.getErr(); + return genError(fog_err.getBody()); + } + return file_path; } dbgTrace(D_ORCHESTRATOR) << "Unnecessary attribute files downloading on hybrid mode"; return string(""); diff --git a/components/security_apps/orchestration/update_communication/local_communication.cc b/components/security_apps/orchestration/update_communication/local_communication.cc index ba5743c..373d5dd 100755 --- a/components/security_apps/orchestration/update_communication/local_communication.cc +++ b/components/security_apps/orchestration/update_communication/local_communication.cc @@ -122,9 +122,14 @@ LocalCommunication::getUpdate(CheckUpdateRequest &request) } Maybe -LocalCommunication::downloadAttributeFile(const GetResourceFile &resource_file) +LocalCommunication::downloadAttributeFile(const GetResourceFile &resource_file, const string &file_path) { auto file_name = resource_file.getFileName(); + dbgTrace(D_ORCHESTRATOR) + << "Download " + << file_name + << " file in local communication, file path is redundant: " + << file_path; I_OrchestrationTools *orchestration_tools = Singleton::Consume::by(); if (file_name.compare("policy") == 0) { diff --git a/components/security_apps/orchestration/update_communication/update_communication.cc b/components/security_apps/orchestration/update_communication/update_communication.cc index b27db9f..e3a52f0 100755 --- a/components/security_apps/orchestration/update_communication/update_communication.cc +++ b/components/security_apps/orchestration/update_communication/update_communication.cc @@ -82,9 +82,9 @@ public: } Maybe - downloadAttributeFile(const GetResourceFile &resourse_file) override + downloadAttributeFile(const GetResourceFile &resourse_file, const string &file_path) override { - return i_update_comm_impl->downloadAttributeFile(resourse_file); + return i_update_comm_impl->downloadAttributeFile(resourse_file, file_path); } void diff --git a/components/security_apps/orchestration/update_communication/update_communication_ut/local_communication_ut.cc b/components/security_apps/orchestration/update_communication/update_communication_ut/local_communication_ut.cc index aec9925..2bae44a 100755 --- a/components/security_apps/orchestration/update_communication/update_communication_ut/local_communication_ut.cc +++ b/components/security_apps/orchestration/update_communication/update_communication_ut/local_communication_ut.cc @@ -45,9 +45,9 @@ public: } Maybe - downloadAttributeFile(const GetResourceFile &resourse_file) + downloadAttributeFile(const GetResourceFile &resourse_file, const string &file_path) { - return local_communication.downloadAttributeFile(resourse_file); + return local_communication.downloadAttributeFile(resourse_file, file_path); } void @@ -127,7 +127,7 @@ TEST_F(LocalCommunicationTest, downloadManifest) string new_manifest_string = "new manifest"; EXPECT_CALL(mock_orc_tools, readFile("/etc/cp/conf/offline_manifest.json")).WillOnce(Return(new_manifest_string)); GetResourceFile resourse_file(GetResourceFile::ResourceFileType::MANIFEST); - auto downloaded_string = downloadAttributeFile(resourse_file); + auto downloaded_string = downloadAttributeFile(resourse_file, "/tmp/orch_files"); EXPECT_TRUE(downloaded_string.ok()); EXPECT_EQ(downloaded_string.unpack(), new_manifest_string); } diff --git a/components/security_apps/rate_limit/rate_limit.cc b/components/security_apps/rate_limit/rate_limit.cc index 2025c9b..1c8cb19 100755 --- a/components/security_apps/rate_limit/rate_limit.cc +++ b/components/security_apps/rate_limit/rate_limit.cc @@ -37,7 +37,7 @@ USE_DEBUG_FLAG(D_RATE_LIMIT); using namespace std; -enum class RateLimitVedict { ACCEPT, DROP, DROP_AND_LOG }; +enum class RateLimitVerdict { ACCEPT, DROP, DROP_AND_LOG }; class RateLimit::Impl : @@ -74,8 +74,100 @@ public: return !should_rule_be_exact_match && str_starts_with(request_uri, rule_uri); } + bool + isRuleMatchingUri(const string &rule_uri, const string &request_uri, const RateLimitRule &rule) + { + if (rule_uri == request_uri || + rule_uri == request_uri + "/" || + rule_uri + "/" == request_uri) { + dbgDebug(D_RATE_LIMIT) + << "Found Exact match to request URI: " + << request_uri + << ", rule URI: " + << rule_uri; + return true; + } + + if (rule_uri == "/") { + dbgDebug(D_RATE_LIMIT) + << "Matched new longest rule, request URI: " + << request_uri + << ", rule URI: " + << rule_uri; + return true; + } + + if (isRuleMatchingUri(rule_uri, request_uri, rule.isExactMatch())) { + dbgDebug(D_RATE_LIMIT) + << "Matched new longest rule, request URI: " + << request_uri + << ", rule URI: " + << rule_uri; + return true; + } + + return false; + } + + bool + shouldUpdateBestMatchingRule( + const RateLimitRule &rule, + const unordered_map> &condition_map, + int full_rule_uri_length, + int rate_limit_longest_match, + float current_matched_rule_limit, + RateLimitAction current_matched_rule_verdict) + { + if (!rule.isMatchAny() && !rule.getRateLimitMatch().matchAttributes(condition_map)) { + dbgTrace(D_RATE_LIMIT) << "The request does not match the rule's condition"; + return false; + } + + RateLimitAction rule_action = calcRuleAction(rule); + if (current_matched_rule_verdict < rule_action) { + dbgTrace(D_RATE_LIMIT) + << "Rule's action is more strict than already matched rule. current rule's action: " + << RateLimitConfig::rate_limit_action_to_string.at(rule_action) + << ", previously matched rule's action: " + << RateLimitConfig::rate_limit_action_to_string.at(current_matched_rule_verdict); + return true; + } + + if (rule_action < current_matched_rule_verdict) { + dbgTrace(D_RATE_LIMIT) + << "Rule's action is less strict than already matched rule. current rule's action: " + << RateLimitConfig::rate_limit_action_to_string.at(rule_action) + << ", previously matched rule's action: " + << RateLimitConfig::rate_limit_action_to_string.at(current_matched_rule_verdict); + return false; + } + + if (full_rule_uri_length < rate_limit_longest_match) { + dbgTrace(D_RATE_LIMIT) + << "rule is shorter than already matched rule. current rule length: " + << full_rule_uri_length + << ", previously longest matched rule length: " + << rate_limit_longest_match; + return false; + } + + if (full_rule_uri_length == rate_limit_longest_match && current_matched_rule_limit < calcRuleLimit(rule)) { + dbgTrace(D_RATE_LIMIT) + << "rule limit is more permissive than already matched rule. current rule limit: " + << limit + << ", previously matched rule limit: " + << current_matched_rule_limit; + return false; + } + + return true; + } + Maybe - findRateLimitRule(const string &matched_uri, string &asset_id) + findRateLimitRule( + const string &matched_uri, + string &asset_id, + const unordered_map> &condition_map) { WaapConfigAPI api_config; WaapConfigApplication application_config; @@ -97,13 +189,14 @@ public: return genError("Failed to get rate limit configuration. Skipping rate limit check."); const auto &rate_limit_config = maybe_rate_limit_config.unpack(); - mode = rate_limit_config.getRateLimitMode(); + practice_action = rate_limit_config.getRateLimitMode(); - if (mode == "Inactive") return genError("Rate limit mode is Inactive in policy"); + if (practice_action == RateLimitAction::INACTIVE) return genError("Rate limit mode is Inactive in policy"); - set rule_set; Maybe matched_rule = genError("URI did not match any rate limit rule."); int rate_limit_longest_match = 0; + float current_matched_rule_limit = 0; + RateLimitAction current_matched_rule_verdict = RateLimitAction::INACTIVE; for (const auto &application_url : site_config->get_applicationUrls()) { dbgTrace(D_RATE_LIMIT) << "Application URL: " << application_url; @@ -120,54 +213,30 @@ public: string full_rule_uri = application_uri + rule.getRateLimitUri(); int full_rule_uri_length = full_rule_uri.length(); - // avoiding duplicates - if (!rule_set.insert(full_rule_uri).second) continue; - dbgTrace(D_RATE_LIMIT) - << "Trying to match rule uri: " + << "Trying to match rule URI: " << full_rule_uri - << " with request uri: " + << " with request URI: " << matched_uri; - if (full_rule_uri_length < rate_limit_longest_match) { - dbgDebug(D_RATE_LIMIT) - << "rule is shorter then already matched rule. current rule length: " - << full_rule_uri_length - << ", previously longest matched rule length: " - << rate_limit_longest_match; + if (!isRuleMatchingUri(full_rule_uri, matched_uri, rule)) { + dbgTrace(D_RATE_LIMIT) << "No match"; continue; } - if (full_rule_uri == matched_uri || - full_rule_uri == matched_uri + "/" || - full_rule_uri + "/" == matched_uri) { - dbgDebug(D_RATE_LIMIT) - << "Found Exact match to request uri: " - << matched_uri - << ", rule uri: " - << full_rule_uri; - return rule; - } + bool should_update_rule = shouldUpdateBestMatchingRule( + rule, + condition_map, + full_rule_uri_length, + rate_limit_longest_match, + current_matched_rule_limit, + current_matched_rule_verdict); - if (rule.getRateLimitUri() == "/") { - dbgDebug(D_RATE_LIMIT) - << "Matched new longest rule, request uri: " - << matched_uri - << ", rule uri: " - << full_rule_uri; - matched_rule = rule; - rate_limit_longest_match = full_rule_uri_length; - continue; - } - - if (isRuleMatchingUri(full_rule_uri, matched_uri, rule.isExactMatch())) { - dbgDebug(D_RATE_LIMIT) - << "Matched new longest rule, request uri: " - << matched_uri - << ", rule uri: " - << full_rule_uri; + if (should_update_rule) { matched_rule = rule; rate_limit_longest_match = full_rule_uri_length; + current_matched_rule_verdict = calcRuleAction(rule); + current_matched_rule_limit = calcRuleLimit(rule); } } } @@ -203,21 +272,27 @@ public: string source_ip = ""; if (maybe_source_ip.ok()) source_ip = ipAddrToStr(maybe_source_ip.unpack()); - if (shouldApplyException(uri, source_identifier, source_ip)) { - dbgDebug(D_RATE_LIMIT) << "found accept exception, not enforcing rate limit on this uri: " << uri; + unordered_map> condition_map = createConditionMap(uri, source_ip, source_identifier); + if (shouldApplyException(condition_map)) { + dbgDebug(D_RATE_LIMIT) << "found accept exception, not enforcing rate limit on this URI: " << uri; return ACCEPT; } string asset_id; - auto maybe_rule = findRateLimitRule(uri, asset_id); + auto maybe_rule = findRateLimitRule(uri, asset_id, condition_map); if (!maybe_rule.ok()) { dbgDebug(D_RATE_LIMIT) << "Not Enforcing Rate Limit: " << maybe_rule.getErr(); return ACCEPT; } const auto &rule = maybe_rule.unpack(); + if (rule.getRateLimitAction() == RateLimitAction::INACTIVE) { + dbgDebug(D_RATE_LIMIT) << "Rule's action is Inactive, rate limit will not be enforced"; + return ACCEPT; + } + burst = rule.getRateLimit(); - limit = static_cast(rule.getRateLimit()) / (rule.getRateLimitScope() == "Minute" ? 60 : 1); + limit = calcRuleLimit(rule); dbgTrace(D_RATE_LIMIT) << "found rate limit rule with: " @@ -226,18 +301,18 @@ public: << (rule.getRateLimitScope() == "Minute" ? 60 : 1) << " seconds"; - string unique_key = asset_id + ":" + source_identifier + ":" + uri; - if (!unique_key.empty() && unique_key.back() == '/') unique_key.pop_back(); + string unique_key = asset_id + ":" + source_identifier + ":" + rule.getRateLimitUri(); + if (unique_key.back() == '/') unique_key.pop_back(); auto verdict = decide(unique_key); - if (verdict == RateLimitVedict::ACCEPT) { + if (verdict == RateLimitVerdict::ACCEPT) { dbgTrace(D_RATE_LIMIT) << "Received ACCEPT verdict."; return ACCEPT; } - if (verdict == RateLimitVedict::DROP_AND_LOG) sendLog(uri, source_identifier, source_ip, rule); + if (verdict == RateLimitVerdict::DROP_AND_LOG) sendLog(uri, source_identifier, source_ip, rule); - if (mode == "Active") { + if (calcRuleAction(rule) == RateLimitAction::PREVENT) { dbgTrace(D_RATE_LIMIT) << "Received DROP verdict, this request will be blocked by rate limit"; return DROP; } @@ -246,19 +321,33 @@ public: return ACCEPT; } + RateLimitAction + calcRuleAction(const RateLimitRule &rule) + { + if (rule.getRateLimitAction() == RateLimitAction::ACCORDING_TO_PRACTICE) return practice_action; + + return rule.getRateLimitAction(); + } + + float + calcRuleLimit(const RateLimitRule &rule) + { + return static_cast(rule.getRateLimit()) / (rule.getRateLimitScope() == "Minute" ? 60 : 1); + } + string getListenerName() const override { return "rate limit"; } - RateLimitVedict + RateLimitVerdict decide(const string &key) { if (redis == nullptr) { dbgDebug(D_RATE_LIMIT) << "there is no connection to the redis at the moment, unable to enforce rate limit"; reconnectRedis(); - return RateLimitVedict::ACCEPT; + return RateLimitVerdict::ACCEPT; } redisReply* reply = static_cast(redisCommand(redis, "EVALSHA %s 1 %s %f %d", @@ -268,26 +357,26 @@ public: dbgDebug(D_RATE_LIMIT) << "Error executing Redis command: No reply received, unable to enforce rate limit"; reconnectRedis(); - return RateLimitVedict::ACCEPT; + return RateLimitVerdict::ACCEPT; } // redis's lua script returned true - accept if (reply->type == REDIS_REPLY_INTEGER) { freeReplyObject(reply); - return RateLimitVedict::ACCEPT; + return RateLimitVerdict::ACCEPT; } // redis's lua script returned false - drop, no need to log if (reply->type == REDIS_REPLY_NIL) { freeReplyObject(reply); - return RateLimitVedict::DROP; + return RateLimitVerdict::DROP; } // redis's lua script returned string - drop and send log const char* log_str = "BLOCK AND LOG"; if (reply->type == REDIS_REPLY_STRING && strncmp(reply->str, log_str, strlen(log_str)) == 0) { freeReplyObject(reply); - return RateLimitVedict::DROP_AND_LOG; + return RateLimitVerdict::DROP_AND_LOG; } dbgDebug(D_RATE_LIMIT) @@ -295,7 +384,7 @@ public: << reply->type << ". not enforcing rate limit for this request."; freeReplyObject(reply); - return RateLimitVedict::ACCEPT; + return RateLimitVerdict::ACCEPT; } void @@ -344,7 +433,7 @@ public: << LogField("ruleName", rule_by_ctx.getRuleName()) << LogField("httpUriPath", uri) << LogField("httpSourceId", source_identifier) - << LogField("securityAction", (mode == "Active" ? "Prevent" : "Detect")) + << LogField("securityAction", (calcRuleAction(rule) == RateLimitAction::PREVENT ? "Prevent" : "Detect")) << LogField("waapIncidentType", "Rate Limit"); auto env = Singleton::Consume::by(); @@ -363,28 +452,33 @@ public: } bool - shouldApplyException(const string &uri, const string &source_identifier, const string &source_ip) + shouldApplyException(const unordered_map> &exceptions_dict) { dbgTrace(D_RATE_LIMIT) << "matching exceptions"; - unordered_map> exceptions_dict; - - // collect sourceip, sourceIdentifier, url - if (!source_ip.empty()) exceptions_dict["sourceIP"].insert(source_ip); - exceptions_dict["sourceIdentifier"].insert(source_identifier); - exceptions_dict["url"].insert(uri); auto behaviors = Singleton::Consume::by()->getBehavior(exceptions_dict); for (auto const &behavior : behaviors) { if (behavior == action_accept) { - dbgTrace(D_RATE_LIMIT) << "matched exceptions for " << uri << " should accept"; + dbgTrace(D_RATE_LIMIT) << "matched exceptions for current request, should accept"; return true; } } - dbgTrace(D_RATE_LIMIT) << "No accept exceptions found for this uri and source ip"; + dbgTrace(D_RATE_LIMIT) << "No accept exceptions found for this request"; return false; } + unordered_map> + createConditionMap(const string &uri, const string &source_ip, const string &source_identifier) + { + unordered_map> condition_map; + if (!source_ip.empty()) condition_map["sourceIP"].insert(source_ip); + condition_map["sourceIdentifier"].insert(source_identifier); + condition_map["url"].insert(uri); + + return condition_map; + } + string ipAddrToStr(const IPAddr& ip_address) const { @@ -398,7 +492,7 @@ public: { disconnectRedis(); - const string &redis_ip = getConfigurationWithDefault("127.0.0.1", "connection", "Redis IP"); + const string redis_ip = getConfigurationWithDefault("127.0.0.1", "connection", "Redis IP"); int redis_port = getConfigurationWithDefault(6379, "connection", "Redis Port"); timeval timeout; @@ -476,7 +570,6 @@ public: false ); } - } void @@ -529,7 +622,7 @@ private: static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; - string mode; + RateLimitAction practice_action; string rate_limit_lua_script_hash; int burst; float limit; diff --git a/components/security_apps/rate_limit/rate_limit_config.cc b/components/security_apps/rate_limit/rate_limit_config.cc index 7084320..b5f5dff 100755 --- a/components/security_apps/rate_limit/rate_limit_config.cc +++ b/components/security_apps/rate_limit/rate_limit_config.cc @@ -1,15 +1,37 @@ #include "rate_limit_config.h" +using namespace std; + +const string RateLimitRule::default_match = + "{\"type\":\"condition\",\"op\":\"equals\",\"key\":\"any\",\"value\":[\"any\"]}"; bool RateLimitConfig::is_active = false; +const map RateLimitConfig::rate_limit_action_to_string = { + {RateLimitAction::INACTIVE, "Inactive"}, + {RateLimitAction::ACCORDING_TO_PRACTICE, "AccordingToPractice"}, + {RateLimitAction::DETECT, "Detect"}, + {RateLimitAction::PREVENT, "Prevent"}, +}; + + +// Actions in mgmt were changed from Active/Learn to Prevent/Detect. Active/Learn are being kept here for BC. +const map RateLimitConfig::rate_limit_string_to_action = { + {"Inactive", RateLimitAction::INACTIVE}, + {"AccordingToPractice", RateLimitAction::ACCORDING_TO_PRACTICE}, + {"Detect", RateLimitAction::DETECT}, + {"Learn", RateLimitAction::DETECT}, + {"Prevent", RateLimitAction::PREVENT}, + {"Active", RateLimitAction::PREVENT} +}; + void RateLimitTrigger::load(cereal::JSONInputArchive &ar) { - dbgTrace(D_REVERSE_PROXY) << "Serializing single Rate Limit rule's triggers"; + dbgTrace(D_RATE_LIMIT) << "Serializing single Rate Limit rule's triggers"; try { ar(cereal::make_nvp("id", id)); } catch (const cereal::Exception &e) { - dbgWarning(D_REVERSE_PROXY) + dbgWarning(D_RATE_LIMIT) << "Failed to load single Rate Limit JSON rule's triggers. Error: " << e.what(); ar.setNextName(nullptr); } @@ -18,39 +40,43 @@ RateLimitTrigger::load(cereal::JSONInputArchive &ar) void RateLimitRule::load(cereal::JSONInputArchive &ar) { - dbgTrace(D_REVERSE_PROXY) << "Serializing single Rate Limit rule"; + dbgTrace(D_RATE_LIMIT) << "Serializing single Rate Limit rule"; try { + string _action; ar(cereal::make_nvp("URI", uri)); ar(cereal::make_nvp("scope", scope)); ar(cereal::make_nvp("limit", limit)); ar(cereal::make_nvp("triggers", rate_limit_triggers)); + ar(cereal::make_nvp("action", _action)); + action = RateLimitConfig::rate_limit_string_to_action.at(_action); + ar(cereal::make_nvp("match", match)); } catch (const cereal::Exception &e) { - dbgWarning(D_REVERSE_PROXY) << "Failed to load single Rate Limit JSON rule. Error: " << e.what(); + dbgWarning(D_RATE_LIMIT) << "Failed to load single Rate Limit JSON rule. Error: " << e.what(); ar.setNextName(nullptr); } } void -RateLimitRule::prepare(const std::string &asset_id, int zone_id) +RateLimitRule::prepare(const string &asset_id, int zone_id) { - std::string zone_id_s = std::to_string(zone_id); - std::string zone; + string zone_id_s = to_string(zone_id); + string zone; if (isRootLocation()) { zone = "root_zone_" + asset_id + "_" + zone_id_s; } else { - std::string zone_name_suffix = uri; - std::replace(zone_name_suffix.begin(), zone_name_suffix.end(), '/', '_'); + string zone_name_suffix = uri; + replace(zone_name_suffix.begin(), zone_name_suffix.end(), '/', '_'); zone = "zone" + zone_name_suffix + "_" + zone_id_s; } - limit_req_template_value = "zone=" + zone + " burst=" + std::to_string(limit) + " nodelay"; + limit_req_template_value = "zone=" + zone + " burst=" + to_string(limit) + " nodelay"; // nginx conf will look like: limit_req_zone zone=_:10m rate=r/; - std::string rate_unit = scope == "Minute" ? "r/m" : "r/s"; + string rate_unit = scope == "Minute" ? "r/m" : "r/s"; limit_req_zone_template_value = - "zone=" + zone + ":" + cache_size + " rate=" + std::to_string(limit) + rate_unit; + "zone=" + zone + ":" + cache_size + " rate=" + to_string(limit) + rate_unit; - dbgTrace(D_REVERSE_PROXY) + dbgTrace(D_RATE_LIMIT) << "limit_req_zone nginx template value: " << limit_req_zone_template_value << ", limit_req nginx template value: " @@ -65,33 +91,58 @@ RateLimitRule::isRootLocation() const } auto non_root = uri.find_first_not_of("/"); - if (non_root != std::string::npos) { + if (non_root != string::npos) { return false; } return true; } +bool +RateLimitRule::isMatchAny() const +{ + return + match.getType() == MatchQuery::MatchType::Condition && + match.getKey() == "any" && + match.getValue().count("any") > 0; +} + void RateLimitConfig::load(cereal::JSONInputArchive &ar) { - dbgTrace(D_REVERSE_PROXY) << "Serializing Rate Limit config"; + dbgTrace(D_RATE_LIMIT) << "Serializing Rate Limit config"; try { + string _mode; ar(cereal::make_nvp("rules", rate_limit_rules)); - ar(cereal::make_nvp("mode", mode)); + ar(cereal::make_nvp("mode", _mode)); + mode = rate_limit_string_to_action.at(_mode); prepare(); } catch (const cereal::Exception &e) { - dbgWarning(D_REVERSE_PROXY) << "Failed to load single Rate Limit JSON config. Error: " << e.what(); + dbgWarning(D_RATE_LIMIT) << "Failed to load single Rate Limit JSON config. Error: " << e.what(); ar.setNextName(nullptr); } } -void -RateLimitConfig::addSiblingRateLimitRule(RateLimitRule &rule) { - rule.setExactMatch(); +RateLimitRule +RateLimitConfig::generateSiblingRateLimitRule(const RateLimitRule &rule) { RateLimitRule sibling_rule(rule); sibling_rule.appendSlash(); sibling_rule.setExactMatch(); - rate_limit_rules.push_back(sibling_rule); + + return sibling_rule; +} + +void +RateLimitConfig::addSiblingRateLimitRules() +{ + std::vector siblings; + for (auto &rule : rate_limit_rules) { + if (rule.isExactMatch()) { + siblings.push_back(generateSiblingRateLimitRule(rule)); + rule.setExactMatch(); + } + } + + rate_limit_rules.insert(rate_limit_rules.end(), siblings.begin(), siblings.end()); } void @@ -99,7 +150,7 @@ RateLimitConfig::prepare() { // Removes invalid rules auto last_valid_rule = - std::remove_if( + remove_if( rate_limit_rules.begin(), rate_limit_rules.end(), [](const RateLimitRule &rule) { return !rule; } @@ -107,35 +158,27 @@ RateLimitConfig::prepare() rate_limit_rules.erase(last_valid_rule, rate_limit_rules.end()); - // Removes duplicates sort(rate_limit_rules.begin(), rate_limit_rules.end()); - rate_limit_rules.erase(std::unique(rate_limit_rules.begin(), rate_limit_rules.end()), rate_limit_rules.end()); - std::for_each( - rate_limit_rules.begin(), - rate_limit_rules.end(), - [this](RateLimitRule &rule) { if (rule.isExactMatch()) { addSiblingRateLimitRule(rule); } } - ); + addSiblingRateLimitRules(); - dbgTrace(D_REVERSE_PROXY) + dbgTrace(D_RATE_LIMIT) << "Final rate-limit rules: " - << makeSeparatedStr(rate_limit_rules, "; ") - << "; Mode: " - << mode; + << makeSeparatedStr(rate_limit_rules, "; "); - setIsActive(mode != "Inactive"); + setIsActive(mode != RateLimitAction::INACTIVE); } const RateLimitRule -RateLimitConfig::findLongestMatchingRule(const std::string &nginx_uri) const +RateLimitConfig::findLongestMatchingRule(const string &nginx_uri) const { - dbgFlow(D_REVERSE_PROXY) << "Trying to find a matching rat-limit rule for NGINX URI: " << nginx_uri; + dbgFlow(D_RATE_LIMIT) << "Trying to find a matching rat-limit rule for NGINX URI: " << nginx_uri; size_t longest_len = 0; RateLimitRule longest_matching_rule; for (const RateLimitRule &rule : rate_limit_rules) { if (rule.getRateLimitUri() == nginx_uri) { - dbgTrace(D_REVERSE_PROXY) << "Found exact rate-limit match: " << rule; + dbgTrace(D_RATE_LIMIT) << "Found exact rate-limit match: " << rule; return rule; } @@ -143,15 +186,15 @@ RateLimitConfig::findLongestMatchingRule(const std::string &nginx_uri) const continue; } - if (std::equal(rule.getRateLimitUri().rbegin(), rule.getRateLimitUri().rend(), nginx_uri.rbegin())) { + if (equal(rule.getRateLimitUri().rbegin(), rule.getRateLimitUri().rend(), nginx_uri.rbegin())) { if (rule.getRateLimitUri().size() > longest_len) { longest_matching_rule = rule; longest_len = rule.getRateLimitUri().size(); - dbgTrace(D_REVERSE_PROXY) << "Longest matching rate-limit rule so far: " << rule; + dbgTrace(D_RATE_LIMIT) << "Longest matching rate-limit rule so far: " << rule; } } } - dbgTrace(D_REVERSE_PROXY) << "Longest matching rate-limit rule: " << longest_matching_rule; + dbgTrace(D_RATE_LIMIT) << "Longest matching rate-limit rule: " << longest_matching_rule; return longest_matching_rule; } diff --git a/components/security_apps/waap/include/i_serialize.h b/components/security_apps/waap/include/i_serialize.h index ec502db..00299a7 100755 --- a/components/security_apps/waap/include/i_serialize.h +++ b/components/security_apps/waap/include/i_serialize.h @@ -143,7 +143,7 @@ protected: size_t getIntervalsCount(); template - bool sendObject(T &obj, I_Messaging::Method method, std::string uri) + bool sendObject(T &obj, HTTPMethod method, std::string uri) { I_Messaging *messaging = Singleton::Consume::by(); I_AgentDetails *agentDetails = Singleton::Consume::by(); @@ -152,33 +152,28 @@ protected: return false; } if (agentDetails->getOrchestrationMode() == OrchestrationMode::HYBRID) { - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::EXTERNAL); - std::string tenant_header = "X-Tenant-Id: " + agentDetails->getTenantId(); - - return messaging->sendObject( - obj, + MessageMetadata req_md(getSharedStorageHost(), 80); + req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); + auto req_status = messaging->sendSyncMessage( method, - getSharedStorageHost(), - 80, - conn_flags, uri, - tenant_header, - nullptr, - MessageTypeTag::WAAP_LEARNING); + obj, + MessageCategory::GENERIC, + req_md + ); + return req_status.ok(); } - return messaging->sendObject( - obj, + auto req_status = messaging->sendSyncMessage( method, uri, - "", - nullptr, - true, - MessageTypeTag::WAAP_LEARNING); + obj, + MessageCategory::GENERIC + ); + return req_status.ok(); } template - bool sendObjectWithRetry(T &obj, I_Messaging::Method method, std::string uri) + bool sendObjectWithRetry(T &obj, HTTPMethod method, std::string uri) { I_MainLoop *mainloop = Singleton::Consume::by(); for (uint i = 0; i < max_send_obj_retries; i++) @@ -198,7 +193,7 @@ protected: } template - bool sendNoReplyObject(T &obj, I_Messaging::Method method, std::string uri) + bool sendNoReplyObject(T &obj, HTTPMethod method, std::string uri) { I_Messaging *messaging = Singleton::Consume::by(); I_AgentDetails *agentDetails = Singleton::Consume::by(); @@ -207,32 +202,26 @@ protected: return false; } if (agentDetails->getOrchestrationMode() == OrchestrationMode::HYBRID) { - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::EXTERNAL); - std::string tenant_header = "X-Tenant-Id: " + agentDetails->getTenantId(); - return messaging->sendNoReplyObject( - obj, + MessageMetadata req_md(getSharedStorageHost(), 80); + req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); + return messaging->sendSyncMessageWithoutResponse( method, - getSharedStorageHost(), - 80, - conn_flags, uri, - tenant_header, - nullptr, - MessageTypeTag::WAAP_LEARNING); + obj, + MessageCategory::GENERIC, + req_md + ); } - return messaging->sendNoReplyObject( - obj, + return messaging->sendSyncMessageWithoutResponse( method, uri, - "", - nullptr, - true, - MessageTypeTag::WAAP_LEARNING); + obj, + MessageCategory::GENERIC + ); } template - bool sendNoReplyObjectWithRetry(T &obj, I_Messaging::Method method, std::string uri) + bool sendNoReplyObjectWithRetry(T &obj, HTTPMethod method, std::string uri) { I_MainLoop *mainloop= Singleton::Consume::by(); for (uint i = 0; i < max_send_obj_retries; i++) @@ -273,6 +262,7 @@ private: size_t m_intervalsCounter; bool m_remoteSyncEnabled; const std::string m_assetId; + const bool m_isAssetIdUuid; std::string m_type; std::string m_lastProcessedModified; Maybe m_shared_storage_host; diff --git a/components/security_apps/waap/reputation/reputation_features_agg.cc b/components/security_apps/waap/reputation/reputation_features_agg.cc index fde179d..7d44cca 100755 --- a/components/security_apps/waap/reputation/reputation_features_agg.cc +++ b/components/security_apps/waap/reputation/reputation_features_agg.cc @@ -375,12 +375,12 @@ ReputationFeaturesAgg::Impl::reportReputationFeatures() string uri = "/storage/waap/" + tenantId + "/reputation/" + to_string(chrono::duration_cast(currentTime).count()) + "/" + agentId + "/data.data"; - msg->sendObjectWithPersistence(report, - I_Messaging::Method::PUT, + + msg->sendAsyncMessage( + HTTPMethod::PUT, uri, - "", - true, - MessageTypeTag::WAAP_LEARNING); + report + ); } } diff --git a/components/security_apps/waap/reputation/reputation_features_agg.h b/components/security_apps/waap/reputation/reputation_features_agg.h index c774b6d..f402a56 100755 --- a/components/security_apps/waap/reputation/reputation_features_agg.h +++ b/components/security_apps/waap/reputation/reputation_features_agg.h @@ -22,6 +22,7 @@ #include "i_table.h" #include "i_agent_details.h" #include "i_instance_awareness.h" +#include "i_messaging.h" class ReputationFeaturesEntry : public TableOpaqueSerialize { diff --git a/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc b/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc index fe4648a..78fa7a2 100755 --- a/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc +++ b/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc @@ -16,7 +16,6 @@ #include "waap.h" #include "ConfidenceFile.h" #include "i_agent_details.h" -#include "i_messaging.h" #include "i_mainloop.h" #include @@ -138,7 +137,7 @@ bool ConfidenceCalculator::postData() WindowLogPost currentWindow(m_time_window_logger_backup); bool ok = sendNoReplyObjectWithRetry(currentWindow, - I_Messaging::Method::PUT, + HTTPMethod::PUT, url); if (!ok) { dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to post collected data to: " << url; @@ -164,7 +163,7 @@ void ConfidenceCalculator::pullData(const std::vector& files) dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Pulling the file: " << file; WindowLogGet getWindow; bool ok = sendObjectWithRetry(getWindow, - I_Messaging::Method::GET, + HTTPMethod::GET, getUri() + "/" + file); if (!ok) { @@ -213,7 +212,7 @@ void ConfidenceCalculator::pullProcessedData(const std::vector& fil { ConfidenceFileDecryptor getConfFile; bool res = sendObjectWithRetry(getConfFile, - I_Messaging::Method::GET, + HTTPMethod::GET, getUri() + "/" + file); is_ok |= res; if (res && getConfFile.getConfidenceSet().ok()) @@ -243,7 +242,7 @@ void ConfidenceCalculator::postProcessedData() dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Posting the confidence set object to: " << postUrl; ConfidenceFileEncryptor postConfFile(m_confident_sets, m_confidence_level); sendNoReplyObjectWithRetry(postConfFile, - I_Messaging::Method::PUT, + HTTPMethod::PUT, postUrl); } diff --git a/components/security_apps/waap/waap_clib/DeepParser.cc b/components/security_apps/waap/waap_clib/DeepParser.cc index d331cfa..d224a0e 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.cc +++ b/components/security_apps/waap/waap_clib/DeepParser.cc @@ -187,6 +187,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f bool isCookiePayload = (m_key.first().size() == 6 && m_key.first() == "cookie"); bool isBodyPayload = (m_key.first().size() == 4 && m_key.first() == "body"); + // If csrf/antibot cookie - send to Waf2Transaction for collection of cookie value. if (m_depth == 1 && isCookiePayload && (m_key.str() == "x-chkp-csrf-token" || m_key.str() == "__fn1522082288")) { std::string cur_val = std::string(v, v_len); @@ -388,21 +389,17 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f && m_parsersDeque.size() > parser_depth &&!m_parsersDeque.at(parser_depth)->getRecursionFlag() ) { - ScopedContext ctx; - ctx.registerValue("waap_transaction", m_pTransaction); rc = pushValueToTopParser(cur_val, flags, base64ParamFound, offset, parser_depth); if (rc != CONTINUE_PARSING) { if (shouldUpdateKeyStack) { m_key.pop("deep parser key"); } - m_depth--; return rc; } } - if (rc == CONTINUE_PARSING) { // Try to eliminate m_multipart_boundary to allow other parser to work instead of multipart if (m_depth == 1 @@ -853,21 +850,17 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned( && m_parsersDeque.size() > parser_depth &&!m_parsersDeque.at(parser_depth)->getRecursionFlag() ) { - ScopedContext ctx; - ctx.registerValue("waap_transaction", m_pTransaction); rc = pushValueToTopParser(cur_val, flags, base64ParamFound, offset, parser_depth); if (rc != CONTINUE_PARSING) { if (shouldUpdateKeyStack) { m_key.pop("deep parser key"); } - m_depth--; return rc; } } - return rc; } @@ -1091,7 +1084,12 @@ DeepParser::createInternalParser( ) { // Graphql value detected dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse graphql"; - m_parsersDeque.push_back(std::make_shared>(*this, parser_depth + 1)); + + m_parsersDeque.push_back(std::make_shared>( + *this, + parser_depth + 1, + m_pTransaction)); + offset = 0; } else if (cur_val.length() > 0 && (cur_val[0] == '[' || cur_val[0] == '{') @@ -1123,12 +1121,12 @@ DeepParser::createInternalParser( // but only if the JSON is passed in body and on the top level. bool should_collect_for_oa_schema_updater = false; - m_parsersDeque.push_back( - std::make_shared>( - *this, - should_collect_for_oa_schema_updater, - parser_depth + 1 -)); + m_parsersDeque.push_back(std::make_shared>( + *this, + parser_depth + 1, + m_pTransaction, + should_collect_for_oa_schema_updater + )); offset = 0; } } @@ -1326,7 +1324,6 @@ DeepParser::createInternalParser( return offset; } - void DeepParser::apiProcessKey(const char *v, size_t v_len) { diff --git a/components/security_apps/waap/waap_clib/IndicatorsFilterBase.h b/components/security_apps/waap/waap_clib/IndicatorsFilterBase.h index fbce3e9..183b3e0 100755 --- a/components/security_apps/waap/waap_clib/IndicatorsFilterBase.h +++ b/components/security_apps/waap/waap_clib/IndicatorsFilterBase.h @@ -14,7 +14,6 @@ #pragma once #include "i_indicatorsFilter.h" -#include "i_messaging.h" #include "waap.h" #include "TrustedSources.h" #include "TrustedSourcesConfidence.h" diff --git a/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.h b/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.h index e83733f..852a175 100755 --- a/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.h +++ b/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.h @@ -19,7 +19,6 @@ #include "TypeIndicatorsFilter.h" #include "WaapParameters.h" #include "i_waapConfig.h" -#include "i_messaging.h" #include "ScannersDetector.h" #include "TuningDecisions.h" #include diff --git a/components/security_apps/waap/waap_clib/ParserGql.cc b/components/security_apps/waap/waap_clib/ParserGql.cc index 0b75d18..d95de1c 100644 --- a/components/security_apps/waap/waap_clib/ParserGql.cc +++ b/components/security_apps/waap/waap_clib/ParserGql.cc @@ -16,15 +16,23 @@ #include "graphqlparser/AstVisitor.h" #include "graphqlparser/GraphQLParser.h" #include "debug.h" +#include "oas_updater_entry_saver.h" USE_DEBUG_FLAG(D_WAAP_PARSER_GQL); +USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); const std::string ParserGql::m_parserName = "gqlParser"; -ParserGql::ParserGql(IParserReceiver &receiver, size_t parser_depth) : +ParserGql::ParserGql( + IParserReceiver &receiver, + size_t parser_depth, + IWaf2Transaction *pTransaction) + : m_receiver(receiver), m_error(false), m_curNameValues(0), + m_pTransaction(pTransaction), + field_depth(0), m_parser_depth(parser_depth) { dbgFlow(D_WAAP_PARSER_GQL); @@ -93,6 +101,7 @@ bool ParserGql::visitValue(const char *value) bool ParserGql::visitName(const facebook::graphql::ast::Name &node) { dbgTrace(D_WAAP_PARSER_GQL) << node.getValue() << "'"; + bool ret = true; if (m_curNameValues == 0 && !m_curNodeName.empty()) { ret = m_receiver.onKv( @@ -105,6 +114,13 @@ bool ParserGql::visitName(const facebook::graphql::ast::Name &node) return ret; } +bool ParserGql::visitOperationDefinition(const facebook::graphql::ast::OperationDefinition &node) +{ + dbgFlow(D_OA_SCHEMA_UPDATER) << "getOperation()"; + return true; +} + + bool ParserGql::visitIntValue(const facebook::graphql::ast::IntValue &node) { dbgFlow(D_WAAP_PARSER_GQL); diff --git a/components/security_apps/waap/waap_clib/ParserGql.h b/components/security_apps/waap/waap_clib/ParserGql.h index 1616bf5..2d5b235 100644 --- a/components/security_apps/waap/waap_clib/ParserGql.h +++ b/components/security_apps/waap/waap_clib/ParserGql.h @@ -22,10 +22,19 @@ #include "graphqlparser/AstNode.h" #include "graphqlparser/AstVisitor.h" #include "KeyStack.h" +#include "i_transaction.h" +#include "singleton.h" +#include "i_oa_schema_updater.h" -class ParserGql : public ParserBase, public facebook::graphql::ast::visitor::AstVisitor { +class ParserGql : + public ParserBase, + public facebook::graphql::ast::visitor::AstVisitor, + Singleton::Consume { public: - ParserGql(IParserReceiver &receiver, size_t parser_depth); + ParserGql( + IParserReceiver &receiver, + size_t parser_depth, + IWaf2Transaction *pTransaction=nullptr); virtual ~ParserGql(); size_t push(const char *data, size_t data_len); void finish(); @@ -38,10 +47,12 @@ private: std::string m_buffer; std::string m_curNodeName; int m_curNameValues; - + IWaf2Transaction *m_pTransaction; + int field_depth; bool visitValue(const char *value); // Callbacks from the parser + bool visitOperationDefinition(const facebook::graphql::ast::OperationDefinition &node) override; bool visitName(const facebook::graphql::ast::Name &node) override; bool visitIntValue(const facebook::graphql::ast::IntValue &node) override; bool visitFloatValue(const facebook::graphql::ast::FloatValue &node) override; diff --git a/components/security_apps/waap/waap_clib/ParserJson.cc b/components/security_apps/waap/waap_clib/ParserJson.cc index 3959c7a..8fb930b 100755 --- a/components/security_apps/waap/waap_clib/ParserJson.cc +++ b/components/security_apps/waap/waap_clib/ParserJson.cc @@ -249,8 +249,9 @@ ParserJson::p_end_array(void *ctx) ParserJson::ParserJson( IParserReceiver &receiver, - bool should_collect_oas, size_t parser_depth, + IWaf2Transaction *pTransaction, + bool should_collect_oas, IParserReceiver2 *receiver2) : m_receiver(receiver), @@ -260,10 +261,12 @@ ParserJson::ParserJson( m_key("json_parser"), m_jsonHandler(NULL), is_map_empty(false), - should_collect_for_oa_schema_updater(should_collect_oas), - m_parser_depth(parser_depth) + m_pTransaction(pTransaction), + m_parser_depth(parser_depth), + is_graphql_operation_name(false) { dbgTrace(D_WAAP_PARSER_JSON) << "parser_depth= " << parser_depth; + should_collect_for_oa_schema_updater = should_collect_oas; // TODO:: do we really want to clear this? memset(m_buf, 0, sizeof(m_buf)); diff --git a/components/security_apps/waap/waap_clib/ParserJson.h b/components/security_apps/waap/waap_clib/ParserJson.h index f113831..0ae3010 100755 --- a/components/security_apps/waap/waap_clib/ParserJson.h +++ b/components/security_apps/waap/waap_clib/ParserJson.h @@ -20,17 +20,22 @@ #include "ParserBase.h" #include "KeyStack.h" #include "yajl/yajl_parse.h" +#include "singleton.h" +#include "i_oa_schema_updater.h" #define FIRST_JSON_BUFFER_SIZE 4 // must buffer at least 4 first bytes to allow unicode autodetection (BOM). typedef size_t yajl_size_t; -class ParserJson : public ParserBase { +class ParserJson : + public ParserBase, + Singleton::Consume { public: ParserJson( IParserReceiver &receiver, - bool should_collect_for_oa_schema_updater=false, size_t parser_depth=0, + IWaf2Transaction *pTransaction = nullptr, + bool should_collect_for_oa_schema_updater=false, IParserReceiver2 *receiver2=NULL); virtual ~ParserJson(); size_t push(const char *data, size_t data_len); @@ -86,8 +91,9 @@ private: yajl_handle m_jsonHandler; bool is_map_empty; bool should_collect_for_oa_schema_updater; - + IWaf2Transaction *m_pTransaction; size_t m_parser_depth; + bool is_graphql_operation_name; public: static const std::string m_parserName; }; diff --git a/components/security_apps/waap/waap_clib/ScannerDetector.cc b/components/security_apps/waap/waap_clib/ScannerDetector.cc index fd2c3b2..19d0bf2 100755 --- a/components/security_apps/waap/waap_clib/ScannerDetector.cc +++ b/components/security_apps/waap/waap_clib/ScannerDetector.cc @@ -13,7 +13,6 @@ #include "ScannersDetector.h" #include "waap.h" -#include "i_messaging.h" #include USE_DEBUG_FLAG(D_WAAP); @@ -101,7 +100,7 @@ bool ScannerDetector::postData() SourcesMonitorPost currentWindow(m_sources_monitor_backup); bool ok = sendNoReplyObjectWithRetry(currentWindow, - I_Messaging::Method::PUT, + HTTPMethod::PUT, url); if (!ok) { dbgError(D_WAAP) << "Failed to post collected data to: " << url; @@ -123,7 +122,7 @@ void ScannerDetector::pullData(const std::vector& files) dbgTrace(D_WAAP) << "Pulling the file: " << file; SourcesMonitorGet getMonitor; bool ok = sendObjectWithRetry(getMonitor, - I_Messaging::Method::GET, + HTTPMethod::GET, getUri() + "/" + file); if (!ok) { diff --git a/components/security_apps/waap/waap_clib/Serializator.cc b/components/security_apps/waap/waap_clib/Serializator.cc index cd49cb6..8452d0e 100755 --- a/components/security_apps/waap/waap_clib/Serializator.cc +++ b/components/security_apps/waap/waap_clib/Serializator.cc @@ -16,6 +16,7 @@ #include "Waf2Util.h" #include "WaapAssetState.h" #include "i_instance_awareness.h" +#include #include #include #include @@ -387,15 +388,14 @@ const vector& RemoteFilesList::getFilesMetadataList() const return files.get(); } - SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase( ch::minutes interval, ch::seconds waitForSync, - const string& filePath, - const string& remotePath, - const string& assetId, - const string& owner) - : + const string &filePath, + const string &remotePath, + const string &assetId, + const string &owner +) : SerializeToFileBase(filePath), m_remotePath(remotePath), m_interval(0), @@ -408,6 +408,7 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase( m_intervalsCounter(0), m_remoteSyncEnabled(true), m_assetId(assetId), + m_isAssetIdUuid(Waap::Util::isUuid(assetId)), m_shared_storage_host(genError("not set")), m_learning_host(genError("not set")) { @@ -600,7 +601,7 @@ bool SerializeToLocalAndRemoteSyncBase::localSyncAndProcess() dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Getting files of all agents"; bool isSuccessful = sendObjectWithRetry(rawDataFiles, - I_Messaging::Method::GET, + HTTPMethod::GET, getUri() + "/?list-type=2&prefix=" + m_remotePath + "/" + getWindowId() + "/"); if (!isSuccessful) @@ -655,14 +656,16 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() OrchestrationMode mode = Singleton::exists() ? Singleton::Consume::by()->getOrchestrationMode() : OrchestrationMode::ONLINE; - if (!m_remoteSyncEnabled || isBase() || !postData() || - mode == OrchestrationMode::OFFLINE) - { + if (mode == OrchestrationMode::OFFLINE || !m_remoteSyncEnabled || isBase() || + (mode == OrchestrationMode::ONLINE && !m_isAssetIdUuid) || !postData()) { dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) - << "Did not synchronize the data. Remote URL: " + << "Did not synchronize the data. for asset: " + << m_assetId + << " Remote URL: " << m_remotePath << " is enabled: " - << to_string(m_remoteSyncEnabled); + << to_string(m_remoteSyncEnabled) + << ", mode: " << int(mode); processData(); saveData(); return; @@ -701,16 +704,15 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() SyncLearningObject syncObj(m_assetId, m_type, getWindowId()); - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::EXTERNAL); - string tenant_header = "X-Tenant-Id: " + agentDetails->getTenantId(); - bool ok = messaging->sendNoReplyObject(syncObj, - I_Messaging::Method::POST, - getLearningHost(), - 80, - conn_flags, - "/api/sync", - tenant_header); + MessageMetadata req_md(getLearningHost(), 80); + req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); + bool ok = messaging->sendSyncMessageWithoutResponse( + HTTPMethod::POST, + "/api/sync", + syncObj, + MessageCategory::GENERIC, + req_md + ); dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "sent learning sync notification ok: " << ok; if (!ok) { dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "failed to send learning notification"; @@ -725,7 +727,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() ReportIS::AudienceTeam::WAAP, syncNotification, false, - MessageTypeTag::WAAP_LEARNING, + MessageCategory::GENERIC, ReportIS::Tags::WAF, ReportIS::Notification::SYNC_LEARNING ); @@ -762,7 +764,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getRemoteProcessedFilesList() bool isSuccessful = sendObject( remoteFiles, - I_Messaging::Method::GET, + HTTPMethod::GET, getUri() + "/?list-type=2&prefix=" + m_remotePath + "/remote"); if (!isSuccessful) @@ -795,7 +797,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList() bool isSuccessful = sendObject( processedFilesList, - I_Messaging::Method::GET, + HTTPMethod::GET, getUri() + "/?list-type=2&prefix=" + m_remotePath + "/processed"); if (!isSuccessful) @@ -832,7 +834,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList() isSuccessful = sendObject( processedFilesList, - I_Messaging::Method::GET, + HTTPMethod::GET, getUri() + "/?list-type=2&prefix=" + bcRemotePath + "/processed"); if (!isSuccessful) diff --git a/components/security_apps/waap/waap_clib/Telemetry.cc b/components/security_apps/waap/waap_clib/Telemetry.cc index a5e9def..45e37ae 100755 --- a/components/security_apps/waap/waap_clib/Telemetry.cc +++ b/components/security_apps/waap/waap_clib/Telemetry.cc @@ -23,8 +23,6 @@ USE_DEBUG_FLAG(D_WAAP); -#define LOGGING_INTERVAL_IN_MINUTES 10 - using namespace std; const static string default_host = "open-appsec-tuning-svc"; @@ -34,26 +32,25 @@ WaapTelemetryBase::sendLog(const LogRest &metric_client_rest) const { OrchestrationMode mode = Singleton::Consume::by()->getOrchestrationMode(); + GenericMetric::sendLog(metric_client_rest); + if (mode == OrchestrationMode::ONLINE) { - GenericMetric::sendLog(metric_client_rest); return; } auto svc_host = getConfigurationWithDefault(default_host, "Logging", "K8sSvc Log host"); - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::EXTERNAL); string fog_metric_uri = getConfigurationWithDefault("/api/v1/agents/events", "metric", "fogMetricUri"); - std::string tenant_header = "X-Tenant-Id: " + - Singleton::Consume::by()->getTenantId(); - Singleton::Consume::by()->sendNoReplyObject( - metric_client_rest, - I_Messaging::Method::POST, - svc_host, - 80, - conn_flags, + MessageMetadata req_md(svc_host, 80); + req_md.insertHeader( + "X-Tenant-Id", + Singleton::Consume::by()->getTenantId() + ); + Singleton::Consume::by()->sendSyncMessageWithoutResponse( + HTTPMethod::POST, fog_metric_uri, - tenant_header, - nullptr, - MessageTypeTag::METRIC); + metric_client_rest, + MessageCategory::METRIC, + req_md + ); } void @@ -140,6 +137,56 @@ WaapTelemetrics::updateMetrics(const string &asset_id, const DecisionTelemetryDa } } +void +WaapTrafficTelemetrics::initMetrics() +{ + post_requests.report(0); + get_requests.report(0); + put_requests.report(0); + patch_requests.report(0); + delete_requests.report(0); + other_requests.report(0); + + response_2xx.report(0); + response_4xx.report(0); + response_5xx.report(0); +} + +void +WaapTrafficTelemetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data) +{ + initMetrics(); + switch (data.method) + { + case POST: + post_requests.report(1); + break; + case GET: + get_requests.report(1); + break; + case PUT: + put_requests.report(1); + break; + case PATCH: + patch_requests.report(1); + break; + case DELETE: + delete_requests.report(1); + break; + default: + other_requests.report(1); + break; + } + + if (data.responseCode >= 500) { + response_5xx.report(1);; + } else if (data.responseCode >= 400) { + response_4xx.report(1); + } else if (data.responseCode >= 200 && data.responseCode < 300) { + response_2xx.report(1); + } +} + void WaapAttackTypesMetrics::initMetrics() { @@ -195,85 +242,17 @@ WaapMetricWrapper::upon(const WaapTelemetryEvent &event) << ", Threat level: " << data.threat; - if (!telemetries.count(asset_id)) { - telemetries.emplace(asset_id, make_shared()); - telemetries[asset_id]->init( - "WAAP telemetry", - ReportIS::AudienceTeam::WAAP, - ReportIS::IssuingEngine::AGENT_CORE, - chrono::minutes(LOGGING_INTERVAL_IN_MINUTES), - true, - ReportIS::Audience::SECURITY - ); - - telemetries[asset_id]->registerContext( - "pracitceType", - string("Threat Prevention"), - EnvKeyAttr::LogSection::SOURCE - ); - telemetries[asset_id]->registerContext( - "practiceSubType", - string("Web Application"), - EnvKeyAttr::LogSection::SOURCE - ); - telemetries[asset_id]->registerContext("assetId", asset_id, EnvKeyAttr::LogSection::SOURCE); - telemetries[asset_id]->registerContext("assetName", data.assetName, EnvKeyAttr::LogSection::SOURCE); - telemetries[asset_id]->registerContext("practiceId", data.practiceId, EnvKeyAttr::LogSection::SOURCE); - telemetries[asset_id]->registerContext( - "practiceName", - data.practiceName, - EnvKeyAttr::LogSection::SOURCE - ); - - telemetries[asset_id]->registerListener(); - } - if (!attack_types_telemetries.count(asset_id)) { - attack_types_telemetries.emplace(asset_id, make_shared()); - attack_types_telemetries[asset_id]->init( - "WAAP attack type telemetry", - ReportIS::AudienceTeam::WAAP, - ReportIS::IssuingEngine::AGENT_CORE, - chrono::minutes(LOGGING_INTERVAL_IN_MINUTES), - true, - ReportIS::Audience::SECURITY - ); - - attack_types_telemetries[asset_id]->registerContext( - "pracitceType", - string("Threat Prevention"), - EnvKeyAttr::LogSection::SOURCE - ); - attack_types_telemetries[asset_id]->registerContext( - "practiceSubType", - string("Web Application"), - EnvKeyAttr::LogSection::SOURCE - ); - attack_types_telemetries[asset_id]->registerContext( - "assetId", - asset_id, - EnvKeyAttr::LogSection::SOURCE - ); - attack_types_telemetries[asset_id]->registerContext( - "assetName", - data.assetName, - EnvKeyAttr::LogSection::SOURCE - ); - attack_types_telemetries[asset_id]->registerContext( - "practiceId", - data.practiceId, - EnvKeyAttr::LogSection::SOURCE - ); - attack_types_telemetries[asset_id]->registerContext( - "practiceName", - data.practiceName, - EnvKeyAttr::LogSection::SOURCE - ); - - attack_types_telemetries[asset_id]->registerListener(); - } + initializeTelemetryData(asset_id, data, "WAAP telemetry", telemetries); + initializeTelemetryData( + asset_id, data, + "WAAP attack type telemetry", + attack_types_telemetries + ); + initializeTelemetryData(asset_id, data, "WAAP traffic telemetry", traffic_telemetries); telemetries[asset_id]->updateMetrics(asset_id, data); attack_types_telemetries[asset_id]->updateMetrics(asset_id, data); + traffic_telemetries[asset_id]->updateMetrics(asset_id, data); auto agent_mode = Singleton::Consume::by()->getOrchestrationMode(); string tenant_id = Singleton::Consume::by()->getTenantId(); diff --git a/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc b/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc index 2a64ac4..3947532 100755 --- a/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc +++ b/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc @@ -12,7 +12,6 @@ // limitations under the License. #include "TrustedSourcesConfidence.h" -#include "i_messaging.h" #include "waap.h" #include "Waf2Util.h" @@ -97,7 +96,7 @@ bool TrustedSourcesConfidenceCalculator::postData() TrsutedSourcesLogger logger(m_logger); bool ok = sendNoReplyObjectWithRetry(logger, - I_Messaging::Method::PUT, + HTTPMethod::PUT, url); if (!ok) { dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to post collected data to: " << url; @@ -118,7 +117,7 @@ void TrustedSourcesConfidenceCalculator::pullData(const std::vector } GetTrustedFile getTrustFile; bool res = sendObjectWithRetry(getTrustFile, - I_Messaging::Method::GET, + HTTPMethod::GET, getUri() + "/" + file); if (!res) { @@ -149,7 +148,7 @@ void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector - bool sendObject(T &obj, I_Messaging::Method method, std::string uri) + bool sendObject(T &obj, HTTPMethod method, std::string uri) { I_Messaging *messaging = Singleton::Consume::by(); I_AgentDetails *agentDetails = Singleton::Consume::by(); if (agentDetails->getOrchestrationMode() != OrchestrationMode::ONLINE) { - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::EXTERNAL); - std::string tenant_header = "X-Tenant-Id: " + agentDetails->getTenantId(); - - return messaging->sendObject( - obj, + MessageMetadata req_md(getSharedStorageHost(), 80); + req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); + auto req_status = messaging->sendSyncMessage( method, - getSharedStorageHost(), - 80, - conn_flags, uri, - tenant_header, - nullptr, - MessageTypeTag::WAAP_LEARNING); + obj, + MessageCategory::GENERIC, + req_md + ); + return req_status.ok(); } - return messaging->sendObject( - obj, + auto req_status = messaging->sendSyncMessage( method, uri, - "", - nullptr, - true, - MessageTypeTag::WAAP_LEARNING); + obj, + MessageCategory::GENERIC + ); + return req_status.ok(); } std::string m_remotePath; diff --git a/components/security_apps/waap/waap_clib/WaapConfigBase.cc b/components/security_apps/waap/waap_clib/WaapConfigBase.cc index 0e60a0b..04122cf 100755 --- a/components/security_apps/waap/waap_clib/WaapConfigBase.cc +++ b/components/security_apps/waap/waap_clib/WaapConfigBase.cc @@ -255,6 +255,8 @@ void WaapConfigBase::loadOpenRedirectPolicy(cereal::JSONInputArchive& ar) } + + const std::vector & WaapConfigBase::get_applicationUrls() const { diff --git a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc index f13ddc4..092203b 100755 --- a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc +++ b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc @@ -49,7 +49,7 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& boost::cmatch what; try { if (tag == "url") { - return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUriStr().c_str(), what, rx); + return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUri().c_str(), what, rx); } else if (tag == "hostname") { return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getHost().c_str(), what, rx); diff --git a/components/security_apps/waap/waap_clib/WaapScanner.cc b/components/security_apps/waap/waap_clib/WaapScanner.cc index 4875b3b..17ec1f6 100755 --- a/components/security_apps/waap/waap_clib/WaapScanner.cc +++ b/components/security_apps/waap/waap_clib/WaapScanner.cc @@ -181,7 +181,6 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len std::string key = std::string(k, k_len); std::string value = std::string(v, v_len); - res.clear(); dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: k='" << key << "' v='" << value << "'"; diff --git a/components/security_apps/waap/waap_clib/WaapScanner.h b/components/security_apps/waap/waap_clib/WaapScanner.h index 7d75b10..b76da81 100644 --- a/components/security_apps/waap/waap_clib/WaapScanner.h +++ b/components/security_apps/waap/waap_clib/WaapScanner.h @@ -33,7 +33,6 @@ namespace Waap { { } - bool suspiciousHit(Waf2ScanResult &res, DeepParser &dp, const std::string &location, const std::string ¶m_name, const std::string &key); int onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags, size_t parser_depth) override; diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.cc b/components/security_apps/waap/waap_clib/Waf2Engine.cc index 188a264..903c064 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.cc +++ b/components/security_apps/waap/waap_clib/Waf2Engine.cc @@ -274,7 +274,6 @@ void Waf2Transaction::setCurrentAssetState(IWaapConfig* sitePolicy) "couldn't set waapAssetState for asset... using original waapAssetState"; return; } - m_pWaapAssetState = pCurrentWaapAssetState; } @@ -314,18 +313,7 @@ Waf2Transaction::Waf2Transaction() : m_index(-1), m_triggerLog(), m_waf2TransactionFlags() -{ - is_hybrid_mode = - Singleton::exists() ? - Singleton::Consume::by()->getOrchestrationMode() == OrchestrationMode::HYBRID - : false; - if (is_hybrid_mode) { - max_grace_logs = getProfileAgentSettingWithDefault( - 10, - "rulebase.initialForcedSecurityLogsToLocalStorage.count" - ); - } -} +{} Waf2Transaction::Waf2Transaction(std::shared_ptr pWaapAssetState) : TableOpaqueSerialize(this), @@ -356,18 +344,7 @@ Waf2Transaction::Waf2Transaction(std::shared_ptr pWaapAssetState m_index(-1), m_triggerLog(), m_waf2TransactionFlags() -{ - is_hybrid_mode = - Singleton::exists() ? - Singleton::Consume::by()->getOrchestrationMode() == OrchestrationMode::HYBRID - : false; - if (is_hybrid_mode) { - max_grace_logs = getProfileAgentSettingWithDefault( - 10, - "rulebase.initialForcedSecurityLogsToLocalStorage.count" - ); - } -} +{} Waf2Transaction::~Waf2Transaction() { dbgTrace(D_WAAP) << "Waf2Transaction::~Waf2Transaction: deleting m_requestBodyParser"; @@ -470,7 +447,6 @@ HeaderType Waf2Transaction::checkCleanHeader(const char* name, int name_len, con } } } - } return UNKNOWN_HEADER; } @@ -539,6 +515,7 @@ bool Waf2Transaction::checkIsScanningRequired() m_siteConfig = &m_ngenAPIConfig; auto rateLimitingPolicy = m_siteConfig ? m_siteConfig->get_RateLimitingPolicy() : NULL; result |= m_siteConfig->get_WebAttackMitigation(); + if(rateLimitingPolicy) { result |= m_siteConfig->get_RateLimitingPolicy()->getRateLimitingEnforcementStatus(); } @@ -547,15 +524,14 @@ bool Waf2Transaction::checkIsScanningRequired() if (userLimitsPolicy) { result = true; } - } - - if (WaapConfigApplication::getWaapSiteConfig(m_ngenSiteConfig)) { + } else if (WaapConfigApplication::getWaapSiteConfig(m_ngenSiteConfig)) { m_siteConfig = &m_ngenSiteConfig; auto rateLimitingPolicy = m_siteConfig ? m_siteConfig->get_RateLimitingPolicy() : NULL; auto errorLimitingPolicy = m_siteConfig ? m_siteConfig->get_ErrorLimitingPolicy() : NULL; auto csrfPolicy = m_siteConfig ? m_siteConfig->get_CsrfPolicy() : NULL; auto userLimitsPolicy = m_siteConfig ? m_siteConfig->get_UserLimitsPolicy() : nullptr; result |= m_siteConfig->get_WebAttackMitigation(); + if (rateLimitingPolicy) { result |= m_siteConfig->get_RateLimitingPolicy()->getRateLimitingEnforcementStatus(); } @@ -1592,7 +1568,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog, } // Count of bytes available to send to the log - std::string requestBodyToLog = (send_extended_log || triggerLog->webBody) ? + std::string requestBodyToLog = (triggerLog->webBody) ? m_request_body : std::string(); std::string responseBodyToLog = m_response_body; if (!shouldBlock && responseBodyToLog.empty()) @@ -1671,6 +1647,25 @@ Waf2Transaction::sendLog() const auto& autonomousSecurityDecision = std::dynamic_pointer_cast( m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION)); + if (m_methodStr == "POST") { + telemetryData.method = POST; + } else if (m_methodStr == "GET") { + telemetryData.method = GET; + } else if (m_methodStr == "PUT") { + telemetryData.method = PUT; + } else if (m_methodStr == "PATCH") { + telemetryData.method = PATCH; + } else if (m_methodStr == "DELETE") { + telemetryData.method = DELETE; + } else { + telemetryData.method = OTHER; + } + + if (m_responseStatus != 0) { + telemetryData.responseCode = m_responseStatus; + } + + telemetryData.source = getSourceIdentifier(); telemetryData.assetName = m_siteConfig->get_AssetName(); telemetryData.practiceId = m_siteConfig->get_PracticeId(); @@ -1739,17 +1734,7 @@ Waf2Transaction::sendLog() return; } - static int cur_grace_logs = 0; - bool grace_period = is_hybrid_mode && cur_grace_logs < max_grace_logs; bool send_extended_log = shouldSendExtendedLog(triggerLog); - if (grace_period) { - dbgTrace(D_WAAP) - << "Waf2Transaction::sendLog: current grace log index: " - << cur_grace_logs + 1 - << " out of " - << max_grace_logs; - } - shouldBlock |= m_waapDecision.getShouldBlockFromHighestPriorityDecision(); // Do not send Detect log if trigger disallows it if (!send_extended_log && shouldBlock == false && !triggerLog->tpDetect && @@ -1787,13 +1772,6 @@ Waf2Transaction::sendLog() if (decision_type == DecisionType::NO_WAAP_DECISION) { if (send_extended_log || autonomousSecurityDecision->getOverridesLog()) { sendAutonomousSecurityLog(triggerLog, shouldBlock, logOverride, attackTypes); - if (grace_period) { - dbgTrace(D_WAAP) - << "Waf2Transaction::sendLog: Sending log in grace period. Log " - << ++cur_grace_logs - << "out of " - << max_grace_logs; - } } dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: decisions marked for block only"; return; @@ -1833,13 +1811,6 @@ Waf2Transaction::sendLog() appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType); waap_log << LogField("waapIncidentDetails", incidentDetails); waap_log << LogField("eventConfidence", "High"); - if (grace_period) { - dbgTrace(D_WAAP) - << "Waf2Transaction::sendLog: Sending log in grace period. Log " - << ++cur_grace_logs - << "out of " - << max_grace_logs; - } break; } case OPEN_REDIRECT_DECISION: @@ -1872,15 +1843,7 @@ Waf2Transaction::sendLog() appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType); - waap_log << LogField("waapIncidentDetails", incidentDetails); - if (grace_period) { - dbgTrace(D_WAAP) - << "Waf2Transaction::sendLog: Sending log in grace period. Log " - << ++cur_grace_logs - << "out of " - << max_grace_logs; - } break; } case CSRF_DECISION: { @@ -1896,13 +1859,6 @@ Waf2Transaction::sendLog() LogGen& waap_log = logGenWrapper.getLogGen(); appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, "Cross Site Request Forgery"); waap_log << LogField("waapIncidentDetails", "CSRF Attack discovered."); - if (grace_period) { - dbgTrace(D_WAAP) - << "Waf2Transaction::sendLog: Sending log in grace period. Log " - << ++cur_grace_logs - << "out of " - << max_grace_logs; - } break; } case AUTONOMOUS_SECURITY_DECISION: { @@ -1911,13 +1867,6 @@ Waf2Transaction::sendLog() autonomousSecurityDecision->getThreatLevel() != ThreatLevel::NO_THREAT || autonomousSecurityDecision->getOverridesLog()) { sendAutonomousSecurityLog(triggerLog, shouldBlock, logOverride, attackTypes); - if (grace_period) { - dbgTrace(D_WAAP) - << "Waf2Transaction::sendLog: Sending log in grace period. Log " - << ++cur_grace_logs - << "out of " - << max_grace_logs; - } } break; } @@ -2204,7 +2153,7 @@ bool Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) { auto exceptions = getConfiguration("rulebase", "exception"); if (!exceptions.ok()) { - dbgInfo(D_WAAP_OVERRIDE) << "matching exceptions error:" << exceptions.getErr(); + dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error:" << exceptions.getErr(); return false; } dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions"; diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.h b/components/security_apps/waap/waap_clib/Waf2Engine.h index ea81548..b02f030 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.h +++ b/components/security_apps/waap/waap_clib/Waf2Engine.h @@ -346,10 +346,6 @@ private: // Cached pointer to const triggerLog (hence mutable) mutable std::shared_ptr m_triggerLog; Waf2TransactionFlags m_waf2TransactionFlags; - - // Grace period for logging - int max_grace_logs; - bool is_hybrid_mode = false; }; #endif // __WAF2_TRANSACTION_H__99e4201a diff --git a/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc b/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc index f485fa7..ad39478 100755 --- a/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc +++ b/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc @@ -624,3 +624,4 @@ ReportIS::Severity Waf2Transaction::computeEventSeverityFromDecision() const return ReportIS::Severity::INFO; } + diff --git a/components/security_apps/waap/waap_clib/Waf2Util.cc b/components/security_apps/waap/waap_clib/Waf2Util.cc index 4295772..931cb9c 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.cc +++ b/components/security_apps/waap/waap_clib/Waf2Util.cc @@ -42,7 +42,6 @@ USE_DEBUG_FLAG(D_WAAP); USE_DEBUG_FLAG(D_WAAP_EVASIONS); USE_DEBUG_FLAG(D_WAAP_BASE64); USE_DEBUG_FLAG(D_WAAP_JSON); -USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); #define MIN_HEX_LENGTH 6 #define charToDigit(c) (c - '0') @@ -2056,6 +2055,33 @@ string extractForwardedIp(const string &x_forwarded_hdr_val) return forward_ip; } + +bool isUuid(const string& str) { + if (str.length() != 36) { + return false; + } + + static bool err; + static const SingleRegex uuid_detector_re( + "[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}", + err, + "uuid_detector" + ); + // Check if the string matches the UUID format + return uuid_detector_re.hasMatch(str); +/* + boost::cmatch what; + try { + static const boost::regex uuidRegex("[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}"); + // Check if the string matches the UUID format + return boost::regex_match(str.c_str(), what, uuidRegex); + } catch (std::runtime_error &e) { + dbgError(D_WAAP) << e.what(); + } + return false; +*/ +} + bool isIpAddress(const string &ip_address) { struct in_addr source_inaddr; diff --git a/components/security_apps/waap/waap_clib/Waf2Util.h b/components/security_apps/waap/waap_clib/Waf2Util.h index aade3ac..9ef2d0a 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.h +++ b/components/security_apps/waap/waap_clib/Waf2Util.h @@ -1185,6 +1185,7 @@ namespace Util { std::string stripOptionalPort(const std::string::const_iterator &first, const std::string::const_iterator &last); std::string extractKeyValueFromCookie(const std::string &cookie, const std::string &key); bool isIpAddress(const std::string &ip_address); + bool isUuid(const std::string& str); bool vectorStringContain(const std::vector& vec, const std::string& str); bool isIpTrusted(const std::string &ip, const std::vector &trusted_ips); diff --git a/components/security_apps/waap/waap_clib/events_for_oa_schema.h b/components/security_apps/waap/waap_clib/events_for_oa_schema.h new file mode 100644 index 0000000..e69de29 diff --git a/components/security_apps/waap/waap_clib/oasu_key_types.h b/components/security_apps/waap/waap_clib/oasu_key_types.h new file mode 100644 index 0000000..dbf0e93 --- /dev/null +++ b/components/security_apps/waap/waap_clib/oasu_key_types.h @@ -0,0 +1,20 @@ +#ifndef __OA_SCHEMA_UPDATER_KEYS_H__ +#define __OA_SCHEMA_UPDATER_KEYS_H__ + +enum SchemaKeyType +{ + NullKeyName, + BooleanKeyName, + NumberKeyName, + StringKeyName, + ObjectMapKeyName, + ObjectInArrayKeyName, + EmptyObjectKeyName, + EndObjectKeyName, + StartObjectKeyName, + StartArrayKeyName, + EndArrayKeyName, + OtherKey +}; + +#endif // __OA_SCHEMA_UPDATER_KEYS_H__ diff --git a/components/security_apps/waap/waap_component_impl.cc b/components/security_apps/waap/waap_component_impl.cc index baf16ca..8cd1d6b 100755 --- a/components/security_apps/waap/waap_component_impl.cc +++ b/components/security_apps/waap/waap_component_impl.cc @@ -88,10 +88,8 @@ WaapComponent::Impl::init(const std::string &waapDataFileName) //waf2_set_log_target(WAF2_LOGTARGET_STDERR); dbgTrace(D_WAAP) << "WaapComponent::Impl::init() ..."; - reputationAggregator.init(); - waapStateTable = Singleton::Consume::by(); - + bool success = waf2_proc_start(waapDataFileName); if (!success) { dbgWarning(D_WAAP) << "WAF2 engine FAILED to initialize (probably failed to load signatures). Aborting!"; @@ -169,6 +167,7 @@ WaapComponent::Impl::respond(const NewHttpTransactionEvent &event) waf2TransactionFlags.endResponseHeadersCalled = false; waf2TransactionFlags.responseDataPushStarted = false; + waf2Transaction.start(); char sourceIpStr[INET_ADDRSTRLEN]; @@ -187,7 +186,6 @@ WaapComponent::Impl::respond(const NewHttpTransactionEvent &event) // Tell waf2 API that request headers started waf2Transaction.start_request_hdrs(); - return pending_response; } @@ -242,7 +240,7 @@ WaapComponent::Impl::respond(const HttpRequestHeaderEvent &event) } // Delete state before returning any verdict which is not pending - if ((verdict.getVerdict() != pending_response.getVerdict()) && waapStateTable->hasState()) { + if ((verdict.getVerdict() != pending_response.getVerdict()) && waapStateTable->hasState()) { finishTransaction(waf2Transaction); } else { } @@ -344,6 +342,7 @@ WaapComponent::Impl::respond(const ResponseCodeEvent &event) } IWaf2Transaction& waf2Transaction = waapStateTable->getState(); + // TODO:: extract HTTP version from attachment? static const int http_version = 0x11; diff --git a/components/security_apps/waap/waap_component_impl.h b/components/security_apps/waap/waap_component_impl.h index 4d21f21..91ab2c3 100755 --- a/components/security_apps/waap/waap_component_impl.h +++ b/components/security_apps/waap/waap_component_impl.h @@ -81,7 +81,6 @@ private: // instance of singleton classes DeepAnalyzer deepAnalyzer; WaapAssetStatesManager waapAssetStatesManager; - ReputationFeaturesAgg reputationAggregator; std::unordered_set m_seen_assets_id; }; diff --git a/components/signal_handler/signal_handler.cc b/components/signal_handler/signal_handler.cc index 1f09ccd..994ba98 100755 --- a/components/signal_handler/signal_handler.cc +++ b/components/signal_handler/signal_handler.cc @@ -215,13 +215,11 @@ private: LogRest signalHandler_client_rest(message_to_fog); - Singleton::Consume::by()->sendObjectWithPersistence( - signalHandler_client_rest, - I_Messaging::Method::POST, + Singleton::Consume::by()->sendAsyncMessage( + HTTPMethod::POST, fog_signalHandler_uri, - "", - true, - MessageTypeTag::REPORT + signalHandler_client_rest, + MessageCategory::LOG ); dbgInfo(D_SIGNAL_HANDLER) << "Sent crash log to fog" << endl; diff --git a/components/utils/CMakeLists.txt b/components/utils/CMakeLists.txt index 32b02f9..a71b851 100644 --- a/components/utils/CMakeLists.txt +++ b/components/utils/CMakeLists.txt @@ -1,3 +1,4 @@ +#add_subdirectory(geo_location) add_subdirectory(http_transaction_data) add_subdirectory(ip_utilities) add_subdirectory(keywords) diff --git a/components/utils/geo_location/CMakeLists.txt b/components/utils/geo_location/CMakeLists.txt new file mode 100644 index 0000000..ae280bb --- /dev/null +++ b/components/utils/geo_location/CMakeLists.txt @@ -0,0 +1 @@ +add_library(geo_location geo_location.cc) diff --git a/components/utils/geo_location/geo_location.cc b/components/utils/geo_location/geo_location.cc new file mode 100644 index 0000000..1fab700 --- /dev/null +++ b/components/utils/geo_location/geo_location.cc @@ -0,0 +1,194 @@ +#include "geo_location.h" + +#include + +#include "config.h" +#include "debug.h" +#include "enum_range.h" + +using namespace std; + +USE_DEBUG_FLAG(D_GEO_DB); + +class GeoLocation::Impl : Singleton::Provide::From +{ +public: + void + preload() + { + registerExpectedConfigFile("agentGeoDb", Config::ConfigFileType::RawData); + } + + void + init() + { + registerConfigLoadCb([this]() { initGeoDbObj(); }); + initGeoDbObj(); + } + + void + fini() + { + if (mind_db_obj_status != MMDB_SUCCESS) return; + MMDB_close(&max_mind_db_obj); + dbgTrace(D_GEO_DB) << "Closed geo location DB"; + } + + Maybe> + lookupLocation(const string &ip) override + { + dbgFlow(D_GEO_DB) << "Geo location lookup by string"; + + if (mind_db_obj_status != MMDB_SUCCESS) { + dbgDebug(D_GEO_DB) << "Maxmind db is uninitialized"; + return genError("Maxmind db is uninitialized"); + } + + Maybe maybe_ip_addr = IPAddr::createIPAddr(ip); + + if (!maybe_ip_addr.ok()) { + dbgWarning(D_GEO_DB) + << "Error in creating IPAddr from string: " + << ip + << ", error: " + << maybe_ip_addr.getErr(); + return genError( + "Error in creating IPAddr from string: " + + ip + + ", error: " + + maybe_ip_addr.getErr() + ); + } + + return lookupLocation(maybe_ip_addr.unpack()); + } + + Maybe> + lookupLocation(const IPAddr &ip) override + { + dbgFlow(D_GEO_DB) << "Geo location lookup by IPAddr"; + + if (mind_db_obj_status != MMDB_SUCCESS) { + dbgDebug(D_GEO_DB) << "Maxmind db is uninitialized"; + return genError("Maxmind db is uninitialized"); + } + + int maxminddb_error; + struct sockaddr sockaddr_to_search = convertIPAddrtoSockaddr(ip); + + MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&max_mind_db_obj, &sockaddr_to_search, &maxminddb_error); + if (maxminddb_error != MMDB_SUCCESS) { + dbgWarning(D_GEO_DB) << "maxMindDB error: " << MMDB_strerror(maxminddb_error); + return genError("maxMindDB error: " + string(MMDB_strerror(maxminddb_error))); + } + if (result.found_entry) { + return getGeoLocationDetails(result); + } + return genError("No results were found by lookup geo location"); + } + +private: + void + initGeoDbObj() + { + if (mind_db_obj_status == MMDB_SUCCESS) { + dbgTrace(D_GEO_DB) << "Closing an open geo location DB file"; + MMDB_close(&max_mind_db_obj); + mind_db_obj_status = -1; + } + + string geo_location_db_file = getPolicyConfigPath("agentGeoDb", Config::ConfigFileType::RawData); + dbgDebug(D_GEO_DB) << "Path to GeoDb file" << geo_location_db_file; + if (geo_location_db_file == "") { + dbgWarning(D_GEO_DB) << "No geo location db file specified"; + return; + } + + mind_db_obj_status = MMDB_open(geo_location_db_file.c_str(), MMDB_MODE_MMAP, &max_mind_db_obj); + if (mind_db_obj_status != MMDB_SUCCESS) { + dbgWarning(D_GEO_DB) << "maxMindDB error: " << MMDB_strerror(mind_db_obj_status); + if (mind_db_obj_status == MMDB_IO_ERROR) { + dbgWarning(D_GEO_DB) << "maxMindDB IO error: " << strerror(mind_db_obj_status); + } + return; + } + dbgDebug(D_GEO_DB) << "Successfully Opened geo location DB"; + } + + EnumArray + getGeoLocationDetails(MMDB_lookup_result_s &result) + { + EnumArray geo_location_details; + for (I_GeoLocation::GeoLocationField geo_field : makeRange()) { + geo_location_details[geo_field] = getGeoLocationValueResults(result, geo_field); + } + return geo_location_details; + } + + string + getGeoLocationValueResults(MMDB_lookup_result_s &result, I_GeoLocation::GeoLocationField field_type) + { + MMDB_entry_data_s entry_data; + int status = -1; + switch (field_type) { + case I_GeoLocation::GeoLocationField::COUNTRY_NAME: { + status = MMDB_get_value(&result.entry, &entry_data, "country", "names", "en", NULL); + break; + } + case I_GeoLocation::GeoLocationField::COUNTRY_CODE: { + status = MMDB_get_value(&result.entry, &entry_data, "country", "iso_code", NULL); + break; + } + case I_GeoLocation::GeoLocationField::CONTINENT_NAME: { + status = MMDB_get_value(&result.entry, &entry_data, "continent", "names", "en", NULL); + break; + } + case I_GeoLocation::GeoLocationField::CONTINENT_CODE: { + status = MMDB_get_value(&result.entry, &entry_data, "continent", "code", NULL); + break; + } + default: { + dbgError(D_GEO_DB) << "Invalid geo location field type"; + break; + } + } + if (status != MMDB_SUCCESS) { + dbgWarning(D_GEO_DB) << "maxMindDB error: " << MMDB_strerror(status); + } else if (!entry_data.has_data) { + dbgWarning(D_GEO_DB) << "maxMindDB Entry has no data"; + } else { + string search_result(entry_data.utf8_string, entry_data.data_size); + return search_result; + } + return ""; + } + + struct sockaddr + convertIPAddrtoSockaddr(const IPAddr &address) + { + if (address.getType() == IPType::V6) { + struct sockaddr_in6 sa6; + sa6.sin6_family = AF_INET6; + sa6.sin6_addr = address.getIPv6(); + return *(struct sockaddr *)&sa6; + } + + struct sockaddr_in sa; + sa.sin_family = AF_INET; + sa.sin_addr = address.getIPv4(); + return *(struct sockaddr *)&sa; + } + + MMDB_s max_mind_db_obj; + int mind_db_obj_status = -1; +}; + +GeoLocation::GeoLocation() : Component("GeoLocation"), pimpl(make_unique()) {} + +GeoLocation::~GeoLocation() {} + +void GeoLocation::preload() { pimpl->preload(); } + +void GeoLocation::init() { pimpl->init(); } + +void GeoLocation::fini() { pimpl->fini(); } diff --git a/components/utils/keywords/keywords_ut/CMakeLists.txt b/components/utils/keywords/keywords_ut/CMakeLists.txt index 260a6f8..041bb22 100644 --- a/components/utils/keywords/keywords_ut/CMakeLists.txt +++ b/components/utils/keywords/keywords_ut/CMakeLists.txt @@ -1,5 +1,5 @@ add_unit_test( keywords_ut "keywords_ut.cc;single_keyword_ut.cc" - "keywords;pcre2-8;buffers;singleton;table;event_is;metric;-lboost_regex" + "keywords;messaging;pcre2-8;buffers;singleton;table;event_is;metric;-lboost_regex" ) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 2b7d2de..8ee29c4 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -12,7 +12,7 @@ add_subdirectory(rest) add_subdirectory(report) add_subdirectory(logging) add_subdirectory(connkey) -add_subdirectory(message) +add_subdirectory(messaging) add_subdirectory(config) add_subdirectory(agent_details) add_subdirectory(event_is) @@ -25,7 +25,6 @@ add_subdirectory(shm_pkt_queue) add_subdirectory(instance_awareness) add_subdirectory(socket_is) add_subdirectory(agent_details_reporter) -add_subdirectory(messaging_buffer) add_subdirectory(metric) add_subdirectory(version) add_subdirectory(tenant_manager) @@ -36,9 +35,9 @@ add_library(ngen_core SHARED ".") target_link_libraries( ngen_core -Wl,-whole-archive - "table;debug_is;shell_cmd;metric;tenant_manager;message;encryptor;time_proxy;singleton;mainloop;environment;logging;report;rest" + "table;debug_is;shell_cmd;metric;tenant_manager;messaging;encryptor;time_proxy;singleton;mainloop;environment;logging;report;rest" "config;intelligence_is_v2;event_is;memory_consumption;connkey" - "instance_awareness;socket_is;messaging_buffer;agent_details;agent_details_reporter;buffers;cpu;agent_core_utilities" + "instance_awareness;socket_is;agent_details;agent_details_reporter;buffers;cpu;agent_core_utilities" -Wl,-no-whole-archive ) diff --git a/core/agent_core_utilities/agent_core_utilities.cc b/core/agent_core_utilities/agent_core_utilities.cc old mode 100755 new mode 100644 diff --git a/core/agent_core_utilities/agent_core_utilities_ut/CMakeLists.txt b/core/agent_core_utilities/agent_core_utilities_ut/CMakeLists.txt index 5b05edc..00cd936 100644 --- a/core/agent_core_utilities/agent_core_utilities_ut/CMakeLists.txt +++ b/core/agent_core_utilities/agent_core_utilities_ut/CMakeLists.txt @@ -3,5 +3,5 @@ include_directories(${CMAKE_SOURCE_DIR}/cptest/include) add_unit_test( agent_core_utilities_ut "agent_core_utilities_ut.cc" - "agent_core_utilities;shell_cmd;config;time_proxy;-lboost_regex" + "agent_core_utilities;shell_cmd;messaging;config;time_proxy;-lboost_regex" ) diff --git a/core/agent_details/agent_details.cc b/core/agent_details/agent_details.cc old mode 100755 new mode 100644 index 0659797..0435aa0 --- a/core/agent_details/agent_details.cc +++ b/core/agent_details/agent_details.cc @@ -27,6 +27,7 @@ using namespace std; USE_DEBUG_FLAG(D_ORCHESTRATOR); + const map AgentDetails::machineTypes({ { "Amazon EC2", I_AgentDetails::MachineType::AWS }, { "Xen", I_AgentDetails::MachineType::AWS }, @@ -466,6 +467,7 @@ AgentDetails::loadProxyType(const string &proxy_type) Maybe AgentDetails::loadProxyType(ProxyProtocol protocol) { + dbgFlow(D_ORCHESTRATOR) << "Loading proxy type: " << convertProxyProtocolToString(protocol); dbgAssert(protocol == ProxyProtocol::HTTP || protocol == ProxyProtocol::HTTPS) << "Unsupported Proxy Protocol " << static_cast(protocol); @@ -537,7 +539,7 @@ AgentDetails::getProxyDomain(ProxyProtocol protocol) const } Maybe -AgentDetails::getProxyCredentials(ProxyProtocol protocol) const +AgentDetails::getProxyAuthentication(ProxyProtocol protocol) const { if (proxies.find(protocol) == proxies.end()) { return genError("Proxy type is not loaded in map, type: " + convertProxyProtocolToString(protocol)); diff --git a/core/agent_details_reporter/agent_details_reporter.cc b/core/agent_details_reporter/agent_details_reporter.cc index c34ea5b..c433c58 100644 --- a/core/agent_details_reporter/agent_details_reporter.cc +++ b/core/agent_details_reporter/agent_details_reporter.cc @@ -228,35 +228,37 @@ AgentDetailsReporter::Impl::sendAttributes() if (is_server) { AttrSerializer(attributes, "save"); - messaging->sendObjectWithPersistence(attr_to_send, I_Messaging::Method::PATCH, "/agents"); + messaging->sendAsyncMessage(HTTPMethod::PATCH, "/agents", attr_to_send); dbgDebug(D_AGENT_DETAILS) << "Triggered persistent message request with attributes to the Fog"; new_attributes.clear(); return true; } for (uint retry = 3; retry > 0; retry--) { - ::Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN); - bool is_success = messaging->sendObject( + MessageMetadata add_agent_details_req_md("127.0.0.1", 7777); + add_agent_details_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); + add_agent_details_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + auto add_agent_details_status = messaging->sendSyncMessage( + HTTPMethod::POST, + "add-agent-details-attr", attr_to_send, - I_Messaging::Method::POST, - "127.0.0.1", - 7777, // primary Orchestrator's port - conn_flags, - "add-agent-details-attr" + MessageCategory::GENERIC, + add_agent_details_req_md ); - if (!is_success) { - is_success = messaging->sendObject( + if (!add_agent_details_status.ok()) { + MessageMetadata secondary_port_req_md("127.0.0.1", 7778); + secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); + secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + add_agent_details_status = messaging->sendSyncMessage( + HTTPMethod::POST, + "add-agent-details-attr", attr_to_send, - I_Messaging::Method::POST, - "127.0.0.1", - 7778, // secondary Orchestrator's port - conn_flags, - "add-agent-details-attr" + MessageCategory::GENERIC, + secondary_port_req_md ); } - if (is_success) { + if (add_agent_details_status.ok()) { dbgDebug(D_AGENT_DETAILS) << "Successfully sent attributes to the Orchestrator"; new_attributes.clear(); return true; @@ -390,7 +392,7 @@ AgentDetailsReporter::Impl::sendReport( additional_metadata.setAdditionalAttributes(attributes); } - messaging->sendObjectWithPersistence(additional_metadata, I_Messaging::Method::PATCH, "/agents"); + messaging->sendAsyncMessage(HTTPMethod::PATCH, "/agents", additional_metadata); } AgentDetailsReporter::AgentDetailsReporter() diff --git a/core/agent_details_reporter/agent_details_reporter_ut/CMakeLists.txt b/core/agent_details_reporter/agent_details_reporter_ut/CMakeLists.txt index 3e53306..5848a02 100644 --- a/core/agent_details_reporter/agent_details_reporter_ut/CMakeLists.txt +++ b/core/agent_details_reporter/agent_details_reporter_ut/CMakeLists.txt @@ -3,5 +3,5 @@ link_directories(${BOOST_ROOT}/lib) add_unit_test( agent_details_reporter_ut "agent_details_reporter_ut.cc" - "environment;config;rest;event_is;metric;agent_details_reporter;-lboost_regex" + "environment;messaging;config;rest;event_is;metric;agent_details_reporter;-lboost_regex" ) diff --git a/core/agent_details_reporter/agent_details_reporter_ut/agent_details_reporter_ut.cc b/core/agent_details_reporter/agent_details_reporter_ut/agent_details_reporter_ut.cc index cae019b..248a0d4 100644 --- a/core/agent_details_reporter/agent_details_reporter_ut/agent_details_reporter_ut.cc +++ b/core/agent_details_reporter/agent_details_reporter_ut/agent_details_reporter_ut.cc @@ -86,44 +86,34 @@ public: TEST_F(AgentReporterTest, dataReport) { string custom_data = "Linux version 24.00.15F"; - EXPECT_CALL( - mock_messaging, - mockSendPersistentMessage( - _, - "{\n" - " \"additionalMetaData\": {\n" - " \"custom_data\": \"Linux version 24.00.15F\"\n" - " }" - "\n}", - I_Messaging::Method::PATCH, - "/agents", - _, - _, - MessageTypeTag::GENERIC - ) - ).WillOnce(Return(string())); + EXPECT_CALL(mock_messaging, sendAsyncMessage( + HTTPMethod::PATCH, + "/agents", + "{\n" + " \"additionalMetaData\": {\n" + " \"custom_data\": \"Linux version 24.00.15F\"\n" + " }" + "\n}", + MessageCategory::GENERIC, + _ + )).Times(1); AgentDataReport() << AgentReportField(custom_data);; } TEST_F(AgentReporterTest, labeledDataReport) { string data = "Linux version 24.00.15F"; - EXPECT_CALL( - mock_messaging, - mockSendPersistentMessage( - _, - "{\n" - " \"additionalMetaData\": {\n" - " \"this_is_custom_label\": \"Linux version 24.00.15F\"\n" - " }" - "\n}", - I_Messaging::Method::PATCH, - "/agents", - _, - _, - MessageTypeTag::GENERIC - ) - ).WillOnce(Return(string())); + EXPECT_CALL(mock_messaging, sendAsyncMessage( + HTTPMethod::PATCH, + "/agents", + "{\n" + " \"additionalMetaData\": {\n" + " \"this_is_custom_label\": \"Linux version 24.00.15F\"\n" + " }" + "\n}", + MessageCategory::GENERIC, + _ + )).Times(1); AgentDataReport() << AgentReportFieldWithLabel("this_is_custom_label", data); } @@ -131,24 +121,18 @@ TEST_F(AgentReporterTest, multiDataReport) { string custom_data = "Linux version 24.00.15F"; string data_to_report = "Agent Version 95.95.95.00A"; - - EXPECT_CALL( - mock_messaging, - mockSendPersistentMessage( - _, - "{\n" - " \"additionalMetaData\": {\n" - " \"custom_data\": \"Linux version 24.00.15F\",\n" - " \"this_is_custom_label\": \"Agent Version 95.95.95.00A\"\n" - " }" - "\n}", - I_Messaging::Method::PATCH, - "/agents", - _, - _, - MessageTypeTag::GENERIC - ) - ).WillOnce(Return(string())); + EXPECT_CALL(mock_messaging, sendAsyncMessage( + HTTPMethod::PATCH, + "/agents", + "{\n" + " \"additionalMetaData\": {\n" + " \"custom_data\": \"Linux version 24.00.15F\",\n" + " \"this_is_custom_label\": \"Agent Version 95.95.95.00A\"\n" + " }" + "\n}", + MessageCategory::GENERIC, + _ + )).Times(1); AgentDataReport() << AgentReportField(custom_data) @@ -160,27 +144,22 @@ TEST_F(AgentReporterTest, multiDataReportWithRegistrationData) string custom_data = "Linux version 24.00.15F"; string data_to_report = "Agent Version 95.95.95.00A"; - EXPECT_CALL( - mock_messaging, - mockSendPersistentMessage( - _, - "{\n" - " \"additionalMetaData\": {\n" - " \"custom_data\": \"Linux version 24.00.15F\",\n" - " \"this_is_custom_label\": \"Agent Version 95.95.95.00A\"\n" - " },\n" - " \"agentVersion\": \"1.15.9\",\n" - " \"policyVersion\": \"ccc\",\n" - " \"platform\": \"bbb\",\n" - " \"architecture\": \"aaa\"\n" - "}", - I_Messaging::Method::PATCH, - "/agents", - _, - _, - MessageTypeTag::GENERIC - ) - ).WillOnce(Return(string())); + EXPECT_CALL(mock_messaging, sendAsyncMessage( + HTTPMethod::PATCH, + "/agents", + "{\n" + " \"additionalMetaData\": {\n" + " \"custom_data\": \"Linux version 24.00.15F\",\n" + " \"this_is_custom_label\": \"Agent Version 95.95.95.00A\"\n" + " },\n" + " \"agentVersion\": \"1.15.9\",\n" + " \"policyVersion\": \"ccc\",\n" + " \"platform\": \"bbb\",\n" + " \"architecture\": \"aaa\"\n" + "}", + MessageCategory::GENERIC, + _ + )).Times(1); AgentDataReport agent_data; agent_data @@ -195,44 +174,34 @@ TEST_F(AgentReporterTest, multiDataReportWithRegistrationData) TEST_F(AgentReporterTest, basicAttrTest) { - EXPECT_CALL( - mock_messaging, - mockSendPersistentMessage( - _, - "{\n" - " \"additionalMetaData\": {}\n" - "}", - I_Messaging::Method::PATCH, - "/agents", - _, - _, - MessageTypeTag::GENERIC - ) - ).WillOnce(Return(string())); + EXPECT_CALL(mock_messaging, sendAsyncMessage( + HTTPMethod::PATCH, + "/agents", + "{\n" + " \"additionalMetaData\": {}\n" + "}", + MessageCategory::GENERIC, + _ + )).Times(1); { AgentDataReport agent_data; } - EXPECT_CALL( - mock_messaging, - mockSendPersistentMessage( - _, - "{\n" - " \"additionalMetaData\": {},\n" - " \"attributes\": {\n" - " \"1\": \"2\",\n" - " \"a\": \"1\",\n" - " \"c\": \"d\"\n" - " }\n" - "}", - I_Messaging::Method::PATCH, - "/agents", - _, - _, - MessageTypeTag::GENERIC - ) - ).WillOnce(Return(string())); + EXPECT_CALL(mock_messaging, sendAsyncMessage( + HTTPMethod::PATCH, + "/agents", + "{\n" + " \"additionalMetaData\": {},\n" + " \"attributes\": {\n" + " \"1\": \"2\",\n" + " \"a\": \"1\",\n" + " \"c\": \"d\"\n" + " }\n" + "}", + MessageCategory::GENERIC, + _ + )).Times(1); EXPECT_TRUE(report->addAttr("a", "b")); EXPECT_TRUE(report->addAttr({{"c", "d"}, {"1", "2"}, {"delete", "me"}})); @@ -243,20 +212,15 @@ TEST_F(AgentReporterTest, basicAttrTest) AgentDataReport agent_data; } - EXPECT_CALL( - mock_messaging, - mockSendPersistentMessage( - _, - "{\n" - " \"additionalMetaData\": {}\n" - "}", - I_Messaging::Method::PATCH, - "/agents", - _, - _, - MessageTypeTag::GENERIC - ) - ).WillOnce(Return(string())); + EXPECT_CALL(mock_messaging, sendAsyncMessage( + HTTPMethod::PATCH, + "/agents", + "{\n" + " \"additionalMetaData\": {}\n" + "}", + MessageCategory::GENERIC, + _ + )).Times(1); { AgentDataReport agent_data; @@ -271,25 +235,20 @@ TEST_F(AgentReporterTest, advancedAttrTest) EXPECT_TRUE(report->addAttr({{"c", "d"}, {"1", "2"}, {"send", "me"}})); EXPECT_TRUE(report->addAttr("a", "b")); - EXPECT_CALL( - mock_messaging, - mockSendPersistentMessage( - _, - "{\n" - " \"attributes\": {\n" - " \"1\": \"2\",\n" - " \"a\": \"b\",\n" - " \"c\": \"d\",\n" - " \"send\": \"me\"\n" - " }\n" - "}", - I_Messaging::Method::PATCH, - "/agents", - _, - _, - MessageTypeTag::GENERIC - ) - ).WillOnce(Return(string())); + EXPECT_CALL(mock_messaging, sendAsyncMessage( + HTTPMethod::PATCH, + "/agents", + "{\n" + " \"attributes\": {\n" + " \"1\": \"2\",\n" + " \"a\": \"b\",\n" + " \"c\": \"d\",\n" + " \"send\": \"me\"\n" + " }\n" + "}", + MessageCategory::GENERIC, + _ + )).Times(1); periodic_report(); @@ -300,26 +259,21 @@ TEST_F(AgentReporterTest, advancedAttrTest) EXPECT_TRUE(report->addAttr("new", "key val")); EXPECT_TRUE(report->addAttr("a", "key val override", true)); - EXPECT_CALL( - mock_messaging, - mockSendPersistentMessage( - _, - "{\n" - " \"attributes\": {\n" - " \"1\": \"2\",\n" - " \"a\": \"key val override\",\n" - " \"c\": \"d\",\n" - " \"new\": \"key val\",\n" - " \"send\": \"me\"\n" - " }\n" - "}", - I_Messaging::Method::PATCH, - "/agents", - _, - _, - MessageTypeTag::GENERIC - ) - ).WillOnce(Return(string())); + EXPECT_CALL(mock_messaging, sendAsyncMessage( + HTTPMethod::PATCH, + "/agents", + "{\n" + " \"attributes\": {\n" + " \"1\": \"2\",\n" + " \"a\": \"key val override\",\n" + " \"c\": \"d\",\n" + " \"new\": \"key val\",\n" + " \"send\": \"me\"\n" + " }\n" + "}", + MessageCategory::GENERIC, + _ + )).Times(1); periodic_report(); } @@ -338,35 +292,29 @@ TEST_F(AgentReporterTest, RestDetailsTest) << "}"; add_details_rest_cb->performRestCall(rest_call_parameters); - EXPECT_CALL( - mock_messaging, - mockSendPersistentMessage( - _, - rest_call_parameters.str(), - I_Messaging::Method::PATCH, - "/agents", - _, - _, - MessageTypeTag::GENERIC - ) - ).WillOnce(Return(string())); + EXPECT_CALL(mock_messaging, sendAsyncMessage( + HTTPMethod::PATCH, + "/agents", + rest_call_parameters.str(), + MessageCategory::GENERIC, + _ + )).Times(1); EXPECT_TRUE(report->sendAttributes()); is_server_mode = false; - EXPECT_CALL(mock_mainloop, addRecurringRoutine(_, _, _, "Report agent details attributes", _)).WillOnce(Return(2)); + EXPECT_CALL( + mock_mainloop, + addRecurringRoutine(_, _, _, "Report agent details attributes", _) + ).WillOnce(Return(2)); EXPECT_CALL(mock_rest, mockRestCall(RestAction::ADD, "agent-details-attr", _)).Times(0); - agent_details_reporter_comp.init(); - EXPECT_TRUE(report->addAttr("new", "key val")); - ::Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN); - EXPECT_CALL( - mock_messaging, - sendMessage( - true, + EXPECT_CALL(mock_messaging, + sendSyncMessage( + HTTPMethod::POST, + "add-agent-details-attr", "{\n" " \"attributes\": {\n" " \"1\": \"2\",\n" @@ -376,16 +324,13 @@ TEST_F(AgentReporterTest, RestDetailsTest) " \"send\": \"me\"\n" " }\n" "}", - I_Messaging::Method::POST, - string("127.0.0.1"), - 7777, - conn_flags, - string("add-agent-details-attr"), - string(), _, - MessageTypeTag::GENERIC + _ ) - ).WillOnce(Return(Maybe(string("{\"status\":true}")))); + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "{\"status\":true}"))); + agent_details_reporter_comp.init(); + + EXPECT_TRUE(report->addAttr("new", "key val")); periodic_report(); } @@ -416,19 +361,13 @@ TEST_F(AgentReporterTest, PersistenceAttrTest) EXPECT_CALL(mock_rest, mockRestCall(RestAction::ADD, "agent-details-attr", _)).WillOnce(Return(true)); agent_details_reporter_comp.init(); - - EXPECT_CALL( - mock_messaging, - mockSendPersistentMessage( - _, - expected_attributes, - I_Messaging::Method::PATCH, - "/agents", - _, - _, - MessageTypeTag::GENERIC - ) - ).WillOnce(Return(string())); + EXPECT_CALL(mock_messaging, sendAsyncMessage( + HTTPMethod::PATCH, + "/agents", + expected_attributes, + MessageCategory::GENERIC, + _ + )).Times(1); EXPECT_TRUE(report->sendAttributes()); EXPECT_TRUE(report->addAttr("new attr", "to add before fini")); diff --git a/core/buffers/buffer.cc b/core/buffers/buffer.cc old mode 100755 new mode 100644 diff --git a/core/buffers/buffers_ut/CMakeLists.txt b/core/buffers/buffers_ut/CMakeLists.txt index 742ad59..20ca8a5 100644 --- a/core/buffers/buffers_ut/CMakeLists.txt +++ b/core/buffers/buffers_ut/CMakeLists.txt @@ -3,5 +3,5 @@ link_directories(${BOOST_ROOT}/lib) add_unit_test( buffers_ut "buffers_ut.cc;buffer_eval_ut.cc" - "buffers;event_is;metric;-lboost_regex" + "buffers;messaging;event_is;metric;-lboost_regex" ) diff --git a/core/buffers/char_iterator.cc b/core/buffers/char_iterator.cc old mode 100755 new mode 100644 diff --git a/core/buffers/data_container.cc b/core/buffers/data_container.cc old mode 100755 new mode 100644 diff --git a/core/buffers/segment.cc b/core/buffers/segment.cc old mode 100755 new mode 100644 diff --git a/core/compression/compression_utils.cc b/core/compression/compression_utils.cc old mode 100755 new mode 100644 index 2b79d45..0aaa656 --- a/core/compression/compression_utils.cc +++ b/core/compression/compression_utils.cc @@ -167,7 +167,7 @@ struct CompressionStream basic_string res; int retries = 0; - while (stream.avail_in != 0) { + while (stream.avail_in != 0 || is_last_chunk) { stream.avail_out = work_space.capacity(); stream.next_out = work_space.data(); diff --git a/core/compression/compression_utils_ut/compression_utils_ut.cc b/core/compression/compression_utils_ut/compression_utils_ut.cc old mode 100755 new mode 100644 diff --git a/core/config/config.cc b/core/config/config.cc index f7a69b7..920c1b8 100644 --- a/core/config/config.cc +++ b/core/config/config.cc @@ -199,27 +199,30 @@ private: config_updates.expected_configurations = move(files); I_Messaging *messaging = Singleton::Consume::by(); - ::Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN); - bool is_success = messaging->sendObject( + MessageMetadata service_config_req_md("127.0.0.1", 7777); + service_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); + service_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + auto service_config_status = messaging->sendSyncMessage( + HTTPMethod::POST, + "/set-nano-service-config", config_updates, - I_Messaging::Method::POST, - "127.0.0.1", - 7777, // primary Orchestrator's port - conn_flags, - "/set-nano-service-config" + MessageCategory::GENERIC, + service_config_req_md ); - if (!is_success) { - is_success = messaging->sendObject( + if (!service_config_status.ok()) { + MessageMetadata secondary_port_req_md("127.0.0.1", 7778); + secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); + secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + service_config_status = messaging->sendSyncMessage( + HTTPMethod::POST, + "/set-nano-service-config", config_updates, - I_Messaging::Method::POST, - "127.0.0.1", - 7778, // secondary Orchestrator's port - conn_flags, - "/set-nano-service-config" + MessageCategory::GENERIC, + secondary_port_req_md ); } - return is_success && config_updates.status.get(); + + return service_config_status.ok() && config_updates.status.get(); } void @@ -244,24 +247,26 @@ private: sendOrchestatorReloadStatusMsg(const LoadNewConfigurationStatus &status) { I_Messaging *messaging = Singleton::Consume::by(); - ::Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN); - bool is_success = messaging->sendNoReplyObject( + MessageMetadata service_config_req_md("127.0.0.1", 7777); + service_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); + service_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + bool service_config_status = messaging->sendSyncMessageWithoutResponse( + HTTPMethod::POST, + "/set-reconf-status", status, - I_Messaging::Method::POST, - "127.0.0.1", - 7777, // primary Orchestrator's port - conn_flags, - "/set-reconf-status" + MessageCategory::GENERIC, + service_config_req_md ); - if (!is_success) { - messaging->sendNoReplyObject( + if (!service_config_status) { + MessageMetadata secondary_port_req_md("127.0.0.1", 7778); + secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); + secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + messaging->sendSyncMessageWithoutResponse( + HTTPMethod::POST, + "/set-reconf-status", status, - I_Messaging::Method::POST, - "127.0.0.1", - 7778, // secondary Orchestrator's port - conn_flags, - "/set-reconf-status" + MessageCategory::GENERIC, + secondary_port_req_md ); } } diff --git a/core/config/include/profile_settings.h b/core/config/include/profile_settings.h old mode 100755 new mode 100644 diff --git a/core/connkey/connkey.cc b/core/connkey/connkey.cc old mode 100755 new mode 100644 diff --git a/core/core_ut/maybe_res_ut.cc b/core/core_ut/maybe_res_ut.cc old mode 100755 new mode 100644 index ec79ecf..20e9629 --- a/core/core_ut/maybe_res_ut.cc +++ b/core/core_ut/maybe_res_ut.cc @@ -359,6 +359,15 @@ TEST(Maybe, maybe_void_error_passing) EXPECT_EQ("error", err2.getErr()); } +TEST(Maybe, maybe_void_error_passing2) +{ + Maybe err1 = genError("error"); + Maybe err2 = err1.passErr(); + + EXPECT_FALSE(err2.ok()); + EXPECT_EQ("error", err2.getErr()); +} + TEST(Maybe, printing) { ostringstream os; diff --git a/core/cptest/cptest.cc b/core/cptest/cptest.cc old mode 100755 new mode 100644 diff --git a/core/cptest/cptest_data_buf.cc b/core/cptest/cptest_data_buf.cc old mode 100755 new mode 100644 diff --git a/core/cptest/cptest_tcppacket.cc b/core/cptest/cptest_tcppacket.cc old mode 100755 new mode 100644 diff --git a/core/cptest/cptest_ut/cptest_packet_ut.cc b/core/cptest/cptest_ut/cptest_packet_ut.cc old mode 100755 new mode 100644 diff --git a/core/cpu/cpu.cc b/core/cpu/cpu.cc old mode 100755 new mode 100644 diff --git a/core/cpu/cpu_ut/CMakeLists.txt b/core/cpu/cpu_ut/CMakeLists.txt index 4d35658..9139d3f 100755 --- a/core/cpu/cpu_ut/CMakeLists.txt +++ b/core/cpu/cpu_ut/CMakeLists.txt @@ -1,5 +1,5 @@ -add_unit_test( - cpu_ut - "cpu_ut.cc" - "cpu;event_is;metric;-lboost_regex" +add_unit_test( + cpu_ut + "cpu_ut.cc" + "cpu;messaging;event_is;metric;-lboost_regex" ) \ No newline at end of file diff --git a/core/cpu/cpu_ut/cpu_ut.cc b/core/cpu/cpu_ut/cpu_ut.cc old mode 100755 new mode 100644 diff --git a/core/debug_is/debug.cc b/core/debug_is/debug.cc old mode 100755 new mode 100644 index 69fb745..51a9a5c --- a/core/debug_is/debug.cc +++ b/core/debug_is/debug.cc @@ -754,7 +754,13 @@ Debug::startStreams( void Debug::isCommunicationFlag(const DebugFlags &flag) { - is_communication |= (flag == D_HTTP_REQUEST || flag == D_COMMUNICATION); + is_communication |= ( + flag == D_MESSAGING || + flag == D_COMMUNICATION || + flag == D_CONNECTION || + flag == D_MESSAGING_BUFFER || + flag == D_HTTP_REQUEST + ); } Debug::DebugLevel Debug::lowest_global_level = default_level; diff --git a/core/debug_is/debug_is_ut/CMakeLists.txt b/core/debug_is/debug_is_ut/CMakeLists.txt index b30ed1b..4c63c62 100644 --- a/core/debug_is/debug_is_ut/CMakeLists.txt +++ b/core/debug_is/debug_is_ut/CMakeLists.txt @@ -3,4 +3,4 @@ include_directories(${CMAKE_SOURCE_DIR}/cptest/include) include_directories(${Boost_INCLUDE_DIRS}) link_directories(${BOOST_ROOT}/lib) -add_unit_test(debug_is_ut "debug_ut.cc" "agent_details;metric;event_is;-lboost_regex") +add_unit_test(debug_is_ut "debug_ut.cc" "agent_details;metric;messaging;event_is;-lboost_regex") diff --git a/core/debug_is/debug_is_ut/debug_ut.cc b/core/debug_is/debug_is_ut/debug_ut.cc old mode 100755 new mode 100644 index 03d2fdd..3fffa62 --- a/core/debug_is/debug_is_ut/debug_ut.cc +++ b/core/debug_is/debug_is_ut/debug_ut.cc @@ -802,7 +802,11 @@ TEST(DebugFogTest, fog_stream) StrictMock mock_mainloop; StrictMock mock_time; - StrictMock mock_agent_details; + NiceMock mock_agent_details; + + ON_CALL(mock_agent_details, getFogDomain()).WillByDefault(Return(Maybe(string("fog_domain.com")))); + ON_CALL(mock_agent_details, getFogPort()).WillByDefault(Return(Maybe(443))); + EXPECT_CALL(mock_agent_details, getAgentId()).WillRepeatedly(Return("Unknown")); EXPECT_CALL(mock_agent_details, getOrchestrationMode()).WillRepeatedly(Return(OrchestrationMode::ONLINE)); @@ -814,15 +818,14 @@ TEST(DebugFogTest, fog_stream) StrictMock messaging_mock; string message_body; - EXPECT_CALL(messaging_mock, mockSendPersistentMessage( - false, - _, + + EXPECT_CALL(messaging_mock, sendAsyncMessage( _, "/api/v1/agents/events/bulk", _, _, - MessageTypeTag::DEBUG - )).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); Singleton::Consume::from(conf)->loadConfiguration( vector{"--orchestration-mode=online_mode"} @@ -953,7 +956,7 @@ TEST(DebugFogTest, fog_stream) " \"agentId\": \"Unknown\",\n" " \"issuingFunction\": \"handleThresholdReach\",\n" " \"issuingFile\": \"debug_streams.cc\",\n" - " \"issuingLine\": 345,\n" + " \"issuingLine\": 344,\n" " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" @@ -977,12 +980,9 @@ TEST(DebugFogTest, fog_stream) .WillOnce(DoAll(InvokeMainLoopCB(), Return(0))); string message_body_1, message_body_2; - EXPECT_CALL( - messaging_mock, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, MessageTypeTag::DEBUG) - ).WillOnce(DoAll(SaveArg<1>(&message_body_1), Return(Maybe(string(""))))).WillOnce( - DoAll(SaveArg<1>(&message_body_2), Return(Maybe(string("")))) - ); + EXPECT_CALL(messaging_mock, sendAsyncMessage(_, "/api/v1/agents/events", _, MessageCategory::DEBUG, _)) + .WillOnce(SaveArg<2>(&message_body_1)) + .WillOnce(SaveArg<2>(&message_body_2)); doFWError(); line1 = line; diff --git a/core/debug_is/debug_streams.cc b/core/debug_is/debug_streams.cc index a9fa83c..503c64a 100644 --- a/core/debug_is/debug_streams.cc +++ b/core/debug_is/debug_streams.cc @@ -296,8 +296,7 @@ DebugFogStream::sendBufferedMessages() while (!reports.empty()) { auto rest = reports.pop(); - using Method = I_Messaging::Method; - i_msg->sendObjectWithPersistence(rest, Method::POST, fog_debug_uri, "", true, MessageTypeTag::DEBUG); + i_msg->sendAsyncMessage(HTTPMethod::POST, fog_debug_uri, rest, MessageCategory::DEBUG); } } void @@ -310,7 +309,7 @@ DebugFogStream::sendSingleMessage(const LogRest &rest) ); auto i_msg = Singleton::Consume::by(); - i_msg->sendObjectWithPersistence(rest, I_Messaging::Method::POST, fog_debug_uri, "", true, MessageTypeTag::DEBUG); + i_msg->sendAsyncMessage(HTTPMethod::POST, fog_debug_uri, rest, MessageCategory::DEBUG); } void diff --git a/core/encryptor/cpnano_base64/base64.cc b/core/encryptor/cpnano_base64/base64.cc old mode 100755 new mode 100644 diff --git a/core/encryptor/cpnano_base64/base64.h b/core/encryptor/cpnano_base64/base64.h old mode 100755 new mode 100644 diff --git a/core/encryptor/cpnano_base64/cpnano_base64.cc b/core/encryptor/cpnano_base64/cpnano_base64.cc old mode 100755 new mode 100644 diff --git a/core/encryptor/encryptor_ut/CMakeLists.txt b/core/encryptor/encryptor_ut/CMakeLists.txt index d147d78..330cbd0 100755 --- a/core/encryptor/encryptor_ut/CMakeLists.txt +++ b/core/encryptor/encryptor_ut/CMakeLists.txt @@ -3,5 +3,5 @@ link_directories(${BOOST_ROOT}/lib) add_unit_test( encryptor_ut "encryptor_ut.cc" - "encryptor;config;singleton;metric;event_is;-lboost_context;-lboost_regex;-lresolv;-lcrypto" + "encryptor;messaging;config;singleton;metric;event_is;-lboost_context;-lboost_regex;-lresolv;-lcrypto" ) diff --git a/core/environment/environment.cc b/core/environment/environment.cc index 4c4af12..d0278b3 100644 --- a/core/environment/environment.cc +++ b/core/environment/environment.cc @@ -59,6 +59,7 @@ public: string getCurrentTrace() const override; string getCurrentSpan() const override; string getCurrentHeaders() override; + map getCurrentHeadersMap() override; void startNewTrace(bool new_span, const string &_trace_id) override; void startNewSpan(Span::ContextType _type, const string &prev_span, const string &trace) override; @@ -266,6 +267,34 @@ Environment::Impl::getCurrentHeaders() return tracing_headers; } +map +Environment::Impl::getCurrentHeadersMap() +{ + map tracing_headers; + auto trace_id = getCurrentTrace(); + if (!trace_id.empty()) { + tracing_headers["X-Trace-Id"] = trace_id; + } else { + string correlation_id_string = "00000000-0000-0000-0000-000000000000"; + try { + boost::uuids::random_generator uuid_random_gen; + correlation_id_string = boost::uuids::to_string(uuid_random_gen()); + } catch (const boost::uuids::entropy_error &e) { + dbgTrace(D_ENVIRONMENT) + << "Failed to generate random correlation id - entropy exception. Exception: " + << e.what(); + tracing_status = TracingStatus::DISABLED; + } + tracing_headers["X-Trace-Id"] = correlation_id_string; + } + + auto span_id = getCurrentSpan(); + if (!span_id.empty()) { + tracing_headers["X-Span-Id"] = span_id; + } + return tracing_headers; +} + void Environment::Impl::startNewTrace(bool new_span, const string &_trace_id) { diff --git a/core/environment/environment_ut/CMakeLists.txt b/core/environment/environment_ut/CMakeLists.txt index 014a215..8325f3c 100755 --- a/core/environment/environment_ut/CMakeLists.txt +++ b/core/environment/environment_ut/CMakeLists.txt @@ -1,5 +1,5 @@ add_unit_test( environment_ut "context_ut.cc;parsing_ut.cc;base_evaluators_ut.cc;environment_rest_ut.cc;span_ut.cc;trace_ut.cc;tracing_ut.cc;environment_ut.cc" - "environment;singleton;rest;mainloop;metric;-lboost_context;event_is;-lboost_regex" + "environment;messaging;singleton;rest;mainloop;metric;-lboost_context;event_is;-lboost_regex" ) diff --git a/core/environment/environment_ut/base_evaluators_ut.cc b/core/environment/environment_ut/base_evaluators_ut.cc old mode 100755 new mode 100644 diff --git a/core/environment/environment_ut/context_ut.cc b/core/environment/environment_ut/context_ut.cc old mode 100755 new mode 100644 diff --git a/core/environment/environment_ut/parsing_ut.cc b/core/environment/environment_ut/parsing_ut.cc old mode 100755 new mode 100644 diff --git a/core/environment/environment_ut/span_ut.cc b/core/environment/environment_ut/span_ut.cc old mode 100755 new mode 100644 diff --git a/core/environment/environment_ut/trace_ut.cc b/core/environment/environment_ut/trace_ut.cc old mode 100755 new mode 100644 diff --git a/core/environment/environment_ut/tracing_ut.cc b/core/environment/environment_ut/tracing_ut.cc old mode 100755 new mode 100644 index 764a835..33ad799 --- a/core/environment/environment_ut/tracing_ut.cc +++ b/core/environment/environment_ut/tracing_ut.cc @@ -344,8 +344,6 @@ public: TEST_F(TracingCompRoutinesTest, 2SpansDifFlow) { - ON_CALL(mock_messaging, mockSendPersistentMessage(_, _, _, _, _, _, _)).WillByDefault(Return(string())); - I_MainLoop::Routine routine = [&] () { i_env->startNewTrace(true, "a687b388-1108-4083-9852-07c33b1074e9"); trace_id = i_env->getCurrentTrace(); diff --git a/core/environment/span.cc b/core/environment/span.cc old mode 100755 new mode 100644 diff --git a/core/environment/trace.cc b/core/environment/trace.cc old mode 100755 new mode 100644 diff --git a/core/include/attachments/compression_utils.h b/core/include/attachments/compression_utils.h old mode 100755 new mode 100644 diff --git a/core/include/attachments/nginx_attachment_common.h b/core/include/attachments/nginx_attachment_common.h old mode 100755 new mode 100644 diff --git a/core/include/attachments/shmem_ipc.h b/core/include/attachments/shmem_ipc.h old mode 100755 new mode 100644 diff --git a/core/include/general/buffer.h b/core/include/general/buffer.h old mode 100755 new mode 100644 diff --git a/core/include/general/buffer/char_iterator.h b/core/include/general/buffer/char_iterator.h old mode 100755 new mode 100644 diff --git a/core/include/general/buffer/data_container.h b/core/include/general/buffer/data_container.h old mode 100755 new mode 100644 diff --git a/core/include/general/buffer/helper_functions.h b/core/include/general/buffer/helper_functions.h old mode 100755 new mode 100644 diff --git a/core/include/general/buffer/internal_ptr.h b/core/include/general/buffer/internal_ptr.h old mode 100755 new mode 100644 diff --git a/core/include/general/buffer/segment.h b/core/include/general/buffer/segment.h old mode 100755 new mode 100644 diff --git a/core/include/general/c_common/network_defs.h b/core/include/general/c_common/network_defs.h old mode 100755 new mode 100644 diff --git a/core/include/general/c_common/networking_headers.h b/core/include/general/c_common/networking_headers.h old mode 100755 new mode 100644 diff --git a/core/include/general/cptest.h b/core/include/general/cptest.h old mode 100755 new mode 100644 diff --git a/core/include/general/cptest/cptest_tcppacket.h b/core/include/general/cptest/cptest_tcppacket.h old mode 100755 new mode 100644 diff --git a/core/include/general/debug.h b/core/include/general/debug.h old mode 100755 new mode 100644 diff --git a/core/include/general/intelligence_comp_v2.h b/core/include/general/intelligence_comp_v2.h old mode 100755 new mode 100644 diff --git a/core/include/general/scope_exit.h b/core/include/general/scope_exit.h index 7bb86b7..d5636d5 100644 --- a/core/include/general/scope_exit.h +++ b/core/include/general/scope_exit.h @@ -29,7 +29,7 @@ // Code taken from N4189 pending standard - www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4189.pdf // (with some minor adaptations for C++11) -#include // For ::std::remove_reference +#include namespace std { @@ -56,7 +56,6 @@ public: if (execute_on_destruction) this->exit_function(); } -// LCOV_EXCL_START Reason: coverage upgrade // move scope_exit(scope_exit &&rhs) noexcept : @@ -65,7 +64,6 @@ public: { rhs.release(); } -// LCOV_EXCL_STOP void release() noexcept diff --git a/core/include/general/time_proxy.h b/core/include/general/time_proxy.h old mode 100755 new mode 100644 diff --git a/core/include/internal/http_encoder.h b/core/include/internal/http_encoder.h deleted file mode 100755 index c80caf2..0000000 --- a/core/include/internal/http_encoder.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __HTTP_ENCODER_H__ -#define __HTTP_ENCODER_H__ - -#include - -#include "messaging/http_core.h" - -class HTTPRequest -{ -public: - HTTPRequest() = default; - HTTPRequest(const std::string &_method_statement); - HTTPRequest(const std::string &_method_statement, const std::string &_host, const bool to_proxy); - - HTTPRequest & insertHeader(const std::string &header_key, const std::string &header_val); - HTTPRequest & insertHeader(const std::string &header); - HTTPRequest & insertHeaders(const std::string &rec_headers); - HTTPRequest & insertBody(const std::string &body); - - std::string toString() const; - -private: - std::string method_statement; - HTTPHeaders headers; - std::string body; -}; - -class ConnectRequest: public HTTPRequest -{ -public: - ConnectRequest(const std::string &_host, const std::string &_port); -}; - -class PostRequest: public HTTPRequest -{ -public: - PostRequest(const std::string &_post_path, const std::string &_host, bool to_proxy); -}; - -class PutRequest : public HTTPRequest -{ -public: - PutRequest(const std::string &_put_path, const std::string &_host, bool to_proxy); -}; - -class GetRequest: public HTTPRequest -{ -public: - GetRequest(const std::string &_get_path, const std::string &_host, bool to_proxy); -}; - -class PatchRequest: public HTTPRequest -{ -public: - PatchRequest(const std::string &_patch_path, const std::string &_host, bool to_proxy); -}; - -class HTTPEncoder -{ -public: - HTTPEncoder(const std::string &_host, const std::string &_port); - - HTTPRequest & Connect(); - HTTPRequest & Post(const std::string &post_path); - HTTPRequest & Put(const std::string &put_path); - HTTPRequest & Patch(const std::string &patch_path); - HTTPRequest & Get(const std::string &get_path); - - HTTPEncoder & isOverProxy(); - HTTPEncoder & isOverSSL(); - - std::string build() const { return request.toString(); } - -private: - HTTPRequest request; - std::string host; - std::string port; - bool over_ssl = false; - bool over_proxy = false; -}; - -#endif // __HTTP_ENCODER_H__ diff --git a/core/include/internal/ioctl_is.h b/core/include/internal/ioctl_is.h old mode 100755 new mode 100644 diff --git a/core/include/internal/mainloop/mainloop_metric.h b/core/include/internal/mainloop/mainloop_metric.h old mode 100755 new mode 100644 diff --git a/core/include/internal/proto_message_comp.h b/core/include/internal/messaging.h similarity index 76% rename from core/include/internal/proto_message_comp.h rename to core/include/internal/messaging.h index 0453eb8..3051aaf 100644 --- a/core/include/internal/proto_message_comp.h +++ b/core/include/internal/messaging.h @@ -11,44 +11,43 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef __PROTO_MESSAGE_COMP_H__ -#define __PROTO_MESSAGE_COMP_H__ +#ifndef __MESSAGING_H__ +#define __MESSAGING_H__ -#include -#include - -#include "singleton.h" -#include "i_mainloop.h" -#include "i_encryptor.h" #include "i_messaging.h" -#include "i_agent_details.h" -#include "i_environment.h" -#include "i_rest_api.h" -#include "i_messaging_buffer.h" -#include "i_shell_cmd.h" -#include "i_proxy_configuration.h" -#include "component.h" -class ProtoMessageComp +#include "component.h" +#include "singleton.h" + +#include "i_environment.h" +#include "i_proxy_configuration.h" +#include "i_agent_details.h" +#include "i_mainloop.h" +#include "i_time_get.h" +#include "i_encryptor.h" +#include "i_shell_cmd.h" +#include "i_instance_awareness.h" + +#include "config.h" + +class Messaging : public Component, Singleton::Provide, - Singleton::Consume, - Singleton::Consume, - Singleton::Consume, - Singleton::Consume, + Singleton::Consume, Singleton::Consume, - Singleton::Consume, + Singleton::Consume, + Singleton::Consume, + Singleton::Consume, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: - ProtoMessageComp(); - ~ProtoMessageComp(); + Messaging(); + ~Messaging(); void init(); - void fini(); - void preload(); private: @@ -56,4 +55,4 @@ private: std::unique_ptr pimpl; }; -#endif // __PROTO_MESSAGE_COMP_H__ +#endif // __MESSAGING_H__ diff --git a/core/include/internal/messaging_buffer/bucket_manager.h b/core/include/internal/messaging_buffer/bucket_manager.h deleted file mode 100644 index 8d2cbef..0000000 --- a/core/include/internal/messaging_buffer/bucket_manager.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __BUCKET_MANAGER_H__ -#define __BUCKET_MANAGER_H__ - -#include -#include -#include - -#include "event_queue.h" -#include "instance_awareness.h" -#include "i_time_get.h" -#include "i_encryptor.h" - -using bucketName = std::string; - -class BucketManager -{ -public: - void init(const std::string &service_name); - void fini(); - bool doesExist(const bucketName &); - void push(const bucketName &, std::string &&); - bool handleNextBucket(); - bool hasValue(); - EventQueue & peek(); - - void flush(); - -private: - std::string resolveFilesName(const std::string &file_name); - - std::string buffer_directory = ""; - std::string next_bucket = ""; - std::string service_name = ""; - std::string management_file_path = ""; - - uint buffer_max_size = 0; // in MB - uint max_buffer_files = 0; - - EventQueue iterator; - std::unordered_map buckets; - I_InstanceAwareness *instance_awareness = nullptr; - I_Encryptor *encryptor = nullptr; -}; - -#endif // __BUCKET_MANAGER_H__ diff --git a/core/include/internal/messaging_buffer/event_queue.h b/core/include/internal/messaging_buffer/event_queue.h deleted file mode 100644 index 6199f10..0000000 --- a/core/include/internal/messaging_buffer/event_queue.h +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __EVENT_QUEUE_H__ -#define __EVENT_QUEUE_H__ - -#include - -#include -#include -#include -#include -#include -#include - -#include "i_time_get.h" -#include "maybe_res.h" - -USE_DEBUG_FLAG(D_EVENT_BUFFER); - -class EventQueueFile -{ -public: - EventQueueFile(const std::string &_file_path) - : - file_path(_file_path) - {} - - EventQueueFile( - const std::string &file_location_path, - const std::string &file_extention, - bool is_file_compressed); - - EventQueueFile(const EventQueueFile &other_event, int _num_of_events_in_file, uint64_t _size_of_file) - : - file_path(other_event.file_path), - suffix(other_event.suffix), - num_of_events_in_file(_num_of_events_in_file), - size_of_file(_size_of_file) - {} - - static const std::string zip_file_suffix; - const std::string & getFilePath() const { return file_path; } - bool isCompressed() const { return is_compressed; } - int getSuffix() const { return suffix; } - int getNumOfEvents() const { return num_of_events_in_file; } - uint64_t getFileSizeInBytes() const { return size_of_file; } - - void restoreNumberOfLines(); - void incFileSize(uint64_t size_to_add); - void handleCompression(int size_of_files_list); - void decompress(const std::string &infilename, const std::string &outfilename, bool remove_old = true); - void compress(); - -private: - std::string file_path; - int suffix = -1; - bool is_compressed = false; - int num_of_events_in_file = 0; - size_t size_of_file = 0; -}; - -class EventQueue -{ -public: - EventQueue() = default; - ~EventQueue(); - - void init(const std::string &path, uint max_buff_size); - void fini(); - bool isEmpty() const; - const std::string & peek(); - void push(std::string &&event_data); - void reloadEventsIntoList(const std::string &path); - - Maybe refreshBufferFile(); - void refreshReadBuff(); - - void trim(); - void flush(); - -private: - void rotate(); - void updateReadFile(); - void setReaderFileAndOpen(const EventQueueFile &file); - void sortEventFilesBySuffix(std::vector &tmp_vec); - void pushNewEventQueueFile(EventQueueFile &eventFile, std::vector &tmp_vec); - double getSizeMB(double size_in_B) const; - Maybe writeCachesToFile(); - void enforceMaxNumberOfFiles(); - - // File management - std::list files; //front is write, back is read - std::ifstream reader; - std::ofstream writer; - - // Read & write management - double max_size; // in MB - uint64_t size_on_disk; // in B - uint64_t write_cache_size; // in B - uint64_t read_cache_size; // in B - std::list write_cache_buff; - std::list read_cache_buff; - - unsigned int num_of_events_on_disk; - unsigned int read_events_on_disk; - - // Timing management - std::chrono::microseconds next_sync_freq_in_sec; - I_TimeGet *timer = nullptr; - bool is_pending_rotate = false; - bool is_pending_write = false; -}; - -#endif // __EVENT_QUEUE_H__ diff --git a/core/include/internal/shell_cmd.h b/core/include/internal/shell_cmd.h old mode 100755 new mode 100644 diff --git a/core/include/internal/trap_handler.h b/core/include/internal/trap_handler.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/i_agent_details.h b/core/include/services_sdk/interfaces/i_agent_details.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/i_cpu.h b/core/include/services_sdk/interfaces/i_cpu.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/i_environment.h b/core/include/services_sdk/interfaces/i_environment.h index 420aa6f..88f0f04 100644 --- a/core/include/services_sdk/interfaces/i_environment.h +++ b/core/include/services_sdk/interfaces/i_environment.h @@ -80,6 +80,7 @@ public: virtual std::string getCurrentTrace() const = 0; virtual std::string getCurrentSpan() const = 0; virtual std::string getCurrentHeaders() = 0; + virtual std::map getCurrentHeadersMap() = 0; virtual void startNewTrace(bool new_span = true, const std::string &_trace_id = std::string()) = 0; virtual void startNewSpan( Span::ContextType _type, diff --git a/core/include/services_sdk/interfaces/i_failopen.h b/core/include/services_sdk/interfaces/i_failopen.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/i_health_check_manager.h b/core/include/services_sdk/interfaces/i_health_check_manager.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h old mode 100755 new mode 100644 index 365120a..0ab65ee --- a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h +++ b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h @@ -131,20 +131,18 @@ private: IntelligenceQuery &intelligence_query, const std::string &query_uri, I_Messaging *i_message, - Flags conn_flags, + Flags conn_flags, const std::string &ip, uint server_port ) { if (ip == "" && server_port == 0) { - return i_message->sendObject( - intelligence_query, - I_Messaging::Method::POST, + auto req_status = i_message->sendSyncMessage( + HTTPMethod::POST, query_uri, - "", - nullptr, - true, - MessageTypeTag::INTELLIGENCE + intelligence_query, + MessageCategory::INTELLIGENCE ); + return req_status.ok(); } dbgTrace(D_INTELLIGENCE) @@ -154,18 +152,15 @@ private: << server_port << " query_uri: " << query_uri; - - return i_message->sendObject( - intelligence_query, - I_Messaging::Method::POST, - ip, - server_port, - conn_flags, + MessageMetadata req_md(ip, server_port, conn_flags); + auto req_status = i_message->sendSyncMessage( + HTTPMethod::POST, query_uri, - "", - nullptr, - MessageTypeTag::INTELLIGENCE + intelligence_query, + MessageCategory::INTELLIGENCE, + req_md ); + return req_status.ok(); } template @@ -174,7 +169,7 @@ private: IntelligenceQuery &intelligence_query, const std::string &query_uri, I_Messaging *i_message, - Flags conn_flags, + Flags conn_flags, const std::string &ip = "", uint server_port = 0 ) { @@ -233,7 +228,7 @@ private: const std::string &query_uri, int assets_limit, I_Messaging *i_message, - Flags conn_flags, + Flags conn_flags, const std::string &ip = "", uint server_port = 0 ) { @@ -265,7 +260,7 @@ private: bool is_primary_port, int assets_limit, I_Messaging *i_message, - Flags conn_flags + Flags conn_flags ) { static const std::string primary_port_setting = "local intelligence server primary port"; static const std::string secondary_port_setting = "local intelligence server secondary port"; @@ -299,7 +294,7 @@ private: sendQueryObject(IntelligenceQuery &intelligence_query, const std::string &query_uri, int assets_limit) { auto i_message = getMessaging(); - Flags conn_flags; + Flags conn_flags; bool crowdsec_enabled = std::getenv("CROWDSEC_ENABLED") ? std::string(std::getenv("CROWDSEC_ENABLED")) == "true" : diff --git a/core/include/services_sdk/interfaces/i_ioctl.h b/core/include/services_sdk/interfaces/i_ioctl.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/i_messaging.h b/core/include/services_sdk/interfaces/i_messaging.h old mode 100755 new mode 100644 index 0cb9bd4..52a3bf6 --- a/core/include/services_sdk/interfaces/i_messaging.h +++ b/core/include/services_sdk/interfaces/i_messaging.h @@ -10,172 +10,98 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - -#ifndef __I_FOG_MESSAGING_H__ -#define __I_FOG_MESSAGING_H__ +#ifndef __I_MESSAGING_H__ +#define __I_MESSAGING_H__ #include +#include #include #include -#include +#include -#include "cereal/archives/json.hpp" -#include "cereal/types/common.hpp" -#include "cereal/types/string.hpp" -#include "cereal/types/vector.hpp" - -#include "maybe_res.h" -#include "debug.h" -#include "messaging/http_core.h" +#include "i_agent_details.h" +#include "i_proxy_configuration.h" #include "flags.h" +#include "maybe_res.h" +#include "messaging/http_response.h" +#include "messaging/messaging_metadata.h" -USE_DEBUG_FLAG(D_COMMUNICATION); - -enum class MessageTypeTag -{ - GENERIC, - LOG, - DEBUG, - METRIC, - REPORT, - WAAP_LEARNING, - INTELLIGENCE, - BUFFERED_MESSAGES, - - COUNT -}; - -enum class MessageConnConfig -{ - SECURE_CONN, - ONE_TIME_CONN, - EXPECT_REPLY, - EXTERNAL, - IGNORE_SSL_VALIDATION, - - COUNT -}; +USE_DEBUG_FLAG(D_MESSAGING); class I_Messaging + : + Singleton::Consume, + Singleton::Consume { public: - using string = std::string; - using ErrorCB = std::function; + template + Maybe sendSyncMessage( + HTTPMethod method, + const std::string &uri, + serializableObject &req_obj, + MessageCategory category = MessageCategory::GENERIC, + MessageMetadata message_metadata = MessageMetadata()); - enum class Method { GET, POST, PATCH, CONNECT, PUT }; + template + bool sendSyncMessageWithoutResponse( + const HTTPMethod method, + const std::string &uri, + serializableObject &req_obj, + const MessageCategory category = MessageCategory::GENERIC, + MessageMetadata message_metadata = MessageMetadata()); - template - bool - sendObject(T &obj, Args ...args) - { - auto req = obj.genJson(); - if (!req.ok()) { - dbgWarning(D_COMMUNICATION) << "Failed to create a request. Error: " << req.getErr(); - return false; - } + template + void sendAsyncMessage( + const HTTPMethod method, + const std::string &uri, + serializableObject &req_obj, + const MessageCategory category = MessageCategory::GENERIC, + MessageMetadata message_metadata = MessageMetadata()); - dbgTrace(D_COMMUNICATION) << "Request generated from json. Request: " << req.unpack(); - auto res = sendMessage(true, *req, args...); - if (!res.ok()) { - dbgWarning(D_COMMUNICATION) << "Failed to send request. Error: " << res.getErr(); - return false; - } - dbgTrace(D_COMMUNICATION) << "Successfully got response: " << res.unpack(); + virtual void sendAsyncMessage( + const HTTPMethod method, + const std::string &uri, + const std::string &body, + const MessageCategory category, + MessageMetadata message_metadata + ) = 0; + virtual Maybe sendSyncMessage( + const HTTPMethod method, + const std::string &uri, + const std::string &body, + const MessageCategory category = MessageCategory::GENERIC, + MessageMetadata message_metadata = MessageMetadata() + ) = 0; - auto res_json = obj.loadJson(res.unpack()); - if (!res_json) { - dbgWarning(D_COMMUNICATION) << "Failed to parse response body. Content: " << res.unpack(); - } else { - dbgTrace(D_COMMUNICATION) << "Successfully parsed response body"; - } - return res_json; - } + virtual Maybe downloadFile( + const HTTPMethod method, + const std::string &uri, + const std::string &download_file_path, + const MessageCategory category = MessageCategory::GENERIC, + MessageMetadata message_metadata = MessageMetadata() + ) = 0; - template - bool - sendNoReplyObject(T &obj, Args ...args) - { - auto req = obj.genJson(); - if (!req.ok()) { - dbgWarning(D_COMMUNICATION) << "Failed to create a request. Error: " << req.getErr();; - return false; - } + virtual Maybe uploadFile( + const std::string & uri, + const std::string & upload_file_path, + const MessageCategory category = MessageCategory::GENERIC, + MessageMetadata message_metadata = MessageMetadata() + ) = 0; - auto res = sendMessage(false, *req, args...); - if (!res.ok()) { - dbgWarning(D_COMMUNICATION) << "Failed to send request. Error: " << res.getErr(); - } - return res.ok(); - } - - template - void - sendObjectWithPersistence(T &obj, Args ...args) - { - auto req = obj.genJson(); - if (!req.ok()) { - dbgWarning(D_COMMUNICATION) << "Failed to create a request. Error: " << req.getErr();; - return; - } - - sendPersistentMessage(false, req.unpackMove(), args...); - } - - template - Maybe - downloadFile(T &obj, Args ...args) - { - auto req = obj.genJson(); - if (!req.ok()) return genError("Invalid request"); - auto response = sendMessage(true, *req, args...); - if (response.ok()) { - return response.unpack(); - } - return genError("Failed to download file. Error: " + response.getErr()); - } - - virtual bool setActiveFog(MessageTypeTag tag) = 0; - virtual bool setActiveFog(const string &host, const uint16_t port, bool is_secure, MessageTypeTag tag) = 0; - -protected: - ~I_Messaging() {} - -private: - virtual Maybe - sendPersistentMessage( - bool get_reply, - const string &&body, - Method method, - const string &uri, - const string &headers = "", - bool should_yield = true, - MessageTypeTag tag = MessageTypeTag::GENERIC, - bool skip_sending = false) = 0; - - virtual Maybe - sendMessage( - bool get_reply, - const string &body, - Method method, - const string &uri, - const string &headers = "", - ErrorCB err_call_back = nullptr, - bool should_yield = true, - MessageTypeTag tag = MessageTypeTag::GENERIC) = 0; - - virtual Maybe - sendMessage( - bool get_reply, - const string &body, - Method method, + virtual bool setFogConnection( const std::string &host, uint16_t port, - Flags &conn_flags, - const string &uri, - const string &headers = "", - ErrorCB err_call_back = nullptr, - MessageTypeTag tag = MessageTypeTag::GENERIC) = 0; + bool is_secure, + MessageCategory category + ) = 0; + + virtual bool setFogConnection(MessageCategory category = MessageCategory::GENERIC) = 0; + +protected: + virtual ~I_Messaging() {} }; -#endif // __I_FOG_MESSAGING_H__ +#include "messaging/interface_impl.h" + +#endif // __I_MESSAGING_H__ diff --git a/core/include/services_sdk/interfaces/i_proxy_configuration.h b/core/include/services_sdk/interfaces/i_proxy_configuration.h index d10f451..eac3bea 100644 --- a/core/include/services_sdk/interfaces/i_proxy_configuration.h +++ b/core/include/services_sdk/interfaces/i_proxy_configuration.h @@ -27,12 +27,12 @@ enum class ProxyProtocol class I_ProxyConfiguration { public: - virtual Maybe getProxyDomain(ProxyProtocol protocol) const = 0; - virtual Maybe getProxyCredentials(ProxyProtocol protocol) const = 0; - virtual Maybe getProxyPort(ProxyProtocol protocol) const = 0; - virtual bool getProxyExists(ProxyProtocol protocol) const = 0; - virtual Maybe getProxyAddress(ProxyProtocol protocol) const = 0; - virtual Maybe loadProxy() = 0; + virtual Maybe getProxyDomain(ProxyProtocol protocol) const = 0; + virtual Maybe getProxyAuthentication(ProxyProtocol protocol) const = 0; + virtual Maybe getProxyPort(ProxyProtocol protocol) const = 0; + virtual bool getProxyExists(ProxyProtocol protocol) const = 0; + virtual Maybe getProxyAddress(ProxyProtocol protocol) const = 0; + virtual Maybe loadProxy() = 0; }; #endif // __I_PROXY_CONFIGURATION_H__ diff --git a/core/include/services_sdk/interfaces/i_shell_cmd.h b/core/include/services_sdk/interfaces/i_shell_cmd.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/i_signal_handler.h b/core/include/services_sdk/interfaces/i_signal_handler.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/i_socket_is.h b/core/include/services_sdk/interfaces/i_socket_is.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/i_trap_handler.h b/core/include/services_sdk/interfaces/i_trap_handler.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2_impl.h b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2_impl.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h old mode 100755 new mode 100644 index dd2ba45..aab8369 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h @@ -45,7 +45,8 @@ enum class Condition IN, NOT_IN, GREATER_THAN, - LESS_THAN + LESS_THAN, + RANGE }; enum class CursorState { @@ -60,7 +61,7 @@ enum class ResponseStatus IN_PROGRESS }; -enum class ObjectType { ASSET, ZONE, CONFIGURATION, COUNT }; +enum class ObjectType { ASSET, ZONE, CONFIGURATION, SHORTLIVED, COUNT}; const std::string & convertConditionTypeToString(const Condition &condition_type); const std::string & convertOperationTypeToString(const Operator &operation_type); diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h old mode 100755 new mode 100644 index 7fb173b..15fdceb --- a/core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h @@ -30,7 +30,7 @@ using namespace Intelligence_IS_V2; class SerializableQueryCondition { public: - typedef boost::variant ValueVariant; + typedef boost::variant> ValueVariant; SerializableQueryCondition() {} @@ -48,6 +48,13 @@ public: value(_value) {} + SerializableQueryCondition(Condition _condition_type, std::string _key, std::vector _value) + : + condition_type(_condition_type), + key(_key), + value(_value) + {} + void save(cereal::JSONOutputArchive &ar) const; Condition getConditionType() const { return condition_type; } @@ -66,11 +73,13 @@ public: SerializableQueryFilter() {} SerializableQueryFilter(Condition condition_type, const std::string &key, const std::string &value); SerializableQueryFilter(Condition condition_type, const std::string &key, const int64_t &value); + SerializableQueryFilter(Condition condition_type, const std::string &key, const std::vector &value); void save(cereal::JSONOutputArchive &ar) const; void addCondition(Condition condition_type, const std::string &key, const std::string &value); void addCondition(Condition condition_type, const std::string &key, const int64_t &value); + void addCondition(Condition condition_type, const std::string &key, const std::vector &value); Operator getOperator() const { return operator_type; } const std::vector & getConditionOperands() const { return condition_operands; } diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h old mode 100755 new mode 100644 index ecf90bf..337b384 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h @@ -45,6 +45,14 @@ public: AttributeKeyType type = AttributeKeyType::MAIN ); + QueryRequest( + Condition condition_type, + const std::string &key, + const std::vector &value, + bool full_response, + AttributeKeyType type = AttributeKeyType::MAIN + ); + void saveToJson(cereal::JSONOutputArchive &ar) const; void save(cereal::JSONOutputArchive &ar) const; @@ -66,6 +74,13 @@ public: AttributeKeyType attribute_type = AttributeKeyType::MAIN ); + void addCondition( + Condition condition_type, + const std::string &key, + const std::vector &value, + AttributeKeyType attribute_type = AttributeKeyType::MAIN + ); + void setRequestedAttr( const std::string &attr, AttributeKeyType attribute_type = AttributeKeyType::REGULAR diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2_impl.h b/core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2_impl.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/requested_attributes_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/requested_attributes_v2.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/messaging/http_response.h b/core/include/services_sdk/interfaces/messaging/http_response.h new file mode 100644 index 0000000..5b4b813 --- /dev/null +++ b/core/include/services_sdk/interfaces/messaging/http_response.h @@ -0,0 +1,45 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __HTTP_RESPONSE_H__ +#define __HTTP_RESPONSE_H__ + +#include +#include +#include + +#include "singleton.h" +#include "i_env_details.h" +#include "i_agent_details.h" +#include "i_encryptor.h" +#include "messaging/messaging_enums.h" + +class HTTPResponse +{ +public: +// LCOV_EXCL_START Reason: Not actually called but is required by the caching interface + HTTPResponse() = default; +// LCOV_EXCL_STOP + + HTTPResponse(HTTPStatusCode _status_code, const std::string &_body) : status_code(_status_code), body(_body) {} + + HTTPStatusCode getHTTPStatusCode() const; + const std::string & getBody() const; + std::string toString() const; + +private: + HTTPStatusCode status_code; + std::string body; +}; + +#endif // __HTTP_RESPONSE_H__ diff --git a/core/include/services_sdk/interfaces/messaging/interface_impl.h b/core/include/services_sdk/interfaces/messaging/interface_impl.h new file mode 100644 index 0000000..1995dc1 --- /dev/null +++ b/core/include/services_sdk/interfaces/messaging/interface_impl.h @@ -0,0 +1,148 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef __INTERFACE_IMPL_H__ +#define __INTERFACE_IMPL_H__ + +#ifndef __I_MESSAGING_H__ +#error "interface_impl.h should not be included directly" +#endif // __I_MESSAGING_H__ + +USE_DEBUG_FLAG(D_MESSAGING); + +MessageMetadata::MessageMetadata() +{ + if (!Singleton::exists() || !Singleton::exists()) return; + auto i_agent_details = Singleton::Consume::by(); + auto i_proxy_configuration = Singleton::Consume::by(); + + is_to_fog = true; + host_name = i_agent_details->getFogDomain().ok() ? i_agent_details->getFogDomain().unpack() : ""; + port_num = i_agent_details->getFogPort().ok() ? i_agent_details->getFogPort().unpack() : 0; + + ProxyProtocol protocol = i_agent_details->getSSLFlag() ? ProxyProtocol::HTTPS : ProxyProtocol::HTTP; + + auto maybe_proxy_domain = i_proxy_configuration->getProxyDomain(protocol); + std::string proxy_domain = maybe_proxy_domain.ok() ? *maybe_proxy_domain : ""; + + dbgTrace(D_MESSAGING) << "Created message metadata. Host name: " << host_name << ", Port num: " << port_num; + if (proxy_domain.empty()) return; + + auto maybe_proxy_port = i_proxy_configuration->getProxyPort(protocol); + uint16_t proxy_port = maybe_proxy_port.ok() ? *maybe_proxy_port : 0; + + auto maybe_proxy_auth = i_proxy_configuration->getProxyAuthentication(protocol); + std::string proxy_auth = maybe_proxy_auth.ok() ? *maybe_proxy_auth : ""; + + setProxySettings(MessageProxySettings(proxy_domain, proxy_auth, proxy_port)); + + dbgTrace(D_MESSAGING) << "Proxy : " << proxy_domain << ":" << proxy_port; +} + +template +Maybe +I_Messaging::sendSyncMessage( + HTTPMethod method, + const std::string &uri, + serializableObject &req_obj, + MessageCategory category, + MessageMetadata message_metadata) +{ + Maybe req_body = req_obj.genJson(); + if (!req_body.ok()) { + return genError( + HTTPResponse( + HTTPStatusCode::NO_HTTP_RESPONSE, + "Failed to create a request. Error: " + req_body.getErr() + ) + ); + } + + Maybe response_data = sendSyncMessage( + method, + uri, + req_body.unpack(), + category, + message_metadata + ); + if (!response_data.ok()) return response_data.passErr(); + + auto res_obj = req_obj.loadJson(response_data.unpack().getBody()); + if (!res_obj) { + return genError( + HTTPResponse( + HTTPStatusCode::NO_HTTP_RESPONSE, + "Failed to parse response body. Body: " + response_data.unpack().getBody() + ) + ); + } + return {}; +} + +template +bool +I_Messaging::sendSyncMessageWithoutResponse( + const HTTPMethod method, + const std::string &uri, + serializableObject &req_obj, + const MessageCategory category, + MessageMetadata message_metadata) +{ + Maybe req_body = req_obj.genJson(); + if (!req_body.ok()) { + dbgWarning(D_MESSAGING) << "Failed to create a request. Error: " << req_body.getErr(); + return false; + } + + Maybe response_data = sendSyncMessage( + method, + uri, + req_body.unpack(), + category, + message_metadata + ); + if (!response_data.ok()) { + dbgWarning(D_MESSAGING) + << "Received error from server. Status code: " + << int(response_data.getErr().getHTTPStatusCode()) + << ", error response: " + << response_data.getErr().getBody(); + return false; + } + return true; +} + +template +void +I_Messaging::sendAsyncMessage( + const HTTPMethod method, + const std::string &uri, + serializableObject &req_obj, + const MessageCategory category, + MessageMetadata message_metadata) +{ + Maybe req_body = req_obj.genJson(); + if (!req_body.ok()) { + dbgWarning(D_MESSAGING) << "Failed to create a request. Error: " << req_body.getErr(); + return; + } + + sendAsyncMessage( + method, + uri, + req_body.unpack(), + category, + message_metadata + ); +} + +#endif // __INTERFACE_IMPL_H__ diff --git a/core/include/services_sdk/interfaces/messaging/http_core.h b/core/include/services_sdk/interfaces/messaging/messaging_enums.h similarity index 53% rename from core/include/services_sdk/interfaces/messaging/http_core.h rename to core/include/services_sdk/interfaces/messaging/messaging_enums.h index 7a1cb41..6caa60b 100644 --- a/core/include/services_sdk/interfaces/messaging/http_core.h +++ b/core/include/services_sdk/interfaces/messaging/messaging_enums.h @@ -11,20 +11,45 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef __HTTP_CORE_H__ -#define __HTTP_CORE_H__ +#ifndef __MESSAGING_ENUMS_H__ +#define __MESSAGING_ENUMS_H__ -#include -#include +enum class MessageCategory +{ + GENERIC, + LOG, + DEBUG, + METRIC, + INTELLIGENCE, -#include "maybe_res.h" -#include "cereal/archives/json.hpp" -#include "cereal/types/string.hpp" + COUNT +}; + +enum class MessageConnectionConfig +{ + UNSECURE_CONN, + ONE_TIME_CONN, + IGNORE_SSL_VALIDATION, + + COUNT +}; + +enum class HTTPMethod +{ + GET, + POST, + PATCH, + CONNECT, + PUT, + + COUNT +}; enum class HTTPStatusCode { // 10X - Information responses. Not supported yet. // 20x - Successful responses. + NO_HTTP_RESPONSE = 0, HTTP_OK = 200, HTTP_NO_CONTENT = 204, HTTP_MULTI_STATUS = 207, @@ -51,53 +76,8 @@ enum class HTTPStatusCode HTTP_NOT_EXTENDED = 510, HTTP_NETWORK_AUTHENTICATION_REQUIRED = 511, // Not supported status code. - HTTP_UNKNOWN + HTTP_UNKNOWN = -1, + HTTP_SUSPEND = -2 }; -class HTTPResponse -{ -public: - HTTPResponse(const HTTPStatusCode status_code, const std::string &&body); - - Maybe getResponse() const; - - HTTPStatusCode getStatusCode() const { return status_code; } - std::string getBody() const { return body; } - - class BadRequestResponse - { - public: - void serialize(cereal::JSONInputArchive &ar); - - std::string getMsg() const { return message; } - std::string getID() const { return message_id; } - - private: - std::string message; - std::string message_id; - }; - -private: - HTTPStatusCode status_code; - std::string body; -}; - -class HTTPHeaders -{ -public: - HTTPHeaders() = default; - static Maybe createHTTPHeader(const std::string &http_data); - - void insertHeader(const std::string &header_key, const std::string &header_val); - void insertHeader(const std::string &header); - void insertHeaders(const std::string &headers); - Maybe getHeaderVal(const std::string &header_key); - std::string toString() const; - -private: - HTTPHeaders(const std::string &http_data); - - std::unordered_map headers; -}; - -#endif // __HTTP_CORE_H__ +#endif // __MESSAGING_ENUMS_H__ diff --git a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h new file mode 100644 index 0000000..7318b12 --- /dev/null +++ b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h @@ -0,0 +1,198 @@ +#ifndef __MESSAGING_METADATA_H__ +#define __MESSAGING_METADATA_H__ + +#include +#include + +#include "flags.h" +#include "singleton.h" +#include "i_agent_details.h" + +class MessageProxySettings +{ +public: + MessageProxySettings() {} + + MessageProxySettings(const std::string &_proxy_host, const std::string &_proxy_auth, uint16_t _proxy_port) + : + proxy_host(_proxy_host), proxy_authentication(_proxy_auth), proxy_port(_proxy_port) + {} + + const std::string & + getProxyHost() const + { + return proxy_host; + } + + const std::string & + getProxyAuth() const + { + return proxy_authentication; + } + + uint16_t + getProxyPort() const + { + return proxy_port; + } + + template + void + serialize(Archive &ar) + { + ar( + cereal::make_nvp("proxy_host", proxy_host), + cereal::make_nvp("proxy_authentication", proxy_authentication), + cereal::make_nvp("proxy_port", proxy_port) + ); + } + +private: + std::string proxy_host = ""; + std::string proxy_authentication = ""; + uint16_t proxy_port = 0; +}; + +class MessageMetadata +{ +public: + inline MessageMetadata(); + + MessageMetadata(const std::string &_host_name, uint16_t _port_num, bool _buffer = false, bool _fog = false) : + host_name(_host_name), port_num(_port_num), should_buffer(_buffer), is_to_fog(_fog) + {} + + MessageMetadata( + std::string _host_name, + uint16_t _port_num, + Flags _conn_flags, + bool _should_buffer = false, + bool _is_to_fog = false + ) : + host_name(_host_name), + port_num(_port_num), + conn_flags(_conn_flags), + should_buffer(_should_buffer), + is_to_fog(_is_to_fog) + {} + + const std::string & + getHostName() const + { + return host_name; + } + + const uint16_t & + getPort() const + { + return port_num; + } + + void + setConnectioFlag(MessageConnectionConfig flag) + { + conn_flags.setFlag(flag); + } + + const Flags & + getConnectionFlags() const + { + return conn_flags; + } + + const MessageProxySettings & + getProxySettings() const + { + return proxy_settings; + } + + const std::string & + getExternalCertificate() const + { + return external_certificate; + } + + const std::map & + getHeaders() const + { + return headers; + } + + void + insertHeader(const std::string &header_key, const std::string &header_val) + { + headers[header_key] = header_val; + } + + void + insertHeaders(const std::map &_headers) + { + headers.insert(_headers.begin(), _headers.end()); + } + + void + setProxySettings(const MessageProxySettings &_proxy_settings) + { + proxy_settings = _proxy_settings; + is_proxy_set = true; + } + + void + setExternalCertificate(const std::string &_external_certificate) + { + external_certificate = _external_certificate; + } + + void + setShouldBufferMessage(bool _should_buffer) + { + should_buffer = _should_buffer; + } + + bool + shouldBufferMessage() const + { + return should_buffer; + } + + bool + isProxySet() const + { + return is_proxy_set; + } + + bool + isToFog() const + { + return is_to_fog; + } + + template + void + serialize(Archive &ar) + { + ar( + cereal::make_nvp("host_name", host_name), + cereal::make_nvp("port_num", port_num), + cereal::make_nvp("is_proxy_set", is_proxy_set), + cereal::make_nvp("headers", headers), + cereal::make_nvp("conn_flags", conn_flags), + cereal::make_nvp("external_certificate", external_certificate), + cereal::make_nvp("should_buffer", should_buffer), + cereal::make_nvp("is_to_fog", is_to_fog) + ); + } + +private: + std::string host_name = ""; + uint16_t port_num = 0; + bool is_proxy_set = false; + std::map headers; + Flags conn_flags; + MessageProxySettings proxy_settings; + std::string external_certificate = ""; + bool should_buffer = false; + bool is_to_fog = false; +}; + +#endif // __MESSAGING_METADATA_H__ diff --git a/core/include/services_sdk/interfaces/mock/mock_agent_details.h b/core/include/services_sdk/interfaces/mock/mock_agent_details.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/mock/mock_cpu.h b/core/include/services_sdk/interfaces/mock/mock_cpu.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/mock/mock_environment.h b/core/include/services_sdk/interfaces/mock/mock_environment.h index 5f20cca..3f08bf8 100644 --- a/core/include/services_sdk/interfaces/mock/mock_environment.h +++ b/core/include/services_sdk/interfaces/mock/mock_environment.h @@ -29,6 +29,7 @@ public: MOCK_CONST_METHOD0(getCurrentTrace, std::string()); MOCK_CONST_METHOD0(getCurrentSpan, std::string()); MOCK_METHOD0(getCurrentHeaders, std::string()); + MOCK_METHOD0(getCurrentHeadersMap, std::map()); MOCK_METHOD2(startNewTrace, void(bool, const std::string &)); MOCK_METHOD3(startNewSpan, void(Span::ContextType, const std::string &, const std::string &)); diff --git a/core/include/services_sdk/interfaces/mock/mock_logging.h b/core/include/services_sdk/interfaces/mock/mock_logging.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/mock/mock_messaging.h b/core/include/services_sdk/interfaces/mock/mock_messaging.h old mode 100755 new mode 100644 index 9a56d7e..8060db6 --- a/core/include/services_sdk/interfaces/mock/mock_messaging.h +++ b/core/include/services_sdk/interfaces/mock/mock_messaging.h @@ -2,67 +2,69 @@ #define __MOCK_MESSAGING_H__ #include "i_messaging.h" - #include "cptest.h" + class MockMessaging : public Singleton::Provide::From> { public: using string = std::string; - MOCK_METHOD10( - sendMessage, - Maybe ( - bool, - const string &, - Method, - const string &, - uint16_t, - Flags &, + MOCK_METHOD5( + sendSyncMessage, + Maybe ( + HTTPMethod, const string &, const string &, - I_Messaging::ErrorCB, - MessageTypeTag + MessageCategory, + MessageMetadata + ) + ); + MOCK_METHOD5( + sendAsyncMessage, + void ( + HTTPMethod, + const string &, + const string &, + MessageCategory, + MessageMetadata ) ); - MOCK_METHOD7( - mockSendPersistentMessage, - Maybe(bool, const string &, Method, const string &, const string &, bool, MessageTypeTag) - ); - - Maybe - sendPersistentMessage( - bool get_reply, - const string &&body, - Method method, - const string &url, - const string &headers, - bool should_yield, - MessageTypeTag tag, - bool) - { - return mockSendPersistentMessage(get_reply, body, method, url, headers, should_yield, tag); - } - - MOCK_METHOD8( - sendMessage, - Maybe ( - bool, - const string &, - Method, + MOCK_METHOD5( + downloadFile, + Maybe ( + HTTPMethod, const string &, const string &, - I_Messaging::ErrorCB, - bool, - MessageTypeTag + MessageCategory, + MessageMetadata ) ); - MOCK_METHOD0(setActiveFog, bool()); - MOCK_METHOD1(setActiveFog, bool(MessageTypeTag)); - MOCK_METHOD0(unsetFogProxy, void()); - MOCK_METHOD0(loadFogProxy, void()); - MOCK_METHOD4(setActiveFog, bool(const string &, const uint16_t, const bool, MessageTypeTag)); + MOCK_METHOD4( + uploadFile, + Maybe ( + const string &, + const string &, + MessageCategory, + MessageMetadata + ) + ); + MOCK_METHOD4(setFogConnection, bool(const string &, uint16_t, bool, MessageCategory)); + MOCK_METHOD0(setFogConnection, bool()); + MOCK_METHOD1(setFogConnection, bool(MessageCategory)); }; +static std::ostream & +operator<<(std::ostream &os, const HTTPResponse &) +{ + return os; +} + +static std::ostream & +operator<<(std::ostream &os, const HTTPStatusCode &) +{ + return os; +} + #endif // __MOCK_MESSAGING_H__ diff --git a/core/include/services_sdk/interfaces/mock/mock_proxy_configuration.h b/core/include/services_sdk/interfaces/mock/mock_proxy_configuration.h new file mode 100644 index 0000000..a308aa6 --- /dev/null +++ b/core/include/services_sdk/interfaces/mock/mock_proxy_configuration.h @@ -0,0 +1,23 @@ +#ifndef __MOCK_PROXY_CONFIGURATION_H__ +#define __MOCK_PROXY_CONFIGURATION_H__ + +#include "i_proxy_configuration.h" + +#include "cptest.h" + +class MockProxyConfiguration + : + public Singleton::Provide::From> +{ +public: + using string = std::string; + + MOCK_CONST_METHOD1(getProxyDomain, Maybe(ProxyProtocol protocol)); + MOCK_CONST_METHOD1(getProxyAuthentication, Maybe(ProxyProtocol protocol)); + MOCK_CONST_METHOD1(getProxyPort, Maybe(ProxyProtocol protocol)); + MOCK_CONST_METHOD1(getProxyExists, bool(ProxyProtocol protocol)); + MOCK_CONST_METHOD1(getProxyAddress, Maybe(ProxyProtocol protocol)); + MOCK_METHOD0(loadProxy, Maybe()); +}; + +#endif // __MOCK_PROXY_CONFIGURATION_H__ diff --git a/core/include/services_sdk/interfaces/mock/mock_shell_cmd.h b/core/include/services_sdk/interfaces/mock/mock_shell_cmd.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/mock/mock_socket_is.h b/core/include/services_sdk/interfaces/mock/mock_socket_is.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/mock/mock_table.h b/core/include/services_sdk/interfaces/mock/mock_table.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/interfaces/mock/mock_time_get.h b/core/include/services_sdk/interfaces/mock/mock_time_get.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/agent_details.h b/core/include/services_sdk/resources/agent_details.h old mode 100755 new mode 100644 index 7baa58b..5d026e3 --- a/core/include/services_sdk/resources/agent_details.h +++ b/core/include/services_sdk/resources/agent_details.h @@ -96,7 +96,7 @@ public: void setClusterId(const std::string &_cluster_id); Maybe getProxyDomain(ProxyProtocol protocol) const; - Maybe getProxyCredentials(ProxyProtocol protocol) const; + Maybe getProxyAuthentication(ProxyProtocol protocol) const; Maybe getProxyPort(ProxyProtocol protocol) const; bool getProxyExists(ProxyProtocol protocol) const; Maybe getProxyAddress(ProxyProtocol protocol) const; diff --git a/core/include/services_sdk/resources/component.h b/core/include/services_sdk/resources/component.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/component_is/components_list_impl.h b/core/include/services_sdk/resources/component_is/components_list_impl.h old mode 100755 new mode 100644 index 7ae12bc..c043257 --- a/core/include/services_sdk/resources/component_is/components_list_impl.h +++ b/core/include/services_sdk/resources/component_is/components_list_impl.h @@ -31,7 +31,6 @@ #include "rest_server.h" #include "logging_comp.h" #include "log_generator.h" -#include "proto_message_comp.h" #include "table.h" #include "agent_details.h" #include "encryptor.h" @@ -42,12 +41,12 @@ #include "socket_is.h" #include "generic_rulebase/generic_rulebase.h" #include "generic_rulebase/generic_rulebase_context.h" -#include "messaging_buffer.h" #include "shell_cmd.h" #include "generic_metric.h" #include "tenant_manager.h" #include "buffer.h" #include "intelligence_comp_v2.h" +#include "messaging.h" USE_DEBUG_FLAG(D_COMP_IS); @@ -209,6 +208,7 @@ class ComponentListCore Buffer, ShellCmd, GenericMetric, + Messaging, ConfigComponent, InstanceAwareness, IntelligenceComponentV2, @@ -220,11 +220,9 @@ class ComponentListCore RestServer, Encryptor, SocketIS, - ProtoMessageComp, CPUCalculator, CPUManager, MemoryCalculator, - MessagingBuffer, TenantManager, GenericRulebase, Components... diff --git a/core/include/services_sdk/resources/component_is/node_components_impl.h b/core/include/services_sdk/resources/component_is/node_components_impl.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/components_list.h b/core/include/services_sdk/resources/components_list.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/config/i_config_iterator.h b/core/include/services_sdk/resources/config/i_config_iterator.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/cpu.h b/core/include/services_sdk/resources/cpu.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/cpu/cpu_metric.h b/core/include/services_sdk/resources/cpu/cpu_metric.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/cpu/failopen_mode_status.h b/core/include/services_sdk/resources/cpu/failopen_mode_status.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h old mode 100755 new mode 100644 index bf408c5..9c30422 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -44,8 +44,10 @@ DEFINE_FLAG(D_INFRA, D_ALL) DEFINE_FLAG(D_SOCKET, D_COMMUNICATION) DEFINE_FLAG(D_SYNC, D_COMMUNICATION) DEFINE_FLAG(D_UPGRADE, D_SYNC) - DEFINE_FLAG(D_EVENT_BUFFER, D_COMMUNICATION) - DEFINE_FLAG(D_HTTP_REQUEST, D_COMMUNICATION) + DEFINE_FLAG(D_MESSAGING, D_COMMUNICATION) + DEFINE_FLAG(D_CONNECTION, D_MESSAGING) + DEFINE_FLAG(D_MESSAGING_BUFFER, D_MESSAGING) + DEFINE_FLAG(D_HTTP_REQUEST, D_MESSAGING) DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_PRELOAD, D_COMPONENT) @@ -99,6 +101,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_IPS, D_COMPONENT) DEFINE_FLAG(D_FILE_UPLOAD, D_COMPONENT) DEFINE_FLAG(D_RATE_LIMIT, D_COMPONENT) + DEFINE_FLAG(D_ROLLBACK_TESTING, D_COMPONENT) DEFINE_FLAG(D_PARSER, D_COMPONENT) DEFINE_FLAG(D_WS, D_COMPONENT) @@ -131,7 +134,6 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_NGINX_POLICY, D_ORCHESTRATOR) DEFINE_FLAG(D_GRADUAL_DEPLOYMENT, D_COMPONENT) - DEFINE_FLAG(D_C8_CONTROLLER, D_COMPONENT) DEFINE_FLAG(D_SDWAN, D_COMPONENT) DEFINE_FLAG(D_SDWAN_POLICY, D_SDWAN) DEFINE_FLAG(D_SDWAN_DATA, D_SDWAN) @@ -144,7 +146,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_UPSTREAM_KEEPALIVE, D_REVERSE_PROXY) DEFINE_FLAG(D_FORWARD_PROXY, D_REVERSE_PROXY) - DEFINE_FLAG(D_IDA, D_COMPONENT) + DEFINE_FLAG(D_IDA_SAML, D_COMPONENT) DEFINE_FLAG(D_IOT_NEXT, D_COMPONENT) DEFINE_FLAG(D_IOT_AUXILIARY, D_IOT_NEXT) @@ -171,7 +173,9 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_GEO_DB, D_COMPONENT) DEFINE_FLAG(D_CPVIEW_METRIC_PROVIDER, D_COMPONENT) DEFINE_FLAG(D_GEO_FILTER, D_COMPONENT) - DEFINE_FLAG(D_URL_FILTERING, D_COMPONENT) + DEFINE_FLAG(D_EGRESS_PROTECTION, D_COMPONENT) + DEFINE_FLAG(D_URL_FILTERING, D_EGRESS_PROTECTION) + DEFINE_FLAG(D_ANTIBOT, D_EGRESS_PROTECTION) DEFINE_FLAG(D_L7_ACCESS_CONTROL, D_COMPONENT) DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT) DEFINE_FLAG(D_HORIZON_TELEMETRY, D_COMPONENT) diff --git a/core/include/services_sdk/resources/environment/span.h b/core/include/services_sdk/resources/environment/span.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/environment/trace.h b/core/include/services_sdk/resources/environment/trace.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/environment/tracing_metric.h b/core/include/services_sdk/resources/environment/tracing_metric.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/generic_metric.h b/core/include/services_sdk/resources/generic_metric.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/health_check_status/health_check_status.h b/core/include/services_sdk/resources/health_check_status/health_check_status.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/intelligence_filter.h b/core/include/services_sdk/resources/intelligence_filter.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/intelligence_is/data_string.h b/core/include/services_sdk/resources/intelligence_is/data_string.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/intelligence_is/read_attribute_impl.h b/core/include/services_sdk/resources/intelligence_is/read_attribute_impl.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/intelligence_is_v2/data_string_v2.h b/core/include/services_sdk/resources/intelligence_is_v2/data_string_v2.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/intelligence_is_v2/data_vector_v2.h b/core/include/services_sdk/resources/intelligence_is_v2/data_vector_v2.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/intelligence_is_v2/read_attribute_v2_impl.h b/core/include/services_sdk/resources/intelligence_is_v2/read_attribute_v2_impl.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/log_generator.h b/core/include/services_sdk/resources/log_generator.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/memory_consumption.h b/core/include/services_sdk/resources/memory_consumption.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/metric/all_metric_event.h b/core/include/services_sdk/resources/metric/all_metric_event.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/metric/average.h b/core/include/services_sdk/resources/metric/average.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/metric/counter.h b/core/include/services_sdk/resources/metric/counter.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/metric/last_reported_value.h b/core/include/services_sdk/resources/metric/last_reported_value.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/metric/max.h b/core/include/services_sdk/resources/metric/max.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/metric/metric_calc.h b/core/include/services_sdk/resources/metric/metric_calc.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/metric/metric_map.h b/core/include/services_sdk/resources/metric/metric_map.h index e3be1c8..d4cfd1e 100644 --- a/core/include/services_sdk/resources/metric/metric_map.h +++ b/core/include/services_sdk/resources/metric/metric_map.h @@ -27,16 +27,30 @@ namespace MetricCalculations template class MetricMap : public MetricCalc { - class InnerMap : public std::map + class InnerMap { public: void save(cereal::JSONOutputArchive &ar) const { - for (auto &metric : *this) { + for (auto &metric : inner_map) { metric.second.save(ar); } } + + std::pair::iterator, bool> + emplace(const std::string &key, Metric &&metric) + { + return inner_map.emplace(key, std::move(metric)); + } + + void clear() { inner_map.clear(); } + + typename std::map::const_iterator begin() const { return inner_map.begin(); } + typename std::map::const_iterator end() const { return inner_map.end(); } + + private: + std::map inner_map; }; public: diff --git a/core/include/services_sdk/resources/metric/min.h b/core/include/services_sdk/resources/metric/min.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/metric/top_values.h b/core/include/services_sdk/resources/metric/top_values.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/read_attribute.h b/core/include/services_sdk/resources/read_attribute.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/read_attribute_v2.h b/core/include/services_sdk/resources/read_attribute_v2.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/report/base_field.h b/core/include/services_sdk/resources/report/base_field.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/report/log_rest.h b/core/include/services_sdk/resources/report/log_rest.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/report/report.h b/core/include/services_sdk/resources/report/report.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/report/report_bulks.h b/core/include/services_sdk/resources/report/report_bulks.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/resources/report/report_enums.h b/core/include/services_sdk/resources/report/report_enums.h old mode 100755 new mode 100644 index 5b7fe1c..796e0ca --- a/core/include/services_sdk/resources/report/report_enums.h +++ b/core/include/services_sdk/resources/report/report_enums.h @@ -155,8 +155,8 @@ enum class IssuingEngine { IOT_NEXT, SDWAN, FILE_UPLOAD, - IDA_NEXT_BLADE_REGISTRATION, - IDA_NEXT_CLIENT_IP_NOTIFY, + IDA_SAML_IDN_BLADE_REGISTRATION, + IDA_SAML_IDN_CLIENT_IP_NOTIFY, HORIZON_TELEMETRY_METRICS, API_DISCOVERY }; diff --git a/core/include/services_sdk/utilities/agent_core_utilities.h b/core/include/services_sdk/utilities/agent_core_utilities.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/utilities/connkey.h b/core/include/services_sdk/utilities/connkey.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/utilities/customized_cereal_map.h b/core/include/services_sdk/utilities/customized_cereal_map.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/utilities/customized_cereal_multimap.h b/core/include/services_sdk/utilities/customized_cereal_multimap.h old mode 100755 new mode 100644 diff --git a/core/include/services_sdk/utilities/flags.h b/core/include/services_sdk/utilities/flags.h index 0938c38..bc9931f 100644 --- a/core/include/services_sdk/utilities/flags.h +++ b/core/include/services_sdk/utilities/flags.h @@ -18,7 +18,7 @@ #include "cereal/types/bitset.hpp" -template (EnumClass::COUNT) + 1> +template (EnumClass::COUNT) + 1> class Flags { static_assert(NumberOfValues > 0, "Number of possible Flags must be positive"); @@ -39,7 +39,7 @@ public: void serialize(Archive &ar, uint32_t) { ar(flags); } private: - uint getIndex(const EnumClass &flag) const { return static_cast(flag); } + uint32_t getIndex(const EnumClass &flag) const { return static_cast(flag); } std::bitset flags; }; diff --git a/core/include/services_sdk/utilities/sasal.h b/core/include/services_sdk/utilities/sasal.h deleted file mode 100755 index 24ca1e7..0000000 --- a/core/include/services_sdk/utilities/sasal.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __FW_INCLUDE_SASAL_H__ -#define __FW_INCLUDE_SASAL_H__ - -// Comment this out if no SASAL is allowed. -#define SASAL_ALLOWED - -#if defined(SASAL_ALLOWED) -#define SASAL_START -#define SASAL_END -#else // SASAL_ALLOWED -#define SASAL_START ERROR: SASAL NOT ALLOWED -#define SASAL_END ERROR: SASAL NOT ALLOWED -#endif // SASAL_ALLOWED - -#endif // __FW_INCLUDE_SASAL_H__ diff --git a/core/include/services_sdk/utilities/socket_is.h b/core/include/services_sdk/utilities/socket_is.h old mode 100755 new mode 100644 diff --git a/core/instance_awareness/instance_awareness_ut/CMakeLists.txt b/core/instance_awareness/instance_awareness_ut/CMakeLists.txt index 38bd543..9a4d45d 100644 --- a/core/instance_awareness/instance_awareness_ut/CMakeLists.txt +++ b/core/instance_awareness/instance_awareness_ut/CMakeLists.txt @@ -1,5 +1,5 @@ add_unit_test( instance_awareness_ut "instance_awareness_ut.cc" - "instance_awareness;environment;metric;event_is;-lboost_regex;agent_core_utilities" + "instance_awareness;messaging;environment;metric;event_is;-lboost_regex;agent_core_utilities" ) diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc old mode 100755 new mode 100644 index b5ec870..609c752 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -80,7 +80,7 @@ public: { stringstream res; - res << "{ \"apiVersion\": \"v2\", \"communicationType\": \"sync\", "; + res << "{ \"apiVersion\": \"v2\", \"communicationType\": \"sync\", \"callbackType\": \"invalidation\", "; auto details = Singleton::Consume::by(); res << "\"name\": \"" << details->getAgentId() << "\", "; auto rest = Singleton::Consume::by(); @@ -388,16 +388,14 @@ private: << "Invalidation value: " << (invalidation.genJson().ok() ? invalidation.genJson().unpack() : invalidation.genJson().getErr()); - return message->sendNoReplyObject( - invalidation, - I_Messaging::Method::POST, - server, - *port, - Flags(), + MessageMetadata invalidation_req_md(server, *port); + invalidation_req_md.insertHeaders(getHTTPHeaders()); + return message->sendSyncMessageWithoutResponse( + HTTPMethod::POST, invalidation_uri, - getHTTPHeaders(), - nullptr, - MessageTypeTag::INTELLIGENCE + invalidation, + MessageCategory::INTELLIGENCE, + invalidation_req_md ); } @@ -409,27 +407,29 @@ private: dbgTrace(D_INTELLIGENCE) << "Invalidation value: " << (invalidation.genJson().ok() ? invalidation.genJson().unpack() : invalidation.genJson().getErr()); - - return message->sendNoReplyObject( - invalidation, - I_Messaging::Method::POST, + MessageMetadata global_invalidation_req_md; + global_invalidation_req_md.insertHeaders(getHTTPHeaders()); + return message->sendSyncMessageWithoutResponse( + HTTPMethod::POST, invalidation_uri, - getHTTPHeaders(), - nullptr, - true, - MessageTypeTag::INTELLIGENCE + invalidation, + MessageCategory::INTELLIGENCE, + global_invalidation_req_md ); } - string + map getHTTPHeaders() const { + map headers; auto details = Singleton::Consume::by(); auto tenant = details->getTenantId(); if (tenant == "") tenant = "Global"; + headers["X-Tenant-Id"] = tenant; auto agent = details->getAgentId(); + headers["X-Source-Id"] = agent; - return "X-Tenant-Id: " + tenant + "\r\nX-Source-Id: " + agent; + return headers; } bool @@ -472,17 +472,13 @@ private: } dbgTrace(D_INTELLIGENCE) << "Invalidation value: " << registration.genJson(); - - return message->sendNoReplyObject( - registration, - I_Messaging::Method::POST, - server, - *port, - Flags(), + MessageMetadata registration_req_md(server, *port); + return message->sendSyncMessageWithoutResponse( + HTTPMethod::POST, registration_uri, - "", - nullptr, - MessageTypeTag::INTELLIGENCE + registration, + MessageCategory::INTELLIGENCE, + registration_req_md ); } @@ -497,7 +493,7 @@ private: OfflineIntelligeceHandler offline_intelligence; bool offline_mode_only = false; InvalidationCallBack invalidations; - I_Messaging *message = nullptr; + I_Messaging *message = nullptr; I_TimeGet *timer = nullptr; I_MainLoop *mainloop = nullptr; }; diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/CMakeLists.txt b/core/intelligence_is_v2/intelligence_is_v2_ut/CMakeLists.txt index 8ef0668..b721e52 100755 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/CMakeLists.txt +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/CMakeLists.txt @@ -4,4 +4,4 @@ link_directories(${BOOST_ROOT}/lib) add_unit_test( intelligence_is_v2_ut "query_request_v2_ut.cc;query_response_v2_ut.cc;intelligence_comp_v2_ut.cc" - "intelligence_is_v2;singleton;shell_cmd;event_is;metric;message;agent_details;connkey;-lboost_regex") + "intelligence_is_v2;singleton;shell_cmd;event_is;metric;messaging;agent_details;connkey;-lboost_regex") diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc old mode 100755 new mode 100644 index 0dc746f..57e376e --- a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc @@ -142,10 +142,9 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest) "}\n" ); - EXPECT_CALL( - messaging_mock, - sendMessage(true, _, I_Messaging::Method::POST, _, _, _, _, MessageTypeTag::INTELLIGENCE) - ).WillOnce(Return(response_str)); + EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str))); + auto maybe_ans = intell->queryIntelligence(request); EXPECT_TRUE(maybe_ans.ok()); auto vec = maybe_ans.unpack(); @@ -285,10 +284,9 @@ TEST_F(IntelligenceComponentTestV2, multiAssetsIntelligenceTest) "}\n" ); - EXPECT_CALL( - messaging_mock, - sendMessage(true, _, I_Messaging::Method::POST, _, _, _, _, MessageTypeTag::INTELLIGENCE) - ).WillOnce(Return(response_str1)); + EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str1))); + auto maybe_ans = intell->queryIntelligence(request); EXPECT_TRUE(maybe_ans.ok()); auto vec = maybe_ans.unpack(); @@ -415,10 +413,9 @@ TEST_F(IntelligenceComponentTestV2, inProgressQueryTest) "}\n" ); - EXPECT_CALL( - messaging_mock, - sendMessage(true, _, I_Messaging::Method::POST, _, _, _, _, MessageTypeTag::INTELLIGENCE) - ).WillOnce(Return(in_progress_response_str)).WillOnce(Return(done_response_str)); + EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, in_progress_response_str)) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, done_response_str))); EXPECT_CALL( mock_ml, @@ -605,10 +602,8 @@ TEST_F(IntelligenceComponentTestV2, pagingQueryTest) "}\n" ); - EXPECT_CALL( - messaging_mock, - sendMessage(true, _, I_Messaging::Method::POST, _, _, _, _, MessageTypeTag::INTELLIGENCE) - ).WillOnce(Return(paging_in_progress_response_str1)); + EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, paging_in_progress_response_str1))); request.setAssetsLimit(2); EXPECT_EQ(request.getAssetsLimit(), 2); @@ -618,10 +613,8 @@ TEST_F(IntelligenceComponentTestV2, pagingQueryTest) EXPECT_EQ(vec1.size(), 1); EXPECT_EQ(request.isPagingFinished(), false); - EXPECT_CALL( - messaging_mock, - sendMessage(true, _, I_Messaging::Method::POST, _, _, _, _, MessageTypeTag::INTELLIGENCE) - ).WillOnce(Return(paging_in_progress_response_str2)); + EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, paging_in_progress_response_str2))); auto maybe_ans2 = intell->queryIntelligence(request); EXPECT_TRUE(maybe_ans2.ok()); @@ -629,10 +622,9 @@ TEST_F(IntelligenceComponentTestV2, pagingQueryTest) EXPECT_EQ(vec2.size(), 2); EXPECT_EQ(request.isPagingFinished(), false); - EXPECT_CALL( - messaging_mock, - sendMessage(true, _, I_Messaging::Method::POST, _, _, _, _, MessageTypeTag::INTELLIGENCE) - ).WillOnce(Return(paging_done_response_str)); + EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, paging_done_response_str))); + auto maybe_ans3 = intell->queryIntelligence(request); EXPECT_TRUE(maybe_ans3.ok()); auto vec3 = maybe_ans3.unpack(); @@ -840,10 +832,9 @@ TEST_F(IntelligenceComponentTestV2, bulkOnlineIntelligenceTest) "}\n" ); Debug::setNewDefaultStdout(&cout); - EXPECT_CALL( - messaging_mock, - sendMessage(true, _, I_Messaging::Method::POST, _, _, _, _, MessageTypeTag::INTELLIGENCE) - ).WillOnce(Return(response_str)); + EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str))); + auto maybe_ans = intell->queryIntelligence(requests); EXPECT_TRUE(maybe_ans.ok()); auto vec = maybe_ans.unpack(); @@ -1004,11 +995,9 @@ TEST_F(IntelligenceComponentTestV2, ignoreInProgressQueryTest_2) "}\n" ); - EXPECT_CALL( - messaging_mock, - sendMessage(true, _, I_Messaging::Method::POST, _, _, _, _, MessageTypeTag::INTELLIGENCE)) - .WillOnce(Return(paging_in_progress_response_str)) - .WillOnce(Return(paging_done_response_str)); + EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _)) + .WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, paging_in_progress_response_str))) + .WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, paging_done_response_str))); I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); QueryRequest request(Condition::EQUALS, "category", "cloud", true, AttributeKeyType::NONE); diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc index d3e8996..8cd7763 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc @@ -124,6 +124,9 @@ public: getListeningPort() ).WillRepeatedly(Return(7000)); + ON_CALL(mock_details, getFogDomain()).WillByDefault(Return(Maybe(string("fog_domain.com")))); + ON_CALL(mock_details, getFogPort()).WillByDefault(Return(Maybe(443))); + conf.preload(); intelligence.preload(); intelligence.init(); @@ -175,8 +178,12 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation) string invalidation_json; EXPECT_CALL( messaging_mock, - sendMessage(false, _, I_Messaging::Method::POST, invalidation_uri, _, _, true, MessageTypeTag::INTELLIGENCE) - ).WillOnce(DoAll(SaveArg<1>(&invalidation_json), Return(string()))); + sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(DoAll( + SaveArg<2>(&invalidation_json), + Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")) + )); + EXPECT_TRUE(invalidation.report(i_intelligence)); string expected_json = @@ -216,8 +223,12 @@ TEST_F(IntelligenceInvalidation, sending_private_invalidation) string invalidation_json; EXPECT_CALL( messaging_mock, - sendMessage(false, _, I_Messaging::Method::POST, "127.0.0.1", 9090, _, invalidation_uri, _, _, _) - ).WillOnce(DoAll(SaveArg<1>(&invalidation_json), Return(string()))); + sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(DoAll( + SaveArg<2>(&invalidation_json), + Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")) + )); + EXPECT_TRUE(invalidation.report(i_intelligence)); string expected_json = @@ -256,10 +267,10 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation) string body; EXPECT_CALL( messaging_mock, - sendMessage(_, _, _, "127.0.0.1", 9090, _, "/api/v2/intelligence/invalidation/register", _, _, _) + sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _) ).WillOnce(DoAll( - SaveArg<1>(&body), - Return(string()) + SaveArg<2>(&body), + Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")) )); EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0); @@ -292,8 +303,8 @@ TEST_F(IntelligenceInvalidation, invalidation_callback) EXPECT_CALL( messaging_mock, - sendMessage(_, _, _, "127.0.0.1", 9090, _, "/api/v2/intelligence/invalidation/register", _, _, _) - ).WillOnce(Return(string())); + sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0); @@ -336,8 +347,8 @@ TEST_F(IntelligenceInvalidation, delete_invalidation_callback) EXPECT_CALL( messaging_mock, - sendMessage(_, _, _, "127.0.0.1", 9090, _, "/api/v2/intelligence/invalidation/register", _, _, _) - ).WillOnce(Return(string())); + sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); auto callback_id = i_intelligence->registerInvalidation(invalidation, callback); i_intelligence->unregisterInvalidation(*callback_id); @@ -380,8 +391,9 @@ TEST_F(IntelligenceInvalidation, invalidation_short_handling) EXPECT_CALL( messaging_mock, - sendMessage(_, _, _, "127.0.0.1", 9090, _, "/api/v2/intelligence/invalidation/register", _, _, _) - ).WillOnce(Return(string())); + sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); + invalidation.startListening(i_intelligence, callback); invalidation.stopListening(i_intelligence); @@ -426,18 +438,18 @@ TEST_F(IntelligenceInvalidation, routine_registration) EXPECT_CALL( messaging_mock, - sendMessage(_, _, _, "127.0.0.1", 9090, _, "/api/v2/intelligence/invalidation/register", _, _, _) - ).WillOnce(Return(string())); + sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _) + ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); i_intelligence->registerInvalidation(invalidation, callback); string body; EXPECT_CALL( messaging_mock, - sendMessage(_, _, _, "127.0.0.1", 9090, _, "/api/v2/intelligence/invalidation/register", _, _, _) + sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _) ).WillOnce(DoAll( - SaveArg<1>(&body), - Return(string()) + SaveArg<2>(&body), + Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")) )); routine(); diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc old mode 100755 new mode 100644 index a64f6da..7e92420 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc @@ -78,6 +78,52 @@ TEST(QueryRequestTestV2, QueryTest) request2.saveToJson(out_ar2); } EXPECT_EQ(out2.str(), output_json2); + + vector range1; + range1.push_back("224.10.10.16"); + range1.push_back("224.10.10.31"); + + vector range2; + range2.push_back("224.11.10.16"); + range2.push_back("224.11.10.31"); + + QueryRequest request3(Condition::RANGE, "ipv4AddressesRange", range1, true); + request3.addCondition(Condition::RANGE, "ipv4AddressesRange", range2); + + string output_json3= + "{\n" + " \"limit\": 20,\n" + " \"fullResponse\": true,\n" + " \"query\": {\n" + " \"operator\": \"and\",\n" + " \"operands\": [\n" + " {\n" + " \"operator\": \"range\",\n" + " \"key\": \"mainAttributes.ipv4AddressesRange\",\n" + " \"value\": [\n" + " \"224.10.10.16\",\n" + " \"224.10.10.31\"\n" + " ]\n" + " },\n" + " {\n" + " \"operator\": \"range\",\n" + " \"key\": \"mainAttributes.ipv4AddressesRange\",\n" + " \"value\": [\n" + " \"224.11.10.16\",\n" + " \"224.11.10.31\"\n" + " ]\n" + " }\n" + " ]\n" + " }\n" + "}"; + + + stringstream out3; + { + cereal::JSONOutputArchive out_ar3(out3); + request3.saveToJson(out_ar3); + } + EXPECT_EQ(out3.str(), output_json3); } TEST(QueryRequestTestV2, AttributesTest) diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc old mode 100755 new mode 100644 diff --git a/core/intelligence_is_v2/intelligence_types_v2.cc b/core/intelligence_is_v2/intelligence_types_v2.cc old mode 100755 new mode 100644 index 0703b5e..94930c1 --- a/core/intelligence_is_v2/intelligence_types_v2.cc +++ b/core/intelligence_is_v2/intelligence_types_v2.cc @@ -38,6 +38,7 @@ Intelligence_IS_V2::convertConditionTypeToString(const Condition &condition_type {Condition::NOT_IN, "notIn"}, {Condition::GREATER_THAN, "greaterThan"}, {Condition::LESS_THAN, "lessThan"}, + {Condition::RANGE, "range"}, }; auto condition_str = condition_type_to_string_map.find(condition_type); diff --git a/core/intelligence_is_v2/query_filter_v2.cc b/core/intelligence_is_v2/query_filter_v2.cc old mode 100755 new mode 100644 index 8d2bec3..847d1a0 --- a/core/intelligence_is_v2/query_filter_v2.cc +++ b/core/intelligence_is_v2/query_filter_v2.cc @@ -65,6 +65,16 @@ SerializableQueryFilter::SerializableQueryFilter( condition_operands.emplace_back(condition_type, key, value); } + +SerializableQueryFilter::SerializableQueryFilter( + Condition condition_type, + const std::string &key, + const vector &value +) { + condition_operands.emplace_back(condition_type, key, value); +} + + void SerializableQueryFilter::save(cereal::JSONOutputArchive &ar) const { @@ -97,6 +107,17 @@ SerializableQueryFilter::addCondition(Condition condition_type, const string &ke condition_operands.emplace_back(condition_type, key, value); } +void +SerializableQueryFilter::addCondition(Condition condition_type, const string &key, const vector &value) +{ + if (queries_operands.size() > 0) { + queries_operands.emplace_back(condition_type, key, value); + return; + } + if (condition_operands.size() == 1 && operator_type == Operator::NONE) operator_type = Operator::AND; + condition_operands.emplace_back(condition_type, key, value); +} + void SerializableQueryFilter::saveCondition(cereal::JSONOutputArchive &ar) const { diff --git a/core/intelligence_is_v2/query_request_v2.cc b/core/intelligence_is_v2/query_request_v2.cc old mode 100755 new mode 100644 index 38d3184..fe95566 --- a/core/intelligence_is_v2/query_request_v2.cc +++ b/core/intelligence_is_v2/query_request_v2.cc @@ -23,7 +23,7 @@ using namespace Intelligence_IS_V2; USE_DEBUG_FLAG(D_INTELLIGENCE); -static const EnumArray object_type_to_string_array{ "asset", "zone", "configuration" }; +static const EnumArray object_type_to_string_array{"asset", "zone", "configuration", "shortLived"}; BulkQueryRequest::BulkQueryRequest(QueryRequest &_request, int _index) : @@ -70,6 +70,18 @@ QueryRequest::QueryRequest( full_response = full_reponse; } +QueryRequest::QueryRequest( + Condition condition_type, + const string &key, + const vector &value, + bool full_reponse, + AttributeKeyType attribute_type +) { + query = SerializableQueryFilter(condition_type, createAttributeString(key, attribute_type), value); + assets_limit = default_assets_limit; + full_response = full_reponse; +} + Maybe QueryRequest::convertObjectTypeToString() const { @@ -161,6 +173,16 @@ QueryRequest::addCondition ( query.addCondition(condition_type, createAttributeString(key, attribute_type), value); } +void +QueryRequest::addCondition ( + Condition condition_type, + const string &key, + const vector &value, + AttributeKeyType attribute_type +) { + query.addCondition(condition_type, createAttributeString(key, attribute_type), value); +} + void QueryRequest::setRequestedAttr(const string &attr, AttributeKeyType attr_type) { diff --git a/core/intelligence_is_v2/requested_attributes_v2.cc b/core/intelligence_is_v2/requested_attributes_v2.cc old mode 100755 new mode 100644 diff --git a/core/logging/cef_stream.cc b/core/logging/cef_stream.cc old mode 100755 new mode 100644 diff --git a/core/logging/debug_stream.cc b/core/logging/debug_stream.cc old mode 100755 new mode 100644 diff --git a/core/logging/file_stream.cc b/core/logging/file_stream.cc old mode 100755 new mode 100644 diff --git a/core/logging/fog_stream.cc b/core/logging/fog_stream.cc old mode 100755 new mode 100644 index a78689c..1a47e32 --- a/core/logging/fog_stream.cc +++ b/core/logging/fog_stream.cc @@ -38,7 +38,7 @@ FogStream::sendLog(const Report &log) ctx.registerValue("Obfuscate log field", true); LogRest rest(log); - i_msg->sendObjectWithPersistence(rest, I_Messaging::Method::POST, fog_log_uri, "", true, MessageTypeTag::LOG); + i_msg->sendAsyncMessage(HTTPMethod::POST, fog_log_uri, rest, MessageCategory::LOG); } void @@ -49,16 +49,13 @@ FogStream::sendLog(const LogBulkRest &logs, bool persistence_only) auto fog_log_uri = getConfigurationWithDefault("/api/v1/agents/events/bulk", "Logging", "Fog Log URI"); if (!persistence_only) { - i_msg->sendObjectWithPersistence(logs, I_Messaging::Method::POST, fog_log_uri, "", true, MessageTypeTag::LOG); + i_msg->sendAsyncMessage(HTTPMethod::POST, fog_log_uri, logs, MessageCategory::LOG); } else { - i_msg->sendObjectWithPersistence( - logs, - I_Messaging::Method::POST, + i_msg->sendAsyncMessage( + HTTPMethod::POST, fog_log_uri, - "", - false, - MessageTypeTag::LOG, - true + logs, + MessageCategory::LOG ); } } diff --git a/core/logging/k8s_svc_stream.cc b/core/logging/k8s_svc_stream.cc index 8ad9daf..96f080b 100644 --- a/core/logging/k8s_svc_stream.cc +++ b/core/logging/k8s_svc_stream.cc @@ -32,31 +32,21 @@ K8sSvcStream::~K8sSvcStream() { } -string -K8sSvcStream::genHeader() -{ - return "X-Tenant-Id: " + Singleton::Consume::by()->getTenantId(); -} - void K8sSvcStream::sendLog(const Report &log) { auto svc_host = getConfigurationWithDefault(default_host, "Logging", "K8sSvc Log host"); auto K8sSvc_log_uri = getConfigurationWithDefault(default_log_uri, "Logging", "K8sSvc Log URI"); LogRest rest(log); - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::EXTERNAL); - bool ok = i_msg->sendNoReplyObject( - rest, - I_Messaging::Method::POST, - svc_host, - 80, - conn_flags, + MessageMetadata rest_req_md(svc_host, 80); + rest_req_md.insertHeader("X-Tenant-Id", Singleton::Consume::by()->getTenantId()); + bool ok = i_msg->sendSyncMessageWithoutResponse( + HTTPMethod::POST, K8sSvc_log_uri, - genHeader(), - nullptr, - MessageTypeTag::LOG + rest, + MessageCategory::LOG, + rest_req_md ); if (!ok) { @@ -76,19 +66,15 @@ K8sSvcStream::sendLog(const LogBulkRest &logs, bool persistence_only) auto svc_host = getConfigurationWithDefault(default_host, "Logging", "K8sSvc Log host"); auto K8sSvc_log_uri = getConfigurationWithDefault(default_bulk_uri, "Logging", "K8sSvc Bulk Log URI"); - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::EXTERNAL); - bool ok = i_msg->sendNoReplyObject( - logs, - I_Messaging::Method::POST, - svc_host, - 80, - conn_flags, + MessageMetadata rest_req_md(svc_host, 80); + rest_req_md.insertHeader("X-Tenant-Id", Singleton::Consume::by()->getTenantId()); + bool ok = i_msg->sendSyncMessageWithoutResponse( + HTTPMethod::POST, K8sSvc_log_uri, - genHeader(), - nullptr, - MessageTypeTag::LOG + logs, + MessageCategory::LOG, + rest_req_md ); if (!ok) { diff --git a/core/logging/log_generator.cc b/core/logging/log_generator.cc old mode 100755 new mode 100644 diff --git a/core/logging/log_streams.h b/core/logging/log_streams.h old mode 100755 new mode 100644 index 0948604..0afa819 --- a/core/logging/log_streams.h +++ b/core/logging/log_streams.h @@ -90,7 +90,6 @@ public: void sendLog(const LogBulkRest &logs, bool persistance_only) override; private: - std::string genHeader(); I_Messaging *i_msg = nullptr; }; diff --git a/core/logging/logging.cc b/core/logging/logging.cc old mode 100755 new mode 100644 index a4eac16..f43a208 --- a/core/logging/logging.cc +++ b/core/logging/logging.cc @@ -98,7 +98,7 @@ public: addStream(StreamType type) override { if (streams_preperation.find(type) != streams_preperation.end()) { - dbgWarning(D_REPORT) + dbgDebug(D_REPORT) << "Cannot add second instance of the same stream. Stream type: " << TagAndEnumManagement::convertToString(type); return false; @@ -119,7 +119,7 @@ public: { string log_type = TagAndEnumManagement::convertToString(type); if (streams_preperation.find(type) != streams_preperation.end()) { - dbgWarning(D_REPORT) + dbgDebug(D_REPORT) << "Cannot add second instance of the same stream. Stream type: " << log_type; return false; @@ -319,6 +319,7 @@ LoggingComp::preload() registerExpectedConfiguration("Logging", "Log file name"); registerExpectedConfiguration("Logging", "Log file line separator"); registerExpectedConfiguration("Logging", "Fog Log URI"); + registerExpectedConfiguration("Logging", "K8sSvc Log host"); registerExpectedConfiguration("Logging", "Syslog IP"); registerExpectedConfiguration("Logging", "Syslog port"); registerExpectedConfiguration("Logging", "CEF IP"); diff --git a/core/logging/logging_metric.h b/core/logging/logging_metric.h old mode 100755 new mode 100644 diff --git a/core/logging/logging_ut/CMakeLists.txt b/core/logging/logging_ut/CMakeLists.txt index 0f4ae83..3044a76 100755 --- a/core/logging/logging_ut/CMakeLists.txt +++ b/core/logging/logging_ut/CMakeLists.txt @@ -3,5 +3,5 @@ link_directories(${BOOST_ROOT}/lib) add_unit_test( logging_ut "logging_ut.cc" - "logging;singleton;connkey;rest;report;agent_details;event_is;metric;version;-lboost_regex;" + "logging;singleton;messaging;connkey;rest;report;agent_details;event_is;metric;version;-lboost_regex;" ) diff --git a/core/logging/logging_ut/logging_ut.cc b/core/logging/logging_ut/logging_ut.cc old mode 100755 new mode 100644 index eedf040..447ebbf --- a/core/logging/logging_ut/logging_ut.cc +++ b/core/logging/logging_ut/logging_ut.cc @@ -166,12 +166,14 @@ public: Debug::setUnitTestFlag(D_REPORT, Debug::DebugLevel::DEBUG); Debug::setNewDefaultStdout(&capture_debug); - EXPECT_CALL( - mock_fog_msg, - mockSendPersistentMessage(_, _, _, _, _, _, MessageTypeTag::LOG) - ).WillRepeatedly(DoAll(SaveArg<1>(&body), Return(string()))); + EXPECT_CALL(mock_msg, sendAsyncMessage( + _, + _, + _, + MessageCategory::LOG, + _ + )).WillRepeatedly(SaveArg<2>(&body)); } - ~LogTest() { is_domain = false; @@ -273,9 +275,9 @@ public: return Singleton::Consume::from(config)->loadConfiguration(str_stream); } - StrictMock mock_mainloop; - StrictMock mock_fog_msg; - StrictMock mock_socket_is; + StrictMock mock_mainloop; + StrictMock mock_msg; + StrictMock mock_socket_is; ostringstream capture_debug; LoggingComp log_comp; @@ -738,11 +740,13 @@ TEST_F(LogTest, FogBulkLogs) { loadFakeConfiguration(true); string local_body; - string res("[{\"id\": 1, \"code\": 400, \"message\": \"yes\"}]"); - EXPECT_CALL( - mock_fog_msg, - mockSendPersistentMessage(_, _, _, _, _, _, MessageTypeTag::LOG) - ).WillRepeatedly(DoAll(SaveArg<1>(&local_body), Return(res))); + EXPECT_CALL(mock_msg, sendAsyncMessage( + _, + _, + _, + MessageCategory::LOG, + _ + )).WillRepeatedly(SaveArg<2>(&local_body)); Tags tag1 = Tags::POLICY_INSTALLATION; Tags tag2 = Tags::ACCESS_CONTROL; @@ -799,9 +803,12 @@ TEST_F(LogTest, OfflineK8sSvcTest) string local_body; string res("[{\"id\": 1, \"code\": 400, \"message\": \"yes\"}]"); EXPECT_CALL( - mock_fog_msg, - sendMessage(_, _, _, "open-appsec-tuning-svc", _, _, "/api/v1/agents/events", _, _, MessageTypeTag::LOG) - ).WillRepeatedly(DoAll(SaveArg<1>(&local_body), Return(res))); + mock_msg, + sendSyncMessage(_, "/api/v1/agents/events", _, MessageCategory::LOG, _) + ).WillRepeatedly(DoAll( + SaveArg<2>(&local_body), + Return(HTTPResponse(HTTPStatusCode::HTTP_OK, res)) + )); string str1( "{\n" @@ -846,9 +853,12 @@ TEST_F(LogTest, OfflineK8sSvcBulkLogs) string local_body; string res("[{\"id\": 1, \"code\": 400, \"message\": \"yes\"}]"); EXPECT_CALL( - mock_fog_msg, - sendMessage(_, _, _, "open-appsec-tuning-svc", _, _, "/api/v1/agents/events/bulk", _, _, MessageTypeTag::LOG) - ).WillRepeatedly(DoAll(SaveArg<1>(&local_body), Return(res))); + mock_msg, + sendSyncMessage(_, "/api/v1/agents/events/bulk", _, MessageCategory::LOG, _) + ).WillRepeatedly(DoAll( + SaveArg<2>(&local_body), + Return(HTTPResponse(HTTPStatusCode::HTTP_OK, res)) + )); Tags tag1 = Tags::POLICY_INSTALLATION; Tags tag2 = Tags::ACCESS_CONTROL; @@ -1123,15 +1133,19 @@ TEST(LogTestInstanceAwareness, LogGenInstanceAwareness) ConfigComponent config; StrictMock mock_mainloop; StrictMock mock_timer; - StrictMock mock_fog_msg; StrictMock mock_socket_is; + StrictMock mock_msg; AgentDetails agent_details; LoggingComp log_comp; - EXPECT_CALL( - mock_fog_msg, - mockSendPersistentMessage(_, _, _, _, _, _, MessageTypeTag::LOG) - ).WillRepeatedly(Return(string())); + EXPECT_CALL(mock_msg, sendAsyncMessage( + _, + _, + _, + MessageCategory::LOG, + _ + )).Times(AnyNumber()); + EXPECT_CALL(mock_socket_is, genSocket(_, _, _, _)).WillRepeatedly(Return(1)); EXPECT_CALL(mock_socket_is, closeSocket(_)).Times(AnyNumber()); EXPECT_CALL(mock_mainloop, doesRoutineExist(_)).WillRepeatedly(Return(true)); @@ -1225,14 +1239,17 @@ TEST(LogTestWithoutComponent, RegisterBasicConfig) ConfigComponent config; NiceMock mock_mainloop; NiceMock mock_timer; - StrictMock mock_fog_msg; StrictMock mock_agent_details; + StrictMock mock_msg; EXPECT_CALL(mock_agent_details, getOrchestrationMode()).WillRepeatedly(Return(OrchestrationMode::ONLINE)); - EXPECT_CALL( - mock_fog_msg, - mockSendPersistentMessage(_, _, _, _, _, _, MessageTypeTag::LOG) - ).WillRepeatedly(Return(string())); + EXPECT_CALL(mock_msg, sendAsyncMessage( + _, + _, + _, + MessageCategory::LOG, + _ + )).Times(AnyNumber()); LoggingComp log_comp; log_comp.preload(); @@ -1269,14 +1286,17 @@ TEST(LogTestWithoutComponent, RegisterAdvancedConfig) ConfigComponent config; NiceMock mock_mainloop; NiceMock mock_timer; - StrictMock mock_fog_msg; StrictMock mock_agent_details; + StrictMock mock_msg; EXPECT_CALL(mock_agent_details, getOrchestrationMode()).WillRepeatedly(Return(OrchestrationMode::ONLINE)); - EXPECT_CALL( - mock_fog_msg, - mockSendPersistentMessage(_, _, _, _, _, _, MessageTypeTag::LOG) - ).WillRepeatedly(Return(string())); + EXPECT_CALL(mock_msg, sendAsyncMessage( + _, + _, + _, + MessageCategory::LOG, + _ + )).Times(AnyNumber()); LoggingComp log_comp; log_comp.preload(); @@ -1340,10 +1360,14 @@ TEST_F(LogTest, BulkModification) { string local_body; string res("[{\"id\": 1, \"code\": 400, \"message\": \"yes\"}]"); - EXPECT_CALL( - mock_fog_msg, - mockSendPersistentMessage(_, _, _, _, _, _, MessageTypeTag::LOG) - ).WillRepeatedly(DoAll(SaveArg<1>(&local_body), Return(res))); + + EXPECT_CALL(mock_msg, sendAsyncMessage( + _, + _, + _, + MessageCategory::LOG, + _ + )).WillRepeatedly(SaveArg<2>(&local_body)); logger->addGeneralModifier(changeOne); logger->addGeneralModifier(changeTwo); diff --git a/core/logging/syslog_stream.cc b/core/logging/syslog_stream.cc old mode 100755 new mode 100644 diff --git a/core/mainloop/mainloop.cc b/core/mainloop/mainloop.cc index 388de22..28639d9 100644 --- a/core/mainloop/mainloop.cc +++ b/core/mainloop/mainloop.cc @@ -198,13 +198,11 @@ MainloopComponent::Impl::reportStartupEvent() LogRest startup_message_client_rest(startup_message); - Singleton::Consume::by()->sendObjectWithPersistence( - startup_message_client_rest, - I_Messaging::Method::POST, + Singleton::Consume::by()->sendAsyncMessage( + HTTPMethod::POST, fog_event_uri, - "", - true, - MessageTypeTag::REPORT + startup_message_client_rest, + MessageCategory::LOG ); dbgInfo(D_MAINLOOP) << "Startup report was successfully sent to fog"; diff --git a/core/mainloop/mainloop_ut/CMakeLists.txt b/core/mainloop/mainloop_ut/CMakeLists.txt index 2a9c399..dd0b43f 100644 --- a/core/mainloop/mainloop_ut/CMakeLists.txt +++ b/core/mainloop/mainloop_ut/CMakeLists.txt @@ -6,6 +6,6 @@ link_directories(${Boost_LIBRARY_DIRS}) add_unit_test( mainloop_ut "mainloop_ut.cc" - "mainloop;singleton;boost_context;rest;metric;event_is;-lboost_regex" + "mainloop;messaging;singleton;boost_context;rest;metric;event_is;-lboost_regex" ) diff --git a/core/mainloop/mainloop_ut/mainloop_ut.cc b/core/mainloop/mainloop_ut/mainloop_ut.cc index a4d81ab..44667d5 100644 --- a/core/mainloop/mainloop_ut/mainloop_ut.cc +++ b/core/mainloop/mainloop_ut/mainloop_ut.cc @@ -34,6 +34,9 @@ public: Debug::setUnitTestFlag(D_MAINLOOP, Debug::DebugLevel::DEBUG); Debug::setNewDefaultStdout(&capture_debug); + + ON_CALL(mock_agent_details, getFogDomain()).WillByDefault(Return(Maybe(string("fog_domain.com")))); + ON_CALL(mock_agent_details, getFogPort()).WillByDefault(Return(Maybe(443))); } ~MainloopTest() @@ -48,14 +51,14 @@ public: { EXPECT_CALL( mock_msg, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, _) + sendAsyncMessage(_, "/api/v1/agents/events", _, _, _) ).Times(2).WillRepeatedly( - WithArgs<1, 6>( + WithArgs<2, 3>( Invoke( - [this](const string &req_body, MessageTypeTag tag) + [this](const string &req_body, MessageCategory tag) { - EXPECT_TRUE(tag == MessageTypeTag::REPORT || tag == MessageTypeTag::METRIC); - if (tag == MessageTypeTag::REPORT) startup_report_body = req_body; + EXPECT_TRUE(tag == MessageCategory::LOG || tag == MessageCategory::METRIC); + if (tag == MessageCategory::LOG) startup_report_body = req_body; static bool should_throw = false; if (should_throw) { should_throw = false; @@ -64,7 +67,7 @@ public: should_throw = true; } - return string(); + return; } ) ) diff --git a/core/memory_consumption/memory_consumption.cc b/core/memory_consumption/memory_consumption.cc old mode 100755 new mode 100644 diff --git a/core/memory_consumption/memory_consumption_ut/CMakeLists.txt b/core/memory_consumption/memory_consumption_ut/CMakeLists.txt index 6f09377..6d6d67a 100755 --- a/core/memory_consumption/memory_consumption_ut/CMakeLists.txt +++ b/core/memory_consumption/memory_consumption_ut/CMakeLists.txt @@ -1,7 +1,7 @@ -link_directories(${BOOST_ROOT}/lib) - -add_unit_test( - memory_consumption_ut - "memory_consumption_ut.cc" - "memory_consumption;event_is;metric;-lboost_regex;" -) +link_directories(${BOOST_ROOT}/lib) + +add_unit_test( + memory_consumption_ut + "memory_consumption_ut.cc" + "memory_consumption;messaging;event_is;metric;-lboost_regex;" +) diff --git a/core/memory_consumption/memory_consumption_ut/memory_consumption_ut.cc b/core/memory_consumption/memory_consumption_ut/memory_consumption_ut.cc old mode 100755 new mode 100644 diff --git a/core/memory_consumption/memory_metric.h b/core/memory_consumption/memory_metric.h old mode 100755 new mode 100644 diff --git a/core/message/CMakeLists.txt b/core/message/CMakeLists.txt deleted file mode 100644 index 9008ea2..0000000 --- a/core/message/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -link_directories(${ng_module_osrc_openssl_path}/lib) -include_directories(${ng_module_osrc_openssl_path}/include) - -add_library(message message.cc http_core.cc http_encoder.cc http_decoder.cc) diff --git a/core/message/http_core.cc b/core/message/http_core.cc deleted file mode 100644 index ed4b3fb..0000000 --- a/core/message/http_core.cc +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "messaging/http_core.h" - -#include -#include - -using namespace std; - -USE_DEBUG_FLAG(D_HTTP_REQUEST); - -HTTPHeaders::HTTPHeaders(const string &http_data) -{ - static const string end_of_headers = "\r\n\r\n"; - if (http_data.find(end_of_headers) == string::npos) throw invalid_argument("Invalid headers"); - - insertHeaders(http_data); -} - -void -HTTPHeaders::insertHeader(const string &_header_key, const string &_header_val) -{ - string header_key = _header_key; - string header_val = _header_val; - // Removing \n from end of the value. - if (header_val.back() == '\n') header_val.pop_back(); - // Removing \r from end of the value. - if (header_val.back() == '\r') header_val.pop_back(); - // Transforming all the keys to lower case. - // RFC 2616 - "Hypertext Transfer Protocol -- HTTP/1.1", Section 4.2, "Message Headers": - // Each header field consists of a name followed by a colon (":") and the field value. - // Field names are case-insensitive. - transform(header_key.begin(), header_key.end(), header_key.begin(), ::tolower); - dbgTrace(D_HTTP_REQUEST) << "Added HTTP header :'" << header_key << ": " << header_val << "'"; - headers[header_key] = move(header_val); -} - -void -HTTPHeaders::insertHeader(const string &header) -{ - if (header.empty()) return; - - auto colon_index = header.find_first_of(":"); - - if (colon_index == string::npos) throw invalid_argument(header + " is invalid headers"); - - auto header_key = header.substr(0, colon_index); - // Including characters of colon and space. - auto header_val = header.substr(colon_index + 2); - insertHeader(header_key, header_val); -} - -void -HTTPHeaders::insertHeaders(const string &headers) -{ - string header; - stringstream ss(headers); - while (getline(ss, header) && header != "\r") { insertHeader(header); } -} - -Maybe -HTTPHeaders::getHeaderVal(const string &header_key) -{ - auto header = headers.find(header_key); - if (header == headers.end()) return genError("Header not found."); - return header->second; -} - -string -HTTPHeaders::toString() const -{ - string headers_as_string; - for_each( - headers.begin(), - headers.end(), - [&headers_as_string] (const pair &header) - { - headers_as_string += header.first + ": " + header.second + "\r\n"; - } - ); - return headers_as_string + "\r\n"; -} - - -Maybe -HTTPHeaders::createHTTPHeader(const string &http_data) -{ - try { - return move(HTTPHeaders(http_data)); - } catch(const std::exception& e) { - return genError(e.what()); - } - dbgAssert(false) << "Failed to create HTTP headers"; - // To justify the compiler - return HTTPHeaders(); -} diff --git a/core/message/http_decoder.cc b/core/message/http_decoder.cc deleted file mode 100755 index a744a37..0000000 --- a/core/message/http_decoder.cc +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "http_decoder.h" - -using namespace std; - -USE_DEBUG_FLAG(D_COMMUNICATION); - -const map errorCodeMapper = { - { HTTPStatusCode::HTTP_OK, "OK" }, - { HTTPStatusCode::HTTP_NO_CONTENT, "No Content" }, - { HTTPStatusCode::HTTP_MULTI_STATUS, "Multi Status" }, - { HTTPStatusCode::HTTP_BAD_REQUEST, "Bad Request" }, - { HTTPStatusCode::HTTP_UNAUTHORIZED, "Unauthorized" }, - { HTTPStatusCode::HTTP_FORBIDDEN, "Forbidden" }, - { HTTPStatusCode::HTTP_NOT_FOUND, "Not Found" }, - { HTTPStatusCode::HTTP_METHOD_NOT_ALLOWED, "Method Not Allowed" }, - { HTTPStatusCode::HTTP_PROXY_AUTHENTICATION_REQUIRED, "Proxy Authentication Required"}, - { HTTPStatusCode::HTTP_REQUEST_TIME_OUT, "Request Timeout" }, - { HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE, "Payload Too Large" }, - { HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error" }, - { HTTPStatusCode::HTTP_NOT_IMPLEMENTED, "Not Implemented" }, - { HTTPStatusCode::HTTP_BAD_GATEWAY, "Bad Gateway" }, - { HTTPStatusCode::HTTP_SERVICE_UNABAILABLE, "Service Unavailable" }, - { HTTPStatusCode::HTTP_GATEWAY_TIMEOUT, "Gateway Timeout" }, - { HTTPStatusCode::HTTP_UNKNOWN, "Not supported." } -}; - -ostream & -operator<<(ostream &os, const HTTPResponse::BadRequestResponse &response) -{ - return os - << "[Message]: " << response.getMsg() << " " - << "[Message-ID]: " << response.getID(); -} - -void -HTTPResponse::BadRequestResponse::serialize(cereal::JSONInputArchive &ar) -{ - ar(cereal::make_nvp("message", message)); - ar(cereal::make_nvp("messageId", message_id)); -} - -HTTPDecoder::HTTPDecoder(I_Messaging::Method _method) - : - method(_method), - status_code(genError("Not received")), - headers(genError("Not received")), - response(), - body() -{ -} - -Maybe -HTTPDecoder::decodeBytes(const string &data) -{ - connection_is_closed = data.empty(); - response += data; - if (!status_code.ok()) status_code = parseStatusLine(); - if (!status_code.ok()) genError("Failed to parse the status line"); - if (!headers.ok()) headers = handleHeaders(); - - if (handleBody()) return HTTPResponse(status_code.unpack(), move(body)); - - return genError("Response not ready!"); -} - -Maybe -HTTPDecoder::handleHeaders() -{ - auto end_of_headers = response.find("\r\n\r\n"); - if (end_of_headers == string::npos) return genError("Headers data not found."); - end_of_headers += 4; - - auto headers = response.substr(0, end_of_headers); - response = response.substr(end_of_headers); - - return HTTPHeaders::createHTTPHeader(headers); -} - -bool -HTTPDecoder::handleBody() -{ - if (!status_code.ok()) return false; - - if (status_code.unpack() == HTTPStatusCode::HTTP_OK) { - if (method == I_Messaging::Method::CONNECT) return true; - } - - if (!headers.ok()) return false; - - body_size += response.size(); - if (status_code.unpack() == HTTPStatusCode::HTTP_NO_CONTENT) { - if (body_size != 0) { - dbgDebug(D_COMMUNICATION) << "Invalid body."; - return false; - } - return true; - } - - auto unpacked_headers = headers.unpack(); - auto content_length = unpacked_headers.getHeaderVal("content-length"); - if (content_length.ok()) { - size_t body_length; - try{ - body_length = stoi(content_length.unpack()); - } catch (const exception& err) { - dbgDebug(D_COMMUNICATION) - << "Failed to convert body length to a number. Body length: " - << content_length.unpack(); - return false; - } - body += response; - response.clear(); - return body_size == body_length; - } - - auto maybe_transfer_encoding = unpacked_headers.getHeaderVal("transfer-encoding"); - if (maybe_transfer_encoding.ok()) { - auto transfer_encoding_type = maybe_transfer_encoding.unpack(); - if (transfer_encoding_type == "chunked") return getChunkedResponse(); - } - - auto connection_header = unpacked_headers.getHeaderVal("connection"); - if (connection_header.ok()) { - if (connection_header.unpack() == "close" && connection_is_closed) { - return true; - } - } - dbgDebug(D_COMMUNICATION) << "Transfer-Encoding method isn't supported."; - return false; -} - -bool -HTTPDecoder::getChunkedResponse() -{ - if(!isLegalChunkedResponse(response)) return false; - - string line; - string res = response; - string chunk_body = ""; - string CRLF = "\r\n"; - size_t chunk_size = 0; - - for (auto end_of_line = res.find(CRLF); end_of_line != string::npos; end_of_line = res.find(CRLF)) { - line = res.substr(0, end_of_line); - try { - chunk_size = stoi(line, nullptr, 16); - } catch (const exception& err) { - dbgDebug(D_COMMUNICATION) << "Failed to convert chunk length to a number. Line: " << line; - return false; - } - - if (end_of_line + 2 + chunk_size > res.length()) { - dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure - chunk-size is bigger than chunk-data"; - return false; - } - chunk_body = res.substr(end_of_line + 2, chunk_size); - res = res.substr(end_of_line + 2 + chunk_size); - - if (res.find(CRLF) != 0) { - dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure - chunk-data missing final CRLF sequence"; - return false; - } - res = res.substr(2); - body += chunk_body; - } - - if (chunk_size != 0) { - dbgDebug(D_COMMUNICATION) << "Invalid chunked data structure - last-chunk of the body is not sized 0"; - return false; - } - return true; -} - -bool -HTTPDecoder::isLegalChunkedResponse(const string &res) -{ - auto end_of_data = res.find("0\r\n\r\n"); - return end_of_data != string::npos && res.length() == end_of_data + 5; -} - -Maybe -HTTPDecoder::parseStatusLine() -{ - auto end_of_first_line = response.find("\r\n"); - if (end_of_first_line == string::npos) return genError("No Status Line was received."); - auto status = response.substr(0, end_of_first_line); - // Removing the status - response = response.substr(end_of_first_line + 2); - // Also status text can be supported at the future. - if (status.find("HTTP/1.") != string::npos) { - int status_num; - try { - status_num = stoi(status.substr(9, 3)); - } catch (const exception& err) { - return genError("Failed to convert status code to a number. Status code: " + status.substr(9, 3)); - } - switch (status_num) - { - case 200: { - return HTTPStatusCode::HTTP_OK; - } - case 204: { - return HTTPStatusCode::HTTP_NO_CONTENT; - } - case 207: { - return HTTPStatusCode::HTTP_MULTI_STATUS; - } - case 400: { - return HTTPStatusCode::HTTP_BAD_REQUEST; - } - case 401: { - return HTTPStatusCode::HTTP_UNAUTHORIZED; - } - case 403: { - return HTTPStatusCode::HTTP_FORBIDDEN; - } - case 404: { - return HTTPStatusCode::HTTP_NOT_FOUND; - } - case 405: { - return HTTPStatusCode::HTTP_METHOD_NOT_ALLOWED; - } - case 408: { - return HTTPStatusCode::HTTP_REQUEST_TIME_OUT; - } - case 413: { - return HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE; - } - case 500: { - return HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR; - } - case 501: { - return HTTPStatusCode::HTTP_NOT_IMPLEMENTED; - } - case 502: { - return HTTPStatusCode::HTTP_BAD_GATEWAY; - } - case 503: { - return HTTPStatusCode::HTTP_SERVICE_UNABAILABLE; - } - case 504: { - return HTTPStatusCode::HTTP_GATEWAY_TIMEOUT; - } - default: { - dbgWarning(D_COMMUNICATION) << "Unknown HTTP status code: " << status_num; - return HTTPStatusCode::HTTP_UNKNOWN; - } - } - } - return genError("Status code not found."); -} - -HTTPResponse::HTTPResponse(const HTTPStatusCode _status_code, const string &&_body) - : - status_code(_status_code), - body(move(_body)) -{ -} - -Maybe -HTTPResponse::getResponse() const -{ - if (status_code == HTTPStatusCode::HTTP_OK || status_code == HTTPStatusCode::HTTP_NO_CONTENT) return body; - - try { - stringstream in; - in << body; - cereal::JSONInputArchive in_ar(in); - BadRequestResponse response_details; - response_details.serialize(in_ar); - dbgWarning(D_COMMUNICATION) << "Response details: " << response_details; - } catch (...) {} - - string status_code_str = to_string(static_cast(status_code)); - auto status_code_message = errorCodeMapper.find(status_code); - if(status_code_message == errorCodeMapper.end()) { - dbgWarning(D_COMMUNICATION) << "Failed to parse HTTP status code message. Status code: " << status_code_str; - return genError(string("Request failed, Status code: ") + status_code_str); - } - return genError(string("Request failed, Error: ") + status_code_str + " " + status_code_message->second); -} diff --git a/core/message/http_decoder.h b/core/message/http_decoder.h deleted file mode 100755 index 23c2bd3..0000000 --- a/core/message/http_decoder.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __HTTP_DECODDER_H__ -#define __HTTP_DECODDER_H__ - -#include - -#include "singleton.h" -#include "maybe_res.h" -#include "messaging/http_core.h" -#include "i_message_decoder.h" -#include "i_messaging.h" -#include "i_env_details.h" - -class HTTPDecoder - : - public I_MessageDecoder , - Singleton::Consume -{ -public: - HTTPDecoder(I_Messaging::Method _method); - - Maybe decodeBytes(const std::string &data) override; - -private: - Maybe handleHeaders(); - Maybe parseStatusLine(); - bool handleBody(); - - bool getChunkedResponse(); - bool isLegalChunkedResponse(const std::string &res); - - I_Messaging::Method method; - bool connection_is_closed = false; - - Maybe status_code; - Maybe headers; - std::string response; - std::string body; - size_t body_size = 0; -}; - -#endif // __HTTP_DECODDER_H__ diff --git a/core/message/http_encoder.cc b/core/message/http_encoder.cc deleted file mode 100755 index 78f858f..0000000 --- a/core/message/http_encoder.cc +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "http_encoder.h" -#include "debug.h" - -using namespace std; - -USE_DEBUG_FLAG(D_COMMUNICATION); - -HTTPRequest::HTTPRequest(const string &_method_statement, const string &_host, const bool to_proxy) - : - method_statement(_method_statement) -{ - if (to_proxy) { - insertHeader("Accept: */*"); - insertHeader("Proxy-Connection: Keep-Alive"); - } - insertHeader("Host", _host); -} - -HTTPRequest::HTTPRequest(const string &_method_statement) - : - method_statement(_method_statement) -{ -} - -HTTPRequest & -HTTPRequest::insertHeader(const string &header_key, const string &header_val) -{ - headers.insertHeader(header_key, header_val); - return *this; -} - -HTTPRequest & -HTTPRequest::insertHeader(const string &header) -{ - try { - headers.insertHeader(header); - } catch(const std::exception& e) { - dbgWarning(D_COMMUNICATION) << "Failed to insert header. Header: " << header; - } - - return *this; -} - -HTTPRequest & -HTTPRequest::insertHeaders(const string &rec_headers) -{ - try { - headers.insertHeaders(rec_headers); - } catch(const std::exception& e) { - dbgWarning(D_COMMUNICATION) << "Failed to insert headers. Headers: " << rec_headers; - } - - return *this; -} - -HTTPRequest & -HTTPRequest::insertBody(const string &reqest_body) -{ - body = reqest_body; - return *this; -} - -string -HTTPRequest::toString() const -{ - string ret = method_statement + "\r\n"; - ret += headers.toString(); - ret += body; - return ret; -} - -ConnectRequest::ConnectRequest(const string &_host, const string &_port) - : - HTTPRequest("CONNECT " + _host + ":" + _port + " HTTP/1.1") -{ - insertHeader("Host", _host + ":" + _port); -} - -PostRequest::PostRequest(const string &_post_path, const string &_host, bool to_proxy) - : - HTTPRequest("POST " + (to_proxy ? "http://" + _host: "") + _post_path + " HTTP/1.1", _host, to_proxy) -{ -} - -PutRequest::PutRequest(const string &_put_path, const string &_host, bool to_proxy) - : - HTTPRequest("PUT " + (to_proxy ? "http://" + _host : "") + _put_path + " HTTP/1.1", _host, to_proxy) -{ -} - -GetRequest::GetRequest(const string &_get_path, const string &_host, bool to_proxy) - : - HTTPRequest("GET " + (to_proxy ? "http://" + _host: "") + _get_path + " HTTP/1.1", _host, to_proxy) -{ -} - -PatchRequest::PatchRequest(const string &_patch_path, const string &_host, bool to_proxy) - : - HTTPRequest("PATCH " + (to_proxy ? "http://" + _host: "") + _patch_path + " HTTP/1.1", _host, to_proxy) -{ -} - -HTTPEncoder::HTTPEncoder::HTTPEncoder(const string &_host, const string &_port) - : - host(_host), - port(_port) -{ -} - -HTTPRequest & -HTTPEncoder::Connect() -{ - request = ConnectRequest(host, port); - request.insertHeader("Proxy-Connection: Keep-Alive"); - return request; -} - -HTTPRequest & -HTTPEncoder::Post(const string &_post_path) -{ - request = PostRequest(_post_path, host, over_proxy && !over_ssl); - return request; -} - -HTTPRequest & -HTTPEncoder::Put(const string &_put_path) -{ - request = PutRequest(_put_path, host, over_proxy && !over_ssl); - return request; -} - -HTTPRequest & -HTTPEncoder::Patch(const string &_patch_path) -{ - request = PatchRequest(_patch_path, host, over_proxy && !over_ssl); - return request; -} - -HTTPRequest & -HTTPEncoder::Get(const string &_get_path) -{ - request = GetRequest(_get_path, host, over_proxy && !over_ssl); - return request; -} - -HTTPEncoder & -HTTPEncoder::isOverProxy() -{ - over_proxy = true; - return *this; -} - -HTTPEncoder & -HTTPEncoder::isOverSSL() -{ - over_ssl = true; - return *this; -}; diff --git a/core/message/message.cc b/core/message/message.cc deleted file mode 100755 index 9b2e790..0000000 --- a/core/message/message.cc +++ /dev/null @@ -1,1793 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "proto_message_comp.h" - -#include -#include -#include -#include -#include -#include "boost/regex.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common.h" -#include "singleton.h" -#include "debug.h" -#include "rest.h" -#include "config.h" -#include "cache.h" -#include "messaging/http_core.h" -#include "http_encoder.h" -#include "http_decoder.h" -#include "agent_details.h" -#include "messaging_buffer/http_request_event.h" -#include "boost/asio.hpp" -#include "message_metric.h" -#include "smart_bio.h" -#include "connkey.h" -#include "agent_core_utilities.h" - -using namespace std; -using namespace chrono; -using namespace smartBIO; - -USE_DEBUG_FLAG(D_COMMUNICATION); -USE_DEBUG_FLAG(D_HTTP_REQUEST); - -static string -tagToString(MessageTypeTag tag) -{ - switch(tag) { - case MessageTypeTag::GENERIC: return "generic"; - case MessageTypeTag::LOG: return "log"; - case MessageTypeTag::DEBUG: return "debug"; - case MessageTypeTag::METRIC: return "metric"; - case MessageTypeTag::REPORT: return "report"; - case MessageTypeTag::WAAP_LEARNING: return "waap learning"; - case MessageTypeTag::INTELLIGENCE: return "intelligence"; - case MessageTypeTag::BUFFERED_MESSAGES: return "buffered messages"; - - case MessageTypeTag::COUNT: break; - } - dbgAssert(false) << "Unsupported message type tag " << static_cast(tag); - return ""; -} - -class MessageConnection -{ -public: - MessageConnection(MessageConnection &&other); - ~MessageConnection(); - // To prevent wrong usage - MessageConnection() = delete; - MessageConnection(const MessageConnection &other) = delete; - MessageConnection& operator=(const MessageConnection &other) = delete; - MessageConnection& operator=(MessageConnection &&other) = delete; - - static Maybe - startNewConnection( - const string &host, - const uint16_t port, - const bool _is_secure, - MessageTypeTag _tag, - const bool over_proxy = false, - bool _is_external = false, - bool is_ssl_ignore_validation = false - ); - - bool lock(); - bool unlock(); - - bool sendData(const string &data) const; - - template - Maybe receiveResponse(I_MessageDecoder &decoder); - - Maybe reconnect(bool should_lock = true); - const string & getHost() const { return host; } - uint16_t getPort() const { return port_num; } - MessageTypeTag getTag() const { return tag; } - bool shouldYieldOnFailure() const { return should_yield_on_failure; } - bool overProxy() const { return over_proxy; } - bool isSecure() const { return is_secure; } - bool isExternal() const { return is_external; } - bool isReady() const; - - void setShouldYieldOnFailure(const bool should_yield) { should_yield_on_failure = should_yield; } - void waitForQueue(); - void releaseQueue(); - - ostream & print(ostream &os) const; - - static I_MainLoop *mainloop; - static I_TimeGet *timer; - static I_Encryptor *encryptor; - static string proxy_host; - static uint16_t proxy_port; - static string proxy_auth; - -private: - MessageConnection( - const string &_host, - const uint16_t _port, - bool _is_secure, - const bool _over_proxy, - bool _is_external, - MessageTypeTag _tag, - bool _is_ssl_ignore_validation = false - ) - : - is_secure(_is_secure), - is_external(_is_external), - over_proxy(_over_proxy), - host(_host), - port_num(_port), - current_messaging_queue_pos(0), - available_messaging_queue_pos(0), - tag(_tag), - is_ssl_ignore_validation(_is_ssl_ignore_validation) - {} - - Maybe doHandshake(const BioUniquePtr &bio); - bool verifyCert(); - bool encrypt(); - bool setCnVerification(); - bool setCtx(); - bool setSocket(); - bool connect(const string &host, const string &overwrite_port); - bool shouldIgnoreSslValidation()const; - bool isBioSocketReady() const; - Maybe calculatePublicKey(const BioUniquePtr &cert) const; - Maybe getPinnedCertificate(); - bool verifyCertPinning(const BioUniquePtr &cert); - Maybe establishConnection(); - Maybe establishConnectionOverProxy(); - void getIpFromHostname(const string &hostname); - static string printData(const string &data); - static uint getConnectionTimeout(); - - bool is_secure; - bool is_external; - bool over_proxy; - bool connection_lock = false; - bool should_yield_on_failure = false; - string ca_chain_dir; - string host; - string pinned_cert_pub_key; - uint16_t port_num; - uint64_t current_messaging_queue_pos = 0; - uint64_t available_messaging_queue_pos = 0; - MessageTypeTag tag; - string filesystem_prefix = ""; - vector current_ips; - bool is_ssl_ignore_validation; - uint connection_closed_count = 0; - -public: - SSL *ssl_socket = nullptr; - BioUniquePtr ssl_ctx = nullptr; - BioUniquePtr bio = nullptr; - static uint64_t metrics_current_size; -}; - -I_MainLoop * MessageConnection::mainloop = nullptr; -I_TimeGet * MessageConnection::timer = nullptr; -I_Encryptor * MessageConnection::encryptor = nullptr; -string MessageConnection::proxy_host = ""; -uint16_t MessageConnection::proxy_port = 0; -string MessageConnection::proxy_auth = ""; -uint64_t MessageConnection::metrics_current_size = 0; - -class ProtoMessageComp::Impl - : - Singleton::Provide::From -{ - using Method = I_Messaging::Method; - using MessageConnKey = tuple; -public: - Impl() : active_connections() {} - - ~Impl() {} - - void - init() - { - initSSL(); - timer = Singleton::Consume::by(); - encryptor = Singleton::Consume::by(); - MessageConnection::encryptor = Singleton::Consume::by(); - msg_buffer = Singleton::Consume::by(); - MessageConnection::timer = Singleton::Consume::by(); - agent_details = Singleton::Consume::by(); - proxy_configuration = Singleton::Consume::by(); - agent_details->readAgentDetails(); - - if (!setActiveFog()) { - dbgDebug(D_COMMUNICATION) << "Could not initialize active fog connection"; - } - - mainloop = Singleton::Consume::by(); - MessageConnection::mainloop = Singleton::Consume::by(); - - auto cache_timeout = getConfigurationWithDefault(2, "message", "Cache timeout"); - cache.startExpiration(seconds(cache_timeout), mainloop, timer); - - auto metrics_debugs_interval = - chrono::seconds(getConfigurationWithDefault( - 600, - "message", - "Metrics Routine Interval" - ) - ); - message_queue_metric.init( - "Message queue elements", - ReportIS::AudienceTeam::AGENT_CORE, - ReportIS::IssuingEngine::AGENT_CORE, - metrics_debugs_interval, - false - ); - message_queue_metric.registerListener(); - - mainloop->addOneTimeRoutine( - I_MainLoop::RoutineType::System, - [&] () - { - while (true) { - if ( - agent_details->getOrchestrationMode() == OrchestrationMode::OFFLINE || - handleBufferedEvents() == 0 - ) { - uint tmo = getConfigurationWithDefault(5, "message", "send event retry in sec"); - mainloop->yield(chrono::seconds(tmo)); - } else { - mainloop->yield(false); - } - } - }, - "Persistent messaging stream", - false - ); - } - - void - fini() - { - MessageConnection::proxy_host = ""; - MessageConnection::proxy_port = 0; - MessageConnection::proxy_auth = ""; - MessageConnection::encryptor = nullptr; - MessageConnection::mainloop = nullptr; - MessageConnection::timer = nullptr; - } - - // LCOV_EXCL_START Reason: No proxy for ut - void - setFogProxy(const string &host, const uint16_t port, ProxyProtocol proto) - { - dbgTrace(D_COMMUNICATION) << "Proxy was set. Proxy: " << host << ":" << port; - MessageConnection::proxy_host = host; - MessageConnection::proxy_port = port; - auto proxy_auth = proxy_configuration->getProxyCredentials(proto); - if (proxy_auth.ok()) { - MessageConnection::proxy_auth = proxy_auth.unpack(); - } - } - // LCOV_EXCL_STOP - - bool - setActiveFog(const string &host, const uint16_t port, bool is_secure, MessageTypeTag tag) override - { - MessageConnKey fog_key = make_tuple("fog", 0, tag); - proxy_protocol = is_secure ? ProxyProtocol::HTTPS : ProxyProtocol::HTTP; - - auto load_env_proxy = proxy_configuration->loadProxy(); - if (!load_env_proxy.ok()) { - dbgDebug(D_COMMUNICATION) - << "Could not initialize load proxy from environment, Error: " - << load_env_proxy.getErr(); - } - - if (proxy_configuration->getProxyExists(proxy_protocol)) { - auto proxy_host = proxy_configuration->getProxyDomain(proxy_protocol); - auto proxy_port = proxy_configuration->getProxyPort(proxy_protocol); - if (proxy_host.ok() && proxy_port.ok()) { - setFogProxy(proxy_host.unpack(), proxy_port.unpack(), proxy_protocol); - } - } - - Maybe conn = MessageConnection::startNewConnection( - host, port, is_secure, tag, proxy_configuration->getProxyExists(proxy_protocol) - ); - if (!conn.ok()) { - dbgWarning(D_COMMUNICATION) - << "Failed to establish connection to the Fog: " - << conn.getErr(); - return false; - } - - if (active_connections.find(fog_key) == active_connections.end()) { - active_connections.emplace(fog_key, conn.unpackMove()); - } - - dbgInfo(D_COMMUNICATION) - << "Successfully connected to the Fog: " - << host - << ":" - << port - << " via " - << (proxy_configuration->getProxyExists(proxy_protocol) ? "proxy, using " : "") - << (is_secure ? "secure" : "clear") - << " connection"; - - tag_to_active_conn_key[tag] = fog_key; - return true; - } - - bool - setActiveFog(MessageTypeTag tag = MessageTypeTag::GENERIC) override - { - string fog_host = ""; - uint16_t fog_port = 0; - bool is_secure_connection = false; - if (agent_details->readAgentDetails()) { - auto domain = agent_details->getFogDomain(); - auto port = agent_details->getFogPort(); - is_secure_connection = agent_details->getSSLFlag(); - if (domain.ok() && port.ok()) { - fog_host = domain.unpack(); - fog_port = port.unpack(); - } - } - - if (agent_details->getOrchestrationMode() == OrchestrationMode::OFFLINE) { - dbgDebug(D_COMMUNICATION) << "Agent Is in offline mode and would not attempt connecting to the fog"; - return true; - } - - if(fog_host.empty() || fog_port == 0) { - dbgWarning(D_COMMUNICATION) - << "Cannot establish connection to the Fog: " - << "failed to get host and port details"; - return false; - } - - return setActiveFog( - fog_host, - fog_port, - is_secure_connection, - tag - ); - } - - string - buildFogHeaders(const string &headers) - { - string modified_headers = headers; - modified_headers += "User-Agent: Infinity Next (a7030abf93a4c13)\r\n"; - auto i_env = Singleton::Consume::by(); - modified_headers += i_env->getCurrentHeaders(); - return modified_headers; - } - - Maybe - sendPersistentMessage( - bool get_reply, - const string &&body, - Method method, - const string &url, - const string &headers, - bool should_yield, - MessageTypeTag tag = MessageTypeTag::GENERIC, - bool skip_sending = false) override - { - if (agent_details->getOrchestrationMode() == OrchestrationMode::OFFLINE) { - return genError("Agent is in offline mode and cannot communicate with the fog"); - } - - string method_as_string; - switch (method) - { - case Method::GET: { - method_as_string = "GET"; - break; - } - case Method::POST: { - method_as_string = "POST"; - break; - } - case Method::PUT: { - method_as_string = "PUT"; - break; - } - case Method::PATCH: { - method_as_string = "PATCH"; - break; - } - case Method::CONNECT: { - method_as_string = "CONNECT"; - break; - } - } - HTTPRequestSignature req_sig(method_as_string, url, tagToString(tag)); - - bool should_buffer = false; - if (pending_signatures.find(req_sig) != pending_signatures.end()) { - dbgDebug(D_COMMUNICATION) << "Previous HTTP Request is already in queue. Buffering the request"; - should_buffer = true; - } - - bool is_rejected = false; - if (!should_buffer && !skip_sending) { - ErrorCB fog_server_err = [&] (HTTPStatusCode http_status_code) mutable - { - is_rejected = - http_status_code == HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE || - http_status_code == HTTPStatusCode::HTTP_MULTI_STATUS || - http_status_code == HTTPStatusCode::HTTP_BAD_REQUEST; - }; - pending_signatures.insert(req_sig); - try { - auto res = sendMessage(get_reply, body, method, url, headers, fog_server_err, should_yield, tag); - pending_signatures.erase(req_sig); - if (res.ok()) return res; - - bool should_buffer_default = getProfileAgentSettingWithDefault( - true, - "eventBuffer.bufferFailedRequests" - ); - if (!getConfigurationWithDefault(should_buffer_default, "message", "Buffer Failed Requests")) { - dbgWarning(D_COMMUNICATION) << "Failed to send Request."; - return res; - } - } catch (...) { - dbgWarning(D_COMMUNICATION) << "Can't send a persistent message, mainloop has been stopped"; - return genError("mainloop has been stopped"); - } - dbgWarning(D_COMMUNICATION) << "Failed to send Request. Buffering the request."; - } - - HTTPRequestEvent request_event(move(req_sig), headers, move(body)); - msg_buffer->bufferNewRequest(request_event, is_rejected); - return genError("HTTP Request is buffered"); - } - - Maybe - sendMessage( - bool get_reply, - const string &body, - Method method, - const string &url, - const string &headers, - ErrorCB err_callback, - bool should_yield, - MessageTypeTag tag = MessageTypeTag::GENERIC) override - { - bool reuse_conns = getConfigurationWithDefault(true, "message", "Reuse connection"); - - if (agent_details->getOrchestrationMode() == OrchestrationMode::OFFLINE) { - return genError("Agent is in offline mode and cannot communicate with the fog"); - } - - if (tag_to_active_conn_key.find(tag) == tag_to_active_conn_key.end()) { - if (!setActiveFog(tag)) { - dbgWarning(D_COMMUNICATION) - << "Connection to fog for tag " - << tagToString(tag) - << " does not exist."; - return genError("Cannot send message to the Fog"); - } - reuse_conns = true; - } - - MessageConnection &curr_conn = active_connections.at(tag_to_active_conn_key[tag]); - - if (!reuse_conns) { - Maybe res = curr_conn.reconnect(); - if(!res.ok()) { - active_connections.erase(tag_to_active_conn_key[tag]); - tag_to_active_conn_key.erase(tag); - return genError( - "Cannot send message after failure in establishing new connection with the fog: " + - res.getErr() - ); - } - } - - ErrorCB fog_server_err = [this, err_callback, &curr_conn] (HTTPStatusCode http_status_code) { - bool is_server_error = ( - http_status_code >= HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR && - http_status_code <= HTTPStatusCode::HTTP_NETWORK_AUTHENTICATION_REQUIRED - ); - - if (is_server_error) { - if (last_fog_server_error == chrono::microseconds(0)) { - last_fog_server_error = timer->getMonotonicTime(); - } - chrono::seconds dead_fog_timeout( - getConfigurationWithDefault( - 300, - "message", - "Internal Fog error timeout" - ) - ); - if (last_fog_server_error + chrono::microseconds(dead_fog_timeout) < timer->getMonotonicTime()) { - curr_conn.reconnect(); - dbgWarning(D_COMMUNICATION) - << "Restarting the Fog connection after Fog error persists for more than " - << dead_fog_timeout.count() - << " seconds"; - last_fog_server_error == chrono::microseconds(0); - } - } - if (err_callback != nullptr) err_callback(http_status_code); - }; - - auto fog_res = sendMessage( - curr_conn, - get_reply, - body, - method, - url, - buildFogHeaders(headers), - fog_server_err, - should_yield - ); - - if (fog_res.ok()) last_fog_server_error = chrono::microseconds(0); - - return fog_res; - } - - Maybe - sendMessage( - bool get_reply, - const string &body, - Method method, - const string &host, - uint16_t port, - Flags &conn_flags, - const string &url, - const string &headers, - ErrorCB err_call_back, - MessageTypeTag tag = MessageTypeTag::GENERIC) override - { - const MessageConnKey key = make_tuple(host, port, tag); - bool is_one_time_conn = conn_flags.isSet(MessageConnConfig::ONE_TIME_CONN); - bool is_secure_conn = conn_flags.isSet(MessageConnConfig::SECURE_CONN); - bool is_external = conn_flags.isSet(MessageConnConfig::EXTERNAL); - bool is_ssl_ignore_validation = conn_flags.isSet(MessageConnConfig::IGNORE_SSL_VALIDATION); - - auto reuse_conns = getConfigurationWithDefault(true, "message", "Reuse connection"); - if (reuse_conns) { - map::iterator conn_iter = active_connections.find(key); - if (conn_iter != active_connections.end()) { - return sendMessage(conn_iter->second, get_reply, body, method, url, headers, err_call_back); - } - } - auto load_env_proxy = proxy_configuration->loadProxy(); - if (!load_env_proxy.ok()) return genError(load_env_proxy.getErr()); - - Maybe conn = MessageConnection::startNewConnection( - host, - port, - is_secure_conn, - tag, - false, - is_external, - is_ssl_ignore_validation - ); - if (!conn.ok()) return conn.passErr(); - - if (!is_one_time_conn) { - active_connections.emplace(key, conn.unpackMove()); - return sendMessage( - active_connections.find(key)->second, - get_reply, - body, - method, - url, - headers, - err_call_back - ); - } - MessageConnection active_conn = conn.unpackMove(); - return sendMessage(active_conn, get_reply, body, method, url, headers, err_call_back); - } - -private: - Maybe - sendMessage( - MessageConnection &conn, - bool get_reply, - const string &body, - Method method, - const string &url, - const string &headers, - ErrorCB err_call_back, - bool should_yield = true) - { - dbgDebug(D_COMMUNICATION) << "Sending a new message"; - - if (conn.getHost() == "") return genError("No host provided"); - - if (mainloop && should_yield) mainloop->yield(false); - - string full_url = conn.getHost() + url; - if (method == Method::GET && cache.doesKeyExists(full_url)) return cache.getEntry(full_url); - - while (!conn.lock()) { mainloop->yield(true); } - - conn.setShouldYieldOnFailure(should_yield); - Maybe response = sendHTTPRequest(conn, body, method, url, headers); - conn.unlock(); - - if (response.ok()) { - auto response_data = response->getResponse(); - if (response_data.ok()) { - if (get_reply && method == Method::GET) cache.emplaceEntry(full_url, response_data.unpack()); - } else { - if (err_call_back != nullptr) err_call_back(response->getStatusCode()); - } - return response_data; - } - number_of_send_failure += 1; - dbgTrace(D_COMMUNICATION) << "Number of a failed attempt to send a message " << number_of_send_failure; - return response.passErr(); - } - - Maybe - sendHTTPRequest( - MessageConnection &conn, - const string &body, - Method method, - const string &url, - const string &headers) - { - auto maybe_data = buildHTTPRequest(method, url, headers, body, conn); - if (!maybe_data.ok()) { - return maybe_data.passErr(); - } - string data = maybe_data.unpack(); - - uint num_of_retries = 0; - const uint max_retries = 2; - while (num_of_retries < max_retries) { - Maybe response = sendMessage(conn, data); - if (response.ok()) return response; - - dbgDebug(D_COMMUNICATION) - << "Failed to send HTTP request, trying to restart the connection. " - << "Error: " - << response.getErr(); - - Maybe connection_result = conn.reconnect(false); - number_of_reconnects += 1; - dbgTrace(D_COMMUNICATION) << "Number of an attempt to reconnect is " << number_of_reconnects; - if (!connection_result.ok()) { - number_of_reconnect_failures += 1; - dbgTrace(D_COMMUNICATION) - << "Number of a failed attempt to reconnect is " - << number_of_reconnect_failures; - return - genError( - string("Failed to reconnect after send request failure. Error: ") + - connection_result.getErr() - ); - } - dbgDebug(D_COMMUNICATION) << "Successfully reconnected after a failure to send a request."; - num_of_retries++; - } - - return genError("Failed to send an HTTP request, reached the maximum number of retries " + max_retries); - } - - Maybe - sendMessage(MessageConnection &conn, const string &data) - { - dbgTrace(D_COMMUNICATION) << "Acquiring connection lock. Connection: " << conn; - - if (!conn.isReady()) { - dbgTrace(D_COMMUNICATION) << "Cannot send data over uninitialized connection"; - return genError("Failed to send HTTP request. The connection is uninitialized."); - } - - conn.waitForQueue(); - auto release_queue_on_exit = make_scope_exit([&conn] () { conn.releaseQueue(); }); - if (conn.sendData(data)) { - return getHttpResponse(conn); - } - - return genError("Failed to send HTTP request"); - } - - string - base64Decode(const string &input) const - { - string out; - vector T(256, -1); - static const string base = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - for (int i = 0; i < 64; i++) { T[base[i]] = i; } - - int val = 0, val_balancer = -8; - for (unsigned char c : input) { - if (T[c] == -1) break; - val = (val << 6) + T[c]; - val_balancer += 6; - if (val_balancer >= 0) { - out.push_back(char((val >> val_balancer) & 0xFF)); - val_balancer -= 8; - } - } - return out; - } - - void - initSSL() - { - SSL_library_init(); - SSL_load_error_strings(); - OpenSSL_add_all_algorithms(); - } - - Maybe - getHttpResponse(MessageConnection &conn) - { - HTTPDecoder http_decoder(Method::GET); - return conn.receiveResponse(http_decoder); - } - - Maybe - buildHTTPRequest( - Method _method, - const string &url, - const string &headers, - const string &body, - const MessageConnection &conn) - { - HTTPRequest req; - HTTPEncoder http_encoder(conn.getHost(), to_string(conn.getPort())); - - if (conn.isSecure()) http_encoder.isOverSSL(); - if (conn.overProxy()) http_encoder.isOverProxy(); - - switch (_method) - { - case Method::GET: { - req = http_encoder.Get(url); - break; - } - case Method::POST: { - req = http_encoder.Post(url); - break; - } - case Method::PATCH: { - req = http_encoder.Patch(url); - break; - } - case Method::PUT: { - req = http_encoder.Put(url); - break; - } - case Method::CONNECT: { - return http_encoder.Connect().toString(); - } - } - - const string &access_token = agent_details->getAccessToken(); - - if (!conn.isExternal() && !access_token.empty() && headers.find("Authorization") == std::string::npos) { - req.insertHeader("Authorization", "Bearer " + access_token); - } - // Proxy-Authorization - if (conn.overProxy() && !conn.isSecure()) { - if (!MessageConnection::proxy_auth.empty()) { - req.insertHeader( - "Proxy-Authorization", - "Basic " + encryptor->base64Encode(MessageConnection::proxy_auth) - ); - } else { - return genError("Failed to authenticate on a proxy with empty token."); - } - } - - req.insertHeader("Content-Length", to_string(body.size())); - req.insertHeader("Content-type: application/json"); - req.insertHeader("Accept-Encoding: identity"); - if(headers.find("Connection:") == std::string::npos){ - req.insertHeader("Connection: keep-alive"); - } - - req.insertHeaders(headers); - req.insertBody(body); - - return req.toString(); - } - - int - handleBufferedEvents() - { - bool should_buffer_default = getProfileAgentSettingWithDefault( - true, - "eventBuffer.bufferFailedRequests" - ); - if (!getConfigurationWithDefault( - should_buffer_default, - "message", - "Buffer Failed Requests" - ) - ) { - return 0; - } - - int count = 0; - Maybe event = genError("empty"); - while ((event = msg_buffer->peekRequest()).ok()) { - dbgTrace(D_COMMUNICATION) << "Trying to send HTTPEvent " << event.unpack().getSignature(); - - bool is_rejected = false; - // LCOV_EXCL_START Reason: We can't check it, since we don't control the response on ut yet - ErrorCB fog_server_err = [&] (HTTPStatusCode http_status_code) mutable - { - is_rejected = - http_status_code == HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE || - http_status_code == HTTPStatusCode::HTTP_MULTI_STATUS || - http_status_code == HTTPStatusCode::HTTP_BAD_REQUEST; - }; - // LCOV_EXCL_STOP - - auto maybe_method = stringToMethod(event.unpack().getMethod()); - if (!maybe_method.ok()) { - dbgTrace(D_COMMUNICATION) << "Failed to sent the buffered request. Error: " << maybe_method.getErr(); - msg_buffer->popRequest(); - count++; - mainloop->yield(false); - continue; - } - - auto resp = sendMessage( - false, - event.unpack().getBody(), - *maybe_method, - event.unpack().getURL(), - event.unpack().getHeaders(), - fog_server_err, - false, - MessageTypeTag::BUFFERED_MESSAGES - ); - - if (resp.ok()) { - dbgTrace(D_COMMUNICATION) << "Successfully sent the buffered request" << event.unpack().getSignature(); - msg_buffer->popRequest(); - count++; - mainloop->yield(false); - } else { - if (!is_rejected) { - dbgWarning(D_COMMUNICATION) << "Failed to send HTTPEvent " << event.unpack().getSignature(); - return count; - } - msg_buffer->popRequest(); - msg_buffer->bufferNewRequest(*event, is_rejected); - dbgWarning(D_COMMUNICATION) << "HTTPEvent " << event.unpack().getSignature() << " was rejected"; - mainloop->yield(false); - } - } - return count; - } - - Maybe - stringToMethod(const string &name) - { - if (name == "GET") return Method::GET; - if (name == "POST") return Method::POST; - if (name == "PATCH") return Method::PATCH; - if (name == "CONNECT") return Method::CONNECT; - if (name == "PUT") return Method::PUT; - - return genError("Cannot convert unknown HTTP method to Enum. Method name: " + name); - } - - bool is_proxy_configured_via_settings = false; - uint64_t number_of_reconnects = 0; - uint64_t number_of_reconnect_failures = 0; - uint64_t number_of_send_failure = 0; - I_AgentDetails *agent_details = nullptr; - I_MainLoop *mainloop = nullptr; - I_TimeGet *timer = nullptr; - I_Encryptor *encryptor = nullptr; - I_MessagingBuffer *msg_buffer = nullptr; - I_ProxyConfiguration *proxy_configuration = nullptr; - map active_connections; - map tag_to_active_conn_key; - ProxyProtocol proxy_protocol; - TemporaryCache cache; - static const map proxyProtocolToString; - set pending_signatures; - chrono::microseconds last_fog_server_error = chrono::microseconds(0); - MessageQueueMetric message_queue_metric; - string filesystem_prefix = ""; -}; - -MessageConnection::MessageConnection(MessageConnection &&other) - : - is_secure(other.is_secure), - is_external(other.is_external), - over_proxy(other.over_proxy), - host(other.host), - pinned_cert_pub_key(other.pinned_cert_pub_key), - port_num(other.port_num), - current_messaging_queue_pos(other.current_messaging_queue_pos), - available_messaging_queue_pos(other.available_messaging_queue_pos), - tag(other.tag), - is_ssl_ignore_validation(other.is_ssl_ignore_validation), - connection_closed_count(other.connection_closed_count), - ssl_socket(move(other.ssl_socket)), - ssl_ctx(move(other.ssl_ctx)), - bio(move(other.bio)) -{ - other.ssl_socket = nullptr; - other.ssl_ctx = nullptr; - other.bio = nullptr; -} - -bool -MessageConnection::lock() -{ - if (connection_lock) return false; - connection_lock = true; - dbgTrace(D_COMMUNICATION) << "The connection lock was taken. Connection: " << this; - return true; -} - -bool -MessageConnection::unlock() -{ - if (!connection_lock) return false; - connection_lock = false; - dbgTrace(D_COMMUNICATION) << "The connection lock was released. Connection: " << *this; - return true; -} - -// LCOV_EXCL_START Reason: No proxy for ut -bool -MessageConnection::shouldIgnoreSslValidation() const -{ - if(is_ssl_ignore_validation) { - dbgTrace(D_COMMUNICATION) << "Ignoring SSL validation"; - return true; - } - - bool ignore_ssl_validation = getProfileAgentSettingWithDefault( - false, - "agent.config.message.ignoreSslValidation"); - - if (ignore_ssl_validation) { - dbgTrace(D_COMMUNICATION) - << "ignoreSslValidation: " - << (ignore_ssl_validation ? "true, Ignoring ssl validation of the current connection" : "false"); - } - - return ignore_ssl_validation; -} -// LCOV_EXCL_START Reason: No proxy for ut - -bool -MessageConnection::verifyCert() -{ - dbgFlow(D_COMMUNICATION); - BioUniquePtr cert = BioUniquePtr(SSL_get_peer_certificate(ssl_socket)); - // In this case cert returned null from SSL_get_peer_certificate - - if (shouldIgnoreSslValidation()) return true; - - if (cert.get() == nullptr) { - dbgWarning(D_COMMUNICATION) << "Server did not provide a certificate during handshake"; - return false; - } - // Verify the result of chain verification - int res = SSL_get_verify_result(ssl_socket); - if (res != X509_V_OK) { - dbgWarning(D_COMMUNICATION) - << "Failed to verify server certificate. OpenSSL error: " - << string(ERR_error_string(res, nullptr)) - << ", OpenSSL error code: " << res; - return false; - } - - auto verify_pining_required = getConfigurationWithDefault(false, "message", "Verify SSL pinning"); - if (verify_pining_required && !verifyCertPinning(cert)) { - dbgWarning(D_COMMUNICATION) << "Couldn't verify server public certificate (pinning)"; - return false; - } - - return true; -} - -Maybe -MessageConnection::calculatePublicKey(const BioUniquePtr &cert) const -{ - if (cert.get() == nullptr) return genError("Certificate is null"); - - BioUniquePtr outbio = BioUniquePtr(BIO_new(BIO_s_mem())); - BioUniquePtr pkey = BioUniquePtr(X509_get_pubkey(cert.get())); - - if (pkey.get() == nullptr) { - return genError("Error getting public key from certificate"); - } - if(!PEM_write_bio_PUBKEY(outbio.get(), pkey.get())) { - return genError("Error writing public key data in PEM format"); - } - - char *buf; - size_t len = BIO_get_mem_data(outbio.get(), &buf); - dbgTrace(D_COMMUNICATION) << "Provide public key has been loaded"; - return move(string(buf, len)); -} - -Maybe -MessageConnection::getPinnedCertificate() -{ - if (!pinned_cert_pub_key.empty()) return pinned_cert_pub_key; - - filesystem_prefix = getFilesystemPathConfig(); - dbgTrace(D_COMMUNICATION) << "MessageConnection, file systen prefix: " << filesystem_prefix << endl; - string public_key_path = - getConfigurationWithDefault( - filesystem_prefix + "/certs/public-key.pem", - "message", - "Public key path" - ); - dbgTrace(D_COMMUNICATION) << "Load public key path. Path: " << public_key_path; - ifstream pinned_public_file(public_key_path); - if (!pinned_public_file.is_open()) { - return genError("Failed to open pinned public key file"); - } - stringstream pinned_public_key_steam; - pinned_public_key_steam << pinned_public_file.rdbuf(); - pinned_cert_pub_key = pinned_public_key_steam.str(); - - dbgTrace(D_COMMUNICATION) << "Pinned public key has been loaded"; - return pinned_cert_pub_key; -} - -bool -MessageConnection::verifyCertPinning(const BioUniquePtr &cert) -{ - dbgFlow(D_COMMUNICATION); - - if (cert.get() == nullptr) { - dbgWarning(D_COMMUNICATION) << "Certificate is missing"; - return false; - } - - auto public_key = calculatePublicKey(cert); - if (!public_key.ok()) { - dbgWarning(D_COMMUNICATION) << "The provided public key is not valid. Error: " << public_key.getErr(); - return false; - } - - auto pinned_key = getPinnedCertificate(); - if (!pinned_key.ok()) { - dbgWarning(D_COMMUNICATION) << "The pinned public key is not valid. Error: " << pinned_key.getErr(); - return false; - } - - if(public_key.unpackMove().compare(pinned_key.unpack()) != 0) { - dbgWarning(D_COMMUNICATION) << "The provided public key and the pinned public key are diffrent"; - return false; - } - - dbgTrace(D_COMMUNICATION) << "The provided public key is valid"; - return true; -} - -Maybe -MessageConnection::doHandshake(const BioUniquePtr &bio) -{ - auto timeout = chrono::microseconds( - getConfigurationWithDefault(500000, "message", "Connection handshake timeout") - ); - auto end_time = timer->getMonotonicTime() + timeout; - while (timer->getMonotonicTime() < end_time) { - if (!isBioSocketReady()) { - dbgDebug(D_COMMUNICATION) << "Socket is not ready for use."; - if (mainloop != nullptr) mainloop->yield(true); - continue; - } - if (BIO_do_handshake(bio.get()) > 0 || shouldIgnoreSslValidation()) { - return Maybe(); - } - if (!BIO_should_retry(bio.get())) { - unsigned long ssl_err = ERR_get_error(); - return genError( - "Failed to obtain a successful SSL handshake. OpenSSL error: " - + string(ERR_error_string(ssl_err, nullptr)) - + ", OpenSSL error code: " - + to_string(ssl_err) - ); - } - } - return genError("SSL handshake timed out"); -} - -bool -MessageConnection::setCnVerification() -{ - SSL_set_hostflags(ssl_socket, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); - return SSL_set1_host(ssl_socket, host.c_str()) != 0; -} - -bool -MessageConnection::encrypt() -{ - if (ssl_ctx.get() == nullptr) { - dbgWarning(D_COMMUNICATION) << "SSL context does not exist"; - return false; - } - - BioUniquePtr s_bio = BioUniquePtr(BIO_new_ssl(ssl_ctx.get(), 1)); - if (s_bio.get() == nullptr) { - dbgWarning(D_COMMUNICATION) << "Failed to create encrypted BIO socket"; - return false; - } - - bio = BioUniquePtr(BIO_push(s_bio.release(), bio.release())); - BIO_get_ssl(bio.get(), &ssl_socket); - if (!ssl_socket) { - dbgWarning(D_COMMUNICATION) << "Failed to locate SSL pointer"; - return false; - } - - if (!setCnVerification()) { - dbgWarning(D_COMMUNICATION) << "Failed to set host name (CN) verification"; - return false; - } - - auto handshake_result = doHandshake(bio); - if (!handshake_result.ok()) { - dbgWarning(D_COMMUNICATION) << handshake_result.getErr(); - return false; - } - - if (!verifyCert()) { - dbgWarning(D_COMMUNICATION) << "Failed to verify the certificate"; - return false; - } - - dbgTrace(D_COMMUNICATION) << "Successfully secured BIO socket for connection " << *this; - return true; -} -// LCOV_EXCL_STOP - -bool -MessageConnection::setSocket() -{ - auto is_secure_conn = (is_secure && !over_proxy); - bio = is_secure_conn ? - BioUniquePtr(BIO_new_ssl_connect(ssl_ctx.get())) : - BioUniquePtr(BIO_new(BIO_s_connect())); - - if (!bio.get()) { - dbgWarning(D_COMMUNICATION) - << "Failed to create new " - << (is_secure_conn ? "secure" : "clear") - << " BIO connection"; - return false; - } - - if (is_secure_conn) { - BIO_get_ssl(bio.get(), &ssl_socket); - if (!ssl_socket) { - dbgWarning(D_COMMUNICATION) << "Failed to locate SSL pointer"; - return false; - } - - SSL_set_mode(ssl_socket, SSL_MODE_AUTO_RETRY); - if (!setCnVerification()) { - dbgWarning(D_COMMUNICATION) << "Failed to set host name (CN) verification"; - return false; - } - if (!SSL_set_tlsext_host_name(ssl_socket, host.c_str())) { - dbgWarning(D_COMMUNICATION) << "Failed to set TLS host name extension (SNI)"; - return false; - } - } - - return true; -} - -bool -MessageConnection::isBioSocketReady() const -{ - auto fd = BIO_get_fd(bio.get(), nullptr); - - fd_set rfds; - FD_ZERO(&rfds); - FD_SET(fd, &rfds); - - struct timeval tv = { 0, 0 }; - - return select(fd + 1, nullptr, &rfds, nullptr, &tv) == 1; -} - - -bool -MessageConnection::connect(const string &host, const string &overwrite_port) -{ - string address = host + ":" + overwrite_port; - BIO_set_conn_hostname(bio.get(), address.c_str()); - BIO_set_nbio(bio.get(), 1); - - auto timer = Singleton::Consume::by(); - auto conn_timeout = chrono::microseconds(getConnectionTimeout()); - auto end_time = timer->getMonotonicTime() + conn_timeout; - int counter = 0; - - while (timer->getMonotonicTime() < end_time) { - counter++; - if (BIO_do_connect(bio.get()) > 0) { - dbgDebug(D_COMMUNICATION) - << "Successfully established new BIO connection. " - << "Number of attempts: " - << counter; - if (is_secure && !over_proxy) { - auto handshake_result = doHandshake(bio); - if (!handshake_result.ok()) { - dbgWarning(D_COMMUNICATION) << handshake_result.getErr(); - return false; - } - return verifyCert(); - } - return true; - } - - if (!BIO_should_retry(bio.get())) { - int bio_err = ERR_get_error(); - dbgWarning(D_COMMUNICATION) - << "Failed completely to establish new BIO connection (BIO won't retry!)." - << "trying next address. OpenSSL error: " - << string(ERR_error_string(bio_err, nullptr)) - << ", OpenSSL error code: " << bio_err - << ", Number of attempts: " - << counter; - return false; - } - - if (mainloop != nullptr && (counter % 10) == 0) mainloop->yield(true); - } - dbgWarning(D_COMMUNICATION) - << "Failed to establish new connection after reaching timeout. " - << "address: " - << address - << ", Number of attempts: " - << counter; - return false; -} - -template -Maybe -MessageConnection::receiveResponse(I_MessageDecoder &decoder) -{ - auto end_time = - timer->getMonotonicTime() + chrono::microseconds(getConnectionTimeout()); - uint counter = 0; - char buf[1000]; - while (timer->getMonotonicTime() < end_time) { - if (!isBioSocketReady()) { - dbgDebug(D_COMMUNICATION) << "Socket is not ready for use."; - if (mainloop != nullptr) mainloop->yield(true); - continue; - } - int len_or_error_ret_val = BIO_read(bio.get(), buf, sizeof(buf) - 1); - if (len_or_error_ret_val <= 0) { - if (!BIO_should_retry(bio.get())) { - if (len_or_error_ret_val == 0) { - if (connection_closed_count == 1) { - dbgWarning(D_COMMUNICATION) - << "Connection closed. Type: " - << tagToString(tag) - << ", Count: " - << connection_closed_count; - } else { - dbgDebug(D_COMMUNICATION) - << "Connection closed. Type: " - << tagToString(tag) - << ", Count: " - << connection_closed_count; - } - auto maybe_message = decoder.decodeBytes(string()); - if (maybe_message.ok()) { - return maybe_message.unpackMove(); - } - } - - if (connection_closed_count == 1) { - dbgWarning(D_COMMUNICATION) - << "Failed to read data from BIO socket. Type: " - << tagToString(tag) - << ", Count: " - << connection_closed_count - << ", Error code: " - << len_or_error_ret_val; - } else { - dbgDebug(D_COMMUNICATION) - << "Failed to read data from BIO socket. Type: " - << tagToString(tag) - << ", Count: " - << connection_closed_count - << ", Error code: " - << len_or_error_ret_val; - } - - connection_closed_count++; - - return genError("Error reading from BIO socket"); - } - if (mainloop != nullptr) mainloop->yield(true); - continue; - } - - if (connection_closed_count > 0) { - dbgTrace(D_COMMUNICATION) - << "Connection was reconnected. Type: " - << tagToString(tag) - << ", number of attempts: " - << connection_closed_count; - connection_closed_count = 0; - } - - string data = string(buf, len_or_error_ret_val); - dbgTrace(D_HTTP_REQUEST) << "Received the following data:\n" << data; - - auto maybe_message = decoder.decodeBytes(data); - if (maybe_message.ok()) { - return maybe_message.unpackMove(); - } - - if (mainloop != nullptr && (counter++ % 5) == 0) mainloop->yield(true); - } - - dbgWarning(D_COMMUNICATION) << "Failed to receive data after reaching timeout"; - return genError("Reading took too long"); -} - -Maybe -MessageConnection::startNewConnection( - const string &host, - const uint16_t port_num, - const bool is_secure, - MessageTypeTag tag, - const bool over_proxy, - bool is_external, - bool is_ssl_ignore_validation -) -{ - MessageConnection conn = MessageConnection( - host, - port_num, - is_secure, - over_proxy, - is_external, - tag, - is_ssl_ignore_validation - ); - Maybe conn_res = conn.establishConnection(); - if (!conn_res.ok()) return conn_res.passErr(); - dbgTrace(D_COMMUNICATION) << "Started new connection for tag: " << tagToString(tag); - return move(conn); -} - -MessageConnection::~MessageConnection() {} - -bool -MessageConnection::setCtx() -{ - if (!is_secure) return true; - - ssl_ctx = BioUniquePtr(SSL_CTX_new(TLS_client_method())); - if (ssl_ctx.get() == nullptr) { - dbgWarning(D_COMMUNICATION) << "Failed to initialize SSL context"; - return false; - } - - if (shouldIgnoreSslValidation()) return true; - - SSL_CTX_set_verify(ssl_ctx.get(), SSL_VERIFY_PEER, nullptr); - - filesystem_prefix = getFilesystemPathConfig(); - dbgTrace(D_COMMUNICATION) << "MessageConnection, file systen prefix: " << filesystem_prefix << endl; - string cert_file_path = getConfigurationWithDefault( - filesystem_prefix + "/certs/fog.pem", - "message", - "Certificate chain file path" - ); - - string openssl_dir = "/usr/lib/ssl/certs/"; - auto openssl_dir_maybe = Singleton::Consume::by()->getOpenSSLDir(); - if (openssl_dir_maybe.ok()) openssl_dir = openssl_dir_maybe.unpack(); - - auto trusted_ca_directory = getConfigurationWithDefault( - openssl_dir, - "message", - "Trusted CA directory" - ); - const char *ca_dir_path = nullptr; - if (!trusted_ca_directory.empty()) { - ca_dir_path = trusted_ca_directory.c_str(); - } - - if (SSL_CTX_load_verify_locations(ssl_ctx.get(), cert_file_path.c_str(), ca_dir_path) == 1) { - return true; - } - - dbgWarning(D_COMMUNICATION) << "Failed to load fog's certificate file. Path: " << cert_file_path; - - return false; -} - -bool -MessageConnection::isReady() const -{ - dbgFlow(D_COMMUNICATION); - if (!bio.get()) { - dbgTrace(D_COMMUNICATION) << "Bio is uninitialized"; - return false; - } - - if (!is_secure) return true; - - if (!ssl_socket) { - dbgTrace(D_COMMUNICATION) << "SSL socket is uninitialized"; - return false; - } - - if (!ssl_ctx.get()) { - dbgTrace(D_COMMUNICATION) << "SSL context is uninitialized"; - return false; - } - - return true; -} - -string -MessageConnection::printData(const string &data) -{ - auto type = getConfigurationWithDefault("chopped", "message", "Data printout type"); - if (type == "chopped") return data.substr(0, 10) + (data.size() > 10 ? " ..." : ""); - if (type == "full") return data; - if (type == "size") return to_string(data.size()) + " bytes"; - if (type == "none") return ""; - - dbgWarning(D_COMMUNICATION) << "Unknown data printout option '" << type << "' - going with 'chopped' instead."; - return data.substr(0, 10) + (data.size() > 10 ? " ..." : ""); -} - -uint -MessageConnection::getConnectionTimeout() -{ - I_Environment *environment = Singleton::Consume::by(); - auto tmo_override = environment->get("Connection timeout Override"); - uint conf_tmo = - tmo_override.ok() ? - *tmo_override : - getConfigurationWithDefault(2000000, "message", "Connection timeout"); - - uint profile_setting_tmo = getProfileAgentSettingWithDefault( - conf_tmo, - "agent.config.message.connectionTimeout" - ); - - auto executable = environment->get("Service Name"); - auto nano_service_name_tmo = getProfileAgentSetting("agent.config.message.connectionTimeoutServiceName"); - if (!nano_service_name_tmo.ok() || !executable.ok()) { - dbgTrace(D_COMMUNICATION) - << "Could not identify service name. Executable env state: " - << (executable.ok() ? "true" : "false") - << ", state of nano service name from settings: " - << (nano_service_name_tmo.ok() ? "true" : "false") - << ", timeout value to use: " - << conf_tmo; - return conf_tmo; - } - if (*nano_service_name_tmo == *executable) { - dbgTrace(D_COMMUNICATION) - << "Using profile setting for specific nano service. " - << " nano service name: " - << *nano_service_name_tmo - << ", timeout value used: " - << profile_setting_tmo; - return profile_setting_tmo; - } - - dbgTrace(D_COMMUNICATION) - << "Using non profile config setting for nano service. " - << " profile configuration for nano service name: " - << *nano_service_name_tmo - << ", actual service name: " - << *executable - << ", timeout value used: " - << conf_tmo; - return conf_tmo; -} - -bool -MessageConnection::sendData(const string &data) const -{ - dbgTrace(D_HTTP_REQUEST) << "Sending the following data " << *this << ":\n" << printData(data); - - auto end_time = timer->getMonotonicTime() + chrono::microseconds(getConnectionTimeout()); - - uint counter = 0; - int remaining_data_len = data.length(); - while (timer->getMonotonicTime() < end_time) { - int offset = data.length() - remaining_data_len; - if (!isBioSocketReady()) { - dbgDebug(D_COMMUNICATION) << "Socket is not ready for use."; - if (mainloop != nullptr) mainloop->yield(true); - continue; - } - int data_sent_len = BIO_write(bio.get(), data.c_str() + offset, remaining_data_len); - if (data_sent_len > 0) { - if (remaining_data_len - data_sent_len < 0) { - dbgWarning(D_COMMUNICATION) - << "Sent data length exceeded actual data length (" - << to_string(data_sent_len) - << " > " - << to_string(remaining_data_len) - << ")"; - - return false; - } - - dbgTrace(D_COMMUNICATION) - << "Successfully sent " - << to_string(data_sent_len) - << " bytes of data out of total " - << to_string(data.length()) - << " bytes."; - - remaining_data_len -= data_sent_len; - if (remaining_data_len == 0) return true; - if (mainloop != nullptr && (counter++ % 5) == 0) mainloop->yield(true); - continue; - } - - if(!BIO_should_retry(bio.get())) { - dbgWarning(D_COMMUNICATION) << "Failed to Write data into BIO socket. Error code: " << data_sent_len; - return false; - } - dbgTrace(D_COMMUNICATION) << "Temporarily cannot send data. Will retry."; - if (mainloop != nullptr) mainloop->yield(true); - } - - dbgWarning(D_COMMUNICATION) << "Failed to send data after reaching timeout"; - return false; -} - -ostream & -MessageConnection::print(ostream &os) const -{ - os << "<" << host << ":" << port_num << " over " << (is_secure? "secure" : "clear") << " socket>"; - return os; -} - -ostream & operator<<(ostream &os, const MessageConnection &conn) { return conn.print(os); } - - -void -MessageConnection::getIpFromHostname(const string &hostname) -{ - struct addrinfo *servinfo = nullptr; - auto __scope_exit = make_scope_exit([&servinfo] () { if (servinfo) freeaddrinfo(servinfo); }); - - struct addrinfo hints; - memset (&hints, 0, sizeof (hints)); - hints.ai_family = PF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - hints.ai_flags |= AI_CANONNAME; - hints.ai_protocol = IPPROTO_TCP; - vector res; - if (IPAddr::isValidIPAddr(hostname)) { - dbgDebug(D_COMMUNICATION) << "Provided host name is already an IP address. Host: " << hostname; - current_ips.clear(); - current_ips.push_back(hostname); - return; - } - - if (getaddrinfo(hostname.c_str(), nullptr, &hints, &servinfo) != 0) { - dbgWarning(D_COMMUNICATION) << "IP address was not found for the given host name. Host: " << hostname; - return; - } - - for(struct addrinfo *addr_iter = servinfo; addr_iter != nullptr; addr_iter = addr_iter->ai_next) { - char buf[INET6_ADDRSTRLEN]; - const char *formatted_addr; - - if (addr_iter->ai_addr->sa_family == AF_INET) { - struct in_addr addr = reinterpret_cast(addr_iter->ai_addr)->sin_addr; - formatted_addr = inet_ntop(AF_INET, &addr, buf, sizeof(buf)); - } else if (addr_iter->ai_addr->sa_family == AF_INET6) { - struct in6_addr addr = reinterpret_cast(addr_iter->ai_addr)->sin6_addr; - formatted_addr = inet_ntop(AF_INET6, &addr, buf, sizeof(buf)); - } else { - continue; - } - - res.push_back(string(formatted_addr)); - dbgDebug(D_COMMUNICATION) - << "Successfully resolved host name to IP address. Host: " - << hostname - << ", IP: " - << res.back(); - } - - if (res.empty()) { - dbgWarning(D_COMMUNICATION) << "No IPv4 / IPv6 addresses were found for the given host. Host: " << hostname; - return; - } - - current_ips = res; - return; -} - -Maybe -MessageConnection::establishConnection() -{ - if (!setCtx()) return genError("Failed to initialize SSL context"); - dbgDebug(D_COMMUNICATION) << "Succesfully initialized SSL context"; - - if (!setSocket()) return genError("Failed to create new socket"); - dbgDebug(D_COMMUNICATION) << "Succesfully created new socket"; - - string conn_host = over_proxy ? proxy_host : host; - string conn_port = over_proxy ? to_string(proxy_port) : to_string(port_num); - - getIpFromHostname(conn_host); - Maybe is_connected = genError("Failed to establish new connection with: " + conn_host + ":" + conn_port); - - for (const string &address : current_ips) { - if (is_connected.ok()) break; - dbgDebug(D_COMMUNICATION) << "Trying to connect to " << address << ":" << conn_port; - if (!connect(address, conn_port)) { - dbgWarning(D_COMMUNICATION) << "Failed to connect " << address << ":" << conn_port; - continue; - } - is_connected = (over_proxy && is_secure) ? establishConnectionOverProxy() : Maybe(); - if (!is_connected.ok()) { - dbgWarning(D_COMMUNICATION) << "Failed to connect " << address << ":" << conn_port; - continue; - } - - dbgDebug(D_COMMUNICATION) << "Successfully connected to " << address << ":" << conn_port; - } - return is_connected; -} - -// LCOV_EXCL_START Reason: No proxy for ut -Maybe -MessageConnection::establishConnectionOverProxy() -{ - HTTPEncoder http_encoder(host, to_string(port_num)); - HTTPRequest req = http_encoder.Connect(); - if (!proxy_auth.empty()) req.insertHeader("Proxy-Authorization", "Basic " + encryptor->base64Encode(proxy_auth)); - waitForQueue(); - auto release_queue_on_exit = make_scope_exit([this] () { releaseQueue(); }); - if (!sendData(req.toString())) { - return genError("Failed to send CONNECT request to proxy"); - } - - HTTPDecoder http_decoder(I_Messaging::Method::CONNECT); - Maybe response = receiveResponse(http_decoder); - if (!response.ok()) { - return genError("Failed to receive a response from proxy"); - } - - auto response_data = response->getResponse(); - if (!response_data.ok()) { - return genError("Failed to connect via proxy"); - } - - if (!encrypt()) { - return genError("Failed to encrypt the socket after the CONNECT request"); - } - return Maybe(); -} -// LCOV_EXCL_STOP - -Maybe -MessageConnection::reconnect(bool should_lock) -{ - if (should_lock) { - while (!lock()) { mainloop->yield(true); } - } - - auto res = establishConnection(); - if (should_lock) { - unlock(); - } - return res; -} - -void -MessageConnection::waitForQueue() -{ - dbgTrace(D_COMMUNICATION) << "Pending queue position"; - while (available_messaging_queue_pos == UINT64_MAX) { - mainloop->yield(true); - } - - uint64_t messaging_queue_pos = available_messaging_queue_pos++; - dbgTrace(D_COMMUNICATION) << "Received an available queue position: " << messaging_queue_pos; - metrics_current_size++; - - MessageQueueEvent queue_event; - queue_event.setMessageQueueSize(metrics_current_size); - queue_event.notify(); - - while (messaging_queue_pos != current_messaging_queue_pos) { - mainloop->yield(true); - } - - dbgTrace(D_COMMUNICATION) << "Reached the current queue position: " << messaging_queue_pos; - return; -} - -void -MessageConnection::releaseQueue() -{ - dbgTrace(D_COMMUNICATION) << "Released the queue position " << current_messaging_queue_pos; - - current_messaging_queue_pos++; - if (current_messaging_queue_pos == UINT64_MAX) { - current_messaging_queue_pos = 0; - available_messaging_queue_pos = 0; - } - metrics_current_size--; - dbgTrace(D_COMMUNICATION) << "Queue position was advanced"; -} - -ProtoMessageComp::ProtoMessageComp() : Component("ProtoMessageComp"), pimpl(make_unique()) {} -ProtoMessageComp::~ProtoMessageComp() {} - -void ProtoMessageComp::init() { pimpl->init(); } -void ProtoMessageComp::fini() { pimpl->fini(); } - -void -ProtoMessageComp::preload() -{ - registerExpectedConfiguration("message", "Cache timeout"); - registerExpectedConfiguration("message", "Connection timeout"); - registerExpectedConfiguration("message", "send event retry in sec"); - registerExpectedConfiguration("message", "Reuse connection"); - registerExpectedConfiguration("message", "Verify SSL pinning"); - registerExpectedConfiguration("message", "Buffer Failed Requests"); - registerExpectedConfiguration("message", "Certificate chain file path"); - registerExpectedConfiguration("message", "Trusted CA directory"); - registerExpectedConfiguration("message", "Public key path"); - registerExpectedConfiguration("message", "Metrics Routine Interval"); - registerExpectedConfiguration("message", "Data printout type"); - registerExpectedConfiguration("message", "Internal Fog error timeout"); -} diff --git a/core/message/message_metric.h b/core/message/message_metric.h deleted file mode 100755 index dd568c9..0000000 --- a/core/message/message_metric.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __MESSAGE_METRIC_H__ -#define __MESSAGE_METRIC_H__ - -#include "generic_metric.h" - -class MessageQueueEvent : public Event -{ -public: - void - setMessageQueueSize(uint64_t _queue_size) - { - queue_size = _queue_size; - } - - uint64_t - getMessageQueueSize() const - { - return queue_size; - } - -private: - uint64_t queue_size; -}; - -class MessageQueueMetric - : - public GenericMetric, - public Listener -{ -public: - void - upon(const MessageQueueEvent &event) override - { - max_queue_size.report(event.getMessageQueueSize()); - avg_queue_size.report(double(event.getMessageQueueSize())); - current_queue_size.report(event.getMessageQueueSize()); - } - -private: - MetricCalculations::Max max_queue_size{this, "messageQueueMaxSizeSample", 0}; - MetricCalculations::Average avg_queue_size{this, "messageQueueAvgSizeSample"}; - MetricCalculations::LastReportedValue current_queue_size{this, "messageQueueCurrentSizeSample"}; -}; - -#endif // __MESSAGE_METRIC_H__ diff --git a/core/messaging/CMakeLists.txt b/core/messaging/CMakeLists.txt new file mode 100644 index 0000000..9edf53a --- /dev/null +++ b/core/messaging/CMakeLists.txt @@ -0,0 +1,12 @@ +include_directories(include) + +add_subdirectory(messaging_comp) +add_subdirectory(connection) +add_subdirectory(messaging_buffer_comp) +add_library(messaging messaging.cc) +target_link_libraries( + messaging + "messaging_comp" + "connection" + "messaging_buffer_comp" +) diff --git a/core/messaging/connection/CMakeLists.txt b/core/messaging/connection/CMakeLists.txt new file mode 100644 index 0000000..843ce98 --- /dev/null +++ b/core/messaging/connection/CMakeLists.txt @@ -0,0 +1,2 @@ +add_library(connection connection_comp.cc connection.cc) +add_subdirectory(connection_ut) diff --git a/core/messaging/connection/connection.cc b/core/messaging/connection/connection.cc new file mode 100644 index 0000000..640ebcc --- /dev/null +++ b/core/messaging/connection/connection.cc @@ -0,0 +1,694 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "time_print.h" +#include "connection.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "http_request.h" +#include "maybe_res.h" +#include "messaging.h" +#include "response_parser.h" +#include "scope_exit.h" +#include "smart_bio.h" + +using namespace std; +using namespace smartBIO; + +USE_DEBUG_FLAG(D_CONNECTION); + +static const HTTPResponse sending_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to send all data in time"); +static const HTTPResponse receving_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to receive all data in time"); +static const HTTPResponse parsing_error(HTTPStatusCode::HTTP_UNKNOWN, "Failed to parse the HTTP response"); + +const string & +MessageConnectionKey::getHostName() const +{ + return host_name; +} + +uint16_t +MessageConnectionKey::getPort() const +{ + return port; +} + +const MessageCategory & +MessageConnectionKey::getCategory() const +{ + return category; +} + +bool +MessageConnectionKey::operator<(const MessageConnectionKey &other) const +{ + if (host_name != other.host_name) return host_name < other.host_name; + if (port != other.port) return port < other.port; + return category < other.category; +} + +enum class ConnectionFlags +{ + UNSECURE, + ONE_TIME, + IGNORE_SSL_VALIDATION, + PROXY, + + COUNT +}; + +class Connection::Impl +{ +public: + Impl(const MessageConnectionKey &_key, const MessageMetadata &metadata) : key(_key) + { + auto metadata_flags = metadata.getConnectionFlags(); + if (metadata_flags.isSet(MessageConnectionConfig::UNSECURE_CONN)) flags.setFlag(ConnectionFlags::UNSECURE); + if (metadata_flags.isSet(MessageConnectionConfig::ONE_TIME_CONN)) flags.setFlag(ConnectionFlags::ONE_TIME); + if (metadata_flags.isSet(MessageConnectionConfig::IGNORE_SSL_VALIDATION)) { + flags.setFlag(ConnectionFlags::IGNORE_SSL_VALIDATION); + } + } + + void + setProxySettings(const MessageProxySettings &_settings) + { + flags.setFlag(ConnectionFlags::PROXY); + settings = _settings; + } + + void + setConnectMessage(const string &connect_msg) + { + connect_message = connect_msg; + } + + void + setExternalCertificate(const string &_certificate) + { + certificate = _certificate; + } + + const MessageProxySettings & + getProxySettings() const + { + return settings; + } + + const string & + getExternalCertificate() const + { + return certificate; + } + + const MessageConnectionKey & + getConnKey() const + { + return key; + } + + bool + isOverProxy() const + { + return flags.isSet(ConnectionFlags::PROXY); + } + + bool + isUnsecure() const + { + return flags.isSet(ConnectionFlags::UNSECURE); + } + + bool + isSuspended() + { + if (active.ok()) return false; + + I_TimeGet *i_time = Singleton::Consume::by(); + auto curr_time = chrono::duration_cast(i_time->getMonotonicTime()); + + if (active.getErr() > curr_time) { + dbgTrace(D_MESSAGING) << "Connection is suspended for another " << (active.getErr() - curr_time); + return true; + } + + if (establishConnection().ok()) { + dbgDebug(D_MESSAGING) << "Reestablish connection"; + return true; + } + + dbgWarning(D_MESSAGING) << "Reestablish connection failed"; + active = genError(curr_time + chrono::seconds(300)); + return false; + } + + Maybe + establishConnection() + { + dbgFlow(D_CONNECTION) << "Establishing a new connection"; + auto set_socket = setSocket(); + if (!set_socket.ok()) { + dbgWarning(D_CONNECTION) << "Failed to set socket: " << set_socket.getErr(); + return set_socket; + } + + auto connect = connectToHost(); + if (!connect.ok()) { + dbgWarning(D_CONNECTION) << "Failed to connect to host: " << connect.getErr(); + return connect; + } + + if (flags.isSet(ConnectionFlags::PROXY)) { + dbgDebug(D_CONNECTION) << "Sending a CONNECT request: " << connect_message; + auto res = sendAndReceiveData(connect_message, true); + if (!res.ok()) { + string connect_error = res.getErr().getBody(); + dbgWarning(D_CONNECTION) << "Failed to connect to proxy: " << connect_error; + return genError(connect_error); + } + + if (!isUnsecure()) { + auto encrypt_res = encryptProxyConnection(); + if (!encrypt_res.ok()) { + return genError("Failed to encrypt the socket after the CONNECT request" + encrypt_res.getErr()); + } + } + } + + dbgDebug(D_CONNECTION) + << "Successfully connected to " + << key.getHostName() + << ':' + << key.getPort() + << (isOverProxy() ? ", Over proxy: " + settings.getProxyHost() + ":" + to_string(key.getPort()) : ""); + active = Maybe(); + return Maybe(); + } + + Maybe + sendRequest(const string &request) + { + dbgFlow(D_CONNECTION) + << "Send request to " + << key.getHostName() + << ':' + << key.getPort() + << ":\n" + << printOut(request); + + auto result = sendAndReceiveData(request, false); + if (!result.ok()) { + establishConnection(); + result = sendAndReceiveData(request, false); + } + + if (!result.ok()) { + ++failed_attempts; + if (failed_attempts > 10) { + I_TimeGet *i_time = Singleton::Consume::by(); + auto curr_time = chrono::duration_cast(i_time->getMonotonicTime()); + active = genError(curr_time + chrono::seconds(300)); + } + return result.passErr(); + } + + failed_attempts = 0; + return result; + } + +private: + Maybe + setSSLContext() + { + dbgFlow(D_CONNECTION) << "Setting SSL context"; + if (isUnsecure()) { + dbgTrace(D_CONNECTION) << "Connection is unsecure. Skipping SSL context creation"; + return Maybe(); + } + ssl_ctx = BioUniquePtr(SSL_CTX_new(TLS_client_method())); + if (!ssl_ctx.get()) return genError("Failed to initialize SSL context"); + if (shouldIgnoreSslValidation()) { + dbgTrace(D_CONNECTION) << "Ignoring SSL validation"; + return Maybe(); + } + + SSL_CTX_set_verify(ssl_ctx.get(), SSL_VERIFY_PEER, nullptr); + + auto defualt_cert_path = getFilesystemPathConfig() + "/certs/fog.pem"; + auto cert_path = getConfigurationWithDefault(defualt_cert_path, "message", "Certificate chain file path"); + const char *cert = cert_path.c_str(); + + auto details_ssl_dir = Singleton::Consume::by()->getOpenSSLDir(); + auto openssl_dir = details_ssl_dir.ok() ? *details_ssl_dir : "/usr/lib/ssl/certs/"; + auto configured_ssl_dir = getConfigurationWithDefault(openssl_dir, "message", "Trusted CA directory"); + const char *ca_dir = configured_ssl_dir.empty() ? nullptr : configured_ssl_dir.c_str(); + + if (SSL_CTX_load_verify_locations(ssl_ctx.get(), cert, ca_dir) != 1) { + return genError("Failed to load certificate locations"); + } + + dbgDebug(D_CONNECTION) << "SSL context set successfully. Certificate: " << cert << ", CA dir: " << ca_dir; + return Maybe(); + } + + Maybe + setSocket() + { + dbgFlow(D_CONNECTION) << "Setting socket"; + if (isUnsecure()) { + bio = BioUniquePtr(BIO_new(BIO_s_connect())); + if (!bio.get()) return genError("Failed to create new BIO connection"); + return Maybe(); + } + + auto build_ssl = setSSLContext(); + if (!build_ssl.ok()) return build_ssl; + + if (isOverProxy()) { + bio = BioUniquePtr(BIO_new(BIO_s_connect())); + if (!bio.get()) return genError("Failed to create new BIO connection"); + return Maybe(); + } + + bio = BioUniquePtr(BIO_new_ssl_connect(ssl_ctx.get())); + BIO_get_ssl(bio.get(), &ssl_socket); + if (!ssl_socket) return genError("Failed to locate SSL pointer"); + + SSL_set_mode(ssl_socket, SSL_MODE_AUTO_RETRY); + SSL_set_hostflags(ssl_socket, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); + + auto host = key.getHostName().c_str(); + if (!SSL_set1_host(ssl_socket, host)) { + return genError("Failed to set host name verification. Host: " + string(host)); + } + if (!SSL_set_tlsext_host_name(ssl_socket, host)) { + return genError("Failed to set TLS host name extension. Host: " + string(host)); + } + + return Maybe(); + } + + static chrono::microseconds + getConnectionTimeout() + { + I_Environment *env = Singleton::Consume::by(); + + auto executable = env->get("Service Name"); + auto service_name = getProfileAgentSetting("agent.config.message.connectionTimeoutServiceName"); + + if (executable.ok() && service_name.ok() && *executable == *service_name) { + auto service_timeout = getProfileAgentSetting("agent.config.message.connectionTimeout"); + if (service_timeout.ok()) return chrono::microseconds(*service_timeout); + } + + auto env_timeout = env->get("Connection timeout Override"); + if (env_timeout.ok()) return chrono::microseconds(*env_timeout); + + return chrono::microseconds(getConfigurationWithDefault(10000000, "message", "Connection timeout")); + } + + bool + shouldIgnoreSslValidation() const + { + if (flags.isSet(ConnectionFlags::UNSECURE)) return true; + if (flags.isSet(ConnectionFlags::IGNORE_SSL_VALIDATION)) return true; + return getProfileAgentSettingWithDefault(false, "agent.config.message.ignoreSslValidation"); + } + + bool + isBioSocketReady() const + { + if (!bio.get()) return false; + + auto fd = BIO_get_fd(bio.get(), nullptr); + fd_set rfds; + FD_ZERO(&rfds); + FD_SET(fd, &rfds); + struct timeval tv = { 0, 0 }; + + return select(fd + 1, nullptr, &rfds, nullptr, &tv) == 1; + } + +// LCOV_EXCL_START Reason: No ssl ut + Maybe + verifyCertPinning(const BioUniquePtr &cert) const + { + BioUniquePtr outbio = BioUniquePtr(BIO_new(BIO_s_mem())); + if (!outbio.get()) return genError("Failed to allocate new BIO"); + BioUniquePtr pkey = BioUniquePtr(X509_get_pubkey(cert.get())); + if (!pkey.get()) return genError("Error getting public key from certificate"); + + if (!PEM_write_bio_PUBKEY(outbio.get(), pkey.get())) return genError("Error writing key in PEM format"); + + char *buf; + auto len = BIO_get_mem_data(outbio.get(), &buf); + string recieved_public_key(buf, len); + dbgTrace(D_CONNECTION) << "Received public key " << recieved_public_key; + + auto defualt_key_path = getFilesystemPathConfig() + "/certs/public-key.pem"; + auto key_path = getConfigurationWithDefault(defualt_key_path, "message", "Public key path"); + dbgTrace(D_CONNECTION) << "Load public key path. Path: " << key_path; + + ifstream pinned_public_file(key_path); + if (!pinned_public_file.is_open()) return genError("Failed to open pinned public key file"); + + stringstream pinned_key; + pinned_key << pinned_public_file.rdbuf(); + dbgTrace(D_CONNECTION) << "Saved public key: " << pinned_key.str(); + + if (recieved_public_key != pinned_key.str()) return genError("Received and pinned keys don't match"); + + return Maybe(); + } + + Maybe + verifyCert() + { + dbgFlow(D_CONNECTION) << "Verifying certificate"; + + if (shouldIgnoreSslValidation()) { + dbgTrace(D_CONNECTION) << "Ignoring SSL validation"; + return Maybe(); + } + + BioUniquePtr cert = BioUniquePtr(SSL_get_peer_certificate(ssl_socket)); + if (!cert.get() || cert.get() == nullptr) return genError("Server did not provide a cert during handshake"); + + if (SSL_get_verify_result(ssl_socket) != X509_V_OK) { + string error = ERR_error_string(ERR_get_error(), nullptr); + return genError("Failed to verify server certificate. OpenSSL error: " + error); + } + + if (!getConfigurationWithDefault(false, "message", "Verify SSL pinning")) return Maybe(); + + return verifyCertPinning(cert); + } + + Maybe + performHandshakeAndVerifyCert(I_TimeGet *i_time, I_MainLoop *i_mainloop) + { + dbgFlow(D_CONNECTION) << "Performing SSL handshake"; + auto handshake_timeout = getConfigurationWithDefault(500000, "message", "Connection handshake timeout"); + auto handshake_end_time = i_time->getMonotonicTime() + chrono::microseconds(handshake_timeout); + + while (i_time->getMonotonicTime() < handshake_end_time) { + if (!isBioSocketReady()) { + dbgTrace(D_CONNECTION) << "Socket is not ready for use."; + i_mainloop->yield(true); + continue; + } + + if (BIO_do_handshake(bio.get()) > 0) return verifyCert(); + if (!BIO_should_retry(bio.get())) { + string error = ERR_error_string(ERR_get_error(), nullptr); + return genError("Failed to obtain a successful SSL handshake. OpenSSL error: " + error); + } + } + + return genError("SSL handshake reached timed out"); + } +// LCOV_EXCL_STOP + + Maybe + connectToHost() + { + string full_address; + if (isOverProxy()) { + full_address = settings.getProxyHost() + ":" + to_string(settings.getProxyPort()); + } else { + full_address = key.getHostName() + ":" + to_string(key.getPort()); + } + + dbgFlow(D_CONNECTION) << "Connecting to " << full_address; + BIO_set_conn_hostname(bio.get(), full_address.c_str()); + BIO_set_nbio(bio.get(), 1); + + I_MainLoop *i_mainloop = Singleton::Consume::by(); + I_TimeGet *i_time = Singleton::Consume::by(); + auto conn_end_time = i_time->getMonotonicTime() + getConnectionTimeout(); + + uint attempts_count = 0; + while (i_time->getMonotonicTime() < conn_end_time) { + attempts_count++; + if (BIO_do_connect(bio.get()) > 0) { + if (isUnsecure() || isOverProxy()) return Maybe(); + return performHandshakeAndVerifyCert(i_time, i_mainloop); + } + + if (!BIO_should_retry(bio.get())) { + string bio_error = ERR_error_string(ERR_get_error(), nullptr); + return genError("Failed to connect (BIO won't retry!): " + bio_error); + } + + if ((attempts_count % 10) == 0) i_mainloop->yield(true); + } + + return genError("Failed to establish new connection to " + full_address + " after reaching timeout."); + } + + Maybe + sendData(const string &request, size_t data_left_to_send) const + { + if (!isBioSocketReady()) return 0; + + dbgTrace(D_MESSAGING) << "Sending request: " << printOut(request); + size_t offset = request.length() - data_left_to_send; + auto curr_data_to_send = request.c_str() + offset; + int data_sent_len = BIO_write(bio.get(), curr_data_to_send, data_left_to_send); + + if (data_sent_len >= 0) { + dbgTrace(D_CONNECTION) << "Sent " << data_sent_len << " bytes, out of: " << data_left_to_send << " bytes."; + return data_sent_len; + } + + if (BIO_should_retry(bio.get())) { + dbgTrace(D_CONNECTION) << "Failed to send data - retrying"; + return 0; + } + + char error_buf[256]; + ERR_error_string(ERR_get_error(), error_buf); + string error = "Failed to write data into BIO socket. Error: " + string(error_buf); + dbgWarning(D_CONNECTION) << error; + return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, error)); + } + + Maybe + receiveData() const + { + if (!isBioSocketReady()) return string(); + + char buffer[1000]; + int receive_len = BIO_read(bio.get(), buffer, sizeof(buffer) - 1); + + if (receive_len > 0) { + dbgTrace(D_CONNECTION) << "Received " << receive_len << " bytes"; + return string(buffer, receive_len); + } + + if (BIO_should_retry(bio.get())) return string(); + + char error_buf[256]; + ERR_error_string(ERR_get_error(), error_buf); + string error = receive_len == 0 ? + "Connection closed by peer" : + "Failed to read data from BIO socket. Error: " + string(error_buf); + dbgWarning(D_CONNECTION) << error; + return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, error)); + } + +// LCOV_EXCL_START Reason: Fix in a future commit + Maybe + encryptProxyConnection() + { + dbgFlow(D_CONNECTION) << "Encrypting BIO socket"; + if (ssl_ctx.get() == nullptr) return genError("SSL context does not exist"); + + BioUniquePtr s_bio = BioUniquePtr(BIO_new_ssl(ssl_ctx.get(), 1)); + if (s_bio.get() == nullptr) return genError("Failed to create encrypted BIO socket"); + + bio = BioUniquePtr(BIO_push(s_bio.release(), bio.release())); + BIO_get_ssl(bio.get(), &ssl_socket); + if (!ssl_socket) return genError("Failed to locate SSL pointer"); + + SSL_set_hostflags(ssl_socket, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); + auto host = key.getHostName().c_str(); + if (!SSL_set1_host(ssl_socket, host)) { + return genError("Failed to set host name verification. Host: " + string(host)); + } + + I_MainLoop *i_mainloop = Singleton::Consume::by(); + I_TimeGet *i_time = Singleton::Consume::by(); + return performHandshakeAndVerifyCert(i_time, i_mainloop); + } +// LCOV_EXCL_STOP + + Maybe + sendAndReceiveData(const string &request, bool is_connect) + { + I_MainLoop *i_mainloop = Singleton::Consume::by(); + while (lock) { + i_mainloop->yield(true); + } + lock = true; + auto unlock = make_scope_exit([&] () { lock = false; }); + + I_TimeGet *i_time = Singleton::Consume::by(); + auto sending_end_time = i_time->getMonotonicTime() + getConnectionTimeout(); + size_t data_left_to_send = request.length(); + + while (data_left_to_send > 0) { + if (i_time->getMonotonicTime() > sending_end_time) return genError(sending_timeout); + auto send_size = sendData(request, data_left_to_send); + if (!send_size.ok()) return send_size.passErr(); + data_left_to_send -= *send_size; + i_mainloop->yield(*send_size == 0); // We want to force waiting if we failed to send the data + } + + auto receiving_end_time = i_time->getMonotonicTime() + getConnectionTimeout(); + HTTPResponseParser http_parser; + dbgTrace(D_CONNECTION) << "Sent the message, now waiting for response"; + while (!http_parser.hasReachedError()) { + if (i_time->getMonotonicTime() > receiving_end_time) return genError(receving_timeout); + auto receieved = receiveData(); + if (!receieved.ok()) return receieved.passErr(); + auto response = http_parser.parseData(*receieved, is_connect); + if (response.ok()) { + dbgTrace(D_MESSAGING) << printOut(response.unpack().toString()); + return response.unpack(); + } + i_mainloop->yield(receieved.unpack().empty()); + } + return genError(parsing_error); + } + + static string + printOut(const string &data) + { + string type = getConfigurationWithDefault("chopped", "message", "Data printout type"); + uint length = getConfigurationWithDefault(10, "message", "Data printout length"); + if (type == "full") return data; + if (type == "size") return to_string(data.size()) + " bytes"; + if (type == "none") return ""; + string chopped_str = data.substr(0, length) + (data.size() > length ? " ..." : ""); + if (type == "chopped") return chopped_str; + + + dbgWarning(D_CONNECTION) << "Unknown data printout option '" << type << "' - going with 'chopped' instead."; + return chopped_str; + } + + MessageConnectionKey key; + Flags flags; + + MessageProxySettings settings; + string connect_message; + string certificate; + + smartBIO::BioUniquePtr bio = nullptr; + smartBIO::BioUniquePtr ssl_ctx = nullptr; + SSL *ssl_socket = nullptr; + + Maybe active = genError(0); + uint failed_attempts = 0; + + bool lock = false; +}; + +Connection::Connection(const MessageConnectionKey &key, const MessageMetadata &metadata) + : + pimpl(make_shared(key, metadata)) +{} + +Connection::~Connection() +{} + +Maybe +Connection::setProxySettings(const MessageProxySettings &settings) +{ + pimpl->setProxySettings(settings); + + map headers; + auto i_encrypt = Singleton::Consume::by(); + if (!settings.getProxyAuth().empty()) { + headers["Proxy-Authorization"] = i_encrypt->base64Encode(settings.getProxyAuth()); + } + + auto req = HTTPRequest::prepareRequest(*this, HTTPMethod::CONNECT, "", headers, ""); + if (!req.ok()) return genError("Failed to create connect request. Error: " + req.getErr()); + + pimpl->setConnectMessage(req.unpack().toString()); + return Maybe(); +} + +void +Connection::setExternalCertificate(const string &certificate) +{ + pimpl->setExternalCertificate(certificate); +} + +const MessageProxySettings & +Connection::getProxySettings() const +{ + return pimpl->getProxySettings(); +} + +const string & +Connection::getExternalCertificate() const +{ + return pimpl->getExternalCertificate(); +} + +const MessageConnectionKey & +Connection::getConnKey() const +{ + return pimpl->getConnKey(); +} + +bool +Connection::isOverProxy() const +{ + return pimpl->isOverProxy(); +} + +bool +Connection::isUnsecure() const +{ + return pimpl->isUnsecure(); +} + +bool +Connection::isSuspended() +{ + return pimpl->isSuspended(); +} + +Maybe +Connection::establishConnection() +{ + return pimpl->establishConnection(); +} + +Maybe +Connection::sendRequest(const string &request) +{ + return pimpl->sendRequest(request); +} diff --git a/core/messaging/connection/connection_comp.cc b/core/messaging/connection/connection_comp.cc new file mode 100644 index 0000000..26421f8 --- /dev/null +++ b/core/messaging/connection/connection_comp.cc @@ -0,0 +1,157 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "connection_comp.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "connection.h" +#include "maybe_res.h" +#include "messaging.h" +#include "smart_bio.h" + +using namespace std; + +USE_DEBUG_FLAG(D_CONNECTION); + +class ConnectionComponent::Impl : Singleton::Provide::From +{ +public: + void + init() + { + SSL_library_init(); + SSL_load_error_strings(); + OpenSSL_add_all_algorithms(); + } + + Maybe + establishConnection(const MessageMetadata &metadata, MessageCategory category) override + { + if (metadata.isProxySet()) return establishNewProxyConnection(metadata, category); + return establishNewConnection(metadata, category); + } + + Maybe + getPersistentConnection(const string &host_name, uint16_t port, MessageCategory category) override + { + auto conn = persistent_connections.find(MessageConnectionKey(host_name, port, category)); + if (conn == persistent_connections.end()) return genError("No persistent connection found"); + return conn->second; + } + + Maybe + getFogConnectionByCategory(MessageCategory category) override + { + auto maybe_fog_domain = Singleton::Consume::by()->getFogDomain(); + if (!maybe_fog_domain.ok()) { + return genError("Failed to retrieve FOG domain " + maybe_fog_domain.getErr()); + } + auto maybe_fog_port = Singleton::Consume::by()->getFogPort(); + if (!maybe_fog_port.ok()) { + return genError("Failed to retrieve FOG port " + maybe_fog_port.getErr()); + } + + return getPersistentConnection(*maybe_fog_domain, *maybe_fog_port, category); + } + + Maybe + sendRequest(Connection &connection, HTTPRequest request) override + { + return connection.sendRequest(request.toString()); + } + +private: + Maybe + establishNewConnection(const MessageMetadata &metadata, MessageCategory category) + { + dbgFlow(D_CONNECTION) + << "Establishing a new connection. Host: " + << metadata.getHostName() + << ", port: " + << metadata.getPort(); + MessageConnectionKey conn_key(metadata.getHostName(), metadata.getPort(), category); + Connection conn(conn_key, metadata); + + const auto &external_certificate = metadata.getExternalCertificate(); + if (!external_certificate.empty()) conn.setExternalCertificate(external_certificate); + + auto connected = conn.establishConnection(); + + if (!connected.ok()) { + string connection_err = "Failed to establish connection. Error: " + connected.getErr(); + dbgWarning(D_CONNECTION) << connection_err; + return genError(connection_err); + } + + dbgTrace(D_CONNECTION) << "Connection establish succssesfuly"; + persistent_connections.emplace(conn_key, conn); + return conn; + } + + Maybe + establishNewProxyConnection(const MessageMetadata &metadata, MessageCategory category) + { + dbgTrace(D_CONNECTION) + << "Establishing a new connection over proxy. Host: " + << metadata.getHostName() + << ", port: " + << metadata.getPort() + << ", proxy host: " + << metadata.getProxySettings().getProxyHost() + << ", proxy port: " + << metadata.getProxySettings().getProxyPort(); + + const auto &proxy = metadata.getProxySettings(); + MessageConnectionKey conn_key(metadata.getHostName(), metadata.getPort(), category); + Connection conn(conn_key, metadata); + + auto is_proxy = conn.setProxySettings(proxy); + if (!is_proxy.ok()) { + string proxy_err = "Failed to set proxy settings. Error: " + is_proxy.getErr(); + return genError(proxy_err); + } + + auto is_connected = conn.establishConnection(); + if (!is_connected.ok()) { + string connection_err = "Failed to establish connection over proxy. Error: " + is_connected.getErr(); + dbgWarning(D_CONNECTION) << connection_err; + return genError(connection_err); + } + + dbgTrace(D_CONNECTION) << "Connection over proxy established succssesfuly"; + + persistent_connections.emplace(conn_key, conn); + return conn; + } + + std::map persistent_connections; +}; + +ConnectionComponent::ConnectionComponent() : pimpl(make_unique()) +{} + +ConnectionComponent::~ConnectionComponent() +{} + +void +ConnectionComponent::init() +{ + pimpl->init(); +} diff --git a/core/messaging/connection/connection_ut/CMakeLists.txt b/core/messaging/connection/connection_ut/CMakeLists.txt new file mode 100644 index 0000000..8e444e3 --- /dev/null +++ b/core/messaging/connection/connection_ut/CMakeLists.txt @@ -0,0 +1,5 @@ +add_unit_test( + connection_comp_ut + "connection_comp_ut.cc" + "connection;ssl;connkey;singleton;boost_context;rest;metric;event_is;-lboost_regex;agent_details;-lcrypto;messaging;" +) diff --git a/core/messaging/connection/connection_ut/connection_comp_ut.cc b/core/messaging/connection/connection_ut/connection_comp_ut.cc new file mode 100644 index 0000000..af4f3a9 --- /dev/null +++ b/core/messaging/connection/connection_ut/connection_comp_ut.cc @@ -0,0 +1,298 @@ +#include "connection_comp.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "agent_core_utilities.h" +#include "agent_details.h" +#include "config.h" +#include "config_component.h" +#include "cptest.h" +#include "environment.h" +#include "mainloop.h" +#include "connection.h" +#include "mocks/mock_messaging_buffer.h" +#include "mock/mock_agent_details.h" +#include "mock/mock_mainloop.h" +#include "mock/mock_time_get.h" +#include "mock/mock_encryptor.h" +#include "rest.h" +#include "rest_server.h" + +using namespace std; +using namespace testing; + +USE_DEBUG_FLAG(D_CONNECTION); + +class DummySocket : Singleton::Consume +{ +public: + ~DummySocket() + { + if (server_fd != -1) close(server_fd); + if (connection_fd != -1) close(connection_fd); + } + + void + init() + { + server_fd = socket(AF_INET, SOCK_STREAM, 0); + dbgAssert(server_fd >= 0) << "Failed to open a socket"; + int socket_enable = 1; + setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)); + + struct sockaddr_in addr; + bzero(&addr, sizeof(addr)); + + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = htons(8080); + bind(server_fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)); + listen(server_fd, 100); + } + + void + acceptSocket() + { + if (connection_fd == -1) connection_fd = accept(server_fd, nullptr, nullptr); + } + + string + readFromSocket() + { + acceptSocket(); + + string res; + char buffer[1024]; + while (int bytesRead = readRaw(buffer, sizeof(buffer))) { + res += string(buffer, bytesRead); + } + return res; + } + + void + writeToSocket(const string &msg) + { + acceptSocket(); + EXPECT_EQ(write(connection_fd, msg.data(), msg.size()), msg.size()); + } + +private: + int + readRaw(char *buf, uint len) + { + struct pollfd s_poll; + s_poll.fd = connection_fd; + s_poll.events = POLLIN; + s_poll.revents = 0; + + if (poll(&s_poll, 1, 0) <= 0 || (s_poll.revents & POLLIN) == 0) return 0; + + return read(connection_fd, buf, len); + } + + int server_fd = -1; + int connection_fd = -1; +}; + +static ostream & +operator<<(ostream &os, const BufferedMessage &) +{ + return os; +} + +class TestConnectionComp : public testing::Test +{ +public: + TestConnectionComp() + { + Debug::setUnitTestFlag(D_CONNECTION, Debug::DebugLevel::TRACE); + connection_comp.init(); + i_conn = Singleton::Consume::from(connection_comp); + setAgentDetails(); + dummy_socket.init(); + } + + void + setAgentDetails() + { + EXPECT_CALL(mock_agent_details, getFogDomain()).WillRepeatedly(Return(string(fog_addr))); + EXPECT_CALL(mock_agent_details, getFogPort()).WillRepeatedly(Return(fog_port)); + EXPECT_CALL(mock_agent_details, getOpenSSLDir()).WillRepeatedly(Return(string("/usr/lib/ssl/certs/"))); + EXPECT_CALL(mock_agent_details, getAccessToken()).WillRepeatedly(Return(string("accesstoken"))); + } + + const string fog_addr = "127.0.0.1"; + int fog_port = 8080; + CPTestTempfile agent_details_file; + ConnectionComponent connection_comp; + I_MessagingConnection *i_conn; + ::Environment env; + ConfigComponent config; + NiceMock mock_messaging_buffer; + NiceMock mock_agent_details; + NiceMock mock_timer; + NiceMock mock_mainloop; + StrictMock mock_encryptor; + DummySocket dummy_socket; +}; + +TEST_F(TestConnectionComp, testSetAndGetFogConnection) +{ + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); + MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags); + auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::GENERIC); + ASSERT_TRUE(maybe_connection.ok()); + + auto maybe_get_connection = i_conn->getFogConnectionByCategory(MessageCategory::GENERIC); + ASSERT_TRUE(maybe_get_connection.ok()); +} + +TEST_F(TestConnectionComp, testSetAndGetConnection) +{ + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); + MessageMetadata conn_metadata("127.0.0.1", 8080, conn_flags); + auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG); + ASSERT_TRUE(maybe_connection.ok()); + + auto maybe_get_connection = i_conn->getPersistentConnection("127.0.0.1", 8080, MessageCategory::LOG); + ASSERT_TRUE(maybe_get_connection.ok()); + auto get_conn = maybe_get_connection.unpack(); + EXPECT_EQ(get_conn.getConnKey().getHostName(), "127.0.0.1"); + EXPECT_EQ(get_conn.getConnKey().getPort(), 8080); + EXPECT_EQ(get_conn.getConnKey().getCategory(), MessageCategory::LOG); +} + +TEST_F(TestConnectionComp, testEstablishNewConnection) +{ + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); + conn_flags.setFlag(MessageConnectionConfig::ONE_TIME_CONN); + MessageMetadata conn_metadata("127.0.0.1", 8080, conn_flags); + conn_metadata.setExternalCertificate("external cert"); + + auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG); + cout << "[OREN] read: " << endl; + ASSERT_TRUE(maybe_connection.ok()); + auto get_conn = maybe_connection.unpack(); + EXPECT_EQ(get_conn.getConnKey().getHostName(), "127.0.0.1"); +} + +TEST_F(TestConnectionComp, testSendRequest) +{ + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); + MessageMetadata conn_metadata("127.0.0.1", 8080, conn_flags); + + auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG); + ASSERT_TRUE(maybe_connection.ok()); + auto conn = maybe_connection.unpack(); + + auto req = HTTPRequest::prepareRequest(conn, HTTPMethod::POST, "/test", conn_metadata.getHeaders(), "test-body"); + ASSERT_TRUE(req.ok()); + + ON_CALL(mock_mainloop, yield(false)) + .WillByDefault( + InvokeWithoutArgs( + [&] () + { + dummy_socket.acceptSocket(); + dummy_socket.writeToSocket("HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\nmy-test"); + } + ) + ); + + EXPECT_CALL(mock_timer, getMonotonicTime()) + .WillRepeatedly(Invoke([] () {static int j = 0; return chrono::microseconds(++j * 1000 * 1000);})); + auto maybe_response = i_conn->sendRequest(conn, *req); + ASSERT_TRUE(maybe_response.ok()); + EXPECT_EQ((*maybe_response).getBody(), "my-test"); + + string expected_msg = + "POST /test HTTP/1.1\r\n" + "Accept-Encoding: identity\r\n" + "Authorization: Bearer accesstoken\r\n" + "Connection: keep-alive\r\n" + "Content-Length: 9\r\n" + "Content-type: application/json\r\n" + "Host: 127.0.0.1\r\n" + "\r\n" + "test-body"; + EXPECT_EQ(dummy_socket.readFromSocket(), expected_msg); +} + +TEST_F(TestConnectionComp, testSendRequestReplyChunked) +{ + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); + MessageMetadata conn_metadata("127.0.0.1", 8080, conn_flags); + + auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG); + ASSERT_TRUE(maybe_connection.ok()); + auto conn = maybe_connection.unpack(); + + auto req = HTTPRequest::prepareRequest(conn, HTTPMethod::POST, "/test", conn_metadata.getHeaders(), "test-body"); + ASSERT_TRUE(req.ok()); + + ON_CALL(mock_mainloop, yield(false)) + .WillByDefault( + InvokeWithoutArgs( + [&] () + { + dummy_socket.acceptSocket(); + string msg = + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "3\r\n" + "my-\r\n" + "4\r\n" + "test\r\n" + "0\r\n" + "\r\n"; + dummy_socket.writeToSocket(msg); + } + ) + ); + + EXPECT_CALL(mock_timer, getMonotonicTime()) + .WillRepeatedly(Invoke([] () {static int j = 0; return chrono::microseconds(++j * 1000 * 1000);})); + auto maybe_response = i_conn->sendRequest(conn, *req); + ASSERT_TRUE(maybe_response.ok()); + EXPECT_EQ((*maybe_response).getHTTPStatusCode(), HTTPStatusCode::HTTP_OK); + EXPECT_EQ((*maybe_response).getBody(), "my-test"); + EXPECT_EQ((*maybe_response).toString(), "[Status-code]: 200 - HTTP_OK, [Body]: my-test"); +} + +TEST_F(TestConnectionComp, testEstablishNewProxyConnection) +{ + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); + MessageMetadata conn_metadata("1.1.1.1", 9000, conn_flags); + + MessageProxySettings proxy_settings("127.0.0.1", "oren", 8080); + conn_metadata.setProxySettings(proxy_settings); + + //ON_CALL(mock_encryptor, base64Encode("oren")).WillByDefault(Return("encoded_oren")); + EXPECT_CALL(mock_encryptor, base64Encode("oren")).WillRepeatedly(Return("encoded_oren")); + + ON_CALL(mock_mainloop, yield(false)) + .WillByDefault( + InvokeWithoutArgs( + [&] () + { + dummy_socket.acceptSocket(); + dummy_socket.writeToSocket("HTTP/1.1 200 OK\r\n\r\n"); + } + ) + ); + + auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG); +} diff --git a/core/messaging/include/buffered_message.h b/core/messaging/include/buffered_message.h new file mode 100644 index 0000000..d8fe1fc --- /dev/null +++ b/core/messaging/include/buffered_message.h @@ -0,0 +1,63 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __BUFFERED_MESSAGE_H__ +#define __BUFFERED_MESSAGE_H__ + +#include +#include +#include + +#include "i_messaging.h" + +#include "cereal/archives/json.hpp" +#include "cereal/types/string.hpp" +#include "maybe_res.h" + +class BufferedMessage +{ +public: + BufferedMessage() {} + + BufferedMessage( + std::string _body, + HTTPMethod _method, + std::string _uri, + MessageCategory _category, + MessageMetadata _message_metadata + ) : + body(_body), method(_method), uri(_uri), category(_category), message_metadata(_message_metadata) + {} + + void save(cereal::JSONOutputArchive &out_ar) const; + void load(cereal::JSONInputArchive &archive_in); + + std::string toString() const; + bool operator==(const BufferedMessage &other) const; + + const std::string &getBody() const; + const std::string &getURI() const; + HTTPMethod getMethod() const; + MessageCategory getCategory() const; + const MessageMetadata &getMessageMetadata() const; + +private: + std::string body; + HTTPMethod method; + std::string uri; + MessageCategory category; + MessageMetadata message_metadata; + uint16_t retries_number = 0; +}; + +#endif // __BUFFERED_MESSAGE_H__ diff --git a/core/messaging/include/connection.h b/core/messaging/include/connection.h new file mode 100644 index 0000000..28ca1b5 --- /dev/null +++ b/core/messaging/include/connection.h @@ -0,0 +1,75 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __CONNECTION_H__ +#define __CONNECTION_H__ + +#include +#include +#include + +#include "i_agent_details.h" +#include "i_environment.h" +#include "i_mainloop.h" +#include "i_time_get.h" +#include "messaging/http_response.h" +#include "messaging/messaging_metadata.h" + +#include "maybe_res.h" + +class MessageConnectionKey +{ +public: + MessageConnectionKey() {} + + MessageConnectionKey(const std::string &_host_name, uint16_t _port, MessageCategory _category) : + host_name(_host_name), port(_port), category(_category) + {} + + const std::string & getHostName() const; + uint16_t getPort() const; + const MessageCategory & getCategory() const; + + bool operator<(const MessageConnectionKey &other) const; + +private: + std::string host_name; + uint16_t port; + MessageCategory category; +}; + +class Connection +{ +public: + Connection(const MessageConnectionKey &conn_key, const MessageMetadata &metadata); + ~Connection(); + + Maybe setProxySettings(const MessageProxySettings &settings); + void setExternalCertificate(const std::string &certificate); + const MessageProxySettings & getProxySettings() const; + const std::string & getExternalCertificate() const; + + const MessageConnectionKey &getConnKey() const; + bool isOverProxy() const; + bool isUnsecure() const; + bool isSuspended(); + + Maybe establishConnection(); + Maybe sendRequest(const std::string &request); + +private: + class Impl; + std::shared_ptr pimpl; +}; + +#endif // __CONNECTION_H__ diff --git a/core/include/internal/messaging_buffer.h b/core/messaging/include/connection_comp.h similarity index 56% rename from core/include/internal/messaging_buffer.h rename to core/messaging/include/connection_comp.h index 20b7152..1195652 100644 --- a/core/include/internal/messaging_buffer.h +++ b/core/messaging/include/connection_comp.h @@ -11,42 +11,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef __MESSAGING_BUFFER_H__ -#define __MESSAGING_BUFFER_H__ +#ifndef __CONNECTION_COMPONENT_H__ +#define __CONNECTION_COMPONENT_H__ -#include +#include +#include -#include "i_messaging_buffer.h" -#include "i_time_get.h" -#include "i_instance_awareness.h" -#include "singleton.h" +#include "i_agent_details.h" #include "i_encryptor.h" #include "i_environment.h" #include "i_mainloop.h" -#include "component.h" +#include "i_messaging.h" +#include "i_time_get.h" +#include "interfaces/i_messaging_connection.h" -class MessagingBuffer - : - public Component, - Singleton::Provide, - Singleton::Consume, - Singleton::Consume, - Singleton::Consume, - Singleton::Consume, - Singleton::Consume +#include "maybe_res.h" + +class ConnectionComponent : Singleton::Provide { public: - MessagingBuffer(); - ~MessagingBuffer(); - - void preload(); - + ConnectionComponent(); + ~ConnectionComponent(); void init(); - void fini(); private: class Impl; std::unique_ptr pimpl; }; -#endif // __MESSAGING_BUFFER_H__ +#endif // __CONNECTION_COMPONENT_H__ diff --git a/core/messaging/include/http_request.h b/core/messaging/include/http_request.h new file mode 100644 index 0000000..9eeca07 --- /dev/null +++ b/core/messaging/include/http_request.h @@ -0,0 +1,68 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __HTTP_REQUEST_H__ +#define __HTTP_REQUEST_H__ + +#include +#include + +#include "connection.h" +#include "i_agent_details.h" +#include "i_encryptor.h" +#include "i_messaging.h" + +class HTTPRequest +{ +public: + static Maybe prepareRequest( + const Connection &conn, + HTTPMethod method, + const std::string &uri, + const std::map &headers, + const std::string &body + ); + + Maybe setConnectionHeaders(const Connection &conn, bool is_access_token_needed); + + bool + isConnect() const + { + return method == HTTPMethod::CONNECT; + } + + std::string toString() const; + +private: + HTTPRequest( + HTTPMethod _method, + const std::string &_uri, + const std::map &_headers, + const std::string &_body + ) : + body(_body), uri(_uri), headers(_headers), method(_method) + {} + + bool setConnectionHeaders(const Connection &conn); + void insertHeader(const std::string &header_key, const std::string &header_val); + Maybe addAccessToken(const Connection &conn, bool is_registration); + Maybe addProxyAuthorization(const Connection &conn); + + std::string body; + std::string uri; + std::string method_statement; + std::map headers; + HTTPMethod method; +}; + +#endif // __HTTP_REQUEST_H__ diff --git a/core/include/internal/messaging_buffer/http_request_event.h b/core/messaging/include/http_request_event.h similarity index 98% rename from core/include/internal/messaging_buffer/http_request_event.h rename to core/messaging/include/http_request_event.h index 006ce20..51848d4 100644 --- a/core/include/internal/messaging_buffer/http_request_event.h +++ b/core/messaging/include/http_request_event.h @@ -18,6 +18,8 @@ #include "cereal/archives/json.hpp" +// LCOV_EXCL_START + class HTTPRequestSignature { public: @@ -117,4 +119,6 @@ private: std::string body; }; +// LCOV_EXCL_STOP + #endif // __HTTP_REQUEST_EVENT_H__ diff --git a/core/include/services_sdk/interfaces/i_messaging_buffer.h b/core/messaging/include/interfaces/i_messaging_buffer.h similarity index 57% rename from core/include/services_sdk/interfaces/i_messaging_buffer.h rename to core/messaging/include/interfaces/i_messaging_buffer.h index f7be084..539f43a 100644 --- a/core/include/services_sdk/interfaces/i_messaging_buffer.h +++ b/core/messaging/include/interfaces/i_messaging_buffer.h @@ -14,20 +14,32 @@ #ifndef __I_MESSAGING_BUFFER_H__ #define __I_MESSAGING_BUFFER_H__ -#include "messaging_buffer/http_request_event.h" +#include +#include +#include + +#include "i_messaging.h" + +#include "../buffered_message.h" +#include "cereal/archives/json.hpp" +#include "cereal/types/string.hpp" #include "maybe_res.h" -class I_MessagingBuffer +class I_MessageBuffer { public: - virtual Maybe peekRequest() = 0; - virtual void popRequest() = 0; - virtual void bufferNewRequest(const HTTPRequestEvent &request, bool is_rejected = false) = 0; - virtual bool isPending(const HTTPRequestSignature &request) = 0; - virtual void cleanBuffer() = 0; + virtual void pushNewBufferedMessage( + const std::string &body, + HTTPMethod method, + const std::string &uri, + MessageCategory category, + MessageMetadata message_metadata, + bool force_immediate_writing + ) = 0; -protected: - ~I_MessagingBuffer() {} + virtual Maybe peekMessage() = 0; + virtual void popMessage() = 0; + virtual void cleanBuffer() = 0; }; #endif // __I_MESSAGING_BUFFER_H__ diff --git a/core/messaging/include/interfaces/i_messaging_connection.h b/core/messaging/include/interfaces/i_messaging_connection.h new file mode 100644 index 0000000..5fa050a --- /dev/null +++ b/core/messaging/include/interfaces/i_messaging_connection.h @@ -0,0 +1,42 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __I_MESSAGING_CONNECTION_H__ +#define __I_MESSAGING_CONNECTION_H__ + +#include +#include + +#include "http_request.h" +#include "i_messaging.h" + +#include "connection.h" +#include "maybe_res.h" + +class I_MessagingConnection +{ +public: + virtual Maybe establishConnection(const MessageMetadata &metadata, MessageCategory category) = 0; + + virtual Maybe getPersistentConnection( + const std::string &host_name, uint16_t port, MessageCategory category + ) = 0; + + virtual Maybe getFogConnectionByCategory(MessageCategory category) = 0; + virtual Maybe sendRequest(Connection &connection, HTTPRequest request) = 0; + +protected: + virtual ~I_MessagingConnection() {} +}; + +#endif // __I_MESSAGING_CONNECTION_H__ diff --git a/core/message/i_message_decoder.h b/core/messaging/include/messaging_buffer.h similarity index 59% rename from core/message/i_message_decoder.h rename to core/messaging/include/messaging_buffer.h index 549133d..7860013 100644 --- a/core/message/i_message_decoder.h +++ b/core/messaging/include/messaging_buffer.h @@ -11,16 +11,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef __I_MESSAGE_DECODER_H__ -#define __I_MESSAGE_DECODER_H__ +#ifndef __MESSAGNIG_BUFFER_H__ +#define __MESSAGNIG_BUFFER_H__ +#include #include -template -class I_MessageDecoder +#include "i_environment.h" +#include "interfaces/i_messaging_buffer.h" +#include "singleton.h" + +class MessagingBufferComponent : Singleton::Provide { public: - virtual Maybe decodeBytes(const std::string &data) = 0; + MessagingBufferComponent(); + ~MessagingBufferComponent(); + + void init(); + +private: + class Impl; + std::unique_ptr pimpl; }; -#endif // __I_MESSAGE_DECODER_H__ +#endif // __MESSAGNIG_BUFFER_H__ diff --git a/core/messaging/include/messaging_comp.h b/core/messaging/include/messaging_comp.h new file mode 100644 index 0000000..f432004 --- /dev/null +++ b/core/messaging/include/messaging_comp.h @@ -0,0 +1,101 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __MESSAGNIG_COMP_H__ +#define __MESSAGNIG_COMP_H__ + +#include "messaging.h" + +#include +#include +#include +#include +#include + +#include "cache.h" +#include "connection.h" +#include "connection_comp.h" +#include "flags.h" +#include "interfaces/i_messaging_buffer.h" +#include "interfaces/i_messaging_connection.h" +#include "maybe_res.h" +#include "messaging_buffer.h" +#include "singleton.h" + +class MessagingComp +{ +public: + void init(); + + Maybe sendSyncMessage( + HTTPMethod method, + const std::string &uri, + const std::string &body, + MessageCategory category, + const MessageMetadata &message_metadata + ); + + void sendAsyncMessage( + HTTPMethod method, + const std::string &uri, + const std::string &body, + MessageCategory category, + const MessageMetadata &message_metadata + ); + + Maybe downloadFile( + HTTPMethod method, + const std::string &uri, + const std::string &download_file_path, + MessageCategory category = MessageCategory::GENERIC, + const MessageMetadata &message_metadata = MessageMetadata() + ); + + Maybe uploadFile( + const std::string &uri, + const std::string &upload_file_path, + MessageCategory category, + const MessageMetadata &message_metadata + ); + + bool setFogConnection(const std::string &host, uint16_t port, bool is_secure, MessageCategory category); + bool setFogConnection(MessageCategory category); + +private: + Maybe getConnection(MessageCategory category, const MessageMetadata &message_metadata); + Maybe getPersistentConnection(const MessageMetadata &metadata, MessageCategory category) const; + + Maybe sendMessage( + HTTPMethod method, + const std::string &uri, + const std::string &body, + MessageCategory category, + const MessageMetadata &message_metadata + ); + + Maybe suspendMessage( + const std::string &body, + HTTPMethod method, + const std::string &uri, + MessageCategory category, + const MessageMetadata &message_metadata + ) const; + + I_MessagingConnection *i_conn; + I_MessageBuffer *i_messaging_buffer; + I_AgentDetails *agent_details; + bool should_buffer_failed_messages; + TemporaryCache fog_get_requests_cache; +}; + +#endif //__MESSAGNIG_COMP_H__ diff --git a/core/messaging/include/mocks/mock_messaging_buffer.h b/core/messaging/include/mocks/mock_messaging_buffer.h new file mode 100644 index 0000000..8c04d80 --- /dev/null +++ b/core/messaging/include/mocks/mock_messaging_buffer.h @@ -0,0 +1,25 @@ +#ifndef __MOCK_MESSAGING_BUFFER_H__ +#define __MOCK_MESSAGING_BUFFER_H__ + +#include "cptest.h" +#include "interfaces/i_messaging_buffer.h" + +// LCOV_EXCL_START Reason: No need to test mocks + +class MockMessagingBuffer : public Singleton::Provide::From> +{ +public: + using string = std::string; + MOCK_METHOD6( + pushNewBufferedMessage, + void(const std::string &, HTTPMethod, const std::string &, MessageCategory, MessageMetadata, bool) + ); + + MOCK_METHOD0(peekMessage, Maybe()); + MOCK_METHOD0(popMessage, void()); + MOCK_METHOD0(cleanBuffer, void()); +}; + +// LCOV_EXCL_STOP + +#endif // __MOCK_MESSAGING_BUFFER_H__ diff --git a/core/messaging/include/mocks/mock_messaging_connection.h b/core/messaging/include/mocks/mock_messaging_connection.h new file mode 100644 index 0000000..5572cc1 --- /dev/null +++ b/core/messaging/include/mocks/mock_messaging_connection.h @@ -0,0 +1,38 @@ +#ifndef __MOCK_MESSAGING_CONNECTION_H__ +#define __MOCK_MESSAGING_CONNECTION_H__ + +#include "cptest.h" +#include "interfaces/i_messaging_connection.h" + +// LCOV_EXCL_START Reason: No need to test mocks + +class MockMessagingConnection : + public Singleton::Provide::From> +{ +public: + using string = std::string; + + MOCK_METHOD2(establishConnection, Maybe(const MessageMetadata &, MessageCategory)); + + MOCK_METHOD3( + establishNewConnection, + Maybe(MessageConnectionKey, Flags, const string &) + ); + + MOCK_METHOD2(establishNewProxyConnection, Maybe(Flags, MessageProxySettings)); + + Maybe + sendRequest(Connection &conn, HTTPRequest req) + { + return mockSendRequest(conn, req, false); + } + + MOCK_METHOD3(mockSendRequest, Maybe(Connection &, HTTPRequest, bool)); + + MOCK_METHOD3(getPersistentConnection, Maybe(const string &, uint16_t, MessageCategory)); + MOCK_METHOD1(getFogConnectionByCategory, Maybe(MessageCategory)); +}; + +// LCOV_EXCL_STOP + +#endif // __MOCK_MESSAGING_CONNECTION_H__ diff --git a/core/messaging/include/response_parser.h b/core/messaging/include/response_parser.h new file mode 100644 index 0000000..7e86e87 --- /dev/null +++ b/core/messaging/include/response_parser.h @@ -0,0 +1,34 @@ +#ifndef __RESPONSE_PARSER_H__ +#define __RESPONSE_PARSER_H__ + +#include "i_messaging.h" + +class HTTPResponseParser +{ +public: + Maybe parseData(const std::string &data, bool is_connect); + + bool + hasReachedError() const + { + return error; + } + +private: + bool parseStatusLine(); + bool handleHeaders(); + bool handleBody(bool is_connect); + + Maybe getHeaderVal(const std::string &header_key); + + bool getChunkedResponse(); + bool isLegalChunkedResponse(const std::string &res); + + Maybe status_code = genError("Not received"); + Maybe> headers = genError("Not received"); + std::string body; + std::string raw_response; + bool error = false; +}; + +#endif // __RESPONSE_PARSER_H__ diff --git a/core/message/smart_bio.h b/core/messaging/include/smart_bio.h similarity index 91% rename from core/message/smart_bio.h rename to core/messaging/include/smart_bio.h index 34c9367..6b4284a 100644 --- a/core/message/smart_bio.h +++ b/core/messaging/include/smart_bio.h @@ -14,17 +14,18 @@ #ifndef __SMART_BIO_H__ #define __SMART_BIO_H__ -#include "openssl/ssl.h" -#include "openssl/err.h" #include "openssl/bio.h" +#include "openssl/err.h" +#include "openssl/ssl.h" #include "openssl/x509v3.h" namespace smartBIO { -template struct Destrctor; +template +struct Destrctor; -template<> +template <> struct Destrctor { void @@ -35,7 +36,7 @@ struct Destrctor }; // LCOV_EXCL_START Reason: No ssl ut -template<> +template <> struct Destrctor { void @@ -45,7 +46,7 @@ struct Destrctor } }; -template<> +template <> struct Destrctor { void @@ -55,7 +56,7 @@ struct Destrctor } }; -template<> +template <> struct Destrctor { void @@ -66,9 +67,9 @@ struct Destrctor }; // LCOV_EXCL_STOP -template +template using BioUniquePtr = std::unique_ptr>; -} // namespace SmartBIO +} // namespace smartBIO #endif // __SMART_BIO_H__ diff --git a/core/messaging/messaging.cc b/core/messaging/messaging.cc new file mode 100644 index 0000000..bacc71e --- /dev/null +++ b/core/messaging/messaging.cc @@ -0,0 +1,132 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "messaging.h" + +#include + +#include "connection_comp.h" +#include "interfaces/i_messaging_connection.h" +#include "messaging_buffer.h" +#include "messaging_comp.h" + +using namespace std; + +USE_DEBUG_FLAG(D_MESSAGING); + +// LCOV_EXCL_START Reason: This wrapper for the other components, all logic is tested there. + +class Messaging::Impl : Singleton::Provide::From +{ +public: + void + init() + { + messaging_comp.init(); + connection_comp.init(); + messaging_buffer_comp.init(); + } + + Maybe + sendSyncMessage( + HTTPMethod method, + const std::string &uri, + const std::string &body, + MessageCategory category, + MessageMetadata message_metadata + ) override + { + return messaging_comp.sendSyncMessage(method, uri, body, category, message_metadata); + } + + void + sendAsyncMessage( + const HTTPMethod method, + const std::string &uri, + const std::string &body, + const MessageCategory category, + MessageMetadata message_metadata + ) override + { + return messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata); + } + + Maybe + downloadFile( + const HTTPMethod method, + const std::string &uri, + const std::string &download_file_path, + const MessageCategory category, + MessageMetadata message_metadata + ) override + { + return messaging_comp.downloadFile(method, uri, download_file_path, category, message_metadata); + } + + Maybe + uploadFile( + const std::string &uri, + const std::string &upload_file_path, + const MessageCategory category, + MessageMetadata message_metadata + ) override + { + return messaging_comp.uploadFile(uri, upload_file_path, category, message_metadata); + } + + bool + setFogConnection(const string &host, uint16_t port, bool is_secure, MessageCategory category) override + { + return messaging_comp.setFogConnection(host, port, is_secure, category); + } + + bool + setFogConnection(MessageCategory category = MessageCategory::GENERIC) override + { + return messaging_comp.setFogConnection(category); + } + +private: + MessagingComp messaging_comp; + ConnectionComponent connection_comp; + MessagingBufferComponent messaging_buffer_comp; +}; + +Messaging::Messaging() : Component("Messaging"), pimpl(make_unique()) +{} + +Messaging::~Messaging() +{} + +void +Messaging::init() +{ + pimpl->init(); +} + +void +Messaging::preload() +{ + registerExpectedConfiguration("message", "Cache timeout"); + registerExpectedConfiguration("message", "Connection timeout"); + registerExpectedConfiguration("message", "Connection handshake timeout"); + registerExpectedConfiguration("message", "Verify SSL pinning"); + registerExpectedConfiguration("message", "Buffer Failed Requests"); + registerExpectedConfiguration("message", "Certificate chain file path"); + registerExpectedConfiguration("message", "Trusted CA directory"); + registerExpectedConfiguration("message", "Public key path"); + registerExpectedConfiguration("message", "Data printout type"); + registerExpectedConfiguration("message", "Data printout length"); +} + +// LCOV_EXCL_STOP diff --git a/core/messaging/messaging_buffer_comp/CMakeLists.txt b/core/messaging/messaging_buffer_comp/CMakeLists.txt new file mode 100644 index 0000000..ca2a467 --- /dev/null +++ b/core/messaging/messaging_buffer_comp/CMakeLists.txt @@ -0,0 +1,2 @@ +add_library(messaging_buffer_comp messaging_buffer_comp.cc buffered_message.cc) +add_subdirectory(messaging_buffer_comp_ut) diff --git a/core/messaging/messaging_buffer_comp/buffered_message.cc b/core/messaging/messaging_buffer_comp/buffered_message.cc new file mode 100644 index 0000000..0ae6897 --- /dev/null +++ b/core/messaging/messaging_buffer_comp/buffered_message.cc @@ -0,0 +1,128 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "buffered_message.h" +#include "customized_cereal_map.h" + +using namespace std; + +static const std::map string_to_category = { + {"generic", MessageCategory::GENERIC }, + { "log", MessageCategory::LOG }, + { "debug", MessageCategory::DEBUG }, + { "metric", MessageCategory::METRIC }, + { "intelligence", MessageCategory::INTELLIGENCE} +}; + +static const std::map category_to_string = { + {MessageCategory::GENERIC, "generic" }, + { MessageCategory::LOG, "log" }, + { MessageCategory::DEBUG, "debug" }, + { MessageCategory::METRIC, "metric" }, + { MessageCategory::INTELLIGENCE, "intelligence"} +}; + +static const std::map string_to_method = { + {"get", HTTPMethod::GET }, + { "post", HTTPMethod::POST }, + { "patch", HTTPMethod::PATCH }, + { "connect", HTTPMethod::CONNECT}, + { "put", HTTPMethod::PUT } +}; + +static const std::map method_to_string = { + {HTTPMethod::GET, "get" }, + { HTTPMethod::POST, "post" }, + { HTTPMethod::PATCH, "patch" }, + { HTTPMethod::CONNECT, "connect"}, + { HTTPMethod::PUT, "put" } +}; + +void +BufferedMessage::save(cereal::JSONOutputArchive &out_ar) const +{ + string category_str = category_to_string.find(category)->second; + string method_str = method_to_string.find(method)->second; + + out_ar( + cereal::make_nvp("body", body), + cereal::make_nvp("uri", uri), + cereal::make_nvp("method", method_str), + cereal::make_nvp("category", category_str), + cereal::make_nvp("message_metadata", message_metadata) + ); +} + +void +BufferedMessage::load(cereal::JSONInputArchive &archive_in) +{ + string method_str; + string category_str; + archive_in( + cereal::make_nvp("body", body), + cereal::make_nvp("uri", uri), + cereal::make_nvp("method", method_str), + cereal::make_nvp("category", category_str), + cereal::make_nvp("message_metadata", message_metadata) + ); + method = string_to_method.find(method_str)->second; + category = string_to_category.find(category_str)->second; +} + +string +BufferedMessage::toString() const +{ + stringstream ss; + { + cereal::JSONOutputArchive ar(ss); + save(ar); + } + + return ss.str(); +} + +bool +BufferedMessage::operator==(const BufferedMessage &other) const +{ + return body == other.body && uri == other.uri; +} + +const string & +BufferedMessage::getBody() const +{ + return body; +} + +const string & +BufferedMessage::getURI() const +{ + return uri; +} + +HTTPMethod +BufferedMessage::getMethod() const +{ + return method; +} + +MessageCategory +BufferedMessage::getCategory() const +{ + return category; +} + +const MessageMetadata & +BufferedMessage::getMessageMetadata() const +{ + return message_metadata; +} diff --git a/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc b/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc new file mode 100644 index 0000000..a5d7fc3 --- /dev/null +++ b/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc @@ -0,0 +1,544 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "messaging_buffer.h" +#include "messaging.h" +#include "http_request_event.h" + +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "debug.h" + +using namespace std; + +USE_DEBUG_FLAG(D_MESSAGING_BUFFER); + +#ifndef smb +static constexpr uint buffer_max_size_MB = 100; +#else +static constexpr uint buffer_max_size_MB = 3; +#endif + +static bool +checkExistence(const string &path) +{ + try { + struct stat info; + if (stat(path.c_str(), &info) != 0) return false; + return info.st_mode & S_IFREG; + } catch (exception &e) { + return false; + } +} + +class MessagingBufferComponent::Impl : Singleton::Provide::From +{ +public: + void init(); + + void pushNewBufferedMessage( + const string &body, + HTTPMethod method, + const string &uri, + MessageCategory category, + MessageMetadata message_metadata, + bool force_immediate_writing + ) override; + + Maybe peekMessage() override; + + void popMessage() override; + + void cleanBuffer() override; + +private: + void handleBufferedMessages(); + bool sendMessage(); + HTTPStatusCode sendMessage(const BufferedMessage &message) const; + + void handleInMemoryMessages(); + + void writeToDisk(const BufferedMessage &message); + + static Maybe seekStartOfMessage(FILE *file); + static bool readBytes(FILE *file, uint size_to_read, char *output_bytes); + + bool canWriteToDisk(size_t message_size) const; + Maybe getDirectorySize() const; + + // LCOV_EXCL_START - Converting old formats to new format will be added later + static Maybe convertStringToHTTPMethod(const string &method_string); + void removeLegacyBuffer(const string &root_path, const string &exec_name); + void convertLegacyBuffer(const string &body_file_path); + Maybe serializeOldData(const string &data); + // LCOV_EXCL_STOP + + vector memory_messages; + string buffer_input; + string buffer_output; + string buffer_root_path; + uint max_size_on_disk_MB = 0; + uint curr_no_retries = 0; + I_ShellCmd *shell_cmd = nullptr; + I_Encryptor *encryptor = nullptr; + I_MainLoop *mainloop = nullptr; + I_Messaging *messaging = nullptr; +}; + +void +MessagingBufferComponent::Impl::init() +{ + max_size_on_disk_MB = getProfileAgentSettingWithDefault(buffer_max_size_MB, "eventBuffer.maxSizeOnDiskInMB"); + shell_cmd = Singleton::Consume::by(); + encryptor = Singleton::Consume::by(); + mainloop = Singleton::Consume::by(); + messaging = Singleton::Consume::from(); + + auto sub_path = getProfileAgentSettingWithDefault("nano_agent/event_buffer/", "eventBuffer.baseFolder"); + buffer_root_path = getLogFilesPathConfig() + "/" + sub_path; + string full_executable_name = + Singleton::Consume::by()->get("Executable Name").unpack(); + string executable_name = full_executable_name.substr(full_executable_name.find_last_of("/") + 1); + removeLegacyBuffer(buffer_root_path, executable_name); + mkdir(buffer_root_path.c_str(), 0644); + + auto *instance_awareness = Singleton::Consume::by(); + string unique_id = instance_awareness->getInstanceID().ok() ? instance_awareness->getInstanceID().unpack() : ""; + buffer_input = buffer_root_path + "/" + executable_name + unique_id + ".input"; + buffer_output = buffer_root_path + "/" + executable_name + unique_id + ".output"; + memory_messages.reserve(32); + + uint tmo = getConfigurationWithDefault(5, "message", "Send event retry in sec"); + mainloop->addRecurringRoutine( + I_MainLoop::RoutineType::Timer, + chrono::seconds(tmo), + [this] () { handleBufferedMessages(); }, + "A-sync messaging routine", + false + ); + mainloop->addRecurringRoutine( + I_MainLoop::RoutineType::Timer, + chrono::seconds(2), + [this] () { handleInMemoryMessages(); }, + "Handling in-memory messages", + false + ); +} + +void +MessagingBufferComponent::Impl::pushNewBufferedMessage( + const string &body, + HTTPMethod method, + const string &uri, + MessageCategory category, + MessageMetadata message_metadata, + bool force_immediate_writing +) +{ + dbgTrace(D_MESSAGING_BUFFER) << "Pushing new message to buffer"; + + message_metadata.setShouldBufferMessage(false); + + if (!force_immediate_writing) { + dbgDebug(D_MESSAGING_BUFFER) << "Holding message temporarily in memory"; + memory_messages.emplace_back(body, method, uri, category, message_metadata); + return; + } + + BufferedMessage buffered_message(body, method, uri, category, message_metadata); + writeToDisk(buffered_message); +} + +Maybe +MessagingBufferComponent::Impl::peekMessage() +{ + auto move_cmd = + "if [ -s " + buffer_input + " ] && [ ! -s " + buffer_output + " ];" + "then mv " + buffer_input + " " + buffer_output + ";" + "fi"; + + shell_cmd->getExecOutput(move_cmd); + + if (!checkExistence(buffer_output)) return genError(buffer_output + " does not exist"); + + FILE *file = fopen(buffer_output.c_str(), "rb"); + if (file == nullptr) { + dbgWarning(D_MESSAGING_BUFFER) << "Failed to open file for reading. File: " << buffer_output; + cleanBuffer(); + return genError("Failed to open file"); + } + + auto possition = seekStartOfMessage(file); + if (!possition.ok()) { + fclose(file); + dbgDebug(D_MESSAGING_BUFFER) << "Failed to find message start: " << possition.getErr(); + cleanBuffer(); + return possition.passErr(); + } + + string buffer; + buffer.resize(*possition); + auto read = readBytes(file, *possition, const_cast(buffer.data())); + fclose(file); + if (!read) { + cleanBuffer(); + return genError("Filed to read the message"); + } + + BufferedMessage message; + try { + stringstream ss(buffer); + cereal::JSONInputArchive ar(ss); + message.load(ar); + } catch (const cereal::Exception &e) { + string err = e.what(); + dbgError(D_MESSAGING_BUFFER) << "Parsing backlog error: " << err; + cleanBuffer(); + return genError("Filed to parse the message: " + err); + } + + return message; +} + +void +MessagingBufferComponent::Impl::popMessage() +{ + dbgTrace(D_MESSAGING_BUFFER) << "Popping message from buffer"; + + FILE *file = fopen(buffer_output.c_str(), "rb"); + if (file == nullptr) { + dbgWarning(D_MESSAGING_BUFFER) << "Failed to open file for reading. File: " << buffer_input; + return; + } + + auto possition = seekStartOfMessage(file); + auto new_size = ftell(file); + fclose(file); + if (!possition.ok()) { + dbgDebug(D_MESSAGING_BUFFER) << "Failed to find message start: " << possition.getErr(); + return; + } + + int result = truncate(buffer_output.c_str(), new_size); + if (result == 0) { + dbgTrace(D_MESSAGING_BUFFER) << "File truncated successfully."; + } else { + dbgTrace(D_MESSAGING_BUFFER) << "Error truncating the file: " << strerror(errno); + } +} + +void +MessagingBufferComponent::Impl::cleanBuffer() +{ + dbgTrace(D_MESSAGING_BUFFER) << "Cleaning buffer"; + remove(buffer_input.c_str()); + remove(buffer_output.c_str()); +} + +void +MessagingBufferComponent::Impl::handleBufferedMessages() +{ + while (true) { + if (!sendMessage()) return; + mainloop->yield(); + } +} + +bool +MessagingBufferComponent::Impl::sendMessage() +{ + const Maybe &maybe_msg_to_send = peekMessage(); + if (!maybe_msg_to_send.ok()) { + dbgDebug(D_MESSAGING) << "Peeking failed: " << maybe_msg_to_send.getErr(); + return false; + } + + auto res = sendMessage(*maybe_msg_to_send); + + if (res == HTTPStatusCode::HTTP_OK) { + dbgDebug(D_MESSAGING) << "Successfully sent buffered message"; + popMessage(); + curr_no_retries = 0; + return true; + } + + if (res == HTTPStatusCode::HTTP_SUSPEND) { + dbgDebug(D_MESSAGING) << "Suspended connection - sleeping for a while"; + mainloop->yield(chrono::seconds(1)); + return true; + } + + ++curr_no_retries; + if (curr_no_retries >= getProfileAgentSettingWithDefault(10, "eventBuffer.maxNumOfSendigRetries")) { + dbgWarning(D_MESSAGING) << "Reached maximum number of retries - poping message"; + popMessage(); + curr_no_retries = 0; + } + return true; +} + +HTTPStatusCode +MessagingBufferComponent::Impl::sendMessage(const BufferedMessage &message) const +{ + auto res = messaging->sendSyncMessage( + message.getMethod(), + message.getURI(), + message.getBody(), + message.getCategory(), + message.getMessageMetadata() + ); + + if (res.ok()) return HTTPStatusCode::HTTP_OK; + if (res.getErr().getHTTPStatusCode() == HTTPStatusCode::HTTP_SUSPEND) return HTTPStatusCode::HTTP_SUSPEND; + return HTTPStatusCode::HTTP_UNKNOWN; +} + +void +MessagingBufferComponent::Impl::handleInMemoryMessages() +{ + auto messages = move(memory_messages); + memory_messages.reserve(32); + + for (const auto &message : messages) { + if (sendMessage(message) != HTTPStatusCode::HTTP_OK) writeToDisk(message); + mainloop->yield(); + } +} + +void +MessagingBufferComponent::Impl::writeToDisk(const BufferedMessage &message) +{ + auto serialized_message = message.toString(); + + if (!canWriteToDisk(serialized_message.size())) { + dbgWarning(D_MESSAGING_BUFFER) << "Buffer is full. Message will not be written to disk: " << message.getURI(); + return; + } + + ofstream file(buffer_input, ios::app); + if (!file.is_open()) { + dbgWarning(D_MESSAGING_BUFFER) << "Failed to open file for writing. File: " << buffer_input; + return; + } + + uint32_t size = serialized_message.size(); + file.write(serialized_message.data(), size); + file.write(reinterpret_cast(&size), sizeof(size)); + char type = 0; + file.write(&type, 1); +} + +Maybe +MessagingBufferComponent::Impl::seekStartOfMessage(FILE *file) +{ + int type_size = sizeof(char); + int lenght_size = sizeof(uint32_t); + + if (fseek(file, -type_size, SEEK_END) != 0) return genError("Failed to get to type byte"); + char type; + if (!readBytes(file, type_size, &type)) return genError("Failed to read type"); + if (type != 0) return genError("Only type 0 is currently supported"); + + if (fseek(file, -(type_size + lenght_size), SEEK_END) != 0) return genError("Failed to get to length bytes"); + uint32_t length; + if (!readBytes(file, lenght_size, reinterpret_cast(&length))) return genError("Failed to read length"); + + int total_offset = type_size + lenght_size + length; + if (ftell(file) == total_offset) { + if (fseek(file, 0, SEEK_SET) != 0) return genError("Failed to get to the start of the file"); + } else { + if (fseek(file, -total_offset, SEEK_END) != 0) return genError("Failed to get to message start"); + } + + return length; +} + +bool +MessagingBufferComponent::Impl::readBytes(FILE *file, uint size_to_read, char *output) +{ + for (uint index = 0; index < size_to_read; ++index) { + int ch = fgetc(file); + if (ch == EOF) return false; + output[index] = static_cast(ch); + } + return true; +} + +Maybe +MessagingBufferComponent::Impl::getDirectorySize() const +{ + DIR *dir = opendir(buffer_root_path.c_str()); + if (dir == nullptr) { + return genError("Unable to open directory: " + buffer_root_path); + } + + uint total_size = 0; + struct dirent *entry; + while ((entry = readdir(dir)) != nullptr) { + if (entry->d_type == DT_REG) { + struct stat file_info; + string tmp_file_path = buffer_root_path + "/" + entry->d_name; + if (stat(tmp_file_path.c_str(), &file_info) == 0) { + total_size += file_info.st_size; + } else { + return genError("Error retrieving file size. " + tmp_file_path); + } + } + } + + closedir(dir); + return total_size; +} + +// LCOV_EXCL_START - Converting old formats to new format will be added later +Maybe +MessagingBufferComponent::Impl::convertStringToHTTPMethod(const string &method_string) +{ + if (method_string == "GET") { + return HTTPMethod::GET; + } else if (method_string == "POST") { + return HTTPMethod::POST; + } else if (method_string == "PATCH") { + return HTTPMethod::PATCH; + } else if (method_string == "CONNECT") { + return HTTPMethod::CONNECT; + } else if (method_string == "PUT") { + return HTTPMethod::PUT; + } else { + return genError("Unknown HTTP method"); + } +} + +Maybe +MessagingBufferComponent::Impl::serializeOldData(const string &data) +{ + try { + stringstream in; + in.str(data); + cereal::JSONInputArchive in_ar(in); + + HTTPRequestEvent req; + req.load(in_ar); + return req; + } catch (cereal::Exception &e) { + return genError("JSON parsing failed: " + string(e.what())); + } catch (exception &e) { + return genError(e.what()); + } +} + +void +MessagingBufferComponent::Impl::convertLegacyBuffer(const string &body_file_path) +{ + ifstream file(body_file_path); + if (!file.is_open()) { + dbgTrace(D_MESSAGING_BUFFER) << "No body file found: " << body_file_path; + return; + } + + string request; + while (getline(file, request)) { + auto http_request_event = serializeOldData(encryptor->base64Decode(request)); + if (!http_request_event.ok()) { + dbgWarning(D_MESSAGING_BUFFER) << "Error to serialize http_request_event: " << http_request_event.getErr(); + continue; + } + + auto http_method = convertStringToHTTPMethod(http_request_event.unpack().getMethod()); + if (!http_method.ok()) { + dbgWarning(D_MESSAGING_BUFFER) << "Error to convert http_method: " << http_method.getErr(); + continue; + } + + pushNewBufferedMessage( + http_request_event.unpack().getBody(), + http_method.unpack(), + http_request_event.unpack().getURL(), + MessageCategory::GENERIC, + MessageMetadata(), + true + ); + } +} + +void +MessagingBufferComponent::Impl::removeLegacyBuffer(const string &root_path, const string &executable_name) +{ + string file_path = root_path + "manager" + executable_name; + ifstream file(file_path); + if (!file.is_open()) { + dbgTrace(D_MESSAGING_BUFFER) << "No legacy MessagingBuffer buffers found: " << file_path; + return; + } + + string line; + while (getline(file, line)) { + dbgTrace(D_MESSAGING_BUFFER) << "Line: " << line; + string body_file_path = root_path + line + executable_name; + convertLegacyBuffer(body_file_path); + if (remove(body_file_path.c_str()) == 0) { + dbgDebug(D_MESSAGING_BUFFER) << "File successfully removed: " << body_file_path; + } else { + dbgWarning(D_MESSAGING_BUFFER) << "Failed to remove file: " << body_file_path; + } + } + + file.close(); + if (remove(file_path.c_str()) == 0) { + dbgDebug(D_MESSAGING_BUFFER) << "Manager file successfully removed: " << file_path; + } else { + dbgWarning(D_MESSAGING_BUFFER) << "Failed to remove file manager: " << file_path; + } +} +// LCOV_EXCL_STOP + +bool +MessagingBufferComponent::Impl::canWriteToDisk(size_t message_size) const +{ + dbgTrace(D_MESSAGING_BUFFER) << "Handling buffer size in disk"; + auto maybe_directory_size = getDirectorySize(); + if (!maybe_directory_size.ok()) { + dbgWarning(D_MESSAGING_BUFFER) << "Failed to get directory size. " << maybe_directory_size.getErr(); + return false; + } + if ((*maybe_directory_size + message_size) < (max_size_on_disk_MB * 1024 * 1024)) { + return true; + } + + dbgWarning(D_MESSAGING_BUFFER) + << "Buffer size is full. Directry size: " + << *maybe_directory_size + << ", Message size: " + << message_size + << ", Max size: " + << max_size_on_disk_MB * 1024 * 1024; + return false; +} + +void +MessagingBufferComponent::init() +{ + pimpl->init(); +} + +MessagingBufferComponent::MessagingBufferComponent() : pimpl(make_unique()) +{} + +MessagingBufferComponent::~MessagingBufferComponent() +{} diff --git a/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/CMakeLists.txt b/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/CMakeLists.txt new file mode 100644 index 0000000..7484680 --- /dev/null +++ b/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/CMakeLists.txt @@ -0,0 +1,7 @@ +add_unit_test( + messaging_buffer_comp_ut + "messaging_buffer_comp_ut.cc" + "messaging_buffer_comp;messaging_comp;metric;event_is;boost_regex;shell_cmd;time_proxy;agent_details;instance_awareness" +) + +add_subdirectory(test_data) diff --git a/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc b/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc new file mode 100644 index 0000000..b3fbfa5 --- /dev/null +++ b/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc @@ -0,0 +1,264 @@ +#include "messaging_buffer.h" + +#include "agent_core_utilities.h" +#include "agent_details.h" +#include "config.h" +#include "config_component.h" +#include "cptest.h" +#include "environment.h" +#include "agent_details.h" +#include "instance_awareness.h" +#include "mock/mock_mainloop.h" +#include "mock/mock_messaging.h" +#include "mock/mock_tenant_manager.h" +#include "mock/mock_encryptor.h" +#include "shell_cmd.h" +#include "time_proxy.h" + +using namespace std; +using namespace testing; + +string removeWhitespaces(const std::string &str); + +class TestMessagingBuffer : public Test +{ +public: + TestMessagingBuffer() + { + env.preload(); + Singleton::Consume::from(env)->registerValue("Executable Name", "tmp_test_file"); + + config.preload(); + config.init(); + + string config_json = + "{" + " \"agentSettings\": [\n" + " {\n" + " \"id\": \"123\",\n" + " \"key\": \"eventBuffer.maxSizeOnDiskInMB\",\n" + " \"value\": \"1\"\n" + " },\n" + " {\n" + " \"id\": \"123\",\n" + " \"key\": \"eventBuffer.baseFolder\",\n" + " \"value\": \"../.." + cptestFnameInExeDir("test_data") + "\"\n" + " }]\n" + "}"; + + istringstream ss(config_json); + Singleton::Consume::from(config)->loadConfiguration({"--id=8"}); + Singleton::Consume::from(config)->loadConfiguration(ss); + + EXPECT_CALL(mock_mainloop, addRecurringRoutine(_, _, _, "A-sync messaging routine", _)) + .WillOnce(DoAll(SaveArg<2>(&async_routine), Return(0))); + + EXPECT_CALL(mock_mainloop, addRecurringRoutine(_, _, _, "Handling in-memory messages", _)) + .WillOnce(DoAll(SaveArg<2>(&memory_routine), Return(0))); + + buffer_comp.init(); + buffer_provider = Singleton::Consume::from(buffer_comp); + + agent_details.setFogDomain("fog_domain"); + agent_details.setFogPort(443); + } + + ~TestMessagingBuffer() { buffer_provider->cleanBuffer(); } + + NiceMock tenant_manager; + NiceMock mock_mainloop; + NiceMock mock_messaging; + ConfigComponent config; + MessagingBufferComponent buffer_comp; + ::Environment env; + ShellCmd shell_cmd; + StrictMock m_encryptor; + TimeProxyComponent time_proxy; + AgentDetails agent_details; + InstanceAwareness instance_awareness; + I_MessageBuffer *buffer_provider; + I_MainLoop::Routine async_routine; + I_MainLoop::Routine memory_routine; +}; + +TEST_F(TestMessagingBuffer, testPeekingEmptyBuffer) +{ + auto msg = buffer_provider->peekMessage(); + ASSERT_FALSE(msg.ok()); +} + +static bool +checkExistence(const string &path) +{ + try { + struct stat info; + if (stat(path.c_str(), &info) != 0) return false; + return info.st_mode & S_IFREG; + } catch (exception &e) { + return false; + } +} + +TEST_F(TestMessagingBuffer, testPushOneBuffer) +{ + string body = "body"; + string uri = "uri"; + MessageCategory category = MessageCategory::GENERIC; + MessageMetadata message_metadata = MessageMetadata(); + HTTPMethod method = HTTPMethod::POST; + + buffer_provider->pushNewBufferedMessage(body, method, uri, category, message_metadata, true); + + auto msg = buffer_provider->peekMessage(); + ASSERT_TRUE(msg.ok()); + + BufferedMessage expected(body, method, uri, category, message_metadata); + EXPECT_EQ(*msg, expected); + EXPECT_TRUE(checkExistence(cptestFnameInExeDir("test_data") + "/tmp_test_file8.output")); +} + +TEST_F(TestMessagingBuffer, testMultiplePushesAndPulls) +{ + string uri = "uri"; + MessageCategory category = MessageCategory::GENERIC; + MessageMetadata message_metadata = MessageMetadata(); + HTTPMethod method = HTTPMethod::POST; + + string body1 = "body1"; + string body2 = "body2"; + string body3 = "body3"; + string body4 = "body4"; + string body5 = "body5"; + + BufferedMessage expected1(body1, method, uri, category, message_metadata); + BufferedMessage expected2(body2, method, uri, category, message_metadata); + BufferedMessage expected3(body3, method, uri, category, message_metadata); + BufferedMessage expected4(body4, method, uri, category, message_metadata); + BufferedMessage expected5(body5, method, uri, category, message_metadata); + + buffer_provider->pushNewBufferedMessage(body1, method, uri, category, message_metadata, true); + buffer_provider->pushNewBufferedMessage(body2, method, uri, category, message_metadata, true); + + auto msg = buffer_provider->peekMessage(); + ASSERT_TRUE(msg.ok()); + EXPECT_EQ(*msg, expected2); + buffer_provider->popMessage(); + + buffer_provider->pushNewBufferedMessage(body3, method, uri, category, message_metadata, true); + buffer_provider->pushNewBufferedMessage(body4, method, uri, category, message_metadata, true); + + msg = buffer_provider->peekMessage(); + ASSERT_TRUE(msg.ok()); + EXPECT_EQ(*msg, expected1); + buffer_provider->popMessage(); + + msg = buffer_provider->peekMessage(); + ASSERT_TRUE(msg.ok()); + EXPECT_EQ(*msg, expected4); + buffer_provider->popMessage(); + + + msg = buffer_provider->peekMessage(); + ASSERT_TRUE(msg.ok()); + EXPECT_EQ(*msg, expected3); + buffer_provider->popMessage(); + + buffer_provider->pushNewBufferedMessage(body5, method, uri, category, message_metadata, true); + + msg = buffer_provider->peekMessage(); + ASSERT_TRUE(msg.ok()); + EXPECT_EQ(*msg, expected5); + buffer_provider->popMessage(); + + msg = buffer_provider->peekMessage(); + ASSERT_FALSE(msg.ok()); +} + +TEST_F(TestMessagingBuffer, testPushMoreThanAllowed) +{ + string body_1 = "body"; + string body_2 = string(1024 * 1024 * 1, 'a'); // 1MB + string body_3 = "body"; + string uri_1 = "uri_1"; + string uri_2 = "uri_2"; + string uri_3 = "uri_3"; + MessageCategory category = MessageCategory::GENERIC; + MessageMetadata message_metadata = MessageMetadata(); + HTTPMethod method = HTTPMethod::POST; + + BufferedMessage expected1(body_1, method, uri_1, category, message_metadata); + BufferedMessage expected3(body_3, method, uri_3, category, message_metadata); + + buffer_provider->pushNewBufferedMessage(body_1, method, uri_1, category, message_metadata, true); + buffer_provider->pushNewBufferedMessage(body_2, method, uri_2, category, message_metadata, true); + buffer_provider->pushNewBufferedMessage(body_3, method, uri_3, category, message_metadata, true); + + auto msg = buffer_provider->peekMessage(); + ASSERT_TRUE(msg.ok()); + EXPECT_EQ(*msg, expected3); + buffer_provider->popMessage(); + + msg = buffer_provider->peekMessage(); + ASSERT_TRUE(msg.ok()); + EXPECT_EQ(*msg, expected1); + buffer_provider->popMessage(); + + msg = buffer_provider->peekMessage(); + ASSERT_FALSE(msg.ok()); +} + +TEST_F(TestMessagingBuffer, testRoutinePulling) +{ + string body_1 = "body1"; + string body_2 = "body2"; + string uri_1 = "uri_1"; + string uri_2 = "uri_2"; + MessageCategory category = MessageCategory::GENERIC; + MessageMetadata message_metadata = MessageMetadata(); + HTTPMethod method = HTTPMethod::POST; + + buffer_provider->pushNewBufferedMessage(body_1, method, uri_1, category, message_metadata, true); + buffer_provider->pushNewBufferedMessage(body_2, method, uri_2, category, message_metadata, true); + + HTTPResponse res(HTTPStatusCode::HTTP_OK, ""); + + EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_1, body_1, _, _)).WillOnce(Return(res)); + EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_2, body_2, _, _)).WillOnce(Return(res)); + + async_routine(); +} + +TEST_F(TestMessagingBuffer, testRoutinInMemory) +{ + string body_1 = "body1"; + string body_2 = "body2"; + string body_3 = "body3"; + string uri_1 = "uri_1"; + string uri_2 = "uri_2"; + string uri_3 = "uri_3"; + + MessageCategory category = MessageCategory::GENERIC; + MessageMetadata message_metadata = MessageMetadata(); + HTTPMethod method = HTTPMethod::POST; + + buffer_provider->pushNewBufferedMessage(body_1, method, uri_1, category, message_metadata, false); + buffer_provider->pushNewBufferedMessage(body_2, method, uri_2, category, message_metadata, false); + buffer_provider->pushNewBufferedMessage(body_3, method, uri_3, category, message_metadata, false); + + HTTPResponse res(HTTPStatusCode::HTTP_OK, ""); + Maybe err = genError(res); + + EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_1, body_1, _, _)).WillOnce(Return(res)); + EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_2, body_2, _, _)).WillOnce(Return(err)); + EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_3, body_3, _, _)).WillOnce(Return(res)); + + memory_routine(); + + auto msg = buffer_provider->peekMessage(); + ASSERT_TRUE(msg.ok()); + EXPECT_EQ(*msg, BufferedMessage(body_2, method, uri_2, category, message_metadata)); + buffer_provider->popMessage(); + + msg = buffer_provider->peekMessage(); + ASSERT_FALSE(msg.ok()); +} diff --git a/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/test_data/CMakeLists.txt b/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/test_data/CMakeLists.txt new file mode 100644 index 0000000..139597f --- /dev/null +++ b/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/test_data/CMakeLists.txt @@ -0,0 +1,2 @@ + + diff --git a/core/messaging/messaging_comp/CMakeLists.txt b/core/messaging/messaging_comp/CMakeLists.txt new file mode 100644 index 0000000..d2d6eed --- /dev/null +++ b/core/messaging/messaging_comp/CMakeLists.txt @@ -0,0 +1,5 @@ +add_library(messaging_comp + messaging_comp.cc + http_request.cc + http_response.cc) +add_subdirectory(messaging_comp_ut) diff --git a/core/messaging/messaging_comp/http_request.cc b/core/messaging/messaging_comp/http_request.cc new file mode 100644 index 0000000..27451f2 --- /dev/null +++ b/core/messaging/messaging_comp/http_request.cc @@ -0,0 +1,149 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "http_request.h" + +#include +#include + +#include "messaging.h" + +using namespace std; + +USE_DEBUG_FLAG(D_MESSAGING); + +void +HTTPRequest::insertHeader(const string &header_key, const string &header_val) +{ + headers[header_key] = header_val; +} + +bool +HTTPRequest::setConnectionHeaders(const Connection &conn) +{ + string host = conn.getConnKey().getHostName(); + string uri_prefix = conn.isOverProxy() ? "http://" + host : ""; + + switch (method) { + case HTTPMethod::GET: { + method_statement = "GET " + uri_prefix + uri + " HTTP/1.1"; + break; + } + case HTTPMethod::POST: { + method_statement = "POST " + uri_prefix + uri + " HTTP/1.1"; + break; + } + case HTTPMethod::PATCH: { + method_statement = "PATCH " + uri_prefix + uri + " HTTP/1.1"; + break; + } + case HTTPMethod::PUT: { + method_statement = "PUT " + uri_prefix + uri + " HTTP/1.1"; + break; + } + case HTTPMethod::CONNECT: { + host = host + ":" + to_string(conn.getConnKey().getPort()); + method_statement = "CONNECT " + host + " HTTP/1.1"; + break; + } + default: { + return false; + } + } + + insertHeader("Host", host); + insertHeader("Content-Length", to_string(body.size())); + insertHeader("Content-type", "application/json"); + insertHeader("Accept-Encoding", "identity"); + if (headers.find("Connection") == headers.end()) { + insertHeader("Connection", "keep-alive"); + } + return true; +} + +Maybe +HTTPRequest::prepareRequest( + const Connection &conn, + HTTPMethod method, + const string &uri, + const map &headers, + const string &body +) +{ + HTTPRequest req(method, uri, headers, body); + + if (!req.setConnectionHeaders(conn)) return genError("Failed to identify the HTTP method"); + + string agent_registration_query = R"("authenticationMethod": "token")"; + bool dont_add_access_token = false; + if (method == HTTPMethod::CONNECT || body.find(agent_registration_query) != string::npos) { + dont_add_access_token = true; + dbgTrace(D_MESSAGING) << "Request is for agent authentication"; + } + auto res = req.addAccessToken(conn, dont_add_access_token); + if (!res.ok()) return res.passErr(); + + if (conn.isOverProxy()) { + auto res = req.addProxyAuthorization(conn); + if (!res.ok()) return res.passErr(); + } + + return req; +} + +string +HTTPRequest::toString() const +{ + stringstream res; + res << method_statement << "\r\n"; + for (const auto &header : headers) { + res << header.first << ": " << header.second << "\r\n"; + } + res << "\r\n" << body; + return res.str(); +} + +Maybe +HTTPRequest::addAccessToken(const Connection &conn, bool dont_add_access_token) +{ + if (headers.find("Authorization") != headers.end() || dont_add_access_token) return Maybe(); + + if (!conn.getExternalCertificate().empty()) { + insertHeader("Authorization", conn.getExternalCertificate()); + return Maybe(); + } + + string access_token = Singleton::Consume::by()->getAccessToken(); + if (access_token.empty()) return genError("Access token is missing."); + insertHeader("Authorization", "Bearer " + access_token); + return Maybe(); +} + +Maybe +HTTPRequest::addProxyAuthorization(const Connection &conn) +{ + insertHeader("Accept", "*/*"); + insertHeader("Proxy-Connection", "Keep-Alive"); + + if (!conn.isUnsecure()) return Maybe(); + + MessageProxySettings proxy_settings = conn.getProxySettings(); + if (proxy_settings.getProxyAuth().empty()) { + dbgTrace(D_MESSAGING) << "No proxy authentication was set"; + return Maybe(); + } + + I_Encryptor *encryptor = Singleton::Consume::by(); + insertHeader("Proxy-Authorization", "Basic " + encryptor->base64Encode(proxy_settings.getProxyAuth())); + return Maybe(); +} diff --git a/core/messaging/messaging_comp/http_response.cc b/core/messaging/messaging_comp/http_response.cc new file mode 100644 index 0000000..3fcf2d9 --- /dev/null +++ b/core/messaging/messaging_comp/http_response.cc @@ -0,0 +1,286 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "response_parser.h" + +#include +#include +#include + +using namespace std; + +USE_DEBUG_FLAG(D_MESSAGING); + +static const string CRLF = "\r\n"; + +static const map status_code_to_string = { + {HTTPStatusCode::NO_HTTP_RESPONSE, "0 - NO_HTTP_RESPONSE" }, + { HTTPStatusCode::HTTP_OK, "200 - HTTP_OK" }, + { HTTPStatusCode::HTTP_NO_CONTENT, "204 - HTTP_NO_CONTENT" }, + { HTTPStatusCode::HTTP_MULTI_STATUS, "207 - HTTP_MULTI_STATUS" }, + { HTTPStatusCode::HTTP_BAD_REQUEST, "400 - HTTP_BAD_REQUEST" }, + { HTTPStatusCode::HTTP_UNAUTHORIZED, "401 - HTTP_UNAUTHORIZED" }, + { HTTPStatusCode::HTTP_FORBIDDEN, "403 - HTTP_FORBIDDEN" }, + { HTTPStatusCode::HTTP_NOT_FOUND, "404 - HTTP_NOT_FOUND" }, + { HTTPStatusCode::HTTP_METHOD_NOT_ALLOWED, "405 - HTTP_METHOD_NOT_ALLOWED" }, + { HTTPStatusCode::HTTP_PROXY_AUTHENTICATION_REQUIRED, "407 - HTTP_PROXY_AUTHENTICATION_REQUIRED" }, + { HTTPStatusCode::HTTP_REQUEST_TIME_OUT, "408 - HTTP_REQUEST_TIME_OUT" }, + { HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE, "413 - HTTP_PAYLOAD_TOO_LARGE" }, + { HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR, "500 - HTTP_INTERNAL_SERVER_ERROR" }, + { HTTPStatusCode::HTTP_NOT_IMPLEMENTED, "501 - HTTP_NOT_IMPLEMENTED" }, + { HTTPStatusCode::HTTP_BAD_GATEWAY, "502 - HTTP_BAD_GATEWAY" }, + { HTTPStatusCode::HTTP_SERVICE_UNABAILABLE, "503 - HTTP_SERVICE_UNABAILABLE" }, + { HTTPStatusCode::HTTP_GATEWAY_TIMEOUT, "504 - HTTP_GATEWAY_TIMEOUT" }, + { HTTPStatusCode::HTTP_VERSION_NOT_SUPPORTED, "505 - HTTP_VERSION_NOT_SUPPORTED" }, + { HTTPStatusCode::HTTP_VARIANT_ALSO_NEGOTIATES, "506 - HTTP_VARIANT_ALSO_NEGOTIATES" }, + { HTTPStatusCode::HTTP_INSUFFICIENT_STORAGE, "507 - HTTP_INSUFFICIENT_STORAGE" }, + { HTTPStatusCode::HTTP_LOOP_DETECTED, "508 - HTTP_LOOP_DETECTED" }, + { HTTPStatusCode::HTTP_NOT_EXTENDED, "510 - HTTP_NOT_EXTENDED" }, + { HTTPStatusCode::HTTP_NETWORK_AUTHENTICATION_REQUIRED, "511 - HTTP_NETWORK_AUTHENTICATION_REQUIRED"}, + { HTTPStatusCode::HTTP_UNKNOWN, "-1 - HTTP_UNKNOWN" }, + { HTTPStatusCode::HTTP_SUSPEND, "-2 - HTTP_SUSPEND" } +}; + +static const map num_to_status_code = { + {200, HTTPStatusCode::HTTP_OK }, + { 204, HTTPStatusCode::HTTP_NO_CONTENT }, + { 207, HTTPStatusCode::HTTP_MULTI_STATUS }, + { 400, HTTPStatusCode::HTTP_BAD_REQUEST }, + { 401, HTTPStatusCode::HTTP_UNAUTHORIZED }, + { 403, HTTPStatusCode::HTTP_FORBIDDEN }, + { 404, HTTPStatusCode::HTTP_NOT_FOUND }, + { 405, HTTPStatusCode::HTTP_METHOD_NOT_ALLOWED }, + { 407, HTTPStatusCode::HTTP_PROXY_AUTHENTICATION_REQUIRED }, + { 408, HTTPStatusCode::HTTP_REQUEST_TIME_OUT }, + { 413, HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE }, + { 500, HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR }, + { 501, HTTPStatusCode::HTTP_NOT_IMPLEMENTED }, + { 502, HTTPStatusCode::HTTP_BAD_GATEWAY }, + { 503, HTTPStatusCode::HTTP_SERVICE_UNABAILABLE }, + { 504, HTTPStatusCode::HTTP_GATEWAY_TIMEOUT }, + { 505, HTTPStatusCode::HTTP_VERSION_NOT_SUPPORTED }, + { 506, HTTPStatusCode::HTTP_VARIANT_ALSO_NEGOTIATES }, + { 507, HTTPStatusCode::HTTP_INSUFFICIENT_STORAGE }, + { 508, HTTPStatusCode::HTTP_LOOP_DETECTED }, + { 510, HTTPStatusCode::HTTP_NOT_EXTENDED }, + { 511, HTTPStatusCode::HTTP_NETWORK_AUTHENTICATION_REQUIRED} +}; + +const string & +HTTPResponse::getBody() const +{ + return body; +} + +HTTPStatusCode +HTTPResponse::getHTTPStatusCode() const +{ + return status_code; +} + +string +HTTPResponse::toString() const +{ + auto code = status_code_to_string.find(status_code); + dbgAssert(code != status_code_to_string.end()) << "Unknown status code " << int(status_code); + + return "[Status-code]: " + code->second + ", [Body]: " + (body.empty() ? "{}" : body); +} + +Maybe +HTTPResponseParser::parseData(const string &data, bool is_connect) +{ + if (data.empty()) return genError("Data is empty"); + raw_response += data; + + if (!status_code.ok()) { + if (!parseStatusLine()) return genError("Failed to parse the status line. Error: " + status_code.getErr()); + } + + if (!headers.ok()) { + if (!handleHeaders()) return genError("Failed to parse the headers. Error: " + headers.getErr()); + } + + if (!handleBody(is_connect)) return genError("Response not ready!"); + + return HTTPResponse(status_code.unpack(), body); +} + +static string +strip(const string &str) +{ + string res; + for (auto ch : str) { + if (!isspace(ch)) res += tolower(ch); + } + return res; +} + +bool +HTTPResponseParser::handleHeaders() +{ + stringstream ss(raw_response); + map header_map; + + while (true) { + string header; + getline(ss, header); + + if (header.empty()) { + headers = genError("Headers not complete"); + return false; + } + + if (header == "\r") { + headers = header_map; + ss.sync(); + raw_response = raw_response.substr(ss.tellg()); + return true; + } + + auto colon_index = header.find_first_of(":"); + if (colon_index == string::npos) { + // The only case where not finding a colon isn't an error is if we run out of data + error = !ss.str().empty(); + headers = genError(error ? "Invalid headers: " + header : "Did not reach end of headers"); + return false; + } + + auto header_key = header.substr(0, colon_index); + auto header_val = header.substr(colon_index + 2); + header_map[strip(header_key)] = strip(header_val); + } +} + +Maybe +HTTPResponseParser::getHeaderVal(const string &header_key) +{ + auto headers_map = headers.unpack(); + auto header = headers_map.find(header_key); + if (header == headers_map.end()) { + return genError("Header\'" + header_key + "\' not found."); + } + return header->second; +} + +bool +HTTPResponseParser::handleBody(bool is_connect) +{ + if (*status_code == HTTPStatusCode::HTTP_OK && is_connect) return true; + + if (*status_code == HTTPStatusCode::HTTP_NO_CONTENT) return raw_response.empty(); + + auto content_length = getHeaderVal("content-length"); + if (content_length.ok()) { + size_t body_length; + try { + body_length = stoi(content_length.unpack()); + } catch (const exception &err) { + return false; + } + + body += raw_response; + raw_response.clear(); + return body.size() == body_length; + } + + auto transfer_encoding = getHeaderVal("transfer-encoding"); + if (transfer_encoding.ok() && *transfer_encoding == "chunked") return getChunkedResponse(); + + dbgError(D_MESSAGING) << "Response has neither content-lenght nor chunked encoded"; + return false; +} + +bool +HTTPResponseParser::getChunkedResponse() +{ + if (!isLegalChunkedResponse(raw_response)) return false; + + size_t chunk_size = 0; + + for (auto line_end = raw_response.find(CRLF); line_end != string::npos; line_end = raw_response.find(CRLF)) { + string line = raw_response.substr(0, line_end); + try { + chunk_size = stoi(line, nullptr, 16); + } catch (const exception &) { + dbgWarning(D_MESSAGING) << "Failed to convert chunk length to a number. Line: " << line; + return false; + } + + if (line_end + 2 + chunk_size > raw_response.length()) { + dbgWarning(D_MESSAGING) << "Invalid chunked data structure - chunk-size is bigger than chunk-data"; + return false; + } + string chunk_body = raw_response.substr(line_end + 2, chunk_size); + raw_response = raw_response.substr(line_end + 2 + chunk_size); + + if (raw_response.find(CRLF) != 0) { + dbgWarning(D_MESSAGING) << "Invalid chunked data structure - chunk-data missing final CRLF sequence"; + return false; + } + raw_response = raw_response.substr(2); + + body += chunk_body; + } + + if (chunk_size != 0) { + dbgDebug(D_MESSAGING) << "Invalid chunked data structure - last-chunk of the body is not sized 0"; + return false; + } + return true; +} + +bool +HTTPResponseParser::isLegalChunkedResponse(const string &res) +{ + auto end_of_data = res.find("0\r\n\r\n"); + return end_of_data != string::npos && res.length() == end_of_data + 5; +} + +bool +HTTPResponseParser::parseStatusLine() +{ + auto end_of_first_line = raw_response.find(CRLF); + if (end_of_first_line == string::npos) { + status_code = genError("No Status Line was received."); + return false; + } + + auto status_line = raw_response.substr(0, end_of_first_line); + raw_response = raw_response.substr(end_of_first_line + 2); + + // Also status text can be supported at the future. + if (status_line.find("HTTP/1.") == string::npos) { + status_code = genError("Status code not found."); + error = true; + return false; + } + + int status_num; + try { + status_num = stoi(status_line.substr(9, 3)); + } catch (const exception &err) { + status_code = genError("Failed to convert status code to a number. Status code: " + status_line.substr(9, 3)); + error = true; + return false; + } + + auto status = num_to_status_code.find(status_num); + if (status != num_to_status_code.end()) { + status_code = status->second; + } else { + dbgWarning(D_MESSAGING) << "Unknown HTTP status code: " << status_num; + status_code = HTTPStatusCode::HTTP_UNKNOWN; + } + return true; +} diff --git a/core/messaging/messaging_comp/messaging_comp.cc b/core/messaging/messaging_comp/messaging_comp.cc new file mode 100644 index 0000000..5ad5df9 --- /dev/null +++ b/core/messaging/messaging_comp/messaging_comp.cc @@ -0,0 +1,320 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "messaging_comp.h" + +#include +#include + +#include "agent_core_utilities.h" +#include "connection_comp.h" +#include "debug.h" +#include "messaging_buffer.h" + +using namespace std; + +USE_DEBUG_FLAG(D_MESSAGING); + +void +MessagingComp::init() +{ + i_conn = Singleton::Consume::from(); + i_messaging_buffer = Singleton::Consume::from(); + agent_details = Singleton::Consume::by(); + + auto i_mainloop = Singleton::Consume::by(); + auto i_time_get = Singleton::Consume::by(); + auto cache_timeout = getConfigurationWithDefault(40, "message", "Cache timeout"); + fog_get_requests_cache.startExpiration(chrono::seconds(cache_timeout), i_mainloop, i_time_get); + + should_buffer_failed_messages = getConfigurationWithDefault( + getProfileAgentSettingWithDefault(true, "eventBuffer.bufferFailedRequests"), + "message", + "Buffer Failed Requests" + ); +} + +static bool +isMessageToFog(const MessageMetadata message_metadata) +{ + return message_metadata.isToFog(); +} + +Maybe +MessagingComp::getConnection(MessageCategory category, const MessageMetadata &metadata) +{ + auto persistant_conn = getPersistentConnection(metadata, category); + if (persistant_conn.ok()) { + dbgTrace(D_MESSAGING) << "Found a persistant connection"; + return persistant_conn; + } + dbgDebug(D_MESSAGING) << persistant_conn.getErr(); + + auto maybe_conn = i_conn->establishConnection(metadata, category); + if (!maybe_conn.ok()) { + dbgWarning(D_MESSAGING) << "Failed to establish connection: " << maybe_conn.getErr(); + } + return maybe_conn; +} + +Maybe +MessagingComp::sendMessage( + HTTPMethod method, + const string &uri, + const string &body, + MessageCategory category, + const MessageMetadata &message_metadata +) +{ + auto maybe_conn = getConnection(category, message_metadata); + if (!maybe_conn.ok()) { + dbgWarning(D_MESSAGING) << "Failed to get connection. Error: " << maybe_conn.getErr(); + return genError(HTTPStatusCode::HTTP_UNKNOWN, maybe_conn.getErr()); + } + + Connection conn = maybe_conn.unpack(); + if (conn.isSuspended()) return suspendMessage(body, method, uri, category, message_metadata); + + bool is_to_fog = isMessageToFog(message_metadata); + auto metadata = message_metadata; + if (is_to_fog) { + if (method == HTTPMethod::GET && fog_get_requests_cache.doesKeyExists(uri)) { + HTTPResponse res = fog_get_requests_cache.getEntry(uri); + dbgTrace(D_MESSAGING) << "Response returned from Fog cache. res body: " << res.getBody(); + + return fog_get_requests_cache.getEntry(uri); + } + + auto i_env = Singleton::Consume::by(); + metadata.insertHeader("User-Agent", "Infinity Next (a7030abf93a4c13)"); + metadata.insertHeaders(i_env->getCurrentHeadersMap()); + } + + auto req = HTTPRequest::prepareRequest(conn, method, uri, metadata.getHeaders(), body); + if (!req.ok()) return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, req.getErr())); + + auto response = i_conn->sendRequest(conn, *req); + if (!response.ok()) return response.passErr(); + + if (is_to_fog && method == HTTPMethod::GET) fog_get_requests_cache.emplaceEntry(uri, *response); + return response; +} + +Maybe +MessagingComp::sendSyncMessage( + HTTPMethod method, + const string &uri, + const string &body, + MessageCategory category, + const MessageMetadata &message_metadata +) +{ + Maybe is_msg_send = sendMessage(method, uri, body, category, message_metadata); + + if (is_msg_send.ok()) return *is_msg_send; + + if (should_buffer_failed_messages && message_metadata.shouldBufferMessage()) { + dbgTrace(D_MESSAGING) << "After sending error, buffering the message"; + i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false); + } + return is_msg_send.passErr(); +} + +void +MessagingComp::sendAsyncMessage( + HTTPMethod method, + const string &uri, + const string &body, + MessageCategory category, + const MessageMetadata &message_metadata +) +{ + i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false); +} + +Maybe +MessagingComp::downloadFile( + HTTPMethod method, + const string &uri, + const string &download_file_path, + MessageCategory category, + const MessageMetadata &message_metadata +) +{ + dbgTrace(D_MESSAGING) << "Send download file message"; + string parent_directory = download_file_path.substr(0, download_file_path.find_last_of("/\\")); + if (!NGEN::Filesystem::exists(parent_directory)) { + if (!NGEN::Filesystem::makeDirRecursive(parent_directory)) { + string creation_err = "Failed to create the parent directory. Path: " + parent_directory; + dbgWarning(D_MESSAGING) << creation_err; + return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, creation_err)); + } + } + + auto response = sendSyncMessage(method, uri, "", category, message_metadata); + if (!response.ok()) return response.passErr(); + + ofstream file_stream(download_file_path); + if (!file_stream.is_open()) { + string open_err = "Failed to open the destination file. Path: " + download_file_path; + dbgWarning(D_MESSAGING) << open_err; + return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, open_err)); + } + file_stream << response.unpack().getBody(); + file_stream.close(); + + dbgTrace(D_MESSAGING) << "Successfully downloaded and save file to: " << download_file_path; + return HTTPStatusCode::HTTP_OK; +} + +Maybe +MessagingComp::uploadFile( + const string &uri, + const string &upload_file_path, + MessageCategory category, + const MessageMetadata &message_metadata +) +{ + dbgTrace(D_MESSAGING) << "Send upload file message"; + + ifstream file(upload_file_path); + if (!file.is_open()) { + string open_err = "Failed to open the file to upload. Path: " + upload_file_path; + dbgWarning(D_MESSAGING) << open_err; + return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, open_err)); + } + + stringstream buffer; + buffer << file.rdbuf(); + file.close(); + + Maybe response = + sendSyncMessage(HTTPMethod::PUT, uri, buffer.str(), category, message_metadata); + + if (!response.ok()) return response.passErr(); + + dbgTrace(D_MESSAGING) << "Successfully upload file from: " << upload_file_path; + return HTTPStatusCode::HTTP_OK; +} + +bool +MessagingComp::setFogConnection(const string &host, uint16_t port, bool is_secure, MessageCategory category) +{ + dbgTrace(D_MESSAGING) << "Setting a fog connection to " << host << ":" << port; + MessageMetadata metadata(host, port, true); + + I_ProxyConfiguration *proxy_configuration = Singleton::Consume::by(); + auto load_env_proxy = proxy_configuration->loadProxy(); + if (!load_env_proxy.ok()) { + dbgDebug(D_MESSAGING) + << "Could not initialize load proxy from environment, Error: " + << load_env_proxy.getErr(); + } + + ProxyProtocol proxy_protocol = is_secure ? ProxyProtocol::HTTPS : ProxyProtocol::HTTP; + if (proxy_configuration->getProxyExists(proxy_protocol)) { + auto proxy_host = proxy_configuration->getProxyDomain(proxy_protocol); + auto proxy_port = proxy_configuration->getProxyPort(proxy_protocol); + auto maybe_proxy_auth = proxy_configuration->getProxyAuthentication(proxy_protocol); + + if (proxy_host.ok() && proxy_port.ok()) { + string proxy_auth = maybe_proxy_auth.ok() ? *maybe_proxy_auth : ""; + dbgDebug(D_MESSAGING) << "Setting proxy address: " << *proxy_host << ":" << *proxy_port; + MessageProxySettings proxy_settings(proxy_host.unpack(), proxy_auth, proxy_port.unpack()); + metadata.setProxySettings(proxy_settings); + } + } + + I_MessagingConnection *i_conn = Singleton::Consume::from(); + auto conn = i_conn->establishConnection(metadata, category); + if (!conn.ok()) { + dbgWarning(D_MESSAGING) << "Failed to establish connection to fog: " << conn.getErr(); + return false; + } + + dbgInfo(D_MESSAGING) + << "Successfully connected to the Fog: " + << host + << ":" + << port + << " via " + << (metadata.isProxySet() ? "proxy, using " : "") + << (is_secure ? "secure" : "clear") + << " connection"; + + return true; +} + +bool +MessagingComp::setFogConnection(MessageCategory category) +{ + I_AgentDetails *agent_details = Singleton::Consume::by(); + + if (agent_details->getOrchestrationMode() == OrchestrationMode::OFFLINE) { + dbgDebug(D_MESSAGING) << "Agent Is in offline mode and would not attempt connecting to the fog"; + return true; + } + + if (!agent_details->readAgentDetails()) { + dbgWarning(D_MESSAGING) << "Cannot establish connection to the Fog, failed to read agent details"; + return false; + } + + auto domain = agent_details->getFogDomain(); + auto port = agent_details->getFogPort(); + auto is_secure_connection = agent_details->getSSLFlag(); + + if (!domain.ok() || *domain == "" || !port.ok() || port == 0) { + dbgWarning(D_MESSAGING) << "Cannot establish connection to the Fog, failed to get host and port details"; + return false; + } + + return setFogConnection(*domain, *port, is_secure_connection, category); +} + +Maybe +MessagingComp::getPersistentConnection(const MessageMetadata &metadata, MessageCategory category) const +{ + if (!metadata.isToFog()) { + auto maybe_conn = i_conn->getPersistentConnection(metadata.getHostName(), metadata.getPort(), category); + if (maybe_conn.ok()) return *maybe_conn; + return genError("Failed to get persistant connection based on host and port"); + } + + auto maybe_conn = i_conn->getFogConnectionByCategory(category); + if (maybe_conn.ok()) return maybe_conn; + return genError("Failed to get persistant connection to the fog"); +} + +Maybe +MessagingComp::suspendMessage( + const string &body, + HTTPMethod method, + const string &uri, + MessageCategory category, + const MessageMetadata &message_metadata +) const +{ + if (message_metadata.shouldBufferMessage()) { + dbgWarning(D_MESSAGING) << "Buffering message due to connection suspended"; + i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false); + return genError( + HTTPStatusCode::HTTP_SUSPEND, + "The connection is suspended due to consecutive message sending errors, message is buffered." + ); + } + + return genError( + HTTPStatusCode::HTTP_SUSPEND, "The connection is suspended due to consecutive message sending errors." + ); +} diff --git a/core/messaging/messaging_comp/messaging_comp_ut/CMakeLists.txt b/core/messaging/messaging_comp/messaging_comp_ut/CMakeLists.txt new file mode 100644 index 0000000..4eb959e --- /dev/null +++ b/core/messaging/messaging_comp/messaging_comp_ut/CMakeLists.txt @@ -0,0 +1,5 @@ +add_unit_test( + messaging_comp_ut + "messaging_comp_ut.cc" + "messaging_comp;connection;messaging_buffer_comp;ssl;connkey;singleton;boost_context;rest;metric;event_is;-lboost_regex;agent_details;-lcrypto;" +) diff --git a/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc new file mode 100644 index 0000000..ec6e8d4 --- /dev/null +++ b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc @@ -0,0 +1,238 @@ +#include "messaging_comp.h" + +#include "agent_core_utilities.h" +#include "config.h" +#include "config_component.h" +#include "connection.h" +#include "cptest.h" +#include "environment.h" +#include "mainloop.h" +#include "mock/mock_agent_details.h" +#include "mock/mock_mainloop.h" +#include "mock/mock_time_get.h" +#include "mock/mock_proxy_configuration.h" +#include "mocks/mock_messaging_buffer.h" +#include "mocks/mock_messaging_connection.h" +#include "rest.h" +#include "rest_server.h" + +using namespace std; +using namespace testing; + +static ostream & +operator<<(ostream &os, const Maybe &) +{ + return os; +} + +static std::ostream & +operator<<(std::ostream &os, const HTTPResponse &) +{ + return os; +} + +static std::ostream & +operator<<(std::ostream &os, const HTTPStatusCode &) +{ + return os; +} + +static std::ostream & +operator<<(std::ostream &os, const Connection &) +{ + return os; +} + +class TestMessagingComp : public testing::Test +{ +public: + TestMessagingComp() + { + EXPECT_CALL(mock_time_get, getMonotonicTime()).WillRepeatedly(Return(chrono::microseconds(0))); + + ON_CALL(mock_agent_details, getFogDomain()).WillByDefault(Return(Maybe(fog_addr))); + ON_CALL(mock_agent_details, getFogPort()).WillByDefault(Return(Maybe(fog_port))); + messaging_comp.init(); + } + + void + setAgentDetails() + { + EXPECT_CALL(mock_agent_details, getFogDomain()).WillRepeatedly(Return(string(fog_addr))); + EXPECT_CALL(mock_agent_details, getFogPort()).WillRepeatedly(Return(fog_port)); + EXPECT_CALL(mock_agent_details, getOpenSSLDir()).WillRepeatedly(Return(string("/usr/lib/ssl/certs/"))); + EXPECT_CALL(mock_agent_details, getAccessToken()).WillRepeatedly(Return(string("accesstoken"))); + EXPECT_CALL(mock_agent_details, readAgentDetails()).WillRepeatedly(Return(true)); + EXPECT_CALL(mock_proxy_conf, loadProxy()).WillRepeatedly(Return(Maybe())); + EXPECT_CALL(mock_proxy_conf, getProxyExists(_)).WillRepeatedly(Return(true)); + EXPECT_CALL(mock_proxy_conf, getProxyDomain(_)).WillRepeatedly(Return(string("7.7.7.7"))); + EXPECT_CALL(mock_proxy_conf, getProxyPort(_)).WillRepeatedly(Return(8080)); + EXPECT_CALL(mock_proxy_conf, getProxyAuthentication(_)).WillRepeatedly(Return(string("cred"))); + } + + const string fog_addr = "1.2.3.4"; + uint16_t fog_port = 80; + CPTestTempfile agent_details_file; + MessagingComp messaging_comp; + ::Environment env; + ConfigComponent config; + NiceMock mock_messaging_connection; + NiceMock mock_messaging_buffer; + NiceMock mock_mainloop; + NiceMock mock_time_get; + NiceMock mock_agent_details; + NiceMock mock_proxy_conf; +}; + +TEST_F(TestMessagingComp, testInitComp) +{ + EXPECT_CALL( + mock_mainloop, addRecurringRoutine(I_MainLoop::RoutineType::Timer, _, _, "Delete expired cache entries", _) + ) + .WillOnce(Return(0)); + messaging_comp.init(); +} + +TEST_F(TestMessagingComp, testSendSyncMessage) +{ + setAgentDetails(); + string body = "test body"; + HTTPMethod method = HTTPMethod::POST; + string uri = "/test-uri"; + MessageCategory category = MessageCategory::GENERIC; + MessageMetadata message_metadata; + + MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC); + Connection conn(conn_key, message_metadata); + EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC)) + .WillOnce(Return(conn)); + + HTTPResponse res(HTTPStatusCode::HTTP_OK, "response!!"); + EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res)); + auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, message_metadata); + ASSERT_TRUE(sending_res.ok()); + HTTPResponse http_res = sending_res.unpack(); + EXPECT_EQ(http_res.getBody(), "response!!"); + EXPECT_EQ(http_res.getHTTPStatusCode(), HTTPStatusCode::HTTP_OK); +} + +TEST_F(TestMessagingComp, testSendAsyncMessage) +{ + setAgentDetails(); + string body = "test body"; + HTTPMethod method = HTTPMethod::POST; + string uri = "/test-uri"; + MessageCategory category = MessageCategory::GENERIC; + MessageMetadata message_metadata; + + EXPECT_CALL(mock_messaging_buffer, pushNewBufferedMessage(body, method, uri, category, _, _)).Times(1); + messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata); +} + +TEST_F(TestMessagingComp, testSendSyncMessageOnSuspendedConn) +{ + setAgentDetails(); + string body = "test body"; + HTTPMethod method = HTTPMethod::POST; + string uri = "/test-uri"; + MessageCategory category = MessageCategory::GENERIC; + MessageMetadata message_metadata; + + MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC); + Connection conn(conn_key, message_metadata); + + EXPECT_CALL(mock_time_get, getMonotonicTime()) + .WillRepeatedly(Invoke([] () { static int j = 0; return chrono::microseconds(++j * 1000 * 1000); })); + for (int i = 0; i < 20; i++) { + conn.sendRequest("."); + } + EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC)) + .WillOnce(Return(conn)); + + auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, message_metadata); + ASSERT_FALSE(sending_res.ok()); + HTTPResponse http_res = sending_res.getErr(); + EXPECT_EQ(http_res.getBody(), "The connection is suspended due to consecutive message sending errors."); + EXPECT_EQ(http_res.getHTTPStatusCode(), HTTPStatusCode::HTTP_SUSPEND); +} + +TEST_F(TestMessagingComp, testUploadFile) +{ + string path = cptestFnameInSrcDir("tests_files/file_to_send.txt"); + + setAgentDetails(); + string uri = "/test-uri"; + MessageCategory category = MessageCategory::GENERIC; + MessageMetadata message_metadata; + + MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC); + Connection conn(conn_key, message_metadata); + EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC)) + .WillOnce(Return(conn)); + + HTTPResponse res(HTTPStatusCode::HTTP_OK, ""); + EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res)); + auto upload_res = messaging_comp.uploadFile(uri, path, category, message_metadata); + ASSERT_TRUE(upload_res.ok()); + EXPECT_EQ(upload_res.unpack(), HTTPStatusCode::HTTP_OK); +} + +TEST_F(TestMessagingComp, testDownloadFile) +{ + string path = cptestFnameInSrcDir("tests_files/file_to_send.txt"); + + setAgentDetails(); + string uri = "/test-uri"; + HTTPMethod method = HTTPMethod::GET; + MessageCategory category = MessageCategory::GENERIC; + MessageMetadata message_metadata; + + MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC); + Connection conn(conn_key, message_metadata); + EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC)) + .WillOnce(Return(conn)); + + HTTPResponse res(HTTPStatusCode::HTTP_OK, ""); + EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res)); + auto upload_res = messaging_comp.downloadFile(method, uri, "/tmp/test.txt", category, message_metadata); + ASSERT_TRUE(upload_res.ok()); + EXPECT_EQ(upload_res.unpack(), HTTPStatusCode::HTTP_OK); +} + +bool +operator==(const MessageProxySettings &one, const MessageProxySettings &two) +{ + return + one.getProxyHost() == two.getProxyHost() && + one.getProxyAuth() == two.getProxyAuth() && + one.getProxyPort() == two.getProxyPort(); +} + +bool +operator==(const MessageMetadata &one, const MessageMetadata &two) +{ + return + one.getHostName() == two.getHostName() && + one.getPort() == two.getPort() && + one.getConnectionFlags() == two.getConnectionFlags() && + one.getProxySettings() == two.getProxySettings() && + one.getExternalCertificate() == two.getExternalCertificate() && + one.getHeaders() == two.getHeaders() && + one.shouldBufferMessage() == two.shouldBufferMessage() && + one.isProxySet() == two.isProxySet(); +} + +TEST_F(TestMessagingComp, testSetFogConnection) +{ + setAgentDetails(); + + MessageCategory category = MessageCategory::GENERIC; + MessageConnectionKey conn_key(fog_addr, fog_port, category); + MessageMetadata metadata(fog_addr, fog_port, true); + MessageProxySettings proxy_settings("7.7.7.7", "cred", 8080); + metadata.setProxySettings(proxy_settings); + Connection conn(conn_key, metadata); + + EXPECT_CALL(mock_messaging_connection, establishConnection(metadata, category)).WillOnce(Return(conn)); + EXPECT_TRUE(messaging_comp.setFogConnection(category)); +} diff --git a/core/messaging/messaging_comp/messaging_comp_ut/tests_files/file_to_send.txt b/core/messaging/messaging_comp/messaging_comp_ut/tests_files/file_to_send.txt new file mode 100644 index 0000000..5047baf --- /dev/null +++ b/core/messaging/messaging_comp/messaging_comp_ut/tests_files/file_to_send.txt @@ -0,0 +1 @@ +file to send \ No newline at end of file diff --git a/core/messaging_buffer/CMakeLists.txt b/core/messaging_buffer/CMakeLists.txt deleted file mode 100644 index 249e646..0000000 --- a/core/messaging_buffer/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -include_directories(${ng_module_osrc_zlib_path}/include) -include_directories(${Boost_INCLUDE_DIRS}) - -add_library(messaging_buffer messaging_buffer.cc event_queue.cc bucket_manager.cc) - -add_subdirectory(messaging_buffer_ut) diff --git a/core/messaging_buffer/bucket_manager.cc b/core/messaging_buffer/bucket_manager.cc deleted file mode 100644 index 9722abd..0000000 --- a/core/messaging_buffer/bucket_manager.cc +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "messaging_buffer/bucket_manager.h" - -#include "messaging_buffer.h" -#include "debug.h" -#include "config.h" - -using namespace std; - -USE_DEBUG_FLAG(D_EVENT_BUFFER); - -void -BucketManager::init(const string &_service_name) -{ - dbgTrace(D_EVENT_BUFFER) << "Initializing Bucket Manager: Service name: " << _service_name; - encryptor = Singleton::Consume::by(); - instance_awareness = Singleton::Consume::by(); - string log_files_prefix = getLogFilesPathConfig(); - - string buffer_dir_base_folder_setting = getProfileAgentSettingWithDefault( - log_files_prefix + "/nano_agent/event_buffer", - "eventBuffer.baseFolder" - ); - dbgTrace(D_EVENT_BUFFER) << "buffer dir base folder setting path: " << buffer_dir_base_folder_setting; - buffer_directory = getConfigurationWithDefault( - buffer_dir_base_folder_setting, - "Event Buffer", - "base folder" - ); - - uint buffer_max_size_base_settings = getProfileAgentSettingWithDefault( - 1000, - "eventBuffer.maxBufferSizeInMB" - ); - buffer_max_size = getConfigurationWithDefault( - buffer_max_size_base_settings, - "Event Buffer", - "max buffer size in MB" - ); - - uint max_buffer_files_base_settings = getProfileAgentSettingWithDefault(10, "eventBuffer.maxBufferFiles"); - max_buffer_files = getConfigurationWithDefault( - max_buffer_files_base_settings, - "Event Buffer", - "max buffer files" - ); - - service_name = _service_name; - management_file_path = resolveFilesName(buffer_directory + "/manager"); - iterator.init(management_file_path, buffer_max_size/max_buffer_files); -} - -void -BucketManager::fini() -{ - dbgTrace(D_EVENT_BUFFER) << "Finalizing Bucket Manager"; - iterator.fini(); - for (auto &bucket : buckets) { - bucket.second.fini(); - } -} - -bool -BucketManager::doesExist(const bucketName &name) -{ - dbgTrace(D_EVENT_BUFFER) << "Checking if bucket exists and containing data: Bucket name: " << name; - string base64_name = encryptor->base64Encode(name); - - if (buckets.find(base64_name) == buckets.end()) { - string management_file = resolveFilesName(buffer_directory + "/" + base64_name); - buckets.emplace( - piecewise_construct, - forward_as_tuple(base64_name), - forward_as_tuple() - ); - buckets[base64_name].init(management_file, buffer_max_size/max_buffer_files); - } - - return !buckets[base64_name].isEmpty(); -} - -void -BucketManager::push(const bucketName &name, string &&data) -{ - dbgTrace(D_EVENT_BUFFER) << "Pushing data into bucket: Bucket name: " << name; - string base64_name = encryptor->base64Encode(name); - if (buckets.find(base64_name) == buckets.end()) { - dbgTrace(D_EVENT_BUFFER) << "Bucket does not exist, creating new. Bucket name: " << name; - string management_file = resolveFilesName(buffer_directory + "/" + base64_name); - buckets.emplace( - piecewise_construct, - forward_as_tuple(base64_name), - forward_as_tuple() - ); - buckets[base64_name].init(management_file, buffer_max_size/max_buffer_files); - } - - string copy_name = base64_name; - string copy_data = encryptor->base64Encode(data); - buckets[base64_name].push(move(copy_data)); - iterator.push(move(copy_name)); - if (next_bucket.empty()) { - next_bucket = base64_name; - } -} - -bool -BucketManager::handleNextBucket() -{ - if (!next_bucket.empty()) { - const string &iterator_peek = iterator.peek(); - if (next_bucket != iterator_peek) { - dbgWarning(D_EVENT_BUFFER) - << "Invalid Iteration value, current iteration value does not equal to next bucket" - << endl - << "Current iteration value:" - << iterator_peek - << endl - << "Next bucket value:" - << next_bucket; - } - - if (!iterator_peek.empty()) { - iterator.trim(); - } - - buckets[next_bucket].trim(); - } - - if (iterator.isEmpty()) { - next_bucket.clear(); - dbgTrace(D_EVENT_BUFFER) << "Iteration bucket is empty"; - return false; - } - - const string &next_req_bucket = iterator.peek(); - if (next_req_bucket.empty()) { - dbgDebug(D_EVENT_BUFFER) - << "Next request within iteration bucket is empty, removing sent messages from file:" - << management_file_path; - iterator.refreshBufferFile(); - next_bucket.clear(); - return false; - } - dbgDebug(D_EVENT_BUFFER) - << "Next request within iteration bucket is :" - << next_req_bucket; - - string bucket_path = resolveFilesName(buffer_directory + "/" + next_req_bucket); - auto bucket = buckets.find(next_req_bucket); - if (bucket == buckets.end()) { - dbgDebug(D_EVENT_BUFFER) - << "Next request bucket was not found within the manager. trying to load it, bucket: " - << next_req_bucket; - buckets.emplace( - std::piecewise_construct, - std::forward_as_tuple(next_req_bucket), - std::forward_as_tuple() - ); - buckets[next_req_bucket].init(bucket_path, buffer_max_size/max_buffer_files); - } - next_bucket = next_req_bucket; - - return true; -} - -bool -BucketManager::hasValue() -{ - if (iterator.isEmpty()) { - dbgDebug(D_EVENT_BUFFER) << "Iterator is empty"; - return false; - } - if (next_bucket.empty()) { - dbgDebug(D_EVENT_BUFFER) << "Next bucket is empty"; - return handleNextBucket(); - } - - return true; -} - -EventQueue & -BucketManager::peek() -{ - dbgAssert(!next_bucket.empty()) << "Invalid call, handleNextBucket must be called before"; - return buckets[next_bucket]; -} - -void -BucketManager::flush() -{ - dbgTrace(D_EVENT_BUFFER) << "Flushing all data from the Bucket Manager"; - iterator.flush(); - for (auto &bucket : buckets) { - bucket.second.flush(); - } -} - -string -BucketManager::resolveFilesName(const string &file_name) -{ - string new_name = file_name; - if (instance_awareness != nullptr) new_name = new_name + instance_awareness->getUniqueID(""); - return new_name + service_name; -} diff --git a/core/messaging_buffer/event_queue.cc b/core/messaging_buffer/event_queue.cc deleted file mode 100644 index 7beeb07..0000000 --- a/core/messaging_buffer/event_queue.cc +++ /dev/null @@ -1,846 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "messaging_buffer/event_queue.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "zlib.h" -#include "config.h" -#include "debug.h" -#include "messaging_buffer.h" -#include "i_mainloop.h" - -using namespace std; - -USE_DEBUG_FLAG(D_EVENT_BUFFER); - -class MessagingBuffer; - -const string EventQueueFile::zip_file_suffix = ".cpz"; - -static const size_t size_of_new_line = strlen("\n"); -static const uint MB_in_bytes = 1<<20; - -string -parseFilePath(int suffix, string file_path) -{ - return suffix == -1 ? file_path : file_path + string(".") + to_string(suffix); -} - -static void -performSafeYield() -{ - auto env = Singleton::Consume::by(); - auto can_not_yield = env->get("Not part of coroutine"); - if (can_not_yield.ok() && *can_not_yield) return; - Singleton::Consume::by()->yield(); -} - -// If the program crashed during rotation, a tmp file was created without removing the old one -// Thus need to remove non tmp file and rename the tmp file -void -restoreTmpFile(const string &file_name) -{ - dbgTrace(D_EVENT_BUFFER) << "Handling a temporary file during init. File: " << file_name; - string new_file_name = file_name.substr(0, strlen(file_name.c_str()) - strlen(".tmp")); - remove(new_file_name.c_str()); - if (rename(file_name.c_str(), new_file_name.c_str()) != 0) { - dbgWarning(D_EVENT_BUFFER) - << "Couldn't handle a temporary file during init. Couldn't rename: " - << file_name - << ", to: " - << new_file_name - << ". Errno: " - << strerror(errno); - } -} - -EventQueue::~EventQueue() -{ - if (reader.is_open()) reader.close(); - if (writer.is_open()) writer.close(); -} - -void -EventQueue::setReaderFileAndOpen(const EventQueueFile &file) -{ - string file_path = parseFilePath(file.getSuffix(), file.getFilePath()); - reader.open(file_path); - if (!reader.is_open() && ifstream(file_path).good()) { - dbgWarning(D_EVENT_BUFFER) - << "Failed to open the file for read. File name: " - << file_path - << ". Errno: " - << errno - << ", Errno: " - << strerror(errno); - return; - } -} - -void -EventQueue::sortEventFilesBySuffix(std::vector &tmp_vec) -{ - sort( - tmp_vec.begin(), - tmp_vec.end(), - [](const EventQueueFile &first, const EventQueueFile &second) - { - return first.getSuffix() < second.getSuffix(); - } - ); - for (const EventQueueFile &file: tmp_vec) { - files.emplace_back(file); - } - for (auto &file: files) { - file.restoreNumberOfLines(); - num_of_events_on_disk += file.getNumOfEvents(); - size_on_disk += file.getFileSizeInBytes(); - } -} - -void -EventQueue::enforceMaxNumberOfFiles() -{ - uint max_files_to_rotate = getProfileAgentSettingWithDefault( - 10, - "eventBuffer.maxNumberOfEventQueueFiles" - ); - - while (files.size() >= max_files_to_rotate) { - performSafeYield(); - string file_to_delete = files.back().getFilePath() + string(".") + to_string(files.back().getSuffix()); - dbgDebug(D_EVENT_BUFFER) - << "Event Queue passed the maximum number of files it should contain by " - << files.size() - max_files_to_rotate - << " files. Removing the file: " - << file_to_delete - << ". This action reduced the number of events on disk by " - << files.back().getNumOfEvents() - << " and reduced the events buffers' size of disk by " - << files.back().getFileSizeInBytes() - << " bytes."; - num_of_events_on_disk -= files.back().getNumOfEvents(); - size_on_disk -= files.back().getFileSizeInBytes(); - updateReadFile(); - } -} - -// In case the agent crashed or stopped, iterate over all files. -// if files with same path prefix exist, insert them to the list. -void -EventQueue::reloadEventsIntoList(const string &path) { - dbgFlow(D_EVENT_BUFFER) << "Trying to reload event buffer from persistent location. Path: " << path; - string dir_path; - if (path.find("/") != string::npos) { - dir_path = path.substr(0, path.find_last_of("/")); - } else { - dir_path = path; - } - - dbgDebug(D_EVENT_BUFFER) << "Event queue directory to iterate: " << dir_path; - - struct dirent *entry = nullptr; - DIR *directory = opendir(dir_path.c_str()); - - if (directory == nullptr) { - int orig_errno = errno; - dbgWarning(D_EVENT_BUFFER) << "Failed to open directory. Path: " << dir_path << ", Errno: " << orig_errno; - return; - } - - vector tmp_vec; - while ((entry = readdir(directory))) { - string entry_file_name = entry->d_name; - static const string curr_dir("."); - static const string parent_dir(".."); - if (entry_file_name == curr_dir || entry_file_name == parent_dir) { - dbgTrace(D_EVENT_BUFFER) - << "Skipping reload of events from irrelevant directory entries. Entry name: " - << entry_file_name; - continue; - } - - bool file_has_extension = entry_file_name.find(".") != string::npos; - if (!file_has_extension) { - dbgTrace(D_EVENT_BUFFER) - << "Skipping reload of events who's entry lack extension. Entry name: " - << entry_file_name - << ", Path: " - << path; - continue; - } - string file_extension = entry_file_name.substr(entry_file_name.find_last_of(".")); - - dbgDebug(D_EVENT_BUFFER) << "Event queue file current extension: " << file_extension; - - if (file_extension == ".tmp") { - restoreTmpFile(entry_file_name); - continue; - } - - bool is_compressed = file_extension == EventQueueFile::zip_file_suffix; - string base_name = - file_has_extension ? - entry_file_name.substr(0, entry_file_name.find_last_of(".")) : - entry_file_name; - if (is_compressed && base_name.find(".") != string::npos) { - file_extension = base_name.substr(base_name.find_last_of(".") + 1); - base_name = base_name.substr(0, base_name.find_last_of(".")); - } - - dbgDebug(D_EVENT_BUFFER) - << "Trying to load event queue file from directory. File name: " - << entry_file_name - << ", does file has extension: " - << (file_has_extension ? "true" : "false") - << ", base name: " - << base_name - << ", is compressed: " - << (is_compressed ? "true" : "false"); - - if (path.find(base_name) == string::npos) { - dbgTrace(D_EVENT_BUFFER) - << "Skipping reload of events from irrelevant directory entries. Entry name: " - << entry_file_name - << ", Entry path: " - << path - << ", Entry file base name: " - << base_name; - continue; - } - - int max_files_to_rotate = getProfileAgentSettingWithDefault( - 10, - "eventBuffer.maxNumberOfEventQueueFiles" - ); - EventQueueFile new_file(path, file_extension, is_compressed); - if (new_file.getSuffix() < max_files_to_rotate) { - dbgDebug(D_EVENT_BUFFER) - << "Reloading file " - << new_file.getFilePath() - << " with suffix " - << new_file.getSuffix(); - tmp_vec.push_back(new_file); - } else { - dbgWarning(D_EVENT_BUFFER) - << "File " - << new_file.getFilePath() - << " with suffix " - << new_file.getSuffix() - << " will not be reloaded due to limitation of maximum number of event queue files."; - } - } - sortEventFilesBySuffix(tmp_vec); -} - -void -EventQueueFile::restoreNumberOfLines() -{ - string tmp_name; - if (isCompressed()) { - string compressed_name = getFilePath() + "." + to_string(getSuffix()) + zip_file_suffix; - tmp_name = getFilePath() + "." + to_string(getSuffix()); - decompress(compressed_name, tmp_name, false); - } - string line; - ifstream reader(parseFilePath(getSuffix(), getFilePath())); - while (getline(reader, line)) { - incFileSize(line.size() + size_of_new_line); - } - remove(tmp_name.c_str()); -} - -void -EventQueue::init(const string &path, uint max_buff_size) -{ - dbgTrace(D_EVENT_BUFFER) << "Initializing Event Queue. Path: " << path << ", Max buffer size: " << max_buff_size; - max_size = max_buff_size; - files.emplace_front(EventQueueFile(path)); - reloadEventsIntoList(path); - if (timer == nullptr) timer = Singleton::Consume::by();; - dbgAssert(timer != nullptr) << "Failed to find the time component"; - - uint next_sync_in_sec_base_settings = getProfileAgentSettingWithDefault( - 10, - "eventBuffer.syncToDiskFrequencyInSec" - ); - next_sync_freq_in_sec = - timer->getMonotonicTime() + - chrono::seconds(getConfigurationWithDefault( - next_sync_in_sec_base_settings, - "Event Buffer", - "sync to disk frequency in sec" - )); - - setReaderFileAndOpen(files.back()); - reader.seekg(0, ios::beg); -} - -// if current reader file is empty, iterate over to the next one -Maybe -EventQueue::refreshBufferFile() -{ - if (read_events_on_disk == 0) { - dbgDebug(D_EVENT_BUFFER) << "Nothing to refresh: all events on the disk still pending"; - return Maybe(); - } - - if (!reader.is_open()) return genError("nothing to trim since the file is still unopened"); - - int num_of_events_to_transfare = 0; - uint64_t size_of_events_to_transfare = 0; - string line; - vector file_content; - while (getline(reader, line)) { - performSafeYield(); - file_content.push_back(line); - num_of_events_to_transfare++; - size_of_events_to_transfare += (line.size() + size_of_new_line); - } - reader.close(); - - string read_file = parseFilePath(files.back().getSuffix(), files.back().getFilePath()); - string temp_file = read_file + ".tmp"; - remove(temp_file.c_str()); - writer.open(temp_file, ios_base::app); - if (!writer.is_open()) { - dbgWarning(D_EVENT_BUFFER) - << "Failed to open the file for write (append): " - << temp_file - << ". Errno: " - << errno - << ", Errno: " - << strerror(errno); - for (auto &line : file_content) { - performSafeYield(); - read_cache_buff.push_back(line); - read_cache_size += line.size(); - } - return genError("cannot open new cache file"); - } - num_of_events_on_disk -= files.back().getNumOfEvents(); - size_on_disk -= files.back().getFileSizeInBytes(); - for (const string &single_event: file_content) { - performSafeYield(); - writer << single_event << '\n'; - num_of_events_on_disk++; - size_on_disk += (single_event.size() + size_of_new_line); - } - writer.close(); - remove(read_file.c_str()); - rename(temp_file.c_str(), read_file.c_str()); - - reader.open(read_file); - if (!reader.is_open()) return genError("failed to open cache file to skip cached events"); - EventQueueFile updated_file{ - files.back(), - num_of_events_to_transfare, - size_of_events_to_transfare - }; - files.pop_back(); - files.emplace_back(updated_file); - return Maybe(); -} - -void -EventQueue::push(string &&event_data) -{ - if (files.front().getFilePath() == "") { - dbgWarning(D_EVENT_BUFFER) << "Cannot save events to a non-existent file"; - return; - } - event_data.erase(remove(event_data.begin(), event_data.end(), '\n'), event_data.end()); // remove all new-line - - write_cache_size += event_data.size(); - write_cache_buff.push_back(move(event_data)); // hold data in RAM in case write will fail - - if (is_pending_rotate) { - dbgDebug(D_EVENT_BUFFER) - << "Rotation pending. Accumulating events (write_cache_buff size=" - << write_cache_buff.size() - << ")"; - return; - } - - uint cache_buff_max_size_base_settings = getProfileAgentSettingWithDefault( - 100, - "eventBuffer.syncToDiskWriteCacheBufferSize" - ); - uint cache_buff_max_size = getConfigurationWithDefault( - cache_buff_max_size_base_settings, - "Event Buffer", - "sync to disk write cache buffer size" - ); - - if (timer->getMonotonicTime() < next_sync_freq_in_sec && write_cache_buff.size() < cache_buff_max_size) { - dbgTrace(D_EVENT_BUFFER) - << "Not writing event to disk because cache buffer is not full and time is before sync time interval "; - return; - } - - uint next_sync_in_sec_base_settings = getProfileAgentSettingWithDefault( - 10, - "eventBuffer.syncToDiskFrequencyInSec" - ); - next_sync_freq_in_sec = - timer->getMonotonicTime() + - chrono::seconds(getConfigurationWithDefault( - next_sync_in_sec_base_settings, - "Event Buffer", - "sync to disk frequency in sec" - )); - - if ( - files.front().getNumOfEvents() != 0 && - getSizeMB(write_cache_size + files.front().getFileSizeInBytes()) >= max_size - ) { - dbgTrace(D_EVENT_BUFFER) << "Event buffer queue reached max size, pending files rotation."; - is_pending_rotate = true; - - Singleton::Consume::by()->addOneTimeRoutine( - I_MainLoop::RoutineType::System, - [&] () - { - dbgWarning(D_EVENT_BUFFER) - << "Failed to buffer a message after reaching the maximum buffer size." - << "Compressing the buffer and creating a new one."; - rotate(); - - files.push_front(EventQueueFile(files.front().getFilePath())); - dbgInfo(D_EVENT_BUFFER) << "Successfully appended new buffer to list"; - is_pending_rotate = false; - }, - "Event queue rotation", - false - ); - - return; - } - - if (is_pending_write) { - dbgDebug(D_EVENT_BUFFER) - << "Writing events pending. Accumulating events (write_cache_buff size=" - << write_cache_buff.size() - << ")"; - return; - } - - is_pending_write = true; - - Singleton::Consume::by()->addOneTimeRoutine( - I_MainLoop::RoutineType::System, - [&] () - { - writer.open(files.front().getFilePath(), ios_base::app); - if (!writer.is_open()) { - dbgWarning(D_EVENT_BUFFER) - << "Failed to open the file for write (append):" - << files.front().getFilePath() - << ". Errno: " - << errno - << ", Errno: " - << strerror(errno); - return; - } - - for_each( - write_cache_buff.begin(), - write_cache_buff.end(), - [this](string &single_event) - { - size_t event_size = single_event.size(); - write_cache_size -= event_size; - writer << single_event << '\n'; - num_of_events_on_disk++; - files.front().incFileSize(event_size + size_of_new_line); - size_on_disk += (event_size + size_of_new_line); - performSafeYield(); - } - ); - - write_cache_buff.clear(); - writer.close(); - is_pending_write = false; - }, - "Event queue rotation", - false - ); -} - -Maybe -EventQueue::writeCachesToFile() -{ - vector file_content(read_cache_buff.begin(), read_cache_buff.end()); - if (num_of_events_on_disk > 0) { - reader.close(); - reader.open(files.front().getFilePath()); - if (!reader.is_open()) { - return genError("Failed to open the file for read: " + files.front().getFilePath()); - } - string line; - reader.clear(); - while (getline(reader, line)) { - file_content.push_back(line); - } - } - file_content.insert(file_content.end(), write_cache_buff.begin(), write_cache_buff.end()); - - string temp_file_name = files.front().getFilePath() + ".tmp"; - writer.open(temp_file_name, ios_base::app|ios_base::out); - - if (!writer.is_open()) { - return genError("Failed to open the file for write, file: " + temp_file_name); - } - - int current_num_of_events = 0; - uint64_t current_size_of_events = 0; - for (const string &single_event: file_content) { - writer << single_event << '\n'; - current_num_of_events++; - current_size_of_events = (single_event.size() + size_of_new_line); - } - writer.close(); - - remove(files.front().getFilePath().c_str()); // File possibly can be uncreated by this point - if (rename(temp_file_name.c_str(), files.front().getFilePath().c_str()) != 0) { - return genError("Error renaming temp file " + temp_file_name + " to " + files.front().getFilePath()); - } - EventQueueFile new_file{files.front(), current_num_of_events, current_size_of_events}; - files.pop_front(); - files.emplace_front(new_file); - return Maybe(); -} - -bool -EventQueue::isEmpty() const -{ - return num_of_events_on_disk + read_events_on_disk + read_cache_buff.size() + write_cache_buff.size() == 0; -} - -void -EventQueue::fini() -{ - auto write_caches = writeCachesToFile(); - if (!write_caches.ok()) { - dbgWarning(D_EVENT_BUFFER) - << "Failed to write cache to file, Error: " - << write_caches.getErr(); - } -} - -const string & -EventQueue::peek() -{ - static const string error_reading = ""; - if (isEmpty()) { - dbgDebug(D_EVENT_BUFFER) - << "Number of events on disk: " - << num_of_events_on_disk - << endl - << "Number of read events on disk: " - << read_events_on_disk - << endl - << "Read cache size: " - << read_cache_buff.size() - << endl - << "Write cache size: " - << write_cache_buff.size(); - dbgWarning(D_EVENT_BUFFER) - << "Cannot peek at an empty queue. file: " - << files.back().getFilePath(); - return error_reading; - } - if (read_cache_buff.empty()) { - refreshReadBuff(); - if (read_cache_buff.empty()) { - dbgDebug(D_EVENT_BUFFER) << "Read cache buffer is empty"; - return error_reading; - } - } - return read_cache_buff.front(); -} - -void -EventQueue::refreshReadBuff() -{ - if (files.empty()) { - dbgDebug(D_EVENT_BUFFER) << "Buffer files are empty"; - return; - } - if (files.back().getNumOfEvents() == 0) { - updateReadFile(); - if (files.empty() | (files.back().getNumOfEvents() == 0)) { - dbgDebug(D_EVENT_BUFFER) << "Buffered events file is empty."; - read_cache_buff.splice(read_cache_buff.begin(), write_cache_buff); - read_cache_size += write_cache_size; - write_cache_size = 0; - return; - } - } - if (!reader.is_open()) { - dbgTrace(D_EVENT_BUFFER) - << "Buffered events file is closed trying to open it. file: " - << files.back().getFilePath(); - setReaderFileAndOpen(files.back()); - } - - uint cache_buff_max_size_base_settings = getProfileAgentSettingWithDefault( - 100, - "eventBuffer.syncToDiskWriteCacheBufferSize" - ); - uint cache_buff_max_size = getConfigurationWithDefault( - cache_buff_max_size_base_settings, - "Event Buffer", - "sync to disk write cache buffer size" - ); - - int counter = 0; - while (read_cache_buff.size() < cache_buff_max_size && counter < files.back().getNumOfEvents()) { - performSafeYield(); - string line; - if (!getline(reader, line)) { - reader.clear(); - break; - } - read_events_on_disk ++; - counter++; - read_cache_buff.push_back(line); - read_cache_size += line.size(); - } - refreshBufferFile(); -} - -void -EventQueue::updateReadFile() -{ - if (files.back().getSuffix() == -1) { - return; - } - - if (!reader.is_open()) { - dbgTrace(D_EVENT_BUFFER) - << "Buffered events file is closed trying to open it. file: " - << files.back().getFilePath(); - setReaderFileAndOpen(files.back()); - } - - string file_to_delete = files.back().getFilePath() + string(".") + to_string(files.back().getSuffix()); - - string new_file = - files.back().getSuffix() == 0 ? - files.back().getFilePath() : - files.back().getFilePath() + string(".") + to_string(files.back().getSuffix() - 1); - - dbgDebug(D_EVENT_BUFFER) - << "Updating the reader file. Current file: " - << file_to_delete - << ", New file: " - << new_file; - - reader.close(); - files.pop_back(); - remove(file_to_delete.c_str()); - if (files.back().isCompressed()) files.back().decompress(new_file + files.back().zip_file_suffix, new_file); - reader.open(new_file); - if (!reader.is_open() && ifstream(new_file).good()) { - dbgWarning(D_EVENT_BUFFER) - << "Failed to open the file for read: " - << new_file - << ". Errno: " - << errno - << ", Errno: " - << strerror(errno); - return; - } -} - -void -EventQueue::trim() -{ - if (!read_cache_buff.empty()) { - read_cache_size -= read_cache_buff.front().size(); - read_cache_buff.pop_front(); - dbgTrace(D_EVENT_BUFFER) << "Removed first element in read cache buffer"; - if (!read_cache_buff.empty()) return; - } - - refreshReadBuff(); -} - -void -EventQueue::flush() -{ - for (auto &file: files) { - string file_path = parseFilePath(file.getSuffix(), file.getFilePath()); - remove(file_path.c_str()); - } - write_cache_buff.clear(); - read_cache_buff.clear(); - size_on_disk = 0; - num_of_events_on_disk = 0; - write_cache_size = 0; - read_cache_size = 0; - read_events_on_disk = 0; - reader.close(); - writer.close(); -} - -double -EventQueue::getSizeMB(double size_in_B) const -{ - return size_in_B/MB_in_bytes; -} - -void -EventQueue::rotate() -{ - enforceMaxNumberOfFiles(); - - for_each( - files.rbegin(), - files.rend(), - [&](EventQueueFile &file) - { - file.handleCompression(files.size()); - }); -} - -void -EventQueueFile::handleCompression(int list_length) -{ - bool should_rename = true; - suffix++; - string old_name = suffix == 0 ? file_path : file_path + string(".") + to_string(suffix - 1); - string new_name = file_path + string(".") + to_string(suffix); - - auto rename_on_exit = make_scope_exit( - [&]() - { - if (should_rename) rename(old_name.c_str(), new_name.c_str()); - dbgTrace(D_EVENT_BUFFER) - << "Renamed a file during rotation. Old file name: " - << old_name - << ". New file name: " - << new_name; - } - ); - if (suffix != list_length - 1) { // not the read file - new_name = new_name + zip_file_suffix; - if (is_compressed) { - old_name = old_name + zip_file_suffix; - return; - } - compress(); - should_rename = false; - return; - } - if (is_compressed) { - old_name = old_name + zip_file_suffix; - decompress(old_name, new_name); - should_rename = false; - } -} - -void -EventQueueFile::decompress(const string &infilename, const string &outfilename, bool remove_old) -{ - gzFile infile = gzopen(infilename.c_str(), "rb"); - FILE *outfile = fopen(outfilename.c_str(), "wb"); - char buffer[128]; - int num_read = 0; - while ((num_read = gzread(infile, buffer, sizeof(buffer))) > 0) { - fwrite(buffer, 1, num_read, outfile); - performSafeYield(); - } - gzclose(infile); - fclose(outfile); - if (remove_old) { - remove(infilename.c_str()); - is_compressed = false; - } -} - -void -EventQueueFile::compress() -{ - string infilename = - suffix == 0 ? file_path : file_path + string(".") + to_string(suffix - 1); - string outfilename = file_path + string(".") + to_string(suffix) + zip_file_suffix; - FILE *infile = fopen(infilename.c_str(), "rb"); - gzFile outfile = gzopen(outfilename.c_str(), "wb"); - char inbuffer[128]; - int num_read = 0; - unsigned long total_read = 0; - while ((num_read = fread(inbuffer, 1, sizeof(inbuffer), infile)) > 0) { - total_read += num_read; - gzwrite(outfile, inbuffer, num_read); - performSafeYield(); - } - fclose(infile); - gzclose(outfile); - dbgTrace(D_EVENT_BUFFER) - << "After file compression: Read " - << total_read - << "bytes, Wrote " - << getFileSizeInBytes() - << "bytes, Compression factor " - << ((1.0-getFileSizeInBytes()*1.0/total_read)*100.0); - remove(infilename.c_str()); - is_compressed = true; -} - -void -EventQueueFile::incFileSize(uint64_t size_to_add) -{ - size_of_file += size_to_add; - num_of_events_in_file++; -} - -EventQueueFile::EventQueueFile( - const string &file_location_path, - const string &file_extension_raw, - bool is_file_compressed) -{ - dbgInfo(D_EVENT_BUFFER) - << "Creating new event queue file. File's location path: " - << file_location_path - << ", File extension: " - << file_extension_raw - << "Is Compressed: " - << (is_file_compressed ? "true" : "false"); - - file_path = file_location_path; - is_compressed = is_file_compressed; - string file_extension = file_extension_raw; - try { - if (!file_extension.empty() && file_extension.front() == '.') { - file_extension.erase(0, 1); // delete the '.' before the suffix - } - suffix = stoi(file_extension); - } catch (const exception &e) { - dbgWarning(D_EVENT_BUFFER) - << "Error reloading event files. File: " - << file_path - << ", Error: " - << e.what(); - } -} diff --git a/core/messaging_buffer/messaging_buffer.cc b/core/messaging_buffer/messaging_buffer.cc deleted file mode 100644 index 6f108ab..0000000 --- a/core/messaging_buffer/messaging_buffer.cc +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "messaging_buffer.h" - -#include -#include -#include -#include -#include -#include "cereal/archives/json.hpp" - -#include "messaging_buffer/event_queue.h" -#include "messaging_buffer/bucket_manager.h" -#include "maybe_res.h" -#include "config.h" -#include "debug.h" - -USE_DEBUG_FLAG(D_EVENT_BUFFER); - -using namespace std; - -class MessagingBuffer::Impl - : - Singleton::Provide::From -{ -public: - void init(); - void fini(); - Maybe peekRequest() override; - void popRequest() override; - void bufferNewRequest(const HTTPRequestEvent &request, bool is_rejected) override; - bool isPending(const HTTPRequestSignature &request) override; - void cleanBuffer() override; -private: - void loadConfig(); - void initComponents(); - void initRejectedQueue(const string &process_path); - - Maybe deserialize(const HTTPRequestEvent &req); - Maybe serialize(const string &data); - - string buffer_directory = ""; - string log_files_prefix = ""; - I_TimeGet *timer = nullptr; - I_Encryptor *encryptor = nullptr; - I_InstanceAwareness *instance_awareness = nullptr; - BucketManager bucket_manager; - EventQueue rejected_events; -}; - -void -MessagingBuffer::Impl::init() -{ - ScopedContext ctx; - ctx.registerValue("Not part of coroutine", true); - - log_files_prefix = getLogFilesPathConfig(); - dbgTrace(D_EVENT_BUFFER) << "Messaging buffer init, log files prefix: " << log_files_prefix; - I_Environment *env = Singleton::Consume::by(); - string process_path = env->get("Executable Name").unpack(); - string service_name = process_path.substr(process_path.find_last_of("/") + 1); - loadConfig(); - initComponents(); - mkdir(buffer_directory.c_str(), 0644); - bucket_manager.init(service_name); - initRejectedQueue(service_name); -} - -void -MessagingBuffer::Impl::fini() -{ - ScopedContext ctx; - ctx.registerValue("Not part of coroutine", true); - - bucket_manager.fini(); -} - -Maybe -MessagingBuffer::Impl::peekRequest() -{ - if (!bucket_manager.hasValue()) { - dbgDebug(D_EVENT_BUFFER) << "No data avaliable"; - return genError("No data avaliable"); - } - EventQueue &tmp = bucket_manager.peek(); - if (tmp.isEmpty()) { - dbgDebug(D_EVENT_BUFFER) << "Next bucket returned empty queue"; - return genError("No data available in empty bucket"); - } - auto request = tmp.peek(); - if (request.empty()) { - popRequest(); - return genError("Request is empty, message is popped"); - } - return serialize(encryptor->base64Decode(request)); -} - -void -MessagingBuffer::Impl::popRequest() -{ - bucket_manager.handleNextBucket(); -} - -void -MessagingBuffer::Impl::bufferNewRequest(const HTTPRequestEvent &request, bool is_rejected) -{ - auto raw_data = deserialize(request); - if (!raw_data.ok()) { - string dbg_msg = - "Cannot buffer the request. Error: " + - raw_data.getErr() + - ". Request: " - + request.getSignature(); - - dbgWarning(D_EVENT_BUFFER) << dbg_msg; - dbgDebug(D_EVENT_BUFFER) - << dbg_msg - << ", headers: " - << request.getHeaders() - << ", body: " - << request.getBody(); - return; - } - - if (is_rejected) { - rejected_events.push(raw_data.unpackMove()); - return; - } - - string req_bucket_name = request.getSignature(); - bucket_manager.push(req_bucket_name, raw_data.unpackMove()); -} - -bool -MessagingBuffer::Impl::isPending(const HTTPRequestSignature &request) -{ - string req_bucket_name = request.getSignature(); - return bucket_manager.doesExist(req_bucket_name); -} - -void -MessagingBuffer::Impl::cleanBuffer() -{ - bucket_manager.flush(); - rejected_events.flush(); -} - -void -MessagingBuffer::Impl::initComponents() -{ - encryptor = Singleton::Consume::by(); - instance_awareness = Singleton::Consume::by(); -} - -void -MessagingBuffer::Impl::loadConfig() -{ - string base_folder_setting = getProfileAgentSettingWithDefault( - log_files_prefix + "/nano_agent/event_buffer", - "eventBuffer.baseFolder" - ); - buffer_directory = getConfigurationWithDefault( - base_folder_setting, - "Event Buffer", - "base folder" - ); -} - -void -MessagingBuffer::Impl::initRejectedQueue(const string &service_name) -{ - string buffer_dir_base_folder_setting = getProfileAgentSettingWithDefault( - log_files_prefix + "/nano_agent/event_buffer", - "eventBuffer.baseFolder" - ); - string buffer_directory = getConfigurationWithDefault( - buffer_dir_base_folder_setting, - "Event Buffer", - "base folder" - ); - - uint buffer_max_size_base_settings = getProfileAgentSettingWithDefault( - 1000, - "eventBuffer.maxBufferSizeInMB" - ); - uint buffer_max_size = getConfigurationWithDefault( - buffer_max_size_base_settings, - "Event Buffer", - "max buffer size in MB" - ); - - uint max_buffer_files_base_settings = getProfileAgentSettingWithDefault( - 10, - "eventBuffer.maxBufferFiles" - ); - uint max_buffer_files = getConfigurationWithDefault( - max_buffer_files_base_settings, - "Event Buffer", - "max buffer files" - ); - - string service_file_name = instance_awareness->getUniqueID("") + service_name; - rejected_events.init( - buffer_directory + "/rejected_events" + service_file_name, buffer_max_size/max_buffer_files - ); -} - -Maybe -MessagingBuffer::Impl::deserialize(const HTTPRequestEvent &req) -{ - try { - stringstream out; - { - cereal::JSONOutputArchive out_ar(out); - req.save(out_ar); - } - return out.str(); - } catch (cereal::Exception &e) { - return genError(e.what()); - } -} - -Maybe -MessagingBuffer::Impl::serialize(const string &data) -{ - try { - HTTPRequestEvent req; - stringstream in; - in.str(data); - try { - cereal::JSONInputArchive in_ar(in); - req.load(in_ar); - } catch (cereal::Exception &e) { - return genError("JSON parsing failed: " + string(e.what())); - } - return req; - } catch (exception &e) { - return genError(e.what()); - } -} - -MessagingBuffer::MessagingBuffer() - : - Component("MessagingBuffer"), - pimpl(make_unique()) -{ -} - -void MessagingBuffer::init() { pimpl->init(); } - -void MessagingBuffer::fini() { pimpl->fini(); } - -MessagingBuffer::~MessagingBuffer() {} - -void -MessagingBuffer::preload() -{ - registerExpectedConfiguration("Event Buffer", "base folder"); - registerExpectedConfiguration("Event Buffer", "base file name"); - registerExpectedConfiguration("Event Buffer", "max buffer size in MB"); - registerExpectedConfiguration("Event Buffer", "max buffer files"); - registerExpectedConfiguration("Event Buffer", "sync to disk frequency in sec"); - registerExpectedConfiguration("Event Buffer", "send event retry in sec"); -} diff --git a/core/messaging_buffer/messaging_buffer_ut/CMakeLists.txt b/core/messaging_buffer/messaging_buffer_ut/CMakeLists.txt deleted file mode 100644 index eaa3fde..0000000 --- a/core/messaging_buffer/messaging_buffer_ut/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -link_directories(${BOOST_ROOT}/lib) - -include_directories(${ng_module_osrc_zlib_path}/include) -link_directories(${ng_module_osrc_zlib_path}/lib) - -add_unit_test(messaging_buffer_ut "messaging_buffer_ut.cc" "singleton;environment;time_proxy;instance_awareness;messaging_buffer;-lz;encryptor;event_is;metric;-lboost_regex;-lboost_filesystem;-lcrypto;-lboost_system") diff --git a/core/messaging_buffer/messaging_buffer_ut/messaging_buffer_ut.cc b/core/messaging_buffer/messaging_buffer_ut/messaging_buffer_ut.cc deleted file mode 100644 index ffc62b8..0000000 --- a/core/messaging_buffer/messaging_buffer_ut/messaging_buffer_ut.cc +++ /dev/null @@ -1,755 +0,0 @@ -#include "messaging_buffer.h" - -#include -#include -#include -#include - -#include "singleton.h" -#include "environment.h" -#include "config.h" -#include "config_component.h" -#include "messaging_buffer/event_queue.h" -#include "messaging_buffer/http_request_event.h" - -#include "cptest.h" -#include "mock/mock_instance_awareness.h" -#include "mock/mock_time_get.h" -#include "encryptor.h" -#include "mock/mock_mainloop.h" - -using namespace testing; -using namespace std; - -USE_DEBUG_FLAG(D_EVENT_BUFFER); - -bool -operator==(const HTTPRequestEvent &a, const HTTPRequestEvent &b) -{ - return - a.getBody() == b.getBody() && - a.getHeaders() == b.getHeaders() && - a.getMethod() == b.getMethod() && - a.getURL() == b.getURL(); -} - -class MessagingBufferTest : public Test -{ -public: - MessagingBufferTest() - { - mkdir("/tmp/event_buffer/", 0777); - instance_awareness_value = "ia"; - process_name_value = "pn"; - Debug::setNewDefaultStdout(&capture_debug); - EXPECT_CALL(instance_awareness, getUniqueID(_)).WillRepeatedly(Return(instance_awareness_value)); - i_messaging_buffer = Singleton::Consume::from(messaging_buffer); - i_encryptor = Singleton::Consume::from(encryptor); - env.preload(); - } - - void - init(bool proccess_name_value = true) - { - setConfiguration("/tmp/event_buffer", "Event Buffer", "base folder"); - string process_path = ""; - if (proccess_name_value) process_path = "a/b/" + process_name_value; - Singleton::Consume::from(env)->registerValue("Executable Name", process_path); - messaging_buffer.init(); - } - - ~MessagingBufferTest() - { - i_messaging_buffer->cleanBuffer(); - Debug::setNewDefaultStdout(&cout); - boost::filesystem::path dir_path("/tmp/event_buffer/"); - remove_all(dir_path); - rmdir(dir_path.filename().c_str()); - } - - I_MessagingBuffer *i_messaging_buffer; - ostringstream capture_debug; - string instance_awareness_value; - string process_name_value; - Encryptor encryptor; - I_Encryptor *i_encryptor; - StrictMock timer; - NaggyMock mock_mainloop; - StrictMock instance_awareness; - MessagingBuffer messaging_buffer; - -private: - ::Environment env; - ConfigComponent config; -}; - -TEST_F(MessagingBufferTest, doNothing) -{ -} - -TEST_F(MessagingBufferTest, init) -{ - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - static int counter = 1; - return chrono::microseconds(10000001 * (++counter)); - } - ) - ); - init(); -} - -TEST_F(MessagingBufferTest, popRequestFromEmpty) -{ - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - static int counter = 1; - return chrono::microseconds(10000001 * (++counter)); - } - ) - ); - init(); - auto req = i_messaging_buffer->peekRequest(); - EXPECT_FALSE(req.ok()); -} - -TEST_F(MessagingBufferTest, popRequestFromNonEmpty) -{ - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - static int counter = 1; - return chrono::microseconds(10000001 * (++counter)); - } - ) - ); - - init(); - auto empty_req = i_messaging_buffer->peekRequest(); - EXPECT_FALSE(empty_req.ok()); - - HTTPRequestEvent req("0", "1", "2", "3"); - i_messaging_buffer->bufferNewRequest(req); - - auto req_1 = i_messaging_buffer->peekRequest(); - EXPECT_TRUE(req_1.ok()); - i_messaging_buffer->popRequest(); - - auto req_2 = i_messaging_buffer->peekRequest(); - EXPECT_FALSE(req_2.ok()); -} - -TEST_F(MessagingBufferTest, MultiRequestBuffering) -{ - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - static int counter = 1; - return chrono::microseconds(10000001 * (++counter)); - } - ) - ); - init(); - - HTTPRequestEvent req0123("0", "1", "2", "3"); - i_messaging_buffer->bufferNewRequest(req0123); - - HTTPRequestEvent req0124("0", "1", "2", "4"); - i_messaging_buffer->bufferNewRequest(req0124); - - HTTPRequestEvent req1124("1", "1", "2", "4"); - i_messaging_buffer->bufferNewRequest(req1124); - - auto req_1 = i_messaging_buffer->peekRequest(); - EXPECT_TRUE(req_1.ok()); - i_messaging_buffer->popRequest(); - - auto req_2 = i_messaging_buffer->peekRequest(); - EXPECT_TRUE(req_2.ok()); - i_messaging_buffer->popRequest(); - - auto req_3 = i_messaging_buffer->peekRequest(); - EXPECT_TRUE(req_3.ok()); - i_messaging_buffer->popRequest(); - - EXPECT_EQ(req_1.unpack(), req0123); - EXPECT_EQ(req_2.unpack(), req0124); - EXPECT_EQ(req_3.unpack(), req1124); -} - -TEST_F(MessagingBufferTest, isPendingTrue) -{ - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly(Return(chrono::microseconds(10000000))); - init(); - - HTTPRequestEvent req0123("0", "1", "2", "3"); - i_messaging_buffer->bufferNewRequest(req0123); - - HTTPRequestEvent req0124("0", "1", "2", "4"); - - EXPECT_TRUE(i_messaging_buffer->isPending(req0124)); - - i_messaging_buffer->bufferNewRequest(req0124); - - HTTPRequestEvent req1124("1", "1", "2", "4"); - i_messaging_buffer->bufferNewRequest(req1124); -} - -TEST_F(MessagingBufferTest, isPendingFalse) -{ - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly(Return(chrono::microseconds(10000000))); - init(); - - HTTPRequestEvent req0123("0", "1", "2", "3"); - i_messaging_buffer->bufferNewRequest(req0123); - - HTTPRequestEvent req0124("0", "1", "2", "4"); - i_messaging_buffer->bufferNewRequest(req0124); - - HTTPRequestEvent req1124("1", "1", "2", "4"); - - EXPECT_FALSE(i_messaging_buffer->isPending(req1124)); - - i_messaging_buffer->bufferNewRequest(req1124); -} - -TEST_F(MessagingBufferTest, noPopGivesSameRequest) -{ - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly(Return(chrono::microseconds(10000000))); - init(); - - HTTPRequestEvent req0123("0", "1", "2", "3"); - i_messaging_buffer->bufferNewRequest(req0123); - - HTTPRequestEvent req0124("0", "1", "2", "4"); - i_messaging_buffer->bufferNewRequest(req0124); - - HTTPRequestEvent req1124("1", "1", "2", "4"); - i_messaging_buffer->bufferNewRequest(req1124); - - - auto req_1 = i_messaging_buffer->peekRequest(); - EXPECT_TRUE(req_1.ok()); - - auto req_2 = i_messaging_buffer->peekRequest(); - EXPECT_TRUE(req_2.ok()); - - auto req_3 = i_messaging_buffer->peekRequest(); - EXPECT_TRUE(req_3.ok()); - - EXPECT_EQ(req_1.unpack(), req0123); - EXPECT_EQ(req_2.unpack(), req0123); - EXPECT_EQ(req_3.unpack(), req0123); -} - -TEST_F(MessagingBufferTest, nothingLeft) -{ - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - static int counter = 1; - return chrono::microseconds(10000001 * (++counter)); - } - ) - ); - init(); - - HTTPRequestEvent req0123("0", "1", "2", "3"); - i_messaging_buffer->bufferNewRequest(req0123); - - HTTPRequestEvent req0124("0", "1", "2", "4"); - i_messaging_buffer->bufferNewRequest(req0124); - - HTTPRequestEvent req1124("1", "1", "2", "4"); - i_messaging_buffer->bufferNewRequest(req1124); - - - i_messaging_buffer->popRequest(); - i_messaging_buffer->popRequest(); - i_messaging_buffer->popRequest(); - - auto req_1 = i_messaging_buffer->peekRequest(); - EXPECT_FALSE(req_1.ok()); -} - -TEST_F(MessagingBufferTest, hugeBuffering) -{ - messaging_buffer.preload(); - setConfiguration(0, "Event Buffer", "max buffer size in MB"); - setConfiguration(1, "Event Buffer", "max buffer files"); - - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - return chrono::microseconds(10000001); - } - ) - ); - init(); - const size_t events_size = 499; - vector events; - - for (size_t i = 0; i < events_size; i++) { - events.push_back(HTTPRequestEvent("0", std::to_string(i/10), "2", std::to_string(i))); - i_messaging_buffer->bufferNewRequest(events[i]); - } - - for (size_t i = 0; i < events_size; i++) { - auto req = i_messaging_buffer->peekRequest(); - i_messaging_buffer->popRequest(); - ASSERT_TRUE(req.ok()); - EXPECT_EQ(req.unpack(), events[i]); - } - - events.clear(); - for (size_t i = 0; i < events_size; i++) { - events.push_back(HTTPRequestEvent("0", std::to_string(i/10), "2", std::to_string(i))); - i_messaging_buffer->bufferNewRequest(events[i]); - } - - for (size_t i = 0; i < events_size; i++) { - auto req = i_messaging_buffer->peekRequest(); - i_messaging_buffer->popRequest(); - ASSERT_TRUE(req.ok()); - EXPECT_EQ(req.unpack(), events[i]); - } -} - -TEST_F(MessagingBufferTest, rejectedBufferOk) -{ - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - static int counter = 1; - return chrono::microseconds(10000001 * (++counter)); - } - ) - ); - init(); - - I_MainLoop::Routine timer_routine; - EXPECT_CALL(mock_mainloop, addOneTimeRoutine( - I_MainLoop::RoutineType::System, - _, - _, - _ - )).WillOnce(DoAll(SaveArg<1>(&timer_routine), Return(0))); - - HTTPRequestEvent req("0", "1", "2", "2"); - const size_t events_size = 3; - for (size_t i = 0; i < events_size; i++) { - i_messaging_buffer->bufferNewRequest(req, true); - } - - timer_routine(); - - ifstream buffer_file( - "/tmp/event_buffer/rejected_events" + instance_awareness_value + process_name_value - ); - - ASSERT_TRUE(buffer_file.is_open()); - - string line; - vector file_content; - while (getline(buffer_file, line)) { - file_content.push_back(line); - } - buffer_file.close(); - ASSERT_FALSE(buffer_file.is_open()); - - for (auto content_line: file_content) { - HTTPRequestEvent rejected_req; - stringstream in; - in.str(content_line); - cereal::JSONInputArchive in_ar(in); - rejected_req.load(in_ar); - EXPECT_EQ(rejected_req, req); - } -} - -TEST_F(MessagingBufferTest, startFromFile) -{ - string event_as_string; - HTTPRequestEvent event("0", "1", "2", "3"); - - stringstream out; - { - cereal::JSONOutputArchive out_ar(out); - event.save(out_ar); - } - event_as_string = i_encryptor->base64Encode(out.str()); - - ofstream write_initial_file( - "/tmp/event_buffer/" + i_encryptor->base64Encode("01") + instance_awareness_value + process_name_value, - ios_base::app - ); - - ofstream manager_file( - "/tmp/event_buffer/manager" + instance_awareness_value + process_name_value, ios_base::app - ); - - ASSERT_TRUE(write_initial_file.is_open()); - ASSERT_TRUE(manager_file.is_open()); - for (int i = 0 ; i < 101 ; i++) { - write_initial_file << event_as_string << "\n"; - manager_file << i_encryptor->base64Encode("01") << "\n"; - } - - write_initial_file.close(); - ASSERT_FALSE(write_initial_file.is_open()); - - manager_file.close(); - ASSERT_FALSE(manager_file.is_open()); - - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - return chrono::microseconds(10000001); - } - ) - ); - init(); - - for (int i = 0 ; i < 101 ; i++) { - auto req = i_messaging_buffer->peekRequest(); - i_messaging_buffer->popRequest(); - EXPECT_TRUE(req.ok()); - EXPECT_EQ(req.unpack(), event); - } - - i_messaging_buffer->bufferNewRequest(event); -} - -TEST_F(MessagingBufferTest, PushToBufferedFile) -{ - messaging_buffer.preload(); - setConfiguration(0, "Event Buffer", "max buffer size in MB"); - setConfiguration(1, "Event Buffer", "max buffer files"); - - string event_as_string; - HTTPRequestEvent event("0", "1", "2", "3"); - - stringstream out; - { - cereal::JSONOutputArchive out_ar(out); - event.save(out_ar); - } - event_as_string = i_encryptor->base64Encode(out.str()); - - ofstream write_initial_file( - "/tmp/event_buffer/" + i_encryptor->base64Encode("01") + instance_awareness_value + process_name_value, - ios_base::app - ); - ofstream manager_file( - "/tmp/event_buffer/manager" + instance_awareness_value + process_name_value, ios_base::app - ); - - ASSERT_TRUE(write_initial_file.is_open()); - ASSERT_TRUE(manager_file.is_open()); - for (int i = 0 ; i < 101 ; i++) { - write_initial_file << event_as_string << "\n"; - manager_file << i_encryptor->base64Encode("01") << "\n"; - } - - write_initial_file.close(); - ASSERT_FALSE(write_initial_file.is_open()); - - manager_file.close(); - ASSERT_FALSE(manager_file.is_open()); - - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - return chrono::microseconds(10000001); - } - ) - ); - init(); - - i_messaging_buffer->bufferNewRequest(event); - for (int i = 0 ; i < 101 ; i++) { - auto req = i_messaging_buffer->peekRequest(); - i_messaging_buffer->popRequest(); - EXPECT_TRUE(req.ok()); - EXPECT_EQ(req.unpack(), event); - } -} - -TEST_F(MessagingBufferTest, max_buffer_size) -{ - messaging_buffer.preload(); - setConfiguration(0, "Event Buffer", "max buffer size in MB"); - setConfiguration(1, "Event Buffer", "max buffer files"); - ostringstream capture_debug; - Debug::setNewDefaultStdout(&capture_debug); - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - static int counter = 1; - return chrono::microseconds(10000001 * (++counter)); - } - ) - ); - init(); - - vector timer_routines; - EXPECT_CALL(mock_mainloop, addOneTimeRoutine( - I_MainLoop::RoutineType::System, - _, - _, - _ - )).WillRepeatedly( - WithArgs<1>( - Invoke( - [&](const I_MainLoop::Routine &routine) - { - timer_routines.push_back(routine); - return 0; - } - ) - )); - - const size_t events_size = 3; - vector events; - for (size_t i = 0; i < events_size; i++) { - events.push_back( - HTTPRequestEvent( - "0", - to_string(i), - "00", - to_string(i) - ) - ); - i_messaging_buffer->bufferNewRequest(events[i]); - - // Run all pending timers - for (auto &routine : timer_routines) routine(); - timer_routines.clear(); - } - EXPECT_THAT( - capture_debug.str(), - HasSubstr("Failed to buffer a message after reaching the maximum buffer size") - ); -} - -class MessagingBufferFiniTest : public Test -{ -public: - MessagingBufferFiniTest() - { - messaging_buffer = make_unique(); - mkdir("/tmp/event_buffer/", 0777); - instance_awareness_value = "ia"; - process_name_value = "pn"; - Debug::setUnitTestFlag(D_EVENT_BUFFER, Debug::DebugLevel::DEBUG); - EXPECT_CALL(instance_awareness, getUniqueID(_)).WillRepeatedly(Return(instance_awareness_value)); - i_messaging_buffer = Singleton::Consume::from(*messaging_buffer); - i_encryptor = Singleton::Consume::from(encryptor); - env.preload(); - env.init(); - } - - void - init(bool proccess_name_value = true) - { - setConfiguration("/tmp/event_buffer", "Event Buffer", "base folder"); - string process_path = ""; - if (proccess_name_value) process_path = "a/b/" + process_name_value; - Singleton::Consume::from(env)->registerValue("Executable Name", process_path); - messaging_buffer->init(); - } - - ~MessagingBufferFiniTest() - { - i_messaging_buffer->cleanBuffer(); - Debug::setUnitTestFlag(D_EVENT_BUFFER, Debug::DebugLevel::INFO); - Debug::setNewDefaultStdout(&cout); - boost::filesystem::path dir_path("/tmp/event_buffer/"); - remove_all(dir_path); - rmdir(dir_path.filename().c_str()); - } - - void - preload() - { - messaging_buffer->preload(); - } - - void - release() - { - messaging_buffer->fini(); - delete messaging_buffer.release(); - messaging_buffer = make_unique(); - i_messaging_buffer = Singleton::Consume::from(*messaging_buffer); - } - - I_MessagingBuffer *i_messaging_buffer; - - string instance_awareness_value; - string process_name_value; - Encryptor encryptor; - I_Encryptor *i_encryptor; - StrictMock timer; - NiceMock mock_mainloop; - StrictMock instance_awareness; - unique_ptr messaging_buffer; - -private: - ::Environment env; - ConfigComponent config; -}; - -TEST_F(MessagingBufferFiniTest, fini) -{ - messaging_buffer->preload(); - setConfiguration(1, "Event Buffer", "max buffer size in MB"); - setConfiguration(1, "Event Buffer", "max buffer files"); - - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - return chrono::microseconds(10000001); - } - ) - ); - init(); - - const size_t events_size = 101; - vector events; - for (size_t i = 0; i < events_size; i++) { - events.push_back(HTTPRequestEvent("0", to_string(i), "2", to_string(i))); - i_messaging_buffer->bufferNewRequest(events[i]); - } - - release(); - - init(); - for (size_t i = 0; i < events_size; i++) { - auto req = i_messaging_buffer->peekRequest(); - i_messaging_buffer->popRequest(); - ASSERT_TRUE(req.ok()); - EXPECT_EQ(req.unpack(), events[i]); - } -} - -static inline ostream & -operator<<(ostream &os, const HTTPRequestEvent &req) -{ - return os - << "Signature: " - << req.getSignature() - << ", Headers: " - << req.getHeaders() - << ", Body " - << req.getBody(); -} - -TEST_F(MessagingBufferFiniTest, hugeBufferingDoubleInit) -{ - messaging_buffer->preload(); - setConfiguration(0, "Event Buffer", "max buffer size in MB"); - setConfiguration(1, "Event Buffer", "max buffer files"); - - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - return chrono::microseconds(10000001); - } - ) - ); - init(); - - vector timer_routines; - EXPECT_CALL(mock_mainloop, addOneTimeRoutine( - I_MainLoop::RoutineType::System, - _, - _, - _ - )).WillRepeatedly( - WithArgs<1>( - Invoke( - [&](const I_MainLoop::Routine &routine) - { - timer_routines.push_back(routine); - return 0; - } - ) - )); - - const size_t events_size = 499; - vector events; - - for (size_t i = 0; i < events_size; i++) { - events.push_back(HTTPRequestEvent("0", "1", "2", std::to_string(i))); - i_messaging_buffer->bufferNewRequest(events[i]); - - // Run all pending timers - for (auto &routine : timer_routines) routine(); - timer_routines.clear(); - } - - release(); - init(); - - for (size_t i = 0; i < events_size; i++) { - auto req = i_messaging_buffer->peekRequest(); - i_messaging_buffer->popRequest(); - ASSERT_TRUE(req.ok()); - EXPECT_EQ(req.unpack(), events[i]); - } -} - -TEST_F(MessagingBufferFiniTest, initTempFile) -{ - messaging_buffer->preload(); - setConfiguration(0, "Event Buffer", "max buffer size in MB"); - setConfiguration(1, "Event Buffer", "max buffer files"); - - EXPECT_CALL(timer, getMonotonicTime()).WillRepeatedly( - InvokeWithoutArgs( - [&]() - { - return chrono::microseconds(10000001); - } - ) - ); - init(); - const size_t events_size = 1; - vector events; - - for (size_t i = 0; i < events_size; i++) { - events.push_back(HTTPRequestEvent("0", "1", "2", "temp_file")); - i_messaging_buffer->bufferNewRequest(events[i]); - } - - release(); - std::ofstream outfile("/tmp/event_buffer/MDFidWZmZXJlZCBtZXNzYWdlcw==iapn.tmp"); - string tmp_file = - "ewogICAgInRhZyI6ICJidWZmZXJlZCBtZXNzYWdlcyIsCiAgICAidmFsdWUwIjogIjAiLAo" \ - "gICAgInZhbHVlMSI6ICIxIiwKICAgICJ2YWx1ZTIiOiAiMiIsCiAgICAidmFsdWUzIjogInRlbXBfZmlsZSIKfQ=="; - outfile << tmp_file; - outfile.close(); - init(); - - for (size_t i = 0; i < events_size; i++) { - auto req = i_messaging_buffer->peekRequest(); - i_messaging_buffer->popRequest(); - ASSERT_TRUE(req.ok()); - EXPECT_EQ(req.unpack(), events[i]); - } -} diff --git a/core/metric/generic_metric.cc b/core/metric/generic_metric.cc old mode 100755 new mode 100644 index 617055c..b2233a8 --- a/core/metric/generic_metric.cc +++ b/core/metric/generic_metric.cc @@ -229,13 +229,11 @@ void GenericMetric::sendLog(const LogRest &metric_client_rest) const { string fog_metric_uri = getConfigurationWithDefault("/api/v1/agents/events", "metric", "fogMetricUri"); - Singleton::Consume::by()->sendObjectWithPersistence( - metric_client_rest, - I_Messaging::Method::POST, + Singleton::Consume::by()->sendAsyncMessage( + HTTPMethod::POST, fog_metric_uri, - "", - true, - MessageTypeTag::METRIC + metric_client_rest, + MessageCategory::METRIC ); } diff --git a/core/metric/metric_ut/CMakeLists.txt b/core/metric/metric_ut/CMakeLists.txt index 948cfa7..be74ed2 100755 --- a/core/metric/metric_ut/CMakeLists.txt +++ b/core/metric/metric_ut/CMakeLists.txt @@ -1,5 +1,5 @@ -add_unit_test( - metric_ut - "metric_ut.cc;" - "singleton;mainloop;event_is;metric;agent_details;-lboost_regex" +add_unit_test( + metric_ut + "metric_ut.cc;" + "singleton;messaging;mainloop;event_is;metric;agent_details;-lboost_regex" ) \ No newline at end of file diff --git a/core/metric/metric_ut/metric_ut.cc b/core/metric/metric_ut/metric_ut.cc old mode 100755 new mode 100644 index ee25f46..e66f156 --- a/core/metric/metric_ut/metric_ut.cc +++ b/core/metric/metric_ut/metric_ut.cc @@ -210,10 +210,13 @@ TEST_F(MetricTest, basicMetricTest) "}"; string message_body; - EXPECT_CALL( - messaging_mock, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, MessageTypeTag::METRIC) - ).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); string expected_message = "{\n" @@ -275,10 +278,13 @@ TEST_F(MetricTest, basicMetricTest) " ]\n" "}"; - EXPECT_CALL( - messaging_mock, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, MessageTypeTag::METRIC) - ).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); expected_message = "{\n" @@ -342,10 +348,13 @@ TEST_F(MetricTest, basicMetricTest) " ]\n" "}"; - EXPECT_CALL( - messaging_mock, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, MessageTypeTag::METRIC) - ).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); expected_message = "{\n" @@ -419,10 +428,13 @@ TEST_F(MetricTest, printMetricsTest) cpu_event.setProcessCPU(89); cpu_event.notify(); - EXPECT_CALL( - messaging_mock, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, MessageTypeTag::METRIC) - ).WillRepeatedly(Return(Maybe(string("")))); + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _ + )).Times(AnyNumber()); string metric_str = "{\n" @@ -486,10 +498,14 @@ TEST_F(MetricTest, metricTestWithReset) "}"; string message_body; - EXPECT_CALL( - messaging_mock, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, MessageTypeTag::METRIC) - ).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); + + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); string expected_message = "{\n" @@ -550,10 +566,13 @@ TEST_F(MetricTest, metricTestWithReset) " ]\n" "}"; - EXPECT_CALL( - messaging_mock, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, MessageTypeTag::METRIC) - ).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); expected_message = "{\n" @@ -614,10 +633,13 @@ TEST_F(MetricTest, metricTestWithReset) " ]\n" "}"; - EXPECT_CALL( - messaging_mock, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, MessageTypeTag::METRIC) - ).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); expected_message = "{\n" @@ -699,10 +721,14 @@ TEST_F(MetricTest, generateReportWithReset) "}"; string message_body; - EXPECT_CALL( - messaging_mock, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, MessageTypeTag::METRIC) - ).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); + + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); string expected_message = "{\n" @@ -770,10 +796,13 @@ TEST_F(MetricTest, generateReportWithReset) " ]\n" "}"; - EXPECT_CALL( - messaging_mock, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, MessageTypeTag::METRIC) - ).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); expected_message = "{\n" @@ -835,10 +864,13 @@ TEST_F(MetricTest, generateReportWithReset) " ]\n" "}"; - EXPECT_CALL( - messaging_mock, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, MessageTypeTag::METRIC) - ).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); expected_message = "{\n" @@ -980,10 +1012,14 @@ TEST_F(MetricTest, testMapMetric) HttpTransaction("/index.html", 40).notify(); string message_body; - EXPECT_CALL( - messaging_mock, - mockSendPersistentMessage(false, _, _, "/api/v1/agents/events", _, _, MessageTypeTag::METRIC) - ).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe(string(""))))); + + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); routine(); string msg_str = diff --git a/core/report/report_ut/CMakeLists.txt b/core/report/report_ut/CMakeLists.txt index 6748657..28af030 100644 --- a/core/report/report_ut/CMakeLists.txt +++ b/core/report/report_ut/CMakeLists.txt @@ -1,5 +1,5 @@ add_unit_test( report_ut "report_ut.cc" - "report;config;encryptor;environment;metric;event_is;-lboost_regex;-lcrypto" + "report;messaging;config;encryptor;environment;metric;event_is;-lboost_regex;-lcrypto" ) diff --git a/core/report/tag_and_enum_management.cc b/core/report/tag_and_enum_management.cc old mode 100755 new mode 100644 index 85a0028..fd9aaaf --- a/core/report/tag_and_enum_management.cc +++ b/core/report/tag_and_enum_management.cc @@ -270,8 +270,8 @@ TagAndEnumManagement::convertToString(const IssuingEngine &issuing_engine) case IssuingEngine::IOT_NEXT: return "iotNext"; case IssuingEngine::SDWAN: return "sdwanGwSharing"; case IssuingEngine::FILE_UPLOAD: return "fileUpload"; - case IssuingEngine::IDA_NEXT_BLADE_REGISTRATION: return "quantumMetaNotifyIdn"; - case IssuingEngine::IDA_NEXT_CLIENT_IP_NOTIFY: return "quantumIPNotifyIdn"; + case IssuingEngine::IDA_SAML_IDN_BLADE_REGISTRATION: return "quantumMetaNotifyIdn"; + case IssuingEngine::IDA_SAML_IDN_CLIENT_IP_NOTIFY: return "quantumIPNotifyIdn"; case IssuingEngine::API_DISCOVERY: return "apiDiscoveryCloudMessaging"; case IssuingEngine::HORIZON_TELEMETRY_METRICS: return "horizonTelemetryMetrics"; } diff --git a/core/rest/rest_server.cc b/core/rest/rest_server.cc index db9cb87..a751fc0 100644 --- a/core/rest/rest_server.cc +++ b/core/rest/rest_server.cc @@ -121,6 +121,7 @@ RestServer::Impl::init() struct sockaddr_in addr; bzero(&addr, sizeof(addr)); + addr.sin_family = AF_INET; addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); diff --git a/core/rest/rest_ut/CMakeLists.txt b/core/rest/rest_ut/CMakeLists.txt index b3171dc..b677797 100755 --- a/core/rest/rest_ut/CMakeLists.txt +++ b/core/rest/rest_ut/CMakeLists.txt @@ -4,5 +4,5 @@ link_directories(${ng_module_osrc_zlib_path}/lib) add_unit_test( rest_server_ut "rest_schema_ut.cc;rest_must_param_ut.cc;rest_config_ut.cc" - "singleton;rest;environment;-lz;shell_cmd;-lboost_filesystem;message;instance_awareness;messaging_buffer;-lz;debug_is;time_proxy;mainloop;agent_details;encryptor;event_is;metric;-lboost_context;-lboost_regex;-lboost_system;-lssl;-lcrypto;connkey" + "singleton;messaging;tenant_manager;rest;environment;-lz;shell_cmd;-lboost_filesystem;instance_awareness;-lz;debug_is;time_proxy;mainloop;agent_details;encryptor;event_is;metric;-lboost_context;-lboost_regex;-lboost_system;-lssl;-lcrypto;connkey" ) diff --git a/core/rest/rest_ut/rest_config_ut.cc b/core/rest/rest_ut/rest_config_ut.cc old mode 100755 new mode 100644 index 9a30bb3..346196e --- a/core/rest/rest_ut/rest_config_ut.cc +++ b/core/rest/rest_ut/rest_config_ut.cc @@ -10,11 +10,15 @@ #include "time_proxy.h" #include "mainloop.h" #include "rest_server.h" +#include "agent_details.h" +#include "mock/mock_messaging.h" +#include "tenant_manager.h" using namespace std; using namespace testing; USE_DEBUG_FLAG(D_API); +USE_DEBUG_FLAG(D_MAINLOOP); class RestConfigTest : public Test { @@ -46,6 +50,7 @@ public: Singleton::Consume::from(config)->loadConfiguration(ss); Debug::setUnitTestFlag(D_API, Debug::DebugLevel::NOISE); + Debug::setUnitTestFlag(D_MAINLOOP, Debug::DebugLevel::NOISE); Debug::setNewDefaultStdout(&capture_debug); } @@ -62,6 +67,9 @@ public: ::Environment env; ConfigComponent config; RestServer rest_server; + TenantManager tenant_manager; + AgentDetails agent_details; + NiceMock messaging; }; TEST_F(RestConfigTest, alternative_port_used) @@ -107,4 +115,67 @@ TEST_F(RestConfigTest, alternative_port_used) EXPECT_THAT(capture_debug.str(), HasSubstr("REST server started: " + to_string(alternative_port.unpack()))); rest_server.fini(); + close(file_descriptor); +} + +class TestServer : public ServerRest +{ + void doCall() override { g_num = num; } + + C2S_PARAM(int, num); +public: + static int g_num; +}; + +int TestServer::g_num = 0; + +TEST_F(RestConfigTest, basic_flow) +{ + env.preload(); + Singleton::Consume::from(env)->registerValue("Executable Name", "tmp_test_file"); + + config.preload(); + config.init(); + + rest_server.init(); + time_proxy.init(); + mainloop_comp.init(); + + auto i_rest = Singleton::Consume::from(rest_server); + ASSERT_TRUE(i_rest->addRestCall(RestAction::ADD, "test")); + + int file_descriptor = socket(AF_INET, SOCK_STREAM, 0); + EXPECT_NE(file_descriptor, -1); + + auto primary_port = getConfiguration("connection", "Nano service API Port Alternative"); + struct sockaddr_in sa; + sa.sin_family = AF_INET; + sa.sin_port = htons(primary_port.unpack()); + sa.sin_addr.s_addr = inet_addr("127.0.0.1"); + int socket_enable = 1; + EXPECT_EQ(setsockopt(file_descriptor, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0); + + EXPECT_CALL(messaging, sendSyncMessage(_, _, _, _, _)) + .WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); + + auto mainloop = Singleton::Consume::from(mainloop_comp); + I_MainLoop::Routine stop_routine = [&] () { + EXPECT_EQ(connect(file_descriptor, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0); + string msg = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}"; + EXPECT_EQ(write(file_descriptor, msg.data(), msg.size()), msg.size()); + + while(!TestServer::g_num) { + mainloop->yield(true); + } + mainloop->stopAll(); + }; + mainloop->addOneTimeRoutine( + I_MainLoop::RoutineType::RealTime, + stop_routine, + "RestConfigTest-alternative_port_used stop routine", + true + ); + mainloop->run(); + + EXPECT_EQ(TestServer::g_num, 5); } diff --git a/core/rest/rest_ut/rest_must_param_ut.cc b/core/rest/rest_ut/rest_must_param_ut.cc old mode 100755 new mode 100644 diff --git a/core/rest/rest_ut/rest_schema_ut.cc b/core/rest/rest_ut/rest_schema_ut.cc index 08b1a30..1a2ab71 100644 --- a/core/rest/rest_ut/rest_schema_ut.cc +++ b/core/rest/rest_ut/rest_schema_ut.cc @@ -5,14 +5,15 @@ #include "singleton.h" #include "mainloop.h" #include "encryptor.h" -#include "proto_message_comp.h" +#include "i_messaging.h" +#include "messaging.h" #include "time_proxy.h" #include "environment.h" #include "config.h" #include "config_component.h" #include "agent_details.h" -#include "messaging_buffer.h" #include "instance_awareness.h" +#include "tenant_manager.h" #include #include @@ -20,6 +21,7 @@ #include #include "customized_cereal_map.h" #include "customized_cereal_multimap.h" +#include "mock/mock_agent_details.h" using namespace std; @@ -419,26 +421,53 @@ public: TEST(RestSchema, server_schema) { - AgentDetails agent_details; + Debug::setUnitTestFlag(D_MESSAGING, Debug::DebugLevel::TRACE); + ::Environment env; TimeProxyComponent time_proxy; MainloopComponent mainloop_comp; - ::Environment env; Encryptor encryptor; - MessagingBuffer messaging_buffer; InstanceAwareness instance_awareness; ShellCmd cmd; - ProtoMessageComp message; + I_Messaging *i_message; + Messaging message; RestServer server; ConfigComponent config; + TenantManager tenant_manager; + testing::NiceMock mock_agent_details; + env.preload(); + Singleton::Consume::from(env)->registerValue("Executable Name", "tmp_test_file"); + + config.preload(); + config.init(); + EXPECT_CALL(mock_agent_details, getAccessToken()).WillRepeatedly(testing::Return(string("accesstoken"))); + EXPECT_CALL(mock_agent_details, getFogDomain()).WillRepeatedly(testing::Return(string("127.0.0.1"))); + EXPECT_CALL(mock_agent_details, getFogPort()).WillRepeatedly(testing::Return(9777)); + + string config_json = + "{" + " \"agentSettings\": [\n" + " {\n" + " \"id\": \"123\",\n" + " \"key\": \"eventBuffer.maxSizeOnDiskInMB\",\n" + " \"value\": \"1\"\n" + " },\n" + " {\n" + " \"id\": \"123\",\n" + " \"key\": \"eventBuffer.baseFolder\",\n" + " \"value\": \"/test_data/\"\n" + " }]\n" + "}"; + + istringstream ss(config_json); + Singleton::Consume::from(config)->loadConfiguration(ss); setConfiguration(false, string("message"), string("HTTPS connection")); setConfiguration(uint(9777), string("connection"), string("Nano service API Port Primary")); setConfiguration(uint(9778), string("connection"), string("Nano service API Port Alternative")); - Singleton::Consume::from(env)->registerValue("Executable Name", "a/b/"); - - messaging_buffer.init(); message.init(); + + server.init(); cmd.init(); time_proxy.init(); @@ -467,14 +496,21 @@ TEST(RestSchema, server_schema) true ); - auto i_message = Singleton::Consume::from(message); + i_message = Singleton::Consume::from(message); I_MainLoop::Routine action = [&stop, i_message] () { GetSchema schema; - Flags conn_flags; - conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN); - EXPECT_TRUE( - i_message->sendObject(schema, I_Messaging::Method::GET, "127.0.0.1", 9777, conn_flags, "/add-int") + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::ONE_TIME_CONN); + MessageMetadata message_metadata("127.0.0.1", 9777, conn_flags); + message_metadata.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + auto is_message_sent = i_message->sendSyncMessage( + HTTPMethod::GET, + "/add-int", + schema, + MessageCategory::GENERIC, + message_metadata ); + EXPECT_TRUE(is_message_sent.ok()); vector expected_req = { "must_int" }; EXPECT_EQ(schema.required.get(), expected_req); ProperitiesSchema properties(schema.properties.get()); @@ -494,7 +530,6 @@ TEST(RestSchema, server_schema) TEST(RestSchema, short_connection_server) { TimeProxyComponent time_proxy; - ProtoMessageComp message; AgentDetails agent_details; MainloopComponent mainloop_comp; ::Environment env; diff --git a/core/shell_cmd/shell_cmd.cc b/core/shell_cmd/shell_cmd.cc old mode 100755 new mode 100644 diff --git a/core/shmem_ipc/shared_ipc_debug.h b/core/shmem_ipc/shared_ipc_debug.h old mode 100755 new mode 100644 diff --git a/core/shmem_ipc/shared_ring_queue.h b/core/shmem_ipc/shared_ring_queue.h old mode 100755 new mode 100644 diff --git a/core/shmem_ipc/shmem_ipc_ut/shared_ring_queue_ut.cc b/core/shmem_ipc/shmem_ipc_ut/shared_ring_queue_ut.cc old mode 100755 new mode 100644 diff --git a/core/shmem_ipc/shmem_ipc_ut/shmem_ipc_ut.cc b/core/shmem_ipc/shmem_ipc_ut/shmem_ipc_ut.cc old mode 100755 new mode 100644 diff --git a/core/singleton/singleton_ut/CMakeLists.txt b/core/singleton/singleton_ut/CMakeLists.txt index 024fda6..5f02076 100644 --- a/core/singleton/singleton_ut/CMakeLists.txt +++ b/core/singleton/singleton_ut/CMakeLists.txt @@ -1,5 +1,5 @@ add_unit_test( singleton_ut "singleton_ut.cc" - "singleton;rest;environment;metric;event_is;-lboost_regex" + "singleton;messaging;rest;environment;metric;event_is;-lboost_regex" ) diff --git a/core/socket_is/socket_is.cc b/core/socket_is/socket_is.cc old mode 100755 new mode 100644 diff --git a/core/tenant_manager/tenant_manager.cc b/core/tenant_manager/tenant_manager.cc index 14b2acd..731d5fc 100644 --- a/core/tenant_manager/tenant_manager.cc +++ b/core/tenant_manager/tenant_manager.cc @@ -117,7 +117,7 @@ private: I_Messaging *i_messaging = nullptr; TenantManagerType type; - ::Flags conn_flags; + ::Flags conn_flags; }; class LoadNewTenants : public ServerRest @@ -205,7 +205,7 @@ TenantManager::Impl::init() auto is_orchestrator = Singleton::Consume::by()->get("Is Orchestrator"); type = (is_orchestrator.ok() && *is_orchestrator) ? TenantManagerType::SERVER : TenantManagerType::CLIENT; - conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN); + conn_flags.setFlag(MessageConnectionConfig::ONE_TIME_CONN); i_messaging = Singleton::Consume::by(); if (type == TenantManagerType::SERVER) { @@ -229,27 +229,34 @@ TenantManager::Impl::getAllTenants() const GetActiveTenants active_tenant; - auto res = i_messaging->sendObject( + MessageMetadata get_active_tenants_req_md("127.0.0.1", 7777, conn_flags); + get_active_tenants_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + auto get_active_tenants_req_status = i_messaging->sendSyncMessage( + HTTPMethod::POST, + "/show-active-tenants", active_tenant, - I_Messaging::Method::POST, - "127.0.0.1", - 7777, - conn_flags, - "/show-active-tenants" + MessageCategory::GENERIC, + get_active_tenants_req_md ); - if (!res) { - i_messaging->sendObject( + if (!get_active_tenants_req_status.ok()) { + MessageMetadata secondery_port_req_md("127.0.0.1", 7778, conn_flags); + secondery_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + get_active_tenants_req_status = i_messaging->sendSyncMessage( + HTTPMethod::POST, + "/show-active-tenants", active_tenant, - I_Messaging::Method::POST, - "127.0.0.1", - 7778, - conn_flags, - "/show-active-tenants" + MessageCategory::GENERIC, + secondery_port_req_md ); } + if (!get_active_tenants_req_status.ok()) { + auto err = get_active_tenants_req_status.getErr(); + dbgWarning(D_TENANT_MANAGER) << "Failed to get all active tenants. Error: " << err.getBody(); + } return active_tenant.active_tenants.get(); + } set @@ -259,26 +266,32 @@ TenantManager::Impl::getProfileIds(const string &_tenant_id) const GetProfileIds tenant_id(_tenant_id); - auto res = i_messaging->sendObject( + MessageMetadata get_profile_id_req_md("127.0.0.1", 7777, conn_flags); + get_profile_id_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + auto get_profile_id_req_status = i_messaging->sendSyncMessage( + HTTPMethod::POST, + "/show-profile-ids", tenant_id, - I_Messaging::Method::POST, - "127.0.0.1", - 7777, - conn_flags, - "/show-profile-ids" + MessageCategory::GENERIC, + get_profile_id_req_md ); - if (!res) { - i_messaging->sendObject( + + if (!get_profile_id_req_status.ok()) { + MessageMetadata secondery_port_req_md("127.0.0.1", 7778, conn_flags); + secondery_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + get_profile_id_req_status = i_messaging->sendSyncMessage( + HTTPMethod::POST, + "/show-profile-ids", tenant_id, - I_Messaging::Method::POST, - "127.0.0.1", - 7778, - conn_flags, - "/show-profile-ids" + MessageCategory::GENERIC, + secondery_port_req_md ); } - + if (!get_profile_id_req_status.ok()) { + auto err = get_profile_id_req_status.getErr(); + dbgWarning(D_TENANT_MANAGER) << "Failed to get all active tenants. Error: " << err.getBody(); + } return tenant_id.profile_ids.get(); } diff --git a/core/time_proxy/time_proxy_ut/time_print_ut.cc b/core/time_proxy/time_proxy_ut/time_print_ut.cc old mode 100755 new mode 100644 diff --git a/core/version/version.cc b/core/version/version.cc old mode 100755 new mode 100644 diff --git a/core/version/version_ut/version_ut.cc b/core/version/version_ut/version_ut.cc old mode 100755 new mode 100644 diff --git a/nodes/orchestration/main.cc b/nodes/orchestration/main.cc index f524c81..213a66b 100755 --- a/nodes/orchestration/main.cc +++ b/nodes/orchestration/main.cc @@ -30,7 +30,6 @@ #include "rest_server.h" #include "logging_comp.h" #include "rest.h" -#include "proto_message_comp.h" #include "encryptor.h" #include "downloader.h" #include "orchestration_tools.h" @@ -44,7 +43,6 @@ #include "signal_handler.h" #include "cpu.h" #include "memory_consumption.h" -#include "messaging_buffer.h" #include "agent_details_reporter.h" #include "instance_awareness.h" #include "socket_is.h" diff --git a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc index 70789ac..28d990c 100755 --- a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc +++ b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc @@ -93,6 +93,7 @@ enum class Service { DEDICATED_NETWORK_HANDLER, HELLO_WORLD, IDA, + IDA_SAML, IOT_ACCESS_CONTROL, HORIZON_TELEMETRY, @@ -169,6 +170,7 @@ getServiceString(const Service service) case (Service::IOT_WLP): return "workload-protection"; case (Service::HELLO_WORLD): return "hello-world"; case (Service::IDA): return "identity-awareness"; + case (Service::IDA_SAML): return "ida-saml"; case (Service::IOT_ACCESS_CONTROL): return "iot-access-control"; case (Service::HORIZON_TELEMETRY): return "horizon-telemetry"; default: @@ -344,6 +346,11 @@ getServiceConfig (const Service service) filesystem_path + "/conf/cp-nano-ida-debug-conf.json", log_files_path + "/nano_agent/cp-nano-ida.dbg" ); + case (Service::IDA_SAML): + return ServiceConfig( + filesystem_path + "/conf/cp-nano-ida-saml-debug-conf.json", + log_files_path + "/nano_agent/cp-nano-ida-saml.dbg" + ); case (Service::HELLO_WORLD): return ServiceConfig( filesystem_path + "/conf/cp-nano-hello-world-conf.json", @@ -1284,6 +1291,8 @@ extractServices(const vector &args) services.push_back(Service::IOT_WLP); } else if (getServiceString(Service::IDA).find(maybe_service) == 0) { services.push_back(Service::IDA); + } else if (getServiceString(Service::IDA_SAML).find(maybe_service) == 0) { + services.push_back(Service::IDA_SAML); } else if (getServiceString(Service::IOT_ACCESS_CONTROL).find(maybe_service) == 0) { services.push_back(Service::IOT_ACCESS_CONTROL); } else if (getServiceString(Service::HORIZON_TELEMETRY).find(maybe_service) == 0) { diff --git a/nodes/orchestration/package/open-appsec-ctl.sh b/nodes/orchestration/package/open-appsec-ctl.sh index 84ccb69..84bace9 100644 --- a/nodes/orchestration/package/open-appsec-ctl.sh +++ b/nodes/orchestration/package/open-appsec-ctl.sh @@ -129,7 +129,11 @@ CUSTOM_POLICY_CONF_FILE="${FILESYSTEM_PATH}/${cp_nano_conf_location}/custom_poli if [ -f ${CUSTOM_POLICY_CONF_FILE} ]; then . $CUSTOM_POLICY_CONF_FILE else - var_policy_file="${FILESYSTEM_PATH}/${cp_nano_conf_location}/local_policy.yaml" + if [ -f /ext/appsec/local_policy.yaml ]; then + var_policy_file=/ext/appsec/local_policy.yaml + else + var_policy_file="${FILESYSTEM_PATH}/${cp_nano_conf_location}/local_policy.yaml" + fi; fi is_arm32= @@ -1617,7 +1621,7 @@ stop_service() # Initials - stops eval "$stops_cmd" stops_exit_code=$? if [ $stops_exit_code -eq 0 ]; then - echo "Successfully stoped the $stops_service_to_stop service" + echo "Successfully stopped the $stops_service_to_stop service" exit 0 fi echo "Failed to stop the $stops_service_to_stop service" @@ -1759,7 +1763,11 @@ run() # Initials - r shift if [ ! -z $1 ]; then if [ "-d" = "$1" ] || [ "--default-policy" = "$1" ]; then - var_new_policy_file="${FILESYSTEM_PATH}/${cp_nano_conf_location}/local_policy.yaml" + if [ -f /ext/appsec/local_policy.yaml ]; then + var_new_policy_file=/ext/appsec/local_policy.yaml + else + var_new_policy_file="${FILESYSTEM_PATH}/${cp_nano_conf_location}/local_policy.yaml" + fi elif [ -f $1 ]; then var_new_policy_file=$1 else @@ -1767,7 +1775,11 @@ run() # Initials - r exit 1 fi else - var_new_policy_file="${FILESYSTEM_PATH}/${cp_nano_conf_location}/local_policy.yaml" + if [ -f /ext/appsec/local_policy.yaml ]; then + var_new_policy_file=/ext/appsec/local_policy.yaml + else + var_new_policy_file="${FILESYSTEM_PATH}/${cp_nano_conf_location}/local_policy.yaml" + fi fi is_apply_policy_needed @@ -1782,7 +1794,13 @@ run() # Initials - r http://127.0.0.1:"$(extract_api_port 'orchestration')"/set-apply-policy 2>&1) is_policy_file_changed is_changed=$? + counter=0 while [ ${is_changed} -eq 0 ]; do + counter=$((counter+1)) + if [ ${counter} -gt 40 ]; then + echo "\nPolicy didn't change please verify that you have a legal new policy" + exit 1 + fi echo -n "." sleep 3 is_policy_file_changed @@ -1799,7 +1817,11 @@ run() # Initials - r echo $var_policy_file elif [ "-vl" = "$1" ] || [ "--view-logs" = "$1" ]; then record_command $@ - less $LOG_FILE_PATH/nano_agent/cp-nano-http-transaction-handler.log? + if ls /var/log/nano_agent/cp-nano-http-transaction-handler.log? 1>dev/null 2>&1; then + less $LOG_FILE_PATH/nano_agent/cp-nano-http-transaction-handler.log? + else + echo "No logs found" + fi else usage fi diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index 92fd5a6..0221c3f 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -3,6 +3,7 @@ # Copyright Check Point Software Technologies LTD FILESYSTEM_PATH="/etc/cp" LOG_FILE_PATH="/var/log" +SMB_LOG_FILE_PATH="/storage" USR_LIB_PATH="/usr/lib" USR_SBIN_PATH="/usr/sbin" INIT_D_PATH="/etc/init.d" @@ -383,7 +384,7 @@ cp_print() if [ "$is_smb" != "1" ]; then printf "%b\n" "$1" >> ${LOG_FILE_PATH}/${LOG_PATH}/${INSTALLATION_LOG_FILE} else - printf "%b\n" "$1" > ${LOG_FILE_PATH}/${LOG_PATH}/${INSTALLATION_LOG_FILE} + printf "%b\n" "$1" >> ${SMB_LOG_FILE_PATH}/${LOG_PATH}/${INSTALLATION_LOG_FILE} fi } @@ -488,7 +489,12 @@ install_watchdog() fi if [ "$old_cp_nano_watchdog_md5" = "$new_cp_nano_watchdog_md5" ]; then # Watchdog did not changed - cp_print "There is no update in watchdog. Everything is up to date." + cp_print "There is no update in watchdog. Everything is up to date. Reregistering services to be on the sae side." + cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration $var_arch_flag" + if [ "$IS_K8S_ENV" = "true" ]; then + cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh" + fi + return fi cp_print "Installing the watchdog" ${FORCE_STDOUT}