From 9ac186f4830fe9bce2cc366a1a441fe7948f78e7 Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Wed, 23 Aug 2023 14:40:32 +0000 Subject: [PATCH] Aug_23_2023-Dev --- .../nginx/ngx_module/ngx_cp_hook_threads.c | 36 ++++++++++-- .../nginx/ngx_module/ngx_cp_hook_threads.h | 20 +++++++ attachments/nginx/ngx_module/ngx_cp_hooks.c | 22 +++++--- .../nginx/ngx_module/ngx_cp_initializer.c | 56 +++++++++---------- attachments/nginx/ngx_module/ngx_cp_utils.c | 13 ++++- attachments/nginx/ngx_module/ngx_cp_utils.h | 13 ++++- 6 files changed, 116 insertions(+), 44 deletions(-) diff --git a/attachments/nginx/ngx_module/ngx_cp_hook_threads.c b/attachments/nginx/ngx_module/ngx_cp_hook_threads.c index ce87e7e..eacdfcb 100644 --- a/attachments/nginx/ngx_module/ngx_cp_hook_threads.c +++ b/attachments/nginx/ngx_module/ngx_cp_hook_threads.c @@ -55,18 +55,43 @@ static ngx_int_t already_registered = 0; ///< Registration status with the nano static const ngx_int_t inspection_irrelevant = INSPECTION_IRRELEVANT; extern struct timeval metric_timeout; ///< Holds per-session metric timeout. -void +#define MIN_REGISTRATION_DURATION_MSEC 100 +#define MAX_REGISTRATION_DURATION_MSEC 3200 +static uint current_registration_duration_msec = MIN_REGISTRATION_DURATION_MSEC; +static struct timeval registration_timeout = (struct timeval){0}; + +inline void set_already_registered(ngx_int_t value) { already_registered = value; } -ngx_int_t +inline ngx_int_t get_already_registered() { return already_registered; } +inline void +reset_registration_timeout(void) +{ + registration_timeout = get_timeout_val_msec(current_registration_duration_msec); + if (current_registration_duration_msec < MAX_REGISTRATION_DURATION_MSEC) + current_registration_duration_msec *= 2; +} + +inline void +reset_registration_timeout_duration(void) +{ + current_registration_duration_msec = MIN_REGISTRATION_DURATION_MSEC; +} + +inline ngx_int_t +is_registration_timeout_reached(void) +{ + return is_timeout_reached(®istration_timeout); +} + void init_thread_ctx( struct ngx_http_cp_event_thread_ctx_t *ctx, @@ -88,14 +113,15 @@ ngx_http_cp_registration_thread(void *_ctx) { struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t *)_ctx; ngx_int_t res = ngx_cp_attachment_init_process(ctx->request); - if (res == NGX_ABORT && already_registered) { - already_registered = 0; + if (res == NGX_ABORT && get_already_registered()) { + set_already_registered(0); disconnect_communication(); reset_transparent_mode(); } if (res != NGX_OK) { // failed to register to the attachment service. - if (already_registered) handle_inspection_failure(registration_failure_weight, fail_mode_verdict, ctx->session_data_p); + if (get_already_registered()) + handle_inspection_failure(registration_failure_weight, fail_mode_verdict, ctx->session_data_p); write_dbg(DBG_LEVEL_DEBUG, "Communication with nano service is not ready yet"); THREAD_CTX_RETURN(NGX_OK); } diff --git a/attachments/nginx/ngx_module/ngx_cp_hook_threads.h b/attachments/nginx/ngx_module/ngx_cp_hook_threads.h index 5619130..2c29df1 100644 --- a/attachments/nginx/ngx_module/ngx_cp_hook_threads.h +++ b/attachments/nginx/ngx_module/ngx_cp_hook_threads.h @@ -65,6 +65,26 @@ void set_already_registered(ngx_int_t value); /// ngx_int_t get_already_registered(void); +/// +/// @brief Resets the registration timeout duration to its minimal value. +/// @returns NULL. +/// +void reset_registration_timeout_duration(void); + +/// +/// @brief Resets the registration timeout. +/// The timeout is being reset to now + timeout duration, where the timeout duration gets doubled every reset. +/// The initial timeout duration is 100 msec, and the maximum is 3200 msec. +/// @returns NULL. +/// +void reset_registration_timeout(void); + +/// +/// @brief Checks if registration timeout has elapsed. +/// @returns 1 if timeout has elapsed, 0 if not. +/// +ngx_int_t is_registration_timeout_reached(void); + /// /// @brief Initates ngx_http_cp_event_thread_ctx_t struct. /// @param[in, out] ctx struct to initiate. diff --git a/attachments/nginx/ngx_module/ngx_cp_hooks.c b/attachments/nginx/ngx_module/ngx_cp_hooks.c index 9121f5d..dfe5e13 100644 --- a/attachments/nginx/ngx_module/ngx_cp_hooks.c +++ b/attachments/nginx/ngx_module/ngx_cp_hooks.c @@ -372,13 +372,20 @@ ngx_http_cp_req_header_handler(ngx_http_request_t *request) } if (!get_already_registered() || !isIpcReady()) { - write_dbg(DBG_LEVEL_DEBUG, "spawn ngx_http_cp_registration_thread"); - res = ngx_cp_run_in_thread_timeout( - ngx_http_cp_registration_thread, - (void *)&ctx, - registration_thread_timeout_msec, - "ngx_http_cp_registration_thread" - ); + if (is_registration_timeout_reached()) { + write_dbg(DBG_LEVEL_DEBUG, "spawn ngx_http_cp_registration_thread"); + reset_registration_timeout(); + res = ngx_cp_run_in_thread_timeout( + ngx_http_cp_registration_thread, + (void *)&ctx, + registration_thread_timeout_msec, + "ngx_http_cp_registration_thread" + ); + } else { + res = 0; + write_dbg(DBG_LEVEL_DEBUG, "Attachment registration has recently started, wait for timeout"); + } + if (!res) { // failed to execute thread task, or it timed out session_data_p->verdict = fail_mode_verdict == NGX_OK ? TRAFFIC_VERDICT_ACCEPT : TRAFFIC_VERDICT_DROP; @@ -405,6 +412,7 @@ ngx_http_cp_req_header_handler(ngx_http_request_t *request) } set_already_registered(1); + reset_registration_timeout_duration(); if (handle_shmem_corruption() == NGX_ERROR) { session_data_p->verdict = fail_mode_verdict == NGX_OK ? TRAFFIC_VERDICT_ACCEPT : TRAFFIC_VERDICT_DROP; diff --git a/attachments/nginx/ngx_module/ngx_cp_initializer.c b/attachments/nginx/ngx_module/ngx_cp_initializer.c index 381fddb..7764d4f 100644 --- a/attachments/nginx/ngx_module/ngx_cp_initializer.c +++ b/attachments/nginx/ngx_module/ngx_cp_initializer.c @@ -91,7 +91,7 @@ exchange_communication_data_with_service( if (res < 0) { close(socket); socket = -1; - write_dbg_if_needed(DBG_LEVEL_TRACE, "Failed to communicate with the socket"); + write_dbg(DBG_LEVEL_TRACE, "Failed to communicate with the socket, Error: %s", strerror(errno)); break; } @@ -102,7 +102,7 @@ exchange_communication_data_with_service( if (is_timeout_reached(remaining_timeout)) { close(socket); socket = -1; - write_dbg_if_needed(DBG_LEVEL_TRACE, "Reached timeout while communicating with the socket"); + write_dbg(DBG_LEVEL_TRACE, "Reached timeout while communicating with the socket"); break; } } @@ -132,7 +132,7 @@ init_signaling_socket() // Setup a new socket comm_socket = socket(AF_UNIX, SOCK_STREAM, 0); if (comm_socket < 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Could not create socket"); + write_dbg(DBG_LEVEL_WARNING, "Could not create socket, Error: %s", strerror(errno)); return NGX_ERROR; } @@ -142,7 +142,7 @@ init_signaling_socket() if (connect(comm_socket, (struct sockaddr *)&server, sizeof(struct sockaddr_un)) == -1) { close(comm_socket); comm_socket = -1; - write_dbg_if_needed( + write_dbg( DBG_LEVEL_DEBUG, "Could not connect to nano service. Path: %s, Error: %s", server.sun_path, @@ -165,7 +165,7 @@ init_signaling_socket() &timeout ); if (res <= 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Failed to send unique id size"); + write_dbg(DBG_LEVEL_WARNING, "Failed to send unique id size"); return NGX_ERROR; } @@ -177,13 +177,13 @@ init_signaling_socket() &timeout ); if (res <= 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Failed to send unique id %s", unique_id); + write_dbg(DBG_LEVEL_WARNING, "Failed to send unique id %s", unique_id); return NGX_ERROR; } res = exchange_communication_data_with_service(comm_socket, &nginx_user_id, sizeof(uint32_t), WRITE_TO_SOCKET, &timeout); if (res <= 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Failed to send nginx user id"); + write_dbg(DBG_LEVEL_WARNING, "Failed to send nginx user id"); return NGX_ERROR; } @@ -195,7 +195,7 @@ init_signaling_socket() &timeout ); if (res <= 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Failed to send nginx group id"); + write_dbg(DBG_LEVEL_WARNING, "Failed to send nginx group id"); return NGX_ERROR; } @@ -209,11 +209,11 @@ init_signaling_socket() &timeout ); if (res <= 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Failed to read registration ack"); + write_dbg(DBG_LEVEL_WARNING, "Failed to read registration ack"); return NGX_ERROR; } - write_dbg_if_needed(DBG_LEVEL_DEBUG, "Successfully connected on client socket %d", comm_socket); + write_dbg(DBG_LEVEL_DEBUG, "Successfully connected on client socket %d", comm_socket); return NGX_OK; } @@ -298,7 +298,7 @@ register_to_attachments_manager(ngx_http_request_t *request) // Connect a new socket. registration_socket = socket(AF_UNIX, SOCK_STREAM, 0); if (registration_socket < 0) { - write_dbg(DBG_LEVEL_WARNING, "Could not create socket"); + write_dbg(DBG_LEVEL_WARNING, "Could not create socket, Error: %s", strerror(errno)); return NGX_ERROR; } @@ -309,7 +309,7 @@ register_to_attachments_manager(ngx_http_request_t *request) cur_errno = errno; close(registration_socket); registration_socket = -1; - write_dbg_if_needed( + write_dbg( DBG_LEVEL_DEBUG, "Could not connect to nano service. Path: %s, Error: %s, Errno: %d", server.sun_path, @@ -338,7 +338,7 @@ register_to_attachments_manager(ngx_http_request_t *request) &timeout ); if (res <= 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Failed to send attachment type"); + write_dbg(DBG_LEVEL_WARNING, "Failed to send attachment type"); close(registration_socket); registration_socket = -1; return NGX_ERROR; @@ -352,7 +352,7 @@ register_to_attachments_manager(ngx_http_request_t *request) &timeout ); if (res <= 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Failed to send worker ID"); + write_dbg(DBG_LEVEL_WARNING, "Failed to send worker ID"); close(registration_socket); registration_socket = -1; return NGX_ERROR; @@ -366,7 +366,7 @@ register_to_attachments_manager(ngx_http_request_t *request) &timeout ); if (res <= 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Failed to send workers amount"); + write_dbg(DBG_LEVEL_WARNING, "Failed to send workers amount"); close(registration_socket); registration_socket = -1; return NGX_ERROR; @@ -380,7 +380,7 @@ register_to_attachments_manager(ngx_http_request_t *request) &timeout ); if (res <= 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Failed to send family name size"); + write_dbg(DBG_LEVEL_WARNING, "Failed to send family name size"); close(registration_socket); registration_socket = -1; return NGX_ERROR; @@ -395,7 +395,7 @@ register_to_attachments_manager(ngx_http_request_t *request) &timeout ); if (res <= 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Failed to send family name"); + write_dbg(DBG_LEVEL_WARNING, "Failed to send family name"); close(registration_socket); registration_socket = -1; return NGX_ERROR; @@ -416,7 +416,7 @@ register_to_attachments_manager(ngx_http_request_t *request) ); if (res <= 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Failed to read path length"); + write_dbg(DBG_LEVEL_WARNING, "Failed to read path length"); close(registration_socket); registration_socket = -1; return NGX_ERROR; @@ -430,7 +430,7 @@ register_to_attachments_manager(ngx_http_request_t *request) &timeout ); if (res <= 0) { - write_dbg_if_needed(DBG_LEVEL_WARNING, "Failed to read socket path"); + write_dbg(DBG_LEVEL_WARNING, "Failed to read socket path"); close(registration_socket); registration_socket = -1; return NGX_ERROR; @@ -440,7 +440,7 @@ register_to_attachments_manager(ngx_http_request_t *request) shared_verdict_signal_path[path_length] = '\0'; int32_t dbg_id = worker_id; int32_t dbg_size = workers_amount_to_send; - write_dbg_if_needed( + write_dbg( DBG_LEVEL_DEBUG, "Successfully registered on client. socket: %d, instance ID: %d, instances amount: %d, received path: %s", registration_socket, @@ -535,23 +535,23 @@ ngx_cp_attachment_init_process(ngx_http_request_t *request) if (need_registration) { if (register_to_attachments_manager(request) == NGX_ERROR) { - write_dbg_if_needed(DBG_LEVEL_INFO, "Failed to register to Attachments Manager service"); + write_dbg(DBG_LEVEL_INFO, "Failed to register to Attachments Manager service"); return NGX_ERROR; } need_registration = 0; } if (comm_socket < 0) { - write_dbg_if_needed(DBG_LEVEL_DEBUG, "Registering to nano service"); + write_dbg(DBG_LEVEL_DEBUG, "Registering to nano service"); if (init_signaling_socket() == NGX_ERROR) { - write_dbg_if_needed(DBG_LEVEL_DEBUG, "Failed to register to the Nano Service"); + write_dbg(DBG_LEVEL_DEBUG, "Failed to register to the Nano Service"); need_registration = 1; return NGX_ERROR; } } if (init_general_config(SHARED_ATTACHMENT_CONF_PATH) == NGX_ERROR) { - write_dbg_if_needed(DBG_LEVEL_INFO, "Failed to initialize attachment's configuration"); + write_dbg(DBG_LEVEL_INFO, "Failed to initialize attachment's configuration"); return NGX_ERROR; } @@ -560,7 +560,7 @@ ngx_cp_attachment_init_process(ngx_http_request_t *request) static const int max_ipc_init_retry_count = 10; static int max_retry_count = max_ipc_init_retry_count; if (nano_service_ipc == NULL) { - write_dbg_if_needed(DBG_LEVEL_INFO, "Initializing IPC channel"); + write_dbg(DBG_LEVEL_INFO, "Initializing IPC channel"); nano_service_ipc = initIpc( unique_id, nginx_user_id, @@ -574,7 +574,7 @@ ngx_cp_attachment_init_process(ngx_http_request_t *request) restart_communication(request); max_retry_count = max_ipc_init_retry_count; } - write_dbg_if_needed(DBG_LEVEL_INFO, "Failed to initialize IPC with nano service"); + write_dbg(DBG_LEVEL_INFO, "Failed to initialize IPC with nano service"); return NGX_ERROR; } } @@ -611,7 +611,7 @@ ngx_cp_attachment_init_process(ngx_http_request_t *request) int restart_communication(ngx_http_request_t *request) { - write_dbg_if_needed(DBG_LEVEL_TRACE, "Restarting communication channels with nano service"); + write_dbg(DBG_LEVEL_TRACE, "Restarting communication channels with nano service"); if (nano_service_ipc != NULL) { destroyIpc(nano_service_ipc, 0); nano_service_ipc = NULL; @@ -619,7 +619,7 @@ restart_communication(ngx_http_request_t *request) if (init_signaling_socket() == NGX_ERROR) { if (register_to_attachments_manager(request) == NGX_ERROR) { - write_dbg_if_needed(DBG_LEVEL_DEBUG, "Failed to register to Attachments Manager service"); + write_dbg(DBG_LEVEL_DEBUG, "Failed to register to Attachments Manager service"); return -1; } diff --git a/attachments/nginx/ngx_module/ngx_cp_utils.c b/attachments/nginx/ngx_module/ngx_cp_utils.c index 801110b..21ac8bf 100644 --- a/attachments/nginx/ngx_module/ngx_cp_utils.c +++ b/attachments/nginx/ngx_module/ngx_cp_utils.c @@ -491,6 +491,17 @@ get_timeout_val_usec(const int delta_time_in_usec) return time; } +struct timeval +get_timeout_val_msec(const int delta_time_in_msec) +{ + struct timeval time; + + time = getCurrTimeFast(); + time.tv_sec += delta_time_in_msec / 1000; + time.tv_usec += (delta_time_in_msec % 1000) * 1000; + return time; +} + void set_custom_response(const ngx_str_t *title, const ngx_str_t *body, const ngx_str_t *uuid, ngx_uint_t response_code) { @@ -577,7 +588,7 @@ get_response_page(ngx_http_request_t *request, ngx_chain_t (*out_chain)[7]) for (idx = 0; idx < 7; idx++) { buf[idx] = ngx_calloc_buf(request->pool); - if (buf == NULL) { + if (buf[idx] == NULL) { for (; idx >= 0; idx--) { ngx_pfree(request->pool, buf[idx]); } diff --git a/attachments/nginx/ngx_module/ngx_cp_utils.h b/attachments/nginx/ngx_module/ngx_cp_utils.h index 294712d..46b9452 100644 --- a/attachments/nginx/ngx_module/ngx_cp_utils.h +++ b/attachments/nginx/ngx_module/ngx_cp_utils.h @@ -145,7 +145,7 @@ ngx_int_t free_list_from_pool(ngx_pool_t *memory_pool, ngx_list_t *list); /// @param[in] bucket_size Bucket size to set in the hash table. /// @param[in, out] keys Keys initiate and put into the hash_table. /// @param[in, out] initial_value Initial hash value. -/// @param[in, out] initial_value_size Initial hash value's size. +/// @param[in, out] initial_value_size Initial hash value's size. /// @return ngx_int_t /// - #NGX_OK. /// - #NGX_ERROR. @@ -281,6 +281,13 @@ struct timeval get_timeout_val_sec(const int delta_time_in_sec); /// struct timeval get_timeout_val_usec(const int delta_time_in_usec); +/// +/// @brief Get delta current time + delta_time_in_msec value in msec. +/// @param[in] delta_time_in_msec Delta time to return +/// @returns timeval struct with tv_sec, tv_usec set accordingly +/// +struct timeval get_timeout_val_msec(const int delta_time_in_msec); + /// /// @brief Get the currently set response page. /// @param[in, out] request NGINX request, used to get the NGINX pool to allocate buffer needed for out_chain. @@ -420,12 +427,12 @@ u_char *reverse_strnchr(u_char *string, const u_char char_to_find, const size_t ngx_msec_t get_keep_alive_interval_msec(void); /// -/// @brief Update CPU's max, average metrics and time usage metric. +/// @brief Update CPU's max, average metrics and time usage metric. /// void set_metric_cpu_usage(void); /// -/// @brief Update memory's max, average metrics and time usage metric. +/// @brief Update memory's max, average metrics and time usage metric. /// void set_metric_memory_usage(void);