From 26e2a387133b9640173c0a26246e65b3298c0d9d Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Mon, 10 Feb 2025 16:27:27 +0000 Subject: [PATCH] sync code --- .../nginx_attachment_util.cc | 12 +++++ .../nginx/ngx_module/ngx_cp_compression.c | 12 ++++- attachments/nginx/ngx_module/ngx_cp_hooks.c | 47 +++++++++---------- attachments/nginx/ngx_module/ngx_cp_io.c | 5 +- attachments/nginx/ngx_module/ngx_cp_utils.c | 23 ++++++++- attachments/nginx/ngx_module/ngx_cp_utils.h | 2 + .../http_configuration/http_configuration.cc | 4 ++ .../attachments/nginx_attachment_util.h | 3 ++ 8 files changed, 80 insertions(+), 28 deletions(-) diff --git a/attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc b/attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc index ddf3a01..85cd5b6 100644 --- a/attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc +++ b/attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc @@ -95,6 +95,18 @@ getFailOpenHoldTimeout() return conf_data.getNumericalValue("fail_open_hold_timeout"); } +unsigned int +getHoldVerdictPollingTime() +{ + return conf_data.getNumericalValue("hold_verdict_polling_time"); +} + +unsigned int +getHoldVerdictRetries() +{ + return conf_data.getNumericalValue("hold_verdict_retries"); +} + unsigned int getMaxSessionsPerMinute() { diff --git a/attachments/nginx/ngx_module/ngx_cp_compression.c b/attachments/nginx/ngx_module/ngx_cp_compression.c index 5cc39c2..99a91a7 100644 --- a/attachments/nginx/ngx_module/ngx_cp_compression.c +++ b/attachments/nginx/ngx_module/ngx_cp_compression.c @@ -370,7 +370,17 @@ compression_chain_filter( ); if (compression_result != NGX_OK) { // Failed to decompress or compress. - free_chain(pool, *body); + if (curr_original_contents_link != NULL) { + write_dbg( + DBG_LEVEL_WARNING, + "Failed to %s chain: free unused chain link " + "and copy original chain back to body up to current link", + should_compress ? "compress" : "decompress" + ); + ngx_free_chain(pool, curr_original_contents_link); + curr_original_contents_link = NULL; + copy_chain_buffers(*body, *original_body_contents); + } return NGX_ERROR; } diff --git a/attachments/nginx/ngx_module/ngx_cp_hooks.c b/attachments/nginx/ngx_module/ngx_cp_hooks.c index 55a1f84..9141f3d 100644 --- a/attachments/nginx/ngx_module/ngx_cp_hooks.c +++ b/attachments/nginx/ngx_module/ngx_cp_hooks.c @@ -118,15 +118,10 @@ ngx_session_data_cleanup(void *data) { if (data == NULL) return; ngx_http_cp_session_data *session_data = (ngx_http_cp_session_data *)data; - write_dbg(DBG_LEVEL_TRACE, "Cleaning up session data for session ID %d", session_data->session_id); + write_dbg(DBG_LEVEL_DEBUG, "Cleaning up session data for session ID %d", session_data->session_id); if (session_data->response_data.original_compressed_body != NULL) { - ngx_chain_t *current = session_data->response_data.original_compressed_body; - while (current != NULL) { - ngx_chain_t *next = current->next; - ngx_free_chain(session_data->response_data.request_pool, current); - current = next; - } + free_chain(session_data->response_data.request_pool, session_data->response_data.original_compressed_body); session_data->response_data.original_compressed_body = NULL; } @@ -220,8 +215,8 @@ ngx_int_t ngx_http_cp_hold_verdict(struct ngx_http_cp_event_thread_ctx_t *ctx) { ngx_http_cp_session_data *session_data_p = ctx->session_data_p; - for (uint i = 0; i < 3; i++) { - sleep(1); + for (uint i = 0; i < hold_verdict_retries; i++) { + sleep(hold_verdict_polling_time); int res = ngx_cp_run_in_thread_timeout( ngx_http_cp_hold_verdict_thread, (void *)ctx, @@ -436,16 +431,16 @@ ngx_http_cp_req_header_handler(ngx_http_request_t *request) if (is_in_transparent_mode()) { updateMetricField(TRANSPARENTS_COUNT, 1); - return fail_mode_verdict; + return fail_mode_verdict == NGX_OK ? NGX_DECLINED : NGX_ERROR; } if (is_ngx_cp_attachment_disabled(request)) { write_dbg(DBG_LEVEL_TRACE, "Ignoring inspection of request on a disabled location"); - return NGX_OK; + return NGX_DECLINED; } session_data_p = init_cp_session_data(request); - if (session_data_p == NULL) return NGX_OK; + if (session_data_p == NULL) return NGX_DECLINED; set_current_session_id(session_data_p->session_id); write_dbg(DBG_LEVEL_DEBUG, "Request header filter handling session ID: %d", session_data_p->session_id); @@ -455,7 +450,7 @@ ngx_http_cp_req_header_handler(ngx_http_request_t *request) sessions_per_minute_verdict = enforce_sessions_rate(); if (sessions_per_minute_verdict != TRAFFIC_VERDICT_INSPECT) { session_data_p->verdict = sessions_per_minute_verdict; - return sessions_per_minute_verdict == TRAFFIC_VERDICT_ACCEPT ? NGX_OK : NGX_ERROR; + return sessions_per_minute_verdict == TRAFFIC_VERDICT_ACCEPT ? NGX_DECLINED : NGX_ERROR; } if (!get_already_registered() || !isIpcReady()) { @@ -484,7 +479,7 @@ ngx_http_cp_req_header_handler(ngx_http_request_t *request) ); updateMetricField(REG_THREAD_TIMEOUT, 1); - return fail_mode_verdict; + return fail_mode_verdict == NGX_OK ? NGX_DECLINED : fail_mode_verdict; } write_dbg( DBG_LEVEL_DEBUG, @@ -494,7 +489,7 @@ ngx_http_cp_req_header_handler(ngx_http_request_t *request) ); if (ctx.should_return) { session_data_p->verdict = TRAFFIC_VERDICT_ACCEPT; - return ctx.res; + return ctx.res == NGX_OK ? NGX_DECLINED : ctx.res; } } @@ -509,7 +504,7 @@ ngx_http_cp_req_header_handler(ngx_http_request_t *request) session_data_p->session_id, session_data_p->verdict == TRAFFIC_VERDICT_ACCEPT ? "accept" : "drop" ); - return fail_mode_verdict; + return fail_mode_verdict == NGX_OK ? NGX_DECLINED : fail_mode_verdict; } handle_static_resource_result = handle_static_resource_request( @@ -538,7 +533,7 @@ ngx_http_cp_req_header_handler(ngx_http_request_t *request) ); updateMetricField(REQ_HEADER_THREAD_TIMEOUT, 1); - return fail_mode_verdict; + return fail_mode_verdict == NGX_OK ? NGX_DECLINED : fail_mode_verdict; } write_dbg( DBG_LEVEL_DEBUG, @@ -552,22 +547,23 @@ ngx_http_cp_req_header_handler(ngx_http_request_t *request) if (!res) { session_data_p->verdict = fail_mode_hold_verdict == NGX_OK ? TRAFFIC_VERDICT_ACCEPT : TRAFFIC_VERDICT_DROP; updateMetricField(HOLD_THREAD_TIMEOUT, 1); - return fail_mode_verdict == NGX_OK ? TRAFFIC_VERDICT_ACCEPT : TRAFFIC_VERDICT_DROP; + return fail_mode_verdict == NGX_OK ? NGX_DECLINED : fail_mode_verdict; } } calcProcessingTime(session_data_p, &hook_time_begin, 1); if (ctx.should_return) { - return ctx.res; + return ctx.res == NGX_OK ? NGX_DECLINED : ctx.res; } // There's no body for inspection - return ngx_http_cp_finalize_request_headers_hook( + ngx_int_t result = ngx_http_cp_finalize_request_headers_hook( request, session_data_p, ctx.modifications, ctx.res ); + return result == NGX_OK ? NGX_DECLINED : result; } ngx_int_t @@ -686,7 +682,7 @@ ngx_http_cp_req_body_filter(ngx_http_request_t *request, ngx_chain_t *request_bo if (!res) { session_data_p->verdict = fail_mode_hold_verdict == NGX_OK ? TRAFFIC_VERDICT_ACCEPT : TRAFFIC_VERDICT_DROP; updateMetricField(HOLD_THREAD_TIMEOUT, 1); - return fail_mode_verdict == NGX_OK ? ngx_http_next_request_body_filter(request, request_body_chain) : NGX_HTTP_FORBIDDEN; + return fail_mode_hold_verdict == NGX_OK ? ngx_http_next_request_body_filter(request, request_body_chain) : NGX_HTTP_FORBIDDEN; } } @@ -725,7 +721,7 @@ ngx_http_cp_req_body_filter(ngx_http_request_t *request, ngx_chain_t *request_bo } if (ctx.should_return) { - return ctx.res; + return ctx.res == NGX_OK ? NGX_DECLINED : ctx.res; } if (was_transaction_timedout(session_data_p)) { session_data_p->verdict = fail_mode_verdict == NGX_OK ? TRAFFIC_VERDICT_ACCEPT : TRAFFIC_VERDICT_DROP; @@ -1029,6 +1025,7 @@ ngx_http_cp_res_body_filter(ngx_http_request_t *request, ngx_chain_t *body_chain } if (body_chain->buf->pos != NULL && session_data_p->response_data.new_compression_type != NO_COMPRESSION) { + write_dbg(DBG_LEVEL_TRACE, "Decompressing response body"); if (init_cp_session_original_body(session_data_p, request->pool) == NGX_OK) { if (session_data_p->response_data.decompression_stream == NULL) { session_data_p->response_data.decompression_stream = initCompressionStream(); @@ -1045,7 +1042,7 @@ ngx_http_cp_res_body_filter(ngx_http_request_t *request, ngx_chain_t *body_chain } if (compression_result != NGX_OK) { - copy_chain_buffers(body_chain, session_data_p->response_data.original_compressed_body); + write_dbg(DBG_LEVEL_WARNING, "Failed to decompress response body"); handle_inspection_failure(inspection_failure_weight, fail_mode_verdict, session_data_p); fini_cp_session_data(session_data_p); session_data_p->response_data.response_data_status = NGX_ERROR; @@ -1056,6 +1053,7 @@ ngx_http_cp_res_body_filter(ngx_http_request_t *request, ngx_chain_t *body_chain } if (session_data_p->verdict == TRAFFIC_VERDICT_ACCEPT) { + write_dbg(DBG_LEVEL_TRACE, "Compressing response body"); if (session_data_p->response_data.compression_stream == NULL) { session_data_p->response_data.compression_stream = initCompressionStream(); } @@ -1070,6 +1068,7 @@ ngx_http_cp_res_body_filter(ngx_http_request_t *request, ngx_chain_t *body_chain request->pool ); if (compression_result != NGX_OK) { + write_dbg(DBG_LEVEL_WARNING, "Failed to compress response body"); // Failed to compress body. handle_inspection_failure(inspection_failure_weight, fail_mode_verdict, session_data_p); fini_cp_session_data(session_data_p); @@ -1167,7 +1166,7 @@ ngx_http_cp_res_body_filter(ngx_http_request_t *request, ngx_chain_t *body_chain calcProcessingTime(session_data_p, &hook_time_begin, 0); if (ctx.should_return) { - return ctx.res; + return ctx.res == NGX_OK ? NGX_DECLINED : ctx.res; } if (ctx.should_return_next_filter) { diff --git a/attachments/nginx/ngx_module/ngx_cp_io.c b/attachments/nginx/ngx_module/ngx_cp_io.c index 3f049db..cc0bc3b 100644 --- a/attachments/nginx/ngx_module/ngx_cp_io.c +++ b/attachments/nginx/ngx_module/ngx_cp_io.c @@ -1116,8 +1116,9 @@ ngx_http_cp_body_sender( write_dbg( DBG_LEVEL_TRACE, - "Sending %s body chunk for inspection", - body_type == REQUEST_BODY ? "request" : "response" + "Sending %s body chunk from session id %d for inspection", + body_type == REQUEST_BODY ? "request" : "response", + session_data->session_id ); // Sets fragments identifier to the provided body type. diff --git a/attachments/nginx/ngx_module/ngx_cp_utils.c b/attachments/nginx/ngx_module/ngx_cp_utils.c index 8bbe369..ff5a151 100644 --- a/attachments/nginx/ngx_module/ngx_cp_utils.c +++ b/attachments/nginx/ngx_module/ngx_cp_utils.c @@ -102,6 +102,8 @@ ngx_uint_t num_of_nginx_ipc_elements = 200; ///< Number of NGINX IPC elements. ngx_msec_t keep_alive_interval_msec = DEFAULT_KEEP_ALIVE_INTERVAL_MSEC; ngx_uint_t min_retries_for_verdict = 3; ///< Minimum number of retries for verdict. ngx_uint_t max_retries_for_verdict = 15; ///< Maximum number of retries for verdict. +ngx_uint_t hold_verdict_retries = 3; ///< Number of retries for hold verdict. +ngx_uint_t hold_verdict_polling_time = 1; ///< Polling time for hold verdict. ngx_uint_t body_size_trigger = 200000; ///< Request body size in bytes to switch to maximum retries for verdict. ngx_uint_t remove_res_server_header = 0; ///< Remove server header flag. @@ -322,6 +324,15 @@ copy_chain_buffers(ngx_chain_t *dest, ngx_chain_t *src) ngx_chain_t *curr_src = src; ngx_chain_t *curr_dst = dest; while (curr_src != NULL && curr_dst != NULL) { + if (curr_src->buf == NULL || curr_dst->buf == NULL) { + write_dbg( + DBG_LEVEL_WARNING, + "Failed to copy chain buffers: NULL buffer found. src: %p, dst: %p", + curr_src, + curr_dst + ); + return; + } ngx_memcpy(curr_dst->buf, curr_src->buf, sizeof(ngx_buf_t)); curr_src = curr_src->next; curr_dst = curr_dst->next; @@ -459,10 +470,12 @@ free_chain(ngx_pool_t *pool, ngx_chain_t *chain) while (chain) { ngx_pfree(pool, chain->buf->start); + chain->buf->start = NULL; ngx_pfree(pool, chain->buf); + chain->buf = NULL; next_chain = chain->next; - ngx_pfree(pool, chain); + ngx_free_chain(pool, chain); chain = next_chain; } } @@ -940,6 +953,10 @@ init_general_config(const char *conf_path) fail_mode_hold_verdict = isFailOpenHoldMode() == 1 ? NGX_OK : NGX_HTTP_FORBIDDEN; fail_open_hold_timeout = getFailOpenHoldTimeout(); + // Setting hold verdict polling time and retries. + hold_verdict_polling_time = getHoldVerdictPollingTime(); + hold_verdict_retries = getHoldVerdictRetries(); + // Setting attachment's variables. sessions_per_minute_limit_verdict = isFailOpenOnSessionLimit() ? TRAFFIC_VERDICT_ACCEPT : TRAFFIC_VERDICT_DROP; max_sessions_per_minute = getMaxSessionsPerMinute(); @@ -987,6 +1004,8 @@ init_general_config(const char *conf_path) "keep alive interval msec: %u msec" "min retries for verdict: %u" "max retries for verdict: %u" + "num retries for hold verdict: %u" + "polling time for hold verdict: %u" "body size trigger for request: %u", inspection_mode, new_dbg_level, @@ -1009,6 +1028,8 @@ init_general_config(const char *conf_path) keep_alive_interval_msec, min_retries_for_verdict, max_retries_for_verdict, + hold_verdict_retries, + hold_verdict_polling_time, body_size_trigger ); diff --git a/attachments/nginx/ngx_module/ngx_cp_utils.h b/attachments/nginx/ngx_module/ngx_cp_utils.h index 5c944ad..74cd1f8 100644 --- a/attachments/nginx/ngx_module/ngx_cp_utils.h +++ b/attachments/nginx/ngx_module/ngx_cp_utils.h @@ -64,6 +64,8 @@ extern ngx_http_inspection_mode_e inspection_mode; extern ngx_uint_t num_of_nginx_ipc_elements; extern ngx_uint_t min_retries_for_verdict; extern ngx_uint_t max_retries_for_verdict; +extern ngx_uint_t hold_verdict_retries; +extern ngx_uint_t hold_verdict_polling_time; extern ngx_uint_t body_size_trigger; extern ngx_uint_t remove_res_server_header; diff --git a/core/attachments/http_configuration/http_configuration.cc b/core/attachments/http_configuration/http_configuration.cc index 9a1ddf0..18639ea 100644 --- a/core/attachments/http_configuration/http_configuration.cc +++ b/core/attachments/http_configuration/http_configuration.cc @@ -111,6 +111,8 @@ HttpAttachmentConfiguration::save(cereal::JSONOutputArchive &archive) const cereal::make_nvp("keep_alive_interval_msec", getNumericalValue("keep_alive_interval_msec")), cereal::make_nvp("min_retries_for_verdict", getNumericalValue("min_retries_for_verdict")), cereal::make_nvp("max_retries_for_verdict", getNumericalValue("max_retries_for_verdict")), + cereal::make_nvp("hold_verdict_retries", getNumericalValue("hold_verdict_retries")), + cereal::make_nvp("hold_verdict_polling_time", getNumericalValue("hold_verdict_polling_time")), cereal::make_nvp("body_size_trigger", getNumericalValue("body_size_trigger")), cereal::make_nvp("remove_server_header", getNumericalValue("remove_server_header")) ); @@ -167,6 +169,8 @@ HttpAttachmentConfiguration::load(cereal::JSONInputArchive &archive) loadNumericalValue(archive, "keep_alive_interval_msec", DEFAULT_KEEP_ALIVE_INTERVAL_MSEC); loadNumericalValue(archive, "min_retries_for_verdict", 3); loadNumericalValue(archive, "max_retries_for_verdict", 15); + loadNumericalValue(archive, "hold_verdict_retries", 3); + loadNumericalValue(archive, "hold_verdict_polling_time", 1); loadNumericalValue(archive, "body_size_trigger", 200000); loadNumericalValue(archive, "remove_server_header", 0); } diff --git a/core/include/attachments/nginx_attachment_util.h b/core/include/attachments/nginx_attachment_util.h index 33885d4..6a2b5e9 100644 --- a/core/include/attachments/nginx_attachment_util.h +++ b/core/include/attachments/nginx_attachment_util.h @@ -42,6 +42,9 @@ unsigned int getFailOpenTimeout(); int isFailOpenHoldMode(); unsigned int getFailOpenHoldTimeout(); +unsigned int getHoldVerdictPollingTime(); +unsigned int getHoldVerdictRetries(); + unsigned int getMaxSessionsPerMinute(); int isFailOpenOnSessionLimit();