github_mirrors/attachment
3
0
Fork 1
mirror of https://github.com/openappsec/attachment.git synced 2025-12-31 05:39:07 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix kong response body (#49)

Browse Source 
* fix large response body

* fix large response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix response body

* fix specific path

* return correct path

* fresh start

* fix send_bodu in lua_attachment_wrapper.c

* change branch

* add timeout

* add timeout

* remove the header filter

* try chunk hashing

* proper fini session

* add more check

* try setting last chunk

* remove finilizing session

* return to basic version

* proper fini of session data

* add some check for internal traffic

* envoy functionality...

* proper fini session

* proper fini session

* fini session on accept also

* for testing change the log level

* remove fini session from header

* remove fini session from header

* remove fini session from header

* remove inpect check

* fix wrong close statement

* correct inspection_complete handling

* fix oom

* fix oom

* fix oom

* fix oom

* fix oom

* fix init attachment

* nano_ffi free memoty

* GC implementation

* optimize

* clean

* increase timeout

* increase timeout

* using is_session_fini, the commit before this one is working ...

* returning to last working version ...

* just for testing

* fix block page

* fix block page

* fix block page

* fix block page

* fix block page

* fix fini session

* add collect(stop)

* return to default

* fix oom caused by using freed values

* fix oom caused by using freed values

* add large response body handling

* implement timeout

* implement timeout

* remove GC

* testing

* add end_inspection

* remove comments

* ngx.arg[1] = nil

* Skip inspection for health checks and internal requests

* add new flag for bypass inspections

* move ngx.arg[1] to the beginning

* try without coolectgarbage

* remove internal traffic check

* remove internal traffic check

* I don't know

* I don't know

* try add chunk assignment

* check why data session is nil in header but availablein body

* remove end inspection

* remove logs

* ctx.timeout_passthrough

* remove ctx.timeout_passthrough

* add GC

* remove the logs

* return the logs

* last modification....

* add logs

* revert to working version

* remove stupid cat

* fini_session only in log phase

* last try

* remove processed_requests

* remove unused variable

* remove nano_ffi changes

* add debuging message for testing

* get worker workers inside the init_attachmetn

* test now

* move check

* add accept

* add accept

* remove endinspection

* fix typo

* get req body in chunks

* test body chunks:

* add timeout in req body

* fixing

* add logs for test

* fix accept verdic in body_filter

* add more logs

* fix ngx time

* add more logs

* add getter functions for timeout

* add more logs

* ready for review

* use right content-length

* add missing content-length update

* try content-length nil

* add nil to header filter

* revert shcema changes

* try exit in header filter

* try content-length nil

* remove the test

* refuse connection when prevnet occures in response body

---------

Co-authored-by: wiaamm <wiaamm@checkpoint.com>
This commit is contained in:
wiaam-mhameed
2025-12-15 13:51:44 +02:00
committed by GitHub
parent 1ed86d4cee
commit f778b851c1
6 changed files with 292 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

252
attachments/kong/plugins/open-appsec-waf-kong-plugin/handler.lua
View File

@@ -8,102 +8,113 @@ local NanoHandler = {}
NanoHandler.PRIORITY = 3000
NanoHandler.VERSION = "1.0.0"
NanoHandler.sessions = {}
NanoHandler.processed_requests = {}
function NanoHandler.init_worker()
nano.init_attachment()
end
-- **Handles Request Headers (DecodeHeaders Equivalent)**
function NanoHandler.access(conf)
local ctx = kong.ctx.plugin
local headers = kong.request.get_headers()
local session_id = nano.generate_session_id()
kong.service.request.set_header("x-session-id", tostring(session_id))
if NanoHandler.processed_requests[session_id] then
kong.ctx.plugin.blocked = true
return
end
local session_data = nano.init_session(session_id)
if not session_data then
kong.log.err("Failed to initialize session - failing open")
kong.ctx.plugin.cleanup_needed = false
return
end
kong.ctx.plugin.session_data = session_data
kong.ctx.plugin.session_id = session_id
ctx.session_data = session_data
ctx.session_id = session_id
if nano.is_session_finalized(session_id) then
kong.log.debug("Session has already been inspected, no need for further inspection")
return
end
local meta_data = nano.handle_start_transaction()
if not meta_data then
kong.log.err("Failed to handle start transaction - failing open")
kong.log.debug("Failed to handle start transaction - failing mode")
ctx.cleanup_needed = true
return
end
local req_headers = nano.handleHeaders(headers)
if not req_headers then
kong.log.debug("Failed to handle request headers - failing mode")
ctx.cleanup_needed = true
return
end
local has_content_length = tonumber(ngx.var.http_content_length) and tonumber(ngx.var.http_content_length) > 0
local contains_body = has_content_length and 1 or 0
local verdict, response = nano.send_data(session_id, session_data, meta_data, req_headers, contains_body, nano.HttpChunkType.HTTP_REQUEST_FILTER)
if verdict ~= nano.AttachmentVerdict.INSPECT then
ctx.cleanup_needed = true
if verdict == nano.AttachmentVerdict.DROP then
nano.fini_session(session_data)
kong.ctx.plugin.blocked = true
local result = nano.handle_custom_response(session_data, response)
nano.cleanup_all()
return result
return nano.handle_custom_response(session_data, response)
end
return
end
if contains_body == 1 then
local body = kong.request.get_raw_body()
if body and #body > 0 then
verdict, response = nano.send_body(session_id, session_data, body, nano.HttpChunkType.HTTP_REQUEST_BODY)
if verdict ~= nano.AttachmentVerdict.INSPECT then
ctx.cleanup_needed = true
if verdict == nano.AttachmentVerdict.DROP then
nano.fini_session(session_data)
kong.ctx.plugin.blocked = true
local result = nano.handle_custom_response(session_data, response)
nano.cleanup_all()
return result
end
else
kong.log.debug("Request body not in memory, attempting to read from buffer/file")
local body_data = ngx.var.request_body
if body_data and #body_data > 0 then
kong.log.debug("Found request body in nginx var, size: ", #body_data)
verdict, response = nano.send_body(session_id, session_data, body_data, nano.HttpChunkType.HTTP_REQUEST_BODY)
if verdict == nano.AttachmentVerdict.DROP then
nano.fini_session(session_data)
kong.ctx.plugin.blocked = true
return nano.handle_custom_response(session_data, response)
end
return
end
else
local body_file = ngx.var.request_body_file
if body_file then
kong.log.debug("Reading request body from file: ", body_file)
local file = io.open(body_file, "rb")
if file then
local entire_body = file:read("*all")
file:close()
local chunk_size = 8192
local chunk_count = 0
local start_time = ngx.now()
local timeout_sec = nano.get_request_processing_timeout_sec()
kong.log.debug("Request body reading timeout set to ", timeout_sec, " seconds")
if entire_body and #entire_body > 0 then
kong.log.debug("Sending entire body of size ", #entire_body, " bytes to C module")
verdict, response = nano.send_body(session_id, session_data, entire_body, nano.HttpChunkType.HTTP_REQUEST_BODY)
while true do
ngx.update_time()
local current_time = ngx.now()
local elapsed = current_time - start_time
if elapsed > timeout_sec then
ctx.cleanup_needed = true
kong.log.warn("Request body reading timeout after ", elapsed, " seconds")
file:close()
return
end
local chunk = file:read(chunk_size)
if not chunk or #chunk == 0 then
kong.log.debug("End of request body file reached")
break
end
chunk_count = chunk_count + 1
kong.log.debug("Sending request body chunk ", chunk_count, " of size ", #chunk, " bytes to C module")
verdict, response = nano.send_body(session_id, session_data, chunk, nano.HttpChunkType.HTTP_REQUEST_BODY)
if verdict ~= nano.AttachmentVerdict.INSPECT then
file:close()
ctx.cleanup_needed = true
if verdict == nano.AttachmentVerdict.DROP then
nano.fini_session(session_data)
kong.ctx.plugin.blocked = true
local result = nano.handle_custom_response(session_data, response)
nano.cleanup_all()
return result
return nano.handle_custom_response(session_data, response)
end
return
end
end
file:close()
kong.log.debug("Sent ", chunk_count, " chunks from request body file")
end
else
kong.log.debug("Empty body file")
end
end
else
kong.log.warn("Request body expected but no body data or file available")
end
kong.log.err("Request body expected but no body data or file available")
end
end
@@ -112,119 +123,150 @@ function NanoHandler.access(conf)
end)
if not ok then
kong.log.err("Error ending request inspection: ", verdict, " - failing open")
nano.fini_session(session_data)
nano.cleanup_all()
kong.log.debug("Error ending request inspection: ", verdict, " - failing open")
ctx.cleanup_needed = true
return
end
if verdict ~= nano.AttachmentVerdict.INSPECT then
ctx.cleanup_needed = true
if verdict == nano.AttachmentVerdict.DROP then
nano.fini_session(session_data)
kong.ctx.plugin.blocked = true
local result = nano.handle_custom_response(session_data, response)
nano.cleanup_all()
return result
return nano.handle_custom_response(session_data, response)
end
else
verdict, response = nano.end_inspection(session_id, session_data, nano.HttpChunkType.HTTP_REQUEST_END)
if verdict == nano.AttachmentVerdict.DROP then
nano.fini_session(session_data)
kong.ctx.plugin.blocked = true
local result = nano.handle_custom_response(session_data, response)
nano.cleanup_all()
return result
return
end
end
NanoHandler.processed_requests[session_id] = true
end
function NanoHandler.header_filter(conf)
local ctx = kong.ctx.plugin
if ctx.blocked then
if nano.is_session_finalized(ctx.session_data) then
kong.log.debug("Session has already been inspected, no need for further inspection")
return
end
if ctx.cleanup_needed then
kong.log.debug("cleanup in header_filter, passing through")
return
end
local session_id = ctx.session_id
local session_data = ctx.session_data
if not session_id or not session_data then
kong.log.err("No session data found in header_filter")
local headers = kong.response.get_headers()
local header_data = nano.handleHeaders(headers)
if not header_data then
kong.log.debug("Failed to handle response headers - failing open")
ctx.cleanup_needed = true
return
end
local headers = kong.response.get_headers()
local header_data = nano.handleHeaders(headers)
local status_code = kong.response.get_status()
local content_length = tonumber(headers["content-length"]) or 0
local verdict, response = nano.send_response_headers(session_id, session_data, header_data, status_code, content_length)
if verdict ~= nano.AttachmentVerdict.INSPECT then
ctx.cleanup_needed = true
if verdict == nano.AttachmentVerdict.DROP then
kong.ctx.plugin.blocked = true
nano.fini_session(session_data)
nano.cleanup_all()
kong.log.debug("DROP verdict in header_filter - sending block response immediately")
return nano.handle_custom_response(session_data, response)
end
ngx.header["Content-Length"] = nil
return
end
ngx.header["Content-Length"] = nil
ctx.expect_body = not (status_code == 204 or status_code == 304 or (100 <= status_code and status_code < 200) or content_length == 0)
end
function NanoHandler.body_filter(conf)
local ctx = kong.ctx.plugin
if ctx.blocked then
return
end
local chunk = ngx.arg[1]
local eof = ngx.arg[2]
local session_id = ctx.session_id
local session_data = ctx.session_data
if not session_id or not session_data or ctx.session_finalized then
if nano.is_session_finalized(session_data) then
kong.log.debug("Session has already been inspected, no need for further inspection")
return
end
local body = kong.response.get_raw_body()
if ctx.cleanup_needed then
kong.log.debug("cleanup chunk without inspection, passing through")
return
end
if body then
ctx.body_seen = true
local verdict, response, modifications = nano.send_body(session_id, session_data, body, nano.HttpChunkType.HTTP_RESPONSE_BODY)
if not ctx.body_filter_start_time then
ctx.body_filter_start_time = ngx.now()
ctx.body_filter_timeout_sec = nano.get_response_processing_timeout_sec()
kong.log.debug("body_filter timeout set to ", ctx.body_filter_timeout_sec, " seconds")
end
-- Initialize if not exists
local elapsed_time = ngx.now() - ctx.body_filter_start_time
if elapsed_time > ctx.body_filter_timeout_sec then
kong.log.warn("Body filter timeout after ", elapsed_time, " seconds - failing open")
ctx.cleanup_needed = true
return
end
if chunk and #chunk > 0 then
ctx.body_buffer_chunk = ctx.body_buffer_chunk or 0
ctx.body_seen = true
local verdict, response, modifications = nano.send_body(session_id, session_data, chunk, nano.HttpChunkType.HTTP_RESPONSE_BODY)
-- Handle body modifications if any
if modifications then
body = nano.handle_body_modifications(body, modifications, ctx.body_buffer_chunk)
kong.response.set_raw_body(body)
chunk = nano.handle_body_modifications(chunk, modifications, ctx.body_buffer_chunk)
end
ctx.body_buffer_chunk = ctx.body_buffer_chunk + 1
if verdict ~= nano.AttachmentVerdict.INSPECT then
ctx.cleanup_needed = true
if verdict == nano.AttachmentVerdict.DROP then
nano.fini_session(session_data)
ctx.session_finalized = true
local result = nano.handle_custom_response(session_data, response)
-- Clean up allocated memory
nano.cleanup_all()
return result
kong.log.debug("DROP verdict during response streaming - closing connection")
ngx.header["Connection"] = "close"
ngx.arg[1] = ""
ngx.arg[2] = true
return
end
end
ngx.arg[1] = chunk
return
end
if eof then
if ctx.body_seen or ctx.expect_body == false then
ctx.cleanup_needed = true
local verdict, response = nano.end_inspection(session_id, session_data, nano.HttpChunkType.HTTP_RESPONSE_END)
if verdict ~= nano.AttachmentVerdict.INSPECT then
kong.log.debug("Final verdict after end_inspection: ", verdict)
ctx.cleanup_needed = true
if verdict == nano.AttachmentVerdict.DROP then
nano.fini_session(session_data)
ctx.session_finalized = true
local result = nano.handle_custom_response(session_data, response)
-- Clean up allocated memory
nano.cleanup_all()
return result
kong.log.debug("DROP verdict at EOF - closing connection")
ngx.header["Connection"] = "close"
ngx.arg[1] = ""
ngx.arg[2] = true
return
end
end
end
nano.fini_session(session_data)
-- Clean up allocated memory
end
end
function NanoHandler.log(conf)
local ctx = kong.ctx.plugin
if ctx.cleanup_needed then
nano.fini_session(ctx.session_data)
nano.cleanup_all()
ctx.session_finalized = true
ctx.session_data = nil
ctx.session_id = nil
collectgarbage("collect")
end
end

26
attachments/kong/plugins/open-appsec-waf-kong-plugin/lua_attachment_wrapper.c
View File

@@ -507,6 +507,30 @@ static int lua_free_verdict_response(lua_State *L) {
return 0;
}
static int lua_get_request_processing_timeout_msec(lua_State *L) {
NanoAttachment* attachment = (NanoAttachment*)lua_touserdata(L, 1);
if (!attachment) {
lua_pushinteger(L, 3000);
return 1;
}
uint32_t timeout = GetRequestProcessingTimeout(attachment);
lua_pushinteger(L, timeout);
return 1;
}
static int lua_get_response_processing_timeout_msec(lua_State *L) {
NanoAttachment* attachment = (NanoAttachment*)lua_touserdata(L, 1);
if (!attachment) {
lua_pushinteger(L, 3000);
return 1;
}
uint32_t timeout = GetResponseProcessingTimeout(attachment);
lua_pushinteger(L, timeout);
return 1;
}
static const struct luaL_Reg nano_attachment_lib[] = {
{"init_nano_attachment", lua_init_nano_attachment},
{"get_web_response_type", lua_get_web_response_type},
@@ -529,6 +553,8 @@ static const struct luaL_Reg nano_attachment_lib[] = {
{"free_verdict_response", lua_free_verdict_response},
{"send_body", lua_send_body},
{"end_inspection", lua_end_inspection},
{"get_request_processing_timeout_msec", lua_get_request_processing_timeout_msec},
{"get_response_processing_timeout_msec", lua_get_response_processing_timeout_msec},
{NULL, NULL}
};

60
attachments/kong/plugins/open-appsec-waf-kong-plugin/nano_ffi.lua
View File

@@ -5,7 +5,6 @@ local nano = {}
nano.session_counter = 0
nano.attachments = {}
nano.num_workers = ngx.worker.count() or 1
nano.allocated_strings = {}
nano.allocate_headers = {}
nano.allocated_metadata = {}
@@ -82,13 +81,14 @@ function nano.generate_session_id()
return tonumber(string.format("%d%05d", worker_id, nano.session_counter))
end
function nano.handle_custom_response(session_data, response)
-- Returns: status_code, body, headers
function nano.get_custom_response_data(session_data, response)
local worker_id = ngx.worker.id()
local attachment = nano.attachments[worker_id]
if not attachment then
kong.log.warn("Cannot handle custom response: Attachment not available for worker ", worker_id, " - failing open")
return kong.response.exit(200, "Request allowed due to attachment unavailability")
return 200, "Request allowed due to attachment unavailability", {}
end
local response_type = nano_attachment.get_web_response_type(attachment, session_data, response)
@@ -100,18 +100,19 @@ function nano.handle_custom_response(session_data, response)
code = 403
end
kong.log.debug("Response code only: ", code)
return kong.response.exit(code, "")
return code, "", {}
end
if response_type == nano.WebResponseType.REDIRECT_WEB_RESPONSE then
local location = nano_attachment.get_redirect_page(attachment, session_data, response)
return kong.response.exit(307, "", { ["Location"] = location })
local code = nano_attachment.get_response_code(response) or 307
return code, "", { ["Location"] = location }
end
local block_page = nano_attachment.get_block_page(attachment, session_data, response)
if not block_page then
kong.log.err("Failed to retrieve custom block page for session ", session_data)
return kong.response.exit(500, { message = "Internal Server Error" })
kong.log.debug("Failed to retrieve custom block page for session ", session_data)
return 403, "", {}
end
local code = nano_attachment.get_response_code(response)
if not code or code < 100 or code > 599 then
@@ -119,8 +120,13 @@ function nano.handle_custom_response(session_data, response)
code = 403
end
kong.log.debug("Block page response with code: ", code)
return kong.response.exit(code, block_page, { ["Content-Type"] = "text/html" })
return code, block_page, { ["Content-Type"] = "text/html" }
end
-- Wrapper for backward compatibility - calls kong.response.exit() in access phase
function nano.handle_custom_response(session_data, response)
local code, body, headers = nano.get_custom_response_data(session_data, response)
return kong.response.exit(code, body, headers)
end
@@ -178,15 +184,15 @@ end
function nano.init_attachment()
local worker_id = ngx.worker.id()
local num_workers = ngx.worker.count() or 1 -- Get count dynamically
local attachment, err
local retries = 3
for attempt = 1, retries do
attachment, err = nano_attachment.init_nano_attachment(worker_id, nano.num_workers)
attachment, err = nano_attachment.init_nano_attachment(worker_id, num_workers)
if attachment then
break
end
kong.log.err("Worker ", worker_id, " failed to initialize attachment (attempt ", attempt, "/", retries, "): ", err)
end
@@ -228,8 +234,8 @@ function nano.is_session_finalized(session_data)
local attachment = nano.attachments[worker_id]
if not attachment or not session_data then
kong.log.err("Cannot check session finalization: Invalid attachment or session_data")
return false
kong.log.debug("Cannot check session finalization: Invalid attachment or session_data")
return true
end
return nano_attachment.is_session_finalized(attachment, session_data)
@@ -256,7 +262,6 @@ function nano.handle_start_transaction()
)
table.insert(nano.allocated_metadata, metadata)
collectgarbage("stop")
return metadata
@@ -295,7 +300,6 @@ function nano.handleHeaders(headers)
end
nano_attachment.setHeaderCount(header_data, index)
return header_data
end
@@ -468,7 +472,7 @@ function nano.end_inspection(session_id, session_data, chunk_type)
end
if not session_data then
kong.log.err("Cannot end inspection: Invalid session_data for session ", session_id)
kong.log.debug("Cannot end inspection: Invalid session_data for session ", session_id)
return nano.AttachmentVerdict.INSPECT, nil
end
@@ -481,4 +485,30 @@ function nano.end_inspection(session_id, session_data, chunk_type)
return verdict, response
end
function nano.get_request_processing_timeout_sec()
local worker_id = ngx.worker.id()
local attachment = nano.attachments[worker_id]
if not attachment then
kong.log.warn("Attachment not available for worker ", worker_id, " - using default timeout")
return 3
end
local timeout_msec = nano_attachment.get_request_processing_timeout_msec(attachment)
return timeout_msec / 1000.0
end
function nano.get_response_processing_timeout_sec()
local worker_id = ngx.worker.id()
local attachment = nano.attachments[worker_id]
if not attachment then
kong.log.warn("Attachment not available for worker ", worker_id, " - using default timeout")
return 3
end
local timeout_msec = nano_attachment.get_response_processing_timeout_msec(attachment)
return timeout_msec / 1000.0
end
return nano

2
attachments/kong/plugins/open-appsec-waf-kong-plugin/open-appsec-waf-kong-plugin-1.0.0-1.rockspec
View File

@@ -3,7 +3,7 @@ version = "1.0.0-1"
source = {
url = "git://github.com/openappsec/attachment.git",
tag = "main"
branch = "fix-kong-response-body"
}
description = {

19
attachments/nano_attachment/nano_attachment.c
View File

@@ -622,3 +622,22 @@ freeCompressedBody(NanoAttachment *attachment, HttpSessionData *session_data, Ht
{
nano_free_compressed_body(attachment, bodies, session_data);
}
uint32_t
GetRequestProcessingTimeout(NanoAttachment *attachment)
{
if (attachment == NULL) {
return 3000;
}
return attachment->req_max_proccessing_ms_time;
}
uint32_t
GetResponseProcessingTimeout(NanoAttachment *attachment)
{
if (attachment == NULL) {
return 3000;
}
return attachment->res_max_proccessing_ms_time;
}

24
core/include/attachments/nano_attachment.h
View File

@@ -266,4 +266,28 @@ freeCompressedBody(
HttpBody *bodies
);
///
/// @brief Gets the request processing timeout in milliseconds.
///
/// This function retrieves the configured timeout value for request processing
/// from the NanoAttachment configuration.
///
/// @param attachment A pointer to the NanoAttachment structure.
///
/// @return The request processing timeout in milliseconds.
///
uint32_t GetRequestProcessingTimeout(NanoAttachment *attachment);
///
/// @brief Gets the response processing timeout in milliseconds.
///
/// This function retrieves the configured timeout value for response processing
/// from the NanoAttachment configuration.
///
/// @param attachment A pointer to the NanoAttachment structure.
///
/// @return The response processing timeout in milliseconds.
///
uint32_t GetResponseProcessingTimeout(NanoAttachment *attachment);
#endif // __NANO_ATTACHMENT_H__