mirror of
https://github.com/openappsec/attachment.git
synced 2025-08-13 13:25:59 +03:00
fix large file inspection and free memory
This commit is contained in:
wiaamm
parent
479f605800
commit
b6250149c2
@ -27,9 +27,8 @@ function NanoHandler.access(conf)
|
||||
|
||||
local session_data = nano.init_session(session_id)
|
||||
if not session_data then
|
||||
kong.log.err("Failed to initialize session")
|
||||
kong.ctx.plugin.blocked = true
|
||||
return kong.response.exit(500, { message = "Session Initialization Failed" })
|
||||
kong.log.err("Failed to initialize session - failing open")
|
||||
return
|
||||
end
|
||||
|
||||
kong.ctx.plugin.session_data = session_data
|
||||
@ -41,9 +40,14 @@ function NanoHandler.access(conf)
|
||||
local has_content_length = tonumber(ngx.var.http_content_length) and tonumber(ngx.var.http_content_length) > 0
|
||||
local contains_body = has_content_length and 1 or 0
|
||||
|
||||
if has_content_length then
|
||||
kong.log.debug("Request has content-length: ", ngx.var.http_content_length)
|
||||
end
|
||||
|
||||
local verdict, response = nano.send_data(session_id, session_data, meta_data, req_headers, contains_body, nano.HttpChunkType.HTTP_REQUEST_FILTER)
|
||||
if verdict == nano.AttachmentVerdict.DROP then
|
||||
nano.fini_session(session_data)
|
||||
nano.cleanup_all()
|
||||
kong.ctx.plugin.blocked = true
|
||||
return nano.handle_custom_response(session_data, response)
|
||||
end
|
||||
@ -54,9 +58,62 @@ function NanoHandler.access(conf)
|
||||
verdict, response = nano.send_body(session_id, session_data, body, nano.HttpChunkType.HTTP_REQUEST_BODY)
|
||||
if verdict == nano.AttachmentVerdict.DROP then
|
||||
nano.fini_session(session_data)
|
||||
nano.cleanup_all()
|
||||
kong.ctx.plugin.blocked = true
|
||||
return nano.handle_custom_response(session_data, response)
|
||||
end
|
||||
else
|
||||
-- Body might be buffered to file, try to read it using nginx variables
|
||||
kong.log.debug("Request body not in memory, attempting to read from buffer/file")
|
||||
|
||||
-- Try to read the request body using nginx.var
|
||||
local body_data = ngx.var.request_body
|
||||
if body_data and #body_data > 0 then
|
||||
kong.log.debug("Found request body in nginx var, size: ", #body_data)
|
||||
verdict, response = nano.send_body(session_id, session_data, body_data, nano.HttpChunkType.HTTP_REQUEST_BODY)
|
||||
if verdict == nano.AttachmentVerdict.DROP then
|
||||
nano.fini_session(session_data)
|
||||
kong.ctx.plugin.blocked = true
|
||||
return nano.handle_custom_response(session_data, response)
|
||||
end
|
||||
else
|
||||
-- Try to read from the temporary file if available
|
||||
local body_file = ngx.var.request_body_file
|
||||
if body_file then
|
||||
kong.log.debug("Reading request body from file: ", body_file)
|
||||
local file = io.open(body_file, "rb")
|
||||
if file then
|
||||
local chunk_size = 8192 -- 8KB chunks
|
||||
local chunks_processed = 0
|
||||
local total_bytes = 0
|
||||
|
||||
while true do
|
||||
local chunk = file:read(chunk_size)
|
||||
if not chunk then break end
|
||||
|
||||
chunks_processed = chunks_processed + 1
|
||||
total_bytes = total_bytes + #chunk
|
||||
kong.log.debug("Processing body chunk ", chunks_processed, " of size ", #chunk)
|
||||
|
||||
verdict, response = nano.send_body(session_id, session_data, chunk, nano.HttpChunkType.HTTP_REQUEST_BODY)
|
||||
if verdict == nano.AttachmentVerdict.DROP then
|
||||
file:close()
|
||||
nano.fini_session(session_data)
|
||||
nano.cleanup_all()
|
||||
kong.ctx.plugin.blocked = true
|
||||
return nano.handle_custom_response(session_data, response)
|
||||
end
|
||||
end
|
||||
|
||||
file:close()
|
||||
kong.log.debug("Processed ", chunks_processed, " chunks, total bytes: ", total_bytes)
|
||||
else
|
||||
kong.log.err("Failed to open request body file: ", body_file)
|
||||
end
|
||||
else
|
||||
kong.log.warn("Request body expected but no body data or file available")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
local ok, verdict, response = pcall(function()
|
||||
@ -64,15 +121,15 @@ function NanoHandler.access(conf)
|
||||
end)
|
||||
|
||||
if not ok then
|
||||
kong.log.err("Error ending request inspection: ", verdict)
|
||||
kong.log.err("Error ending request inspection: ", verdict, " - failing open")
|
||||
nano.fini_session(session_data)
|
||||
kong.ctx.plugin.blocked = true
|
||||
return kong.response.exit(500, { message = "Error completing request processing" })
|
||||
nano.cleanup_all()
|
||||
return
|
||||
end
|
||||
|
||||
|
||||
if verdict == nano.AttachmentVerdict.DROP then
|
||||
nano.fini_session(session_data)
|
||||
nano.cleanup_all()
|
||||
kong.ctx.plugin.blocked = true
|
||||
return nano.handle_custom_response(session_data, response)
|
||||
end
|
||||
@ -80,6 +137,7 @@ function NanoHandler.access(conf)
|
||||
verdict, response = nano.end_inspection(session_id, session_data, nano.HttpChunkType.HTTP_REQUEST_END)
|
||||
if verdict == nano.AttachmentVerdict.DROP then
|
||||
nano.fini_session(session_data)
|
||||
nano.cleanup_all()
|
||||
kong.ctx.plugin.blocked = true
|
||||
return nano.handle_custom_response(session_data, response)
|
||||
end
|
||||
@ -111,6 +169,7 @@ function NanoHandler.header_filter(conf)
|
||||
if verdict == nano.AttachmentVerdict.DROP then
|
||||
kong.ctx.plugin.blocked = true
|
||||
nano.fini_session(session_data)
|
||||
nano.cleanup_all()
|
||||
return nano.handle_custom_response(session_data, response)
|
||||
end
|
||||
|
||||
@ -149,6 +208,8 @@ function NanoHandler.body_filter(conf)
|
||||
|
||||
if verdict == nano.AttachmentVerdict.DROP then
|
||||
nano.fini_session(session_data)
|
||||
-- Clean up allocated memory
|
||||
nano.cleanup_all()
|
||||
ctx.session_finalized = true
|
||||
return nano.handle_custom_response(session_data, response)
|
||||
end
|
||||
@ -159,11 +220,15 @@ function NanoHandler.body_filter(conf)
|
||||
local verdict, response = nano.end_inspection(session_id, session_data, nano.HttpChunkType.HTTP_RESPONSE_END)
|
||||
if verdict == nano.AttachmentVerdict.DROP then
|
||||
nano.fini_session(session_data)
|
||||
-- Clean up allocated memory
|
||||
nano.cleanup_all()
|
||||
ctx.session_finalized = true
|
||||
return nano.handle_custom_response(session_data, response)
|
||||
end
|
||||
|
||||
nano.fini_session(session_data)
|
||||
-- Clean up allocated memory
|
||||
nano.cleanup_all()
|
||||
ctx.session_finalized = true
|
||||
end
|
||||
end
|
||||
|
||||
@ -98,6 +98,24 @@ static int lua_get_redirect_page(lua_State *L) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Free HttpMetaData memory
|
||||
static int lua_free_http_metadata(lua_State *L) {
|
||||
HttpMetaData *metadata = (HttpMetaData *)lua_touserdata(L, 1);
|
||||
if (!metadata) return 0;
|
||||
|
||||
if (metadata->http_protocol.data) free(metadata->http_protocol.data);
|
||||
if (metadata->method_name.data) free(metadata->method_name.data);
|
||||
if (metadata->host.data) free(metadata->host.data);
|
||||
if (metadata->listening_ip.data) free(metadata->listening_ip.data);
|
||||
if (metadata->uri.data) free(metadata->uri.data);
|
||||
if (metadata->client_ip.data) free(metadata->client_ip.data);
|
||||
if (metadata->parsed_host.data) free(metadata->parsed_host.data);
|
||||
if (metadata->parsed_uri.data) free(metadata->parsed_uri.data);
|
||||
free(metadata);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
// Allocate memory for nano_str_t (long-lived, must be freed later)
|
||||
static int lua_createNanoStrAlloc(lua_State *L) {
|
||||
@ -531,6 +549,17 @@ static int lua_send_response_headers(lua_State *L) {
|
||||
return 2;
|
||||
}
|
||||
|
||||
// Free verdict response memory
|
||||
static int lua_free_verdict_response(lua_State *L) {
|
||||
AttachmentVerdictResponse *response = (AttachmentVerdictResponse *)lua_touserdata(L, 1);
|
||||
if (!response) return 0;
|
||||
|
||||
// Free the AttachmentVerdictResponse structure
|
||||
free(response);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Register functions in Lua
|
||||
static const struct luaL_Reg nano_attachment_lib[] = {
|
||||
{"init_nano_attachment", lua_init_nano_attachment},
|
||||
@ -550,6 +579,8 @@ static const struct luaL_Reg nano_attachment_lib[] = {
|
||||
{"freeHttpHeaders", lua_freeHttpHeaders},
|
||||
{"setHeaderCount", lua_setHeaderCount},
|
||||
{"create_http_metadata", lua_create_http_metadata},
|
||||
{"free_http_metadata", lua_free_http_metadata},
|
||||
{"free_verdict_response", lua_free_verdict_response},
|
||||
{"send_body", lua_send_body},
|
||||
{"end_inspection", lua_end_inspection},
|
||||
{NULL, NULL}
|
||||
|
||||
@ -8,6 +8,8 @@ nano.attachments = {} -- Store attachments per worker
|
||||
nano.num_workers = ngx.worker.count() or 1 -- Detect number of workers
|
||||
nano.allocated_strings = {}
|
||||
nano.allocate_headers = {}
|
||||
nano.allocated_metadata = {} -- Track metadata allocations
|
||||
nano.allocated_responses = {} -- Track response allocations
|
||||
nano.AttachmentVerdict = {
|
||||
INSPECT = 0,
|
||||
ACCEPT = 1,
|
||||
@ -83,7 +85,12 @@ end
|
||||
|
||||
function nano.handle_custom_response(session_data, response)
|
||||
local worker_id = ngx.worker.id()
|
||||
local attachment = nano.attachments[worker_id]
|
||||
local attachment = nano.ensure_attachment()
|
||||
|
||||
if not attachment then
|
||||
kong.log.err("Cannot handle custom response: Attachment not available for worker ", worker_id, " - failing open")
|
||||
return kong.response.exit(200, "Request allowed due to attachment unavailability")
|
||||
end
|
||||
|
||||
local response_type = nano_attachment.get_web_response_type(attachment, session_data, response)
|
||||
|
||||
@ -136,46 +143,82 @@ end
|
||||
|
||||
function nano.free_http_headers(header_data)
|
||||
for _, nano_header in ipairs(nano.allocate_headers) do
|
||||
nano_attachment.freeNanoStr(nano_header) -- Free memory in C
|
||||
nano_attachment.freeHttpHeaders(nano_header) -- Free memory in C
|
||||
end
|
||||
|
||||
nano.allocate_headers = {} -- Reset the list
|
||||
end
|
||||
|
||||
-- Free all allocated metadata
|
||||
function nano.free_all_metadata()
|
||||
for _, metadata in ipairs(nano.allocated_metadata) do
|
||||
nano_attachment.free_http_metadata(metadata)
|
||||
end
|
||||
nano.allocated_metadata = {}
|
||||
end
|
||||
|
||||
-- Free all allocated responses
|
||||
function nano.free_all_responses()
|
||||
for _, response in ipairs(nano.allocated_responses) do
|
||||
nano_attachment.free_verdict_response(response)
|
||||
end
|
||||
nano.allocated_responses = {}
|
||||
end
|
||||
|
||||
-- Free all allocations (call this on cleanup)
|
||||
function nano.cleanup_all()
|
||||
nano.free_all_nano_str()
|
||||
nano.free_all_metadata()
|
||||
nano.free_all_responses()
|
||||
nano.free_http_headers()
|
||||
end
|
||||
|
||||
-- Initialize worker attachment
|
||||
function nano.init_attachment()
|
||||
local worker_id = ngx.worker.id()
|
||||
local attachment, err
|
||||
local retries = 3
|
||||
local retries = 5 -- Increased retries
|
||||
local base_delay = 1 -- Start with 1 second delay
|
||||
|
||||
for attempt = 1, retries do
|
||||
attachment, err = nano_attachment.init_nano_attachment(worker_id, nano.num_workers)
|
||||
if attachment then break end
|
||||
kong.log.err("Worker ", worker_id, " failed to initialize attachment (attempt ", attempt, "): ", err)
|
||||
ngx.sleep(2)
|
||||
if attachment then
|
||||
break
|
||||
end
|
||||
|
||||
kong.log.err("Worker ", worker_id, " failed to initialize attachment (attempt ", attempt, "/", retries, "): ", err)
|
||||
|
||||
if attempt < retries then
|
||||
local delay = base_delay * (2 ^ (attempt - 1)) + math.random(0, 1000) / 1000
|
||||
kong.log.info("Worker ", worker_id, " retrying attachment initialization in ", delay, " seconds...")
|
||||
ngx.sleep(delay)
|
||||
end
|
||||
end
|
||||
|
||||
if not attachment then
|
||||
kong.log.err("Worker ", worker_id, " failed to initialize attachment after ", retries, " attempts.")
|
||||
kong.log.crit("Worker ", worker_id, " CRITICAL: Failed to initialize attachment after ", retries, " attempts. Worker will operate in fail-open mode.")
|
||||
-- Don't store nil attachment - let other functions handle the missing attachment gracefully
|
||||
return false
|
||||
else
|
||||
nano.attachments[worker_id] = attachment
|
||||
kong.log.info("Worker ", worker_id, " successfully initialized nano_attachment.")
|
||||
return true
|
||||
end
|
||||
end
|
||||
|
||||
-- Initialize a session for a given request
|
||||
function nano.init_session(session_id)
|
||||
local worker_id = ngx.worker.id()
|
||||
local attachment = nano.attachments[worker_id]
|
||||
local attachment = nano.ensure_attachment()
|
||||
|
||||
if not attachment then
|
||||
kong.log.err("Cannot initialize session: Attachment not found for worker ", worker_id)
|
||||
kong.log.warn("Cannot initialize session: Attachment not available for worker ", worker_id, " - failing open")
|
||||
return nil
|
||||
end
|
||||
|
||||
local session_data, err = nano_attachment.init_session(attachment, session_id)
|
||||
if not session_data then
|
||||
kong.log.err("Failed to initialize session for session_id ", session_id, ": ", err)
|
||||
kong.log.err("Failed to initialize session for session_id ", session_id, ": ", err, " - failing open")
|
||||
return nil
|
||||
end
|
||||
|
||||
@ -208,8 +251,18 @@ function nano.handle_start_transaction()
|
||||
local client_ip = kong.client.get_ip()
|
||||
local client_port = kong.client.get_port()
|
||||
|
||||
local listening_address = kong.request.get_host()
|
||||
local listening_ip, listening_port = listening_address:match("([^:]+):?(%d*)")
|
||||
local listening_ip = ngx.var.server_addr or "127.0.0.1"
|
||||
local listening_port = ngx.var.server_port or 80
|
||||
|
||||
kong.log.err("The host is: ", host)
|
||||
kong.log.err("The scheme is: ", scheme)
|
||||
kong.log.err("The method is: ", method)
|
||||
kong.log.err("The uri is: ", uri)
|
||||
kong.log.err("The client IP is: ", client_ip)
|
||||
kong.log.err("The client port is: ", client_port)
|
||||
kong.log.err("The listening IP is: ", listening_ip)
|
||||
kong.log.err("The listening port is: ", listening_port)
|
||||
|
||||
|
||||
-- Call the C function with extracted metadata
|
||||
local metadata = nano_attachment.create_http_metadata(
|
||||
@ -217,6 +270,9 @@ function nano.handle_start_transaction()
|
||||
uri, client_ip, tonumber(client_port) or 0, "", ""
|
||||
)
|
||||
|
||||
-- Track metadata for cleanup
|
||||
table.insert(nano.allocated_metadata, metadata)
|
||||
|
||||
collectgarbage("stop")
|
||||
|
||||
return metadata
|
||||
@ -260,10 +316,10 @@ end
|
||||
-- Send data to NanoAttachment
|
||||
function nano.send_data(session_id, session_data, meta_data, header_data, contains_body, chunk_type)
|
||||
local worker_id = ngx.worker.id()
|
||||
local attachment = nano.attachments[worker_id]
|
||||
local attachment = nano.ensure_attachment()
|
||||
|
||||
if not attachment then
|
||||
kong.log.err("Attachment not initialized for worker ", worker_id, ". Dropping request.")
|
||||
kong.log.warn("Attachment not available for worker ", worker_id, " - failing open")
|
||||
return nano.AttachmentVerdict.INSPECT
|
||||
end
|
||||
|
||||
@ -271,20 +327,32 @@ function nano.send_data(session_id, session_data, meta_data, header_data, contai
|
||||
contains_body = (contains_body > 0) and 1 or 0 -- Force strict 0 or 1
|
||||
|
||||
local verdict, response = nano_attachment.send_data(attachment, session_id, session_data, chunk_type, meta_data, header_data, contains_body)
|
||||
|
||||
-- Track response for cleanup
|
||||
if response then
|
||||
table.insert(nano.allocated_responses, response)
|
||||
end
|
||||
|
||||
return verdict, response
|
||||
end
|
||||
|
||||
function nano.send_body(session_id, session_data, body_chunk, chunk_type)
|
||||
local worker_id = ngx.worker.id()
|
||||
local attachment = nano.attachments[worker_id]
|
||||
local attachment = nano.ensure_attachment()
|
||||
|
||||
if not attachment then
|
||||
kong.log.err("Attachment not initialized for worker ", worker_id, ". Dropping request.")
|
||||
kong.log.warn("Attachment not available for worker ", worker_id, " - failing open")
|
||||
return nano.AttachmentVerdict.INSPECT
|
||||
end
|
||||
|
||||
-- Send the body chunk directly as a string
|
||||
local verdict, response, modifications = nano_attachment.send_body(attachment, session_id, session_data, body_chunk, chunk_type)
|
||||
|
||||
-- Track response for cleanup
|
||||
if response then
|
||||
table.insert(nano.allocated_responses, response)
|
||||
end
|
||||
|
||||
return verdict, response, modifications
|
||||
end
|
||||
|
||||
@ -327,26 +395,13 @@ function nano.send_response_body(session_id, session_data, body_chunk)
|
||||
return verdict, response, modifications
|
||||
end
|
||||
|
||||
function nano.end_inspection(session_id, session_data, chunk_type)
|
||||
local worker_id = ngx.worker.id()
|
||||
local attachment = nano.attachments[worker_id]
|
||||
|
||||
if not attachment then
|
||||
kong.log.err("Attachment not initialized for worker ", worker_id, ". Dropping request.")
|
||||
return nano.AttachmentVerdict.INSPECT
|
||||
end
|
||||
|
||||
local verdict, response = nano_attachment.end_inspection(attachment, session_id, session_data, chunk_type)
|
||||
return verdict, response
|
||||
end
|
||||
|
||||
-- Finalize session cleanup
|
||||
function nano.fini_session(session_data)
|
||||
local worker_id = ngx.worker.id()
|
||||
local attachment = nano.attachments[worker_id]
|
||||
local attachment = nano.ensure_attachment()
|
||||
|
||||
if not attachment or not session_data then
|
||||
kong.log.err("Cannot finalize session: Invalid attachment or session_data")
|
||||
kong.log.warn("Cannot finalize session: Invalid attachment or session_data for worker ", worker_id)
|
||||
return false
|
||||
end
|
||||
|
||||
@ -358,10 +413,10 @@ end
|
||||
-- Send response headers for inspection
|
||||
function nano.send_response_headers(session_id, session_data, headers, status_code, content_length)
|
||||
local worker_id = ngx.worker.id()
|
||||
local attachment = nano.attachments[worker_id]
|
||||
local attachment = nano.ensure_attachment()
|
||||
|
||||
if not attachment then
|
||||
kong.log.err("Attachment not initialized for worker ", worker_id, ". Dropping request.")
|
||||
kong.log.warn("Attachment not available for worker ", worker_id, " - failing open")
|
||||
return nano.AttachmentVerdict.INSPECT
|
||||
end
|
||||
|
||||
@ -373,6 +428,12 @@ function nano.send_response_headers(session_id, session_data, headers, status_co
|
||||
status_code,
|
||||
content_length
|
||||
)
|
||||
|
||||
-- Track response for cleanup
|
||||
if response then
|
||||
table.insert(nano.allocated_responses, response)
|
||||
end
|
||||
|
||||
return verdict, response
|
||||
end
|
||||
|
||||
@ -423,4 +484,48 @@ function nano.handle_header_modifications(headers, modifications)
|
||||
return modified_headers
|
||||
end
|
||||
|
||||
-- End inspection for a session
|
||||
function nano.end_inspection(session_id, session_data, chunk_type)
|
||||
local worker_id = ngx.worker.id()
|
||||
local attachment = nano.ensure_attachment()
|
||||
|
||||
if not attachment then
|
||||
kong.log.warn("Attachment not available for worker ", worker_id, " - failing open during end_inspection")
|
||||
return nano.AttachmentVerdict.INSPECT, nil
|
||||
end
|
||||
|
||||
if not session_data then
|
||||
kong.log.err("Cannot end inspection: Invalid session_data for session ", session_id)
|
||||
return nano.AttachmentVerdict.INSPECT, nil
|
||||
end
|
||||
|
||||
local verdict, response = nano_attachment.end_inspection(attachment, session_id, session_data, chunk_type)
|
||||
|
||||
-- Track response for cleanup if allocated
|
||||
if response then
|
||||
table.insert(nano.allocated_responses, response)
|
||||
end
|
||||
|
||||
return verdict, response
|
||||
end
|
||||
|
||||
-- Helper function to ensure attachment is available
|
||||
function nano.ensure_attachment()
|
||||
local worker_id = ngx.worker.id()
|
||||
local attachment = nano.attachments[worker_id]
|
||||
|
||||
if not attachment then
|
||||
kong.log.warn("Attachment not found for worker ", worker_id, ", attempting to reinitialize...")
|
||||
if nano.init_attachment() then
|
||||
attachment = nano.attachments[worker_id]
|
||||
kong.log.info("Successfully reinitialized attachment for worker ", worker_id)
|
||||
else
|
||||
kong.log.err("Failed to reinitialize attachment for worker ", worker_id)
|
||||
return nil
|
||||
end
|
||||
end
|
||||
|
||||
return attachment
|
||||
end
|
||||
|
||||
return nano
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user