Kong fail open (#39)

* fix large file inspection and free memory

* remove duplicated function

* fix cleanup calls

* return the old functionality

* send the all the body

* fix cleanup

* remove debug messages

---------

Co-authored-by: wiaamm <wiaamm@checkpoint.com>

attachments/kong/handler.lua

@@ -27,15 +27,15 @@ function NanoHandler.access(conf)
 
     local session_data = nano.init_session(session_id)
     if not session_data then
-        kong.log.err("Failed to initialize session")
-        kong.ctx.plugin.blocked = true
-        return kong.response.exit(500, { message = "Session Initialization Failed" })
+        kong.log.err("Failed to initialize session - failing open")
+        return
     end
 
     kong.ctx.plugin.session_data = session_data
     kong.ctx.plugin.session_id = session_id
 
     local meta_data = nano.handle_start_transaction()
+    kong.log.err("Failed to handle start transaction - failing open")
     local req_headers = nano.handleHeaders(headers)
 
     local has_content_length = tonumber(ngx.var.http_content_length) and tonumber(ngx.var.http_content_length) > 0
@@ -45,7 +45,9 @@ function NanoHandler.access(conf)
     if verdict == nano.AttachmentVerdict.DROP then
         nano.fini_session(session_data)
         kong.ctx.plugin.blocked = true
-        return nano.handle_custom_response(session_data, response)
+        local result = nano.handle_custom_response(session_data, response)
+        nano.cleanup_all()
+        return result
     end
 
     if contains_body == 1 then
@@ -55,7 +57,52 @@ function NanoHandler.access(conf)
             if verdict == nano.AttachmentVerdict.DROP then
                 nano.fini_session(session_data)
                 kong.ctx.plugin.blocked = true
-                return nano.handle_custom_response(session_data, response)
+                local result = nano.handle_custom_response(session_data, response)
+                nano.cleanup_all()
+                return result
+            end
+        else
+            -- Body might be buffered to file, try to read it using nginx variables
+            kong.log.debug("Request body not in memory, attempting to read from buffer/file")
+
+            -- Try to read the request body using nginx.var
+            local body_data = ngx.var.request_body
+            if body_data and #body_data > 0 then
+                kong.log.debug("Found request body in nginx var, size: ", #body_data)
+                verdict, response = nano.send_body(session_id, session_data, body_data, nano.HttpChunkType.HTTP_REQUEST_BODY)
+                if verdict == nano.AttachmentVerdict.DROP then
+                    nano.fini_session(session_data)
+                    kong.ctx.plugin.blocked = true
+                    return nano.handle_custom_response(session_data, response)
+                end
+            else
+                -- Try to read from the temporary file if available
+                local body_file = ngx.var.request_body_file
+                if body_file then
+                    kong.log.debug("Reading request body from file: ", body_file)
+                    local file = io.open(body_file, "rb")
+                    if file then
+                        -- Read entire body at once
+                        local entire_body = file:read("*all")
+                        file:close()
+
+                        if entire_body and #entire_body > 0 then
+                            kong.log.debug("Sending entire body of size ", #entire_body, " bytes to C module")
+                            verdict, response = nano.send_body(session_id, session_data, entire_body, nano.HttpChunkType.HTTP_REQUEST_BODY)
+                            if verdict == nano.AttachmentVerdict.DROP then
+                                nano.fini_session(session_data)
+                                kong.ctx.plugin.blocked = true
+                                local result = nano.handle_custom_response(session_data, response)
+                                nano.cleanup_all()
+                                return result
+                            end
+                        else
+                            kong.log.debug("Empty body file")
+                        end
+                    end
+                else
+                    kong.log.warn("Request body expected but no body data or file available")
+                end
             end
         end
 
@@ -64,24 +111,27 @@ function NanoHandler.access(conf)
         end)
 
         if not ok then
-            kong.log.err("Error ending request inspection: ", verdict)
+            kong.log.err("Error ending request inspection: ", verdict, " - failing open")
             nano.fini_session(session_data)
-            kong.ctx.plugin.blocked = true
-            return kong.response.exit(500, { message = "Error completing request processing" })
+            nano.cleanup_all()
+            return
         end
 
-
         if verdict == nano.AttachmentVerdict.DROP then
             nano.fini_session(session_data)
             kong.ctx.plugin.blocked = true
-            return nano.handle_custom_response(session_data, response)
+            local result = nano.handle_custom_response(session_data, response)
+            nano.cleanup_all()
+            return result
         end
     else
         verdict, response = nano.end_inspection(session_id, session_data, nano.HttpChunkType.HTTP_REQUEST_END)
         if verdict == nano.AttachmentVerdict.DROP then
             nano.fini_session(session_data)
             kong.ctx.plugin.blocked = true
-            return nano.handle_custom_response(session_data, response)
+            local result = nano.handle_custom_response(session_data, response)
+            nano.cleanup_all()
+            return result
         end
     end
 
@@ -111,6 +161,7 @@ function NanoHandler.header_filter(conf)
     if verdict == nano.AttachmentVerdict.DROP then
         kong.ctx.plugin.blocked = true
         nano.fini_session(session_data)
+        nano.cleanup_all()
         return nano.handle_custom_response(session_data, response)
     end
 
@@ -150,7 +201,10 @@ function NanoHandler.body_filter(conf)
         if verdict == nano.AttachmentVerdict.DROP then
             nano.fini_session(session_data)
             ctx.session_finalized = true
-            return nano.handle_custom_response(session_data, response)
+            local result = nano.handle_custom_response(session_data, response)
+            -- Clean up allocated memory
+            nano.cleanup_all()
+            return result
         end
         return
     end
@@ -160,10 +214,15 @@ function NanoHandler.body_filter(conf)
         if verdict == nano.AttachmentVerdict.DROP then
             nano.fini_session(session_data)
             ctx.session_finalized = true
-            return nano.handle_custom_response(session_data, response)
+            local result = nano.handle_custom_response(session_data, response)
+            -- Clean up allocated memory
+            nano.cleanup_all()
+            return result
         end
 
         nano.fini_session(session_data)
+        -- Clean up allocated memory
+        nano.cleanup_all()
         ctx.session_finalized = true
     end
 end

attachments/kong/lua_attachment_wrapper.c

@@ -98,6 +98,24 @@ static int lua_get_redirect_page(lua_State *L) {
     return 1;
 }
 
+// Free HttpMetaData memory
+static int lua_free_http_metadata(lua_State *L) {
+    HttpMetaData *metadata = (HttpMetaData *)lua_touserdata(L, 1);
+    if (!metadata) return 0;
+
+    if (metadata->http_protocol.data) free(metadata->http_protocol.data);
+    if (metadata->method_name.data) free(metadata->method_name.data);
+    if (metadata->host.data) free(metadata->host.data);
+    if (metadata->listening_ip.data) free(metadata->listening_ip.data);
+    if (metadata->uri.data) free(metadata->uri.data);
+    if (metadata->client_ip.data) free(metadata->client_ip.data);
+    if (metadata->parsed_host.data) free(metadata->parsed_host.data);
+    if (metadata->parsed_uri.data) free(metadata->parsed_uri.data);
+    free(metadata);
+
+    return 0;
+}
+
 
 // Allocate memory for nano_str_t (long-lived, must be freed later)
 static int lua_createNanoStrAlloc(lua_State *L) {
@@ -209,23 +227,6 @@ static void lua_fill_nano_str(lua_State *L, int index, nano_str_t *nano_str) {
     nano_str->len = len;
 }
 
-static int lua_free_http_metadata(lua_State *L) {
-    HttpMetaData *metadata = *(HttpMetaData **)lua_touserdata(L, 1);
-    if (!metadata) return 0;
-
-    free(metadata->http_protocol.data);
-    free(metadata->method_name.data);
-    free(metadata->host.data);
-    free(metadata->listening_ip.data);
-    free(metadata->uri.data);
-    free(metadata->client_ip.data);
-    free(metadata->parsed_host.data);
-    free(metadata->parsed_uri.data);
-    free(metadata);
-
-    return 0;
-}
-
 // Set a header element in HttpHeaderData
 static int lua_setHeaderElement(lua_State *L) {
     HttpHeaders *headers = (HttpHeaders *)lua_touserdata(L, 1);
@@ -403,7 +404,6 @@ static int lua_send_body(lua_State *L) {
         lua_pushinteger(L, res_ptr->verdict);
         lua_pushlightuserdata(L, res_ptr);
 
-        // Push modifications if they exist
         if (res_ptr->modifications) {
             lua_pushlightuserdata(L, res_ptr->modifications);
         } else {
@@ -413,27 +413,22 @@ static int lua_send_body(lua_State *L) {
         return 3;
     }
 
-    // Calculate number of chunks (8KB each, like Envoy)
     const size_t CHUNK_SIZE = 8 * 1024;
     size_t num_chunks = ((body_len - 1) / CHUNK_SIZE) + 1;
 
-    // Limit number of chunks like Envoy
     if (num_chunks > 10000) {
         num_chunks = 10000;
     }
 
-    // Create HttpBody structure
     HttpBody http_chunks;
     http_chunks.bodies_count = num_chunks;
 
-    // Allocate memory for chunks array
     http_chunks.data = (nano_str_t*)malloc(num_chunks * sizeof(nano_str_t));
     if (!http_chunks.data) {
         lua_pushstring(L, "Error: Failed to allocate memory for chunks");
         return lua_error(L);
     }
 
-    // Prepare chunks using pointer arithmetic like Envoy
     for (size_t i = 0; i < num_chunks; i++) {
         nano_str_t* chunk_ptr = (nano_str_t*)((char*)http_chunks.data + (i * sizeof(nano_str_t)));
         size_t chunk_start = i * CHUNK_SIZE;
@@ -443,25 +438,20 @@ static int lua_send_body(lua_State *L) {
         chunk_ptr->len = chunk_len;
     }
 
-    // Prepare attachment data
     AttachmentData attachment_data;
     attachment_data.session_id = session_id;
     attachment_data.session_data = session_data;
     attachment_data.chunk_type = chunk_type;
     attachment_data.data = &http_chunks;
 
-    // Send all chunks at once
     AttachmentVerdictResponse* res_ptr = malloc(sizeof(AttachmentVerdictResponse));
     *res_ptr = SendDataNanoAttachment(attachment, &attachment_data);
 
-    // Free allocated memory
     free(http_chunks.data);
 
-    // Push verdict
     lua_pushinteger(L, res_ptr->verdict);
     lua_pushlightuserdata(L, res_ptr);
 
-    // Push modifications if they exist
     if (res_ptr->modifications) {
         lua_pushlightuserdata(L, res_ptr->modifications);
     } else {
@@ -531,6 +521,17 @@ static int lua_send_response_headers(lua_State *L) {
     return 2;
 }
 
+// Free verdict response memory
+static int lua_free_verdict_response(lua_State *L) {
+    AttachmentVerdictResponse *response = (AttachmentVerdictResponse *)lua_touserdata(L, 1);
+    if (!response) return 0;
+
+    // Free the AttachmentVerdictResponse structure
+    free(response);
+
+    return 0;
+}
+
 // Register functions in Lua
 static const struct luaL_Reg nano_attachment_lib[] = {
     {"init_nano_attachment", lua_init_nano_attachment},
@@ -550,6 +551,8 @@ static const struct luaL_Reg nano_attachment_lib[] = {
     {"freeHttpHeaders", lua_freeHttpHeaders},
     {"setHeaderCount", lua_setHeaderCount},
     {"create_http_metadata", lua_create_http_metadata},
+    {"free_http_metadata", lua_free_http_metadata},
+    {"free_verdict_response", lua_free_verdict_response},
     {"send_body", lua_send_body},
     {"end_inspection", lua_end_inspection},
     {NULL, NULL}

attachments/kong/nano_ffi.lua

@@ -8,6 +8,8 @@ nano.attachments = {} -- Store attachments per worker
 nano.num_workers = ngx.worker.count() or 1 -- Detect number of workers
 nano.allocated_strings = {}
 nano.allocate_headers = {}
+nano.allocated_metadata = {} -- Track metadata allocations
+nano.allocated_responses = {} -- Track response allocations
 nano.AttachmentVerdict = {
     INSPECT = 0,
     ACCEPT = 1,
@@ -85,10 +87,21 @@ function nano.handle_custom_response(session_data, response)
     local worker_id = ngx.worker.id()
     local attachment = nano.attachments[worker_id]
 
+    if not attachment then
+        kong.log.warn("Cannot handle custom response: Attachment not available for worker ", worker_id, " - failing open")
+        return kong.response.exit(200, "Request allowed due to attachment unavailability")
+    end
+
     local response_type = nano_attachment.get_web_response_type(attachment, session_data, response)
 
     if response_type == nano.WebResponseType.RESPONSE_CODE_ONLY then
         local code = nano_attachment.get_response_code(response)
+        -- Validate HTTP status code
+        if not code or code < 100 or code > 599 then
+            kong.log.warn("Invalid response code received: ", code, " - using 403 instead")
+            code = 403
+        end
+        kong.log.debug("Response code only: ", code)
         return kong.response.exit(code, "")
     end
 
@@ -103,6 +116,12 @@ function nano.handle_custom_response(session_data, response)
         return kong.response.exit(500, { message = "Internal Server Error" })
     end
     local code = nano_attachment.get_response_code(response) -- Get the intended status code
+    -- Validate HTTP status code
+    if not code or code < 100 or code > 599 then
+        kong.log.warn("Invalid response code received: ", code, " - using 403 instead")
+        code = 403
+    end
+    kong.log.debug("Block page response with code: ", code)
     return kong.response.exit(code, block_page, { ["Content-Type"] = "text/html" })
 
 end
@@ -136,30 +155,64 @@ end
 
 function nano.free_http_headers(header_data)
     for _, nano_header in ipairs(nano.allocate_headers) do
-        nano_attachment.freeNanoStr(nano_header) -- Free memory in C
+        nano_attachment.freeHttpHeaders(nano_header) -- Free memory in C
     end
 
     nano.allocate_headers = {} -- Reset the list
 end
 
+-- Free all allocated metadata
+function nano.free_all_metadata()
+    for _, metadata in ipairs(nano.allocated_metadata) do
+        nano_attachment.free_http_metadata(metadata)
+    end
+    nano.allocated_metadata = {}
+end
+
+-- Free all allocated responses
+function nano.free_all_responses()
+    for _, response in ipairs(nano.allocated_responses) do
+        nano_attachment.free_verdict_response(response)
+    end
+    nano.allocated_responses = {}
+end
+
+-- Free all allocations (call this on cleanup)
+function nano.cleanup_all()
+    nano.free_all_nano_str()
+    nano.free_all_metadata()
+    nano.free_all_responses()
+    nano.free_http_headers()
+end
+
 -- Initialize worker attachment
 function nano.init_attachment()
     local worker_id = ngx.worker.id()
     local attachment, err
-    local retries = 3
+    local retries = 5  -- Increased retries
+    local base_delay = 1  -- Start with 1 second delay
 
     for attempt = 1, retries do
         attachment, err = nano_attachment.init_nano_attachment(worker_id, nano.num_workers)
-        if attachment then break end
-        kong.log.err("Worker ", worker_id, " failed to initialize attachment (attempt ", attempt, "): ", err)
-        ngx.sleep(2)
+        if attachment then
+            break
+        end
+
+        kong.log.err("Worker ", worker_id, " failed to initialize attachment (attempt ", attempt, "/", retries, "): ", err)
+
+        local delay = base_delay * (2 ^ (attempt - 1)) + math.random(0, 1000) / 1000
+        kong.log.info("Worker ", worker_id, " retrying attachment initialization in ", delay, " seconds...")
+        ngx.sleep(delay)
     end
 
     if not attachment then
-        kong.log.err("Worker ", worker_id, " failed to initialize attachment after ", retries, " attempts.")
+        kong.log.crit("Worker ", worker_id, " CRITICAL: Failed to initialize attachment after ", retries, " attempts. Worker will operate in fail-open mode.")
+        -- Don't store nil attachment - let other functions handle the missing attachment gracefully
+        return false
     else
         nano.attachments[worker_id] = attachment
         kong.log.info("Worker ", worker_id, " successfully initialized nano_attachment.")
+        return true
     end
 end
 
@@ -169,13 +222,13 @@ function nano.init_session(session_id)
     local attachment = nano.attachments[worker_id]
 
     if not attachment then
-        kong.log.err("Cannot initialize session: Attachment not found for worker ", worker_id)
+        kong.log.warn("Cannot initialize session: Attachment not available for worker ", worker_id, " - failing open")
         return nil
     end
 
     local session_data, err = nano_attachment.init_session(attachment, session_id)
     if not session_data then
-        kong.log.err("Failed to initialize session for session_id ", session_id, ": ", err)
+        kong.log.err("Failed to initialize session for session_id ", session_id, ": ", err, " - failing open")
         return nil
     end
 
@@ -208,15 +261,16 @@ function nano.handle_start_transaction()
     local client_ip = kong.client.get_ip()
     local client_port = kong.client.get_port()
 
-    local listening_address = kong.request.get_host()
-    local listening_ip, listening_port = listening_address:match("([^:]+):?(%d*)")
+    local listening_ip = ngx.var.server_addr or "127.0.0.1"
+    local listening_port = ngx.var.server_port or 80
 
-    -- Call the C function with extracted metadata
     local metadata = nano_attachment.create_http_metadata(
         scheme, method, host, listening_ip, tonumber(listening_port) or 0,
         uri, client_ip, tonumber(client_port) or 0, "", ""
     )
 
+    table.insert(nano.allocated_metadata, metadata)
+
     collectgarbage("stop")
 
     return metadata
@@ -263,7 +317,7 @@ function nano.send_data(session_id, session_data, meta_data, header_data, contai
     local attachment = nano.attachments[worker_id]
 
     if not attachment then
-        kong.log.err("Attachment not initialized for worker ", worker_id, ". Dropping request.")
+        kong.log.warn("Attachment not available for worker ", worker_id, " - failing open")
         return nano.AttachmentVerdict.INSPECT
     end
 
@@ -271,6 +325,12 @@ function nano.send_data(session_id, session_data, meta_data, header_data, contai
     contains_body = (contains_body > 0) and 1 or 0  -- Force strict 0 or 1
 
     local verdict, response = nano_attachment.send_data(attachment, session_id, session_data, chunk_type, meta_data, header_data, contains_body)
+
+    -- Track response for cleanup
+    if response then
+        table.insert(nano.allocated_responses, response)
+    end
+
     return verdict, response
 end
 
@@ -279,12 +339,18 @@ function nano.send_body(session_id, session_data, body_chunk, chunk_type)
     local attachment = nano.attachments[worker_id]
 
     if not attachment then
-        kong.log.err("Attachment not initialized for worker ", worker_id, ". Dropping request.")
+        kong.log.warn("Attachment not available for worker ", worker_id, " - failing open")
         return nano.AttachmentVerdict.INSPECT
     end
 
     -- Send the body chunk directly as a string
     local verdict, response, modifications = nano_attachment.send_body(attachment, session_id, session_data, body_chunk, chunk_type)
+
+    -- Track response for cleanup
+    if response then
+        table.insert(nano.allocated_responses, response)
+    end
+
     return verdict, response, modifications
 end
 
@@ -327,26 +393,13 @@ function nano.send_response_body(session_id, session_data, body_chunk)
     return verdict, response, modifications
 end
 
-function nano.end_inspection(session_id, session_data, chunk_type)
-    local worker_id = ngx.worker.id()
-    local attachment = nano.attachments[worker_id]
-
-    if not attachment then
-        kong.log.err("Attachment not initialized for worker ", worker_id, ". Dropping request.")
-        return nano.AttachmentVerdict.INSPECT
-    end
-
-    local verdict, response = nano_attachment.end_inspection(attachment, session_id, session_data, chunk_type)
-    return verdict, response
-end
-
 -- Finalize session cleanup
 function nano.fini_session(session_data)
     local worker_id = ngx.worker.id()
     local attachment = nano.attachments[worker_id]
 
     if not attachment or not session_data then
-        kong.log.err("Cannot finalize session: Invalid attachment or session_data")
+        kong.log.warn("Cannot finalize session: Invalid attachment or session_data for worker ", worker_id)
         return false
     end
 
@@ -361,7 +414,7 @@ function nano.send_response_headers(session_id, session_data, headers, status_co
     local attachment = nano.attachments[worker_id]
 
     if not attachment then
-        kong.log.err("Attachment not initialized for worker ", worker_id, ". Dropping request.")
+        kong.log.warn("Attachment not available for worker ", worker_id, " - failing open")
         return nano.AttachmentVerdict.INSPECT
     end
 
@@ -373,6 +426,12 @@ function nano.send_response_headers(session_id, session_data, headers, status_co
         status_code,
         content_length
     )
+
+    -- Track response for cleanup
+    if response then
+        table.insert(nano.allocated_responses, response)
+    end
+
     return verdict, response
 end
 
@@ -423,4 +482,30 @@ function nano.handle_header_modifications(headers, modifications)
     return modified_headers
 end
 
+-- End inspection for a session
+function nano.end_inspection(session_id, session_data, chunk_type)
+    local worker_id = ngx.worker.id()
+    local attachment = nano.attachments[worker_id]
+
+    if not attachment then
+        kong.log.warn("Attachment not available for worker ", worker_id, " - failing open during end_inspection")
+        return nano.AttachmentVerdict.INSPECT, nil
+    end
+
+    if not session_data then
+        kong.log.err("Cannot end inspection: Invalid session_data for session ", session_id)
+        return nano.AttachmentVerdict.INSPECT, nil
+    end
+
+    local verdict, response = nano_attachment.end_inspection(attachment, session_id, session_data, chunk_type)
+
+    -- Track response for cleanup if allocated
+    if response then
+        table.insert(nano.allocated_responses, response)
+    end
+
+    return verdict, response
+end
+
+-- Helper function to ensure attachment is available
 return nano