change gzipped detection

* sync code

* sync code

* sync code

---------

Co-authored-by: Ned Wright <nedwright@proton.me>

README.md

@@ -6,7 +6,7 @@
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/6629/badge)](https://bestpractices.coreinfrastructure.org/projects/6629)
 
 # About
-[open-appsec](https://www.openappsec.io) (openappsec.io) builds on machine learning to provide preemptive web app & API threat protection against OWASP-Top-10 and zero-day attacks. It can be deployed as an add-on to Kubernetes Ingress, NGINX, Envoy (soon), and API Gateways.
+[open-appsec](https://www.openappsec.io) (openappsec.io) builds on machine learning to provide preemptive web app & API threat protection against OWASP-Top-10 and zero-day attacks. It can be deployed as an add-on to Linux, Docker or K8s deployments, on NGINX, Kong, APISIX, or Envoy.
 
 The open-appsec engine learns how users normally interact with your web application. It then uses this information to automatically detect requests that fall outside of normal operations, and conducts further analysis to decide whether the request is malicious or not.
 
@@ -39,13 +39,13 @@ open-appsec can be managed using multiple methods:
 * [Using SaaS Web Management](https://docs.openappsec.io/getting-started/using-the-web-ui-saas)
 
 open-appsec Web UI:
-![image](https://github.com/openappsec/openappsec/assets/114033741/22d99379-df52-45c8-984f-1b820635f3b9)
+<img width="1854" height="775" alt="image" src="https://github.com/user-attachments/assets/4c6f7b0a-14f3-4f02-9ab0-ddadc9979b8d" />
+
 
 
 ## Deployment Playgrounds (Virtual labs)
 You can experiment with open-appsec using [Playgrounds](https://www.openappsec.io/playground)
-
+<img width="781" height="878" alt="image" src="https://github.com/user-attachments/assets/0ddee216-5cdf-4288-8c41-cc28cfbf3297" />
-![image](https://github.com/openappsec/openappsec/assets/114033741/14d35d69-4577-48fc-ae87-ea344888e94d)
 
 # Resources
 * [Project Website](https://openappsec.io)
@@ -54,21 +54,15 @@ You can experiment with open-appsec using [Playgrounds](https://www.openappsec.i
 
 # Installation
 
-For Kubernetes (NGINX Ingress) using the installer:
+For Kubernetes (NGINX /Kong / APISIX / Istio) using Helm: follow [documentation](https://docs.openappsec.io/getting-started/start-with-kubernetes)
 
-```bash
+For Linux (NGINX / Kong / APISIX) using the installer (list of supported/pre-compiled NGINX attachments is available [here](https://downloads.openappsec.io/packages/supported-nginx.txt)):
-$ wget https://downloads.openappsec.io/open-appsec-k8s-install && chmod +x open-appsec-k8s-install
-$ ./open-appsec-k8s-install
-```
-
-For Kubernetes (NGINX or Kong) using Helm: follow [documentation](https://docs.openappsec.io/getting-started/start-with-kubernetes/install-using-helm-ingress-nginx-and-kong) – use this method if you’ve built your own containers. 
-
-For Linux (NGINX or Kong) using the installer (list of supported/pre-compiled NGINX attachments is available [here](https://downloads.openappsec.io/packages/supported-nginx.txt)):
 
 ```bash
 $ wget https://downloads.openappsec.io/open-appsec-install && chmod +x open-appsec-install
 $ ./open-appsec-install --auto
 ```
+For kong Lua Based plug in follow [documentation](https://docs.openappsec.io/getting-started/start-with-linux)
 
 For Linux, if you’ve built your own package use the following commands:
 

build_system/docker/Dockerfile

@@ -13,8 +13,10 @@ RUN apk add --no-cache libunwind
 RUN apk add --no-cache gdb
 RUN apk add --no-cache libxml2
 RUN apk add --no-cache pcre2
+RUN apk add --no-cache ca-certificates
 RUN apk add --update coreutils
 
+
 COPY self_managed_openappsec_manifest.json /tmp/self_managed_openappsec_manifest.json
 
 COPY install*.sh /nano-service-installers/

components/security_apps/waap/waap_clib/CMakeLists.txt

@@ -12,6 +12,7 @@ add_library(waap_clib
     ParserJson.cc
     ParserMultipartForm.cc
     ParserRaw.cc
+    ParserGzip.cc
     ParserUrlEncode.cc
     ParserXML.cc
     ParserDelimiter.cc

components/security_apps/waap/waap_clib/DeepParser.cc

@@ -22,6 +22,7 @@
 #include "ParserXML.h"
 #include "ParserHTML.h"
 #include "ParserBinary.h"
+#include "ParserGzip.h"
 #include "ParserMultipartForm.h"
 #include "ParserPercentEncode.h"
 #include "ParserPairs.h"
@@ -1261,6 +1262,10 @@ DeepParser::createInternalParser(
         dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse an HTML file";
         m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserHTML>>(*this, parser_depth + 1));
         offset = 0;
+    } else if (isBodyPayload && Waap::Util::isGzipped(cur_val)){
+        dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a gzip file";
+        m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserGzip>>(*this, parser_depth + 1));
+        offset = 0;
     } else if (cur_val.size() > 0 && signatures->php_serialize_identifier.hasMatch(cur_val)) {
         // PHP value detected
         dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse phpSerializedData";

components/security_apps/waap/waap_clib/ParserGzip.cc

@@ -0,0 +1,115 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "ParserGzip.h"
+#include "debug.h"
+
+USE_DEBUG_FLAG(D_WAAP_PARSER_GZIP);
+
+const std::string ParserGzip::m_parserName = "ParserGzip";
+
+ParserGzip::ParserGzip(IParserStreamReceiver &receiver, size_t parser_depth)
+:m_receiver(receiver), m_key("gzip"), m_state(s_start), m_stream(nullptr) {
+}
+
+ParserGzip::~ParserGzip() {
+    if (m_stream != nullptr) {
+        finiCompressionStream(m_stream);
+        m_stream = nullptr;
+    }
+}
+
+size_t ParserGzip::push(const char *buf, size_t len) {
+    dbgTrace(D_WAAP_PARSER_GZIP) << "len=" << (unsigned long int)len << ")";
+
+    if (len == 0) {
+        dbgTrace(D_WAAP_PARSER_GZIP) << "end of data signal! m_state=" << m_state;
+
+        // flush
+        if (m_state != s_start) { // only emit if at least something was pushed
+            if (m_receiver.onKvDone() != 0) {
+                m_state = s_error;
+            }
+        }
+
+        return 0;
+    }
+    DecompressionResult res;
+    switch (m_state) {
+    case s_start:
+        dbgTrace(D_WAAP_PARSER_GZIP) << "s_start";
+        if (m_receiver.onKey(m_key.data(), m_key.size()) != 0) {
+            m_state = s_error;
+            return 0;
+        }
+        m_stream = initCompressionStream();
+        m_state = s_forward;
+        // fallthrough //
+        CP_FALL_THROUGH;
+    case s_forward:
+        dbgTrace(D_WAAP_PARSER_GZIP) << "s_forward";
+        res = decompressData(
+            m_stream,
+            len,
+            reinterpret_cast<const unsigned char *>(buf));
+        dbgTrace(D_WAAP_PARSER_GZIP) << "res: " << res.ok
+            << ", size: " << res.num_output_bytes
+            << ", is last: " << res.is_last_chunk;
+
+        if (!res.ok) {
+            m_state = s_error;
+            break;
+        }
+
+        if (res.num_output_bytes != 0 &&
+                m_receiver.onValue(reinterpret_cast<const char *>(res.output), res.num_output_bytes) != 0) {
+            m_state = s_error;
+            break;
+        }
+
+        if (res.is_last_chunk) {
+            m_state = s_done;
+            break;
+        }
+        break;
+    case s_done:
+        if (len > 0) {
+            dbgTrace(D_WAAP_PARSER_GZIP) << " unexpected data after completion, len=" << len;
+            m_state = s_error;
+            return 0; // Return 0 to indicate error
+        }
+        break;
+    case s_error:
+        dbgTrace(D_WAAP_PARSER_GZIP) << "s_error";
+        return 0;
+    }
+
+    return len;
+}
+
+void ParserGzip::finish() {
+    push(NULL, 0);
+    if (m_state != s_done) {
+        m_state = s_error;
+        return;
+    }
+}
+
+const std::string &
+ParserGzip::name() const {
+    return m_parserName;
+}
+
+bool ParserGzip::error() const {
+    return m_state == s_error;
+}

components/security_apps/waap/waap_clib/ParserGzip.h

@@ -0,0 +1,46 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __PARSER_GZIP_H_
+#define __PARSER_GZIP_H_
+
+#include "ParserBase.h"
+#include <string.h>
+#include "compression_utils.h"
+
+class ParserGzip : public ParserBase {
+public:
+    ParserGzip(IParserStreamReceiver &receiver, size_t parser_depth);
+    virtual ~ParserGzip();
+    size_t push(const char *data, size_t data_len);
+    void finish();
+    virtual const std::string &name() const;
+    bool error() const;
+    virtual size_t depth() { return 1; }
+private:
+    enum state {
+        s_start,
+        s_forward,
+        s_done,
+        s_error
+    };
+
+    IParserStreamReceiver &m_receiver;
+    std::string m_key;
+    state m_state;
+    CompressionStream * m_stream;
+
+    static const std::string m_parserName;
+};
+
+#endif // __PARSER_GZIP_H_

components/security_apps/waap/waap_clib/Serializator.cc

@@ -44,14 +44,6 @@ static const string defaultSharedStorageHost = "appsec-shared-storage-svc";
 #define SHARED_STORAGE_HOST_ENV_NAME "SHARED_STORAGE_HOST"
 #define LEARNING_HOST_ENV_NAME "LEARNING_HOST"
 
-static bool
-isGZipped(const string &stream)
-{
-    if (stream.size() < 2) return false;
-    auto unsinged_stream = reinterpret_cast<const u_char *>(stream.data());
-    return unsinged_stream[0] == 0x1f && unsinged_stream[1] == 0x8b;
-}
-
 void yieldIfPossible(const string& func, int line)
 {
     // Check if we are in the main loop
@@ -73,7 +65,7 @@ bool RestGetFile::loadJson(const string& json)
     string json_str;
 
     json_str = json;
-    if (!isGZipped(json_str))
+    if (!Waap::Util::isGzipped(json_str))
     {
         return ClientRest::loadJson(json_str);
     }
@@ -343,7 +335,7 @@ void SerializeToFileBase::saveData()
 }
 
 string decompress(string fileContent) {
-    if (!isGZipped(fileContent)) {
+    if (!Waap::Util::isGzipped(fileContent)) {
         dbgTrace(D_WAAP_SERIALIZE) << "file note zipped";
         return fileContent;
     }

components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc

@@ -103,7 +103,7 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val)
     bool lastNul = false; // whether last processed character was ASCII NUL
     size_t curValLength = cur_val.length();
 
-    if (curValLength == 0) {
+    if (curValLength == 0 || Waap::Util::isGzipped(cur_val)) {
         canSplitSemicolon = false;
         canSplitPipe = false;
         return;

components/security_apps/waap/waap_clib/Waf2Util.cc

@@ -1912,6 +1912,17 @@ base64Decode(const string &input)
     return out;
 }
 
+bool
+isGzipped(const string &stream)
+{
+    if (stream.size() < 2) return false;
+    auto unsinged_stream = reinterpret_cast<const u_char *>(stream.data());
+    dbgTrace(D_WAAP) << "isGzipped: first two bytes: "
+        << std::hex << static_cast<int>(unsinged_stream[0]) << " "
+        << std::hex << static_cast<int>(unsinged_stream[1]);
+    return unsinged_stream[0] == 0x1f && unsinged_stream[1] == 0x8b;
+}
+
 bool
 containsInvalidUtf8(const string &payload)
 {

components/security_apps/waap/waap_clib/Waf2Util.h

@@ -1135,6 +1135,7 @@ namespace Util {
     std::string obfuscateXor(const std::string& toEncrypt);
     std::string obfuscateXorBase64(const std::string& toEncrypt);
 
+    bool isGzipped(const std::string &stream);
     bool containsInvalidUtf8(const std::string &payload);
 
     bool containsPercentEncoding(const std::string &payload);

core/include/services_sdk/resources/debug_flags.h

@@ -106,6 +106,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
             DEFINE_FLAG(D_WAAP_PARSER_GQL, D_WAAP_PARSER)
             DEFINE_FLAG(D_WAAP_PARSER_MULTIPART_FORM, D_WAAP_PARSER)
             DEFINE_FLAG(D_WAAP_PARSER_RAW, D_WAAP_PARSER)
+            DEFINE_FLAG(D_WAAP_PARSER_GZIP, D_WAAP_PARSER)
             DEFINE_FLAG(D_WAAP_PARSER_URLENCODE, D_WAAP_PARSER)
             DEFINE_FLAG(D_WAAP_PARSER_PHPSERIALIZE, D_WAAP_PARSER)
             DEFINE_FLAG(D_WAAP_PARSER_PERCENT, D_WAAP_PARSER)

core/messaging/connection/connection.cc

@@ -262,6 +262,29 @@ public:
     }
 
 private:
+    string
+    getCertificateDirectory()
+    {
+        auto details_ssl_dir = Singleton::Consume<I_AgentDetails>::by<Messaging>()->getOpenSSLDir();
+
+        if (details_ssl_dir.ok()) {
+            return *details_ssl_dir;
+        }
+
+        // Use detail_resolver to determine platform-specific certificate directory
+#if defined(alpine)
+       string platform = "alpine";
+#else
+       string platform = "linux";
+#endif
+
+        if (platform == "alpine") {
+            return "/etc/ssl/certs/";
+        }
+
+        return "/usr/lib/ssl/certs/";
+    }
+
     Maybe<void>
     setSSLContext()
     {
@@ -296,10 +319,11 @@ private:
         }
 
         dbgTrace(D_CONNECTION) << "Setting CA authentication";
-        auto details_ssl_dir = Singleton::Consume<I_AgentDetails>::by<Messaging>()->getOpenSSLDir();
+
-        auto openssl_dir = details_ssl_dir.ok() ? *details_ssl_dir : "/usr/lib/ssl/certs/";
+        auto default_ssl_dir = getCertificateDirectory();
-        auto configured_ssl_dir = getConfigurationWithDefault(openssl_dir, "message", "Trusted CA directory");
+        auto configured_ssl_dir =
-        const char *ca_dir = configured_ssl_dir.empty() ? nullptr : configured_ssl_dir.c_str();
+            getProfileAgentSettingWithDefault<string>(default_ssl_dir, "agent.config.message.capath");
+        const char *ca_dir = configured_ssl_dir.empty() ? "/usr/lib/ssl/certs/" : configured_ssl_dir.c_str();
 
         if (SSL_CTX_load_verify_locations(ssl_ctx.get(), ca_path.c_str(), ca_dir) != 1) {
             return genError("Failed to load certificate locations");

deployment/docker-compose/kong-lua-plugin/docker-compose.yaml

@@ -28,7 +28,7 @@ services:
       - user_email=${APPSEC_USER_EMAIL}
       - AGENT_TOKEN=${APPSEC_AGENT_TOKEN}
       - autoPolicyLoad=${APPSEC_AUTO_POLICY_LOAD}
-      - registered_server=Kong
+      - registered_server=KongLua
     ipc: shareable
     restart: unless-stopped
     volumes: