update embedded nginx support for openappsec and bug fixes and schema changes

components/security_apps/orchestration/local_policy_mgmt_gen/include/appsec_practice_section.h

@@ -119,9 +119,9 @@ public:
     load(cereal::JSONInputArchive &archive_in)
     {
         dbgTrace(D_K8S_POLICY) << "Loading AppSec Web Attack Protections";
-        parseAppsecJSONKey<std::string>("csrf-protection", csrf_protection, archive_in, "Inactive");
+        parseAppsecJSONKey<std::string>("csrf-enabled", csrf_protection, archive_in, "inactive");
-        parseAppsecJSONKey<std::string>("error-disclosure", error_disclosure, archive_in, "Inactive");
+        parseAppsecJSONKey<std::string>("error-disclosure-enabled", error_disclosure, archive_in, "inactive");
-        parseAppsecJSONKey<std::string>("open-redirect", open_redirect, archive_in, "Inactive");
+        parseAppsecJSONKey<std::string>("open-redirect-enabled", open_redirect, archive_in, "inactive");
         parseAppsecJSONKey<bool>("non-valid-http-methods", non_valid_http_methods, archive_in, false);
     }
 
@@ -253,7 +253,7 @@ public:
     {
         dbgTrace(D_K8S_POLICY) << "Loading AppSec Snort Signatures practice";
         parseAppsecJSONKey<std::string>("override-mode", override_mode, archive_in, "Inactive");
-        parseAppsecJSONKey<std::vector<std::string>>("configmap", config_map, archive_in);
+        parseAppsecJSONKey<std::vector<std::string>>("files", config_map, archive_in);
     }
 
     const std::string & getOverrideMode() const { return override_mode; }
@@ -285,7 +285,7 @@ public:
     {
         dbgTrace(D_K8S_POLICY) << "Loading AppSecPracticeOpenSchemaAPI practice";
         parseAppsecJSONKey<std::string>("override-mode", override_mode, archive_in, "Inactive");
-        parseAppsecJSONKey<std::vector<std::string>>("configmap", config_map, archive_in);
+        parseAppsecJSONKey<std::vector<std::string>>("files", config_map, archive_in);
     }
 
     const std::string & getOverrideMode() const { return override_mode; }
@@ -777,12 +777,12 @@ public:
         dbgTrace(D_K8S_POLICY) << "Loading AppSec policy spec";
         parseAppsecJSONKey<AppsecPolicySpec>("policies", policies, archive_in);
         parseAppsecJSONKey<std::vector<AppSecPracticeSpec>>("practices", practices, archive_in);
-        parseAppsecJSONKey<std::vector<AppsecTriggerSpec>>("logtriggers", log_triggers, archive_in);
+        parseAppsecJSONKey<std::vector<AppsecTriggerSpec>>("log-triggers", log_triggers, archive_in);
-        parseAppsecJSONKey<std::vector<AppSecCustomResponseSpec>>("customresponses", custom_responses, archive_in);
+        parseAppsecJSONKey<std::vector<AppSecCustomResponseSpec>>("custom-responses", custom_responses, archive_in);
         parseAppsecJSONKey<std::vector<AppsecExceptionSpec>>("exceptions", exceptions, archive_in);
-        parseAppsecJSONKey<std::vector<TrustedSourcesSpec>>("trustedsources", trusted_sources, archive_in);
+        parseAppsecJSONKey<std::vector<TrustedSourcesSpec>>("trusted-sources", trusted_sources, archive_in);
         parseAppsecJSONKey<std::vector<SourceIdentifierSpecWrapper>>(
-            "sourceidentifiers",
+            "source-identifier",
             sources_identifier,
             archive_in
         );

components/security_apps/orchestration/local_policy_mgmt_gen/include/trusted_sources_section.h

@@ -34,7 +34,7 @@ public:
     {
         dbgTrace(D_K8S_POLICY) << "Loading trusted sources spec";
         parseAppsecJSONKey<int>("minNumOfSources", min_num_of_sources, archive_in, 3);
-        parseAppsecJSONKey<std::vector<std::string>>("sourcesIdentifiers", sources_identifiers, archive_in);
+        parseAppsecJSONKey<std::vector<std::string>>("sources-identifiers", sources_identifiers, archive_in);
         parseAppsecJSONKey<std::string>("name", name, archive_in);
     }
 
@@ -110,7 +110,7 @@ public:
     load(cereal::JSONInputArchive &archive_in)
     {
         dbgTrace(D_K8S_POLICY) << "Loading trusted sources spec";
-        parseAppsecJSONKey<std::string>("sourceIdentifier", source_identifier, archive_in);
+        parseAppsecJSONKey<std::string>("source-identifier", source_identifier, archive_in);
         parseAppsecJSONKey<std::vector<std::string>>("value", value, archive_in);
     }
 

components/security_apps/orchestration/local_policy_mgmt_gen/local_policy_mgmt_gen.cc

@@ -28,6 +28,7 @@
 #include <boost/uuid/uuid_generators.hpp>
 
 #include "rest.h"
+#include "report/report.h"
 #include "debug.h"
 #include "config.h"
 #include "connkey.h"
@@ -1218,27 +1219,10 @@ private:
     LocalPolicyEnv env_type;
     map<string, string> practice_name_to_id_map;
 
-    bool 
-    isPlaygroundEnv()
-    {
-        string playground_variable = "PLAYGROUND";
-        const char* env_string = getenv(playground_variable.c_str());
-        
-        if (env_string)
-        {
-          string env_value = env_string;
-          std::transform(env_value.begin(), env_value.end(), env_value.begin(), 
-                         [](unsigned char c){ return std::tolower(c); });
-          return env_value == "true";           
-        }
-         
-        return false;
-    }
-
     bool
     getClusterId()
     {
-        string playground_uid = isPlaygroundEnv() ? "playground-" : "";
+        string playground_uid = Report::isPlaygroundEnv() ? "playground-" : "";
 
         dbgTrace(D_K8S_POLICY) << "Getting cluster UID";
         auto maybe_namespaces_data = getObjectFromCluster<NamespaceData>("/api/v1/namespaces/");

components/security_apps/orchestration/orchestration_comp.cc

@@ -596,6 +596,10 @@ private:
                 auto team = i_env->get<AudienceTeam>("Audience Team");
                 if (team.ok()) audience_team = *team;
 
+                string agent_uid =
+                    (Report::isPlaygroundEnv() ? "playground-" : "") +
+                    Singleton::Consume<I_AgentDetails>::by<OrchestrationComp>()->getAgentId();
+
                 Report policy_update_message(
                     "Agent's policy has been updated",
                     curr_time,
@@ -607,7 +611,7 @@ private:
                     Severity::INFO,
                     Priority::LOW,
                     chrono::seconds(0),
-                    LogField("agentId", Singleton::Consume<I_AgentDetails>::by<OrchestrationComp>()->getAgentId()),
+                    LogField("agentId", agent_uid),
                     Tags::ORCHESTRATOR
                 );
                 policy_update_message.addToOrigin(LogField("policyVersion", new_policy_version));

components/signal_handler/signal_handler.cc

@@ -207,6 +207,9 @@ private:
         }
 
         set<ReportIS::Tags> tags;
+        string agent_uid =
+            (Report::isPlaygroundEnv() ? "playground-" : "") +
+            Singleton::Consume<I_AgentDetails>::by<SignalHandler>()->getAgentId();
         Report message_to_fog(
             "Nano service startup after crash",
             curr_time,
@@ -218,7 +221,7 @@ private:
             Severity::HIGH,
             Priority::HIGH,
             chrono::seconds(0),
-            LogField("agentId", Singleton::Consume<I_AgentDetails>::by<SignalHandler>()->getAgentId()),
+            LogField("agentId", agent_uid),
             tags,
             Tags::INFORMATIONAL
         );

core/debug_is/debug_streams.cc

@@ -245,6 +245,9 @@ DebugFogStream::finishMessage()
         if (team.ok()) audience_team = *team;
     }
 
+    string agent_uid =
+        (Report::isPlaygroundEnv() ? "playground-" : "") +
+        Singleton::Consume<I_AgentDetails>::by<DebugFogStream>()->getAgentId();
     Report message_to_fog(
         "Debug message",
         curr_time,
@@ -256,7 +259,7 @@ DebugFogStream::finishMessage()
         getSeverity(),
         Priority::LOW,
         chrono::seconds(0),
-        LogField("agentId", Singleton::Consume<I_AgentDetails>::by<DebugFogStream>()->getAgentId()),
+        LogField("agentId", agent_uid),
         LogField("issuingFunction", func_name),
         LogField("issuingFile", file_name),
         LogField("issuingLine", line),
@@ -328,6 +331,9 @@ DebugFogStream::handleThresholdReach()
         if (team.ok()) audience_team = *team;
     }
 
+    string agent_uid =
+        (Report::isPlaygroundEnv() ? "playground-" : "") +
+        Singleton::Consume<I_AgentDetails>::by<DebugFogStream>()->getAgentId();
     Report message_to_fog(
         "Debug message",
         curr_time,
@@ -339,7 +345,7 @@ DebugFogStream::handleThresholdReach()
         Severity::MEDIUM,
         Priority::LOW,
         chrono::seconds(0),
-        LogField("agentId", Singleton::Consume<I_AgentDetails>::by<DebugFogStream>()->getAgentId()),
+        LogField("agentId", agent_uid),
         LogField("issuingFunction", __FUNCTION__),
         LogField("issuingFile", "debug_streams.cc"),
         LogField("issuingLine", __LINE__),

core/include/services_sdk/resources/log_generator.h

@@ -80,7 +80,11 @@ public:
             _severity,
             _priority,
             std::chrono::seconds(0),
-            LogField("agentId", Singleton::Consume<I_AgentDetails>::by<LogGen>()->getAgentId()),
+            LogField(
+                "agentId",
+                (Report::isPlaygroundEnv() ? "playground-" : "") +
+                Singleton::Consume<I_AgentDetails>::by<LogGen>()->getAgentId()
+            ),
             std::forward<Args>(args)...
         )
     {

core/include/services_sdk/resources/report/report.h

@@ -16,6 +16,10 @@
 
 #include <set>
 #include <chrono>
+#include <stdlib.h>
+#include <algorithm>
+#include <cctype>
+#include <iostream>
 
 #include "report/base_field.h"
 #include "report/report_enums.h"
@@ -71,6 +75,26 @@ public:
         setInstanceAwareness();
     }
 
+    static bool
+    isPlaygroundEnv()
+    {
+        std::string playground_variable = "PLAYGROUND";
+        const char *env_string = getenv(playground_variable.c_str());
+
+        if (env_string) {
+            std::string env_value = env_string;
+            std::transform(
+                env_value.begin(),
+                env_value.end(),
+                env_value.begin(),
+                [](unsigned char c){ return std::tolower(c); }
+            );
+            return env_value == "true";
+        }
+
+        return false;
+    }
+
     void serialize(cereal::JSONOutputArchive &ar) const;
     std::string getSyslog() const;
     std::string getCef() const;

core/mainloop/mainloop.cc

@@ -174,6 +174,9 @@ MainloopComponent::Impl::reportStartupEvent()
     auto team = i_env->get<ReportIS::AudienceTeam>("Audience Team");
     if (team.ok()) audience_team = *team;
 
+    string agent_uid =
+        (Report::isPlaygroundEnv() ? "playground-" : "") +
+        Singleton::Consume<I_AgentDetails>::by<MainloopComponent>()->getAgentId();
     Report startup_message(
         "Nano service successfully started",
         curr_time,
@@ -185,7 +188,7 @@ MainloopComponent::Impl::reportStartupEvent()
         ReportIS::Severity::INFO,
         ReportIS::Priority::HIGH,
         chrono::seconds(0),
-        LogField("agentId", Singleton::Consume<I_AgentDetails>::by<MainloopComponent>()->getAgentId()),
+        LogField("agentId", agent_uid),
         ReportIS::Tags::INFORMATIONAL
     );
 

core/metric/generic_metric.cc

@@ -170,6 +170,9 @@ string GenericMetric::getListenerName() const { return metric_name; }
 void
 GenericMetric::generateLog()
 {
+    string agent_uid =
+        (Report::isPlaygroundEnv() ? "playground-" : "") +
+        Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId();
     set<ReportIS::Tags> tags;
     Report metric_to_fog(
         metric_name,
@@ -182,7 +185,7 @@ GenericMetric::generateLog()
         Severity::INFO,
         Priority::LOW,
         report_interval,
-        LogField("agentId", Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId()),
+        LogField("agentId", agent_uid),
         tags,
         Tags::INFORMATIONAL,
         issuing_engine

nodes/orchestration/package/cp-nano-cli.sh

@@ -899,6 +899,25 @@ print_single_service_status() # Initials - psss
     fi
 }
 
+get_status_content()
+{
+    if [ "${remove_curl_ld_path}" = "true" ]; then
+        gsc_orch_status=$(LD_LIBRARY_PATH="" ${curl_cmd} -sS -m 1 --noproxy "*" --header "Content-Type: application/json" --request POST --data {} http://127.0.0.1:"$(extract_api_port 'orchestration')"/show-orchestration-status 2>&1)
+    else
+        gsc_orch_status=$(${curl_cmd} -sS -m 1 --noproxy "*" --header "Content-Type: application/json" --request POST --data {} http://127.0.0.1:"$(extract_api_port 'orchestration')"/show-orchestration-status 2>&1)
+    fi
+
+    if echo "$gsc_orch_status" | grep -q "update status"; then
+        gsc_line_count=$(echo "$gsc_orch_status" | grep -c '^')
+
+        gsc_temp_old_status=$(echo "$gsc_orch_status" | sed -r "${gsc_line_count},${gsc_line_count}d; "' 1,1d; s/^\s*//g; s/^\n//g; s/\"//g; s/\\n/\n/g; s/\,//g')
+    else
+        gsc_temp_old_status=$(sed 's/{//g' <${FILESYSTEM_PATH}/$cp_nano_conf_location/orchestrations_status.json | sed 's/}//g' | sed 's/"//g' | sed 's/,//g' | sed -r '/^\s*$/d' | sed -r 's/^    //g')
+    fi
+
+    echo ${gsc_temp_old_status}
+}
+
 run_status() # Initials - rs
 {
     rs_orch_service_full_path=$(get_nano_service_path 'orchestration')
@@ -1463,7 +1482,7 @@ set_mode()
         time_sleep=2
         time_out=60
         echo "Registering open-appsec Nano Agent to Fog.."
-        until $USR_SBIN_PATH/${CP_NANO_CTL} -s 2> /dev/null | grep -q "Registration status: Succeeded"; do
+        until get_status_content | grep -q "Registration status: Succeeded"; do
             time_out=$(( time_out - time_sleep ))
             if [ $time_out -le 0 ]; then
                 echo "open-appsec Nano Agent registration failed. Failed to register to Fog: $fog_address"

nodes/orchestration/package/local-default-policy.yaml

@@ -6,7 +6,7 @@ policies:
     practices:
     - webapp-default-practice
     custom-response: appsec-default-web-user-response
-  specific-rules:
+  specific-rules: []
 
 practices:
   - name: webapp-default-practice
@@ -33,7 +33,7 @@ practices:
       validated-URIs: []
       override-mode: detect-learn
 
-logtriggers:
+log-triggers:
   - name: appsec-default-log-trigger
     access-control-logging:
       allow-events: false
@@ -53,18 +53,11 @@ logtriggers:
       url-query: false
     log-destination:
       cloud: false
-      file:
       stdout:
         format: json
-      syslog-service:
-      cef-service:
 
-customresponses:
+custom-responses:
   - name: appsec-default-web-user-response
     mode: response-code-only
     http-response-code: 403
 
-exceptions:
-trustedsources:
-sourceidentifiers:
-

nodes/orchestration/package/watchdog/watchdog

@@ -63,7 +63,6 @@ load_paths()
 	if [ -z "${MAX_ROTATION}" ]; then
         MAX_ROTATION=$DEFAULT_MAX_ROTATION
 	fi
-
 }
 
 load_paths