Create open-appsec-crd-latest.yaml

Jul 23rd update

attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc

@@ -155,6 +155,24 @@ getWaitingForVerdictThreadTimeout()
     return conf_data.getNumericalValue("waiting_for_verdict_thread_timeout_msec");
 }
 
+unsigned int
+getMinRetriesForVerdict()
+{
+    return conf_data.getNumericalValue("min_retries_for_verdict");
+}
+
+unsigned int
+getMaxRetriesForVerdict()
+{
+    return conf_data.getNumericalValue("max_retries_for_verdict");
+}
+
+unsigned int
+getReqBodySizeTrigger()
+{
+    return conf_data.getNumericalValue("body_size_trigger");
+}
+
 int
 isIPAddress(c_str ip_str)
 {

attachments/nginx/nginx_attachment_util/nginx_attachment_util_ut/nginx_attachment_util_ut.cc

@@ -63,7 +63,10 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration)
             "\"waiting_for_verdict_thread_timeout_msec\": 75,\n"
             "\"req_header_thread_timeout_msec\": 10,\n"
             "\"ip_ranges\": " + createIPRangesString(ip_ranges) + ",\n"
-            "\"static_resources_path\": \"" + static_resources_path + "\""
+            "\"static_resources_path\": \"" + static_resources_path + "\",\n"
+            "\"min_retries_for_verdict\": 1,\n"
+            "\"max_retries_for_verdict\": 3,\n"
+            "\"body_size_trigger\": 777\n"
         "}\n";
     ofstream valid_configuration_file(attachment_configuration_file_name);
     valid_configuration_file << valid_configuration;
@@ -87,6 +90,9 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration)
     EXPECT_EQ(getReqBodyThreadTimeout(), 155);
     EXPECT_EQ(getResHeaderThreadTimeout(), 1);
     EXPECT_EQ(getResBodyThreadTimeout(), 0);
+    EXPECT_EQ(getMinRetriesForVerdict(), 1);
+    EXPECT_EQ(getMaxRetriesForVerdict(), 3);
+    EXPECT_EQ(getReqBodySizeTrigger(), 777);
     EXPECT_EQ(getWaitingForVerdictThreadTimeout(), 75);
     EXPECT_EQ(getInspectionMode(), ngx_http_inspection_mode::BLOCKING_THREAD);
 

components/attachment-intakers/nginx_attachment/nginx_attachment_config.cc

@@ -42,6 +42,7 @@ HttpAttachmentConfig::init()
     setNumOfNginxIpcElements();
     setDebugByContextValues();
     setKeepAliveIntervalMsec();
+    setRetriesForVerdict();
 }
 
 bool
@@ -215,6 +216,31 @@ HttpAttachmentConfig::setFailOpenTimeout()
     conf_data.setNumericalValue("nginx_inspection_mode", inspection_mode);
 }
 
+void
+HttpAttachmentConfig::setRetriesForVerdict()
+{
+    conf_data.setNumericalValue("min_retries_for_verdict", getAttachmentConf<uint>(
+        3,
+        "agent.minRetriesForVerdict.nginxModule",
+        "HTTP manager",
+        "Min retries for verdict"
+    ));
+
+    conf_data.setNumericalValue("max_retries_for_verdict", getAttachmentConf<uint>(
+        15,
+        "agent.maxRetriesForVerdict.nginxModule",
+        "HTTP manager",
+        "Max retries for verdict"
+    ));
+
+    conf_data.setNumericalValue("body_size_trigger", getAttachmentConf<uint>(
+        200000,
+        "agent.reqBodySizeTrigger.nginxModule",
+        "HTTP manager",
+        "Request body size trigger"
+    ));
+}
+
 void
 HttpAttachmentConfig::setFailOpenWaitMode()
 {

components/attachment-intakers/nginx_attachment/nginx_attachment_config.h

@@ -70,6 +70,8 @@ private:
 
     void setDebugByContextValues();
 
+    void setRetriesForVerdict();
+
     WebTriggerConf web_trigger_conf;
     HttpAttachmentConfiguration conf_data;
 };

components/security_apps/local_policy_mgmt_gen/triggers_section.cc

@@ -96,8 +96,8 @@ LogTriggerSection::save(cereal::JSONOutputArchive &out_ar) const
         cereal::make_nvp("acDrop",                   acDrop),
         cereal::make_nvp("complianceViolations",     false),
         cereal::make_nvp("complianceWarnings",       false),
-        cereal::make_nvp("extendloggingMinSeverity", extendloggingMinSeverity),
-        cereal::make_nvp("extendlogging",            extendlogging),
+        cereal::make_nvp("extendLoggingMinSeverity", extendloggingMinSeverity),
+        cereal::make_nvp("extendLogging",            extendlogging),
         cereal::make_nvp("logToAgent",               logToAgent),
         cereal::make_nvp("logToCef",                 logToCef),
         cereal::make_nvp("logToCloud",               logToCloud),

components/security_apps/orchestration/details_resolver/details_resolver.cc

@@ -152,6 +152,7 @@ DetailsResolver::Impl::isCloudStorageEnabled()
 bool
 DetailsResolver::Impl::isKernelVersion3OrHigher()
 {
+#if defined(gaia) || defined(smb)
     static const string cmd =
         "clish -c 'show version os kernel' | awk '{print $4}' "
         "| cut -d '.' -f 1 | awk -F: '{ if ( $1 >= 3 ) {print 1} else {print 0}}'";
@@ -160,12 +161,14 @@ DetailsResolver::Impl::isKernelVersion3OrHigher()
     if (is_gogo.ok() && !is_gogo.unpack().empty()) {
         return is_gogo.unpack().front() == '1';
     }
+#endif
     return false;
 }
 
 bool
 DetailsResolver::Impl::isGwNotVsx()
 {
+#if defined(gaia) || defined(smb)
     static const string is_gw_cmd = "cpprod_util FwIsFirewallModule";
     static const string is_vsx_cmd = "cpprod_util FWisVSX";
     auto is_gw = DetailsResolvingHanlder::getCommandOutput(is_gw_cmd);
@@ -173,6 +176,7 @@ DetailsResolver::Impl::isGwNotVsx()
     if (is_gw.ok() && is_vsx.ok() && !is_gw.unpack().empty() && !is_vsx.unpack().empty()) {
         return is_gw.unpack().front() == '1' && is_vsx.unpack().front() == '0';
     }
+#endif
     return false;
 }
 

components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h

@@ -24,14 +24,16 @@
 Maybe<string>
 checkSAMLSupportedBlade(const string &command_output)
 {
-    string supportedBlades[3] = {"identityServer", "vpn", "cvpn"};
+    // uncomment when vpn will support SAML authentication
+    // string supportedBlades[3] = {"identityServer", "vpn", "cvpn"};
+    string supportedBlades[1] = {"identityServer"};
     for(const string &blade : supportedBlades) {
         if (command_output.find(blade) != string::npos) {
             return string("true");
         }
     }
 
-    return genError("Current host does not have SAML capability");
+    return string("false");
 }
 
 Maybe<string>
@@ -42,7 +44,7 @@ checkIDABlade(const string &command_output)
         return string("true");
     }
 
-    return genError("Current host does not have IDA installed");
+    return string("false");
 }
 
 Maybe<string>
@@ -52,7 +54,7 @@ checkSAMLPortal(const string &command_output)
         return string("true");
     }
 
-    return genError("Current host does not have SAML Portal configured");
+    return string("false");
 }
 
 Maybe<string>
@@ -61,7 +63,7 @@ checkPepIdaIdnStatus(const string &command_output)
     if (command_output.find("nac_pep_scaled_sharing_enabled = 1") != string::npos) {
         return string("true");
     }
-    return genError("Current host does not have PEP control scaled_sharing enabled");
+    return string("false");
 }
 
 Maybe<string>
@@ -87,7 +89,7 @@ checkIDP(shared_ptr<istream> file_stream)
         }
     }
 
-    return genError("Identity Provider was not found");
+    return string("false");
 }
 
 #endif // gaia

components/security_apps/orchestration/details_resolver/details_resolving_handler.cc

@@ -99,6 +99,7 @@ map<string, string>
 DetailsResolvingHanlder::Impl::getResolvedDetails() const
 {
     I_ShellCmd *shell = Singleton::Consume<I_ShellCmd>::by<DetailsResolvingHanlder>();
+    I_AgentDetailsReporter *reporter = Singleton::Consume<I_AgentDetailsReporter>::by<DetailsResolvingHanlder>();
     uint32_t timeout = getConfigurationWithDefault<uint32_t>(5000, "orchestration", "Details resolver time out");
 
     for (auto &shell_pre_command : shell_pre_commands) {
@@ -122,7 +123,15 @@ DetailsResolvingHanlder::Impl::getResolvedDetails() const
         Maybe<string> shell_command_output = getCommandOutput(command);
         if (!shell_command_output.ok()) continue;
         Maybe<string> handler_ret = handler(*shell_command_output);
-        if (handler_ret.ok()) resolved_details[attr] = *handler_ret;
+
+        if (handler_ret.ok()) {
+            resolved_details[attr] = *handler_ret;
+        } else {
+            if (reporter->isPersistantAttr(attr)) {
+                dbgTrace(D_AGENT_DETAILS)<< "Persistent attribute changed, removing old value";
+                reporter->deleteAttr(attr);
+            }
+        }
     }
 
     for (auto file_handler : file_content_handlers) {
@@ -157,7 +166,6 @@ DetailsResolvingHanlder::Impl::getResolvedDetails() const
         }
     }
 
-    I_AgentDetailsReporter *reporter = Singleton::Consume<I_AgentDetailsReporter>::by<DetailsResolvingHanlder>();
     reporter->addAttr(resolved_details, true);
 
     return resolved_details;

components/security_apps/orchestration/include/fog_communication.h

@@ -51,6 +51,7 @@ public:
 
 private:
     I_DeclarativePolicy *i_declarative_policy = nullptr;
+    std::string profile_mode;
 };
 
 #endif // __FOG_COMMUNICATION_H__

components/security_apps/orchestration/update_communication/declarative_policy_utils.cc

@@ -141,7 +141,7 @@ DeclarativePolicyUtils::sendUpdatesToFog(
     auto shell_cmd = Singleton::Consume<I_ShellCmd>::by<DeclarativePolicyUtils>();
     string exec_command =
         getFilesystemPathConfig()
-        + "/scripts/open-appsec-cloud-mgmt --upload_policy_only"
+        + "/scripts/open-appsec-cloud-mgmt --config-upload-only"
         + " --access_token " + access_token
         + " --tenant_id " + tenant_id
         + " --profile_id " + profile_id;

components/security_apps/orchestration/update_communication/fog_communication.cc

@@ -32,6 +32,7 @@ FogCommunication::init()
 {
     FogAuthenticator::init();
     i_declarative_policy = Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>();
+    profile_mode = getSettingWithDefault<string>("management", "profileManagedMode");
 }
 
 Maybe<void>
@@ -66,6 +67,16 @@ FogCommunication::getUpdate(CheckUpdateRequest &request)
         Maybe<string> maybe_new_data = request.getData();
         string data_checksum = maybe_new_data.ok() ? maybe_new_data.unpack() : "";
 
+        if (profile_mode != policy_mgmt_mode) {
+            dbgTrace(D_ORCHESTRATOR)
+                << "The profile managed mode was changed from: "
+                << profile_mode
+                << " to: "
+                << policy_mgmt_mode;
+            profile_mode = policy_mgmt_mode;
+            i_declarative_policy->turnOnApplyPolicyFlag();
+        }
+
         if (i_declarative_policy->shouldApplyPolicy()) {
             string policy_response = i_declarative_policy->getUpdate(request);
             if (!policy_response.empty()) {

components/security_apps/waap/waap_clib/ScanResult.cc

@@ -23,6 +23,7 @@ unescaped_line(),
 param_name(),
 location(),
 score(0.0f),
+scoreNoFilter(0.0f),
 scoreArray(),
 keywordCombinations(),
 attack_types(),
@@ -40,6 +41,7 @@ void Waf2ScanResult::clear()
     param_name.clear();
     location.clear();
     score = 0;
+    scoreNoFilter = 0;
     scoreArray.clear();
     keywordCombinations.clear();
     attack_types.clear();

components/security_apps/waap/waap_clib/ScanResult.h

@@ -29,6 +29,7 @@ struct Waf2ScanResult {
     std::string param_name;
     std::string location;
     double score;
+    double scoreNoFilter;
     std::vector<double> scoreArray;
     std::vector<std::string> keywordCombinations;
     std::set<std::string> attack_types;

components/security_apps/waap/waap_clib/WaapConversions.cc

@@ -39,7 +39,7 @@ namespace Conversions {
         return HIGH_THREAT;
     }
 
-    bool shouldDoWafBlocking(const IWaapConfig* pWaapConfig, ThreatLevel threatLevel)
+    bool shouldDoWafBlocking(const IWaapConfig* const pWaapConfig, ThreatLevel threatLevel)
     {
         if (pWaapConfig == NULL)
         {

components/security_apps/waap/waap_clib/WaapConversions.h

@@ -20,7 +20,7 @@
 namespace Waap {
 namespace Conversions {
     ThreatLevel convertFinalScoreToThreatLevel(double finalScore);
-    bool shouldDoWafBlocking(const IWaapConfig* pSitePolicy, ThreatLevel threatLevel);
+    bool shouldDoWafBlocking(const IWaapConfig* const pSitePolicy, ThreatLevel threatLevel);
 }
 }
 

components/security_apps/waap/waap_clib/WaapScanner.cc

@@ -25,7 +25,7 @@ USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
 // id generated by xml parser for an entity attribute
 const std::string Waap::Scanner::xmlEntityAttributeId = "08a80340-06d3-11ea-9f87-0242ac11000f";
 
-double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolName)
+double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolName, bool applyLearning)
 {
     std::string source = m_transaction->getSourceIdentifier();
 
@@ -33,21 +33,24 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
     Waap::Keywords::KeywordsSet keywordsSet;
     Waap::Keywords::computeKeywordsSet(keywordsSet, res.keyword_matches, res.found_patterns);
 
-    std::string param_name = IndicatorsFiltersManager::generateKey(res.location, res.param_name, m_transaction);
-    dbgTrace(D_WAAP_SCANNER) << "filter processing for parameter: " << param_name;
-    m_transaction->getAssetState()->logIndicatorsInFilters(param_name, keywordsSet, m_transaction);
+    if (applyLearning) {
+        std::string param_name = IndicatorsFiltersManager::generateKey(res.location, res.param_name, m_transaction);
+        dbgTrace(D_WAAP_SCANNER) << "filter processing for parameter: " << param_name <<
+        ", indicators count: " << keywordsSet.size();
+        m_transaction->getAssetState()->logIndicatorsInFilters(param_name, keywordsSet, m_transaction);
 
-    m_transaction->getAssetState()->filterKeywords(param_name, keywordsSet, res.filtered_keywords);
-    if (m_transaction->getSiteConfig() != nullptr)
-    {
-        auto waapParams = m_transaction->getSiteConfig()->get_WaapParametersPolicy();
-        if (waapParams != nullptr && waapParams->getParamVal("filtersVerbose", "false") == "true") {
-            m_transaction->getAssetState()->filterVerbose(param_name, res.filtered_keywords);
+        m_transaction->getAssetState()->filterKeywords(param_name, keywordsSet, res.filtered_keywords);
+        if (m_transaction->getSiteConfig() != nullptr)
+        {
+            auto waapParams = m_transaction->getSiteConfig()->get_WaapParametersPolicy();
+            if (waapParams != nullptr && waapParams->getParamVal("filtersVerbose", "false") == "true") {
+                m_transaction->getAssetState()->filterVerbose(param_name, res.filtered_keywords);
+            }
         }
+        m_transaction->getAssetState()->filterKeywordsByParameters(res.param_name, keywordsSet);
+
+        dbgTrace(D_WAAP_SCANNER) << "post filtering indicators count: " << keywordsSet.size();
     }
-    m_transaction->getAssetState()->filterKeywordsByParameters(res.param_name, keywordsSet);
-
-
     // The keywords are only removed in production, they are still used while building scores
     if (!m_transaction->get_ignoreScore()) {
         m_transaction->getAssetState()->removeKeywords(keywordsSet);
@@ -148,9 +151,16 @@ bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, DeepParser &dp,
     // Select scores pool by location
     std::string poolName = Waap::Scores::getScorePoolNameByLocation(location);
 
+    Waf2ScanResult nonFilterRes = res;
+    res.scoreNoFilter = getScoreData(nonFilterRes, poolName, false);
+
     double score = getScoreData(res, poolName);
 
-    dbgTrace(D_WAAP_SCANNER) << "score: " << score;
+    // call shouldIgnoreOverride post score calculation and filtering to evaluate ignore override effectivness
+    res.score = score;
+    m_transaction->shouldIgnoreOverride(res);
+
+    dbgTrace(D_WAAP_SCANNER) << "score: " << score << " should ignore: " << ignoreOverride;
     // Add record about scores to the notes[] log (also reported in logs)
     if (score > 1.0f) {
         DetectionEvent(location, res.keyword_matches).notify();
@@ -166,6 +176,7 @@ bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, DeepParser &dp,
     if (isKeyCspReport(key, res, dp) || ignoreOverride) {
         dbgTrace(D_WAAP_SCANNER) << "Ignoring parameter key/value " << res.param_name <<
             " due to ignore action in override";
+        res.score = 0;
         m_bIgnoreOverride = true;
         return false;
     }

components/security_apps/waap/waap_clib/WaapScanner.h

@@ -43,7 +43,7 @@ namespace Waap {
 
         static const std::string xmlEntityAttributeId;
     private:
-        double getScoreData(Waf2ScanResult& res, const std::string &poolName);
+        double getScoreData(Waf2ScanResult& res, const std::string &poolName, bool applyLearning = true);
         bool shouldIgnoreOverride(const Waf2ScanResult &res);
         bool isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp);
         

components/security_apps/waap/waap_clib/Waf2Engine.cc

@@ -329,6 +329,7 @@ Waf2Transaction::Waf2Transaction() :
     is_schema_validation(false),
     m_waf2TransactionFlags()
 {
+    m_overrideOriginalMaxScore[OVERRIDE_ACCEPT] = 0;
     I_TimeGet *timeGet = Singleton::Consume<I_TimeGet>::by<Waf2Transaction>();
     m_entry_time = chrono::duration_cast<chrono::milliseconds>(timeGet->getMonotonicTime());
 }
@@ -1729,6 +1730,11 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
             std::copy(m_effectiveOverrideIds.begin(), m_effectiveOverrideIds.end(), vEffectiveOverrideIds.begin());
             waapLog.addToOrigin(LogField("effectiveExceptionIdList", vEffectiveOverrideIds));
         }
+        if (!m_exceptionLearned.empty()) {
+            std::vector<std::string> vLearningAffected(m_exceptionLearned.size());
+            std::copy(m_exceptionLearned.begin(), m_exceptionLearned.end(), vLearningAffected.begin());
+            waapLog.addToOrigin(LogField("redundantExceptionIdList", vLearningAffected));
+        }
     }
 }
 
@@ -1809,12 +1815,6 @@ Waf2Transaction::sendLog()
         return;
     }
 
-    dbgTrace(D_WAAP) << "force exception: " << m_overrideState.bForceException <<
-        " force block: " << m_overrideState.bForceBlock <<
-        " matched overrides count: " << m_matchedOverrideIds.size() <<
-        " effective overrides count: " << m_effectiveOverrideIds.size();
-
-
     bool shouldBlock = false;
     if (m_overrideState.bForceBlock) {
         // If override forces "reject" decision, mention it in the "override" log field.
@@ -2091,7 +2091,30 @@ Waf2Transaction::decideAutonomousSecurity(
         transactionResult.threatLevel = threat;
     }
 
+    dbgTrace(D_WAAP_OVERRIDE) << "override ids count: " << m_matchedOverrideIds.size();
     // Apply overrides
+    for (auto it = m_overridePostFilterMaxScore.begin(); it != m_overridePostFilterMaxScore.end(); it++) {
+        const string id = it->first;
+        if (m_overrideState.forceBlockIds.find(id) != m_overrideState.forceBlockIds.end()) {
+            // blocked effectivness is calculates later from the force block exception ids list
+            continue;
+        }
+        ThreatLevel threat = Waap::Conversions::convertFinalScoreToThreatLevel(it->second);
+        bool shouldBlock = Waap::Conversions::shouldDoWafBlocking(m_siteConfig, threat);
+        dbgTrace(D_WAAP_OVERRIDE) << "checking effectivness of override: " << id << ", should have blocked: " << shouldBlock
+            << ", scores: " << m_overridePostFilterMaxScore[id] << ", " << m_overrideOriginalMaxScore[id];
+        if (shouldBlock) {
+            m_effectiveOverrideIds.insert(id);
+        } else {
+            ThreatLevel threatNoFilter = Waap::Conversions::convertFinalScoreToThreatLevel(
+                m_overrideOriginalMaxScore[id]
+            );
+            if (Waap::Conversions::shouldDoWafBlocking(m_siteConfig, threatNoFilter)) {
+                m_exceptionLearned.insert(id);
+            }
+        }
+    }
+
     if (m_overrideState.bForceBlock) {
         dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
             " and override forces REJECT ...";
@@ -2105,25 +2128,25 @@ Waf2Transaction::decideAutonomousSecurity(
         }
     }
     else if (m_overrideState.bForceException) {
-        dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
+        dbgTrace(D_WAAP) << "de cideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
             " and override forces ALLOW ...";
-        if (m_scanResult) {
-            // on accept exception the decision is not set and needs to be calculated to determine effectivness
-            ThreatLevel threat = Waap::Conversions::convertFinalScoreToThreatLevel(m_scanResult->score);
-            bool shouldBlock = Waap::Conversions::shouldDoWafBlocking(&sitePolicy, threat);
-            if (shouldBlock) {
-                m_effectiveOverrideIds.insert(
-                    m_overrideState.forceExceptionIds.begin(), m_overrideState.forceExceptionIds.end()
-                );
-            }
-        }
-
         decision->setBlock(false);
         if (!m_overrideState.bIgnoreLog)
         {
             decision->setOverridesLog(true);
         }
+    } else if (!m_matchedOverrideIds.empty()) {
+        if (!m_overrideState.bIgnoreLog)
+        {
+            decision->setOverridesLog(true);
+        }
     }
+    dbgTrace(D_WAAP_OVERRIDE) << "force exception: " << m_overrideState.bForceException <<
+    " force block: " << m_overrideState.bForceBlock <<
+    " matched overrides count: " << m_matchedOverrideIds.size() <<
+    " effective overrides count: " << m_effectiveOverrideIds.size() <<
+    " learned overrides count: " << m_exceptionLearned.size();
+
 
 
     bool log_all = false;
@@ -2262,7 +2285,7 @@ bool
 Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
     auto exceptions = getConfiguration<ParameterException>("rulebase", "exception");
     if (!exceptions.ok()) {
-        dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error:" << exceptions.getErr();
+        dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error: " << exceptions.getErr();
         return false;
     }
     dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions";
@@ -2305,6 +2328,24 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
         auto behaviors = exceptions.unpack().getBehavior(exceptions_dict,
                 getAssetState()->m_filtersMngr->getMatchedOverrideKeywords());
         for (const auto &behavior : behaviors) {
+            if (!res.filtered_keywords.empty() || res.score > 0) {
+                dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " with filtered indicators";
+                std::string overrideId = behavior.getId();
+                if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){
+                    m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
+                    m_overridePostFilterMaxScore[overrideId] = res.score;
+                } else {
+                    if (res.scoreNoFilter > m_overrideOriginalMaxScore[overrideId]) {
+                        m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
+                    }
+                    if (res.score > m_overridePostFilterMaxScore[overrideId]) {
+                        m_overridePostFilterMaxScore[overrideId] = res.score;
+                    }
+                }
+                if (res.scoreNoFilter > m_overrideOriginalMaxScore[OVERRIDE_ACCEPT]) {
+                    m_overrideOriginalMaxScore[OVERRIDE_ACCEPT] = res.scoreNoFilter;
+                }
+            }
             if (behavior == action_ignore)
             {
                 dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " should ignore.";
@@ -2312,12 +2353,6 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
                 if (!overrideId.empty()) {
                     m_matchedOverrideIds.insert(overrideId);
                 }
-                if (!res.keyword_matches.empty() || res.unescaped_line == Waap::Scanner::xmlEntityAttributeId)
-                {
-                    if (!overrideId.empty()) {
-                        m_effectiveOverrideIds.insert(overrideId);
-                    }
-                }
                 return true;
             }
         }

components/security_apps/waap/waap_clib/Waf2Engine.h

@@ -293,6 +293,9 @@ private:
     // Matched override IDs
     std::set<std::string> m_matchedOverrideIds;
     std::set<std::string> m_effectiveOverrideIds;
+    std::set<std::string> m_exceptionLearned;
+    std::map<std::string, double> m_overrideOriginalMaxScore;
+    std::map<std::string, double> m_overridePostFilterMaxScore;
 
     //csrf state
     Waap::CSRF::State m_csrfState;

components/security_apps/waap/waap_clib/Waf2EngineGetters.cc

@@ -459,9 +459,15 @@ Waf2Transaction::getUserLimitVerdict()
     }
     else if (mode == AttackMitigationMode::PREVENT) {
         decision->setLog(true);
-        decision->setBlock(true);
-        dbgInfo(D_WAAP_ULIMITS) << msg << "BLOCK" << reason;
-        verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
+        if (!m_overrideState.bForceException) {
+            decision->setBlock(true);
+            dbgInfo(D_WAAP_ULIMITS) << msg << "BLOCK" << reason;
+            verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
+        } else {
+            decision->setBlock(true);
+            dbgInfo(D_WAAP_ULIMITS) << msg << "Override Accept" << reason;
+            verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
+        }
     }
 
     return verdict;

components/utils/generic_rulebase/evaluators/parameter_eval.cc

@@ -22,6 +22,8 @@
 
 using namespace std;
 
+USE_DEBUG_FLAG(D_RULEBASE_CONFIG);
+
 string ParameterMatcher::ctx_key = "parameters";
 
 ParameterMatcher::ParameterMatcher(const vector<string> &params)
@@ -33,6 +35,17 @@ ParameterMatcher::ParameterMatcher(const vector<string> &params)
 Maybe<bool, Context::Error>
 ParameterMatcher::evalVariable() const
 {
+    I_Environment *env = Singleton::Consume<I_Environment>::by<ParameterMatcher>();
+    auto bc_param_id_ctx = env->get<set<GenericConfigId>>(ParameterMatcher::ctx_key);
+    dbgTrace(D_RULEBASE_CONFIG)
+        << "Trying to match parameter. ID: "
+        << parameter_id << ", Current set IDs: "
+        << makeSeparatedStr(bc_param_id_ctx.ok() ? *bc_param_id_ctx : set<GenericConfigId>(), ", ");
+    if (bc_param_id_ctx.ok()) return bc_param_id_ctx.unpack().count(parameter_id) > 0;
+
+    dbgTrace(D_RULEBASE_CONFIG)
+        << "Did not find current parameter in context."
+        << " Match parameter from current rule";
     auto rule = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
     return rule.ok() && rule.unpack().isParameterActive(parameter_id);
 }

config/crds/open-appsec-crd-v1beta1.yaml

@@ -0,0 +1,525 @@
+Enter file contents hereapiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata :
+  name : customresponses.openappsec.io
+
+spec:
+  group: openappsec.io
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                mode:
+                  type: string
+                  enum:
+                    - block-page
+                    #- redirect
+                    - response-code-only
+                message-title:
+                  type: string
+                message-body:
+                  type: string
+                http-response-code:
+                  type: integer
+                  minimum: 100
+                  maximum: 599
+
+  scope: Cluster
+  names:
+    plural: customresponses
+    singular: customresponse
+    kind: CustomResponse
+    shortNames:
+    - customresponse
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: exceptions.openappsec.io
+
+spec:
+  group: openappsec.io
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: array
+              items:
+                type: object
+                required:
+                  - action
+                properties:
+                  action:
+                    type: string
+                    enum:
+                      - skip
+                      - accept
+                      - drop
+                      - suppressLog
+                  sourceIp:
+                    type: array
+                    items:
+                      type: string
+                  url:
+                    type: array
+                    items:
+                      type: string
+                  sourceIdentifier:
+                    type: array
+                    items:
+                      type: string
+                  protectionName:
+                    type: array
+                    items:
+                      type: string
+                  paramValue:
+                    type: array
+                    items:
+                      type: string
+                  paramName:
+                    type: array
+                    items:
+                      type: string
+                  hostName:
+                    type: array
+                    items:
+                      type: string
+                  countryCode:
+                    type: array
+                    items:
+                      type: string
+                  countryName:
+                    type: array
+                    items:
+                      type: string
+                  comment:
+                    type: string
+
+  scope: Cluster
+  names:
+    plural: exceptions
+    singular: exception
+    kind: Exception
+    shortNames:
+      - exception
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata :
+  name : logtriggers.openappsec.io
+
+spec:
+  group: openappsec.io
+  versions:
+    - name: v1beta1
+      # Each version can be enabled/disabled by Served flag.
+      served: true
+      # One and only one version must be marked as the storage version.
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                access-control-logging:
+                  type: object
+                  properties:
+                    allow-events:
+                      type: boolean
+                    drop-events:
+                      type: boolean
+                appsec-logging:
+                  type: object
+                  properties:
+                    detect-events:
+                      type: boolean
+                    prevent-events:
+                      type: boolean
+                    all-web-requests:
+                      type: boolean
+                additional-suspicious-events-logging:
+                  type: object
+                  properties:
+                    enabled:
+                      type: boolean
+                    minimum-severity:
+                      type: string
+                      enum:
+                        - high
+                        - critical
+                    response-body:
+                      type: boolean
+                    response-code:
+                      type: boolean
+                extended-logging:
+                  type: object
+                  properties:
+                    url-path:
+                      type: boolean
+                    url-query:
+                      type: boolean
+                    http-headers:
+                      type: boolean
+                    request-body:
+                      type: boolean
+                log-destination:
+                  type: object
+                  properties:
+                    cloud:
+                      type: boolean
+                    syslog-service: #change to object array
+                      type: array
+                      items:
+                        type: object
+                        properties:
+                          address:
+                            type: string
+                          port:
+                            type: integer
+                    file:
+                      type: string
+                    stdout:
+                      type: object
+                      properties:
+                        format:
+                          type: string
+                          enum:
+                            - json
+                            - json-formatted
+                    cef-service:
+                      type: array
+                      items:
+                        type: object
+                        properties:
+                          address:
+                            type: string
+                          port:
+                            type: integer
+                          proto:
+                            type: string
+                            enum:
+                              - tcp
+                              - udp
+
+  scope: Cluster
+  names:
+    plural: logtriggers
+    singular: logtrigger
+    kind: LogTrigger
+    shortNames:
+    - logtrigger
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata :
+  name : policies.openappsec.io
+
+spec:
+  group: openappsec.io
+  versions:
+    - name: v1beta1
+      # Each version can be enabled/disabled by Served flag.
+      served: true
+      # One and only one version must be marked as the storage version.
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                default:
+                  type: object
+                  properties:
+                    mode:
+                      type: string
+                      enum:
+                        - prevent-learn
+                        - detect-learn
+                        - prevent
+                        - detect
+                        - inactive
+                    practices:
+                      type: array
+                      items:
+                        type: string
+                    triggers:
+                      type: array
+                      items:
+                        type: string
+                    custom-response:
+                      type: string
+                    source-identifiers:
+                      type: string
+                    trusted-sources:
+                      type: string
+                    exceptions:
+                      type: array
+                      items:
+                        type: string
+                specific-rules:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      host:
+                        type: string
+                      mode:
+                        type: string
+                        enum:
+                          - prevent-learn
+                          - detect-learn
+                          - prevent
+                          - detect
+                          - inactive
+                      practices:
+                        type: array
+                        items:
+                          type: string
+                      triggers:
+                        type: array
+                        items:
+                          type: string
+                      custom-response:
+                        type: string
+                      source-identifiers:
+                        type: string
+                      trusted-sources:
+                        type: string
+                      exceptions:
+                        type: array
+                        items:
+                          type: string
+
+  scope: Cluster
+  names:
+    plural: policies
+    singular: policy
+    kind: Policy
+    shortNames:
+    - policy
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata :
+  name : practices.openappsec.io
+
+spec:
+  group: openappsec.io
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                web-attacks:
+                  type: object
+                  properties:
+                    override-mode:
+                      type: string
+                      enum:
+                        - prevent-learn
+                        - detect-learn
+                        - prevent
+                        - detect
+                        - inactive
+                    minimum-confidence:
+                      type: string
+                      enum:
+                        - medium
+                        - high
+                        - critical
+                    max-url-size-bytes:
+                      type: integer
+                    max-object-depth:
+                      type: integer
+                    max-body-size-kb:
+                      type: integer
+                    max-header-size-bytes:
+                      type: integer
+                    protections:
+                      type: object
+                      properties:
+                        csrf-enabled:
+                          type: string
+                          enum:
+                            - prevent-learn
+                            - detect-learn
+                            - prevent
+                            - detect
+                            - inactive
+                        error-disclosure-enabled:
+                          type: string
+                          enum:
+                            - prevent-learn
+                            - detect-learn
+                            - prevent
+                            - detect
+                            - inactive
+                        open-redirect-enabled:
+                          type: string
+                          enum:
+                            - prevent-learn
+                            - detect-learn
+                            - prevent
+                            - detect
+                            - inactive
+                        non-valid-http-methods:
+                          type: boolean
+                anti-bot:
+                  type: object
+                  properties:
+                    override-mode:
+                      type: string
+                      enum:
+                        - prevent-learn
+                        - detect-learn
+                        - prevent
+                        - detect
+                        - inactive
+                    injected-URIs:
+                      type: array
+                      items:
+                        type: object
+                        properties:
+                          uri:
+                            type: string
+                    validated-URIs:
+                      type: array
+                      items:
+                        type: object
+                        properties:
+                          uri:
+                            type: string
+                snort-signatures:
+                  type: object
+                  properties:
+                    override-mode:
+                      type: string
+                      enum:
+                        - prevent-learn
+                        - detect-learn
+                        - prevent
+                        - detect
+                        - inactive
+                    configmap:
+                      type: array
+                      items:
+                        type: string
+                openapi-schema-validation:
+                  type: object
+                  properties:
+                    override-mode:
+                      type: string
+                      enum:
+                        - prevent-learn
+                        - detect-learn
+                        - prevent
+                        - detect
+                        - inactive
+                    configmap:
+                      type: array
+                      items:
+                        type: string
+
+  scope: Cluster
+  names:
+    plural: practices
+    singular: practice
+    kind: Practice
+    shortNames:
+    - practice
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata :
+  name : sourcesidentifiers.openappsec.io
+
+spec:
+  group: openappsec.io
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: array
+              items:
+                type: object
+                properties:
+                  sourceIdentifier:
+                    type: string
+                    enum:
+                      - headerkey
+                      - JWTKey
+                      - cookie
+                      - sourceip
+                      - x-forwarded-for
+                  value:
+                    type: array
+                    items:
+                      type: string
+
+  scope: Cluster
+  names:
+    plural: sourcesidentifiers
+    singular: sourcesidentifier
+    kind: SourcesIdentifier
+    shortNames:
+    - sourcesidentifier
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata :
+  name : trustedsources.openappsec.io
+
+spec:
+  group: openappsec.io
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                minNumOfSources:
+                  type: integer
+                sourcesIdentifiers:
+                  type: array
+                  items:
+                    type: string
+
+  scope: Cluster
+  names:
+    plural: trustedsources
+    singular: trustedsource
+    kind: TrustedSource
+    shortNames:
+    - trustedsource

core/agent_details_reporter/agent_details_reporter.cc

@@ -71,6 +71,7 @@ public:
     bool addAttr(const string &key, const string &val, bool allow_override = false) override;
     bool addAttr(const map<string, string> &attr, bool allow_override = false) override;
     void deleteAttr(const string &key) override;
+    bool isPersistantAttr(const string &key) override;
 
     bool sendAttributes() override;
 
@@ -130,6 +131,7 @@ private:
     map<string, string> persistant_attributes;
     map<string, string> new_attributes;
     map<string, string> attributes;
+    bool is_attr_deleted = false;
 
     I_Messaging *messaging = nullptr;
     bool is_server;
@@ -207,6 +209,13 @@ AgentDetailsReporter::Impl::deleteAttr(const string &key)
     attributes.erase(key);
     new_attributes.erase(key);
     persistant_attributes.erase(key);
+    is_attr_deleted = true;
+}
+
+bool
+AgentDetailsReporter::Impl::isPersistantAttr(const std::string &key)
+{
+    return persistant_attributes.count(key) > 0;
 }
 
 bool
@@ -214,7 +223,7 @@ AgentDetailsReporter::Impl::sendAttributes()
 {
     dbgDebug(D_AGENT_DETAILS) << "Trying to send attributes";
 
-    if (new_attributes.empty()) {
+    if (new_attributes.empty() && !is_attr_deleted) {
         dbgDebug(D_AGENT_DETAILS) << "Skipping current attempt since no new attributes were added";
         return true;
     }
@@ -261,6 +270,7 @@ AgentDetailsReporter::Impl::sendAttributes()
         if (add_agent_details_status.ok()) {
             dbgDebug(D_AGENT_DETAILS) << "Successfully sent attributes to the Orchestrator";
             new_attributes.clear();
+            is_attr_deleted = false;
             return true;
         }
 

core/agent_details_reporter/agent_details_reporter_ut/agent_details_reporter_ut.cc

@@ -213,6 +213,7 @@ TEST_F(AgentReporterTest, basicAttrTest)
     EXPECT_TRUE(report->addAttr({{"c", "d"}, {"1", "2"}, {"delete", "me"}}));
     EXPECT_FALSE(report->addAttr("a", "d"));
     EXPECT_TRUE(report->addAttr("a", "1", true));
+    EXPECT_TRUE(report->isPersistantAttr("a"));
     report->deleteAttr("delete");
     {
         AgentDataReport agent_data;

core/attachments/http_configuration/http_configuration.cc

@@ -108,7 +108,10 @@ HttpAttachmentConfiguration::save(cereal::JSONOutputArchive &archive) const
         ),
         cereal::make_nvp("nginx_inspection_mode", getNumericalValue("inspection_mode")),
         cereal::make_nvp("num_of_nginx_ipc_elements", getNumericalValue("num_of_nginx_ipc_elements")),
-        cereal::make_nvp("keep_alive_interval_msec", getNumericalValue("keep_alive_interval_msec"))
+        cereal::make_nvp("keep_alive_interval_msec", getNumericalValue("keep_alive_interval_msec")),
+        cereal::make_nvp("min_retries_for_verdict", getNumericalValue("min_retries_for_verdict")),
+        cereal::make_nvp("max_retries_for_verdict", getNumericalValue("max_retries_for_verdict")),
+        cereal::make_nvp("body_size_trigger", getNumericalValue("body_size_trigger"))
     );
 }
 
@@ -161,6 +164,9 @@ HttpAttachmentConfiguration::load(cereal::JSONInputArchive &archive)
     loadNumericalValue(archive, "nginx_inspection_mode", 0);
     loadNumericalValue(archive, "num_of_nginx_ipc_elements", 200);
     loadNumericalValue(archive, "keep_alive_interval_msec", DEFAULT_KEEP_ALIVE_INTERVAL_MSEC);
+    loadNumericalValue(archive, "min_retries_for_verdict", 3);
+    loadNumericalValue(archive, "max_retries_for_verdict", 15);
+    loadNumericalValue(archive, "body_size_trigger", 200000);
 }
 
 bool

core/include/attachments/nginx_attachment_util.h

@@ -54,6 +54,9 @@ unsigned int getReqBodyThreadTimeout();
 unsigned int getResProccessingTimeout();
 unsigned int getResHeaderThreadTimeout();
 unsigned int getResBodyThreadTimeout();
+unsigned int getMinRetriesForVerdict();
+unsigned int getMaxRetriesForVerdict();
+unsigned int getReqBodySizeTrigger();
 
 unsigned int getWaitingForVerdictThreadTimeout();
 

core/include/services_sdk/interfaces/i_agent_details_reporter.h

@@ -48,6 +48,7 @@ public:
     virtual bool addAttr(const std::map<std::string, std::string> &attr, bool allow_override = false) = 0;
     virtual void deleteAttr(const std::string &key) = 0;
     virtual bool sendAttributes() = 0;
+    virtual bool isPersistantAttr(const std::string &key) = 0;
 
 protected:
     ~I_AgentDetailsReporter() = default;

core/include/services_sdk/interfaces/mock/mock_agent_details_reporter.h

@@ -26,6 +26,7 @@ public:
     MOCK_METHOD3(addAttr, bool(const std::string &key, const std::string &val, bool allow_override));
     MOCK_METHOD2(addAttr, bool(const std::map<std::string, std::string> &attr, bool allow_override));
     MOCK_METHOD1(deleteAttr, void(const std::string &key));
+    MOCK_METHOD1(isPersistantAttr, bool(const std::string &key));
     MOCK_METHOD0(sendAttributes, bool());
 };
 

core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc

@@ -109,7 +109,7 @@ TEST(IntelligenceQueryTestV2, genJsonPrettyBulkRequests) {
     Intelligence::IntelligenceRequest query(requests, true, true, false, MessageMetadata("", 0));
 
     std::string expected = "{\n"
-        "    \"queryTypes\": {\n"
+        "    \"queriesTypes\": {\n"
         "        \"proxyToCloud\": false\n"
         "    },\n"
         "    \"queries\": [\n"
@@ -150,7 +150,7 @@ TEST(IntelligenceQueryTestV2, genJsonPrettyBulkRequestsProxied) {
     Intelligence::IntelligenceRequest query(requests, true, true, true, MessageMetadata("", 0));
 
     std::string expected = "{\n"
-        "    \"queryTypes\": {\n"
+        "    \"queriesTypes\": {\n"
         "        \"proxyToCloud\": true\n"
         "    },\n"
         "    \"queries\": [\n"
@@ -191,7 +191,7 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettyBulkRequest) {
     Intelligence::IntelligenceRequest query(requests, false, true, false, MessageMetadata("", 0));
 
     std::string expected = "{"
-            "\"queryTypes\":{"
+            "\"queriesTypes\":{"
             "\"proxyToCloud\":false"
             "},"
             "\"queries\":[{"

core/intelligence_is_v2/intelligence_request.cc

@@ -90,7 +90,7 @@ IntelligenceRequest::genJson() const
     {
         cereal::JSONOutputArchive out_ar(json_stream);
 
-        out_ar.setNextName("queryTypes");
+        out_ar.setNextName(isBulk() ? "queriesTypes" : "queryTypes");
         out_ar.startNode();
         out_ar(cereal::make_nvp("proxyToCloud", is_proxy));
         out_ar.finishNode();

nodes/orchestration/package/k8s-check-update-listener.sh

@@ -69,7 +69,7 @@ while true; do
         exception_pid=$!
         saveRuningPids
     fi
-        if [ ! -d /proc/${exception_pid} ]; then
+    if [ ! -d /proc/${policy_pid} ]; then
         runGetResourceListener policies
         policy_pid=$!
         saveRuningPids

nodes/orchestration/package/open-appsec-cloud-mgmt

@@ -3,6 +3,7 @@
 POLICY_TEMP_PATH="/tmp/policy_temp.json"
 DECLARATIVE_CONFIG_PATH="/etc/cp/conf/declarative_config.cfg"
 CHANGE_AGENT_MODE=true
+UPLOAD_AGENT_POLICY=false
 ra_token=
 tenant_id=
 agent_id=
@@ -10,15 +11,16 @@ profile_id=
 
 load_agent_details()
 {
-    tenant_id=$(cat /etc/cp/conf/agent_details.json | sed "s|Tenant ID|TenantID|g" | /etc/cp/bin/yq -P '.TenantID')
-    profile_id=$(cat /etc/cp/conf/agent_details.json | sed "s|Profile ID|ProfileID|g" | /etc/cp/bin/yq -P '.ProfileID')
+    tenant_id=$(awk -F\" '/Tenant ID/{print $4}' /etc/cp/conf/agent_details.json)
+    profile_id=$(awk -F\" '/Profile ID/{print $4}' /etc/cp/conf/agent_details.json)
+    agent_id=$(awk -F\" '/Agent ID/{print $4}' /etc/cp/conf/agent_details.json)
 }
 
 generate_policy()
 {
     cp -f /etc/cp/conf/local_policy.yaml /tmp/tmp_local_policy.yaml
     sed -i "s|\"\*\"|\"Any\"|g" /tmp/tmp_local_policy.yaml
-    POLICY=$(/etc/cp/bin/yq /tmp/tmp_local_policy.yaml -o json)
+    POLICY=$(/etc/cp/bin/yq eval /tmp/tmp_local_policy.yaml -o json)
     echo $POLICY > $POLICY_TEMP_PATH
     rm -f /tmp/tmp_local_policy.yaml
 }
@@ -193,6 +195,7 @@ upload_policy_to_the_cloud()
         fi
     fi
 
+
     upload_the_policy_to_s3
     if [ "$?" = "1" ]; then
         echo "Failed uploading the policy to S3"
@@ -228,7 +231,8 @@ usage()
     echo "Options:"
     echo "    --namespace <namespace> : Namespace with the relevant Helm Chart"
     echo "    --fog <fog address>     : Namespace with the relevant Helm Chart"
-    echo "    --upload_policy_only    : Upload policy to the fog, withput changing agent mode"
+    echo "    --config-upload         : Upload policy to the fog"
+    echo "    --config-upload-only    : Upload policy to the fog, withput changing agent mode"
     exit 255
 }
 
@@ -254,7 +258,10 @@ while true; do
         validate_arg_value_exists "$1" "$#"
         shift
         var_fog="$1"
-    elif [ "$1" = "--upload_policy_only" ]; then
+    elif [ "$1" = "--config-upload" ]; then
+        UPLOAD_AGENT_POLICY=true
+    elif [ "$1" = "--config-upload-only" ]; then
+        UPLOAD_AGENT_POLICY=true
         CHANGE_AGENT_MODE=false
         source $DECLARATIVE_CONFIG_PATH
     elif [ "$1" = "--access_token" ] || [ "$1" = "-at" ]; then
@@ -271,6 +278,9 @@ while true; do
         profile_id="$1"
     elif [ -z "$1" ]; then
         break
+    else
+        usage
+        exit 1
     fi
     shift
 done
@@ -279,7 +289,11 @@ if [ -z "$var_fog" ]; then
     var_fog="https://inext-agents.cloud.ngen.checkpoint.com"
 fi
 
-upload_policy_to_the_cloud
+if [ $UPLOAD_AGENT_POLICY = true ]; then
+    upload_policy_to_the_cloud
+else
+    open-appsec-ctl --set-mode --online_mode --token $var_token --fog $var_fog
+fi
 if [ "$?" = "0" ]; then
     echo "SUCCESS"
 fi

nodes/orchestration/package/open-appsec-cloud-mgmt-k8s

@@ -15,11 +15,13 @@ latest_policy_version=1
 
 load_agent_details()
 {
-    tenant_id=$(cat /etc/cp/conf/agent_details.json | sed "s|Tenant ID|TenantID|g" | /etc/cp/bin/yq -P '.TenantID')
-    agent_id=$(cat /etc/cp/conf/agent_details.json | sed "s|Agent ID|AgentID|g" | /etc/cp/bin/yq -P '.AgentID')
-    profile_id=$(cat /etc/cp/conf/agent_details.json | sed "s|Profile ID|ProfileID|g" | /etc/cp/bin/yq -P '.ProfileID')
+    tenant_id=$(awk -F\" '/Tenant ID/{print $4}' /etc/cp/conf/agent_details.json)
+    agent_id=$(awk -F\" '/Agent ID/{print $4}' /etc/cp/conf/agent_details.json)
+    profile_id=$(awk -F\" '/Profile ID/{print $4}' /etc/cp/conf/agent_details.json)
     cluster_id=$(echo $(curl -s --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" -X GET ${APISERVER}/api/v1/namespaces/ ) \
-        | /etc/cp/bin/yq .items | /etc/cp/bin/yq  '.[] | select(.metadata.name  | contains("kube-system"))' | /etc/cp/bin/yq .metadata.uid)
+        | /etc/cp/bin/yq eval '.items' - \
+        | /etc/cp/bin/yq eval '.[] | select(.metadata.name | contains("kube-system"))' - \
+        | /etc/cp/bin/yq eval '.metadata.uid' -)
 }
 
 get_latest_policy_version()
@@ -44,15 +46,16 @@ get_latest_policy_version()
 
 concat_to_policy()
 {
-    crd_to_concat="$1"
-    is_first=$2
+    api_version="$1"
+    crd_to_concat="$2"
+    is_first=$3
     if [ ! -z $is_first ]; then
-        POLICY="$POLICY \"$1\": "
+        POLICY="$POLICY \"$crd_to_concat\": "
     else
-        POLICY="$POLICY, \"$1\": "
+        POLICY="$POLICY, \"$crd_to_concat\": "
     fi
     CRD=$(curl -s --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" \
-        -X GET ${APISERVER}/apis/openappsec.io/v1beta1/$crd_to_concat)
+        -X GET ${APISERVER}/apis/openappsec.io/$api_version/$crd_to_concat)
     CRD=$(echo $CRD|tr -d '\n')
     if [ -z "$CRD" ]; then
         CRD="{}"
@@ -60,28 +63,49 @@ concat_to_policy()
     POLICY="$POLICY $CRD"
 }
 
+get_api_version()
+{
+    CRD=$(curl -s --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" \
+        -X GET ${APISERVER}/apis/openappsec.io/v1beta2/policies)
+    CRD=$(echo $CRD|tr -d '\n')
+    # if CRD is not empty and does not contain "page not found" then it is v1beta2
+    if [ ! -z "$CRD" ] && ! echo "$CRD" | grep -q "page not found"; then
+        echo "v1beta2"
+    else
+        echo "v1beta1"
+    fi
+}
+
 generate_policy()
 {
     POLICY="{ \"Policy\": {"
-    concat_to_policy policies true
-    concat_to_policy practices
-    concat_to_policy logtriggers
-    concat_to_policy customresponses
-    concat_to_policy exceptions
-    concat_to_policy sourcesidentifiers
-    concat_to_policy trustedsources
+
+    api_version=$(get_api_version)
+
+    concat_to_policy $api_version "policies" true
+    if [ "$api_version" = "v1beta2" ]; then
+        concat_to_policy $api_version "threatpreventionpractices"
+        concat_to_policy $api_version "accesscontrolpractices"
+    else
+        concat_to_policy $api_version "practices"
+    fi
+    concat_to_policy $api_version "logtriggers"
+    concat_to_policy $api_version "customresponses"
+    concat_to_policy $api_version "exceptions"
+    concat_to_policy $api_version "sourcesidentifiers"
+    concat_to_policy $api_version "trustedsources"
 
     POLICY="$POLICY, \"assets\": { \"items\":[ "
 
     FIRST="1"
     all_ingresses=$(curl -s --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" \
         -X GET ${APISERVER}/apis/networking.k8s.io/v1/ingresses)
-    namespaces=$(echo $all_ingresses | /etc/cp/bin/yq -P '.items[].metadata.namespace')
+    namespaces=$(echo $all_ingresses | /etc/cp/bin/yq eval '.items[].metadata.namespace' -)
 
     for ns in ${namespaces}; do
         ingress_in_ns=$(curl -s --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" \
             -X GET ${APISERVER}/apis/networking.k8s.io/v1/namespaces/${ns}/ingresses)
-        ingress_list=$(echo $ingress_in_ns | /etc/cp/bin/yq -P '.items[].metadata.name')
+        ingress_list=$(echo $ingress_in_ns | /etc/cp/bin/yq eval '.items[].metadata.name' -)
         for ingress_name in ${ingress_list}; do
             ingress_crd=$(curl -s --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" \
                 -X GET ${APISERVER}/apis/networking.k8s.io/v1/namespaces/${ns}/ingresses/${ingress_name})
@@ -273,7 +297,7 @@ while true; do
 done
 
 if [ -z "$var_fog" ]; then
-    var_fog=$(cat /etc/cp/conf/agent_details.json | sed "s|Fog domain|Fogdomain|g" | /etc/cp/bin/yq -P '.Fogdomain')
+    var_fog=$(awk -F\" '/Fog domain/{print $4}' /etc/cp/conf/agent_details.json)
     var_fog="https://$var_fog"
 fi
 

nodes/orchestration/package/open-appsec-ctl.sh

@@ -1363,9 +1363,12 @@ run_ai() # Initials - ra
         exit 1
     fi
     if [ "$ra_upload_to_fog" = "true" ]; then
-        ra_token_data=$(curl_to_orchestration "show-access-token")
-        ra_token_hex=$(echo "$ra_token_data" | grep "token" | cut -d '"' -f4 | base64 -d | od -t x1 -An)
-        ra_token_hex_formatted=$(echo $ra_token_hex | tr -d ' ')
+        ra_token_data=$(curl_to_orchestration "show-access-token" | grep "token" | cut -d '"' -f4)
+        if [ -z "${ra_token_data}" ]; then
+            echo "Failed to get crediantials to upload the file to the cloud."
+            exit 1;
+        fi
+        ra_token_hex_formatted=$(echo $ra_token_data | base64 -d | od -t x1 -An | tr -d '[:space:]')
         ra_token="$(xor_decrypt "${ra_token_hex_formatted}")"
 
         ra_proxy_val=""

nodes/orchestration/package/orchestration_package.sh

@@ -302,13 +302,15 @@ while true; do
         echo "Filesystem paths: ${FILESYSTEM_PATH}"
     elif [ "$1" = "--vs_id" ]; then
         shift
-        VS_ID=$1
-        export FILESYSTEM_PATH="/etc/cp/vs${VS_ID}"
-        NANO_AGENT_SERVICE_NAME="nano_agent_${VS_ID}"
-        NANO_AGENT_SERVICE_FILE="${NANO_AGENT_SERVICE_NAME}.service"
-        VS_LIB_SUB_FOLDER="/vs${VS_ID}"
-        LOG_FILE_PATH="${LOG_FILE_PATH}/vs${VS_ID}"
-        TMP_FOLDER="${TMP_FOLDER}/vs${VS_ID}"
+        if [ "$1" != "0" ]; then
+            VS_ID=$1
+            export FILESYSTEM_PATH="/etc/cp/vs${VS_ID}"
+            NANO_AGENT_SERVICE_NAME="nano_agent_${VS_ID}"
+            NANO_AGENT_SERVICE_FILE="${NANO_AGENT_SERVICE_NAME}.service"
+            VS_LIB_SUB_FOLDER="/vs${VS_ID}"
+            LOG_FILE_PATH="${LOG_FILE_PATH}/vs${VS_ID}"
+            TMP_FOLDER="${TMP_FOLDER}/vs${VS_ID}"
+        fi
     elif [ "$1" = "--log_files_path" ]; then
         shift
         var=$1
@@ -360,6 +362,16 @@ if [ -z "$VS_ID" ]; then
     fi
 fi
 
+if [ -n "${VS_ID}" ]; then
+    if [ "$VS_ID" != "$INSTANCE_VSID" ]; then
+        echo "Error: Incorrect context, switch to VS${VS_ID} context first."
+        exit 1
+    fi
+elif [ -n "$INSTANCE_VSID" ] && [ "$INSTANCE_VSID" != "0" ]; then
+    echo "Error: Incorrect context, exit vs${INSTANCE_VSID} first."
+    exit 1
+fi
+
 if [ "$RUN_MODE" = "install" ] && [ $var_offline_mode = false ]; then
     if [ -n "$OTP_TOKEN" ] && [ -z "$var_token" ] && [ "$var_no_otp" = "false" ]; then
         var_token=$OTP_TOKEN
@@ -846,7 +858,7 @@ install_public_key()
 
     fog_address=${var_fog_address}
     if [ -n "${var_upgrade_mode}" ]; then
-        # Upgradde - look in policy.json
+        # Upgrade - look in policy.json
         fog_address=$(cat ${FILESYSTEM_PATH}/${CONF_PATH}/${SERVICE_PATH}/orchestration.policy)
     fi