First release of open-appsec source code

components/security_apps/waap/waap_clib/FpMitigation.cc

@@ -0,0 +1,226 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "FpMitigation.h"
+#include <memory>
+#include <algorithm>
+#include <string.h>
+
+
+#define DEFAULT_SCORE 10.0
+#define TRUE_POSITIVE_REPUTATION_THRESHOLD 1.5
+#define FALSE_POSITIVE_REPUTATION_THRESHOLD 5
+
+USE_DEBUG_FLAG(D_WAAP);
+
+using namespace std::chrono;
+
+FpMitigationScore::FpMitigationScore(const std::string& backupFilePath) :
+    SerializeToFilePeriodically(duration_cast<seconds>(minutes(10)), backupFilePath),
+    m_policyDataUrl(),
+    m_policyDataParam(),
+    m_history(),
+    m_counter(0)
+{
+    dbgTrace(D_WAAP) << "False positive mitigation constructor";
+    restore();
+}
+
+FpMitigationScore::~FpMitigationScore()
+{
+    reset();
+}
+
+void FpMitigationScore::reset() {
+    m_policyDataParam.clear();
+
+    m_policyDataUrl.clear();
+
+    m_history.clear();
+}
+
+void FpMitigationScore::serialize(std::ostream& stream) {
+    cereal::JSONOutputArchive archive(stream);
+
+    archive(cereal::make_nvp("version", 1),
+        cereal::make_nvp("policyDataUrl", m_policyDataUrl),
+        cereal::make_nvp("policyDataParam", m_policyDataParam));
+}
+
+void FpMitigationScore::deserialize(std::istream& stream) {
+    cereal::JSONInputArchive archive(stream);
+
+    size_t version = 0;
+
+    try
+    {
+        archive(cereal::make_nvp("version", version));
+    }
+    catch (std::runtime_error & e) {
+        archive.setNextName(nullptr);
+        version = 0;
+        dbgDebug(D_WAAP) << "Can't load file version: " << e.what();
+    }
+
+    switch (version)
+    {
+    case 0:
+        archive(cereal::make_nvp("m_policyDataUrl", m_policyDataUrl),
+            cereal::make_nvp("m_policyDataParam", m_policyDataParam));
+        break;
+    case 1:
+        archive(cereal::make_nvp("policyDataUrl", m_policyDataUrl),
+            cereal::make_nvp("policyDataParam", m_policyDataParam));
+        break;
+    default:
+        dbgWarning(D_WAAP) << "unknown file format version: " << version;
+        break;
+    }
+}
+
+
+double FpMitigationScore::calculateFpMitigationScore(const std::string& shortUri,
+    const std::string& canonisedParam)
+{
+    double urlScore = DEFAULT_SCORE, paramScore = DEFAULT_SCORE;
+
+    if (m_policyDataUrl.find(shortUri) != m_policyDataUrl.end())
+    {
+        urlScore = m_policyDataUrl[shortUri]->getScore();
+    }
+
+    if (m_policyDataParam.find(canonisedParam) != m_policyDataParam.end())
+    {
+        paramScore = m_policyDataParam[canonisedParam]->getScore();
+    }
+
+    return ((int)(paramScore * 2) / 3 + 3.3) * ((int)(urlScore * 2) / 3 + 3.3) / 10;
+}
+
+template<typename T>
+bool hasElement(std::vector<T> vec, T& elem) {
+    return (std::find(vec.begin(), vec.end(), elem) != vec.end());
+}
+
+void FpMitigationScore::learnFalsePositive(
+    const std::vector<std::string>& keywordMatches,
+    PolicyCounterType rep,
+    const std::string& shortUri,
+    const std::string& canonisedParam)
+{
+    static std::string probing = "probing";
+
+    if (keywordMatches.size() > 3 && hasElement(keywordMatches, probing))
+    {
+        return;
+    }
+    if (rep != UNKNOWN_TYPE)
+    {
+        if (m_policyDataUrl.find(shortUri) == m_policyDataUrl.end())
+        {
+            m_policyDataUrl[shortUri] = std::make_shared<PolicyDataCounter>();
+        }
+        if (m_policyDataParam.find(canonisedParam) == m_policyDataParam.end())
+        {
+            m_policyDataParam[canonisedParam] = std::make_shared<PolicyDataCounter>();
+        }
+
+        incrementCounter(shortUri, canonisedParam, rep);
+        m_counter++;
+
+        if (m_counter % FP_SCORE_CALCULATION_INTERVALS == 0)
+        {
+            dbgTrace(D_WAAP) << "evaluating fp mitigation scores";
+            evaluatePolicyDataCounterScore();
+        }
+    }
+
+}
+
+
+PolicyCounterType FpMitigationScore::IdentifyFalseTruePositive(double relativeReputation,
+    const std::string& shortUri, const std::string& canonisedParam, const std::string& userAgentIp)
+{
+    std::string uriParamCat = shortUri + canonisedParam;
+    if (relativeReputation < TRUE_POSITIVE_REPUTATION_THRESHOLD && m_history.find(uriParamCat) == m_history.end())
+    {
+        m_history.insert(uriParamCat);
+        return TRUE_POSITIVE;
+    }
+    if (relativeReputation > FALSE_POSITIVE_REPUTATION_THRESHOLD && m_history.find(userAgentIp) == m_history.end())
+    {
+        m_history.insert(userAgentIp);
+        return FALSE_POSITIVE;
+    }
+
+    return UNKNOWN_TYPE;
+}
+
+void FpMitigationScore::incrementCounter(const std::string& shortUri,
+    const std::string& canonisedParam,
+    PolicyCounterType counterType)
+{
+    // It is assumed that m_policyDataUrl contains shortUrl and
+    // m_policyDataParam contains canonisedParam. See caller.
+    std::shared_ptr<PolicyDataCounter> urlCounter = m_policyDataUrl[shortUri];
+    std::shared_ptr<PolicyDataCounter> paramCounter = m_policyDataParam[canonisedParam];
+
+    urlCounter->incrementCounter(counterType);
+    paramCounter->incrementCounter(counterType);
+}
+
+void FpMitigationScore::evaluatePolicyDataCounterScore()
+{
+    for (auto urlPolicy : m_policyDataUrl) {
+        urlPolicy.second->evaluateScore();
+    }
+
+    for (auto paramPolicy : m_policyDataParam) {
+        paramPolicy.second->evaluateScore();
+    }
+}
+
+PolicyDataCounter::PolicyDataCounter() : falsePositive(0), truePositive(0), score(10.0)
+{
+}
+
+double PolicyDataCounter::getScore()
+{
+    return score;
+}
+
+void PolicyDataCounter::incrementCounter(PolicyCounterType counterType)
+{
+    switch (counterType)
+    {
+    case UNKNOWN_TYPE:
+        // add assert
+        break;
+    case FALSE_POSITIVE:
+    case HTML_CONTENT:
+        falsePositive++;
+        break;
+    case TRUE_POSITIVE:
+    case SPAM:
+        truePositive++;
+        break;
+    default:
+        break;
+    }
+}
+
+void PolicyDataCounter::evaluateScore()
+{
+    size_t tp = truePositive + 50 + 1, fp = falsePositive;
+    score = (double)(10.0 * tp) / (10.0 * fp + tp);
+}