First release of open-appsec source code

core/cpu/CMakeLists.txt

@@ -0,0 +1,3 @@
+add_library(cpu cpu.cc)
+
+add_subdirectory(cpu_ut)

core/cpu/cpu.cc

@@ -0,0 +1,330 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cpu.h"
+
+#include <sys/resource.h>
+#include <fstream>
+#include <sstream>
+
+#include "debug.h"
+#include "log_generator.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_MONITORING);
+
+static const int micro_seconds_in_second = 1000000;
+
+CPUCalculator::CPUCalculator() : Component("CPUCalculator")
+{
+    last_cpu_process_time = chrono::microseconds(0);
+    last_cpu_general_time = chrono::microseconds(0);
+    last_cpu_general_time_active = 0;
+    i_time_get = nullptr;
+}
+
+void CPUCalculator::init() { i_time_get = Singleton::Consume<I_TimeGet>::by<CPUCalculator>(); }
+
+void CPUCalculator::fini() { i_time_get = nullptr; }
+
+// LCOV_EXCL_START Reason: Compilation server dependency
+double
+CPUCalculator::GetGeneralCPUActiveTime(const cpu_data_array &cpu_data)
+{
+    double current_time_active =
+        cpu_data[CPUCalculator::CPUGeneralDataEntryType::USER] +
+        cpu_data[CPUCalculator::CPUGeneralDataEntryType::NICE] +
+        cpu_data[CPUCalculator::CPUGeneralDataEntryType::SYS] +
+        cpu_data[CPUCalculator::CPUGeneralDataEntryType::IRQ] +
+        cpu_data[CPUCalculator::CPUGeneralDataEntryType::SOFTIRQ] +
+        cpu_data[CPUCalculator::CPUGeneralDataEntryType::STEAL] +
+        cpu_data[CPUCalculator::CPUGeneralDataEntryType::GUEST] +
+        cpu_data[CPUCalculator::CPUGeneralDataEntryType::GUEST_NICE];
+
+    return (current_time_active - last_cpu_general_time_active);
+}
+
+Maybe<cpu_data_array>
+CPUCalculator::getGeneralCPUData()
+{
+    static const string cpu_data_file = "/proc/stat";
+    ifstream fileStat(cpu_data_file);
+    string line;
+
+    static const int max_lines_nead_to_read = 9;
+    int lines_count = 0;
+    while (lines_count < max_lines_nead_to_read && getline(fileStat, line))
+    {
+        lines_count++;
+        static const string cpu_str = "cpu";
+        if (line.compare(0, cpu_str.size(), cpu_str)) continue;
+        istringstream iss(line);
+        string ignore;
+        iss >> ignore;
+        cpu_data_array tmp_cpu_data;
+        for (CPUGeneralDataEntryType cpu_type : NGEN::Range<CPUGeneralDataEntryType>() ) {
+            string entry;
+            iss >> entry;
+            tmp_cpu_data[cpu_type] = atof(entry.c_str());
+        }
+        return tmp_cpu_data;
+    }
+
+    return genError("Could not fill general cpu data array.");
+}
+
+Maybe<double>
+CPUCalculator::getCurrentGeneralCPUUsage()
+{
+    Maybe<cpu_data_array> current_cpu_data = getGeneralCPUData();
+
+    if (!current_cpu_data.ok()) return genError(current_cpu_data.getErr());
+
+    if (last_cpu_general_time == chrono::microseconds(0)) {
+        last_cpu_general_time = i_time_get->getMonotonicTime();
+        last_cpu_general_time_active = GetGeneralCPUActiveTime(current_cpu_data.unpack());
+        return 0;
+    }
+
+    auto current_time = i_time_get->getMonotonicTime();
+    auto elapsed_time = current_time - last_cpu_general_time;
+
+    double cpu_usage_active_time = GetGeneralCPUActiveTime(current_cpu_data.unpack());
+    double elapsed_time_count = static_cast<double>(elapsed_time.count());
+    double general_cpu_perc = cpu_usage_active_time/elapsed_time_count;
+
+    last_cpu_general_time = current_time;
+    last_cpu_general_time_active += cpu_usage_active_time;
+
+    return general_cpu_perc * 100;
+}
+
+double
+CPUCalculator::getCurrentProcessCPUUsage()
+{
+    struct rusage usage;
+    if (last_cpu_process_time == chrono::microseconds(0)) {
+        last_cpu_process_time = i_time_get->getMonotonicTime();
+        getrusage (RUSAGE_SELF, &usage);
+        last_cpu_usage_time_in_user_mod = usage.ru_utime;
+        last_cpu_usage_time_in_kernel = usage.ru_stime;
+        return 0;
+    }
+    auto current_time = i_time_get->getMonotonicTime();
+    auto elapsed_time = current_time - last_cpu_process_time;
+
+    getrusage(RUSAGE_SELF, &usage);
+    chrono::microseconds cpu_usage_time_in_user_mod = calcTimeDiff(usage.ru_utime, last_cpu_usage_time_in_user_mod);
+    chrono::microseconds cpu_usage_time_in_kernel = calcTimeDiff(usage.ru_stime, last_cpu_usage_time_in_kernel);
+
+    double general_cpu_time =
+        static_cast<double>(cpu_usage_time_in_kernel.count() + cpu_usage_time_in_user_mod.count());
+    double elapsed_time_count = static_cast<double>(elapsed_time.count());
+    double general_cpu_perc = general_cpu_time/elapsed_time_count;
+
+    last_cpu_process_time = current_time;
+    last_cpu_usage_time_in_user_mod = usage.ru_utime;
+    last_cpu_usage_time_in_kernel = usage.ru_stime;
+
+    return general_cpu_perc * 100;
+}
+
+chrono::microseconds
+CPUCalculator::calcTimeDiff(const timeval &current_cpu_time, const timeval &last_cpu_time) const
+{
+    auto diff_in_usec = current_cpu_time.tv_usec - last_cpu_time.tv_usec;
+    auto diff_in_sec = current_cpu_time.tv_sec - last_cpu_time.tv_sec;
+    if (diff_in_usec < 0) {
+        diff_in_usec += micro_seconds_in_second;
+        diff_in_sec -= 1;
+    }
+
+    return static_cast<chrono::microseconds>(diff_in_sec * micro_seconds_in_second + diff_in_usec);
+}
+// LCOV_EXCL_STOP
+
+void
+CPUManager::loadCPUConfig()
+{
+    high_watermark = getConfigurationWithDefault<uint>(85, "CPU", "high watermark");
+    low_watermark = getConfigurationWithDefault<uint>(60, "CPU", "low watermark");
+    watermark_period = chrono::seconds(getConfigurationWithDefault<uint>(30, "CPU", "watermark period"));
+    sampling_interval = chrono::seconds(getConfigurationWithDefault<uint>(5, "CPU", "sampling interval"));
+    debug_period = chrono::seconds(getConfigurationWithDefault<uint>(30, "CPU", "debug period"));
+    metric_report_interval = chrono::seconds(
+        getConfigurationWithDefault<uint>(600, "CPU", "metric reporting interval")
+    );
+    failopen_counter = watermark_period/sampling_interval;
+}
+
+void
+CPUManager::init()
+{
+    loadCPUConfig();
+
+    i_mainloop = Singleton::Consume<I_MainLoop>::by<CPUManager>();
+    i_time_get = Singleton::Consume<I_TimeGet>::by<CPUManager>();
+    i_cpu = Singleton::Consume<I_CPU>::by<CPUManager>();
+    i_env = Singleton::Consume<I_Environment>::by<CPUManager>();
+
+    current_counter = 0;
+    is_failopen_mode = false;
+    i_env->registerValue("Failopen Status", is_failopen_mode);
+
+    cpu_process_metric.init(
+        "CPU process usage",
+        ReportIS::AudienceTeam::AGENT_CORE,
+        ReportIS::IssuingEngine::AGENT_CORE,
+        metric_report_interval,
+        true
+    );
+    cpu_process_metric.registerListener();
+
+    if (Singleton::exists<I_Environment>()) {
+        auto name = Singleton::Consume<I_Environment>::by<CPUManager>()->get<string>("Service Name");
+        string orch_service_name = getConfigurationWithDefault<string>(
+            "Orchestration",
+            "orchestration",
+            "Service name"
+        );
+        if (name.ok() && *name == orch_service_name) {
+            cpu_general_metric.init(
+                "CPU general usage",
+                ReportIS::AudienceTeam::AGENT_CORE,
+                ReportIS::IssuingEngine::AGENT_CORE,
+                metric_report_interval,
+                false
+            );
+            cpu_general_metric.registerContext<string>("Service Name", "all");
+            cpu_general_metric.registerListener();
+        }
+    }
+
+    i_mainloop->addOneTimeRoutine(
+        I_MainLoop::RoutineType::Timer,
+        [this]() { checkCPUStatus(); },
+        "CPU manager status check",
+        false
+    );
+}
+
+bool
+CPUManager::isFailOpenMode() const
+{
+    return is_failopen_mode;
+}
+
+void
+CPUManager::preload()
+{
+    registerExpectedConfiguration<uint>("CPU", "high watermark");
+    registerExpectedConfiguration<uint>("CPU", "low watermark");
+    registerExpectedConfiguration<uint>("CPU", "watermark period");
+    registerExpectedConfiguration<uint>("CPU", "sampling interval");
+    registerExpectedConfiguration<uint>("CPU", "metric reporting interval");
+    registerExpectedConfiguration<uint>("CPU", "debug period");
+    registerExpectedConfiguration<string>("orchestration", "Service name");
+}
+
+bool
+CPUManager::isCPUAboveHighWatermark(double current_cpu) const
+{
+    return
+        current_cpu > high_watermark &&
+        current_counter >= 0 &&
+        current_counter < failopen_counter;
+}
+
+bool
+CPUManager::isCPUUnderHighWatermark(double current_cpu) const
+{
+    return
+        current_cpu < high_watermark &&
+        current_counter > 0 &&
+        !is_failopen_mode;
+}
+
+bool
+CPUManager::isCPUUnderLowWatermark(double current_cpu) const
+{
+    return current_cpu <= low_watermark && is_failopen_mode;
+}
+
+void
+CPUManager::checkCPUStatus()
+{
+    while (true) {
+        loadCPUConfig();
+
+        auto is_orchestrator = Singleton::Consume<I_Environment>::by<CPUManager>()->get<bool>("Is Orchestrator");
+        if (is_orchestrator.ok() && is_orchestrator.unpack()) {
+            Maybe<double> current_general_cpu = i_cpu->getCurrentGeneralCPUUsage();
+            if (!current_general_cpu.ok()) {
+                dbgWarning(D_MONITORING) << current_general_cpu.getErr();
+            } else {
+                CPUEvent(current_general_cpu.unpack(), true).notify();
+            }
+        }
+
+        auto current_process_cpu = i_cpu->getCurrentProcessCPUUsage();
+        dbgTrace(D_MONITORING) << "Current process CPU usage: " << current_process_cpu;
+        CPUEvent(current_process_cpu, false).notify();
+
+        if (isCPUAboveHighWatermark(current_process_cpu)) {
+            current_counter++;
+        } else {
+            if (isCPUUnderHighWatermark(current_process_cpu)) {
+                current_counter=0;
+            } else {
+                if (isCPUUnderLowWatermark(current_process_cpu)) {
+                    current_counter--;
+                }
+            }
+        }
+
+        if (current_counter == failopen_counter && !is_failopen_mode) {
+            is_failopen_mode = true;
+            i_env->registerValue("Failopen Status", is_failopen_mode);
+            failopen_mode_event.setFailopenMode(is_failopen_mode);
+            failopen_mode_event.notify();
+
+            dbgInfo(D_MONITORING) << "Failopen mode is ON, CPU usage is above "
+                << high_watermark
+                << "% for "
+                << watermark_period.count()
+                << " seconds";
+
+            if (debug_period == chrono::seconds::zero()) {
+                dbgInfo(D_MONITORING) << "Debug period for Failopen mode is zero seconds";
+            } else {
+                Debug::failOpenDebugMode(debug_period);
+            }
+        }
+
+        if (current_counter == 0 && is_failopen_mode) {
+            is_failopen_mode = false;
+            i_env->registerValue("Failopen Status", is_failopen_mode);
+            failopen_mode_event.setFailopenMode(is_failopen_mode);
+            failopen_mode_event.notify();
+
+            dbgInfo(D_MONITORING) << "Failopen mode is OFF, CPU usage is below "
+                << low_watermark
+                << "% for "
+                << watermark_period.count()
+                << " seconds";
+        }
+
+        i_mainloop->yield(sampling_interval);
+    }
+}

core/cpu/cpu_ut/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_unit_test(
+    cpu_ut
+    "cpu_ut.cc"
+    "cpu;event_is;metric;-lboost_regex"
+)

core/cpu/cpu_ut/cpu_ut.cc

@@ -0,0 +1,643 @@
+#include "cpu.h"
+
+#include "mock/mock_cpu.h"
+#include "cptest.h"
+#include "cptest.h"
+#include "config.h"
+#include "config_component.h"
+#include "mock/mock_mainloop.h"
+#include "mock/mock_time_get.h"
+#include "environment.h"
+#include "event.h"
+#include "listener.h"
+
+using namespace std;
+using namespace testing;
+using namespace chrono;
+
+USE_DEBUG_FLAG(D_FW);
+USE_DEBUG_FLAG(D_CONFIG);
+
+string line = "";
+
+void doFWError() { dbgError(D_FW) << "FW error message"; line = to_string(__LINE__); }
+void doFWWarning() { dbgWarning(D_FW) << "FW warning message"; line = to_string(__LINE__); }
+void doFWInfo() { dbgInfo(D_FW) << "FW info message"; line = to_string(__LINE__); }
+void doFWDebug() { dbgDebug(D_FW) << "FW debug message"; line = to_string(__LINE__); }
+void doFWTrace() { dbgTrace(D_FW) << "FW trace message"; line = to_string(__LINE__); }
+
+class TestEnd {};
+
+static ostream & operator<<(ostream &os, const Context::Error &) { return os; }
+
+class CPUTest : public Test
+{
+public:
+    CPUTest()
+    {
+        env.preload();
+        env.init();
+        i_env = Singleton::Consume<I_Environment>::from(env);
+        i_env->registerValue<bool>("Is Orchestrator", true);
+        EXPECT_CALL(mock_ml, getCurrentRoutineId()).WillRepeatedly(Return(5));
+    }
+
+    ~CPUTest() { Debug::setNewDefaultStdout(&cout); }
+
+    StrictMock<MockMainLoop> mock_ml;
+    StrictMock<MockTimeGet> mock_time;
+    I_Environment *i_env;
+
+private:
+    ConfigComponent conf;
+    ::Environment env;
+};
+
+class FailopenModeListener : public Listener<FailopenModeEvent>
+{
+public:
+    void
+    upon(const FailopenModeEvent &event) override
+    {
+        current_failopen_status = event.getFailopenMode();
+    }
+
+    bool
+    isFailopenMode() const
+    {
+        return current_failopen_status;
+    }
+
+private:
+    bool current_failopen_status = false;
+};
+
+TEST_F(CPUTest, basicTest)
+{
+    seconds time = seconds(0);
+
+    Debug::init();
+    stringstream debug_output;
+    Debug::setNewDefaultStdout(&debug_output);
+    Debug::setUnitTestFlag(D_FW, Debug::DebugLevel::ERROR);
+    Debug::setUnitTestFlag(D_CONFIG, Debug::DebugLevel::ERROR);
+
+    FailopenModeListener failopen_mode_listener;
+    failopen_mode_listener.registerListener();
+
+    EXPECT_CALL(mock_time, getMonotonicTime()).WillRepeatedly(Return(microseconds(time+=seconds(1))));
+    EXPECT_CALL(mock_time, getWalltime()).WillRepeatedly(Return(microseconds(1)));
+    EXPECT_CALL(mock_time, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087")));
+
+    I_MainLoop::Routine cpu_routine = nullptr;
+    I_MainLoop::Routine debug_routine = nullptr;
+
+    EXPECT_CALL(mock_ml, addOneTimeRoutine(I_MainLoop::RoutineType::Timer, _, _, _))
+        .WillOnce(DoAll(SaveArg<1>(&cpu_routine), Return(0)));
+
+    EXPECT_CALL(mock_ml, addOneTimeRoutine(I_MainLoop::RoutineType::System, _, _, _))
+        .WillOnce(DoAll(SaveArg<1>(&debug_routine), Return(0)));
+
+    EXPECT_CALL(
+        mock_ml,
+        addRecurringRoutine(
+            I_MainLoop::RoutineType::System,
+            chrono::microseconds(600000000),
+            _,
+            _,
+            _
+        )
+    ).WillRepeatedly(Return(1));
+
+    StrictMock<MockCPU> mock_cpu;
+    CPUManager cpu;
+    cpu.init();
+
+    doFWInfo();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWWarning();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWDebug();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWTrace();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWError();
+    EXPECT_THAT(debug_output.str(), HasSubstr("!!!] FW error message"));
+    debug_output.str("");
+
+    EXPECT_CALL(mock_cpu, getCurrentProcessCPUUsage()).WillOnce(Return(90));
+    EXPECT_CALL(mock_cpu, getCurrentGeneralCPUUsage()).WillOnce(Return(90));
+
+    EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>())).WillRepeatedly(Invoke(
+        [&] (chrono::microseconds duration) {
+            EXPECT_EQ(duration.count(), chrono::microseconds(chrono::seconds(5)).count());
+            static int count = 0;
+            count++;
+            if (count <= 5) {
+                //Getting 90% CPU for 30 seconds
+                EXPECT_CALL(mock_cpu, getCurrentProcessCPUUsage()).WillOnce(Return(90));
+                EXPECT_CALL(mock_cpu, getCurrentGeneralCPUUsage()).WillOnce(Return(90));
+                EXPECT_FALSE(cpu.isFailOpenMode());
+                EXPECT_THAT(i_env->get<bool>("Failopen Status"), IsValue(false));
+                EXPECT_FALSE(failopen_mode_listener.isFailopenMode());
+            }
+            if (count > 5 && count <= 11) {
+                //Getting 50% CPU for 30 seconds
+                EXPECT_CALL(mock_cpu, getCurrentProcessCPUUsage()).WillOnce(Return(50));
+                EXPECT_CALL(mock_cpu, getCurrentGeneralCPUUsage()).WillOnce(Return(50));
+                EXPECT_TRUE(cpu.isFailOpenMode());
+                EXPECT_THAT(i_env->get<bool>("Failopen Status"), IsValue(true));
+                EXPECT_TRUE(failopen_mode_listener.isFailopenMode());
+            }
+            if (count == 12) {
+                EXPECT_FALSE(cpu.isFailOpenMode());
+                EXPECT_THAT(i_env->get<bool>("Failopen Status"), IsValue(false));
+                EXPECT_FALSE(failopen_mode_listener.isFailopenMode());
+                throw TestEnd();
+            }
+        }
+    ));
+
+    try {
+        cpu_routine();
+    } catch(const TestEnd &T) {
+        //During Failopen mode debugs will be ON
+        EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>()))
+            .WillOnce(
+                Invoke(
+                    [&] (chrono::microseconds duration)
+                    {
+                        EXPECT_EQ(duration.count(), chrono::microseconds(chrono::seconds(30)).count());
+                        debug_output.str("");
+                        doFWError();
+                        EXPECT_THAT(debug_output.str(), HasSubstr("!!!] FW error message\n"));
+                        debug_output.str("");
+
+                        doFWInfo();
+                        EXPECT_THAT(debug_output.str(), HasSubstr("---] FW info message\n"));
+                        debug_output.str("");
+
+                        doFWWarning();
+                        EXPECT_THAT(debug_output.str(), HasSubstr("###] FW warning message\n"));
+                        debug_output.str("");
+
+                        doFWDebug();
+                        EXPECT_THAT(debug_output.str(), HasSubstr("@@@] FW debug message\n"));
+                        debug_output.str("");
+
+                        doFWTrace();
+                        EXPECT_THAT(debug_output.str(), HasSubstr(">>>] FW trace message\n"));
+                        debug_output.str("");
+                    }
+                )
+            );
+        debug_routine();
+    }
+
+    //Exiting Failopen mode - debugs will be back to pervious state
+    doFWInfo();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWWarning();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWDebug();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWTrace();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWError();
+    EXPECT_THAT(debug_output.str(), HasSubstr("!!!] FW error message\n"));
+    debug_output.str("");
+
+    failopen_mode_listener.unregisterListener();
+    Debug::fini();
+}
+
+TEST_F(CPUTest, noDebugTest)
+{
+    seconds time = seconds(0);
+
+    Debug::init();
+    stringstream debug_output;
+    Debug::setNewDefaultStdout(&debug_output);
+    Debug::setUnitTestFlag(D_FW, Debug::DebugLevel::INFO);
+
+    FailopenModeListener failopen_mode_listener;
+    failopen_mode_listener.registerListener();
+
+    EXPECT_CALL(mock_time, getMonotonicTime()).WillRepeatedly(Return(microseconds(time+=seconds(1))));
+    EXPECT_CALL(mock_time, getWalltime()).WillRepeatedly(Return(microseconds(1)));
+    EXPECT_CALL(mock_time, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087")));
+
+    I_MainLoop::Routine cpu_routine = nullptr;
+    EXPECT_CALL(mock_ml, addOneTimeRoutine(I_MainLoop::RoutineType::Timer, _, _, _))
+        .WillOnce(DoAll(SaveArg<1>(&cpu_routine), Return(0)));
+
+    EXPECT_CALL(
+        mock_ml,
+        addRecurringRoutine(
+            I_MainLoop::RoutineType::System,
+            chrono::microseconds(600000000),
+            _,
+            _,
+            _
+        )
+    ).WillRepeatedly(Return(1));
+
+    StrictMock<MockCPU> mock_cpu;
+    CPUManager cpu;
+    cpu.preload();
+    setConfiguration<uint>(0, string("CPU"), string("debug period"));
+    cpu.init();
+
+    doFWError();
+    EXPECT_THAT(debug_output.str(), HasSubstr("!!!] FW error message\n"));
+
+    debug_output.str("");
+    doFWInfo();
+    EXPECT_THAT(debug_output.str(), HasSubstr("---] FW info message\n"));
+    debug_output.str("");
+
+    doFWWarning();
+    EXPECT_THAT(debug_output.str(), HasSubstr("###] FW warning message\n"));
+    debug_output.str("");
+
+    doFWDebug();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWTrace();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    EXPECT_CALL(mock_cpu, getCurrentProcessCPUUsage()).WillOnce(Return(90));
+    EXPECT_CALL(mock_cpu, getCurrentGeneralCPUUsage()).WillOnce(Return(90));
+
+    EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>())).WillRepeatedly(Invoke(
+        [&] (chrono::microseconds duration) {
+            EXPECT_EQ(duration.count(), chrono::microseconds(chrono::seconds(5)).count());
+            static int count = 0;
+            count++;
+            if (count <= 5) {
+                //Getting 90% CPU for 30 seconds
+                EXPECT_CALL(mock_cpu, getCurrentProcessCPUUsage()).WillOnce(Return(90));
+                EXPECT_CALL(mock_cpu, getCurrentGeneralCPUUsage()).WillOnce(Return(90));
+                EXPECT_FALSE(cpu.isFailOpenMode());
+                EXPECT_THAT(i_env->get<bool>("Failopen Status"), IsValue(false));
+                EXPECT_FALSE(failopen_mode_listener.isFailopenMode());
+            }
+            if (count > 5 && count <= 11) {
+                //Getting 50% CPU for 30 seconds
+                EXPECT_CALL(mock_cpu, getCurrentProcessCPUUsage()).WillOnce(Return(50));
+                EXPECT_CALL(mock_cpu, getCurrentGeneralCPUUsage()).WillOnce(Return(50));
+                EXPECT_TRUE(cpu.isFailOpenMode());
+                EXPECT_THAT(i_env->get<bool>("Failopen Status"), IsValue(true));
+                EXPECT_TRUE(failopen_mode_listener.isFailopenMode());
+            }
+            if (count == 12) {
+                EXPECT_FALSE(cpu.isFailOpenMode());
+                EXPECT_THAT(i_env->get<bool>("Failopen Status"), IsValue(false));
+                EXPECT_FALSE(failopen_mode_listener.isFailopenMode());
+                throw TestEnd();
+            }
+        }
+    ));
+
+    try {
+        cpu_routine();
+    } catch(const TestEnd &T) {}
+
+    EXPECT_THAT(
+        debug_output.str(),
+        HasSubstr("Failopen mode is ON, CPU usage is above 85% for 30 seconds")
+    );
+
+    EXPECT_THAT(
+        debug_output.str(),
+        HasSubstr("Debug period for Failopen mode is zero seconds")
+    );
+
+    EXPECT_THAT(
+        debug_output.str(),
+        HasSubstr("Failopen mode is OFF, CPU usage is below 60% for 30 seconds")
+    );
+
+    debug_output.str("");
+    doFWError();
+    EXPECT_THAT(debug_output.str(), HasSubstr("!!!] FW error message\n"));
+    debug_output.str("");
+
+    doFWInfo();
+    EXPECT_THAT(debug_output.str(), HasSubstr("---] FW info message\n"));
+    debug_output.str("");
+
+    doFWWarning();
+    EXPECT_THAT(debug_output.str(), HasSubstr("###] FW warning message\n"));
+    debug_output.str("");
+
+    doFWDebug();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWTrace();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    failopen_mode_listener.unregisterListener();
+    Debug::fini();
+}
+
+TEST_F(CPUTest, CPUCalculatorConstructor)
+{
+    seconds time = seconds(0);
+    Debug::init();
+    stringstream debug_output;
+    Debug::setNewDefaultStdout(&debug_output);
+    Debug::setUnitTestFlag(D_FW, Debug::DebugLevel::INFO);
+
+    EXPECT_CALL(mock_time, getMonotonicTime()).WillRepeatedly(Return(microseconds(time+=seconds(1))));
+    EXPECT_CALL(mock_time, getWalltime()).WillRepeatedly(Return(microseconds(1)));
+    EXPECT_CALL(mock_time, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087")));
+    I_MainLoop::Routine cpu_routine = nullptr;
+    EXPECT_CALL(mock_ml, addOneTimeRoutine(I_MainLoop::RoutineType::Timer, _, _, _))
+        .WillOnce(DoAll(SaveArg<1>(&cpu_routine), Return(0)));
+
+    EXPECT_CALL(
+        mock_ml,
+        addRecurringRoutine(
+            I_MainLoop::RoutineType::System,
+            chrono::microseconds(600000000),
+            _,
+            _,
+            _
+        )
+    ).WillRepeatedly(Return(1));
+
+    CPUCalculator cpu_calc;
+    CPUManager cpu;
+    cpu.preload();
+    cpu_calc.init();
+    cpu.init();
+
+    doFWError();
+    EXPECT_THAT(debug_output.str(), HasSubstr("!!!] FW error message\n"));
+
+    debug_output.str("");
+    doFWInfo();
+    EXPECT_THAT(debug_output.str(), HasSubstr("---] FW info message\n"));
+    debug_output.str("");
+
+    doFWWarning();
+    EXPECT_THAT(debug_output.str(), HasSubstr("###] FW warning message\n"));
+    debug_output.str("");
+
+    doFWDebug();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWTrace();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    cpu_calc.fini();
+}
+
+TEST_F(CPUTest, TwoFailopenDebugTest)
+{
+    seconds time = seconds(0);
+
+    Debug::init();
+    stringstream debug_output;
+    Debug::setNewDefaultStdout(&debug_output);
+    Debug::setUnitTestFlag(D_FW, Debug::DebugLevel::ERROR);
+
+    FailopenModeListener failopen_mode_listener;
+    failopen_mode_listener.registerListener();
+
+    EXPECT_CALL(mock_time, getMonotonicTime()).WillRepeatedly(Return(microseconds(time+=seconds(1))));
+    EXPECT_CALL(mock_time, getWalltime()).WillRepeatedly(Return(microseconds(1)));
+    EXPECT_CALL(mock_time, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087")));
+
+    I_MainLoop::Routine cpu_routine = nullptr;
+    EXPECT_CALL(mock_ml, addOneTimeRoutine(I_MainLoop::RoutineType::Timer, _, _, _))
+        .WillOnce(DoAll(SaveArg<1>(&cpu_routine), Return(0)));
+
+    I_MainLoop::Routine first_debug_routine = nullptr;
+    I_MainLoop::Routine second_debug_routine = nullptr;
+    EXPECT_CALL(mock_ml, addOneTimeRoutine(I_MainLoop::RoutineType::System, _, _, _))
+        .WillOnce(DoAll(SaveArg<1>(&first_debug_routine), Return(0)))
+        .WillOnce(DoAll(SaveArg<1>(&second_debug_routine), Return(0)));
+
+    EXPECT_CALL(
+        mock_ml,
+        addRecurringRoutine(
+            I_MainLoop::RoutineType::System,
+            chrono::microseconds(600000000),
+            _,
+            _,
+            _
+        )
+    ).WillRepeatedly(Return(1));
+
+    StrictMock<MockCPU> mock_cpu;
+    CPUManager cpu;
+    setConfiguration<uint>(90, string("CPU"), string("debug period"));
+    setConfiguration<uint>(25, "CPU", "watermark period");
+    cpu.init();
+
+    doFWInfo();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWWarning();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWDebug();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWTrace();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWError();
+    EXPECT_THAT(debug_output.str(), HasSubstr("!!!] FW error message\n"));
+    debug_output.str("");
+
+    EXPECT_CALL(mock_cpu, getCurrentProcessCPUUsage()).WillOnce(Return(90));
+    EXPECT_CALL(mock_cpu, getCurrentGeneralCPUUsage()).WillOnce(Return(90));
+
+    EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>())).WillRepeatedly(Invoke(
+        [&] (chrono::microseconds duration) {
+            EXPECT_EQ(duration.count(), chrono::microseconds(chrono::seconds(5)).count());
+            static int count = 0;
+            count++;
+            if (count <= 4) {
+                //Getting 90% CPU for 5 seconds
+                EXPECT_CALL(mock_cpu, getCurrentProcessCPUUsage()).WillOnce(Return(90));
+                EXPECT_CALL(mock_cpu, getCurrentGeneralCPUUsage()).WillOnce(Return(90));
+                EXPECT_FALSE(cpu.isFailOpenMode());
+                EXPECT_THAT(i_env->get<bool>("Failopen Status"), IsValue(false));
+                EXPECT_FALSE(failopen_mode_listener.isFailopenMode());
+            }
+            if (count > 4 && count <= 9) {
+                //Getting 50% CPU for 5 seconds
+                EXPECT_CALL(mock_cpu, getCurrentProcessCPUUsage()).WillOnce(Return(50));
+                EXPECT_CALL(mock_cpu, getCurrentGeneralCPUUsage()).WillOnce(Return(50));
+                EXPECT_TRUE(cpu.isFailOpenMode());
+                EXPECT_THAT(i_env->get<bool>("Failopen Status"), IsValue(true));
+                EXPECT_TRUE(failopen_mode_listener.isFailopenMode());
+            }
+            if (count > 9 && count <= 14) {
+                //Getting 90% CPU for 5 seconds
+                EXPECT_CALL(mock_cpu, getCurrentProcessCPUUsage()).WillOnce(Return(90));
+                EXPECT_CALL(mock_cpu, getCurrentGeneralCPUUsage()).WillOnce(Return(90));
+                EXPECT_FALSE(cpu.isFailOpenMode());
+                EXPECT_THAT(i_env->get<bool>("Failopen Status"), IsValue(false));
+                EXPECT_FALSE(failopen_mode_listener.isFailopenMode());
+            }
+            if (count > 14 && count <= 19) {
+                //Getting 50% CPU for 5 seconds
+                EXPECT_CALL(mock_cpu, getCurrentProcessCPUUsage()).WillOnce(Return(50));
+                EXPECT_CALL(mock_cpu, getCurrentGeneralCPUUsage()).WillOnce(Return(50));
+                EXPECT_TRUE(cpu.isFailOpenMode());
+                EXPECT_THAT(i_env->get<bool>("Failopen Status"), IsValue(true));
+                EXPECT_TRUE(failopen_mode_listener.isFailopenMode());
+            }
+            if (count == 20) {
+                EXPECT_FALSE(cpu.isFailOpenMode());
+                EXPECT_THAT(i_env->get<bool>("Failopen Status"), IsValue(false));
+                EXPECT_FALSE(failopen_mode_listener.isFailopenMode());
+                throw TestEnd();
+            }
+        }
+    ));
+
+    try {
+        cpu_routine();
+    } catch(const TestEnd &T) {
+        //During Failopen mode debugs will be ON
+        EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>()))
+            .WillOnce(
+                Invoke(
+                    [&] (chrono::microseconds duration)
+                    {
+                        EXPECT_EQ(duration.count(), chrono::microseconds(chrono::seconds(90)).count());
+                        debug_output.str("");
+                        doFWError();
+                        EXPECT_THAT(debug_output.str(), HasSubstr("!!!] FW error message\n"));
+                        debug_output.str("");
+
+                        doFWInfo();
+                        EXPECT_THAT(debug_output.str(), HasSubstr("---] FW info message\n"));
+                        debug_output.str("");
+
+                        doFWWarning();
+                        EXPECT_THAT(debug_output.str(), HasSubstr("###] FW warning message\n"));
+                        debug_output.str("");
+
+                        doFWDebug();
+                        EXPECT_THAT(debug_output.str(), HasSubstr("@@@] FW debug message\n"));
+                        debug_output.str("");
+
+                        doFWTrace();
+                        EXPECT_THAT(debug_output.str(), HasSubstr(">>>] FW trace message\n"));
+                        debug_output.str("");
+                    }
+                )
+            )
+            .WillOnce(
+                Invoke(
+                    [&] (chrono::microseconds duration)
+                    {
+                        EXPECT_EQ(duration.count(), chrono::microseconds(chrono::seconds(90)).count());
+                        debug_output.str("");
+                        doFWError();
+                        EXPECT_THAT(debug_output.str(), HasSubstr("!!!] FW error message\n"));
+                        debug_output.str("");
+
+                        doFWInfo();
+                        EXPECT_THAT(debug_output.str(), HasSubstr("---] FW info message\n"));
+                        debug_output.str("");
+
+                        doFWWarning();
+                        EXPECT_THAT(debug_output.str(), HasSubstr("###] FW warning message\n"));
+                        debug_output.str("");
+
+                        doFWDebug();
+                        EXPECT_THAT(debug_output.str(), HasSubstr("@@@] FW debug message\n"));
+                        debug_output.str("");
+
+                        doFWTrace();
+                        EXPECT_THAT(debug_output.str(), HasSubstr(">>>] FW trace message\n"));
+                        debug_output.str("");
+                    }
+                )
+            );
+
+        first_debug_routine();
+
+        //Exiting first Failopen mode - debugs are still enabled as only second failopen end will turn them off
+        debug_output.str("");
+        doFWError();
+        EXPECT_THAT(debug_output.str(), HasSubstr("!!!] FW error message\n"));
+        debug_output.str("");
+
+        doFWInfo();
+        EXPECT_THAT(debug_output.str(), HasSubstr("---] FW info message\n"));
+        debug_output.str("");
+
+        doFWWarning();
+        EXPECT_THAT(debug_output.str(), HasSubstr("###] FW warning message\n"));
+        debug_output.str("");
+
+        doFWDebug();
+        EXPECT_THAT(debug_output.str(), HasSubstr("@@@] FW debug message\n"));
+        debug_output.str("");
+
+        doFWTrace();
+        EXPECT_THAT(debug_output.str(), HasSubstr(">>>] FW trace message\n"));
+        debug_output.str("");
+
+        second_debug_routine();
+    }
+
+    // Back to previous debug state
+    doFWInfo();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWWarning();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWDebug();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWTrace();
+    EXPECT_THAT(debug_output.str(), "");
+    debug_output.str("");
+
+    doFWError();
+    EXPECT_THAT(debug_output.str(), HasSubstr("!!!] FW error message\n"));
+    debug_output.str("");
+
+    failopen_mode_listener.unregisterListener();
+    Debug::fini();
+}