From 7a468a8fbec09fd02706a0303ed12ec09b39efa9 Mon Sep 17 00:00:00 2001 From: Felipe Zimmerle Date: Wed, 16 Sep 2015 13:35:30 -0300 Subject: [PATCH] Cosmetic: Prints regression test results in a better shape --- test/common/colors.h | 2 +- test/regression/regression.cc | 67 ++++++++++++++++++++++--------- test/regression/regression_test.h | 11 +++++ test/unit/unit.cc | 11 ----- 4 files changed, 59 insertions(+), 32 deletions(-) diff --git a/test/common/colors.h b/test/common/colors.h index eaf048fc..ad878ed5 100644 --- a/test/common/colors.h +++ b/test/common/colors.h @@ -24,7 +24,7 @@ #define KBLU "\x1B[34m" #define KMAG "\x1B[35m" #define KCYN "\x1B[36m" -#define KWHT "\x1B[37m" +#define KWHT "\x1B[97m" #define RESET "\033[0m" diff --git a/test/regression/regression.cc b/test/regression/regression.cc index 671085cd..958bb35a 100644 --- a/test/regression/regression.cc +++ b/test/regression/regression.cc @@ -24,6 +24,7 @@ #include "modsecurity/rules.h" #include "common/modsecurity_test.h" +#include "common/colors.h" #include "regression/regression_test.h" #include "common/modsecurity_test_results.h" #include "regression/custom_debug_log.h" @@ -33,6 +34,7 @@ using modsecurity_test::CustomDebugLog; using modsecurity_test::ModSecurityTest; using modsecurity_test::ModSecurityTestResults; using modsecurity_test::RegressionTest; +using modsecurity_test::RegressionTestResult; using ModSecurity::Utils::regex_search; using ModSecurity::Utils::SMatch; @@ -67,7 +69,7 @@ void actions(ModSecurityTestResults *r, void perform_unit_test(std::vector *tests, - ModSecurityTestResults *res, int *count) { + ModSecurityTestResults *res, int *count) { for (RegressionTest *t : *tests) { @@ -75,7 +77,9 @@ void perform_unit_test(std::vector *tests, ModSecurity::ModSecurity *modsec = NULL; ModSecurity::Rules *modsec_rules = NULL; ModSecurity::Assay *modsec_assay = NULL; - ModSecurityTestResults r; + ModSecurityTestResults r; + RegressionTestResult *testRes = new RegressionTestResult(); + testRes->test = t; r.status = 200; (*count)++; @@ -195,26 +199,31 @@ end: if (d != NULL) { if (!d->contains(t->debug_log)) { - std::cout << "failed!" << std::endl; - std::cout << "Debug log was not matching the expected results."; - std::cout << std::endl; + std::cout << KRED << "failed!" << RESET << std::endl; + testRes->reason << "Debug log was not matching the expected results."; + testRes->passed = false; } else if (r.status != t->http_code) { - std::cout << "failed!" << std::endl; - std::cout << "HTTP code mismatch. expecting: " + \ + std::cout << KRED << "failed!" << RESET << std::endl; + testRes->reason << "HTTP code mismatch. expecting: " + \ std::to_string(t->http_code) + \ " got: " + std::to_string(r.status) + "\n"; + testRes->passed = false; } else { - std::cout << "passed!" << std::endl; + std::cout << KGRN << "passed!" << RESET << std::endl; + testRes->passed = true; goto after_debug_log; } - std::cout << "Debug log:" << std::endl; - std::cout << "" << d->log_messages() << "" <passed == false) { + testRes->reason << std::endl; + testRes->reason << KWHT << "Debug log:" << RESET << std::endl; + testRes->reason << d->log_messages() << std::endl; + } } after_debug_log: if (d != NULL) { - res->log_raw_debug_log = d->log_messages(); + r.log_raw_debug_log = d->log_messages(); } delete modsec_assay; @@ -222,7 +231,7 @@ after_debug_log: delete modsec; /* delete debug_log; */ - res->insert(res->end(), r.begin(), r.end()); + res->push_back(testRes); } } @@ -236,9 +245,6 @@ int main(int argc, char **argv) { test.load_tests(); - std::ofstream test_log; - test_log.open("regression_test_log.txt"); - std::cout << std::setw(4) << std::right << "# "; std::cout << std::setw(50) << std::left << "File Name"; std::cout << std::setw(70) << std::left << "Test Name"; @@ -257,15 +263,36 @@ int main(int argc, char **argv) { } keyList.sort(); + ModSecurityTestResults res; for (std::string &a : keyList) { std::vector *tests = test[a]; - ModSecurityTestResults res; - perform_unit_test(tests, &res, &counter); - - test_log << res.log_raw_debug_log; } - test_log.close(); + + std::cout << std::endl; + + int passed = 0; + int failed = 0; + for (RegressionTestResult *r : res) { + if (r->passed) { + passed++; + } else { + std::cout << KRED << "Test failed." << RESET << KWHT << " From: " \ + << RESET << r->test->filename << "." << std::endl; + std::cout << KWHT << "Test name: " << RESET << r->test->name << "." << std::endl; + std::cout << KWHT << "Reason: " << RESET << std::endl; + std::cout << r->reason.str() << std::endl; + failed++; + } + } + + std::cout << "Ran a total of: " << std::to_string(failed + passed) \ + << " regression tests - "; + if (failed == 0) { + std::cout << KGRN << "All tests passed" << RESET << std::endl; + } else { + std::cout << KRED << failed << " failed." << RESET << std::endl; + } for (std::pair *> a : test) { std::vector *vec = a.second; diff --git a/test/regression/regression_test.h b/test/regression/regression_test.h index fab2b0e0..afdfce45 100644 --- a/test/regression/regression_test.h +++ b/test/regression/regression_test.h @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -25,6 +26,7 @@ namespace modsecurity_test { + class RegressionTest { public: static RegressionTest *from_yajl_node(const yajl_val &); @@ -71,6 +73,15 @@ class RegressionTest { std::string redirect_url; }; + +class RegressionTestResult { + public: + bool passed; + RegressionTest *test; + std::stringstream reason; +}; + + } // namespace modsecurity_test #endif // TEST_REGRESSION_REGRESSION_TEST_H_ diff --git a/test/unit/unit.cc b/test/unit/unit.cc index b38a7a89..15ef447d 100644 --- a/test/unit/unit.cc +++ b/test/unit/unit.cc @@ -39,18 +39,7 @@ using ModSecurity::operators::Operator; std::string default_test_path = "test-cases/secrules-language-tests/operators"; void print_help() { -#ifdef HAS_GETOPT - std::cout << "Use ./unit [--no-color] -t /path/to/test/files "; - std::cout << std::endl; - std::cout << std::endl; - std::cout << std::endl; - std::cout << " -h\t\tThis help message" << std::endl; - std::cout << " -v\t\tVerbose" << std::endl; - std::cout << " -c\t\tNo color" << std::endl; - std::cout << " -t\t\tPath to test cases" << std::endl; -#else std::cout << "Use ./unit /path/to/file" << std::endl; -#endif std::cout << std::endl; std::cout << std::endl; }