From 2a950a435b9c1e35b608c8345fd36322415822a7 Mon Sep 17 00:00:00 2001 From: Felipe Zimmerle Date: Thu, 10 Dec 2015 18:36:20 -0300 Subject: [PATCH] Fix various minor bugs in the regression test suite Now if a test fails it keep testing the others tests from the same family. The output was also improved. --- test/regression/regression.cc | 22 ++++++++++++++----- test/test-cases/regression/action-id.json | 4 ++-- .../regression/config-include-bad.json | 10 ++++----- .../regression/operator-ipMatchFromFile.json | 2 +- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/test/regression/regression.cc b/test/regression/regression.cc index f93f3879..4ecfbbc1 100644 --- a/test/regression/regression.cc +++ b/test/regression/regression.cc @@ -129,14 +129,17 @@ void perform_unit_test(std::vector *tests, } if (modsec_rules->load(t->rules.c_str(), filename) < 0) { + /* Parser error */ if (t->parser_error.empty() == true) { + /* Not expecting any error, thus return the error to the user. */ + std::cout << KRED << "failed!" << RESET << std::endl; testRes->reason << KRED << "parse failed." << RESET \ << std::endl; testRes->reason << modsec_rules->getParserError() \ << std::endl; testRes->passed = false; res->push_back(testRes); - return; + continue; } Regex re(t->parser_error); @@ -144,10 +147,16 @@ void perform_unit_test(std::vector *tests, std::string s = modsec_rules->getParserError(); if (regex_search(s, &match, re) && match.size() >= 1) { + std::cout << KGRN << "passed!" << RESET << std::endl; + /* Parser error was expected, thus, the test passed. */ testRes->reason << KGRN << "passed!" << RESET << std::endl; testRes->passed = true; - return; + res->push_back(testRes); + continue; } else { + /* Parser error was expected, but with a different content */ + std::cout << KRED << "failed!" << RESET << std::endl; + testRes->reason << KRED << "failed!" << RESET << std::endl; testRes->reason << KWHT << "Expected a parser error." \ << RESET << std::endl; @@ -156,17 +165,20 @@ void perform_unit_test(std::vector *tests, testRes->reason << KWHT << "Produced: " << RESET \ << s << std::endl; testRes->passed = false; - return; + res->push_back(testRes); + continue; } - } else { + /* Parser error was expected but never happened */ if (t->parser_error.empty() == false) { - std::cout << KRED << "failed!" << std::endl; + std::cout << KRED << "failed!" << RESET << std::endl; std::cout << KWHT << "Expected a parser error." \ << RESET << std::endl; std::cout << KWHT << "Expected: " << RESET \ << t->parser_error << std::endl; testRes->passed = false; + res->push_back(testRes); + continue; } } diff --git a/test/test-cases/regression/action-id.json b/test/test-cases/regression/action-id.json index c2dfe557..46ad559e 100644 --- a/test/test-cases/regression/action-id.json +++ b/test/test-cases/regression/action-id.json @@ -82,7 +82,7 @@ ] }, "expected":{ - "parser_error": "Rules error. File: action-id.json. Line: 4. Column: 31. id:-1" + "parser_error": "Rules error. File: action-id.json. Line: 4. Column: 31. .*" }, "rules":[ "SecRuleEngine On", @@ -220,7 +220,7 @@ ] }, "expected":{ - "parser_error": "Rules error. File: action-id.json. Line: 4. Column: 31. id:'1" + "parser_error": "Rules error. File: action-id.json. Line: 4. Column: 31. .*" }, "rules":[ "SecRuleEngine On", diff --git a/test/test-cases/regression/config-include-bad.json b/test/test-cases/regression/config-include-bad.json index 86a3dbba..e5a029e8 100644 --- a/test/test-cases/regression/config-include-bad.json +++ b/test/test-cases/regression/config-include-bad.json @@ -4,7 +4,7 @@ "version_min":300000, "title":"Include - bad rule", "expected":{ - "parser_error": "Rules error. File: test-cases/data/config_example3.txt. Line: 2. Column: 42. ops" + "parser_error": "Rules error. File: test-cases/data/config_example3.txt. Line: 2. Column: 42. invalid character .*" }, "rules":[ "SecRuleEngine On", @@ -19,7 +19,7 @@ "version_min":300000, "title":"Include - missing file", "expected":{ - "parser_error": "Rules error. File: config-include-bad.json. Line: 4. Column: 47. Include test-cases/data/config_example-ops.txt: Not able to open file." + "parser_error": "Rules error. File: config-include-bad.json. Line: 4. Column: 47. test-cases/data/config_example-ops.txt: Not able to open file." }, "rules":[ "SecRuleEngine On", @@ -34,7 +34,7 @@ "version_min":300000, "title":"Include - missing at include", "expected":{ - "parser_error": "Rules error. File: test-cases/data/config_example-ops-include.txt. Line: 1. Column: 53. Include test-cases/data/config_example-not-exist.txt: Not able to open file." + "parser_error": "Rules error. File: test-cases/data/config_example-ops-include.txt. Line: 1. Column: 53. test-cases/data/config_example-not-exist.txt: Not able to open file." }, "rules":[ "SecRuleEngine On", @@ -49,7 +49,7 @@ "version_min":300000, "title":"Include - bad operator inside include", "expected":{ - "parser_error": "Rules error. File: test-cases/data/config_example-bad-op-include.txt. Line: 2. Column: 32." + "parser_error": "Rules error. File: test-cases/data/config_example-bad-op-include.txt. Line: 2. Column: 67. .*" }, "rules":[ "SecRuleEngine On", @@ -64,7 +64,7 @@ "version_min":300000, "title":"Include - duplicate id", "expected":{ - "parser_error": "Rule id: 40.000000 is duplicated" + "parser_error": "Rule id: 40 is duplicated" }, "rules":[ "SecRuleEngine On", diff --git a/test/test-cases/regression/operator-ipMatchFromFile.json b/test/test-cases/regression/operator-ipMatchFromFile.json index 8a817cc3..be27a018 100644 --- a/test/test-cases/regression/operator-ipMatchFromFile.json +++ b/test/test-cases/regression/operator-ipMatchFromFile.json @@ -82,7 +82,7 @@ ] }, "expected":{ - "parser_error":"Failed to open file: file-not-found.txt" + "parser_error":"Rules error. File: operator-ipMatchFromFile.json. Line: 4. Column: 20. .*" }, "rules":[ "SecRuleEngine On",