From f0155e3f3204e05b0137fd7d431f27cef9785dbe Mon Sep 17 00:00:00 2001 From: Felipe Zimmerle Date: Tue, 14 Jun 2016 09:45:15 -0300 Subject: [PATCH] Adds support to `make check` The regression and unit tests are now integrated with `make check`. It is possible to use make check -jN to have multiple tests running in parallel. --- Makefile.am | 176 +++++++++++++++++++++++++++++++- configure.ac | 6 +- test/common/modsecurity_test.cc | 42 ++++---- test/common/modsecurity_test.h | 6 ++ test/custom-test-driver | 133 ++++++++++++++++++++++++ test/regression/regression.cc | 158 +++++++++++++++++++--------- test/test-suite.sh | 11 ++ test/unit/unit.cc | 76 +++++++++----- 8 files changed, 510 insertions(+), 98 deletions(-) create mode 100755 test/custom-test-driver create mode 100755 test/test-suite.sh diff --git a/Makefile.am b/Makefile.am index 75842340..62707b8c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -34,5 +34,179 @@ MAINTAINERCLEANFILES = \ ltmain.sh \ ylwrap \ missing \ - depcomp + depcomp +LOG_DRIVER = env $(SHELL) $(top_srcdir)/test/custom-test-driver +AM_TESTS_ENVIRONMENT=AUTOMAKE_TESTS=true; export AUTOMAKE_TESTS; +LOG_COMPILER=test/test-suite.sh +# for i in `find test/test-cases -iname *.json`; do echo TESTS+=$i; done +TESTS= +TESTS+=test/test-cases/regression/actions.json +TESTS+=test/test-cases/regression/debug_log.json +TESTS+=test/test-cases/regression/config-include.json +TESTS+=test/test-cases/regression/variable-PATH_INFO.json +TESTS+=test/test-cases/regression/action-xmlns.json +TESTS+=test/test-cases/regression/collection-tx.json +TESTS+=test/test-cases/regression/config-body_limits.json +TESTS+=test/test-cases/regression/variable-RESPONSE_BODY.json +TESTS+=test/test-cases/regression/action-ctl_request_body_processor.json +TESTS+=test/test-cases/regression/variable-QUERY_STRING.json +TESTS+=test/test-cases/regression/variable-REQUEST_LINE.json +TESTS+=test/test-cases/regression/variable-RESPONSE_HEADERS.json +TESTS+=test/test-cases/regression/operator-rx.json +TESTS+=test/test-cases/regression/variable-ARGS.json +TESTS+=test/test-cases/regression/issue-394.json +TESTS+=test/test-cases/regression/variable-TIME_MON.json +TESTS+=test/test-cases/regression/misc.json +TESTS+=test/test-cases/regression/variable-REQUEST_HEADERS.json +TESTS+=test/test-cases/regression/variable-FULL_REQUEST_LENGTH.json +TESTS+=test/test-cases/regression/variable-ARGS_NAMES.json +TESTS+=test/test-cases/regression/variable-REQUEST_COOKIES_NAMES.json +TESTS+=test/test-cases/regression/variable-ARGS_GET_NAMES.json +TESTS+=test/test-cases/regression/variable-REMOTE_USER.json +TESTS+=test/test-cases/regression/action-msg.json +TESTS+=test/test-cases/regression/variable-UNIQUE_ID.json +TESTS+=test/test-cases/regression/variable-REQUEST_URI.json +TESTS+=test/test-cases/regression/request-body-parser-xml.json +TESTS+=test/test-cases/regression/secaction.json +TESTS+=test/test-cases/regression/variable-TIME_WDAY.json +TESTS+=test/test-cases/regression/issue-960.json +TESTS+=test/test-cases/regression/request-body-parser-xml-validade-dtd.json +TESTS+=test/test-cases/regression/variable-TIME_MIN.json +TESTS+=test/test-cases/regression/action-setuid.json +TESTS+=test/test-cases/regression/config-calling_phases_by_name.json +TESTS+=test/test-cases/regression/variable-USERID.json +TESTS+=test/test-cases/regression/request-body-parser-multipart.json +TESTS+=test/test-cases/regression/variable-RESPONSE_CONTENT_TYPE.json +TESTS+=test/test-cases/regression/action-disruptive.json +TESTS+=test/test-cases/regression/variable-REQUEST_BASENAME.json +TESTS+=test/test-cases/regression/sec_component_signature.json +TESTS+=test/test-cases/regression/variable-REQUEST_METHOD.json +TESTS+=test/test-cases/regression/action-initcol.json +TESTS+=test/test-cases/regression/variable-FILES.json +TESTS+=test/test-cases/regression/variable-REQUEST_BODY.json +TESTS+=test/test-cases/regression/request-body-parser-multipart-crlf.json +TESTS+=test/test-cases/regression/variable-TIME_EPOCH.json +TESTS+=test/test-cases/regression/variable-HIGHEST_SEVERITY.json +TESTS+=test/test-cases/regression/variable-variation-exclusion.json +TESTS+=test/test-cases/regression/variable-GEO.json +TESTS+=test/test-cases/regression/variable-SESSIONID.json +TESTS+=test/test-cases/regression/variable-SERVER_PORT.json +TESTS+=test/test-cases/regression/variable-DURATION.json +TESTS+=test/test-cases/regression/secruleengine.json +TESTS+=test/test-cases/regression/variable-FILES_COMBINED_SIZE.json +TESTS+=test/test-cases/regression/secmarker.json +TESTS+=test/test-cases/regression/action-id.json +TESTS+=test/test-cases/regression/variable-AUTH_TYPE.json +TESTS+=test/test-cases/regression/variable-ARGS_COMBINED_SIZE.json +TESTS+=test/test-cases/regression/variable-REMOTE_PORT.json +TESTS+=test/test-cases/regression/variable-TIME_SEC.json +TESTS+=test/test-cases/regression/config-secremoterules.json +TESTS+=test/test-cases/regression/variable-FILES_NAMES.json +TESTS+=test/test-cases/regression/variable-MULTIPART_CRLF_LF_LINES.json +TESTS+=test/test-cases/regression/variable-MULTIPART_UNMATCHED_BOUNDARY.json +TESTS+=test/test-cases/regression/variable-REQUEST_HEADERS_NAMES.json +TESTS+=test/test-cases/regression/variable-INBOUND_DATA_ERROR.json +TESTS+=test/test-cases/regression/action-setsid.json +TESTS+=test/test-cases/regression/variable-MATCHED_VARS.json +TESTS+=test/test-cases/regression/operator-ipMatchFromFile.json +TESTS+=test/test-cases/regression/variable-FILES_SIZES.json +TESTS+=test/test-cases/regression/variable-MATCHED_VAR_NAME.json +TESTS+=test/test-cases/regression/collection-tx-with-macro.json +TESTS+=test/test-cases/regression/variable-MODSEC_BUILD.json +TESTS+=test/test-cases/regression/variable-ARGS_POST_NAMES.json +TESTS+=test/test-cases/regression/variable-REMOTE_ADDR.json +TESTS+=test/test-cases/regression/auditlog.json +TESTS+=test/test-cases/regression/config-include-bad.json +TESTS+=test/test-cases/regression/variable-OUTBOUND_DATA_ERROR.json +TESTS+=test/test-cases/regression/config-xml_external_entity.json +TESTS+=test/test-cases/regression/variable-MATCHED_VAR.json +TESTS+=test/test-cases/regression/variable-FULL_REQUEST.json +TESTS+=test/test-cases/regression/variable-TIME_HOUR.json +TESTS+=test/test-cases/regression/variable-ARGS_POST.json +TESTS+=test/test-cases/regression/variable-variation-count.json +TESTS+=test/test-cases/regression/variable-REQUEST_URI_RAW.json +TESTS+=test/test-cases/regression/variable-TIME_YEAR.json +TESTS+=test/test-cases/regression/variable-RESPONSE_HEADERS_NAMES.json +TESTS+=test/test-cases/regression/variable-MULTIPART_NAME.json +TESTS+=test/test-cases/regression/variable-ARGS_GET.json +TESTS+=test/test-cases/regression/transformations.json +TESTS+=test/test-cases/regression/config-response_type.json +TESTS+=test/test-cases/regression/variable-TIME.json +TESTS+=test/test-cases/regression/variable-MATCHED_VARS_NAMES.json +TESTS+=test/test-cases/regression/variable-REQUEST_COOKIES.json +TESTS+=test/test-cases/regression/variable-REQUEST_FILENAME.json +TESTS+=test/test-cases/regression/variable-RESPONSE_CONTENT_LENGTH.json +TESTS+=test/test-cases/regression/variable-REMOTE_HOST.json +TESTS+=test/test-cases/regression/variable-MULTIPART_STRICT_ERROR.json +TESTS+=test/test-cases/regression/variable-SERVER_ADDR.json +TESTS+=test/test-cases/regression/variable-MULTIPART_FILENAME.json +TESTS+=test/test-cases/regression/variable-MULTIPART_INVALID_HEADER_FOLDING.json +TESTS+=test/test-cases/regression/variable-REQUEST_BODY_LENGTH.json +TESTS+=test/test-cases/regression/config-secdefaultaction.json +TESTS+=test/test-cases/regression/variable-REQUEST_PROTOCOL.json +TESTS+=test/test-cases/regression/variable-TIME_DAY.json +TESTS+=test/test-cases/regression/transformation-none.json +TESTS+=test/test-cases/regression/variable-TX.json +TESTS+=test/test-cases/regression/variable-ENV.json +TESTS+=test/test-cases/regression/action-tag.json +TESTS+=test/test-cases/secrules-language-tests/transformations/base64Encode.json +TESTS+=test/test-cases/secrules-language-tests/transformations/trimRight.json +TESTS+=test/test-cases/secrules-language-tests/transformations/parityEven7bit.json +TESTS+=test/test-cases/secrules-language-tests/transformations/length.json +TESTS+=test/test-cases/secrules-language-tests/transformations/urlDecodeUni.json +TESTS+=test/test-cases/secrules-language-tests/transformations/base64Decode.json +TESTS+=test/test-cases/secrules-language-tests/transformations/utf8toUnicode.json +TESTS+=test/test-cases/secrules-language-tests/transformations/cssDecode.json +TESTS+=test/test-cases/secrules-language-tests/transformations/trim.json +TESTS+=test/test-cases/secrules-language-tests/transformations/ge.json +TESTS+=test/test-cases/secrules-language-tests/transformations/replaceComments.json +TESTS+=test/test-cases/secrules-language-tests/transformations/htmlEntityDecode.json +TESTS+=test/test-cases/secrules-language-tests/transformations/escapeSeqDecode.json +TESTS+=test/test-cases/secrules-language-tests/transformations/parityZero7bit.json +TESTS+=test/test-cases/secrules-language-tests/transformations/replaceNulls.json +TESTS+=test/test-cases/secrules-language-tests/transformations/sqlHexDecode.json +TESTS+=test/test-cases/secrules-language-tests/transformations/md5.json +TESTS+=test/test-cases/secrules-language-tests/transformations/removeWhitespace.json +TESTS+=test/test-cases/secrules-language-tests/transformations/trimLeft.json +TESTS+=test/test-cases/secrules-language-tests/transformations/hexDecode.json +TESTS+=test/test-cases/secrules-language-tests/transformations/hexEncode.json +TESTS+=test/test-cases/secrules-language-tests/transformations/normalisePathWin.json +TESTS+=test/test-cases/secrules-language-tests/transformations/cmdLine.json +TESTS+=test/test-cases/secrules-language-tests/transformations/parityOdd7bit.json +TESTS+=test/test-cases/secrules-language-tests/transformations/urlDecode.json +TESTS+=test/test-cases/secrules-language-tests/transformations/base64DecodeExt.json +TESTS+=test/test-cases/secrules-language-tests/transformations/lowercase.json +TESTS+=test/test-cases/secrules-language-tests/transformations/sha1.json +TESTS+=test/test-cases/secrules-language-tests/transformations/normalisePath.json +TESTS+=test/test-cases/secrules-language-tests/transformations/jsDecode.json +TESTS+=test/test-cases/secrules-language-tests/transformations/removeNulls.json +TESTS+=test/test-cases/secrules-language-tests/transformations/removeCommentsChar.json +TESTS+=test/test-cases/secrules-language-tests/transformations/removeComments.json +TESTS+=test/test-cases/secrules-language-tests/transformations/urlEncode.json +TESTS+=test/test-cases/secrules-language-tests/transformations/compressWhitespace.json +TESTS+=test/test-cases/secrules-language-tests/operators/geoLookup.json +TESTS+=test/test-cases/secrules-language-tests/operators/validateUrlEncoding.json +TESTS+=test/test-cases/secrules-language-tests/operators/endsWith.json +TESTS+=test/test-cases/secrules-language-tests/operators/ge.json +TESTS+=test/test-cases/secrules-language-tests/operators/gt.json +TESTS+=test/test-cases/secrules-language-tests/operators/contains.json +TESTS+=test/test-cases/secrules-language-tests/operators/validateUtf8Encoding.json +TESTS+=test/test-cases/secrules-language-tests/operators/detectSQLi.json +TESTS+=test/test-cases/secrules-language-tests/operators/validateByteRange.json +TESTS+=test/test-cases/secrules-language-tests/operators/pmFromFile.json +TESTS+=test/test-cases/secrules-language-tests/operators/streq.json +TESTS+=test/test-cases/secrules-language-tests/operators/le.json +TESTS+=test/test-cases/secrules-language-tests/operators/within.json +TESTS+=test/test-cases/secrules-language-tests/operators/noMatch.json +TESTS+=test/test-cases/secrules-language-tests/operators/unconditionalMatch.json +TESTS+=test/test-cases/secrules-language-tests/operators/beginsWith.json +TESTS+=test/test-cases/secrules-language-tests/operators/containsWord.json +TESTS+=test/test-cases/secrules-language-tests/operators/pm.json +TESTS+=test/test-cases/secrules-language-tests/operators/verifyCC.json +TESTS+=test/test-cases/secrules-language-tests/operators/rx.json +TESTS+=test/test-cases/secrules-language-tests/operators/lt.json +TESTS+=test/test-cases/secrules-language-tests/operators/ipMatch.json +TESTS+=test/test-cases/secrules-language-tests/operators/strmatch.json +TESTS+=test/test-cases/secrules-language-tests/operators/detectXSS.json +TESTS+=test/test-cases/secrules-language-tests/operators/eq.json diff --git a/configure.ac b/configure.ac index 9365bcc0..b6b5f77a 100644 --- a/configure.ac +++ b/configure.ac @@ -116,7 +116,6 @@ CHECK_LIBXML2 CHECK_PCRE - # Checks for header files. AC_HEADER_STDC AC_CHECK_HEADERS([string]) @@ -244,6 +243,11 @@ fi AM_CONDITIONAL([TEST_UTILITIES], [test $buildTestUtilities = true]) +if test $buildTestUtilities = true; then + if test $debugLogs = true; then + TEST_CASES=`./test/test-list.sh` + fi +fi # General link options diff --git a/test/common/modsecurity_test.cc b/test/common/modsecurity_test.cc index 9706fbbe..ea9c7907 100644 --- a/test/common/modsecurity_test.cc +++ b/test/common/modsecurity_test.cc @@ -136,34 +136,28 @@ std::pair>* ModSecurityTest::load_tests() { template void ModSecurityTest::cmd_options(int argc, char **argv) { -#if HAS_GETOPT - int option_char; - GetOpt getopt(argc, argv, "hvct:"); - - while ((option_char = getopt()) != EOF) { - switch (option_char) { - case 'h': - print_help(); - return; - break; - case 'v': - this->verbose = true; - break; - case 'c': - this->color = false; - break; - case 't': - this->target_folder = getopt.optarg; - break; - } + int i = 1; + if (argc > i && strcmp(argv[i], "automake") == 0) { + i++; + m_automake_output = true; } -#else - if (argv[1]) { - this->target = argv[1]; + + if(const char* env_p = std::getenv("AUTOMAKE_TESTS")) { + m_automake_output = true; + } + + if (argc > i && argv[i]) { + this->target = argv[i]; + size_t pos = this->target.find(":"); + if (pos != std::string::npos) { + std::string test_numbers = std::string(this->target, pos + 1, + this->target.length() - pos); + this->target = std::string(this->target, 0, pos); + m_test_number = std::atoi(test_numbers.c_str()); + } } else { this->target = default_test_path; } -#endif } } // namespace modsecurity_test diff --git a/test/common/modsecurity_test.h b/test/common/modsecurity_test.h index 617dfbbf..258442ce 100644 --- a/test/common/modsecurity_test.h +++ b/test/common/modsecurity_test.h @@ -31,6 +31,10 @@ namespace modsecurity_test { template class ModSecurityTest : public std::unordered_map *> { public: + ModSecurityTest() + : m_test_number(0), + m_automake_output(false) { } + std::string header(); void cmd_options(int, char **); std::pair>* load_tests(); @@ -40,6 +44,8 @@ template class ModSecurityTest : std::string target; bool verbose = false; bool color = false; + int m_test_number; + bool m_automake_output; }; } // namespace modsecurity_test diff --git a/test/custom-test-driver b/test/custom-test-driver new file mode 100755 index 00000000..4522a0b3 --- /dev/null +++ b/test/custom-test-driver @@ -0,0 +1,133 @@ +#! /bin/sh +# test-driver - basic testsuite driver script. + +scriptversion=2013-07-13.22-modsec; # UTC + +# Copyright (C) 2011-2014 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to or send patches to +# . + +# Make unconditional expansion of undefined variables an error. This +# helps a lot in preventing typo-related bugs. +set -u + +usage_error () +{ + echo "$0: $*" >&2 + print_usage >&2 + exit 2 +} + +print_usage () +{ + cat <$log_file 2>&1 +estatus=$? +cat $log_file >> $trs_file +tfail=`cat $log_file | egrep "^:test-result: FAIL" | wc -l` +tfail=`printf "%3d" $tfail` +tpass=`cat $log_file | egrep "^:test-result: PASS" | wc -l` +tpass=`printf "%4d" $tpass` +ttotal=`cat $log_file | egrep "^:test-result: " | wc -l` +ttotal=`printf "%4d" $ttotal` + +# Report outcome to console. +if test ${tfail} -eq 0; then + echo "(${grn}${tpass}$std/${red}${tfail}$std/${wht}${ttotal}${std}): $test_name" +else + echo "(${grn}${tpass}$std/${red}${tfail}$std/${wht}${ttotal}${std}): ${red}$test_name${std}" +fi + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/test/regression/regression.cc b/test/regression/regression.cc index 77d4e728..a49913a8 100644 --- a/test/regression/regression.cc +++ b/test/regression/regression.cc @@ -75,7 +75,8 @@ void logCb(void *data, const char *msg) { } -void perform_unit_test(std::vector *tests, +void perform_unit_test(ModSecurityTest *test, + std::vector *tests, ModSecurityTestResults *res, int *count) { @@ -99,10 +100,13 @@ void perform_unit_test(std::vector *tests, } else { filename = t->filename; } - std::cout << std::setw(3) << std::right << - std::to_string(*count) << " "; - std::cout << std::setw(50) << std::left << filename; - std::cout << std::setw(70) << std::left << t->name; + + if (!test->m_automake_output) { + std::cout << std::setw(3) << std::right << + std::to_string(*count) << " "; + std::cout << std::setw(50) << std::left << filename; + std::cout << std::setw(70) << std::left << t->name; + } modsec = new modsecurity::ModSecurity(); modsec->setConnectorInformation("ModSecurity-regression v0.0.1-alpha" \ @@ -123,7 +127,12 @@ void perform_unit_test(std::vector *tests, testRes->reason << KCYN << "compiled with support " << std::endl; testRes->reason << KCYN << "to: " << t->resource << std::endl; testRes->reason << RESET << std::endl; - std::cout << KCYN << "skipped!" << RESET << std::endl; + if (test->m_automake_output) { + std::cout << ":test-result: SKIP " << filename \ + << ":" << t->name << std::endl; + } else { + std::cout << KCYN << "skipped!" << RESET << std::endl; + } res->push_back(testRes); continue; } @@ -135,7 +144,12 @@ void perform_unit_test(std::vector *tests, * Not expecting any error, thus return the error to * the user. */ - std::cout << KRED << "failed!" << RESET << std::endl; + if (test->m_automake_output) { + std::cout << ":test-result: FAIL " << filename \ + << ":" << t->name << std::endl; + } else { + std::cout << KRED << "failed!" << RESET << std::endl; + } testRes->reason << KRED << "parse failed." << RESET \ << std::endl; testRes->reason << modsec_rules->getParserError() \ @@ -150,7 +164,12 @@ void perform_unit_test(std::vector *tests, std::string s = modsec_rules->getParserError(); if (regex_search(s, &match, re) && match.size() >= 1) { - std::cout << KGRN << "passed!" << RESET << std::endl; + if (test->m_automake_output) { + std::cout << ":test-result: PASS " << filename \ + << ":" << t->name << std::endl; + } else { + std::cout << KGRN << "passed!" << RESET << std::endl; + } /* Parser error was expected, thus, the test passed. */ testRes->reason << KGRN << "passed!" << RESET << std::endl; testRes->passed = true; @@ -158,7 +177,12 @@ void perform_unit_test(std::vector *tests, continue; } else { /* Parser error was expected, but with a different content */ - std::cout << KRED << "failed!" << RESET << std::endl; + if (test->m_automake_output) { + std::cout << ":test-result: FAIL " << filename \ + << ":" << t->name << std::endl; + } else { + std::cout << KRED << "failed!" << RESET << std::endl; + } testRes->reason << KRED << "failed!" << RESET << std::endl; testRes->reason << KWHT << "Expected a parser error." \ @@ -174,11 +198,16 @@ void perform_unit_test(std::vector *tests, } else { /* Parser error was expected but never happened */ if (t->parser_error.empty() == false) { - std::cout << KRED << "failed!" << RESET << std::endl; - std::cout << KWHT << "Expected a parser error." \ - << RESET << std::endl; - std::cout << KWHT << "Expected: " << RESET \ - << t->parser_error << std::endl; + if (test->m_automake_output) { + std::cout << ":test-result: FAIL " << filename \ + << ":" << t->name << std::endl; + } else { + std::cout << KRED << "failed!" << RESET << std::endl; + std::cout << KWHT << "Expected a parser error." \ + << RESET << std::endl; + std::cout << KWHT << "Expected: " << RESET \ + << t->parser_error << std::endl; + } testRes->passed = false; res->push_back(testRes); continue; @@ -254,20 +283,35 @@ end: if (d != NULL) { if (!d->contains(t->debug_log)) { - std::cout << KRED << "failed!" << RESET << std::endl; + if (test->m_automake_output) { + std::cout << ":test-result: FAIL " << filename \ + << ":" << t->name << std::endl; + } else { + std::cout << KRED << "failed!" << RESET << std::endl; + } testRes->reason << "Debug log was not matching the " \ << "expected results." << std::endl; testRes->reason << KWHT << "Expecting: " << RESET \ << t->debug_log + "."; testRes->passed = false; } else if (r.status != t->http_code) { - std::cout << KRED << "failed!" << RESET << std::endl; + if (test->m_automake_output) { + std::cout << ":test-result: FAIL " << filename \ + << ":" << t->name << std::endl; + } else { + std::cout << KRED << "failed!" << RESET << std::endl; + } testRes->reason << "HTTP code mismatch. expecting: " + \ std::to_string(t->http_code) + \ " got: " + std::to_string(r.status) + "\n"; testRes->passed = false; } else { - std::cout << KGRN << "passed!" << RESET << std::endl; + if (test->m_automake_output) { + std::cout << ":test-result: PASS " << filename \ + << ":" << t->name << std::endl; + } else { + std::cout << KGRN << "passed!" << RESET << std::endl; + } testRes->passed = true; goto after_debug_log; } @@ -297,6 +341,7 @@ after_debug_log: int main(int argc, char **argv) { ModSecurityTest test; ModSecurityTestResults results; + int test_number = 0; #ifdef WITH_GEOIP resources.push_back("geoip"); @@ -307,20 +352,24 @@ int main(int argc, char **argv) { << std::endl; #else test.cmd_options(argc, argv); - std::cout << test.header(); + if (!test.m_automake_output) { + std::cout << test.header(); + } test.load_tests(); - std::cout << std::setw(4) << std::right << "# "; - std::cout << std::setw(50) << std::left << "File Name"; - std::cout << std::setw(70) << std::left << "Test Name"; - std::cout << std::setw(10) << std::left << "Passed?"; - std::cout << std::endl; - std::cout << std::setw(4) << std::right << "--- "; - std::cout << std::setw(50) << std::left << "---------"; - std::cout << std::setw(70) << std::left << "---------"; - std::cout << std::setw(10) << std::left << "-------"; - std::cout << std::endl; + if (!test.m_automake_output) { + std::cout << std::setw(4) << std::right << "# "; + std::cout << std::setw(50) << std::left << "File Name"; + std::cout << std::setw(70) << std::left << "Test Name"; + std::cout << std::setw(10) << std::left << "Passed?"; + std::cout << std::endl; + std::cout << std::setw(4) << std::right << "--- "; + std::cout << std::setw(50) << std::left << "---------"; + std::cout << std::setw(70) << std::left << "---------"; + std::cout << std::setw(10) << std::left << "-------"; + std::cout << std::endl; + } int counter = 0; std::list keyList; @@ -331,8 +380,12 @@ int main(int argc, char **argv) { ModSecurityTestResults res; for (std::string &a : keyList) { - std::vector *tests = test[a]; - perform_unit_test(tests, &res, &counter); + test_number++; + if ((test.m_test_number == 0) + || (test.m_test_number != 0 && test_number == test.m_test_number)) { + std::vector *tests = test[a]; + perform_unit_test(&test, tests, &res, &counter); + } } std::cout << std::endl; @@ -347,31 +400,38 @@ int main(int argc, char **argv) { if (r->passed == true && r->skipped == false) { passed++; } else if (r->skipped == false) { - std::cout << KRED << "Test failed." << RESET << KWHT << " From: " \ - << RESET << r->test->filename << "." << std::endl; - std::cout << KWHT << "Test name: " << RESET << r->test->name \ - << "." << std::endl; - std::cout << KWHT << "Reason: " << RESET << std::endl; - std::cout << r->reason.str() << std::endl; + if (test.m_automake_output && 1 == 0) { + // m_automake_output + } else { + std::cout << KRED << "Test failed." << RESET << KWHT \ + << " From: " \ + << RESET << r->test->filename << "." << std::endl; + std::cout << KWHT << "Test name: " << RESET \ + << r->test->name \ + << "." << std::endl; + std::cout << KWHT << "Reason: " << RESET << std::endl; + std::cout << r->reason.str() << std::endl; + } failed++; } } - std::cout << "Ran a total of: " << std::to_string(failed + passed) \ - << " regression tests - "; - if (failed == 0) { - std::cout << KGRN << "All tests passed." << RESET; - } else { - std::cout << KRED << failed << " failed." << RESET; - } + if (!test.m_automake_output) { + std::cout << "Ran a total of: " << std::to_string(failed + passed) \ + << " regression tests - "; + if (failed == 0) { + std::cout << KGRN << "All tests passed." << RESET; + } else { + std::cout << KRED << failed << " failed." << RESET; + } - if (skipped > 0) { - std::cout << KCYN << " " << std::to_string(skipped) << " "; - std::cout << "skipped tests." << RESET << std::endl; - } else { - std::cout << std::endl; + if (skipped > 0) { + std::cout << KCYN << " " << std::to_string(skipped) << " "; + std::cout << "skipped tests." << RESET << std::endl; + } else { + std::cout << std::endl; + } } - for (std::pair *> a : test) { std::vector *vec = a.second; for (int i = 0; i < vec->size(); i++) { diff --git a/test/test-suite.sh b/test/test-suite.sh new file mode 100755 index 00000000..44ce996d --- /dev/null +++ b/test/test-suite.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +TEST=$1 1> /dev/null +cd test 1> /dev/null +if [[ $TEST == *"test-cases/regression/"* ]] +then + ./regression_tests ../$* +else + ./unit_tests ../$* +fi +cd - 1> /dev/null diff --git a/test/unit/unit.cc b/test/unit/unit.cc index 8b94734e..b4e006a2 100644 --- a/test/unit/unit.cc +++ b/test/unit/unit.cc @@ -28,7 +28,7 @@ #include "common/modsecurity_test_results.h" #include "common/colors.h" #include "unit/unit_test.h" - +#include "src/utils.h" using modsecurity_test::UnitTest; using modsecurity_test::ModSecurityTest; @@ -45,34 +45,53 @@ void print_help() { } -void perform_unit_test(UnitTest *t, ModSecurityTestResults* res) { +void perform_unit_test(ModSecurityTest *test, UnitTest *t, + ModSecurityTestResults* res) { const char *error = NULL; + if (test->m_automake_output) { + std::cout << ":test-result: "; + } + if (t->type == "op") { Operator *op = Operator::instantiate("\"@" + t->name + \ " " + t->param + "\""); op->init(t->filename, &error); int ret = op->evaluate(NULL, t->input); + t->obtained = ret; if (ret != t->ret) { - t->obtained = ret; res->push_back(t); + if (test->m_automake_output) { + std::cout << "FAIL "; + } + } else if (test->m_automake_output) { + std::cout << "PASS "; } - delete op; } else if (t->type == "tfn") { Transformation *tfn = Transformation::instantiate("t:" + t->name); std::string ret = tfn->evaluate(t->input, NULL); t->obtained = 1; + t->obtainedOutput = ret; if (ret != t->output) { - t->obtainedOutput = ret; res->push_back(t); + if (test->m_automake_output) { + std::cout << "FAIL "; + } + } else if (test->m_automake_output) { + std::cout << "PASS "; } - delete tfn; } else { std::cerr << "Failed. Test type is unknown: << " << t->type; std::cerr << std::endl; } + + if (test->m_automake_output) { + std::cout << t->name << " " + << modsecurity::toHexIfNeeded(t->input) << std::endl; + } + } @@ -82,7 +101,9 @@ int main(int argc, char **argv) { ModSecurityTestResults results; test.cmd_options(argc, argv); - std::cout << test.header(); + if (!test.m_automake_output) { + std::cout << test.header(); + } test.load_tests(); if (test.target == default_test_path) { @@ -96,33 +117,42 @@ int main(int argc, char **argv) { for (UnitTest *t : *tests) { ModSecurityTestResults r; - std::cout << " " << a.first << "...\t"; - - perform_unit_test(t, &r); - - if (r.size() == 0) { - std::cout << KGRN << r.size() << " tests failed."; - } else { - std::cout << KRED << r.size() << " tests failed."; + if (!test.m_automake_output) { + std::cout << " " << a.first << "...\t"; + } + perform_unit_test(&test, t, &r); + + if (!test.m_automake_output) { + if (r.size() == 0) { + std::cout << KGRN << r.size() << " tests failed."; + } else { + std::cout << KRED << r.size() << " tests failed."; + } + std::cout << RESET << std::endl; } - std::cout << RESET << std::endl; results.insert(results.end(), r.begin(), r.end()); } } - std::cout << "Total >> " << total << std::endl; + + if (!test.m_automake_output) { + std::cout << "Total >> " << total << std::endl; + } for (UnitTest *t : results) { std::cout << t->print() << std::endl; } - std::cout << std::endl; + if (!test.m_automake_output) { + std::cout << std::endl; + + std::cout << "Ran a total of: " << total << " unit tests - "; + if (results.size() == 0) { + std::cout << KGRN << "All tests passed" << RESET << std::endl; + } else { + std::cout << KRED << results.size() << " failed." << RESET << std::endl; + } - std::cout << "Ran a total of: " << total << " unit tests - "; - if (results.size() == 0) { - std::cout << KGRN << "All tests passed" << RESET << std::endl; - } else { - std::cout << KRED << results.size() << " failed." << RESET << std::endl; } for (std::pair *> a : test) {