mirror of
https://github.com/owasp-modsecurity/ModSecurity.git
synced 2025-08-13 13:26:01 +03:00
Adds support to make check
The regression and unit tests are now integrated with `make check`. It is possible to use make check -jN to have multiple tests running in parallel.
This commit is contained in:
Felipe Zimmerle
parent
2e3da7ea24
commit
f0155e3f32
176
Makefile.am
176
Makefile.am
@ -34,5 +34,179 @@ MAINTAINERCLEANFILES = \
|
||||
ltmain.sh \
|
||||
ylwrap \
|
||||
missing \
|
||||
depcomp
|
||||
depcomp
|
||||
|
||||
LOG_DRIVER = env $(SHELL) $(top_srcdir)/test/custom-test-driver
|
||||
AM_TESTS_ENVIRONMENT=AUTOMAKE_TESTS=true; export AUTOMAKE_TESTS;
|
||||
LOG_COMPILER=test/test-suite.sh
|
||||
# for i in `find test/test-cases -iname *.json`; do echo TESTS+=$i; done
|
||||
TESTS=
|
||||
TESTS+=test/test-cases/regression/actions.json
|
||||
TESTS+=test/test-cases/regression/debug_log.json
|
||||
TESTS+=test/test-cases/regression/config-include.json
|
||||
TESTS+=test/test-cases/regression/variable-PATH_INFO.json
|
||||
TESTS+=test/test-cases/regression/action-xmlns.json
|
||||
TESTS+=test/test-cases/regression/collection-tx.json
|
||||
TESTS+=test/test-cases/regression/config-body_limits.json
|
||||
TESTS+=test/test-cases/regression/variable-RESPONSE_BODY.json
|
||||
TESTS+=test/test-cases/regression/action-ctl_request_body_processor.json
|
||||
TESTS+=test/test-cases/regression/variable-QUERY_STRING.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_LINE.json
|
||||
TESTS+=test/test-cases/regression/variable-RESPONSE_HEADERS.json
|
||||
TESTS+=test/test-cases/regression/operator-rx.json
|
||||
TESTS+=test/test-cases/regression/variable-ARGS.json
|
||||
TESTS+=test/test-cases/regression/issue-394.json
|
||||
TESTS+=test/test-cases/regression/variable-TIME_MON.json
|
||||
TESTS+=test/test-cases/regression/misc.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_HEADERS.json
|
||||
TESTS+=test/test-cases/regression/variable-FULL_REQUEST_LENGTH.json
|
||||
TESTS+=test/test-cases/regression/variable-ARGS_NAMES.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_COOKIES_NAMES.json
|
||||
TESTS+=test/test-cases/regression/variable-ARGS_GET_NAMES.json
|
||||
TESTS+=test/test-cases/regression/variable-REMOTE_USER.json
|
||||
TESTS+=test/test-cases/regression/action-msg.json
|
||||
TESTS+=test/test-cases/regression/variable-UNIQUE_ID.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_URI.json
|
||||
TESTS+=test/test-cases/regression/request-body-parser-xml.json
|
||||
TESTS+=test/test-cases/regression/secaction.json
|
||||
TESTS+=test/test-cases/regression/variable-TIME_WDAY.json
|
||||
TESTS+=test/test-cases/regression/issue-960.json
|
||||
TESTS+=test/test-cases/regression/request-body-parser-xml-validade-dtd.json
|
||||
TESTS+=test/test-cases/regression/variable-TIME_MIN.json
|
||||
TESTS+=test/test-cases/regression/action-setuid.json
|
||||
TESTS+=test/test-cases/regression/config-calling_phases_by_name.json
|
||||
TESTS+=test/test-cases/regression/variable-USERID.json
|
||||
TESTS+=test/test-cases/regression/request-body-parser-multipart.json
|
||||
TESTS+=test/test-cases/regression/variable-RESPONSE_CONTENT_TYPE.json
|
||||
TESTS+=test/test-cases/regression/action-disruptive.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_BASENAME.json
|
||||
TESTS+=test/test-cases/regression/sec_component_signature.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_METHOD.json
|
||||
TESTS+=test/test-cases/regression/action-initcol.json
|
||||
TESTS+=test/test-cases/regression/variable-FILES.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_BODY.json
|
||||
TESTS+=test/test-cases/regression/request-body-parser-multipart-crlf.json
|
||||
TESTS+=test/test-cases/regression/variable-TIME_EPOCH.json
|
||||
TESTS+=test/test-cases/regression/variable-HIGHEST_SEVERITY.json
|
||||
TESTS+=test/test-cases/regression/variable-variation-exclusion.json
|
||||
TESTS+=test/test-cases/regression/variable-GEO.json
|
||||
TESTS+=test/test-cases/regression/variable-SESSIONID.json
|
||||
TESTS+=test/test-cases/regression/variable-SERVER_PORT.json
|
||||
TESTS+=test/test-cases/regression/variable-DURATION.json
|
||||
TESTS+=test/test-cases/regression/secruleengine.json
|
||||
TESTS+=test/test-cases/regression/variable-FILES_COMBINED_SIZE.json
|
||||
TESTS+=test/test-cases/regression/secmarker.json
|
||||
TESTS+=test/test-cases/regression/action-id.json
|
||||
TESTS+=test/test-cases/regression/variable-AUTH_TYPE.json
|
||||
TESTS+=test/test-cases/regression/variable-ARGS_COMBINED_SIZE.json
|
||||
TESTS+=test/test-cases/regression/variable-REMOTE_PORT.json
|
||||
TESTS+=test/test-cases/regression/variable-TIME_SEC.json
|
||||
TESTS+=test/test-cases/regression/config-secremoterules.json
|
||||
TESTS+=test/test-cases/regression/variable-FILES_NAMES.json
|
||||
TESTS+=test/test-cases/regression/variable-MULTIPART_CRLF_LF_LINES.json
|
||||
TESTS+=test/test-cases/regression/variable-MULTIPART_UNMATCHED_BOUNDARY.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_HEADERS_NAMES.json
|
||||
TESTS+=test/test-cases/regression/variable-INBOUND_DATA_ERROR.json
|
||||
TESTS+=test/test-cases/regression/action-setsid.json
|
||||
TESTS+=test/test-cases/regression/variable-MATCHED_VARS.json
|
||||
TESTS+=test/test-cases/regression/operator-ipMatchFromFile.json
|
||||
TESTS+=test/test-cases/regression/variable-FILES_SIZES.json
|
||||
TESTS+=test/test-cases/regression/variable-MATCHED_VAR_NAME.json
|
||||
TESTS+=test/test-cases/regression/collection-tx-with-macro.json
|
||||
TESTS+=test/test-cases/regression/variable-MODSEC_BUILD.json
|
||||
TESTS+=test/test-cases/regression/variable-ARGS_POST_NAMES.json
|
||||
TESTS+=test/test-cases/regression/variable-REMOTE_ADDR.json
|
||||
TESTS+=test/test-cases/regression/auditlog.json
|
||||
TESTS+=test/test-cases/regression/config-include-bad.json
|
||||
TESTS+=test/test-cases/regression/variable-OUTBOUND_DATA_ERROR.json
|
||||
TESTS+=test/test-cases/regression/config-xml_external_entity.json
|
||||
TESTS+=test/test-cases/regression/variable-MATCHED_VAR.json
|
||||
TESTS+=test/test-cases/regression/variable-FULL_REQUEST.json
|
||||
TESTS+=test/test-cases/regression/variable-TIME_HOUR.json
|
||||
TESTS+=test/test-cases/regression/variable-ARGS_POST.json
|
||||
TESTS+=test/test-cases/regression/variable-variation-count.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_URI_RAW.json
|
||||
TESTS+=test/test-cases/regression/variable-TIME_YEAR.json
|
||||
TESTS+=test/test-cases/regression/variable-RESPONSE_HEADERS_NAMES.json
|
||||
TESTS+=test/test-cases/regression/variable-MULTIPART_NAME.json
|
||||
TESTS+=test/test-cases/regression/variable-ARGS_GET.json
|
||||
TESTS+=test/test-cases/regression/transformations.json
|
||||
TESTS+=test/test-cases/regression/config-response_type.json
|
||||
TESTS+=test/test-cases/regression/variable-TIME.json
|
||||
TESTS+=test/test-cases/regression/variable-MATCHED_VARS_NAMES.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_COOKIES.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_FILENAME.json
|
||||
TESTS+=test/test-cases/regression/variable-RESPONSE_CONTENT_LENGTH.json
|
||||
TESTS+=test/test-cases/regression/variable-REMOTE_HOST.json
|
||||
TESTS+=test/test-cases/regression/variable-MULTIPART_STRICT_ERROR.json
|
||||
TESTS+=test/test-cases/regression/variable-SERVER_ADDR.json
|
||||
TESTS+=test/test-cases/regression/variable-MULTIPART_FILENAME.json
|
||||
TESTS+=test/test-cases/regression/variable-MULTIPART_INVALID_HEADER_FOLDING.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_BODY_LENGTH.json
|
||||
TESTS+=test/test-cases/regression/config-secdefaultaction.json
|
||||
TESTS+=test/test-cases/regression/variable-REQUEST_PROTOCOL.json
|
||||
TESTS+=test/test-cases/regression/variable-TIME_DAY.json
|
||||
TESTS+=test/test-cases/regression/transformation-none.json
|
||||
TESTS+=test/test-cases/regression/variable-TX.json
|
||||
TESTS+=test/test-cases/regression/variable-ENV.json
|
||||
TESTS+=test/test-cases/regression/action-tag.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/base64Encode.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/trimRight.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/parityEven7bit.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/length.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/urlDecodeUni.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/base64Decode.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/utf8toUnicode.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/cssDecode.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/trim.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/ge.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/replaceComments.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/htmlEntityDecode.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/escapeSeqDecode.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/parityZero7bit.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/replaceNulls.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/sqlHexDecode.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/md5.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/removeWhitespace.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/trimLeft.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/hexDecode.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/hexEncode.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/normalisePathWin.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/cmdLine.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/parityOdd7bit.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/urlDecode.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/base64DecodeExt.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/lowercase.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/sha1.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/normalisePath.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/jsDecode.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/removeNulls.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/removeCommentsChar.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/removeComments.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/urlEncode.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/transformations/compressWhitespace.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/geoLookup.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/validateUrlEncoding.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/endsWith.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/ge.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/gt.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/contains.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/validateUtf8Encoding.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/detectSQLi.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/validateByteRange.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/pmFromFile.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/streq.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/le.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/within.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/noMatch.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/unconditionalMatch.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/beginsWith.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/containsWord.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/pm.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/verifyCC.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/rx.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/lt.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/ipMatch.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/strmatch.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/detectXSS.json
|
||||
TESTS+=test/test-cases/secrules-language-tests/operators/eq.json
|
||||
|
||||
@ -116,7 +116,6 @@ CHECK_LIBXML2
|
||||
CHECK_PCRE
|
||||
|
||||
|
||||
|
||||
# Checks for header files.
|
||||
AC_HEADER_STDC
|
||||
AC_CHECK_HEADERS([string])
|
||||
@ -244,6 +243,11 @@ fi
|
||||
|
||||
|
||||
AM_CONDITIONAL([TEST_UTILITIES], [test $buildTestUtilities = true])
|
||||
if test $buildTestUtilities = true; then
|
||||
if test $debugLogs = true; then
|
||||
TEST_CASES=`./test/test-list.sh`
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
# General link options
|
||||
|
||||
@ -136,34 +136,28 @@ std::pair<std::string, std::vector<T *>>* ModSecurityTest<T>::load_tests() {
|
||||
|
||||
template <class T>
|
||||
void ModSecurityTest<T>::cmd_options(int argc, char **argv) {
|
||||
#if HAS_GETOPT
|
||||
int option_char;
|
||||
GetOpt getopt(argc, argv, "hvct:");
|
||||
|
||||
while ((option_char = getopt()) != EOF) {
|
||||
switch (option_char) {
|
||||
case 'h':
|
||||
print_help();
|
||||
return;
|
||||
break;
|
||||
case 'v':
|
||||
this->verbose = true;
|
||||
break;
|
||||
case 'c':
|
||||
this->color = false;
|
||||
break;
|
||||
case 't':
|
||||
this->target_folder = getopt.optarg;
|
||||
break;
|
||||
}
|
||||
int i = 1;
|
||||
if (argc > i && strcmp(argv[i], "automake") == 0) {
|
||||
i++;
|
||||
m_automake_output = true;
|
||||
}
|
||||
#else
|
||||
if (argv[1]) {
|
||||
this->target = argv[1];
|
||||
|
||||
if(const char* env_p = std::getenv("AUTOMAKE_TESTS")) {
|
||||
m_automake_output = true;
|
||||
}
|
||||
|
||||
if (argc > i && argv[i]) {
|
||||
this->target = argv[i];
|
||||
size_t pos = this->target.find(":");
|
||||
if (pos != std::string::npos) {
|
||||
std::string test_numbers = std::string(this->target, pos + 1,
|
||||
this->target.length() - pos);
|
||||
this->target = std::string(this->target, 0, pos);
|
||||
m_test_number = std::atoi(test_numbers.c_str());
|
||||
}
|
||||
} else {
|
||||
this->target = default_test_path;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace modsecurity_test
|
||||
|
||||
@ -31,6 +31,10 @@ namespace modsecurity_test {
|
||||
template <class T> class ModSecurityTest :
|
||||
public std::unordered_map<std::string, std::vector<T *> *> {
|
||||
public:
|
||||
ModSecurityTest()
|
||||
: m_test_number(0),
|
||||
m_automake_output(false) { }
|
||||
|
||||
std::string header();
|
||||
void cmd_options(int, char **);
|
||||
std::pair<std::string, std::vector<T *>>* load_tests();
|
||||
@ -40,6 +44,8 @@ template <class T> class ModSecurityTest :
|
||||
std::string target;
|
||||
bool verbose = false;
|
||||
bool color = false;
|
||||
int m_test_number;
|
||||
bool m_automake_output;
|
||||
};
|
||||
|
||||
} // namespace modsecurity_test
|
||||
|
||||
133
test/custom-test-driver
Executable file
133
test/custom-test-driver
Executable file
@ -0,0 +1,133 @@
|
||||
#! /bin/sh
|
||||
# test-driver - basic testsuite driver script.
|
||||
|
||||
scriptversion=2013-07-13.22-modsec; # UTC
|
||||
|
||||
# Copyright (C) 2011-2014 Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2, or (at your option)
|
||||
# any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# As a special exception to the GNU General Public License, if you
|
||||
# distribute this file as part of a program that contains a
|
||||
# configuration script generated by Autoconf, you may include it under
|
||||
# the same distribution terms that you use for the rest of that program.
|
||||
|
||||
# This file is maintained in Automake, please report
|
||||
# bugs to <bug-automake@gnu.org> or send patches to
|
||||
# <automake-patches@gnu.org>.
|
||||
|
||||
# Make unconditional expansion of undefined variables an error. This
|
||||
# helps a lot in preventing typo-related bugs.
|
||||
set -u
|
||||
|
||||
usage_error ()
|
||||
{
|
||||
echo "$0: $*" >&2
|
||||
print_usage >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
print_usage ()
|
||||
{
|
||||
cat <<END
|
||||
Usage:
|
||||
test-driver --test-name=NAME --log-file=PATH --trs-file=PATH
|
||||
[--expect-failure={yes|no}] [--color-tests={yes|no}]
|
||||
[--enable-hard-errors={yes|no}] [--]
|
||||
TEST-SCRIPT [TEST-SCRIPT-ARGUMENTS]
|
||||
The '--test-name', '--log-file' and '--trs-file' options are mandatory.
|
||||
END
|
||||
}
|
||||
|
||||
test_name= # Used for reporting.
|
||||
log_file= # Where to save the output of the test script.
|
||||
trs_file= # Where to save the metadata of the test run.
|
||||
expect_failure=no
|
||||
color_tests=no
|
||||
enable_hard_errors=yes
|
||||
while test $# -gt 0; do
|
||||
case $1 in
|
||||
--help) print_usage; exit $?;;
|
||||
--version) echo "test-driver $scriptversion"; exit $?;;
|
||||
--test-name) test_name=$2; shift;;
|
||||
--log-file) log_file=$2; shift;;
|
||||
--trs-file) trs_file=$2; shift;;
|
||||
--color-tests) color_tests=$2; shift;;
|
||||
--expect-failure) expect_failure=$2; shift;;
|
||||
--enable-hard-errors) enable_hard_errors=$2; shift;;
|
||||
--) shift; break;;
|
||||
-*) usage_error "invalid option: '$1'";;
|
||||
*) break;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
missing_opts=
|
||||
test x"$test_name" = x && missing_opts="$missing_opts --test-name"
|
||||
test x"$log_file" = x && missing_opts="$missing_opts --log-file"
|
||||
test x"$trs_file" = x && missing_opts="$missing_opts --trs-file"
|
||||
if test x"$missing_opts" != x; then
|
||||
usage_error "the following mandatory options are missing:$missing_opts"
|
||||
fi
|
||||
|
||||
if test $# -eq 0; then
|
||||
usage_error "missing argument"
|
||||
fi
|
||||
|
||||
if test $color_tests = yes; then
|
||||
# Keep this in sync with 'lib/am/check.am:$(am__tty_colors)'.
|
||||
red='[0;31m' # Red.
|
||||
grn='[0;32m' # Green.
|
||||
lgn='[1;32m' # Light green.
|
||||
blu='[1;34m' # Blue.
|
||||
mgn='[0;35m' # Magenta.
|
||||
wht='[1;40m' # White.
|
||||
std='[m' # No color.
|
||||
else
|
||||
red= grn= lgn= blu= mgn= std=
|
||||
fi
|
||||
|
||||
do_exit='rm -f $log_file $trs_file; (exit $st); exit $st'
|
||||
trap "st=129; $do_exit" 1
|
||||
trap "st=130; $do_exit" 2
|
||||
trap "st=141; $do_exit" 13
|
||||
trap "st=143; $do_exit" 15
|
||||
|
||||
# Test script is run here.
|
||||
"$@" >$log_file 2>&1
|
||||
estatus=$?
|
||||
cat $log_file >> $trs_file
|
||||
tfail=`cat $log_file | egrep "^:test-result: FAIL" | wc -l`
|
||||
tfail=`printf "%3d" $tfail`
|
||||
tpass=`cat $log_file | egrep "^:test-result: PASS" | wc -l`
|
||||
tpass=`printf "%4d" $tpass`
|
||||
ttotal=`cat $log_file | egrep "^:test-result: " | wc -l`
|
||||
ttotal=`printf "%4d" $ttotal`
|
||||
|
||||
# Report outcome to console.
|
||||
if test ${tfail} -eq 0; then
|
||||
echo "(${grn}${tpass}$std/${red}${tfail}$std/${wht}${ttotal}${std}): $test_name"
|
||||
else
|
||||
echo "(${grn}${tpass}$std/${red}${tfail}$std/${wht}${ttotal}${std}): ${red}$test_name${std}"
|
||||
fi
|
||||
|
||||
# Local Variables:
|
||||
# mode: shell-script
|
||||
# sh-indentation: 2
|
||||
# eval: (add-hook 'write-file-hooks 'time-stamp)
|
||||
# time-stamp-start: "scriptversion="
|
||||
# time-stamp-format: "%:y-%02m-%02d.%02H"
|
||||
# time-stamp-time-zone: "UTC"
|
||||
# time-stamp-end: "; # UTC"
|
||||
# End:
|
||||
@ -75,7 +75,8 @@ void logCb(void *data, const char *msg) {
|
||||
}
|
||||
|
||||
|
||||
void perform_unit_test(std::vector<RegressionTest *> *tests,
|
||||
void perform_unit_test(ModSecurityTest<RegressionTest> *test,
|
||||
std::vector<RegressionTest *> *tests,
|
||||
ModSecurityTestResults<RegressionTestResult> *res, int *count) {
|
||||
|
||||
|
||||
@ -99,10 +100,13 @@ void perform_unit_test(std::vector<RegressionTest *> *tests,
|
||||
} else {
|
||||
filename = t->filename;
|
||||
}
|
||||
std::cout << std::setw(3) << std::right <<
|
||||
std::to_string(*count) << " ";
|
||||
std::cout << std::setw(50) << std::left << filename;
|
||||
std::cout << std::setw(70) << std::left << t->name;
|
||||
|
||||
if (!test->m_automake_output) {
|
||||
std::cout << std::setw(3) << std::right <<
|
||||
std::to_string(*count) << " ";
|
||||
std::cout << std::setw(50) << std::left << filename;
|
||||
std::cout << std::setw(70) << std::left << t->name;
|
||||
}
|
||||
|
||||
modsec = new modsecurity::ModSecurity();
|
||||
modsec->setConnectorInformation("ModSecurity-regression v0.0.1-alpha" \
|
||||
@ -123,7 +127,12 @@ void perform_unit_test(std::vector<RegressionTest *> *tests,
|
||||
testRes->reason << KCYN << "compiled with support " << std::endl;
|
||||
testRes->reason << KCYN << "to: " << t->resource << std::endl;
|
||||
testRes->reason << RESET << std::endl;
|
||||
std::cout << KCYN << "skipped!" << RESET << std::endl;
|
||||
if (test->m_automake_output) {
|
||||
std::cout << ":test-result: SKIP " << filename \
|
||||
<< ":" << t->name << std::endl;
|
||||
} else {
|
||||
std::cout << KCYN << "skipped!" << RESET << std::endl;
|
||||
}
|
||||
res->push_back(testRes);
|
||||
continue;
|
||||
}
|
||||
@ -135,7 +144,12 @@ void perform_unit_test(std::vector<RegressionTest *> *tests,
|
||||
* Not expecting any error, thus return the error to
|
||||
* the user.
|
||||
*/
|
||||
std::cout << KRED << "failed!" << RESET << std::endl;
|
||||
if (test->m_automake_output) {
|
||||
std::cout << ":test-result: FAIL " << filename \
|
||||
<< ":" << t->name << std::endl;
|
||||
} else {
|
||||
std::cout << KRED << "failed!" << RESET << std::endl;
|
||||
}
|
||||
testRes->reason << KRED << "parse failed." << RESET \
|
||||
<< std::endl;
|
||||
testRes->reason << modsec_rules->getParserError() \
|
||||
@ -150,7 +164,12 @@ void perform_unit_test(std::vector<RegressionTest *> *tests,
|
||||
std::string s = modsec_rules->getParserError();
|
||||
|
||||
if (regex_search(s, &match, re) && match.size() >= 1) {
|
||||
std::cout << KGRN << "passed!" << RESET << std::endl;
|
||||
if (test->m_automake_output) {
|
||||
std::cout << ":test-result: PASS " << filename \
|
||||
<< ":" << t->name << std::endl;
|
||||
} else {
|
||||
std::cout << KGRN << "passed!" << RESET << std::endl;
|
||||
}
|
||||
/* Parser error was expected, thus, the test passed. */
|
||||
testRes->reason << KGRN << "passed!" << RESET << std::endl;
|
||||
testRes->passed = true;
|
||||
@ -158,7 +177,12 @@ void perform_unit_test(std::vector<RegressionTest *> *tests,
|
||||
continue;
|
||||
} else {
|
||||
/* Parser error was expected, but with a different content */
|
||||
std::cout << KRED << "failed!" << RESET << std::endl;
|
||||
if (test->m_automake_output) {
|
||||
std::cout << ":test-result: FAIL " << filename \
|
||||
<< ":" << t->name << std::endl;
|
||||
} else {
|
||||
std::cout << KRED << "failed!" << RESET << std::endl;
|
||||
}
|
||||
|
||||
testRes->reason << KRED << "failed!" << RESET << std::endl;
|
||||
testRes->reason << KWHT << "Expected a parser error." \
|
||||
@ -174,11 +198,16 @@ void perform_unit_test(std::vector<RegressionTest *> *tests,
|
||||
} else {
|
||||
/* Parser error was expected but never happened */
|
||||
if (t->parser_error.empty() == false) {
|
||||
std::cout << KRED << "failed!" << RESET << std::endl;
|
||||
std::cout << KWHT << "Expected a parser error." \
|
||||
<< RESET << std::endl;
|
||||
std::cout << KWHT << "Expected: " << RESET \
|
||||
<< t->parser_error << std::endl;
|
||||
if (test->m_automake_output) {
|
||||
std::cout << ":test-result: FAIL " << filename \
|
||||
<< ":" << t->name << std::endl;
|
||||
} else {
|
||||
std::cout << KRED << "failed!" << RESET << std::endl;
|
||||
std::cout << KWHT << "Expected a parser error." \
|
||||
<< RESET << std::endl;
|
||||
std::cout << KWHT << "Expected: " << RESET \
|
||||
<< t->parser_error << std::endl;
|
||||
}
|
||||
testRes->passed = false;
|
||||
res->push_back(testRes);
|
||||
continue;
|
||||
@ -254,20 +283,35 @@ end:
|
||||
|
||||
if (d != NULL) {
|
||||
if (!d->contains(t->debug_log)) {
|
||||
std::cout << KRED << "failed!" << RESET << std::endl;
|
||||
if (test->m_automake_output) {
|
||||
std::cout << ":test-result: FAIL " << filename \
|
||||
<< ":" << t->name << std::endl;
|
||||
} else {
|
||||
std::cout << KRED << "failed!" << RESET << std::endl;
|
||||
}
|
||||
testRes->reason << "Debug log was not matching the " \
|
||||
<< "expected results." << std::endl;
|
||||
testRes->reason << KWHT << "Expecting: " << RESET \
|
||||
<< t->debug_log + ".";
|
||||
testRes->passed = false;
|
||||
} else if (r.status != t->http_code) {
|
||||
std::cout << KRED << "failed!" << RESET << std::endl;
|
||||
if (test->m_automake_output) {
|
||||
std::cout << ":test-result: FAIL " << filename \
|
||||
<< ":" << t->name << std::endl;
|
||||
} else {
|
||||
std::cout << KRED << "failed!" << RESET << std::endl;
|
||||
}
|
||||
testRes->reason << "HTTP code mismatch. expecting: " + \
|
||||
std::to_string(t->http_code) + \
|
||||
" got: " + std::to_string(r.status) + "\n";
|
||||
testRes->passed = false;
|
||||
} else {
|
||||
std::cout << KGRN << "passed!" << RESET << std::endl;
|
||||
if (test->m_automake_output) {
|
||||
std::cout << ":test-result: PASS " << filename \
|
||||
<< ":" << t->name << std::endl;
|
||||
} else {
|
||||
std::cout << KGRN << "passed!" << RESET << std::endl;
|
||||
}
|
||||
testRes->passed = true;
|
||||
goto after_debug_log;
|
||||
}
|
||||
@ -297,6 +341,7 @@ after_debug_log:
|
||||
int main(int argc, char **argv) {
|
||||
ModSecurityTest<RegressionTest> test;
|
||||
ModSecurityTestResults<RegressionTest> results;
|
||||
int test_number = 0;
|
||||
|
||||
#ifdef WITH_GEOIP
|
||||
resources.push_back("geoip");
|
||||
@ -307,20 +352,24 @@ int main(int argc, char **argv) {
|
||||
<< std::endl;
|
||||
#else
|
||||
test.cmd_options(argc, argv);
|
||||
std::cout << test.header();
|
||||
if (!test.m_automake_output) {
|
||||
std::cout << test.header();
|
||||
}
|
||||
|
||||
test.load_tests();
|
||||
|
||||
std::cout << std::setw(4) << std::right << "# ";
|
||||
std::cout << std::setw(50) << std::left << "File Name";
|
||||
std::cout << std::setw(70) << std::left << "Test Name";
|
||||
std::cout << std::setw(10) << std::left << "Passed?";
|
||||
std::cout << std::endl;
|
||||
std::cout << std::setw(4) << std::right << "--- ";
|
||||
std::cout << std::setw(50) << std::left << "---------";
|
||||
std::cout << std::setw(70) << std::left << "---------";
|
||||
std::cout << std::setw(10) << std::left << "-------";
|
||||
std::cout << std::endl;
|
||||
if (!test.m_automake_output) {
|
||||
std::cout << std::setw(4) << std::right << "# ";
|
||||
std::cout << std::setw(50) << std::left << "File Name";
|
||||
std::cout << std::setw(70) << std::left << "Test Name";
|
||||
std::cout << std::setw(10) << std::left << "Passed?";
|
||||
std::cout << std::endl;
|
||||
std::cout << std::setw(4) << std::right << "--- ";
|
||||
std::cout << std::setw(50) << std::left << "---------";
|
||||
std::cout << std::setw(70) << std::left << "---------";
|
||||
std::cout << std::setw(10) << std::left << "-------";
|
||||
std::cout << std::endl;
|
||||
}
|
||||
int counter = 0;
|
||||
|
||||
std::list<std::string> keyList;
|
||||
@ -331,8 +380,12 @@ int main(int argc, char **argv) {
|
||||
|
||||
ModSecurityTestResults<RegressionTestResult> res;
|
||||
for (std::string &a : keyList) {
|
||||
std::vector<RegressionTest *> *tests = test[a];
|
||||
perform_unit_test(tests, &res, &counter);
|
||||
test_number++;
|
||||
if ((test.m_test_number == 0)
|
||||
|| (test.m_test_number != 0 && test_number == test.m_test_number)) {
|
||||
std::vector<RegressionTest *> *tests = test[a];
|
||||
perform_unit_test(&test, tests, &res, &counter);
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << std::endl;
|
||||
@ -347,31 +400,38 @@ int main(int argc, char **argv) {
|
||||
if (r->passed == true && r->skipped == false) {
|
||||
passed++;
|
||||
} else if (r->skipped == false) {
|
||||
std::cout << KRED << "Test failed." << RESET << KWHT << " From: " \
|
||||
<< RESET << r->test->filename << "." << std::endl;
|
||||
std::cout << KWHT << "Test name: " << RESET << r->test->name \
|
||||
<< "." << std::endl;
|
||||
std::cout << KWHT << "Reason: " << RESET << std::endl;
|
||||
std::cout << r->reason.str() << std::endl;
|
||||
if (test.m_automake_output && 1 == 0) {
|
||||
// m_automake_output
|
||||
} else {
|
||||
std::cout << KRED << "Test failed." << RESET << KWHT \
|
||||
<< " From: " \
|
||||
<< RESET << r->test->filename << "." << std::endl;
|
||||
std::cout << KWHT << "Test name: " << RESET \
|
||||
<< r->test->name \
|
||||
<< "." << std::endl;
|
||||
std::cout << KWHT << "Reason: " << RESET << std::endl;
|
||||
std::cout << r->reason.str() << std::endl;
|
||||
}
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "Ran a total of: " << std::to_string(failed + passed) \
|
||||
<< " regression tests - ";
|
||||
if (failed == 0) {
|
||||
std::cout << KGRN << "All tests passed." << RESET;
|
||||
} else {
|
||||
std::cout << KRED << failed << " failed." << RESET;
|
||||
}
|
||||
if (!test.m_automake_output) {
|
||||
std::cout << "Ran a total of: " << std::to_string(failed + passed) \
|
||||
<< " regression tests - ";
|
||||
if (failed == 0) {
|
||||
std::cout << KGRN << "All tests passed." << RESET;
|
||||
} else {
|
||||
std::cout << KRED << failed << " failed." << RESET;
|
||||
}
|
||||
|
||||
if (skipped > 0) {
|
||||
std::cout << KCYN << " " << std::to_string(skipped) << " ";
|
||||
std::cout << "skipped tests." << RESET << std::endl;
|
||||
} else {
|
||||
std::cout << std::endl;
|
||||
if (skipped > 0) {
|
||||
std::cout << KCYN << " " << std::to_string(skipped) << " ";
|
||||
std::cout << "skipped tests." << RESET << std::endl;
|
||||
} else {
|
||||
std::cout << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
for (std::pair<std::string, std::vector<RegressionTest *> *> a : test) {
|
||||
std::vector<RegressionTest *> *vec = a.second;
|
||||
for (int i = 0; i < vec->size(); i++) {
|
||||
|
||||
11
test/test-suite.sh
Executable file
11
test/test-suite.sh
Executable file
@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
|
||||
TEST=$1 1> /dev/null
|
||||
cd test 1> /dev/null
|
||||
if [[ $TEST == *"test-cases/regression/"* ]]
|
||||
then
|
||||
./regression_tests ../$*
|
||||
else
|
||||
./unit_tests ../$*
|
||||
fi
|
||||
cd - 1> /dev/null
|
||||
@ -28,7 +28,7 @@
|
||||
#include "common/modsecurity_test_results.h"
|
||||
#include "common/colors.h"
|
||||
#include "unit/unit_test.h"
|
||||
|
||||
#include "src/utils.h"
|
||||
|
||||
using modsecurity_test::UnitTest;
|
||||
using modsecurity_test::ModSecurityTest;
|
||||
@ -45,34 +45,53 @@ void print_help() {
|
||||
}
|
||||
|
||||
|
||||
void perform_unit_test(UnitTest *t, ModSecurityTestResults<UnitTest>* res) {
|
||||
void perform_unit_test(ModSecurityTest<UnitTest> *test, UnitTest *t,
|
||||
ModSecurityTestResults<UnitTest>* res) {
|
||||
const char *error = NULL;
|
||||
|
||||
if (test->m_automake_output) {
|
||||
std::cout << ":test-result: ";
|
||||
}
|
||||
|
||||
if (t->type == "op") {
|
||||
Operator *op = Operator::instantiate("\"@" + t->name + \
|
||||
" " + t->param + "\"");
|
||||
op->init(t->filename, &error);
|
||||
int ret = op->evaluate(NULL, t->input);
|
||||
t->obtained = ret;
|
||||
if (ret != t->ret) {
|
||||
t->obtained = ret;
|
||||
res->push_back(t);
|
||||
if (test->m_automake_output) {
|
||||
std::cout << "FAIL ";
|
||||
}
|
||||
} else if (test->m_automake_output) {
|
||||
std::cout << "PASS ";
|
||||
}
|
||||
|
||||
delete op;
|
||||
} else if (t->type == "tfn") {
|
||||
Transformation *tfn = Transformation::instantiate("t:" + t->name);
|
||||
std::string ret = tfn->evaluate(t->input, NULL);
|
||||
t->obtained = 1;
|
||||
t->obtainedOutput = ret;
|
||||
if (ret != t->output) {
|
||||
t->obtainedOutput = ret;
|
||||
res->push_back(t);
|
||||
if (test->m_automake_output) {
|
||||
std::cout << "FAIL ";
|
||||
}
|
||||
} else if (test->m_automake_output) {
|
||||
std::cout << "PASS ";
|
||||
}
|
||||
|
||||
delete tfn;
|
||||
} else {
|
||||
std::cerr << "Failed. Test type is unknown: << " << t->type;
|
||||
std::cerr << std::endl;
|
||||
}
|
||||
|
||||
if (test->m_automake_output) {
|
||||
std::cout << t->name << " "
|
||||
<< modsecurity::toHexIfNeeded(t->input) << std::endl;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -82,7 +101,9 @@ int main(int argc, char **argv) {
|
||||
ModSecurityTestResults<UnitTest> results;
|
||||
|
||||
test.cmd_options(argc, argv);
|
||||
std::cout << test.header();
|
||||
if (!test.m_automake_output) {
|
||||
std::cout << test.header();
|
||||
}
|
||||
|
||||
test.load_tests();
|
||||
if (test.target == default_test_path) {
|
||||
@ -96,33 +117,42 @@ int main(int argc, char **argv) {
|
||||
for (UnitTest *t : *tests) {
|
||||
ModSecurityTestResults<UnitTest> r;
|
||||
|
||||
std::cout << " " << a.first << "...\t";
|
||||
|
||||
perform_unit_test(t, &r);
|
||||
|
||||
if (r.size() == 0) {
|
||||
std::cout << KGRN << r.size() << " tests failed.";
|
||||
} else {
|
||||
std::cout << KRED << r.size() << " tests failed.";
|
||||
if (!test.m_automake_output) {
|
||||
std::cout << " " << a.first << "...\t";
|
||||
}
|
||||
perform_unit_test(&test, t, &r);
|
||||
|
||||
if (!test.m_automake_output) {
|
||||
if (r.size() == 0) {
|
||||
std::cout << KGRN << r.size() << " tests failed.";
|
||||
} else {
|
||||
std::cout << KRED << r.size() << " tests failed.";
|
||||
}
|
||||
std::cout << RESET << std::endl;
|
||||
}
|
||||
std::cout << RESET << std::endl;
|
||||
|
||||
results.insert(results.end(), r.begin(), r.end());
|
||||
}
|
||||
}
|
||||
std::cout << "Total >> " << total << std::endl;
|
||||
|
||||
if (!test.m_automake_output) {
|
||||
std::cout << "Total >> " << total << std::endl;
|
||||
}
|
||||
|
||||
for (UnitTest *t : results) {
|
||||
std::cout << t->print() << std::endl;
|
||||
}
|
||||
|
||||
std::cout << std::endl;
|
||||
if (!test.m_automake_output) {
|
||||
std::cout << std::endl;
|
||||
|
||||
std::cout << "Ran a total of: " << total << " unit tests - ";
|
||||
if (results.size() == 0) {
|
||||
std::cout << KGRN << "All tests passed" << RESET << std::endl;
|
||||
} else {
|
||||
std::cout << KRED << results.size() << " failed." << RESET << std::endl;
|
||||
}
|
||||
|
||||
std::cout << "Ran a total of: " << total << " unit tests - ";
|
||||
if (results.size() == 0) {
|
||||
std::cout << KGRN << "All tests passed" << RESET << std::endl;
|
||||
} else {
|
||||
std::cout << KRED << results.size() << " failed." << RESET << std::endl;
|
||||
}
|
||||
|
||||
for (std::pair<std::string, std::vector<UnitTest *> *> a : test) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user