Configure test fixture using CTest for Windows build

- Added new test/test_suite.in with list of regression and unit tests
  previously in Makefile.am, to be shared between Unix and Windows
  builds.
- Updated regression.cc & unit.cc to return the number of failed tests
  to indicate to CTest that the test failed. Similarly, a crash or
  unhandled exception terminates the process with a non-zero exit code.
  - This change doesn't affect running the tests with autotest in Unix
    builds because this processes test output from custom-test-driver &
    test-suite.sh, and ignores the exit code of the test runner.
- Removed comment in test/test-cases/regression-offset-variable.json as
  this is not supported by JSON and prevents strict parsers to read and
  process the file.
- Minor change in regression.cc's clearAuditLog to replace std::ifstream
  with std::ofstream as the mode to open the flag applies to an output
  stream.
- Minor change in unit.cc to simplify code that deletes tests.
- Minor changes to test/custom-test-driver to correct usage information.

build/win32/CMakeLists.txt

@@ -151,6 +151,39 @@ add_regression_test_capability(WITH_CURL HAVE_CURL)
 add_regression_test_capability(WITH_LMDB HAVE_LMDB)
 add_regression_test_capability(WITH_MAXMIND HAVE_MAXMIND)
 
+enable_testing()
+
+file(READ ${BASE_DIR}/test/test-suite.in TEST_FILES_RAW)
+string(REPLACE "\n" ";" TEST_FILES ${TEST_FILES_RAW})
+
+foreach(TEST_FILE ${TEST_FILES})
+  # ignore comment lines
+  string(FIND ${TEST_FILE} "#" is_comment)
+  if(NOT is_comment EQUAL 0)
+    string(FIND ${TEST_FILE} "TESTS+=" is_valid_prefix)
+    if(NOT is_valid_prefix EQUAL 0)
+      message(FATAL_ERROR "Invalid prefix in line: ${TEST_FILE}")
+    endif()
+
+    # remove 'TESTS+=' prefix and 'test/' too because tests are launched
+    # from that directory
+    string(SUBSTRING ${TEST_FILE} 12 -1 TEST_FILE)
+
+    # test name
+    get_filename_component(TEST_NAME ${TEST_FILE} NAME_WE)
+
+    # determine test runner based on test path prefix
+    string(FIND ${TEST_FILE} "test-cases/regression/" is_regression_test)
+    if(is_regression_test EQUAL 0)
+        set(TEST_RUNNER "regression_tests")
+    else()
+        set(TEST_RUNNER "unit_tests")
+    endif()
+
+    add_test(NAME ${TEST_NAME} COMMAND ${TEST_RUNNER} ${TEST_FILE} WORKING_DIRECTORY ${BASE_DIR}/test)
+  endif()
+endforeach()
+
 # benchmark
 add_executable(benchmark ${BASE_DIR}/test/benchmark/benchmark.cc)
 setTestTargetProperties(benchmark)

build/win32/docker/Dockerfile

@@ -77,6 +77,35 @@ ARG USE_ASAN=
 
 RUN C:\BuildTools\VC\Auxiliary\Build\vcvars64.bat && vcbuild.bat %BUILD_TYPE% %ARCH% %USE_ASAN%
 
+# test suite
+#
+
+# setup test environment
+RUN cmd.exe /C md \tmp
+RUN cmd.exe /C md \bin
+RUN cmd.exe /C copy "C:\Program Files\GIT\usr\bin" \bin > NUL
+RUN cmd.exe /C copy "C:\Program Files\GIT\usr\bin\echo.exe" \bin\echo > NUL
+
+# disable tests that don't work on windows
+ARG JQ_VERSION=1.7.1
+ARG JQ_BINARY=jq-windows-amd64.exe
+ARG JQ_URL=https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/${JQ_BINARY}
+
+ARG JQ_BIN=C:\TEMP\jq.exe
+ADD ${JQ_URL} ${JQ_BIN}
+
+WORKDIR ${MOD_SECURITY_DIR}\test\test-cases\regression
+
+RUN %JQ_BIN% "map(if .title == \"Test match variable (1/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json
+RUN %JQ_BIN% "map(if .title == \"Test match variable (2/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json
+RUN %JQ_BIN% "map(if .title == \"Test match variable (3/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json
+RUN %JQ_BIN% "map(if .title == \"Variable offset - FILES_NAMES\" then .enabled = 0 else . end)" offset-variable.json > tmp.json && move /Y tmp.json offset-variable.json
+
+# run tests
+WORKDIR ${MOD_SECURITY_DIR}\build\win32\build
+
+RUN C:\BuildTools\VC\Auxiliary\Build\vcvars64.bat && ctest -C %BUILD_TYPE% --output-on-failure
+
 # setup container's entrypoint
 #