Windows porting: port hyperscan and chimera tools to windows.

This commit is contained in:
Lu, Qi 2018-05-08 12:05:44 -04:00 committed by Wang, Xiang W
parent bf87f8c003
commit 5a0885d235
34 changed files with 563 additions and 109 deletions

View File

@ -70,16 +70,6 @@ include_directories(SYSTEM include)
include (${CMAKE_MODULE_PATH}/boost.cmake)
# PCRE check, we have a fixed requirement for PCRE to use Chimera
# and hscollider
set(PCRE_REQUIRED_MAJOR_VERSION 8)
set(PCRE_REQUIRED_MINOR_VERSION 41)
set(PCRE_REQUIRED_VERSION ${PCRE_REQUIRED_MAJOR_VERSION}.${PCRE_REQUIRED_MINOR_VERSION})
include (${CMAKE_MODULE_PATH}/pcre.cmake)
if (NOT CORRECT_PCRE_VERSION)
message(STATUS "PCRE ${PCRE_REQUIRED_VERSION} not found")
endif()
# -- make this work? set(python_ADDITIONAL_VERSIONS 2.7 2.6)
find_package(PythonInterp)
find_program(RAGEL ragel)
@ -455,20 +445,33 @@ else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
endif()
add_subdirectory(util)
add_subdirectory(doc/dev-reference)
if (NOT WIN32)
# PCRE check, we have a fixed requirement for PCRE to use Chimera
# and hscollider
set(PCRE_REQUIRED_MAJOR_VERSION 8)
set(PCRE_REQUIRED_MINOR_VERSION 41)
set(PCRE_REQUIRED_VERSION ${PCRE_REQUIRED_MAJOR_VERSION}.${PCRE_REQUIRED_MINOR_VERSION})
include (${CMAKE_MODULE_PATH}/pcre.cmake)
if (NOT CORRECT_PCRE_VERSION)
message(STATUS "PCRE ${PCRE_REQUIRED_VERSION} not found")
endif()
# we need static libs for Chimera - too much deep magic for shared libs
if (CORRECT_PCRE_VERSION AND BUILD_STATIC_LIBS)
set(BUILD_CHIMERA TRUE)
endif()
add_subdirectory(util)
add_subdirectory(unit)
add_subdirectory(doc/dev-reference)
if (EXISTS ${CMAKE_SOURCE_DIR}/tools/CMakeLists.txt)
add_subdirectory(tools)
endif()
if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA)
add_subdirectory(chimera)
endif()
endif()
# do substitutions
configure_file(${CMAKE_MODULE_PATH}/config.h.in ${PROJECT_BINARY_DIR}/config.h)
@ -497,6 +500,31 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_C_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXX_FLAGS}")
endif()
if (WIN32)
# PCRE check, we have a fixed requirement for PCRE to use Chimera
# and hscollider
set(PCRE_REQUIRED_MAJOR_VERSION 8)
set(PCRE_REQUIRED_MINOR_VERSION 41)
set(PCRE_REQUIRED_VERSION ${PCRE_REQUIRED_MAJOR_VERSION}.${PCRE_REQUIRED_MINOR_VERSION})
include (${CMAKE_MODULE_PATH}/pcre.cmake)
if (NOT CORRECT_PCRE_VERSION)
message(STATUS "PCRE ${PCRE_REQUIRED_VERSION} not found")
endif()
# we need static libs for Chimera - too much deep magic for shared libs
if (CORRECT_PCRE_VERSION AND BUILD_STATIC_LIBS)
set(BUILD_CHIMERA TRUE)
endif()
add_subdirectory(unit)
if (EXISTS ${CMAKE_SOURCE_DIR}/tools/CMakeLists.txt)
add_subdirectory(tools)
endif()
if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA)
add_subdirectory(chimera)
endif()
endif()
if(NOT WIN32)
set(RAGEL_C_FLAGS "-Wno-unused")
endif()

View File

@ -92,7 +92,7 @@ void pq_pop_nice(struct match_pq *pq) {
/** dummy event handler for use when user does not provide one */
static
int null_onEvent(UNUSED unsigned id, UNUSED unsigned long long from,
int HS_CDECL null_onEvent(UNUSED unsigned id, UNUSED unsigned long long from,
UNUSED unsigned long long to, UNUSED unsigned flags,
UNUSED unsigned size, UNUSED const ch_capture_t *captured,
UNUSED void *ctxt) {
@ -108,12 +108,12 @@ struct HybridContext {
struct ch_scratch *scratch;
struct match_pq *pq;
/** \brief user-supplied match callback */
int (*match_callback)(unsigned int id, unsigned long long from,
int (HS_CDECL *match_callback)(unsigned int id, unsigned long long from,
unsigned long long to, unsigned int flags,
unsigned int size, const ch_capture_t *capture,
void *ctx);
/** \brief user-supplied error callback */
int (*error_callback)(ch_error_event_t error_type, unsigned int id,
int (HS_CDECL *error_callback)(ch_error_event_t error_type, unsigned int id,
void *info, void *ctx);
/** \brief user-supplied context */
void *context;
@ -363,7 +363,7 @@ ch_error_t catchupPcre(struct HybridContext *hyctx, unsigned int id,
/** \brief Callback used for internal Hyperscan multi-matcher. */
static
int multiCallback(unsigned int id, unsigned long long from,
int HS_CDECL multiCallback(unsigned int id, unsigned long long from,
unsigned long long to, UNUSED unsigned int flags,
void *ctx) {
assert(ctx);

View File

@ -201,7 +201,7 @@ typedef struct ch_capture {
* pattern, while a return value of @ref CH_CALLBACK_SKIP_PATTERN will
* cease matching this pattern but continue matching the next pattern.
*/
typedef ch_callback_t (*ch_match_event_handler)(unsigned int id,
typedef ch_callback_t (HS_CDECL *ch_match_event_handler)(unsigned int id,
unsigned long long from,
unsigned long long to,
unsigned int flags,
@ -233,11 +233,12 @@ typedef ch_callback_t (*ch_match_event_handler)(unsigned int id,
* function.
*
* @return
* The callback can return @ref CH_CALLBACK_SKIP_PATTERN to cease matching this
* pattern but continue matching the next pattern. Otherwise, we stop
* The callback can return @ref CH_CALLBACK_SKIP_PATTERN to cease matching
* this pattern but continue matching the next pattern. Otherwise, we stop
* matching for all patterns with @ref CH_CALLBACK_TERMINATE.
*/
typedef ch_callback_t (*ch_error_event_handler)(ch_error_event_t error_type,
typedef ch_callback_t (HS_CDECL *ch_error_event_handler)(
ch_error_event_t error_type,
unsigned int id, void *info,
void *ctx);

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -122,7 +122,7 @@ typedef struct hs_scratch hs_scratch_t;
* subsequent calls to @ref hs_scan_stream() for that stream will
* immediately return with @ref HS_SCAN_TERMINATED.
*/
typedef int (*match_event_handler)(unsigned int id,
typedef int (HS_CDECL *match_event_handler)(unsigned int id,
unsigned long long from,
unsigned long long to,
unsigned int flags,

View File

@ -1437,7 +1437,19 @@ void mergeLeftfixesVariableLag(RoseBuildImpl &build) {
assert(!parents.empty());
#ifndef _WIN32
engine_groups[MergeKey(left, parents)].push_back(left);
#else
// On windows, when passing MergeKey object into map 'engine_groups',
// it will not be copied, but will be freed along with
// engine_groups.clear().
// If we construct MergeKey object on the stack, it will be destructed
// on its life cycle ending, then on engine_groups.clear(), which
// will cause is_block_type_valid() assertion error in MergeKey
// destructor.
MergeKey *mk = new MergeKey(left, parents);
engine_groups[*mk].push_back(left);
#endif
}
vector<vector<left_id>> chunks;

View File

@ -67,7 +67,7 @@ void prefetch_data(const char *data, unsigned length) {
/** dummy event handler for use when user does not provide one */
static
int null_onEvent(UNUSED unsigned id, UNUSED unsigned long long from,
int HS_CDECL null_onEvent(UNUSED unsigned id, UNUSED unsigned long long from,
UNUSED unsigned long long to, UNUSED unsigned flags,
UNUSED void *ctxt) {
return 0;

View File

@ -36,6 +36,7 @@
#ifndef SCRATCH_H_DA6D4FC06FF410
#define SCRATCH_H_DA6D4FC06FF410
#include "hs_common.h"
#include "ue2common.h"
#include "rose/rose_types.h"
@ -88,8 +89,9 @@ struct core_info {
void *userContext; /**< user-supplied context */
/** \brief user-supplied match callback */
int (*userCallback)(unsigned int id, unsigned long long from,
unsigned long long to, unsigned int flags, void *ctx);
int (HS_CDECL *userCallback)(unsigned int id, unsigned long long from,
unsigned long long to, unsigned int flags,
void *ctx);
const struct RoseEngine *rose;
char *state; /**< full stream state */

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -66,8 +66,13 @@ typedef signed int s32;
/* We append the 'a' for aligned, since these aren't common, garden variety
* 64 bit values. The alignment is necessary for structs on some platforms,
* so we don't end up performing accidental unaligned accesses. */
#if defined(_WIN32) && ! defined(_WIN64)
typedef unsigned long long ALIGN_ATTR(4) u64a;
typedef signed long long ALIGN_ATTR(4) s64a;
#else
typedef unsigned long long ALIGN_ATTR(8) u64a;
typedef signed long long ALIGN_ATTR(8) s64a;
#endif
/* get the SIMD types */
#include "util/simd_types.h"

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -305,9 +305,10 @@ public:
}
/// Bitwise OR.
bitfield operator|(bitfield a) const {
a |= *this;
return a;
bitfield operator|(const bitfield &a) const {
bitfield b = a;
b |= *this;
return b;
}
/// Bitwise OR-equals.
@ -325,9 +326,10 @@ public:
}
/// Bitwise AND.
bitfield operator&(bitfield a) const {
a &= *this;
return a;
bitfield operator&(const bitfield &a) const {
bitfield b = a;
b &= *this;
return b;
}
/// Bitwise AND-equals.

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -56,7 +56,11 @@ void describeChar(ostream &os, char c, enum cc_output_t out_type) {
const string backslash((out_type == CC_OUT_DOT ? 2 : 1), '\\');
#ifdef _WIN32
if (c >= 0x21 && c < 0x7F && c != '\\') {
#else
if (isgraph(c) && c != '\\') {
#endif
if (escaped.find(c) != string::npos) {
os << backslash << c;
} else if (out_type == CC_OUT_DOT

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -1197,7 +1197,11 @@ u32 mmbit_sparse_iter_begin(const u8 *bits, u32 total_bits, u32 *idx,
assert(ISALIGNED_N(it_root, alignof(struct mmbit_sparse_iter)));
// Our state _may_ be on the stack
#ifndef _WIN32
assert(ISALIGNED_N(s, alignof(struct mmbit_sparse_state)));
#else
assert(ISALIGNED_N(s, 4));
#endif
MDEBUG_PRINTF("%p total_bits %u\n", bits, total_bits);
// iterator should have _something_ at the root level
@ -1305,7 +1309,11 @@ u32 mmbit_sparse_iter_next(const u8 *bits, u32 total_bits, u32 last_key,
assert(ISALIGNED_N(it_root, alignof(struct mmbit_sparse_iter)));
// Our state _may_ be on the stack
#ifndef _WIN32
assert(ISALIGNED_N(s, alignof(struct mmbit_sparse_state)));
#else
assert(ISALIGNED_N(s, 4));
#endif
MDEBUG_PRINTF("%p total_bits %u\n", bits, total_bits);
MDEBUG_PRINTF("NEXT (total_bits=%u, last_key=%u)\n", total_bits, last_key);
@ -1458,7 +1466,11 @@ void mmbit_sparse_iter_unset(u8 *bits, u32 total_bits,
assert(ISALIGNED_N(it, alignof(struct mmbit_sparse_iter)));
// Our state _may_ be on the stack
#ifndef _WIN32
assert(ISALIGNED_N(s, alignof(struct mmbit_sparse_state)));
#else
assert(ISALIGNED_N(s, 4));
#endif
MDEBUG_PRINTF("%p total_bits %u\n", bits, total_bits);

View File

@ -46,7 +46,7 @@ using namespace std;
namespace ue2 {
u32 HS_CDECL mmbit_size(u32 total_bits) {
u32 mmbit_size(u32 total_bits) {
if (total_bits > MMB_MAX_BITS) {
throw ResourceLimitError();
}

View File

@ -63,10 +63,8 @@ namespace ue2 {
*
* This will throw a resource limit assertion if the requested mmbit is too
* large.
*
* TODO:add temporary HS_CDECL for chimera on Windows, need improve this.
*/
u32 HS_CDECL mmbit_size(u32 total_bits);
u32 mmbit_size(u32 total_bits);
/** \brief Construct a sparse iterator over the values in \a bits for a
* multibit of size \a total_bits. */

View File

@ -1,6 +1,3 @@
if (WIN32)
return()
endif()
find_package(Threads)
# remove some warnings
@ -12,11 +9,18 @@ include_directories(${PROJECT_SOURCE_DIR})
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)
include_directories(${PROJECT_SOURCE_DIR}/util)
# add any subdir with a cmake file
file(GLOB dirents RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *)
foreach(e ${dirents})
if (WIN32)
add_subdirectory(hscheck)
add_subdirectory(hsbench)
add_subdirectory(hsdump)
add_subdirectory(hscollider)
else()
# add any subdir with a cmake file
file(GLOB dirents RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *)
foreach(e ${dirents})
if(IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${e} AND
EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${e}/CMakeLists.txt)
add_subdirectory(${e})
endif ()
endforeach ()
endforeach ()
endif()

View File

@ -61,8 +61,13 @@ endif()
add_executable(hsbench ${hsbench_SOURCES})
if (BUILD_CHIMERA)
include_directories(${PCRE_INCLUDE_DIRS})
if(NOT WIN32)
target_link_libraries(hsbench hs chimera ${PCRE_LDFLAGS} databaseutil
expressionutil ${SQLITE3_LDFLAGS} ${CMAKE_THREAD_LIBS_INIT})
else()
target_link_libraries(hsbench hs chimera pcre databaseutil
expressionutil ${SQLITE3_LDFLAGS} ${CMAKE_THREAD_LIBS_INIT})
endif()
else()
target_link_libraries(hsbench hs databaseutil expressionutil
${SQLITE3_LDFLAGS} ${CMAKE_THREAD_LIBS_INIT})

View File

@ -68,8 +68,9 @@ struct ScanCHContext {
* "echo matches" is off.
*/
static
int onMatch(unsigned int, unsigned long long, unsigned long long, unsigned int,
unsigned int, const ch_capture_t *, void *ctx) {
int HS_CDECL onMatch(unsigned int, unsigned long long, unsigned long long,
unsigned int, unsigned int, const ch_capture_t *,
void *ctx) {
ScanCHContext *sc = static_cast<ScanCHContext *>(ctx);
assert(sc);
sc->result.matches++;
@ -82,8 +83,9 @@ int onMatch(unsigned int, unsigned long long, unsigned long long, unsigned int,
* matches" is enabled.
*/
static
int onMatchEcho(unsigned int id, unsigned long long, unsigned long long to,
unsigned int, unsigned int, const ch_capture_t *, void *ctx) {
int HS_CDECL onMatchEcho(unsigned int id, unsigned long long,
unsigned long long to, unsigned int, unsigned int,
const ch_capture_t *, void *ctx) {
ScanCHContext *sc = static_cast<ScanCHContext *>(ctx);
assert(sc);
sc->result.matches++;
@ -166,12 +168,23 @@ void EngineChimera::printStats() const {
}
printf("Signatures: %s\n", compile_stats.signatures.c_str());
printf("Chimera info: %s\n", compile_stats.db_info.c_str());
#ifndef _WIN32
printf("Expression count: %'zu\n", compile_stats.expressionCount);
printf("Bytecode size: %'zu bytes\n", compile_stats.compiledSize);
#else
printf("Expression count: %zu\n", compile_stats.expressionCount);
printf("Bytecode size: %zu bytes\n", compile_stats.compiledSize);
#endif
printf("Database CRC: 0x%x\n", compile_stats.crc32);
#ifndef _WIN32
printf("Scratch size: %'zu bytes\n", compile_stats.scratchSize);
printf("Compile time: %'0.3Lf seconds\n", compile_stats.compileSecs);
printf("Peak heap usage: %'u bytes\n", compile_stats.peakMemorySize);
#else
printf("Scratch size: %zu bytes\n", compile_stats.scratchSize);
printf("Compile time: %0.3Lf seconds\n", compile_stats.compileSecs);
printf("Peak heap usage: %u bytes\n", compile_stats.peakMemorySize);
#endif
}
void EngineChimera::sqlStats(SqlDB &sqldb) const {

View File

@ -87,8 +87,8 @@ struct ScanHSContext {
* "echo matches" is off.
*/
static
int onMatch(unsigned int, unsigned long long, unsigned long long, unsigned int,
void *ctx) {
int HS_CDECL onMatch(unsigned int, unsigned long long,
unsigned long long, unsigned int, void *ctx) {
ScanHSContext *sc = static_cast<ScanHSContext *>(ctx);
assert(sc);
sc->result.matches++;
@ -101,8 +101,8 @@ int onMatch(unsigned int, unsigned long long, unsigned long long, unsigned int,
* matches" is enabled.
*/
static
int onMatchEcho(unsigned int id, unsigned long long, unsigned long long to,
unsigned int, void *ctx) {
int HS_CDECL onMatchEcho(unsigned int id, unsigned long long,
unsigned long long to, unsigned int, void *ctx) {
ScanHSContext *sc = static_cast<ScanHSContext *>(ctx);
assert(sc);
sc->result.matches++;
@ -250,15 +250,30 @@ void EngineHyperscan::printStats() const {
}
printf("Signatures: %s\n", compile_stats.signatures.c_str());
printf("Hyperscan info: %s\n", compile_stats.db_info.c_str());
#ifndef _WIN32
printf("Expression count: %'zu\n", compile_stats.expressionCount);
printf("Bytecode size: %'zu bytes\n", compile_stats.compiledSize);
#else
printf("Expression count: %zu\n", compile_stats.expressionCount);
printf("Bytecode size: %zu bytes\n", compile_stats.compiledSize);
#endif
printf("Database CRC: 0x%x\n", compile_stats.crc32);
if (compile_stats.streaming) {
#ifndef _WIN32
printf("Stream state size: %'zu bytes\n", compile_stats.streamSize);
#else
printf("Stream state size: %zu bytes\n", compile_stats.streamSize);
#endif
}
#ifndef _WIN32
printf("Scratch size: %'zu bytes\n", compile_stats.scratchSize);
printf("Compile time: %'0.3Lf seconds\n", compile_stats.compileSecs);
printf("Peak heap usage: %'u bytes\n", compile_stats.peakMemorySize);
#else
printf("Scratch size: %zu bytes\n", compile_stats.scratchSize);
printf("Compile time: %0.3Lf seconds\n", compile_stats.compileSecs);
printf("Peak heap usage: %u bytes\n", compile_stats.peakMemorySize);
#endif
}
void EngineHyperscan::sqlStats(SqlDB &sqldb) const {

View File

@ -26,6 +26,9 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifdef _WIN32
#define PCRE_STATIC
#endif
#include "config.h"
#include "common.h"
@ -38,6 +41,8 @@
#include "util/make_unique.h"
#include "util/unicode_def.h"
#include <algorithm>
using namespace std;
EnginePCREContext::EnginePCREContext(int capture_cnt) {
@ -207,11 +212,19 @@ void EnginePCRE::printStats() const {
}
printf("Signatures: %s\n", compile_stats.signatures.c_str());
printf("PCRE info: %s\n", compile_stats.db_info.c_str());
#ifndef _WIN32
printf("Expression count: %'zu\n", compile_stats.expressionCount);
printf("Bytecode size: %'zu bytes\n", compile_stats.compiledSize);
printf("Scratch size: %'zu bytes\n", compile_stats.scratchSize);
printf("Compile time: %'0.3Lf seconds\n", compile_stats.compileSecs);
printf("Peak heap usage: %'u bytes\n", compile_stats.peakMemorySize);
#else
printf("Expression count: %zu\n", compile_stats.expressionCount);
printf("Bytecode size: %zu bytes\n", compile_stats.compiledSize);
printf("Scratch size: %zu bytes\n", compile_stats.scratchSize);
printf("Compile time: %0.3Lf seconds\n", compile_stats.compileSecs);
printf("Peak heap usage: %u bytes\n", compile_stats.peakMemorySize);
#endif
}
void EnginePCRE::sqlStats(SqlDB &sqldb) const {

View File

@ -58,7 +58,11 @@
#include <set>
#include <thread>
#ifndef _WIN32
#include <getopt.h>
#else
#include "win_getopt.h"
#endif
#ifndef _WIN32
#include <pthread.h>
#if defined(HAVE_PTHREAD_NP_H)
@ -138,6 +142,16 @@ public:
// Apply processor affinity (if available) to this thread.
bool affine(UNUSED int cpu) {
#if defined(_WIN32)
SYSTEM_INFO system_info;
GetSystemInfo(&system_info);
assert(cpu >= 0 && (DWORD)cpu < system_info.dwNumberOfProcessors);
DWORD_PTR mask = 1 << cpu;
DWORD_PTR rv = SetThreadAffinityMask(thr.native_handle(), mask);
return rv != 0;
#endif
#ifdef HAVE_DECL_PTHREAD_SETAFFINITY_NP
#if defined(__FreeBSD__)
cpuset_t cpuset;
@ -191,7 +205,7 @@ void usage(const char *error) {
printf(" -H Benchmark using Chimera (if supported).\n");
printf(" -P Benchmark using PCRE (if supported).\n");
#endif
#ifdef HAVE_DECL_PTHREAD_SETAFFINITY_NP
#if defined(HAVE_DECL_PTHREAD_SETAFFINITY_NP) || defined(_WIN32)
printf(" -T CPU,CPU,... Benchmark with threads on these CPUs.\n");
#endif
printf(" -i DIR Don't compile, load from files in DIR"
@ -225,7 +239,7 @@ static
void processArgs(int argc, char *argv[], vector<BenchmarkSigs> &sigSets,
UNUSED unique_ptr<Grey> &grey) {
const char options[] = "-b:c:Cd:e:E:G:hHi:n:No:p:PsS:Vw:z:"
#ifdef HAVE_DECL_PTHREAD_SETAFFINITY_NP
#if defined(HAVE_DECL_PTHREAD_SETAFFINITY_NP) || defined(_WIN32)
"T:" // add the thread flag
#endif
;
@ -332,7 +346,7 @@ void processArgs(int argc, char *argv[], vector<BenchmarkSigs> &sigSets,
case 'S':
sigName.assign(optarg);
break;
#ifdef HAVE_DECL_PTHREAD_SETAFFINITY_NP
#if defined(HAVE_DECL_PTHREAD_SETAFFINITY_NP) || defined(_WIN32)
case 'T':
if (!strToList(optarg, threadCores)) {
usage("Couldn't parse argument to -T flag, should be"
@ -704,7 +718,11 @@ void displayPerScanResults(const vector<unique_ptr<ThreadContext>> &threads,
for (size_t j = 0; j != results.size(); j++) {
const auto &r = results[j];
double mbps = calc_mbps(r.seconds, bytesPerRun);
#ifndef _WIN32
printf("T %2u Scan %2zu: %'0.2f Mbit/sec\n", t->num, j, mbps);
#else
printf("T %2u Scan %2zu: %0.2f Mbit/sec\n", t->num, j, mbps);
#endif
}
}
printf("\n");
@ -749,6 +767,7 @@ void displayResults(const vector<unique_ptr<ThreadContext>> &threads,
}
}
#ifndef _WIN32
printf("Time spent scanning: %'0.3f seconds\n", totalSecs);
printf("Corpus size: %'llu bytes ", bytesPerRun);
switch (scan_mode) {
@ -764,22 +783,56 @@ void displayResults(const vector<unique_ptr<ThreadContext>> &threads,
printf("(%'zu blocks)\n", corpus_blocks.size());
break;
}
#else
printf("Time spent scanning: %0.3f seconds\n", totalSecs);
printf("Corpus size: %llu bytes ", bytesPerRun);
switch (scan_mode) {
case ScanMode::STREAMING:
printf("(%zu blocks in %llu streams)\n", corpus_blocks.size(),
count_streams(corpus_blocks));
break;
case ScanMode::VECTORED:
printf("(%zu blocks in %llu vectors)\n", corpus_blocks.size(),
count_streams(corpus_blocks));
break;
case ScanMode::BLOCK:
printf("(%zu blocks)\n", corpus_blocks.size());
break;
}
#endif
u64a totalBytes = bytesPerRun * repeats * threads.size();
u64a totalBlocks = corpus_blocks.size() * repeats * threads.size();
double matchRate = ((double)matchesPerRun * 1024) / bytesPerRun;
#ifndef _WIN32
printf("Matches per iteration: %'llu (%'0.3f matches/kilobyte)\n",
matchesPerRun, matchRate);
#else
printf("Matches per iteration: %llu (%0.3f matches/kilobyte)\n",
matchesPerRun, matchRate);
#endif
double blockRate = (double)totalBlocks / (double)totalSecs;
#ifndef _WIN32
printf("Overall block rate: %'0.2f blocks/sec\n", blockRate);
printf("Mean throughput (overall): %'0.2Lf Mbit/sec\n",
calc_mbps(totalSecs, totalBytes));
#else
printf("Overall block rate: %0.2f blocks/sec\n", blockRate);
printf("Mean throughput (overall): %0.2Lf Mbit/sec\n",
calc_mbps(totalSecs, totalBytes));
#endif
double lowestScanTime = fastestResult(threads);
#ifndef _WIN32
printf("Max throughput (per core): %'0.2Lf Mbit/sec\n",
calc_mbps(lowestScanTime, bytesPerRun));
#else
printf("Max throughput (per core): %0.2Lf Mbit/sec\n",
calc_mbps(lowestScanTime, bytesPerRun));
#endif
printf("\n");
if (display_per_scan) {
@ -892,7 +945,7 @@ void runBenchmark(const Engine &db,
numThreads = 1;
} else {
numThreads = threadCores.size();
#ifdef HAVE_DECL_PTHREAD_SETAFFINITY_NP
#if defined(HAVE_DECL_PTHREAD_SETAFFINITY_NP) || defined(_WIN32)
useAffinity = true;
#else
useAffinity = false;
@ -932,7 +985,7 @@ void runBenchmark(const Engine &db,
} // namespace
/** Main driver. */
int main(int argc, char *argv[]) {
int HS_CDECL main(int argc, char *argv[]) {
unique_ptr<Grey> grey;
#if !defined(RELEASE_BUILD)
grey = make_unique<Grey>();

View File

@ -10,9 +10,16 @@ if (BUILD_CHIMERA)
include_directories(${PCRE_INCLUDE_DIRS})
add_definitions(-DHS_HYBRID)
add_executable(hscheck ${hscheck_SOURCES})
if(NOT WIN32)
target_link_libraries(hscheck hs chimera ${PCRE_LDFLAGS} expressionutil pthread)
else()
target_link_libraries(hscheck hs chimera pcre expressionutil)
endif()
else()
add_executable(hscheck ${hscheck_SOURCES})
if(NOT WIN32)
target_link_libraries(hscheck hs expressionutil pthread)
else()
target_link_libraries(hscheck hs expressionutil)
endif()
endif()

View File

@ -70,8 +70,11 @@
#include <stdexcept>
#include <string>
#include <thread>
#ifndef _WIN32
#include <getopt.h>
#else
#include "win_getopt.h"
#endif
#include <boost/algorithm/string/trim.hpp>
using namespace std;
@ -625,7 +628,7 @@ void loadSignatureBuildSigs(const string &inFile,
}
}
int main(int argc, char **argv) {
int HS_CDECL main(int argc, char **argv) {
num_of_threads = max(1u, std::thread::hardware_concurrency());
#if !defined(RELEASE_BUILD)

View File

@ -74,7 +74,8 @@ if(HAVE_BACKTRACE)
"${BACKTRACE_CFLAGS}")
endif()
else() # WIN32
target_link_libraries(hscollider hs chimera ${PCRE_LDFLAGS} databaseutil
set_target_properties(hscollider PROPERTIES LINK_FLAGS "/STACK:8388608,8388608")
target_link_libraries(hscollider hs chimera pcre databaseutil
expressionutil corpusomatic crosscompileutil)
endif()

View File

@ -26,6 +26,9 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifdef _WIN32
#define PCRE_STATIC
#endif
#include "config.h"
#include "common.h"

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -35,8 +35,7 @@
#include <cstdlib>
#include <iostream>
#include <pthread.h>
#ifndef _WIN32
static const size_t COLLIDER_THREAD_STACK_SIZE = 8192 * 1024;
void Thread::start() {
@ -79,6 +78,16 @@ create_thread:
}
}
void Thread::join() { pthread_join(thread, nullptr); }
#else // windows
void Thread::start() { thread = std::thread(&runThread, this); }
void Thread::join() { thread.join(); }
#endif
// Dispatch
void *Thread::runThread(void *thr) {
if (!no_signal_handler) {
@ -88,7 +97,6 @@ void *Thread::runThread(void *thr) {
return nullptr;
}
void Thread::join() { pthread_join(thread, nullptr); }
Thread::Thread(size_t num) : thread_id(num) {}

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -31,7 +31,11 @@
#include <cstdlib>
#ifndef _WIN32
#include <pthread.h>
#else
#include <thread>
#endif
#include <boost/core/noncopyable.hpp>
@ -54,7 +58,11 @@ protected:
const size_t thread_id;
private:
#ifndef _WIN32
pthread_t thread;
#else
std::thread thread;
#endif
};
#endif // UE2COLLIDER_THREAD_H

View File

@ -169,8 +169,9 @@ struct MultiContext {
// Callback used for all (both single and multi-mode) scans.
static
int callbackMulti(unsigned int id, unsigned long long from,
unsigned long long to, UNUSED unsigned int flags, void *ctx) {
int HS_CDECL callbackMulti(unsigned int id, unsigned long long from,
unsigned long long to,
UNUSED unsigned int flags, void *ctx) {
MultiContext *mctx = static_cast<MultiContext *>(ctx);
assert(mctx);
assert(mctx->rs);
@ -269,7 +270,7 @@ int callbackMulti(unsigned int id, unsigned long long from,
// Hybrid matcher callback.
static
ch_callback_t callbackHybrid(unsigned id, unsigned long long from,
ch_callback_t HS_CDECL callbackHybrid(unsigned id, unsigned long long from,
unsigned long long to, unsigned, unsigned size,
const ch_capture_t *captured, void *ctx) {
MultiContext *mctx = static_cast<MultiContext *>(ctx);
@ -322,7 +323,8 @@ ch_callback_t callbackHybrid(unsigned id, unsigned long long from,
// Hybrid matcher error callback.
static
ch_callback_t errorCallback(UNUSED ch_error_event_t errorType, UNUSED unsigned int id, void *,
ch_callback_t HS_CDECL errorCallback(UNUSED ch_error_event_t errorType,
UNUSED unsigned int id, void *,
void *ctx) {
UNUSED MultiContext *mctx = static_cast<MultiContext *>(ctx);
assert(mctx);

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -46,7 +46,11 @@
#include <sstream>
#include <string>
#include <vector>
#ifndef _WIN32
#include <getopt.h>
#else
#include "win_getopt.h"
#endif
#define xstr(s) str(s)
#define str(s) #s
@ -467,7 +471,7 @@ void processArgs(int argc, char *argv[], CorpusProperties &corpus_gen_prop,
exit(1);
}
break;
case 'Z':
case 'Z': { // Parentheses save VS C2360
static constexpr unsigned ALIGN_LIMIT = MAX_MAX_UE2_ALIGN - 1;
if (optarg == string("R")) {
// Random min alignment selected.
@ -481,6 +485,7 @@ void processArgs(int argc, char *argv[], CorpusProperties &corpus_gen_prop,
}
max_ue2_align = min_ue2_align + 1;
break;
}
case '8':
force_utf8 = true;
break;

View File

@ -1840,13 +1840,17 @@ unique_ptr<CorporaSource> buildCorpora(const vector<string> &corporaFiles,
static
bool needsQuotes(const char *s) {
size_t len = strlen(s);
// don't confuse the correct isblank for the one in locale
int (*blank)(int) = &std::isblank;
if (len == 0) {
return true;
}
#ifndef _WIN32
// don't confuse the correct isblank for the one in locale
int (*blank)(int) = &std::isblank;
if (find_if(s, s + len, blank) != s + len) {
#else
if (find_if(s, s + len, [](unsigned char c){ return std::isblank(c); }) != s + len) {
#endif
return true;
}
@ -1910,7 +1914,7 @@ bool runTests(CorporaSource &corpora_source, const ExpressionMap &exprMap,
return !summary.hasFailure();
}
int main(int argc, char *argv[]) {
int HS_CDECL main(int argc, char *argv[]) {
Grey grey;
vector<string> corporaFiles;

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -36,7 +36,7 @@
#include <ctype.h>
#include <string>
#ifdef HAVE_SIGACTION
#if defined(HAVE_SIGACTION) || defined(_WIN32)
#include <signal.h>
#endif
@ -56,8 +56,12 @@ TLS_VARIABLE volatile size_t debug_corpus_len = 0;
extern std::string g_cmdline;
#ifdef HAVE_SIGACTION
#if defined(_WIN32)
static void __cdecl sighandler(int signum) {
#elif defined(HAVE_SIGACTION)
static void sighandler(int signum) {
#endif
#if defined(HAVE_SIGACTION) || defined(_WIN32)
/* NOTE: This signal handler is designed solely to provide more information
* when a crash occurs in ue2collider -- it makes calls to signal-unsafe
* functions like printf() and backtrace() by design, since we're already
@ -141,7 +145,13 @@ static void sighandler(int signum) {
#endif // HAVE_SIGACTION
void installSignalHandler(void) {
#ifdef HAVE_SIGACTION
#ifdef _WIN32
signal(SIGABRT, sighandler);
signal(SIGFPE, sighandler);
signal(SIGILL, sighandler);
signal(SIGSEGV, sighandler);
#elif defined(HAVE_SIGACTION)
struct sigaction act;
memset(&act, 0, sizeof(act));
act.sa_handler = sighandler;

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -40,7 +40,11 @@
#define STAGE_GRAPH_COMPILE 6
#define STAGE_GRAPH_RUN 7
#ifndef WIN32
#define TLS_VARIABLE __thread
#else
#define TLS_VARIABLE __declspec(thread)
#endif
extern TLS_VARIABLE volatile int debug_stage;
extern TLS_VARIABLE volatile int debug_expr;

View File

@ -3,10 +3,6 @@ if (NOT DUMP_SUPPORT)
return()
endif ()
if (WIN32)
return()
endif ()
include_directories(${PROJECT_SOURCE_DIR})
include_directories(${PROJECT_SOURCE_DIR}/util)

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -58,9 +58,19 @@
#include <string>
#include <vector>
#ifndef _WIN32
#include <getopt.h>
#else
#include "win_getopt.h"
#endif
#include <sys/stat.h>
#ifndef _WIN32
#include <dirent.h>
#else
#include <direct.h>
#define stat _stat
#endif
#include <boost/ptr_container/ptr_vector.hpp>
@ -318,6 +328,7 @@ u32 buildDumpFlags(void) {
return flags;
}
#ifndef _WIN32
static
void clearDir(const string &path) {
DIR *dir = opendir(path.c_str());
@ -341,15 +352,54 @@ void clearDir(const string &path) {
}
closedir(dir);
}
#else // windows
static
void clearDir(const string &path) {
WIN32_FIND_DATA ffd;
HANDLE hFind = INVALID_HANDLE_VALUE;
string glob = path + "/*";
hFind = FindFirstFile(glob.c_str(), &ffd);
if (hFind == INVALID_HANDLE_VALUE) {
printf("ERROR: couldn't open location %s\n", path.c_str());
exit(1);
}
do {
string basename(ffd.cFileName);
string fname(path);
fname.push_back('/');
fname.append(basename);
// Ignore '.' and '..'
if (basename == "." || basename == "..") {
continue;
}
if (!DeleteFile(fname.c_str())) {
printf("ERROR: couldn't remove file %s\n", fname.c_str());
}
} while (FindNextFile(hFind, &ffd) != 0);
FindClose(hFind);
}
#endif
static
int makeDirectory(const string &dirName) {
#ifndef _WIN32
mode_t mode = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP |
S_IROTH | S_IXOTH;
return mkdir(dirName.c_str(), mode);
#else
return _mkdir(dirName.c_str());
#endif
}
static
void prepareDumpLoc(string parent, string path, u32 flags, Grey &grey) {
struct stat st;
if (stat(parent.c_str(), &st)) {
// Create dump location if not found
mode_t mode = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP |
S_IROTH | S_IXOTH;
if (mkdir(parent.c_str(), mode) < 0) {
if (makeDirectory(parent) < 0) {
printf("ERROR: could not create dump location %s: %s\n",
parent.c_str(), strerror(errno));
exit(1);
@ -365,9 +415,7 @@ void prepareDumpLoc(string parent, string path, u32 flags, Grey &grey) {
path = parent.append(path);
if (stat(path.c_str(), &st)) {
// Create dump location if not found
mode_t mode = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP |
S_IROTH | S_IXOTH;
if (mkdir(path.c_str(), mode) < 0) {
if (makeDirectory(path) < 0) {
printf("ERROR: could not create dump location %s: %s\n",
path.c_str(), strerror(errno));
exit(1);
@ -546,7 +594,7 @@ unsigned int dumpData(const ExpressionMap &exprMap, Grey &grey) {
return dumpDataMulti(patterns, flags, ids, ext, grey);
}
int main(int argc, char *argv[]) {
int HS_CDECL main(int argc, char *argv[]) {
Grey grey;
grey.dumpFlags = Grey::DUMP_BASICS;

View File

@ -1,6 +1,13 @@
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_C_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXX_FLAGS}")
if(CMAKE_C_FLAGS MATCHES "/Gv" )
string(REPLACE "/Gv" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
endif()
if(CMAKE_CXX_FLAGS MATCHES "/Gv" )
string(REPLACE "/Gv" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
endif()
set(gtest_SOURCES gtest/gtest-all.cc gtest/gtest.h)
include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR})
include_directories(${PROJECT_SOURCE_DIR})
@ -31,6 +38,10 @@ endif()
add_definitions(-DGTEST_HAS_PTHREAD=0 -DSRCDIR=${PROJECT_SOURCE_DIR})
if (WIN32)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4309 /wd4018")
endif()
set(unit_hyperscan_SOURCES
${gtest_SOURCES}
hyperscan/allocators.cpp

177
util/win_getopt.h Normal file
View File

@ -0,0 +1,177 @@
/*
* Copyright (c) 2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef WIN_GETOPT_H
#define WIN_GETOPT_H
#include <windows.h>
#define ILLEGAL (int)'?'
#define END -1
#define SPECIAL_OPT 1
int optind = 0;
char *optarg;
static char EMPT[] = "";
static char *ptr = EMPT;
static int no_argument = 0;
static int required_argument = 1;
static const char no_arg[] = "option doesn't take an argument --%.*s";
static const char non_opt_string[] = "not an option : %s";
static const char ill_shortopt_char[] = "unknown option -%c";
static const char ill_longopt_string[] = "unknown option --%s";
static const char req_arg_string[] = "option requires an argument --%s";
static const char req_arg_char[] = "option requires an argument -%c";
struct option {
const char *name;
int has_arg;
int *flag;
int value;
};
static
void warn(const char *fmt, ...) {
va_list args;
va_start(args, fmt);
vfprintf(stdout, fmt, args);
fprintf(stdout, "\n");
va_end(args);
}
int getopt_long(int nargc, char *const *nargv, const char *options,
const struct option *long_options, int *idx) {
char *check, *equal;
size_t current_opt_len;
bool all_flag = false;
int match = -1;
// illegal
if (options == NULL) {
return ILLEGAL;
}
if (optind == 0) {
optind = 1;
}
if (optind >= nargc) {
return END;
}
if (*options == '-') {
all_flag = true;
++options;
}
optarg = NULL;
// illegal
if (*(ptr = nargv[optind]) != '-') {
ptr = EMPT;
if (all_flag) {
optarg = nargv[optind++];
return SPECIAL_OPT;
} else {
warn(non_opt_string, nargv[optind]);
return ILLEGAL;
}
}
// likely a short option ?
if (ptr[1] != '\0' && *++ptr != '-' && ptr[1] == '\0') {
char opt_char = *ptr;
ptr = EMPT;
// really short option ?
if ((check = (char *)strchr(options, opt_char)) != NULL) {
if (check[1] == ':') {
++optind;
if (optind >= nargc) {
warn(req_arg_char, opt_char);
return ILLEGAL;
} else {
optarg = nargv[optind];
}
}
++optind;
return opt_char;
} else { // illegal
warn(ill_shortopt_char, opt_char);
return ILLEGAL;
}
}
// we meet '--'
if (*ptr == '-' && ptr[1] == '\0') {
ptr = EMPT;
return END;
}
// we meet '--foo' , long option
if (long_options != NULL && *ptr == '-' && ptr[1] != '\0') {
++ptr;
if ((equal = strchr(ptr, '=')) != NULL) {
// found --option=arg
current_opt_len = equal - ptr;
++equal;
} else {
current_opt_len = strlen(ptr);
}
for (int i = 0; long_options[i].name; i++) {
if (!strcmp(ptr, long_options[i].name )) {
match = i;
break;
}
}
if (match == -1) { // don't match
warn(ill_longopt_string, ptr);
ptr = EMPT;
return ILLEGAL;
} else {
++optind;
if (long_options[match].has_arg == required_argument) {
if (equal) {
optarg = equal;
} else if (optind < nargc) {
optarg = nargv[optind++];
} else {
warn(req_arg_string, ptr);
ptr = EMPT;
return ILLEGAL;
}
}
if (long_options[match].has_arg == no_argument && equal) {
warn(no_arg, (int)current_opt_len, ptr);
ptr = EMPT;
return ILLEGAL;
}
ptr = EMPT;
if (long_options[match].flag) {
*long_options[match].flag = long_options[match].value;
return 0;
} else {
return (long_options[match].value);
}
}
}
warn(non_opt_string, ptr);
ptr = EMPT;
return ILLEGAL;
}
#endif