github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-09-29 19:24:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

Windows porting: port hyperscan and chimera tools to windows.

Browse Source
This commit is contained in:
Lu, Qi
2018-05-08 12:05:44 -04:00
committed by Wang, Xiang W
parent bf87f8c003
commit 5a0885d235
34 changed files with 563 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tools/hsbench/CMakeLists.txt
View File

@@ -61,8 +61,13 @@ endif()
add_executable(hsbench ${hsbench_SOURCES})
if (BUILD_CHIMERA)
include_directories(${PCRE_INCLUDE_DIRS})
target_link_libraries(hsbench hs chimera ${PCRE_LDFLAGS} databaseutil
expressionutil ${SQLITE3_LDFLAGS} ${CMAKE_THREAD_LIBS_INIT})
if(NOT WIN32)
target_link_libraries(hsbench hs chimera ${PCRE_LDFLAGS} databaseutil
expressionutil ${SQLITE3_LDFLAGS} ${CMAKE_THREAD_LIBS_INIT})
else()
target_link_libraries(hsbench hs chimera pcre databaseutil
expressionutil ${SQLITE3_LDFLAGS} ${CMAKE_THREAD_LIBS_INIT})
endif()
else()
target_link_libraries(hsbench hs databaseutil expressionutil
${SQLITE3_LDFLAGS} ${CMAKE_THREAD_LIBS_INIT})

21
tools/hsbench/engine_chimera.cpp
View File

@@ -68,8 +68,9 @@ struct ScanCHContext {
* "echo matches" is off.
*/
static
int onMatch(unsigned int, unsigned long long, unsigned long long, unsigned int,
unsigned int, const ch_capture_t *, void *ctx) {
int HS_CDECL onMatch(unsigned int, unsigned long long, unsigned long long,
unsigned int, unsigned int, const ch_capture_t *,
void *ctx) {
ScanCHContext *sc = static_cast<ScanCHContext *>(ctx);
assert(sc);
sc->result.matches++;
@@ -82,8 +83,9 @@ int onMatch(unsigned int, unsigned long long, unsigned long long, unsigned int,
* matches" is enabled.
*/
static
int onMatchEcho(unsigned int id, unsigned long long, unsigned long long to,
unsigned int, unsigned int, const ch_capture_t *, void *ctx) {
int HS_CDECL onMatchEcho(unsigned int id, unsigned long long,
unsigned long long to, unsigned int, unsigned int,
const ch_capture_t *, void *ctx) {
ScanCHContext *sc = static_cast<ScanCHContext *>(ctx);
assert(sc);
sc->result.matches++;
@@ -166,12 +168,23 @@ void EngineChimera::printStats() const {
}
printf("Signatures: %s\n", compile_stats.signatures.c_str());
printf("Chimera info: %s\n", compile_stats.db_info.c_str());
#ifndef _WIN32
printf("Expression count: %'zu\n", compile_stats.expressionCount);
printf("Bytecode size: %'zu bytes\n", compile_stats.compiledSize);
#else
printf("Expression count: %zu\n", compile_stats.expressionCount);
printf("Bytecode size: %zu bytes\n", compile_stats.compiledSize);
#endif
printf("Database CRC: 0x%x\n", compile_stats.crc32);
#ifndef _WIN32
printf("Scratch size: %'zu bytes\n", compile_stats.scratchSize);
printf("Compile time: %'0.3Lf seconds\n", compile_stats.compileSecs);
printf("Peak heap usage: %'u bytes\n", compile_stats.peakMemorySize);
#else
printf("Scratch size: %zu bytes\n", compile_stats.scratchSize);
printf("Compile time: %0.3Lf seconds\n", compile_stats.compileSecs);
printf("Peak heap usage: %u bytes\n", compile_stats.peakMemorySize);
#endif
}
void EngineChimera::sqlStats(SqlDB &sqldb) const {

23
tools/hsbench/engine_hyperscan.cpp
View File

@@ -87,8 +87,8 @@ struct ScanHSContext {
* "echo matches" is off.
*/
static
int onMatch(unsigned int, unsigned long long, unsigned long long, unsigned int,
void *ctx) {
int HS_CDECL onMatch(unsigned int, unsigned long long,
unsigned long long, unsigned int, void *ctx) {
ScanHSContext *sc = static_cast<ScanHSContext *>(ctx);
assert(sc);
sc->result.matches++;
@@ -101,8 +101,8 @@ int onMatch(unsigned int, unsigned long long, unsigned long long, unsigned int,
* matches" is enabled.
*/
static
int onMatchEcho(unsigned int id, unsigned long long, unsigned long long to,
unsigned int, void *ctx) {
int HS_CDECL onMatchEcho(unsigned int id, unsigned long long,
unsigned long long to, unsigned int, void *ctx) {
ScanHSContext *sc = static_cast<ScanHSContext *>(ctx);
assert(sc);
sc->result.matches++;
@@ -250,15 +250,30 @@ void EngineHyperscan::printStats() const {
}
printf("Signatures: %s\n", compile_stats.signatures.c_str());
printf("Hyperscan info: %s\n", compile_stats.db_info.c_str());
#ifndef _WIN32
printf("Expression count: %'zu\n", compile_stats.expressionCount);
printf("Bytecode size: %'zu bytes\n", compile_stats.compiledSize);
#else
printf("Expression count: %zu\n", compile_stats.expressionCount);
printf("Bytecode size: %zu bytes\n", compile_stats.compiledSize);
#endif
printf("Database CRC: 0x%x\n", compile_stats.crc32);
if (compile_stats.streaming) {
#ifndef _WIN32
printf("Stream state size: %'zu bytes\n", compile_stats.streamSize);
#else
printf("Stream state size: %zu bytes\n", compile_stats.streamSize);
#endif
}
#ifndef _WIN32
printf("Scratch size: %'zu bytes\n", compile_stats.scratchSize);
printf("Compile time: %'0.3Lf seconds\n", compile_stats.compileSecs);
printf("Peak heap usage: %'u bytes\n", compile_stats.peakMemorySize);
#else
printf("Scratch size: %zu bytes\n", compile_stats.scratchSize);
printf("Compile time: %0.3Lf seconds\n", compile_stats.compileSecs);
printf("Peak heap usage: %u bytes\n", compile_stats.peakMemorySize);
#endif
}
void EngineHyperscan::sqlStats(SqlDB &sqldb) const {

13
tools/hsbench/engine_pcre.cpp
View File

@@ -26,6 +26,9 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifdef _WIN32
#define PCRE_STATIC
#endif
#include "config.h"
#include "common.h"
@@ -38,6 +41,8 @@
#include "util/make_unique.h"
#include "util/unicode_def.h"
#include <algorithm>
using namespace std;
EnginePCREContext::EnginePCREContext(int capture_cnt) {
@@ -207,11 +212,19 @@ void EnginePCRE::printStats() const {
}
printf("Signatures: %s\n", compile_stats.signatures.c_str());
printf("PCRE info: %s\n", compile_stats.db_info.c_str());
#ifndef _WIN32
printf("Expression count: %'zu\n", compile_stats.expressionCount);
printf("Bytecode size: %'zu bytes\n", compile_stats.compiledSize);
printf("Scratch size: %'zu bytes\n", compile_stats.scratchSize);
printf("Compile time: %'0.3Lf seconds\n", compile_stats.compileSecs);
printf("Peak heap usage: %'u bytes\n", compile_stats.peakMemorySize);
#else
printf("Expression count: %zu\n", compile_stats.expressionCount);
printf("Bytecode size: %zu bytes\n", compile_stats.compiledSize);
printf("Scratch size: %zu bytes\n", compile_stats.scratchSize);
printf("Compile time: %0.3Lf seconds\n", compile_stats.compileSecs);
printf("Peak heap usage: %u bytes\n", compile_stats.peakMemorySize);
#endif
}
void EnginePCRE::sqlStats(SqlDB &sqldb) const {

63
tools/hsbench/main.cpp
View File

@@ -58,7 +58,11 @@
#include <set>
#include <thread>
#ifndef _WIN32
#include <getopt.h>
#else
#include "win_getopt.h"
#endif
#ifndef _WIN32
#include <pthread.h>
#if defined(HAVE_PTHREAD_NP_H)
@@ -138,6 +142,16 @@ public:
// Apply processor affinity (if available) to this thread.
bool affine(UNUSED int cpu) {
#if defined(_WIN32)
SYSTEM_INFO system_info;
GetSystemInfo(&system_info);
assert(cpu >= 0 && (DWORD)cpu < system_info.dwNumberOfProcessors);
DWORD_PTR mask = 1 << cpu;
DWORD_PTR rv = SetThreadAffinityMask(thr.native_handle(), mask);
return rv != 0;
#endif
#ifdef HAVE_DECL_PTHREAD_SETAFFINITY_NP
#if defined(__FreeBSD__)
cpuset_t cpuset;
@@ -191,7 +205,7 @@ void usage(const char *error) {
printf(" -H Benchmark using Chimera (if supported).\n");
printf(" -P Benchmark using PCRE (if supported).\n");
#endif
#ifdef HAVE_DECL_PTHREAD_SETAFFINITY_NP
#if defined(HAVE_DECL_PTHREAD_SETAFFINITY_NP) || defined(_WIN32)
printf(" -T CPU,CPU,... Benchmark with threads on these CPUs.\n");
#endif
printf(" -i DIR Don't compile, load from files in DIR"
@@ -225,7 +239,7 @@ static
void processArgs(int argc, char *argv[], vector<BenchmarkSigs> &sigSets,
UNUSED unique_ptr<Grey> &grey) {
const char options[] = "-b:c:Cd:e:E:G:hHi:n:No:p:PsS:Vw:z:"
#ifdef HAVE_DECL_PTHREAD_SETAFFINITY_NP
#if defined(HAVE_DECL_PTHREAD_SETAFFINITY_NP) || defined(_WIN32)
"T:" // add the thread flag
#endif
;
@@ -332,7 +346,7 @@ void processArgs(int argc, char *argv[], vector<BenchmarkSigs> &sigSets,
case 'S':
sigName.assign(optarg);
break;
#ifdef HAVE_DECL_PTHREAD_SETAFFINITY_NP
#if defined(HAVE_DECL_PTHREAD_SETAFFINITY_NP) || defined(_WIN32)
case 'T':
if (!strToList(optarg, threadCores)) {
usage("Couldn't parse argument to -T flag, should be"
@@ -704,7 +718,11 @@ void displayPerScanResults(const vector<unique_ptr<ThreadContext>> &threads,
for (size_t j = 0; j != results.size(); j++) {
const auto &r = results[j];
double mbps = calc_mbps(r.seconds, bytesPerRun);
#ifndef _WIN32
printf("T %2u Scan %2zu: %'0.2f Mbit/sec\n", t->num, j, mbps);
#else
printf("T %2u Scan %2zu: %0.2f Mbit/sec\n", t->num, j, mbps);
#endif
}
}
printf("\n");
@@ -749,6 +767,7 @@ void displayResults(const vector<unique_ptr<ThreadContext>> &threads,
}
}
#ifndef _WIN32
printf("Time spent scanning: %'0.3f seconds\n", totalSecs);
printf("Corpus size: %'llu bytes ", bytesPerRun);
switch (scan_mode) {
@@ -764,22 +783,56 @@ void displayResults(const vector<unique_ptr<ThreadContext>> &threads,
printf("(%'zu blocks)\n", corpus_blocks.size());
break;
}
#else
printf("Time spent scanning: %0.3f seconds\n", totalSecs);
printf("Corpus size: %llu bytes ", bytesPerRun);
switch (scan_mode) {
case ScanMode::STREAMING:
printf("(%zu blocks in %llu streams)\n", corpus_blocks.size(),
count_streams(corpus_blocks));
break;
case ScanMode::VECTORED:
printf("(%zu blocks in %llu vectors)\n", corpus_blocks.size(),
count_streams(corpus_blocks));
break;
case ScanMode::BLOCK:
printf("(%zu blocks)\n", corpus_blocks.size());
break;
}
#endif
u64a totalBytes = bytesPerRun * repeats * threads.size();
u64a totalBlocks = corpus_blocks.size() * repeats * threads.size();
double matchRate = ((double)matchesPerRun * 1024) / bytesPerRun;
#ifndef _WIN32
printf("Matches per iteration: %'llu (%'0.3f matches/kilobyte)\n",
matchesPerRun, matchRate);
#else
printf("Matches per iteration: %llu (%0.3f matches/kilobyte)\n",
matchesPerRun, matchRate);
#endif
double blockRate = (double)totalBlocks / (double)totalSecs;
#ifndef _WIN32
printf("Overall block rate: %'0.2f blocks/sec\n", blockRate);
printf("Mean throughput (overall): %'0.2Lf Mbit/sec\n",
calc_mbps(totalSecs, totalBytes));
#else
printf("Overall block rate: %0.2f blocks/sec\n", blockRate);
printf("Mean throughput (overall): %0.2Lf Mbit/sec\n",
calc_mbps(totalSecs, totalBytes));
#endif
double lowestScanTime = fastestResult(threads);
#ifndef _WIN32
printf("Max throughput (per core): %'0.2Lf Mbit/sec\n",
calc_mbps(lowestScanTime, bytesPerRun));
#else
printf("Max throughput (per core): %0.2Lf Mbit/sec\n",
calc_mbps(lowestScanTime, bytesPerRun));
#endif
printf("\n");
if (display_per_scan) {
@@ -892,7 +945,7 @@ void runBenchmark(const Engine &db,
numThreads = 1;
} else {
numThreads = threadCores.size();
#ifdef HAVE_DECL_PTHREAD_SETAFFINITY_NP
#if defined(HAVE_DECL_PTHREAD_SETAFFINITY_NP) || defined(_WIN32)
useAffinity = true;
#else
useAffinity = false;
@@ -932,7 +985,7 @@ void runBenchmark(const Engine &db,
} // namespace
/** Main driver. */
int main(int argc, char *argv[]) {
int HS_CDECL main(int argc, char *argv[]) {
unique_ptr<Grey> grey;
#if !defined(RELEASE_BUILD)
grey = make_unique<Grey>();