From 5f6291529f4db5d40ad005f738266e6169e8b4ab Mon Sep 17 00:00:00 2001 From: Alex Coyte Date: Mon, 6 Mar 2017 11:58:53 +1100 Subject: [PATCH] hsbench: add stream compress functionality --- tools/hsbench/common.h | 1 + tools/hsbench/engine_hyperscan.cpp | 29 ++++++++++++++++++ tools/hsbench/engine_hyperscan.h | 6 +++- tools/hsbench/main.cpp | 49 +++++++++++++++++++++++++++--- 4 files changed, 80 insertions(+), 5 deletions(-) diff --git a/tools/hsbench/common.h b/tools/hsbench/common.h index efff3f99..a8295911 100644 --- a/tools/hsbench/common.h +++ b/tools/hsbench/common.h @@ -40,5 +40,6 @@ extern std::string serializePath; extern unsigned int somPrecisionMode; extern bool forceEditDistance; extern unsigned editDistance; +extern bool printCompressSize; #endif // COMMON_H diff --git a/tools/hsbench/engine_hyperscan.cpp b/tools/hsbench/engine_hyperscan.cpp index 9674e5c8..5f188472 100644 --- a/tools/hsbench/engine_hyperscan.cpp +++ b/tools/hsbench/engine_hyperscan.cpp @@ -205,6 +205,35 @@ void EngineHyperscan::streamScan(EngineStream &stream, const char *data, } } +void EngineHyperscan::streamCompressExpand(EngineStream &stream, + vector &temp) const { + size_t used = 0; + hs_error_t err = hs_compress_stream(stream.id, temp.data(), temp.size(), + &used); + if (err == HS_INSUFFICIENT_SPACE) { + temp.resize(used); + err = hs_compress_stream(stream.id, temp.data(), temp.size(), &used); + } + + if (err != HS_SUCCESS) { + printf("Fatal error: hs_compress_stream returned error %d\n", err); + abort(); + } + + if (printCompressSize) { + printf("stream %u: compressed to %zu\n", stream.sn, used); + } + + err = hs_reset_and_expand_stream(stream.id, temp.data(), temp.size(), + nullptr, nullptr, nullptr); + + if (err != HS_SUCCESS) { + printf("Fatal error: hs_reset_and expand_stream returned error %d\n", + err); + abort(); + } +} + static unsigned makeModeFlags(ScanMode scan_mode) { switch (scan_mode) { diff --git a/tools/hsbench/engine_hyperscan.h b/tools/hsbench/engine_hyperscan.h index 7875decc..2c93959b 100644 --- a/tools/hsbench/engine_hyperscan.h +++ b/tools/hsbench/engine_hyperscan.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Intel Corporation + * Copyright (c) 2016-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,6 +34,7 @@ #include "hs_runtime.h" #include +#include /** Structure for the result of a single complete scan. */ struct ResultEntry { @@ -79,6 +80,9 @@ public: void streamClose(std::unique_ptr stream, ResultEntry &result) const; + void streamCompressExpand(EngineStream &stream, + std::vector &temp) const; + void streamScan(EngineStream &stream, const char *data, unsigned int len, unsigned int id, ResultEntry &result) const; diff --git a/tools/hsbench/main.cpp b/tools/hsbench/main.cpp index 9c5fd6cb..f2ea8e7e 100644 --- a/tools/hsbench/main.cpp +++ b/tools/hsbench/main.cpp @@ -77,10 +77,13 @@ string serializePath(""); unsigned int somPrecisionMode = HS_MODE_SOM_HORIZON_LARGE; bool forceEditDistance = false; unsigned editDistance = 0; +bool printCompressSize = false; + +// Globals local to this file. +static bool compressStream = false; namespace /* anonymous */ { -// Globals local to this file. bool display_per_scan = false; ScanMode scan_mode = ScanMode::STREAMING; unsigned repeats = 20; @@ -212,11 +215,15 @@ void processArgs(int argc, char *argv[], vector &sigSets, int in_sigfile = 0; int do_per_scan = 0; int do_echo_matches = 0; + int do_compress = 0; + int do_compress_size = 0; vector sigFiles; static struct option longopts[] = { {"per-scan", 0, &do_per_scan, 1}, {"echo-matches", 0, &do_echo_matches, 1}, + {"compress-stream", 0, &do_compress, 1}, + {"print-compress-size", 0, &do_compress_size, 1}, {nullptr, 0, nullptr, 0} }; @@ -338,6 +345,12 @@ void processArgs(int argc, char *argv[], vector &sigSets, if (do_per_scan) { display_per_scan = true; } + if (do_compress) { + compressStream = true; + } + if (do_compress_size) { + printCompressSize = true; + } if (exprPath.empty() && !sigFiles.empty()) { /* attempt to infer an expression directory */ @@ -470,10 +483,12 @@ vector prepStreamingData(const ThreadContext *ctx) { } static -void benchStreamingInternal(ThreadContext *ctx, vector &streams) { +void benchStreamingInternal(ThreadContext *ctx, vector &streams, + bool do_compress) { assert(ctx); const EngineHyperscan &e = ctx->engine; const vector &blocks = ctx->corpus_data; + vector compress_buf(do_compress ? 1000 : 0); for (ResultEntry &r : ctx->results) { ctx->timer.start(); @@ -491,6 +506,8 @@ void benchStreamingInternal(ThreadContext *ctx, vector &streams) { printf("Fatal error: stream open failed!\n"); exit(1); } + } else if (do_compress) { + e.streamCompressExpand(*stream.eng_handle, compress_buf); } assert(stream.eng_handle); @@ -521,7 +538,7 @@ void benchStreaming(void *context) { startTotalTimer(ctx); - benchStreamingInternal(ctx, streams); + benchStreamingInternal(ctx, streams, false); // Synchronization point ctx->barrier(); @@ -530,6 +547,26 @@ void benchStreaming(void *context) { stopTotalTimer(ctx); } +static +void benchStreamingCompress(void *context) { + ThreadContext *ctx = (ThreadContext *)context; + vector streams = prepStreamingData(ctx); + + // Synchronization point + ctx->barrier(); + + startTotalTimer(ctx); + + benchStreamingInternal(ctx, streams, true); + + // Synchronization point + ctx->barrier(); + + // Now that all threads are finished, we can stop the clock. + stopTotalTimer(ctx); +} + + /** In-memory structure for a data block to be scanned in vectored mode. */ struct VectoredInfo { vector data; @@ -704,7 +741,11 @@ unique_ptr makeThreadContext(const EngineHyperscan &db, thread_func_t fn = nullptr; switch (scan_mode) { case ScanMode::STREAMING: - fn = benchStreaming; + if (compressStream) { + fn = benchStreamingCompress; + } else { + fn = benchStreaming; + } break; case ScanMode::VECTORED: fn = benchVectored;