github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-09-29 11:16:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

hsbench: add Hyperscan benchmarker

Browse Source 
The hsbench tool provides an easy way to measure Hyperscan's
performance for a particular set of patterns and corpus of data
to be scanned.
This commit is contained in:
Matthew Barr
2016-12-14 15:26:01 +11:00
parent 06cde4c94d
commit f626276271
26 changed files with 3145 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
util/CMakeLists.txt
View File

@@ -1,7 +1,10 @@
# utility libs
CHECK_FUNCTION_EXISTS(mmap HAVE_MMAP)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXX_FLAGS}")
include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}
${PROJECT_SOURCE_DIR})
set_source_files_properties(
${CMAKE_BINARY_DIR}/tools/ExpressionParser.cpp
@@ -31,3 +34,14 @@ SET(corpusomatic_SRCS
)
add_library(corpusomatic STATIC ${corpusomatic_SRCS})
set(databaseutil_SRCS
database_util.cpp
database_util.h
)
add_library(databaseutil STATIC ${databaseutil_SRCS})
set(crosscompileutil_SRCS
cross_compile.cpp
cross_compile.h
)
add_library(crosscompileutil STATIC ${crosscompileutil_SRCS})

115
util/cross_compile.cpp Normal file
View File

@@ -0,0 +1,115 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "cross_compile.h"
#include "src/ue2common.h"
#include "src/hs_compile.h"
#include "src/util/make_unique.h"
#include <sstream>
#include <string>
using namespace std;
struct XcompileMode {
const char *name;
unsigned long long cpu_features;
};
static const XcompileMode xcompile_options[] = {
{ "avx2", HS_CPU_FEATURES_AVX2 },
{ "base", 0 },
};
unique_ptr<hs_platform_info> xcompileReadMode(const char *s) {
hs_platform_info rv;
UNUSED hs_error_t err;
err = hs_populate_platform(&rv);
assert(!err);
string str(s);
string mode = str.substr(0, str.find(":"));
string opt = str.substr(str.find(":")+1, str.npos);
bool found_mode = false;
if (!opt.empty()) {
const size_t numOpts = ARRAY_LENGTH(xcompile_options);
for (size_t i = 0; i < numOpts; i++) {
if (opt.compare(xcompile_options[i].name) == 0) {
DEBUG_PRINTF("found opt %zu:%llu\n", i,
xcompile_options[i].cpu_features);
rv.cpu_features = xcompile_options[i].cpu_features;
found_mode = true;
break;
}
}
}
if (!found_mode) {
return nullptr;
} else {
DEBUG_PRINTF("cpu_features %llx\n", rv.cpu_features);
return ue2::make_unique<hs_platform_info>(rv);
}
}
string to_string(const hs_platform_info &p) {
ostringstream out;
if (p.tune) {
out << p.tune;
}
if (p.cpu_features) {
u64a features = p.cpu_features;
if (features & HS_CPU_FEATURES_AVX2) {
out << " avx2";
features &= ~HS_CPU_FEATURES_AVX2;
}
if (features) {
out << " " << "?cpu_features?:" << features;
}
}
return out.str();
}
string xcompileUsage(void) {
string variants = "Instruction set options: ";
const size_t numOpts = ARRAY_LENGTH(xcompile_options);
for (size_t i = 0; i < numOpts; i++) {
variants += xcompile_options[i].name;
if (i + 1 != numOpts) {
variants += ", ";
}
}
return variants;
}

42
util/cross_compile.h Normal file
View File

@@ -0,0 +1,42 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef CROSS_COMPILE_H
#define CROSS_COMPILE_H
#include <memory>
#include <string>
struct hs_platform_info;
std::unique_ptr<hs_platform_info> xcompileReadMode(const char *s);
std::string xcompileUsage(void);
std::string to_string(const hs_platform_info &p);
#endif /* CROSS_COMPILE_H */

155
util/database_util.cpp Normal file
View File

@@ -0,0 +1,155 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "database_util.h"
#include "hs_common.h"
#include <cassert>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#if defined(HAVE_MMAP)
#include <sys/mman.h> // for mmap
#include <unistd.h> // for close
#include <sys/fcntl.h>
#include <sys/stat.h>
#endif
using namespace std;
bool saveDatabase(const hs_database_t *db, const char *filename, bool verbose) {
assert(db);
assert(filename);
if (verbose) {
cout << "Saving database to: " << filename << endl;
}
char *bytes = nullptr;
size_t length = 0;
hs_error_t err = hs_serialize_database(db, &bytes, &length);
if (err != HS_SUCCESS) {
return false;
}
assert(bytes);
assert(length > 0);
ofstream out(filename, ios::binary);
out.write(bytes, length);
out.close();
::free(bytes);
return true;
}
hs_database_t * loadDatabase(const char *filename, bool verbose) {
assert(filename);
if (verbose) {
cout << "Loading database from: " << filename << endl;
}
char *bytes = nullptr;
#if defined(HAVE_MMAP)
// Use mmap to read the file
int fd = open(filename, O_RDONLY);
if (fd < 0) {
return nullptr;
}
struct stat st;
if (fstat(fd, &st) < 0) {
close(fd);
return nullptr;
}
size_t len = st.st_size;
bytes = (char *)mmap(nullptr, len, PROT_READ, MAP_SHARED, fd, 0);
if (bytes == MAP_FAILED) {
cout << "mmap failed" << endl;
close(fd);
return nullptr;
}
#else
// Fall back on stream IO
ifstream is;
is.open(filename, ios::in | ios::binary);
if (!is.is_open()) {
return nullptr;
}
is.seekg(0, ios::end);
size_t len = is.tellg();
if (verbose) {
cout << "Reading " << len << " bytes" << endl;
}
is.seekg(0, ios::beg);
bytes = new char[len];
is.read(bytes, len);
is.close();
#endif
assert(bytes);
if (verbose) {
char *info = nullptr;
hs_error_t err = hs_serialized_database_info(bytes, len, &info);
if (err) {
cout << "Unable to decode serialized database info: " << err
<< endl;
} else if (info) {
cout << "Serialized database info: " << info << endl;
std::free(info);
} else {
cout << "Unable to decode serialized database info." << endl;
}
}
hs_database_t *db = nullptr;
hs_error_t err = hs_deserialize_database(bytes, len, &db);
#if defined(HAVE_MMAP)
munmap(bytes, len);
close(fd);
#else
delete [] bytes;
#endif
if (err != HS_SUCCESS) {
cout << "hs_deserialize_database call failed: " << err << endl;
return nullptr;
}
assert(db);
return db;
}

39
util/database_util.h Normal file
View File

@@ -0,0 +1,39 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef DATABASE_UTIL_H
#define DATABASE_UTIL_H
struct hs_database;
bool saveDatabase(const hs_database *db, const char *filename,
bool verbose = false);
hs_database *loadDatabase(const char *filename, bool verbose = false);
#endif /* DATABASE_UTIL_H */

107
util/expression_path.h Normal file
View File

@@ -0,0 +1,107 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef EXPRESSION_PATH_H
#define EXPRESSION_PATH_H
#include "ue2common.h"
#include <cerrno>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
#include <sys/stat.h>
#if !defined(_WIN32)
#include <unistd.h>
#include <libgen.h>
#endif
//
// Utility functions
//
/**
* Given a path to a signature file, infer the path of the pcre directory.
*/
static inline
std::string inferExpressionPath(const std::string &sigFile) {
#ifndef _WIN32
// POSIX variant.
// dirname() may modify its argument, so we must make a copy.
std::vector<char> path(sigFile.size() + 1);
memcpy(path.data(), sigFile.c_str(), sigFile.size());
path[sigFile.size()] = 0; // ensure null termination.
std::string rv = dirname(path.data());
#else
// Windows variant.
if (sigFile.size() >= _MAX_DIR) {
return std::string();
}
char path[_MAX_DIR];
_splitpath(sigFile.c_str(), nullptr, path, nullptr, nullptr);
std::string rv(path);
#endif
rv += "/../pcre";
return rv;
}
#if defined(_WIN32)
#define stat _stat
#define S_IFREG _S_IFREG
#endif
static inline
bool isDir(const std::string &filename) {
struct stat s;
if (stat(filename.c_str(), &s) == -1) {
std::cerr << "stat: " << strerror(errno) << std::endl;
return false;
}
return (S_IFDIR & s.st_mode);
}
static inline
bool isFile(const std::string &filename) {
struct stat s;
if (stat(filename.c_str(), &s) == -1) {
std::cerr << "stat: " << strerror(errno) << std::endl;
return false;
}
return (S_IFREG & s.st_mode);
}
#endif /* EXPRESSION_PATH_H */