github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-09-29 19:24:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

Initial commit of Hyperscan

Browse Source
This commit is contained in:
Matthew Barr
2015-10-20 09:13:35 +11:00
commit 904e436f11
610 changed files with 213627 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
unit/hyperscan/allocators.cpp Normal file
View File

@@ -0,0 +1,151 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "gtest/gtest.h"
#include "hs.h"
#include "test_util.h"
#include <cstdlib>
static void *null_malloc(size_t) { return nullptr; }
TEST(CustomAllocator, DatabaseInfoBadAlloc) {
hs_database_t *db = buildDB("foobar", 0, 0, HS_MODE_BLOCK);
ASSERT_TRUE(db != nullptr);
hs_set_allocator(null_malloc, nullptr);
char *info = nullptr;
hs_error_t err = hs_database_info(db, &info);
ASSERT_EQ(HS_NOMEM, err);
hs_set_allocator(nullptr, nullptr);
hs_free_database(db);
}
static
void * two_aligned_malloc(size_t len) {
void *mem = malloc(len + 2);
if (!mem) {
return nullptr;
}
return (char *)mem + 2;
}
static
void two_aligned_free(void *mem) {
if (!mem) {
return;
}
// Allocated with two_aligned_malloc above.
free((char *)mem - 2);
}
TEST(CustomAllocator, TwoAlignedCompile) {
hs_set_database_allocator(two_aligned_malloc, two_aligned_free);
hs_database_t *db = nullptr;
hs_compile_error_t *compile_err = nullptr;
const hs_platform_info_t *platform = nullptr;
hs_error_t err =
hs_compile("foobar", 0, HS_MODE_BLOCK, platform, &db, &compile_err);
ASSERT_EQ(HS_COMPILER_ERROR, err);
ASSERT_EQ(nullptr, db);
ASSERT_NE(nullptr, compile_err);
hs_free_compile_error(compile_err);
hs_set_database_allocator(nullptr, nullptr);
}
TEST(CustomAllocator, TwoAlignedDatabaseInfo) {
hs_database_t *db = buildDB("foobar", 0, 0, HS_MODE_BLOCK);
ASSERT_TRUE(db != nullptr);
hs_set_misc_allocator(two_aligned_malloc, two_aligned_free);
char *info = nullptr;
hs_error_t err = hs_database_info(db, &info);
ASSERT_EQ(HS_BAD_ALLOC, err);
hs_set_misc_allocator(nullptr, nullptr);
hs_free_database(db);
}
TEST(CustomAllocator, TwoAlignedSerialize) {
hs_database_t *db = buildDB("foobar", 0, 0, HS_MODE_BLOCK);
ASSERT_TRUE(db != nullptr);
hs_set_misc_allocator(two_aligned_malloc, two_aligned_free);
char *bytes = nullptr;
size_t serialized_len = 0;
hs_error_t err = hs_serialize_database(db, &bytes, &serialized_len);
ASSERT_EQ(HS_BAD_ALLOC, err);
hs_set_misc_allocator(nullptr, nullptr);
hs_free_database(db);
}
TEST(CustomAllocator, TwoAlignedDeserialize) {
hs_database_t *db = buildDB("foobar", 0, 0, HS_MODE_BLOCK);
ASSERT_TRUE(db != nullptr);
char *bytes = nullptr;
size_t serialized_len = 0;
hs_error_t err = hs_serialize_database(db, &bytes, &serialized_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(nullptr, bytes);
hs_free_database(db);
db = nullptr;
hs_set_database_allocator(two_aligned_malloc, two_aligned_free);
err = hs_deserialize_database(bytes, serialized_len, &db);
ASSERT_EQ(HS_BAD_ALLOC, err);
ASSERT_EQ(nullptr, db);
hs_set_database_allocator(nullptr, nullptr);
free(bytes);
}
TEST(CustomAllocator, TwoAlignedAllocScratch) {
hs_database_t *db = buildDB("foobar", 0, 0, HS_MODE_BLOCK);
ASSERT_TRUE(db != nullptr);
hs_set_scratch_allocator(two_aligned_malloc, two_aligned_free);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_BAD_ALLOC, err);
hs_set_scratch_allocator(nullptr, nullptr);
hs_free_database(db);
}

2241
unit/hyperscan/arg_checks.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

402
unit/hyperscan/bad_patterns.cpp Normal file
View File

@@ -0,0 +1,402 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "gtest/gtest.h"
#include "hs.h"
#include <fstream>
#include <string>
#include <boost/algorithm/string/trim.hpp>
#include "util/expressions.h"
#include "util/ExpressionParser.h"
using namespace std;
using namespace testing;
class HyperscanFailBadPattern
: public TestWithParam<tuple<bool, const char*>> {
protected:
virtual void SetUp() {
// compiles a database for this test instantiation
bool isStreaming;
tie(isStreaming, pattern) = GetParam();
err = hs_compile(pattern, 0,
isStreaming ? HS_MODE_STREAM : HS_MODE_BLOCK, nullptr,
&db, &compile_err);
}
virtual void TearDown() {
hs_free_database(db);
hs_free_compile_error(compile_err);
}
hs_error_t err;
const char *pattern;
hs_compile_error_t *compile_err;
hs_database_t *db;
};
static const char *badPatterns[] = {
// vacuous patterns
"",
"a?",
"a*",
"(foo)?",
"(foo)*(bar)*",
"^arg|(foo)*(bar)*",
"foo|bar|",
"a*(b?)*c*",
"a{0,3}",
"[\\s\\S]*",
"[^\\s\\S]*",
// too big for our largest NFA
"(ewh|m?uit|f|snmv.g.gx[yofl]|.[^g][hbd])((.h|((y|vypfw|dfg{4}|x+|o.|y{8,}))+|k{9}t|cgp...gsk+)){17,}",
// illegal bounds
"fooa{0}",
"a{4,3}",
"a{2,1}",
// nothing to repeat
"a++",
"a+?+",
"a??",
"a?+",
"?qa",
"*abc",
"+abc",
// repeating boundaries is not allowed (UE-1007)
"^?0",
"^*0",
"^+0",
"^{1,3}0",
"0$?",
"0$*",
"0$+",
"0${1,3}",
// zero width asserts ("lookarounds")
"[a-z]+(?=;)", // positive lookahead
".((?!\\x00\\x00)..)*?foo", // negative lookahead
"(?<=bullock|donkey)", // positive lookbehind
"(?<!foo)bar", // negative lookbehind
// embedded anchors
"foo^bar",
"foo$bar",
"(foo^bar|other)",
"(foo$bar|other)",
"$test",
"test^",
"foo(a|^)bar",
"a$^b",
"(^foo)+bar",
"foo(bar$)+",
"a^{3}=",
// atomic groups
"foobar(?>.{3,})bar",
// possessive quantifiers
"\\d++foo",
"(abc|xyz){2,3}+",
// back-reference inside a repeat (also too big, actually)
"^..\x02.{10,522}([^\00])\1{16}",
// char classes
"[]",
"[]foobar",
"[`-\\80",
"[A-\\K]",
// bad named classes
"[[:foo:]]",
"[[:1234:]]",
"[[:f\\oo:]]",
"[[: :]]",
"[[:...:]]",
"[[:l\\ower:]]",
"[[:abc\\:]]",
"[abc[:x\\]pqr:]]",
"[[:a\\dz:]]",
// unhandled subroutines and backrefs
"foo\\g''bar",
"foo\\g'45'bar",
"foo\\g'hatstand'bar",
"foo\\g<>bar",
"foo\\g<45>bar",
"foo\\g<teakettle>bar",
"((?i)rah)\\s+\\1",
"(?<p1>(?i)rah)\\s+\\k<p1>",
"(?'p1'(?i)rah)\\s+\\k{p1}",
"(?P<p1>(?i)rah)\\s+(?P=p1)",
"(?<p1>(?i)rah)\\s+\\g{p1}",
// truly enormous and with complicated assert resolution (UE-1107)
"((c(p|p)h{2,}bh.|p|((((cq|j|c|(\\b)|.[^nbgn]|(\\B)[qfh]a)){10,12}|ih|a|mnde[pa].|.g)){5,8})){21,29}",
// conditional subpatterns
"(?(?=[^a-z]*[a-z])\\d{2}-[a-z]{3}-\\d{2}|\\d{2}-\\d{2}-\\d{2)}",
// unmatched parens
"(foo",
"foo)",
"((foo)",
"(foo))",
// unterminated comment
"/foo(?#comment/",
// bogus \g backrefs
"A\\g",
"A(.*)\\ga",
// malformed \g backrefs (see UE-950)
"^(a)\\g",
"^(a)\\g{3",
"\\g{A",
"[\\g6666666666]",
"(^(a|b\\g<-1'c))",
// oniguruma subroutine calls (UE-950 as well)
"^(?<name>a|b\\g'name'c)",
"^(a|b\\g'1'c)",
"^(a|b\\g'-1'c)",
// backtracking control verbs
"A((?:A|B(*ACCEPT)|C)D)",
"(*FAIL)",
"(*F)",
"a+(*COMMIT)b",
"(*PRUNE)",
"a+(*SKIP)b",
// other unsupported PCRE features
"\\R",
"foo\\Kbar",
"\\Gfoo",
"(?|(Sat)ur|(Sun))day", // duplicate subpatterns, see UE-958
"foobar\\", // trailing unescaped backslash
"(?x)abc(#i)def" // unterminated extended-mode comment
};
// Did we correctly fail the compile?
TEST_P(HyperscanFailBadPattern, Compile) {
ASSERT_NE(HS_SUCCESS, err) << "Compile should have failed for expr: " << pattern;
EXPECT_EQ(HS_COMPILER_ERROR, err);
EXPECT_TRUE(db == nullptr);
EXPECT_TRUE(compile_err != nullptr);
// We shouldn't fail with the following messagess
EXPECT_STRNE("An invalid flag was specified.", compile_err->message);
EXPECT_STRNE("Unable to allocate memory.", compile_err->message);
EXPECT_STRNE("Internal error.", compile_err->message);
EXPECT_STRNE("Match can be raised on EOD", compile_err->message);
}
INSTANTIATE_TEST_CASE_P(CompileBadPatterns,
HyperscanFailBadPattern,
Combine(Bool(), ValuesIn(badPatterns)));
struct BadPatternParam {
BadPatternParam(const string &expr_in, unsigned int flags_in,
const hs_expr_ext &ext_in,
const string &expected_error_in)
: expr(expr_in), flags(flags_in), ext(ext_in),
expected_error(expected_error_in) {}
string expr;
unsigned int flags;
hs_expr_ext ext;
string expected_error;
// Wrap hs_compile_ext_multi for single patterns.
hs_error_t compile(unsigned int mode, hs_database_t **db,
hs_compile_error_t **compile_err) const {
const char *regex = expr.c_str();
const hs_expr_ext *extp = &ext;
return hs_compile_ext_multi(&regex, &flags, nullptr, &extp, 1,
mode, nullptr, db, compile_err);
}
};
void PrintTo(const BadPatternParam &p, ::std::ostream *os) {
*os << "expr: \"" << p.expr << "\", expected error: " << p.expected_error;
}
class BadPattern : public TestWithParam<BadPatternParam> {
};
TEST_P(BadPattern, Block) {
const BadPatternParam &p = GetParam();
SCOPED_TRACE(p.expr);
hs_compile_error_t *compile_err;
hs_database_t *db;
hs_error_t err = p.compile(HS_MODE_NOSTREAM, &db, &compile_err);
EXPECT_EQ(HS_COMPILER_ERROR, err);
EXPECT_TRUE(db == nullptr);
EXPECT_TRUE(compile_err != nullptr);
if (compile_err) {
EXPECT_STREQ(p.expected_error.c_str(), compile_err->message);
}
hs_free_database(db);
hs_free_compile_error(compile_err);
}
TEST_P(BadPattern, Stream) {
const BadPatternParam &p = GetParam();
SCOPED_TRACE(p.expr);
hs_compile_error_t *compile_err;
hs_database_t *db;
hs_error_t err = p.compile(HS_MODE_STREAM | HS_MODE_SOM_HORIZON_LARGE, &db,
&compile_err);
EXPECT_EQ(HS_COMPILER_ERROR, err);
EXPECT_TRUE(db == nullptr);
EXPECT_TRUE(compile_err != nullptr);
if (compile_err) {
EXPECT_STREQ(p.expected_error.c_str(), compile_err->message);
}
hs_free_database(db);
hs_free_compile_error(compile_err);
}
// happy fun preprocessor hoop jumping
#define xstr(s) str(s)
#define str(s) #s
#define SRCDIR_PREFIX xstr(SRCDIR)
static
vector<BadPatternParam> getBadPatterns() {
string filename = "unit/hyperscan/bad_patterns.txt";
ifstream f;
f.open(filename.c_str(), ifstream::in);
if (!f.good()) {
// try it with the src prefix
f.open((string(SRCDIR_PREFIX) + "/" + filename).c_str(), ifstream::in);
}
vector<BadPatternParam> rv;
if (!f.good()) {
string expr("couldn't find input file:" + filename);
cerr << expr << endl;
abort();
return rv;
}
string line;
while (f.good()) {
getline(f, line);
if (line.empty()) {
continue;
}
size_t hashIdx = line.find_first_of('#');
size_t colonIdx = line.find_first_of(':');
assert(hashIdx != string::npos);
assert(colonIdx != string::npos);
if (!hashIdx) {
continue;
}
string error = line.substr(hashIdx + 1);
string expr = line.substr(colonIdx + 1, hashIdx - colonIdx - 1);
boost::trim(expr);
unsigned int flags;
string regex;
hs_expr_ext ext;
if (!readExpression(expr, regex, &flags, &ext)) {
cerr << expr << " failed in readExpression" << endl;
abort();
}
rv.push_back(BadPatternParam(regex, flags, ext, error));
}
f.close();
return rv;
}
INSTANTIATE_TEST_CASE_P(Hyperscan, BadPattern, ValuesIn(getBadPatterns()));
TEST(ResourceLimits, longPattern) {
#define LONG_PATTERN_LEN 16384
hs_database_t *db = nullptr;
hs_compile_error_t *compile_err = nullptr;
char pattern[LONG_PATTERN_LEN];
memset(pattern, 'a', LONG_PATTERN_LEN);
pattern[LONG_PATTERN_LEN - 1] = 0;
hs_error_t err = hs_compile(pattern, HS_FLAG_DOTALL, HS_MODE_BLOCK, nullptr,
&db, &compile_err);
EXPECT_EQ(HS_COMPILER_ERROR, err);
EXPECT_TRUE(db == nullptr);
EXPECT_TRUE(compile_err != nullptr);
if (compile_err) {
EXPECT_STREQ("Pattern length exceeds limit.", compile_err->message);
}
hs_free_database(db);
hs_free_compile_error(compile_err);
}
TEST(ResourceLimits, longPatternInfo) {
#define LONG_PATTERN_LEN 16384
hs_expr_info_t *info = nullptr;
hs_compile_error_t *compile_err = nullptr;
char pattern[LONG_PATTERN_LEN];
memset(pattern, 'a', LONG_PATTERN_LEN);
pattern[LONG_PATTERN_LEN - 1] = 0;
hs_error_t err =
hs_expression_info(pattern, HS_FLAG_DOTALL, &info, &compile_err);
EXPECT_EQ(HS_COMPILER_ERROR, err);
EXPECT_TRUE(compile_err != nullptr);
if (compile_err) {
EXPECT_STREQ("Pattern length exceeds limit.", compile_err->message);
}
free(info);
hs_free_compile_error(compile_err);
}
TEST(ResourceLimits, longLiteral) {
#define LONG_PATTERN_LEN 16384
hs_database_t *db = nullptr;
hs_compile_error_t *compile_err = nullptr;
const char *pattern = "(abcd){4096}";
hs_error_t err = hs_compile(pattern, HS_FLAG_DOTALL, HS_MODE_BLOCK, nullptr,
&db, &compile_err);
EXPECT_EQ(HS_COMPILER_ERROR, err);
EXPECT_TRUE(db == nullptr);
EXPECT_TRUE(compile_err != nullptr);
if (compile_err) {
EXPECT_STREQ("Resource limit exceeded.", compile_err->message);
}
hs_free_database(db);
hs_free_compile_error(compile_err);
}

129
unit/hyperscan/bad_patterns.txt Normal file
View File

@@ -0,0 +1,129 @@
1:/\c空/ #\c must be followed by an ASCII character at index 0.
2:/\c/ #\c must be followed by an ASCII character at index 0.
3:/[\c空]/ #\c must be followed by an ASCII character at index 1.
4:/[\c]/ #Unterminated character class starting at index 0.
5:/\c空/8 #\c must be followed by an ASCII character at index 0.
6:/<([^>+i)>.*?</\1>/sP #Unterminated character class starting at index 2.
6:/[foo/ #Unterminated character class starting at index 0.
7:/[\p{X}]/8 #Unknown property at index 4.
8:/[\p{^X}]/8 #Unknown property at index 5.
9:/[\p{L]/8 #Malformed property at index 0.
10:/[\p{^L]/8 #Malformed property at index 0.
11:/[\P{L]/8 #Malformed property at index 0.
12:/[\P{^L]/8 #Malformed property at index 0.
13:/\p/8 #Malformed property at index 0.
14:/\P/8 #Malformed property at index 0.
15:/\p{/8 #Malformed property at index 0.
16:/\P{/8 #Malformed property at index 0.
17:/\p{^/8 #Malformed property at index 0.
18:/\P{^/8 #Malformed property at index 0.
19:/[\p/8 #Malformed property at index 1.
20:/[\P/8 #Malformed property at index 1.
21:/[\p{/8 #Malformed property at index 0.
22:/[\P{/8 #Malformed property at index 0.
23:/[\p{^/8 #Malformed property at index 0.
24:/[\P{^/8 #Malformed property at index 0.
25:/\pl/8 #Unknown property at index 2.
26:/\p{any}/8 #Unknown property at index 3.
27:/\p{greek}/8 #Unknown property at index 3.
28:/\b/8W #\b unsupported in UCP mode at index 0.
29:/(*UCP)\b/8 #\b unsupported in UCP mode at index 6.
30:/\B/8W #\B unsupported in UCP mode at index 0.
31:/\B/W #\B unsupported in UCP mode at index 0.
32:/foo(?{print "Hello world\n";})bar/ #Embedded code is not supported at index 3.
33:/the (\S+)(?{ $color = $^N }) (\S+)(?{ $animal = $^N })/i #Embedded code is not supported at index 9.
34:/foobar\E/s #Unmatched \E at index 6.
35:/\X/8 #\X unsupported at index 0.
36:/\B+/ #Invalid repeat at index 2.
37:/\B?/ #Invalid repeat at index 2.
38:/\B*/ #Invalid repeat at index 2.
39:/\B{0,6}/ #Invalid repeat at index 2.
40:/\b+/ #Invalid repeat at index 2.
41:/\b?/ #Invalid repeat at index 2.
42:/\b*/ #Invalid repeat at index 2.
43:/\b{0,6}/ #Invalid repeat at index 2.
44:/[.ch.]/ #Unsupported POSIX collating element at index 0.
45:/[=ch=]/ #Unsupported POSIX collating element at index 0.
46:/[:digit:]/ #POSIX named classes are only supported inside a class at index 0.
47:/[[.ch.]]/ #Unsupported POSIX collating element at index 1.
48:/[[=ch=]]/ #Unsupported POSIX collating element at index 1.
49:/foo(?m)?bar/ #Invalid repeat at index 7.
50:/.(?)+/ #Invalid repeat at index 4.
51:/(abc)\2/P #Invalid back reference to expression 2.
52:/\x{100000000}/ #Value in \x{...} sequence is too large at index 0.
53:/^foo/{min_offset=5} #Expression is anchored and cannot satisfy min_offset=5 as it can only produce matches of length 3 bytes at most.
54:/foobar/{min_length=20} #Expression has min_length=20 but can only produce matches of length 6 bytes at most.
55:/foobar/{max_offset=3} #Expression has max_offset=3 but requires 6 bytes to match.
56:/mkdzo(x|u)(\b)kd/{max_offset=29} #Pattern can never match.
57:/[^\x00-\xff]/ #Pattern can never match.
58:/[^\x00-\xff]foo/ #Pattern can never match.
59:/^\Bfoo/ #Pattern can never match.
60:/^\B\Bfoo/ #Pattern can never match.
61:/can't_match\b\B/ #Pattern can never match.
62:/\b\Bcan't_match/ #Pattern can never match.
63:/^\b$/m #Pattern can never match.
64:/^\b\Z/m #Pattern can never match.
65:/^\b\z/m #Pattern can never match.
66:/\A\b$/m #Pattern can never match.
67:/\A\b\Z/m #Pattern can never match.
68:/\A\b\z/m #Pattern can never match.
69:/^[^\x00-\xff]foo/ #Pattern can never match.
70:/foo[^\x00-\xff]/ #Pattern can never match.
71:/foo[^\x00-\xff]$/ #Pattern can never match.
72:/\Bd\B/i{min_length=2,min_offset=4,max_offset=54} #Expression has min_length=2 but can only produce matches of length 1 bytes at most.
73:/(((.|aaa)aaaaaa.aaa){14,19}a((a|a{5,6}|aa){3,11}|aa.|a){2}){4}\Z/sm #Pattern is too large.
74:/(((.|aaa)aaaaaa.aaa){14,19}a((a|a{5,6}|aa){3,11}|aa.|a){2}){4}\Z/smL #Pattern is too large.
75:/\B/s8{min_length=1} #Expression has min_length=1 but can only produce matches of length 0 bytes at most.
76:/(f|d|(\b)|i|a\Z)/mHV8{min_length=2,min_offset=9,max_offset=14} #Expression has min_length=2 but can only produce matches of length 1 bytes at most.
77:/(f|e|d{19,}|h\Z|^j|\Aa)/smi{min_length=7,min_offset=8,max_offset=18} #Extended parameter constraints can not be satisfied for any match from this expression.
78:/(i{13,}|i\Z)/s{min_length=3,max_offset=5} #Extended parameter constraints can not be satisfied for any match from this expression.
79:/(?P<dupename>foo).*(?P<dupename>bar)/ #Two named subpatterns use the name 'dupename' at index 19.
80:/_W{0,3}bazr_W{0,3}(ac[_a-z]{22}a)?e_W{0,3}bazr[_a-z](ac[a-z]{4}c{14}[a-z]{5})?e_W{0,3}bazr[_a-z](e|ac[_a-z]{4}c{16}([_a-z]|[a-p]W|[o-z]WW){3}([_a-z]|WWW))_W{0,3}bazr([_a-z]|[a-p]WW?|[o-z]WWW)a(foobar|c([a-z]W{0,3})bc([a-z]W{0,3})c{14}([_a-z]W{0,3}){6})((fooaa|[_a-z]W{0,3})bazr[_a-z]W{0,5}a(foobar|c([_a-z]|[a-z]W{1,3})bc([_a-z]|[o-z]W{1,5})c{14}([_a-f]|[A-Z0]W|~WW|;WWW){6})){40}(fooaa|_)bazr[_a-z]/sL #Pattern is too large.
81:/[..]/ #Unsupported POSIX collating element at index 0.
82:/[==]/ #Unsupported POSIX collating element at index 0.
83:/[.\].]/ #Unsupported POSIX collating element at index 0.
84:/[=\]=]/ #Unsupported POSIX collating element at index 0.
85:/A(?!)+Z/ #Invalid repeat at index 5.
86:/\X/ #\X unsupported at index 0.
87:/[a\Qz\E]/ #\Q..\E sequences in character classes not supported at index 2.
88:/[A-\d]/ #Invalid range in character class at index 3.
89:/[A-[:digit:]]/ #Invalid range in character class at index 3.
90:/B[--[:digit:]--]+/ #Invalid range in character class at index 4.
91:/a\owibble/ #Value in \o{...} sequence is non-octal or missing braces at index 1.
92:/a\o{wibble/ #Value in \o{...} sequence is non-octal or missing braces at index 1.
93:/a\o{777}/ #Value in \o{...} sequence is too large at index 1.
94:/(*UTF16)foo/ #(*UTF16) not supported at index 2.
95:/(*BSR_UNICODE)abc/ #Unknown control verb at index 2.
96:/a+(*SKIP)b/ #Unknown control verb at index 4.
97:/foo(*/ #Invalid repeat at index 4.
98:/[:\]:]/ #POSIX named classes are only supported inside a class at index 0.
99:/[[:[:]/ #Invalid POSIX named class at index 1.
100:/abc(?(1)def|ghi)/P #Invalid conditional reference to expression 1.
101:/abc(?(<blah>)def|ghi)/P #Invalid conditional reference to label 'blah'.
102:/(?(DEFINE)foo|bar)/P #DEFINE conditional group with more than one branch at index 17.
103:/(?<1name>group)/ #Group name cannot begin with a digit at index 0.
104:/abc((def)?(?(R)bar))+/P #Pattern recursion not supported at index 10.
105:/abc((def)?(?(R2)bar))+/P #Pattern recursion not supported at index 10.
106:/abc((def)(?(R&label)bar))+/P #Pattern recursion not supported at index 9.
107:/\o{4200000}/8 #Value in \o{...} sequence is too large at index 0.
108:/\o{19}/ #Value in \o{...} sequence is non-octal or missing braces at index 0.
109:/\o{/ #Value in \o{...} sequence is non-octal or missing braces at index 0.
110:/\o{1/ #Value in \o{...} sequence is non-octal or missing braces at index 0.
111:/\x{0x110000}/8 #Value in \x{...} sequence is non-hex or missing } at index 0.
112:/\cÀ/ #\c must be followed by an ASCII character at index 0.
113:/[\cÀ]/ #\c must be followed by an ASCII character at index 1.
114:/[\o{4200000}]/8 #Value in \o{...} sequence is too large at index 1.
115:/[\x{0x110000}]/8 #Value in \x{...} sequence is non-hex or missing } at index 1.
116:/[\o{70]/ #Value in \o{...} sequence is non-octal or missing braces at index 1.
117:/[\x{ff]/ #Value in \x{...} sequence is non-hex or missing } at index 1.
118:/foo/{min_offset=10,max_offset=9} #In hs_expr_ext, min_offset must be less than or equal to max_offset.
120:/foo/{min_length=10,max_offset=9} #In hs_expr_ext, min_length must be less than or equal to max_offset.
121:/.e(?:(((eEbd..(d[^Be]{1,7}|A)){8,22}aD.){7}|EecA?(?:\b)c|bB[Dd])){29,37}[adb](?:.|A|c|[BEA]|D)..((?:c|[Cba]))?([Ee]|D)B+(.|[dbB]|E|E).[EcCe]ce(?:C|D)dD[EA]Ac.[aE]d/smiHWP #Pattern too large.
122:/<2F><>/8 #Expression is not valid UTF-8.
123:/hello \6 world/P #Invalid back reference to expression 6.
124:/hello \6 world|dog/P #Invalid back reference to expression 6.
125:/[~-\V]/8 #Invalid range in character class at index 3.
126:/(*UTF8)<29><>/ #Expression is not valid UTF-8.
127:/^fo?ob{ro|nax_off\Qt=10omnax+8Wnah/<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>0}l.{1,60}Car*k|npanomnax+8Wnah/8 #Expression is not valid UTF-8.
128:/(*UTF8)^fo?ob{ro|nax_off\Qt=10omnax+8Wnah/<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>0}l.{1,60}Car*k|npanomnax+8Wnah/ #Expression is not valid UTF-8.
129:/bignum \1111111111111111111/ #Number is too big at index 7.

1513
unit/hyperscan/behaviour.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

118
unit/hyperscan/expr_info.cpp Normal file
View File

@@ -0,0 +1,118 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include <limits.h>
#include <vector>
#include "gtest/gtest.h"
#include "hs.h"
#include "test_util.h"
using namespace std;
using namespace testing;
namespace /* anonymous */ {
struct expected_info {
const char *pattern;
unsigned min;
unsigned max;
char unordered_matches;
char matches_at_eod;
char matches_only_at_eod;
};
class ExprInfop : public TestWithParam<expected_info> {
};
TEST_P(ExprInfop, width) {
const expected_info &ei = GetParam();
SCOPED_TRACE(ei.pattern);
hs_expr_info_t *info = nullptr;
hs_compile_error_t *c_err = nullptr;
hs_error_t err = hs_expression_info(ei.pattern, 0, &info, &c_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(info != nullptr);
ASSERT_TRUE(c_err == nullptr);
EXPECT_EQ(ei.min, info->min_width);
EXPECT_EQ(ei.max, info->max_width);
EXPECT_EQ(ei.unordered_matches, info->unordered_matches);
EXPECT_EQ(ei.matches_at_eod, info->matches_at_eod);
EXPECT_EQ(ei.matches_only_at_eod, info->matches_only_at_eod);
free(info);
}
static const expected_info ei_test[] = {
{"abc", 3, 3, 0, 0, 0},
{"abc.*def", 6, UINT_MAX, 0, 0, 0},
{"abc|defghi", 3, 6, 0, 0, 0},
{"abc(def)?", 3, 6, 0, 0, 0},
{"abc(def){0,3}", 3, 12, 0, 0, 0},
{"abc(def){1,4}", 6, 15, 0, 0, 0},
{"", 0, 0, 0, 0, 0},
{"^", 0, 0, 0, 0, 0},
{"^\\b", 0, 0, 1, 0, 0},
{"\\b$", 0, 0, 1, 1, 1},
{"(?m)\\b$", 0, 0, 1, 1, 0},
{"\\A", 0, 0, 0, 0, 0},
{"\\z", 0, 0, 0, 1, 1},
{"\\Z", 0, 0, 1, 1, 1},
{"$", 0, 0, 1, 1, 1},
{"(?m)$", 0, 0, 1, 1, 0},
{"^foo", 3, 3, 0, 0, 0},
{"^foo.*bar", 6, UINT_MAX, 0, 0, 0},
{"^foo.*bar?", 5, UINT_MAX, 0, 0, 0},
{"^foo.*bar$", 6, UINT_MAX, 1, 1, 1},
{"^foobar$", 6, 6, 1, 1, 1},
{"foobar$", 6, 6, 1, 1, 1},
{"^.*foo", 3, UINT_MAX, 0, 0, 0},
{"foo\\b", 3, 3, 1, 1, 0},
{"foo.{1,13}bar", 7, 19, 0, 0, 0},
{"foo.{10,}bar", 16, UINT_MAX, 0, 0, 0},
{"foo.{0,10}bar", 6, 16, 0, 0, 0},
{"foo.{,10}bar", 12, 12, 0, 0, 0},
{"foo.{10}bar", 16, 16, 0, 0, 0},
{"(^|\n)foo", 3, 4, 0, 0, 0},
{"(^\n|)foo", 3, 4, 0, 0, 0},
{"(?m)^foo", 3, 3, 0, 0, 0},
{"\\bfoo", 3, 3, 0, 0, 0},
{"^\\bfoo", 3, 3, 0, 0, 0},
{"(?m)^\\bfoo", 3, 3, 0, 0, 0},
{"\\Bfoo", 3, 3, 0, 0, 0},
{"(foo|bar\\z)", 3, 3, 0, 1, 0},
{"(foo|bar)\\z", 3, 3, 0, 1, 1},
};
INSTANTIATE_TEST_CASE_P(ExprInfo, ExprInfop, ValuesIn(ei_test));
}

158
unit/hyperscan/extparam.cpp Normal file
View File

@@ -0,0 +1,158 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include <cstring>
#include "gtest/gtest.h"
#include "hs.h"
#include "test_util.h"
using namespace std;
using namespace testing;
TEST(ExtParam, LargeMinOffset) {
hs_expr_ext ext;
memset(&ext, 0, sizeof(ext));
ext.min_offset = 100000;
ext.flags = HS_EXT_FLAG_MIN_OFFSET;
pattern p("hatstand.*teakettle", 0, 0, ext);
hs_database_t *db = buildDB(p, HS_MODE_NOSTREAM);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(scratch != nullptr);
CallBackContext c;
// First, scan a matching corpus that's shorter than our min_offset and
// ensure it doesn't match.
string corpus = "hatstand" + string(80000, '_') + "teakettle";
err = hs_scan(db, corpus.c_str(), corpus.length(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0U, c.matches.size());
// Try exactly at the min_offset.
corpus = "hatstand" + string(99983, '_') + "teakettle";
err = hs_scan(db, corpus.c_str(), corpus.length(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(100000, 0), c.matches[0]);
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(ExtParam, LargeExactOffset) {
hs_expr_ext ext;
memset(&ext, 0, sizeof(ext));
ext.min_offset = 200000;
ext.max_offset = 200000;
ext.flags = HS_EXT_FLAG_MIN_OFFSET | HS_EXT_FLAG_MAX_OFFSET;
pattern p("hatstand.*teakettle", 0, 0, ext);
hs_database_t *db = buildDB(p, HS_MODE_NOSTREAM);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(scratch != nullptr);
CallBackContext c;
// First, scan a matching corpus that's shorter than our min_offset and
// ensure it doesn't match.
string corpus = "hatstand" + string(199982, '_') + "teakettle";
err = hs_scan(db, corpus.c_str(), corpus.length(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0U, c.matches.size());
// Try the exact match.
corpus = "hatstand" + string(199983, '_') + "teakettle";
err = hs_scan(db, corpus.c_str(), corpus.length(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(200000, 0), c.matches[0]);
// Try one byte too far.
c.clear();
corpus = "hatstand" + string(199984, '_') + "teakettle";
err = hs_scan(db, corpus.c_str(), corpus.length(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0U, c.matches.size());
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(ExtParam, LargeMinLength) {
hs_expr_ext ext;
memset(&ext, 0, sizeof(ext));
ext.min_length = 100000;
ext.flags = HS_EXT_FLAG_MIN_LENGTH;
pattern p("hatstand.*teakettle", 0, 0, ext);
hs_database_t *db = buildDB(p, HS_MODE_NOSTREAM);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(scratch != nullptr);
CallBackContext c;
// First, scan a matching corpus that contains a match that's a bit too
// short.
string corpus = string(10000, '_') + "hatstand" + string(80000, '_') + "teakettle";
err = hs_scan(db, corpus.c_str(), corpus.length(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0U, c.matches.size());
// Now, a match of the right length.
corpus = string(10000, '_') + "hatstand" + string(99983, '_') + "teakettle";
err = hs_scan(db, corpus.c_str(), corpus.length(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(110000, 0), c.matches[0]);
hs_free_scratch(scratch);
hs_free_database(db);
}

194
unit/hyperscan/identical.cpp Normal file
View File

@@ -0,0 +1,194 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "gtest/gtest.h"
#include "test_util.h"
#include "hs.h"
namespace {
struct PatternInfo {
std::string expr;
unsigned flags;
std::string corpus;
unsigned long long match;
};
class IdenticalTest : public testing::TestWithParam<PatternInfo> {};
TEST_P(IdenticalTest, Block) {
const PatternInfo &info = GetParam();
std::vector<pattern> patterns;
for (unsigned i = 0; i < 100; i++) {
patterns.push_back(pattern(info.expr, info.flags, i));
}
hs_database_t *db = buildDB(patterns, HS_MODE_BLOCK);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(nullptr, scratch);
CallBackContext cb;
err = hs_scan(db, info.corpus.c_str(), info.corpus.size(), 0, scratch,
record_cb, &cb);
ASSERT_EQ(HS_SUCCESS, err);
hs_free_scratch(scratch);
hs_free_database(db);
ASSERT_EQ(patterns.size(), cb.matches.size());
std::set<unsigned> ids;
for (size_t i = 0; i < cb.matches.size(); i++) {
ASSERT_EQ(info.match, cb.matches[i].to);
ids.insert(cb.matches[i].id);
}
ASSERT_EQ(patterns.size(), ids.size());
ASSERT_EQ(0, *ids.begin());
ASSERT_EQ(patterns.size() - 1, *ids.rbegin());
}
TEST_P(IdenticalTest, Stream) {
const PatternInfo &info = GetParam();
std::vector<pattern> patterns;
for (unsigned i = 0; i < 100; i++) {
patterns.push_back(pattern(info.expr, info.flags, i));
}
hs_database_t *db =
buildDB(patterns, HS_MODE_STREAM | HS_MODE_SOM_HORIZON_LARGE);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(nullptr, scratch);
CallBackContext cb;
hs_stream_t *stream = nullptr;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(nullptr, stream);
err = hs_scan_stream(stream, info.corpus.c_str(), info.corpus.size(), 0,
scratch, record_cb, &cb);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_close_stream(stream, scratch, record_cb, &cb);
ASSERT_EQ(HS_SUCCESS, err);
hs_free_scratch(scratch);
hs_free_database(db);
ASSERT_EQ(patterns.size(), cb.matches.size());
std::set<unsigned> ids;
for (size_t i = 0; i < cb.matches.size(); i++) {
ASSERT_EQ(info.match, cb.matches[i].to);
ids.insert(cb.matches[i].id);
}
ASSERT_EQ(patterns.size(), ids.size());
ASSERT_EQ(0, *ids.begin());
ASSERT_EQ(patterns.size() - 1, *ids.rbegin());
}
TEST_P(IdenticalTest, Vectored) {
const PatternInfo &info = GetParam();
std::vector<pattern> patterns;
for (unsigned i = 0; i < 100; i++) {
patterns.push_back(pattern(info.expr, info.flags, i));
}
hs_database_t *db = buildDB(patterns, HS_MODE_VECTORED);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(nullptr, scratch);
CallBackContext cb;
const char * const data[] = { info.corpus.c_str() };
const unsigned datalen[] = { (unsigned)info.corpus.size() };
err = hs_scan_vector(db, data, datalen, 1, 0, scratch, record_cb, &cb);
ASSERT_EQ(HS_SUCCESS, err);
hs_free_scratch(scratch);
hs_free_database(db);
ASSERT_EQ(patterns.size(), cb.matches.size());
std::set<unsigned> ids;
for (size_t i = 0; i < cb.matches.size(); i++) {
ASSERT_EQ(info.match, cb.matches[i].to);
ids.insert(cb.matches[i].id);
}
ASSERT_EQ(patterns.size(), ids.size());
ASSERT_EQ(0, *ids.begin());
ASSERT_EQ(patterns.size() - 1, *ids.rbegin());
}
static const PatternInfo patterns[] = {
{ "a", 0, "a", 1 },
{ "a", HS_FLAG_SINGLEMATCH, "a", 1 },
{ "handbasket", 0, "__handbasket__", 12 },
{ "handbasket", HS_FLAG_SINGLEMATCH, "__handbasket__", 12 },
{ "handbasket", HS_FLAG_SOM_LEFTMOST, "__handbasket__", 12 },
{ "foo.*bar", 0, "a foolish embarrassment", 15 },
{ "foo.*bar", HS_FLAG_SINGLEMATCH, "a foolish embarrassment", 15 },
{ "foo.*bar", HS_FLAG_SOM_LEFTMOST, "a foolish embarrassment", 15 },
{ "\\bword\\b(..)+\\d{3,7}", 0, " word 012", 15 },
{ "\\bword\\b(..)+\\d{3,7}", HS_FLAG_SINGLEMATCH, " word 012", 15 },
{ "\\bword\\b(..)+\\d{3,7}", HS_FLAG_SOM_LEFTMOST, " word 012", 15 },
{ "eod\\z", 0, "eod", 3 },
{ "eod\\z", HS_FLAG_SINGLEMATCH, "eod", 3 },
{ "eod\\z", HS_FLAG_SOM_LEFTMOST, "eod", 3 },
};
INSTANTIATE_TEST_CASE_P(Identical, IdenticalTest, testing::ValuesIn(patterns));
// teach google-test how to print a param
void PrintTo(const PatternInfo &p, ::std::ostream *os) {
*os << p.expr << ":" << p.flags << ", " << p.corpus;
}
} // namespace

36
unit/hyperscan/main.cpp Normal file
View File

@@ -0,0 +1,36 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "gtest/gtest.h"
// Driver: run all the tests (defined in other source files in this directory)
int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

326
unit/hyperscan/multi.cpp Normal file
View File

@@ -0,0 +1,326 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <algorithm>
#include <array>
#include <iostream>
#include <vector>
#include "gtest/gtest.h"
#include "hs.h"
#include "config.h"
#include "test_util.h"
using namespace std;
TEST(MMAdaptor, norm_cont1) { // UE-901
hs_database_t *db = nullptr;
hs_compile_error_t *compile_err = nullptr;
CallBackContext c;
string data = "aooAaooAbarZ";
const char *expr[] = {"aoo[A-K]", "bar[L-Z]"};
unsigned flags[] = {0, 0};
unsigned ids[] = {30, 31};
hs_error_t err = hs_compile_multi(expr, flags, ids, 2, HS_MODE_NOSTREAM,
nullptr, &db, &compile_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(scratch != nullptr);
c.halt = 0;
err = hs_scan(db, data.c_str(), data.size(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(3U, c.matches.size());
ASSERT_EQ(MatchRecord(4, 30), c.matches[0]);
ASSERT_EQ(MatchRecord(8, 30), c.matches[1]);
ASSERT_EQ(MatchRecord(12, 31), c.matches[2]);
hs_free_database(db);
hs_free_scratch(scratch);
}
TEST(MMAdaptor, norm_cont2) {
hs_database_t *db = nullptr;
hs_compile_error_t *compile_err = nullptr;
CallBackContext c;
string data = "aooAaooAbarZ ";
const char *expr[] = {"aoo[A-K][^\n]{16}", "bar[L-Z][^\n]{16}"};
unsigned flags[] = {0, 0};
unsigned ids[] = {30, 31};
hs_error_t err = hs_compile_multi(expr, flags, ids, 2, HS_MODE_NOSTREAM,
nullptr, &db, &compile_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(scratch != nullptr);
c.halt = 0;
err = hs_scan(db, data.c_str(), data.size(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(3U, c.matches.size());
ASSERT_TRUE(c.matches.end() != find(c.matches.begin(), c.matches.end(), MatchRecord(20, 30)));
ASSERT_TRUE(c.matches.end() != find(c.matches.begin(), c.matches.end(), MatchRecord(24, 30)));
ASSERT_TRUE(c.matches.end() != find(c.matches.begin(), c.matches.end(), MatchRecord(28, 31)));
hs_free_database(db);
hs_free_scratch(scratch);
}
TEST(MMAdaptor, norm_halt1) {
hs_database_t *db = nullptr;
hs_compile_error_t *compile_err = nullptr;
CallBackContext c;
string data = "aooAaooAbarZ";
const char *expr[] = {"aoo[A-K]", "bar[L-Z]"};
unsigned flags[] = {0, 0};
unsigned ids[] = {30, 31};
hs_error_t err = hs_compile_multi(expr, flags, ids, 2, HS_MODE_NOSTREAM,
nullptr, &db, &compile_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(scratch != nullptr);
c.halt = 1;
err = hs_scan(db, data.c_str(), data.size(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SCAN_TERMINATED, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(4, 30), c.matches[0]);
hs_free_database(db);
hs_free_scratch(scratch);
}
TEST(MMAdaptor, norm_halt2) { // UE-901
hs_database_t *db = nullptr;
hs_compile_error_t *compile_err = nullptr;
CallBackContext c;
string data = "aooAaooAbarZ ";
const char *expr[] = {"aoo[A-K][^\n]{16}", "bar[L-Z][^\n]{16}"};
unsigned flags[] = {0, 0};
unsigned ids[] = {30, 31};
hs_error_t err = hs_compile_multi(expr, flags, ids, 2, HS_MODE_NOSTREAM,
nullptr, &db, &compile_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(scratch != nullptr);
c.halt = 1;
err = hs_scan(db, data.c_str(), data.size(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SCAN_TERMINATED, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(20, 30), c.matches[0]);
hs_free_database(db);
hs_free_scratch(scratch);
}
TEST(MMAdaptor, high_cont1) { // UE-901
hs_database_t *db = nullptr;
hs_compile_error_t *compile_err = nullptr;
CallBackContext c;
string data = "aooAaooAbarZ";
const char *expr[] = {"aoo[A-K]", "bar[L-Z]"};
unsigned flags[] = {HS_FLAG_SINGLEMATCH, 0};
unsigned ids[] = {30, 31};
hs_error_t err = hs_compile_multi(expr, flags, ids, 2, HS_MODE_NOSTREAM,
nullptr, &db, &compile_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(scratch != nullptr);
c.halt = 0;
err = hs_scan(db, data.c_str(), data.size(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(2U, c.matches.size());
ASSERT_TRUE(c.matches.end() != find(c.matches.begin(), c.matches.end(), MatchRecord(4, 30)));
ASSERT_TRUE(c.matches.end() != find(c.matches.begin(), c.matches.end(), MatchRecord(12, 31)));
hs_free_database(db);
hs_free_scratch(scratch);
}
TEST(MMAdaptor, high_cont2) {
hs_database_t *db = nullptr;
hs_compile_error_t *compile_err = nullptr;
CallBackContext c;
string data = "aooAaooAbarZ ";
const char *expr[] = {"aoo[A-K][^\n]{16}", "bar[L-Z][^\n]{16}"};
unsigned flags[] = {HS_FLAG_SINGLEMATCH, 0};
unsigned ids[] = {30, 31};
hs_error_t err = hs_compile_multi(expr, flags, ids, 2, HS_MODE_NOSTREAM,
nullptr, &db, &compile_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(scratch != nullptr);
c.halt = 0;
err = hs_scan(db, data.c_str(), data.size(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(2U, c.matches.size());
ASSERT_TRUE(c.matches.end() != find(c.matches.begin(), c.matches.end(), MatchRecord(20, 30)));
ASSERT_TRUE(c.matches.end() != find(c.matches.begin(), c.matches.end(), MatchRecord(28, 31)));
hs_free_database(db);
hs_free_scratch(scratch);
}
TEST(MMAdaptor, high_halt1) {
hs_database_t *db = nullptr;
hs_compile_error_t *compile_err = nullptr;
CallBackContext c;
string data = "aooAaooAbarZ";
const char *expr[] = {"aoo[A-K]", "bar[L-Z]"};
unsigned flags[] = {HS_FLAG_SINGLEMATCH, 0};
unsigned ids[] = {30, 31};
hs_error_t err = hs_compile_multi(expr, flags, ids, 2, HS_MODE_NOSTREAM,
nullptr, &db, &compile_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(scratch != nullptr);
c.halt = 1;
err = hs_scan(db, data.c_str(), data.size(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SCAN_TERMINATED, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(4, 30), c.matches[0]);
hs_free_database(db);
hs_free_scratch(scratch);
}
TEST(MMAdaptor, high_halt2) {
hs_database_t *db = nullptr;
hs_compile_error_t *compile_err = nullptr;
CallBackContext c;
string data = "aooAaooAbarZbarZaooA ";
const char *expr[] = {"aoo[A-K][^\n]{16}", "bar[L-Z][^\n]{16}"};
unsigned flags[] = {HS_FLAG_SINGLEMATCH, 0};
unsigned ids[] = {30, 31};
hs_error_t err = hs_compile_multi(expr, flags, ids, 2, HS_MODE_NOSTREAM,
nullptr, &db, &compile_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(scratch != nullptr);
c.halt = 1;
err = hs_scan(db, data.c_str(), data.size(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SCAN_TERMINATED, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_TRUE(MatchRecord(20, 30) == c.matches[0]
|| MatchRecord(28, 31) == c.matches[0]);
hs_free_database(db);
hs_free_scratch(scratch);
}
TEST(MPV, UE_2395) {
vector<pattern> patterns;
patterns.push_back(pattern("^.{200}", HS_FLAG_DOTALL, 1));
patterns.push_back(pattern(".{40,}", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aaa", HS_FLAG_DOTALL, 3));
hs_database_t *db = buildDB(patterns, HS_MODE_BLOCK);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
array<char, 300> data;
data.fill('a');
CallBackContext c;
err = hs_scan(db, data.data(), data.size(), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
unsigned seen = 39;
for (vector<MatchRecord>::const_iterator it = c.matches.begin();
it != c.matches.end(); ++it) {
if (it->id != 2) {
if (it->id == 1) {
ASSERT_EQ(200, it->to);
}
continue;
}
ASSERT_EQ(seen + 1, it->to);
seen = it->to;
}
ASSERT_EQ(300, seen);
hs_free_database(db);
hs_free_scratch(scratch);
}

356
unit/hyperscan/order.cpp Normal file
View File

@@ -0,0 +1,356 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include <algorithm>
#include <string>
#include <vector>
#include "gtest/gtest.h"
#include "hs.h"
#include "test_util.h"
using namespace std;
using namespace testing;
namespace /* anonymous */ {
bool matchesOrdered(const vector<MatchRecord> &matches) {
unsigned long lastMatch = 0;
for (vector<MatchRecord>::const_iterator it = matches.begin();
it != matches.end(); ++it) {
if (lastMatch > it->to) {
return false;
}
lastMatch = it->to;
}
return true;
}
unsigned countMatchesById(const vector<MatchRecord> &matches, int id) {
unsigned count = 0;
for (vector<MatchRecord>::const_iterator it = matches.begin();
it != matches.end(); ++it) {
if (id == it->id) {
count++;
}
}
return count;
}
TEST(order, ordering1) {
vector<pattern> patterns;
patterns.push_back(pattern("aa", HS_FLAG_DOTALL, 1));
patterns.push_back(pattern("aa.", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aa..", HS_FLAG_DOTALL, 3));
patterns.push_back(pattern("^.{0,4}aa..", HS_FLAG_DOTALL, 4));
patterns.push_back(pattern("^.{0,4}aa", HS_FLAG_DOTALL, 5));
const char *data = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
hs_database_t *db = buildDB(patterns, HS_MODE_NOSTREAM);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
CallBackContext c;
err = hs_scan(db, data, strlen(data), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_EQ(31U, countMatchesById(c.matches, 1));
EXPECT_EQ(30U, countMatchesById(c.matches, 2));
EXPECT_EQ(29U, countMatchesById(c.matches, 3));
EXPECT_EQ(5U, countMatchesById(c.matches, 4));
EXPECT_EQ(5U, countMatchesById(c.matches, 5));
ASSERT_TRUE(matchesOrdered(c.matches));
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(order, ordering2) {
vector<pattern> patterns;
patterns.push_back(pattern("aa.", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aa..", HS_FLAG_DOTALL, 3));
patterns.push_back(pattern("^.{0,4}aa..", HS_FLAG_DOTALL, 4));
patterns.push_back(pattern("^.{0,4}aa", HS_FLAG_DOTALL, 5));
const char *data = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
hs_database_t *db = buildDB(patterns, HS_MODE_NOSTREAM);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
CallBackContext c;
err = hs_scan(db, data, strlen(data), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_EQ(0U, countMatchesById(c.matches, 1));
EXPECT_EQ(30U, countMatchesById(c.matches, 2));
EXPECT_EQ(29U, countMatchesById(c.matches, 3));
EXPECT_EQ(5U, countMatchesById(c.matches, 4));
EXPECT_EQ(5U, countMatchesById(c.matches, 5));
ASSERT_TRUE(matchesOrdered(c.matches));
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(order, ordering3) {
vector<pattern> patterns;
patterns.push_back(pattern("aa.", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aa..", HS_FLAG_DOTALL, 3));
patterns.push_back(pattern("^.{0,4}aa..", HS_FLAG_DOTALL, 4));
const char *data = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
hs_database_t *db = buildDB(patterns, HS_MODE_NOSTREAM);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
CallBackContext c;
err = hs_scan(db, data, strlen(data), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_EQ(0U, countMatchesById(c.matches, 1));
EXPECT_EQ(30U, countMatchesById(c.matches, 2));
EXPECT_EQ(29U, countMatchesById(c.matches, 3));
EXPECT_EQ(5U, countMatchesById(c.matches, 4));
EXPECT_EQ(0U, countMatchesById(c.matches, 5));
ASSERT_TRUE(matchesOrdered(c.matches));
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(order, ordering4) {
vector<pattern> patterns;
patterns.push_back(pattern("aa", HS_FLAG_DOTALL, 1));
patterns.push_back(pattern("aa.", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aa..", HS_FLAG_DOTALL, 3));
const char *data = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
hs_database_t *db = buildDB(patterns, HS_MODE_NOSTREAM);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
CallBackContext c;
err = hs_scan(db, data, strlen(data), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_EQ(31U, countMatchesById(c.matches, 1));
EXPECT_EQ(30U, countMatchesById(c.matches, 2));
EXPECT_EQ(29U, countMatchesById(c.matches, 3));
EXPECT_EQ(0U, countMatchesById(c.matches, 4));
EXPECT_EQ(0U, countMatchesById(c.matches, 5));
ASSERT_TRUE(matchesOrdered(c.matches));
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(order, ordering5) {
vector<pattern> patterns;
patterns.push_back(pattern("aa", HS_FLAG_DOTALL, 1));
patterns.push_back(pattern("aa.", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aa..", HS_FLAG_DOTALL, 3));
patterns.push_back(pattern("^.{0,4}aa..", HS_FLAG_DOTALL, 4));
patterns.push_back(pattern("^.{0,4}aa", HS_FLAG_DOTALL, 5));
const char *data = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
hs_database_t *db = buildDB(patterns, HS_MODE_STREAM);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
hs_stream_t *stream = nullptr;
CallBackContext c;
for (size_t jump = 1; jump <= 8; jump++) {
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
for (unsigned i = 0; i < strlen(data); i += jump) {
err = hs_scan_stream(stream, data + i, min(jump, strlen(data) - i),
0, scratch, record_cb, (void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
}
EXPECT_EQ(31U, countMatchesById(c.matches, 1));
EXPECT_EQ(30U, countMatchesById(c.matches, 2));
EXPECT_EQ(29U, countMatchesById(c.matches, 3));
EXPECT_EQ(5U, countMatchesById(c.matches, 4));
EXPECT_EQ(5U, countMatchesById(c.matches, 5));
ASSERT_TRUE(matchesOrdered(c.matches));
c.matches.clear();
hs_close_stream(stream, scratch, record_cb, (void *)&c);
}
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(order, ordering6) {
vector<pattern> patterns;
patterns.push_back(pattern("aa.", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aa..", HS_FLAG_DOTALL, 3));
patterns.push_back(pattern("^.{0,4}aa..", HS_FLAG_DOTALL, 4));
patterns.push_back(pattern("^.{0,4}aa", HS_FLAG_DOTALL, 5));
const char *data = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
hs_database_t *db = buildDB(patterns, HS_MODE_STREAM);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
hs_stream_t *stream = nullptr;
CallBackContext c;
for (size_t jump = 1; jump <= 8; jump++) {
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
for (unsigned i = 0; i < strlen(data); i += jump) {
err = hs_scan_stream(stream, data + i, min(jump, strlen(data) - i),
0, scratch, record_cb, (void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
}
EXPECT_EQ(0U, countMatchesById(c.matches, 1));
EXPECT_EQ(30U, countMatchesById(c.matches, 2));
EXPECT_EQ(29U, countMatchesById(c.matches, 3));
EXPECT_EQ(5U, countMatchesById(c.matches, 4));
EXPECT_EQ(5U, countMatchesById(c.matches, 5));
ASSERT_TRUE(matchesOrdered(c.matches));
c.matches.clear();
hs_close_stream(stream, scratch, record_cb, (void *)&c);
}
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(order, ordering7) {
vector<pattern> patterns;
patterns.push_back(pattern("aa.", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aa..", HS_FLAG_DOTALL, 3));
patterns.push_back(pattern("^.{0,4}aa..", HS_FLAG_DOTALL, 4));
const char *data = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
hs_database_t *db = buildDB(patterns, HS_MODE_STREAM);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
hs_stream_t *stream = nullptr;
CallBackContext c;
for (size_t jump = 1; jump <= 8; jump++) {
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
for (unsigned i = 0; i < strlen(data); i += jump) {
err = hs_scan_stream(stream, data + i, min(jump, strlen(data) - i),
0, scratch, record_cb, (void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
}
EXPECT_EQ(0U, countMatchesById(c.matches, 1));
EXPECT_EQ(30U, countMatchesById(c.matches, 2));
EXPECT_EQ(29U, countMatchesById(c.matches, 3));
EXPECT_EQ(5U, countMatchesById(c.matches, 4));
EXPECT_EQ(0U, countMatchesById(c.matches, 5));
ASSERT_TRUE(matchesOrdered(c.matches));
c.matches.clear();
hs_close_stream(stream, scratch, record_cb, (void *)&c);
}
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(order, ordering8) {
vector<pattern> patterns;
patterns.push_back(pattern("aa", HS_FLAG_DOTALL, 1));
patterns.push_back(pattern("aa.", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aa..", HS_FLAG_DOTALL, 3));
const char *data = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
hs_database_t *db = buildDB(patterns, HS_MODE_STREAM);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
hs_stream_t *stream = nullptr;
CallBackContext c;
for (size_t jump = 1; jump <= 8; jump++) {
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
for (unsigned i = 0; i < strlen(data); i += jump) {
err = hs_scan_stream(stream, data + i, min(jump, strlen(data) - i),
0, scratch, record_cb, (void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
}
EXPECT_EQ(31U, countMatchesById(c.matches, 1));
EXPECT_EQ(30U, countMatchesById(c.matches, 2));
EXPECT_EQ(29U, countMatchesById(c.matches, 3));
EXPECT_EQ(0U, countMatchesById(c.matches, 4));
EXPECT_EQ(0U, countMatchesById(c.matches, 5));
ASSERT_TRUE(matchesOrdered(c.matches));
c.matches.clear();
hs_close_stream(stream, scratch, record_cb, (void *)&c);
}
hs_free_scratch(scratch);
hs_free_database(db);
}
}

289
unit/hyperscan/scratch_op.cpp Normal file
View File

@@ -0,0 +1,289 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include <stdlib.h>
#include <string>
#include <vector>
#include "gtest/gtest.h"
#include "hs.h"
#include "test_util.h"
using namespace std;
using namespace testing;
namespace /* anonymous */ {
static size_t last_alloc_size;
static void *log_malloc(size_t n) {
last_alloc_size = n;
return malloc(n);
}
static void *bad_alloc(size_t) { return nullptr; }
TEST(scratch, testAlloc) {
vector<pattern> patterns;
patterns.push_back(pattern("aa", HS_FLAG_DOTALL, 1));
patterns.push_back(pattern("aa.", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aa..", HS_FLAG_DOTALL, 3));
patterns.push_back(pattern("^.{0,4}aa..", HS_FLAG_DOTALL, 4));
patterns.push_back(pattern("^.{0,4}aa", HS_FLAG_DOTALL, 5));
hs_database_t *db = buildDB(patterns, HS_MODE_NOSTREAM);
ASSERT_NE(nullptr, db);
hs_error_t err;
err = hs_set_allocator(log_malloc, free);
ASSERT_EQ(HS_SUCCESS, err);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
size_t curr_size;
err = hs_scratch_size(scratch, &curr_size);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(last_alloc_size, curr_size);
hs_free_scratch(scratch);
hs_free_database(db);
hs_set_allocator(nullptr, nullptr);
}
TEST(scratch, testScratchAlloc) {
vector<pattern> patterns;
allocated_count = 0;
allocated_count_b = 0;
hs_error_t err = hs_set_allocator(count_malloc_b, count_free_b);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_set_scratch_allocator(count_malloc, count_free);
ASSERT_EQ(HS_SUCCESS, err);
patterns.push_back(pattern("aa", HS_FLAG_DOTALL, 1));
patterns.push_back(pattern("aa.", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aa..", HS_FLAG_DOTALL, 3));
patterns.push_back(pattern("^.{0,4}aa..", HS_FLAG_DOTALL, 4));
patterns.push_back(pattern("^.{0,4}aa", HS_FLAG_DOTALL, 5));
hs_database_t *db = buildDB(patterns, HS_MODE_NOSTREAM);
ASSERT_NE(nullptr, db);
ASSERT_EQ(0, allocated_count);
ASSERT_NE(0, allocated_count_b);
size_t old_b = allocated_count_b;
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
size_t curr_size;
err = hs_scratch_size(scratch, &curr_size);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(allocated_count, curr_size);
ASSERT_EQ(allocated_count_b, old_b);
hs_free_scratch(scratch);
hs_free_database(db);
hs_set_allocator(nullptr, nullptr);
ASSERT_EQ(0, allocated_count);
ASSERT_EQ(0, allocated_count_b);
}
TEST(scratch, badAlloc) {
vector<pattern> patterns;
patterns.push_back(pattern("aa", HS_FLAG_DOTALL, 1));
patterns.push_back(pattern("aa.", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aa..", HS_FLAG_DOTALL, 3));
patterns.push_back(pattern("^.{0,4}aa..", HS_FLAG_DOTALL, 4));
patterns.push_back(pattern("^.{0,4}aa", HS_FLAG_DOTALL, 5));
hs_database_t *db = buildDB(patterns, HS_MODE_NOSTREAM);
ASSERT_NE(nullptr, db);
hs_error_t err;
err = hs_set_scratch_allocator(bad_alloc, free);
ASSERT_EQ(HS_SUCCESS, err);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_NE(HS_SUCCESS, err);
ASSERT_TRUE(scratch == nullptr);
hs_free_database(db);
hs_set_scratch_allocator(nullptr, nullptr);
}
TEST(scratch, testScratchRealloc) {
vector<pattern> patterns;
patterns.push_back(pattern("aa", HS_FLAG_DOTALL, 1));
patterns.push_back(pattern("aa.", HS_FLAG_DOTALL, 2));
patterns.push_back(pattern("aa..", HS_FLAG_DOTALL, 3));
patterns.push_back(pattern("^.{0,4}aa..", HS_FLAG_DOTALL, 4));
patterns.push_back(pattern("^.{0,4}aa", HS_FLAG_DOTALL, 5));
hs_database_t *db = buildDB(patterns, HS_MODE_NOSTREAM);
ASSERT_NE(nullptr, db);
patterns.push_back(pattern("^.{0,4}aa{0,4}", HS_FLAG_DOTALL, 6));
patterns.push_back(pattern("^.{0,4}aa{0,4}a..", HS_FLAG_DOTALL, 7));
hs_database_t *db2 = buildDB(patterns, HS_MODE_NOSTREAM);
ASSERT_NE(nullptr, db2);
hs_error_t err;
err = hs_set_scratch_allocator(log_malloc, free);
ASSERT_EQ(HS_SUCCESS, err);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
size_t curr_size;
err = hs_scratch_size(scratch, &curr_size);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(last_alloc_size, curr_size);
err = hs_alloc_scratch(db2, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scratch_size(scratch, &curr_size);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(last_alloc_size, curr_size);
hs_free_scratch(scratch);
hs_free_database(db);
hs_free_database(db2);
hs_set_scratch_allocator(nullptr, nullptr);
}
TEST(scratch, tooSmallForDatabase) {
hs_database_t *db1 = buildDB("foobar", 0, 0, HS_MODE_BLOCK, nullptr);
ASSERT_NE(nullptr, db1);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db1, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
hs_free_database(db1);
hs_database_t *db2 =
buildDB("(a.?b.?c.?d.?e.?f.?g)|(hatstand(..)+teakettle)", 0, 0,
HS_MODE_BLOCK, nullptr);
ASSERT_NE(nullptr, db2);
// Try and scan some data using db2 and a scratch that has only been
// allocated for db1.
err = hs_scan(db2, "somedata", 8, 0, scratch, dummy_cb, nullptr);
ASSERT_EQ(HS_INVALID, err);
// Alloc scratch correctly and try again.
err = hs_alloc_scratch(db2, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan(db2, "somedata", 8, 0, scratch, dummy_cb, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
hs_free_scratch(scratch);
hs_free_database(db2);
}
TEST(scratch, tooSmallForDatabase2) {
hs_database_t *db1 = buildDB("foobar", 0, 0, HS_MODE_STREAM, nullptr);
ASSERT_NE(nullptr, db1);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db1, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
hs_free_database(db1);
hs_database_t *db2 =
buildDB("(a.?b.?c.?d.?e.?f.?g)|(hatstand(..)+teakettle)", 0, 0,
HS_MODE_STREAM, nullptr);
ASSERT_NE(nullptr, db2);
// Try and scan some data using db2 and a scratch that has only been
// allocated for db1.
hs_stream_t *stream = nullptr;
err = hs_open_stream(db2, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan_stream(stream, "somedata", 8, 0, scratch, dummy_cb, nullptr);
ASSERT_EQ(HS_INVALID, err);
// Alloc scratch correctly and try again.
err = hs_alloc_scratch(db2, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan_stream(stream, "somedata", 8, 0, scratch, dummy_cb, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_close_stream(stream, scratch, nullptr, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
hs_free_scratch(scratch);
hs_free_database(db2);
}
TEST(scratch, damagedScratch) {
hs_database_t *db = buildDB("foobar", 0, 0, HS_MODE_BLOCK, nullptr);
ASSERT_NE(nullptr, db);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
size_t scratch_size = 0;
err = hs_scratch_size(scratch, &scratch_size);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_LT(4, scratch_size);
// Take a temp copy and then scribble over the first four bytes.
char tmp[4];
memcpy(tmp, scratch, 4);
memset(scratch, 0xff, 4);
err = hs_scan(db, "somedata", 8, 0, scratch, dummy_cb, nullptr);
ASSERT_EQ(HS_INVALID, err);
// Restore first four bytes so we can free.
memcpy(scratch, tmp, 4);
err = hs_free_scratch(scratch);
ASSERT_EQ(HS_SUCCESS, err);
hs_free_database(db);
}
} // namespace

486
unit/hyperscan/serialize.cpp Normal file
View File

@@ -0,0 +1,486 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/**
* Unit tests for serialization functions.
*/
#include "config.h"
#include <cstring>
#include <string>
#include <vector>
#include "gtest/gtest.h"
#include "hs.h"
#include "hs_internal.h"
#include "test_util.h"
namespace {
using namespace std;
using namespace testing;
static const unsigned validModes[] = {
HS_MODE_STREAM,
HS_MODE_NOSTREAM
};
class Serializep : public TestWithParam<unsigned> {
};
// Check that we can deserialize from a char array at any alignment and the info
// is consistent
TEST_P(Serializep, DeserializeFromAnyAlignment) {
const unsigned mode = GetParam();
SCOPED_TRACE(mode);
hs_error_t err;
hs_database_t *db = buildDB("hatstand.*teakettle.*badgerbrush",
HS_FLAG_CASELESS, 1000, mode);
ASSERT_TRUE(db != nullptr) << "database build failed.";
char *original_info = nullptr;
err = hs_database_info(db, &original_info);
ASSERT_EQ(HS_SUCCESS, err);
const char *mode_string = nullptr;
switch (mode) {
case HS_MODE_STREAM:
mode_string = "STREAM";
break;
case HS_MODE_NOSTREAM:
mode_string = "BLOCK";
}
ASSERT_NE(nullptr, original_info) << "hs_serialized_database_info returned null.";
ASSERT_STREQ("Version:", string(original_info).substr(0, 8).c_str());
ASSERT_TRUE(strstr(original_info, mode_string) != nullptr);
char *bytes = nullptr;
size_t length = 0;
err = hs_serialize_database(db, &bytes, &length);
ASSERT_EQ(HS_SUCCESS, err) << "serialize failed.";
ASSERT_NE(nullptr, bytes);
ASSERT_LT(0U, length);
hs_free_database(db);
db = nullptr;
const size_t maxalign = 16;
char *copy = new char[length + maxalign];
// Deserialize from char arrays at a range of alignments.
for (size_t i = 0; i < maxalign; i++) {
SCOPED_TRACE(i);
memset(copy, 0, length + maxalign);
char *mycopy = copy + i;
memcpy(mycopy, bytes, length);
// We should be able to call hs_serialized_database_info and get back a
// reasonable string.
char *info;
err = hs_serialized_database_info(mycopy, length, &info);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(nullptr, original_info);
ASSERT_STREQ(original_info, info);
free(info);
// We should be able to deserialize as well.
err = hs_deserialize_database(mycopy, length, &db);
ASSERT_EQ(HS_SUCCESS, err) << "deserialize failed.";
ASSERT_TRUE(db != nullptr);
// And the info there should match.
err = hs_database_info(db, &info);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_STREQ(original_info, info);
free(info);
hs_free_database(db);
db = nullptr;
}
free(original_info);
free(bytes);
delete[] copy;
}
// Check that we can deserialize_at from a char array at any alignment and the
// info is consistent
TEST_P(Serializep, DeserializeAtFromAnyAlignment) {
const unsigned mode = GetParam();
SCOPED_TRACE(mode);
hs_error_t err;
hs_database_t *db = buildDB("hatstand.*teakettle.*badgerbrush",
HS_FLAG_CASELESS, 1000, mode);
ASSERT_TRUE(db != nullptr) << "database build failed.";
char *original_info;
err = hs_database_info(db, &original_info);
ASSERT_EQ(HS_SUCCESS, err);
const char *mode_string = nullptr;
switch (mode) {
case HS_MODE_STREAM:
mode_string = "STREAM";
break;
case HS_MODE_NOSTREAM:
mode_string = "BLOCK";
}
ASSERT_NE(nullptr, original_info) << "hs_serialized_database_info returned null.";
ASSERT_STREQ("Version:", string(original_info).substr(0, 8).c_str());
ASSERT_TRUE(strstr(original_info, mode_string) != nullptr);
char *bytes = nullptr;
size_t length = 0;
err = hs_serialize_database(db, &bytes, &length);
ASSERT_EQ(HS_SUCCESS, err) << "serialize failed.";
ASSERT_NE(nullptr, bytes);
ASSERT_LT(0U, length);
hs_free_database(db);
db = nullptr;
size_t slength;
err = hs_serialized_database_size(bytes, length, &slength);
ASSERT_EQ(HS_SUCCESS, err);
const size_t maxalign = 16;
char *copy = new char[length + maxalign];
char *mem = new char[slength];
db = (hs_database_t *)mem;
// Deserialize from char arrays at a range of alignments.
for (size_t i = 0; i < maxalign; i++) {
SCOPED_TRACE(i);
memset(copy, 0, length + maxalign);
char *mycopy = copy + i;
memcpy(mycopy, bytes, length);
// We should be able to call hs_serialized_database_info and get back a
// reasonable string.
char *info;
err = hs_serialized_database_info(mycopy, length, &info);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(nullptr, original_info);
ASSERT_STREQ(original_info, info);
free(info);
// Scrub target memory.
memset(mem, 0xff, length);
// We should be able to deserialize as well.
err = hs_deserialize_database_at(mycopy, length, db);
ASSERT_EQ(HS_SUCCESS, err) << "deserialize failed.";
ASSERT_TRUE(db != nullptr);
// And the info there should match.
err = hs_database_info(db, &info);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(info != nullptr);
ASSERT_STREQ(original_info, info);
free(info);
}
free(original_info);
free(bytes);
delete[] copy;
delete[] mem;
}
INSTANTIATE_TEST_CASE_P(Serialize, Serializep,
ValuesIn(validModes));
// Attempt to reproduce the scenario in UE-1946.
TEST(Serialize, CrossCompileSom) {
hs_platform_info plat;
plat.cpu_features = 0;
plat.tune = HS_TUNE_FAMILY_GENERIC;
static const char *pattern = "hatstand.*(badgerbrush|teakettle)";
const unsigned mode = HS_MODE_STREAM
| HS_MODE_SOM_HORIZON_LARGE;
hs_database_t *db = buildDB(pattern, HS_FLAG_SOM_LEFTMOST, 1000, mode,
&plat);
ASSERT_TRUE(db != nullptr) << "database build failed.";
size_t db_len;
hs_error_t err = hs_database_size(db, &db_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0, db_len);
char *bytes = nullptr;
size_t bytes_len = 0;
err = hs_serialize_database(db, &bytes, &bytes_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0, bytes_len);
hs_free_database(db);
// Relocate to misaligned block.
char *copy = (char *)malloc(bytes_len + 1);
ASSERT_TRUE(copy != nullptr);
memcpy(copy + 1, bytes, bytes_len);
free(bytes);
size_t ser_len;
err = hs_serialized_database_size(copy + 1, bytes_len, &ser_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0, ser_len);
ASSERT_EQ(db_len, ser_len);
free(copy);
}
static void *null_malloc(size_t) {
return nullptr;
}
static void *misaligned_malloc(size_t s) {
char *c = (char *)malloc(s + 1);
return (void *)(c + 1);
}
static void misaligned_free(void *p) {
char *c = (char *)p;
free(c - 1);
}
// make sure that serializing/deserializing to null or an unaligned address fails
TEST(Serialize, CompileNullMalloc) {
hs_database_t *db;
hs_compile_error_t *c_err;
static const char *pattern = "hatstand.*(badgerbrush|teakettle)";
// mallocing null should fail compile
hs_set_allocator(null_malloc, nullptr);
hs_error_t err = hs_compile(pattern, 0, HS_MODE_BLOCK, nullptr, &db, &c_err);
ASSERT_NE(HS_SUCCESS, err);
ASSERT_TRUE(db == nullptr);
ASSERT_TRUE(c_err != nullptr);
hs_free_compile_error(c_err);
hs_set_allocator(nullptr, nullptr);
}
TEST(Serialize, CompileErrorAllocator) {
hs_database_t *db;
hs_compile_error_t *c_err;
static const char *pattern = "hatsta^nd.*(badgerbrush|teakettle)";
// failing to compile should use the misc allocator
allocated_count = 0;
allocated_count_b = 0;
hs_set_allocator(count_malloc_b, count_free_b);
hs_set_misc_allocator(count_malloc, count_free);
hs_error_t err = hs_compile(pattern, 0, HS_MODE_BLOCK, nullptr, &db, &c_err);
ASSERT_NE(HS_SUCCESS, err);
ASSERT_TRUE(db == nullptr);
ASSERT_TRUE(c_err != nullptr);
ASSERT_EQ(0, allocated_count_b);
ASSERT_NE(0, allocated_count);
hs_free_compile_error(c_err);
hs_set_allocator(nullptr, nullptr);
ASSERT_EQ(0, allocated_count);
}
TEST(Serialize, AllocatorsUsed) {
hs_database_t *db;
hs_compile_error_t *c_err;
static const char *pattern = "hatstand.*(badgerbrush|teakettle)";
allocated_count = 0;
allocated_count_b = 0;
hs_set_allocator(count_malloc_b, count_free_b);
hs_set_database_allocator(count_malloc, count_free);
hs_error_t err = hs_compile(pattern, 0, HS_MODE_BLOCK, nullptr, &db, &c_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
ASSERT_TRUE(c_err == nullptr);
ASSERT_EQ(0, allocated_count_b);
ASSERT_NE(0, allocated_count);
/* serialize should use the misc allocator */
char *bytes = nullptr;
size_t bytes_len = 0;
err = hs_serialize_database(db, &bytes, &bytes_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0, bytes_len);
ASSERT_EQ(bytes_len, allocated_count_b);
count_free_b(bytes);
hs_free_database(db);
hs_set_allocator(nullptr, nullptr);
ASSERT_EQ(0, allocated_count);
ASSERT_EQ(0, allocated_count_b);
}
TEST(Serialize, CompileUnalignedMalloc) {
hs_database_t *db;
hs_compile_error_t *c_err;
static const char *pattern = "hatstand.*(badgerbrush|teakettle)";
// unaligned malloc should fail compile
hs_set_allocator(misaligned_malloc, misaligned_free);
hs_error_t err = hs_compile(pattern, 0, HS_MODE_BLOCK, nullptr, &db, &c_err);
ASSERT_NE(HS_SUCCESS, err);
ASSERT_TRUE(db == nullptr);
ASSERT_TRUE(c_err != nullptr);
hs_free_compile_error(c_err);
hs_set_allocator(nullptr, nullptr);
}
TEST(Serialize, SerializeNullMalloc) {
hs_database_t *db;
hs_compile_error_t *c_err;
static const char *pattern = "hatstand.*(badgerbrush|teakettle)";
hs_error_t err = hs_compile(pattern, 0, HS_MODE_BLOCK, nullptr, &db, &c_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
size_t db_len;
err = hs_database_size(db, &db_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0, db_len);
char *bytes = nullptr;
size_t bytes_len = 0;
// fail when serialize gets a null malloc
hs_set_allocator(null_malloc, nullptr);
err = hs_serialize_database(db, &bytes, &bytes_len);
ASSERT_NE(HS_SUCCESS, err);
hs_set_allocator(nullptr, nullptr);
hs_free_database(db);
}
// make sure that serializing/deserializing to null or an unaligned address fails
TEST(Serialize, SerializeUnalignedMalloc) {
hs_database_t *db;
hs_compile_error_t *c_err;
static const char *pattern = "hatstand.*(badgerbrush|teakettle)";
hs_error_t err = hs_compile(pattern, 0, HS_MODE_BLOCK, nullptr, &db, &c_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
size_t db_len;
err = hs_database_size(db, &db_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0, db_len);
char *bytes = nullptr;
size_t bytes_len = 0;
// fail when serialize gets a misaligned malloc
hs_set_allocator(misaligned_malloc, misaligned_free);
err = hs_serialize_database(db, &bytes, &bytes_len);
ASSERT_NE(HS_SUCCESS, err);
hs_set_allocator(nullptr, nullptr);
hs_free_database(db);
}
TEST(Serialize, DeserializeNullMalloc) {
hs_database_t *db;
hs_compile_error_t *c_err;
static const char *pattern = "hatstand.*(badgerbrush|teakettle)";
hs_error_t err = hs_compile(pattern, 0, HS_MODE_BLOCK, nullptr, &db, &c_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
size_t db_len;
err = hs_database_size(db, &db_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0, db_len);
char *bytes = nullptr;
size_t bytes_len = 0;
err = hs_serialize_database(db, &bytes, &bytes_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0, bytes_len);
hs_free_database(db);
size_t ser_len;
err = hs_serialized_database_size(bytes, bytes_len, &ser_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0, ser_len);
err = hs_deserialize_database_at(bytes, ser_len, nullptr);
ASSERT_NE(HS_SUCCESS, err);
free(bytes);
}
TEST(Serialize, DeserializeUnalignedMalloc) {
hs_database_t *db;
hs_compile_error_t *c_err;
static const char *pattern = "hatstand.*(badgerbrush|teakettle)";
hs_error_t err = hs_compile(pattern, 0, HS_MODE_BLOCK, nullptr, &db, &c_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(db != nullptr);
size_t db_len;
err = hs_database_size(db, &db_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0, db_len);
char *bytes = nullptr;
size_t bytes_len = 0;
err = hs_serialize_database(db, &bytes, &bytes_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0, bytes_len);
hs_free_database(db);
size_t ser_len;
err = hs_serialized_database_size(bytes, bytes_len, &ser_len);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0, ser_len);
// and now fail when deserialize addr is unaligned
char *new_db = (char *)malloc(ser_len + 8);
for (int i = 1; i < 8; i++) {
err = hs_deserialize_database_at(bytes, ser_len,
(hs_database_t *)(new_db + i));
ASSERT_NE(HS_SUCCESS, err);
}
free(new_db);
free(bytes);
}
}

622
unit/hyperscan/single.cpp Normal file
View File

@@ -0,0 +1,622 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "gtest/gtest.h"
#include "hs.h"
#include "test_util.h"
#include <string>
#include <tuple>
using namespace std;
using namespace testing;
namespace /* anonymous */ {
class HyperscanSingleBase {
public:
virtual ~HyperscanSingleBase() {}
// Simplest case: did we successfully produce a non-zero length database
void compile() {
SCOPED_TRACE("Compile");
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_TRUE(db != nullptr);
EXPECT_TRUE(compile_err == nullptr);
size_t db_size;
err = hs_database_size(db, &db_size);
EXPECT_EQ(HS_SUCCESS, err);
EXPECT_NE(0U, db_size);
}
// Can we query the database for various information?
void queryDatabase() {
SCOPED_TRACE("QueryDatabase");
// info about the database, should begin with "Version:"
char *info;
err = hs_database_info(db, &info);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(nullptr, info);
ASSERT_STREQ("Version:", string(info).substr(0, 8).c_str());
free(info);
// stream size, which is non-zero if streaming
size_t stream_size;
err = hs_stream_size(db, &stream_size);
if (mode & HS_MODE_STREAM) {
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_LT(0U, stream_size);
ASSERT_GT(100000, stream_size); /* more than 100k of stream size
* and we are probably returning
* rubbish. */
} else {
ASSERT_EQ(HS_DB_MODE_ERROR, err);
}
}
protected:
unsigned int mode; // HS_MODE_* mode flags
hs_error_t err;
hs_compile_error_t *compile_err;
hs_database_t *db;
};
class HyperscanTestRuntime
: public HyperscanSingleBase,
public TestWithParam<
tuple<const char *, unsigned int, unsigned int, unsigned long long>> {
protected:
virtual void SetUp() {
// compiles a database for this test instantiation
const char *pattern;
unsigned int flags;
unsigned long long feature_mask;
tie(pattern, flags, mode, feature_mask) = GetParam();
hs_platform_info_t plat;
err = hs_populate_platform(&plat);
ASSERT_EQ(HS_SUCCESS, err);
plat.cpu_features &= feature_mask;
err = hs_compile(pattern, flags, mode, &plat, &db, &compile_err);
}
virtual void TearDown() {
hs_free_database(db);
hs_free_compile_error(compile_err);
}
// does a simple scan with a dummy handler (ignores matches)
virtual void simpleScan() {
const size_t datalen = 2048;
char data[datalen];
memset(data, 'X', datalen);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_TRUE(scratch != nullptr);
if (mode & HS_MODE_STREAM) {
// streaming mode scan
hs_stream_t *stream = nullptr;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
// just for fun, scan a zero-byte block.
err = hs_scan_stream(stream, data, 0, 0, scratch, dummy_cb,
nullptr);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan_stream(stream, data, datalen, 0, scratch,
dummy_cb, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_close_stream(stream, scratch, dummy_cb, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
} else if (mode & HS_MODE_VECTORED) {
const char * const vec_bufs[] = { data };
const unsigned int vec_lens[] = { datalen };
err = hs_scan_vector(db, vec_bufs, vec_lens, 1, 0, scratch, dummy_cb, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
} else {
// block mode scan
ASSERT_TRUE(mode & HS_MODE_BLOCK);
err = hs_scan(db, data, datalen, 0, scratch, dummy_cb, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
}
// teardown
hs_free_scratch(scratch);
}
virtual void zeroLengthScan() {
const size_t datalen = 20;
char data[datalen];
memset(data, 'X', datalen);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_TRUE(scratch != nullptr);
if (mode & HS_MODE_STREAM) {
// streaming mode scan
hs_stream_t *stream = nullptr;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
err = hs_scan_stream(stream, data, 0, 0, scratch, dummy_cb, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_close_stream(stream, scratch, dummy_cb, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
} else if (mode & HS_MODE_VECTORED) {
const char * const vec_bufs[] = { data };
const unsigned int vec_lens[] = { 0 };
err = hs_scan_vector(db, vec_bufs, vec_lens, 1, 0, scratch, dummy_cb, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
} else {
// block mode scan
ASSERT_TRUE(mode & HS_MODE_BLOCK);
err = hs_scan(db, data, 0, 0, scratch, dummy_cb, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
}
// teardown
hs_free_scratch(scratch);
}
// Can we allocate and clone scratch
void allocateScratch() {
SCOPED_TRACE("AllocateScratch");
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_TRUE(scratch != nullptr);
// Try cloning the scratch
hs_scratch_t *cloned = nullptr;
err = hs_clone_scratch(scratch, &cloned);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_TRUE(cloned != nullptr);
hs_free_scratch(scratch);
hs_free_scratch(cloned);
}
// Can we scan with the database (ignoring the matches, and using
// streaming/block mode appropriately)
void scanIgnoreMatches() {
SCOPED_TRACE("ScanIgnoreMatches");
simpleScan();
}
// Can we serialize and deserialize the database
void serialiseAndDeserializeCombo() {
SCOPED_TRACE("SerialiseAndDeserializeCombo");
char *bytes = nullptr;
size_t len = 0;
// serialise
err = hs_serialize_database(db, &bytes, &len);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_TRUE(bytes != nullptr);
EXPECT_NE(0U, len);
// destroy original database
size_t origSize;
err = hs_database_size(db, &origSize);
ASSERT_EQ(HS_SUCCESS, err);
memset(db, 0xff, origSize);
free(db); /* hs_free_database not used as it is no longer a valid db */
// relocate to 16 different alignments, ensuring that we can
// deserialize from any string
char *buffer = new char[len + 16];
for (size_t i = 0; i < 16; i++) {
size_t newSize;
memset(buffer, 0, len + 16);
// relocate
char *copy = buffer + i;
memcpy(copy, bytes, len);
// deserialise
hs_database_t *newdb;
err = hs_deserialize_database(copy, len, &newdb);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_TRUE(newdb != nullptr);
err = hs_database_size(newdb, &newSize);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_EQ(origSize, newSize);
// deserialiseAt
// alloc a new place for this DB
size_t buflen;
err = hs_serialized_database_size(copy, len, &buflen);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_NE(0U, buflen);
char *buf = new char[buflen];
memset(buf, 0, buflen);
// deserialise to our buffer
hs_database_t *newdbAt = (hs_database_t *)buf;
err = hs_deserialize_database_at(copy, len, newdbAt);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_database_size(newdb, &newSize);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_EQ(origSize, newSize);
{
SCOPED_TRACE("DeserializeAt");
// Replace test harness DB and run a simple scan as a test
db = newdbAt;
simpleScan();
delete[] buf;
db = nullptr;
}
{
SCOPED_TRACE("Deserialize");
// Replace test harness DB and run a simple scan as a test
db = newdb;
simpleScan();
hs_free_database(newdb);
db = nullptr;
}
}
delete[] buffer;
free(bytes);
}
};
TEST_P(HyperscanTestRuntime, Combo) {
compile();
queryDatabase();
allocateScratch();
zeroLengthScan();
scanIgnoreMatches();
serialiseAndDeserializeCombo();
}
static const char *validPatterns[] = {
"foobar",
"abd.*def",
"abc[123]def",
"[pqr]",
".",
"\\s",
"hatstand.*(teakettle|badgerbrush)",
"abc{1,3}",
"abc",
"^.{1,10}flibble",
"(foobar)+",
"(foo){2,5}",
"((foo){2}){3}",
"^.*test[a-f]{3}pattern.*$",
"(([^u]|.){16}|x){1,2}", // from bug UE-500
// UE-1553: these should compile without eating all your RAM.
"fooa?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?",
"fooa?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?",
// UE-1816 (makes libpcre hit its match limit)
"((aaa|[aaa]aa|aa[^aaaa]a|a.aaa.|a)){27}",
// UE-2370 (makes libpcre hit its match limit)
"^.{0,40}((e{0,9}|b){16}a.A|[da]..ecbcbcc[^e]de{10,20}[Bb]{3}[dEe]*){2}",
};
static const unsigned validFlags[] = {
0,
HS_FLAG_CASELESS,
HS_FLAG_DOTALL,
HS_FLAG_MULTILINE,
HS_FLAG_CASELESS | HS_FLAG_DOTALL | HS_FLAG_MULTILINE,
HS_FLAG_SINGLEMATCH,
};
static const unsigned validModes[] = {
HS_MODE_BLOCK,
HS_MODE_STREAM,
HS_MODE_VECTORED,
};
// Mode bits for switching off various architecture features
static const unsigned long long featureMask[] = {
~0ULL, /* native */
~HS_CPU_FEATURES_AVX2, /* no avx2 */
};
INSTANTIATE_TEST_CASE_P(Single,
HyperscanTestRuntime,
Combine(ValuesIn(validPatterns),
ValuesIn(validFlags),
ValuesIn(validModes),
ValuesIn(featureMask)));
struct TestPlatform {
TestPlatform() : features(0ULL) {}
TestPlatform(unsigned long long f) : features(f) {}
unsigned long long features;
};
class HyperscanTestCrossCompile
: public HyperscanSingleBase,
public TestWithParam<
tuple<const char *, unsigned, unsigned, TestPlatform>> {
protected:
virtual void SetUp() {
// compiles a database for this test instantiation
const char *pattern;
unsigned flags;
TestPlatform tp;
tie(pattern, flags, mode, tp) = GetParam();
hs_platform_info_t plat;
plat.tune = HS_TUNE_FAMILY_GENERIC;
plat.cpu_features = tp.features;
err = hs_compile(pattern, flags, mode, &plat, &db, &compile_err);
}
virtual void TearDown() {
hs_free_database(db);
hs_free_compile_error(compile_err);
}
// Attempt to allocate a scratch region. This should succeed for the local
// platform and return HS_DB_PLATFORM_ERROR for all the others.
void attemptScratchAlloc() {
SCOPED_TRACE("AttemptScratchAlloc");
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
// XXX: it'd be nice to be a bit less cavalier about this and actually
// find out what the local platform is, then check explicitly that only
// that platform produces HS_SUCCESS.
if (err == HS_SUCCESS) {
// host platform.
hs_free_scratch(scratch);
} else {
ASSERT_EQ(HS_DB_PLATFORM_ERROR, err);
ASSERT_TRUE(!scratch);
}
}
};
TEST_P(HyperscanTestCrossCompile, Build) {
compile();
queryDatabase();
attemptScratchAlloc();
}
static const TestPlatform validPlatforms[] = {
TestPlatform(0),
TestPlatform(HS_CPU_FEATURES_AVX2),
};
INSTANTIATE_TEST_CASE_P(Single,
HyperscanTestCrossCompile,
Combine(ValuesIn(validPatterns),
ValuesIn(validFlags),
ValuesIn(validModes),
ValuesIn(validPlatforms)));
// Also instantiate some normal and cross-compile tests for SOM patterns.
static const char *validSomPatterns[] = {
"foobar",
"abd.*def",
"abc[123]def",
"[pqr]",
".",
"\\s",
"hatstand.*(teakettle|badgerbrush)",
"abc{1,3}",
"abc",
"^.{1,10}flibble",
"(foobar)+",
"(foo){2,5}",
"((foo){2}){3}",
"^.*test[a-f]{3}pattern.*$",
};
static const unsigned validSomFlags[] = {
HS_FLAG_SOM_LEFTMOST,
};
static const unsigned validSomModes[] = {
HS_MODE_BLOCK,
HS_MODE_STREAM | HS_MODE_SOM_HORIZON_LARGE,
HS_MODE_VECTORED,
};
INSTANTIATE_TEST_CASE_P(SingleSom, HyperscanTestRuntime,
Combine(ValuesIn(validSomPatterns),
ValuesIn(validSomFlags),
ValuesIn(validSomModes),
ValuesIn(featureMask)));
INSTANTIATE_TEST_CASE_P(SingleSom, HyperscanTestCrossCompile,
Combine(ValuesIn(validSomPatterns),
ValuesIn(validSomFlags),
ValuesIn(validSomModes),
ValuesIn(validPlatforms)));
struct TerminateMatchData {
TerminateMatchData(const char *pattern_, unsigned int flags_,
const char *corpus_) : pattern(pattern_), flags(flags_),
corpus(corpus_) {}
const char *pattern;
unsigned int flags;
const char *corpus;
};
class HyperscanTestMatchTerminate : public TestWithParam<TerminateMatchData> {
// empty
};
// Simple non-terminating callback: increments the int pointed to by the context
// and tells the matcher to keep going.
int countHandler(unsigned, unsigned long long, unsigned long long,
unsigned, void *ctx) {
int *count = (int *)ctx;
(*count)++;
return 0; // keep going
}
// Simple terminating callback: increments the int pointed to by the context
// and tells the matcher to stop.
int terminateHandler(unsigned, unsigned long long, unsigned long long,
unsigned, void *ctx) {
int *count = (int *)ctx;
(*count)++;
return 1; // stop matching
}
TEST_P(HyperscanTestMatchTerminate, MoreThanOne) {
const TerminateMatchData &data = GetParam();
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch(data.pattern, data.flags, 0,
HS_MODE_BLOCK, &scratch);
int count = 0;
err = hs_scan(db, data.corpus, strlen(data.corpus), 0, scratch,
countHandler, &count);
ASSERT_EQ(HS_SUCCESS, err) << "hs_scan didn't return HS_SCAN_TERMINATED";
ASSERT_LT(1, count) << "Number of matches returned was not greater than 1.";
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST_P(HyperscanTestMatchTerminate, Block) {
const TerminateMatchData &data = GetParam();
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch(data.pattern, data.flags, 0,
HS_MODE_BLOCK, &scratch);
int count = 0;
err = hs_scan(db, data.corpus, strlen(data.corpus), 0, scratch,
terminateHandler, &count);
ASSERT_EQ(HS_SCAN_TERMINATED, err)
<< "hs_scan didn't return HS_SCAN_TERMINATED";
ASSERT_EQ(1, count) << "Number of matches returned was not 1.";
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST_P(HyperscanTestMatchTerminate, StreamWhole) {
const TerminateMatchData& data = GetParam();
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch(data.pattern, data.flags, 0,
HS_MODE_STREAM, &scratch);
int count = 0;
hs_stream_t *stream = nullptr;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err) << "Stream open failed";
ASSERT_TRUE(stream != nullptr);
err = hs_scan_stream(stream, data.corpus, strlen(data.corpus), 0, scratch,
terminateHandler, &count);
ASSERT_TRUE(err == HS_SUCCESS || err == HS_SCAN_TERMINATED);
err = hs_close_stream(stream, scratch, terminateHandler, &count);
ASSERT_EQ(1, count) << "Number of matches returned was not 1.";
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST_P(HyperscanTestMatchTerminate, StreamByteByByte) {
const TerminateMatchData& data = GetParam();
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch(data.pattern, data.flags, 0,
HS_MODE_STREAM, &scratch);
int count = 0;
hs_stream_t *stream = nullptr;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err) << "Stream open failed";
ASSERT_TRUE(stream != nullptr);
size_t len = strlen(data.corpus);
for (size_t i = 0; i < len; i++) {
err = hs_scan_stream(stream, data.corpus + i, 1, 0, scratch,
terminateHandler, &count);
ASSERT_TRUE(err == HS_SUCCESS || err == HS_SCAN_TERMINATED);
}
err = hs_close_stream(stream, scratch, terminateHandler, &count);
ASSERT_EQ(1, count) << "Number of matches returned was not 1.";
hs_free_scratch(scratch);
hs_free_database(db);
}
// Each of these cases must match multiple times
const TerminateMatchData terminateCases[] = {
TerminateMatchData("foobar", 0, "foobarfoobarfoobar"),
TerminateMatchData("a", 0, "a a a a a a a a a a a"),
TerminateMatchData(".", 0, "zzzzzzzzzzzzaaaaaaaaaaaaa"),
TerminateMatchData("...", 0, "zzzzzzzzzzzzaaaaaaaaaaaaa"),
TerminateMatchData("[a-z]{3,7}", 0, "zzzzzzzzzzzzaaaaaaaaaaaaa"),
TerminateMatchData("a", HS_FLAG_CASELESS, " xAaAa"),
TerminateMatchData("xyzzy", HS_FLAG_CASELESS, "abcdef XYZZy xyzzy XyZzY"),
TerminateMatchData("abc.*def", 0, "abc abc abc def def"),
TerminateMatchData("abc.*def", HS_FLAG_DOTALL, "abc abc abc def def"),
TerminateMatchData("(01234|abcde).*(foo|bar)", 0, "abcde xxxx bar foo abcde foo"),
TerminateMatchData("(01234|abcde).*(foo|bar)", HS_FLAG_DOTALL, "abcde xxxx bar foo abcde foo"),
TerminateMatchData("[0-9a-f]{4,10}.*(foobar|bazbaz)", 0, "0123456789abcdef bazbaz foobar"),
TerminateMatchData("[0-9a-f]{4,10}.*(foobar|bazbaz)", HS_FLAG_DOTALL, "0123456789abcdef bazbaz foobar"),
TerminateMatchData("^foobar[^z]{20,}", 0, "foobarxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"),
TerminateMatchData("hatstand|teakettle|badgerbrush|mnemosyne", 0, "hatstand teakettle badgerbrush mnemosyne"),
TerminateMatchData("a|b|c|d", 0, "a b c d a b c d a b c d"),
TerminateMatchData("bat|cat|mat|rat|fat|sat|pat|hat|vat", HS_FLAG_CASELESS, "VAt hat pat sat fat rat mat caT BAT"),
};
INSTANTIATE_TEST_CASE_P(Single, HyperscanTestMatchTerminate, ValuesIn(terminateCases));
} // namespace

200
unit/hyperscan/som.cpp Normal file
View File

@@ -0,0 +1,200 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include <cstring>
#include <vector>
#include "gtest/gtest.h"
#include "hs.h"
#include "test_util.h"
using namespace std;
using namespace testing;
namespace {
struct Match {
Match(unsigned i, unsigned long long f, unsigned long long t) :
id(i), from(f), to(t) {}
unsigned int id;
unsigned long long from;
unsigned long long to;
};
}
static
int vectorCallback(unsigned id, unsigned long long from,
unsigned long long to, unsigned, void *ctx) {
//printf("match id %u at (%llu,%llu)\n", id, from, to);
vector<Match> *matches = (vector<Match> *)ctx;
matches->push_back(Match(id, from, to));
return 0;
}
class SomTest : public TestWithParam<unsigned int> {
protected:
virtual void SetUp() {
som_mode = GetParam();
}
unsigned long long getSomHorizon() const {
switch (som_mode) {
case HS_MODE_SOM_HORIZON_SMALL:
return 1ULL << 16;
case HS_MODE_SOM_HORIZON_MEDIUM:
return 1ULL << 32;
default:
return ~(0ULL);
}
}
unsigned int som_mode;
};
TEST_P(SomTest, PastHorizon) {
hs_database_t *db = buildDB("foo.*bar", HS_FLAG_SOM_LEFTMOST, 1000,
HS_MODE_STREAM | som_mode);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_TRUE(scratch != nullptr);
vector<Match> matches;
hs_stream_t *stream = nullptr;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
const string prefix(" foo");
const string suffix("bar");
const string filler(4096, 'X');
unsigned long long scanned_len = 0;
err = hs_scan_stream(stream, prefix.c_str(), prefix.length(), 0, scratch,
vectorCallback, &matches);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0, matches.size());
scanned_len += prefix.length();
const unsigned long long blocks = getSomHorizon() / filler.length();
for (unsigned long long i = 0; i < blocks; i += 1) {
err = hs_scan_stream(stream, filler.c_str(), filler.length(), 0,
scratch, vectorCallback, &matches);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0, matches.size());
scanned_len += filler.length();
}
err = hs_scan_stream(stream, suffix.c_str(), suffix.length(), 0, scratch,
vectorCallback, &matches);
ASSERT_EQ(HS_SUCCESS, err);
scanned_len += suffix.length();
// We receive one match with the correct 'to' offset, but a sentinel value
// for 'from'.
ASSERT_EQ(1, matches.size());
ASSERT_EQ(1000, matches[0].id);
ASSERT_EQ(HS_OFFSET_PAST_HORIZON, matches[0].from);
ASSERT_EQ(scanned_len, matches[0].to);
err = hs_close_stream(stream, scratch, vectorCallback, &matches);
ASSERT_EQ(HS_SUCCESS, err);
// teardown
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST_P(SomTest, NearHorizon) {
hs_database_t *db = buildDB("foo.*bar", HS_FLAG_SOM_LEFTMOST, 1000,
HS_MODE_STREAM | som_mode);
ASSERT_TRUE(db != nullptr);
hs_scratch_t *scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_TRUE(scratch != nullptr);
vector<Match> matches;
hs_stream_t *stream = nullptr;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
const string prefix(" foo");
const string suffix("bar");
const string filler(4096, 'X');
unsigned long long scanned_len = 0;
err = hs_scan_stream(stream, prefix.c_str(), prefix.length(), 0, scratch,
vectorCallback, &matches);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0, matches.size());
scanned_len += prefix.length();
const unsigned long long blocks = getSomHorizon() / filler.length() - 1;
for (unsigned long long i = 0; i < blocks; i += 1) {
err = hs_scan_stream(stream, filler.c_str(), filler.length(), 0,
scratch, vectorCallback, &matches);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0, matches.size());
scanned_len += filler.length();
}
err = hs_scan_stream(stream, suffix.c_str(), suffix.length(), 0, scratch,
vectorCallback, &matches);
ASSERT_EQ(HS_SUCCESS, err);
scanned_len += suffix.length();
// Both 'to' and 'from' should be accurate.
ASSERT_EQ(1, matches.size());
ASSERT_EQ(1000, matches[0].id);
ASSERT_EQ(1, matches[0].from);
ASSERT_EQ(scanned_len, matches[0].to);
err = hs_close_stream(stream, scratch, vectorCallback, &matches);
ASSERT_EQ(HS_SUCCESS, err);
// teardown
hs_free_scratch(scratch);
hs_free_database(db);
}
INSTANTIATE_TEST_CASE_P(Som, SomTest,
Values(HS_MODE_SOM_HORIZON_SMALL,
HS_MODE_SOM_HORIZON_MEDIUM));

767
unit/hyperscan/stream_op.cpp Normal file
View File

@@ -0,0 +1,767 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <vector>
#include "gtest/gtest.h"
#include "hs.h"
#include "test_util.h"
using namespace std;
namespace /* anonymous */ {
int record_cb2(unsigned id, unsigned long long, unsigned long long to,
unsigned, void *ctxt) {
CallBackContext *c = (CallBackContext *)ctxt;
c->matches.push_back(MatchRecord(to + 1000, id));
return (int)c->halt;
}
static const char data1[] = "barfoobar";
TEST(StreamUtil, reset1) {
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch("foo.*bar", 0, 0, HS_MODE_STREAM,
&scratch);
hs_stream_t *stream = nullptr;
CallBackContext c, c2;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
c.matches.clear();
err = hs_reset_stream(stream, 0, scratch, nullptr, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb2,
(void *)&c2);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0U, c.matches.size());
ASSERT_EQ(1U, c2.matches.size());
ASSERT_EQ(MatchRecord(1009, 0), c2.matches[0]);
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(StreamUtil, reset2) {
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch("foo.*bar", 0, 0, HS_MODE_STREAM,
&scratch);
hs_stream_t *stream = nullptr;
CallBackContext c, c2;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
c.halt = 1;
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SCAN_TERMINATED, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
c.matches.clear();
err = hs_reset_stream(stream, 0, scratch, nullptr, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c2);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0U, c.matches.size());
ASSERT_EQ(1U, c2.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c2.matches[0]);
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(StreamUtil, reset_matches) {
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch("foo.*bar$", 0, 0, HS_MODE_STREAM,
&scratch);
hs_stream_t *stream = nullptr;
CallBackContext c;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
err = hs_scan_stream(stream, data1, strlen(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0U, c.matches.size());
err = hs_reset_stream(stream, 0, scratch, record_cb, (void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(StreamUtil, copy1) {
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch("foo.*bar", 0, 0, HS_MODE_STREAM,
&scratch);
hs_stream_t *stream = nullptr;
hs_stream_t *stream2 = nullptr;
CallBackContext c;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
c.matches.clear();
err = hs_copy_stream(&stream2, stream);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(2U, c.matches.size());
ASSERT_EQ(MatchRecord(13, 0), c.matches[0]);
ASSERT_EQ(MatchRecord(19, 0), c.matches[1]);
c.matches.clear();
err = hs_scan_stream(stream2, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(2U, c.matches.size());
ASSERT_EQ(MatchRecord(13, 0), c.matches[0]);
ASSERT_EQ(MatchRecord(19, 0), c.matches[1]);
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_close_stream(stream2, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(StreamUtil, copy2) {
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch("foo.*bar", 0, 0, HS_MODE_STREAM,
&scratch);
hs_stream_t *stream = nullptr;
hs_stream_t *stream2 = nullptr;
CallBackContext c;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
c.halt = 1;
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SCAN_TERMINATED, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
c.matches.clear();
err = hs_copy_stream(&stream2, stream);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SCAN_TERMINATED, err);
ASSERT_EQ(0U, c.matches.size());
err = hs_scan_stream(stream2, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SCAN_TERMINATED, err);
ASSERT_EQ(0U, c.matches.size());
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_close_stream(stream2, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(StreamUtil, copy_reset1) {
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch("foo.*bar", 0, 0, HS_MODE_STREAM,
&scratch);
hs_stream_t *stream = nullptr;
hs_stream_t *stream2 = nullptr;
CallBackContext c, c2;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
err = hs_open_stream(db, 0, &stream2);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream2 != nullptr);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
c.matches.clear();
err = hs_reset_and_copy_stream(stream, stream2, scratch, nullptr, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb2,
(void *)&c2);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0U, c.matches.size());
ASSERT_EQ(1U, c2.matches.size());
ASSERT_EQ(MatchRecord(1009, 0), c2.matches[0]);
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_close_stream(stream2, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(StreamUtil, copy_reset2) {
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch("foo.*bar", 0, 0, HS_MODE_STREAM,
&scratch);
hs_stream_t *stream = nullptr;
hs_stream_t *stream2 = nullptr;
CallBackContext c, c2;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
err = hs_open_stream(db, 0, &stream2);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream2 != nullptr);
c.halt = 1;
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SCAN_TERMINATED, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
c.matches.clear();
err = hs_reset_and_copy_stream(stream, stream2, scratch, nullptr, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c2);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0U, c.matches.size());
ASSERT_EQ(1U, c2.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c2.matches[0]);
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_close_stream(stream2, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(StreamUtil, copy_reset3) {
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch("foo.*bar", 0, 0, HS_MODE_STREAM,
&scratch);
hs_stream_t *stream = nullptr;
hs_stream_t *stream2 = nullptr;
CallBackContext c;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
err = hs_open_stream(db, 0, &stream2);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
c.matches.clear();
err = hs_reset_and_copy_stream(stream2, stream, scratch, nullptr, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(2U, c.matches.size());
ASSERT_EQ(MatchRecord(13, 0), c.matches[0]);
ASSERT_EQ(MatchRecord(19, 0), c.matches[1]);
c.matches.clear();
err = hs_scan_stream(stream2, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(2U, c.matches.size());
ASSERT_EQ(MatchRecord(13, 0), c.matches[0]);
ASSERT_EQ(MatchRecord(19, 0), c.matches[1]);
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_close_stream(stream2, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(StreamUtil, copy_reset4) {
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch("foo.*bar", 0, 0, HS_MODE_STREAM,
&scratch);
hs_stream_t *stream = nullptr;
hs_stream_t *stream2 = nullptr;
CallBackContext c;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
err = hs_open_stream(db, 0, &stream2);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream2 != nullptr);
c.halt = 1;
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SCAN_TERMINATED, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
c.matches.clear();
err = hs_reset_and_copy_stream(stream2, stream, scratch, nullptr, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SCAN_TERMINATED, err);
ASSERT_EQ(0U, c.matches.size());
err = hs_scan_stream(stream2, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SCAN_TERMINATED, err);
ASSERT_EQ(0U, c.matches.size());
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_close_stream(stream2, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(StreamUtil, copy_reset5) {
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch("foo.*bar", 0, 0, HS_MODE_STREAM,
&scratch);
hs_stream_t *stream = nullptr;
hs_stream_t *stream2 = nullptr;
CallBackContext c;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
err = hs_open_stream(db, 0, &stream2);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
c.matches.clear();
err = hs_scan_stream(stream2, "foo", 3, 0, scratch, record_cb, (void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0U, c.matches.size());
err = hs_reset_and_copy_stream(stream2, stream, scratch, nullptr, nullptr);
ASSERT_EQ(HS_SUCCESS, err);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(2U, c.matches.size());
ASSERT_EQ(MatchRecord(13, 0), c.matches[0]);
ASSERT_EQ(MatchRecord(19, 0), c.matches[1]);
c.matches.clear();
err = hs_scan_stream(stream2, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(2U, c.matches.size());
ASSERT_EQ(MatchRecord(13, 0), c.matches[0]);
ASSERT_EQ(MatchRecord(19, 0), c.matches[1]);
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_close_stream(stream2, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
}
TEST(StreamUtil, copy_reset_matches) {
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch("foo.*bar$", 0, 0, HS_MODE_STREAM,
&scratch);
hs_stream_t *stream = nullptr;
hs_stream_t *stream2 = nullptr;
CallBackContext c;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
err = hs_open_stream(db, 0, &stream2);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream2 != nullptr);
err = hs_scan_stream(stream, data1, strlen(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(0U, c.matches.size());
err = hs_reset_and_copy_stream(stream, stream2, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_close_stream(stream2, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
}
static size_t last_alloc;
static
void *wrap_m(size_t s) {
last_alloc = s;
return malloc(s);
}
TEST(StreamUtil, size) {
hs_error_t err;
hs_scratch_t *scratch = nullptr;
hs_database_t *db = buildDBAndScratch("foo.*bar", 0, 0, HS_MODE_STREAM,
&scratch);
hs_stream_t *stream = nullptr;
CallBackContext c, c2;
hs_set_allocator(wrap_m, nullptr);
last_alloc = 0;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
size_t stream_size;
err = hs_stream_size(db, &stream_size);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(last_alloc, stream_size);
hs_set_allocator(nullptr, nullptr);
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
}
int alloc_called = 0;
int alloc2_called = 0;
int alloc3_called = 0;
void *bad_alloc(size_t) {
return nullptr;
}
void *my_alloc(size_t s) {
++alloc_called;
return malloc(s);
}
void my_free(void *p) {
free(p);
--alloc_called;
}
void *my_alloc2(size_t s) {
++alloc2_called;
return malloc(s);
}
void my_free2(void *p) {
free(p);
--alloc2_called;
}
void *my_alloc3(size_t s) {
++alloc3_called;
return malloc(s);
}
void my_free3(void *p) {
free(p);
--alloc3_called;
}
TEST(StreamUtil, Alloc) {
hs_compile_error_t *compile_err = nullptr;
hs_database_t *db;
hs_set_stream_allocator(my_alloc, my_free);
alloc_called = 0;
hs_error_t err = hs_compile("foo.*bar", 0, HS_MODE_STREAM, nullptr, &db,
&compile_err);
ASSERT_EQ(HS_SUCCESS, err);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
hs_stream_t *stream = nullptr;
CallBackContext c;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
ASSERT_NE(alloc_called, 0);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
hs_close_stream(stream, scratch, record_cb, (void *)&c);
ASSERT_EQ(alloc_called, 0);
hs_free_scratch(scratch);
hs_free_database(db);
hs_free_compile_error(compile_err);
hs_set_allocator(nullptr, nullptr);
}
TEST(StreamUtil, MoreAlloc) {
hs_compile_error_t *compile_err = nullptr;
hs_database_t *db;
hs_set_allocator(my_alloc, my_free);
alloc_called = 0;
hs_error_t err = hs_compile("foo.*bar", 0, HS_MODE_STREAM, nullptr, &db,
&compile_err);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1, alloc_called);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(2, alloc_called);
ASSERT_EQ(HS_SUCCESS, err);
hs_stream_t *stream = nullptr;
CallBackContext c;
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_TRUE(stream != nullptr);
ASSERT_EQ(alloc_called, 3);
err = hs_scan_stream(stream, data1, sizeof(data1), 0, scratch, record_cb,
(void *)&c);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1U, c.matches.size());
ASSERT_EQ(MatchRecord(9, 0), c.matches[0]);
hs_close_stream(stream, scratch, nullptr, nullptr);
ASSERT_EQ(alloc_called, 2);
hs_free_scratch(scratch);
ASSERT_EQ(alloc_called, 1);
hs_free_database(db);
ASSERT_EQ(alloc_called, 0);
hs_free_compile_error(compile_err);
hs_set_allocator(nullptr, nullptr);
}
TEST(StreamUtil, BadStreamAlloc) {
hs_compile_error_t *compile_err = nullptr;
hs_database_t *db;
hs_set_stream_allocator(bad_alloc, free);
alloc_called = 0;
hs_error_t err = hs_compile("foo.*bar", 0, HS_MODE_STREAM, nullptr, &db,
&compile_err);
ASSERT_EQ(HS_SUCCESS, err);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
hs_stream_t *stream = nullptr;
CallBackContext c;
// should go boom
err = hs_open_stream(db, 0, &stream);
ASSERT_NE(HS_SUCCESS, err);
hs_free_scratch(scratch);
hs_free_database(db);
hs_free_compile_error(compile_err);
hs_set_stream_allocator(nullptr, nullptr);
}
TEST(StreamUtil, StreamAllocUsage) {
hs_compile_error_t *compile_err = nullptr;
hs_database_t *db;
hs_set_allocator(my_alloc, my_free);
hs_set_stream_allocator(my_alloc2, my_free2);
hs_set_scratch_allocator(my_alloc3, my_free3);
alloc_called = 0;
alloc2_called = 0;
alloc3_called = 0;
hs_error_t err = hs_compile("foo.*bar", 0, HS_MODE_STREAM, nullptr, &db,
&compile_err);
ASSERT_EQ(HS_SUCCESS, err);
hs_scratch_t *scratch = nullptr;
err = hs_alloc_scratch(db, &scratch);
ASSERT_EQ(HS_SUCCESS, err);
hs_stream_t *stream = nullptr;
hs_stream_t *stream2 = nullptr;
hs_stream_t *stream3 = nullptr;
CallBackContext c;
ASSERT_EQ(1, alloc_called);
ASSERT_EQ(0, alloc2_called);
ASSERT_EQ(1, alloc3_called);
err = hs_open_stream(db, 0, &stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1, alloc_called);
ASSERT_EQ(1, alloc2_called);
ASSERT_EQ(1, alloc3_called);
err = hs_copy_stream(&stream2, stream);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1, alloc_called);
ASSERT_EQ(2, alloc2_called);
ASSERT_EQ(1, alloc3_called);
err = hs_open_stream(db, 0, &stream3);
ASSERT_EQ(HS_SUCCESS, err);
ASSERT_EQ(1, alloc_called);
ASSERT_EQ(3, alloc2_called);
ASSERT_EQ(1, alloc3_called);
hs_close_stream(stream, scratch, nullptr, nullptr);
hs_close_stream(stream2, scratch, nullptr, nullptr);
hs_close_stream(stream3, scratch, nullptr, nullptr);
hs_free_scratch(scratch);
hs_free_database(db);
hs_free_compile_error(compile_err);
hs_set_allocator(nullptr, nullptr);
ASSERT_EQ(0, alloc_called);
ASSERT_EQ(0, alloc2_called);
ASSERT_EQ(0, alloc3_called);
}
}

234
unit/hyperscan/test_util.cpp Normal file
View File

@@ -0,0 +1,234 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
#include "hs.h"
#include "test_util.h"
#include "gtest/gtest.h"
#include "util/expressions.h"
#include "util/ExpressionParser.h"
using namespace std;
int record_cb(unsigned id, unsigned long long, unsigned long long to,
unsigned, void *ctxt) {
CallBackContext *c = (CallBackContext *)ctxt;
c->matches.push_back(MatchRecord(to, id));
return (int)c->halt;
}
std::ostream &operator<< (std::ostream &o, const MatchRecord &m) {
return o << "[" << m.to << ", " << m.id << "]";
}
hs_database_t *buildDB(const vector<pattern> &patterns, unsigned int mode,
hs_platform_info *plat) {
vector<const char *> expressions;
vector<unsigned int> flags;
vector<unsigned int> ids;
vector<const hs_expr_ext *> ext;
for (vector<pattern>::const_iterator it = patterns.begin();
it != patterns.end(); ++it) {
expressions.push_back(it->expression.c_str());
flags.push_back(it->flags);
ids.push_back(it->id);
ext.push_back(&it->ext);
}
hs_database_t *db = nullptr;
hs_compile_error_t *compile_err = nullptr;
hs_error_t err;
err = hs_compile_ext_multi(&expressions[0], &flags[0], &ids[0], &ext[0],
patterns.size(), mode, plat, &db, &compile_err);
if (err != HS_SUCCESS) {
return nullptr;
}
return db;
}
hs_database_t *buildDB(const pattern &expr, unsigned int mode) {
return buildDB(vector<pattern>(1, expr), mode);
}
hs_database_t *buildDB(const char *expression, unsigned int flags,
unsigned int id, unsigned int mode,
hs_platform_info_t *plat) {
vector<pattern> patterns;
patterns.push_back(pattern(expression, flags, id));
return buildDB(patterns, mode, plat);
}
hs_database_t *buildDB(const char *filename, unsigned int mode,
unsigned int extra_flags) {
vector<pattern> patterns;
ExpressionMap expressions;
loadExpressionsFromFile(filename, expressions);
for (ExpressionMap::iterator it = expressions.begin();
it != expressions.end(); ++it) {
unsigned int flags = 0;
string regex;
hs_expr_ext ext;
if (!readExpression(it->second, regex, &flags, &ext)) {
return nullptr;
}
patterns.push_back(pattern(regex, flags | extra_flags, it->first,
ext));
}
return buildDB(patterns, mode);
}
static
bool isOrdered(const string &expr, unsigned int flags) {
// SOM doesn't produce ordered matches?
if (flags & HS_FLAG_SOM_LEFTMOST) {
return false;
}
hs_expr_info_t *info = nullptr;
hs_compile_error_t *error = nullptr;
hs_error_t err = hs_expression_info(expr.c_str(), flags, &info, &error);
if (err != HS_SUCCESS) {
// Expression will fail compilation and report error elsewhere.
free(info);
hs_free_compile_error(error);
return false;
}
assert(info);
// Any pattern that does not require offset adjustment should produce
// matches in order.
bool ordered = !info->unordered_matches;
free(info);
return ordered;
}
hs_database_t *buildDB(const char *filename, unsigned int mode,
bool check_ordering) {
vector<pattern> patterns;
ExpressionMap expressions;
loadExpressionsFromFile(filename, expressions);
for (ExpressionMap::iterator it = expressions.begin();
it != expressions.end(); ++it) {
unsigned int flags = 0;
string regex;
hs_expr_ext ext;
bool must_be_ordered;
if (!readExpression(it->second, regex, &flags, &ext, &must_be_ordered)) {
return nullptr;
}
if (check_ordering && must_be_ordered && !isOrdered(regex, flags)) {
return nullptr;
}
patterns.emplace_back(regex, flags, it->first, ext);
}
return buildDB(patterns, mode);
}
hs_database_t *buildDBAndScratch(const char *expression, unsigned int flags,
unsigned int id, unsigned int mode,
hs_scratch_t **scratch) {
hs_database_t *db = buildDB(expression, flags, id, mode);
EXPECT_TRUE(db != nullptr);
*scratch = nullptr;
hs_error_t err = hs_alloc_scratch(db, scratch);
EXPECT_EQ(HS_SUCCESS, err);
EXPECT_TRUE(*scratch != nullptr);
return db;
}
size_t allocated_count;
size_t allocated_count_b;
void *count_malloc(size_t n) {
void *pp = malloc(n + 16);
if (!pp) {
return nullptr;
}
allocated_count += n;
*(size_t *)pp = n;
void *p = (char *)pp + 16;
return p;
}
void count_free(void *p) {
if (!p) {
return;
}
void *pp = (char *)p - 16;
size_t n = *(size_t *)pp;
allocated_count -= n;
free(pp);
}
void *count_malloc_b(size_t n) {
void *pp = malloc(n + 32);
if (!pp) {
return nullptr;
}
allocated_count_b += n;
*(size_t *)pp = n;
void *p = (char *)pp + 32;
return p;
}
void count_free_b(void *p) {
if (!p) {
return;
}
void *pp = (char *)p - 32;
size_t n = *(size_t *)pp;
allocated_count_b -= n;
free(pp);
}

120
unit/hyperscan/test_util.h Normal file
View File

@@ -0,0 +1,120 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef TEST_UTIL_H
#define TEST_UTIL_H
#include <iosfwd>
#include <vector>
#include "hs.h"
#ifndef UNUSED
#if defined(_WIN32) || defined(_WIN64)
#define UNUSED
#else
#define UNUSED __attribute__ ((unused))
#endif
#endif
struct MatchRecord {
MatchRecord(unsigned long long t, int i) : to(t), id(i) {}
bool operator==(const MatchRecord &o) const {
return to == o.to && id == o.id;
}
unsigned long long to;
int id;
};
std::ostream &operator<< (std::ostream &o, const MatchRecord &m);
struct CallBackContext {
CallBackContext() : halt(false) {}
bool halt;
std::vector<MatchRecord> matches;
void clear() {
halt = false;
matches.clear();
}
};
int record_cb(unsigned id, unsigned long long, unsigned long long to,
unsigned, void *ctxt);
// Dummy callback: does nothing, returns 0 (keep matching)
static UNUSED
int dummy_cb(unsigned, unsigned long long, unsigned long long, unsigned,
void *) {
// empty
return 0;
}
struct pattern {
std::string expression;
unsigned int flags;
unsigned int id;
hs_expr_ext ext;
pattern(const std::string &expression_in, unsigned int flags_in = 0,
unsigned int id_in = 0) : expression(expression_in),
flags(flags_in), id(id_in) {
memset(&ext, 0, sizeof(ext));
}
pattern(const std::string &expression_in, unsigned int flags_in,
unsigned int id_in, const hs_expr_ext &ext_in) :
expression(expression_in), flags(flags_in), id(id_in),
ext(ext_in) { }
};
hs_database_t *buildDB(const std::vector<pattern> &patterns, unsigned int mode,
hs_platform_info *plat = nullptr);
hs_database_t *buildDB(const pattern &pat, unsigned int mode);
hs_database_t *buildDB(const char *expression, unsigned int flags,
unsigned int id, unsigned int mode,
hs_platform_info *plat = nullptr);
hs_database_t *buildDB(const char *filename, unsigned int mode,
unsigned int extra_flags = 0);
hs_database_t *buildDB(const char *filename, unsigned int mode,
bool check_ordering);
hs_database_t *buildDBAndScratch(const char *expression, unsigned int flags,
unsigned int id, unsigned int mode,
hs_scratch_t **scratch);
extern size_t allocated_count;
extern size_t allocated_count_b;
void *count_malloc(size_t n);
void *count_malloc_b(size_t n);
void count_free(void *p);
void count_free_b(void *p);
#endif