From 86c5f7feb1efede55a8f93cc35d852063049688e Mon Sep 17 00:00:00 2001 From: "Wang, Xiang W" Date: Thu, 22 Jun 2017 04:50:45 -0400 Subject: [PATCH] FDR: Squash buckets of included literals in FDR confirm - Change the compile of literal matchers to two passes. - Reverse the bucket assignment in FDR, bucket with longer literals has smaller bucket id. - Squash the buckets of included literals and jump to the the program of included literals directly from parent literal program without going through FDR confirm for included iterals. --- src/fdr/fdr.c | 2 +- src/fdr/fdr_compile.cpp | 291 ++++++++++++++++++++++++--- src/fdr/fdr_compile.h | 22 +- src/fdr/fdr_compile_internal.h | 9 +- src/fdr/fdr_confirm_compile.cpp | 6 +- src/fdr/fdr_confirm_runtime.h | 10 +- src/fdr/teddy_compile.cpp | 94 +++++---- src/fdr/teddy_compile.h | 10 +- src/fdr/teddy_runtime_common.h | 3 +- src/hwlm/hwlm_build.cpp | 111 +++++++--- src/hwlm/hwlm_build.h | 62 +++++- src/hwlm/hwlm_literal.h | 17 ++ src/rose/program_runtime.h | 17 ++ src/rose/rose_build_bytecode.cpp | 216 +++++++++++++++++--- src/rose/rose_build_dump.cpp | 6 + src/rose/rose_build_instructions.cpp | 8 + src/rose/rose_build_instructions.h | 28 +++ src/rose/rose_build_matchers.cpp | 151 +++++++------- src/rose/rose_build_matchers.h | 85 ++++++-- src/rose/rose_build_program.cpp | 8 + src/rose/rose_build_program.h | 1 + src/rose/rose_program.h | 13 +- src/scratch.c | 1 + src/scratch.h | 3 + unit/internal/fdr.cpp | 79 ++++++-- unit/internal/fdr_flood.cpp | 26 ++- 26 files changed, 1017 insertions(+), 262 deletions(-) diff --git a/src/fdr/fdr.c b/src/fdr/fdr.c index 8d072ea2..f7da6981 100644 --- a/src/fdr/fdr.c +++ b/src/fdr/fdr.c @@ -359,7 +359,7 @@ void do_confirm_fdr(u64a *conf, u8 offset, hwlmcb_rv_t *control, } u64a confVal = unaligned_load_u64a(confLoc + byte - sizeof(u64a) + 1); confWithBit(fdrc, a, ptr_main - a->buf + byte, control, - last_match_id, confVal); + last_match_id, confVal, conf, bit); } while (unlikely(!!*conf)); } diff --git a/src/fdr/fdr_compile.cpp b/src/fdr/fdr_compile.cpp index 6f2de3d9..181f9512 100644 --- a/src/fdr/fdr_compile.cpp +++ b/src/fdr/fdr_compile.cpp @@ -42,10 +42,13 @@ #include "ue2common.h" #include "hwlm/hwlm_build.h" #include "util/compare.h" +#include "util/container.h" #include "util/dump_mask.h" +#include "util/make_unique.h" #include "util/math.h" #include "util/noncopyable.h" #include "util/target_info.h" +#include "util/ue2_containers.h" #include "util/ue2string.h" #include "util/verify_types.h" @@ -81,7 +84,6 @@ private: bool make_small; u8 *tabIndexToMask(u32 indexInTable); - void assignStringsToBuckets(); #ifdef DEBUG void dumpMasks(const u8 *defaultMask); #endif @@ -90,10 +92,13 @@ private: void createInitialState(FDR *fdr); public: - FDRCompiler(vector lits_in, const FDREngineDescription &eng_in, + FDRCompiler(vector lits_in, + map> bucketToLits_in, + const FDREngineDescription &eng_in, bool make_small_in, const Grey &grey_in) : eng(eng_in), grey(grey_in), tab(eng_in.getTabSizeBytes()), - lits(move(lits_in)), make_small(make_small_in) {} + lits(move(lits_in)), bucketToLits(move(bucketToLits_in)), + make_small(make_small_in) {} bytecode_ptr build(); }; @@ -309,7 +314,10 @@ next_literal: return chunks; } -void FDRCompiler::assignStringsToBuckets() { +static +map> assignStringsToBuckets( + vector &lits, + const FDREngineDescription &eng) { const double MAX_SCORE = numeric_limits::max(); assert(!lits.empty()); // Shouldn't be called with no literals. @@ -393,6 +401,7 @@ void FDRCompiler::assignStringsToBuckets() { // our best score is in t[0][N_BUCKETS-1] and we can follow the links // to find where our buckets should start and what goes into them + vector> buckets; for (u32 i = 0, n = numBuckets; n && (i != numChunks - 1); n--) { u32 j = t[i][n - 1].second; if (j == 0) { @@ -403,21 +412,33 @@ void FDRCompiler::assignStringsToBuckets() { u32 first_id = chunks[i].first_id; u32 last_id = chunks[j].first_id; assert(first_id < last_id); - u32 bucket = numBuckets - n; UNUSED const auto &first_lit = lits[first_id]; UNUSED const auto &last_lit = lits[last_id - 1]; - DEBUG_PRINTF("placing [%u-%u) in bucket %u (%u lits, len %zu-%zu, " + DEBUG_PRINTF("placing [%u-%u) in one bucket (%u lits, len %zu-%zu, " "score %0.4f)\n", - first_id, last_id, bucket, last_id - first_id, + first_id, last_id, last_id - first_id, first_lit.s.length(), last_lit.s.length(), getScoreUtil(first_lit.s.length(), last_id - first_id)); - auto &bucket_lits = bucketToLits[bucket]; - for (u32 k = first_id; k < last_id; k++) { - bucket_lits.push_back(k); + vector litIds; + u32 cnt = last_id - first_id; + // long literals first for included literals checking + for (u32 k = 0; k < cnt; k++) { + litIds.push_back(last_id - k - 1); } + i = j; + buckets.push_back(litIds); } + + // reverse bucket id, longer literals come first + map> bucketToLits; + size_t bucketCnt = buckets.size(); + for (size_t i = 0; i < bucketCnt; i++) { + bucketToLits.emplace(bucketCnt - i - 1, move(buckets[i])); + } + + return bucketToLits; } #ifdef DEBUG @@ -541,24 +562,216 @@ void FDRCompiler::setupTab() { } bytecode_ptr FDRCompiler::build() { - assignStringsToBuckets(); setupTab(); return setupFDR(); } +static +bool isSuffix(const hwlmLiteral &lit1, const hwlmLiteral &lit2) { + auto s1 = lit1.s; + auto s2 = lit2.s; + if (lit1.nocase || lit2.nocase) { + upperString(s1); + upperString(s2); + } + size_t len1 = s1.length(); + size_t len2 = s2.length(); + assert(len1 >= len2); + return equal(s2.begin(), s2.end(), s1.begin() + len1 - len2); +} + +/* + * if lit2 is a suffix of lit1 but the case sensitivity, groups or mask info + * of lit2 is a subset of lit1, then lit1 can't squash lit2 and lit2 can + * possibly match when lit1 matches. In this case, we can't do bucket + * squashing. e.g. AAA(no case) in bucket 0, AA(no case) and aa in bucket 1, + * we can't squash bucket 1 if we have input like "aaa" as aa can also match. + */ +static +bool includedCheck(const hwlmLiteral &lit1, const hwlmLiteral &lit2) { + /* lit1 is caseless and lit2 is case sensitive */ + if ((lit1.nocase && !lit2.nocase)) { + return true; + } + + /* lit2's group is a subset of lit1 */ + if (lit1.groups != lit2.groups && + (lit2.groups == (lit1.groups & lit2.groups))) { + return true; + } + + /* TODO: narrow down cases for mask check */ + if (lit1.cmp != lit2.cmp || lit1.msk != lit2.msk) { + return true; + } + + return false; +} + +/* + * if lit2 is an included literal of both lit1 and lit0, and lit1 is an + * exceptional literal of lit0 - lit1 sometimes matches when lit0 matches, + * then we give up squashing for lit1. e.g. lit0:AAA(no case), lit1:aa, + * lit2:A(no case). We can have duplicate matches for input "aaa" if lit0 + * and lit1 both squash lit2. + */ +static +bool checkParentLit( + u32 pos1, const unordered_set &parent_map, + const unordered_map> &exception_map) { + for (const auto pos2 : parent_map) { + if (contains(exception_map, pos2)) { + const auto &exception_pos = exception_map.at(pos2); + if (contains(exception_pos, pos1)) { + return false; + } + } + } + + return true; +} + +static +void buildSquashMask(vector &lits, u32 id1, u32 bucket1, + size_t start, const vector> &group, + unordered_map> &parent_map, + unordered_map> &exception_map) { + auto &lit1 = lits[id1]; + DEBUG_PRINTF("b:%u len:%zu\n", bucket1, lit1.s.length()); + + size_t cnt = group.size(); + bool included = false; + bool exception = false; + u32 child_id = ~0U; + for (size_t i = start; i < cnt; i++) { + u32 bucket2 = group[i].first; + assert(bucket2 >= bucket1); + + u32 id2 = group[i].second; + auto &lit2 = lits[id2]; + // check if lit2 is a suffix of lit1 + if (isSuffix(lit1, lit2)) { + /* if we have a included literal in the same bucket, + * quit and let the included literal to do possible squashing + */ + if (bucket1 == bucket2) { + DEBUG_PRINTF("same bucket\n"); + return; + } + /* + * if lit2 is a suffix but doesn't pass included checks for + * extra info, we give up sqaushing + */ + if (includedCheck(lit1, lit2)) { + DEBUG_PRINTF("find exceptional suffix %u\n", lit2.id); + exception_map[id1].insert(id2); + exception = true; + } else if (checkParentLit(id1, parent_map[id2], exception_map)) { + if (lit1.included_id == INVALID_LIT_ID) { + DEBUG_PRINTF("find suffix lit1 %u lit2 %u\n", + lit1.id, lit2.id); + lit1.included_id = lit2.id; + } else { + /* + * if we have multiple included literals in one bucket, + * give up squashing. + */ + DEBUG_PRINTF("multiple included literals\n"); + lit1.included_id = INVALID_LIT_ID; + return; + } + child_id = id2; + included = true; + } + } + + size_t next = i + 1; + u32 nextBucket = next < cnt ? group[next].first : ~0U; + if (bucket2 != nextBucket) { + if (included) { + if (exception) { + /* + * give up if we have exception literals + * in the same bucket as the included literal + */ + lit1.included_id = INVALID_LIT_ID; + } else { + parent_map[child_id].insert(id1); + + lit1.squash |= 1U << bucket2; + DEBUG_PRINTF("build squash mask %2x for %u\n", + lit1.squash, lit1.id); + } + return; + } + exception = false; + } + } +} + +static constexpr u32 INCLUDED_LIMIT = 1000; + +static +void findIncludedLits(vector &lits, + const vector>> &lastCharMap) { + /** Map for finding the positions of literal which includes a literal + * in FDR hwlm literal vector. + */ + unordered_map> parent_map; + + /** Map for finding the positions of exception literals which could + * sometimes match if a literal matches in FDR hwlm literal vector. + */ + unordered_map> exception_map; + for (const auto &group : lastCharMap) { + size_t cnt = group.size(); + if (cnt > INCLUDED_LIMIT) { + continue; + } + for (size_t i = 0; i < cnt; i++) { + u32 bucket1 = group[i].first; + u32 id1 = group[i].second; + buildSquashMask(lits, id1, bucket1, i + 1, group, parent_map, + exception_map); + } + } +} + +static +void addIncludedInfo( + vector &lits, u32 nBuckets, + map> &bucketToLits) { + vector>> lastCharMap(256); + + for (BucketIndex b = 0; b < nBuckets; b++) { + if (!bucketToLits[b].empty()) { + for (const LiteralIndex &lit_idx : bucketToLits[b]) { + const auto &lit = lits[lit_idx]; + u8 c = mytoupper(lit.s.back()); + lastCharMap[c].emplace_back(b, lit_idx); + } + } + } + + findIncludedLits(lits, lastCharMap); +} + } // namespace static -bytecode_ptr fdrBuildTableInternal(const vector &lits, - bool make_small, const target_t &target, - const Grey &grey, u32 hint) { +unique_ptr fdrBuildProtoInternal(u8 engType, + vector &lits, + bool make_small, + const target_t &target, + const Grey &grey, u32 hint) { DEBUG_PRINTF("cpu has %s\n", target.has_avx2() ? "avx2" : "no-avx2"); if (grey.fdrAllowTeddy) { - auto fdr = teddyBuildTableHinted(lits, make_small, hint, target, grey); - if (fdr) { + auto proto = teddyBuildProtoHinted(engType, lits, make_small, hint, + target); + if (proto) { DEBUG_PRINTF("build with teddy succeeded\n"); - return fdr; + return proto; } else { DEBUG_PRINTF("build with teddy failed, will try with FDR\n"); } @@ -576,23 +789,47 @@ bytecode_ptr fdrBuildTableInternal(const vector &lits, des->stride = 1; } - FDRCompiler fc(lits, *des, make_small, grey); + auto bucketToLits = assignStringsToBuckets(lits, *des); + addIncludedInfo(lits, des->getNumBuckets(), bucketToLits); + auto proto = + ue2::make_unique(engType, move(des), lits, bucketToLits, + make_small); + return proto; +} + +unique_ptr fdrBuildProto(u8 engType, vector lits, + bool make_small, const target_t &target, + const Grey &grey) { + return fdrBuildProtoInternal(engType, lits, make_small, target, grey, + HINT_INVALID); +} + +static +bytecode_ptr fdrBuildTableInternal(const HWLMProto &proto, + const Grey &grey) { + + if (proto.teddyEng) { + return teddyBuildTable(proto, grey); + } + + FDRCompiler fc(proto.lits, proto.bucketToLits, *(proto.fdrEng), + proto.make_small, grey); return fc.build(); } -bytecode_ptr fdrBuildTable(const vector &lits, - bool make_small, const target_t &target, - const Grey &grey) { - return fdrBuildTableInternal(lits, make_small, target, grey, HINT_INVALID); +bytecode_ptr fdrBuildTable(const HWLMProto &proto, const Grey &grey) { + return fdrBuildTableInternal(proto, grey); } #if !defined(RELEASE_BUILD) -bytecode_ptr fdrBuildTableHinted(const vector &lits, - bool make_small, u32 hint, - const target_t &target, - const Grey &grey) { - return fdrBuildTableInternal(lits, make_small, target, grey, hint); +unique_ptr fdrBuildProtoHinted(u8 engType, + vector lits, + bool make_small, u32 hint, + const target_t &target, + const Grey &grey) { + return fdrBuildProtoInternal(engType, lits, make_small, target, grey, + hint); } #endif diff --git a/src/fdr/fdr_compile.h b/src/fdr/fdr_compile.h index 58047600..f0ce4925 100644 --- a/src/fdr/fdr_compile.h +++ b/src/fdr/fdr_compile.h @@ -34,6 +34,7 @@ #define FDR_COMPILE_H #include "ue2common.h" +#include "hwlm/hwlm_build.h" #include "util/bytecode_ptr.h" #include @@ -46,18 +47,23 @@ struct hwlmLiteral; struct Grey; struct target_t; -bytecode_ptr fdrBuildTable(const std::vector &lits, - bool make_small, const target_t &target, - const Grey &grey); +bytecode_ptr fdrBuildTable(const HWLMProto &proto, const Grey &grey); #if !defined(RELEASE_BUILD) - -bytecode_ptr fdrBuildTableHinted(const std::vector &lits, - bool make_small, u32 hint, - const target_t &target, const Grey &grey); - +std::unique_ptr fdrBuildProtoHinted( + u8 engType, + std::vector lits, + bool make_small, u32 hint, + const target_t &target, + const Grey &grey); #endif +std::unique_ptr fdrBuildProto( + u8 engType, + std::vector lits, + bool make_small, const target_t &target, + const Grey &grey); + /** \brief Returns size in bytes of the given FDR engine. */ size_t fdrSize(const struct FDR *fdr); diff --git a/src/fdr/fdr_compile_internal.h b/src/fdr/fdr_compile_internal.h index 756fe8e7..3879960a 100644 --- a/src/fdr/fdr_compile_internal.h +++ b/src/fdr/fdr_compile_internal.h @@ -57,10 +57,11 @@ class FDREngineDescription; struct hwlmStreamingControl; struct Grey; -bytecode_ptr setupFullConfs(const std::vector &lits, - const EngineDescription &eng, - std::map> &bucketToLits, - bool make_small); +bytecode_ptr setupFullConfs( + const std::vector &lits, + const EngineDescription &eng, + const std::map> &bucketToLits, + bool make_small); // all suffixes include an implicit max_bucket_width suffix to ensure that // we always read a full-scale flood "behind" us in terms of what's in our diff --git a/src/fdr/fdr_confirm_compile.cpp b/src/fdr/fdr_confirm_compile.cpp index a6eee4cf..c75f8d17 100644 --- a/src/fdr/fdr_confirm_compile.cpp +++ b/src/fdr/fdr_confirm_compile.cpp @@ -292,7 +292,7 @@ bytecode_ptr getFDRConfirm(const vector &lits, bytecode_ptr setupFullConfs(const vector &lits, const EngineDescription &eng, - map> &bucketToLits, + const map> &bucketToLits, bool make_small) { unique_ptr teddyDescr = getTeddyDescription(eng.getID()); @@ -300,9 +300,9 @@ setupFullConfs(const vector &lits, BC2CONF bc2Conf; u32 totalConfirmSize = 0; for (BucketIndex b = 0; b < eng.getNumBuckets(); b++) { - if (!bucketToLits[b].empty()) { + if (contains(bucketToLits, b)) { vector vl; - for (const LiteralIndex &lit_idx : bucketToLits[b]) { + for (const LiteralIndex &lit_idx : bucketToLits.at(b)) { vl.push_back(lits[lit_idx]); } diff --git a/src/fdr/fdr_confirm_runtime.h b/src/fdr/fdr_confirm_runtime.h index 86a3bfa4..067e50e2 100644 --- a/src/fdr/fdr_confirm_runtime.h +++ b/src/fdr/fdr_confirm_runtime.h @@ -29,6 +29,7 @@ #ifndef FDR_CONFIRM_RUNTIME_H #define FDR_CONFIRM_RUNTIME_H +#include "scratch.h" #include "fdr_internal.h" #include "fdr_loadval.h" #include "hwlm/hwlm.h" @@ -41,7 +42,7 @@ static really_inline void confWithBit(const struct FDRConfirm *fdrc, const struct FDR_Runtime_Args *a, size_t i, hwlmcb_rv_t *control, u32 *last_match, - u64a conf_key) { + u64a conf_key, u64a *conf, u8 bit) { assert(i < a->len); assert(i >= a->start_offset); assert(ISALIGNED(fdrc)); @@ -57,6 +58,10 @@ void confWithBit(const struct FDRConfirm *fdrc, const struct FDR_Runtime_Args *a const struct LitInfo *li = (const struct LitInfo *)((const u8 *)fdrc + start); + struct hs_scratch *scratch = a->scratch; + assert(!scratch->fdr_conf); + scratch->fdr_conf = conf; + scratch->fdr_conf_offset = bit; u8 oldNext; // initialized in loop do { assert(ISALIGNED(li)); @@ -88,11 +93,12 @@ void confWithBit(const struct FDRConfirm *fdrc, const struct FDR_Runtime_Args *a } *last_match = li->id; - *control = a->cb(i, li->id, a->scratch); + *control = a->cb(i, li->id, scratch); out: oldNext = li->next; // oldNext is either 0 or an 'adjust' value li++; } while (oldNext); + scratch->fdr_conf = NULL; } #endif diff --git a/src/fdr/teddy_compile.cpp b/src/fdr/teddy_compile.cpp index a35e5900..bb02f759 100644 --- a/src/fdr/teddy_compile.cpp +++ b/src/fdr/teddy_compile.cpp @@ -42,9 +42,11 @@ #include "teddy_engine_description.h" #include "grey.h" #include "ue2common.h" +#include "hwlm/hwlm_build.h" #include "util/alloc.h" #include "util/compare.h" #include "util/container.h" +#include "util/make_unique.h" #include "util/noncopyable.h" #include "util/popcount.h" #include "util/target_info.h" @@ -77,17 +79,18 @@ class TeddyCompiler : noncopyable { const TeddyEngineDescription ŋ const Grey &grey; const vector &lits; + map> bucketToLits; bool make_small; public: TeddyCompiler(const vector &lits_in, + map> bucketToLits_in, const TeddyEngineDescription &eng_in, bool make_small_in, const Grey &grey_in) - : eng(eng_in), grey(grey_in), lits(lits_in), make_small(make_small_in) { - } + : eng(eng_in), grey(grey_in), lits(lits_in), + bucketToLits(move(bucketToLits_in)), make_small(make_small_in) {} bytecode_ptr build(); - bool pack(map> &bucketToLits); }; class TeddySet { @@ -216,8 +219,10 @@ public: } }; -bool TeddyCompiler::pack(map> &bucketToLits) { +static +bool pack(const vector &lits, + const TeddyEngineDescription &eng, + map> &bucketToLits) { set sts; for (u32 i = 0; i < lits.size(); i++) { @@ -473,30 +478,6 @@ void fillReinforcedTable(const map TeddyCompiler::build() { - assert(eng.numMasks <= MAX_NUM_MASKS); - - if (lits.size() > eng.getNumBuckets() * TEDDY_BUCKET_LOAD) { - DEBUG_PRINTF("too many literals: %zu\n", lits.size()); - return nullptr; - } - -#ifdef TEDDY_DEBUG - for (size_t i = 0; i < lits.size(); i++) { - printf("lit %zu (len = %zu, %s) is ", i, lits[i].s.size(), - lits[i].nocase ? "caseless" : "caseful"); - for (size_t j = 0; j < lits[i].s.size(); j++) { - printf("%02x", ((u32)lits[i].s[j])&0xff); - } - printf("\n"); - } -#endif - - map> bucketToLits; - if (!pack(bucketToLits)) { - DEBUG_PRINTF("more lits (%zu) than buckets (%u), can't pack.\n", - lits.size(), eng.getNumBuckets()); - return nullptr; - } u32 maskWidth = eng.getNumBuckets() / 8; size_t headerSize = sizeof(Teddy); @@ -565,12 +546,49 @@ bytecode_ptr TeddyCompiler::build() { return fdr; } + +static +bool assignStringsToBuckets( + const vector &lits, + TeddyEngineDescription &eng, + map> &bucketToLits) { + assert(eng.numMasks <= MAX_NUM_MASKS); + if (lits.size() > eng.getNumBuckets() * TEDDY_BUCKET_LOAD) { + DEBUG_PRINTF("too many literals: %zu\n", lits.size()); + return false; + } + +#ifdef TEDDY_DEBUG + for (size_t i = 0; i < lits.size(); i++) { + printf("lit %zu (len = %zu, %s) is ", i, lits[i].s.size(), + lits[i].nocase ? "caseless" : "caseful"); + for (size_t j = 0; j < lits[i].s.size(); j++) { + printf("%02x", ((u32)lits[i].s[j])&0xff); + } + printf("\n"); + } +#endif + + if (!pack(lits, eng, bucketToLits)) { + DEBUG_PRINTF("more lits (%zu) than buckets (%u), can't pack.\n", + lits.size(), eng.getNumBuckets()); + return false; + } + return true; +} + } // namespace -bytecode_ptr teddyBuildTableHinted(const vector &lits, - bool make_small, u32 hint, - const target_t &target, - const Grey &grey) { +bytecode_ptr teddyBuildTable(const HWLMProto &proto, const Grey &grey) { + TeddyCompiler tc(proto.lits, proto.bucketToLits, *(proto.teddyEng), + proto.make_small, grey); + return tc.build(); +} + + +unique_ptr teddyBuildProtoHinted( + u8 engType, const vector &lits, + bool make_small, u32 hint, const target_t &target) { unique_ptr des; if (hint == HINT_INVALID) { des = chooseTeddyEngine(target, lits); @@ -580,8 +598,14 @@ bytecode_ptr teddyBuildTableHinted(const vector &lits, if (!des) { return nullptr; } - TeddyCompiler tc(lits, *des, make_small, grey); - return tc.build(); + + map> bucketToLits; + if (!assignStringsToBuckets(lits, *des, bucketToLits)) { + return nullptr; + } + + return ue2::make_unique(engType, move(des), lits, + bucketToLits, make_small); } } // namespace ue2 diff --git a/src/fdr/teddy_compile.h b/src/fdr/teddy_compile.h index 5ff4d839..ec251310 100644 --- a/src/fdr/teddy_compile.h +++ b/src/fdr/teddy_compile.h @@ -35,6 +35,7 @@ #define TEDDY_COMPILE_H #include "ue2common.h" +#include "hwlm/hwlm_build.h" #include "util/bytecode_ptr.h" #include @@ -46,12 +47,13 @@ namespace ue2 { struct Grey; struct hwlmLiteral; struct target_t; +struct TeddyEngineDescription; -bytecode_ptr teddyBuildTableHinted(const std::vector &lits, - bool make_small, u32 hint, - const target_t &target, - const Grey &grey); +bytecode_ptr teddyBuildTable(const HWLMProto &proto, const Grey &grey); +std::unique_ptr teddyBuildProtoHinted( + u8 engType, const std::vector &lits, + bool make_small, u32 hint, const target_t &target); } // namespace ue2 #endif // TEDDY_COMPILE_H diff --git a/src/fdr/teddy_runtime_common.h b/src/fdr/teddy_runtime_common.h index 6b809cce..5332423e 100644 --- a/src/fdr/teddy_runtime_common.h +++ b/src/fdr/teddy_runtime_common.h @@ -419,9 +419,10 @@ void do_confWithBit_teddy(TEDDY_CONF_TYPE *conf, u8 bucket, u8 offset, if (!(fdrc->groups & *control)) { continue; } + u64a tmp = 0; u64a confVal = getConfVal(a, ptr, byte, reason); confWithBit(fdrc, a, ptr - a->buf + byte, control, - last_match, confVal); + last_match, confVal, &tmp, 0); } while (unlikely(*conf)); } diff --git a/src/hwlm/hwlm_build.cpp b/src/hwlm/hwlm_build.cpp index c2db5480..1b332815 100644 --- a/src/hwlm/hwlm_build.cpp +++ b/src/hwlm/hwlm_build.cpp @@ -41,8 +41,12 @@ #include "scratch.h" #include "ue2common.h" #include "fdr/fdr_compile.h" +#include "fdr/fdr_compile_internal.h" +#include "fdr/fdr_engine_description.h" +#include "fdr/teddy_engine_description.h" #include "util/compile_context.h" #include "util/compile_error.h" +#include "util/make_unique.h" #include "util/ue2string.h" #include @@ -53,6 +57,28 @@ using namespace std; namespace ue2 { +HWLMProto::HWLMProto(u8 engType_in, vector lits_in) + : engType(engType_in), lits(move(lits_in)) {} + +HWLMProto::HWLMProto(u8 engType_in, + unique_ptr eng_in, + vector lits_in, + map> bucketToLits_in, + bool make_small_in) + : engType(engType_in), fdrEng(move(eng_in)), lits(move(lits_in)), + bucketToLits(move(bucketToLits_in)), make_small(make_small_in) {} + +HWLMProto::HWLMProto(u8 engType_in, + unique_ptr eng_in, + vector lits_in, + map> bucketToLits_in, + bool make_small_in) + : engType(engType_in), teddyEng(move(eng_in)), + lits(move(lits_in)), + bucketToLits(move(bucketToLits_in)), make_small(make_small_in) {} + +HWLMProto::~HWLMProto() {} + static void dumpLits(UNUSED const vector &lits) { #ifdef DEBUG @@ -92,9 +118,52 @@ bool isNoodleable(const vector &lits, return true; } -bytecode_ptr hwlmBuild(const vector &lits, bool make_small, - const CompileContext &cc, +bytecode_ptr hwlmBuild(const HWLMProto &proto, const CompileContext &cc, UNUSED hwlm_group_t expected_groups) { + size_t engSize = 0; + shared_ptr eng; + + const auto &lits = proto.lits; + DEBUG_PRINTF("building table with %zu strings\n", lits.size()); + + if (proto.engType == HWLM_ENGINE_NOOD) { + DEBUG_PRINTF("build noodle table\n"); + const hwlmLiteral &lit = lits.front(); + auto noodle = noodBuildTable(lit); + if (noodle) { + engSize = noodle.size(); + } + eng = move(noodle); + } else { + DEBUG_PRINTF("building a new deal\n"); + auto fdr = fdrBuildTable(proto, cc.grey); + if (fdr) { + engSize = fdr.size(); + } + eng = move(fdr); + } + + if (!eng) { + return nullptr; + } + + assert(engSize); + if (engSize > cc.grey.limitLiteralMatcherSize) { + throw ResourceLimitError(); + } + + const size_t hwlm_len = ROUNDUP_CL(sizeof(HWLM)) + engSize; + auto h = make_zeroed_bytecode_ptr(hwlm_len, 64); + + h->type = proto.engType; + memcpy(HWLM_DATA(h.get()), eng.get(), engSize); + + return h; +} + +unique_ptr +hwlmBuildProto(vector &lits, bool make_small, + const CompileContext &cc) { assert(!lits.empty()); dumpLits(lits); @@ -124,9 +193,7 @@ bytecode_ptr hwlmBuild(const vector &lits, bool make_small, } } - u8 engType = 0; - size_t engSize = 0; - shared_ptr eng; + unique_ptr proto; DEBUG_PRINTF("building table with %zu strings\n", lits.size()); @@ -134,39 +201,17 @@ bytecode_ptr hwlmBuild(const vector &lits, bool make_small, if (isNoodleable(lits, cc)) { DEBUG_PRINTF("build noodle table\n"); - engType = HWLM_ENGINE_NOOD; - const hwlmLiteral &lit = lits.front(); - auto noodle = noodBuildTable(lit); - if (noodle) { - engSize = noodle.size(); - } - eng = move(noodle); + proto = ue2::make_unique(HWLM_ENGINE_NOOD, lits); } else { DEBUG_PRINTF("building a new deal\n"); - engType = HWLM_ENGINE_FDR; - auto fdr = fdrBuildTable(lits, make_small, cc.target_info, cc.grey); - if (fdr) { - engSize = fdr.size(); + proto = fdrBuildProto(HWLM_ENGINE_FDR, lits, make_small, + cc.target_info, cc.grey); + if (!proto) { + return nullptr; } - eng = move(fdr); } - if (!eng) { - return nullptr; - } - - assert(engSize); - if (engSize > cc.grey.limitLiteralMatcherSize) { - throw ResourceLimitError(); - } - - const size_t hwlm_len = ROUNDUP_CL(sizeof(HWLM)) + engSize; - auto h = make_zeroed_bytecode_ptr(hwlm_len, 64); - - h->type = engType; - memcpy(HWLM_DATA(h.get()), eng.get(), engSize); - - return h; + return proto; } size_t hwlmSize(const HWLM *h) { diff --git a/src/hwlm/hwlm_build.h b/src/hwlm/hwlm_build.h index f2691496..4aefc364 100644 --- a/src/hwlm/hwlm_build.h +++ b/src/hwlm/hwlm_build.h @@ -34,9 +34,11 @@ #define HWLM_BUILD_H #include "hwlm.h" +#include "hwlm_literal.h" #include "ue2common.h" #include "util/bytecode_ptr.h" +#include #include #include @@ -44,15 +46,62 @@ struct HWLM; namespace ue2 { +class FDREngineDescription; +class TeddyEngineDescription; struct CompileContext; struct Grey; -struct hwlmLiteral; + +/** \brief Class representing a literal matcher prototype. */ +struct HWLMProto { + /** + * \brief Engine type to distinguish noodle from FDR and Teddy. + */ + u8 engType; + + /** + * \brief FDR engine description. + */ + std::unique_ptr fdrEng; + + /** + * \brief Teddy engine description. + */ + std::unique_ptr teddyEng; + + /** + * \brief HWLM literals passed from Rose. + */ + std::vector lits; + + /** + * \brief Bucket assignment info in FDR and Teddy + */ + std::map> bucketToLits; + + /** + * \brief Flag to optimise matcher for small size from Rose. + */ + bool make_small; + + HWLMProto(u8 engType_in, std::vector lits_in); + + HWLMProto(u8 engType_in, std::unique_ptr eng_in, + std::vector lits_in, + std::map> bucketToLits_in, + bool make_small_in); + + HWLMProto(u8 engType_in, std::unique_ptr eng_in, + std::vector lits_in, + std::map> bucketToLits_in, + bool make_small_in); + + ~HWLMProto(); +}; /** \brief Build an \ref HWLM literal matcher runtime structure for a group of * literals. * - * \param lits The group of literals. - * \param make_small Optimise matcher for small size. + * \param proto Literal matcher prototype. * \param cc Compile context. * \param expected_groups FIXME: document me! * @@ -60,10 +109,13 @@ struct hwlmLiteral; * may result in a nullptr return value, or a std::bad_alloc exception being * thrown. */ -bytecode_ptr hwlmBuild(const std::vector &lits, - bool make_small, const CompileContext &cc, +bytecode_ptr hwlmBuild(const HWLMProto &proto, const CompileContext &cc, hwlm_group_t expected_groups = HWLM_ALL_GROUPS); +std::unique_ptr +hwlmBuildProto(std::vector &lits, bool make_small, + const CompileContext &cc); + /** * Returns an estimate of the number of repeated characters on the end of a * literal that will make a literal set of size \a numLiterals suffer diff --git a/src/hwlm/hwlm_literal.h b/src/hwlm/hwlm_literal.h index 9ae7744d..08510fb0 100644 --- a/src/hwlm/hwlm_literal.h +++ b/src/hwlm/hwlm_literal.h @@ -45,6 +45,8 @@ namespace ue2 { /** \brief Max length of the hwlmLiteral::msk and hwlmLiteral::cmp vectors. */ #define HWLM_MASKLEN 8 +#define INVALID_LIT_ID ~0U + /** \brief Class representing a literal, fed to \ref hwlmBuild. */ struct hwlmLiteral { std::string s; //!< \brief The literal itself. @@ -64,6 +66,21 @@ struct hwlmLiteral { * can be quashed by the literal matcher. */ bool noruns; + /** \brief included literal id. */ + u32 included_id = INVALID_LIT_ID; + + /** \brief Squash mask for FDR's confirm mask for included literals. + * + * In FDR confirm, if we have included literal in another bucket, + * we can use this mask to squash the bit for the bucket in FDR confirm + * mask and then run programs of included literal directly and avoid + * confirm work. + * + * This value is calculated in FDR compile code once bucket assignment is + * completed + */ + u8 squash = 0; + /** \brief Set of groups that literal belongs to. * * Use \ref HWLM_ALL_GROUPS for a literal that could match regardless of diff --git a/src/rose/program_runtime.h b/src/rose/program_runtime.h index 83a34a39..ab0934de 100644 --- a/src/rose/program_runtime.h +++ b/src/rose/program_runtime.h @@ -2570,6 +2570,23 @@ hwlmcb_rv_t roseRunProgram_i(const struct RoseEngine *t, } } PROGRAM_NEXT_INSTRUCTION + + PROGRAM_CASE(INCLUDED_JUMP) { + if (scratch->fdr_conf) { + // squash the bucket of included literal + u8 shift = scratch->fdr_conf_offset & ~7U; + u64a mask = ((~(u64a)ri->squash) << shift); + *(scratch->fdr_conf) &= mask; + + pc = getByOffset(t, ri->child_offset); + pc_base = pc; + programOffset = (const u8 *)pc_base -(const u8 *)t; + DEBUG_PRINTF("pc_base %p pc %p child_offset %u\n", + pc_base, pc, ri->child_offset); + continue; + } + } + PROGRAM_NEXT_INSTRUCTION } } diff --git a/src/rose/rose_build_bytecode.cpp b/src/rose/rose_build_bytecode.cpp index 4d0793bf..a41f0322 100644 --- a/src/rose/rose_build_bytecode.cpp +++ b/src/rose/rose_build_bytecode.cpp @@ -49,6 +49,7 @@ #include "rose_internal.h" #include "rose_program.h" #include "hwlm/hwlm.h" /* engine types */ +#include "hwlm/hwlm_build.h" #include "hwlm/hwlm_literal.h" #include "nfa/castlecompile.h" #include "nfa/goughcompile.h" @@ -2803,7 +2804,7 @@ vector groupByFragment(const RoseBuildImpl &build) { auto groups = info.group_mask; if (lit.s.length() < ROSE_SHORT_LITERAL_LEN_MAX) { - fragments.emplace_back(frag_id, groups, lit_id); + fragments.emplace_back(frag_id, lit.s, groups, lit_id); frag_id++; continue; } @@ -2816,10 +2817,11 @@ vector groupByFragment(const RoseBuildImpl &build) { } for (auto &m : frag_info) { + auto &lit = m.first; auto &fi = m.second; DEBUG_PRINTF("frag %s -> ids: %s\n", dumpString(m.first.s).c_str(), as_string_list(fi.lit_ids).c_str()); - fragments.emplace_back(frag_id, fi.groups, move(fi.lit_ids)); + fragments.emplace_back(frag_id, lit.s, fi.groups, move(fi.lit_ids)); frag_id++; assert(frag_id == fragments.size()); } @@ -2827,33 +2829,181 @@ vector groupByFragment(const RoseBuildImpl &build) { return fragments; } +static +void buildIncludedIdMap(unordered_map> &includedIdMap, + const LitProto *litProto) { + if (!litProto) { + return; + } + const auto &proto = *litProto->hwlmProto; + for (const auto &lit : proto.lits) { + if (lit.included_id != INVALID_LIT_ID) { + includedIdMap[lit.id] = make_pair(lit.included_id, lit.squash); + } + } +} + +static +void findInclusionGroups(vector &fragments, + LitProto *fproto, LitProto *drproto, + LitProto *eproto, LitProto *sbproto) { + unordered_map> includedIdMap; + unordered_map> includedDelayIdMap; + buildIncludedIdMap(includedIdMap, fproto); + buildIncludedIdMap(includedDelayIdMap, drproto); + buildIncludedIdMap(includedIdMap, eproto); + buildIncludedIdMap(includedIdMap, sbproto); + + size_t fragNum = fragments.size(); + vector candidates; + for (size_t j = 0; j < fragNum; j++) { + DEBUG_PRINTF("frag id %lu\n", j); + u32 id = j; + if (contains(includedIdMap, id) || + contains(includedDelayIdMap, id)) { + candidates.push_back(j); + DEBUG_PRINTF("find candidate\n"); + } + } + + for (const auto &c : candidates) { + auto &frag = fragments[c]; + u32 id = c; + if (contains(includedIdMap, id)) { + const auto &childId = includedIdMap[id]; + frag.included_frag_id = childId.first; + frag.squash = childId.second; + DEBUG_PRINTF("frag id %u child frag id %u\n", c, + frag.included_frag_id); + } + + if (contains(includedDelayIdMap, id)) { + const auto &childId = includedDelayIdMap[id]; + frag.included_delay_frag_id = childId.first; + frag.delay_squash = childId.second; + + DEBUG_PRINTF("delay frag id %u child frag id %u\n", c, + frag.included_delay_frag_id); + } + } +} + +static +void buildFragmentPrograms(const RoseBuildImpl &build, + vector &fragments, + build_context &bc, ProgramBuild &prog_build, + const map> &lit_edge_map) { + // Sort fragments based on literal length and case info to build + // included literal programs before their parent programs. + vector ordered_fragments(fragments); + stable_sort(begin(ordered_fragments), end(ordered_fragments), + [](const LitFragment &a, const LitFragment &b) { + auto len1 = a.s.length(); + auto caseful1 = !a.s.any_nocase(); + auto len2 = b.s.length(); + auto caseful2 = !b.s.any_nocase(); + return tie(len1, caseful1) < tie(len2, caseful2); + }); + + for (auto &frag : ordered_fragments) { + auto &pfrag = fragments[frag.fragment_id]; + DEBUG_PRINTF("frag_id=%u, lit_ids=[%s]\n", pfrag.fragment_id, + as_string_list(pfrag.lit_ids).c_str()); + + auto lit_prog = makeFragmentProgram(build, bc, prog_build, + pfrag.lit_ids, lit_edge_map); + if (pfrag.included_frag_id != INVALID_FRAG_ID && + !lit_prog.empty()) { + auto &cfrag = fragments[pfrag.included_frag_id]; + assert(pfrag.s.length() >= cfrag.s.length() && + !pfrag.s.any_nocase() >= !cfrag.s.any_nocase()); + u32 child_offset = cfrag.lit_program_offset; + DEBUG_PRINTF("child %u offset %u\n", cfrag.fragment_id, + child_offset); + addIncludedJumpProgram(lit_prog, child_offset, pfrag.squash); + } + pfrag.lit_program_offset = writeProgram(bc, move(lit_prog)); + + // We only do delayed rebuild in streaming mode. + if (!build.cc.streaming) { + continue; + } + + auto rebuild_prog = makeDelayRebuildProgram(build, prog_build, + pfrag.lit_ids); + if (pfrag.included_delay_frag_id != INVALID_FRAG_ID && + !rebuild_prog.empty()) { + auto &cfrag = fragments[pfrag.included_delay_frag_id]; + assert(pfrag.s.length() >= cfrag.s.length() && + !pfrag.s.any_nocase() >= !cfrag.s.any_nocase()); + u32 child_offset = cfrag.delay_program_offset; + DEBUG_PRINTF("child %u offset %u\n", cfrag.fragment_id, + child_offset); + addIncludedJumpProgram(rebuild_prog, child_offset, + pfrag.delay_squash); + } + pfrag.delay_program_offset = writeProgram(bc, move(rebuild_prog)); + } +} + +static +void updateLitProtoProgramOffset(vector &fragments, + LitProto &litProto, bool delay) { + auto &proto = *litProto.hwlmProto; + for (auto &lit : proto.lits) { + auto fragId = lit.id; + auto &frag = fragments[fragId]; + if (delay) { + DEBUG_PRINTF("delay_program_offset:%u\n", + frag.delay_program_offset); + lit.id = frag.delay_program_offset; + } else { + DEBUG_PRINTF("lit_program_offset:%u\n", + frag.lit_program_offset); + lit.id = frag.lit_program_offset; + } + } +} + +static +void updateLitProgramOffset(vector &fragments, + LitProto *fproto, LitProto *drproto, + LitProto *eproto, LitProto *sbproto) { + if (fproto) { + updateLitProtoProgramOffset(fragments, *fproto, false); + } + + if (drproto) { + updateLitProtoProgramOffset(fragments, *drproto, true); + } + + if (eproto) { + updateLitProtoProgramOffset(fragments, *eproto, false); + } + + if (sbproto) { + updateLitProtoProgramOffset(fragments, *sbproto, false); + } +} + /** * \brief Build the interpreter programs for each literal. */ static void buildLiteralPrograms(const RoseBuildImpl &build, vector &fragments, build_context &bc, - ProgramBuild &prog_build) { + ProgramBuild &prog_build, LitProto *fproto, + LitProto *drproto, LitProto *eproto, + LitProto *sbproto) { DEBUG_PRINTF("%zu fragments\n", fragments.size()); auto lit_edge_map = findEdgesByLiteral(build); - for (auto &frag : fragments) { - DEBUG_PRINTF("frag_id=%u, lit_ids=[%s]\n", frag.fragment_id, - as_string_list(frag.lit_ids).c_str()); + findInclusionGroups(fragments, fproto, drproto, eproto, sbproto); - auto lit_prog = makeFragmentProgram(build, bc, prog_build, frag.lit_ids, - lit_edge_map); - frag.lit_program_offset = writeProgram(bc, move(lit_prog)); + buildFragmentPrograms(build, fragments, bc, prog_build, lit_edge_map); - // We only do delayed rebuild in streaming mode. - if (!build.cc.streaming) { - continue; - } - - auto rebuild_prog = makeDelayRebuildProgram(build, prog_build, - frag.lit_ids); - frag.delay_program_offset = writeProgram(bc, move(rebuild_prog)); - } + // update literal program offsets for literal matcher prototypes + updateLitProgramOffset(fragments, fproto, drproto, eproto, sbproto); } /** @@ -3470,7 +3620,24 @@ bytecode_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { tie(proto.delayProgramOffset, proto.delay_count) = writeDelayPrograms(*this, fragments, bc, prog_build); - buildLiteralPrograms(*this, fragments, bc, prog_build); + // Build floating HWLM matcher prototype. + rose_group fgroups = 0; + auto fproto = buildFloatingMatcherProto(*this, fragments, + longLitLengthThreshold, + &fgroups, &historyRequired); + + // Build delay rebuild HWLM matcher prototype. + auto drproto = buildDelayRebuildMatcherProto(*this, fragments, + longLitLengthThreshold); + + // Build EOD-anchored HWLM matcher prototype. + auto eproto = buildEodAnchoredMatcherProto(*this, fragments); + + // Build small-block HWLM matcher prototype. + auto sbproto = buildSmallBlockMatcherProto(*this, fragments); + + buildLiteralPrograms(*this, fragments, bc, prog_build, fproto.get(), + drproto.get(), eproto.get(), sbproto.get()); auto eod_prog = makeEodProgram(*this, bc, prog_build, eodNfaIterOffset); proto.eodProgramOffset = writeProgram(bc, move(eod_prog)); @@ -3497,29 +3664,26 @@ bytecode_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { } // Build floating HWLM matcher. - rose_group fgroups = 0; - auto ftable = buildFloatingMatcher(*this, fragments, longLitLengthThreshold, - &fgroups, &historyRequired); + auto ftable = buildHWLMMatcher(*this, fproto.get()); if (ftable) { proto.fmatcherOffset = bc.engine_blob.add(ftable); bc.resources.has_floating = true; } // Build delay rebuild HWLM matcher. - auto drtable = buildDelayRebuildMatcher(*this, fragments, - longLitLengthThreshold); + auto drtable = buildHWLMMatcher(*this, drproto.get()); if (drtable) { proto.drmatcherOffset = bc.engine_blob.add(drtable); } // Build EOD-anchored HWLM matcher. - auto etable = buildEodAnchoredMatcher(*this, fragments); + auto etable = buildHWLMMatcher(*this, eproto.get()); if (etable) { proto.ematcherOffset = bc.engine_blob.add(etable); } // Build small-block HWLM matcher. - auto sbtable = buildSmallBlockMatcher(*this, fragments); + auto sbtable = buildHWLMMatcher(*this, sbproto.get()); if (sbtable) { proto.sbmatcherOffset = bc.engine_blob.add(sbtable); } diff --git a/src/rose/rose_build_dump.cpp b/src/rose/rose_build_dump.cpp index 5e9f95f2..e98308ac 100644 --- a/src/rose/rose_build_dump.cpp +++ b/src/rose/rose_build_dump.cpp @@ -1463,6 +1463,12 @@ void dumpProgram(ofstream &os, const RoseEngine *t, const char *pc) { } PROGRAM_NEXT_INSTRUCTION + PROGRAM_CASE(INCLUDED_JUMP) { + os << " child_offset " << ri->child_offset << endl; + os << " squash " << ri->squash << endl; + } + PROGRAM_NEXT_INSTRUCTION + default: os << " UNKNOWN (code " << int{code} << ")" << endl; os << " " << endl; diff --git a/src/rose/rose_build_instructions.cpp b/src/rose/rose_build_instructions.cpp index b00c36be..8af08298 100644 --- a/src/rose/rose_build_instructions.cpp +++ b/src/rose/rose_build_instructions.cpp @@ -636,4 +636,12 @@ void RoseInstrCheckMultipathShufti64::write(void *dest, RoseEngineBlob &blob, inst->fail_jump = calc_jump(offset_map, this, target); } +void RoseInstrIncludedJump::write(void *dest, RoseEngineBlob &blob, + const OffsetMap &offset_map) const { + RoseInstrBase::write(dest, blob, offset_map); + auto *inst = static_cast(dest); + inst->child_offset = child_offset; + inst->squash = squash; +} + } diff --git a/src/rose/rose_build_instructions.h b/src/rose/rose_build_instructions.h index 025f6a67..3bc3266b 100644 --- a/src/rose/rose_build_instructions.h +++ b/src/rose/rose_build_instructions.h @@ -2121,6 +2121,34 @@ public: } }; +class RoseInstrIncludedJump + : public RoseInstrBaseNoTargets { +public: + u32 child_offset; + u8 squash; + + RoseInstrIncludedJump(u32 child_offset_in, u8 squash_in) + : child_offset(child_offset_in), squash(squash_in) {} + + bool operator==(const RoseInstrIncludedJump &ri) const { + return child_offset == ri.child_offset && squash == ri.squash; + } + + size_t hash() const override { + return hash_all(static_cast(opcode), child_offset, squash); + } + + void write(void *dest, RoseEngineBlob &blob, + const OffsetMap &offset_map) const override; + + bool equiv_to(const RoseInstrIncludedJump &ri, const OffsetMap &, + const OffsetMap &) const { + return child_offset == ri.child_offset && squash == ri.squash; + } +}; + class RoseInstrEnd : public RoseInstrBaseTrivial { diff --git a/src/rose/rose_build_matchers.cpp b/src/rose/rose_build_matchers.cpp index 57269747..2c302a85 100644 --- a/src/rose/rose_build_matchers.cpp +++ b/src/rose/rose_build_matchers.cpp @@ -46,6 +46,7 @@ #include "util/compile_context.h" #include "util/compile_error.h" #include "util/dump_charclass.h" +#include "util/make_unique.h" #include "util/report.h" #include "util/report_manager.h" #include "util/verify_types.h" @@ -699,8 +700,7 @@ struct MatcherProto { static void addFragmentLiteral(const RoseBuildImpl &build, MatcherProto &mp, - const LitFragment &f, u32 id, bool delay_rebuild, - size_t max_len) { + const LitFragment &f, u32 id, size_t max_len) { const rose_literal_id &lit = build.literals.at(id); DEBUG_PRINTF("lit='%s' (len %zu)\n", dumpString(lit.s).c_str(), @@ -737,12 +737,10 @@ void addFragmentLiteral(const RoseBuildImpl &build, MatcherProto &mp, return; } - u32 prog_offset = - delay_rebuild ? f.delay_program_offset : f.lit_program_offset; const auto &groups = f.groups; - mp.lits.emplace_back(move(s_final), nocase, noruns, prog_offset, groups, - msk, cmp); + mp.lits.emplace_back(move(s_final), nocase, noruns, f.fragment_id, + groups, msk, cmp); } static @@ -837,8 +835,7 @@ MatcherProto makeMatcherProto(const RoseBuildImpl &build, } // Build our fragment (for the HWLM matcher) from the first literal. - addFragmentLiteral(build, mp, f, used_lit_ids.front(), delay_rebuild, - max_len); + addFragmentLiteral(build, mp, f, used_lit_ids.front(), max_len); for (u32 id : used_lit_ids) { const rose_literal_id &lit = build.literals.at(id); @@ -876,8 +873,8 @@ void MatcherProto::insert(const MatcherProto &a) { } static -void buildAccel(const RoseBuildImpl &build, const MatcherProto &mp, - HWLM &hwlm) { +void buildAccel(const RoseBuildImpl &build, + const vector &accel_lits, HWLM &hwlm) { if (!build.cc.grey.hamsterAccelForward) { return; } @@ -886,49 +883,68 @@ void buildAccel(const RoseBuildImpl &build, const MatcherProto &mp, return; } - buildForwardAccel(&hwlm, mp.accel_lits, build.getInitialGroups()); + buildForwardAccel(&hwlm, accel_lits, build.getInitialGroups()); } -bytecode_ptr buildFloatingMatcher(const RoseBuildImpl &build, - const vector &fragments, - size_t longLitLengthThreshold, - rose_group *fgroups, - size_t *historyRequired) { - *fgroups = 0; - - auto mp = makeMatcherProto(build, fragments, ROSE_FLOATING, false, - longLitLengthThreshold); - if (mp.lits.empty()) { - DEBUG_PRINTF("empty floating matcher\n"); +bytecode_ptr +buildHWLMMatcher(const RoseBuildImpl &build, LitProto *litProto) { + if (!litProto) { return nullptr; } - dumpMatcherLiterals(mp.lits, "floating", build.cc.grey); - - for (const hwlmLiteral &lit : mp.lits) { - *fgroups |= lit.groups; - } - - auto hwlm = hwlmBuild(mp.lits, false, build.cc, build.getInitialGroups()); + auto hwlm = hwlmBuild(*litProto->hwlmProto, build.cc, + build.getInitialGroups()); if (!hwlm) { throw CompileError("Unable to generate bytecode."); } - buildAccel(build, mp, *hwlm); + buildAccel(build, litProto->accel_lits, *hwlm); - if (build.cc.streaming) { - DEBUG_PRINTF("history_required=%zu\n", mp.history_required); - assert(mp.history_required <= build.cc.grey.maxHistoryAvailable); - *historyRequired = max(*historyRequired, mp.history_required); - } - - DEBUG_PRINTF("built floating literal table size %zu bytes\n", hwlm.size()); + DEBUG_PRINTF("built eod-anchored literal table size %zu bytes\n", + hwlm.size()); return hwlm; } -bytecode_ptr -buildDelayRebuildMatcher(const RoseBuildImpl &build, - const vector &fragments, - size_t longLitLengthThreshold) { +unique_ptr +buildFloatingMatcherProto(const RoseBuildImpl &build, + const vector &fragments, + size_t longLitLengthThreshold, + rose_group *fgroups, + size_t *historyRequired) { + DEBUG_PRINTF("Floating literal matcher\n"); + *fgroups = 0; + + auto mp = makeMatcherProto(build, fragments, ROSE_FLOATING, false, + longLitLengthThreshold); + if (mp.lits.empty()) { + DEBUG_PRINTF("empty floating matcher\n"); + return nullptr; + } + dumpMatcherLiterals(mp.lits, "floating", build.cc.grey); + + for (const hwlmLiteral &lit : mp.lits) { + *fgroups |= lit.groups; + } + + if (build.cc.streaming) { + DEBUG_PRINTF("history_required=%zu\n", mp.history_required); + assert(mp.history_required <= build.cc.grey.maxHistoryAvailable); + *historyRequired = max(*historyRequired, mp.history_required); + } + + auto proto = hwlmBuildProto(mp.lits, false, build.cc); + + if (!proto) { + throw CompileError("Unable to generate literal matcher proto."); + } + + return ue2::make_unique(move(proto), mp.accel_lits); +} + +unique_ptr +buildDelayRebuildMatcherProto(const RoseBuildImpl &build, + const vector &fragments, + size_t longLitLengthThreshold) { + DEBUG_PRINTF("Delay literal matcher\n"); if (!build.cc.streaming) { DEBUG_PRINTF("not streaming\n"); return nullptr; @@ -942,20 +958,20 @@ buildDelayRebuildMatcher(const RoseBuildImpl &build, } dumpMatcherLiterals(mp.lits, "delay_rebuild", build.cc.grey); - auto hwlm = hwlmBuild(mp.lits, false, build.cc, build.getInitialGroups()); - if (!hwlm) { - throw CompileError("Unable to generate bytecode."); + + auto proto = hwlmBuildProto(mp.lits, false, build.cc); + + if (!proto) { + throw CompileError("Unable to generate literal matcher proto."); } - buildAccel(build, mp, *hwlm); - - DEBUG_PRINTF("built delay rebuild table size %zu bytes\n", hwlm.size()); - return hwlm; + return ue2::make_unique(move(proto), mp.accel_lits); } -bytecode_ptr -buildSmallBlockMatcher(const RoseBuildImpl &build, - const vector &fragments) { +unique_ptr +buildSmallBlockMatcherProto(const RoseBuildImpl &build, + const vector &fragments) { + DEBUG_PRINTF("Small block literal matcher\n"); if (build.cc.streaming) { DEBUG_PRINTF("streaming mode\n"); return nullptr; @@ -1000,21 +1016,19 @@ buildSmallBlockMatcher(const RoseBuildImpl &build, return nullptr; } - auto hwlm = hwlmBuild(mp.lits, true, build.cc, build.getInitialGroups()); - if (!hwlm) { - throw CompileError("Unable to generate bytecode."); + auto proto = hwlmBuildProto(mp.lits, false, build.cc); + + if (!proto) { + throw CompileError("Unable to generate literal matcher proto."); } - buildAccel(build, mp, *hwlm); - - DEBUG_PRINTF("built small block literal table size %zu bytes\n", - hwlm.size()); - return hwlm; + return ue2::make_unique(move(proto), mp.accel_lits); } -bytecode_ptr -buildEodAnchoredMatcher(const RoseBuildImpl &build, - const vector &fragments) { +unique_ptr +buildEodAnchoredMatcherProto(const RoseBuildImpl &build, + const vector &fragments) { + DEBUG_PRINTF("Eod anchored literal matcher\n"); auto mp = makeMatcherProto(build, fragments, ROSE_EOD_ANCHORED, false, build.ematcher_region_size); @@ -1027,16 +1041,13 @@ buildEodAnchoredMatcher(const RoseBuildImpl &build, assert(build.ematcher_region_size); - auto hwlm = hwlmBuild(mp.lits, true, build.cc, build.getInitialGroups()); - if (!hwlm) { - throw CompileError("Unable to generate bytecode."); + auto proto = hwlmBuildProto(mp.lits, false, build.cc); + + if (!proto) { + throw CompileError("Unable to generate literal matcher proto."); } - buildAccel(build, mp, *hwlm); - - DEBUG_PRINTF("built eod-anchored literal table size %zu bytes\n", - hwlm.size()); - return hwlm; + return ue2::make_unique(move(proto), mp.accel_lits); } } // namespace ue2 diff --git a/src/rose/rose_build_matchers.h b/src/rose/rose_build_matchers.h index 2b1afc8c..9668ebc9 100644 --- a/src/rose/rose_build_matchers.h +++ b/src/rose/rose_build_matchers.h @@ -35,7 +35,10 @@ #define ROSE_BUILD_MATCHERS_H #include "rose_build_impl.h" +#include "rose_build_lit_accel.h" +#include "hwlm/hwlm_build.h" #include "util/bytecode_ptr.h" +#include "util/ue2string.h" #include @@ -44,38 +47,80 @@ struct HWLM; namespace ue2 { +static constexpr u32 INVALID_FRAG_ID = ~0U; + struct LitFragment { - LitFragment(u32 fragment_id_in, rose_group groups_in, u32 lit_id) - : fragment_id(fragment_id_in), groups(groups_in), lit_ids({lit_id}) {} - LitFragment(u32 fragment_id_in, rose_group groups_in, - std::vector lit_ids_in) - : fragment_id(fragment_id_in), groups(groups_in), - lit_ids(std::move(lit_ids_in)) {} + LitFragment(u32 fragment_id_in, ue2_literal s_in, + rose_group groups_in, u32 lit_id) + : fragment_id(fragment_id_in), s(s_in), groups(groups_in), + lit_ids({lit_id}) {} + LitFragment(u32 fragment_id_in, ue2_literal s_in, + rose_group groups_in, std::vector lit_ids_in) + : fragment_id(fragment_id_in), s(s_in), groups(groups_in), + lit_ids(std::move(lit_ids_in)) {} u32 fragment_id; + + /** + * \brief literal fragment. + */ + ue2_literal s; + + /** + * \brief FDR confirm squash mask for included literals. + */ + u8 squash; + + /** + * \brief FDR confirm squash mask for included literals (Delayed + * literals only). + */ + u8 delay_squash; + + /** + * \brief Fragment id of included literal. + */ + u32 included_frag_id = INVALID_FRAG_ID; + + /** + * \brief Fragment Id of included literal (Delayed literals only). + */ + u32 included_delay_frag_id = INVALID_FRAG_ID; rose_group groups; std::vector lit_ids; u32 lit_program_offset = ROSE_INVALID_PROG_OFFSET; u32 delay_program_offset = ROSE_INVALID_PROG_OFFSET; }; -bytecode_ptr -buildFloatingMatcher(const RoseBuildImpl &build, - const std::vector &fragments, - size_t longLitLengthThreshold, rose_group *fgroups, - size_t *historyRequired); +struct LitProto { + LitProto(std::unique_ptr hwlmProto_in, + std::vector &accel_lits_in) + : hwlmProto(std::move(hwlmProto_in)), accel_lits(accel_lits_in) {} + + std::unique_ptr hwlmProto; + std::vector accel_lits; +}; bytecode_ptr -buildDelayRebuildMatcher(const RoseBuildImpl &build, - const std::vector &fragments, - size_t longLitLengthThreshold); +buildHWLMMatcher(const RoseBuildImpl &build, LitProto *proto); -bytecode_ptr -buildSmallBlockMatcher(const RoseBuildImpl &build, - const std::vector &fragments); +std::unique_ptr +buildFloatingMatcherProto(const RoseBuildImpl &build, + const std::vector &fragments, + size_t longLitLengthThreshold, + rose_group *fgroups, + size_t *historyRequired); -bytecode_ptr -buildEodAnchoredMatcher(const RoseBuildImpl &build, - const std::vector &fragments); +std::unique_ptr +buildDelayRebuildMatcherProto(const RoseBuildImpl &build, + const std::vector &fragments, + size_t longLitLengthThreshold); +std::unique_ptr +buildSmallBlockMatcherProto(const RoseBuildImpl &build, + const std::vector &fragments); + +std::unique_ptr +buildEodAnchoredMatcherProto(const RoseBuildImpl &build, + const std::vector &fragments); void findMoreLiteralMasks(RoseBuildImpl &build); diff --git a/src/rose/rose_build_program.cpp b/src/rose/rose_build_program.cpp index 562ddb20..01bd7c54 100644 --- a/src/rose/rose_build_program.cpp +++ b/src/rose/rose_build_program.cpp @@ -2164,6 +2164,14 @@ RoseProgram makeBoundaryProgram(const RoseBuildImpl &build, return prog; } +void addIncludedJumpProgram(RoseProgram &program, u32 child_offset, + u8 squash) { + RoseProgram block; + block.add_before_end(make_unique(child_offset, + squash)); + program.add_block(move(block)); +} + static void addPredBlockSingle(u32 pred_state, RoseProgram &pred_block, RoseProgram &program) { diff --git a/src/rose/rose_build_program.h b/src/rose/rose_build_program.h index 8758ef64..afbaa36e 100644 --- a/src/rose/rose_build_program.h +++ b/src/rose/rose_build_program.h @@ -282,6 +282,7 @@ void recordLongLiterals(std::vector &longLiterals, void recordResources(RoseResources &resources, const RoseProgram &program); +void addIncludedJumpProgram(RoseProgram &program, u32 child_offset, u8 squash); } // namespace ue2 #endif // ROSE_BUILD_PROGRAM_H diff --git a/src/rose/rose_program.h b/src/rose/rose_program.h index 78b123d5..eeebfed1 100644 --- a/src/rose/rose_program.h +++ b/src/rose/rose_program.h @@ -178,7 +178,12 @@ enum RoseInstructionCode { */ ROSE_INSTR_CHECK_MULTIPATH_SHUFTI_64, - LAST_ROSE_INSTRUCTION = ROSE_INSTR_CHECK_MULTIPATH_SHUFTI_64 //!< Sentinel. + /** + * \brief Jump to the program of included literal. + */ + ROSE_INSTR_INCLUDED_JUMP, + + LAST_ROSE_INSTRUCTION = ROSE_INSTR_INCLUDED_JUMP //!< Sentinel. }; struct ROSE_STRUCT_END { @@ -625,4 +630,10 @@ struct ROSE_STRUCT_CHECK_MULTIPATH_SHUFTI_64 { s32 last_start; //!< The latest start offset among 8 paths. u32 fail_jump; //!< Jump forward this many bytes on failure. }; + +struct ROSE_STRUCT_INCLUDED_JUMP { + u8 code; //!< From enum RoseInstructionCode. + u8 squash; //!< FDR confirm squash mask for included literal. + u32 child_offset; //!< Program offset of included literal. +}; #endif // ROSE_ROSE_PROGRAM_H diff --git a/src/scratch.c b/src/scratch.c index 84d23ced..8e082c77 100644 --- a/src/scratch.c +++ b/src/scratch.c @@ -136,6 +136,7 @@ hs_error_t alloc_scratch(const hs_scratch_t *proto, hs_scratch_t **scratch) { s->in_use = 0; s->scratchSize = alloc_size; s->scratch_alloc = (char *)s_tmp; + s->fdr_conf = NULL; // each of these is at an offset from the previous char *current = (char *)s + sizeof(*s); diff --git a/src/scratch.h b/src/scratch.h index 1d4b849e..fa998e84 100644 --- a/src/scratch.h +++ b/src/scratch.h @@ -200,6 +200,9 @@ struct ALIGN_CL_DIRECTIVE hs_scratch { u32 delay_fatbit_size; /**< size of each delay fatbit in bytes */ u32 scratchSize; char *scratch_alloc; /* user allocated scratch object */ + u64a *fdr_conf; /**< FDR confirm value */ + u8 fdr_conf_offset; /**< offset where FDR/Teddy front end matches + * in buffer */ }; /* array of fatbit ptr; TODO: why not an array of fatbits? */ diff --git a/unit/internal/fdr.cpp b/unit/internal/fdr.cpp index 399147e2..87ab0974 100644 --- a/unit/internal/fdr.cpp +++ b/unit/internal/fdr.cpp @@ -36,6 +36,7 @@ #include "fdr/fdr_engine_description.h" #include "fdr/teddy_compile.h" #include "fdr/teddy_engine_description.h" +#include "hwlm/hwlm_internal.h" #include "util/alloc.h" #include "database.h" @@ -135,6 +136,31 @@ vector getValidFdrEngines() { return ret; } + +static +bytecode_ptr buildFDREngineHinted(std::vector &lits, + bool make_small, u32 hint, + const target_t &target, + const Grey &grey) { + auto proto = fdrBuildProtoHinted(HWLM_ENGINE_FDR, lits, make_small, hint, + target, grey); + if (!proto) { + return nullptr; + } + return fdrBuildTable(*proto, grey); +} + +static +bytecode_ptr buildFDREngine(std::vector &lits, + bool make_small, const target_t &target, + const Grey &grey) { + auto proto = fdrBuildProto(HWLM_ENGINE_FDR, lits, make_small, target, grey); + if (!proto) { + return nullptr; + } + return fdrBuildTable(*proto, grey); +} + class FDRp : public TestWithParam { }; @@ -147,10 +173,12 @@ TEST_P(FDRp, Simple) { vector lits; lits.push_back(hwlmLiteral("mnopqr", 0, 0)); - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); struct hs_scratch scratch; + scratch.fdr_conf = NULL; fdrExec(fdr.get(), (const u8 *)data, sizeof(data), 0, decentCallback, &scratch, HWLM_ALL_GROUPS); @@ -170,10 +198,12 @@ TEST_P(FDRp, SimpleSingle) { vector lits; lits.push_back(hwlmLiteral("m", 0, 0)); - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); struct hs_scratch scratch; + scratch.fdr_conf = NULL; fdrExec(fdr.get(), (const u8 *)data, sizeof(data) - 1 /* skip nul */, 0, decentCallback, &scratch, HWLM_ALL_GROUPS); @@ -192,7 +222,8 @@ TEST_P(FDRp, MultiLocation) { vector lits; lits.push_back(hwlmLiteral("abc", 0, 1)); - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); const u32 testSize = 128; @@ -200,6 +231,7 @@ TEST_P(FDRp, MultiLocation) { vector data(testSize, 0); struct hs_scratch scratch; + scratch.fdr_conf = NULL; for (u32 i = 0; i < testSize - 3; i++) { memcpy(data.data() + i, "abc", 3); fdrExec(fdr.get(), data.data(), testSize, 0, decentCallback, &scratch, @@ -220,10 +252,12 @@ TEST_P(FDRp, NoRepeat1) { vector lits = { hwlmLiteral("m", 0, 1, 0, HWLM_ALL_GROUPS, {}, {}) }; - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); struct hs_scratch scratch; + scratch.fdr_conf = NULL; fdrExec(fdr.get(), (const u8 *)data, sizeof(data) - 1 /* skip nul */, 0, decentCallback, &scratch, HWLM_ALL_GROUPS); @@ -242,10 +276,12 @@ TEST_P(FDRp, NoRepeat2) { = { hwlmLiteral("m", 0, 1, 0, HWLM_ALL_GROUPS, {}, {}), hwlmLiteral("A", 0, 42) }; - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); struct hs_scratch scratch; + scratch.fdr_conf = NULL; fdrExec(fdr.get(), (const u8 *)data, sizeof(data) - 1 /* skip nul */, 0, decentCallback, &scratch, HWLM_ALL_GROUPS); @@ -265,10 +301,12 @@ TEST_P(FDRp, NoRepeat3) { = { hwlmLiteral("90m", 0, 1, 0, HWLM_ALL_GROUPS, {}, {}), hwlmLiteral("zA", 0, 1, 0, HWLM_ALL_GROUPS, {}, {}) }; - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); struct hs_scratch scratch; + scratch.fdr_conf = NULL; fdrExec(fdr.get(), (const u8 *)data, sizeof(data) - 1 /* skip nul */, 0, decentCallback, &scratch, HWLM_ALL_GROUPS); @@ -293,6 +331,7 @@ hwlm_error_t safeExecStreaming(const FDR *fdr, const u8 *hbuf, size_t hlen, hbuf = new_hbuf; } struct hs_scratch scratch; + scratch.fdr_conf = NULL; return fdrExecStreaming(fdr, hbuf, hlen, buf, len, start, cb, &scratch, groups); } @@ -304,7 +343,8 @@ TEST_P(FDRp, SmallStreaming) { vector lits = {hwlmLiteral("a", 1, 1), hwlmLiteral("aardvark", 0, 10)}; - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); vector expected; @@ -342,7 +382,8 @@ TEST_P(FDRp, SmallStreaming2) { hwlmLiteral("kk", 1, 2), hwlmLiteral("aardvark", 0, 10)}; - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); vector expected; @@ -373,7 +414,8 @@ TEST_P(FDRp, moveByteStream) { vector lits; lits.push_back(hwlmLiteral("mnopqr", 0, 0)); - auto fdrTable0 = fdrBuildTableHinted(lits, false, hint, get_current_target(), Grey()); + auto fdrTable0 = buildFDREngineHinted(lits, false, hint, + get_current_target(), Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdrTable0, hint); size_t size = fdrSize(fdrTable0.get()); @@ -390,6 +432,7 @@ TEST_P(FDRp, moveByteStream) { // check matches struct hs_scratch scratch; + scratch.fdr_conf = NULL; hwlm_error_t fdrStatus = fdrExec(fdrTable.get(), (const u8 *)data, data_len, 0, decentCallback, &scratch, @@ -414,7 +457,8 @@ TEST_P(FDRp, Stream1) { lits.push_back(hwlmLiteral("f", 0, 0)); lits.push_back(hwlmLiteral("literal", 0, 1)); - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); // check matches @@ -470,12 +514,13 @@ TEST_P(FDRpp, AlignAndTooEarly) { vector lits; struct hs_scratch scratch; + scratch.fdr_conf = NULL; for (size_t litLen = 1; litLen <= patLen; litLen++) { // building literal from pattern substring of variable length 1-patLen lits.push_back(hwlmLiteral(string(pattern, 0, litLen), 0, 0)); - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), - Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); // check with buffer offset from aligned start from 0 to 31 @@ -592,6 +637,7 @@ TEST_P(FDRpa, ShortWritings) { // run the literal matching through all generated literals struct hs_scratch scratch; + scratch.fdr_conf = NULL; for (size_t patIdx = 0; patIdx < pats.size();) { // group them in the sets of 32 vector testSigs; @@ -599,8 +645,8 @@ TEST_P(FDRpa, ShortWritings) { testSigs.push_back(hwlmLiteral(pats[patIdx], false, patIdx)); } - auto fdr = fdrBuildTableHinted(testSigs, false, hint, - get_current_target(), Grey()); + auto fdr = buildFDREngineHinted(testSigs, false, hint, + get_current_target(), Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); @@ -659,7 +705,7 @@ TEST(FDR, FDRTermS) { lits.push_back(hwlmLiteral("f", 0, 0)); lits.push_back(hwlmLiteral("ff", 0, 1)); - auto fdr = fdrBuildTable(lits, false, get_current_target(), Grey()); + auto fdr = buildFDREngine(lits, false, get_current_target(), Grey()); ASSERT_TRUE(fdr != nullptr); // check matches @@ -682,11 +728,12 @@ TEST(FDR, FDRTermB) { lits.push_back(hwlmLiteral("f", 0, 0)); lits.push_back(hwlmLiteral("ff", 0, 1)); - auto fdr = fdrBuildTable(lits, false, get_current_target(), Grey()); + auto fdr = buildFDREngine(lits, false, get_current_target(), Grey()); ASSERT_TRUE(fdr != nullptr); // check matches struct hs_scratch scratch; + scratch.fdr_conf = NULL; fdrStatus = fdrExec(fdr.get(), (const u8 *)data1, data_len1, 0, decentCallbackT, &scratch, HWLM_ALL_GROUPS); diff --git a/unit/internal/fdr_flood.cpp b/unit/internal/fdr_flood.cpp index 8bdd0763..81afbeaa 100644 --- a/unit/internal/fdr_flood.cpp +++ b/unit/internal/fdr_flood.cpp @@ -36,6 +36,7 @@ #include "fdr/fdr_engine_description.h" #include "fdr/teddy_compile.h" #include "fdr/teddy_engine_description.h" +#include "hwlm/hwlm_internal.h" #include "scratch.h" #include "util/alloc.h" #include "util/bitutils.h" @@ -131,6 +132,16 @@ static vector getValidFdrEngines() { return ret; } +static +bytecode_ptr buildFDREngineHinted(std::vector &lits, + bool make_small, u32 hint, + const target_t &target, + const Grey &grey) { + auto proto = fdrBuildProtoHinted(HWLM_ENGINE_FDR, lits, make_small, hint, + target, grey); + return fdrBuildTable(*proto, grey); +} + class FDRFloodp : public TestWithParam { }; @@ -142,6 +153,7 @@ TEST_P(FDRFloodp, NoMask) { u8 c = 0; struct hs_scratch scratch; + scratch.fdr_conf = NULL; while (1) { SCOPED_TRACE((unsigned int)c); u8 bit = 1 << (c & 0x7); @@ -169,8 +181,8 @@ TEST_P(FDRFloodp, NoMask) { lits.push_back(hwlmLiteral(sAlt, false, i * 8 + 7)); } - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), - Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); hwlm_error_t fdrStatus = fdrExec(fdr.get(), &data[0], dataSize, @@ -235,6 +247,7 @@ TEST_P(FDRFloodp, WithMask) { u8 c = '\0'; struct hs_scratch scratch; + scratch.fdr_conf = NULL; while (1) { u8 bit = 1 << (c & 0x7); u8 cAlt = c ^ bit; @@ -305,8 +318,8 @@ TEST_P(FDRFloodp, WithMask) { HWLM_ALL_GROUPS, msk, cmp)); } } - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), - Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); hwlm_error_t fdrStatus = fdrExec(fdr.get(), &data[0], dataSize, @@ -400,6 +413,7 @@ TEST_P(FDRFloodp, StreamingMask) { u8 c = '\0'; struct hs_scratch scratch; + scratch.fdr_conf = NULL; while (1) { u8 bit = 1 << (c & 0x7); u8 cAlt = c ^ bit; @@ -470,8 +484,8 @@ TEST_P(FDRFloodp, StreamingMask) { HWLM_ALL_GROUPS, msk, cmp)); } } - auto fdr = fdrBuildTableHinted(lits, false, hint, get_current_target(), - Grey()); + auto fdr = buildFDREngineHinted(lits, false, hint, get_current_target(), + Grey()); CHECK_WITH_TEDDY_OK_TO_FAIL(fdr, hint); hwlm_error_t fdrStatus;