From eb14792a63805ffc3aaff2a574f95dd71ca89844 Mon Sep 17 00:00:00 2001 From: Justin Viiret Date: Mon, 12 Dec 2016 17:08:06 +1100 Subject: [PATCH] rose: group final ids by fragment --- src/hwlm/hwlm_literal.h | 14 ++ src/rose/match.c | 42 +++--- src/rose/program_runtime.h | 4 +- src/rose/rose_build_anchored.cpp | 18 ++- src/rose/rose_build_anchored.h | 6 +- src/rose/rose_build_bytecode.cpp | 220 ++++++++++++++++++++++++++----- src/rose/rose_build_dump.cpp | 18 ++- src/rose/rose_build_impl.h | 2 + src/rose/rose_build_matchers.cpp | 93 +++++++++---- src/rose/rose_build_matchers.h | 23 ++-- src/rose/rose_build_program.cpp | 1 + src/rose/rose_build_program.h | 19 +-- src/rose/rose_dump.cpp | 1 + src/rose/rose_internal.h | 13 +- src/rose/rose_program.h | 2 +- 15 files changed, 354 insertions(+), 122 deletions(-) diff --git a/src/hwlm/hwlm_literal.h b/src/hwlm/hwlm_literal.h index a08b2ff6..0e2a1ea5 100644 --- a/src/hwlm/hwlm_literal.h +++ b/src/hwlm/hwlm_literal.h @@ -37,6 +37,7 @@ #include "ue2common.h" #include +#include #include namespace ue2 { @@ -111,6 +112,19 @@ struct hwlmLiteral { : hwlmLiteral(s_in, nocase_in, false, id_in, HWLM_ALL_GROUPS, {}, {}) {} }; +inline +bool operator<(const hwlmLiteral &a, const hwlmLiteral &b) { + return std::tie(a.id, a.s, a.nocase, a.noruns, a.groups, a.msk, a.cmp) < + std::tie(b.id, b.s, b.nocase, b.noruns, b.groups, b.msk, b.cmp); +} + +inline +bool operator==(const hwlmLiteral &a, const hwlmLiteral &b) { + return a.id == b.id && a.s == b.s && a.nocase == b.nocase && + a.noruns == b.noruns && a.groups == b.groups && a.msk == b.msk && + a.cmp == b.cmp; +} + /** * Consistency test; returns false if the given msk/cmp test can never match * the literal string s. diff --git a/src/rose/match.c b/src/rose/match.c index b641e39d..9a702804 100644 --- a/src/rose/match.c +++ b/src/rose/match.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -253,24 +253,6 @@ hwlmcb_rv_t roseProcessMatchInline(const struct RoseEngine *t, flags); } -/** - * \brief Run the program for the given literal ID, with the interpreter - * out of line. - * - * Assumes not in_anchored. - */ -static really_inline -hwlmcb_rv_t roseProcessMatch(const struct RoseEngine *t, - struct hs_scratch *scratch, u64a end, - size_t match_len, u32 id) { - DEBUG_PRINTF("id=%u\n", id); - const u32 *programs = getByOffset(t, t->litProgramOffset); - assert(id < t->literalCount); - const u64a som = 0; - const u8 flags = 0; - return roseRunProgram(t, scratch, programs[id], som, end, match_len, flags); -} - static rose_inline hwlmcb_rv_t playDelaySlot(const struct RoseEngine *t, struct hs_scratch *scratch, @@ -290,14 +272,17 @@ hwlmcb_rv_t playDelaySlot(const struct RoseEngine *t, roseFlushLastByteHistory(t, scratch, offset); tctxt->lastEndOffset = offset; + const u32 *programs = getByOffset(t, t->delayProgramOffset); + for (u32 it = fatbit_iterate(vicSlot, delay_count, MMB_INVALID); it != MMB_INVALID; it = fatbit_iterate(vicSlot, delay_count, it)) { - u32 literal_id = t->delay_base_id + it; - UNUSED rose_group old_groups = tctxt->groups; - DEBUG_PRINTF("DELAYED MATCH id=%u offset=%llu\n", literal_id, offset); - hwlmcb_rv_t rv = roseProcessMatch(t, scratch, offset, 0, literal_id); + DEBUG_PRINTF("DELAYED MATCH id=%u offset=%llu\n", it, offset); + const u64a som = 0; + const u8 flags = 0; + hwlmcb_rv_t rv = roseRunProgram(t, scratch, programs[it], som, offset, + 0, flags); DEBUG_PRINTF("DONE groups=0x%016llx\n", tctxt->groups); /* delayed literals can't safely set groups. @@ -322,16 +307,19 @@ hwlmcb_rv_t flushAnchoredLiteralAtLoc(const struct RoseEngine *t, struct fatbit *curr_row = getAnchoredLiteralLog(scratch)[curr_loc - 1]; u32 region_width = t->anchored_count; + const u32 *programs = getByOffset(t, t->anchoredProgramOffset); + DEBUG_PRINTF("report matches at curr loc\n"); for (u32 it = fatbit_iterate(curr_row, region_width, MMB_INVALID); it != MMB_INVALID; it = fatbit_iterate(curr_row, region_width, it)) { DEBUG_PRINTF("it = %u/%u\n", it, region_width); - u32 literal_id = t->anchored_base_id + it; rose_group old_groups = tctxt->groups; - DEBUG_PRINTF("ANCH REPLAY MATCH id=%u offset=%u\n", literal_id, - curr_loc); - hwlmcb_rv_t rv = roseProcessMatch(t, scratch, curr_loc, 0, literal_id); + DEBUG_PRINTF("ANCH REPLAY MATCH id=%u offset=%u\n", it, curr_loc); + const u64a som = 0; + const u8 flags = 0; + hwlmcb_rv_t rv = roseRunProgram(t, scratch, programs[it], som, curr_loc, + 0, flags); DEBUG_PRINTF("DONE groups=0x%016llx\n", tctxt->groups); /* anchored literals can't safely set groups. diff --git a/src/rose/program_runtime.h b/src/rose/program_runtime.h index 1a5f25e9..8f4c528d 100644 --- a/src/rose/program_runtime.h +++ b/src/rose/program_runtime.h @@ -1554,7 +1554,9 @@ hwlmcb_rv_t roseRunProgram_i(const struct RoseEngine *t, if (end < ri->min_offset) { DEBUG_PRINTF("halt: before min_offset=%u\n", ri->min_offset); - return HWLM_CONTINUE_MATCHING; + assert(ri->fail_jump); // must progress + pc += ri->fail_jump; + continue; } } PROGRAM_NEXT_INSTRUCTION diff --git a/src/rose/rose_build_anchored.cpp b/src/rose/rose_build_anchored.cpp index 3d0affc6..ea565eaa 100644 --- a/src/rose/rose_build_anchored.cpp +++ b/src/rose/rose_build_anchored.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -208,7 +208,8 @@ void remapAnchoredReports(RoseBuildImpl &build) { * raw_dfa with program offsets. */ static -void remapIdsToPrograms(raw_dfa &rdfa, const vector &litPrograms) { +void remapIdsToPrograms(raw_dfa &rdfa, const vector &litPrograms, + const map &final_to_frag_map) { for (dstate &ds : rdfa.states) { assert(ds.reports_eod.empty()); // Not used in anchored matcher. if (ds.reports.empty()) { @@ -216,9 +217,11 @@ void remapIdsToPrograms(raw_dfa &rdfa, const vector &litPrograms) { } flat_set new_reports; - for (auto id : ds.reports) { - assert(id < litPrograms.size()); - new_reports.insert(litPrograms.at(id)); + for (auto final_id : ds.reports) { + assert(contains(final_to_frag_map, final_id)); + auto frag_id = final_to_frag_map.at(final_id); + assert(frag_id < litPrograms.size()); + new_reports.insert(litPrograms.at(frag_id)); } ds.reports = move(new_reports); } @@ -846,7 +849,8 @@ vector buildAnchoredDfas(RoseBuildImpl &build) { aligned_unique_ptr buildAnchoredMatcher(RoseBuildImpl &build, vector &dfas, - const vector &litPrograms, size_t *asize) { + const vector &litPrograms, + const map &final_to_frag_map, size_t *asize) { const CompileContext &cc = build.cc; if (dfas.empty()) { @@ -856,7 +860,7 @@ buildAnchoredMatcher(RoseBuildImpl &build, vector &dfas, } for (auto &rdfa : dfas) { - remapIdsToPrograms(rdfa, litPrograms); + remapIdsToPrograms(rdfa, litPrograms, final_to_frag_map); } vector> nfas; diff --git a/src/rose/rose_build_anchored.h b/src/rose/rose_build_anchored.h index ef06fcbb..fa379ff6 100644 --- a/src/rose/rose_build_anchored.h +++ b/src/rose/rose_build_anchored.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -59,7 +59,9 @@ std::vector buildAnchoredDfas(RoseBuildImpl &build); */ aligned_unique_ptr buildAnchoredMatcher(RoseBuildImpl &build, std::vector &dfas, - const std::vector &litPrograms, size_t *asize); + const std::vector &litPrograms, + const std::map &final_to_frag_map, + size_t *asize); u32 anchoredStateSize(const anchored_matcher_info &atable); diff --git a/src/rose/rose_build_bytecode.cpp b/src/rose/rose_build_bytecode.cpp index 9f978134..6f996979 100644 --- a/src/rose/rose_build_bytecode.cpp +++ b/src/rose/rose_build_bytecode.cpp @@ -4346,7 +4346,9 @@ void makeCheckLitEarlyInstruction(const RoseBuildImpl &build, build_context &bc, assert(min_offset < UINT32_MAX); DEBUG_PRINTF("adding lit early check, min_offset=%u\n", min_offset); - program.add_before_end(make_unique(min_offset)); + const auto *end_inst = program.end_instruction(); + program.add_before_end( + make_unique(min_offset, end_inst)); } static @@ -4528,9 +4530,33 @@ RoseProgram buildLiteralProgram(RoseBuildImpl &build, build_context &bc, } static -u32 writeLiteralProgram(RoseBuildImpl &build, build_context &bc, u32 final_id, - const vector &lit_edges) { - RoseProgram program = buildLiteralProgram(build, bc, final_id, lit_edges); +RoseProgram buildLiteralProgram(RoseBuildImpl &build, build_context &bc, + const flat_set &final_ids, + const map> &lit_edges) { + assert(!final_ids.empty()); + + DEBUG_PRINTF("entry, %zu final ids\n", final_ids.size()); + const vector no_edges; + + RoseProgram program; + for (const auto &final_id : final_ids) { + const auto *edges_ptr = &no_edges; + if (contains(lit_edges, final_id)) { + edges_ptr = &(lit_edges.at(final_id)); + } + auto prog = buildLiteralProgram(build, bc, final_id, *edges_ptr); + DEBUG_PRINTF("final_id=%u, prog has %zu entries\n", final_id, + prog.size()); + program.add_block(move(prog)); + } + return program; +} + +static +u32 writeLiteralProgram(RoseBuildImpl &build, build_context &bc, + const flat_set &final_ids, + const map> &lit_edges) { + RoseProgram program = buildLiteralProgram(build, bc, final_ids, lit_edges); if (program.empty()) { return 0; } @@ -4540,18 +4566,26 @@ u32 writeLiteralProgram(RoseBuildImpl &build, build_context &bc, u32 final_id, static u32 buildDelayRebuildProgram(RoseBuildImpl &build, build_context &bc, - u32 final_id) { - const auto &lit_infos = getLiteralInfoByFinalId(build, final_id); - const auto &arb_lit_info = **lit_infos.begin(); - if (arb_lit_info.delayed_ids.empty()) { - return 0; // No delayed IDs, no work to do. + const flat_set &final_ids) { + RoseProgram program; + + for (const auto &final_id : final_ids) { + const auto &lit_infos = getLiteralInfoByFinalId(build, final_id); + const auto &arb_lit_info = **lit_infos.begin(); + if (arb_lit_info.delayed_ids.empty()) { + continue; // No delayed IDs, no work to do. + } + + RoseProgram prog; + makeCheckLiteralInstruction(build, bc, final_id, prog); + makeCheckLitMaskInstruction(build, bc, final_id, prog); + makePushDelayedInstructions(build, final_id, prog); + program.add_block(move(prog)); } - RoseProgram program; - makeCheckLiteralInstruction(build, bc, final_id, program); - makeCheckLitMaskInstruction(build, bc, final_id, program); - makePushDelayedInstructions(build, final_id, program); - assert(!program.empty()); + if (program.empty()) { + return 0; + } applyFinalSpecialisation(program); return writeProgram(bc, move(program)); } @@ -4590,27 +4624,104 @@ map> findEdgesByLiteral(const RoseBuildImpl &build) { return lit_edge_map; } +static +rose_literal_id getFragment(const rose_literal_id &lit) { + if (lit.s.length() <= ROSE_SHORT_LITERAL_LEN_MAX) { + DEBUG_PRINTF("whole lit is frag\n"); + return lit; + } + + rose_literal_id frag = lit; + frag.s = frag.s.substr(frag.s.length() - ROSE_SHORT_LITERAL_LEN_MAX); + + DEBUG_PRINTF("fragment: %s\n", dumpString(frag.s).c_str()); + return frag; +} + +map groupByFragment(const RoseBuildImpl &build) { + u32 frag_id = 0; + map final_to_frag; + + map> frag_lits; + for (const auto &m : build.final_id_to_literal) { + u32 final_id = m.first; + const auto &lit_ids = m.second; + assert(!lit_ids.empty()); + + if (lit_ids.size() > 1) { + final_to_frag.emplace(final_id, frag_id++); + continue; + } + + const auto lit_id = *lit_ids.begin(); + const auto &lit = build.literals.right.at(lit_id); + if (lit.s.length() < ROSE_SHORT_LITERAL_LEN_MAX) { + final_to_frag.emplace(final_id, frag_id++); + continue; + } + + // Combining exploded fragments with others is unsafe. + const auto &info = build.literal_info[lit_id]; + if (info.requires_explode) { + final_to_frag.emplace(final_id, frag_id++); + continue; + } + + DEBUG_PRINTF("fragment candidate: final_id=%u %s\n", final_id, + dumpString(lit.s).c_str()); + auto frag = getFragment(lit); + frag_lits[frag].push_back(final_id); + } + + for (const auto &m : frag_lits) { + DEBUG_PRINTF("frag %s -> ids: %s\n", dumpString(m.first.s).c_str(), + as_string_list(m.second).c_str()); + for (const auto final_id : m.second) { + assert(!contains(final_to_frag, final_id)); + final_to_frag.emplace(final_id, frag_id); + } + frag_id++; + } + + return final_to_frag; +} + /** * \brief Build the interpreter programs for each literal. * - * Returns the base of the literal program list and the base of the delay - * rebuild program list. + * Returns the following as a tuple: + * + * - base of the literal program list + * - base of the delay rebuild program list + * - total number of literal fragments */ static -pair buildLiteralPrograms(RoseBuildImpl &build, build_context &bc) { - const u32 num_literals = build.final_id_to_literal.size(); +tuple +buildLiteralPrograms(RoseBuildImpl &build, build_context &bc, + const map &final_to_frag_map) { + // Build a reverse mapping from fragment -> final_id. + map> frag_to_final_map; + for (const auto &m : final_to_frag_map) { + frag_to_final_map[m.second].insert(m.first); + } + + const u32 num_fragments = verify_u32(frag_to_final_map.size()); + DEBUG_PRINTF("%u fragments\n", num_fragments); + auto lit_edge_map = findEdgesByLiteral(build); - bc.litPrograms.resize(num_literals); - vector delayRebuildPrograms(num_literals); + bc.litPrograms.resize(num_fragments); + vector delayRebuildPrograms(num_fragments); - for (u32 finalId = 0; finalId != num_literals; ++finalId) { - const auto &lit_edges = lit_edge_map[finalId]; + for (u32 frag_id = 0; frag_id != num_fragments; ++frag_id) { + const auto &final_ids = frag_to_final_map[frag_id]; + DEBUG_PRINTF("frag_id=%u, final_ids=[%s]\n", frag_id, + as_string_list(final_ids).c_str()); - bc.litPrograms[finalId] = - writeLiteralProgram(build, bc, finalId, lit_edges); - delayRebuildPrograms[finalId] = - buildDelayRebuildProgram(build, bc, finalId); + bc.litPrograms[frag_id] = + writeLiteralProgram(build, bc, final_ids, lit_edge_map); + delayRebuildPrograms[frag_id] = + buildDelayRebuildProgram(build, bc, final_ids); } u32 litProgramsOffset = @@ -4618,7 +4729,40 @@ pair buildLiteralPrograms(RoseBuildImpl &build, build_context &bc) { u32 delayRebuildProgramsOffset = bc.engine_blob.add( begin(delayRebuildPrograms), end(delayRebuildPrograms)); - return {litProgramsOffset, delayRebuildProgramsOffset}; + return tuple{litProgramsOffset, delayRebuildProgramsOffset, + num_fragments}; +} + +static +u32 buildDelayPrograms(RoseBuildImpl &build, build_context &bc) { + auto lit_edge_map = findEdgesByLiteral(build); + + vector programs; + + for (u32 final_id = build.delay_base_id; + final_id < build.final_id_to_literal.size(); final_id++) { + u32 offset = writeLiteralProgram(build, bc, {final_id}, lit_edge_map); + programs.push_back(offset); + } + + DEBUG_PRINTF("%zu delay programs\n", programs.size()); + return bc.engine_blob.add(begin(programs), end(programs)); +} + +static +u32 buildAnchoredPrograms(RoseBuildImpl &build, build_context &bc) { + auto lit_edge_map = findEdgesByLiteral(build); + + vector programs; + + for (u32 final_id = build.anchored_base_id; + final_id < build.delay_base_id; final_id++) { + u32 offset = writeLiteralProgram(build, bc, {final_id}, lit_edge_map); + programs.push_back(offset); + } + + DEBUG_PRINTF("%zu anchored programs\n", programs.size()); + return bc.engine_blob.add(begin(programs), end(programs)); } /** @@ -5253,6 +5397,7 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { DEBUG_PRINTF("longLitLengthThreshold=%zu\n", longLitLengthThreshold); allocateFinalLiteralId(*this); + auto final_to_frag_map = groupByFragment(*this); auto anchored_dfas = buildAnchoredDfas(*this); @@ -5316,8 +5461,12 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { u32 litProgramOffset; u32 litDelayRebuildProgramOffset; - tie(litProgramOffset, litDelayRebuildProgramOffset) = - buildLiteralPrograms(*this, bc); + u32 litProgramCount; + tie(litProgramOffset, litDelayRebuildProgramOffset, litProgramCount) = + buildLiteralPrograms(*this, bc, final_to_frag_map); + + u32 delayProgramOffset = buildDelayPrograms(*this, bc); + u32 anchoredProgramOffset = buildAnchoredPrograms(*this, bc); u32 eodProgramOffset = writeEodProgram(*this, bc, eodNfaIterOffset); @@ -5354,7 +5503,7 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { size_t asize = 0; u32 amatcherOffset = 0; auto atable = buildAnchoredMatcher(*this, anchored_dfas, bc.litPrograms, - &asize); + final_to_frag_map, &asize); if (atable) { currOffset = ROUNDUP_CL(currOffset); amatcherOffset = currOffset; @@ -5365,7 +5514,8 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { rose_group fgroups = 0; size_t fsize = 0; auto ftable = buildFloatingMatcher(*this, bc.longLitLengthThreshold, - &fgroups, &fsize, &historyRequired); + final_to_frag_map, &fgroups, &fsize, + &historyRequired); u32 fmatcherOffset = 0; if (ftable) { currOffset = ROUNDUP_CL(currOffset); @@ -5375,7 +5525,7 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { // Build EOD-anchored HWLM matcher. size_t esize = 0; - auto etable = buildEodAnchoredMatcher(*this, &esize); + auto etable = buildEodAnchoredMatcher(*this, final_to_frag_map, &esize); u32 ematcherOffset = 0; if (etable) { currOffset = ROUNDUP_CL(currOffset); @@ -5385,7 +5535,7 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { // Build small-block HWLM matcher. size_t sbsize = 0; - auto sbtable = buildSmallBlockMatcher(*this, &sbsize); + auto sbtable = buildSmallBlockMatcher(*this, final_to_frag_map, &sbsize); u32 sbmatcherOffset = 0; if (sbtable) { currOffset = ROUNDUP_CL(currOffset); @@ -5495,11 +5645,13 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { engine->needsCatchup = bc.needs_catchup ? 1 : 0; - engine->literalCount = verify_u32(final_id_to_literal.size()); + engine->literalCount = litProgramCount; engine->litProgramOffset = litProgramOffset; engine->litDelayRebuildProgramOffset = litDelayRebuildProgramOffset; engine->reportProgramOffset = reportProgramOffset; engine->reportProgramCount = reportProgramCount; + engine->delayProgramOffset = delayProgramOffset; + engine->anchoredProgramOffset = anchoredProgramOffset; engine->runtimeImpl = pickRuntimeImpl(*this, bc, outfixEndQueue); engine->mpvTriggeredByLeaf = anyEndfixMpvTriggers(*this); diff --git a/src/rose/rose_build_dump.cpp b/src/rose/rose_build_dump.cpp index e7cef100..495d6f36 100644 --- a/src/rose/rose_build_dump.cpp +++ b/src/rose/rose_build_dump.cpp @@ -505,19 +505,25 @@ void dumpRoseTestLiterals(const RoseBuildImpl &build, const string &base) { size_t longLitLengthThreshold = calcLongLitThreshold(build, historyRequired); - auto mp = makeMatcherProto(build, ROSE_ANCHORED, longLitLengthThreshold); + const auto final_to_frag_map = groupByFragment(build); + + auto mp = makeMatcherProto(build, final_to_frag_map, ROSE_ANCHORED, + longLitLengthThreshold); dumpTestLiterals(base + "rose_anchored_test_literals.txt", mp.lits); - mp = makeMatcherProto(build, ROSE_FLOATING, longLitLengthThreshold); + mp = makeMatcherProto(build, final_to_frag_map, ROSE_FLOATING, + longLitLengthThreshold); dumpTestLiterals(base + "rose_float_test_literals.txt", mp.lits); - mp = makeMatcherProto(build, ROSE_EOD_ANCHORED, build.ematcher_region_size); + mp = makeMatcherProto(build, final_to_frag_map, ROSE_EOD_ANCHORED, + build.ematcher_region_size); dumpTestLiterals(base + "rose_eod_test_literals.txt", mp.lits); if (!build.cc.streaming) { - mp = makeMatcherProto(build, ROSE_FLOATING, ROSE_SMALL_BLOCK_LEN, - ROSE_SMALL_BLOCK_LEN); - auto mp2 = makeMatcherProto(build, ROSE_ANCHORED_SMALL_BLOCK, + mp = makeMatcherProto(build, final_to_frag_map, ROSE_FLOATING, + ROSE_SMALL_BLOCK_LEN, ROSE_SMALL_BLOCK_LEN); + auto mp2 = makeMatcherProto(build, final_to_frag_map, + ROSE_ANCHORED_SMALL_BLOCK, ROSE_SMALL_BLOCK_LEN, ROSE_SMALL_BLOCK_LEN); mp.lits.insert(end(mp.lits), begin(mp2.lits), end(mp2.lits)); dumpTestLiterals(base + "rose_smallblock_test_literals.txt", mp.lits); diff --git a/src/rose/rose_build_impl.h b/src/rose/rose_build_impl.h index 02c5a389..7421dbfa 100644 --- a/src/rose/rose_build_impl.h +++ b/src/rose/rose_build_impl.h @@ -644,6 +644,8 @@ void normaliseLiteralMask(const ue2_literal &s, std::vector &msk, bool canImplementGraphs(const RoseBuildImpl &tbi); #endif +std::map groupByFragment(const RoseBuildImpl &build); + } // namespace ue2 #endif /* ROSE_BUILD_IMPL_H_17E20A3C6935D6 */ diff --git a/src/rose/rose_build_matchers.cpp b/src/rose/rose_build_matchers.cpp index f7c237a7..c51905ca 100644 --- a/src/rose/rose_build_matchers.cpp +++ b/src/rose/rose_build_matchers.cpp @@ -632,7 +632,27 @@ u64a literalMinReportOffset(const RoseBuildImpl &build, return lit_min_offset; } +static +map makeFragGroupMap(const RoseBuildImpl &build, + const map &final_to_frag_map) { + map frag_to_group; + + for (const auto &m : final_to_frag_map) { + u32 final_id = m.first; + u32 frag_id = m.second; + hwlm_group_t groups = 0; + const auto &lits = build.final_id_to_literal.at(final_id); + for (auto lit_id : lits) { + groups |= build.literal_info[lit_id].group_mask; + } + frag_to_group[frag_id] |= groups; + } + + return frag_to_group; +} + MatcherProto makeMatcherProto(const RoseBuildImpl &build, + const map &final_to_frag_map, rose_literal_table table, size_t max_len, u32 max_offset) { MatcherProto mp; @@ -710,23 +730,26 @@ MatcherProto makeMatcherProto(const RoseBuildImpl &build, msk, cmp); } } else { - string s = lit.get_string(); - bool nocase = lit.any_nocase(); + auto lit_final = lit; // copy + + if (lit_final.length() > ROSE_SHORT_LITERAL_LEN_MAX) { + DEBUG_PRINTF("truncating to tail of length %zu\n", + size_t{ROSE_SHORT_LITERAL_LEN_MAX}); + lit_final.erase(0, lit_final.length() - + ROSE_SHORT_LITERAL_LEN_MAX); + // We shouldn't have set a threshold below 8 chars. + assert(msk.size() <= ROSE_SHORT_LITERAL_LEN_MAX); + assert(!noruns); + } + + const auto &s = lit_final.get_string(); + bool nocase = lit_final.any_nocase(); DEBUG_PRINTF("id=%u, s='%s', nocase=%d, noruns=%d, msk=%s, " "cmp=%s\n", final_id, escapeString(s).c_str(), (int)nocase, noruns, dumpMask(msk).c_str(), dumpMask(cmp).c_str()); - if (s.length() > ROSE_SHORT_LITERAL_LEN_MAX) { - DEBUG_PRINTF("truncating to tail of length %zu\n", - size_t{ROSE_SHORT_LITERAL_LEN_MAX}); - s.erase(0, s.length() - ROSE_SHORT_LITERAL_LEN_MAX); - // We shouldn't have set a threshold below 8 chars. - assert(msk.size() <= ROSE_SHORT_LITERAL_LEN_MAX); - assert(!noruns); - } - if (!maskIsConsistent(s, nocase, msk, cmp)) { DEBUG_PRINTF("msk/cmp for literal can't match, skipping\n"); continue; @@ -738,18 +761,32 @@ MatcherProto makeMatcherProto(const RoseBuildImpl &build, } } + auto frag_group_map = makeFragGroupMap(build, final_to_frag_map); + + for (auto &lit : mp.lits) { + u32 final_id = lit.id; + assert(contains(final_to_frag_map, final_id)); + lit.id = final_to_frag_map.at(final_id); + assert(contains(frag_group_map, lit.id)); + lit.groups = frag_group_map.at(lit.id); + } + + sort(begin(mp.lits), end(mp.lits)); + mp.lits.erase(unique(begin(mp.lits), end(mp.lits)), end(mp.lits)); + return mp; } -aligned_unique_ptr buildFloatingMatcher(const RoseBuildImpl &build, - size_t longLitLengthThreshold, - rose_group *fgroups, - size_t *fsize, - size_t *historyRequired) { +aligned_unique_ptr +buildFloatingMatcher(const RoseBuildImpl &build, size_t longLitLengthThreshold, + const map &final_to_frag_map, + rose_group *fgroups, size_t *fsize, + size_t *historyRequired) { *fsize = 0; *fgroups = 0; - auto mp = makeMatcherProto(build, ROSE_FLOATING, longLitLengthThreshold); + auto mp = makeMatcherProto(build, final_to_frag_map, ROSE_FLOATING, + longLitLengthThreshold); if (mp.lits.empty()) { DEBUG_PRINTF("empty floating matcher\n"); return nullptr; @@ -776,8 +813,9 @@ aligned_unique_ptr buildFloatingMatcher(const RoseBuildImpl &build, return hwlm; } -aligned_unique_ptr buildSmallBlockMatcher(const RoseBuildImpl &build, - size_t *sbsize) { +aligned_unique_ptr +buildSmallBlockMatcher(const RoseBuildImpl &build, + const map &final_to_frag_map, size_t *sbsize) { *sbsize = 0; if (build.cc.streaming) { @@ -792,8 +830,8 @@ aligned_unique_ptr buildSmallBlockMatcher(const RoseBuildImpl &build, return nullptr; } - auto mp = makeMatcherProto(build, ROSE_FLOATING, ROSE_SMALL_BLOCK_LEN, - ROSE_SMALL_BLOCK_LEN); + auto mp = makeMatcherProto(build, final_to_frag_map, ROSE_FLOATING, + ROSE_SMALL_BLOCK_LEN, ROSE_SMALL_BLOCK_LEN); if (mp.lits.empty()) { DEBUG_PRINTF("no floating table\n"); return nullptr; @@ -803,8 +841,8 @@ aligned_unique_ptr buildSmallBlockMatcher(const RoseBuildImpl &build, } auto mp_anchored = - makeMatcherProto(build, ROSE_ANCHORED_SMALL_BLOCK, ROSE_SMALL_BLOCK_LEN, - ROSE_SMALL_BLOCK_LEN); + makeMatcherProto(build, final_to_frag_map, ROSE_ANCHORED_SMALL_BLOCK, + ROSE_SMALL_BLOCK_LEN, ROSE_SMALL_BLOCK_LEN); if (mp_anchored.lits.empty()) { DEBUG_PRINTF("no small-block anchored literals\n"); return nullptr; @@ -834,12 +872,13 @@ aligned_unique_ptr buildSmallBlockMatcher(const RoseBuildImpl &build, return hwlm; } -aligned_unique_ptr buildEodAnchoredMatcher(const RoseBuildImpl &build, - size_t *esize) { +aligned_unique_ptr +buildEodAnchoredMatcher(const RoseBuildImpl &build, + const map &final_to_frag_map, size_t *esize) { *esize = 0; - auto mp = - makeMatcherProto(build, ROSE_EOD_ANCHORED, build.ematcher_region_size); + auto mp = makeMatcherProto(build, final_to_frag_map, ROSE_EOD_ANCHORED, + build.ematcher_region_size); if (mp.lits.empty()) { DEBUG_PRINTF("no eod anchored literals\n"); diff --git a/src/rose/rose_build_matchers.h b/src/rose/rose_build_matchers.h index 15ccf278..742e8a14 100644 --- a/src/rose/rose_build_matchers.h +++ b/src/rose/rose_build_matchers.h @@ -36,6 +36,7 @@ #include "rose_build_impl.h" +#include #include struct HWLM; @@ -57,20 +58,26 @@ struct MatcherProto { * only lead to a pattern match after max_offset may be excluded. */ MatcherProto makeMatcherProto(const RoseBuildImpl &build, + const std::map &final_to_frag_map, rose_literal_table table, size_t max_len, u32 max_offset = ROSE_BOUND_INF); aligned_unique_ptr buildFloatingMatcher(const RoseBuildImpl &build, - size_t longLitLengthThreshold, - rose_group *fgroups, - size_t *fsize, - size_t *historyRequired); + size_t longLitLengthThreshold, + const std::map &final_to_frag_map, + rose_group *fgroups, + size_t *fsize, + size_t *historyRequired); -aligned_unique_ptr buildSmallBlockMatcher(const RoseBuildImpl &build, - size_t *sbsize); +aligned_unique_ptr +buildSmallBlockMatcher(const RoseBuildImpl &build, + const std::map &final_to_frag_map, + size_t *sbsize); -aligned_unique_ptr buildEodAnchoredMatcher(const RoseBuildImpl &build, - size_t *esize); +aligned_unique_ptr +buildEodAnchoredMatcher(const RoseBuildImpl &build, + const std::map &final_to_frag_map, + size_t *esize); void findMoreLiteralMasks(RoseBuildImpl &build); diff --git a/src/rose/rose_build_program.cpp b/src/rose/rose_build_program.cpp index 5f7ab0bf..112b93f9 100644 --- a/src/rose/rose_build_program.cpp +++ b/src/rose/rose_build_program.cpp @@ -79,6 +79,7 @@ void RoseInstrCheckLitEarly::write(void *dest, RoseEngineBlob &blob, RoseInstrBase::write(dest, blob, offset_map); auto *inst = static_cast(dest); inst->min_offset = min_offset; + inst->fail_jump = calc_jump(offset_map, this, target); } void RoseInstrCheckGroups::write(void *dest, RoseEngineBlob &blob, diff --git a/src/rose/rose_build_program.h b/src/rose/rose_build_program.h index 440bf4e1..fd966a8d 100644 --- a/src/rose/rose_build_program.h +++ b/src/rose/rose_build_program.h @@ -241,16 +241,18 @@ public: }; class RoseInstrCheckLitEarly - : public RoseInstrBaseNoTargets { public: u32 min_offset; + const RoseInstruction *target; - explicit RoseInstrCheckLitEarly(u32 min) : min_offset(min) {} + RoseInstrCheckLitEarly(u32 min_offset_in, const RoseInstruction *target_in) + : min_offset(min_offset_in), target(target_in) {} bool operator==(const RoseInstrCheckLitEarly &ri) const { - return min_offset == ri.min_offset; + return min_offset == ri.min_offset && target == ri.target; } size_t hash() const override { @@ -260,9 +262,10 @@ public: void write(void *dest, RoseEngineBlob &blob, const OffsetMap &offset_map) const override; - bool equiv_to(const RoseInstrCheckLitEarly &ri, const OffsetMap &, - const OffsetMap &) const { - return min_offset == ri.min_offset; + bool equiv_to(const RoseInstrCheckLitEarly &ri, const OffsetMap &offsets, + const OffsetMap &other_offsets) const { + return min_offset == ri.min_offset && + offsets.at(target) == other_offsets.at(ri.target); } }; @@ -1786,7 +1789,7 @@ public: }; class RoseInstrCheckMedLit - : public RoseInstrBaseNoTargets { public: @@ -1816,7 +1819,7 @@ public: }; class RoseInstrCheckMedLitNocase - : public RoseInstrBaseNoTargets { public: diff --git a/src/rose/rose_dump.cpp b/src/rose/rose_dump.cpp index 5d79da2e..f38d94c8 100644 --- a/src/rose/rose_dump.cpp +++ b/src/rose/rose_dump.cpp @@ -250,6 +250,7 @@ void dumpProgram(ofstream &os, const RoseEngine *t, const char *pc) { PROGRAM_CASE(CHECK_LIT_EARLY) { os << " min_offset " << ri->min_offset << endl; + os << " fail_jump " << offset + ri->fail_jump << endl; } PROGRAM_NEXT_INSTRUCTION diff --git a/src/rose/rose_internal.h b/src/rose/rose_internal.h index 411ce03f..bf6e9a86 100644 --- a/src/rose/rose_internal.h +++ b/src/rose/rose_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -361,6 +361,17 @@ struct RoseEngine { */ u32 reportProgramCount; + /** + * \brief Offset of u32 array of program offsets for delayed replay of + * literals. + */ + u32 delayProgramOffset; + + /** + * \brief Offset of u32 array of program offsets for anchored literals. + */ + u32 anchoredProgramOffset; + /** * \brief Number of entries in the arrays pointed to by litProgramOffset, * litDelayRebuildProgramOffset. diff --git a/src/rose/rose_program.h b/src/rose/rose_program.h index c5ddc942..652b9109 100644 --- a/src/rose/rose_program.h +++ b/src/rose/rose_program.h @@ -154,10 +154,10 @@ struct ROSE_STRUCT_ANCHORED_DELAY { u32 done_jump; //!< Jump forward this many bytes if successful. }; -/** Note: check failure will halt program. */ struct ROSE_STRUCT_CHECK_LIT_EARLY { u8 code; //!< From enum RoseInstructionCode. u32 min_offset; //!< Minimum offset for this literal. + u32 fail_jump; //!< Jump forward this many bytes on failure. }; /** Note: check failure will halt program. */