github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-06-28 16:41:01 +03:00
Code Issues Packages Projects Releases Wiki Activity
vectorscan/src/rose/rose_build_compile.cpp
Justin Viiret 7690881f85 rose: make assignGroupsToLiterals a free function
2016-07-08 10:47:08 +10:00

1692 lines
53 KiB
C++
Raw Blame History

/*
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "rose_build_impl.h"
#include "grey.h"
#include "hs_internal.h"
#include "rose_build_anchored.h"
#include "rose_build_castle.h"
#include "rose_build_convert.h"
#include "rose_build_dump.h"
#include "rose_build_groups.h"
#include "rose_build_matchers.h"
#include "rose_build_merge.h"
#include "rose_build_role_aliasing.h"
#include "rose_build_util.h"
#include "ue2common.h"
#include "nfa/nfa_internal.h"
#include "nfa/rdfa.h"
#include "nfagraph/ng_holder.h"
#include "nfagraph/ng_dump.h"
#include "nfagraph/ng_execute.h"
#include "nfagraph/ng_is_equal.h"
#include "nfagraph/ng_limex.h"
#include "nfagraph/ng_mcclellan.h"
#include "nfagraph/ng_repeat.h"
#include "nfagraph/ng_reports.h"
#include "nfagraph/ng_stop.h"
#include "nfagraph/ng_util.h"
#include "nfagraph/ng_width.h"
#include "util/bitutils.h"
#include "util/charreach.h"
#include "util/charreach_util.h"
#include "util/compare.h"
#include "util/compile_context.h"
#include "util/container.h"
#include "util/dump_charclass.h"
#include "util/graph_range.h"
#include "util/order_check.h"
#include "util/report_manager.h"
#include "util/ue2_containers.h"
#include "util/ue2string.h"
#include "util/verify_types.h"
#include <algorithm>
#include <functional>
#include <map>
#include <set>
#include <string>
#include <vector>
#include <utility>
#include <boost/range/adaptor/map.hpp>
using namespace std;
using boost::adaptors::map_values;
namespace ue2 {
#define ANCHORED_REHOME_MIN_FLOATING 800
#define ANCHORED_REHOME_MIN_FLOATING_SHORT 50
#define ANCHORED_REHOME_ALLOW_SHORT 20
#define ANCHORED_REHOME_DEEP 25
#define ANCHORED_REHOME_SHORT_LEN 3
#ifdef DEBUG
static UNUSED
void printLitInfo(const rose_literal_info &li, u32 id) {
DEBUG_PRINTF("lit_info %u\n", id);
DEBUG_PRINTF(" parent %u%s", li.undelayed_id,
li.delayed_ids.empty() ? "":", children:");
for (u32 d_id : li.delayed_ids) {
printf(" %u", d_id);
}
printf("\n");
DEBUG_PRINTF(" group %llu %s\n", li.group_mask, li.squash_group ? "s":"");
}
#endif
static
void allocateFinalIdToSet(const RoseGraph &g, const set<u32> &lits,
deque<rose_literal_info> *literal_info,
map<u32, set<u32> > *final_id_to_literal,
u32 *next_final_id) {
/* We can allocate the same final id to multiple literals of the same type
* if they share the same vertex set and trigger the same delayed literal
* ids and squash the same roles and have the same group squashing
* behaviour. Benefits literals cannot be merged. */
for (u32 int_id : lits) {
rose_literal_info &curr_info = (*literal_info)[int_id];
const auto &verts = curr_info.vertices;
if (!verts.empty() && !curr_info.requires_benefits
&& curr_info.delayed_ids.empty()) {
vector<u32> cand;
insert(&cand, cand.end(), g[*verts.begin()].literals);
for (auto v : verts) {
vector<u32> temp;
set_intersection(cand.begin(), cand.end(),
g[v].literals.begin(),
g[v].literals.end(),
inserter(temp, temp.end()));
cand.swap(temp);
}
for (u32 cand_id : cand) {
if (cand_id >= int_id) {
break;
}
const rose_literal_info &cand_info = (*literal_info)[cand_id];
if (cand_info.requires_benefits) {
continue;
}
if (!cand_info.delayed_ids.empty()) {
/* TODO: allow cases where delayed ids are equivalent.
* This is awkward currently as the have not had their
* final ids allocated yet */
continue;
}
if (lits.find(cand_id) == lits.end()
|| cand_info.vertices.size() != verts.size()
|| cand_info.squash_group != curr_info.squash_group) {
continue;
}
/* if we are squashing groups we need to check if they are the
* same group */
if (cand_info.squash_group
&& cand_info.group_mask != curr_info.group_mask) {
continue;
}
u32 final_id = cand_info.final_id;
assert(final_id != MO_INVALID_IDX);
assert(curr_info.final_id == MO_INVALID_IDX);
curr_info.final_id = final_id;
(*final_id_to_literal)[final_id].insert(int_id);
goto next_lit;
}
}
/* oh well, have to give it a fresh one, hang the expense */
DEBUG_PRINTF("allocating final id %u to %u\n", *next_final_id, int_id);
assert(curr_info.final_id == MO_INVALID_IDX);
curr_info.final_id = *next_final_id;
(*final_id_to_literal)[*next_final_id].insert(int_id);
(*next_final_id)++;
next_lit:;
}
}
static
bool isUsedLiteral(const RoseBuildImpl &build, u32 lit_id) {
assert(lit_id < build.literal_info.size());
const auto &info = build.literal_info[lit_id];
if (!info.vertices.empty()) {
return true;
}
for (const u32 &delayed_id : info.delayed_ids) {
assert(delayed_id < build.literal_info.size());
const rose_literal_info &delayed_info = build.literal_info[delayed_id];
if (!delayed_info.vertices.empty()) {
return true;
}
}
DEBUG_PRINTF("literal %u has no refs\n", lit_id);
return false;
}
/** \brief Allocate final literal IDs for all literals.
*
* These are the literal ids used in the bytecode.
*/
static
void allocateFinalLiteralId(RoseBuildImpl &tbi) {
RoseGraph &g = tbi.g;
set<u32> anch;
set<u32> norm;
set<u32> delay;
/* undelayed ids come first */
assert(tbi.final_id_to_literal.empty());
u32 next_final_id = 0;
for (u32 i = 0; i < tbi.literal_info.size(); i++) {
assert(!tbi.hasFinalId(i));
if (!isUsedLiteral(tbi, i)) {
/* what is this literal good for? absolutely nothing */
continue;
}
// The special EOD event literal has its own program and does not need
// a real literal ID.
if (i == tbi.eod_event_literal_id) {
assert(tbi.eod_event_literal_id != MO_INVALID_IDX);
continue;
}
if (tbi.isDelayed(i)) {
assert(!tbi.literal_info[i].requires_benefits);
delay.insert(i);
} else if (tbi.literals.right.at(i).table == ROSE_ANCHORED) {
anch.insert(i);
} else {
norm.insert(i);
}
}
/* normal lits */
allocateFinalIdToSet(g, norm, &tbi.literal_info, &tbi.final_id_to_literal,
&next_final_id);
/* next anchored stuff */
tbi.anchored_base_id = next_final_id;
allocateFinalIdToSet(g, anch, &tbi.literal_info, &tbi.final_id_to_literal,
&next_final_id);
/* delayed ids come last */
tbi.delay_base_id = next_final_id;
allocateFinalIdToSet(g, delay, &tbi.literal_info, &tbi.final_id_to_literal,
&next_final_id);
}
#define MAX_EXPLOSION_NC 3
static
bool limited_explosion(const ue2_literal &s) {
u32 nc_count = 0;
for (const auto &e : s) {
if (e.nocase) {
nc_count++;
}
}
return nc_count <= MAX_EXPLOSION_NC;
}
void RoseBuildImpl::handleMixedSensitivity(void) {
for (const auto &e : literals.right) {
u32 id = e.first;
const rose_literal_id &lit = e.second;
if (lit.delay) {
continue; /* delay id's are virtual-ish */
}
if (lit.table == ROSE_ANCHORED || lit.table == ROSE_EVENT) {
continue; /* wrong table */
}
if (!mixed_sensitivity(lit.s)) {
continue;
}
if (limited_explosion(lit.s)) {
DEBUG_PRINTF("need to explode existing string '%s'\n",
dumpString(lit.s).c_str());
literal_info[id].requires_explode = true;
} else {
literal_info[id].requires_benefits = true;
}
}
}
// Returns the length of the longest prefix of s that is (a) also a suffix of s
// and (b) not s itself.
static
size_t maxPeriod(const ue2_literal &s) {
/* overly conservative if only part of the string is nocase */
if (s.empty()) {
return 0;
}
const size_t len = s.length();
const char *begin = s.c_str(), *end = begin + len;
size_t i;
for (i = len - 1; i != 0; i--) {
if (!cmp(begin, end - i, i, s.any_nocase())) {
break;
}
}
return i;
}
bool RoseBuildImpl::isPseudoStar(const RoseEdge &e) const {
return !g[e].minBound && isPseudoStarOrFirstOnly(e);
}
bool RoseBuildImpl::isPseudoStarOrFirstOnly(const RoseEdge &e) const {
RoseVertex u = source(e, g);
RoseVertex v = target(e, g);
if (g[e].maxBound != ROSE_BOUND_INF) {
return false;
}
if (isAnyStart(u)) {
return true;
}
if (isAnchored(u)) {
/* anchored table runs out of order */
return false;
}
if (hasDelayedLiteral(u)) {
return false;
}
if (g[v].left) {
return false;
}
if (g[v].eod_accept) {
return true;
}
assert(!g[v].literals.empty());
if (maxLiteralOverlap(u, v)) {
return false;
}
return true;
}
bool RoseBuildImpl::hasOnlyPseudoStarInEdges(RoseVertex v) const {
for (const auto &e : in_edges_range(v, g)) {
if (!isPseudoStar(e)) {
return false;
}
}
return true;
}
void RoseBuildImpl::renumberVertices() {
vertexIndex = 0;
DEBUG_PRINTF("renumbering vertices\n");
for (auto v : vertices_range(g)) {
g[v].idx = vertexIndex++;
}
}
static
size_t trailerDueToSelf(const rose_literal_id &lit) {
size_t trailer = lit.s.length() - maxPeriod(lit.s);
if (trailer > 255) {
return 255;
}
if (!trailer) {
return 1;
}
return trailer;
}
static
RoseRoleHistory findHistoryScheme(const RoseBuildImpl &tbi, const RoseEdge &e) {
const RoseGraph &g = tbi.g;
const RoseVertex u = source(e, g); /* pred role */
const RoseVertex v = target(e, g); /* current role */
DEBUG_PRINTF("find history for [%zu,%zu]\n", g[u].idx, g[v].idx);
DEBUG_PRINTF("u has min_offset=%u, max_offset=%u\n", g[u].min_offset,
g[u].max_offset);
if (g[v].left) {
if (!tbi.isAnyStart(u)) {
/* infix nfa will track history, treat as pseudo .*. Note: rose lits
* may overlap so rose history track would be wrong anyway */
DEBUG_PRINTF("skipping history as prefix\n");
return ROSE_ROLE_HISTORY_NONE;
}
if (g[e].minBound || g[e].maxBound != ROSE_BOUND_INF) {
DEBUG_PRINTF("rose prefix with external bounds\n");
return ROSE_ROLE_HISTORY_ANCH;
} else {
return ROSE_ROLE_HISTORY_NONE;
}
}
// Handle EOD cases.
if (g[v].eod_accept) {
const u32 minBound = g[e].minBound, maxBound = g[e].maxBound;
DEBUG_PRINTF("EOD edge with bounds [%u,%u]\n", minBound, maxBound);
// Trivial case: we don't need history for {0,inf} bounds
if (minBound == 0 && maxBound == ROSE_BOUND_INF) {
return ROSE_ROLE_HISTORY_NONE;
}
// Event literals store no history.
if (tbi.hasLiteralInTable(u, ROSE_EVENT)) {
return ROSE_ROLE_HISTORY_NONE;
}
// Trivial case: fixed offset from anchor
if (g[u].fixedOffset()) {
return ROSE_ROLE_HISTORY_ANCH;
}
// If the bounds are {0,0}, this role can only match precisely at EOD.
if (minBound == 0 && maxBound == 0) {
return ROSE_ROLE_HISTORY_LAST_BYTE;
}
// XXX: No other history schemes should be possible any longer.
assert(0);
}
// Non-EOD cases.
DEBUG_PRINTF("examining edge [%zu,%zu] with bounds {%u,%u}\n",
g[u].idx, g[v].idx, g[e].minBound, g[e].maxBound);
if (tbi.isAnchored(v)) {
// Matches for literals in the anchored table will always arrive at the
// right offsets, so there's no need for history-based confirmation.
DEBUG_PRINTF("v in anchored table, no need for history\n");
assert(u == tbi.anchored_root);
return ROSE_ROLE_HISTORY_NONE;
}
if (g[u].fixedOffset()) {
DEBUG_PRINTF("fixed offset -> anch\n");
return ROSE_ROLE_HISTORY_ANCH;
}
return ROSE_ROLE_HISTORY_NONE;
}
static
void assignHistories(RoseBuildImpl &tbi) {
for (const auto &e : edges_range(tbi.g)) {
if (tbi.g[e].history == ROSE_ROLE_HISTORY_INVALID) {
tbi.g[e].history = findHistoryScheme(tbi, e);
}
}
}
bool RoseBuildImpl::isDirectReport(u32 id) const {
assert(id < literal_info.size());
// Literal info properties.
const rose_literal_info &info = literal_info[id];
if (info.vertices.empty()) {
return false;
}
if (!info.delayed_ids.empty() /* dr's don't set groups */
|| info.requires_benefits) { /* dr's don't require confirm */
return false;
}
if (isDelayed(id)) { /* can't handle delayed dr atm as we require delay
* ids to be dense */
return false;
}
// Role properties.
// Note that a literal can have multiple roles and still be a direct
// report; it'll become a multi-direct report ("MDR") that fires each
// role's reports from a list.
for (auto v : info.vertices) {
assert(contains(g[v].literals, id));
if (g[v].reports.empty() ||
g[v].eod_accept || // no accept EOD
!g[v].isBoring() ||
!isLeafNode(v, g) || // Must have no out-edges
in_degree(v, g) != 1) { // Role must have exactly one in-edge
return false;
}
// Use the program to handle cases that aren't external reports.
for (const ReportID &id : g[v].reports) {
if (!isExternalReport(rm.getReport(id))) {
return false;
}
}
if (literals.right.at(id).table == ROSE_ANCHORED) {
/* in-edges are irrelevant for anchored region. */
continue;
}
/* The in-edge must be an (0, inf) edge from root. */
assert(in_degree(v, g) != 0);
RoseEdge e = *(in_edges(v, g).first);
if (source(e, g) != root || g[e].minBound != 0 ||
g[e].maxBound != ROSE_BOUND_INF) {
return false;
}
// Note: we allow ekeys; they will result in unused roles being built as
// direct reporting will be used when actually matching in Rose.
/* TODO: prevent roles being created */
}
DEBUG_PRINTF("literal %u ('%s') is a %s report\n", id,
dumpString(literals.right.at(id).s).c_str(),
info.vertices.size() > 1 ? "multi-direct" : "direct");
return true;
}
static
bool checkEodStealFloating(const RoseBuildImpl &tbi,
const vector<u32> &eodLiteralsForFloating,
u32 numFloatingLiterals,
size_t shortestFloatingLen) {
if (eodLiteralsForFloating.empty()) {
DEBUG_PRINTF("no eod literals\n");
return true;
}
if (!numFloatingLiterals) {
DEBUG_PRINTF("no floating table\n");
return false;
}
if (tbi.hasNoFloatingRoots()) {
DEBUG_PRINTF("skipping as floating table is conditional\n");
/* TODO: investigate putting stuff in atable */
return false;
}
DEBUG_PRINTF("%zu are eod literals, %u floating; floating len=%zu\n",
eodLiteralsForFloating.size(), numFloatingLiterals,
shortestFloatingLen);
u32 new_floating_lits = 0;
for (u32 eod_id : eodLiteralsForFloating) {
const rose_literal_id &lit = tbi.literals.right.at(eod_id);
DEBUG_PRINTF("checking '%s'\n", dumpString(lit.s).c_str());
if (tbi.hasLiteral(lit.s, ROSE_FLOATING)) {
DEBUG_PRINTF("skip; there is already a floating version\n");
continue;
}
// Don't want to make the shortest floating literal shorter/worse.
if (trailerDueToSelf(lit) < 4 || lit.s.length() < shortestFloatingLen) {
DEBUG_PRINTF("len=%zu, selfOverlap=%zu\n", lit.s.length(),
trailerDueToSelf(lit));
DEBUG_PRINTF("would shorten, bailing\n");
return false;
}
new_floating_lits++;
}
DEBUG_PRINTF("..would require %u new floating literals\n",
new_floating_lits);
// Magic number thresholds: we only want to get rid of our EOD table if it
// would make no real difference to the FDR.
if (numFloatingLiterals / 8 < new_floating_lits
&& (new_floating_lits > 3 || numFloatingLiterals <= 2)) {
DEBUG_PRINTF("leaving eod table alone.\n");
return false;
}
return true;
}
static
void promoteEodToFloating(RoseBuildImpl &tbi, const vector<u32> &eodLiterals) {
DEBUG_PRINTF("promoting eod literals to floating table\n");
for (u32 eod_id : eodLiterals) {
const rose_literal_id &lit = tbi.literals.right.at(eod_id);
u32 floating_id = tbi.getLiteralId(lit.s, lit.msk, lit.cmp, lit.delay,
ROSE_FLOATING);
auto &float_verts = tbi.literal_info[floating_id].vertices;
auto &eod_verts = tbi.literal_info[eod_id].vertices;
insert(&float_verts, eod_verts);
eod_verts.clear();
DEBUG_PRINTF("eod_lit=%u -> float_lit=%u\n", eod_id, floating_id);
for (auto v : float_verts) {
tbi.g[v].literals.erase(eod_id);
tbi.g[v].literals.insert(floating_id);
}
tbi.literal_info[floating_id].requires_explode
= tbi.literal_info[eod_id].requires_explode;
tbi.literal_info[floating_id].requires_benefits
= tbi.literal_info[eod_id].requires_benefits;
}
}
static
bool promoteEodToAnchored(RoseBuildImpl &tbi, const vector<u32> &eodLiterals) {
DEBUG_PRINTF("promoting eod literals to anchored table\n");
bool rv = true;
for (u32 eod_id : eodLiterals) {
const rose_literal_id &lit = tbi.literals.right.at(eod_id);
NGHolder h;
add_edge(h.start, h.accept, h);
appendLiteral(h, lit.s); /* we only accept cases which are anchored
* hard up against start */
u32 a_id = tbi.getNewLiteralId();
u32 remap_id = 0;
DEBUG_PRINTF(" trying to add dfa stuff\n");
int anch_ok = addToAnchoredMatcher(tbi, h, a_id, &remap_id);
if (anch_ok == ANCHORED_FAIL) {
DEBUG_PRINTF("failed to promote to anchored need to keep etable\n");
rv = false;
continue;
} else if (anch_ok == ANCHORED_REMAP) {
DEBUG_PRINTF("remapped\n");
a_id = remap_id;
} else {
assert(anch_ok == ANCHORED_SUCCESS);
}
// Store the literal itself in a side structure so that we can use it
// for overlap calculations later. This may be obsolete when the old
// Rose construction path (and its history selection code) goes away.
tbi.anchoredLitSuffix.insert(make_pair(a_id, lit));
auto &a_verts = tbi.literal_info[a_id].vertices;
auto &eod_verts = tbi.literal_info[eod_id].vertices;
for (auto v : eod_verts) {
for (const auto &e : in_edges_range(v, tbi.g)) {
assert(tbi.g[e].maxBound != ROSE_BOUND_INF);
tbi.g[e].minBound += lit.s.length();
tbi.g[e].maxBound += lit.s.length();
}
}
insert(&a_verts, eod_verts);
eod_verts.clear();
for (auto v : a_verts) {
tbi.g[v].literals.erase(eod_id);
tbi.g[v].literals.insert(a_id);
}
}
return rv;
}
static
bool suitableForAnchored(const RoseBuildImpl &tbi, const rose_literal_id &l_id,
const rose_literal_info &lit) {
const RoseGraph &g = tbi.g;
bool seen = false;
u32 min_offset = 0;
u32 max_offset = 0;
if (!lit.delayed_ids.empty() || l_id.delay) {
DEBUG_PRINTF("delay\n");
return false;
}
if (!l_id.msk.empty()) {
DEBUG_PRINTF("msk\n");
return false;
}
for (auto v : lit.vertices) {
if (!seen) {
min_offset = g[v].min_offset;
max_offset = g[v].max_offset;
seen = true;
if (max_offset > tbi.cc.grey.maxAnchoredRegion) {
DEBUG_PRINTF("too deep %u\n", max_offset);
return false;
}
}
if (max_offset != g[v].max_offset || min_offset != g[v].min_offset) {
DEBUG_PRINTF(":(\n");
return false;
}
if (!g[v].isBoring()) {
DEBUG_PRINTF(":(\n");
return false;
}
if (g[v].literals.size() != 1) {
DEBUG_PRINTF("shared\n");
return false;
}
if (tbi.isNonRootSuccessor(v)) {
DEBUG_PRINTF("non root\n");
return false;
}
if (max_offset != l_id.s.length() || min_offset != l_id.s.length()) {
DEBUG_PRINTF("|%zu| (%u,%u):(\n", l_id.s.length(), min_offset,
max_offset);
/* TODO: handle cases with small bounds */
return false;
}
for (auto w : adjacent_vertices_range(v, g)) {
if (!g[w].eod_accept) {
DEBUG_PRINTF("non eod accept literal\n");
return false;
}
}
}
return true;
}
// If we've got a small number of long, innocuous EOD literals and a large
// floating table, we consider promoting those EOD literals to the floating
// table to avoid having to run both. See UE-2069, consider deleting this and
// replacing with an elegant reverse DFA.
/* We do not want to do this if we would otherwise avoid running the floating
* table altogether. */
static
void stealEodVertices(RoseBuildImpl &tbi) {
u32 numFloatingLiterals = 0;
u32 numAnchoredLiterals = 0;
size_t shortestFloatingLen = SIZE_MAX;
vector<u32> eodLiteralsForFloating;
vector<u32> eodLiteralsForAnchored;
DEBUG_PRINTF("hi\n");
for (u32 i = 0; i < tbi.literal_info.size(); i++) {
const auto &info = tbi.literal_info[i];
if (info.vertices.empty()) {
continue; // skip unused literals
}
const rose_literal_id &lit = tbi.literals.right.at(i);
if (lit.table == ROSE_EOD_ANCHORED) {
if (suitableForAnchored(tbi, lit, info)) {
eodLiteralsForAnchored.push_back(i);
} else {
eodLiteralsForFloating.push_back(i);
}
} else if (lit.table == ROSE_FLOATING) {
numFloatingLiterals++;
shortestFloatingLen = min(shortestFloatingLen, lit.s.length());
} else if (lit.table == ROSE_ANCHORED) {
numAnchoredLiterals++;
}
}
/* given a choice of having either an eod table or an anchored table, we
* always favour having an anchored table */
if (!checkEodStealFloating(tbi, eodLiteralsForFloating, numFloatingLiterals,
shortestFloatingLen)) {
DEBUG_PRINTF("removing etable weakens ftable\n");
return;
}
promoteEodToFloating(tbi, eodLiteralsForFloating);
if (!promoteEodToAnchored(tbi, eodLiteralsForAnchored)) {
DEBUG_PRINTF("still need ematcher\n");
return;
}
// We're no longer using the EOD matcher.
tbi.ematcher_region_size = 0;
}
bool RoseBuildImpl::isDelayed(u32 id) const {
return literal_info.at(id).undelayed_id != id;
}
bool RoseBuildImpl::hasFinalId(u32 id) const {
return literal_info.at(id).final_id != MO_INVALID_IDX;
}
bool RoseBuildImpl::hasDelayedLiteral(RoseVertex v) const {
for (u32 lit_id : g[v].literals) {
if (literals.right.at(lit_id).delay) {
return true;
}
}
return false;
}
bool RoseBuildImpl::hasDelayPred(RoseVertex v) const {
for (auto u : inv_adjacent_vertices_range(v, g)) {
if (hasDelayedLiteral(u)) {
return true;
}
}
return false;
}
bool RoseBuildImpl::hasAnchoredTablePred(RoseVertex v) const {
for (auto u : inv_adjacent_vertices_range(v, g)) {
if (isAnchored(u)) {
return true;
}
}
return false;
}
void RoseBuildImpl::findTransientLeftfixes(void) {
for (auto v : vertices_range(g)) {
if (!g[v].left) {
continue;
}
/* infixes can never (or at least not yet) be transient */
if (isNonRootSuccessor(v)) {
continue;
}
const left_id &left(g[v].left);
if (::ue2::isAnchored(left) && !isInETable(v)) {
/* etable prefixes currently MUST be transient as we do not know
* where we can safely catch them up to (yet). */
DEBUG_PRINTF("anchored roses in rocky soil are not fleeting\n");
continue;
}
const depth max_width = findMaxWidth(left);
if (!max_width.is_finite()) {
DEBUG_PRINTF("inf max width\n");
continue;
}
u32 his = g[v].left.lag + max_width;
// If this vertex has an event literal, we need to add one to cope
// with it.
if (hasLiteralInTable(v, ROSE_EVENT)) {
his++;
}
/* +1 as trigger must appear in main buffer and no byte is needed to
* decompress the state */
if (his <= cc.grey.maxHistoryAvailable + 1) {
transient.insert(left);
DEBUG_PRINTF("a transient leftfix has been spotted his=%u\n", his);
}
}
}
/** Find all the different roses and their associated literals. */
static
map<left_id, vector<RoseVertex>> findLeftSucc(RoseBuildImpl &tbi) {
map<left_id, vector<RoseVertex>> leftfixes;
for (auto v : vertices_range(tbi.g)) {
if (tbi.g[v].left) {
const LeftEngInfo &lei = tbi.g[v].left;
leftfixes[lei].push_back(v);
}
}
return leftfixes;
}
static
bool triggerKillsRoseGraph(const RoseBuildImpl &tbi, const left_id &left,
const set<ue2_literal> &all_lits,
const RoseEdge &e) {
assert(left.graph());
const NGHolder &h = *left.graph();
ue2::flat_set<NFAVertex> all_states;
insert(&all_states, vertices(h));
assert(out_degree(h.startDs, h) == 1); /* triggered don't use sds */
DEBUG_PRINTF("removing sds\n");
all_states.erase(h.startDs);
ue2::flat_set<NFAVertex> states;
/* check each pred literal to see if they all kill previous graph
* state */
for (u32 lit_id : tbi.g[source(e, tbi.g)].literals) {
const rose_literal_id &pred_lit = tbi.literals.right.at(lit_id);
const ue2_literal s = findNonOverlappingTail(all_lits, pred_lit.s);
DEBUG_PRINTF("running graph %zu\n", states.size());
states = execute_graph(h, s, all_states, true);
DEBUG_PRINTF("ran, %zu states on\n", states.size());
if (!states.empty()) {
return false;
}
}
return true;
}
static
bool triggerKillsRose(const RoseBuildImpl &tbi, const left_id &left,
const set<ue2_literal> &all_lits, const RoseEdge &e) {
if (left.haig()) {
/* TODO: To allow this for som-based engines we would also need to
* ensure as well that no other triggers can occur at the same location
* with a different som. */
return false;
}
if (left.graph()) {
return triggerKillsRoseGraph(tbi, left, all_lits, e);
}
if (left.castle()) {
return triggerKillsRoseCastle(tbi, left, all_lits, e);
}
return false;
}
static
void inspectRoseTops(RoseBuildImpl &tbi) {
/* Sometimes the arrival of a top for a rose infix can ensure that the nfa
* would be dead at that time. In the case of multiple trigger literals we
* can only base our decision on that portion of literal after any
* overlapping literals */
map<left_id, vector<RoseVertex>> roses =
findLeftSucc(tbi); /* rose -> succ verts */
for (const auto &r : roses) {
const left_id &left = r.first;
const vector<RoseVertex> &succs = r.second;
assert(!succs.empty());
if (tbi.isRootSuccessor(*succs.begin())) {
/* a prefix is never an infix */
continue;
}
set<u32> tops_seen;
set<RoseEdge> rose_edges;
set<u32> pred_lit_ids;
for (auto v : succs) {
for (const auto &e : in_edges_range(v, tbi.g)) {
RoseVertex u = source(e, tbi.g);
tops_seen.insert(tbi.g[e].rose_top);
insert(&pred_lit_ids, tbi.g[u].literals);
rose_edges.insert(e);
}
}
set<ue2_literal> all_lits;
if (tops_seen.size() > 1) {
goto next_rose; /* slightly tricky to deal with overlap case */
}
for (u32 lit_id : pred_lit_ids) {
const rose_literal_id &p_lit = tbi.literals.right.at(lit_id);
if (p_lit.delay || p_lit.table == ROSE_ANCHORED) {
goto next_rose;
}
all_lits.insert(p_lit.s);
DEBUG_PRINTF("trigger: '%s'\n", dumpString(p_lit.s).c_str());
}
DEBUG_PRINTF("rose has %zu trigger literals, %zu edges\n",
all_lits.size(), rose_edges.size());
for (const auto &e : rose_edges) {
if (triggerKillsRose(tbi, left, all_lits, e)) {
DEBUG_PRINTF("top will override previous rose state\n");
tbi.g[e].rose_cancel_prev_top = true;
}
}
next_rose:;
}
}
static
void buildRoseSquashMasks(RoseBuildImpl &tbi) {
/* Rose nfa squash masks are applied to the groups when the nfa can no
* longer match */
map<left_id, vector<RoseVertex>> roses =
findLeftSucc(tbi); /* rose -> succ verts */
/* a rose nfa can squash a group if all literals in that group are a
* successor of the nfa and all the literals */
for (const auto &e : roses) {
const left_id &left = e.first;
const vector<RoseVertex> &succs = e.second;
set<u32> lit_ids;
bool anchored_pred = false;
for (auto v : succs) {
lit_ids.insert(tbi.g[v].literals.begin(), tbi.g[v].literals.end());
for (auto u : inv_adjacent_vertices_range(v, tbi.g)) {
anchored_pred |= tbi.isAnchored(u);
}
}
/* Due to the anchored table not being able to set groups again,
* we cannot use a rose nfa for group squashing if it is being triggered
* from the anchored table and can match more than once. */
if (anchored_pred) { /* infix with pred in anchored table */
u32 min_off = ~0U;
u32 max_off = 0U;
for (auto v : succs) {
for (auto u : inv_adjacent_vertices_range(v, tbi.g)) {
min_off = min(min_off, tbi.g[u].min_offset);
max_off = max(max_off, tbi.g[u].max_offset);
}
}
if (min_off != max_off) {
/* leave all groups alone */
tbi.rose_squash_masks[left] = ~0ULL;
continue;
}
}
rose_group unsquashable = 0;
for (u32 lit_id : lit_ids) {
const rose_literal_info &info = tbi.literal_info[lit_id];
if (info.vertices.size() > 1 || !info.delayed_ids.empty()) {
unsquashable |= info.group_mask;
}
}
rose_group squash_mask = ~0ULL; /* leave all groups alone */
for (u32 i = 0; i < ROSE_GROUPS_MAX; i++) {
if (is_subset_of(tbi.group_to_literal[i], lit_ids)) {
squash_mask &= ~(1ULL << i);
}
}
squash_mask |= unsquashable;
tbi.rose_squash_masks[left] = squash_mask;
}
}
static
void countFloatingLiterals(const RoseBuildImpl &tbi, u32 *total_count,
u32 *short_count) {
*total_count = 0;
*short_count = 0;
for (const rose_literal_id &lit : tbi.literals.right | map_values) {
if (lit.delay) {
continue; /* delay id's are virtual-ish */
}
if (lit.table != ROSE_FLOATING) {
continue; /* wrong table */
}
++*total_count;
if (lit.s.length() <= ANCHORED_REHOME_SHORT_LEN) {
++*short_count;
}
}
}
static
void rehomeAnchoredLiteral(RoseBuildImpl &tbi, const simple_anchored_info &sai,
const set<u32> &lit_ids) {
/* TODO: verify that vertices only have a single literal at the moment */
DEBUG_PRINTF("rehoming ^.{%u,%u}%s\n", sai.min_bound, sai.max_bound,
dumpString(sai.literal).c_str());
/* Get a floating literal corresponding to the anchored literal */
u32 new_literal_id = tbi.getLiteralId(sai.literal, 0, ROSE_FLOATING);
rose_literal_info &new_lit_info = tbi.literal_info[new_literal_id];
DEBUG_PRINTF("floating literal id -> %u\n", new_literal_id);
for (u32 lit_id : lit_ids) {
rose_literal_info &old_lit_info = tbi.literal_info[lit_id];
assert(old_lit_info.delayed_ids.empty());
for (auto v : old_lit_info.vertices) {
/* Transfer vertex over to new literal id */
assert(tbi.g[v].literals.size() == 1);
tbi.g[v].literals.clear();
tbi.g[v].literals.insert(new_literal_id);
new_lit_info.vertices.insert(v);
/* ensure bounds on the vertex's in-edge are correct */
assert(in_degree(v, tbi.g) == 1);
const RoseEdge &e = *in_edges(v, tbi.g).first;
assert(tbi.g[e].minBound == sai.min_bound + sai.literal.length());
assert(tbi.g[e].maxBound == sai.max_bound + sai.literal.length());
tbi.g[e].minBound = sai.min_bound;
tbi.g[e].maxBound = sai.max_bound;
}
/* mark the old literal as empty */
old_lit_info.vertices.clear();
}
}
static
void rehomeAnchoredLiterals(RoseBuildImpl &tbi) {
/* if we have many literals in the floating table, we want to push
* literals which are anchored but deep into the floating table as they
* are unlikely to reduce the performance of the floating table. */
u32 total_count;
u32 short_count;
countFloatingLiterals(tbi, &total_count, &short_count);
DEBUG_PRINTF("considering rehoming options\n");
if (total_count < ANCHORED_REHOME_MIN_FLOATING
&& short_count < ANCHORED_REHOME_MIN_FLOATING_SHORT) {
DEBUG_PRINTF("not a heavy case %u %u\n", total_count, short_count);
return;
}
u32 min_rehome_len = ANCHORED_REHOME_SHORT_LEN + 1;
if (short_count >= ANCHORED_REHOME_ALLOW_SHORT) {
min_rehome_len--;
}
for (map<simple_anchored_info, set<u32> >::iterator it
= tbi.anchored_simple.begin();
it != tbi.anchored_simple.end();) {
if (it->first.max_bound < ANCHORED_REHOME_DEEP
|| it->first.literal.length() < min_rehome_len) {
++it;
continue;
}
rehomeAnchoredLiteral(tbi, it->first, it->second);
tbi.anchored_simple.erase(it++);
}
}
/** \brief Maximum number of single-byte literals to add to the small block
* table. */
static const size_t MAX_1BYTE_SMALL_BLOCK_LITERALS = 20;
static
void addSmallBlockLiteral(RoseBuildImpl &tbi, const simple_anchored_info &sai,
const set<u32> &lit_ids) {
DEBUG_PRINTF("anchored ^.{%u,%u}%s\n", sai.min_bound, sai.max_bound,
dumpString(sai.literal).c_str());
u32 lit_id = tbi.getLiteralId(sai.literal, 0, ROSE_ANCHORED_SMALL_BLOCK);
rose_literal_info &lit_info = tbi.literal_info[lit_id];
DEBUG_PRINTF("anchored small block literal id -> %u\n", lit_id);
RoseGraph &g = tbi.g;
const RoseVertex anchored_root = tbi.anchored_root;
for (u32 old_id : lit_ids) {
assert(old_id < tbi.literal_info.size());
const rose_literal_info &li = tbi.literal_info[old_id];
// For compile determinism, operate over literal vertices in index order.
vector<RoseVertex> lit_verts(begin(li.vertices), end(li.vertices));
sort(begin(lit_verts), end(lit_verts), VertexIndexComp(g));
for (auto lit_v : lit_verts) {
// Clone vertex with the new literal ID.
RoseVertex v = add_vertex(g[lit_v], g);
g[v].idx = tbi.vertexIndex++;
g[v].literals.clear();
g[v].literals.insert(lit_id);
g[v].min_offset = sai.min_bound + sai.literal.length();
g[v].max_offset = sai.max_bound + sai.literal.length();
lit_info.vertices.insert(v);
assert(!g[v].reports.empty());
bool doDirectReports = true;
for (ReportID report_id : g[v].reports) {
const Report &old_rep = tbi.rm.getReport(report_id);
if (!isExternalReport(old_rep) || old_rep.hasBounds()) {
doDirectReports = false;
break;
}
}
if (doDirectReports) {
flat_set<ReportID> dr_reports;
for (ReportID report_id : g[v].reports) {
// These new literal roles can be made direct reports, with
// their bounds handled by the bounds on their Report
// structures.
Report rep(tbi.rm.getReport(report_id)); // copy
assert(!rep.hasBounds());
rep.minOffset = sai.literal.length() + sai.min_bound;
rep.maxOffset = sai.literal.length() + sai.max_bound;
dr_reports.insert(tbi.rm.getInternalId(rep));
}
g[v].reports = dr_reports;
RoseEdge e = add_edge(tbi.root, v, g).first;
g[e].minBound = 0; // handled by internal_report
g[e].maxBound = ROSE_BOUND_INF; // handled by internal_report
} else {
// If we have a complex internal report, these must become
// anchored literals with their own roles.
RoseEdge e = add_edge(anchored_root, v, g).first;
g[e].minBound = sai.min_bound;
g[e].maxBound = sai.max_bound;
}
}
}
}
static
void addSmallBlockLiteral(RoseBuildImpl &tbi, const ue2_literal &lit,
const flat_set<ReportID> &reports) {
DEBUG_PRINTF("lit %s, reports: %s\n", dumpString(lit).c_str(),
as_string_list(reports).c_str());
assert(!reports.empty());
u32 lit_id = tbi.getLiteralId(lit, 0, ROSE_ANCHORED_SMALL_BLOCK);
assert(lit_id < tbi.literal_info.size());
rose_literal_info &lit_info = tbi.literal_info[lit_id];
RoseGraph &g = tbi.g;
RoseVertex v = add_vertex(g);
g[v].idx = tbi.vertexIndex++;
g[v].literals.insert(lit_id);
g[v].reports = reports;
RoseEdge e = add_edge(tbi.root, v, g).first;
g[e].minBound = 0;
g[e].maxBound = ROSE_BOUND_INF;
g[v].min_offset = 1;
g[v].max_offset = ROSE_BOUND_INF;
lit_info.vertices.insert(v);
}
static
bool stateIsSEPLiteral(const dstate_id_t &s, const symbol_t &sym,
const raw_dfa &rdfa) {
const dstate &ds = rdfa.states[s];
if (!ds.reports_eod.empty() || ds.reports.empty()) {
DEBUG_PRINTF("badly formed reports\n");
return false;
}
DEBUG_PRINTF("examine state %u reached by sym %u\n", s, sym);
for (symbol_t i = 0; i < rdfa.getImplAlphaSize(); i++) {
const auto &s_next = ds.next[i];
DEBUG_PRINTF("state %u -> %u on sym %u\n", s, s_next, i);
if (s_next == DEAD_STATE) {
continue; // dead, probably pruned
} else if (s_next == s && i == sym) {
continue; // self loop on same symbol
} else if (s_next == rdfa.start_floating) {
continue; // return to floating start
}
// We don't handle any other transitions.
DEBUG_PRINTF("not single-byte\n");
return false;
}
return true;
}
static
bool extractSEPLiterals(const raw_dfa &rdfa,
map<ue2_literal, flat_set<ReportID>> &lits_out) {
if (rdfa.start_floating == DEAD_STATE) {
DEBUG_PRINTF("not floating?\n");
return false;
}
if (rdfa.start_anchored != rdfa.start_floating) {
DEBUG_PRINTF("not all floating?\n");
return false;
}
map<flat_set<ReportID>, vector<u32>> lits; // reports -> symbols
const dstate &start = rdfa.states[rdfa.start_floating];
const symbol_t alpha_size = rdfa.getImplAlphaSize();
for (symbol_t i = 0; i < alpha_size; i++) {
auto next = start.next[i];
if (next == DEAD_STATE || next == rdfa.start_floating) {
continue;
}
if (!stateIsSEPLiteral(next, i, rdfa)) {
return false;
}
lits[rdfa.states[next].reports].push_back(i);
}
// Map from symbols back to character reachability.
vector<CharReach> reach(alpha_size);
for (u32 i = 0; i < N_CHARS; i++) {
assert(rdfa.alpha_remap[i] < alpha_size);
reach[rdfa.alpha_remap[i]].set(i);
}
for (const auto &m : lits) {
const auto &reports = m.first;
const auto &symbols = m.second;
CharReach cr;
for (const auto &sym : symbols) {
cr |= reach[sym];
}
for (size_t i = cr.find_first(); i != cr.npos; i = cr.find_next(i)) {
if (myisupper(i) && cr.test(mytolower(i))) {
// ignore upper half of a nocase pair
continue;
}
bool nocase = myislower(i) && cr.test(mytoupper(i));
insert(&lits_out[ue2_literal((char)i, nocase)], reports);
}
}
return true;
}
static
bool extractSEPLiterals(const OutfixInfo &outfix, const ReportManager &rm,
map<ue2_literal, flat_set<ReportID>> &lits_out) {
if (outfix.minWidth != depth(1) || outfix.maxWidth != depth(1)) {
DEBUG_PRINTF("outfix must be fixed width of one\n");
return false;
}
for (const auto &report_id : all_reports(outfix)) {
const auto &report = rm.getReport(report_id);
if (!isSimpleExhaustible(report)) {
DEBUG_PRINTF("report id %u not simple exhaustible\n", report_id);
return false;
}
}
// SEP cases should always become DFAs, so that's the only extract code we
// have implemented here.
if (outfix.rdfa()) {
return extractSEPLiterals(*outfix.rdfa(), lits_out);
}
DEBUG_PRINTF("cannot extract literals from outfix type\n");
return false;
}
static
void addAnchoredSmallBlockLiterals(RoseBuildImpl &tbi) {
if (tbi.cc.streaming) {
DEBUG_PRINTF("not block mode\n");
return;
}
if (!tbi.anchored_nfas.empty()) {
DEBUG_PRINTF("anchored table is not purely literal\n");
return;
}
// At the moment, we only use the small-block matcher if all our anchored
// literals are direct reports (i.e. leaf nodes in the Rose graph).
for (const set<u32> &lits : tbi.anchored_simple | map_values) {
for (u32 lit_id : lits) {
if (!tbi.isDirectReport(lit_id)) {
DEBUG_PRINTF("not all anchored lits are direct reports\n");
return;
}
}
}
vector<pair<simple_anchored_info, set<u32> > > anchored_lits;
vector<OutfixInfo *> sep_outfixes;
size_t oneByteLiterals = 0;
for (const auto &e : tbi.anchored_simple) {
const simple_anchored_info &sai = e.first;
const set<u32> &lit_ids = e.second;
if (sai.literal.length() + sai.min_bound > ROSE_SMALL_BLOCK_LEN) {
DEBUG_PRINTF("skipping literal '%s' with min bound %u that cannot "
"match inside small block width\n",
dumpString(sai.literal).c_str(), sai.min_bound);
}
anchored_lits.push_back(make_pair(sai, lit_ids));
if (sai.literal.length() == 1) {
oneByteLiterals++;
}
}
// Capture SEP outfixes as well, adding them as literals to the small block
// table.
map<ue2_literal, flat_set<ReportID>> sep_literals;
for (OutfixInfo &oi : tbi.outfixes) {
if (extractSEPLiterals(oi, tbi.rm, sep_literals)) {
sep_outfixes.push_back(&oi);
}
}
oneByteLiterals += sep_literals.size();
DEBUG_PRINTF("%zu one-byte literals\n", oneByteLiterals);
if (oneByteLiterals > MAX_1BYTE_SMALL_BLOCK_LITERALS) {
DEBUG_PRINTF("too many one-byte literals, not building small block "
"table!\n");
return;
}
for (const auto &e : tbi.anchored_simple) {
const simple_anchored_info &sai = e.first;
const set<u32> &lit_ids = e.second;
addSmallBlockLiteral(tbi, sai, lit_ids);
}
for (const auto &m : sep_literals) {
addSmallBlockLiteral(tbi, m.first, m.second);
}
for (OutfixInfo *oi : sep_outfixes) {
assert(oi);
oi->in_sbmatcher = true;
}
}
#ifndef NDEBUG
static
bool historiesAreValid(const RoseGraph &g) {
for (const auto &e : edges_range(g)) {
if (g[e].history == ROSE_ROLE_HISTORY_INVALID) {
DEBUG_PRINTF("edge [%zu,%zu] has invalid history\n",
g[source(e, g)].idx, g[target(e, g)].idx);
return false;
}
}
return true;
}
/**
* Assertion: Returns true if we have a reference hanging around to a vertex
* that no longer exists in the graph.
*/
static
bool danglingVertexRef(RoseBuildImpl &tbi) {
RoseGraph::vertex_iterator vi, ve;
tie(vi, ve) = vertices(tbi.g);
const ue2::unordered_set<RoseVertex> valid_vertices(vi, ve);
if (!contains(valid_vertices, tbi.anchored_root)) {
DEBUG_PRINTF("anchored root vertex %p not in graph\n",
tbi.anchored_root);
return true;
}
for (const auto &e : tbi.ghost) {
if (!contains(valid_vertices, e.first)) {
DEBUG_PRINTF("ghost key vertex %p not in graph\n", e.first);
return true;
}
if (!contains(valid_vertices, e.second)) {
DEBUG_PRINTF("ghost value vertex %p not in graph\n", e.second);
return true;
}
}
return false;
}
static
bool roleOffsetsAreValid(const RoseGraph &g) {
for (auto v : vertices_range(g)) {
if (g[v].min_offset >= ROSE_BOUND_INF) {
DEBUG_PRINTF("invalid min_offset for role %zu\n", g[v].idx);
return false;
}
if (g[v].min_offset > g[v].max_offset) {
DEBUG_PRINTF("min_offset > max_offset for %zu\n", g[v].idx);
return false;
}
}
return true;
}
static UNUSED
bool hasOrphanedTops(const RoseBuildImpl &tbi) {
const RoseGraph &g = tbi.g;
ue2::unordered_map<left_id, set<u32> > roses;
ue2::unordered_map<suffix_id, set<u32> > suffixes;
for (auto v : vertices_range(g)) {
if (g[v].left) {
set<u32> &tops = roses[g[v].left];
if (tbi.isRootSuccessor(v)) {
// Prefix, has only one top.
tops.insert(0);
} else {
// Tops for infixes come from the in-edges.
for (const auto &e : in_edges_range(v, g)) {
tops.insert(g[e].rose_top);
}
}
}
if (g[v].suffix) {
suffixes[g[v].suffix].insert(g[v].suffix.top);
}
}
for (const auto &e : roses) {
if (all_tops(e.first) != e.second) {
DEBUG_PRINTF("rose tops (%s) don't match rose graph (%s)\n",
as_string_list(all_tops(e.first)).c_str(),
as_string_list(e.second).c_str());
return true;
}
}
for (const auto &e : suffixes) {
if (all_tops(e.first) != e.second) {
DEBUG_PRINTF("suffix tops (%s) don't match rose graph (%s)\n",
as_string_list(all_tops(e.first)).c_str(),
as_string_list(e.second).c_str());
return true;
}
}
return false;
}
#endif // NDEBUG
aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildRose(u32 minWidth) {
dumpRoseGraph(*this, nullptr, "rose_early.dot");
// Early check for Rose implementability.
assert(canImplementGraphs(*this));
// Sanity check vertex role offsets.
assert(roleOffsetsAreValid(g));
convertPrefixToBounds(*this);
// Turn flood-prone suffixes into suffix NFAs.
convertFloodProneSuffixes(*this);
// Turn repeats into Castle prototypes.
makeCastles(*this);
rehomeAnchoredLiterals(*this);
// If we've got a very small number of EOD-anchored literals, consider
// moving them into the floating table so that we only have one literal
// matcher to run. Note that this needs to happen before
// addAnchoredSmallBlockLiterals as it may create anchored literals.
assert(roleOffsetsAreValid(g));
stealEodVertices(*this);
addAnchoredSmallBlockLiterals(*this);
// Merge duplicate leaf nodes
dedupeSuffixes(*this);
if (cc.grey.roseGraphReduction) {
mergeDupeLeaves(*this);
uncalcLeaves(*this);
}
assert(roleOffsetsAreValid(g));
handleMixedSensitivity();
assignHistories(*this);
convertAnchPrefixToBounds(*this);
// Do some final graph reduction.
dedupeLeftfixes(*this);
aliasRoles(*this, false); // Don't merge leftfixes.
dedupeLeftfixes(*this);
convertBadLeaves(*this);
uncalcLeaves(*this);
/* note the leftfixes which do not need to keep state across stream
boundaries */
findTransientLeftfixes();
dedupeLeftfixesVariableLag(*this);
mergeLeftfixesVariableLag(*this);
mergeSmallLeftfixes(*this);
mergeCastleLeftfixes(*this);
// Do a rose-merging aliasing pass.
aliasRoles(*this, true);
// Merging of suffixes _below_ role aliasing, as otherwise we'd have to
// teach role aliasing about suffix tops.
mergeCastleSuffixes(*this);
mergePuffixes(*this);
mergeAcyclicSuffixes(*this);
mergeSmallSuffixes(*this);
// Convert Castles that would be better off as NFAs back to NGHolder
// infixes/suffixes.
if (unmakeCastles(*this)) {
// We may be able to save some stream state by merging the newly
// "unmade" Castles.
mergeSmallSuffixes(*this);
mergeSmallLeftfixes(*this);
}
// Do a rose-merging aliasing pass.
aliasRoles(*this, true);
// Run a merge pass over the outfixes as well.
mergeOutfixes(*this);
assert(!danglingVertexRef(*this));
findMoreLiteralMasks(*this);
assignGroupsToLiterals(*this);
assignGroupsToRoles(*this);
findGroupSquashers(*this);
/* final prep work */
remapCastleTops(*this);
allocateFinalLiteralId(*this);
inspectRoseTops(*this);
buildRoseSquashMasks(*this);
rm.assignDkeys(this);
/* transfer mpv outfix to main queue */
if (mpv_outfix) {
outfixes.push_back(move(*mpv_outfix));
mpv_outfix = nullptr;
}
assert(canImplementGraphs(*this));
assert(!hasOrphanedTops(*this));
assert(roleOffsetsAreValid(g));
assert(historiesAreValid(g));
dumpRoseGraph(*this, nullptr, "rose_pre_norm.dot");
return buildFinalEngine(minWidth);
}
} // namespace ue2
Reference in New Issue View Git Blame Copy Permalink