github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-09-29 19:24:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

allow sets of tops on edges

Browse Source
This commit is contained in:
Alex Coyte
2016-09-01 14:58:55 +10:00
committed by Matthew Barr
parent aca89e66d2
commit c94899dd44
31 changed files with 284 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/rose/rose_build_add.cpp
View File

@@ -1619,6 +1619,8 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter,
}
NGHolder *h = in[e].graph.get();
assert(isCorrectlyTopped(*h));
if (!contains(graphs, h)) {
ordered_graphs.push_back(h);
}

4
src/rose/rose_build_add_mask.cpp
View File

@@ -574,7 +574,8 @@ unique_ptr<NGHolder> buildMaskRhs(const ue2::flat_set<ReportID> &reports,
succ = u;
}
add_edge(h.start, succ, h);
NFAEdge e = add_edge(h.start, succ, h).first;
h[e].tops.insert(DEFAULT_TOP);
return rhs;
}
@@ -632,6 +633,7 @@ void doAddMask(RoseBuildImpl &tbi, bool anchored,
= buildMaskLhs(true, minBound - prefix2_len + overlap,
mask3);
mhs->kind = NFA_INFIX;
setTops(*mhs);
add_edge(u, v, RoseInEdgeProps(mhs, delay), ig);
DEBUG_PRINTF("add anch literal too!\n");

12
src/rose/rose_build_bytecode.cpp
View File

@@ -512,7 +512,7 @@ bool nfaStuckOn(const NGHolder &g) {
set<u32> done_tops;
for (const auto &e : out_edges_range(g.start, g)) {
tops.insert(g[e].top);
insert(&tops, g[e].tops);
if (!g[target(e, g)].char_reach.all()) {
continue;
}
@@ -521,7 +521,7 @@ bool nfaStuckOn(const NGHolder &g) {
insert(&asucc, adjacent_vertices(target(e, g), g));
if (asucc == succ) {
done_tops.insert(g[e].top);
insert(&done_tops, g[e].tops);
}
}
@@ -842,8 +842,8 @@ makeLeftNfa(const RoseBuildImpl &tbi, left_id &left,
if (!n && !is_prefix && left.graph() && onlyOneTop(*left.graph())) {
map<u32, vector<vector<CharReach> > > triggers;
findTriggerSequences(tbi, infixTriggers.at(left), &triggers);
assert(contains(triggers, 0)); // single top
n = constructLBR(*left.graph(), triggers[0], cc, rm);
assert(triggers.size() == 1); // single top
n = constructLBR(*left.graph(), triggers.begin()->second, cc, rm);
}
if (!n && left.graph()) {
@@ -1435,7 +1435,7 @@ void findExclusiveInfixes(RoseBuildImpl &build, build_context &bc,
// Sanity check: our NFA should contain each of the tops mentioned on
// our in-edges.
assert(roseHasTops(g, v));
assert(roseHasTops(build, v));
if (contains(leftfixes, leftfix)) {
// NFA already built.
@@ -1504,7 +1504,7 @@ bool buildLeftfixes(RoseBuildImpl &tbi, build_context &bc,
// Sanity check: our NFA should contain each of the tops mentioned on
// our in-edges.
assert(roseHasTops(g, v));
assert(roseHasTops(tbi, v));
bool is_transient = contains(tbi.transient, leftfix);

52
src/rose/rose_build_compile.cpp
View File

@@ -43,7 +43,6 @@
#include "nfa/nfa_internal.h"
#include "nfa/rdfa.h"
#include "nfagraph/ng_holder.h"
#include "nfagraph/ng_dump.h"
#include "nfagraph/ng_execute.h"
#include "nfagraph/ng_is_equal.h"
#include "nfagraph/ng_limex.h"
@@ -1554,53 +1553,6 @@ bool roleOffsetsAreValid(const RoseGraph &g) {
}
return true;
}
static UNUSED
bool hasOrphanedTops(const RoseBuildImpl &tbi) {
const RoseGraph &g = tbi.g;
ue2::unordered_map<left_id, set<u32> > roses;
ue2::unordered_map<suffix_id, set<u32> > suffixes;
for (auto v : vertices_range(g)) {
if (g[v].left) {
set<u32> &tops = roses[g[v].left];
if (tbi.isRootSuccessor(v)) {
// Prefix, has only one top.
tops.insert(0);
} else {
// Tops for infixes come from the in-edges.
for (const auto &e : in_edges_range(v, g)) {
tops.insert(g[e].rose_top);
}
}
}
if (g[v].suffix) {
suffixes[g[v].suffix].insert(g[v].suffix.top);
}
}
for (const auto &e : roses) {
if (all_tops(e.first) != e.second) {
DEBUG_PRINTF("rose tops (%s) don't match rose graph (%s)\n",
as_string_list(all_tops(e.first)).c_str(),
as_string_list(e.second).c_str());
return true;
}
}
for (const auto &e : suffixes) {
if (all_tops(e.first) != e.second) {
DEBUG_PRINTF("suffix tops (%s) don't match rose graph (%s)\n",
as_string_list(all_tops(e.first)).c_str(),
as_string_list(e.second).c_str());
return true;
}
}
return false;
}
#endif // NDEBUG
aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildRose(u32 minWidth) {
@@ -1681,13 +1633,17 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildRose(u32 minWidth) {
mergeSmallLeftfixes(*this);
}
assert(!hasOrphanedTops(*this));
// Do a rose-merging aliasing pass.
aliasRoles(*this, true);
assert(!hasOrphanedTops(*this));
// Run a merge pass over the outfixes as well.
mergeOutfixes(*this);
assert(!danglingVertexRef(*this));
assert(!hasOrphanedTops(*this));
findMoreLiteralMasks(*this);

7
src/rose/rose_build_convert.cpp
View File

@@ -163,6 +163,8 @@ unique_ptr<NGHolder> convertLeafToHolder(const RoseGraph &g,
}
}
setTops(*out);
// Literal vertices wired to accept.
NFAVertex litfirst, litlast;
tie(litfirst, litlast) = addLiteralVertices(g, literals, t_v, *out);
@@ -400,7 +402,10 @@ unique_ptr<NGHolder> makeFloodProneSuffix(const ue2_literal &s, size_t len,
NFAVertex u = h->start;
for (auto it = s.begin() + s.length() - len; it != s.end(); ++it) {
NFAVertex v = addHolderVertex(*it, *h);
add_edge(u, v, *h);
NFAEdge e = add_edge(u, v, *h).first;
if (u == h->start) {
(*h)[e].tops.insert(DEFAULT_TOP);
}
u = v;
}

3
src/rose/rose_build_impl.h
View File

@@ -615,7 +615,8 @@ ue2_literal findNonOverlappingTail(const std::set<ue2_literal> &lits,
void setReportId(NGHolder &g, ReportID id);
#ifndef NDEBUG
bool roseHasTops(const RoseGraph &g, RoseVertex v);
bool roseHasTops(const RoseBuildImpl &build, RoseVertex v);
bool hasOrphanedTops(const RoseBuildImpl &build);
#endif
u64a findMaxOffset(const std::set<ReportID> &reports, const ReportManager &rm);

17
src/rose/rose_build_infix.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -108,15 +108,10 @@ void contractVertex(NGHolder &g, NFAVertex v,
}
static
u32 findMaxInfixMatches(const NGHolder &h, const set<ue2_literal> &lits) {
u32 findMaxLiteralMatches(const NGHolder &h, const set<ue2_literal> &lits) {
DEBUG_PRINTF("h=%p, %zu literals\n", &h, lits.size());
//dumpGraph("infix.dot", h.g);
if (!onlyOneTop(h)) {
DEBUG_PRINTF("more than one top!n");
return NO_MATCH_LIMIT;
}
// Indices of vertices that could terminate any of the literals in 'lits'.
set<u32> terms;
@@ -262,7 +257,11 @@ u32 findMaxInfixMatches(const left_id &left, const set<ue2_literal> &lits) {
return findMaxInfixMatches(*left.castle(), lits);
}
if (left.graph()) {
return findMaxInfixMatches(*left.graph(), lits);
if (!onlyOneTop(*left.graph())) {
DEBUG_PRINTF("more than one top!n");
return NO_MATCH_LIMIT;
}
return findMaxLiteralMatches(*left.graph(), lits);
}
return NO_MATCH_LIMIT;
@@ -315,7 +314,7 @@ void findCountingMiracleInfo(const left_id &left, const vector<u8> &stopTable,
lits.insert(ue2_literal(c, false));
}
u32 count = findMaxInfixMatches(*left.graph(), lits);
u32 count = findMaxLiteralMatches(*left.graph(), lits);
DEBUG_PRINTF("counting miracle %u\n", count + 1);
if (count && count < 50) {
*cm_count = count + 1;

2
src/rose/rose_build_lookaround.cpp
View File

@@ -72,7 +72,7 @@ void getForwardReach(const NGHolder &g, u32 top, map<s32, CharReach> &look) {
if (v == g.startDs) {
continue;
}
if (g[e].top == top) {
if (contains(g[e].tops, top)) {
curr.insert(v);
}
}

9
src/rose/rose_build_merge.cpp
View File

@@ -1762,9 +1762,12 @@ void replaceTops(NGHolder &h, const map<u32, u32> &top_mapping) {
if (v == h.startDs) {
continue;
}
DEBUG_PRINTF("vertex %u has top %u\n", h[v].index, h[e].top);
assert(contains(top_mapping, h[e].top));
h[e].top = top_mapping.at(h[e].top);
flat_set<u32> new_tops;
for (u32 t : h[e].tops) {
DEBUG_PRINTF("vertex %u has top %u\n", h[v].index, t);
new_tops.insert(top_mapping.at(t));
}
h[e].tops = move(new_tops);
}
}

81
src/rose/rose_build_misc.cpp
View File

@@ -903,12 +903,15 @@ RoseVertex RoseBuildImpl::cloneVertex(RoseVertex v) {
}
#ifndef NDEBUG
bool roseHasTops(const RoseGraph &g, RoseVertex v) {
bool roseHasTops(const RoseBuildImpl &build, RoseVertex v) {
const RoseGraph &g = build.g;
assert(g[v].left);
set<u32> graph_tops;
for (const auto &e : in_edges_range(v, g)) {
graph_tops.insert(g[e].rose_top);
if (!build.isRootSuccessor(v)) {
for (const auto &e : in_edges_range(v, g)) {
graph_tops.insert(g[e].rose_top);
}
}
return is_subset_of(graph_tops, all_tops(g[v].left));
@@ -1073,18 +1076,9 @@ bool has_non_eod_accepts(const suffix_id &s) {
set<u32> all_tops(const suffix_id &s) {
assert(s.graph() || s.castle() || s.haig() || s.dfa());
if (s.graph()) {
set<u32> tops;
const NGHolder &h = *s.graph();
for (const auto &e : out_edges_range(h.start, h)) {
if (target(e, h) == h.startDs) {
continue;
}
tops.insert(h[e].top);
}
if (tops.empty()) {
tops.insert(0); // Vacuous graph, triggered on zero top.
}
return tops;
flat_set<u32> tops = getTops(*s.graph());
assert(!tops.empty());
return {tops.begin(), tops.end()};
}
if (s.castle()) {
@@ -1142,18 +1136,8 @@ depth findMaxWidth(const left_id &r) {
set<u32> all_tops(const left_id &r) {
assert(r.graph() || r.castle() || r.haig() || r.dfa());
if (r.graph()) {
set<u32> tops;
const NGHolder &h = *r.graph();
for (const auto &e : out_edges_range(h.start, h)) {
if (target(e, h) == h.startDs) {
continue;
}
tops.insert(h[e].top);
}
if (tops.empty()) {
tops.insert(0); // Vacuous graph, triggered on zero top.
}
return tops;
flat_set<u32> tops = getTops(*r.graph());
return {tops.begin(), tops.end()};
}
if (r.castle()) {
@@ -1348,6 +1332,49 @@ bool canImplementGraphs(const RoseBuildImpl &tbi) {
return true;
}
bool hasOrphanedTops(const RoseBuildImpl &build) {
const RoseGraph &g = build.g;
ue2::unordered_map<left_id, set<u32> > roses;
ue2::unordered_map<suffix_id, set<u32> > suffixes;
for (auto v : vertices_range(g)) {
if (g[v].left) {
set<u32> &tops = roses[g[v].left];
if (!build.isRootSuccessor(v)) {
// Tops for infixes come from the in-edges.
for (const auto &e : in_edges_range(v, g)) {
tops.insert(g[e].rose_top);
}
}
}
if (g[v].suffix) {
suffixes[g[v].suffix].insert(g[v].suffix.top);
}
}
for (const auto &e : roses) {
if (all_tops(e.first) != e.second) {
DEBUG_PRINTF("rose tops (%s) don't match rose graph (%s)\n",
as_string_list(all_tops(e.first)).c_str(),
as_string_list(e.second).c_str());
return true;
}
}
for (const auto &e : suffixes) {
if (all_tops(e.first) != e.second) {
DEBUG_PRINTF("suffix tops (%s) don't match rose graph (%s)\n",
as_string_list(all_tops(e.first)).c_str(),
as_string_list(e.second).c_str());
return true;
}
}
return false;
}
#endif // NDEBUG
} // namespace ue2

33
src/rose/rose_build_role_aliasing.cpp
View File

@@ -863,7 +863,7 @@ void pruneUnusedTops(CastleProto &castle, const RoseGraph &g,
static
void pruneUnusedTops(NGHolder &h, const RoseGraph &g,
const set<RoseVertex> &verts) {
ue2::unordered_set<u32> used_tops;
ue2::flat_set<u32> used_tops;
for (auto v : verts) {
assert(g[v].left.graph.get() == &h);
@@ -879,10 +879,13 @@ void pruneUnusedTops(NGHolder &h, const RoseGraph &g,
if (v == h.startDs) {
continue; // stylised edge, leave it alone.
}
u32 top = h[e].top;
if (!contains(used_tops, top)) {
DEBUG_PRINTF("edge (start,%u) has unused top %u\n",
h[v].index, top);
flat_set<u32> pruned_tops;
auto pt_inserter = inserter(pruned_tops, pruned_tops.end());
set_intersection(h[e].tops.begin(), h[e].tops.end(),
used_tops.begin(), used_tops.end(), pt_inserter);
h[e].tops = move(pruned_tops);
if (h[e].tops.empty()) {
DEBUG_PRINTF("edge (start,%u) has only unused tops\n", h[v].index);
dead.push_back(e);
}
}
@@ -1327,8 +1330,8 @@ bool attemptRoseGraphMerge(RoseBuildImpl &build, bool preds_same, RoseVertex a,
DEBUG_PRINTF("winner %zu states\n", num_vertices(*b_h));
if (!setDistinctRoseTops(g, victim, *b_h, deque<RoseVertex>(1, a))) {
assert(roseHasTops(g, a));
assert(roseHasTops(g, b));
assert(roseHasTops(build, a));
assert(roseHasTops(build, b));
return false;
}
@@ -1341,8 +1344,8 @@ bool attemptRoseGraphMerge(RoseBuildImpl &build, bool preds_same, RoseVertex a,
for (const auto &e : in_edges_range(a, g)) {
g[e] = a_props[source(e, g)];
}
assert(roseHasTops(g, a));
assert(roseHasTops(g, b));
assert(roseHasTops(build, a));
assert(roseHasTops(build, b));
return false;
}
@@ -1365,8 +1368,8 @@ bool attemptRoseGraphMerge(RoseBuildImpl &build, bool preds_same, RoseVertex a,
reduceImplementableGraph(*b_h, SOM_NONE, nullptr, build.cc);
assert(roseHasTops(g, a));
assert(roseHasTops(g, b));
assert(roseHasTops(build, a));
assert(roseHasTops(build, b));
assert(isImplementableNFA(*b_h, nullptr, build.cc));
return true;
}
@@ -1417,8 +1420,8 @@ bool attemptRoseMerge(RoseBuildImpl &build, bool preds_same, RoseVertex a,
return false;
}
assert(roseHasTops(g, a));
assert(roseHasTops(g, b));
assert(roseHasTops(build, a));
assert(roseHasTops(build, b));
if (a_left_id.graph() && b_left_id.graph()) {
return attemptRoseGraphMerge(build, preds_same, a, b, trivialCasesOnly,
@@ -1737,6 +1740,7 @@ void leftMergePass(CandidateSet &candidates, RoseBuildImpl &build,
}
DEBUG_PRINTF("%zu candidates remaining\n", candidates.size());
assert(!hasOrphanedTops(build));
}
// Can't merge vertices with different root predecessors.
@@ -1952,6 +1956,7 @@ void rightMergePass(CandidateSet &candidates, RoseBuildImpl &build,
}
DEBUG_PRINTF("%zu candidates remaining\n", candidates.size());
assert(!hasOrphanedTops(build));
}
/**
@@ -2017,6 +2022,7 @@ void filterDiamondCandidates(RoseGraph &g, CandidateSet &candidates) {
void aliasRoles(RoseBuildImpl &build, bool mergeRoses) {
const CompileContext &cc = build.cc;
RoseGraph &g = build.g;
assert(!hasOrphanedTops(build));
if (!cc.grey.roseRoleAliasing || !cc.grey.roseGraphReduction) {
return;
@@ -2050,6 +2056,7 @@ void aliasRoles(RoseBuildImpl &build, bool mergeRoses) {
DEBUG_PRINTF("killed %zu vertices\n", dead.size());
build.removeVertices(dead);
assert(!hasOrphanedTops(build));
}
} // namespace ue2