github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-06-28 16:41:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

allow sets of tops on edges

Browse Source
This commit is contained in:
Alex Coyte 2016-09-01 14:58:55 +10:00 committed by Matthew Barr
parent aca89e66d2
commit c94899dd44
31 changed files with 284 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/nfa/castlecompile.cpp
View File

@ -904,7 +904,7 @@ void addToHolder(NGHolder &g, u32 top, const PureRepeat &pr) {
if (min_bound == 0) { // Vacuous case, we can only do this once.
assert(!edge(g.start, g.accept, g).second);
NFAEdge e = add_edge(g.start, g.accept, g).first;
g[e].top = top;
g[e].tops.insert(top);
g[u].reports.insert(pr.reports.begin(), pr.reports.end());
min_bound = 1;
}
@ -914,7 +914,7 @@ void addToHolder(NGHolder &g, u32 top, const PureRepeat &pr) {
g[v].char_reach = pr.reach;
NFAEdge e = add_edge(u, v, g).first;
if (u == g.start) {
g[e].top = top;
g[e].tops.insert(top);
}
u = v;
}
@ -933,7 +933,7 @@ void addToHolder(NGHolder &g, u32 top, const PureRepeat &pr) {
}
NFAEdge e = add_edge(u, v, g).first;
if (u == g.start) {
g[e].top = top;
g[e].tops.insert(top);
}
u = v;
}

6
src/nfagraph/ng_dump.cpp
View File

@ -234,9 +234,9 @@ public:
void operator()(ostream& os, const EdgeT& e) const {
// Edge label. Print priority.
os << "[fontsize=9,label=\"";
// If it's an edge from start, print top id.
if (is_any_start(source(e, g), g) && !is_any_start(target(e, g), g)) {
os << "TOP " << g[e].top << "\\n";
// print tops if any set.
if (!g[e].tops.empty()) {
os << "TOP " << as_string_list(g[e].tops) << "\\n";
}
// If it's an assert vertex, then display its info.

29
src/nfagraph/ng_equivalence.cpp
View File

@ -72,7 +72,7 @@ struct VertexInfoPtrCmp {
class VertexInfo {
public:
VertexInfo(NFAVertex v_in, const NGHolder &g)
: v(v_in), vert_index(g[v].index), cr(g[v].char_reach), edge_top(~0),
: v(v_in), vert_index(g[v].index), cr(g[v].char_reach),
equivalence_class(~0), vertex_flags(g[v].assert_flags) {}
flat_set<VertexInfo *, VertexInfoPtrCmp> pred; //!< predecessors of this vertex
@ -82,7 +82,7 @@ public:
CharReach cr;
CharReach pred_cr;
CharReach succ_cr;
unsigned edge_top;
flat_set<u32> edge_tops; /**< tops on edge from start */
unsigned equivalence_class;
unsigned vertex_flags;
};
@ -120,7 +120,7 @@ public:
EquivalenceType eq)
: /* reports only matter for right-equiv */
rs(eq == RIGHT_EQUIVALENCE ? g[vi.v].reports : flat_set<ReportID>()),
vertex_flags(vi.vertex_flags), edge_top(vi.edge_top), cr(vi.cr),
vertex_flags(vi.vertex_flags), edge_tops(vi.edge_tops), cr(vi.cr),
adjacent_cr(eq == LEFT_EQUIVALENCE ? vi.pred_cr : vi.succ_cr),
/* treat non-special vertices the same */
node_type(min(g[vi.v].index, u32{N_SPECIALS})), depth(d_in) {}
@ -128,7 +128,7 @@ public:
bool operator==(const ClassInfo &b) const {
return node_type == b.node_type && depth.d1 == b.depth.d1 &&
depth.d2 == b.depth.d2 && cr == b.cr &&
adjacent_cr == b.adjacent_cr && edge_top == b.edge_top &&
adjacent_cr == b.adjacent_cr && edge_tops == b.edge_tops &&
vertex_flags == b.vertex_flags && rs == b.rs;
}
@ -136,7 +136,6 @@ public:
size_t val = 0;
boost::hash_combine(val, boost::hash_range(begin(c.rs), end(c.rs)));
boost::hash_combine(val, c.vertex_flags);
boost::hash_combine(val, c.edge_top);
boost::hash_combine(val, c.cr);
boost::hash_combine(val, c.adjacent_cr);
boost::hash_combine(val, c.node_type);
@ -148,7 +147,7 @@ public:
private:
flat_set<ReportID> rs; /* for right equiv only */
unsigned vertex_flags;
u32 edge_top;
flat_set<u32> edge_tops;
CharReach cr;
CharReach adjacent_cr;
unsigned node_type;
@ -307,7 +306,7 @@ ptr_vector<VertexInfo> getVertexInfos(const NGHolder &g) {
// also set up edge tops
if (is_triggered(g) && u == g.start) {
cur_vi.edge_top = g[e].top;
cur_vi.edge_tops = g[e].tops;
}
}
@ -544,7 +543,7 @@ void mergeClass(ptr_vector<VertexInfo> &infos, NGHolder &g, unsigned eq_class,
infos.push_back(new_vertex_info_eod);
}
const unsigned edgetop = (*cur_class_vertices.begin())->edge_top;
const auto &edgetops = (*cur_class_vertices.begin())->edge_tops;
for (VertexInfo *old_vertex_info : cur_class_vertices) {
assert(old_vertex_info->equivalence_class == eq_class);
@ -565,9 +564,10 @@ void mergeClass(ptr_vector<VertexInfo> &infos, NGHolder &g, unsigned eq_class,
// if edge doesn't exist, create it
NFAEdge e = add_edge_if_not_present(pred_info->v, new_v, g).first;
// put edge top, if applicable
if (edgetop != (unsigned) -1) {
g[e].top = edgetop;
// put edge tops, if applicable
if (!edgetops.empty()) {
assert(g[e].tops.empty() || g[e].tops == edgetops);
g[e].tops = edgetops;
}
pred_info->succ.insert(new_vertex_info);
@ -576,9 +576,10 @@ void mergeClass(ptr_vector<VertexInfo> &infos, NGHolder &g, unsigned eq_class,
NFAEdge ee = add_edge_if_not_present(pred_info->v, new_v_eod,
g).first;
// put edge top, if applicable
if (edgetop != (unsigned) -1) {
g[ee].top = edgetop;
// put edge tops, if applicable
if (!edgetops.empty()) {
assert(g[e].tops.empty() || g[e].tops == edgetops);
g[ee].tops = edgetops;
}
pred_info->succ.insert(new_vertex_info_eod);

6
src/nfagraph/ng_graph.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -69,8 +69,8 @@ struct NFAGraphEdgeProps {
u32 index = 0;
/** \brief For graphs that will be implemented as multi-top engines, this
* specifies the top event. Only used on edges from the start vertex. */
u32 top = 0;
* specifies the top events. Only used on edges from the start vertex. */
ue2::flat_set<u32> tops;
/** \brief Flags associated with assertions. */
u32 assert_flags = 0;

1
src/nfagraph/ng_holder.cpp
View File

@ -178,7 +178,6 @@ std::pair<NFAEdge, bool> add_edge(NFAVertex u, NFAVertex v, NGHolder &h) {
pair<NFAEdge, bool> e = add_edge(u, v, h.g);
h.g[e.first].index = h.numEdges++;
assert(!h.isValidNumEdges || h.numEdges > 0); // no wrapping
h.g[e.first].top = 0;
return e;
}

2
src/nfagraph/ng_holder.h
View File

@ -315,6 +315,8 @@ void remove_edges(Iter begin, Iter end, NGHolder &h, bool renumber = true) {
}
}
#define DEFAULT_TOP 0U
/** \brief Clear and remove all of the edges pointed to by the edge descriptors
* in the given container.
*

10
src/nfagraph/ng_is_equal.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -153,14 +153,14 @@ bool is_equal_i(const NGHolder &a, const NGHolder &b,
}
/* check top for edges out of start */
vector<pair<u32, u32>> top_a;
vector<pair<u32, u32>> top_b;
vector<pair<u32, flat_set<u32>>> top_a;
vector<pair<u32, flat_set<u32>>> top_b;
for (const auto &e : out_edges_range(a.start, a)) {
top_a.emplace_back(a[target(e, a)].index, a[e].top);
top_a.emplace_back(a[target(e, a)].index, a[e].tops);
}
for (const auto &e : out_edges_range(b.start, b)) {
top_b.emplace_back(b[target(e, b)].index, b[e].top);
top_b.emplace_back(b[target(e, b)].index, b[e].tops);
}
sort(top_a.begin(), top_a.end());

6
src/nfagraph/ng_limex.cpp
View File

@ -148,14 +148,16 @@ void dropRedundantStartEdges(NGHolder &g) {
static
void makeTopStates(NGHolder &g, map<u32, NFAVertex> &tops,
const map<u32, CharReach> &top_reach) {
/* TODO: more intelligent creation of top states */
map<u32, vector<NFAVertex>> top_succs;
for (const auto &e : out_edges_range(g.start, g)) {
NFAVertex v = target(e, g);
if (v == g.startDs) {
continue;
}
u32 t = g[e].top;
top_succs[t].push_back(v);
for (u32 t : g[e].tops) {
top_succs[t].push_back(v);
}
}
for (const auto &top : top_succs) {

2
src/nfagraph/ng_redundancy.cpp
View File

@ -310,7 +310,7 @@ bool hasInEdgeTops(const NGHolder &g, NFAVertex v) {
bool exists;
NFAEdge e;
tie(e, exists) = edge_by_target(g.start, v, g);
if (exists && g[e].top != 0) {
if (exists && !g[e].tops.empty()) {
return true;
}
return false;

62
src/nfagraph/ng_repeat.cpp
View File

@ -215,8 +215,8 @@ bool rogueSuccessor(const NGHolder &g, NFAVertex v,
static
bool hasDifferentTops(const NGHolder &g, const vector<NFAVertex> &verts) {
bool found = false;
u32 top = 0;
/* TODO: check that we need this now that we allow multiple tops */
const flat_set<u32> *tops = nullptr;
for (auto v : verts) {
for (const auto &e : in_edges_range(v, g)) {
@ -224,17 +224,12 @@ bool hasDifferentTops(const NGHolder &g, const vector<NFAVertex> &verts) {
if (u != g.start && u != g.startDs) {
continue; // Only edges from starts have valid top properties.
}
u32 t = g[e].top;
DEBUG_PRINTF("edge (%u,%u) with top %u\n", g[u].index,
g[v].index, t);
assert(t < NFA_MAX_TOP_MASKS);
if (!found) {
found = true;
top = t;
} else {
if (t != top) {
return true; // More than one top.
}
DEBUG_PRINTF("edge (%u,%u) with %zu tops\n", g[u].index, g[v].index,
g[e].tops.size());
if (!tops) {
tops = &g[e].tops;
} else if (g[e].tops != *tops) {
return true; // More than one set of tops.
}
}
}
@ -1123,7 +1118,7 @@ NFAVertex buildTriggerStates(NGHolder &g, const vector<CharReach> &trigger,
g[v].char_reach = cr;
add_edge(u, v, g);
if (u == g.start) {
g[edge(u, v, g).first].top = top;
g[edge(u, v, g).first].tops.insert(top);
}
u = v;
}
@ -1153,18 +1148,21 @@ void addTriggers(NGHolder &g,
continue;
}
const auto &top = g[e].top;
const auto &tops = g[e].tops;
// The caller may not have given us complete trigger information. If we
// don't have any triggers for a particular top, we should just leave
// it alone.
if (!contains(triggers, top)) {
DEBUG_PRINTF("no triggers for top %u\n", top);
continue;
}
for (u32 top : tops) {
if (!contains(triggers, top)) {
DEBUG_PRINTF("no triggers for top %u\n", top);
goto next_edge;
}
starts_by_top[top].push_back(v);
starts_by_top[top].push_back(v);
}
dead.push_back(e);
next_edge:;
}
remove_edges(dead, g);
@ -2105,14 +2103,26 @@ void populateFixedTopInfo(const map<u32, u32> &fixed_depth_tops,
if (v == g.startDs) {
continue;
}
u32 top = g[e].top;
depth td = depth::infinity();
if (contains(fixed_depth_tops, top)) {
td = fixed_depth_tops.at(top);
for (u32 top : g[e].tops) {
if (!contains(fixed_depth_tops, top)) {
td = depth::infinity();
break;
}
depth td_t = fixed_depth_tops.at(top);
if (td == td_t) {
continue;
} else if (td == depth::infinity()) {
td = td_t;
} else {
td = depth::infinity();
break;
}
}
DEBUG_PRINTF("scanning from %u top=%u depth=%s\n",
g[v].index, top, td.str().c_str());
DEBUG_PRINTF("scanning from %u depth=%s\n", g[v].index,
td.str().c_str());
/* for each vertex reachable from v update its map to reflect that it is
* reachable from a top of depth td. */
@ -2428,7 +2438,7 @@ bool isPureRepeat(const NGHolder &g, PureRepeat &repeat) {
}
// Must have precisely one top.
if (!onlyOneTop(g)) {
if (is_triggered(g) && !onlyOneTop(g)) {
DEBUG_PRINTF("Too many tops\n");
return false;
}

7
src/nfagraph/ng_restructuring.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -174,6 +174,7 @@ numberStates(NGHolder &h, const map<u32, NFAVertex> &tops) {
u32 countStates(const NGHolder &g,
const ue2::unordered_map<NFAVertex, u32> &state_ids,
bool addTops) {
/* TODO: smarter top state allocation, move to limex? */
if (state_ids.empty()) {
return 0;
}
@ -188,11 +189,11 @@ u32 countStates(const NGHolder &g,
u32 num_states = max_state + 1;
assert(contains(state_ids, g.start));
if (addTops && state_ids.at(g.start) != NO_STATE) {
if (addTops && is_triggered(g) && state_ids.at(g.start) != NO_STATE) {
num_states--;
set<u32> tops;
for (auto e : out_edges_range(g.start, g)) {
tops.insert(g[e].top);
insert(&tops, g[e].tops);
}
num_states += tops.size();
}

14
src/nfagraph/ng_rose.cpp
View File

@ -811,6 +811,7 @@ bool can_match(const NGHolder &g, const ue2_literal &lit, bool overhang_ok) {
u32 removeTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
u32 max_delay, bool overhang_ok) {
assert(isCorrectlyTopped(g));
if (max_delay == MO_INVALID_IDX) {
max_delay--;
}
@ -878,12 +879,16 @@ u32 removeTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
sort(verts.begin(), verts.end(), VertexIndexOrdering<NGHolder>(g));
for (auto v : verts) {
add_edge(v, g.accept, g);
NFAEdge e = add_edge(v, g.accept, g).first;
g[v].reports.insert(0);
if (is_triggered(g) && v == g.start) {
g[e].tops.insert(DEFAULT_TOP);
}
}
pruneUseless(g);
assert(allMatchStatesHaveReports(g));
assert(isCorrectlyTopped(g));
DEBUG_PRINTF("graph has %zu vertices left\n", num_vertices(g));
return delay;
@ -892,6 +897,7 @@ u32 removeTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
u32 delay, const vector<NFAVertex> &preds) {
assert(delay <= lit.length());
assert(isCorrectlyTopped(g));
DEBUG_PRINTF("adding on '%s' %u\n", dumpString(lit).c_str(), delay);
NFAVertex prev = g.accept;
@ -906,7 +912,10 @@ void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
}
for (auto v : preds) {
add_edge(v, prev, g);
NFAEdge e = add_edge(v, prev, g).first;
if (v == g.start && is_triggered(g)) {
g[e].tops.insert(DEFAULT_TOP);
}
}
// Every predecessor of accept must have a report.
@ -917,6 +926,7 @@ void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
g.renumberVertices();
g.renumberEdges();
assert(allMatchStatesHaveReports(g));
assert(isCorrectlyTopped(g));
}
void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,

6
src/nfagraph/ng_split.cpp
View File

@ -151,7 +151,8 @@ void splitRHS(const NGHolder &base, const vector<NFAVertex> &pivots,
for (auto pivot : pivots) {
assert(contains(*rhs_map, pivot));
add_edge(rhs->start, (*rhs_map)[pivot], *rhs);
NFAEdge e = add_edge(rhs->start, (*rhs_map)[pivot], *rhs).first;
(*rhs)[e].tops.insert(DEFAULT_TOP);
}
/* should do the renumbering unconditionally as we know edges are already
@ -215,6 +216,7 @@ void splitGraph(const NGHolder &base, const vector<NFAVertex> &pivots,
DEBUG_PRINTF("splitting graph at %zu vertices\n", pivots.size());
assert(!has_parallel_edge(base));
assert(isCorrectlyTopped(base));
/* RHS pivots are built from the common set of successors of pivots. */
vector<NFAVertex> rhs_pivots;
@ -228,6 +230,8 @@ void splitGraph(const NGHolder &base, const vector<NFAVertex> &pivots,
assert(!has_parallel_edge(*lhs));
assert(!has_parallel_edge(*rhs));
assert(isCorrectlyTopped(*lhs));
assert(isCorrectlyTopped(*rhs));
}
void splitGraph(const NGHolder &base, NFAVertex pivot,

4
src/nfagraph/ng_split.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -47,6 +47,8 @@ class NGHolder;
* is in the lhs if it is reachable from start without going through the
* pivot. The pivot ends up in the LHS and any adjacent vertices in the RHS.
*
* Note: The RHS is setup to be triggered by TOP 0
*
* When multiple split vertices are provided:
* - RHS contains all vertices reachable from every pivot
* - LHS contains all vertices which are reachable from start ignoring any

13
src/nfagraph/ng_uncalc_components.cpp
View File

@ -205,11 +205,10 @@ u32 commonPrefixLength(const NGHolder &ga,
break;
}
if (ga[*ei].top != gb[b_edge].top) {
if (ga[*ei].tops != gb[b_edge].tops) {
max = i;
ok = false;
DEBUG_PRINTF("tops don't match on edge %zu->%u\n",
i, sid);
DEBUG_PRINTF("tops don't match on edge %zu->%u\n", i, sid);
}
}
@ -318,7 +317,7 @@ void mergeNfa(NGHolder &dest, vector<NFAVertex> &destStateMap,
DEBUG_PRINTF("skipping common edge\n");
assert(edge(u, v, dest).second);
// Should never merge edges with different top values.
assert(vic[e].top == dest[edge(u, v, dest).first].top);
assert(vic[e].tops == dest[edge(u, v, dest).first].tops);
continue;
} else {
assert(is_any_accept(v, dest));
@ -506,11 +505,13 @@ bool mergeableStarts(const NGHolder &h1, const NGHolder &h2) {
return false;
}
/* TODO: relax top checks if reports match */
// If both graphs have edge (start, accept), the tops must match.
auto e1_accept = edge(h1.start, h1.accept, h1);
auto e2_accept = edge(h2.start, h2.accept, h2);
if (e1_accept.second && e2_accept.second &&
h1[e1_accept.first].top != h2[e2_accept.first].top) {
h1[e1_accept.first].tops != h2[e2_accept.first].tops) {
return false;
}
@ -518,7 +519,7 @@ bool mergeableStarts(const NGHolder &h1, const NGHolder &h2) {
auto e1_eod = edge(h1.start, h1.acceptEod, h1);
auto e2_eod = edge(h2.start, h2.acceptEod, h2);
if (e1_eod.second && e2_eod.second &&
h1[e1_eod.first].top != h2[e2_eod.first].top) {
h1[e1_eod.first].tops != h2[e2_eod.first].tops) {
return false;
}

44
src/nfagraph/ng_util.cpp
View File

@ -165,12 +165,7 @@ void clone_in_edges(NGHolder &g, NFAVertex s, NFAVertex dest) {
}
bool onlyOneTop(const NGHolder &g) {
set<u32> tops;
for (const auto &e : out_edges_range(g.start, g)) {
tops.insert(g[e].top);
}
assert(!tops.empty());
return tops.size() == 1;
return getTops(g).size() == 1;
}
namespace {
@ -465,17 +460,21 @@ void appendLiteral(NGHolder &h, const ue2_literal &s) {
ue2::flat_set<u32> getTops(const NGHolder &h) {
ue2::flat_set<u32> tops;
for (const auto &e : out_edges_range(h.start, h)) {
NFAVertex v = target(e, h);
if (v == h.startDs) {
continue;
}
u32 top = h[e].top;
assert(top < NFA_MAX_TOP_MASKS);
tops.insert(top);
insert(&tops, h[e].tops);
}
return tops;
}
void setTops(NGHolder &h, u32 top) {
for (const auto &e : out_edges_range(h.start, h)) {
assert(h[e].tops.empty());
if (target(e, h) == h.startDs) {
continue;
}
h[e].tops.insert(top);
}
}
void clearReports(NGHolder &g) {
DEBUG_PRINTF("clearing reports without an accept edge\n");
ue2::unordered_set<NFAVertex> allow;
@ -694,6 +693,25 @@ bool hasCorrectlyNumberedEdges(const NGHolder &g) {
&& num_edges(g) == num_edges(g.g);
}
bool isCorrectlyTopped(const NGHolder &g) {
if (is_triggered(g)) {
for (const auto &e : out_edges_range(g.start, g)) {
if (g[e].tops.empty() != (target(e, g) == g.startDs)) {
return false;
}
}
} else {
for (const auto &e : out_edges_range(g.start, g)) {
if (!g[e].tops.empty()) {
return false;
}
}
}
return true;
}
#endif // NDEBUG
} // namespace ue2

12
src/nfagraph/ng_util.h
View File

@ -198,9 +198,13 @@ VertexIndexOrdering<Graph> make_index_ordering(const Graph &g) {
bool onlyOneTop(const NGHolder &g);
/** Return a mask of the tops on the given graph. */
/** Return the set of the tops on the given graph. */
flat_set<u32> getTops(const NGHolder &h);
/** Initialise the tops on h to the provide top. Assumes that h is triggered and
* no tops have been set on h. */
void setTops(NGHolder &h, u32 top = DEFAULT_TOP);
/** adds a vertex to g with all the same vertex properties as \p v (aside from
* index) */
NFAVertex clone_vertex(NGHolder &g, NFAVertex v);
@ -319,6 +323,12 @@ bool hasCorrectlyNumberedVertices(const NGHolder &g);
*/
bool hasCorrectlyNumberedEdges(const NGHolder &g);
/**
* Assertion: returns true if the graph is triggered and all edges out of start
* have tops OR if the graph is not-triggered and all edges out of start have no
* tops.
*/
bool isCorrectlyTopped(const NGHolder &g);
#endif // NDEBUG
} // namespace ue2

26
src/nfagraph/ng_violet.cpp
View File

@ -1076,8 +1076,10 @@ bool splitRoseEdge(const NGHolder &base_graph, RoseInGraph &vg,
assert(hasCorrectlyNumberedVertices(*rhs));
assert(hasCorrectlyNumberedEdges(*rhs));
assert(isCorrectlyTopped(*rhs));
assert(hasCorrectlyNumberedVertices(*lhs));
assert(hasCorrectlyNumberedEdges(*lhs));
assert(isCorrectlyTopped(*lhs));
return true;
}
@ -1152,7 +1154,11 @@ void splitEdgesByCut(NGHolder &h, RoseInGraph &vg,
/* want to cut off paths to pivot from things other than the pivot -
* makes a more svelte graphy */
clear_in_edges(temp_map[pivot], *new_lhs);
add_edge(temp_map[prev_v], temp_map[pivot], *new_lhs);
NFAEdge pivot_edge = add_edge(temp_map[prev_v], temp_map[pivot],
*new_lhs).first;
if (is_triggered(h) && prev_v == h.start) {
(*new_lhs)[pivot_edge].tops.insert(DEFAULT_TOP);
}
pruneUseless(*new_lhs, false);
renumber_vertices(*new_lhs);
@ -1162,6 +1168,7 @@ void splitEdgesByCut(NGHolder &h, RoseInGraph &vg,
assert(hasCorrectlyNumberedVertices(*new_lhs));
assert(hasCorrectlyNumberedEdges(*new_lhs));
assert(isCorrectlyTopped(*new_lhs));
const set<ue2_literal> &lits = cut_lits.at(e);
for (const auto &lit : lits) {
@ -1228,6 +1235,7 @@ void splitEdgesByCut(NGHolder &h, RoseInGraph &vg,
DEBUG_PRINTF(" into rhs %s\n",
to_string(new_rhs->kind).c_str());
done_rhs.emplace(adj, new_rhs);
assert(isCorrectlyTopped(*new_rhs));
}
assert(done_rhs[adj].get());
@ -1235,6 +1243,7 @@ void splitEdgesByCut(NGHolder &h, RoseInGraph &vg,
assert(hasCorrectlyNumberedVertices(*new_rhs));
assert(hasCorrectlyNumberedEdges(*new_rhs));
assert(isCorrectlyTopped(*new_rhs));
if (vg[dest].type == RIV_LITERAL
&& !can_match(*new_rhs, vg[dest].s, true)) {
@ -1380,6 +1389,7 @@ void avoidOutfixes(RoseInGraph &vg, const CompileContext &cc) {
RoseInEdge e = *edges(vg).first;
NGHolder &h = *vg[e].graph;
assert(isCorrectlyTopped(h));
renumber_vertices(h);
renumber_edges(h);
@ -1602,6 +1612,7 @@ void removeRedundantLiteralsFromInfix(const NGHolder &h, RoseInGraph &ig,
continue;
}
assert(isCorrectlyTopped(*h_new));
graphs[right] = make_pair(h_new, delay);
}
@ -1720,6 +1731,8 @@ unique_ptr<NGHolder> make_chain(u32 count) {
h[u].reports.insert(0);
add_edge(u, h.accept, h);
setTops(h);
return rv;
}
@ -1777,6 +1790,7 @@ bool makeTransientFromLongLiteral(NGHolder &h, RoseInGraph &vg,
assert(willBeTransient(findMaxWidth(*h_new), cc)
|| willBeAnchoredTable(findMaxWidth(*h_new), cc.grey));
assert(isCorrectlyTopped(*h_new));
graphs[v] = h_new;
}
@ -1811,6 +1825,7 @@ bool improvePrefix(NGHolder &h, RoseInGraph &vg, const vector<RoseInEdge> &ee,
const CompileContext &cc) {
DEBUG_PRINTF("trying to improve prefix %p, %zu verts\n", &h,
num_vertices(h));
assert(isCorrectlyTopped(h));
renumber_vertices(h);
renumber_edges(h);
@ -1860,6 +1875,7 @@ bool improvePrefix(NGHolder &h, RoseInGraph &vg, const vector<RoseInEdge> &ee,
for (const auto &e : ee) {
shared_ptr<NGHolder> hh = cloneHolder(h);
auto succ_lit = vg[target(e, vg)].s;
assert(isCorrectlyTopped(*hh));
u32 delay = removeTrailingLiteralStates(*hh, succ_lit,
succ_lit.length(),
false /* can't overhang start */);
@ -1868,6 +1884,7 @@ bool improvePrefix(NGHolder &h, RoseInGraph &vg, const vector<RoseInEdge> &ee,
continue;
}
assert(isCorrectlyTopped(*hh));
trimmed[hh].emplace_back(e, delay);
}
@ -2110,10 +2127,15 @@ void splitEdgesForSuffix(const NGHolder &base_graph, RoseInGraph &vg,
add_edge(lhs->accept, lhs->acceptEod, *lhs);
clearReports(*lhs);
for (NFAVertex v : splitters) {
add_edge(v_map[v], lhs->accept, *lhs);
NFAEdge e = add_edge(v_map[v], lhs->accept, *lhs).first;
if (v == base_graph.start) {
(*lhs)[e].tops.insert(DEFAULT_TOP);
}
(*lhs)[v_map[v]].reports.insert(0);
}
pruneUseless(*lhs);
assert(isCorrectlyTopped(*lhs));
/* create literal vertices and connect preds */
for (const auto &lit : split.lit) {

4
src/nfagraph/ng_width.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -69,7 +69,7 @@ struct SpecialEdgeFilter {
return false;
}
if (single_top) {
if (u == h->start && g[e].top != top) {
if (u == h->start && !contains(g[e].tops, top)) {
return false;
}
if (u == h->startDs) {

2
src/rose/rose_build_add.cpp
View File

@ -1619,6 +1619,8 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter,
}
NGHolder *h = in[e].graph.get();
assert(isCorrectlyTopped(*h));
if (!contains(graphs, h)) {
ordered_graphs.push_back(h);
}

4
src/rose/rose_build_add_mask.cpp
View File

@ -574,7 +574,8 @@ unique_ptr<NGHolder> buildMaskRhs(const ue2::flat_set<ReportID> &reports,
succ = u;
}
add_edge(h.start, succ, h);
NFAEdge e = add_edge(h.start, succ, h).first;
h[e].tops.insert(DEFAULT_TOP);
return rhs;
}
@ -632,6 +633,7 @@ void doAddMask(RoseBuildImpl &tbi, bool anchored,
= buildMaskLhs(true, minBound - prefix2_len + overlap,
mask3);
mhs->kind = NFA_INFIX;
setTops(*mhs);
add_edge(u, v, RoseInEdgeProps(mhs, delay), ig);
DEBUG_PRINTF("add anch literal too!\n");

12
src/rose/rose_build_bytecode.cpp
View File

@ -512,7 +512,7 @@ bool nfaStuckOn(const NGHolder &g) {
set<u32> done_tops;
for (const auto &e : out_edges_range(g.start, g)) {
tops.insert(g[e].top);
insert(&tops, g[e].tops);
if (!g[target(e, g)].char_reach.all()) {
continue;
}
@ -521,7 +521,7 @@ bool nfaStuckOn(const NGHolder &g) {
insert(&asucc, adjacent_vertices(target(e, g), g));
if (asucc == succ) {
done_tops.insert(g[e].top);
insert(&done_tops, g[e].tops);
}
}
@ -842,8 +842,8 @@ makeLeftNfa(const RoseBuildImpl &tbi, left_id &left,
if (!n && !is_prefix && left.graph() && onlyOneTop(*left.graph())) {
map<u32, vector<vector<CharReach> > > triggers;
findTriggerSequences(tbi, infixTriggers.at(left), &triggers);
assert(contains(triggers, 0)); // single top
n = constructLBR(*left.graph(), triggers[0], cc, rm);
assert(triggers.size() == 1); // single top
n = constructLBR(*left.graph(), triggers.begin()->second, cc, rm);
}
if (!n && left.graph()) {
@ -1435,7 +1435,7 @@ void findExclusiveInfixes(RoseBuildImpl &build, build_context &bc,
// Sanity check: our NFA should contain each of the tops mentioned on
// our in-edges.
assert(roseHasTops(g, v));
assert(roseHasTops(build, v));
if (contains(leftfixes, leftfix)) {
// NFA already built.
@ -1504,7 +1504,7 @@ bool buildLeftfixes(RoseBuildImpl &tbi, build_context &bc,
// Sanity check: our NFA should contain each of the tops mentioned on
// our in-edges.
assert(roseHasTops(g, v));
assert(roseHasTops(tbi, v));
bool is_transient = contains(tbi.transient, leftfix);

52
src/rose/rose_build_compile.cpp
View File

@ -43,7 +43,6 @@
#include "nfa/nfa_internal.h"
#include "nfa/rdfa.h"
#include "nfagraph/ng_holder.h"
#include "nfagraph/ng_dump.h"
#include "nfagraph/ng_execute.h"
#include "nfagraph/ng_is_equal.h"
#include "nfagraph/ng_limex.h"
@ -1554,53 +1553,6 @@ bool roleOffsetsAreValid(const RoseGraph &g) {
}
return true;
}
static UNUSED
bool hasOrphanedTops(const RoseBuildImpl &tbi) {
const RoseGraph &g = tbi.g;
ue2::unordered_map<left_id, set<u32> > roses;
ue2::unordered_map<suffix_id, set<u32> > suffixes;
for (auto v : vertices_range(g)) {
if (g[v].left) {
set<u32> &tops = roses[g[v].left];
if (tbi.isRootSuccessor(v)) {
// Prefix, has only one top.
tops.insert(0);
} else {
// Tops for infixes come from the in-edges.
for (const auto &e : in_edges_range(v, g)) {
tops.insert(g[e].rose_top);
}
}
}
if (g[v].suffix) {
suffixes[g[v].suffix].insert(g[v].suffix.top);
}
}
for (const auto &e : roses) {
if (all_tops(e.first) != e.second) {
DEBUG_PRINTF("rose tops (%s) don't match rose graph (%s)\n",
as_string_list(all_tops(e.first)).c_str(),
as_string_list(e.second).c_str());
return true;
}
}
for (const auto &e : suffixes) {
if (all_tops(e.first) != e.second) {
DEBUG_PRINTF("suffix tops (%s) don't match rose graph (%s)\n",
as_string_list(all_tops(e.first)).c_str(),
as_string_list(e.second).c_str());
return true;
}
}
return false;
}
#endif // NDEBUG
aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildRose(u32 minWidth) {
@ -1681,13 +1633,17 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildRose(u32 minWidth) {
mergeSmallLeftfixes(*this);
}
assert(!hasOrphanedTops(*this));
// Do a rose-merging aliasing pass.
aliasRoles(*this, true);
assert(!hasOrphanedTops(*this));
// Run a merge pass over the outfixes as well.
mergeOutfixes(*this);
assert(!danglingVertexRef(*this));
assert(!hasOrphanedTops(*this));
findMoreLiteralMasks(*this);

7
src/rose/rose_build_convert.cpp
View File

@ -163,6 +163,8 @@ unique_ptr<NGHolder> convertLeafToHolder(const RoseGraph &g,
}
}
setTops(*out);
// Literal vertices wired to accept.
NFAVertex litfirst, litlast;
tie(litfirst, litlast) = addLiteralVertices(g, literals, t_v, *out);
@ -400,7 +402,10 @@ unique_ptr<NGHolder> makeFloodProneSuffix(const ue2_literal &s, size_t len,
NFAVertex u = h->start;
for (auto it = s.begin() + s.length() - len; it != s.end(); ++it) {
NFAVertex v = addHolderVertex(*it, *h);
add_edge(u, v, *h);
NFAEdge e = add_edge(u, v, *h).first;
if (u == h->start) {
(*h)[e].tops.insert(DEFAULT_TOP);
}
u = v;
}

3
src/rose/rose_build_impl.h
View File

@ -615,7 +615,8 @@ ue2_literal findNonOverlappingTail(const std::set<ue2_literal> &lits,
void setReportId(NGHolder &g, ReportID id);
#ifndef NDEBUG
bool roseHasTops(const RoseGraph &g, RoseVertex v);
bool roseHasTops(const RoseBuildImpl &build, RoseVertex v);
bool hasOrphanedTops(const RoseBuildImpl &build);
#endif
u64a findMaxOffset(const std::set<ReportID> &reports, const ReportManager &rm);

17
src/rose/rose_build_infix.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -108,15 +108,10 @@ void contractVertex(NGHolder &g, NFAVertex v,
}
static
u32 findMaxInfixMatches(const NGHolder &h, const set<ue2_literal> &lits) {
u32 findMaxLiteralMatches(const NGHolder &h, const set<ue2_literal> &lits) {
DEBUG_PRINTF("h=%p, %zu literals\n", &h, lits.size());
//dumpGraph("infix.dot", h.g);
if (!onlyOneTop(h)) {
DEBUG_PRINTF("more than one top!n");
return NO_MATCH_LIMIT;
}
// Indices of vertices that could terminate any of the literals in 'lits'.
set<u32> terms;
@ -262,7 +257,11 @@ u32 findMaxInfixMatches(const left_id &left, const set<ue2_literal> &lits) {
return findMaxInfixMatches(*left.castle(), lits);
}
if (left.graph()) {
return findMaxInfixMatches(*left.graph(), lits);
if (!onlyOneTop(*left.graph())) {
DEBUG_PRINTF("more than one top!n");
return NO_MATCH_LIMIT;
}
return findMaxLiteralMatches(*left.graph(), lits);
}
return NO_MATCH_LIMIT;
@ -315,7 +314,7 @@ void findCountingMiracleInfo(const left_id &left, const vector<u8> &stopTable,
lits.insert(ue2_literal(c, false));
}
u32 count = findMaxInfixMatches(*left.graph(), lits);
u32 count = findMaxLiteralMatches(*left.graph(), lits);
DEBUG_PRINTF("counting miracle %u\n", count + 1);
if (count && count < 50) {
*cm_count = count + 1;

2
src/rose/rose_build_lookaround.cpp
View File

@ -72,7 +72,7 @@ void getForwardReach(const NGHolder &g, u32 top, map<s32, CharReach> &look) {
if (v == g.startDs) {
continue;
}
if (g[e].top == top) {
if (contains(g[e].tops, top)) {
curr.insert(v);
}
}

9
src/rose/rose_build_merge.cpp
View File

@ -1762,9 +1762,12 @@ void replaceTops(NGHolder &h, const map<u32, u32> &top_mapping) {
if (v == h.startDs) {
continue;
}
DEBUG_PRINTF("vertex %u has top %u\n", h[v].index, h[e].top);
assert(contains(top_mapping, h[e].top));
h[e].top = top_mapping.at(h[e].top);
flat_set<u32> new_tops;
for (u32 t : h[e].tops) {
DEBUG_PRINTF("vertex %u has top %u\n", h[v].index, t);
new_tops.insert(top_mapping.at(t));
}
h[e].tops = move(new_tops);
}
}

81
src/rose/rose_build_misc.cpp
View File

@ -903,12 +903,15 @@ RoseVertex RoseBuildImpl::cloneVertex(RoseVertex v) {
}
#ifndef NDEBUG
bool roseHasTops(const RoseGraph &g, RoseVertex v) {
bool roseHasTops(const RoseBuildImpl &build, RoseVertex v) {
const RoseGraph &g = build.g;
assert(g[v].left);
set<u32> graph_tops;
for (const auto &e : in_edges_range(v, g)) {
graph_tops.insert(g[e].rose_top);
if (!build.isRootSuccessor(v)) {
for (const auto &e : in_edges_range(v, g)) {
graph_tops.insert(g[e].rose_top);
}
}
return is_subset_of(graph_tops, all_tops(g[v].left));
@ -1073,18 +1076,9 @@ bool has_non_eod_accepts(const suffix_id &s) {
set<u32> all_tops(const suffix_id &s) {
assert(s.graph() || s.castle() || s.haig() || s.dfa());
if (s.graph()) {
set<u32> tops;
const NGHolder &h = *s.graph();
for (const auto &e : out_edges_range(h.start, h)) {
if (target(e, h) == h.startDs) {
continue;
}
tops.insert(h[e].top);
}
if (tops.empty()) {
tops.insert(0); // Vacuous graph, triggered on zero top.
}
return tops;
flat_set<u32> tops = getTops(*s.graph());
assert(!tops.empty());
return {tops.begin(), tops.end()};
}
if (s.castle()) {
@ -1142,18 +1136,8 @@ depth findMaxWidth(const left_id &r) {
set<u32> all_tops(const left_id &r) {
assert(r.graph() || r.castle() || r.haig() || r.dfa());
if (r.graph()) {
set<u32> tops;
const NGHolder &h = *r.graph();
for (const auto &e : out_edges_range(h.start, h)) {
if (target(e, h) == h.startDs) {
continue;
}
tops.insert(h[e].top);
}
if (tops.empty()) {
tops.insert(0); // Vacuous graph, triggered on zero top.
}
return tops;
flat_set<u32> tops = getTops(*r.graph());
return {tops.begin(), tops.end()};
}
if (r.castle()) {
@ -1348,6 +1332,49 @@ bool canImplementGraphs(const RoseBuildImpl &tbi) {
return true;
}
bool hasOrphanedTops(const RoseBuildImpl &build) {
const RoseGraph &g = build.g;
ue2::unordered_map<left_id, set<u32> > roses;
ue2::unordered_map<suffix_id, set<u32> > suffixes;
for (auto v : vertices_range(g)) {
if (g[v].left) {
set<u32> &tops = roses[g[v].left];
if (!build.isRootSuccessor(v)) {
// Tops for infixes come from the in-edges.
for (const auto &e : in_edges_range(v, g)) {
tops.insert(g[e].rose_top);
}
}
}
if (g[v].suffix) {
suffixes[g[v].suffix].insert(g[v].suffix.top);
}
}
for (const auto &e : roses) {
if (all_tops(e.first) != e.second) {
DEBUG_PRINTF("rose tops (%s) don't match rose graph (%s)\n",
as_string_list(all_tops(e.first)).c_str(),
as_string_list(e.second).c_str());
return true;
}
}
for (const auto &e : suffixes) {
if (all_tops(e.first) != e.second) {
DEBUG_PRINTF("suffix tops (%s) don't match rose graph (%s)\n",
as_string_list(all_tops(e.first)).c_str(),
as_string_list(e.second).c_str());
return true;
}
}
return false;
}
#endif // NDEBUG
} // namespace ue2

33
src/rose/rose_build_role_aliasing.cpp
View File

@ -863,7 +863,7 @@ void pruneUnusedTops(CastleProto &castle, const RoseGraph &g,
static
void pruneUnusedTops(NGHolder &h, const RoseGraph &g,
const set<RoseVertex> &verts) {
ue2::unordered_set<u32> used_tops;
ue2::flat_set<u32> used_tops;
for (auto v : verts) {
assert(g[v].left.graph.get() == &h);
@ -879,10 +879,13 @@ void pruneUnusedTops(NGHolder &h, const RoseGraph &g,
if (v == h.startDs) {
continue; // stylised edge, leave it alone.
}
u32 top = h[e].top;
if (!contains(used_tops, top)) {
DEBUG_PRINTF("edge (start,%u) has unused top %u\n",
h[v].index, top);
flat_set<u32> pruned_tops;
auto pt_inserter = inserter(pruned_tops, pruned_tops.end());
set_intersection(h[e].tops.begin(), h[e].tops.end(),
used_tops.begin(), used_tops.end(), pt_inserter);
h[e].tops = move(pruned_tops);
if (h[e].tops.empty()) {
DEBUG_PRINTF("edge (start,%u) has only unused tops\n", h[v].index);
dead.push_back(e);
}
}
@ -1327,8 +1330,8 @@ bool attemptRoseGraphMerge(RoseBuildImpl &build, bool preds_same, RoseVertex a,
DEBUG_PRINTF("winner %zu states\n", num_vertices(*b_h));
if (!setDistinctRoseTops(g, victim, *b_h, deque<RoseVertex>(1, a))) {
assert(roseHasTops(g, a));
assert(roseHasTops(g, b));
assert(roseHasTops(build, a));
assert(roseHasTops(build, b));
return false;
}
@ -1341,8 +1344,8 @@ bool attemptRoseGraphMerge(RoseBuildImpl &build, bool preds_same, RoseVertex a,
for (const auto &e : in_edges_range(a, g)) {
g[e] = a_props[source(e, g)];
}
assert(roseHasTops(g, a));
assert(roseHasTops(g, b));
assert(roseHasTops(build, a));
assert(roseHasTops(build, b));
return false;
}
@ -1365,8 +1368,8 @@ bool attemptRoseGraphMerge(RoseBuildImpl &build, bool preds_same, RoseVertex a,
reduceImplementableGraph(*b_h, SOM_NONE, nullptr, build.cc);
assert(roseHasTops(g, a));
assert(roseHasTops(g, b));
assert(roseHasTops(build, a));
assert(roseHasTops(build, b));
assert(isImplementableNFA(*b_h, nullptr, build.cc));
return true;
}
@ -1417,8 +1420,8 @@ bool attemptRoseMerge(RoseBuildImpl &build, bool preds_same, RoseVertex a,
return false;
}
assert(roseHasTops(g, a));
assert(roseHasTops(g, b));
assert(roseHasTops(build, a));
assert(roseHasTops(build, b));
if (a_left_id.graph() && b_left_id.graph()) {
return attemptRoseGraphMerge(build, preds_same, a, b, trivialCasesOnly,
@ -1737,6 +1740,7 @@ void leftMergePass(CandidateSet &candidates, RoseBuildImpl &build,
}
DEBUG_PRINTF("%zu candidates remaining\n", candidates.size());
assert(!hasOrphanedTops(build));
}
// Can't merge vertices with different root predecessors.
@ -1952,6 +1956,7 @@ void rightMergePass(CandidateSet &candidates, RoseBuildImpl &build,
}
DEBUG_PRINTF("%zu candidates remaining\n", candidates.size());
assert(!hasOrphanedTops(build));
}
/**
@ -2017,6 +2022,7 @@ void filterDiamondCandidates(RoseGraph &g, CandidateSet &candidates) {
void aliasRoles(RoseBuildImpl &build, bool mergeRoses) {
const CompileContext &cc = build.cc;
RoseGraph &g = build.g;
assert(!hasOrphanedTops(build));
if (!cc.grey.roseRoleAliasing || !cc.grey.roseGraphReduction) {
return;
@ -2050,6 +2056,7 @@ void aliasRoles(RoseBuildImpl &build, bool mergeRoses) {
DEBUG_PRINTF("killed %zu vertices\n", dead.size());
build.removeVertices(dead);
assert(!hasOrphanedTops(build));
}
} // namespace ue2

2
src/util/ue2_containers.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met: