github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-06-28 16:41:01 +03:00
Code Issues Packages Projects Releases Wiki Activity
vectorscan/src/nfagraph/ng_rose.cpp
Justin Viiret 4ce268af47 ng: ensure that only match states have reports 
Ensure (and assert) that vertices without an edge to {accept, acceptEod}
do not have reports set.
2016-08-10 15:05:23 +10:00

2973 lines
90 KiB
C++
Raw Blame History

/*
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief Rose construction from NGHolder.
*/
// #define DEBUG
// #define DEBUG_ROSE
#include "ng_rose.h"
#include "grey.h"
#include "ng_depth.h"
#include "ng_dominators.h"
#include "ng_equivalence.h"
#include "ng_holder.h"
#include "ng_is_equal.h"
#include "ng_literal_analysis.h"
#include "ng_netflow.h"
#include "ng_prune.h"
#include "ng_redundancy.h"
#include "ng_region.h"
#include "ng_reports.h"
#include "ng_split.h"
#include "ng_util.h"
#include "ng_width.h"
#include "rose/rose_build.h"
#include "rose/rose_build_util.h"
#include "rose/rose_in_dump.h"
#include "rose/rose_in_graph.h"
#include "rose/rose_in_util.h"
#include "util/compare.h"
#include "util/compile_context.h"
#include "util/container.h"
#include "util/graph.h"
#include "util/graph_range.h"
#include "util/make_unique.h"
#include "util/order_check.h"
#include "util/ue2string.h"
#include "util/ue2_containers.h"
#include <set>
#include <utility>
#include <vector>
#include <boost/core/noncopyable.hpp>
#include <boost/graph/reverse_graph.hpp>
#include <boost/graph/topological_sort.hpp>
#define NDEBUG_PRINTF(x, ...) \
do { if (0) { DEBUG_PRINTF(x, ## __VA_ARGS__); } } while (0)
using namespace std;
namespace ue2 {
/**
* Maps vertices in the original graph to vertices on edge graphs. Each edge
* graph should contain at most one copy of the vertex. Multiple images for a
* vertex arise after we split on multiple literals - in this cases all edges
* should share a common graph.
*
* If, when an edge is split, a vertex ends up in both the LHS and RHS then only
* the LHS is tracked. This is because in general we want to simplify the LHS
* and allow complexity to be pushed further back.
*/
typedef ue2::unordered_map<NFAVertex, vector<pair<RoseInEdge, NFAVertex> > >
vdest_map_t;
typedef ue2::unordered_map<RoseInEdge, vector<NFAVertex> > vsrc_map_t;
/**
* \brief Maximum width of the character class usable as an escape class.
*/
static const u32 MAX_ESCAPE_CHARS = 20;
static
u32 maxDelay(const CompileContext &cc) {
if (!cc.streaming) {
return MO_INVALID_IDX;
}
return cc.grey.maxHistoryAvailable;
}
static
bool createsAnchoredLHS(const NGHolder &g, const vector<NFAVertex> &vv,
const vector<NFAVertexDepth> &depths,
const Grey &grey, depth max_depth = depth::infinity()) {
max_depth = min(max_depth, depth(grey.maxAnchoredRegion));
for (auto v : vv) {
/* avoid issues of self loops blowing out depths:
* look at preds, add 1 */
for (auto u : inv_adjacent_vertices_range(v, g)) {
if (u == v) {
continue;
}
u32 idx = g[u].index;
assert(idx < depths.size());
if (maxDistFromStartOfData(depths.at(idx)) >= max_depth) {
return false;
}
}
}
return true;
}
static
bool createsTransientLHS(const NGHolder &g, const vector<NFAVertex> &vv,
const vector<NFAVertexDepth> &depths,
const Grey &grey) {
const depth max_depth(grey.maxHistoryAvailable);
for (auto v : vv) {
/* avoid issues of self loops blowing out depths:
* look at preds, add 1 */
for (auto u : inv_adjacent_vertices_range(v, g)) {
if (u == v) {
continue;
}
u32 idx = g[u].index;
assert(idx < depths.size());
if (maxDistFromInit(depths.at(idx)) >= max_depth) {
return false;
}
}
}
return true;
}
static
bool isLHSUsablyAnchored(const NGHolder &g,
const vector<NFAVertexDepth> &depths,
const Grey &grey) {
assert(in_degree(g.acceptEod, g) == 1);
vector<NFAVertex> accepts;
insert(&accepts, accepts.end(), inv_adjacent_vertices(g.accept, g));
bool rv = createsAnchoredLHS(g, accepts, depths, grey);
DEBUG_PRINTF("lhs is %susably anchored\n", rv ? "" : "not ");
return rv;
}
static
bool isLHSTransient(const NGHolder &g,
const vector<NFAVertexDepth> &depths,
const Grey &grey) {
assert(in_degree(g.acceptEod, g) == 1);
vector<NFAVertex> accepts;
insert(&accepts, accepts.end(), inv_adjacent_vertices(g.accept, g));
bool rv = createsTransientLHS(g, accepts, depths, grey);
DEBUG_PRINTF("lhs is %stransient\n", rv ? "" : "not ");
return rv;
}
namespace {
/**
* Information on a cut: vertices and literals.
*/
struct VertLitInfo {
VertLitInfo(NFAVertex v, const set<ue2_literal> &litlit)
: vv(vector<NFAVertex>(1, v)), lit(litlit) {}
VertLitInfo(const vector<NFAVertex> &vvvv, const set<ue2_literal> &litlit)
: vv(vvvv), lit(litlit) {}
vector<NFAVertex> vv;
set<ue2_literal> lit;
};
/**
* A factory for candidate simple cuts (literals/vertices).
*/
class LitCollection : boost::noncopyable {
vector<unique_ptr<VertLitInfo>> lits; /**< sorted list of potential cuts */
const NGHolder &g; /**< graph on which cuts are found */
const vector<NFAVertexDepth> &depths; /**< depth information for g */
const ue2::unordered_map<NFAVertex, u32> &region_map; /**< region map for g */
/** Set of vertices to avoid selecting as end vertices for cuts as previous
* cuts overlap them. This is solely to prevent us picking literal sets
* which do not add significant value. */
ue2::unordered_set<NFAVertex> poisoned;
/** Back-edges in g. */
ue2::unordered_map<NFAVertex, vector<NFAVertex> > back_edges;
const Grey &grey;
bool seeking_transient;
bool seeking_anchored;
void poisonLHS(const VertLitInfo &picked);
void poisonLitVerts(const VertLitInfo &picked);
void poisonCandidates(const VertLitInfo &picked);
friend class LitComparator;
public:
LitCollection(const NGHolder &g_in, const vector<NFAVertexDepth> &depths_in,
const ue2::unordered_map<NFAVertex, u32> &region_map_in,
const set<NFAVertex> &ap, const set<NFAVertex> &ap_raw,
u32 min_len, bool desperation, const CompileContext &cc,
bool override_literal_quality_check = false);
/**< Returns the next candidate cut. Cut still needs to be inspected for
* complete envelopment. */
unique_ptr<VertLitInfo> pickNext(void);
};
/**
* \brief Comparator class for sorting LitCollection::lits.
*
* This is separated out from LitCollection itself as passing LitCollection to
* std::sort() would incur a (potentially expensive) copy.
*/
class LitComparator {
public:
explicit LitComparator(const LitCollection &lc_in) : lc(lc_in) {}
bool operator()(const unique_ptr<VertLitInfo> &a,
const unique_ptr<VertLitInfo> &b) const {
assert(a && b);
if (lc.seeking_anchored) {
bool a_anchored =
createsAnchoredLHS(lc.g, a->vv, lc.depths, lc.grey);
bool b_anchored =
createsAnchoredLHS(lc.g, b->vv, lc.depths, lc.grey);
if (a_anchored != b_anchored) {
return a_anchored < b_anchored;
}
}
if (lc.seeking_transient) {
bool a_transient =
createsTransientLHS(lc.g, a->vv, lc.depths, lc.grey);
bool b_transient =
createsTransientLHS(lc.g, b->vv, lc.depths, lc.grey);
if (a_transient != b_transient) {
return a_transient < b_transient;
}
}
u64a score_a = scoreSet(a->lit);
u64a score_b = scoreSet(b->lit);
if (score_a != score_b) {
return score_a > score_b;
}
/* vertices should only be in one candidate cut */
assert(a->vv == b->vv || a->vv.front() != b->vv.front());
return lc.g[a->vv.front()].index >
lc.g[b->vv.front()].index;
}
private:
const LitCollection &lc;
};
static
size_t shorter_than(const set<ue2_literal> &s, size_t limit) {
size_t count = 0;
for (const auto &lit : s) {
if (lit.length() < limit) {
count++;
}
}
return count;
}
static
u32 min_len(const set<ue2_literal> &s) {
u32 rv = ~0U;
for (const auto &lit : s) {
rv = min(rv, (u32)lit.length());
}
return rv;
}
static
u32 max_len(const set<ue2_literal> &s) {
u32 rv = 0;
for (const auto &lit : s) {
rv = max(rv, (u32)lit.length());
}
return rv;
}
static
u32 min_period(const set<ue2_literal> &s) {
u32 rv = ~0U;
for (const auto &lit : s) {
rv = min(rv, (u32)minStringPeriod(lit));
}
DEBUG_PRINTF("min period %u\n", rv);
return rv;
}
static
bool validateRoseLiteralSetQuality(const set<ue2_literal> &s, u64a score,
u32 min_allowed_len, bool desperation,
bool override_literal_quality_check) {
if (!override_literal_quality_check && score >= NO_LITERAL_AT_EDGE_SCORE) {
DEBUG_PRINTF("candidate is too bad %llu/%zu\n", score, s.size());
return false;
}
assert(!s.empty());
if (s.empty()) {
DEBUG_PRINTF("candidate is too bad/something went wrong\n");
return false;
}
u32 s_min_len = min_len(s);
u32 s_min_period = min_period(s);
size_t short_count = shorter_than(s, 5);
DEBUG_PRINTF("cand '%s': score %llu count=%zu min_len=%u min_period=%u"
" short_count=%zu desp=%d\n",
dumpString(*s.begin()).c_str(), score, s.size(), s_min_len,
s_min_period, short_count, (int)desperation);
bool ok = true;
if (s.size() > 10 /* magic number is magic */
|| s_min_len < min_allowed_len
|| (s_min_period <= 1 && !override_literal_quality_check
&& min_allowed_len != 1)) {
ok = false;
}
if (!ok && desperation
&& s.size() <= 20 /* more magic numbers are magical */
&& (s_min_len > 5 || (s_min_len > 2 && short_count <= 10))
&& s_min_period > 1) {
DEBUG_PRINTF("candidate is ok\n");
ok = true;
}
if (!ok && desperation
&& s.size() <= 50 /* more magic numbers are magical */
&& s_min_len > 10
&& s_min_period > 1) {
DEBUG_PRINTF("candidate is ok\n");
ok = true;
}
if (!ok) {
DEBUG_PRINTF("candidate is too bad\n");
return false;
}
return true;
}
static UNUSED
void dumpRoseLiteralSet(const set<ue2_literal> &s) {
for (UNUSED const auto &lit : s) {
DEBUG_PRINTF(" lit: %s\n", dumpString(lit).c_str());
}
}
static
void getSimpleRoseLiterals(const NGHolder &g, const set<NFAVertex> &a_dom,
vector<unique_ptr<VertLitInfo>> *lits,
u32 min_allowed_len, bool desperation,
bool override_literal_quality_check) {
map<NFAVertex, u64a> scores;
map<NFAVertex, unique_ptr<VertLitInfo>> lit_info;
set<ue2_literal> s;
for (auto v : a_dom) {
s = getLiteralSet(g, v, true); /* RHS will take responsibility for any
revisits to the target vertex */
if (s.empty()) {
DEBUG_PRINTF("candidate is too bad\n");
continue;
}
DEBUG_PRINTF("|candidate raw literal set| = %zu\n", s.size());
dumpRoseLiteralSet(s);
u64a score = compressAndScore(s);
if (!validateRoseLiteralSetQuality(s, score, min_allowed_len,
desperation,
override_literal_quality_check)) {
continue;
}
DEBUG_PRINTF("candidate is a candidate\n");
scores[v] = score;
lit_info.insert(make_pair(v, ue2::make_unique<VertLitInfo>(v, s)));
}
/* try to filter out cases where appending some characters produces worse
* literals. Only bother to look back one byte, TODO make better */
for (auto u : a_dom) {
if (out_degree(u, g) != 1 || !scores[u]) {
continue;
}
NFAVertex v = *adjacent_vertices(u, g).first;
if (contains(scores, v) && scores[v] >= scores[u]) {
DEBUG_PRINTF("killing off v as score %llu >= %llu\n",
scores[v], scores[u]);
lit_info.erase(v);
}
}
lits->reserve(lit_info.size());
for (auto &m : lit_info) {
lits->push_back(move(m.second));
}
DEBUG_PRINTF("%zu candidate literal sets\n", lits->size());
}
static
void getRegionRoseLiterals(const NGHolder &g,
const ue2::unordered_map<NFAVertex, u32> &region_map,
const set<NFAVertex> &a_dom_raw,
vector<unique_ptr<VertLitInfo>> *lits,
u32 min_allowed_len, bool desperation,
bool override_literal_quality_check) {
/* This allows us to get more places to chop the graph as we are not limited
to points where there is a single vertex to split. */
/* TODO: operate over 'proto-regions' which ignore back edges */
set<u32> mand, optional;
map<u32, vector<NFAVertex> > exits;
for (auto v : vertices_range(g)) {
assert(contains(region_map, v));
const u32 region = region_map.at(v);
if (is_any_start(v, g) || region == 0) {
continue;
}
if (is_any_accept(v, g)) {
continue;
}
if (isRegionExit(g, v, region_map)) {
exits[region].push_back(v);
}
if (isRegionEntry(g, v, region_map)) {
// Determine whether this region is mandatory or optional. We only
// need to do this check for the first entry vertex we encounter
// for this region.
if (!contains(mand, region) && !contains(optional, region)) {
if (isOptionalRegion(g, v, region_map)) {
optional.insert(region);
} else {
mand.insert(region);
}
}
}
}
for (const auto &m : exits) {
if (0) {
next_cand:
continue;
}
const u32 region = m.first;
const vector<NFAVertex> &vv = m.second;
assert(!vv.empty());
if (!contains(mand, region)) {
continue;
}
for (auto v : vv) {
/* if an exit is in a_dom_raw, the region is already handled well
* by getSimpleRoseLiterals */
if (contains(a_dom_raw, v)) {
goto next_cand;
}
}
/* the final region may not have a neat exit. validate that all exits
* have an edge to each accept or none do */
bool edge_to_a = edge(vv[0], g.accept, g).second;
bool edge_to_aeod = edge(vv[0], g.acceptEod, g).second;
const auto &reports = g[vv[0]].reports;
for (auto v : vv) {
if (edge_to_a != edge(v, g.accept, g).second) {
goto next_cand;
}
if (edge_to_aeod != edge(v, g.acceptEod, g).second) {
goto next_cand;
}
if (g[v].reports != reports) {
goto next_cand;
}
}
DEBUG_PRINTF("inspecting region %u\n", region);
set<ue2_literal> s;
for (auto v : vv) {
DEBUG_PRINTF(" exit vertex: %u\n", g[v].index);
/* Note: RHS can not be depended on to take all subsequent revisits
* to this vertex */
set<ue2_literal> ss = getLiteralSet(g, v, false);
if (ss.empty()) {
DEBUG_PRINTF("candidate is too bad\n");
goto next_cand;
}
insert(&s, ss);
}
assert(!s.empty());
DEBUG_PRINTF("|candidate raw literal set| = %zu\n", s.size());
dumpRoseLiteralSet(s);
u64a score = compressAndScore(s);
DEBUG_PRINTF("|candidate literal set| = %zu\n", s.size());
dumpRoseLiteralSet(s);
if (!validateRoseLiteralSetQuality(s, score, min_allowed_len,
desperation,
override_literal_quality_check)) {
continue;
}
DEBUG_PRINTF("candidate is a candidate\n");
lits->push_back(ue2::make_unique<VertLitInfo>(vv, s));
}
}
static
void gatherBackEdges(const NGHolder &g,
ue2::unordered_map<NFAVertex, vector<NFAVertex>> *out) {
set<NFAEdge> backEdges;
BackEdges<set<NFAEdge>> be(backEdges);
depth_first_search(g.g, visitor(be).root_vertex(g.start).vertex_index_map(
get(&NFAGraphVertexProps::index, g.g)));
for (const auto &e : backEdges) {
(*out)[source(e, g)].push_back(target(e, g));
}
}
LitCollection::LitCollection(const NGHolder &g_in,
const vector<NFAVertexDepth> &depths_in,
const ue2::unordered_map<NFAVertex, u32> &region_map_in,
const set<NFAVertex> &a_dom,
const set<NFAVertex> &a_dom_raw, u32 min_len,
bool desperation, const CompileContext &cc,
bool override_literal_quality_check)
: g(g_in), depths(depths_in), region_map(region_map_in), grey(cc.grey),
seeking_transient(cc.streaming), seeking_anchored(true) {
getSimpleRoseLiterals(g, a_dom, &lits, min_len, desperation,
override_literal_quality_check);
getRegionRoseLiterals(g, region_map, a_dom_raw, &lits, min_len, desperation,
override_literal_quality_check);
DEBUG_PRINTF("lit coll is looking for a%d t%d\n", (int)seeking_anchored,
(int)seeking_transient);
DEBUG_PRINTF("we have %zu candidate literal splits\n", lits.size());
sort(lits.begin(), lits.end(), LitComparator(*this));
gatherBackEdges(g, &back_edges);
}
void LitCollection::poisonLHS(const VertLitInfo &picked) {
DEBUG_PRINTF("found anchored %d transient %d\n",
(int)createsAnchoredLHS(g, picked.vv, depths, grey),
(int)createsTransientLHS(g, picked.vv, depths, grey));
set<NFAVertex> curr;
set<NFAVertex> next;
insert(&curr, picked.vv);
while (!curr.empty()) {
insert(&poisoned, curr);
next.clear();
for (auto v : curr) {
for (auto u : inv_adjacent_vertices_range(v, g)) {
if (!is_special(u, g) && !contains(poisoned, u)) {
next.insert(u);
}
}
}
curr.swap(next);
}
seeking_transient = false;
seeking_anchored = false;
/* reprioritise cuts now that the LHS is taken care off */
sort(lits.begin(), lits.end(), LitComparator(*this));
}
static
void flood_back(const NGHolder &g, u32 len, const set<NFAVertex> &initial,
set<NFAVertex> *visited) {
vector<NFAVertex> curr;
vector<NFAVertex> next;
insert(&curr, curr.end(), initial);
insert(visited, initial);
/* bfs: flood back len vertices */
for (u32 i = 1; i < len; i++) {
next.clear();
DEBUG_PRINTF("poison %u/%u: curr %zu\n", i, len, curr.size());
for (auto v : curr) {
for (auto u : inv_adjacent_vertices_range(v, g)) {
if (!contains(*visited, u)) {
next.push_back(u);
visited->insert(u);
}
}
}
next.swap(curr);
}
}
/**
* Add vertices near a picked literal to the poison set unless it looks
* like they may still add value (ie they are on they other side of cycle).
*/
void LitCollection::poisonLitVerts(const VertLitInfo &picked) {
DEBUG_PRINTF("poisoning vertices associated with picked literals\n");
u32 len = max_len(picked.lit);
/* poison vertices behind */
set<NFAVertex> starters;
insert(&starters, picked.vv);
set<NFAVertex> visited;
flood_back(g, len, starters, &visited);
DEBUG_PRINTF("flood %zu vertices\n", visited.size());
/* inspect any back edges which are in the flooded subgraph; look for any
* destination vertices which are not starters */
set<NFAVertex> anti;
for (auto u : visited) {
if (!contains(back_edges, u) || contains(starters, u)) {
continue;
}
for (auto v : back_edges[u]) {
if (contains(visited, v) && !contains(starters, v)) {
anti.insert(v);
}
}
}
DEBUG_PRINTF("%zu cycle ends\n", visited.size());
/* remove any vertices which lie on the other side of a cycle from the
* visited set */
set<NFAVertex> anti_pred;
flood_back(g, len - 1, anti, &anti_pred);
DEBUG_PRINTF("flood visited %zu vertices; anti %zu\n", visited.size(),
anti_pred.size());
erase_all(&visited, anti_pred);
DEBUG_PRINTF("filtered flood visited %zu vertices\n", visited.size());
insert(&poisoned, visited);
insert(&poisoned, starters); /* complicated back loops can result in start
vertices being removed from the visited
set */
for (UNUSED auto v : picked.vv) {
assert(contains(poisoned, v));
}
/* TODO: poison vertices in front of us? */
}
void LitCollection::poisonCandidates(const VertLitInfo &picked) {
assert(!picked.lit.empty());
if (picked.lit.empty()) {
return;
}
if ((seeking_anchored && createsAnchoredLHS(g, picked.vv, depths, grey))
|| (seeking_transient && createsTransientLHS(g, picked.vv, depths, grey))) {
/* We don't want to pick anything to the LHS of picked.v any more as we
* have something good. We also don't want to provide any bonus for
* remaining literals based on anchoredness/transientness of the lhs.
*/
poisonLHS(picked);
} else {
poisonLitVerts(picked);
}
}
unique_ptr<VertLitInfo> LitCollection::pickNext() {
while (!lits.empty()) {
if (0) {
next_lit:
continue;
}
for (auto v : lits.back()->vv) {
if (contains(poisoned, v)) {
DEBUG_PRINTF("skipping '%s' as overlapped\n",
((const string &)*lits.back()->lit.begin()).c_str());
lits.pop_back();
goto next_lit;
}
}
unique_ptr<VertLitInfo> rv = move(lits.back());
lits.pop_back();
poisonCandidates(*rv);
DEBUG_PRINTF("best is '%s' %u a%d t%d\n",
((const string &)*rv->lit.begin()).c_str(),
g[rv->vv.front()].index,
(int)createsAnchoredLHS(g, rv->vv, depths, grey),
(int)createsTransientLHS(g, rv->vv, depths, grey));
return rv;
}
return nullptr;
}
}
static
bool can_match(const NGHolder &g, const ue2_literal &lit, bool overhang_ok) {
set<NFAVertex> curr, next;
curr.insert(g.accept);
for (auto it = lit.rbegin(); it != lit.rend(); ++it) {
next.clear();
for (auto v : curr) {
for (auto u : inv_adjacent_vertices_range(v, g)) {
if (u == g.start) {
if (overhang_ok) {
DEBUG_PRINTF("bail\n");
return true;
} else {
continue; /* it is not possible for a lhs literal to
* overhang the start */
}
}
const CharReach &cr = g[u].char_reach;
if (!overlaps(*it, cr)) {
DEBUG_PRINTF("skip\n");
continue;
}
next.insert(u);
}
}
curr.swap(next);
}
return !curr.empty();
}
u32 removeTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
u32 max_delay, bool overhang_ok) {
if (max_delay == MO_INVALID_IDX) {
max_delay--;
}
DEBUG_PRINTF("killing off '%s'\n", ((const string &)lit).c_str());
set<NFAVertex> curr, next;
curr.insert(g.accept);
auto it = lit.rbegin();
for (u32 delay = max_delay; delay > 0 && it != lit.rend(); delay--, ++it) {
next.clear();
for (auto v : curr) {
for (auto u : inv_adjacent_vertices_range(v, g)) {
if (u == g.start) {
if (overhang_ok) {
DEBUG_PRINTF("bail\n");
goto bail; /* things got complicated */
} else {
continue; /* it is not possible for a lhs literal to
* overhang the start */
}
}
const CharReach &cr = g[u].char_reach;
if (!overlaps(*it, cr)) {
DEBUG_PRINTF("skip\n");
continue;
}
if (isSubsetOf(*it, cr)) {
next.insert(u);
} else {
DEBUG_PRINTF("bail\n");
goto bail; /* things got complicated */
}
}
}
curr.swap(next);
}
bail:
if (curr.empty()) {
/* This can happen when we have an edge representing a cross from two
* sides of an alternation. This whole edge needs to be marked as
* dead */
assert(0); /* should have been picked up by can match */
return MO_INVALID_IDX;
}
u32 delay = distance(lit.rbegin(), it);
assert(delay <= max_delay);
assert(delay <= lit.length());
DEBUG_PRINTF("managed delay %u (of max %u)\n", delay, max_delay);
// For determinism, we make sure that we create these edges from vertices
// in index-sorted order.
set<NFAVertex> pred;
for (auto v : curr) {
insert(&pred, inv_adjacent_vertices_range(v, g));
}
clear_in_edges(g.accept, g);
clearReports(g);
vector<NFAVertex> verts(pred.begin(), pred.end());
sort(verts.begin(), verts.end(), VertexIndexOrdering<NGHolder>(g));
for (auto v : verts) {
add_edge(v, g.accept, g);
g[v].reports.insert(0);
}
pruneUseless(g);
assert(allMatchStatesHaveReports(g));
DEBUG_PRINTF("graph has %zu vertices left\n", num_vertices(g));
return delay;
}
void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
u32 delay, const vector<NFAVertex> &preds) {
assert(delay <= lit.length());
DEBUG_PRINTF("adding on '%s' %u\n", ((const string &)lit).c_str(), delay);
NFAVertex prev = g.accept;
auto it = lit.rbegin();
while (delay--) {
NFAVertex curr = add_vertex(g);
assert(it != lit.rend());
g[curr].char_reach = *it;
add_edge(curr, prev, g);
++it;
prev = curr;
}
for (auto v : preds) {
add_edge(v, prev, g);
}
// Every predecessor of accept must have a report.
for (auto u : inv_adjacent_vertices_range(g.accept, g)) {
g[u].reports.insert(0);
}
g.renumberVertices();
g.renumberEdges();
assert(allMatchStatesHaveReports(g));
}
void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
u32 delay) {
vector<NFAVertex> preds;
insert(&preds, preds.end(), inv_adjacent_vertices(g.accept, g));
clear_in_edges(g.accept, g);
for (auto v : preds) {
g[v].reports.clear(); /* clear report from old accepts */
}
restoreTrailingLiteralStates(g, lit, delay, preds);
}
/* return false if we should get rid of the edge altogether */
static
bool removeLiteralFromLHS(RoseInGraph &ig, const RoseInEdge &lhs,
const CompileContext &cc) {
unique_ptr<NGHolder> h = cloneHolder(*ig[lhs].graph);
NGHolder &g = *h;
assert(ig[target(lhs, ig)].type == RIV_LITERAL);
const ue2_literal &lit = ig[target(lhs, ig)].s;
/* lhs should be connected to a start */
assert(ig[source(lhs, ig)].type == RIV_START
|| ig[source(lhs, ig)].type == RIV_ANCHORED_START);
if (in_degree(g.acceptEod, g) != 1 /* edge from accept */) {
assert(0);
return true;
}
if (lit.empty()) {
assert(0);
return true;
}
const u32 max_delay = maxDelay(cc);
// In streaming mode, we must limit the depth to the available history
// UNLESS the given literal follows start or startDs and has nothing
// before it that we will need to account for. In that case, we can
// lean on FDR's support for long literals.
if (literalIsWholeGraph(g, lit)) {
assert(!ig[lhs].haig);
assert(ig[lhs].minBound == 0);
assert(ig[lhs].maxBound == ROSE_BOUND_INF);
DEBUG_PRINTF("literal is the whole graph\n");
u32 delay = removeTrailingLiteralStates(g, lit, MO_INVALID_IDX, false);
assert(delay == lit.length());
ig[lhs].graph = move(h);
ig[lhs].graph_lag = delay;
return true;
}
if (!can_match(g, lit, false)) {
/* This is can happen if the literal arises from a large cyclic
to/beyond the pivot. As the LHS graph only cares about the first
reach of the pivot, this literal is junk */
DEBUG_PRINTF("bogus edge\n");
return false;
}
u32 delay = removeTrailingLiteralStates(g, lit, max_delay,
false /* can't overhang start */);
if (delay == MO_INVALID_IDX) {
/* This is can happen if the literal arises from a large cyclic
to/beyond the pivot. As the LHS graph only cares about the first
reach of the pivot, this literal is junk */
DEBUG_PRINTF("bogus edge\n");
return false;
}
if (!delay) {
return true;
}
DEBUG_PRINTF("setting delay %u on lhs %p\n", delay, h.get());
ig[lhs].graph = move(h);
ig[lhs].graph_lag = delay;
return true;
}
static
void handleLhsCliche(RoseInGraph &ig, const RoseInEdge &lhs) {
const NGHolder &h = *ig[lhs].graph;
size_t s_od = out_degree(h.start, h);
size_t sds_od = out_degree(h.startDs, h);
assert(in_degree(h.acceptEod, h) == 1 /* edge from accept */);
/* need to check if simple floating start */
if (edge(h.startDs, h.accept, h).second && sds_od == 2
&& ((s_od == 2 && edge(h.start, h.accept, h).second) || s_od == 1)) {
/* no need for graph */
ig[lhs].graph.reset();
ig[lhs].graph_lag = 0;
DEBUG_PRINTF("lhs is floating start\n");
return;
}
/* need to check if a simple anchor */
/* start would have edges to sds and accept in this case */
if (edge(h.start, h.accept, h).second && s_od == 2 && sds_od == 1) {
if (ig[source(lhs, ig)].type == RIV_ANCHORED_START) {
// assert(ig[lhs].graph_lag == ig[target(lhs, ig)].s.length());
if (ig[lhs].graph_lag != ig[target(lhs, ig)].s.length()) {
DEBUG_PRINTF("oddness\n");
return;
}
ig[lhs].graph.reset();
ig[lhs].graph_lag = 0;
ig[lhs].maxBound = 0;
DEBUG_PRINTF("lhs is anchored start\n");
} else {
DEBUG_PRINTF("lhs rewiring start\n");
assert(ig[source(lhs, ig)].type == RIV_START);
RoseInVertex t = target(lhs, ig);
remove_edge(lhs, ig);
RoseInVertex s2
= add_vertex(RoseInVertexProps::makeStart(true), ig);
add_edge(s2, t, RoseInEdgeProps(0U, 0U), ig);
}
return;
}
}
static
void filterCandPivots(const NGHolder &g, const set<NFAVertex> &cand_raw,
set<NFAVertex> *out) {
for (auto u : cand_raw) {
const CharReach &u_cr = g[u].char_reach;
if (u_cr.count() > 40) {
continue; /* too wide to be plausible */
}
if (u_cr.count() > 2) {
/* include u as a candidate as successor may have backed away from
* expanding through it */
out->insert(u);
continue;
}
NFAVertex v = getSoleDestVertex(g, u);
if (v && in_degree(v, g) == 1 && out_degree(u, g) == 1) {
const CharReach &v_cr = g[v].char_reach;
if (v_cr.count() == 1 || v_cr.isCaselessChar()) {
continue; /* v will always generate better literals */
}
}
out->insert(u);
}
}
/* cand_raw is the candidate set before filtering points which are clearly
* a bad idea. */
static
void getCandidatePivots(const NGHolder &g, set<NFAVertex> *cand,
set<NFAVertex> *cand_raw) {
ue2::unordered_map<NFAVertex, NFAVertex> dominators =
findDominators(g);
set<NFAVertex> accepts;
for (auto v : inv_adjacent_vertices_range(g.accept, g)) {
if (is_special(v, g)) {
continue;
}
accepts.insert(v);
}
for (auto v : inv_adjacent_vertices_range(g.acceptEod, g)) {
if (is_special(v, g)) {
continue;
}
accepts.insert(v);
}
assert(!accepts.empty());
vector<NFAVertex> dom_trace;
auto ait = accepts.begin();
assert(ait != accepts.end());
NFAVertex curr = *ait;
while (curr && !is_special(curr, g)) {
dom_trace.push_back(curr);
curr = dominators[curr];
}
reverse(dom_trace.begin(), dom_trace.end());
for (++ait; ait != accepts.end(); ++ait) {
curr = *ait;
vector<NFAVertex> dom_trace2;
while (curr && !is_special(curr, g)) {
dom_trace2.push_back(curr);
curr = dominators[curr];
}
reverse(dom_trace2.begin(), dom_trace2.end());
auto dti = dom_trace.begin(), dtie = dom_trace.end();
auto dtj = dom_trace2.begin(), dtje = dom_trace2.end();
while (dti != dtie && dtj != dtje && *dti == *dtj) {
++dti;
++dtj;
}
dom_trace.erase(dti, dtie);
}
cand_raw->insert(dom_trace.begin(), dom_trace.end());
filterCandPivots(g, *cand_raw, cand);
}
static
void deanchorIfNeeded(NGHolder &g, bool *orig_anch) {
DEBUG_PRINTF("hi\n");
if (proper_out_degree(g.startDs, g)) {
return;
}
/* look for a non-special dot with a loop following start */
set<NFAVertex> succ_g;
insert(&succ_g, adjacent_vertices(g.start, g));
succ_g.erase(g.startDs);
for (auto v : adjacent_vertices_range(g.start, g)) {
DEBUG_PRINTF("inspecting cand %u || =%zu\n", g[v].index,
g[v].char_reach.size());
if (v == g.startDs || !g[v].char_reach.all()) {
continue;
}
set<NFAVertex> succ_v;
insert(&succ_v, adjacent_vertices(v, g));
if (succ_v == succ_g) {
DEBUG_PRINTF("found ^.*\n");
*orig_anch = true;
for (auto succ : succ_g) {
add_edge(g.startDs, succ, g);
}
clear_vertex(v, g);
remove_vertex(v, g);
g.renumberVertices();
return;
}
if (succ_g.size() == 1 && hasSelfLoop(v, g)) {
DEBUG_PRINTF("found ^.+\n");
*orig_anch = true;
add_edge(g.startDs, v, g);
remove_edge(v, v, g);
return;
}
}
}
static
unique_ptr<RoseInGraph> makeTrivialGraph(const NGHolder &h,
vdest_map_t &v_dest_map,
vsrc_map_t &v_src_map) {
shared_ptr<NGHolder> root_g = cloneHolder(h);
bool orig_anch = isAnchored(*root_g);
deanchorIfNeeded(*root_g, &orig_anch);
DEBUG_PRINTF("orig_anch %d\n", (int)orig_anch);
unique_ptr<RoseInGraph> igp = ue2::make_unique<RoseInGraph>();
RoseInVertex start =
add_vertex(RoseInVertexProps::makeStart(orig_anch), *igp);
RoseInVertex accept =
add_vertex(RoseInVertexProps::makeAccept(set<ReportID>()), *igp);
RoseInEdge e =
add_edge(start, accept, RoseInEdgeProps(root_g, 0), *igp).first;
for (auto v : vertices_range(*root_g)) {
v_dest_map[v].emplace_back(e, v);
v_src_map[e].push_back(v);
}
return igp;
}
static never_inline
void updateVDestMap(const vector<pair<RoseInEdge, NFAVertex> > &images,
const ue2::unordered_map<NFAVertex, NFAVertex> &lhs_map,
const vector<RoseInEdge> &l_e,
const ue2::unordered_map<NFAVertex, NFAVertex> &rhs_map,
const vector<RoseInEdge> &r_e,
vdest_map_t &v_dest_map, vsrc_map_t &v_src_map) {
RoseInEdge e = images.front().first;
set<RoseInEdge> edge_set;
for (const auto &image : images) {
edge_set.insert(image.first);
}
const vector<NFAVertex> &domain = v_src_map[e];
vector<pair<RoseInEdge, NFAVertex> > temp;
for (auto v : domain) {
vdest_map_t::iterator it = v_dest_map.find(v);
assert(it != v_dest_map.end());
temp.clear();
for (const auto &dest : it->second) {
const RoseInEdge &old_e = dest.first;
const NFAVertex old_dest = dest.second;
if (old_e != e) {
if (!contains(edge_set, old_e)) {
temp.emplace_back(old_e, old_dest);
}
} else if (contains(lhs_map, old_dest)) {
for (const auto &e2 : l_e) {
temp.emplace_back(e2, lhs_map.at(old_dest));
}
/* only allow v to be tracked on one side of the split */
} else if (contains(rhs_map, old_dest)) {
for (const auto &e2 : r_e) {
temp.emplace_back(e2, rhs_map.at(old_dest));
}
}
}
NDEBUG_PRINTF("%zu images for vertex; prev %zu\n", temp.size(),
it->second.size());
it->second.swap(temp);
}
}
/** Returns the collection of vertices from the original graph which end up
* having an image in the [lr]hs side of the graph split. */
static never_inline
void fillDomain(const vdest_map_t &v_dest_map, const vsrc_map_t &v_src_map,
RoseInEdge e,
const ue2::unordered_map<NFAVertex, NFAVertex> &split_map,
vector<NFAVertex> *out) {
const vector<NFAVertex> &presplit_domain = v_src_map.at(e);
for (auto v : presplit_domain) {
/* v is in the original graph, need to find its image on e's graph */
typedef vector<pair<RoseInEdge, NFAVertex> > dests_t;
const dests_t &dests = v_dest_map.at(v);
for (const auto &dest : dests) {
if (dest.first == e) {
NFAVertex vv = dest.second;
/* vv is v image on e's graph */
if (contains(split_map, vv)) {
out->push_back(v);
}
}
}
}
}
static
void getSourceVerts(RoseInGraph &ig,
const vector<pair<RoseInEdge, NFAVertex> > &images,
vector<RoseInVertex> *out) {
set<RoseInVertex> seen;
for (const auto &image : images) {
RoseInVertex s = source(image.first, ig);
if (contains(seen, s)) {
continue;
}
seen.insert(s);
out->push_back(s);
}
}
static
void getDestVerts(RoseInGraph &ig,
const vector<pair<RoseInEdge, NFAVertex> > &images,
vector<RoseInVertex> *out) {
set<RoseInVertex> seen;
for (const auto &image : images) {
RoseInVertex t = target(image.first, ig);
if (contains(seen, t)) {
continue;
}
seen.insert(t);
out->push_back(t);
}
}
static
void getSourceVerts(RoseInGraph &ig, const vector<RoseInEdge> &edges,
vector<RoseInVertex> *out) {
set<RoseInVertex> seen;
for (const auto &e : edges) {
RoseInVertex s = source(e, ig);
if (contains(seen, s)) {
continue;
}
seen.insert(s);
out->push_back(s);
}
}
static
void getDestVerts(RoseInGraph &ig, const vector<RoseInEdge> &edges,
vector<RoseInVertex> *out) {
set<RoseInVertex> seen;
for (const auto &e : edges) {
RoseInVertex t = target(e, ig);
if (contains(seen, t)) {
continue;
}
seen.insert(t);
out->push_back(t);
}
}
static
bool splitRoseEdge(RoseInGraph &ig, const VertLitInfo &split,
vdest_map_t &v_dest_map, vsrc_map_t &v_src_map) {
const vector<NFAVertex> &root_splitters = split.vv; /* vertices in the
'root' graph */
assert(!root_splitters.empty());
/* need copy as split rose edge will update orig map */
vector<pair<RoseInEdge, NFAVertex> > images
= v_dest_map[root_splitters[0]];
DEBUG_PRINTF("splitting %zu rose edge with %zu literals\n",
images.size(), split.lit.size());
/* note: as we haven't removed literals yet the graphs on all edges that we
* are going to split should be identical */
const auto &base_graph = ig[images.front().first].graph;
vector<NFAVertex> splitters; /* vertices in the graph being split */
for (auto v : root_splitters) {
if (!contains(v_dest_map, v)) {
DEBUG_PRINTF("vertex to split on is no longer in the graph\n");
return false;
}
/* sanity check: verify all edges have the same underlying graph */
for (UNUSED const auto &m : v_dest_map[v]) {
assert(base_graph == ig[m.first].graph);
}
assert(v_dest_map[v].size() == images.size());
splitters.push_back(v_dest_map[v].front().second);
}
/* note: the set of split edges should form a complete bipartite graph */
vector<RoseInVertex> src_verts;
vector<RoseInVertex> dest_verts;
getSourceVerts(ig, images, &src_verts);
getDestVerts(ig, images, &dest_verts);
assert(images.size() == src_verts.size() * dest_verts.size());
shared_ptr<NGHolder> lhs = make_shared<NGHolder>();
shared_ptr<NGHolder> rhs = make_shared<NGHolder>();
ue2::unordered_map<NFAVertex, NFAVertex> lhs_map;
ue2::unordered_map<NFAVertex, NFAVertex> rhs_map;
assert(base_graph);
splitGraph(*base_graph, splitters, lhs.get(), &lhs_map,
rhs.get(), &rhs_map);
RoseInEdge first_e = images.front().first;
/* all will be suffix or none */
bool suffix = ig[target(first_e, ig)].type == RIV_ACCEPT;
set<ReportID> splitter_reports;
for (auto v : splitters) {
insert(&splitter_reports, (*base_graph)[v].reports);
}
bool do_accept = false;
bool do_accept_eod = false;
assert(rhs);
if (isVacuous(*rhs) && suffix) {
if (edge(rhs->start, rhs->accept, *rhs).second) {
DEBUG_PRINTF("rhs has a cliche\n");
do_accept = true;
remove_edge(rhs->start, rhs->accept, *rhs);
}
if (edge(rhs->start, rhs->acceptEod, *rhs).second) {
DEBUG_PRINTF("rhs has an eod cliche\n");
do_accept_eod = true;
remove_edge(rhs->start, rhs->acceptEod, *rhs);
}
}
bool do_norm = out_degree(rhs->start, *rhs) != 1; /* check if we still have
a graph left over */
vector<NFAVertex> lhs_domain;
vector<NFAVertex> rhs_domain;
fillDomain(v_dest_map, v_src_map, first_e, lhs_map, &lhs_domain);
fillDomain(v_dest_map, v_src_map, first_e, rhs_map, &rhs_domain);
vector<RoseInEdge> l_e;
vector<RoseInEdge> r_e;
for (const auto &lit : split.lit) {
DEBUG_PRINTF("best is '%s'\n", escapeString(lit).c_str());
RoseInVertex v
= add_vertex(RoseInVertexProps::makeLiteral(lit), ig);
/* work out delay later */
if (do_accept) {
DEBUG_PRINTF("rhs has a cliche\n");
RoseInVertex tt = add_vertex(RoseInVertexProps::makeAccept(
splitter_reports), ig);
add_edge(v, tt, RoseInEdgeProps(0U, 0U), ig);
}
if (do_accept_eod) {
DEBUG_PRINTF("rhs has an eod cliche\n");
RoseInVertex tt = add_vertex(RoseInVertexProps::makeAcceptEod(
splitter_reports), ig);
add_edge(v, tt, RoseInEdgeProps(0U, 0U), ig);
}
for (auto src_v : src_verts) {
l_e.push_back(add_edge(src_v, v,
RoseInEdgeProps(lhs, 0U), ig).first);
v_src_map[l_e.back()] = lhs_domain;
}
if (do_norm) {
for (auto dst_v : dest_verts) {
/* work out delay later */
assert(out_degree(rhs->start, *rhs) > 1);
r_e.push_back(
add_edge(v, dst_v, RoseInEdgeProps(rhs, 0U), ig).first);
v_src_map[r_e.back()] = rhs_domain;
}
}
}
updateVDestMap(images, lhs_map, l_e, rhs_map, r_e, v_dest_map, v_src_map);
for (const auto &image : images) {
/* remove old edge */
remove_edge(image.first, ig);
v_src_map.erase(image.first);
}
return true;
}
static
bool isStarCliche(const NGHolder &g) {
DEBUG_PRINTF("checking graph with %zu vertices\n", num_vertices(g));
bool nonspecials_seen = false;
for (auto v : vertices_range(g)) {
if (is_special(v, g)) {
continue;
}
if (nonspecials_seen) {
return false;
}
nonspecials_seen = true;
if (!g[v].char_reach.all()) {
return false;
}
if (!hasSelfLoop(v, g)) {
return false;
}
if (!edge(v, g.accept, g).second) {
return false;
}
}
if (!nonspecials_seen) {
return false;
}
if (!edge(g.start, g.accept, g).second) {
return false;
}
return true;
}
static
void processInfixes(RoseInGraph &ig, const CompileContext &cc) {
/* we want to ensure that every prefix/infix graph is unique at this stage
* as we have not done any analysis to check if they are safe to share */
vector<RoseInEdge> dead;
for (const auto &e : edges_range(ig)) {
if (!ig[e].graph) {
continue;
}
RoseInVertex u = source(e, ig), v = target(e, ig);
// Infixes are edges between two literals.
if (ig[u].type != RIV_LITERAL || ig[v].type != RIV_LITERAL) {
continue;
}
if (ig[e].graph_lag) {
continue; /* already looked at */
}
DEBUG_PRINTF("looking at infix %p\n", ig[e].graph.get());
const ue2_literal &lit1 = ig[u].s;
const ue2_literal &lit2 = ig[v].s;
size_t overlap = maxOverlap(lit1, lit2, 0);
const NGHolder &h = *ig[e].graph;
DEBUG_PRINTF("infix rose between literals '%s' and '%s', overlap %zu,"
"size %zu\n",
dumpString(lit1).c_str(), dumpString(lit2).c_str(),
overlap, num_vertices(h));
if (!can_match(h, lit2, true)) {
DEBUG_PRINTF("found bogus edge\n");
dead.push_back(e);
continue;
}
unique_ptr<NGHolder> h_new = cloneHolder(h);
u32 delay = removeTrailingLiteralStates(*h_new, lit2, MO_INVALID_IDX);
if (delay == MO_INVALID_IDX) {
DEBUG_PRINTF("found bogus edge\n");
dead.push_back(e);
continue;
}
// Delay can be set to at most lit2.length() - overlap, but we must
// truncate to history available in streaming mode.
u32 max_allowed_delay = lit2.length() - overlap;
LIMIT_TO_AT_MOST(&max_allowed_delay, delay);
if (cc.streaming) {
LIMIT_TO_AT_MOST(&max_allowed_delay, cc.grey.maxHistoryAvailable);
}
if (delay != max_allowed_delay) {
restoreTrailingLiteralStates(*h_new, lit2, delay);
delay = removeTrailingLiteralStates(*h_new, lit2, max_allowed_delay);
}
if (isStarCliche(*h_new)) {
DEBUG_PRINTF("is a X star!\n");
ig[e].graph.reset();
ig[e].graph_lag = 0;
} else {
ig[e].graph = move(h_new);
ig[e].graph_lag = delay;
DEBUG_PRINTF("delay increased to %u\n", delay);
}
}
for (const auto &e : dead) {
remove_edge(e, ig);
}
}
static
void poisonNetflowScores(RoseInGraph &ig, RoseInEdge lhs,
vector<u64a> *scores) {
assert(ig[lhs].graph);
NGHolder &h = *ig[lhs].graph;
if (ig[target(lhs, ig)].type != RIV_LITERAL) {
/* nothing to poison in outfixes */
assert(ig[target(lhs, ig)].type == RIV_ACCEPT);
return;
}
set<NFAVertex> curr, next;
insert(&curr, inv_adjacent_vertices(h.accept, h));
set<NFAEdge> poisoned;
u32 len = ig[target(lhs, ig)].s.length();
assert(len);
while (len) {
next.clear();
for (auto v : curr) {
insert(&poisoned, in_edges(v, h));
insert(&next, inv_adjacent_vertices(v, h));
}
curr.swap(next);
len--;
}
for (const auto &e : poisoned) {
(*scores)[h[e].index] = NO_LITERAL_AT_EDGE_SCORE;
}
}
#define MAX_NETFLOW_CUT_WIDTH 40 /* magic number is magic */
#define MAX_LEN_2_LITERALS_PER_CUT 3
static
bool checkValidNetflowLits(NGHolder &h, const vector<u64a> &scores,
const map<NFAEdge, set<ue2_literal>> &cut_lits,
const Grey &grey) {
DEBUG_PRINTF("cut width %zu\n", cut_lits.size());
if (cut_lits.size() > MAX_NETFLOW_CUT_WIDTH) {
return false;
}
u32 len_2_count = 0;
for (const auto &cut : cut_lits) {
if (scores[h[cut.first].index] >= NO_LITERAL_AT_EDGE_SCORE) {
DEBUG_PRINTF("cut uses a forbidden edge\n");
return false;
}
if (min_len(cut.second) < grey.minRoseNetflowLiteralLength) {
DEBUG_PRINTF("cut uses a bad literal\n");
return false;
}
for (const auto &lit : cut.second) {
if (lit.length() == 2) {
len_2_count++;
}
}
}
if (len_2_count > MAX_LEN_2_LITERALS_PER_CUT) {
return false;
}
return true;
}
static
void splitEdgesByCut(RoseInGraph &ig, const vector<RoseInEdge> &to_cut,
const vector<NFAEdge> &cut,
const map<NFAEdge, set<ue2_literal> > &cut_lits) {
assert(!to_cut.empty());
assert(ig[to_cut.front()].graph);
NGHolder &h = *ig[to_cut.front()].graph;
/* note: the set of split edges should form a complete bipartite graph */
vector<RoseInVertex> src_verts;
vector<RoseInVertex> dest_verts;
getSourceVerts(ig, to_cut, &src_verts);
getDestVerts(ig, to_cut, &dest_verts);
assert(to_cut.size() == src_verts.size() * dest_verts.size());
map<vector<NFAVertex>, shared_ptr<NGHolder> > done_rhs;
/* iterate over cut for determinism */
for (const auto &e : cut) {
NFAVertex prev_v = source(e, h);
NFAVertex pivot = target(e, h);
vector<NFAVertex> adj;
insert(&adj, adj.end(), adjacent_vertices(pivot, h));
/* we can ignore presence of accept, accepteod in adj as it is best
effort */
if (!contains(done_rhs, adj)) {
ue2::unordered_map<NFAVertex, NFAVertex> temp_map;
shared_ptr<NGHolder> new_rhs = make_shared<NGHolder>();
splitRHS(h, adj, new_rhs.get(), &temp_map);
remove_edge(new_rhs->start, new_rhs->accept, *new_rhs);
remove_edge(new_rhs->start, new_rhs->acceptEod, *new_rhs);
done_rhs.insert(make_pair(adj, new_rhs));
/* TODO need to update v_mapping (if we were doing more cuts) */
}
DEBUG_PRINTF("splitting on pivot %u\n", h[pivot].index);
ue2::unordered_map<NFAVertex, NFAVertex> temp_map;
shared_ptr<NGHolder> new_lhs = make_shared<NGHolder>();
splitLHS(h, pivot, new_lhs.get(), &temp_map);
/* want to cut of paths to pivot from things other than the pivot -
* makes a more svelte graphy */
clear_in_edges(temp_map[pivot], *new_lhs);
add_edge(temp_map[prev_v], temp_map[pivot], *new_lhs);
pruneUseless(*new_lhs);
const set<ue2_literal> &lits = cut_lits.at(e);
for (const auto &lit : lits) {
RoseInVertex v
= add_vertex(RoseInVertexProps::makeLiteral(lit), ig);
if (edge(pivot, h.accept, h).second) {
/* literal has a direct connection to accept */
assert(ig[dest_verts.front()].type == RIV_ACCEPT);
const auto &reports = h[pivot].reports;
RoseInVertex tt =
add_vertex(RoseInVertexProps::makeAccept(reports), ig);
add_edge(v, tt, RoseInEdgeProps(0U, 0U), ig);
}
if (edge(pivot, h.acceptEod, h).second) {
/* literal has a direct connection to accept */
assert(ig[dest_verts.front()].type == RIV_ACCEPT);
const auto &reports = h[pivot].reports;
RoseInVertex tt = add_vertex(
RoseInVertexProps::makeAcceptEod(reports), ig);
add_edge(v, tt, RoseInEdgeProps(0U, 0U), ig);
}
assert(done_rhs[adj].get());
shared_ptr<NGHolder> new_rhs = done_rhs[adj];
if (out_degree(new_rhs->start, *new_rhs) != 1) {
for (auto dst_v : dest_verts) {
add_edge(v, dst_v, RoseInEdgeProps(done_rhs[adj], 0), ig);
}
}
for (auto src_v : src_verts) {
add_edge(src_v, v, RoseInEdgeProps(new_lhs, 0), ig);
}
}
}
/* TODO need to update v_mapping (if we were doing more cuts) */
for (const auto &e : to_cut) {
assert(ig[e].graph.get() == &h);
remove_edge(e, ig);
}
}
static
bool doNetflowCut(RoseInGraph &ig, const vector<RoseInEdge> &to_cut,
const Grey &grey) {
DEBUG_PRINTF("doing netflow cut\n");
/* TODO: we should really get literals/scores from the full graph as this
* allows us to overlap the graph. Doesn't matter at the moment as we
* are working on the LHS. */
NGHolder &h = *ig[to_cut.front()].graph;
if (num_edges(h) > grey.maxRoseNetflowEdges) {
/* We have a limit on this because scoring edges and running netflow
* gets very slow for big graphs. */
DEBUG_PRINTF("too many edges, skipping netflow cut\n");
return false;
}
h.renumberVertices();
h.renumberEdges();
/* Step 1: Get scores for all edges */
vector<u64a> scores = scoreEdges(h); /* scores by edge_index */
/* Step 2: poison scores for edges covered by successor literal */
for (const auto &e : to_cut) {
assert(&h == ig[e].graph.get());
poisonNetflowScores(ig, e, &scores);
}
/* Step 3: Find cutset based on scores */
vector<NFAEdge> cut = findMinCut(h, scores);
/* Step 4: Get literals corresponding to cut edges */
map<NFAEdge, set<ue2_literal>> cut_lits;
for (const auto &e : cut) {
set<ue2_literal> lits = getLiteralSet(h, e);
compressAndScore(lits);
cut_lits[e] = lits;
DEBUG_PRINTF("cut lit '%s'\n",
((const string &)*cut_lits[e].begin()).c_str());
}
/* if literals are underlength bail or if it involves a forbidden edge*/
if (!checkValidNetflowLits(h, scores, cut_lits, grey)) {
return false;
}
DEBUG_PRINTF("splitting\n");
/* Step 5: Split graph based on cuts */
splitEdgesByCut(ig, to_cut, cut, cut_lits);
return true;
}
/** \brief Returns the number of intermediate vertices in the shortest path
* between (from, to). */
static
u32 min_dist_between(NFAVertex from, NFAVertex to, const NGHolder &g) {
// Check for the trivial case: that way we don't have to set up the
// containers below.
if (edge(from, to, g).second) {
return 0;
}
ue2::unordered_set<NFAVertex> visited;
visited.insert(from);
flat_set<NFAVertex> curr, next;
curr.insert(from);
assert(from != to);
u32 d = 0;
while (!curr.empty()) {
next.clear();
for (auto v : curr) {
for (auto w : adjacent_vertices_range(v, g)) {
if (w == to) {
return d;
}
if (visited.insert(w).second) { // first visit to *ai
next.insert(w);
}
}
}
d++;
curr.swap(next);
}
assert(0);
return ROSE_BOUND_INF;
}
/** Literals which are completely enveloped by a successor are trouble because
* hamsterwheel acceleration can skip past the start of the literal. */
static
bool enveloped(const vector<NFAVertex> &cand_split_v,
const set<ue2_literal> &cand_lit, const NGHolder &g,
const RoseInVertexProps &succ) {
if (succ.type != RIV_LITERAL) {
return false;
}
/* TODO: handle multiple v more precisely: not all candidate v can start all
* candidate literals */
for (auto v : cand_split_v) {
u32 rhs_min_len = min_dist_between(v, g.accept, g);
if (rhs_min_len + min_len(cand_lit) >= succ.s.length()) {
return false;
}
}
return true; /* we are in trouble */
}
static
bool enveloped(const VertLitInfo &cand_split, const RoseInGraph &ig,
const vdest_map_t &v_dest_map) {
for (auto v : cand_split.vv) {
const auto &images = v_dest_map.at(v);
for (const auto &image : images) {
/* check that we aren't enveloped by the successor */
if (enveloped(vector<NFAVertex>(1, image.second), cand_split.lit,
*ig[image.first].graph,
ig[target(image.first, ig)])) {
return true;
}
const RoseInVertexProps &pred = ig[source(image.first, ig)];
if (pred.type != RIV_LITERAL) {
continue;
}
/* check we don't envelop the pred */
const NGHolder &g = *ig[image.first].graph;
u32 lhs_min_len = min_dist_between(g.start, image.second, g);
if (lhs_min_len + pred.s.length() < max_len(cand_split.lit)) {
return true;
}
}
}
return false;
}
static
bool attemptSplit(RoseInGraph &ig, vdest_map_t &v_dest_map,
vsrc_map_t &v_src_map, const vector<RoseInEdge> &v_e,
LitCollection &lits) {
NGHolder &h = *ig[v_e.front()].graph;
unique_ptr<VertLitInfo> split = lits.pickNext();
while (split) {
for (const auto &e : v_e) {
RoseInVertex t = target(e, ig);
if (enveloped(split->vv, split->lit, h, ig[t])) {
DEBUG_PRINTF("enveloped\n");
split = lits.pickNext();
goto next_split;
}
}
break;
next_split:;
}
if (!split) {
return false;
}
for (auto v : split->vv) {
if (edge(v, h.accept, h).second) {
return false;
}
}
DEBUG_PRINTF("saved by a bad literal\n");
splitRoseEdge(ig, *split, v_dest_map, v_src_map);
return true;
}
static
void appendLiteral(const ue2_literal &s, const CharReach &cr,
vector<ue2_literal> *out) {
for (size_t c = cr.find_first(); c != CharReach::npos;
c = cr.find_next(c)) {
bool nocase = ourisalpha(c) && cr.test(mytoupper(c))
&& cr.test(mytolower(c));
if (nocase && (char)c == mytolower(c)) {
continue; /* uppercase already handled us */
}
out->push_back(s);
out->back().push_back(c, nocase);
}
}
static
bool findAnchoredLiterals(const NGHolder &g, vector<ue2_literal> *out,
vector<NFAVertex> *pivots_out) {
DEBUG_PRINTF("trying for anchored\n");
#define MAX_ANCHORED_LITERALS 30
#define MAX_ANCHORED_LITERAL_LEN 30
/* TODO: this could be beefed up by going region-by-region but currently
* that brings back bad memories of ng_rose. OR any AA region we can build
* a dfa out of */
assert(!proper_out_degree(g.startDs, g));
vector<ue2_literal> lits;
lits.push_back(ue2_literal());
set<NFAVertex> curr;
insert(&curr, adjacent_vertices(g.start, g));
curr.erase(g.startDs);
set<NFAVertex> old;
if (contains(curr, g.accept) || curr.empty()) {
DEBUG_PRINTF("surprise accept/voidness\n");
return false;
}
while (!curr.empty()) {
set<NFAVertex> next_verts;
insert(&next_verts, adjacent_vertices(*curr.begin(), g));
bool can_extend
= !next_verts.empty() && !contains(next_verts, g.accept);
CharReach cr;
for (auto v : curr) {
assert(!is_special(v, g));
if (can_extend) {
/* next verts must agree */
set<NFAVertex> next_verts_local;
insert(&next_verts_local, adjacent_vertices(v, g));
can_extend = next_verts_local == next_verts;
}
cr |= g[v].char_reach;
}
if (!can_extend) {
goto bail;
}
/* extend literals */
assert(cr.any());
vector<ue2_literal> next_lits;
for (const auto &lit : lits) {
appendLiteral(lit, cr, &next_lits);
if (next_lits.size() > MAX_ANCHORED_LITERALS) {
goto bail;
}
}
assert(!next_lits.empty());
old.swap(curr);
if (next_lits[0].length() <= MAX_ANCHORED_LITERAL_LEN) {
curr.swap(next_verts);
} else {
curr.clear();
}
lits.swap(next_lits);
}
bail:
assert(!lits.empty());
for (UNUSED const auto &lit : lits) {
DEBUG_PRINTF("found anchored string: %s\n", dumpString(lit).c_str());
}
insert(pivots_out, pivots_out->end(), old);
out->swap(lits);
return !out->empty() && !out->begin()->empty();
}
static
bool tryForAnchoredImprovement(RoseInGraph &ig, RoseInEdge e) {
vector<ue2_literal> lits;
vector<NFAVertex> pivots;
if (!findAnchoredLiterals(*ig[e].graph, &lits, &pivots)) {
DEBUG_PRINTF("unable to find literals\n");
return false;
}
DEBUG_PRINTF("found %zu literals to act as anchors\n", lits.size());
RoseInVertex s = source(e, ig);
RoseInVertex t = target(e, ig);
assert(!ig[e].graph_lag);
shared_ptr<NGHolder> lhs = make_shared<NGHolder>();
shared_ptr<NGHolder> rhs = make_shared<NGHolder>();
ue2::unordered_map<NFAVertex, NFAVertex> temp1;
ue2::unordered_map<NFAVertex, NFAVertex> temp2;
splitGraph(*ig[e].graph, pivots, lhs.get(), &temp1, rhs.get(), &temp2);
for (const auto &lit : lits) {
RoseInVertex v = add_vertex(RoseInVertexProps::makeLiteral(lit),
ig);
add_edge(s, v, RoseInEdgeProps(lhs, 0U), ig);
add_edge(v, t, RoseInEdgeProps(rhs, 0U), ig);
}
remove_edge(e, ig);
return true;
}
#define MAX_SINGLE_BYTE_ANCHORED_DIST 30
/* returns true if we should make another pass */
static
bool lastChanceImproveLHS(RoseInGraph &ig, RoseInEdge lhs,
const CompileContext &cc) {
DEBUG_PRINTF("argh lhs is nasty\n");
assert(ig[lhs].graph);
/* customise the lhs for this literal */
/* TODO better, don't recalc */
if (ig[target(lhs, ig)].type == RIV_LITERAL) {
const NGHolder &h = *ig[lhs].graph;
/* sanitise literal on lhs */
const ue2_literal &s = ig[target(lhs, ig)].s;
if (!can_match(h, s, false)) {
DEBUG_PRINTF("found bogus edge\n");
return false;
}
/* see if we can build some anchored literals out of this */
if (isAnchored(h) && tryForAnchoredImprovement(ig, lhs)) {
return true;
}
unique_ptr<NGHolder> cust = cloneHolder(h);
u32 d = removeTrailingLiteralStates(*cust, s, MO_INVALID_IDX);
if (d == MO_INVALID_IDX) {
DEBUG_PRINTF("found bogus edge\n");
return false;
}
restoreTrailingLiteralStates(*cust, s, d);
ig[lhs].graph = move(cust);
}
NGHolder &lhs_graph = *ig[lhs].graph;
set<NFAVertex> cand;
set<NFAVertex> cand_raw;
getCandidatePivots(lhs_graph, &cand, &cand_raw);
vdest_map_t v_dest_map;
vsrc_map_t v_src_map;
for (auto v : vertices_range(lhs_graph)) {
v_dest_map[v].emplace_back(lhs, v);
v_src_map[lhs].push_back(v);
}
vector<NFAVertexDepth> depths;
calcDepths(lhs_graph, depths);
/* need to ensure regions are valid before we do lit discovery */
auto region_map = assignRegions(lhs_graph);
vector<RoseInEdge> to_cut(1, lhs);
DEBUG_PRINTF("see if we can get a better lhs by another cut\n");
LitCollection lit1(lhs_graph, depths, region_map, cand, cand_raw,
cc.grey.minRoseLiteralLength, true, cc);
if (attemptSplit(ig, v_dest_map, v_src_map, to_cut, lit1)) {
return true;
}
if (doNetflowCut(ig, to_cut, cc.grey)) {
return true;
}
DEBUG_PRINTF("eek last chance try len 1 if it creates an anchored lhs\n");
{
LitCollection lits(lhs_graph, depths, region_map, cand, cand_raw, 1,
true, cc, true);
unique_ptr<VertLitInfo> split = lits.pickNext();
/* TODO fix edge to accept check */
while (split
&& (enveloped(split->vv, split->lit, lhs_graph,
ig[target(lhs, ig)])
|| edge(split->vv.front(), lhs_graph.accept, lhs_graph).second
|| !createsAnchoredLHS(lhs_graph, split->vv, depths, cc.grey,
MAX_SINGLE_BYTE_ANCHORED_DIST))) {
split = lits.pickNext();
}
if (split) {
DEBUG_PRINTF("saved by a really bad literal\n");
splitRoseEdge(ig, *split, v_dest_map, v_src_map);
return true;
}
}
return false;
}
/* returns false if nothing happened */
static
bool lastChanceImproveLHS(RoseInGraph &ig, const vector<RoseInEdge> &to_cut,
const CompileContext &cc) {
DEBUG_PRINTF("argh lhses are nasty\n");
NGHolder &lhs_graph = *ig[to_cut.front()].graph;
set<NFAVertex> cand;
set<NFAVertex> cand_raw;
getCandidatePivots(lhs_graph, &cand, &cand_raw);
vdest_map_t v_dest_map;
vsrc_map_t v_src_map;
for (auto v : vertices_range(lhs_graph)) {
for (const auto &e : to_cut) {
v_dest_map[v].emplace_back(e, v);
v_src_map[e].push_back(v);
}
}
vector<NFAVertexDepth> depths;
calcDepths(lhs_graph, depths);
auto region_map = assignRegions(lhs_graph);
DEBUG_PRINTF("see if we can get a better lhs by allowing another cut\n");
LitCollection lit1(lhs_graph, depths, region_map, cand, cand_raw,
cc.grey.minRoseLiteralLength, true, cc);
if (attemptSplit(ig, v_dest_map, v_src_map, to_cut, lit1)) {
return true;
}
return doNetflowCut(ig, to_cut, cc.grey);
}
static
bool improveLHS(RoseInGraph &ig, const vector<RoseInEdge> &edges,
const CompileContext &cc) {
bool rv = false;
vector<RoseInVertex> src_verts;
getSourceVerts(ig, edges, &src_verts);
map<RoseInVertex, vector<RoseInEdge>> by_src;
for (const auto &e : edges) {
by_src[source(e, ig)].push_back(e);
}
for (auto v : src_verts) {
const vector<RoseInEdge> &local = by_src[v];
vector<NGHolder *> graphs;
map<NGHolder *, vector<RoseInEdge> > by_graph;
for (const auto &e : local) {
NGHolder *gp = ig[e].graph.get();
if (!contains(by_graph, gp)) {
graphs.push_back(gp);
}
by_graph[gp].push_back(e);
}
for (auto h : graphs) {
const vector<RoseInEdge> &local2 = by_graph[h];
if (local2.size() == 1) {
rv |= lastChanceImproveLHS(ig, local2.front(), cc);
continue;
}
bool lrv = lastChanceImproveLHS(ig, local2, cc);
if (lrv) {
rv = true;
} else {
for (const auto &e2 : local2) {
rv |= lastChanceImproveLHS(ig, e2, cc);
}
}
}
}
return rv;
}
static
void processLHS(RoseInGraph &ig, const CompileContext &cc) {
bool redo;
do {
redo = false;
vector<RoseInEdge> to_improve;
for (const auto &lhs : edges_range(ig)) {
if (ig[source(lhs, ig)].type != RIV_START
&& ig[source(lhs, ig)].type != RIV_ANCHORED_START) {
continue;
}
if (ig[target(lhs, ig)].type == RIV_LITERAL) {
DEBUG_PRINTF("checking lhs->'%s'\n",
ig[target(lhs, ig)].s.c_str());
} else {
DEBUG_PRINTF("checking lhs->?\n");
}
/* if check if lhs is nasty */
if (ig[target(lhs, ig)].type == RIV_ACCEPT) {
to_improve.push_back(lhs);
continue;
}
assert(ig[lhs].graph);
const NGHolder *h = ig[lhs].graph.get();
vector<NFAVertexDepth> depths;
calcDepths(*h, depths);
if (!isLHSTransient(*h, depths, cc.grey)
&& !literalIsWholeGraph(*h, ig[target(lhs, ig)].s)
&& !isLHSUsablyAnchored(*h, depths, cc.grey)) {
to_improve.push_back(lhs);
}
}
DEBUG_PRINTF("inspecting %zu lhs\n", to_improve.size());
if (to_improve.size() > 50) {
DEBUG_PRINTF("too big\n");
break;
}
redo = improveLHS(ig, to_improve, cc);
DEBUG_PRINTF("redo = %d\n", (int)redo);
} while (redo);
vector<RoseInEdge> to_inspect; /* to prevent surprises caused by us
* altering the graph while iterating */
for (const auto &e : edges_range(ig)) {
if (ig[source(e, ig)].type == RIV_START
|| ig[source(e, ig)].type == RIV_ANCHORED_START) {
to_inspect.push_back(e);
}
}
for (const auto &lhs : to_inspect) {
if (ig[target(lhs, ig)].type == RIV_LITERAL) {
if (removeLiteralFromLHS(ig, lhs, cc)) {
handleLhsCliche(ig, lhs);
} else {
/* telling us to delete the edge */
remove_edge(lhs, ig);
}
}
}
}
static
void tryNetflowCutForRHS(RoseInGraph &ig, const Grey &grey) {
vector<RoseInEdge> to_improve;
for (const auto &rhs : edges_range(ig)) {
if (ig[target(rhs, ig)].type != RIV_ACCEPT) {
continue;
}
if (ig[source(rhs, ig)].type == RIV_LITERAL) {
DEBUG_PRINTF("checking '%s'->rhs\n", ig[source(rhs, ig)].s.c_str());
} else {
DEBUG_PRINTF("checking ?->rhs\n");
}
if (!ig[rhs].graph) {
continue;
}
DEBUG_PRINTF("%zu vertices\n", num_vertices(*ig[rhs].graph));
if (num_vertices(*ig[rhs].graph) < 512) {
DEBUG_PRINTF("small\n");
continue;
}
/* if check if rhs is nasty */
to_improve.push_back(rhs);
}
DEBUG_PRINTF("inspecting %zu lhs\n", to_improve.size());
if (to_improve.size() > 50) {
DEBUG_PRINTF("too big\n");
return;
}
for (const auto &e : to_improve) {
vector<RoseInEdge> to_cut(1, e);
doNetflowCut(ig, to_cut, grey);
}
}
/* just make the string nocase and get the graph to handle case mask, TODO.
* This could be more nuanced but the effort would probably be better spent
* just making rose less bad. */
static
void makeNocaseWithPrefixMask(RoseInGraph &g, RoseInVertex v) {
for (const auto &e : in_edges_range(v, g)) {
const RoseInVertex u = source(e, g);
if (!g[e].graph) {
g[e].graph = make_shared<NGHolder>(whatRoseIsThis(g, e));
g[e].graph_lag = g[v].s.length();
NGHolder &h = *g[e].graph;
assert(!g[e].maxBound || g[e].maxBound == ROSE_BOUND_INF);
if (g[u].type == RIV_START) {
add_edge(h.startDs, h.accept, h);
h[h.startDs].reports.insert(0);
} else if (g[e].maxBound == ROSE_BOUND_INF) {
add_edge(h.start, h.accept, h);
NFAVertex ds = add_vertex(h);
h[ds].char_reach = CharReach::dot();
add_edge(h.start, ds, h);
add_edge(ds, ds, h);
add_edge(ds, h.accept, h);
h[h.start].reports.insert(0);
h[ds].reports.insert(0);
} else {
add_edge(h.start, h.accept, h);
h[h.start].reports.insert(0);
}
}
if (!g[e].graph_lag) {
continue;
}
unique_ptr<NGHolder> newg = cloneHolder(*g[e].graph);
restoreTrailingLiteralStates(*newg, g[v].s, g[e].graph_lag);
g[e].graph_lag = 0;
g[e].graph = move(newg);
}
make_nocase(&g[v].s);
}
static
unique_ptr<NGHolder> makeGraphCopy(const NGHolder *g) {
if (g) {
return cloneHolder(*g);
} else {
return nullptr;
}
}
static
void explodeLiteral(RoseInGraph &g, RoseInVertex v,
vector<ue2_literal> &exploded) {
for (const auto &lit : exploded) {
RoseInVertex v_new = add_vertex(g[v], g);
g[v_new].s = lit;
for (const auto &e : in_edges_range(v, g)) {
RoseInEdge e2 = add_edge(source(e, g), v_new, g[e], g).first;
// FIXME: are we safe to share graphs here? For now, make our very
// own copy.
g[e2].graph = makeGraphCopy(g[e].graph.get());
}
for (const auto &e : out_edges_range(v, g)) {
RoseInEdge e2 = add_edge(v_new, target(e, g), g[e], g).first;
// FIXME: are we safe to share graphs here? For now, make our very
// own copy.
g[e2].graph = makeGraphCopy(g[e].graph.get());
}
}
clear_vertex(v, g);
remove_vertex(v, g);
}
/* Sadly rose is hacky in terms of mixed case literals. TODO: remove when rose
* becomes less bad */
static
void handleLongMixedSensitivityLiterals(RoseInGraph &g) {
const size_t maxExploded = 8; // only case-explode this far
vector<RoseInVertex> verts;
for (auto v : vertices_range(g)) {
if (g[v].type != RIV_LITERAL) {
continue;
}
ue2_literal &s = g[v].s;
if (!mixed_sensitivity(s)) {
continue;
}
if (s.length() < MAX_MASK2_WIDTH) {
DEBUG_PRINTF("mixed lit will be handled by benefits mask\n");
continue;
}
DEBUG_PRINTF("found mixed lit of len %zu\n", s.length());
verts.push_back(v);
}
for (auto v : verts) {
vector<ue2_literal> exploded;
case_iter cit = caseIterateBegin(g[v].s), cite = caseIterateEnd();
for (; cit != cite; ++cit) {
exploded.emplace_back(*cit, false);
if (exploded.size() > maxExploded) {
goto dont_explode;
}
}
DEBUG_PRINTF("exploding literal into %zu pieces\n", exploded.size());
explodeLiteral(g, v, exploded);
continue;
dont_explode:
DEBUG_PRINTF("converting to nocase with prefix mask\n");
makeNocaseWithPrefixMask(g, v);
}
DEBUG_PRINTF("done!\n");
}
static
void dedupe(RoseInGraph &g) {
/* We know that every prefix/infix is unique after the rose construction.
*
* If a vertex has out-going graphs with the same rewind and they are equal
* we can dedupe the graph.
*
* After this, we may share graphs on out-edges of a vertex. */
map<pair<u32, u64a>, vector<shared_ptr<NGHolder>>> buckets;
for (auto v : vertices_range(g)) {
buckets.clear();
for (const auto &e : out_edges_range(v, g)) {
if (!g[e].graph || g[target(e, g)].type != RIV_LITERAL) {
continue;
}
auto k = make_pair(g[e].graph_lag, hash_holder(*g[e].graph));
auto &bucket = buckets[k];
for (const auto &h : bucket) {
if (is_equal(*g[e].graph, 0U, *h, 0U)) {
g[e].graph = h;
goto next_edge;
}
}
bucket.push_back(g[e].graph);
next_edge:;
}
}
}
static
bool pureReport(NFAVertex v, const NGHolder &g) {
for (auto w : adjacent_vertices_range(v, g)) {
if (w != g.accept && w != g.acceptEod) {
return false;
}
}
return true;
}
static
bool pureReport(const vector<NFAVertex> &vv, const NGHolder &g) {
for (auto v : vv) {
if (!pureReport(v, g)) {
return false;
}
}
return true;
}
/* ensures that a vertex is followed by a start construct AND the cyclic states
* has a reasonably wide reach */
static
bool followedByStar(NFAVertex v, const NGHolder &g) {
set<NFAVertex> succ;
insert(&succ, adjacent_vertices(v, g));
set<NFAVertex> asucc;
for (auto w : adjacent_vertices_range(v, g)) {
if (g[w].char_reach.count() < N_CHARS - MAX_ESCAPE_CHARS) {
continue; /* state is too narrow to be considered as a sane star
cyclic */
}
asucc.clear();
insert(&asucc, adjacent_vertices(w, g));
if (asucc == succ) {
return true;
}
}
return false;
}
static
bool followedByStar(const vector<NFAVertex> &vv, const NGHolder &g) {
for (auto v : vv) {
if (!followedByStar(v, g)) {
return false;
}
}
return true;
}
static
bool isEodPrefixCandidate(const NGHolder &g) {
if (hasGreaterInDegree(0, g.accept, g)) {
DEBUG_PRINTF("graph isn't eod anchored\n");
return false;
}
// TODO: handle more than one report.
if (all_reports(g).size() != 1) {
return false;
}
return true;
}
static
bool isEodWithPrefix(const RoseInGraph &g) {
if (num_vertices(g) != 2) {
return false;
}
for (const auto &e : edges_range(g)) {
RoseInVertex u = source(e, g), v = target(e, g);
DEBUG_PRINTF("edge from %d -> %d\n", g[u].type, g[v].type);
if (g[u].type != RIV_START && g[u].type != RIV_ANCHORED_START) {
DEBUG_PRINTF("source not start, type=%d\n", g[u].type);
return false;
}
if (g[v].type != RIV_ACCEPT && g[v].type != RIV_ACCEPT_EOD) {
DEBUG_PRINTF("target not accept, type=%d\n", g[v].type);
return false;
}
// Haigs not handled.
if (g[e].haig) {
DEBUG_PRINTF("edge has haig\n");
return false;
}
if (!g[e].graph) {
DEBUG_PRINTF("no graph on edge\n");
return false;
}
if (!isEodPrefixCandidate(*g[e].graph)) {
DEBUG_PRINTF("graph is not eod prefix candidate\n");
return false;
}
}
return true;
}
static
void processEodPrefixes(RoseInGraph &g) {
// Find edges to accept with EOD-anchored graphs that we can move over to
// acceptEod.
vector<RoseInEdge> acc_edges;
for (const auto &e : edges_range(g)) {
if (g[target(e, g)].type != RIV_ACCEPT) {
continue;
}
if (g[e].haig || !g[e].graph) {
continue;
}
if (!isEodPrefixCandidate(*g[e].graph)) {
continue;
}
// TODO: handle cases with multiple out-edges.
if (hasGreaterOutDegree(1, source(e, g), g)) {
continue;
}
acc_edges.push_back(e);
}
set<RoseInVertex> accepts;
for (const RoseInEdge &e : acc_edges) {
RoseInVertex u = source(e, g), v = target(e, g);
assert(g[e].graph);
assert(g[v].type == RIV_ACCEPT);
assert(all_reports(*g[e].graph).size() == 1);
// Move this edge from accept to acceptEod and give it the right reports
// from the graph on the edge.
const set<ReportID> reports = all_reports(*g[e].graph);
RoseInVertex w = add_vertex(
RoseInVertexProps::makeAcceptEod(reports), g);
add_edge(u, w, g[e], g);
remove_edge(e, g);
accepts.insert(v);
}
for (auto v : accepts) {
if (!hasGreaterInDegree(0, v, g)) {
remove_vertex(v, g);
}
}
}
/** Run some reduction passes on the graphs on our edges. */
static
void reduceGraphs(RoseInGraph &g, const CompileContext &cc) {
for (const auto &e : edges_range(g)) {
if (!g[e].graph) {
continue;
}
NGHolder &h = *g[e].graph;
assert(h.kind == whatRoseIsThis(g, e));
DEBUG_PRINTF("before, graph %p has %zu vertices, %zu edges\n", &h,
num_vertices(h), num_edges(h));
pruneUseless(h);
reduceGraphEquivalences(h, cc);
removeRedundancy(h, SOM_NONE); /* rose doesn't track som */
DEBUG_PRINTF("after, graph %p has %zu vertices, %zu edges\n", &h,
num_vertices(h), num_edges(h));
// It's possible that one of our graphs may have reduced to a dot-star
// cliche, i.e. it contains a startDs->accept edge. If so, we can
// remove it from the edge and just use edge bounds to represent it.
if (edge(h.startDs, h.accept, h).second) {
DEBUG_PRINTF("graph reduces to dot-star, deleting\n");
g[e].graph.reset();
g[e].graph_lag = 0;
g[e].minBound = 0;
g[e].maxBound = ROSE_BOUND_INF;
}
}
}
static
unique_ptr<RoseInGraph> buildRose(const NGHolder &h, bool desperation,
const CompileContext &cc) {
/* Need to pick a pivot point which splits the graph in two with starts on
* one side and accepts on the other. Thus the pivot needs to dominate all
* the accept vertices */
/* maps a vertex in h to one of its images in the rose graph */
vdest_map_t v_dest_map;
vsrc_map_t v_src_map;
/* create trivial rose graph */
unique_ptr<RoseInGraph> igp = makeTrivialGraph(h, v_dest_map, v_src_map);
RoseInGraph &ig = *igp;
/* root graph is the graph on the only edge in our new RoseInGraph */
assert(num_edges(ig) == 1);
shared_ptr<NGHolder> root_g = ig[*edges(ig).first].graph;
assert(root_g);
/* find the literals */
set<NFAVertex> cand;
set<NFAVertex> cand_raw;
getCandidatePivots(*root_g, &cand, &cand_raw);
DEBUG_PRINTF("|cand| = %zu\n", cand.size());
vector<NFAVertexDepth> depths;
calcDepths(*root_g, depths);
auto region_map = assignRegions(*root_g);
LitCollection lits(*root_g, depths, region_map, cand, cand_raw,
cc.grey.minRoseLiteralLength, desperation, cc);
for (u32 i = 0; i < cc.grey.roseDesiredSplit; ++i) {
DEBUG_PRINTF("attempting split %u (desired %u)\n", i,
cc.grey.roseDesiredSplit);
unique_ptr<VertLitInfo> split = lits.pickNext();
/* need to check we aren't creating any enveloping literals */
while (split && enveloped(*split, ig, v_dest_map)) {
DEBUG_PRINTF("bad cand; getting next split\n");
split = lits.pickNext();
}
if (!split) {
DEBUG_PRINTF("no more lits :(\n");
break;
}
splitRoseEdge(ig, *split, v_dest_map, v_src_map);
}
/* try for more split literals if they are followed by .* or accept */
for (;;) {
DEBUG_PRINTF("attempting bonus split\n");
unique_ptr<VertLitInfo> split = lits.pickNext();
/* need to check we aren't creating any enveloping literals */
while (split
&& (enveloped(*split, ig, v_dest_map)
|| (!pureReport(split->vv, *root_g)
&& !followedByStar(split->vv, *root_g)))) {
DEBUG_PRINTF("bad cand; getting next split\n");
split = lits.pickNext();
}
if (!split) {
DEBUG_PRINTF("no more lits :(\n");
break;
}
DEBUG_PRINTF("got bonus split\n");
splitRoseEdge(ig, *split, v_dest_map, v_src_map);
}
processLHS(ig, cc);
if (num_vertices(ig) <= 2) {
// At present, we don't accept all outfixes.
// However, we do handle the specific case of a rose that precedes an
// acceptEod, which we will support as a prefix to a special EOD event
// "literal".
if (!isEodWithPrefix(ig)) {
igp.reset();
return igp;
}
}
processEodPrefixes(ig);
processInfixes(ig, cc);
handleLongMixedSensitivityLiterals(ig);
dedupe(ig);
pruneUseless(ig);
reduceGraphs(ig, cc);
dumpPreRoseGraph(ig, cc.grey);
calcVertexOffsets(ig);
return igp;
}
static
void desperationImprove(RoseInGraph &ig, const CompileContext &cc) {
DEBUG_PRINTF("rose said no; can we do better?\n");
/* infixes are tricky as we have to worry about delays, enveloping
* literals, etc */
tryNetflowCutForRHS(ig, cc.grey);
processInfixes(ig, cc);
handleLongMixedSensitivityLiterals(ig);
dedupe(ig);
pruneUseless(ig);
calcVertexOffsets(ig);
}
bool splitOffRose(RoseBuild &rose, const NGHolder &h, bool prefilter,
const CompileContext &cc) {
if (!cc.grey.allowRose) {
return false;
}
// We should have at least one edge into accept or acceptEod!
assert(hasGreaterInDegree(0, h.accept, h) ||
hasGreaterInDegree(1, h.acceptEod, h));
unique_ptr<RoseInGraph> igp = buildRose(h, false, cc);
if (igp && rose.addRose(*igp, prefilter)) {
goto ok;
}
igp = buildRose(h, true, cc);
if (igp) {
if (rose.addRose(*igp, prefilter)) {
goto ok;
}
desperationImprove(*igp, cc);
if (rose.addRose(*igp, prefilter)) {
goto ok;
}
}
DEBUG_PRINTF("rose build failed\n");
return false;
ok:
DEBUG_PRINTF("rose build ok\n");
return true;
}
bool finalChanceRose(RoseBuild &rose, const NGHolder &h, bool prefilter,
const CompileContext &cc) {
DEBUG_PRINTF("final chance rose\n");
if (!cc.grey.allowRose) {
return false;
}
assert(h.kind == NFA_OUTFIX);
ue2_literal lit;
bool anch = false;
shared_ptr<NGHolder> rhs = make_shared<NGHolder>();
if (!splitOffLeadingLiteral(h, &lit, &*rhs)) {
DEBUG_PRINTF("no floating literal\n");
anch = true;
if (!splitOffAnchoredLeadingLiteral(h, &lit, &*rhs)) {
DEBUG_PRINTF("no anchored literal\n");
return false;
}
}
if (lit.length() < cc.grey.minRoseLiteralLength
|| minStringPeriod(lit) < 2 ) {
DEBUG_PRINTF("lit too weak\n");
return false;
}
assert(lit.length() <= MAX_MASK2_WIDTH || !mixed_sensitivity(lit));
RoseInGraph ig;
RoseInVertex s
= add_vertex(RoseInVertexProps::makeStart(anch), ig);
RoseInVertex v = add_vertex(RoseInVertexProps::makeLiteral(lit), ig);
add_edge(s, v, RoseInEdgeProps(0, anch ? 0 : ROSE_BOUND_INF), ig);
ue2_literal lit2;
if (getTrailingLiteral(h, &lit2)
&& lit2.length() >= cc.grey.minRoseLiteralLength
&& minStringPeriod(lit2) >= 2) {
/* TODO: handle delay */
size_t overlap = maxOverlap(lit, lit2, 0);
u32 delay2 = lit2.length() - overlap;
delay2 = min(delay2, maxDelay(cc));
delay2 = removeTrailingLiteralStates(*rhs, lit2, delay2);
rhs->kind = NFA_INFIX;
assert(delay2 <= lit2.length());
RoseInVertex w
= add_vertex(RoseInVertexProps::makeLiteral(lit2), ig);
add_edge(v, w, RoseInEdgeProps(rhs, delay2), ig);
NFAVertex reporter = getSoleSourceVertex(h, h.accept);
assert(reporter);
const auto &reports = h[reporter].reports;
RoseInVertex a =
add_vertex(RoseInVertexProps::makeAccept(reports), ig);
add_edge(w, a, RoseInEdgeProps(0U, 0U), ig);
} else {
RoseInVertex a =
add_vertex(RoseInVertexProps::makeAccept(set<ReportID>()), ig);
add_edge(v, a, RoseInEdgeProps(rhs, 0U), ig);
}
calcVertexOffsets(ig);
return rose.addRose(ig, prefilter, true /* final chance */);
}
bool checkRose(const ReportManager &rm, const NGHolder &h, bool prefilter,
const CompileContext &cc) {
if (!cc.grey.allowRose) {
return false;
}
// We should have at least one edge into accept or acceptEod!
assert(hasGreaterInDegree(0, h.accept, h) ||
hasGreaterInDegree(1, h.acceptEod, h));
unique_ptr<RoseInGraph> igp;
// First pass.
igp = buildRose(h, false, cc);
if (igp && roseCheckRose(*igp, prefilter, rm, cc)) {
return true;
}
// Second ("desperation") pass.
igp = buildRose(h, true, cc);
if (igp) {
if (roseCheckRose(*igp, prefilter, rm, cc)) {
return true;
}
desperationImprove(*igp, cc);
if (roseCheckRose(*igp, prefilter, rm, cc)) {
return true;
}
}
return false;
}
} // namespace ue2
Reference in New Issue View Git Blame Copy Permalink