diff --git a/src/nfagraph/ng.cpp b/src/nfagraph/ng.cpp index 4525fd3e..caf631f7 100644 --- a/src/nfagraph/ng.cpp +++ b/src/nfagraph/ng.cpp @@ -255,7 +255,7 @@ bool addComponent(NG &ng, NGHolder &g, const NGWrapper &w, const som_type som, return true; } - if (splitOffRose(*ng.rose, g, w.prefilter, cc)) { + if (splitOffRose(*ng.rose, g, w.prefilter, ng.rm, cc)) { return true; } @@ -276,7 +276,7 @@ bool addComponent(NG &ng, NGHolder &g, const NGWrapper &w, const som_type som, return true; } - if (splitOffRose(*ng.rose, g, w.prefilter, cc)) { + if (splitOffRose(*ng.rose, g, w.prefilter, ng.rm, cc)) { return true; } @@ -291,7 +291,7 @@ bool addComponent(NG &ng, NGHolder &g, const NGWrapper &w, const som_type som, } } - if (finalChanceRose(*ng.rose, g, w.prefilter, cc)) { + if (finalChanceRose(*ng.rose, g, w.prefilter, ng.rm, cc)) { return true; } @@ -533,16 +533,16 @@ bool NG::addHolder(NGHolder &w) { return true; } - if (splitOffRose(*rose, w, prefilter, cc)) { + if (splitOffRose(*rose, w, prefilter, rm, cc)) { return true; } if (splitOffPuffs(*rose, rm, w, prefilter, cc)) { return true; } - if (splitOffRose(*rose, w, prefilter, cc)) { + if (splitOffRose(*rose, w, prefilter, rm, cc)) { return true; } - if (finalChanceRose(*rose, w, prefilter, cc)) { + if (finalChanceRose(*rose, w, prefilter, rm, cc)) { return true; } diff --git a/src/nfagraph/ng_rose.cpp b/src/nfagraph/ng_rose.cpp index 7066ab27..1f79b55f 100644 --- a/src/nfagraph/ng_rose.cpp +++ b/src/nfagraph/ng_rose.cpp @@ -48,6 +48,7 @@ #include "ng_reports.h" #include "ng_split.h" #include "ng_util.h" +#include "ng_violet.h" #include "ng_width.h" #include "rose/rose_build.h" #include "rose/rose_build_util.h" @@ -2833,8 +2834,19 @@ void desperationImprove(RoseInGraph &ig, const CompileContext &cc) { calcVertexOffsets(ig); } +static +bool addRose(RoseBuild &rose, RoseInGraph &ig, bool prefilter, + bool final_chance, const ReportManager &rm, + const CompileContext &cc) { + if (!ensureImplementable(rose, ig, false, final_chance, rm, cc) + && !prefilter) { + return false; + } + return rose.addRose(ig, prefilter); +} + bool splitOffRose(RoseBuild &rose, const NGHolder &h, bool prefilter, - const CompileContext &cc) { + const ReportManager &rm, const CompileContext &cc) { if (!cc.grey.allowRose) { return false; } @@ -2843,20 +2855,20 @@ bool splitOffRose(RoseBuild &rose, const NGHolder &h, bool prefilter, assert(in_degree(h.accept, h) || in_degree(h.acceptEod, h) > 1); unique_ptr igp = buildRose(h, false, cc); - if (igp && rose.addRose(*igp, prefilter)) { + if (igp && addRose(rose, *igp, prefilter, false, rm, cc)) { goto ok; } igp = buildRose(h, true, cc); if (igp) { - if (rose.addRose(*igp, prefilter)) { + if (addRose(rose, *igp, prefilter, false, rm, cc)) { goto ok; } desperationImprove(*igp, cc); - if (rose.addRose(*igp, prefilter)) { + if (addRose(rose, *igp, prefilter, false, rm, cc)) { goto ok; } } @@ -2870,7 +2882,7 @@ ok: } bool finalChanceRose(RoseBuild &rose, const NGHolder &h, bool prefilter, - const CompileContext &cc) { + const ReportManager &rm, const CompileContext &cc) { DEBUG_PRINTF("final chance rose\n"); if (!cc.grey.allowRose) { return false; @@ -2935,7 +2947,7 @@ bool finalChanceRose(RoseBuild &rose, const NGHolder &h, bool prefilter, renumber_vertices(ig); calcVertexOffsets(ig); - return rose.addRose(ig, prefilter, true /* final chance */); + return addRose(rose, ig, prefilter, true /* final chance */, rm, cc); } bool checkRose(const ReportManager &rm, const NGHolder &h, bool prefilter, diff --git a/src/nfagraph/ng_rose.h b/src/nfagraph/ng_rose.h index d180e8a5..1634adf0 100644 --- a/src/nfagraph/ng_rose.h +++ b/src/nfagraph/ng_rose.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -50,13 +50,13 @@ struct ue2_literal; /** \brief Attempt to consume the entire pattern in graph \a h with Rose. * Returns true if successful. */ bool splitOffRose(RoseBuild &rose, const NGHolder &h, bool prefilter, - const CompileContext &cc); + const ReportManager &rm, const CompileContext &cc); /** \brief Attempt to consume the entire pattern in graph \a h with Rose. * This is the last attempt to handle a pattern before we resort to an outfix. * Returns true if successful. */ bool finalChanceRose(RoseBuild &rose, const NGHolder &h, bool prefilter, - const CompileContext &cc); + const ReportManager &rm, const CompileContext &cc); /** \brief True if the pattern in \a h is consumable by Rose. This function * may be conservative (return false even if supported) for efficiency. */ diff --git a/src/nfagraph/ng_violet.cpp b/src/nfagraph/ng_violet.cpp index 66c0253a..0dc010ea 100644 --- a/src/nfagraph/ng_violet.cpp +++ b/src/nfagraph/ng_violet.cpp @@ -2660,8 +2660,8 @@ vector> getDfaTriggers(RoseInGraph &vg, static bool doEarlyDfa(RoseBuild &rose, RoseInGraph &vg, NGHolder &h, - const vector &edges, const ReportManager &rm, - const CompileContext &cc) { + const vector &edges, bool final_chance, + const ReportManager &rm, const CompileContext &cc) { DEBUG_PRINTF("trying for dfa\n"); bool single_trigger; @@ -2680,7 +2680,7 @@ bool doEarlyDfa(RoseBuild &rose, RoseInGraph &vg, NGHolder &h, } shared_ptr dfa = buildMcClellan(h, &rm, single_trigger, triggers, - cc.grey); + cc.grey, final_chance); if (!dfa) { return false; @@ -2695,30 +2695,72 @@ bool doEarlyDfa(RoseBuild &rose, RoseInGraph &vg, NGHolder &h, } static -void ensureImplementable(RoseBuild &rose, RoseInGraph &vg, - const ReportManager &rm, const CompileContext &cc) { - map > edges_by_graph; - vector graphs; - for (const RoseInEdge &ve : edges_range(vg)) { - if (vg[ve].graph) { - NGHolder *h = vg[ve].graph.get(); - if (!contains(edges_by_graph, h)) { - graphs.push_back(h); +bool splitForImplemtabilty(UNUSED RoseInGraph &vg, UNUSED NGHolder &h, + UNUSED const vector &edges, + UNUSED const CompileContext &cc) { + /* TODO: need to add literals back to the graph? */ + return false; +} + +#define MAX_IMPLEMENTABLE_SPLITS 200 + +bool ensureImplementable(RoseBuild &rose, RoseInGraph &vg, bool allow_changes, + bool final_chance, const ReportManager &rm, + const CompileContext &cc) { + DEBUG_PRINTF("checking for impl\n"); + bool changed = false; + u32 added_count = 0; + do { + map > edges_by_graph; + vector graphs; + for (const RoseInEdge &ve : edges_range(vg)) { + if (vg[ve].graph) { + NGHolder *h = vg[ve].graph.get(); + if (!contains(edges_by_graph, h)) { + graphs.push_back(h); + } + edges_by_graph[h].push_back(ve); } - edges_by_graph[h].push_back(ve); } - } - for (NGHolder *h : graphs) { - if (isImplementableNFA(*h, &rm, cc)) { - continue; + for (NGHolder *h : graphs) { + if (isImplementableNFA(*h, &rm, cc)) { + continue; + } + + if (tryForEarlyDfa(*h, cc) + && doEarlyDfa(rose, vg, *h, edges_by_graph[h], final_chance, rm, + cc)) { + continue; + } + + DEBUG_PRINTF("eek\n"); + if (!allow_changes) { + return false; + } + + if (splitForImplemtabilty(vg, *h, edges_by_graph[h], cc)) { + added_count++; + changed = true; + continue; + } + + return false; } - if (tryForEarlyDfa(*h, cc) - && doEarlyDfa(rose, vg, *h, edges_by_graph[h], rm, cc)) { - continue; + if (added_count > MAX_IMPLEMENTABLE_SPLITS) { + return false; } - DEBUG_PRINTF("eek\n"); - } + + if (changed) { + removeRedundantLiterals(vg, cc); + pruneUseless(vg); + renumber_vertices(vg); + calcVertexOffsets(vg); + } + } while (changed); + + DEBUG_PRINTF("ok!\n"); + return true; } bool doViolet(RoseBuild &rose, const NGHolder &h, bool prefilter, @@ -2779,9 +2821,10 @@ bool doViolet(RoseBuild &rose, const NGHolder &h, bool prefilter, /* Step 5: avoid unimplementable, or overly large engines if possible */ - if (last_chance) { - ensureImplementable(rose, vg, rm, cc); + if (!ensureImplementable(rose, vg, last_chance, last_chance, rm, cc)) { + return false; } + dumpPreRoseGraph(vg, cc.grey, "post_ensure_rose.dot"); /* Step 6: send to rose */ bool rv = rose.addRose(vg, prefilter); diff --git a/src/nfagraph/ng_violet.h b/src/nfagraph/ng_violet.h index 6bc42d75..b6ecd028 100644 --- a/src/nfagraph/ng_violet.h +++ b/src/nfagraph/ng_violet.h @@ -42,6 +42,7 @@ class RoseBuild; struct CompileContext; class ReportManager; +struct RoseInGraph; /** \brief Attempt to consume the entire pattern in graph \a h with Rose. * Returns true if successful. */ @@ -49,6 +50,9 @@ bool doViolet(RoseBuild &rose, const NGHolder &h, bool prefilter, bool last_chance, const ReportManager &rm, const CompileContext &cc); +bool ensureImplementable(RoseBuild &rose, RoseInGraph &vg, bool allow_changes, + bool final_chance, const ReportManager &rm, + const CompileContext &cc); } // namespace ue2 #endif diff --git a/src/rose/rose_build.h b/src/rose/rose_build.h index c71671fa..0af8ba57 100644 --- a/src/rose/rose_build.h +++ b/src/rose/rose_build.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -88,8 +88,7 @@ public: virtual void add(bool anchored, bool eod, const ue2_literal &lit, const ue2::flat_set &ids) = 0; - virtual bool addRose(const RoseInGraph &ig, bool prefilter, - bool finalChance = false) = 0; + virtual bool addRose(const RoseInGraph &ig, bool prefilter) = 0; virtual bool addSombeRose(const RoseInGraph &ig) = 0; virtual bool addOutfix(const NGHolder &h) = 0; diff --git a/src/rose/rose_build_add.cpp b/src/rose/rose_build_add.cpp index b601f943..3ac6836f 100644 --- a/src/rose/rose_build_add.cpp +++ b/src/rose/rose_build_add.cpp @@ -85,9 +85,6 @@ struct RoseBuildData : boost::noncopyable { /** Input rose graph. */ const RoseInGraph &ig; - /** Mapping from engine graph to constructed DFA for pre-build DFAs. */ - ue2::unordered_map > early_dfas; - /** Edges we've transformed (in \ref transformAnchoredLiteralOverlap) which * require ANCH history to prevent overlap. */ ue2::unordered_set anch_history_edges; @@ -281,8 +278,8 @@ void createVertices(RoseBuildImpl *tbi, if (prefix_graph) { g[w].left.graph = prefix_graph; - if (contains(bd.early_dfas, prefix_graph.get())) { - g[w].left.dfa = bd.early_dfas.at(prefix_graph.get()); + if (edge_props.dfa) { + g[w].left.dfa = edge_props.dfa; } g[w].left.haig = edge_props.haig; g[w].left.lag = prefix_lag; @@ -769,9 +766,9 @@ void doRoseAcceptVertex(RoseBuildImpl *tbi, assert(!g[u].suffix); if (ig[iv].type == RIV_ACCEPT) { assert(!tbi->isAnyStart(u)); - if (contains(bd.early_dfas, edge_props.graph.get())) { + if (edge_props.dfa) { DEBUG_PRINTF("adding early dfa suffix to i%zu\n", g[u].index); - g[u].suffix.rdfa = bd.early_dfas.at(edge_props.graph.get()); + g[u].suffix.rdfa = edge_props.dfa; g[u].suffix.dfa_min_width = findMinWidth(*edge_props.graph); g[u].suffix.dfa_max_width = findMaxWidth(*edge_props.graph); } else if (edge_props.graph) { @@ -1033,13 +1030,10 @@ bool empty(const GraphT &g) { return vi == ve; } -/* We only try to implement as a dfa if a non-nullptr as_dfa is provided to - * return the raw dfa to. */ static -bool canImplementGraph(RoseBuildImpl *tbi, const RoseInGraph &in, NGHolder &h, +bool canImplementGraph(const RoseInGraph &in, NGHolder &h, const vector &edges, bool prefilter, - const ReportManager &rm, const CompileContext &cc, - bool finalChance, unique_ptr *as_dfa) { + const ReportManager &rm, const CompileContext &cc) { assert(!edges.empty()); assert(&*in[edges[0]].graph == &h); @@ -1061,64 +1055,6 @@ bool canImplementGraph(RoseBuildImpl *tbi, const RoseInGraph &in, NGHolder &h, } } - if (as_dfa) { - switch (h.kind) { - case NFA_OUTFIX: /* 'prefix' of eod */ - case NFA_PREFIX: - if (!cc.grey.earlyMcClellanPrefix) { - return false; - } - break; - case NFA_INFIX: - if (!cc.grey.earlyMcClellanInfix) { - return false; - } - break; - case NFA_SUFFIX: - if (!cc.grey.earlyMcClellanSuffix) { - return false; - } - break; - case NFA_EAGER_PREFIX: - case NFA_REV_PREFIX: - case NFA_OUTFIX_RAW: - DEBUG_PRINTF("kind %u\n", (u32)h.kind); - assert(0); - } - assert(!*as_dfa); - assert(tbi); - vector > triggers; - u32 min_offset = ~0U; - u32 max_offset = 0; - for (const auto &e : edges) { - RoseInVertex s = source(e, in); - RoseInVertex t = target(e, in); - if (in[s].type == RIV_LITERAL) { - triggers.push_back(as_cr_seq(in[s].s)); - } - if (in[t].type == RIV_ACCEPT_EOD) { - /* TODO: support eod prefixes */ - return false; - } - ENSURE_AT_LEAST(&max_offset, in[s].max_offset); - LIMIT_TO_AT_MOST(&min_offset, in[s].min_offset); - } - - if (!generates_callbacks(h)) { - set_report(h, tbi->getNewNfaReport()); - } - - bool single_trigger = min_offset == max_offset; - - DEBUG_PRINTF("trying for mcclellan (%u, %u)\n", min_offset, max_offset); - *as_dfa = buildMcClellan(h, &rm, single_trigger, triggers, cc.grey, - finalChance); - - if (*as_dfa) { - return true; - } - } - DEBUG_PRINTF("unable to build engine\n"); return false; } @@ -1573,8 +1509,7 @@ bool validateKinds(const RoseInGraph &g) { } #endif -bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter, - bool finalChance) { +bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter) { DEBUG_PRINTF("trying to rose\n"); assert(validateKinds(ig)); assert(hasCorrectlyNumberedVertices(ig)); @@ -1606,8 +1541,9 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter, continue; // no graph } - if (in[e].haig) { - // Haigs are always implementable (we've already built the raw DFA). + if (in[e].haig || in[e].dfa) { + /* Early DFAs/Haigs are always implementable (we've already built + * the raw DFA). */ continue; } @@ -1618,11 +1554,6 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter, ordered_graphs.push_back(h); } graphs[h].push_back(e); - if (in[e].dfa) { - assert(!contains(bd.early_dfas, h) - || bd.early_dfas[h] == in[e].dfa); - bd.early_dfas[h] = in[e].dfa; - } } assert(ordered_graphs.size() == graphs.size()); @@ -1631,16 +1562,9 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter, for (auto h : ordered_graphs) { const vector &h_edges = graphs.at(h); - unique_ptr as_dfa; - /* allow finalChance as fallback is basically an outfix at this point */ - if (!contains(bd.early_dfas, h) - && !canImplementGraph(this, in, *h, h_edges, prefilter, rm, cc, - finalChance, &as_dfa)) { + if (!canImplementGraph(in, *h, h_edges, prefilter, rm, cc)) { return false; } - if (as_dfa) { - bd.early_dfas[h] = move(as_dfa); - } insert(&graph_edges, graph_edges.end(), h_edges); } @@ -1655,7 +1579,6 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter, assert(allMatchStatesHaveReports(h)); if (!generates_callbacks(whatRoseIsThis(in, e)) - && !contains(bd.early_dfas, &h) && in[target(e, in)].type != RIV_ACCEPT_EOD) { set_report(h, getNewNfaReport()); } @@ -1716,8 +1639,7 @@ bool roseCheckRose(const RoseInGraph &ig, bool prefilter, } for (const auto &m : graphs) { - if (!canImplementGraph(nullptr, ig, *m.first, m.second, prefilter, rm, - cc, false, nullptr)) { + if (!canImplementGraph(ig, *m.first, m.second, prefilter, rm, cc)) { return false; } } diff --git a/src/rose/rose_build_impl.h b/src/rose/rose_build_impl.h index ce8a859d..2cefb42a 100644 --- a/src/rose/rose_build_impl.h +++ b/src/rose/rose_build_impl.h @@ -448,8 +448,7 @@ public: void add(bool anchored, bool eod, const ue2_literal &lit, const ue2::flat_set &ids) override; - bool addRose(const RoseInGraph &ig, bool prefilter, - bool finalChance = false) override; + bool addRose(const RoseInGraph &ig, bool prefilter) override; bool addSombeRose(const RoseInGraph &ig) override; bool addOutfix(const NGHolder &h) override;