shift all early_dfa creation logic to ng_violet/ng_rose

src/nfagraph/ng.cpp

@@ -255,7 +255,7 @@ bool addComponent(NG &ng, NGHolder &g, const NGWrapper &w, const som_type som,
         return true;
     }
 
-    if (splitOffRose(*ng.rose, g, w.prefilter, cc)) {
+    if (splitOffRose(*ng.rose, g, w.prefilter, ng.rm, cc)) {
         return true;
     }
 
@@ -276,7 +276,7 @@ bool addComponent(NG &ng, NGHolder &g, const NGWrapper &w, const som_type som,
         return true;
     }
 
-    if (splitOffRose(*ng.rose, g, w.prefilter, cc)) {
+    if (splitOffRose(*ng.rose, g, w.prefilter, ng.rm, cc)) {
         return true;
     }
 
@@ -291,7 +291,7 @@ bool addComponent(NG &ng, NGHolder &g, const NGWrapper &w, const som_type som,
         }
     }
 
-    if (finalChanceRose(*ng.rose, g, w.prefilter, cc)) {
+    if (finalChanceRose(*ng.rose, g, w.prefilter, ng.rm, cc)) {
         return true;
     }
 
@@ -533,16 +533,16 @@ bool NG::addHolder(NGHolder &w) {
         return true;
     }
 
-    if (splitOffRose(*rose, w, prefilter, cc)) {
+    if (splitOffRose(*rose, w, prefilter, rm, cc)) {
         return true;
     }
     if (splitOffPuffs(*rose, rm, w, prefilter, cc)) {
         return true;
     }
-    if (splitOffRose(*rose, w, prefilter, cc)) {
+    if (splitOffRose(*rose, w, prefilter, rm, cc)) {
         return true;
     }
-    if (finalChanceRose(*rose, w, prefilter, cc)) {
+    if (finalChanceRose(*rose, w, prefilter, rm, cc)) {
         return true;
     }
 

src/nfagraph/ng_rose.cpp

@@ -48,6 +48,7 @@
 #include "ng_reports.h"
 #include "ng_split.h"
 #include "ng_util.h"
+#include "ng_violet.h"
 #include "ng_width.h"
 #include "rose/rose_build.h"
 #include "rose/rose_build_util.h"
@@ -2833,8 +2834,19 @@ void desperationImprove(RoseInGraph &ig, const CompileContext &cc) {
     calcVertexOffsets(ig);
 }
 
+static
+bool addRose(RoseBuild &rose, RoseInGraph &ig, bool prefilter,
+             bool final_chance, const ReportManager &rm,
+             const CompileContext &cc) {
+    if (!ensureImplementable(rose, ig, false, final_chance, rm, cc)
+        && !prefilter) {
+        return false;
+    }
+    return rose.addRose(ig, prefilter);
+}
+
 bool splitOffRose(RoseBuild &rose, const NGHolder &h, bool prefilter,
-                  const CompileContext &cc) {
+                  const ReportManager &rm, const CompileContext &cc) {
     if (!cc.grey.allowRose) {
         return false;
     }
@@ -2843,20 +2855,20 @@ bool splitOffRose(RoseBuild &rose, const NGHolder &h, bool prefilter,
     assert(in_degree(h.accept, h) || in_degree(h.acceptEod, h) > 1);
 
     unique_ptr<RoseInGraph> igp = buildRose(h, false, cc);
-    if (igp && rose.addRose(*igp, prefilter)) {
+    if (igp && addRose(rose, *igp, prefilter, false, rm, cc)) {
         goto ok;
     }
 
     igp = buildRose(h, true, cc);
 
     if (igp) {
-        if (rose.addRose(*igp, prefilter)) {
+        if (addRose(rose, *igp, prefilter, false, rm, cc)) {
             goto ok;
         }
 
         desperationImprove(*igp, cc);
 
-        if (rose.addRose(*igp, prefilter)) {
+        if (addRose(rose, *igp, prefilter, false, rm, cc)) {
             goto ok;
         }
     }
@@ -2870,7 +2882,7 @@ ok:
 }
 
 bool finalChanceRose(RoseBuild &rose, const NGHolder &h, bool prefilter,
-                     const CompileContext &cc) {
+                     const ReportManager &rm, const CompileContext &cc) {
     DEBUG_PRINTF("final chance rose\n");
     if (!cc.grey.allowRose) {
         return false;
@@ -2935,7 +2947,7 @@ bool finalChanceRose(RoseBuild &rose, const NGHolder &h, bool prefilter,
     renumber_vertices(ig);
     calcVertexOffsets(ig);
 
-    return rose.addRose(ig, prefilter, true /* final chance */);
+    return addRose(rose, ig, prefilter, true /* final chance */, rm, cc);
 }
 
 bool checkRose(const ReportManager &rm, const NGHolder &h, bool prefilter,

src/nfagraph/ng_rose.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2016, Intel Corporation
+ * Copyright (c) 2015-2017, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -50,13 +50,13 @@ struct ue2_literal;
 /** \brief Attempt to consume the entire pattern in graph \a h with Rose.
  * Returns true if successful. */
 bool splitOffRose(RoseBuild &rose, const NGHolder &h, bool prefilter,
-                  const CompileContext &cc);
+                   const ReportManager &rm, const CompileContext &cc);
 
 /** \brief Attempt to consume the entire pattern in graph \a h with Rose.
  * This is the last attempt to handle a pattern before we resort to an outfix.
  * Returns true if successful. */
 bool finalChanceRose(RoseBuild &rose, const NGHolder &h, bool prefilter,
-                     const CompileContext &cc);
+                     const ReportManager &rm, const CompileContext &cc);
 
 /** \brief True if the pattern in \a h is consumable by Rose. This function
  * may be conservative (return false even if supported) for efficiency. */

src/nfagraph/ng_violet.cpp

@@ -2660,8 +2660,8 @@ vector<vector<CharReach>> getDfaTriggers(RoseInGraph &vg,
 
 static
 bool doEarlyDfa(RoseBuild &rose, RoseInGraph &vg, NGHolder &h,
-                const vector<RoseInEdge> &edges, const ReportManager &rm,
-                const CompileContext &cc) {
+                const vector<RoseInEdge> &edges, bool final_chance,
+                const ReportManager &rm, const CompileContext &cc) {
     DEBUG_PRINTF("trying for dfa\n");
 
     bool single_trigger;
@@ -2680,7 +2680,7 @@ bool doEarlyDfa(RoseBuild &rose, RoseInGraph &vg, NGHolder &h,
     }
 
     shared_ptr<raw_dfa> dfa = buildMcClellan(h, &rm, single_trigger, triggers,
-                                             cc.grey);
+                                             cc.grey, final_chance);
 
     if (!dfa) {
         return false;
@@ -2695,30 +2695,72 @@ bool doEarlyDfa(RoseBuild &rose, RoseInGraph &vg, NGHolder &h,
 }
 
 static
-void ensureImplementable(RoseBuild &rose, RoseInGraph &vg,
-                         const ReportManager &rm, const CompileContext &cc) {
-    map<const NGHolder *, vector<RoseInEdge> > edges_by_graph;
-    vector<NGHolder *> graphs;
-    for (const RoseInEdge &ve : edges_range(vg)) {
-        if (vg[ve].graph) {
-            NGHolder *h = vg[ve].graph.get();
-            if (!contains(edges_by_graph, h)) {
-                graphs.push_back(h);
+bool splitForImplemtabilty(UNUSED RoseInGraph &vg, UNUSED NGHolder &h,
+                           UNUSED const vector<RoseInEdge> &edges,
+                           UNUSED const CompileContext &cc) {
+    /* TODO: need to add literals back to the graph? */
+    return false;
+}
+
+#define MAX_IMPLEMENTABLE_SPLITS 200
+
+bool ensureImplementable(RoseBuild &rose, RoseInGraph &vg, bool allow_changes,
+                         bool final_chance, const ReportManager &rm,
+                         const CompileContext &cc) {
+    DEBUG_PRINTF("checking for impl\n");
+    bool changed = false;
+    u32 added_count = 0;
+    do {
+        map<const NGHolder *, vector<RoseInEdge> > edges_by_graph;
+        vector<NGHolder *> graphs;
+        for (const RoseInEdge &ve : edges_range(vg)) {
+            if (vg[ve].graph) {
+                NGHolder *h = vg[ve].graph.get();
+                if (!contains(edges_by_graph, h)) {
+                    graphs.push_back(h);
+                }
+                edges_by_graph[h].push_back(ve);
             }
-            edges_by_graph[h].push_back(ve);
         }
-    }
-    for (NGHolder *h : graphs) {
-        if (isImplementableNFA(*h, &rm, cc)) {
-            continue;
+        for (NGHolder *h : graphs) {
+            if (isImplementableNFA(*h, &rm, cc)) {
+                continue;
+            }
+
+            if (tryForEarlyDfa(*h, cc)
+                && doEarlyDfa(rose, vg, *h, edges_by_graph[h], final_chance, rm,
+                              cc)) {
+                continue;
+            }
+
+            DEBUG_PRINTF("eek\n");
+            if (!allow_changes) {
+                return false;
+            }
+
+            if (splitForImplemtabilty(vg, *h, edges_by_graph[h], cc)) {
+                added_count++;
+                changed = true;
+                continue;
+            }
+
+            return false;
         }
 
-        if (tryForEarlyDfa(*h, cc)
-            && doEarlyDfa(rose, vg, *h, edges_by_graph[h], rm, cc)) {
-            continue;
+        if (added_count > MAX_IMPLEMENTABLE_SPLITS) {
+            return false;
         }
-        DEBUG_PRINTF("eek\n");
-    }
+
+        if (changed) {
+            removeRedundantLiterals(vg, cc);
+            pruneUseless(vg);
+            renumber_vertices(vg);
+            calcVertexOffsets(vg);
+        }
+    } while (changed);
+
+    DEBUG_PRINTF("ok!\n");
+    return true;
 }
 
 bool doViolet(RoseBuild &rose, const NGHolder &h, bool prefilter,
@@ -2779,9 +2821,10 @@ bool doViolet(RoseBuild &rose, const NGHolder &h, bool prefilter,
 
 
     /* Step 5: avoid unimplementable, or overly large engines if possible */
-    if (last_chance) {
-        ensureImplementable(rose, vg, rm, cc);
+    if (!ensureImplementable(rose, vg, last_chance, last_chance, rm, cc)) {
+        return false;
     }
+    dumpPreRoseGraph(vg, cc.grey, "post_ensure_rose.dot");
 
     /* Step 6: send to rose */
     bool rv = rose.addRose(vg, prefilter);

src/nfagraph/ng_violet.h

@@ -42,6 +42,7 @@ class RoseBuild;
 
 struct CompileContext;
 class ReportManager;
+struct RoseInGraph;
 
 /** \brief Attempt to consume the entire pattern in graph \a h with Rose.
  * Returns true if successful. */
@@ -49,6 +50,9 @@ bool doViolet(RoseBuild &rose, const NGHolder &h, bool prefilter,
               bool last_chance, const ReportManager &rm,
               const CompileContext &cc);
 
+bool ensureImplementable(RoseBuild &rose, RoseInGraph &vg, bool allow_changes,
+                         bool final_chance, const ReportManager &rm,
+                         const CompileContext &cc);
 } // namespace ue2
 
 #endif

src/rose/rose_build.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2016, Intel Corporation
+ * Copyright (c) 2015-2017, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -88,8 +88,7 @@ public:
     virtual void add(bool anchored, bool eod, const ue2_literal &lit,
                      const ue2::flat_set<ReportID> &ids) = 0;
 
-    virtual bool addRose(const RoseInGraph &ig, bool prefilter,
-                         bool finalChance = false) = 0;
+    virtual bool addRose(const RoseInGraph &ig, bool prefilter) = 0;
     virtual bool addSombeRose(const RoseInGraph &ig) = 0;
 
     virtual bool addOutfix(const NGHolder &h) = 0;

src/rose/rose_build_add.cpp

@@ -85,9 +85,6 @@ struct RoseBuildData : boost::noncopyable {
     /** Input rose graph. */
     const RoseInGraph &ig;
 
-    /** Mapping from engine graph to constructed DFA for pre-build DFAs. */
-    ue2::unordered_map<const NGHolder *, shared_ptr<raw_dfa> > early_dfas;
-
     /** Edges we've transformed (in \ref transformAnchoredLiteralOverlap) which
      * require ANCH history to prevent overlap. */
     ue2::unordered_set<RoseInEdge> anch_history_edges;
@@ -281,8 +278,8 @@ void createVertices(RoseBuildImpl *tbi,
 
             if (prefix_graph) {
                 g[w].left.graph = prefix_graph;
-                if (contains(bd.early_dfas, prefix_graph.get())) {
-                    g[w].left.dfa = bd.early_dfas.at(prefix_graph.get());
+                if (edge_props.dfa) {
+                    g[w].left.dfa = edge_props.dfa;
                 }
                 g[w].left.haig = edge_props.haig;
                 g[w].left.lag = prefix_lag;
@@ -769,9 +766,9 @@ void doRoseAcceptVertex(RoseBuildImpl *tbi,
         assert(!g[u].suffix);
         if (ig[iv].type == RIV_ACCEPT) {
             assert(!tbi->isAnyStart(u));
-            if (contains(bd.early_dfas, edge_props.graph.get())) {
+            if (edge_props.dfa) {
                 DEBUG_PRINTF("adding early dfa suffix to i%zu\n", g[u].index);
-                g[u].suffix.rdfa = bd.early_dfas.at(edge_props.graph.get());
+                g[u].suffix.rdfa = edge_props.dfa;
                 g[u].suffix.dfa_min_width = findMinWidth(*edge_props.graph);
                 g[u].suffix.dfa_max_width = findMaxWidth(*edge_props.graph);
             } else if (edge_props.graph) {
@@ -1033,13 +1030,10 @@ bool empty(const GraphT &g) {
     return vi == ve;
 }
 
-/* We only try to implement as a dfa if a non-nullptr as_dfa is provided to
- * return the raw dfa to. */
 static
-bool canImplementGraph(RoseBuildImpl *tbi, const RoseInGraph &in, NGHolder &h,
+bool canImplementGraph(const RoseInGraph &in, NGHolder &h,
                        const vector<RoseInEdge> &edges, bool prefilter,
-                       const ReportManager &rm, const CompileContext &cc,
-                       bool finalChance, unique_ptr<raw_dfa> *as_dfa) {
+                       const ReportManager &rm, const CompileContext &cc) {
     assert(!edges.empty());
     assert(&*in[edges[0]].graph == &h);
 
@@ -1061,64 +1055,6 @@ bool canImplementGraph(RoseBuildImpl *tbi, const RoseInGraph &in, NGHolder &h,
         }
     }
 
-    if (as_dfa) {
-        switch (h.kind) {
-        case NFA_OUTFIX: /* 'prefix' of eod */
-        case NFA_PREFIX:
-            if (!cc.grey.earlyMcClellanPrefix) {
-                return false;
-            }
-            break;
-        case NFA_INFIX:
-            if (!cc.grey.earlyMcClellanInfix) {
-                return false;
-            }
-            break;
-        case NFA_SUFFIX:
-            if (!cc.grey.earlyMcClellanSuffix) {
-                return false;
-            }
-            break;
-        case NFA_EAGER_PREFIX:
-        case NFA_REV_PREFIX:
-        case NFA_OUTFIX_RAW:
-            DEBUG_PRINTF("kind %u\n", (u32)h.kind);
-            assert(0);
-        }
-        assert(!*as_dfa);
-        assert(tbi);
-        vector<vector<CharReach> > triggers;
-        u32 min_offset = ~0U;
-        u32 max_offset = 0;
-        for (const auto &e : edges) {
-            RoseInVertex s = source(e, in);
-            RoseInVertex t = target(e, in);
-            if (in[s].type == RIV_LITERAL) {
-                triggers.push_back(as_cr_seq(in[s].s));
-            }
-            if (in[t].type == RIV_ACCEPT_EOD) {
-                /* TODO: support eod prefixes */
-                return false;
-            }
-            ENSURE_AT_LEAST(&max_offset, in[s].max_offset);
-            LIMIT_TO_AT_MOST(&min_offset, in[s].min_offset);
-        }
-
-        if (!generates_callbacks(h)) {
-            set_report(h, tbi->getNewNfaReport());
-        }
-
-        bool single_trigger = min_offset == max_offset;
-
-        DEBUG_PRINTF("trying for mcclellan (%u, %u)\n", min_offset, max_offset);
-        *as_dfa = buildMcClellan(h, &rm, single_trigger, triggers, cc.grey,
-                                 finalChance);
-
-        if (*as_dfa) {
-            return true;
-        }
-    }
-
     DEBUG_PRINTF("unable to build engine\n");
     return false;
 }
@@ -1573,8 +1509,7 @@ bool validateKinds(const RoseInGraph &g) {
 }
 #endif
 
-bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter,
-                            bool finalChance) {
+bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter) {
     DEBUG_PRINTF("trying to rose\n");
     assert(validateKinds(ig));
     assert(hasCorrectlyNumberedVertices(ig));
@@ -1606,8 +1541,9 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter,
             continue; // no graph
         }
 
-        if (in[e].haig) {
-            // Haigs are always implementable (we've already built the raw DFA).
+        if (in[e].haig || in[e].dfa) {
+            /* Early DFAs/Haigs are always implementable (we've already built
+             * the raw DFA). */
             continue;
         }
 
@@ -1618,11 +1554,6 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter,
             ordered_graphs.push_back(h);
         }
         graphs[h].push_back(e);
-        if (in[e].dfa) {
-            assert(!contains(bd.early_dfas, h)
-                   || bd.early_dfas[h] == in[e].dfa);
-            bd.early_dfas[h] = in[e].dfa;
-        }
     }
 
     assert(ordered_graphs.size() == graphs.size());
@@ -1631,16 +1562,9 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter,
 
     for (auto h : ordered_graphs) {
         const vector<RoseInEdge> &h_edges = graphs.at(h);
-        unique_ptr<raw_dfa> as_dfa;
-        /* allow finalChance as fallback is basically an outfix at this point */
-        if (!contains(bd.early_dfas, h)
-            && !canImplementGraph(this, in, *h, h_edges, prefilter, rm, cc,
-                                  finalChance, &as_dfa)) {
+        if (!canImplementGraph(in, *h, h_edges, prefilter, rm, cc)) {
             return false;
         }
-        if (as_dfa) {
-            bd.early_dfas[h] = move(as_dfa);
-        }
         insert(&graph_edges, graph_edges.end(), h_edges);
     }
 
@@ -1655,7 +1579,6 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter,
         assert(allMatchStatesHaveReports(h));
 
         if (!generates_callbacks(whatRoseIsThis(in, e))
-            && !contains(bd.early_dfas, &h)
             && in[target(e, in)].type != RIV_ACCEPT_EOD) {
             set_report(h, getNewNfaReport());
         }
@@ -1716,8 +1639,7 @@ bool roseCheckRose(const RoseInGraph &ig, bool prefilter,
     }
 
     for (const auto &m : graphs) {
-        if (!canImplementGraph(nullptr, ig, *m.first, m.second, prefilter, rm,
-                               cc, false, nullptr)) {
+        if (!canImplementGraph(ig, *m.first, m.second, prefilter, rm, cc)) {
             return false;
         }
     }

src/rose/rose_build_impl.h

@@ -448,8 +448,7 @@ public:
     void add(bool anchored, bool eod, const ue2_literal &lit,
              const ue2::flat_set<ReportID> &ids) override;
 
-    bool addRose(const RoseInGraph &ig, bool prefilter,
-                 bool finalChance = false) override;
+    bool addRose(const RoseInGraph &ig, bool prefilter) override;
     bool addSombeRose(const RoseInGraph &ig) override;
 
     bool addOutfix(const NGHolder &h) override;