violet: initial implementation

CMakeLists.txt

@@ -748,6 +748,8 @@ SET (hs_SRCS
     src/nfagraph/ng_util.h
     src/nfagraph/ng_vacuous.cpp
     src/nfagraph/ng_vacuous.h
+    src/nfagraph/ng_violet.cpp
+    src/nfagraph/ng_violet.h
     src/nfagraph/ng_width.cpp
     src/nfagraph/ng_width.h
     src/parser/AsciiComponentClass.cpp

src/grey.cpp

@@ -34,7 +34,7 @@
 #include <string>
 #include <vector>
 
-#define DEFAULT_MAX_HISTORY 60
+#define DEFAULT_MAX_HISTORY 110
 
 using namespace std;
 
@@ -51,7 +51,9 @@ Grey::Grey(void) :
                    allowLbr(true),
                    allowMcClellan(true),
                    allowPuff(true),
+                   allowLiteral(true),
                    allowRose(true),
+                   allowViolet(true),
                    allowExtendedNFA(true), /* bounded repeats of course */
                    allowLimExNFA(true),
                    allowAnchoredAcyclic(true),
@@ -60,6 +62,13 @@ Grey::Grey(void) :
                    allowDecoratedLiteral(true),
                    allowNoodle(true),
                    fdrAllowTeddy(true),
+                   violetAvoidSuffixes(true),
+                   violetAvoidWeakInfixes(true),
+                   violetDoubleCut(true),
+                   violetExtractStrongLiterals(true),
+                   violetLiteralChains(true),
+                   violetDoubleCutLiteralLen(3),
+                   violetEarlyCleanLiteralLen(6),
                    puffImproveHead(true),
                    castleExclusive(true),
                    mergeSEP(true), /* short exhaustible passthroughs */
@@ -206,7 +215,9 @@ void applyGreyOverrides(Grey *g, const string &s) {
         G_UPDATE(allowLbr);
         G_UPDATE(allowMcClellan);
         G_UPDATE(allowPuff);
+        G_UPDATE(allowLiteral);
         G_UPDATE(allowRose);
+        G_UPDATE(allowViolet);
         G_UPDATE(allowExtendedNFA);
         G_UPDATE(allowLimExNFA);
         G_UPDATE(allowAnchoredAcyclic);
@@ -215,6 +226,13 @@ void applyGreyOverrides(Grey *g, const string &s) {
         G_UPDATE(allowDecoratedLiteral);
         G_UPDATE(allowNoodle);
         G_UPDATE(fdrAllowTeddy);
+        G_UPDATE(violetAvoidSuffixes);
+        G_UPDATE(violetAvoidWeakInfixes);
+        G_UPDATE(violetDoubleCut);
+        G_UPDATE(violetExtractStrongLiterals);
+        G_UPDATE(violetLiteralChains);
+        G_UPDATE(violetDoubleCutLiteralLen);
+        G_UPDATE(violetEarlyCleanLiteralLen);
         G_UPDATE(puffImproveHead);
         G_UPDATE(castleExclusive);
         G_UPDATE(mergeSEP);
@@ -315,7 +333,9 @@ void applyGreyOverrides(Grey *g, const string &s) {
             g->allowLitHaig = false;
             g->allowMcClellan = false;
             g->allowPuff = false;
+            g->allowLiteral = false;
             g->allowRose = false;
+            g->allowViolet = false;
             g->allowSmallLiteralSet = false;
             g->roseMasks = false;
             done = true;
@@ -331,7 +351,9 @@ void applyGreyOverrides(Grey *g, const string &s) {
             g->allowLitHaig = false;
             g->allowMcClellan = true;
             g->allowPuff = false;
+            g->allowLiteral = false;
             g->allowRose = false;
+            g->allowViolet = false;
             g->allowSmallLiteralSet = false;
             g->roseMasks = false;
             done = true;
@@ -347,7 +369,9 @@ void applyGreyOverrides(Grey *g, const string &s) {
             g->allowLitHaig = false;
             g->allowMcClellan = true;
             g->allowPuff = false;
+            g->allowLiteral = false;
             g->allowRose = false;
+            g->allowViolet = false;
             g->allowSmallLiteralSet = false;
             g->roseMasks = false;
             done = true;

src/grey.h

@@ -51,7 +51,9 @@ struct Grey {
     bool allowLbr;
     bool allowMcClellan;
     bool allowPuff;
+    bool allowLiteral;
     bool allowRose;
+    bool allowViolet;
     bool allowExtendedNFA;
     bool allowLimExNFA;
     bool allowAnchoredAcyclic;
@@ -62,6 +64,14 @@ struct Grey {
     bool allowNoodle;
     bool fdrAllowTeddy;
 
+    u32  violetAvoidSuffixes; /* 0=never, 1=sometimes, 2=always */
+    bool violetAvoidWeakInfixes;
+    bool violetDoubleCut;
+    bool violetExtractStrongLiterals;
+    bool violetLiteralChains;
+    u32  violetDoubleCutLiteralLen;
+    u32  violetEarlyCleanLiteralLen;
+
     bool puffImproveHead;
     bool castleExclusive; // enable castle mutual exclusion analysis
 

src/hwlm/hwlm_build.cpp

@@ -35,6 +35,7 @@
 #include "hwlm_internal.h"
 #include "noodle_engine.h"
 #include "noodle_build.h"
+#include "scratch.h"
 #include "ue2common.h"
 #include "fdr/fdr_compile.h"
 #include "nfa/shufticompile.h"
@@ -493,6 +494,10 @@ bool isNoodleable(const vector<hwlmLiteral> &lits,
                          stream_control->history_max);
             return false;
         }
+        if (2 * lits.front().s.length() - 2 > FDR_TEMP_BUF_SIZE) {
+            assert(0);
+            return false;
+        }
     }
 
     if (!lits.front().msk.empty()) {

src/nfa/nfa_kind.h

@@ -37,6 +37,8 @@
 
 #include "ue2common.h"
 
+#include <string>
+
 namespace ue2 {
 
 /** \brief Specify the use-case for an nfa engine. */
@@ -115,6 +117,32 @@ bool has_managed_reports(enum nfa_kind k) {
     }
 }
 
+#if defined(DEBUG) || defined(DUMP_SUPPORT)
+
+inline
+std::string to_string(nfa_kind k) {
+    switch (k) {
+    case NFA_PREFIX:
+        return "PREFIX";
+    case NFA_INFIX:
+        return "INFIX";
+    case NFA_SUFFIX:
+        return "SUFFIX";
+    case NFA_OUTFIX:
+        return "OUTFIX";
+    case NFA_REV_PREFIX:
+        return "REV_PREFIX";
+    case NFA_OUTFIX_RAW:
+        return "OUTFIX_RAW";
+    case NFA_EAGER_PREFIX:
+        return "EAGER_PREFIX";
+    }
+    assert(0);
+    return "?";
+}
+
+#endif
+
 } // namespace ue2
 
 #endif

src/nfagraph/ng.cpp

@@ -57,6 +57,7 @@
 #include "ng_small_literal_set.h"
 #include "ng_som.h"
 #include "ng_vacuous.h"
+#include "ng_violet.h"
 #include "ng_utf8.h"
 #include "ng_util.h"
 #include "ng_width.h"
@@ -244,6 +245,10 @@ bool addComponent(NG &ng, NGHolder &g, const NGWrapper &w, const som_type som,
         return true;
     }
 
+    if (doViolet(*ng.rose, g, w.prefilter, cc)) {
+        return true;
+    }
+
     if (splitOffRose(*ng.rose, g, w.prefilter, cc)) {
         return true;
     }
@@ -261,6 +266,10 @@ bool addComponent(NG &ng, NGHolder &g, const NGWrapper &w, const som_type som,
         return true;
     }
 
+    if (doViolet(*ng.rose, g, w.prefilter, cc)) {
+        return true;
+    }
+
     if (splitOffRose(*ng.rose, g, w.prefilter, cc)) {
         return true;
     }

src/nfagraph/ng_holder.h

@@ -239,6 +239,16 @@ vertices(const NGHolder &h) {
  */
 void clear_graph(NGHolder &h);
 
+inline
+void renumber_edges(NGHolder &h) {
+    h.renumberEdges();
+}
+
+inline
+void renumber_vertices(NGHolder &h) {
+    h.renumberVertices();
+}
+
 /*
  * \brief Clear and remove all of the vertices pointed to by the given iterator
  * range.

src/nfagraph/ng_literal_analysis.cpp

@@ -339,6 +339,12 @@ void processWorkQueue(const NGHolder &g, const NFAEdge &e,
                  g[source(e, g)].index, g[target(e, g)].index, s.size());
 }
 
+bool bad_mixed_sensitivity(const ue2_literal &s) {
+    /* TODO: if the mixed cases is entirely within MAX_MASK2_WIDTH of the end,
+     * we should be able to handle it */
+    return mixed_sensitivity(s) && s.length() > MAX_MASK2_WIDTH;
+}
+
 static
 u64a litUniqueness(const string &s) {
     CharReach seen(s);
@@ -624,6 +630,48 @@ u64a compressAndScore(set<ue2_literal> &s) {
     return score;
 }
 
+/* like compressAndScore, but replaces long mixed sensitivity literals with
+ * something weaker. */
+u64a sanitizeAndCompressAndScore(set<ue2_literal> &lits) {
+    const size_t maxExploded = 8; // only case-explode this far
+
+    /* TODO: the whole compression thing could be made better by systematically
+     * considering replacing literal sets not just by common suffixes but also
+     * by nocase literals. */
+
+    vector<ue2_literal> replacements;
+
+    for (auto it = lits.begin(); it != lits.end();) {
+        auto jt = it;
+        ++it;
+
+        if (!bad_mixed_sensitivity(*jt)) {
+            continue;
+        }
+
+        /* we have to replace *jt with something... */
+        ue2_literal s = *jt;
+        lits.erase(jt);
+
+        vector<ue2_literal> exploded;
+        for (auto cit = caseIterateBegin(s); cit != caseIterateEnd(); ++cit) {
+            exploded.emplace_back(*cit, false);
+            if (exploded.size() > maxExploded) {
+                goto dont_explode;
+            }
+        }
+        insert(&replacements, replacements.end(), exploded);
+
+        continue;
+    dont_explode:
+        make_nocase(&s);
+        replacements.push_back(s);
+    }
+
+    insert(&lits, replacements);
+    return compressAndScore(lits);
+}
+
 u64a scoreSet(const set<ue2_literal> &s) {
     if (s.empty()) {
         return NO_LITERAL_AT_EDGE_SCORE;
@@ -674,7 +722,7 @@ set<ue2_literal> getLiteralSet(const NGHolder &g, const NFAVertex &v,
     return s;
 }
 
-vector<u64a> scoreEdges(const NGHolder &g) {
+vector<u64a> scoreEdges(const NGHolder &g, const flat_set<NFAEdge> &known_bad) {
     assert(hasCorrectlyNumberedEdges(g));
 
     vector<u64a> scores(num_edges(g));
@@ -682,8 +730,12 @@ vector<u64a> scoreEdges(const NGHolder &g) {
     for (const auto &e : edges_range(g)) {
         u32 eidx = g[e].index;
         assert(eidx < scores.size());
-        set<ue2_literal> ls = getLiteralSet(g, e);
+        if (contains(known_bad, e)) {
-        scores[eidx] = compressAndScore(ls);
+            scores[eidx] = NO_LITERAL_AT_EDGE_SCORE;
+        } else {
+            set<ue2_literal> ls = getLiteralSet(g, e);
+            scores[eidx] = compressAndScore(ls);
+        }
     }
 
     return scores;
@@ -842,4 +894,49 @@ bool getTrailingLiteral(const NGHolder &g, ue2_literal *lit_out) {
     return true;
 }
 
+bool literalIsWholeGraph(const NGHolder &g, const ue2_literal &lit) {
+    NFAVertex v = g.accept;
+
+    for (auto it = lit.rbegin(), ite = lit.rend(); it != ite; ++it) {
+        NGHolder::inv_adjacency_iterator ai, ae;
+        tie(ai, ae) = inv_adjacent_vertices(v, g);
+        if (ai == ae) {
+            assert(0); // no predecessors?
+            return false;
+        }
+        v = *ai++;
+        if (ai != ae) {
+            DEBUG_PRINTF("branch, fail\n");
+            return false;
+        }
+
+        if (is_special(v, g)) {
+            DEBUG_PRINTF("special found, fail\n");
+            return false;
+        }
+
+        const CharReach &cr_g = g[v].char_reach;
+        const CharReach &cr_l = *it;
+
+        if (!cr_l.isSubsetOf(cr_g)) {
+            /* running over the prefix is needed to prevent false postives */
+            DEBUG_PRINTF("reach fail\n");
+            return false;
+        }
+    }
+
+    // Our last value for v should have only start states for predecessors.
+    for (auto u : inv_adjacent_vertices_range(v, g)) {
+        if (!is_any_start(u, g)) {
+            DEBUG_PRINTF("pred is not start\n");
+            return false;
+        }
+    }
+
+    assert(num_vertices(g) == lit.length() + N_SPECIALS);
+
+    DEBUG_PRINTF("ok\n");
+    return true;
+}
+
 } // namespace ue2

src/nfagraph/ng_literal_analysis.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -42,9 +42,7 @@
 namespace ue2 {
 
 #define NO_LITERAL_AT_EDGE_SCORE  10000000ULL
-
+#define INVALID_EDGE_CAP         100000000ULL /* special-to-special score */
-/* Score for special-to-special edges */
-#define INVALID_EDGE_CAP 100000000ULL
 
 class NGHolder;
 
@@ -59,9 +57,20 @@ std::set<ue2_literal> getLiteralSet(const NGHolder &g, const NFAVertex &v,
                                     bool only_first_encounter = true);
 std::set<ue2_literal> getLiteralSet(const NGHolder &g, const NFAEdge &e);
 
-/** Score all the edges in the given graph, returning them in \p scores indexed
+/**
+ * Returns true if we are unable to use a mixed sensitivity literal in rose (as
+ * our literal matchers are generally either case sensitive or not).
+ *
+ * Shortish mixed sensitivity literals can be handled by confirm checks in rose
+ * and are not flagged as bad.
+ */
+bool bad_mixed_sensitivity(const ue2_literal &s);
+
+/**
+ * Score all the edges in the given graph, returning them in \p scores indexed
  * by edge_index. */
-std::vector<u64a> scoreEdges(const NGHolder &h);
+std::vector<u64a> scoreEdges(const NGHolder &h,
+                             const flat_set<NFAEdge> &known_bad = {});
 
 /** Returns a score for a literal set. Lower scores are better. */
 u64a scoreSet(const std::set<ue2_literal> &s);
@@ -69,6 +78,12 @@ u64a scoreSet(const std::set<ue2_literal> &s);
 /** Compress a literal set to fewer literals. */
 u64a compressAndScore(std::set<ue2_literal> &s);
 
+/**
+ * Compress a literal set to fewer literals and replace any long mixed
+ * sensitivity literals with supported literals.
+ */
+u64a sanitizeAndCompressAndScore(std::set<ue2_literal> &s);
+
 bool splitOffLeadingLiteral(const NGHolder &g, ue2_literal *lit_out,
                             NGHolder *rhs);
 
@@ -77,6 +92,10 @@ bool splitOffAnchoredLeadingLiteral(const NGHolder &g, ue2_literal *lit_out,
 
 bool getTrailingLiteral(const NGHolder &g, ue2_literal *lit_out);
 
+/** \brief Returns true if the given literal is the only thing in the graph,
+ * from (start or startDs) to accept. */
+bool literalIsWholeGraph(const NGHolder &g, const ue2_literal &lit);
+
 } // namespace ue2
 
 #endif

src/nfagraph/ng_literal_component.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -186,7 +186,7 @@ bool splitOffLiteral(NG &ng, NGWrapper &g, NFAVertex v, const bool anchored,
 
 /** \brief Split off literals. True if any changes were made to the graph. */
 bool splitOffLiterals(NG &ng, NGWrapper &g) {
-    if (!ng.cc.grey.allowRose) {
+    if (!ng.cc.grey.allowLiteral) {
         return false;
     }
 

src/nfagraph/ng_rose.cpp

@@ -773,51 +773,6 @@ unique_ptr<VertLitInfo> LitCollection::pickNext() {
 
 }
 
-/** \brief Returns true if the given literal is the only thing in the graph,
- * from start to accept. */
-static
-bool literalIsWholeGraph(const NGHolder &g, const ue2_literal &lit) {
-    NFAVertex v = g.accept;
-
-    for (auto it = lit.rbegin(), ite = lit.rend(); it != ite; ++it) {
-        NGHolder::inv_adjacency_iterator ai, ae;
-        tie(ai, ae) = inv_adjacent_vertices(v, g);
-        if (ai == ae) {
-            assert(0); // no predecessors?
-            return false;
-        }
-        v = *ai++;
-        if (ai != ae) {
-            DEBUG_PRINTF("branch, fail\n");
-            return false;
-        }
-
-        if (is_special(v, g)) {
-            DEBUG_PRINTF("special found, fail\n");
-            return false;
-        }
-
-        const CharReach &cr = g[v].char_reach;
-        if (cr != *it) {
-            DEBUG_PRINTF("reach fail\n");
-            return false;
-        }
-    }
-
-    // Our last value for v should have only start states for predecessors.
-    for (auto u : inv_adjacent_vertices_range(v, g)) {
-        if (!is_any_start(u, g)) {
-            DEBUG_PRINTF("pred is not start\n");
-            return false;
-        }
-    }
-
-    assert(num_vertices(g) == lit.length() + N_SPECIALS);
-
-    DEBUG_PRINTF("ok\n");
-    return true;
-}
-
 static
 bool can_match(const NGHolder &g, const ue2_literal &lit, bool overhang_ok) {
     set<NFAVertex> curr, next;
@@ -933,20 +888,11 @@ u32 removeTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
     return delay;
 }
 
-static
 void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
-                                  u32 delay) {
+                                  u32 delay, const vector<NFAVertex> &preds) {
     assert(delay <= lit.length());
     DEBUG_PRINTF("adding on '%s' %u\n", ((const string &)lit).c_str(), delay);
 
-    vector<NFAVertex> preds;
-    insert(&preds, preds.end(), inv_adjacent_vertices(g.accept, g));
-    clear_in_edges(g.accept, g);
-
-    for (auto v : preds) {
-        g[v].reports.clear(); /* clear report from old accepts */
-    }
-
     NFAVertex prev = g.accept;
     auto it = lit.rbegin();
     while (delay--) {
@@ -972,6 +918,19 @@ void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
     assert(allMatchStatesHaveReports(g));
 }
 
+void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
+                                  u32 delay) {
+    vector<NFAVertex> preds;
+    insert(&preds, preds.end(), inv_adjacent_vertices(g.accept, g));
+    clear_in_edges(g.accept, g);
+
+    for (auto v : preds) {
+        g[v].reports.clear(); /* clear report from old accepts */
+    }
+
+    restoreTrailingLiteralStates(g, lit, delay, preds);
+}
+
 /* return false if we should get rid of the edge altogether */
 static
 bool removeLiteralFromLHS(RoseInGraph &ig, const RoseInEdge &lhs,

src/nfagraph/ng_rose.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -33,8 +33,11 @@
 #ifndef NG_ROSE_H
 #define NG_ROSE_H
 
+#include "ng_holder.h"
 #include "ue2common.h"
 
+#include <vector>
+
 namespace ue2 {
 
 class NGHolder;
@@ -65,6 +68,13 @@ bool checkRose(const ReportManager &rm, const NGHolder &h, bool prefilter,
 u32 removeTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
                                 u32 max_delay, bool overhang_ok = true);
 
+void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
+                                  u32 delay);
+
+void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
+                                  u32 delay,
+                                  const std::vector<NFAVertex> &preds);
+
 } // namespace ue2
 
 #endif // NG_ROSE_H

src/nfagraph/ng_som.cpp

@@ -2064,8 +2064,7 @@ sombe_rv doHaigLitSom(NG &ng, NGHolder &g, const NGWrapper &w, u32 comp_id,
     ReportManager &rm = ng.rm;
     SomSlotManager &ssm = ng.ssm;
 
-    // This approach relies on Rose.
+    if (!cc.grey.allowHaigLit) {
-    if (!cc.grey.allowRose) {
         return SOMBE_FAIL;
     }
 

src/nfagraph/ng_split.cpp

@@ -100,7 +100,12 @@ void splitLHS(const NGHolder &base, const vector<NFAVertex> &pivots,
         add_edge((*lhs_map)[pivot], lhs->accept, *lhs);
     }
 
-    pruneUseless(*lhs);
+    /* should do the renumbering unconditionally as we know edges are already
+     * misnumbered */
+    pruneUseless(*lhs, false);
+    renumber_edges(*lhs);
+    renumber_vertices(*lhs);
+
     filterSplitMap(*lhs, lhs_map);
 
     switch (base.kind) {
@@ -148,7 +153,12 @@ void splitRHS(const NGHolder &base, const vector<NFAVertex> &pivots,
         assert(contains(*rhs_map, pivot));
         add_edge(rhs->start, (*rhs_map)[pivot], *rhs);
     }
-    pruneUseless(*rhs);
+
+     /* should do the renumbering unconditionally as we know edges are already
+      * misnumbered */
+    pruneUseless(*rhs, false);
+    renumber_edges(*rhs);
+    renumber_vertices(*rhs);
     filterSplitMap(*rhs, rhs_map);
 
     switch (base.kind) {

src/nfagraph/ng_util.cpp

@@ -209,6 +209,15 @@ bool isAnchored(const NGHolder &g) {
     return true;
 }
 
+bool isFloating(const NGHolder &g) {
+    for (auto v : adjacent_vertices_range(g.start, g)) {
+        if (v != g.startDs && !edge(g.startDs, v, g).second) {
+            return false;
+        }
+    }
+    return true;
+}
+
 bool isAcyclic(const NGHolder &g) {
     try {
         depth_first_search(
@@ -657,7 +666,8 @@ bool hasCorrectlyNumberedVertices(const NGHolder &g) {
         }
         ids[id] = true;
     }
-    return find(ids.begin(), ids.end(), false) == ids.end();
+    return find(ids.begin(), ids.end(), false) == ids.end()
+        && num_vertices(g) == num_vertices(g.g);
 }
 
 /** Assertion: returns true if the edges in this graph are contiguously (and
@@ -672,8 +682,10 @@ bool hasCorrectlyNumberedEdges(const NGHolder &g) {
         }
         ids[id] = true;
     }
-    return find(ids.begin(), ids.end(), false) == ids.end();
+    return find(ids.begin(), ids.end(), false) == ids.end()
+        && num_edges(g) == num_edges(g.g);
 }
+
 #endif // NDEBUG
 
 } // namespace ue2

src/nfagraph/ng_util.h

@@ -228,6 +228,10 @@ bool isVacuous(const NGHolder &h);
  * proper successors). */
 bool isAnchored(const NGHolder &h);
 
+/** \brief True if the graph contains no anchored vertices (start has no
+ * successors aside from startDs or vertices connected to startDs). */
+bool isFloating(const NGHolder &h);
+
 /** True if the graph contains no back-edges at all, other than the
  * startDs self-loop. */
 bool isAcyclic(const NGHolder &g);

src/nfagraph/ng_violet.h

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2016, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Violet method of rose construction from NGHolder.
+ */
+
+#ifndef NG_VIOLET_H
+#define NG_VIOLET_H
+
+#include "ue2common.h"
+
+namespace ue2 {
+
+class NGHolder;
+class RoseBuild;
+
+struct CompileContext;
+
+/** \brief Attempt to consume the entire pattern in graph \a h with Rose.
+ * Returns true if successful. */
+bool doViolet(RoseBuild &rose, const NGHolder &h, bool prefilter,
+              const CompileContext &cc);
+
+} // namespace ue2
+
+#endif

src/parser/shortcut_literal.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -162,7 +162,7 @@ ConstructLiteralVisitor::~ConstructLiteralVisitor() {}
 bool shortcutLiteral(NG &ng, const ParsedExpression &expr) {
     assert(expr.component);
 
-    if (!ng.cc.grey.allowRose) {
+    if (!ng.cc.grey.allowLiteral) {
         return false;
     }
 

src/rose/rose_build_add.cpp

@@ -651,26 +651,94 @@ floating:
 }
 
 static
-unique_ptr<NGHolder> makeRoseEodPrefix(const NGHolder &h,
+unique_ptr<NGHolder> makeRoseEodPrefix(const NGHolder &h, RoseBuildImpl &build,
-                                       ReportID prefix_report) {
+                                   map<flat_set<ReportID>, ReportID> &remap) {
     assert(generates_callbacks(h));
-    auto g = cloneHolder(h);
+    assert(!in_degree(h.accept, h));
-    g->kind = is_triggered(h) ? NFA_INFIX : NFA_PREFIX;
+    auto gg = cloneHolder(h);
-    setReportId(*g, prefix_report);
+    NGHolder &g = *gg;
+    g.kind = is_triggered(h) ? NFA_INFIX : NFA_PREFIX;
 
     // Move acceptEod edges over to accept.
     vector<NFAEdge> dead;
-    for (const auto &e : in_edges_range(g->acceptEod, *g)) {
+    for (const auto &e : in_edges_range(g.acceptEod, g)) {
-        NFAVertex u = source(e, *g);
+        NFAVertex u = source(e, g);
-        if (u == g->accept) {
+        if (u == g.accept) {
             continue;
         }
-        add_edge_if_not_present(u, g->accept, *g);
+        add_edge_if_not_present(u, g.accept, g);
         dead.push_back(e);
+
+        if (!contains(remap, g[u].reports)) {
+            remap[g[u].reports] = build.getNewNfaReport();
+        }
+
+        g[u].reports = { remap[g[u].reports] };
     }
 
-    remove_edges(dead, *g);
+    remove_edges(dead, g);
-    return g;
+    return gg;
+}
+
+static
+u32 getEodEventID(RoseBuildImpl &build) {
+    // Allocate the EOD event if it hasn't been already.
+    if (build.eod_event_literal_id == MO_INVALID_IDX) {
+        build.eod_event_literal_id = build.getLiteralId({}, 0, ROSE_EVENT);
+    }
+
+    return build.eod_event_literal_id;
+}
+
+static
+void makeEodEventLeftfix(RoseBuildImpl &build, RoseVertex u,
+                         const NGHolder &h) {
+    assert(!build.isInETable(u));
+
+    RoseGraph &g = build.g;
+    map<flat_set<ReportID>, ReportID> report_remap;
+    shared_ptr<NGHolder> eod_leftfix
+        = makeRoseEodPrefix(h, build, report_remap);
+
+    u32 eod_event = getEodEventID(build);
+
+    for (const auto &report_mapping : report_remap) {
+        RoseVertex v = add_vertex(g);
+        g[v].idx = build.vertexIndex++;
+        g[v].literals.insert(eod_event);
+        build.literal_info[eod_event].vertices.insert(v);
+
+        map<u32, set<ReportID> > report_remap;
+        g[v].left.graph = eod_leftfix;
+        g[v].left.leftfix_report = report_mapping.second;
+        g[v].left.lag = 0;
+        RoseEdge e1 = add_edge(u, v, g).first;
+        g[e1].minBound = 0;
+        g[e1].maxBound = ROSE_BOUND_INF;
+        g[v].min_offset = add_rose_depth(g[u].min_offset,
+                                         findMinWidth(*g[v].left.graph));
+        g[v].max_offset = ROSE_BOUND_INF;
+
+        depth max_width = findMaxWidth(*g[v].left.graph);
+        if (u != build.root && max_width.is_finite()
+            && (!build.isAnyStart(u) || isPureAnchored(*g[v].left.graph))) {
+            g[e1].maxBound = max_width;
+            g[v].max_offset = add_rose_depth(g[u].max_offset, max_width);
+        }
+
+        g[e1].history = ROSE_ROLE_HISTORY_NONE; // handled by prefix
+        RoseVertex w = add_vertex(g);
+        g[w].idx = build.vertexIndex++;
+        g[w].eod_accept = true;
+        g[w].reports = report_mapping.first;
+        g[w].min_offset = g[v].min_offset;
+        g[w].max_offset = g[v].max_offset;
+        RoseEdge e = add_edge(v, w, g).first;
+        g[e].minBound = 0;
+        g[e].maxBound = 0;
+        g[e].history = ROSE_ROLE_HISTORY_LAST_BYTE;
+        DEBUG_PRINTF("accept eod vertex (idx=%zu)\n", g[w].idx);
+    }
 }
 
 static
@@ -686,8 +754,20 @@ void doRoseAcceptVertex(RoseBuildImpl *tbi,
         RoseVertex u = pv.first;
         const RoseInEdgeProps &edge_props = bd.ig[pv.second];
 
+        /* We need to duplicate the parent vertices if:
+         *
+         * 1) It already has a suffix, etc as we are going to add the specified
+         * suffix, etc to the parents and we do not want to overwrite the
+         * existing information.
+         *
+         * 2) We are making the an EOD accept and the vertex already has other
+         * out-edges - The LAST_BYTE history used for EOD accepts is
+         * incompatible with normal successors. As accepts are processed last we
+         * do not need to worry about other normal successors being added later.
+         */
         if (g[u].suffix || !g[u].reports.empty()
-             /* also poss accept eod edge: TODO check properly */
+            || (ig[iv].type == RIV_ACCEPT_EOD && out_degree(u, g)
+                && !edge_props.graph)
             || (!isLeafNode(u, g) && !tbi->isAnyStart(u))) {
             DEBUG_PRINTF("duplicating for parent %zu\n", g[u].idx);
             assert(!tbi->isAnyStart(u));
@@ -719,74 +799,37 @@ void doRoseAcceptVertex(RoseBuildImpl *tbi,
             }
         } else {
             assert(ig[iv].type == RIV_ACCEPT_EOD);
+            assert(!edge_props.haig);
 
-            if (edge_props.graph && tbi->isInETable(u)) {
+            if (!edge_props.graph) {
+                RoseVertex w = add_vertex(g);
+                g[w].idx = tbi->vertexIndex++;
+                g[w].eod_accept = true;
+                g[w].reports = ig[iv].reports;
+                g[w].min_offset = g[u].min_offset;
+                g[w].max_offset = g[u].max_offset;
+                RoseEdge e = add_edge(u, w, g).first;
+                g[e].minBound = 0;
+                g[e].maxBound = 0;
+                g[e].history = ROSE_ROLE_HISTORY_LAST_BYTE;
+                DEBUG_PRINTF("accept eod vertex (idx=%zu)\n", g[w].idx);
+                continue;
+            }
+
+            const NGHolder &h = *edge_props.graph;
+            assert(!in_degree(h.accept, h));
+            assert(generates_callbacks(h));
+
+            if (tbi->isInETable(u)) {
+                assert(h.kind == NFA_SUFFIX);
                 assert(!tbi->isAnyStart(u));
                 /* etable can't/shouldn't use eod event */
                 DEBUG_PRINTF("adding suffix to i%zu\n", g[u].idx);
                 g[u].suffix.graph = edge_props.graph;
-                assert(g[u].suffix.graph->kind == NFA_SUFFIX);
-                dumpHolder(*g[u].suffix.graph, 98, "eod_suffix", tbi->cc.grey);
-                assert(!in_degree(g[u].suffix.graph->accept,
-                                  *g[u].suffix.graph));
-                set<ReportID> reports = all_reports(*g[u].suffix.graph);
-                tbi->rm.getReport(*reports.begin());
-                assert(reports.size() == 1);
-                /* TODO: set dfa_(min|max)_width */
                 continue;
-            } else if (edge_props.graph) {
-                assert(!edge_props.haig);
-                assert(!tbi->isInETable(u));
-
-                // Allocate the EOD event if it hasn't been already.
-                if (tbi->eod_event_literal_id == MO_INVALID_IDX) {
-                    tbi->eod_event_literal_id =
-                        tbi->getLiteralId(ue2_literal(), 0, ROSE_EVENT);
-                }
-
-                RoseVertex v = add_vertex(g);
-                g[v].idx = tbi->vertexIndex++;
-                g[v].literals.insert(tbi->eod_event_literal_id);
-                tbi->literal_info[tbi->eod_event_literal_id].vertices.insert(v);
-
-                ReportID prefix_report = tbi->getNewNfaReport();
-                g[v].left.graph
-                    = makeRoseEodPrefix(*edge_props.graph, prefix_report);
-                g[v].left.leftfix_report = prefix_report;
-                g[v].left.lag = 0;
-                RoseEdge e1 = add_edge(u, v, g).first;
-                g[e1].minBound = 0;
-                g[e1].maxBound = ROSE_BOUND_INF;
-                g[v].min_offset = add_rose_depth(
-                        g[u].min_offset, findMinWidth(*g[v].left.graph));
-                g[v].max_offset = ROSE_BOUND_INF;
-
-                DEBUG_PRINTF("hi\n");
-                depth max_width = findMaxWidth(*g[v].left.graph);
-                if (u != tbi->root
-                    && max_width.is_finite()
-                    && (!tbi->isAnyStart(u)
-                        || isPureAnchored(*g[v].left.graph))) {
-                    g[e1].maxBound = max_width;
-                    g[v].max_offset = add_rose_depth(g[u].max_offset, max_width);
-                }
-
-                g[e1].history = ROSE_ROLE_HISTORY_NONE; // handled by prefix
-                u = v;
             }
-            assert(!edge_props.haig);
 
-            RoseVertex w = add_vertex(g);
+            makeEodEventLeftfix(*tbi, u, h);
-            g[w].idx = tbi->vertexIndex++;
-            g[w].eod_accept = true;
-            g[w].reports = ig[iv].reports;
-            g[w].min_offset = g[u].min_offset;
-            g[w].max_offset = g[u].max_offset;
-            RoseEdge e = add_edge(u, w, g).first;
-            g[e].minBound = 0;
-            g[e].maxBound = 0;
-            g[e].history = ROSE_ROLE_HISTORY_LAST_BYTE;
-            DEBUG_PRINTF("accept eod vertex (idx=%zu)\n", g[w].idx);
         }
     }
 }
@@ -887,7 +930,8 @@ bool suitableForEod(const RoseInGraph &ig, vector<RoseInVertex> topo,
             ENSURE_AT_LEAST(&v_depth, (u32)max_width);
         }
 
-        if (v_depth == ROSE_BOUND_INF || v_depth > cc.grey.maxHistoryAvailable) {
+        if (v_depth == ROSE_BOUND_INF
+            || v_depth > cc.grey.maxHistoryAvailable) {
             DEBUG_PRINTF("not suitable for eod table %u\n", v_depth);
             return false;
         }
@@ -900,6 +944,13 @@ bool suitableForEod(const RoseInGraph &ig, vector<RoseInVertex> topo,
     return true;
 }
 
+static
+void shift_accepts_to_end(const RoseInGraph &ig,
+                          vector<RoseInVertex> &topo_order) {
+    stable_partition(begin(topo_order), end(topo_order),
+                     [&](RoseInVertex v){ return !is_any_accept(v, ig); });
+}
+
 static
 void populateRoseGraph(RoseBuildImpl *tbi, RoseBuildData &bd) {
     const RoseInGraph &ig = bd.ig;
@@ -912,6 +963,7 @@ void populateRoseGraph(RoseBuildImpl *tbi, RoseBuildData &bd) {
     map<RoseInVertex, vector<RoseVertex> > vertex_map;
 
     vector<RoseInVertex> v_order = topo_order(ig);
+    shift_accepts_to_end(ig, v_order);
 
     u32 eod_space_required;
     bool use_eod_table = suitableForEod(ig, v_order, &eod_space_required,

src/rose/rose_build_add_mask.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -336,7 +336,8 @@ void buildLiteralMask(const vector<CharReach> &mask, vector<u8> &msk,
 }
 
 static
-bool validateTransientMask(const vector<CharReach> &mask, bool eod, const Grey &grey) {
+bool validateTransientMask(const vector<CharReach> &mask, bool anchored,
+                           bool eod, const Grey &grey) {
     assert(!mask.empty());
 
     // An EOD anchored mask requires that everything fit into history, while an
@@ -348,6 +349,12 @@ bool validateTransientMask(const vector<CharReach> &mask, bool eod, const Grey &
         return false;
     }
 
+    /* although anchored masks cannot be transient, short masks may be placed
+     * into the atable. */
+    if (anchored && mask.size() > grey.maxAnchoredRegion) {
+        return false;
+    }
+
     vector<ue2_literal> lits;
     u32 lit_minBound; /* minBound of each literal in lit */
     u32 lit_length;   /* length of each literal in lit */
@@ -703,7 +710,7 @@ bool checkAllowMask(const vector<CharReach> &mask, ue2_literal *lit,
 
 bool RoseBuildImpl::add(bool anchored, const vector<CharReach> &mask,
                         const ue2::flat_set<ReportID> &reports) {
-    if (validateTransientMask(mask, false, cc.grey)) {
+    if (validateTransientMask(mask, anchored, false, cc.grey)) {
         bool eod = false;
         addTransientMask(*this, mask, reports, anchored, eod);
         return true;
@@ -726,8 +733,8 @@ bool RoseBuildImpl::add(bool anchored, const vector<CharReach> &mask,
 
 bool RoseBuildImpl::validateMask(const vector<CharReach> &mask,
                                  UNUSED const ue2::flat_set<ReportID> &reports,
-                                 UNUSED bool anchored, bool eod) const {
+                                 bool anchored, bool eod) const {
-    return validateTransientMask(mask, eod, cc.grey);
+    return validateTransientMask(mask, anchored, eod, cc.grey);
 }
 
 static

src/rose/rose_build_compile.cpp

@@ -433,6 +433,9 @@ RoseRoleHistory findHistoryScheme(const RoseBuildImpl &tbi, const RoseEdge &e) {
 
         // If the bounds are {0,0}, this role can only match precisely at EOD.
         if (minBound == 0 && maxBound == 0) {
+            /* last byte history will squash the state byte so cannot have other
+             * succ */
+            assert(out_degree(u, g) == 1);
             return ROSE_ROLE_HISTORY_LAST_BYTE;
         }
 
@@ -915,19 +918,32 @@ void RoseBuildImpl::findTransientLeftfixes(void) {
             continue;
         }
 
-        u32 his = g[v].left.lag + max_width;
+        if (cc.streaming) {
+            /* STREAMING: transient prefixes must be able to run using history
+             * rather than storing state. */
+            u32 his = g[v].left.lag + max_width;
 
-        // If this vertex has an event literal, we need to add one to cope
+            // If this vertex has an event literal, we need to add one to cope
-        // with it.
+            // with it.
-        if (hasLiteralInTable(v, ROSE_EVENT)) {
+            if (hasLiteralInTable(v, ROSE_EVENT)) {
-            his++;
+                his++;
-        }
+            }
 
-        /* +1 as trigger must appear in main buffer and no byte is needed to
+            /* +1 as trigger must appear in main buffer and no byte is needed to
-         * decompress the state */
+             * decompress the state */
-        if (his <= cc.grey.maxHistoryAvailable + 1) {
+            if (his <= cc.grey.maxHistoryAvailable + 1) {
-            transient.insert(left);
+                transient.insert(left);
-            DEBUG_PRINTF("a transient leftfix has been spotted his=%u\n", his);
+                DEBUG_PRINTF("a transient leftfix spotted his=%u\n", his);
+            }
+        } else {
+            /* BLOCK: transientness is less important and more fuzzy, ideally
+             * it should be quick to calculate the state. No need to worry about
+             * history (and hence lag). */
+            if (max_width < depth(ROSE_BLOCK_TRANSIENT_MAX_WIDTH)) {
+                transient.insert(left);
+                DEBUG_PRINTF("a transient block leftfix spotted [%u]\n",
+                             (u32)max_width);
+            }
         }
     }
 }

src/rose/rose_build_dump.cpp

@@ -61,28 +61,6 @@ using namespace std;
 
 namespace ue2 {
 
-static
-string to_string(nfa_kind k) {
-    switch (k) {
-    case NFA_PREFIX:
-        return "PREFIX";
-    case NFA_INFIX:
-        return "INFIX";
-    case NFA_SUFFIX:
-        return "SUFFIX";
-    case NFA_OUTFIX:
-        return "OUTFIX";
-    case NFA_REV_PREFIX:
-        return "REV_PREFIX";
-    case NFA_OUTFIX_RAW:
-        return "OUTFIX_RAW";
-    case NFA_EAGER_PREFIX:
-        return "EAGER_PREFIX";
-    }
-    assert(0);
-    return "?";
-}
-
 /** \brief Return the kind of a left_id or a suffix_id. */
 template<class Graph>
 string render_kind(const Graph &g) {

src/rose/rose_build_lookaround.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -684,6 +684,10 @@ bool makeLeftfixLookaround(const RoseBuildImpl &build, const RoseVertex v,
 
     lookaround.reserve(look.size());
     for (const auto &m : look) {
+        if (m.first < -128 || m.first > 127) {
+            DEBUG_PRINTF("range too big\n");
+            return false;
+        }
         s8 offset = verify_s8(m.first);
         lookaround.emplace_back(offset, m.second);
     }

src/rose/rose_build_util.h

@@ -36,6 +36,9 @@
 
 namespace ue2 {
 
+/** Max allowed width for transient graphs in block mode */
+#define ROSE_BLOCK_TRANSIENT_MAX_WIDTH 255U
+
 // Comparator for vertices using their index property.
 struct VertexIndexComp {
     VertexIndexComp(const RoseGraph &gg) : g(gg) {}

src/rose/rose_in_dump.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -107,7 +107,8 @@ void dumpPreRoseGraph(const RoseInGraph &ig, const Grey &grey,
                 size_t id = graph_ids.size();
                 graph_ids[&*ig[e].graph] = id;
             }
-            fprintf(f, "graph %zu", graph_ids[&*ig[e].graph]);
+            fprintf(f, "graph %zu\n%s", graph_ids[&*ig[e].graph],
+                    to_string(ig[e].graph->kind).c_str());
         }
         if (ig[e].haig) {
             fprintf(f, "haig ");

src/rose/rose_in_graph.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -106,6 +106,12 @@ public:
                                  ROSE_BOUND_INF);
     }
 
+    /* for when there is a suffix graph which handles the reports */
+    static RoseInVertexProps makeAcceptEod() {
+        return RoseInVertexProps(RIV_ACCEPT_EOD, ue2_literal(), 0,
+                                 ROSE_BOUND_INF);
+    }
+
     static RoseInVertexProps makeStart(bool anchored) {
         DEBUG_PRINTF("making %s\n", anchored ? "anchored start" : "start");
         if (anchored) {

src/rose/rose_in_util.h

@@ -46,6 +46,11 @@ void calcVertexOffsets(RoseInGraph &ig);
 enum nfa_kind whatRoseIsThis(const RoseInGraph &in, const RoseInEdge &e);
 void pruneUseless(RoseInGraph &g);
 
+inline
+bool is_any_accept(RoseInVertex v, const RoseInGraph &g) {
+    return g[v].type == RIV_ACCEPT || g[v].type == RIV_ACCEPT_EOD;
+}
+
 }
 
 #endif

src/rose/stream.c

@@ -510,7 +510,8 @@ void runEagerPrefixesStream(const struct RoseEngine *t,
 }
 
 void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch) {
-    DEBUG_PRINTF("OH HAI\n");
+    DEBUG_PRINTF("OH HAI [%llu, %llu)\n", scratch->core_info.buf_offset,
+                 scratch->core_info.buf_offset + (u64a)scratch->core_info.len);
     assert(t);
     assert(scratch->core_info.hbuf);
     assert(scratch->core_info.buf);

src/scratch.h

@@ -45,7 +45,7 @@ extern "C"
 #endif
 
 UNUSED static const u32 SCRATCH_MAGIC = 0x544F4259;
-#define FDR_TEMP_BUF_SIZE 200
+#define FDR_TEMP_BUF_SIZE 220
 
 struct fatbit;
 struct hs_scratch;
@@ -141,7 +141,6 @@ struct match_deduper {
 struct ALIGN_CL_DIRECTIVE hs_scratch {
     u32 magic;
     u8 in_use; /**< non-zero when being used by an API call. */
-    char *scratch_alloc; /* user allocated scratch object */
     u32 queueCount;
     u32 bStateSize; /**< sizeof block mode states */
     u32 tStateSize; /**< sizeof transient rose states */
@@ -161,10 +160,6 @@ struct ALIGN_CL_DIRECTIVE hs_scratch {
     struct match_deduper deduper;
     u32 anchored_literal_region_len;
     u32 anchored_literal_count;
-    u32 delay_count;
-    u32 scratchSize;
-    u8 ALIGN_DIRECTIVE fdr_temp_buf[FDR_TEMP_BUF_SIZE];
-    u32 handledKeyCount;
     struct fatbit *handled_roles; /**< fatbit of ROLES (not states) already
                                    * handled by this literal */
     u64a *som_store; /**< array of som locations */
@@ -176,6 +171,11 @@ struct ALIGN_CL_DIRECTIVE hs_scratch {
                             * location had been writable */
     u64a som_set_now_offset; /**< offset at which som_set_now represents */
     u32 som_store_count;
+    u32 handledKeyCount;
+    u32 delay_count;
+    u32 scratchSize;
+    char *scratch_alloc; /* user allocated scratch object */
+    u8 ALIGN_DIRECTIVE fdr_temp_buf[FDR_TEMP_BUF_SIZE];
 };
 
 /* array of fatbit ptr; TODO: why not an array of fatbits? */

src/util/ue2string.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -55,8 +55,6 @@ size_t maxStringSelfOverlap(const std::string &a, bool nocase);
 /// Compares two strings, returns non-zero if they're different.
 u32 cmp(const char *a, const char *b, size_t len, bool nocase);
 
-class CharReach;
-
 struct ue2_literal {
 public:
     /// Single element proxy, pointed to by our const_iterator.
@@ -124,6 +122,13 @@ public:
     ue2_literal &operator=(const ue2_literal &) = default;
     ue2_literal &operator=(ue2_literal &&) = default;
 
+    template<typename InputIt>
+    ue2_literal(InputIt b, InputIt e) {
+        for (; b != e; ++b) {
+            push_back(*b);
+        }
+    }
+
     size_type length() const { return s.length(); }
     bool empty() const { return s.empty(); }
     ue2_literal substr(size_type pos, size_type n = std::string::npos) const;