Initial commit of Hyperscan

src/compiler/asserts.cpp

@@ -0,0 +1,310 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Convert temporary assert vertices (from construction method) to
+ * edge-based flags.
+ *
+ * This pass converts the temporary assert vertices created by the Glushkov
+ * construction process above (vertices with special assertions flags) into
+ * edges between those vertices' neighbours in the graph.
+ *
+ * These edges have the appropriate flags applied to them -- a path (u,t,v)
+ * through an assert vertex t will be replaced with the edge (u,v) with the
+ * assertion flags from t.
+ *
+ * Edges with mutually incompatible flags (such as the conjunction of
+ * word-to-word and word-to-nonword) are dropped.
+ */
+#include "asserts.h"
+#include "nfagraph/ng.h"
+#include "nfagraph/ng_prune.h"
+#include "nfagraph/ng_redundancy.h"
+#include "nfagraph/ng_util.h"
+#include "parser/position.h" // for POS flags
+#include "util/compile_error.h"
+#include "util/graph_range.h"
+
+#include <queue>
+#include <set>
+
+using namespace std;
+
+namespace ue2 {
+
+/** Hard limit on the maximum number of edges we'll clone before we throw up
+ * our hands and report 'Pattern too large.' */
+static const size_t MAX_ASSERT_EDGES = 300000;
+
+/** Flags representing the word-boundary assertions, \\b or \\B. */
+static const int WORDBOUNDARY_FLAGS = POS_FLAG_ASSERT_WORD_TO_WORD
+                                    | POS_FLAG_ASSERT_WORD_TO_NONWORD
+                                    | POS_FLAG_ASSERT_NONWORD_TO_WORD
+                                    | POS_FLAG_ASSERT_NONWORD_TO_NONWORD
+                                    | POS_FLAG_ASSERT_WORD_TO_WORD_UCP
+                                    | POS_FLAG_ASSERT_WORD_TO_NONWORD_UCP
+                                    | POS_FLAG_ASSERT_NONWORD_TO_WORD_UCP
+                                    | POS_FLAG_ASSERT_NONWORD_TO_NONWORD_UCP;
+
+#define OPEN_EDGE 0U
+#define DEAD_EDGE (~0U)
+
+static
+u32 disjunct(u32 flags1, u32 flags2) {
+    /* from two asserts in parallel */
+    DEBUG_PRINTF("disjunct %x %x\n", flags1, flags2);
+    u32 rv;
+    if (flags1 == DEAD_EDGE) {
+        rv = flags2;
+    } else if (flags2 == DEAD_EDGE) {
+        rv = flags1;
+    } else if (flags1 == OPEN_EDGE || flags2 == OPEN_EDGE) {
+        rv = OPEN_EDGE;
+    } else {
+        rv = flags1 | flags2;
+    }
+    DEBUG_PRINTF("--> %x\n", rv);
+    return rv;
+}
+
+static
+u32 conjunct(u32 flags1, u32 flags2) {
+    /* from two asserts in series */
+    DEBUG_PRINTF("conjunct %x %x\n", flags1, flags2);
+    u32 rv;
+    if (flags1 == OPEN_EDGE) {
+        rv = flags2;
+    } else if (flags2 == OPEN_EDGE) {
+        rv = flags1;
+    } else if (flags1 & flags2) {
+        rv = flags1 & flags2;
+    } else {
+        rv = DEAD_EDGE; /* the conjunction of two different word boundary
+                         * assertion is impassable */
+    }
+
+    DEBUG_PRINTF("--> %x\n", rv);
+    return rv;
+}
+
+typedef map<pair<NFAVertex, NFAVertex>, NFAEdge> edge_cache_t;
+
+static
+void replaceAssertVertex(NGWrapper &g, NFAVertex t, edge_cache_t &edge_cache,
+                         u32 &assert_edge_count) {
+    DEBUG_PRINTF("replacing assert vertex %u\n", g[t].index);
+
+    const u32 flags = g[t].assert_flags;
+    DEBUG_PRINTF("consider assert vertex %u with flags %u\n",
+                 g[t].index, flags);
+
+    // Wire up all the predecessors to all the successors.
+
+    for (const auto &inEdge : in_edges_range(t, g)) {
+        NFAVertex u = source(inEdge, g);
+        if (u == t) {
+            continue; // ignore self-loops
+        }
+
+        const u32 flags_inc_in = conjunct(g[inEdge].assert_flags,
+                                          flags);
+        if (flags_inc_in == DEAD_EDGE) {
+            DEBUG_PRINTF("fail, in-edge has bad flags %d\n",
+                         g[inEdge].assert_flags);
+            continue;
+        }
+
+        for (const auto &outEdge : out_edges_range(t, g)) {
+            NFAVertex v = target(outEdge, g);
+
+            DEBUG_PRINTF("consider path [%u,%u,%u]\n", g[u].index,
+                         g[t].index, g[v].index);
+
+            if (v == t) {
+                continue; // ignore self-loops
+            }
+
+            const u32 flags_final = conjunct(g[outEdge].assert_flags,
+                                             flags_inc_in);
+
+            if (flags_final == DEAD_EDGE) {
+                DEBUG_PRINTF("fail, out-edge has bad flags %d\n",
+                             g[outEdge].assert_flags);
+                continue;
+            }
+
+            if ((g[u].assert_flags & POS_FLAG_MULTILINE_START)
+                && v == g.acceptEod) {
+                DEBUG_PRINTF("fail, (?m)^ does not match \\n at eod\n");
+                continue;
+            }
+
+            /* Replace path (u,t,v) with direct edge (u,v), unless the edge
+             * already exists, in which case we just need to edit its
+             * properties.
+             *
+             * Use edge_cache to prevent us going O(N).
+             */
+            auto cache_key = make_pair(u, v);
+            auto ecit = edge_cache.find(cache_key);
+            if (ecit == edge_cache.end()) {
+                DEBUG_PRINTF("adding edge %u %u\n", g[u].index,
+                              g[v].index);
+                NFAEdge e = add_edge(u, v, g).first;
+                edge_cache.emplace(cache_key, e);
+                g[e].assert_flags = flags;
+                if (++assert_edge_count > MAX_ASSERT_EDGES) {
+                    throw CompileError(g.expressionIndex,
+                                       "Pattern is too large.");
+                }
+            } else {
+                NFAEdge e = ecit->second;
+                DEBUG_PRINTF("updating edge %u %u [a %u]\n", g[u].index,
+                             g[v].index, g[t].index);
+                // Edge already exists.
+                u32 &e_flags = g[e].assert_flags;
+                e_flags = disjunct(e_flags, flags_final);
+                assert(e_flags != DEAD_EDGE);
+            }
+        }
+    }
+
+    // Clear vertex t to remove all the old edges.
+    /* no need to clear the cache, as we will never look up its edge as it is
+     * unreachable */
+    clear_vertex(t, g);
+}
+
+static
+void setReportId(ReportManager &rm, NGWrapper &g, NFAVertex v, s32 adj) {
+    // Don't try and set the report ID of a special vertex.
+    assert(!is_special(v, g));
+
+    // There should be no reports set already.
+    assert(g[v].reports.empty());
+
+    Report r = rm.getBasicInternalReport(g, adj);
+
+    g[v].reports.insert(rm.getInternalId(r));
+    DEBUG_PRINTF("set report id for vertex %u, adj %d\n",
+                 g[v].index, adj);
+}
+
+static
+void checkForMultilineStart(ReportManager &rm, NGWrapper &g) {
+    vector<NFAEdge> dead;
+    for (auto v : adjacent_vertices_range(g.start, g)) {
+        if (!(g[v].assert_flags & POS_FLAG_MULTILINE_START)) {
+            continue;
+        }
+        DEBUG_PRINTF("mls %u %08x\n", g[v].index,
+                     g[v].assert_flags);
+
+        /* we have found a multi-line start (maybe more than one) */
+
+        /* we need to interpose a dummy dot vertex between v and accept if
+         * required so that ^ doesn't match trailing \n */
+         for (const auto &e : out_edges_range(v, g)) {
+            if (target(e, g) == g.accept) {
+                dead.push_back(e);
+            }
+        }
+        /* assert has been resolved; clear flag */
+        g[v].assert_flags &= ~POS_FLAG_MULTILINE_START;
+    }
+
+    for (const auto &e : dead) {
+        NFAVertex dummy = add_vertex(g);
+        g[dummy].char_reach.setall();
+        setReportId(rm, g, dummy, -1);
+        add_edge(source(e, g), dummy, g[e], g);
+        add_edge(dummy, g.accept, g);
+    }
+
+    remove_edges(dead, g);
+}
+
+static
+bool hasAssertVertices(const NGHolder &g) {
+    for (auto v : vertices_range(g)) {
+        int flags = g[v].assert_flags;
+        if (flags & WORDBOUNDARY_FLAGS) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/** \brief Convert temporary assert vertices (from construction method) to
+ * edge-based flags.
+ *
+ * Remove the horrors that are the temporary assert vertices which arise from
+ * our construction method. Allows the rest of our code base to live in
+ * blissful ignorance of their existence. */
+void removeAssertVertices(ReportManager &rm, NGWrapper &g) {
+    size_t num = 0;
+
+    DEBUG_PRINTF("before: graph has %zu vertices\n", num_vertices(g));
+
+    // Sweep over the graph and ascertain that we do actually have vertices
+    // with assertion flags set. Otherwise, we're done.
+    if (!hasAssertVertices(g)) {
+        DEBUG_PRINTF("no assert vertices, done\n");
+        return;
+    }
+
+    u32 assert_edge_count = 0;
+
+    // Build a cache of (u, v) vertex pairs to edge descriptors.
+    edge_cache_t edge_cache;
+    for (const auto &e : edges_range(g)) {
+        edge_cache[make_pair(source(e, g), target(e, g))] = e;
+    }
+
+    for (auto v : vertices_range(g)) {
+        if (g[v].assert_flags & WORDBOUNDARY_FLAGS) {
+            replaceAssertVertex(g, v, edge_cache, assert_edge_count);
+            num++;
+        }
+    }
+
+    checkForMultilineStart(rm, g);
+
+    if (num) {
+        DEBUG_PRINTF("resolved %zu assert vertices\n", num);
+        pruneUseless(g);
+        pruneEmptyVertices(g);
+        g.renumberVertices();
+        g.renumberEdges();
+    }
+
+    DEBUG_PRINTF("after: graph has %zu vertices\n", num_vertices(g));
+    assert(!hasAssertVertices(g));
+}
+
+} // namespace ue2

src/compiler/asserts.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Convert temporary assert vertices (from construction method) to
+ * edge-based flags.
+ */
+#ifndef ASSERTS_H
+#define ASSERTS_H
+
+namespace ue2 {
+
+class ReportManager;
+class NGWrapper;
+
+/** \brief Convert temporary assert vertices (from construction method) to
+ * edge-based flags.
+ *
+ * Remove the horrors that are the temporary assert vertices which arise from
+ * our construction method. Allows the rest of our code base to live in
+ * blissful ignorance of their existence. */
+void removeAssertVertices(ReportManager &rm, NGWrapper &g);
+
+} // namespace ue2
+
+#endif // ASSERTS_H

src/compiler/compiler.cpp

@@ -0,0 +1,459 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Compiler front-end interface.
+ */
+#include "asserts.h"
+#include "compiler.h"
+#include "database.h"
+#include "grey.h"
+#include "hs_internal.h"
+#include "hs_runtime.h"
+#include "ue2common.h"
+#include "nfagraph/ng_builder.h"
+#include "nfagraph/ng_dump.h"
+#include "nfagraph/ng.h"
+#include "nfagraph/ng_util.h"
+#include "parser/buildstate.h"
+#include "parser/dump.h"
+#include "parser/Component.h"
+#include "parser/parse_error.h"
+#include "parser/Parser.h"          // for flags
+#include "parser/position.h"
+#include "parser/position_dump.h"
+#include "parser/position_info.h"
+#include "parser/prefilter.h"
+#include "parser/shortcut_literal.h"
+#include "parser/unsupported.h"
+#include "parser/utf8_validate.h"
+#include "smallwrite/smallwrite_build.h"
+#include "rose/rose_build.h"
+#include "rose/rose_build_dump.h"
+#include "som/slot_manager_dump.h"
+#include "util/alloc.h"
+#include "util/compile_error.h"
+#include "util/target_info.h"
+#include "util/verify_types.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <memory>
+#include <sstream>
+
+using namespace std;
+
+namespace ue2 {
+
+
+static
+void validateExt(const hs_expr_ext &ext) {
+    static const unsigned long long ALL_EXT_FLAGS = HS_EXT_FLAG_MIN_OFFSET |
+                                                    HS_EXT_FLAG_MAX_OFFSET |
+                                                    HS_EXT_FLAG_MIN_LENGTH;
+    if (ext.flags & ~ALL_EXT_FLAGS) {
+        throw CompileError("Invalid hs_expr_ext flag set.");
+    }
+
+    if ((ext.flags & HS_EXT_FLAG_MIN_OFFSET) &&
+        (ext.flags & HS_EXT_FLAG_MAX_OFFSET) &&
+        (ext.min_offset > ext.max_offset)) {
+        throw CompileError("In hs_expr_ext, min_offset must be less than or "
+                           "equal to max_offset.");
+    }
+
+    if ((ext.flags & HS_EXT_FLAG_MIN_LENGTH) &&
+        (ext.flags & HS_EXT_FLAG_MAX_OFFSET) &&
+        (ext.min_length > ext.max_offset)) {
+        throw CompileError("In hs_expr_ext, min_length must be less than or "
+                           "equal to max_offset.");
+    }
+}
+
+ParsedExpression::ParsedExpression(unsigned index_in, const char *expression,
+                                   unsigned flags, ReportID actionId,
+                                   const hs_expr_ext *ext)
+    : utf8(false),
+      allow_vacuous(flags & HS_FLAG_ALLOWEMPTY),
+      highlander(flags & HS_FLAG_SINGLEMATCH),
+      prefilter(flags & HS_FLAG_PREFILTER),
+      som(SOM_NONE),
+      index(index_in),
+      id(actionId),
+      min_offset(0),
+      max_offset(MAX_OFFSET),
+      min_length(0) {
+    ParseMode mode(flags);
+
+    component = parse(expression, mode);
+
+    utf8 = mode.utf8; /* utf8 may be set by parse() */
+
+    if (utf8 && !isValidUtf8(expression)) {
+        throw ParseError("Expression is not valid UTF-8.");
+    }
+
+    if (!component) {
+        assert(0); // parse() should have thrown a ParseError.
+        throw ParseError("Parse error.");
+    }
+
+    if (flags & ~HS_FLAG_ALL) {
+        DEBUG_PRINTF("Unrecognised flag, flags=%u.\n", flags);
+        throw CompileError("Unrecognised flag.");
+    }
+
+    // FIXME: we disallow highlander + SOM, see UE-1850.
+    if ((flags & HS_FLAG_SINGLEMATCH) && (flags & HS_FLAG_SOM_LEFTMOST)) {
+        throw CompileError("HS_FLAG_SINGLEMATCH is not supported in "
+                           "combination with HS_FLAG_SOM_LEFTMOST.");
+    }
+
+    // FIXME: we disallow prefilter + SOM, see UE-1899.
+    if ((flags & HS_FLAG_PREFILTER) && (flags & HS_FLAG_SOM_LEFTMOST)) {
+        throw CompileError("HS_FLAG_PREFILTER is not supported in "
+                           "combination with HS_FLAG_SOM_LEFTMOST.");
+    }
+
+    // Set SOM type.
+    if (flags & HS_FLAG_SOM_LEFTMOST) {
+        som = SOM_LEFT;
+    }
+
+    // Set extended parameters, if we have them.
+    if (ext) {
+        // Ensure that the given parameters make sense.
+        validateExt(*ext);
+
+        if (ext->flags & HS_EXT_FLAG_MIN_OFFSET) {
+            min_offset = ext->min_offset;
+        }
+        if (ext->flags & HS_EXT_FLAG_MAX_OFFSET) {
+            max_offset = ext->max_offset;
+        }
+        if (ext->flags & HS_EXT_FLAG_MIN_LENGTH) {
+            min_length = ext->min_length;
+        }
+    }
+
+    // These are validated in validateExt, so an error will already have been
+    // thrown if these conditions don't hold.
+    assert(max_offset >= min_offset);
+    assert(max_offset >= min_length);
+
+    // Since prefiltering and SOM aren't supported together, we must squash any
+    // min_length constraint as well.
+    if (flags & HS_FLAG_PREFILTER && min_length) {
+        DEBUG_PRINTF("prefiltering mode: squashing min_length constraint\n");
+        min_length = 0;
+    }
+}
+
+#if defined(DUMP_SUPPORT) || defined(DEBUG)
+/**
+ * \brief Dumps the parse tree to screen in debug mode and to disk in dump
+ * mode.
+ */
+void dumpExpression(UNUSED const ParsedExpression &expr,
+                    UNUSED const char *stage, UNUSED const Grey &grey) {
+#if defined(DEBUG)
+    DEBUG_PRINTF("===== Rule ID: %u (internalID:  %u) =====\n", expr.id,
+                 expr.index);
+    ostringstream debug_tree;
+    dumpTree(debug_tree, expr.component.get());
+    printf("%s\n", debug_tree.str().c_str());
+#endif // DEBUG
+
+#if defined(DUMP_SUPPORT)
+    if (grey.dumpFlags & Grey::DUMP_PARSE) {
+        stringstream ss;
+        ss << grey.dumpPath << "Expr_" << expr.index << "_componenttree_"
+           << stage << ".txt";
+        ofstream out(ss.str().c_str());
+        out << "Component Tree for " << expr.id << endl;
+        dumpTree(out, expr.component.get());
+        if (expr.utf8) {
+            out << "UTF8 mode" << endl;
+        }
+    }
+#endif // DEBUG
+}
+#endif
+
+/** \brief Run Component tree optimisations on \a expr. */
+static
+void optimise(ParsedExpression &expr) {
+    if (expr.min_length || expr.som) {
+        return;
+    }
+
+    DEBUG_PRINTF("optimising\n");
+    expr.component->optimise(true /* root is connected to sds */);
+}
+
+void addExpression(NG &ng, unsigned index, const char *expression,
+                   unsigned flags, const hs_expr_ext *ext, ReportID id) {
+    assert(expression);
+    const CompileContext &cc = ng.cc;
+    DEBUG_PRINTF("index=%u, id=%u, flags=%u, expr='%s'\n", index, id, flags,
+                 expression);
+
+    // Ensure that our pattern isn't too long (in characters).
+    if (strlen(expression) > cc.grey.limitPatternLength) {
+        throw CompileError("Pattern length exceeds limit.");
+    }
+
+    // Do per-expression processing: errors here will result in an exception
+    // being thrown up to our caller
+    ParsedExpression expr(index, expression, flags, id, ext);
+    dumpExpression(expr, "orig", cc.grey);
+
+    // Apply prefiltering transformations if desired.
+    if (expr.prefilter) {
+        prefilterTree(expr.component, ParseMode(flags));
+        dumpExpression(expr, "prefiltered", cc.grey);
+    }
+
+    // Expressions containing zero-width assertions and other extended pcre
+    // types aren't supported yet. This call will throw a ParseError exception
+    // if the component tree contains such a construct.
+    checkUnsupported(*expr.component);
+
+    expr.component->checkEmbeddedStartAnchor(true);
+    expr.component->checkEmbeddedEndAnchor(true);
+
+    if (cc.grey.optimiseComponentTree) {
+        optimise(expr);
+        dumpExpression(expr, "opt", cc.grey);
+    }
+
+    DEBUG_PRINTF("component=%p, nfaId=%u, reportId=%u\n",
+                 expr.component.get(), expr.index, expr.id);
+
+    // You can only use the SOM flags if you've also specified an SOM
+    // precision mode.
+    if (expr.som != SOM_NONE && cc.streaming && !ng.ssm.somPrecision()) {
+        throw CompileError("To use a SOM expression flag in streaming mode, "
+                           "an SOM precision mode (e.g. "
+                           "HS_MODE_SOM_HORIZON_LARGE) must be specified.");
+    }
+
+    // If this expression is a literal, we can feed it directly to Rose rather
+    // than building the NFA graph.
+    if (shortcutLiteral(ng, expr)) {
+        DEBUG_PRINTF("took literal short cut\n");
+        return;
+    }
+
+    unique_ptr<NGWrapper> g = buildWrapper(ng.rm, cc, expr);
+
+    if (!g) {
+        DEBUG_PRINTF("NFA build failed on ID %u, but no exception was "
+                     "thrown.\n", expr.id);
+        throw CompileError("Internal error.");
+    }
+
+    if (!expr.allow_vacuous && matches_everywhere(*g)) {
+        throw CompileError("Pattern matches empty buffer; use "
+                           "HS_FLAG_ALLOWEMPTY to enable support.");
+    }
+
+    if (!ng.addGraph(*g)) {
+        DEBUG_PRINTF("NFA addGraph failed on ID %u.\n", expr.id);
+        throw CompileError("Error compiling expression.");
+    }
+}
+
+static
+aligned_unique_ptr<RoseEngine> generateRoseEngine(NG &ng) {
+    const u32 minWidth =
+        ng.minWidth.is_finite() ? verify_u32(ng.minWidth) : ROSE_BOUND_INF;
+    auto rose = ng.rose->buildRose(minWidth);
+
+    if (!rose) {
+        DEBUG_PRINTF("error building rose\n");
+        assert(0);
+        return nullptr;
+    }
+
+    /* avoid building a smwr if just a pure floating case. */
+    if (!roseIsPureLiteral(rose.get())) {
+        u32 qual = roseQuality(rose.get());
+        auto smwr = ng.smwr->build(qual);
+        if (smwr) {
+            rose = roseAddSmallWrite(rose.get(), smwr.get());
+        }
+    }
+
+    dumpRose(*ng.rose, rose.get(), ng.cc.grey);
+    dumpReportManager(ng.rm, ng.cc.grey);
+    dumpSomSlotManager(ng.ssm, ng.cc.grey);
+    dumpSmallWrite(rose.get(), ng.cc.grey);
+
+    return rose;
+}
+
+platform_t target_to_platform(const target_t &target_info) {
+    platform_t p;
+    p = 0;
+
+    if (!target_info.has_avx2()) {
+        p |= HS_PLATFORM_NOAVX2;
+    }
+    return p;
+}
+
+struct hs_database *build(NG &ng, unsigned int *length) {
+    assert(length);
+
+    auto rose = generateRoseEngine(ng);
+    if (!rose) {
+        throw CompileError("Unable to generate bytecode.");
+    }
+    *length = roseSize(rose.get());
+    if (!*length) {
+        DEBUG_PRINTF("RoseEngine has zero length\n");
+        assert(0);
+        throw CompileError("Internal error.");
+    }
+
+    const char *bytecode = (const char *)(rose.get());
+    const platform_t p = target_to_platform(ng.cc.target_info);
+    struct hs_database *db = dbCreate(bytecode, *length, p);
+    if (!db) {
+        throw CompileError("Could not allocate memory for bytecode.");
+    }
+
+    return db;
+}
+
+static
+void stripFromPositions(vector<PositionInfo> &v, Position pos) {
+    auto removed = remove(v.begin(), v.end(), PositionInfo(pos));
+    v.erase(removed, v.end());
+}
+
+static
+void connectInitialStates(GlushkovBuildState &bs,
+                          const ParsedExpression &expr) {
+    vector<PositionInfo> initials = expr.component->first();
+    const NFABuilder &builder = bs.getBuilder();
+    const Position startState = builder.getStart();
+    const Position startDotStarState = builder.getStartDotStar();
+
+    DEBUG_PRINTF("wiring initials = %s\n",
+                 dumpPositions(initials.begin(), initials.end()).c_str());
+
+    vector<PositionInfo> starts = {startState, startDotStarState};
+
+    // strip start and startDs, which can be present due to boundaries
+    stripFromPositions(initials, startState);
+    stripFromPositions(initials, startDotStarState);
+
+    // replace epsilons with accepts
+    for (const auto &s : initials) {
+        if (s.pos != GlushkovBuildState::POS_EPSILON) {
+            continue;
+        }
+
+        assert(starts.size() == 2); /* start, startds */
+        vector<PositionInfo> starts_temp = starts;
+        starts_temp[0].flags = s.flags;
+        starts_temp[1].flags = s.flags;
+        bs.connectAccepts(starts_temp);
+    }
+
+    if (!initials.empty()) {
+        bs.connectRegions(starts, initials);
+    }
+}
+
+static
+void connectFinalStates(GlushkovBuildState &bs, const ParsedExpression &expr) {
+    vector<PositionInfo> finals = expr.component->last();
+
+    DEBUG_PRINTF("wiring finals = %s\n",
+                 dumpPositions(finals.begin(), finals.end()).c_str());
+
+    bs.connectAccepts(finals);
+}
+
+#ifndef NDEBUG
+static
+bool isSupported(const Component &c) {
+    try {
+        checkUnsupported(c);
+        return true;
+    }
+    catch (ParseError &) {
+        return false;
+    }
+}
+#endif
+
+unique_ptr<NGWrapper> buildWrapper(ReportManager &rm, const CompileContext &cc,
+                                   const ParsedExpression &expr) {
+    assert(isSupported(*expr.component));
+
+    const unique_ptr<NFABuilder> builder = makeNFABuilder(rm, cc, expr);
+    assert(builder);
+
+    // Set up START and ACCEPT states; retrieve the special states
+    const auto bs = makeGlushkovBuildState(*builder, expr.prefilter);
+
+    // Map position IDs to characters/components
+    expr.component->notePositions(*bs);
+
+    // Wire the start dotstar state to the firsts
+    connectInitialStates(*bs, expr);
+
+    DEBUG_PRINTF("wire up body of expr\n");
+    // Build the rest of the FOLLOW set
+    vector<PositionInfo> initials = {builder->getStartDotStar(),
+                                     builder->getStart()};
+    expr.component->buildFollowSet(*bs, initials);
+
+    // Wire the lasts to the accept state
+    connectFinalStates(*bs, expr);
+
+    // Create our edges
+    bs->buildEdges();
+
+    auto g = builder->getGraph();
+    assert(g);
+
+    dumpDotWrapper(*g, "00_before_asserts", cc.grey);
+    removeAssertVertices(rm, *g);
+
+    return g;
+}
+
+} // namespace ue2

src/compiler/compiler.h

@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Compiler front-end interface
+ */
+
+#ifndef COMPILER_H
+#define COMPILER_H
+
+#include "ue2common.h"
+#include "database.h"
+#include "parser/Component.h"
+#include "som/som.h"
+
+#include <memory>
+#include <boost/core/noncopyable.hpp>
+
+struct hs_database;
+struct hs_expr_ext;
+
+namespace ue2 {
+
+struct CompileContext;
+struct Grey;
+struct target_t;
+class NG;
+class ReportManager;
+class NGWrapper;
+
+/** Class gathering together the pieces of a parsed expression.
+ * Note: Owns the provided component.
+ */
+class ParsedExpression : boost::noncopyable {
+public:
+    ParsedExpression(unsigned index, const char *expression, unsigned flags,
+                     ReportID actionId, const hs_expr_ext *ext = nullptr);
+
+    bool utf8; //!< UTF-8 mode flag specified
+
+    /** \brief root node of parsed component tree. */
+    std::unique_ptr<ue2::Component> component;
+
+    const bool allow_vacuous;   //!< HS_FLAG_ALLOWEMPTY specified
+    const bool highlander;      //!< HS_FLAG_SINGLEMATCH specified
+    const bool prefilter;       //!< HS_FLAG_PREFILTER specified
+    som_type som;               //!< chosen SOM mode, or SOM_NONE
+
+    /** \brief index in expressions array passed to \ref hs_compile_multi */
+    const unsigned index;
+
+    const ReportID id; //!< user-specified pattern ID
+    u64a min_offset;   //!< 0 if not used
+    u64a max_offset;   //!< MAX_OFFSET if not used
+    u64a min_length;   //!< 0 if not used
+};
+
+/**
+ * Add an expression to the compiler.
+ *
+ * @param ng
+ *      The global NG object.
+ * @param index
+ *      The index of the expression (used for errors)
+ * @param expression
+ *      NULL-terminated PCRE expression
+ * @param flags
+ *      The full set of Hyperscan flags associated with this rule.
+ * @param ext
+ *      Struct containing extra parameters for this expression, or NULL if
+ *      none.
+ * @param actionId
+ *      The identifier to associate with the expression; returned by engine on
+ *      match.
+ */
+void addExpression(NG &ng, unsigned index, const char *expression,
+                   unsigned flags, const hs_expr_ext *ext, ReportID actionId);
+
+/**
+ * Build a Hyperscan database out of the expressions we've been given. A
+ * fatal error will result in an exception being thrown.
+ *
+ * @param ng
+ *      The global NG object.
+ * @param[out] length
+ *      The number of bytes occupied by the compiled structure.
+ * @return
+ *      The compiled structure. Should be deallocated with the
+ *      hs_database_free() function.
+ */
+struct hs_database *build(NG &ng, unsigned int *length);
+
+/**
+ * Constructs an NFA graph from the given expression tree.
+ *
+ * @param rm
+ *      Global ReportManager for this compile.
+ * @param cc
+ *      Global compile context for this compile.
+ * @param expr
+ *      ParsedExpression object.
+ * @return
+ *      nullptr on error.
+ */
+std::unique_ptr<NGWrapper> buildWrapper(ReportManager &rm,
+                                        const CompileContext &cc,
+                                        const ParsedExpression &expr);
+
+/**
+ * Build a platform_t out of a target_t.
+ */
+platform_t target_to_platform(const target_t &target_info);
+
+#if defined(DUMP_SUPPORT) || defined(DEBUG)
+void dumpExpression(const ParsedExpression &expr, const char *stage,
+                    const Grey &grey);
+#else
+static really_inline
+void dumpExpression(UNUSED const ParsedExpression &expr,
+                    UNUSED const char *stage, UNUSED const Grey &grey) {
+}
+
+#endif
+
+} // namespace
+
+#endif // COMPILER_H

src/compiler/error.cpp

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Compile-time error utils.
+ */
+#include "allocator.h"
+#include "error.h"
+#include "ue2common.h"
+#include "hs_compile.h"
+#include "util/compile_error.h"
+
+#include <cstring>
+#include <string>
+
+using std::string;
+
+static const char failureNoMemory[] = "Unable to allocate memory.";
+static const char failureInternal[] = "Internal error.";
+
+extern const hs_compile_error_t hs_enomem = {
+    const_cast<char *>(failureNoMemory), 0
+};
+extern const hs_compile_error_t hs_einternal = {
+    const_cast<char *>(failureInternal), 0
+};
+
+namespace ue2 {
+
+hs_compile_error_t *generateCompileError(const string &err, int expression) {
+    hs_compile_error_t *ret =
+        (struct hs_compile_error *)hs_misc_alloc(sizeof(hs_compile_error_t));
+    if (ret) {
+        char *msg = (char *)hs_misc_alloc(err.size() + 1);
+        if (msg) {
+            memcpy(msg, err.c_str(), err.size() + 1);
+            ret->message = msg;
+        } else {
+            hs_misc_free(ret);
+            ret = nullptr;
+        }
+    }
+
+    if (!ret || !ret->message) {
+        return const_cast<hs_compile_error_t *>(&hs_enomem);
+    }
+
+    ret->expression = expression;
+
+    return ret;
+}
+
+hs_compile_error_t *generateCompileError(const CompileError &e) {
+    return generateCompileError(e.reason, e.hasIndex ? (int)e.index : -1);
+}
+
+void freeCompileError(hs_compile_error_t *error) {
+    if (!error) {
+        return;
+    }
+    if (error == &hs_enomem || error == &hs_einternal) {
+        // These are not allocated.
+        return;
+    }
+
+    hs_misc_free(error->message);
+    hs_misc_free(error);
+}
+
+} // namespace ue2

src/compiler/error.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Compile-time error utils.
+  */
+
+#ifndef COMPILE_ERROR_H
+#define COMPILE_ERROR_H
+
+#include <string>
+
+struct hs_compile_error;
+
+// Special errors that aren't allocated with hs_alloc/hs_free.
+extern const hs_compile_error hs_enomem;
+extern const hs_compile_error hs_einternal;
+
+namespace ue2 {
+
+class CompileError;
+
+hs_compile_error *generateCompileError(const std::string &err, int expression);
+hs_compile_error *generateCompileError(const CompileError &e);
+
+void freeCompileError(hs_compile_error *error);
+
+} // namespace ue2
+
+#endif