Initial commit of Hyperscan

src/util/determinise.h

@@ -0,0 +1,200 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief DFA determinisation code.
+ */
+
+#ifndef DETERMINISE_H
+#define DETERMINISE_H
+
+#include "nfagraph/ng_holder.h"
+#include "charreach.h"
+#include "container.h"
+#include "ue2common.h"
+
+#include <array>
+#include <algorithm>
+#include <vector>
+
+namespace ue2 {
+
+#define DETERMINISE_RESERVE_SIZE 10
+
+/* Automaton details:
+ *
+ * const vector<StateSet> initial()
+ *     returns initial states to start determinising from. StateSets in the
+ *     initial() vector will given consecutive ids starting from 1, in the order
+ *     that they appear.
+ *
+ *  void reports(StateSet s, flat_set<ReportID> *out)
+ *     fills out with any reports that need to be raised for stateset.
+ *
+ *  void reportsEod(StateSet s, flat_set<ReportID> *out)
+ *     fills out with any reports that need to be raised for stateset at EOD.
+ *
+ *  void transition(const StateSet &in, StateSet *next)
+ *     fills the next array such next[i] is the stateset that in transitions to
+ *     on seeing symbol i (i is in the compressed alphabet of the automaton).
+ *
+ *  u16 alphasize
+ *     size of the compressed alphabet
+ */
+
+/** \brief determinises some sort of nfa
+ *  \param n the automaton to determinise
+ *  \param dstates_out output dfa states
+ *  \param state_limit limit on the number of dfa states to construct
+ *  \param statesets_out a mapping from DFA state to the set of NFA states in
+ *         the automaton
+ *  \return zero on success
+ */
+template<class Auto, class ds>
+never_inline
+int determinise(Auto &n, std::vector<ds> &dstates_out, dstate_id_t state_limit,
+                std::vector<typename Auto::StateSet> *statesets_out = nullptr) {
+    DEBUG_PRINTF("the determinator\n");
+    typedef typename Auto::StateSet StateSet;
+    typedef typename Auto::StateMap DstateIdMap;
+    DstateIdMap dstate_ids;
+    std::vector<StateSet> statesets;
+
+    const size_t alphabet_size = n.alphasize;
+
+    std::vector<ds> dstates;
+    dstates.reserve(DETERMINISE_RESERVE_SIZE);
+    statesets.reserve(DETERMINISE_RESERVE_SIZE);
+
+    dstate_ids[n.dead] = DEAD_STATE;
+    dstates.push_back(ds(alphabet_size));
+    std::fill_n(dstates[0].next.begin(), alphabet_size, DEAD_STATE);
+
+    statesets.push_back(n.dead);
+
+    const std::vector<StateSet> &init = n.initial();
+    for (u32 i = 0; i < init.size(); i++) {
+        statesets.push_back(init[i]);
+        assert(!contains(dstate_ids, init[i]));
+        dstate_ids[init[i]] = dstates.size();
+        dstates.push_back(ds(alphabet_size));
+    }
+
+    std::vector<StateSet> succs(alphabet_size, n.dead);
+    for (dstate_id_t curr_id = DEAD_STATE; curr_id < dstates.size();
+         curr_id++) {
+        StateSet &curr = statesets[curr_id];
+
+        DEBUG_PRINTF("curr: %hu\n", curr_id);
+
+        /* fill in accepts */
+        n.reports(curr, dstates[curr_id].reports);
+        n.reportsEod(curr, dstates[curr_id].reports_eod);
+
+        if (!dstates[curr_id].reports.empty()) {
+            DEBUG_PRINTF("curr: %hu: is accept\n", curr_id);
+        }
+
+        if (!dstates[curr_id].reports.empty()) {
+            /* only external reports set ekeys */
+            if (n.canPrune(dstates[curr_id].reports)) {
+                /* we only transition to dead on characters, TOPs leave us
+                 * alone */
+                std::fill_n(dstates[curr_id].next.begin(), alphabet_size,
+                            DEAD_STATE);
+                dstates[curr_id].next[n.alpha[TOP]] = curr_id;
+                continue;
+            }
+        }
+
+        /* fill in successor states */
+        n.transition(curr, &succs[0]);
+        for (symbol_t s = 0; s < n.alphasize; s++) {
+            dstate_id_t succ_id;
+            if (s && succs[s] == succs[s - 1]) {
+                succ_id = dstates[curr_id].next[s - 1];
+            } else {
+                typename DstateIdMap::const_iterator dstate_id_iter;
+                dstate_id_iter = dstate_ids.find(succs[s]);
+
+                if (dstate_id_iter != dstate_ids.end()) {
+                    succ_id = dstate_id_iter->second;
+
+                    if (succ_id > curr_id && !dstates[succ_id].daddy
+                        && n.unalpha[s] < N_CHARS) {
+                        dstates[succ_id].daddy = curr_id;
+                    }
+                } else {
+                    statesets.push_back(succs[s]);
+                    succ_id = dstates.size();
+                    dstate_ids[succs[s]] = succ_id;
+                    dstates.push_back(ds(alphabet_size));
+                    dstates.back().daddy = n.unalpha[s] < N_CHARS ? curr_id : 0;
+                }
+
+                DEBUG_PRINTF("-->%hu on %02hx\n", succ_id, n.unalpha[s]);
+            }
+
+            if (succ_id >= state_limit) {
+                DEBUG_PRINTF("succ_id %hu >= state_limit %hu\n",
+                             succ_id, state_limit);
+                return -2;
+            }
+
+            dstates[curr_id].next[s] = succ_id;
+        }
+    }
+
+    dstates_out = dstates;
+    if (statesets_out) {
+        statesets_out->swap(statesets);
+    }
+    DEBUG_PRINTF("ok\n");
+    return 0;
+}
+
+static inline
+std::vector<CharReach> populateCR(const NGHolder &g,
+                                  const std::vector<NFAVertex> &v_by_index,
+                                  const std::array<u16, ALPHABET_SIZE> &alpha) {
+    std::vector<CharReach> cr_by_index(v_by_index.size());
+
+    for (size_t i = 0; i < v_by_index.size(); i++) {
+        const CharReach &cr = g[v_by_index[i]].char_reach;
+        CharReach &cr_out = cr_by_index[i];
+        for (size_t s = cr.find_first(); s != cr.npos; s = cr.find_next(s)) {
+            cr_out.set(alpha[s]);
+        }
+    }
+
+    return cr_by_index;
+}
+
+} // namespace ue2
+
+#endif