diff --git a/src/nfagraph/ng.cpp b/src/nfagraph/ng.cpp index dff9c7e8..4525fd3e 100644 --- a/src/nfagraph/ng.cpp +++ b/src/nfagraph/ng.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -251,7 +251,7 @@ bool addComponent(NG &ng, NGHolder &g, const NGWrapper &w, const som_type som, return true; } - if (doViolet(*ng.rose, g, w.prefilter, cc)) { + if (doViolet(*ng.rose, g, w.prefilter, false, ng.rm, cc)) { return true; } @@ -272,7 +272,7 @@ bool addComponent(NG &ng, NGHolder &g, const NGWrapper &w, const som_type som, return true; } - if (doViolet(*ng.rose, g, w.prefilter, cc)) { + if (doViolet(*ng.rose, g, w.prefilter, true, ng.rm, cc)) { return true; } diff --git a/src/nfagraph/ng_reports.cpp b/src/nfagraph/ng_reports.cpp index 3d18a620..4e9b498d 100644 --- a/src/nfagraph/ng_reports.cpp +++ b/src/nfagraph/ng_reports.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -65,6 +65,26 @@ bool can_exhaust(const NGHolder &g, const ReportManager &rm) { return true; } +void set_report(NGHolder &g, ReportID internal_report) { + // First, wipe the report IDs on all vertices. + for (auto v : vertices_range(g)) { + g[v].reports.clear(); + } + + // Any predecessors of accept get our id. + for (auto v : inv_adjacent_vertices_range(g.accept, g)) { + g[v].reports.insert(internal_report); + } + + // Same for preds of acceptEod, except accept itself. + for (auto v : inv_adjacent_vertices_range(g.acceptEod, g)) { + if (v == g.accept) { + continue; + } + g[v].reports.insert(internal_report); + } +} + /** Derive a maximum offset for the graph from the max_offset values of its * reports. Returns MAX_OFFSET for inf. */ u64a findMaxOffset(const NGHolder &g, const ReportManager &rm) { diff --git a/src/nfagraph/ng_reports.h b/src/nfagraph/ng_reports.h index 3047ff0b..31c95308 100644 --- a/src/nfagraph/ng_reports.h +++ b/src/nfagraph/ng_reports.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -48,6 +48,10 @@ std::set all_reports(const NGHolder &g); /** True if *all* reports in the graph are exhaustible. */ bool can_exhaust(const NGHolder &g, const ReportManager &rm); +/** Replaces all existing reports on the holder with the provided internal + * report id. */ +void set_report(NGHolder &g, ReportID internal_report); + /** Derive a maximum offset for the graph from the max_offset values of its * reports. Returns MAX_OFFSET for inf. */ u64a findMaxOffset(const NGHolder &g, const ReportManager &rm); diff --git a/src/nfagraph/ng_violet.cpp b/src/nfagraph/ng_violet.cpp index 985246f0..66c0253a 100644 --- a/src/nfagraph/ng_violet.cpp +++ b/src/nfagraph/ng_violet.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Intel Corporation + * Copyright (c) 2016-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,6 +38,8 @@ #include "ng_holder.h" #include "ng_is_equal.h" #include "ng_literal_analysis.h" +#include "ng_limex.h" +#include "ng_mcclellan.h" #include "ng_netflow.h" #include "ng_prune.h" #include "ng_redundancy.h" @@ -47,6 +49,7 @@ #include "ng_split.h" #include "ng_util.h" #include "ng_width.h" +#include "nfa/rdfa.h" #include "rose/rose_build.h" #include "rose/rose_build_util.h" #include "rose/rose_in_dump.h" @@ -2616,7 +2619,110 @@ void rehomeEodSuffixes(RoseInGraph &vg) { /* old accept vertices will be tidied up by final pruneUseless() call */ } +static +bool tryForEarlyDfa(const NGHolder &h, const CompileContext &cc) { + switch (h.kind) { + case NFA_OUTFIX: /* 'prefix' of eod */ + case NFA_PREFIX: + return cc.grey.earlyMcClellanPrefix; + case NFA_INFIX: + return cc.grey.earlyMcClellanInfix; + case NFA_SUFFIX: + return cc.grey.earlyMcClellanSuffix; + default: + DEBUG_PRINTF("kind %u\n", (u32)h.kind); + assert(0); + return false; + } +} + +static +vector> getDfaTriggers(RoseInGraph &vg, + const vector &edges, + bool *single_trigger) { + vector> triggers; + u32 min_offset = ~0U; + u32 max_offset = 0; + for (const auto &e : edges) { + RoseInVertex s = source(e, vg); + if (vg[s].type == RIV_LITERAL) { + triggers.push_back(as_cr_seq(vg[s].s)); + } + ENSURE_AT_LEAST(&max_offset, vg[s].max_offset); + LIMIT_TO_AT_MOST(&min_offset, vg[s].min_offset); + } + + *single_trigger = min_offset == max_offset; + DEBUG_PRINTF("trigger offset (%u, %u)\n", min_offset, max_offset); + + return triggers; +} + +static +bool doEarlyDfa(RoseBuild &rose, RoseInGraph &vg, NGHolder &h, + const vector &edges, const ReportManager &rm, + const CompileContext &cc) { + DEBUG_PRINTF("trying for dfa\n"); + + bool single_trigger; + for (const auto &e : edges) { + if (vg[target(e, vg)].type == RIV_ACCEPT_EOD) { + /* TODO: support eod prefixes */ + return false; + } + } + + auto triggers = getDfaTriggers(vg, edges, &single_trigger); + + /* TODO: literal delay things */ + if (!generates_callbacks(h)) { + set_report(h, rose.getNewNfaReport()); + } + + shared_ptr dfa = buildMcClellan(h, &rm, single_trigger, triggers, + cc.grey); + + if (!dfa) { + return false; + } + + DEBUG_PRINTF("dfa ok\n"); + for (const auto &e : edges) { + vg[e].dfa = dfa; + } + + return true; +} + +static +void ensureImplementable(RoseBuild &rose, RoseInGraph &vg, + const ReportManager &rm, const CompileContext &cc) { + map > edges_by_graph; + vector graphs; + for (const RoseInEdge &ve : edges_range(vg)) { + if (vg[ve].graph) { + NGHolder *h = vg[ve].graph.get(); + if (!contains(edges_by_graph, h)) { + graphs.push_back(h); + } + edges_by_graph[h].push_back(ve); + } + } + for (NGHolder *h : graphs) { + if (isImplementableNFA(*h, &rm, cc)) { + continue; + } + + if (tryForEarlyDfa(*h, cc) + && doEarlyDfa(rose, vg, *h, edges_by_graph[h], rm, cc)) { + continue; + } + DEBUG_PRINTF("eek\n"); + } +} + bool doViolet(RoseBuild &rose, const NGHolder &h, bool prefilter, + bool last_chance, const ReportManager &rm, const CompileContext &cc) { assert(!can_never_match(h)); @@ -2663,10 +2769,6 @@ bool doViolet(RoseBuild &rose, const NGHolder &h, bool prefilter, decomposeLiteralChains(vg, cc); } - /* Step 5: avoid unimplementable, or overly large engines if possible */ - /* TODO: later - ng_rose is currently acting as a backstop */ - - /* Step 6: send to rose */ rehomeEodSuffixes(vg); removeRedundantLiterals(vg, cc); @@ -2674,6 +2776,14 @@ bool doViolet(RoseBuild &rose, const NGHolder &h, bool prefilter, dumpPreRoseGraph(vg, cc.grey); renumber_vertices(vg); calcVertexOffsets(vg); + + + /* Step 5: avoid unimplementable, or overly large engines if possible */ + if (last_chance) { + ensureImplementable(rose, vg, rm, cc); + } + + /* Step 6: send to rose */ bool rv = rose.addRose(vg, prefilter); DEBUG_PRINTF("violet: %s\n", rv ? "success" : "fail"); return rv; diff --git a/src/nfagraph/ng_violet.h b/src/nfagraph/ng_violet.h index fb62bfc0..6bc42d75 100644 --- a/src/nfagraph/ng_violet.h +++ b/src/nfagraph/ng_violet.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Intel Corporation + * Copyright (c) 2016-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,10 +41,12 @@ class NGHolder; class RoseBuild; struct CompileContext; +class ReportManager; /** \brief Attempt to consume the entire pattern in graph \a h with Rose. * Returns true if successful. */ bool doViolet(RoseBuild &rose, const NGHolder &h, bool prefilter, + bool last_chance, const ReportManager &rm, const CompileContext &cc); } // namespace ue2 diff --git a/src/rose/rose_build_add.cpp b/src/rose/rose_build_add.cpp index 8b10bc7d..da26e126 100644 --- a/src/rose/rose_build_add.cpp +++ b/src/rose/rose_build_add.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -1033,8 +1033,8 @@ bool empty(const GraphT &g) { return vi == ve; } -/* We only try to implement as a dfa if a non-nullptr as_dfa is provided to return - * the raw dfa to. */ +/* We only try to implement as a dfa if a non-nullptr as_dfa is provided to + * return the raw dfa to. */ static bool canImplementGraph(RoseBuildImpl *tbi, const RoseInGraph &in, NGHolder &h, const vector &edges, bool prefilter, @@ -1105,7 +1105,7 @@ bool canImplementGraph(RoseBuildImpl *tbi, const RoseInGraph &in, NGHolder &h, } if (!generates_callbacks(h)) { - setReportId(h, tbi->getNewNfaReport()); + set_report(h, tbi->getNewNfaReport()); } bool single_trigger = min_offset == max_offset; @@ -1601,6 +1601,8 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter, for (const auto &e : edges_range(in)) { if (!in[e].graph) { + assert(!in[e].dfa); + assert(!in[e].haig); continue; // no graph } @@ -1616,6 +1618,11 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter, ordered_graphs.push_back(h); } graphs[h].push_back(e); + if (in[e].dfa) { + assert(!contains(bd.early_dfas, h) + || bd.early_dfas[h] == in[e].dfa); + bd.early_dfas[h] = in[e].dfa; + } } assert(ordered_graphs.size() == graphs.size()); @@ -1626,8 +1633,9 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter, const vector &h_edges = graphs.at(h); unique_ptr as_dfa; /* allow finalChance as fallback is basically an outfix at this point */ - if (!canImplementGraph(this, in, *h, h_edges, prefilter, rm, cc, - finalChance, &as_dfa)) { + if (!contains(bd.early_dfas, h) + && !canImplementGraph(this, in, *h, h_edges, prefilter, rm, cc, + finalChance, &as_dfa)) { return false; } if (as_dfa) { @@ -1649,7 +1657,7 @@ bool RoseBuildImpl::addRose(const RoseInGraph &ig, bool prefilter, if (!generates_callbacks(whatRoseIsThis(in, e)) && !contains(bd.early_dfas, &h) && in[target(e, in)].type != RIV_ACCEPT_EOD) { - setReportId(h, getNewNfaReport()); + set_report(h, getNewNfaReport()); } } diff --git a/src/rose/rose_build_add_mask.cpp b/src/rose/rose_build_add_mask.cpp index de3bdf0a..bd8eed0c 100644 --- a/src/rose/rose_build_add_mask.cpp +++ b/src/rose/rose_build_add_mask.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -480,7 +480,7 @@ void addTransientMask(RoseBuildImpl &build, const vector &mask, // Everyone gets the same report ID. ReportID mask_report = build.getNewNfaReport(); - setReportId(*mask_graph, mask_report); + set_report(*mask_graph, mask_report); // Build the HWLM literal mask. vector msk, cmp; diff --git a/src/rose/rose_build_convert.cpp b/src/rose/rose_build_convert.cpp index a15d4dc6..14eec7c7 100644 --- a/src/rose/rose_build_convert.cpp +++ b/src/rose/rose_build_convert.cpp @@ -166,7 +166,7 @@ bool delayLiteralWithPrefix(RoseBuildImpl &tbi, RoseVertex v, u32 lit_id, shared_ptr h = makeRosePrefix(lit.s); ReportID prefix_report = 0; - setReportId(*h, prefix_report); + set_report(*h, prefix_report); if (!isImplementableNFA(*h, &tbi.rm, tbi.cc)) { DEBUG_PRINTF("prefix not implementable\n"); diff --git a/src/rose/rose_build_impl.h b/src/rose/rose_build_impl.h index d5f37b5a..ce8a859d 100644 --- a/src/rose/rose_build_impl.h +++ b/src/rose/rose_build_impl.h @@ -624,8 +624,6 @@ size_t maxOverlap(const rose_literal_id &a, const rose_literal_id &b); ue2_literal findNonOverlappingTail(const std::set &lits, const ue2_literal &s); -void setReportId(NGHolder &g, ReportID id); - #ifndef NDEBUG bool roseHasTops(const RoseBuildImpl &build, RoseVertex v); bool hasOrphanedTops(const RoseBuildImpl &build); diff --git a/src/rose/rose_build_misc.cpp b/src/rose/rose_build_misc.cpp index 97579111..b33192da 100644 --- a/src/rose/rose_build_misc.cpp +++ b/src/rose/rose_build_misc.cpp @@ -859,27 +859,6 @@ bool RoseDedupeAuxImpl::requiresDedupeSupport( return false; } -// Sets the report ID for all vertices connected to an accept to `id`. -void setReportId(NGHolder &g, ReportID id) { - // First, wipe the report IDs on all vertices. - for (auto v : vertices_range(g)) { - g[v].reports.clear(); - } - - // Any predecessors of accept get our id. - for (auto v : inv_adjacent_vertices_range(g.accept, g)) { - g[v].reports.insert(id); - } - - // Same for preds of acceptEod, except accept itself. - for (auto v : inv_adjacent_vertices_range(g.acceptEod, g)) { - if (v == g.accept) { - continue; - } - g[v].reports.insert(id); - } -} - bool operator<(const RoseEdgeProps &a, const RoseEdgeProps &b) { ORDER_CHECK(minBound); ORDER_CHECK(maxBound); diff --git a/src/rose/rose_in_graph.h b/src/rose/rose_in_graph.h index 0e218576..42c59932 100644 --- a/src/rose/rose_in_graph.h +++ b/src/rose/rose_in_graph.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -55,6 +55,7 @@ namespace ue2 { class NGHolder; struct raw_som_dfa; +struct raw_dfa; enum RoseInVertexType { RIV_LITERAL, @@ -166,9 +167,12 @@ struct RoseInEdgeProps { /** \brief Maximum bound on 'dot' repeat between literals. */ u32 maxBound; - /** \brief Prefix graph. Graph is end to (end - lag). */ + /** \brief Graph on edge. Graph is end to (end - lag). */ std::shared_ptr graph; + /** \brief DFA version of graph, if we have already determinised. */ + std::shared_ptr dfa; + /** \brief Haig version of graph, if required. */ std::shared_ptr haig;