github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-06-28 16:41:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

violet: initial implementation

Browse Source
This commit is contained in:
Alex Coyte 2016-07-15 13:07:00 +10:00 committed by Matthew Barr
parent b13a90e5d2
commit 5c5ec905cc
31 changed files with 3171 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -748,6 +748,8 @@ SET (hs_SRCS
src/nfagraph/ng_util.h
src/nfagraph/ng_vacuous.cpp
src/nfagraph/ng_vacuous.h
src/nfagraph/ng_violet.cpp
src/nfagraph/ng_violet.h
src/nfagraph/ng_width.cpp
src/nfagraph/ng_width.h
src/parser/AsciiComponentClass.cpp

26
src/grey.cpp
View File

@ -34,7 +34,7 @@
#include <string>
#include <vector>
#define DEFAULT_MAX_HISTORY 60
#define DEFAULT_MAX_HISTORY 110
using namespace std;
@ -51,7 +51,9 @@ Grey::Grey(void) :
allowLbr(true),
allowMcClellan(true),
allowPuff(true),
allowLiteral(true),
allowRose(true),
allowViolet(true),
allowExtendedNFA(true), /* bounded repeats of course */
allowLimExNFA(true),
allowAnchoredAcyclic(true),
@ -60,6 +62,13 @@ Grey::Grey(void) :
allowDecoratedLiteral(true),
allowNoodle(true),
fdrAllowTeddy(true),
violetAvoidSuffixes(true),
violetAvoidWeakInfixes(true),
violetDoubleCut(true),
violetExtractStrongLiterals(true),
violetLiteralChains(true),
violetDoubleCutLiteralLen(3),
violetEarlyCleanLiteralLen(6),
puffImproveHead(true),
castleExclusive(true),
mergeSEP(true), /* short exhaustible passthroughs */
@ -206,7 +215,9 @@ void applyGreyOverrides(Grey *g, const string &s) {
G_UPDATE(allowLbr);
G_UPDATE(allowMcClellan);
G_UPDATE(allowPuff);
G_UPDATE(allowLiteral);
G_UPDATE(allowRose);
G_UPDATE(allowViolet);
G_UPDATE(allowExtendedNFA);
G_UPDATE(allowLimExNFA);
G_UPDATE(allowAnchoredAcyclic);
@ -215,6 +226,13 @@ void applyGreyOverrides(Grey *g, const string &s) {
G_UPDATE(allowDecoratedLiteral);
G_UPDATE(allowNoodle);
G_UPDATE(fdrAllowTeddy);
G_UPDATE(violetAvoidSuffixes);
G_UPDATE(violetAvoidWeakInfixes);
G_UPDATE(violetDoubleCut);
G_UPDATE(violetExtractStrongLiterals);
G_UPDATE(violetLiteralChains);
G_UPDATE(violetDoubleCutLiteralLen);
G_UPDATE(violetEarlyCleanLiteralLen);
G_UPDATE(puffImproveHead);
G_UPDATE(castleExclusive);
G_UPDATE(mergeSEP);
@ -315,7 +333,9 @@ void applyGreyOverrides(Grey *g, const string &s) {
g->allowLitHaig = false;
g->allowMcClellan = false;
g->allowPuff = false;
g->allowLiteral = false;
g->allowRose = false;
g->allowViolet = false;
g->allowSmallLiteralSet = false;
g->roseMasks = false;
done = true;
@ -331,7 +351,9 @@ void applyGreyOverrides(Grey *g, const string &s) {
g->allowLitHaig = false;
g->allowMcClellan = true;
g->allowPuff = false;
g->allowLiteral = false;
g->allowRose = false;
g->allowViolet = false;
g->allowSmallLiteralSet = false;
g->roseMasks = false;
done = true;
@ -347,7 +369,9 @@ void applyGreyOverrides(Grey *g, const string &s) {
g->allowLitHaig = false;
g->allowMcClellan = true;
g->allowPuff = false;
g->allowLiteral = false;
g->allowRose = false;
g->allowViolet = false;
g->allowSmallLiteralSet = false;
g->roseMasks = false;
done = true;

10
src/grey.h
View File

@ -51,7 +51,9 @@ struct Grey {
bool allowLbr;
bool allowMcClellan;
bool allowPuff;
bool allowLiteral;
bool allowRose;
bool allowViolet;
bool allowExtendedNFA;
bool allowLimExNFA;
bool allowAnchoredAcyclic;
@ -62,6 +64,14 @@ struct Grey {
bool allowNoodle;
bool fdrAllowTeddy;
u32 violetAvoidSuffixes; /* 0=never, 1=sometimes, 2=always */
bool violetAvoidWeakInfixes;
bool violetDoubleCut;
bool violetExtractStrongLiterals;
bool violetLiteralChains;
u32 violetDoubleCutLiteralLen;
u32 violetEarlyCleanLiteralLen;
bool puffImproveHead;
bool castleExclusive; // enable castle mutual exclusion analysis

5
src/hwlm/hwlm_build.cpp
View File

@ -35,6 +35,7 @@
#include "hwlm_internal.h"
#include "noodle_engine.h"
#include "noodle_build.h"
#include "scratch.h"
#include "ue2common.h"
#include "fdr/fdr_compile.h"
#include "nfa/shufticompile.h"
@ -493,6 +494,10 @@ bool isNoodleable(const vector<hwlmLiteral> &lits,
stream_control->history_max);
return false;
}
if (2 * lits.front().s.length() - 2 > FDR_TEMP_BUF_SIZE) {
assert(0);
return false;
}
}
if (!lits.front().msk.empty()) {

28
src/nfa/nfa_kind.h
View File

@ -37,6 +37,8 @@
#include "ue2common.h"
#include <string>
namespace ue2 {
/** \brief Specify the use-case for an nfa engine. */
@ -115,6 +117,32 @@ bool has_managed_reports(enum nfa_kind k) {
}
}
#if defined(DEBUG) || defined(DUMP_SUPPORT)
inline
std::string to_string(nfa_kind k) {
switch (k) {
case NFA_PREFIX:
return "PREFIX";
case NFA_INFIX:
return "INFIX";
case NFA_SUFFIX:
return "SUFFIX";
case NFA_OUTFIX:
return "OUTFIX";
case NFA_REV_PREFIX:
return "REV_PREFIX";
case NFA_OUTFIX_RAW:
return "OUTFIX_RAW";
case NFA_EAGER_PREFIX:
return "EAGER_PREFIX";
}
assert(0);
return "?";
}
#endif
} // namespace ue2
#endif

9
src/nfagraph/ng.cpp
View File

@ -57,6 +57,7 @@
#include "ng_small_literal_set.h"
#include "ng_som.h"
#include "ng_vacuous.h"
#include "ng_violet.h"
#include "ng_utf8.h"
#include "ng_util.h"
#include "ng_width.h"
@ -244,6 +245,10 @@ bool addComponent(NG &ng, NGHolder &g, const NGWrapper &w, const som_type som,
return true;
}
if (doViolet(*ng.rose, g, w.prefilter, cc)) {
return true;
}
if (splitOffRose(*ng.rose, g, w.prefilter, cc)) {
return true;
}
@ -261,6 +266,10 @@ bool addComponent(NG &ng, NGHolder &g, const NGWrapper &w, const som_type som,
return true;
}
if (doViolet(*ng.rose, g, w.prefilter, cc)) {
return true;
}
if (splitOffRose(*ng.rose, g, w.prefilter, cc)) {
return true;
}

10
src/nfagraph/ng_holder.h
View File

@ -239,6 +239,16 @@ vertices(const NGHolder &h) {
*/
void clear_graph(NGHolder &h);
inline
void renumber_edges(NGHolder &h) {
h.renumberEdges();
}
inline
void renumber_vertices(NGHolder &h) {
h.renumberVertices();
}
/*
* \brief Clear and remove all of the vertices pointed to by the given iterator
* range.

103
src/nfagraph/ng_literal_analysis.cpp
View File

@ -339,6 +339,12 @@ void processWorkQueue(const NGHolder &g, const NFAEdge &e,
g[source(e, g)].index, g[target(e, g)].index, s.size());
}
bool bad_mixed_sensitivity(const ue2_literal &s) {
/* TODO: if the mixed cases is entirely within MAX_MASK2_WIDTH of the end,
* we should be able to handle it */
return mixed_sensitivity(s) && s.length() > MAX_MASK2_WIDTH;
}
static
u64a litUniqueness(const string &s) {
CharReach seen(s);
@ -624,6 +630,48 @@ u64a compressAndScore(set<ue2_literal> &s) {
return score;
}
/* like compressAndScore, but replaces long mixed sensitivity literals with
* something weaker. */
u64a sanitizeAndCompressAndScore(set<ue2_literal> &lits) {
const size_t maxExploded = 8; // only case-explode this far
/* TODO: the whole compression thing could be made better by systematically
* considering replacing literal sets not just by common suffixes but also
* by nocase literals. */
vector<ue2_literal> replacements;
for (auto it = lits.begin(); it != lits.end();) {
auto jt = it;
++it;
if (!bad_mixed_sensitivity(*jt)) {
continue;
}
/* we have to replace *jt with something... */
ue2_literal s = *jt;
lits.erase(jt);
vector<ue2_literal> exploded;
for (auto cit = caseIterateBegin(s); cit != caseIterateEnd(); ++cit) {
exploded.emplace_back(*cit, false);
if (exploded.size() > maxExploded) {
goto dont_explode;
}
}
insert(&replacements, replacements.end(), exploded);
continue;
dont_explode:
make_nocase(&s);
replacements.push_back(s);
}
insert(&lits, replacements);
return compressAndScore(lits);
}
u64a scoreSet(const set<ue2_literal> &s) {
if (s.empty()) {
return NO_LITERAL_AT_EDGE_SCORE;
@ -674,7 +722,7 @@ set<ue2_literal> getLiteralSet(const NGHolder &g, const NFAVertex &v,
return s;
}
vector<u64a> scoreEdges(const NGHolder &g) {
vector<u64a> scoreEdges(const NGHolder &g, const flat_set<NFAEdge> &known_bad) {
assert(hasCorrectlyNumberedEdges(g));
vector<u64a> scores(num_edges(g));
@ -682,8 +730,12 @@ vector<u64a> scoreEdges(const NGHolder &g) {
for (const auto &e : edges_range(g)) {
u32 eidx = g[e].index;
assert(eidx < scores.size());
set<ue2_literal> ls = getLiteralSet(g, e);
scores[eidx] = compressAndScore(ls);
if (contains(known_bad, e)) {
scores[eidx] = NO_LITERAL_AT_EDGE_SCORE;
} else {
set<ue2_literal> ls = getLiteralSet(g, e);
scores[eidx] = compressAndScore(ls);
}
}
return scores;
@ -842,4 +894,49 @@ bool getTrailingLiteral(const NGHolder &g, ue2_literal *lit_out) {
return true;
}
bool literalIsWholeGraph(const NGHolder &g, const ue2_literal &lit) {
NFAVertex v = g.accept;
for (auto it = lit.rbegin(), ite = lit.rend(); it != ite; ++it) {
NGHolder::inv_adjacency_iterator ai, ae;
tie(ai, ae) = inv_adjacent_vertices(v, g);
if (ai == ae) {
assert(0); // no predecessors?
return false;
}
v = *ai++;
if (ai != ae) {
DEBUG_PRINTF("branch, fail\n");
return false;
}
if (is_special(v, g)) {
DEBUG_PRINTF("special found, fail\n");
return false;
}
const CharReach &cr_g = g[v].char_reach;
const CharReach &cr_l = *it;
if (!cr_l.isSubsetOf(cr_g)) {
/* running over the prefix is needed to prevent false postives */
DEBUG_PRINTF("reach fail\n");
return false;
}
}
// Our last value for v should have only start states for predecessors.
for (auto u : inv_adjacent_vertices_range(v, g)) {
if (!is_any_start(u, g)) {
DEBUG_PRINTF("pred is not start\n");
return false;
}
}
assert(num_vertices(g) == lit.length() + N_SPECIALS);
DEBUG_PRINTF("ok\n");
return true;
}
} // namespace ue2

31
src/nfagraph/ng_literal_analysis.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -42,9 +42,7 @@
namespace ue2 {
#define NO_LITERAL_AT_EDGE_SCORE 10000000ULL
/* Score for special-to-special edges */
#define INVALID_EDGE_CAP 100000000ULL
#define INVALID_EDGE_CAP 100000000ULL /* special-to-special score */
class NGHolder;
@ -59,9 +57,20 @@ std::set<ue2_literal> getLiteralSet(const NGHolder &g, const NFAVertex &v,
bool only_first_encounter = true);
std::set<ue2_literal> getLiteralSet(const NGHolder &g, const NFAEdge &e);
/** Score all the edges in the given graph, returning them in \p scores indexed
/**
* Returns true if we are unable to use a mixed sensitivity literal in rose (as
* our literal matchers are generally either case sensitive or not).
*
* Shortish mixed sensitivity literals can be handled by confirm checks in rose
* and are not flagged as bad.
*/
bool bad_mixed_sensitivity(const ue2_literal &s);
/**
* Score all the edges in the given graph, returning them in \p scores indexed
* by edge_index. */
std::vector<u64a> scoreEdges(const NGHolder &h);
std::vector<u64a> scoreEdges(const NGHolder &h,
const flat_set<NFAEdge> &known_bad = {});
/** Returns a score for a literal set. Lower scores are better. */
u64a scoreSet(const std::set<ue2_literal> &s);
@ -69,6 +78,12 @@ u64a scoreSet(const std::set<ue2_literal> &s);
/** Compress a literal set to fewer literals. */
u64a compressAndScore(std::set<ue2_literal> &s);
/**
* Compress a literal set to fewer literals and replace any long mixed
* sensitivity literals with supported literals.
*/
u64a sanitizeAndCompressAndScore(std::set<ue2_literal> &s);
bool splitOffLeadingLiteral(const NGHolder &g, ue2_literal *lit_out,
NGHolder *rhs);
@ -77,6 +92,10 @@ bool splitOffAnchoredLeadingLiteral(const NGHolder &g, ue2_literal *lit_out,
bool getTrailingLiteral(const NGHolder &g, ue2_literal *lit_out);
/** \brief Returns true if the given literal is the only thing in the graph,
* from (start or startDs) to accept. */
bool literalIsWholeGraph(const NGHolder &g, const ue2_literal &lit);
} // namespace ue2
#endif

4
src/nfagraph/ng_literal_component.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -186,7 +186,7 @@ bool splitOffLiteral(NG &ng, NGWrapper &g, NFAVertex v, const bool anchored,
/** \brief Split off literals. True if any changes were made to the graph. */
bool splitOffLiterals(NG &ng, NGWrapper &g) {
if (!ng.cc.grey.allowRose) {
if (!ng.cc.grey.allowLiteral) {
return false;
}

69
src/nfagraph/ng_rose.cpp
View File

@ -773,51 +773,6 @@ unique_ptr<VertLitInfo> LitCollection::pickNext() {
}
/** \brief Returns true if the given literal is the only thing in the graph,
* from start to accept. */
static
bool literalIsWholeGraph(const NGHolder &g, const ue2_literal &lit) {
NFAVertex v = g.accept;
for (auto it = lit.rbegin(), ite = lit.rend(); it != ite; ++it) {
NGHolder::inv_adjacency_iterator ai, ae;
tie(ai, ae) = inv_adjacent_vertices(v, g);
if (ai == ae) {
assert(0); // no predecessors?
return false;
}
v = *ai++;
if (ai != ae) {
DEBUG_PRINTF("branch, fail\n");
return false;
}
if (is_special(v, g)) {
DEBUG_PRINTF("special found, fail\n");
return false;
}
const CharReach &cr = g[v].char_reach;
if (cr != *it) {
DEBUG_PRINTF("reach fail\n");
return false;
}
}
// Our last value for v should have only start states for predecessors.
for (auto u : inv_adjacent_vertices_range(v, g)) {
if (!is_any_start(u, g)) {
DEBUG_PRINTF("pred is not start\n");
return false;
}
}
assert(num_vertices(g) == lit.length() + N_SPECIALS);
DEBUG_PRINTF("ok\n");
return true;
}
static
bool can_match(const NGHolder &g, const ue2_literal &lit, bool overhang_ok) {
set<NFAVertex> curr, next;
@ -933,20 +888,11 @@ u32 removeTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
return delay;
}
static
void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
u32 delay) {
u32 delay, const vector<NFAVertex> &preds) {
assert(delay <= lit.length());
DEBUG_PRINTF("adding on '%s' %u\n", ((const string &)lit).c_str(), delay);
vector<NFAVertex> preds;
insert(&preds, preds.end(), inv_adjacent_vertices(g.accept, g));
clear_in_edges(g.accept, g);
for (auto v : preds) {
g[v].reports.clear(); /* clear report from old accepts */
}
NFAVertex prev = g.accept;
auto it = lit.rbegin();
while (delay--) {
@ -972,6 +918,19 @@ void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
assert(allMatchStatesHaveReports(g));
}
void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
u32 delay) {
vector<NFAVertex> preds;
insert(&preds, preds.end(), inv_adjacent_vertices(g.accept, g));
clear_in_edges(g.accept, g);
for (auto v : preds) {
g[v].reports.clear(); /* clear report from old accepts */
}
restoreTrailingLiteralStates(g, lit, delay, preds);
}
/* return false if we should get rid of the edge altogether */
static
bool removeLiteralFromLHS(RoseInGraph &ig, const RoseInEdge &lhs,

12
src/nfagraph/ng_rose.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -33,8 +33,11 @@
#ifndef NG_ROSE_H
#define NG_ROSE_H
#include "ng_holder.h"
#include "ue2common.h"
#include <vector>
namespace ue2 {
class NGHolder;
@ -65,6 +68,13 @@ bool checkRose(const ReportManager &rm, const NGHolder &h, bool prefilter,
u32 removeTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
u32 max_delay, bool overhang_ok = true);
void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
u32 delay);
void restoreTrailingLiteralStates(NGHolder &g, const ue2_literal &lit,
u32 delay,
const std::vector<NFAVertex> &preds);
} // namespace ue2
#endif // NG_ROSE_H

3
src/nfagraph/ng_som.cpp
View File

@ -2064,8 +2064,7 @@ sombe_rv doHaigLitSom(NG &ng, NGHolder &g, const NGWrapper &w, u32 comp_id,
ReportManager &rm = ng.rm;
SomSlotManager &ssm = ng.ssm;
// This approach relies on Rose.
if (!cc.grey.allowRose) {
if (!cc.grey.allowHaigLit) {
return SOMBE_FAIL;
}

14
src/nfagraph/ng_split.cpp
View File

@ -100,7 +100,12 @@ void splitLHS(const NGHolder &base, const vector<NFAVertex> &pivots,
add_edge((*lhs_map)[pivot], lhs->accept, *lhs);
}
pruneUseless(*lhs);
/* should do the renumbering unconditionally as we know edges are already
* misnumbered */
pruneUseless(*lhs, false);
renumber_edges(*lhs);
renumber_vertices(*lhs);
filterSplitMap(*lhs, lhs_map);
switch (base.kind) {
@ -148,7 +153,12 @@ void splitRHS(const NGHolder &base, const vector<NFAVertex> &pivots,
assert(contains(*rhs_map, pivot));
add_edge(rhs->start, (*rhs_map)[pivot], *rhs);
}
pruneUseless(*rhs);
/* should do the renumbering unconditionally as we know edges are already
* misnumbered */
pruneUseless(*rhs, false);
renumber_edges(*rhs);
renumber_vertices(*rhs);
filterSplitMap(*rhs, rhs_map);
switch (base.kind) {

16
src/nfagraph/ng_util.cpp
View File

@ -209,6 +209,15 @@ bool isAnchored(const NGHolder &g) {
return true;
}
bool isFloating(const NGHolder &g) {
for (auto v : adjacent_vertices_range(g.start, g)) {
if (v != g.startDs && !edge(g.startDs, v, g).second) {
return false;
}
}
return true;
}
bool isAcyclic(const NGHolder &g) {
try {
depth_first_search(
@ -657,7 +666,8 @@ bool hasCorrectlyNumberedVertices(const NGHolder &g) {
}
ids[id] = true;
}
return find(ids.begin(), ids.end(), false) == ids.end();
return find(ids.begin(), ids.end(), false) == ids.end()
&& num_vertices(g) == num_vertices(g.g);
}
/** Assertion: returns true if the edges in this graph are contiguously (and
@ -672,8 +682,10 @@ bool hasCorrectlyNumberedEdges(const NGHolder &g) {
}
ids[id] = true;
}
return find(ids.begin(), ids.end(), false) == ids.end();
return find(ids.begin(), ids.end(), false) == ids.end()
&& num_edges(g) == num_edges(g.g);
}
#endif // NDEBUG
} // namespace ue2

4
src/nfagraph/ng_util.h
View File

@ -228,6 +228,10 @@ bool isVacuous(const NGHolder &h);
* proper successors). */
bool isAnchored(const NGHolder &h);
/** \brief True if the graph contains no anchored vertices (start has no
* successors aside from startDs or vertices connected to startDs). */
bool isFloating(const NGHolder &h);
/** True if the graph contains no back-edges at all, other than the
* startDs self-loop. */
bool isAcyclic(const NGHolder &g);

2642
src/nfagraph/ng_violet.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

52
src/nfagraph/ng_violet.h Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright (c) 2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief Violet method of rose construction from NGHolder.
*/
#ifndef NG_VIOLET_H
#define NG_VIOLET_H
#include "ue2common.h"
namespace ue2 {
class NGHolder;
class RoseBuild;
struct CompileContext;
/** \brief Attempt to consume the entire pattern in graph \a h with Rose.
* Returns true if successful. */
bool doViolet(RoseBuild &rose, const NGHolder &h, bool prefilter,
const CompileContext &cc);
} // namespace ue2
#endif

4
src/parser/shortcut_literal.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -162,7 +162,7 @@ ConstructLiteralVisitor::~ConstructLiteralVisitor() {}
bool shortcutLiteral(NG &ng, const ParsedExpression &expr) {
assert(expr.component);
if (!ng.cc.grey.allowRose) {
if (!ng.cc.grey.allowLiteral) {
return false;
}

198
src/rose/rose_build_add.cpp
View File

@ -651,26 +651,94 @@ floating:
}
static
unique_ptr<NGHolder> makeRoseEodPrefix(const NGHolder &h,
ReportID prefix_report) {
unique_ptr<NGHolder> makeRoseEodPrefix(const NGHolder &h, RoseBuildImpl &build,
map<flat_set<ReportID>, ReportID> &remap) {
assert(generates_callbacks(h));
auto g = cloneHolder(h);
g->kind = is_triggered(h) ? NFA_INFIX : NFA_PREFIX;
setReportId(*g, prefix_report);
assert(!in_degree(h.accept, h));
auto gg = cloneHolder(h);
NGHolder &g = *gg;
g.kind = is_triggered(h) ? NFA_INFIX : NFA_PREFIX;
// Move acceptEod edges over to accept.
vector<NFAEdge> dead;
for (const auto &e : in_edges_range(g->acceptEod, *g)) {
NFAVertex u = source(e, *g);
if (u == g->accept) {
for (const auto &e : in_edges_range(g.acceptEod, g)) {
NFAVertex u = source(e, g);
if (u == g.accept) {
continue;
}
add_edge_if_not_present(u, g->accept, *g);
add_edge_if_not_present(u, g.accept, g);
dead.push_back(e);
if (!contains(remap, g[u].reports)) {
remap[g[u].reports] = build.getNewNfaReport();
}
g[u].reports = { remap[g[u].reports] };
}
remove_edges(dead, *g);
return g;
remove_edges(dead, g);
return gg;
}
static
u32 getEodEventID(RoseBuildImpl &build) {
// Allocate the EOD event if it hasn't been already.
if (build.eod_event_literal_id == MO_INVALID_IDX) {
build.eod_event_literal_id = build.getLiteralId({}, 0, ROSE_EVENT);
}
return build.eod_event_literal_id;
}
static
void makeEodEventLeftfix(RoseBuildImpl &build, RoseVertex u,
const NGHolder &h) {
assert(!build.isInETable(u));
RoseGraph &g = build.g;
map<flat_set<ReportID>, ReportID> report_remap;
shared_ptr<NGHolder> eod_leftfix
= makeRoseEodPrefix(h, build, report_remap);
u32 eod_event = getEodEventID(build);
for (const auto &report_mapping : report_remap) {
RoseVertex v = add_vertex(g);
g[v].idx = build.vertexIndex++;
g[v].literals.insert(eod_event);
build.literal_info[eod_event].vertices.insert(v);
map<u32, set<ReportID> > report_remap;
g[v].left.graph = eod_leftfix;
g[v].left.leftfix_report = report_mapping.second;
g[v].left.lag = 0;
RoseEdge e1 = add_edge(u, v, g).first;
g[e1].minBound = 0;
g[e1].maxBound = ROSE_BOUND_INF;
g[v].min_offset = add_rose_depth(g[u].min_offset,
findMinWidth(*g[v].left.graph));
g[v].max_offset = ROSE_BOUND_INF;
depth max_width = findMaxWidth(*g[v].left.graph);
if (u != build.root && max_width.is_finite()
&& (!build.isAnyStart(u) || isPureAnchored(*g[v].left.graph))) {
g[e1].maxBound = max_width;
g[v].max_offset = add_rose_depth(g[u].max_offset, max_width);
}
g[e1].history = ROSE_ROLE_HISTORY_NONE; // handled by prefix
RoseVertex w = add_vertex(g);
g[w].idx = build.vertexIndex++;
g[w].eod_accept = true;
g[w].reports = report_mapping.first;
g[w].min_offset = g[v].min_offset;
g[w].max_offset = g[v].max_offset;
RoseEdge e = add_edge(v, w, g).first;
g[e].minBound = 0;
g[e].maxBound = 0;
g[e].history = ROSE_ROLE_HISTORY_LAST_BYTE;
DEBUG_PRINTF("accept eod vertex (idx=%zu)\n", g[w].idx);
}
}
static
@ -686,8 +754,20 @@ void doRoseAcceptVertex(RoseBuildImpl *tbi,
RoseVertex u = pv.first;
const RoseInEdgeProps &edge_props = bd.ig[pv.second];
/* We need to duplicate the parent vertices if:
*
* 1) It already has a suffix, etc as we are going to add the specified
* suffix, etc to the parents and we do not want to overwrite the
* existing information.
*
* 2) We are making the an EOD accept and the vertex already has other
* out-edges - The LAST_BYTE history used for EOD accepts is
* incompatible with normal successors. As accepts are processed last we
* do not need to worry about other normal successors being added later.
*/
if (g[u].suffix || !g[u].reports.empty()
/* also poss accept eod edge: TODO check properly */
|| (ig[iv].type == RIV_ACCEPT_EOD && out_degree(u, g)
&& !edge_props.graph)
|| (!isLeafNode(u, g) && !tbi->isAnyStart(u))) {
DEBUG_PRINTF("duplicating for parent %zu\n", g[u].idx);
assert(!tbi->isAnyStart(u));
@ -719,74 +799,37 @@ void doRoseAcceptVertex(RoseBuildImpl *tbi,
}
} else {
assert(ig[iv].type == RIV_ACCEPT_EOD);
assert(!edge_props.haig);
if (edge_props.graph && tbi->isInETable(u)) {
if (!edge_props.graph) {
RoseVertex w = add_vertex(g);
g[w].idx = tbi->vertexIndex++;
g[w].eod_accept = true;
g[w].reports = ig[iv].reports;
g[w].min_offset = g[u].min_offset;
g[w].max_offset = g[u].max_offset;
RoseEdge e = add_edge(u, w, g).first;
g[e].minBound = 0;
g[e].maxBound = 0;
g[e].history = ROSE_ROLE_HISTORY_LAST_BYTE;
DEBUG_PRINTF("accept eod vertex (idx=%zu)\n", g[w].idx);
continue;
}
const NGHolder &h = *edge_props.graph;
assert(!in_degree(h.accept, h));
assert(generates_callbacks(h));
if (tbi->isInETable(u)) {
assert(h.kind == NFA_SUFFIX);
assert(!tbi->isAnyStart(u));
/* etable can't/shouldn't use eod event */
DEBUG_PRINTF("adding suffix to i%zu\n", g[u].idx);
g[u].suffix.graph = edge_props.graph;
assert(g[u].suffix.graph->kind == NFA_SUFFIX);
dumpHolder(*g[u].suffix.graph, 98, "eod_suffix", tbi->cc.grey);
assert(!in_degree(g[u].suffix.graph->accept,
*g[u].suffix.graph));
set<ReportID> reports = all_reports(*g[u].suffix.graph);
tbi->rm.getReport(*reports.begin());
assert(reports.size() == 1);
/* TODO: set dfa_(min|max)_width */
continue;
} else if (edge_props.graph) {
assert(!edge_props.haig);
assert(!tbi->isInETable(u));
// Allocate the EOD event if it hasn't been already.
if (tbi->eod_event_literal_id == MO_INVALID_IDX) {
tbi->eod_event_literal_id =
tbi->getLiteralId(ue2_literal(), 0, ROSE_EVENT);
}
RoseVertex v = add_vertex(g);
g[v].idx = tbi->vertexIndex++;
g[v].literals.insert(tbi->eod_event_literal_id);
tbi->literal_info[tbi->eod_event_literal_id].vertices.insert(v);
ReportID prefix_report = tbi->getNewNfaReport();
g[v].left.graph
= makeRoseEodPrefix(*edge_props.graph, prefix_report);
g[v].left.leftfix_report = prefix_report;
g[v].left.lag = 0;
RoseEdge e1 = add_edge(u, v, g).first;
g[e1].minBound = 0;
g[e1].maxBound = ROSE_BOUND_INF;
g[v].min_offset = add_rose_depth(
g[u].min_offset, findMinWidth(*g[v].left.graph));
g[v].max_offset = ROSE_BOUND_INF;
DEBUG_PRINTF("hi\n");
depth max_width = findMaxWidth(*g[v].left.graph);
if (u != tbi->root
&& max_width.is_finite()
&& (!tbi->isAnyStart(u)
|| isPureAnchored(*g[v].left.graph))) {
g[e1].maxBound = max_width;
g[v].max_offset = add_rose_depth(g[u].max_offset, max_width);
}
g[e1].history = ROSE_ROLE_HISTORY_NONE; // handled by prefix
u = v;
}
assert(!edge_props.haig);
RoseVertex w = add_vertex(g);
g[w].idx = tbi->vertexIndex++;
g[w].eod_accept = true;
g[w].reports = ig[iv].reports;
g[w].min_offset = g[u].min_offset;
g[w].max_offset = g[u].max_offset;
RoseEdge e = add_edge(u, w, g).first;
g[e].minBound = 0;
g[e].maxBound = 0;
g[e].history = ROSE_ROLE_HISTORY_LAST_BYTE;
DEBUG_PRINTF("accept eod vertex (idx=%zu)\n", g[w].idx);
makeEodEventLeftfix(*tbi, u, h);
}
}
}
@ -887,7 +930,8 @@ bool suitableForEod(const RoseInGraph &ig, vector<RoseInVertex> topo,
ENSURE_AT_LEAST(&v_depth, (u32)max_width);
}
if (v_depth == ROSE_BOUND_INF || v_depth > cc.grey.maxHistoryAvailable) {
if (v_depth == ROSE_BOUND_INF
|| v_depth > cc.grey.maxHistoryAvailable) {
DEBUG_PRINTF("not suitable for eod table %u\n", v_depth);
return false;
}
@ -900,6 +944,13 @@ bool suitableForEod(const RoseInGraph &ig, vector<RoseInVertex> topo,
return true;
}
static
void shift_accepts_to_end(const RoseInGraph &ig,
vector<RoseInVertex> &topo_order) {
stable_partition(begin(topo_order), end(topo_order),
[&](RoseInVertex v){ return !is_any_accept(v, ig); });
}
static
void populateRoseGraph(RoseBuildImpl *tbi, RoseBuildData &bd) {
const RoseInGraph &ig = bd.ig;
@ -912,6 +963,7 @@ void populateRoseGraph(RoseBuildImpl *tbi, RoseBuildData &bd) {
map<RoseInVertex, vector<RoseVertex> > vertex_map;
vector<RoseInVertex> v_order = topo_order(ig);
shift_accepts_to_end(ig, v_order);
u32 eod_space_required;
bool use_eod_table = suitableForEod(ig, v_order, &eod_space_required,

17
src/rose/rose_build_add_mask.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -336,7 +336,8 @@ void buildLiteralMask(const vector<CharReach> &mask, vector<u8> &msk,
}
static
bool validateTransientMask(const vector<CharReach> &mask, bool eod, const Grey &grey) {
bool validateTransientMask(const vector<CharReach> &mask, bool anchored,
bool eod, const Grey &grey) {
assert(!mask.empty());
// An EOD anchored mask requires that everything fit into history, while an
@ -348,6 +349,12 @@ bool validateTransientMask(const vector<CharReach> &mask, bool eod, const Grey &
return false;
}
/* although anchored masks cannot be transient, short masks may be placed
* into the atable. */
if (anchored && mask.size() > grey.maxAnchoredRegion) {
return false;
}
vector<ue2_literal> lits;
u32 lit_minBound; /* minBound of each literal in lit */
u32 lit_length; /* length of each literal in lit */
@ -703,7 +710,7 @@ bool checkAllowMask(const vector<CharReach> &mask, ue2_literal *lit,
bool RoseBuildImpl::add(bool anchored, const vector<CharReach> &mask,
const ue2::flat_set<ReportID> &reports) {
if (validateTransientMask(mask, false, cc.grey)) {
if (validateTransientMask(mask, anchored, false, cc.grey)) {
bool eod = false;
addTransientMask(*this, mask, reports, anchored, eod);
return true;
@ -726,8 +733,8 @@ bool RoseBuildImpl::add(bool anchored, const vector<CharReach> &mask,
bool RoseBuildImpl::validateMask(const vector<CharReach> &mask,
UNUSED const ue2::flat_set<ReportID> &reports,
UNUSED bool anchored, bool eod) const {
return validateTransientMask(mask, eod, cc.grey);
bool anchored, bool eod) const {
return validateTransientMask(mask, anchored, eod, cc.grey);
}
static

38
src/rose/rose_build_compile.cpp
View File

@ -433,6 +433,9 @@ RoseRoleHistory findHistoryScheme(const RoseBuildImpl &tbi, const RoseEdge &e) {
// If the bounds are {0,0}, this role can only match precisely at EOD.
if (minBound == 0 && maxBound == 0) {
/* last byte history will squash the state byte so cannot have other
* succ */
assert(out_degree(u, g) == 1);
return ROSE_ROLE_HISTORY_LAST_BYTE;
}
@ -915,19 +918,32 @@ void RoseBuildImpl::findTransientLeftfixes(void) {
continue;
}
u32 his = g[v].left.lag + max_width;
if (cc.streaming) {
/* STREAMING: transient prefixes must be able to run using history
* rather than storing state. */
u32 his = g[v].left.lag + max_width;
// If this vertex has an event literal, we need to add one to cope
// with it.
if (hasLiteralInTable(v, ROSE_EVENT)) {
his++;
}
// If this vertex has an event literal, we need to add one to cope
// with it.
if (hasLiteralInTable(v, ROSE_EVENT)) {
his++;
}
/* +1 as trigger must appear in main buffer and no byte is needed to
* decompress the state */
if (his <= cc.grey.maxHistoryAvailable + 1) {
transient.insert(left);
DEBUG_PRINTF("a transient leftfix has been spotted his=%u\n", his);
/* +1 as trigger must appear in main buffer and no byte is needed to
* decompress the state */
if (his <= cc.grey.maxHistoryAvailable + 1) {
transient.insert(left);
DEBUG_PRINTF("a transient leftfix spotted his=%u\n", his);
}
} else {
/* BLOCK: transientness is less important and more fuzzy, ideally
* it should be quick to calculate the state. No need to worry about
* history (and hence lag). */
if (max_width < depth(ROSE_BLOCK_TRANSIENT_MAX_WIDTH)) {
transient.insert(left);
DEBUG_PRINTF("a transient block leftfix spotted [%u]\n",
(u32)max_width);
}
}
}
}

22
src/rose/rose_build_dump.cpp
View File

@ -61,28 +61,6 @@ using namespace std;
namespace ue2 {
static
string to_string(nfa_kind k) {
switch (k) {
case NFA_PREFIX:
return "PREFIX";
case NFA_INFIX:
return "INFIX";
case NFA_SUFFIX:
return "SUFFIX";
case NFA_OUTFIX:
return "OUTFIX";
case NFA_REV_PREFIX:
return "REV_PREFIX";
case NFA_OUTFIX_RAW:
return "OUTFIX_RAW";
case NFA_EAGER_PREFIX:
return "EAGER_PREFIX";
}
assert(0);
return "?";
}
/** \brief Return the kind of a left_id or a suffix_id. */
template<class Graph>
string render_kind(const Graph &g) {

6
src/rose/rose_build_lookaround.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -684,6 +684,10 @@ bool makeLeftfixLookaround(const RoseBuildImpl &build, const RoseVertex v,
lookaround.reserve(look.size());
for (const auto &m : look) {
if (m.first < -128 || m.first > 127) {
DEBUG_PRINTF("range too big\n");
return false;
}
s8 offset = verify_s8(m.first);
lookaround.emplace_back(offset, m.second);
}

3
src/rose/rose_build_util.h
View File

@ -36,6 +36,9 @@
namespace ue2 {
/** Max allowed width for transient graphs in block mode */
#define ROSE_BLOCK_TRANSIENT_MAX_WIDTH 255U
// Comparator for vertices using their index property.
struct VertexIndexComp {
VertexIndexComp(const RoseGraph &gg) : g(gg) {}

5
src/rose/rose_in_dump.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -107,7 +107,8 @@ void dumpPreRoseGraph(const RoseInGraph &ig, const Grey &grey,
size_t id = graph_ids.size();
graph_ids[&*ig[e].graph] = id;
}
fprintf(f, "graph %zu", graph_ids[&*ig[e].graph]);
fprintf(f, "graph %zu\n%s", graph_ids[&*ig[e].graph],
to_string(ig[e].graph->kind).c_str());
}
if (ig[e].haig) {
fprintf(f, "haig ");

8
src/rose/rose_in_graph.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -106,6 +106,12 @@ public:
ROSE_BOUND_INF);
}
/* for when there is a suffix graph which handles the reports */
static RoseInVertexProps makeAcceptEod() {
return RoseInVertexProps(RIV_ACCEPT_EOD, ue2_literal(), 0,
ROSE_BOUND_INF);
}
static RoseInVertexProps makeStart(bool anchored) {
DEBUG_PRINTF("making %s\n", anchored ? "anchored start" : "start");
if (anchored) {

5
src/rose/rose_in_util.h
View File

@ -46,6 +46,11 @@ void calcVertexOffsets(RoseInGraph &ig);
enum nfa_kind whatRoseIsThis(const RoseInGraph &in, const RoseInEdge &e);
void pruneUseless(RoseInGraph &g);
inline
bool is_any_accept(RoseInVertex v, const RoseInGraph &g) {
return g[v].type == RIV_ACCEPT || g[v].type == RIV_ACCEPT_EOD;
}
}
#endif

3
src/rose/stream.c
View File

@ -510,7 +510,8 @@ void runEagerPrefixesStream(const struct RoseEngine *t,
}
void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch) {
DEBUG_PRINTF("OH HAI\n");
DEBUG_PRINTF("OH HAI [%llu, %llu)\n", scratch->core_info.buf_offset,
scratch->core_info.buf_offset + (u64a)scratch->core_info.len);
assert(t);
assert(scratch->core_info.hbuf);
assert(scratch->core_info.buf);

12
src/scratch.h
View File

@ -45,7 +45,7 @@ extern "C"
#endif
UNUSED static const u32 SCRATCH_MAGIC = 0x544F4259;
#define FDR_TEMP_BUF_SIZE 200
#define FDR_TEMP_BUF_SIZE 220
struct fatbit;
struct hs_scratch;
@ -141,7 +141,6 @@ struct match_deduper {
struct ALIGN_CL_DIRECTIVE hs_scratch {
u32 magic;
u8 in_use; /**< non-zero when being used by an API call. */
char *scratch_alloc; /* user allocated scratch object */
u32 queueCount;
u32 bStateSize; /**< sizeof block mode states */
u32 tStateSize; /**< sizeof transient rose states */
@ -161,10 +160,6 @@ struct ALIGN_CL_DIRECTIVE hs_scratch {
struct match_deduper deduper;
u32 anchored_literal_region_len;
u32 anchored_literal_count;
u32 delay_count;
u32 scratchSize;
u8 ALIGN_DIRECTIVE fdr_temp_buf[FDR_TEMP_BUF_SIZE];
u32 handledKeyCount;
struct fatbit *handled_roles; /**< fatbit of ROLES (not states) already
* handled by this literal */
u64a *som_store; /**< array of som locations */
@ -176,6 +171,11 @@ struct ALIGN_CL_DIRECTIVE hs_scratch {
* location had been writable */
u64a som_set_now_offset; /**< offset at which som_set_now represents */
u32 som_store_count;
u32 handledKeyCount;
u32 delay_count;
u32 scratchSize;
char *scratch_alloc; /* user allocated scratch object */
u8 ALIGN_DIRECTIVE fdr_temp_buf[FDR_TEMP_BUF_SIZE];
};
/* array of fatbit ptr; TODO: why not an array of fatbits? */

11
src/util/ue2string.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -55,8 +55,6 @@ size_t maxStringSelfOverlap(const std::string &a, bool nocase);
/// Compares two strings, returns non-zero if they're different.
u32 cmp(const char *a, const char *b, size_t len, bool nocase);
class CharReach;
struct ue2_literal {
public:
/// Single element proxy, pointed to by our const_iterator.
@ -124,6 +122,13 @@ public:
ue2_literal &operator=(const ue2_literal &) = default;
ue2_literal &operator=(ue2_literal &&) = default;
template<typename InputIt>
ue2_literal(InputIt b, InputIt e) {
for (; b != e; ++b) {
push_back(*b);
}
}
size_type length() const { return s.length(); }
bool empty() const { return s.empty(); }
ue2_literal substr(size_type pos, size_type n = std::string::npos) const;