github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-06-28 16:41:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

McClellan refactor

Browse Source 
Taking dfa strat out of McClellan, to be reused by other DFAs
This commit is contained in:
Anatoly Burakov 2016-05-19 14:16:35 +01:00 committed by Matthew Barr
parent a9fddbc400
commit 56bf25b091
9 changed files with 624 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CMakeLists.txt
View File

@ -597,11 +597,15 @@ SET (hs_SRCS
src/hwlm/noodle_build.h
src/hwlm/noodle_internal.h
src/nfa/accel.h
src/nfa/accel_dfa_build_strat.cpp
src/nfa/accel_dfa_build_strat.h
src/nfa/accelcompile.cpp
src/nfa/accelcompile.h
src/nfa/callback.h
src/nfa/castlecompile.cpp
src/nfa/castlecompile.h
src/nfa/dfa_build_strat.cpp
src/nfa/dfa_build_strat.h
src/nfa/dfa_min.cpp
src/nfa/dfa_min.h
src/nfa/goughcompile.cpp
@ -613,8 +617,6 @@ SET (hs_SRCS
src/nfa/mcclellan_internal.h
src/nfa/mcclellancompile.cpp
src/nfa/mcclellancompile.h
src/nfa/mcclellancompile_accel.cpp
src/nfa/mcclellancompile_accel.h
src/nfa/mcclellancompile_util.cpp
src/nfa/mcclellancompile_util.h
src/nfa/limex_compile.cpp

413
src/nfa/mcclellancompile_accel.cpp → src/nfa/accel_dfa_build_strat.cpp Normal file → Executable file
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -26,18 +26,20 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "mcclellancompile_accel.h"
#include "mcclellancompile_util.h"
#include "accel_dfa_build_strat.h"
#include "accel.h"
#include "grey.h"
#include "nfagraph/ng_limex_accel.h"
#include "shufticompile.h"
#include "trufflecompile.h"
#include "util/charreach.h"
#include "util/container.h"
#include "util/dump_charclass.h"
#include "util/verify_types.h"
#include <vector>
#include <sstream>
#include <vector>
#define PATHS_LIMIT 500
@ -46,14 +48,13 @@ using namespace std;
namespace ue2 {
namespace {
struct path {
vector<CharReach> reach;
dstate_id_t dest = DEAD_STATE;
explicit path(dstate_id_t base) : dest(base) {}
explicit path(dstate_id_t base) : dest(base) {
}
};
};
}
static UNUSED
string describeClasses(const vector<CharReach> &v) {
@ -85,8 +86,8 @@ bool is_useful_path(const vector<path> &good, const path &p) {
goto next;
}
}
DEBUG_PRINTF("better: [%s] -> %u\n",
describeClasses(g.reach).c_str(), g.dest);
DEBUG_PRINTF("better: [%s] -> %u\n", describeClasses(g.reach).c_str(),
g.dest);
return false;
next:;
@ -106,8 +107,7 @@ path append(const path &orig, const CharReach &cr, u32 new_dest) {
static
void extend(const raw_dfa &rdfa, const path &p,
map<u32, vector<path> > &all,
vector<path> &out) {
map<u32, vector<path>> &all, vector<path> &out) {
dstate s = rdfa.states[p.dest];
if (!p.reach.empty() && p.reach.back().none()) {
@ -147,17 +147,17 @@ void extend(const raw_dfa &rdfa, const path &p,
}
DEBUG_PRINTF("----good: [%s] -> %u\n",
describeClasses(pp.reach).c_str(), pp.dest);
describeClasses(pp.reach).c_str(), pp.dest);
all[e.first].push_back(pp);
out.push_back(pp);
}
}
static
vector<vector<CharReach> > generate_paths(const raw_dfa &rdfa, dstate_id_t base,
u32 len) {
vector<path> paths{ path(base) };
map<u32, vector<path> > all;
vector<vector<CharReach>> generate_paths(const raw_dfa &rdfa,
dstate_id_t base, u32 len) {
vector<path> paths{path(base)};
map<u32, vector<path>> all;
all[base].push_back(path(base));
for (u32 i = 0; i < len && paths.size() < PATHS_LIMIT; i++) {
vector<path> next_gen;
@ -170,7 +170,7 @@ vector<vector<CharReach> > generate_paths(const raw_dfa &rdfa, dstate_id_t base,
dump_paths(paths);
vector<vector<CharReach> > rv;
vector<vector<CharReach>> rv;
for (auto &p : paths) {
rv.push_back(move(p.reach));
}
@ -181,16 +181,58 @@ static
AccelScheme look_for_offset_accel(const raw_dfa &rdfa, dstate_id_t base,
u32 max_allowed_accel_offset) {
DEBUG_PRINTF("looking for accel for %hu\n", base);
vector<vector<CharReach> > paths = generate_paths(rdfa, base,
max_allowed_accel_offset + 1);
vector<vector<CharReach>> paths =
generate_paths(rdfa, base, max_allowed_accel_offset + 1);
AccelScheme as = findBestAccelScheme(paths, CharReach(), true);
DEBUG_PRINTF("found %s + %u\n", describeClass(as.cr).c_str(), as.offset);
return as;
}
static UNUSED
bool better(const AccelScheme &a, const AccelScheme &b) {
if (!a.double_byte.empty() && b.double_byte.empty()) {
return true;
}
if (!b.double_byte.empty()) {
return false;
}
return a.cr.count() < b.cr.count();
}
static
vector<CharReach> reverse_alpha_remapping(const raw_dfa &rdfa) {
vector<CharReach> rv(rdfa.alpha_size - 1); /* TOP not required */
for (u32 i = 0; i < N_CHARS; i++) {
rv.at(rdfa.alpha_remap[i]).set(i);
}
return rv;
}
static
bool double_byte_ok(const AccelScheme &info) {
return !info.double_byte.empty() &&
info.double_cr.count() < info.double_byte.size() &&
info.double_cr.count() <= 2 && !info.double_byte.empty();
}
static
bool has_self_loop(dstate_id_t s, const raw_dfa &raw) {
u16 top_remap = raw.alpha_remap[TOP];
for (u32 i = 0; i < raw.states[s].next.size(); i++) {
if (i != top_remap && raw.states[s].next[i] == s) {
return true;
}
}
return false;
}
static
vector<u16> find_nonexit_symbols(const raw_dfa &rdfa,
const CharReach &escape) {
const CharReach &escape) {
set<u16> rv;
CharReach nonexit = ~escape;
for (auto i = nonexit.find_first(); i != CharReach::npos;
@ -201,9 +243,58 @@ vector<u16> find_nonexit_symbols(const raw_dfa &rdfa,
return vector<u16>(rv.begin(), rv.end());
}
static
dstate_id_t get_sds_or_proxy(const raw_dfa &raw) {
if (raw.start_floating != DEAD_STATE) {
DEBUG_PRINTF("has floating start\n");
return raw.start_floating;
}
DEBUG_PRINTF("looking for SDS proxy\n");
dstate_id_t s = raw.start_anchored;
if (has_self_loop(s, raw)) {
return s;
}
u16 top_remap = raw.alpha_remap[TOP];
ue2::unordered_set<dstate_id_t> seen;
while (true) {
seen.insert(s);
DEBUG_PRINTF("basis %hu\n", s);
/* check if we are connected to a state with a self loop */
for (u32 i = 0; i < raw.states[s].next.size(); i++) {
dstate_id_t t = raw.states[s].next[i];
if (i != top_remap && t != DEAD_STATE && has_self_loop(t, raw)) {
return t;
}
}
/* find a neighbour to use as a basis for looking for the sds proxy */
dstate_id_t t = DEAD_STATE;
for (u32 i = 0; i < raw.states[s].next.size(); i++) {
dstate_id_t tt = raw.states[s].next[i];
if (i != top_remap && tt != DEAD_STATE && !contains(seen, tt)) {
t = tt;
break;
}
}
if (t == DEAD_STATE) {
/* we were unable to find a state to use as a SDS proxy */
return DEAD_STATE;
}
s = t;
}
}
static
set<dstate_id_t> find_region(const raw_dfa &rdfa, dstate_id_t base,
const AccelScheme &ei) {
const AccelScheme &ei) {
DEBUG_PRINTF("looking for region around %hu\n", base);
set<dstate_id_t> region = {base};
@ -236,98 +327,10 @@ set<dstate_id_t> find_region(const raw_dfa &rdfa, dstate_id_t base,
return region;
}
static
bool better(const AccelScheme &a, const AccelScheme &b) {
if (!a.double_byte.empty() && b.double_byte.empty()) {
return true;
}
if (!b.double_byte.empty()) {
return false;
}
return a.cr.count() < b.cr.count();
}
static
vector<CharReach> reverse_alpha_remapping(const raw_dfa &rdfa) {
vector<CharReach> rv(rdfa.alpha_size - 1); /* TOP not required */
for (u32 i = 0; i < N_CHARS; i++) {
rv.at(rdfa.alpha_remap[i]).set(i);
}
return rv;
}
map<dstate_id_t, AccelScheme> populateAccelerationInfo(const raw_dfa &rdfa,
const dfa_build_strat &strat,
const Grey &grey) {
map<dstate_id_t, AccelScheme> rv;
if (!grey.accelerateDFA) {
return rv;
}
dstate_id_t sds_proxy = get_sds_or_proxy(rdfa);
DEBUG_PRINTF("sds %hu\n", sds_proxy);
for (size_t i = 0; i < rdfa.states.size(); i++) {
if (i == DEAD_STATE) {
continue;
}
/* Note on report acceleration states: While we can't accelerate while we
* are spamming out callbacks, the QR code paths don't raise reports
* during scanning so they can accelerate report states. */
if (generates_callbacks(rdfa.kind) && !rdfa.states[i].reports.empty()) {
continue;
}
size_t single_limit = i == sds_proxy ? ACCEL_DFA_MAX_FLOATING_STOP_CHAR
: ACCEL_DFA_MAX_STOP_CHAR;
DEBUG_PRINTF("inspecting %zu/%hu: %zu\n", i, sds_proxy, single_limit);
AccelScheme ei = strat.find_escape_strings(i);
if (ei.cr.count() > single_limit) {
DEBUG_PRINTF("state %zu is not accelerable has %zu\n", i,
ei.cr.count());
continue;
}
DEBUG_PRINTF("state %zu should be accelerable %zu\n",
i, ei.cr.count());
rv[i] = ei;
}
/* provide accleration states to states in the region of sds */
if (contains(rv, sds_proxy)) {
AccelScheme sds_ei = rv[sds_proxy];
sds_ei.double_byte.clear(); /* region based on single byte scheme
* may differ from double byte */
DEBUG_PRINTF("looking to expand offset accel to nearby states, %zu\n",
sds_ei.cr.count());
auto sds_region = find_region(rdfa, sds_proxy, sds_ei);
for (auto s : sds_region) {
if (!contains(rv, s) || better(sds_ei, rv[s])) {
rv[s] = sds_ei;
}
}
}
return rv;
}
static
bool double_byte_ok(const AccelScheme &info) {
return !info.double_byte.empty()
&& info.double_cr.count() < info.double_byte.size()
&& info.double_cr.count() <= 2 && !info.double_byte.empty();
}
AccelScheme find_mcclellan_escape_info(const raw_dfa &rdfa, dstate_id_t this_idx,
u32 max_allowed_accel_offset) {
AccelScheme
accel_dfa_build_strat::find_escape_strings(dstate_id_t this_idx) const {
AccelScheme rv;
const raw_dfa &rdfa = get_raw();
rv.cr.clear();
rv.offset = 0;
const dstate &raw = rdfa.states[this_idx];
@ -354,7 +357,7 @@ AccelScheme find_mcclellan_escape_info(const raw_dfa &rdfa, dstate_id_t this_idx
if (!raw_next.reports.empty() && generates_callbacks(rdfa.kind)) {
DEBUG_PRINTF("leads to report\n");
outs2_broken = true; /* cannot accelerate over reports */
outs2_broken = true; /* cannot accelerate over reports */
continue;
}
succs[next_id] |= cr_i;
@ -402,14 +405,12 @@ AccelScheme find_mcclellan_escape_info(const raw_dfa &rdfa, dstate_id_t this_idx
DEBUG_PRINTF("this %u, sds proxy %hu\n", this_idx, get_sds_or_proxy(rdfa));
DEBUG_PRINTF("broken %d\n", outs2_broken);
if (!double_byte_ok(rv) && !is_triggered(rdfa.kind)
&& this_idx == rdfa.start_floating
&& this_idx != DEAD_STATE) {
if (!double_byte_ok(rv) && !is_triggered(rdfa.kind) &&
this_idx == rdfa.start_floating && this_idx != DEAD_STATE) {
DEBUG_PRINTF("looking for offset accel at %u\n", this_idx);
auto offset = look_for_offset_accel(rdfa, this_idx,
max_allowed_accel_offset);
DEBUG_PRINTF("width %zu vs %zu\n", offset.cr.count(),
rv.cr.count());
auto offset =
look_for_offset_accel(rdfa, this_idx, max_allowed_offset_accel());
DEBUG_PRINTF("width %zu vs %zu\n", offset.cr.count(), rv.cr.count());
if (double_byte_ok(offset) || offset.cr.count() < rv.cr.count()) {
DEBUG_PRINTF("using offset accel\n");
rv = offset;
@ -419,4 +420,172 @@ AccelScheme find_mcclellan_escape_info(const raw_dfa &rdfa, dstate_id_t this_idx
return rv;
}
void
accel_dfa_build_strat::buildAccel(UNUSED dstate_id_t this_idx,
const AccelScheme &info,
void *accel_out) {
AccelAux *accel = (AccelAux *)accel_out;
DEBUG_PRINTF("accelerations scheme has offset s%u/d%u\n", info.offset,
info.double_offset);
accel->generic.offset = verify_u8(info.offset);
if (double_byte_ok(info) && info.double_cr.none() &&
info.double_byte.size() == 1) {
accel->accel_type = ACCEL_DVERM;
accel->dverm.c1 = info.double_byte.begin()->first;
accel->dverm.c2 = info.double_byte.begin()->second;
accel->dverm.offset = verify_u8(info.double_offset);
DEBUG_PRINTF("state %hu is double vermicelli\n", this_idx);
return;
}
if (double_byte_ok(info) && info.double_cr.none() &&
(info.double_byte.size() == 2 || info.double_byte.size() == 4)) {
bool ok = true;
assert(!info.double_byte.empty());
u8 firstC = info.double_byte.begin()->first & CASE_CLEAR;
u8 secondC = info.double_byte.begin()->second & CASE_CLEAR;
for (const pair<u8, u8> &p : info.double_byte) {
if ((p.first & CASE_CLEAR) != firstC ||
(p.second & CASE_CLEAR) != secondC) {
ok = false;
break;
}
}
if (ok) {
accel->accel_type = ACCEL_DVERM_NOCASE;
accel->dverm.c1 = firstC;
accel->dverm.c2 = secondC;
accel->dverm.offset = verify_u8(info.double_offset);
DEBUG_PRINTF("state %hu is nc double vermicelli\n", this_idx);
return;
}
u8 m1;
u8 m2;
if (buildDvermMask(info.double_byte, &m1, &m2)) {
accel->accel_type = ACCEL_DVERM_MASKED;
accel->dverm.offset = verify_u8(info.double_offset);
accel->dverm.c1 = info.double_byte.begin()->first & m1;
accel->dverm.c2 = info.double_byte.begin()->second & m2;
accel->dverm.m1 = m1;
accel->dverm.m2 = m2;
DEBUG_PRINTF(
"building maskeddouble-vermicelli for 0x%02hhx%02hhx\n",
accel->dverm.c1, accel->dverm.c2);
return;
}
}
if (double_byte_ok(info) &&
shuftiBuildDoubleMasks(info.double_cr, info.double_byte,
&accel->dshufti.lo1, &accel->dshufti.hi1,
&accel->dshufti.lo2, &accel->dshufti.hi2)) {
accel->accel_type = ACCEL_DSHUFTI;
accel->dshufti.offset = verify_u8(info.double_offset);
DEBUG_PRINTF("state %hu is double shufti\n", this_idx);
return;
}
if (info.cr.none()) {
accel->accel_type = ACCEL_RED_TAPE;
DEBUG_PRINTF("state %hu is a dead end full of bureaucratic red tape"
" from which there is no escape\n",
this_idx);
return;
}
if (info.cr.count() == 1) {
accel->accel_type = ACCEL_VERM;
accel->verm.c = info.cr.find_first();
DEBUG_PRINTF("state %hu is vermicelli\n", this_idx);
return;
}
if (info.cr.count() == 2 && info.cr.isCaselessChar()) {
accel->accel_type = ACCEL_VERM_NOCASE;
accel->verm.c = info.cr.find_first() & CASE_CLEAR;
DEBUG_PRINTF("state %hu is caseless vermicelli\n", this_idx);
return;
}
if (info.cr.count() > max_floating_stop_char()) {
accel->accel_type = ACCEL_NONE;
DEBUG_PRINTF("state %hu is too broad\n", this_idx);
return;
}
accel->accel_type = ACCEL_SHUFTI;
if (-1 != shuftiBuildMasks(info.cr, &accel->shufti.lo, &accel->shufti.hi)) {
DEBUG_PRINTF("state %hu is shufti\n", this_idx);
return;
}
assert(!info.cr.none());
accel->accel_type = ACCEL_TRUFFLE;
truffleBuildMasks(info.cr, &accel->truffle.mask1, &accel->truffle.mask2);
DEBUG_PRINTF("state %hu is truffle\n", this_idx);
}
map<dstate_id_t, AccelScheme>
accel_dfa_build_strat::getAccelInfo(const Grey &grey) {
map<dstate_id_t, AccelScheme> rv;
raw_dfa &rdfa = get_raw();
if (!grey.accelerateDFA) {
return rv;
}
dstate_id_t sds_proxy = get_sds_or_proxy(rdfa);
DEBUG_PRINTF("sds %hu\n", sds_proxy);
for (size_t i = 0; i < rdfa.states.size(); i++) {
if (i == DEAD_STATE) {
continue;
}
/* Note on report acceleration states: While we can't accelerate while
* we
* are spamming out callbacks, the QR code paths don't raise reports
* during scanning so they can accelerate report states. */
if (generates_callbacks(rdfa.kind) && !rdfa.states[i].reports.empty()) {
continue;
}
size_t single_limit =
i == sds_proxy ? max_floating_stop_char() : max_stop_char();
DEBUG_PRINTF("inspecting %zu/%hu: %zu\n", i, sds_proxy, single_limit);
AccelScheme ei = find_escape_strings(i);
if (ei.cr.count() > single_limit) {
DEBUG_PRINTF("state %zu is not accelerable has %zu\n", i,
ei.cr.count());
continue;
}
DEBUG_PRINTF("state %zu should be accelerable %zu\n", i, ei.cr.count());
rv[i] = ei;
}
/* provide accleration states to states in the region of sds */
if (contains(rv, sds_proxy)) {
AccelScheme sds_ei = rv[sds_proxy];
sds_ei.double_byte.clear(); /* region based on single byte scheme
* may differ from double byte */
DEBUG_PRINTF("looking to expand offset accel to nearby states, %zu\n",
sds_ei.cr.count());
auto sds_region = find_region(rdfa, sds_proxy, sds_ei);
for (auto s : sds_region) {
if (!contains(rv, s) || better(sds_ei, rv[s])) {
rv[s] = sds_ei;
}
}
}
return rv;
}
};

47
src/nfa/mcclellancompile_accel.h → src/nfa/accel_dfa_build_strat.h Normal file → Executable file
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, Intel Corporation
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -26,36 +26,35 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef MCCLELLANCOMPILE_ACCEL_H
#define MCCLELLANCOMPILE_ACCEL_H
#ifndef ACCEL_DFA_BUILD_STRAT_H
#define ACCEL_DFA_BUILD_STRAT_H
#include "mcclellancompile.h"
#include "rdfa.h"
#include "dfa_build_strat.h"
#include "ue2common.h"
#include "util/accel_scheme.h"
#include <map>
namespace ue2 {
class ReportManager;
struct Grey;
#define ACCEL_DFA_MAX_OFFSET_DEPTH 4
class accel_dfa_build_strat : public dfa_build_strat {
public:
explicit accel_dfa_build_strat(const ReportManager &rm_in)
: dfa_build_strat(rm_in) {}
virtual AccelScheme find_escape_strings(dstate_id_t this_idx) const;
virtual size_t accelSize(void) const = 0;
virtual u32 max_allowed_offset_accel() const = 0;
virtual u32 max_stop_char() const = 0;
virtual u32 max_floating_stop_char() const = 0;
virtual void buildAccel(dstate_id_t this_idx, const AccelScheme &info,
void *accel_out);
virtual std::map<dstate_id_t, AccelScheme> getAccelInfo(const Grey &grey);
};
/** Maximum tolerated number of escape character from an accel state.
* This is larger than nfa, as we don't have a budget and the nfa cheats on stop
* characters for sets of states */
#define ACCEL_DFA_MAX_STOP_CHAR 160
} // namespace ue2
/** Maximum tolerated number of escape character from a sds accel state. Larger
* than normal states as accelerating sds is important. Matches NFA value */
#define ACCEL_DFA_MAX_FLOATING_STOP_CHAR 192
std::map<dstate_id_t, AccelScheme> populateAccelerationInfo(const raw_dfa &rdfa,
const dfa_build_strat &strat,
const Grey &grey);
AccelScheme find_mcclellan_escape_info(const raw_dfa &rdfa,
dstate_id_t this_idx,
u32 max_allowed_accel_offset);
}
#endif
#endif // ACCEL_DFA_BUILD_STRAT_H

211
src/nfa/dfa_build_strat.cpp Executable file
View File

@ -0,0 +1,211 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "dfa_build_strat.h"
#include "accel.h"
#include "accelcompile.h"
#include "grey.h"
#include "mcclellan_internal.h"
#include "mcclellancompile_util.h"
#include "nfa_internal.h"
#include "shufticompile.h"
#include "trufflecompile.h"
#include "ue2common.h"
#include "util/alloc.h"
#include "util/bitutils.h"
#include "util/charreach.h"
#include "util/compare.h"
#include "util/compile_context.h"
#include "util/container.h"
#include "util/make_unique.h"
#include "util/order_check.h"
#include "util/report_manager.h"
#include "util/ue2_containers.h"
#include "util/unaligned.h"
#include "util/verify_types.h"
#include <vector>
using namespace std;
namespace ue2 {
// prevent weak vtables for raw_report_info, dfa_build_strat and raw_dfa
raw_report_info::~raw_report_info() {
}
dfa_build_strat::~dfa_build_strat() {
}
raw_dfa::~raw_dfa() {
}
namespace {
struct raw_report_list {
flat_set<ReportID> reports;
raw_report_list(const flat_set<ReportID> &reports_in,
const ReportManager &rm, bool do_remap) {
if (do_remap) {
for (auto &id : reports_in) {
reports.insert(rm.getProgramOffset(id));
}
} else {
reports = reports_in;
}
}
bool operator<(const raw_report_list &b) const {
return reports < b.reports;
}
};
struct raw_report_info_impl : public raw_report_info {
vector<raw_report_list> rl;
u32 getReportListSize() const override;
size_t size() const override;
void fillReportLists(NFA *n, size_t base_offset,
std::vector<u32> &ro /* out */) const override;
};
}
unique_ptr<raw_report_info>
dfa_build_strat::gatherReports(vector<u32> &reports, vector<u32> &reports_eod,
u8 *isSingleReport, ReportID *arbReport) const {
auto &rdfa = get_raw();
DEBUG_PRINTF("gathering reports\n");
const bool remap_reports = has_managed_reports(rdfa.kind);
auto ri = ue2::make_unique<raw_report_info_impl>();
map<raw_report_list, u32> rev;
for (const dstate &s : rdfa.states) {
if (s.reports.empty()) {
reports.push_back(MO_INVALID_IDX);
continue;
}
raw_report_list rrl(s.reports, rm, remap_reports);
DEBUG_PRINTF("non empty r\n");
if (rev.find(rrl) != rev.end()) {
reports.push_back(rev[rrl]);
} else {
DEBUG_PRINTF("adding to rl %zu\n", ri->size());
rev[rrl] = ri->size();
reports.push_back(ri->size());
ri->rl.push_back(rrl);
}
}
for (const dstate &s : rdfa.states) {
if (s.reports_eod.empty()) {
reports_eod.push_back(MO_INVALID_IDX);
continue;
}
DEBUG_PRINTF("non empty r eod\n");
raw_report_list rrl(s.reports_eod, rm, remap_reports);
if (rev.find(rrl) != rev.end()) {
reports_eod.push_back(rev[rrl]);
continue;
}
DEBUG_PRINTF("adding to rl eod %zu\n", s.reports_eod.size());
rev[rrl] = ri->size();
reports_eod.push_back(ri->size());
ri->rl.push_back(rrl);
}
assert(!ri->rl.empty()); /* all components should be able to generate
reports */
if (!ri->rl.empty()) {
*arbReport = *ri->rl.begin()->reports.begin();
} else {
*arbReport = 0;
}
/* if we have only a single report id generated from all accepts (not eod)
* we can take some short cuts */
set<ReportID> reps;
for (u32 rl_index : reports) {
if (rl_index == MO_INVALID_IDX) {
continue;
}
assert(rl_index < ri->size());
insert(&reps, ri->rl[rl_index].reports);
}
if (reps.size() == 1) {
*isSingleReport = 1;
*arbReport = *reps.begin();
DEBUG_PRINTF("single -- %u\n", *arbReport);
} else {
*isSingleReport = 0;
}
return move(ri);
}
u32 raw_report_info_impl::getReportListSize() const {
u32 rv = 0;
for (const auto &reps : rl) {
rv += sizeof(report_list);
rv += sizeof(ReportID) * reps.reports.size();
}
return rv;
}
size_t raw_report_info_impl::size() const {
return rl.size();
}
void raw_report_info_impl::fillReportLists(NFA *n, size_t base_offset,
vector<u32> &ro) const {
for (const auto &reps : rl) {
ro.push_back(base_offset);
report_list *p = (report_list *)((char *)n + base_offset);
u32 i = 0;
for (const ReportID report : reps.reports) {
p->report[i++] = report;
}
p->count = verify_u32(reps.reports.size());
base_offset += sizeof(report_list);
base_offset += sizeof(ReportID) * reps.reports.size();
}
}
} // namespace ue2

68
src/nfa/dfa_build_strat.h Normal file
View File

@ -0,0 +1,68 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef DFA_BUILD_STRAT_H
#define DFA_BUILD_STRAT_H
#include "rdfa.h"
#include "ue2common.h"
#include <memory>
#include <vector>
struct NFA;
namespace ue2 {
class ReportManager;
struct raw_report_info {
virtual ~raw_report_info();
virtual u32 getReportListSize() const = 0; /* in bytes */
virtual size_t size() const = 0; /* number of lists */
virtual void fillReportLists(NFA *n, size_t base_offset,
std::vector<u32> &ro /* out */) const = 0;
};
class dfa_build_strat {
public:
explicit dfa_build_strat(const ReportManager &rm_in) : rm(rm_in) {}
virtual ~dfa_build_strat();
virtual raw_dfa &get_raw() const = 0;
virtual std::unique_ptr<raw_report_info> gatherReports(
std::vector<u32> &reports /* out */,
std::vector<u32> &reports_eod /* out */,
u8 *isSingleReport /* out */,
ReportID *arbReport /* out */) const = 0;
protected:
const ReportManager &rm;
};
} // namespace ue2
#endif // DFA_BUILD_STRAT_H

157
src/nfa/mcclellancompile.cpp
View File

@ -32,7 +32,6 @@
#include "accelcompile.h"
#include "grey.h"
#include "mcclellan_internal.h"
#include "mcclellancompile_accel.h"
#include "mcclellancompile_util.h"
#include "nfa_internal.h"
#include "shufticompile.h"
@ -65,6 +64,17 @@
using namespace std;
using boost::adaptors::map_keys;
#define ACCEL_DFA_MAX_OFFSET_DEPTH 4
/** Maximum tolerated number of escape character from an accel state.
* This is larger than nfa, as we don't have a budget and the nfa cheats on stop
* characters for sets of states */
#define ACCEL_DFA_MAX_STOP_CHAR 160
/** Maximum tolerated number of escape character from a sds accel state. Larger
* than normal states as accelerating sds is important. Matches NFA value */
#define ACCEL_DFA_MAX_FLOATING_STOP_CHAR 192
namespace ue2 {
namespace /* anon */ {
@ -75,7 +85,7 @@ struct dstate_extra {
};
struct dfa_info {
dfa_build_strat &strat;
accel_dfa_build_strat &strat;
raw_dfa &raw;
vector<dstate> &states;
vector<dstate_extra> extra;
@ -85,7 +95,7 @@ struct dfa_info {
u8 getAlphaShift() const;
explicit dfa_info(dfa_build_strat &s)
explicit dfa_info(accel_dfa_build_strat &s)
: strat(s),
raw(s.get_raw()),
states(raw.states),
@ -128,13 +138,6 @@ mstate_aux *getAux(NFA *n, dstate_id_t i) {
return aux;
}
static
bool double_byte_ok(const AccelScheme &info) {
return !info.double_byte.empty()
&& info.double_cr.count() < info.double_byte.size()
&& info.double_cr.count() <= 2 && !info.double_byte.empty();
}
static
void markEdges(NFA *n, u16 *succ_table, const dfa_info &info) {
assert((size_t)succ_table % 2 == 0);
@ -190,120 +193,12 @@ u32 mcclellan_build_strat::max_allowed_offset_accel() const {
return ACCEL_DFA_MAX_OFFSET_DEPTH;
}
AccelScheme mcclellan_build_strat::find_escape_strings(dstate_id_t this_idx)
const {
return find_mcclellan_escape_info(rdfa, this_idx,
max_allowed_offset_accel());
u32 mcclellan_build_strat::max_stop_char() const {
return ACCEL_DFA_MAX_STOP_CHAR;
}
/** builds acceleration schemes for states */
void mcclellan_build_strat::buildAccel(UNUSED dstate_id_t this_idx,
const AccelScheme &info,
void *accel_out) {
AccelAux *accel = (AccelAux *)accel_out;
DEBUG_PRINTF("accelerations scheme has offset s%u/d%u\n", info.offset,
info.double_offset);
accel->generic.offset = verify_u8(info.offset);
if (double_byte_ok(info) && info.double_cr.none()
&& info.double_byte.size() == 1) {
accel->accel_type = ACCEL_DVERM;
accel->dverm.c1 = info.double_byte.begin()->first;
accel->dverm.c2 = info.double_byte.begin()->second;
accel->dverm.offset = verify_u8(info.double_offset);
DEBUG_PRINTF("state %hu is double vermicelli\n", this_idx);
return;
}
if (double_byte_ok(info) && info.double_cr.none()
&& (info.double_byte.size() == 2 || info.double_byte.size() == 4)) {
bool ok = true;
assert(!info.double_byte.empty());
u8 firstC = info.double_byte.begin()->first & CASE_CLEAR;
u8 secondC = info.double_byte.begin()->second & CASE_CLEAR;
for (const pair<u8, u8> &p : info.double_byte) {
if ((p.first & CASE_CLEAR) != firstC
|| (p.second & CASE_CLEAR) != secondC) {
ok = false;
break;
}
}
if (ok) {
accel->accel_type = ACCEL_DVERM_NOCASE;
accel->dverm.c1 = firstC;
accel->dverm.c2 = secondC;
accel->dverm.offset = verify_u8(info.double_offset);
DEBUG_PRINTF("state %hu is nc double vermicelli\n", this_idx);
return;
}
u8 m1;
u8 m2;
if (buildDvermMask(info.double_byte, &m1, &m2)) {
accel->accel_type = ACCEL_DVERM_MASKED;
accel->dverm.offset = verify_u8(info.double_offset);
accel->dverm.c1 = info.double_byte.begin()->first & m1;
accel->dverm.c2 = info.double_byte.begin()->second & m2;
accel->dverm.m1 = m1;
accel->dverm.m2 = m2;
DEBUG_PRINTF("building maskeddouble-vermicelli for 0x%02hhx%02hhx\n",
accel->dverm.c1, accel->dverm.c2);
return;
}
}
if (double_byte_ok(info)
&& shuftiBuildDoubleMasks(info.double_cr, info.double_byte,
&accel->dshufti.lo1, &accel->dshufti.hi1,
&accel->dshufti.lo2, &accel->dshufti.hi2)) {
accel->accel_type = ACCEL_DSHUFTI;
accel->dshufti.offset = verify_u8(info.double_offset);
DEBUG_PRINTF("state %hu is double shufti\n", this_idx);
return;
}
if (info.cr.none()) {
accel->accel_type = ACCEL_RED_TAPE;
DEBUG_PRINTF("state %hu is a dead end full of bureaucratic red tape"
" from which there is no escape\n", this_idx);
return;
}
if (info.cr.count() == 1) {
accel->accel_type = ACCEL_VERM;
accel->verm.c = info.cr.find_first();
DEBUG_PRINTF("state %hu is vermicelli\n", this_idx);
return;
}
if (info.cr.count() == 2 && info.cr.isCaselessChar()) {
accel->accel_type = ACCEL_VERM_NOCASE;
accel->verm.c = info.cr.find_first() & CASE_CLEAR;
DEBUG_PRINTF("state %hu is caseless vermicelli\n", this_idx);
return;
}
if (info.cr.count() > ACCEL_DFA_MAX_FLOATING_STOP_CHAR) {
accel->accel_type = ACCEL_NONE;
DEBUG_PRINTF("state %hu is too broad\n", this_idx);
return;
}
accel->accel_type = ACCEL_SHUFTI;
if (-1 != shuftiBuildMasks(info.cr, &accel->shufti.lo,
&accel->shufti.hi)) {
DEBUG_PRINTF("state %hu is shufti\n", this_idx);
return;
}
assert(!info.cr.none());
accel->accel_type = ACCEL_TRUFFLE;
truffleBuildMasks(info.cr, &accel->truffle.mask1, &accel->truffle.mask2);
DEBUG_PRINTF("state %hu is truffle\n", this_idx);
u32 mcclellan_build_strat::max_floating_stop_char() const {
return ACCEL_DFA_MAX_FLOATING_STOP_CHAR;
}
static
@ -343,15 +238,6 @@ void populateBasicInfo(size_t state_size, const dfa_info &info,
}
}
raw_dfa::~raw_dfa() {
}
raw_report_info::raw_report_info() {
}
raw_report_info::~raw_report_info() {
}
namespace {
struct raw_report_list {
@ -592,7 +478,7 @@ aligned_unique_ptr<NFA> mcclellanCompile16(dfa_info &info,
auto ri = info.strat.gatherReports(reports, reports_eod, &single, &arb);
map<dstate_id_t, AccelScheme> accel_escape_info
= populateAccelerationInfo(info.raw, info.strat, cc.grey);
= info.strat.getAccelInfo(cc.grey);
size_t tran_size = (1 << info.getAlphaShift())
* sizeof(u16) * count_real_states;
@ -811,7 +697,7 @@ aligned_unique_ptr<NFA> mcclellanCompile8(dfa_info &info,
auto ri = info.strat.gatherReports(reports, reports_eod, &single, &arb);
map<dstate_id_t, AccelScheme> accel_escape_info
= populateAccelerationInfo(info.raw, info.strat, cc.grey);
= info.strat.getAccelInfo(cc.grey);
size_t tran_size = sizeof(u8) * (1 << info.getAlphaShift()) * info.size();
size_t aux_size = sizeof(mstate_aux) * info.size();
@ -1053,7 +939,7 @@ bool is_cyclic_near(const raw_dfa &raw, dstate_id_t root) {
return false;
}
aligned_unique_ptr<NFA> mcclellanCompile_i(raw_dfa &raw, dfa_build_strat &strat,
aligned_unique_ptr<NFA> mcclellanCompile_i(raw_dfa &raw, accel_dfa_build_strat &strat,
const CompileContext &cc,
set<dstate_id_t> *accel_states) {
u16 total_daddy = 0;
@ -1128,7 +1014,4 @@ bool has_accel_dfa(const NFA *nfa) {
return m->has_accel;
}
dfa_build_strat::~dfa_build_strat() {
}
} // namespace ue2

41
src/nfa/mcclellancompile.h
View File

@ -29,6 +29,7 @@
#ifndef MCCLELLANCOMPILE_H
#define MCCLELLANCOMPILE_H
#include "accel_dfa_build_strat.h"
#include "rdfa.h"
#include "ue2common.h"
#include "util/accel_scheme.h"
@ -47,48 +48,20 @@ namespace ue2 {
class ReportManager;
struct CompileContext;
struct raw_report_info {
raw_report_info();
virtual ~raw_report_info();
virtual u32 getReportListSize() const = 0; /* in bytes */
virtual size_t size() const = 0; /* number of lists */
virtual void fillReportLists(NFA *n, size_t base_offset,
std::vector<u32> &ro /* out */) const = 0;
};
class dfa_build_strat {
public:
explicit dfa_build_strat(const ReportManager &rm_in) : rm(rm_in) {}
virtual ~dfa_build_strat();
virtual raw_dfa &get_raw() const = 0;
virtual std::unique_ptr<raw_report_info> gatherReports(
std::vector<u32> &reports /* out */,
std::vector<u32> &reports_eod /* out */,
u8 *isSingleReport /* out */,
ReportID *arbReport /* out */) const = 0;
virtual AccelScheme find_escape_strings(dstate_id_t this_idx) const = 0;
virtual size_t accelSize(void) const = 0;
virtual void buildAccel(dstate_id_t this_idx, const AccelScheme &info,
void *accel_out) = 0;
protected:
const ReportManager &rm;
};
class mcclellan_build_strat : public dfa_build_strat {
class mcclellan_build_strat : public accel_dfa_build_strat {
public:
mcclellan_build_strat(raw_dfa &rdfa_in, const ReportManager &rm_in)
: dfa_build_strat(rm_in), rdfa(rdfa_in) {}
: accel_dfa_build_strat(rm_in), rdfa(rdfa_in) {}
raw_dfa &get_raw() const override { return rdfa; }
std::unique_ptr<raw_report_info> gatherReports(
std::vector<u32> &reports /* out */,
std::vector<u32> &reports_eod /* out */,
u8 *isSingleReport /* out */,
ReportID *arbReport /* out */) const override;
AccelScheme find_escape_strings(dstate_id_t this_idx) const override;
size_t accelSize(void) const override;
void buildAccel(dstate_id_t this_idx,const AccelScheme &info,
void *accel_out) override;
virtual u32 max_allowed_offset_accel() const;
u32 max_allowed_offset_accel() const override;
u32 max_stop_char() const override;
u32 max_floating_stop_char() const override;
private:
raw_dfa &rdfa;
@ -103,7 +76,7 @@ mcclellanCompile(raw_dfa &raw, const CompileContext &cc,
/* used internally by mcclellan/haig/gough compile process */
ue2::aligned_unique_ptr<NFA>
mcclellanCompile_i(raw_dfa &raw, dfa_build_strat &strat,
mcclellanCompile_i(raw_dfa &raw, accel_dfa_build_strat &strat,
const CompileContext &cc,
std::set<dstate_id_t> *accel_states = nullptr);

59
src/nfa/mcclellancompile_util.cpp
View File

@ -336,65 +336,6 @@ size_t hash_dfa(const raw_dfa &rdfa) {
return v;
}
static
bool has_self_loop(dstate_id_t s, const raw_dfa &raw) {
u16 top_remap = raw.alpha_remap[TOP];
for (u32 i = 0; i < raw.states[s].next.size(); i++) {
if (i != top_remap && raw.states[s].next[i] == s) {
return true;
}
}
return false;
}
dstate_id_t get_sds_or_proxy(const raw_dfa &raw) {
if (raw.start_floating != DEAD_STATE) {
DEBUG_PRINTF("has floating start\n");
return raw.start_floating;
}
DEBUG_PRINTF("looking for SDS proxy\n");
dstate_id_t s = raw.start_anchored;
if (has_self_loop(s, raw)) {
return s;
}
u16 top_remap = raw.alpha_remap[TOP];
ue2::unordered_set<dstate_id_t> seen;
while (true) {
seen.insert(s);
DEBUG_PRINTF("basis %hu\n", s);
/* check if we are connected to a state with a self loop */
for (u32 i = 0; i < raw.states[s].next.size(); i++) {
dstate_id_t t = raw.states[s].next[i];
if (i != top_remap && t != DEAD_STATE && has_self_loop(t, raw)) {
return t;
}
}
/* find a neighbour to use as a basis for looking for the sds proxy */
dstate_id_t t = DEAD_STATE;
for (u32 i = 0; i < raw.states[s].next.size(); i++) {
dstate_id_t tt = raw.states[s].next[i];
if (i != top_remap && tt != DEAD_STATE && !contains(seen, tt)) {
t = tt;
break;
}
}
if (t == DEAD_STATE) {
/* we were unable to find a state to use as a SDS proxy */
return DEAD_STATE;
}
s = t;
}
}
static
bool can_die_early(const raw_dfa &raw, dstate_id_t s,
map<dstate_id_t, u32> &visited, u32 age_limit) {

2
src/nfa/mcclellancompile_util.h
View File

@ -55,8 +55,6 @@ size_t hash_dfa_no_reports(const raw_dfa &rdfa);
/** \brief Compute a simple hash of this raw_dfa, including its reports. */
size_t hash_dfa(const raw_dfa &rdfa);
dstate_id_t get_sds_or_proxy(const raw_dfa &raw);
bool can_die_early(const raw_dfa &raw, u32 age_limit);
} // namespace ue2