github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-09-29 19:24:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

detach the sidecar

Browse Source
This commit is contained in:
Alex Coyte
2015-11-26 12:44:56 +11:00
committed by Matthew Barr
parent e065c4d60b
commit a7d8dafb71
37 changed files with 37 additions and 3001 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/rose/block.c
View File

@@ -34,7 +34,6 @@
#include "nfa/nfa_rev_api.h"
#include "nfa/mcclellan.h"
#include "util/fatbit.h"
#include "rose_sidecar_runtime.h"
#include "rose.h"
#include "rose_common.h"
@@ -78,20 +77,6 @@ void runAnchoredTableBlock(const struct RoseEngine *t, const void *atable,
} while (1);
}
static really_inline
void init_sidecar(const struct RoseEngine *t, struct hs_scratch *scratch) {
if (!t->smatcherOffset) {
return;
}
DEBUG_PRINTF("welcome to the sidecar\n");
assert(t->initSideEnableOffset);
// We have to enable some sidecar literals
const char *template = (const char *)t + t->initSideEnableOffset;
memcpy(&scratch->side_enabled, template, t->stateOffsets.sidecar_size);
}
static really_inline
void init_state_for_block(const struct RoseEngine *t, u8 *state) {
assert(t);
@@ -172,15 +157,12 @@ void init_for_block(const struct RoseEngine *t, struct hs_scratch *scratch,
tctxt->next_mpv_offset = 0;
tctxt->curr_anchored_loc = MMB_INVALID;
tctxt->curr_row_offset = 0;
tctxt->side_curr = 0;
scratch->am_log_sum = 0; /* clear the anchored logs */
scratch->al_log_sum = 0;
fatbit_clear(scratch->aqa);
init_sidecar(t, scratch); /* Init the sidecar enabled state */
scratch->catchup_pq.qm_size = 0;
init_outfixes_for_block(t, scratch, state, is_small_block);

15
src/rose/eod.c
View File

@@ -28,7 +28,6 @@
#include "catchup.h"
#include "match.h"
#include "rose_sidecar_runtime.h"
#include "rose.h"
#include "util/fatbit.h"
@@ -98,13 +97,6 @@ hwlmcb_rv_t roseEodRunMatcher(const struct RoseEngine *t, u64a offset,
DEBUG_PRINTF("eod offset=%llu, eod length=%zu\n", offset, eod_len);
struct RoseContext *tctxt = &scratch->tctxt;
/* update side_curr for eod_len */
tctxt->side_curr = offset - eod_len;
/* no need to enable any sidecar groups as they are for .*A.* constructs
* not allowed in the eod table */
const struct HWLM *etable = getELiteralMatcher(t);
hwlmExec(etable, eod_data, eod_len, adj, roseCallback, tctxt, tctxt->groups);
@@ -238,9 +230,6 @@ void cleanupAfterEodMatcher(const struct RoseEngine *t, u8 *state, u64a offset,
// Flush history to make sure it's consistent.
roseFlushLastByteHistory(t, state, offset, tctxt);
// Catch up the sidecar to cope with matches raised in the etable.
catchup_sidecar(tctxt, offset);
}
static rose_inline
@@ -323,7 +312,6 @@ void roseEodExec_i(const struct RoseEngine *t, u8 *state, u64a offset,
// Unset the reports we just fired so we don't fire them again below.
mmbit_clear(getRoleState(state), t->rolesWithStateCount);
mmbit_clear(getActiveLeafArray(t, state), t->activeArrayCount);
sidecar_enabled_populate(t, scratch, state);
hwlmcb_rv_t rv = roseEodRunMatcher(t, offset, scratch, is_streaming);
if (rv == HWLM_TERMINATE_MATCHING) {
@@ -368,9 +356,6 @@ void prepForEod(const struct RoseEngine *t, u8 *state, size_t length,
struct RoseContext *tctxt) {
roseFlushLastByteHistory(t, state, length, tctxt);
tctxt->lastEndOffset = length;
if (t->requiresEodSideCatchup) {
catchup_sidecar(tctxt, length);
}
}
void roseBlockEodExec(const struct RoseEngine *t, u64a offset,

22
src/rose/init.c
View File

@@ -37,8 +37,6 @@
#include "nfa/mcclellan.h"
#include "nfa/nfa_api_util.h"
#include "nfa/nfa_internal.h"
#include "sidecar/sidecar.h"
#include "sidecar/sidecar_internal.h"
#include "util/multibit.h"
#include <string.h>
@@ -55,21 +53,6 @@ void init_rstate(const struct RoseEngine *t, u8 *state) {
rstate->broken = NOT_BROKEN;
}
static really_inline
void init_sidecar(const struct RoseEngine *t, u8 *state) {
assert(getSLiteralMatcher(t));
struct sidecar_enabled *enabled_state
= (struct sidecar_enabled *)(state + t->stateOffsets.sidecar);
DEBUG_PRINTF("welcome to the sidecar\n");
assert(t->initSideEnableOffset);
// We have to enable some sidecar literals
const char *template = (const char *)t + t->initSideEnableOffset;
memcpy(enabled_state, template, t->stateOffsets.sidecar_size);
}
static really_inline
void init_outfixes(const struct RoseEngine *t, u8 *state) {
/* The active leaf array has been init'ed by the scatter with outfix
@@ -105,11 +88,6 @@ void roseInitState(const struct RoseEngine *t, u8 *state) {
init_rstate(t, state);
// Init the sidecar state
if (t->smatcherOffset) {
init_sidecar(t, state);
}
init_state(t, state);
init_outfixes(t, state);

72
src/rose/match.c
View File

@@ -31,7 +31,6 @@
#include "infix.h"
#include "match.h"
#include "miracle.h"
#include "rose_sidecar_runtime.h"
#include "rose.h"
#include "som/som_runtime.h"
#include "util/bitutils.h"
@@ -230,28 +229,6 @@ hwlmcb_rv_t roseDelayRebuildCallback(size_t start, size_t end, u32 id,
return tctx->groups;
}
/* Note: uses the stashed sparse iter state; cannot be called from
* anybody else who is using it
*/
static never_inline
void roseSquashStates(const struct RoseEngine *t, const struct RoseSide *tsb,
struct RoseContext *tctxt) {
DEBUG_PRINTF("attempting to squash states\n");
struct mmbit_sparse_state *s = tctxtToScratch(tctxt)->sparse_iter_state;
u8 *state = tctxt->state;
void *role_state = getRoleState(state);
u32 role_count = t->rolesWithStateCount;
const struct mmbit_sparse_iter *it = getByOffset(t, tsb->squashIterOffset);
assert(ISALIGNED(it));
/* we can squash willy-nilly */
DEBUG_PRINTF("squashing iter off = %u\n", tsb->squashIterOffset);
mmbit_sparse_iter_unset(role_state, role_count, it, s);
DEBUG_PRINTF("squashing groups with = %016llx\n", tsb->squashGroupMask);
tctxt->groups &= tsb->squashGroupMask;
}
static really_inline
hwlmcb_rv_t ensureQueueFlushed_i(const struct RoseEngine *t,
struct hs_scratch *scratch, u32 qi, s64a loc,
@@ -936,9 +913,6 @@ void roseSetRole(const struct RoseEngine *t, u8 *state,
// offset-tracking role.
if (alreadySet) {
DEBUG_PRINTF("role already set\n");
if (tr->sidecarEnableOffset) {
enable_sidecar(tctxt, tr);
}
return;
}
@@ -947,11 +921,6 @@ void roseSetRole(const struct RoseEngine *t, u8 *state,
// Switch on this role's groups
tctxt->groups |= tr->groups;
if (tr->sidecarEnableOffset) {
// We have to enable some sidecar literals
enable_sidecar(tctxt, tr);
}
}
static rose_inline
@@ -1551,20 +1520,6 @@ char roseWalkRootRoles(const struct RoseEngine *t,
}
}
void roseSidecarCallback(UNUSED u64a offset, u32 side_id, void *context) {
struct RoseContext *tctxt = context;
const struct RoseEngine *t = tctxt->t;
DEBUG_PRINTF("SIDE MATCH side_id=%u offset=[%llu, %llu]\n", side_id,
offset, offset + 1);
assert(side_id < t->sideCount);
const struct RoseSide *side = &getSideEntryTable(t)[side_id];
roseSquashStates(t, side, tctxt);
DEBUG_PRINTF("done with sc\n");
}
/* handles catchup, som, cb, etc */
static really_inline
hwlmcb_rv_t roseHandleReport(const struct RoseEngine *t, u8 *state,
@@ -1674,15 +1629,6 @@ int roseAnchoredCallback(u64a end, u32 id, void *ctx) {
tctxt->lastEndOffset = real_end;
}
if (tl->requires_side
&& real_end <= t->floatingMinLiteralMatchOffset) {
/* Catch up the sidecar to the literal location. This means that all
* squashing events are delivered before any 'side involved' literal
* matches at a given location. */
catchup_sidecar(tctxt, real_end);
}
/* anchored literals are root only */
if (!roseWalkRootRoles(t, tl, real_end, tctxt, 1, 0)) {
rv = HWLM_TERMINATE_MATCHING;
@@ -1762,18 +1708,6 @@ hwlmcb_rv_t roseProcessMatch_i(const struct RoseEngine *t, u64a end, u32 id,
return HWLM_CONTINUE_MATCHING;
}
// If the current literal requires sidecar support, run to current
// location.
if (tl->requires_side) {
/* Catch up the sidecar to the literal location. This means that all
* squashing events are delivered before any 'side involved' literal
* matches at a given location. */
if (tl->rootRoleCount || tl->minDepth <= tctxt->depth) {
catchup_sidecar(tctxt, end);
}
}
if (tl->minDepth > tctxt->depth) {
DEBUG_PRINTF("IGNORE: minDepth=%u > %u\n", tl->minDepth, tctxt->depth);
goto root_roles;
@@ -1848,7 +1782,7 @@ hwlmcb_rv_t playDelaySlot(struct RoseContext *tctxt, const u8 *delaySlotBase,
DEBUG_PRINTF("DONE depth=%u, groups=0x%016llx\n", tctxt->depth,
tctxt->groups);
/* delayed literals can't safely set groups, squashing may from side.
/* delayed literals can't safely set groups.
* However we may be setting groups that successors already have
* worked out that we don't need to match the group */
DEBUG_PRINTF("groups in %016llx out %016llx\n", old_groups,
@@ -1881,8 +1815,8 @@ hwlmcb_rv_t flushAnchoredLiteralAtLoc(struct RoseContext *tctxt, u32 curr_loc) {
DEBUG_PRINTF("DONE depth=%u, groups=0x%016llx\n", tctxt->depth,
tctxt->groups);
/* anchored literals can't safely set groups, squashing may from
* side. However we may be setting groups that successors already
/* anchored literals can't safely set groups.
* However we may be setting groups that successors already
* have worked out that we don't need to match the group */
DEBUG_PRINTF("groups in %016llx out %016llx\n", old_groups,
tctxt->groups);

19
src/rose/rose_build_add.cpp
View File

@@ -115,7 +115,6 @@ RoseVertex createVertex(RoseBuildImpl *build, u32 literalId, u32 min_offset,
g[v].idx = build->vertexIndex++;
g[v].min_offset = min_offset;
g[v].max_offset = max_offset;
/* no escapes */
DEBUG_PRINTF("insert vertex %zu into literal %u's vertex set\n", g[v].idx,
literalId);
@@ -201,19 +200,16 @@ RoseVertex duplicate(RoseBuildImpl *build, RoseVertex v) {
namespace {
struct created_key {
explicit created_key(const RoseInEdgeProps &trep)
: prefix(trep.graph.get()), lag(trep.graph_lag), escapes(trep.escapes) {
assert(escapes.none() || !prefix);
: prefix(trep.graph.get()), lag(trep.graph_lag) {
}
bool operator<(const created_key &b) const {
const created_key &a = *this;
ORDER_CHECK(prefix);
ORDER_CHECK(lag);
ORDER_CHECK(escapes);
return false;
}
NGHolder *prefix;
u32 lag;
CharReach escapes;
};
}
@@ -320,15 +316,6 @@ void createVertices(RoseBuildImpl *tbi,
}
NFAVertex p = pv.first;
if (isLeafNode(p, g)) {
DEBUG_PRINTF("setting escapes (reach %s) on parent\n",
describeClass(key.escapes, 20, CC_OUT_TEXT).c_str());
g[p].escapes = key.escapes;
} else if (key.escapes != g[p].escapes) {
DEBUG_PRINTF("creating differently escaped version of parent\n");
p = duplicate(tbi, p);
g[p].escapes = key.escapes;
}
RoseEdge e;
bool added;
@@ -1106,10 +1093,6 @@ bool predsAreDelaySensitive(const RoseInGraph &ig, RoseInVertex v) {
DEBUG_PRINTF("edge bounds\n");
return true;
}
if (ig[e].escapes.any()) {
DEBUG_PRINTF("escapes\n");
return true;
}
RoseInVertex u = source(e, ig);
if (ig[u].type == RIV_START) {

311
src/rose/rose_build_bytecode.cpp
View File

@@ -57,8 +57,6 @@
#include "nfagraph/ng_stop.h"
#include "nfagraph/ng_util.h"
#include "nfagraph/ng_width.h"
#include "sidecar/sidecar.h"
#include "sidecar/sidecar_compile.h"
#include "som/slot_manager.h"
#include "util/alloc.h"
#include "util/bitutils.h"
@@ -329,21 +327,15 @@ u8 pickRuntimeImpl(const RoseBuildImpl &tbi, u32 outfixEndQueue) {
}
static
void fillStateOffsets(const RoseBuildImpl &tbi, const sidecar *side,
u32 rolesWithStateCount, u32 anchorStateSize,
u32 activeArrayCount, u32 activeLeftCount,
u32 laggedRoseCount, u32 floatingStreamStateRequired,
u32 historyRequired, RoseStateOffsets *so) {
void fillStateOffsets(const RoseBuildImpl &tbi, u32 rolesWithStateCount,
u32 anchorStateSize, u32 activeArrayCount,
u32 activeLeftCount, u32 laggedRoseCount,
u32 floatingStreamStateRequired, u32 historyRequired,
RoseStateOffsets *so) {
/* runtime state (including role state) first and needs to be u32-aligned */
u32 curr_offset = sizeof(RoseRuntimeState)
+ mmbit_size(rolesWithStateCount);
so->sidecar = curr_offset;
if (side) {
so->sidecar_size = sidecarEnabledSize(side);
curr_offset += so->sidecar_size;
}
so->activeLeafArray = curr_offset; /* TODO: limit size of array */
curr_offset += mmbit_size(activeArrayCount);
@@ -1478,82 +1470,6 @@ u32 RoseBuildImpl::calcHistoryRequired() const {
return m ? m - 1 : 0;
}
static
u32 sizeSideSuccMasks(const sidecar *stable,
const map<set<u32>, set<RoseVertex> > &side_succ_map) {
if (!stable) {
return 0;
}
return verify_u32((side_succ_map.size() + 1 /* for init */)
* sidecarEnabledSize(stable));
}
static
void populateSideSuccLists(const RoseBuildImpl &tbi, build_context &bc,
const sidecar *stable, RoseEngine *engine, u32 base_offset,
const map<set<u32>, set<RoseVertex> > &sidecar_succ_map) {
const RoseGraph &g = tbi.g;
if (!stable) {
return;
}
u32 enabled_size = sidecarEnabledSize(stable);
char *curr = (char *)engine + base_offset;
for (const auto &e : sidecar_succ_map) {
u32 offset = verify_u32(curr - (char *)engine);
memset(curr, 0, enabled_size);
/* populate the list */
for (u32 side_id : e.first) {
sidecarEnabledAdd(stable, (sidecar_enabled *)curr, side_id);
}
curr += enabled_size;
/* update the role entries */
for (RoseVertex v : e.second) {
if (v == tbi.root) {
DEBUG_PRINTF("setting root emask\n");
engine->initSideEnableOffset = offset;
} else {
DEBUG_PRINTF("setting boring emask\n");
assert(g[v].role < bc.roleTable.size());
bc.roleTable[g[v].role].sidecarEnableOffset = offset;
}
}
}
if (!engine->initSideEnableOffset) {
DEBUG_PRINTF("add a blank enabled for root\n");
engine->initSideEnableOffset = verify_u32(curr - (char *)engine);
memset(curr, 0, enabled_size);
curr += enabled_size;
}
}
/* Also creates a map of sidecar id set to the roles which enables that set
*/
static
void markSideEnablers(RoseBuildImpl &build,
map<set<u32>, set<RoseVertex> > *scmap) {
map<RoseVertex, set<u32> > enablers;
u32 side_id = 0;
for (const auto &e : build.side_squash_roles) {
for (RoseVertex v : e.second) {
enablers[v].insert(side_id);
}
side_id++;
}
for (const auto &e : enablers) {
(*scmap)[e.second].insert(e.first);
}
}
#ifdef DEBUG
static UNUSED
string dumpMask(const vector<u8> &v) {
@@ -1873,11 +1789,6 @@ bool isNoRunsVertex(const RoseBuildImpl &tbi, NFAVertex u) {
return false;
}
if (g[u].escapes.any()) {
DEBUG_PRINTF("u=%zu has escapes\n", g[u].idx);
return false;
}
/* TODO: handle non-root roles as well. It can't be that difficult... */
if (!in_degree_equal_to(u, g, 1)) {
@@ -2176,35 +2087,6 @@ aligned_unique_ptr<HWLM> buildEodAnchoredMatcher(const RoseBuildImpl &tbi,
return etable;
}
static
aligned_unique_ptr<sidecar> buildSideMatcher(const RoseBuildImpl &tbi,
size_t *ssize) {
*ssize = 0;
if (tbi.side_squash_roles.empty()) {
DEBUG_PRINTF("no sidecar\n");
return nullptr;
}
assert(tbi.cc.grey.allowSidecar);
vector<CharReach> sl;
/* TODO: ensure useful sidecar entries only */
for (const CharReach &cr : tbi.side_squash_roles | map_keys) {
sl.push_back(cr);
}
aligned_unique_ptr<sidecar> stable = sidecarCompile(sl);
if (!stable) {
throw CompileError("Unable to generate bytecode.");
}
*ssize = sidecarSize(stable.get());
assert(*ssize);
DEBUG_PRINTF("built sidecar literal table size %zu bytes\n", *ssize);
return stable;
}
// Adds a sparse iterator to the end of the iterator table, returning its
// offset.
static
@@ -3040,122 +2922,6 @@ pair<u32, u32> buildEodAnchorRoles(RoseBuildImpl &tbi, build_context &bc,
return addPredSparseIter(bc, predStates);
}
static
void buildSideEntriesAndIters(const RoseBuildImpl &tbi, build_context &bc,
const set<RoseVertex> &squash_roles,
vector<RoseSide> &sideTable) {
const RoseGraph &g = tbi.g;
sideTable.push_back(RoseSide()); /* index in array gives an implicit id */
RoseSide &tsb = sideTable.back();
memset(&tsb, 0, sizeof(tsb));
if (squash_roles.empty()) {
return;
}
set<RoseVertex> squashed_succ;
// Build a vector of the roles' state IDs
vector<u32> states;
for (RoseVertex v : squash_roles) {
assert(g[v].role < bc.roleTable.size());
const RoseRole &tr = bc.roleTable[g[v].role];
DEBUG_PRINTF("considering role %u, state index %u\n", g[v].role,
tr.stateIndex);
assert(tr.stateIndex != MMB_INVALID);
states.push_back(tr.stateIndex);
DEBUG_PRINTF("side %zu squashes state index %u/role %u\n",
sideTable.size() - 1, tr.stateIndex, g[v].role);
/* we cannot allow groups to be squashed if the source vertex is in an
* anchored table due to ordering issue mean that a literals cannot
* set groups */
if (tbi.isAnchored(v) && g[v].max_offset != 1) {
DEBUG_PRINTF("%u has anchored table pred no squashy\n", g[v].role);
continue;
}
DEBUG_PRINTF("role %u is fine to g squash\n", g[v].role);
for (auto w : adjacent_vertices_range(v, g)) {
if (in_degree(w, g) == 1) { /* TODO: improve: check that each pred
* is in id's squash role */
squashed_succ.insert(w);
}
}
}
// Build sparse iterators and add to table.
assert(!states.empty());
vector<mmbit_sparse_iter> iter;
mmbBuildSparseIterator(iter, states, bc.numStates);
assert(!iter.empty());
tsb.squashIterOffset = addIteratorToTable(bc, iter);
// Build a mask of groups.
rose_group squash_groups = 0;
for (u32 i = 0; i < ROSE_GROUPS_MAX; i++) {
if (!contains(tbi.group_to_literal, i)) {
continue;
}
DEBUG_PRINTF("checking group %u for %zu's squash mask\n", i,
sideTable.size() - 1);
const set<u32> &group_lits = tbi.group_to_literal.find(i)->second;
/* check for each literal in this group if it is squashed by this
* sidecar escape */
for (u32 lit : group_lits) {
DEBUG_PRINTF("inspecting lit %u\n", lit);
const rose_literal_info &this_info = tbi.literal_info.at(lit);
/* check that all roles belonging to this literal are squashed */
for (RoseVertex v : this_info.vertices) {
DEBUG_PRINTF("checking if role is squashed %u...\n", g[v].role);
if (squashed_succ.find(v) != squashed_succ.end()) {
continue;
}
DEBUG_PRINTF("...role not taken %u\n", g[v].role);
/* if the literal is length 1 and anchored (0,0) when can ignore
* it as any matching must have happened before the side lit
* arrived */
if (g[v].max_offset == 1) {
DEBUG_PRINTF("we can ignore this role as 1st byte only\n");
continue;
}
goto fail_group;
}
}
continue;
fail_group:
DEBUG_PRINTF("group %u is not squashed\n", i);
/* we need to keep this group active */
squash_groups |= 1ULL << i;
}
DEBUG_PRINTF("%zu group squash mask: %016llx\n", sideTable.size() - 1,
squash_groups);
tsb.squashGroupMask = squash_groups;
}
// Construct sparse iterators for squashes
static
void buildSideTable(const RoseBuildImpl &build, build_context &bc,
vector<RoseSide> &sideTable) {
for (const auto &e : build.side_squash_roles) {
buildSideEntriesAndIters(build, bc, e.second, sideTable);
}
}
static
void fillLookaroundTables(char *look_base, char *reach_base,
const vector<LookEntry> &look_vec) {
@@ -3194,31 +2960,12 @@ bool hasBoundaryReports(const BoundaryReports &boundary) {
return false;
}
static
bool needsSidecarCatchup(const RoseBuildImpl &build, u32 id) {
const RoseGraph &g = build.g;
for (RoseVertex v : build.literal_info.at(id).vertices) {
if (g[v].escapes.any()) {
return true;
}
for (RoseVertex u : inv_adjacent_vertices_range(v, g)) {
if (g[u].escapes.any()) {
return true;
}
}
}
return false;
}
static
void createLiteralEntry(const RoseBuildImpl &tbi, build_context &bc,
vector<RoseLiteral> &literalTable) {
const u32 final_id = verify_u32(literalTable.size());
assert(contains(tbi.final_id_to_literal, final_id));
const u32 literalId = *tbi.final_id_to_literal.at(final_id).begin();
const UNUSED u32 literalId = *tbi.final_id_to_literal.at(final_id).begin();
/* all literal ids associated with this final id should result in identical
* literal entry */
const auto &lit_infos = getLiteralInfoByFinalId(tbi, final_id);
@@ -3275,8 +3022,6 @@ void createLiteralEntry(const RoseBuildImpl &tbi, build_context &bc,
}
assert(!tbi.literals.right.at(literalId).delay || !tl.delay_mask);
tl.requires_side = needsSidecarCatchup(tbi, literalId);
}
// Construct the literal table.
@@ -3836,19 +3581,13 @@ void fillMatcherDistances(const RoseBuildImpl &build, RoseEngine *engine) {
if (!engine->anchoredDistance) {
return;
}
/* could be improved, if we have any side squash stuff and an anchored table
* set the min float distance to 0 */
if (!build.side_squash_roles.empty()) {
engine->floatingMinDistance = 0;
}
}
aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
DerivedBoundaryReports dboundary(boundary);
// Build literal matchers
size_t asize = 0, fsize = 0, ssize = 0, esize = 0, sbsize = 0;
size_t asize = 0, fsize = 0, esize = 0, sbsize = 0;
size_t floatingStreamStateRequired = 0;
size_t historyRequired = calcHistoryRequired(); // Updated by HWLM.
@@ -3857,7 +3596,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
buildAnchoredAutomataMatcher(*this, &asize);
aligned_unique_ptr<HWLM> ftable = buildFloatingMatcher(
*this, &fsize, &historyRequired, &floatingStreamStateRequired);
aligned_unique_ptr<sidecar> stable = buildSideMatcher(*this, &ssize);
aligned_unique_ptr<HWLM> etable = buildEodAnchoredMatcher(*this, &esize);
aligned_unique_ptr<HWLM> sbtable = buildSmallBlockMatcher(*this, &sbsize);
@@ -3925,9 +3663,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
tie(eodIterMapOffset, eodIterOffset) = buildEodAnchorRoles(*this, bc,
predTable);
vector<RoseSide> sideTable;
buildSideTable(*this, bc, sideTable);
vector<mmbit_sparse_iter> activeLeftIter;
buildActiveLeftIter(leftInfoTable, activeLeftIter);
@@ -3940,7 +3675,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
u32 amatcherOffset = 0;
u32 fmatcherOffset = 0;
u32 smatcherOffset = 0;
u32 ematcherOffset = 0;
u32 sbmatcherOffset = 0;
@@ -3974,12 +3708,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
currOffset += (u32)fsize;
}
if (stable) {
currOffset = ROUNDUP_CL(currOffset);
smatcherOffset = currOffset;
currOffset += (u32)ssize;
}
if (etable) {
currOffset = ROUNDUP_CL(currOffset);
ematcherOffset = currOffset;
@@ -4002,9 +3730,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
u32 literalLen = sizeof(RoseLiteral) * literalTable.size();
currOffset = literalOffset + literalLen;
u32 sideOffset = ROUNDUP_N(currOffset, alignof(RoseSide));
currOffset = sideOffset + byte_length(sideTable);
u32 roleOffset = ROUNDUP_N(currOffset, alignof(RoseRole));
u32 roleLen = sizeof(RoseRole) * bc.roleTable.size();
currOffset = roleOffset + roleLen;
@@ -4048,13 +3773,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
u32 anchoredReportInverseMapOffset = currOffset;
currOffset += arit.size() * sizeof(u32);
/* sidecar may contain sse in silly cases */
currOffset = ROUNDUP_N(currOffset, 16);
u32 sideSuccListOffset = currOffset;
map<set<u32>, set<RoseVertex> > sidecar_succ_map;
markSideEnablers(*this, &sidecar_succ_map);
currOffset += sizeSideSuccMasks(stable.get(), sidecar_succ_map);
currOffset = ROUNDUP_N(currOffset, alignof(ReportID));
u32 multidirectOffset = currOffset;
currOffset += mdr_reports.size() * sizeof(ReportID);
@@ -4089,7 +3807,7 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
RoseStateOffsets stateOffsets;
memset(&stateOffsets, 0, sizeof(stateOffsets));
fillStateOffsets(*this, stable.get(), bc.numStates, anchorStateSize,
fillStateOffsets(*this, bc.numStates, anchorStateSize,
activeArrayCount, activeLeftCount, laggedRoseCount,
floatingStreamStateRequired, historyRequired,
&stateOffsets);
@@ -4125,11 +3843,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
assert(fmatcherOffset >= base_nfa_offset);
memcpy(ptr + fmatcherOffset, ftable.get(), fsize);
}
if (stable) {
assert(smatcherOffset);
assert(smatcherOffset >= base_nfa_offset);
memcpy(ptr + smatcherOffset, stable.get(), ssize);
}
if (etable) {
assert(ematcherOffset);
assert(ematcherOffset >= base_nfa_offset);
@@ -4162,9 +3875,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
engine->runtimeImpl = pickRuntimeImpl(*this, outfixEndQueue);
engine->mpvTriggeredByLeaf = anyEndfixMpvTriggers(*this);
engine->sideOffset = sideOffset;
engine->sideCount = verify_u32(sideTable.size());
engine->activeArrayCount = activeArrayCount;
engine->activeLeftCount = activeLeftCount;
engine->queueCount = queue_count;
@@ -4209,8 +3919,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
engine->nonbenefits_base_id = nonbenefits_base_id;
engine->literalBenefitsOffsets = base_lits_benefits_offset;
populateSideSuccLists(*this, bc, stable.get(), engine.get(),
sideSuccListOffset, sidecar_succ_map);
engine->rosePrefixCount = rosePrefixCount;
engine->activeLeftIterOffset
@@ -4234,7 +3942,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
engine->ematcherOffset = ematcherOffset;
engine->sbmatcherOffset = sbmatcherOffset;
engine->fmatcherOffset = fmatcherOffset;
engine->smatcherOffset = smatcherOffset;
engine->amatcherMinWidth = findMinWidth(*this, ROSE_ANCHORED);
engine->fmatcherMinWidth = findMinWidth(*this, ROSE_FLOATING);
engine->eodmatcherMinWidth = findMinWidth(*this, ROSE_EOD_ANCHORED);
@@ -4251,7 +3958,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
engine->hasFloatingDirectReports = floating_direct_report;
engine->requiresEodCheck = hasEodAnchors(*this, built_nfas,
outfixEndQueue);
engine->requiresEodSideCatchup = hasEodSideLink();
engine->hasOutfixesInSmallBlock = hasNonSmallBlockOutfix(outfixes);
engine->canExhaust = rm.patternSetCanExhaust();
engine->hasSom = hasSom;
@@ -4323,7 +4029,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
copy_bytes(ptr + engine->anchoredReportInverseMapOffset, arit);
copy_bytes(ptr + engine->multidirectOffset, mdr_reports);
copy_bytes(ptr + engine->activeLeftIterOffset, activeLeftIter);
copy_bytes(ptr + engine->sideOffset, sideTable);
DEBUG_PRINTF("rose done %p\n", engine.get());
return engine;

131
src/rose/rose_build_compile.cpp
View File

@@ -485,21 +485,6 @@ size_t trailerDueToSelf(const rose_literal_id &lit) {
return trailer;
}
/* note: last byte cannot conflict as escapes are processed after other
* lits at same offset */
static
bool conflictsWithEscape(const rose_literal_id &litv, const CharReach &cr) {
if (cr.none()) {
return false;
}
if (litv.delay) {
return true;
}
return contains(litv.s, cr);
}
static
RoseRoleHistory findHistoryScheme(const RoseBuildImpl &tbi, const RoseEdge &e) {
const RoseGraph &g = tbi.g;
@@ -618,7 +603,6 @@ bool RoseBuildImpl::isDirectReport(u32 id) const {
if (g[v].reports.empty() ||
g[v].eod_accept || // no accept EOD
g[v].escapes.any() ||
!g[v].isBoring() ||
!isLeafNode(v, g) || // Must have no out-edges
in_degree(v, g) != 1) { // Role must have exactly one in-edge
@@ -937,47 +921,6 @@ bool RoseBuildImpl::hasFinalId(u32 id) const {
return literal_info.at(id).final_id != MO_INVALID_IDX;
}
static
void doSidecarLiterals(RoseBuildImpl &tbi) {
map<CharReach, set<RoseVertex> > escapes;
const RoseGraph &g = tbi.g;
/* find escapes */
for (auto v : vertices_range(g)) {
const CharReach &cr = g[v].escapes;
if (cr.none()) {
continue;
}
DEBUG_PRINTF("vertex %zu has %zu escapes\n", g[v].idx, cr.count());
// We only have an implementation for these escapes if the Sidecar is
// available for use.
assert(tbi.cc.grey.allowSidecar);
assert(!isLeafNode(v, g));
/* Verify that all the successors are floating */
for (UNUSED auto w : adjacent_vertices_range(v, g)) {
assert(!tbi.isAnchored(w));
}
escapes[cr].insert(v);
}
if (escapes.size() > 32) {
/* ensure that a most one sparse iterator is triggered per char */
escapes = make_disjoint(escapes);
}
/* create the squash/escape sidecar entries for the vertices and associate
* with appropriate roles */
for (const auto &e : escapes) {
const CharReach &cr = e.first;
insert(&tbi.side_squash_roles[cr], e.second);
}
}
static
bool eligibleForAlwaysOnGroup(const RoseBuildImpl &tbi, u32 id) {
/* returns true if it or any of its delay versions have root role */
@@ -1310,42 +1253,6 @@ bool coversGroup(const RoseBuildImpl &tbi, const rose_literal_info &lit_info) {
return true;
}
static
bool escapesAllPreds(const RoseGraph &g, RoseVertex v, const CharReach &cr) {
for (auto u : inv_adjacent_vertices_range(v, g)) {
if ((~g[u].escapes & cr).any()) {
return false;
}
}
return true;
}
static
bool mayNotSeeSubsequentPredsInOrder(const RoseBuildImpl &tbi, RoseVertex v) {
const RoseGraph &g = tbi.g;
if (in_degree(v, g) == 1) {
/* if the pred can only match once, there are no subsequent preds */
RoseVertex u = source(*in_edges(v, g).first, g);
if (g[u].max_offset == g[u].min_offset) {
return false;
}
}
for (auto u : inv_adjacent_vertices_range(v, g)) {
for (u32 lit_id : g[u].literals) {
const rose_literal_id &lit = tbi.literals.right.at(lit_id);
if (lit.table == ROSE_ANCHORED) {
return true;
}
}
}
return false;
}
static
bool isGroupSquasher(const RoseBuildImpl &tbi, const u32 id /* literal id */,
rose_group forbidden_squash_group) {
@@ -1402,35 +1309,6 @@ bool isGroupSquasher(const RoseBuildImpl &tbi, const u32 id /* literal id */,
return false;
}
/* Can only squash cases with escapes if all preds have the same escapes
* and none of the literals overlap with the escape
*
* Additionally, if we may not see one of the preds in time to turn on
* the group again we have problems.
*
* ARGHHHH
*/
if (g[v].escapes.any()) {
if (!escapesAllPreds(g, v, g[v].escapes)
|| mayNotSeeSubsequentPredsInOrder(tbi, v)) {
return false;
}
if (g[v].literals.size() == 1) {
if (conflictsWithEscape(tbi.literals.right.at(id),
g[v].escapes)) {
return false;
}
} else {
for (const auto &lit_id : g[v].literals) {
const rose_literal_id &lit = tbi.literals.right.at(lit_id);
if (lit.delay || contains(lit.s, g[v].escapes)) {
return false;
}
}
}
}
// Out-edges must have inf max bound, + no other shenanigans */
for (const auto &e : out_edges_range(v, g)) {
if (g[e].maxBound != ROSE_BOUND_INF) {
@@ -1453,9 +1331,8 @@ bool isGroupSquasher(const RoseBuildImpl &tbi, const u32 id /* literal id */,
for (auto v : lit_info.vertices) {
assert(!tbi.isAnyStart(v));
// Can't squash cases with accepts or escapes
if (!g[v].reports.empty()
|| (g[v].escapes.any() && !escapesAllPreds(g, v, g[v].escapes))) {
// Can't squash cases with accepts
if (!g[v].reports.empty()) {
return false;
}
@@ -1908,7 +1785,6 @@ void addSmallBlockLiteral(RoseBuildImpl &tbi, const simple_anchored_info &sai,
// Clone vertex with the new literal ID.
RoseVertex v = add_vertex(g[lit_v], g);
g[v].idx = tbi.vertexIndex++;
g[v].escapes.clear();
g[v].literals.clear();
g[v].literals.insert(lit_id);
g[v].min_offset = sai.min_bound + sai.literal.length();
@@ -2418,9 +2294,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildRose(u32 minWidth) {
assignGroupsToRoles();
findGroupSquashers(*this);
// Collect squash literals for the sidecar
doSidecarLiterals(*this);
/* final prep work */
remapCastleTops(*this);
allocateFinalLiteralId(*this);

35
src/rose/rose_build_convert.cpp
View File

@@ -123,12 +123,12 @@ static
unique_ptr<NGHolder> convertLeafToHolder(const RoseGraph &g,
const RoseEdge &t_e,
const RoseLiteralMap &literals) {
RoseVertex t_u = source(t_e, g);
RoseVertex t_v = target(t_e, g); // leaf vertex for demolition.
const CharReach escape_cr(~g[t_u].escapes);
u32 minBound = g[t_e].minBound;
u32 maxBound = g[t_e].maxBound;
const CharReach dot = CharReach::dot();
assert(!g[t_v].left);
auto out = ue2::make_unique<NGHolder>(NFA_SUFFIX);
@@ -138,14 +138,14 @@ unique_ptr<NGHolder> convertLeafToHolder(const RoseGraph &g,
u32 i = 1;
NFAVertex last = out->start;
for (; i <= minBound; i++) {
NFAVertex v = addHolderVertex(escape_cr, *out);
NFAVertex v = addHolderVertex(dot, *out);
add_edge(last, v, *out);
last = v;
}
NFAVertex last_mand = last;
if (maxBound != ROSE_BOUND_INF) {
for (; i <= maxBound; i++) {
NFAVertex v = addHolderVertex(escape_cr, *out);
NFAVertex v = addHolderVertex(dot, *out);
add_edge(last_mand, v, *out);
if (last != last_mand) {
add_edge(last, v, *out);
@@ -156,7 +156,7 @@ unique_ptr<NGHolder> convertLeafToHolder(const RoseGraph &g,
if (minBound) {
add_edge(last_mand, last_mand, *out);
} else {
NFAVertex v = addHolderVertex(escape_cr, *out);
NFAVertex v = addHolderVertex(dot, *out);
add_edge(last_mand, v, *out);
add_edge(v, v, *out);
last = v;
@@ -277,28 +277,10 @@ bool isUnconvertibleLeaf(const RoseBuildImpl &tbi, const RoseVertex v) {
return true;
}
/* more arbitrary magic numbers as riskier transform */
if (g[e].maxBound == ROSE_BOUND_INF) {
if (!tbi.cc.grey.roseConvertInfBadLeaves) {
return true;
}
if (g[e].minBound > 20) {
DEBUG_PRINTF("fail minbound (%u)\n", maxbound);
return true;
}
if (max_lit_len > 2) {
DEBUG_PRINTF("fail length\n");
return true;
}
if (g[u].escapes.none()) {
/* slightly risky as nfa won't die and we don't avoid running the
sidecar */
DEBUG_PRINTF("fail: .*\n");
return true;
}
/* slightly risky as nfa won't die */
DEBUG_PRINTF("fail: .*\n");
return true;
}
return false;
@@ -386,7 +368,6 @@ void convertBadLeaves(RoseBuildImpl &tbi) {
RoseVertex u = source(e, g);
assert(!g[u].suffix);
g[u].escapes = CharReach();
g[u].suffix.graph = h;
DEBUG_PRINTF("%zu's nfa holder %p\n", g[u].idx, h.get());

4
src/rose/rose_build_dump.cpp
View File

@@ -181,10 +181,6 @@ public:
}
}
if (g[v].escapes.any()) {
os << "\\nescapes=";
describeClass(os, g[v].escapes, 5, CC_OUT_DOT);
}
if (ghost.find(v) != ghost.end()) {
os << "\\nGHOST";
}

4
src/rose/rose_build_impl.h
View File

@@ -389,8 +389,6 @@ public:
std::unique_ptr<RoseDedupeAux> generateDedupeAux() const override;
bool hasEodSideLink() const;
// Find the maximum bound on the edges to this vertex's successors.
u32 calcSuccMaxBound(RoseVertex u) const;
@@ -494,8 +492,6 @@ public:
u32 group_weak_end;
u32 group_end;
std::map<CharReach, std::set<RoseVertex> > side_squash_roles;
u32 anchored_base_id;
u32 nonbenefits_base_id;

16
src/rose/rose_build_misc.cpp
View File

@@ -210,22 +210,6 @@ bool RoseBuildImpl::hasNoFloatingRoots() const {
return true;
}
bool RoseBuildImpl::hasEodSideLink(void) const {
for (auto v : vertices_range(g)) {
if (!g[v].eod_accept) {
continue;
}
for (auto u : inv_adjacent_vertices_range(v, g)) {
if (g[u].escapes.any()) {
return true;
}
}
}
return false;
}
size_t RoseBuildImpl::maxLiteralLen(RoseVertex v) const {
const auto &lit_ids = g[v].literals;
assert(!lit_ids.empty());

11
src/rose/rose_build_role_aliasing.cpp
View File

@@ -365,8 +365,7 @@ bool sameRoleProperties(const RoseBuildImpl &build, RoseVertex a, RoseVertex b)
const RoseGraph &g = build.g;
const RoseVertexProps &aprops = g[a], &bprops = g[b];
if (aprops.eod_accept != bprops.eod_accept
|| aprops.escapes != bprops.escapes) {
if (aprops.eod_accept != bprops.eod_accept) {
return false;
}
@@ -457,12 +456,6 @@ size_t hashRightRoleProperties(RoseVertex v, const RoseGraph &g) {
static
void removeVertexFromMaps(RoseVertex v, RoseBuildImpl &build, revRoseMap &rrm) {
// Remove vertex 'a' from literal squash roles. Only sidecar literals can
// squash vertices, so they're the only ones we have to check.
for (auto &roles : build.side_squash_roles | map_values) {
roles.erase(v);
}
if (build.g[v].left) {
const left_id left(build.g[v].left);
assert(contains(rrm[left], v));
@@ -558,7 +551,6 @@ void mergeVertices(RoseVertex a, RoseVertex b, RoseBuildImpl &tbi,
// Merge role properties.
assert(g[a].eod_accept == g[b].eod_accept);
assert(g[a].escapes == g[b].escapes);
assert(g[a].left == g[b].left);
insert(&g[b].reports, g[a].reports);
@@ -596,7 +588,6 @@ void mergeVerticesDiamond(RoseVertex a, RoseVertex b, RoseBuildImpl &tbi,
// Merge role properties. For a diamond merge, most properties are already
// the same (with the notable exception of the literal set).
assert(g[a].eod_accept == g[b].eod_accept);
assert(g[a].escapes == g[b].escapes);
assert(g[a].left == g[b].left);
assert(g[a].reports == g[b].reports);
assert(g[a].suffix == g[b].suffix);

40
src/rose/rose_dump.cpp
View File

@@ -39,9 +39,6 @@
#include "nfa/nfa_build_util.h"
#include "nfa/nfa_dump_api.h"
#include "nfa/nfa_internal.h"
#include "sidecar/sidecar.h"
#include "sidecar/sidecar_compile.h"
#include "sidecar/sidecar_dump.h"
#include "util/multibit_internal.h"
#include <algorithm>
@@ -106,11 +103,6 @@ const HWLM *getFloatingMatcher(const RoseEngine *t) {
return (const HWLM *)loadFromByteCodeOffset(t, t->fmatcherOffset);
}
static
const sidecar *getSidecarMatcher(const RoseEngine *t) {
return (const sidecar *)loadFromByteCodeOffset(t, t->smatcherOffset);
}
static
const HWLM *getEodMatcher(const RoseEngine *t) {
return (const HWLM *)loadFromByteCodeOffset(t, t->ematcherOffset);
@@ -582,7 +574,6 @@ void roseDumpText(const RoseEngine *t, FILE *f) {
const void *atable = getAnchoredMatcher(t);
const HWLM *ftable = getFloatingMatcher(t);
const sidecar *stable = getSidecarMatcher(t);
const HWLM *etable = getEodMatcher(t);
const HWLM *sbtable = getSmallBlockMatcher(t);
@@ -634,16 +625,12 @@ void roseDumpText(const RoseEngine *t, FILE *f) {
} else {
fprintf(f, "\n");
}
fprintf(f, " - sidecar matcher : %u bytes\n",
stable ? sidecarSize(stable) : 0);
fprintf(f, " - eod-anch matcher : %zu bytes over last %u bytes\n",
etable ? hwlmSize(etable) : 0, t->ematcherRegionSize);
fprintf(f, " - small-blk matcher : %zu bytes over %u bytes\n",
sbtable ? hwlmSize(sbtable) : 0, t->smallBlockDistance);
fprintf(f, " - literal table : %zu bytes\n",
t->literalCount * sizeof(RoseLiteral));
fprintf(f, " - side table : %zu bytes\n",
t->sideCount * sizeof(RoseSide));
fprintf(f, " - role table : %zu bytes\n",
t->roleCount * sizeof(RoseRole));
fprintf(f, " - pred table : %zu bytes\n",
@@ -666,8 +653,6 @@ void roseDumpText(const RoseEngine *t, FILE *f) {
fprintf(f, " - role state mmbit : %u bytes\n", t->stateSize);
fprintf(f, " - runtime state : %zu bytes\n", sizeof(RoseRuntimeState));
fprintf(f, " - floating matcher : %u bytes\n", t->floatingStreamState);
fprintf(f, " - sidecar : %u bytes\n",
stable ? sidecarEnabledSize(stable) : 0U);
fprintf(f, " - active array : %u bytes\n",
mmbit_size(t->activeArrayCount));
fprintf(f, " - active rose : %u bytes\n",
@@ -690,8 +675,6 @@ void roseDumpText(const RoseEngine *t, FILE *f) {
literalsWithDirectReports(t));
fprintf(f, " - that squash group : %u\n",
literalsWithProp(t, &RoseLiteral::squashesGroup));
fprintf(f, " - need side catchup : %u\n",
literalsWithProp(t, &RoseLiteral::requires_side));
fprintf(f, " - with benefits : %u\n", t->nonbenefits_base_id);
u32 group_weak_end = t->group_weak_end;
@@ -763,12 +746,6 @@ void roseDumpText(const RoseEngine *t, FILE *f) {
hwlmPrintStats(ftable, f);
}
if (stable) {
fprintf(f, "\nSidecar literal matcher stats:\n\n");
fprintf(f, " Side Entries : %u\n", t->sideCount);
sidecarDump(stable, f);
}
if (etable) {
fprintf(f, "\nEOD-anchored literal matcher stats:\n\n");
hwlmPrintStats(etable, f);
@@ -792,7 +769,6 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
DUMP_U8(t, hasFloatingDirectReports);
DUMP_U8(t, noFloatingRoots);
DUMP_U8(t, requiresEodCheck);
DUMP_U8(t, requiresEodSideCatchup);
DUMP_U8(t, hasEodEventLiteral);
DUMP_U8(t, hasOutfixesInSmallBlock);
DUMP_U8(t, runtimeImpl);
@@ -816,7 +792,6 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
DUMP_U32(t, amatcherOffset);
DUMP_U32(t, ematcherOffset);
DUMP_U32(t, fmatcherOffset);
DUMP_U32(t, smatcherOffset);
DUMP_U32(t, sbmatcherOffset);
DUMP_U32(t, amatcherMinWidth);
DUMP_U32(t, fmatcherMinWidth);
@@ -827,8 +802,6 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
DUMP_U32(t, intReportCount);
DUMP_U32(t, literalOffset);
DUMP_U32(t, literalCount);
DUMP_U32(t, sideOffset);
DUMP_U32(t, sideCount);
DUMP_U32(t, multidirectOffset);
DUMP_U32(t, activeArrayCount);
DUMP_U32(t, activeLeftCount);
@@ -872,8 +845,6 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
DUMP_U32(t, delayRebuildLength);
DUMP_U32(t, stateOffsets.history);
DUMP_U32(t, stateOffsets.exhausted);
DUMP_U32(t, stateOffsets.sidecar);
DUMP_U32(t, stateOffsets.sidecar_size);
DUMP_U32(t, stateOffsets.activeLeafArray);
DUMP_U32(t, stateOffsets.activeLeftArray);
DUMP_U32(t, stateOffsets.activeLeftArray_size);
@@ -891,7 +862,6 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
DUMP_U32(t, boundary.reportZeroEodOffset);
DUMP_U32(t, totalNumLiterals);
DUMP_U32(t, asize);
DUMP_U32(t, initSideEnableOffset);
DUMP_U32(t, outfixBeginQueue);
DUMP_U32(t, outfixEndQueue);
DUMP_U32(t, leftfixBeginQueue);
@@ -952,7 +922,6 @@ void roseDumpRoleStructRaw(const RoseEngine *t, FILE *f) {
DUMP_U32(p, leftfixReport);
DUMP_U32(p, leftfixLag);
DUMP_U32(p, leftfixQueue);
DUMP_U32(p, sidecarEnableOffset);
DUMP_U32(p, somAdjust);
DUMP_U32(p, lookaroundIndex);
DUMP_U32(p, lookaroundCount);
@@ -976,7 +945,6 @@ void roseDumpInternals(const RoseEngine *t, const string &base) {
const void *atable = getAnchoredMatcher(t);
const HWLM *ftable = getFloatingMatcher(t);
const sidecar *stable = getSidecarMatcher(t);
const HWLM *etable = getEodMatcher(t);
if (atable) {
@@ -995,14 +963,6 @@ void roseDumpInternals(const RoseEngine *t, const string &base) {
}
}
if (stable) {
FILE *f = fopen((base + "/sidecar.raw").c_str(), "w");
if (f) {
fwrite(stable, 1, sidecarSize(stable), f);
fclose(f);
}
}
if (etable) {
FILE *f = fopen((base + "/eod.raw").c_str(), "w");
if (f) {

3
src/rose/rose_graph.h
View File

@@ -146,9 +146,6 @@ struct RoseVertexProps {
/** \brief Bitmask of groups that this role sets. */
rose_group groups = 0;
/** \brief Characters that escape and squash this role. */
CharReach escapes;
/** \brief Minimum role (end of literal) offset depth in bytes. */
u32 min_offset = ~u32{0};

6
src/rose/rose_in_graph.h
View File

@@ -168,12 +168,6 @@ struct RoseInEdgeProps {
std::shared_ptr<raw_som_dfa> haig;
u32 graph_lag;
/** \brief Escape characters, can be used instead of graph.
*
* currently must not intersect with succ literal and must be a literal -
* literal edge, TODO: handle */
CharReach escapes;
};
typedef boost::adjacency_list<boost::listS, boost::listS, boost::bidirectionalS,

64
src/rose/rose_internal.h
View File

@@ -88,18 +88,11 @@ struct RoseLiteral {
u8 minDepth; // the minimum of this literal's roles' depths (for depths > 1)
u8 squashesGroup; /**< literal switches off its group behind it if it sets a
* role */
u8 requires_side; // need to catch up sidecar for this literal
u32 delay_mask; /**< bit set indicates that the literal inserts a delayed
* match at the given offset */
u32 delayIdsOffset; // offset to array of ids to poke in the delay structure
};
/* properties for sidecar entries, yay */
struct RoseSide {
u32 squashIterOffset; // offset of the squash sparse iterator, rose relative
rose_group squashGroupMask; // squash literal squash masks
};
/* Allocation of Rose literal ids
*
* The rose literal id space is segmented:
@@ -130,8 +123,6 @@ struct RoseSide {
* | |
* | |
* ----
*
* Note: sidecar 'literals' are in a complete separate space
*/
/* Rose Literal Sources
@@ -140,11 +131,10 @@ struct RoseSide {
* 1) The floating table
* 2) The anchored table
* 3) Delayed literals
* 4) Sidecar literal matcher
* 5) suffixes NFAs
* 6) masksv2 (literals with benefits)
* 7) End anchored table
* 8) prefix / infix nfas
* 4) suffixes NFAs
* 5) masksv2 (literals with benefits)
* 6) End anchored table
* 7) prefix / infix nfas
*
* Care is required to ensure that events appear to come into Rose in order
* (or sufficiently ordered for Rose to cope). Generally the progress of the
@@ -162,13 +152,6 @@ struct RoseSide {
* Delayed literal ordering is handled by delivering any pending delayed
* literals before processing any floating match.
*
* Sidecar:
* The sidecar matcher is unique in that it does not return match
* location information. Sidecar literals are escapes between two normal
* roles. The sidecar matcher is caught up to the floating matcher
* before any possible predecessor role, any possible successor role, and
* at stream boundaries^3.
*
* Suffix:
* Suffixes are always pure terminal roles. Prior to raising a match^2, pending
* NFA queues are run to the current point (floating or delayed literal) as
@@ -319,8 +302,6 @@ struct RoseRole {
* leftfix engine status */
u32 leftfixQueue; /**< queue index of the prefix/infix before role */
u32 infixTriggerOffset; /* offset to list of infix roses to trigger */
u32 sidecarEnableOffset; /**< offset to list of sidecar literals to enable
*/
u32 somAdjust; /**< som for the role is offset from end match offset */
u32 lookaroundIndex; /**< index of lookaround offset/reach in table, or
@@ -374,12 +355,6 @@ struct RoseStateOffsets {
* reports with that ekey should not be delivered to the user. */
u32 exhausted;
/** Sidecar state. */
u32 sidecar;
/** Size of sidecar state, in bytes. */
u32 sidecar_size;
/** Multibit for active suffix/outfix engines. */
u32 activeLeafArray;
@@ -460,9 +435,8 @@ struct RoseBoundaryReports {
// In memory, we follow this with:
// 1a. anchored 'literal' matcher table
// 1b. floating literal matcher table
// 1c. sidecar 'literal' matcher table
// 1d. eod-anchored literal matcher table
// 1e. small block table
// 1c. eod-anchored literal matcher table
// 1d. small block table
// 2. array of RoseLiteral (literalCount entries)
// 3. array of RoseRole (roleCount entries)
// 4. array of RosePred (predCount entries)
@@ -480,8 +454,6 @@ struct RoseEngine {
u8 noFloatingRoots; /* only need to run the anchored table if something
* matched in the anchored table */
u8 requiresEodCheck; /* stuff happens at eod time */
u8 requiresEodSideCatchup; /* we need to do a sidecar catchup before eod
* checks */
u8 hasEodEventLiteral; // fires a ROSE_EVENT literal at eod time.
u8 hasOutfixesInSmallBlock; /**< has at least one outfix that must run even
in small block scans. */
@@ -513,7 +485,6 @@ struct RoseEngine {
u32 amatcherOffset; // offset of the anchored literal matcher (bytes)
u32 ematcherOffset; // offset of the eod-anchored literal matcher (bytes)
u32 fmatcherOffset; // offset of the floating literal matcher (bytes)
u32 smatcherOffset; // offset of the sidecar literal matcher (bytes)
u32 sbmatcherOffset; // offset of the small-block literal matcher (bytes)
u32 amatcherMinWidth; /**< minimum number of bytes required for a pattern
* involved with the anchored table to produce a full
@@ -534,9 +505,6 @@ struct RoseEngine {
u32 intReportCount; /**< number of internal_report structures */
u32 literalOffset; // offset of RoseLiteral array (bytes)
u32 literalCount; // number of RoseLiteral entries [NOT number of literals]
u32 sideOffset; /**< offset of RoseSide array (bytes), indexed by
*sidecar ids */
u32 sideCount; /**< number of RoseSide entries */
u32 multidirectOffset; /**< offset of multi-direct report list. */
u32 activeArrayCount; //number of nfas tracked in the active array
u32 activeLeftCount; //number of nfas tracked in the active rose array
@@ -605,7 +573,6 @@ struct RoseEngine {
struct RoseBoundaryReports boundary;
u32 totalNumLiterals; /* total number of literals including dr */
u32 asize; /* size of the atable */
u32 initSideEnableOffset; /* sidecar literals enabled initially */
u32 outfixBeginQueue; /* first outfix queue */
u32 outfixEndQueue; /* one past the last outfix queue */
u32 leftfixBeginQueue; /* first prefix/infix queue */
@@ -683,17 +650,6 @@ const struct HWLM *getFLiteralMatcher(const struct RoseEngine *t) {
return (const struct HWLM *)lt;
}
static really_inline
const void *getSLiteralMatcher(const struct RoseEngine *t) {
if (!t->smatcherOffset) {
return NULL;
}
const char *st = (const char *)t + t->smatcherOffset;
assert(ISALIGNED_N(st, 8));
return st;
}
static really_inline
const void *getELiteralMatcher(const struct RoseEngine *t) {
if (!t->ematcherOffset) {
@@ -724,14 +680,6 @@ const struct RoseLiteral *getLiteralTable(const struct RoseEngine *t) {
return tl;
}
static really_inline
const struct RoseSide *getSideEntryTable(const struct RoseEngine *t) {
const struct RoseSide *rs
= (const struct RoseSide *)((const char *)t + t->sideOffset);
assert(ISALIGNED(rs));
return rs;
}
static really_inline
const struct RoseRole *getRoleTable(const struct RoseEngine *t) {
const struct RoseRole *r

101
src/rose/rose_sidecar_runtime.h
View File

@@ -1,101 +0,0 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef ROSE_SIDECAR_RUNTIME_H_1F746F6F237176
#define ROSE_SIDECAR_RUNTIME_H_1F746F6F237176
#include "hwlm/hwlm.h"
#include "scratch.h"
#include "sidecar/sidecar.h"
#include "rose_common.h"
#include "ue2common.h"
// Callback defined in match.c
void roseSidecarCallback(u64a offset, u32 side_id, void *context);
static really_inline
void catchup_sidecar(struct RoseContext *tctxt, u64a end) {
DEBUG_PRINTF("catching up the sidecar from %llu to %llu\n",
tctxt->side_curr, end);
const struct sidecar *sidecar = getSLiteralMatcher(tctxt->t);
struct hs_scratch *scratch = tctxtToScratch(tctxt);
struct core_info *ci = &scratch->core_info;
if (!sidecar || tctxt->side_curr == end) {
return;
}
const u8 *start;
if (tctxt->side_curr >= ci->buf_offset) {
start = ci->buf + tctxt->side_curr - ci->buf_offset;
assert(end <= ci->buf_offset + ci->len);
} else {
/* at eod time we are called running over the histroy */
start = ci->hbuf + tctxt->side_curr - ci->buf_offset + ci->hlen;
assert(end <= ci->buf_offset);
}
size_t len = end - tctxt->side_curr;
DEBUG_PRINTF("enabled-->%02hhx\n", *(u8 *)&scratch->side_enabled.arb);
sidecarExec(sidecar, start, len, &scratch->side_enabled.arb,
scratch->side_scratch, tctxt->side_curr, roseSidecarCallback,
tctxt);
tctxt->side_curr = end;
DEBUG_PRINTF("finished catching up the sidecar to %llu\n", end);
}
static rose_inline
void enable_sidecar(struct RoseContext *tctxt, const struct RoseRole *tr) {
assert(tr->sidecarEnableOffset);
const struct sidecar *sidecar = getSLiteralMatcher(tctxt->t);
assert(sidecar);
struct hs_scratch *scratch = tctxtToScratch(tctxt);
DEBUG_PRINTF("welcome to the sidecar\n");
sidecarEnabledUnion(sidecar, &scratch->side_enabled.arb,
(const void *)((const char *)tctxt->t + tr->sidecarEnableOffset));
}
static really_inline
void sidecar_enabled_populate(const struct RoseEngine *t,
struct hs_scratch *scratch, const u8 *state) {
DEBUG_PRINTF("enabled-->%02hhx\n", *(state + t->stateOffsets.sidecar));
memcpy(&scratch->side_enabled, state + t->stateOffsets.sidecar,
t->stateOffsets.sidecar_size);
DEBUG_PRINTF("enabled-->%02hhx\n", *(u8 *)&scratch->side_enabled.arb);
}
static really_inline
void sidecar_enabled_preserve(const struct RoseEngine *t,
const struct hs_scratch *scratch, u8 *state) {
memcpy(state + t->stateOffsets.sidecar, &scratch->side_enabled,
t->stateOffsets.sidecar_size);
}
#endif /* ROSE_SIDECAR_RUNTIME_H_1F746F6F237176 */

7
src/rose/stream.c
View File

@@ -37,7 +37,6 @@
#include "nfa/nfa_api_queue.h"
#include "nfa/nfa_internal.h"
#include "util/fatbit.h"
#include "rose_sidecar_runtime.h"
#include "rose.h"
static rose_inline
@@ -407,8 +406,6 @@ void ensureStreamNeatAndTidy(const struct RoseEngine *t, u8 *state,
roseCatchUpLeftfixes(t, state, scratch);
roseFlushLastByteHistory(t, state, offset + length, tctxt);
tctxt->lastEndOffset = offset + length;
catchup_sidecar(tctxt, offset + length);
sidecar_enabled_preserve(t, scratch, state);
storeGroups(t, state, tctxt->groups);
struct RoseRuntimeState *rstate = getRuntimeState(state);
rstate->stored_depth = tctxt->depth;
@@ -473,8 +470,6 @@ void roseStreamExec(const struct RoseEngine *t, u8 *state,
tctxt->next_mpv_offset = 0;
tctxt->curr_anchored_loc = MMB_INVALID;
tctxt->curr_row_offset = 0;
tctxt->side_curr = offset;
DEBUG_PRINTF("BEGIN: history len=%zu, buffer len=%zu\n",
scratch->core_info.hlen, scratch->core_info.len);
@@ -487,8 +482,6 @@ void roseStreamExec(const struct RoseEngine *t, u8 *state,
streamInitSufPQ(t, state, scratch);
}
sidecar_enabled_populate(t, scratch, state);
u8 delay_rb_status = rstate->flags;
u32 alen = t->anchoredDistance > offset ?