diff --git a/src/report.h b/src/report.h index 69497da8..96cea32e 100644 --- a/src/report.h +++ b/src/report.h @@ -68,14 +68,14 @@ enum DedupeResult { static really_inline enum DedupeResult dedupeCatchup(const struct RoseEngine *rose, - const struct internal_report *ri, + const struct internal_report *ir, struct hs_scratch *scratch, u64a offset, u64a from_offset, u64a to_offset, const char do_som) { DEBUG_PRINTF("offset=%llu, match=[%llu,%llu], dkey=%u, do_som=%d\n", offset, - from_offset, to_offset, ri->dkey, do_som); - DEBUG_PRINTF("report type=%u, quashSom=%d\n", ri->type, ri->quashSom); - const u32 dkey = ri->dkey; + from_offset, to_offset, ir->dkey, do_som); + DEBUG_PRINTF("report type=%u, quashSom=%d\n", ir->type, ir->quashSom); + const u32 dkey = ir->dkey; // We should not have been called if there's no dedupe work to do. assert(do_som || dkey != MO_INVALID_IDX); @@ -99,8 +99,8 @@ enum DedupeResult dedupeCatchup(const struct RoseEngine *rose, if (dkey != MO_INVALID_IDX) { const u32 dkeyCount = rose->dkeyCount; - const s32 offset_adj = ri->offsetAdjust; - if (ri->type == EXTERNAL_CALLBACK || ri->quashSom) { + const s32 offset_adj = ir->offsetAdjust; + if (ir->type == EXTERNAL_CALLBACK || ir->quashSom) { DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset); assert(offset_adj == 0 || offset_adj == -1); if (fatbit_set(deduper->log[to_offset % 2], dkeyCount, dkey)) { @@ -136,12 +136,12 @@ enum DedupeResult dedupeCatchup(const struct RoseEngine *rose, static really_inline enum DedupeResult dedupeCatchupSom(const struct RoseEngine *rose, - const struct internal_report *ri, + const struct internal_report *ir, struct hs_scratch *scratch, u64a offset, u64a from_offset, u64a to_offset) { DEBUG_PRINTF("offset=%llu, match=[%llu,%llu], dkey=%u\n", offset, - from_offset, to_offset, ri->dkey); - DEBUG_PRINTF("report type=%u, quashSom=%d\n", ri->type, ri->quashSom); + from_offset, to_offset, ir->dkey); + DEBUG_PRINTF("report type=%u, quashSom=%d\n", ir->type, ir->quashSom); struct match_deduper *deduper = &scratch->deduper; if (offset != deduper->current_report_offset) { @@ -160,11 +160,11 @@ enum DedupeResult dedupeCatchupSom(const struct RoseEngine *rose, deduper->current_report_offset = offset; } - const u32 dkey = ri->dkey; + const u32 dkey = ir->dkey; if (dkey != MO_INVALID_IDX) { const u32 dkeyCount = rose->dkeyCount; - const s32 offset_adj = ri->offsetAdjust; - if (ri->quashSom) { + const s32 offset_adj = ir->offsetAdjust; + if (ir->quashSom) { DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset); assert(offset_adj == 0 || offset_adj == -1); if (fatbit_set(deduper->log[to_offset % 2], dkeyCount, dkey)) { @@ -208,11 +208,11 @@ int roseAdaptor_i(u64a offset, ReportID id, struct hs_scratch *scratch, struct core_info *ci = &scratch->core_info; const struct RoseEngine *rose = ci->rose; DEBUG_PRINTF("internal report %u\n", id); - const struct internal_report *ri = getInternalReport(rose, id); + const struct internal_report *ir = getInternalReport(rose, id); - assert(isExternalReport(ri)); /* only external reports should reach here */ + assert(isExternalReport(ir)); /* only external reports should reach here */ - s32 offset_adj = ri->offsetAdjust; + s32 offset_adj = ir->offsetAdjust; u64a to_offset = offset; u64a from_offset = 0; @@ -225,7 +225,7 @@ int roseAdaptor_i(u64a offset, ReportID id, struct hs_scratch *scratch, #endif DEBUG_PRINTF("internal match at %llu: IID=%u type=%hhu RID=%u " - "offsetAdj=%d\n", offset, id, ri->type, ri->onmatch, + "offsetAdj=%d\n", offset, id, ir->type, ir->onmatch, offset_adj); if (unlikely(can_stop_matching(scratch))) { /* ok - we are from rose */ @@ -233,46 +233,46 @@ int roseAdaptor_i(u64a offset, ReportID id, struct hs_scratch *scratch, return MO_HALT_MATCHING; } - if (!is_simple && ri->hasBounds) { - assert(ri->minOffset || ri->minLength || ri->maxOffset < MAX_OFFSET); - assert(ri->minOffset <= ri->maxOffset); - if (offset < ri->minOffset || offset > ri->maxOffset) { + if (!is_simple && ir->hasBounds) { + assert(ir->minOffset || ir->minLength || ir->maxOffset < MAX_OFFSET); + assert(ir->minOffset <= ir->maxOffset); + if (offset < ir->minOffset || offset > ir->maxOffset) { DEBUG_PRINTF("match fell outside valid range %llu !: [%llu,%llu]\n", - offset, ri->minOffset, ri->maxOffset); + offset, ir->minOffset, ir->maxOffset); return ROSE_CONTINUE_MATCHING_NO_EXHAUST; } } - if (!is_simple && unlikely(isExhausted(ci->exhaustionVector, ri->ekey))) { + if (!is_simple && unlikely(isExhausted(ci->exhaustionVector, ir->ekey))) { DEBUG_PRINTF("ate exhausted match\n"); return MO_CONTINUE_MATCHING; } - if (ri->type == EXTERNAL_CALLBACK) { + if (ir->type == EXTERNAL_CALLBACK) { from_offset = 0; } else if (do_som) { - from_offset = handleSomExternal(scratch, ri, to_offset); + from_offset = handleSomExternal(scratch, ir, to_offset); } to_offset += offset_adj; assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset); - if (do_som && ri->minLength) { - if (!satisfiesMinLength(ri->minLength, from_offset, to_offset)) { + if (do_som && ir->minLength) { + if (!satisfiesMinLength(ir->minLength, from_offset, to_offset)) { return ROSE_CONTINUE_MATCHING_NO_EXHAUST; } - if (ri->quashSom) { + if (ir->quashSom) { from_offset = 0; } } DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n", - from_offset, to_offset, ri->onmatch, ci->userContext); + from_offset, to_offset, ir->onmatch, ci->userContext); int halt = 0; - if (do_som || ri->dkey != MO_INVALID_IDX) { - enum DedupeResult dedupe_rv = dedupeCatchup(rose, ri, scratch, offset, + if (do_som || ir->dkey != MO_INVALID_IDX) { + enum DedupeResult dedupe_rv = dedupeCatchup(rose, ir, scratch, offset, from_offset, to_offset, do_som); switch (dedupe_rv) { case DEDUPE_HALT: @@ -286,7 +286,7 @@ int roseAdaptor_i(u64a offset, ReportID id, struct hs_scratch *scratch, } } - halt = ci->userCallback((unsigned int)ri->onmatch, from_offset, to_offset, + halt = ci->userCallback((unsigned int)ir->onmatch, from_offset, to_offset, flags, ci->userContext); exit: if (halt) { @@ -295,8 +295,8 @@ exit: return MO_HALT_MATCHING; } - if (!is_simple && ri->ekey != END_EXHAUST) { - markAsMatched(ci->exhaustionVector, ri->ekey); + if (!is_simple && ir->ekey != END_EXHAUST) { + markAsMatched(ci->exhaustionVector, ir->ekey); return MO_CONTINUE_MATCHING; } else { return ROSE_CONTINUE_MATCHING_NO_EXHAUST; @@ -310,58 +310,52 @@ exit: * that dedupe catchup has been done. */ static really_inline -int roseDeliverReport(u64a offset, ReportID id, struct hs_scratch *scratch, - char is_exhaustible) { - assert(id != MO_INVALID_IDX); // Should never get an invalid ID. +int roseDeliverReport(u64a offset, UNUSED ReportID id, ReportID onmatch, + s32 offset_adjust, struct hs_scratch *scratch, u32 ekey) { assert(scratch); assert(scratch->magic == SCRATCH_MAGIC); struct core_info *ci = &scratch->core_info; - const struct RoseEngine *rose = ci->rose; - DEBUG_PRINTF("internal report %u\n", id); - const struct internal_report *ri = getInternalReport(rose, id); - assert(isExternalReport(ri)); /* only external reports should reach here */ - - const s32 offset_adj = ri->offsetAdjust; u32 flags = 0; #ifndef RELEASE_BUILD - if (offset_adj) { + if (offset_adjust) { // alert testing tools that we've got adjusted matches flags |= HS_MATCH_FLAG_ADJUSTED; } #endif - DEBUG_PRINTF("internal match at %llu: IID=%u type=%hhu RID=%u " - "offsetAdj=%d\n", offset, id, ri->type, ri->onmatch, - offset_adj); +#ifndef NDEBUG + // Assertions for development builds. + UNUSED const struct internal_report *ir = getInternalReport(ci->rose, id); + assert(isExternalReport(ir)); /* only external reports should reach here */ assert(!can_stop_matching(scratch)); - assert(!ri->hasBounds || - (offset >= ri->minOffset && offset <= ri->maxOffset)); - assert(ri->type == EXTERNAL_CALLBACK); - assert(!ri->minLength); - assert(!ri->quashSom); - assert(ri->ekey == INVALID_EKEY || - !isExhausted(ci->exhaustionVector, ri->ekey)); + assert(!ir->hasBounds || + (offset >= ir->minOffset && offset <= ir->maxOffset)); + assert(ir->type == EXTERNAL_CALLBACK); + assert(!ir->minLength); + assert(!ir->quashSom); +#endif + + assert(ekey == INVALID_EKEY || !isExhausted(ci->exhaustionVector, ekey)); u64a from_offset = 0; - u64a to_offset = offset + offset_adj; + u64a to_offset = offset + offset_adjust; DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n", - from_offset, to_offset, ri->onmatch, ci->userContext); + from_offset, to_offset, onmatch, ci->userContext); - int halt = ci->userCallback((unsigned int)ri->onmatch, from_offset, - to_offset, flags, ci->userContext); + int halt = ci->userCallback(onmatch, from_offset, to_offset, flags, + ci->userContext); if (halt) { DEBUG_PRINTF("callback requested to terminate matches\n"); ci->status |= STATUS_TERMINATED; return MO_HALT_MATCHING; } - if (is_exhaustible) { - assert(ri->ekey != INVALID_EKEY); - markAsMatched(ci->exhaustionVector, ri->ekey); + if (ekey != INVALID_EKEY) { + markAsMatched(ci->exhaustionVector, ekey); return MO_CONTINUE_MATCHING; } else { return ROSE_CONTINUE_MATCHING_NO_EXHAUST; @@ -379,62 +373,62 @@ int roseSomAdaptor_i(u64a from_offset, u64a to_offset, ReportID id, struct core_info *ci = &scratch->core_info; const struct RoseEngine *rose = ci->rose; - const struct internal_report *ri = getInternalReport(rose, id); + const struct internal_report *ir = getInternalReport(rose, id); /* internal events should be handled by rose directly */ - assert(ri->type == EXTERNAL_CALLBACK); + assert(ir->type == EXTERNAL_CALLBACK); DEBUG_PRINTF("internal match at %llu: IID=%u type=%hhu RID=%u " - "offsetAdj=%d\n", to_offset, id, ri->type, ri->onmatch, - ri->offsetAdjust); + "offsetAdj=%d\n", to_offset, id, ir->type, ir->onmatch, + ir->offsetAdjust); if (unlikely(can_stop_matching(scratch))) { DEBUG_PRINTF("pre broken - halting\n"); return MO_HALT_MATCHING; } - if (!is_simple && ri->hasBounds) { - assert(ri->minOffset || ri->minLength || ri->maxOffset < MAX_OFFSET); - if (to_offset < ri->minOffset || to_offset > ri->maxOffset) { + if (!is_simple && ir->hasBounds) { + assert(ir->minOffset || ir->minLength || ir->maxOffset < MAX_OFFSET); + if (to_offset < ir->minOffset || to_offset > ir->maxOffset) { DEBUG_PRINTF("match fell outside valid range %llu !: [%llu,%llu]\n", - to_offset, ri->minOffset, ri->maxOffset); + to_offset, ir->minOffset, ir->maxOffset); return MO_CONTINUE_MATCHING; } } int halt = 0; - if (!is_simple && unlikely(isExhausted(ci->exhaustionVector, ri->ekey))) { + if (!is_simple && unlikely(isExhausted(ci->exhaustionVector, ir->ekey))) { DEBUG_PRINTF("ate exhausted match\n"); goto exit; } u64a offset = to_offset; - to_offset += ri->offsetAdjust; + to_offset += ir->offsetAdjust; assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset); - if (!is_simple && ri->minLength) { - if (!satisfiesMinLength(ri->minLength, from_offset, to_offset)) { + if (!is_simple && ir->minLength) { + if (!satisfiesMinLength(ir->minLength, from_offset, to_offset)) { return MO_CONTINUE_MATCHING; } - if (ri->quashSom) { + if (ir->quashSom) { from_offset = 0; } } DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n", - from_offset, to_offset, ri->onmatch, ci->userContext); + from_offset, to_offset, ir->onmatch, ci->userContext); #ifndef RELEASE_BUILD - if (ri->offsetAdjust != 0) { + if (ir->offsetAdjust != 0) { // alert testing tools that we've got adjusted matches flags |= HS_MATCH_FLAG_ADJUSTED; } #endif enum DedupeResult dedupe_rv = - dedupeCatchupSom(rose, ri, scratch, offset, from_offset, to_offset); + dedupeCatchupSom(rose, ir, scratch, offset, from_offset, to_offset); switch (dedupe_rv) { case DEDUPE_HALT: halt = 1; @@ -446,11 +440,11 @@ int roseSomAdaptor_i(u64a from_offset, u64a to_offset, ReportID id, break; } - halt = ci->userCallback((unsigned int)ri->onmatch, from_offset, to_offset, + halt = ci->userCallback((unsigned int)ir->onmatch, from_offset, to_offset, flags, ci->userContext); if (!is_simple) { - markAsMatched(ci->exhaustionVector, ri->ekey); + markAsMatched(ci->exhaustionVector, ir->ekey); } exit: @@ -470,48 +464,41 @@ exit: * that dedupe catchup has been done. */ static really_inline -int roseDeliverSomReport(u64a from_offset, u64a to_offset, ReportID id, +int roseDeliverSomReport(u64a from_offset, u64a to_offset, + const struct internal_report *ir, struct hs_scratch *scratch, char is_exhaustible) { - assert(id != MO_INVALID_IDX); // Should never get an invalid ID. assert(scratch); assert(scratch->magic == SCRATCH_MAGIC); - - u32 flags = 0; + assert(isExternalReport(ir)); /* only external reports should reach here */ struct core_info *ci = &scratch->core_info; - const struct RoseEngine *rose = ci->rose; - const struct internal_report *ri = getInternalReport(rose, id); - - assert(isExternalReport(ri)); /* only external reports should reach here */ - - DEBUG_PRINTF("internal match at %llu: IID=%u type=%hhu RID=%u " - "offsetAdj=%d\n", to_offset, id, ri->type, ri->onmatch, - ri->offsetAdjust); - - assert(!can_stop_matching(scratch)); - assert(!ri->hasBounds || - (to_offset >= ri->minOffset && to_offset <= ri->maxOffset)); - assert(ri->ekey == INVALID_EKEY || - !isExhausted(ci->exhaustionVector, ri->ekey)); - - to_offset += ri->offsetAdjust; - assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset); - - assert(!ri->minLength || - satisfiesMinLength(ri->minLength, from_offset, to_offset)); - assert(!ri->quashSom || from_offset == 0); - - DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n", - from_offset, to_offset, ri->onmatch, ci->userContext); + u32 flags = 0; #ifndef RELEASE_BUILD - if (ri->offsetAdjust != 0) { + if (ir->offsetAdjust != 0) { // alert testing tools that we've got adjusted matches flags |= HS_MATCH_FLAG_ADJUSTED; } #endif - int halt = ci->userCallback((unsigned int)ri->onmatch, from_offset, + assert(!can_stop_matching(scratch)); + assert(!ir->hasBounds || + (to_offset >= ir->minOffset && to_offset <= ir->maxOffset)); + assert(ir->ekey == INVALID_EKEY || + !isExhausted(ci->exhaustionVector, ir->ekey)); + + to_offset += ir->offsetAdjust; + assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset); + + assert(!ir->minLength || + satisfiesMinLength(ir->minLength, from_offset, to_offset)); + assert(!ir->quashSom || from_offset == 0); + + DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n", + from_offset, to_offset, ir->onmatch, ci->userContext); + + + int halt = ci->userCallback((unsigned int)ir->onmatch, from_offset, to_offset, flags, ci->userContext); if (halt) { @@ -521,8 +508,8 @@ int roseDeliverSomReport(u64a from_offset, u64a to_offset, ReportID id, } if (is_exhaustible) { - assert(ri->ekey != INVALID_EKEY); - markAsMatched(ci->exhaustionVector, ri->ekey); + assert(ir->ekey != INVALID_EKEY); + markAsMatched(ci->exhaustionVector, ir->ekey); return MO_CONTINUE_MATCHING; } else { return ROSE_CONTINUE_MATCHING_NO_EXHAUST; diff --git a/src/rose/block.c b/src/rose/block.c index e081d3ae..853f1ead 100644 --- a/src/rose/block.c +++ b/src/rose/block.c @@ -150,10 +150,7 @@ void init_for_block(const struct RoseEngine *t, struct hs_scratch *scratch, tctxt->minMatchOffset = 0; tctxt->minNonMpvMatchOffset = 0; tctxt->next_mpv_offset = 0; - tctxt->curr_anchored_loc = MMB_INVALID; - tctxt->curr_row_offset = 0; - scratch->am_log_sum = 0; /* clear the anchored logs */ scratch->al_log_sum = 0; fatbit_clear(scratch->aqa); @@ -219,7 +216,6 @@ void roseBlockExec_i(const struct RoseEngine *t, struct hs_scratch *scratch, goto exit; } - resetAnchoredLog(t, scratch); skip_atable:; } @@ -263,5 +259,5 @@ exit:; assert(!can_stop_matching(scratch)); - roseCatchUpTo(t, scratch, length, 0); + roseCatchUpTo(t, scratch, length); } diff --git a/src/rose/catchup.c b/src/rose/catchup.c index b84ca59c..c740fe08 100644 --- a/src/rose/catchup.c +++ b/src/rose/catchup.c @@ -26,6 +26,11 @@ * POSSIBILITY OF SUCH DAMAGE. */ +/** + * \file + * \brief Rose runtime: code for catching up output-exposed engines. + */ + #include "catchup.h" #include "match.h" #include "rose.h" @@ -53,7 +58,7 @@ int handleReportInternally(const struct RoseEngine *t, return 1; } if (ri->type == INTERNAL_ROSE_CHAIN) { - roseHandleChainMatch(t, scratch, id, offset, 0, 1); + roseHandleChainMatch(t, scratch, id, offset, 1); return 1; } @@ -80,66 +85,6 @@ int handleReportInternallyNoChain(const struct RoseEngine *t, return 0; } -static really_inline -void currentAnchoredMatch(const struct RoseEngine *t, - struct RoseContext *tctxt, ReportID *reportId, - u64a *end) { - if (tctxt->curr_anchored_loc == MMB_INVALID) { - *end = ANCHORED_MATCH_SENTINEL; - *reportId = ANCHORED_MATCH_SENTINEL; - DEBUG_PRINTF("curr %u [idx = %u] @%llu\n", *reportId, - tctxt->curr_row_offset, *end); - return; - } - - *end = tctxt->curr_anchored_loc + t->maxSafeAnchoredDROffset + 1; - *reportId = getAnchoredMap(t)[tctxt->curr_row_offset]; - - DEBUG_PRINTF("curr %u [idx = %u] @%llu\n", *reportId, - tctxt->curr_row_offset, *end); -} - -static rose_inline -void nextAnchoredMatch(const struct RoseEngine *t, struct hs_scratch *scratch, - ReportID *reportId, u64a *end) { - struct RoseContext *tctxt = &scratch->tctxt; - assert(tctxt->curr_anchored_loc != MMB_INVALID); - - struct fatbit **anchoredRows = getAnchoredLog(scratch); - - u32 region_width = t->anchoredMatches; - struct fatbit *curr_row = anchoredRows[tctxt->curr_anchored_loc]; - - tctxt->curr_row_offset = fatbit_iterate(curr_row, region_width, - tctxt->curr_row_offset); - DEBUG_PRINTF("next %u [idx = %u] @%llu\n", *reportId, - tctxt->curr_row_offset, *end); - if (tctxt->curr_row_offset != MMB_INVALID) { - *end = tctxt->curr_anchored_loc + t->maxSafeAnchoredDROffset + 1; - *reportId = getAnchoredMap(t)[tctxt->curr_row_offset]; - return; - } - - tctxt->curr_anchored_loc = bf64_iterate(scratch->am_log_sum, - tctxt->curr_anchored_loc); - - if (tctxt->curr_anchored_loc == MMB_INVALID) { - *end = ANCHORED_MATCH_SENTINEL; - *reportId = ANCHORED_MATCH_SENTINEL; - return; - } - - assert(tctxt->curr_anchored_loc < scratch->anchored_region_len); - curr_row = anchoredRows[tctxt->curr_anchored_loc]; - - tctxt->curr_row_offset = fatbit_iterate(curr_row, region_width, - MMB_INVALID); - assert(tctxt->curr_row_offset != MMB_INVALID); - - *end = tctxt->curr_anchored_loc + t->maxSafeAnchoredDROffset + 1; - *reportId = getAnchoredMap(t)[tctxt->curr_row_offset]; -} - static really_inline void deactivateQueue(const struct RoseEngine *t, u8 *aa, u32 qi, struct hs_scratch *scratch) { @@ -767,7 +712,7 @@ hwlmcb_rv_t buildSufPQ_final(const struct RoseEngine *t, s64a report_ok_loc, char alive = blast_queue(t, scratch, q, a_qi, second_place_loc, 0); - /* We have three posible outcomes: + /* We have three possible outcomes: * (1) the nfa died * (2) we completed the queue (implies that second_place_loc == final_loc) * (3) the queue ran to second_place_loc and stopped. In this case we need @@ -1089,119 +1034,7 @@ exit:; return HWLM_CONTINUE_MATCHING; } -static really_inline -hwlmcb_rv_t roseCatchUpNfasAndMpv(const struct RoseEngine *t, - s64a loc, s64a final_loc, - struct hs_scratch *scratch) { - hwlmcb_rv_t rv = roseCatchUpNfas(t, loc, final_loc, scratch); - - if (rv != HWLM_CONTINUE_MATCHING) { - return rv; - } - - return roseCatchUpMPV(t, loc, scratch); -} - - -static really_inline -hwlmcb_rv_t roseCatchUpAll_i(s64a loc, struct hs_scratch *scratch, - char do_full_mpv) { - const struct RoseEngine *t = scratch->core_info.rose; - assert(t->activeArrayCount); /* otherwise use roseCatchUpAnchoredOnly */ - struct RoseContext *tctxt = &scratch->tctxt; - u64a current_offset = scratch->core_info.buf_offset + loc; - - u64a anchored_end; - ReportID anchored_report; - currentAnchoredMatch(t, tctxt, &anchored_report, &anchored_end); - - DEBUG_PRINTF("am current_offset %llu\n", current_offset); - DEBUG_PRINTF("min match offset %llu\n", scratch->tctxt.minMatchOffset); - DEBUG_PRINTF("min non mpv match offset %llu\n", - scratch->tctxt.minNonMpvMatchOffset); - - assert(current_offset > tctxt->minMatchOffset); - assert(anchored_end != ANCHORED_MATCH_SENTINEL); - - hwlmcb_rv_t rv = buildSufPQ(t, scratch->core_info.state, - anchored_end - scratch->core_info.buf_offset, - loc, scratch); - if (rv != HWLM_CONTINUE_MATCHING) { - return rv; - } - - /* buildSufPQ may have caught only part of the pq upto anchored_end */ - rv = roseCatchUpNfas(t, - anchored_end - scratch->core_info.buf_offset, loc, - scratch); - - if (rv != HWLM_CONTINUE_MATCHING) { - return rv; - } - - while (anchored_report != MO_INVALID_IDX - && anchored_end <= current_offset) { - if (anchored_end != tctxt->minMatchOffset) { - rv = roseCatchUpNfasAndMpv(t, - anchored_end - scratch->core_info.buf_offset, - loc, scratch); - if (rv != HWLM_CONTINUE_MATCHING) { - DEBUG_PRINTF("halting\n"); - return rv; - } - } - - assert(anchored_end == tctxt->minMatchOffset); - updateLastMatchOffset(tctxt, anchored_end); - - if (handleReportInternally(t, scratch, anchored_report, anchored_end)) { - goto next; - } - - if (tctxt->cb(anchored_end, anchored_report, scratch) - == MO_HALT_MATCHING) { - DEBUG_PRINTF("termination requested\n"); - return HWLM_TERMINATE_MATCHING; - } - next: - nextAnchoredMatch(t, scratch, &anchored_report, &anchored_end); - DEBUG_PRINTF("catch up %u %llu\n", anchored_report, anchored_end); - } - - if (current_offset == tctxt->minMatchOffset) { - DEBUG_PRINTF("caught up\n"); - assert(scratch->catchup_pq.qm_size <= t->outfixEndQueue); - return HWLM_CONTINUE_MATCHING; - } - - rv = roseCatchUpNfas(t, loc, loc, scratch); - - if (rv != HWLM_CONTINUE_MATCHING) { - return rv; - } - - assert(scratch->catchup_pq.qm_size <= t->outfixEndQueue - || rv == HWLM_TERMINATE_MATCHING); - - if (do_full_mpv) { - /* finish off any outstanding chained matches */ - rv = roseCatchUpMPV(t, loc, scratch); - } - - DEBUG_PRINTF("catchup all done %llu\n", current_offset); - - return rv; -} - hwlmcb_rv_t roseCatchUpAll(s64a loc, struct hs_scratch *scratch) { - return roseCatchUpAll_i(loc, scratch, 1); -} - -hwlmcb_rv_t roseCatchUpAnchoredAndSuf(s64a loc, struct hs_scratch *scratch) { - return roseCatchUpAll_i(loc, scratch, 0); -} - -hwlmcb_rv_t roseCatchUpSufAndChains(s64a loc, struct hs_scratch *scratch) { /* just need suf/outfixes and mpv */ DEBUG_PRINTF("loc %lld mnmmo %llu mmo %llu\n", loc, scratch->tctxt.minNonMpvMatchOffset, @@ -1248,42 +1081,3 @@ hwlmcb_rv_t roseCatchUpSuf(s64a loc, struct hs_scratch *scratch) { return rv; } - -hwlmcb_rv_t roseCatchUpAnchoredOnly(s64a loc, struct hs_scratch *scratch) { - const struct RoseEngine *t = scratch->core_info.rose; - struct RoseContext *tctxt = &scratch->tctxt; - - assert(!t->activeArrayCount); /* otherwise use roseCatchUpAll */ - - u64a current_offset = scratch->core_info.buf_offset + loc; - u64a anchored_end; - ReportID anchored_report; - currentAnchoredMatch(t, tctxt, &anchored_report, &anchored_end); - - DEBUG_PRINTF("am current_offset %llu\n", current_offset); - - assert(current_offset > tctxt->minMatchOffset); - - while (anchored_report != MO_INVALID_IDX - && anchored_end <= current_offset) { - updateLastMatchOffset(tctxt, anchored_end); - - /* as we require that there are no leaf nfas - there must be no nfa */ - if (handleReportInternallyNoChain(t, scratch, anchored_report, - anchored_end)) { - goto next; - } - - if (tctxt->cb(anchored_end, anchored_report, scratch) - == MO_HALT_MATCHING) { - DEBUG_PRINTF("termination requested\n"); - return HWLM_TERMINATE_MATCHING; - } - next: - nextAnchoredMatch(t, scratch, &anchored_report, &anchored_end); - DEBUG_PRINTF("catch up %u %llu\n", anchored_report, anchored_end); - } - - updateMinMatchOffset(tctxt, current_offset); - return HWLM_CONTINUE_MATCHING; -} diff --git a/src/rose/catchup.h b/src/rose/catchup.h index 65fd12c9..910aa8da 100644 --- a/src/rose/catchup.h +++ b/src/rose/catchup.h @@ -26,6 +26,25 @@ * POSSIBILITY OF SUCH DAMAGE. */ +/** + * \file + * \brief Rose runtime: code for catching up output-exposed engines. + * + * Rose has several components which run behind the main (floating table) clock + * and need to be caught up before we report matches. + * + * Currently we have to deal with: + * 1. Suffix/Outfix NFAs + * 2. A single MPV NFA (chained), which may also be triggered by (1). + * + * The approach is to: + * - (A) build a priority queue of the suffix/outfixes based on their first + * match location; + * - (B) process the matches from the priority queue in order; + * - (C) As we report matches from (B) we interleave matches from the MPV if it + * exists. + */ + #ifndef ROSE_CATCHUP_H #define ROSE_CATCHUP_H @@ -35,43 +54,16 @@ #include "rose_common.h" #include "rose_internal.h" #include "ue2common.h" -#include "nfa/nfa_internal.h" -#include "util/bitutils.h" #include "util/multibit.h" -/* - * Rose has several components which run behind the main (floating table) clock - * and need to be caught up before we report matches. - * - * Currently we have to deal with: - * 1) Stored matches from the anchored matcher - * 2) Suffix/Outfix nfas - * 3) a single MPV nfa (chained) (which may also be triggered by (1) and (2)). - * - * The approach is to: - * A) build a priority queue of the suffix/outfixes based on their first match - * location - * B) process the matches from the anchored matches in order - * C) As we report a match from (B) we interleave matches from the suffixes - * D) As we report matches from (B) and (C) we interleave matches from the - * mpv if it exists. - */ /* Callbacks, defined in catchup.c */ -hwlmcb_rv_t roseCatchUpSufAndChains(s64a loc, struct hs_scratch *scratch); - hwlmcb_rv_t roseCatchUpAll(s64a loc, struct hs_scratch *scratch); -hwlmcb_rv_t roseCatchUpAnchoredOnly(s64a loc, struct hs_scratch *scratch); - - -/* will only catch mpv upto last reported external match */ +/* will only catch mpv up to last reported external match */ hwlmcb_rv_t roseCatchUpSuf(s64a loc, struct hs_scratch *scratch); -/* will only catch mpv upto last reported external match */ -hwlmcb_rv_t roseCatchUpAnchoredAndSuf(s64a loc, struct hs_scratch *scratch); - hwlmcb_rv_t roseCatchUpMPV_i(const struct RoseEngine *t, s64a loc, struct hs_scratch *scratch); @@ -81,44 +73,42 @@ void streamInitSufPQ(const struct RoseEngine *t, char *state, struct hs_scratch *scratch); static really_inline -hwlmcb_rv_t roseCatchUpMPV(const struct RoseEngine *t, s64a loc, - struct hs_scratch *scratch) { - u64a cur_offset = loc + scratch->core_info.buf_offset; - assert(cur_offset >= scratch->tctxt.minMatchOffset); - - if (0) { - quick_exit: - updateMinMatchOffsetFromMpv(&scratch->tctxt, cur_offset); - return HWLM_CONTINUE_MATCHING; - } - +int canSkipCatchUpMPV(const struct RoseEngine *t, struct hs_scratch *scratch, + u64a cur_offset) { if (!has_chained_nfas(t)) { - goto quick_exit; + return 1; } /* note: we may have to run at less than tctxt.minMatchOffset as we may * have a full queue of postponed events that we need to flush */ if (cur_offset < scratch->tctxt.next_mpv_offset) { - DEBUG_PRINTF("skipping cur_offset %lld min %lld, mpv %lld\n", + DEBUG_PRINTF("skipping cur_offset %llu min %llu, mpv %llu\n", cur_offset, scratch->tctxt.minMatchOffset, scratch->tctxt.next_mpv_offset); - goto quick_exit; + return 1; } assert(t->activeArrayCount); - DEBUG_PRINTF("cur offset offset: %lld\n", cur_offset); + DEBUG_PRINTF("cur offset offset: %llu\n", cur_offset); DEBUG_PRINTF("min match offset %llu\n", scratch->tctxt.minMatchOffset); - DEBUG_PRINTF("roseCatchUpMPV to %lld\n", loc); - assert(t->outfixBeginQueue == 1); /* if it exists mpv is queue 0 */ - u8 *aa = getActiveLeafArray(t, scratch->core_info.state); - u32 aaCount = t->activeArrayCount; + const u8 *aa = getActiveLeafArray(t, scratch->core_info.state); + return !mmbit_isset(aa, t->activeArrayCount, 0); +} - if (!mmbit_isset(aa, aaCount, 0)){ - goto quick_exit; +/** \brief Catches up the MPV. */ +static really_inline +hwlmcb_rv_t roseCatchUpMPV(const struct RoseEngine *t, s64a loc, + struct hs_scratch *scratch) { + u64a cur_offset = loc + scratch->core_info.buf_offset; + assert(cur_offset >= scratch->tctxt.minMatchOffset); + + if (canSkipCatchUpMPV(t, scratch, cur_offset)) { + updateMinMatchOffsetFromMpv(&scratch->tctxt, cur_offset); + return HWLM_CONTINUE_MATCHING; } /* Note: chained tails MUST not participate in the priority queue as @@ -128,20 +118,10 @@ hwlmcb_rv_t roseCatchUpMPV(const struct RoseEngine *t, s64a loc, return roseCatchUpMPV_i(t, loc, scratch); } -static really_inline -u64a currentAnchoredEnd(const struct RoseEngine *t, struct RoseContext *tctxt) { - if (tctxt->curr_anchored_loc == MMB_INVALID) { - return ANCHORED_MATCH_SENTINEL; - } else { - return tctxt->curr_anchored_loc + t->maxSafeAnchoredDROffset + 1; - } -} - -/* catches up nfas, anchored matches and the mpv */ +/** \brief Catches up NFAs and the MPV. */ static rose_inline hwlmcb_rv_t roseCatchUpTo(const struct RoseEngine *t, - struct hs_scratch *scratch, u64a end, - char in_anchored) { + struct hs_scratch *scratch, u64a end) { /* no need to catch up if we are at the same offset as last time */ if (end <= scratch->tctxt.minMatchOffset) { /* we must already be up to date */ @@ -158,24 +138,13 @@ hwlmcb_rv_t roseCatchUpTo(const struct RoseEngine *t, } assert(scratch->tctxt.minMatchOffset >= scratch->core_info.buf_offset); - u64a curr_anchored_end = currentAnchoredEnd(t, &scratch->tctxt); hwlmcb_rv_t rv; - if (in_anchored - || curr_anchored_end == ANCHORED_MATCH_SENTINEL - || curr_anchored_end > end) { - if (!t->activeArrayCount - || !mmbit_any(getActiveLeafArray(t, state), t->activeArrayCount)) { - updateMinMatchOffset(&scratch->tctxt, end); - rv = HWLM_CONTINUE_MATCHING; - } else { - rv = roseCatchUpSufAndChains(loc, scratch); - } + if (!t->activeArrayCount + || !mmbit_any(getActiveLeafArray(t, state), t->activeArrayCount)) { + updateMinMatchOffset(&scratch->tctxt, end); + rv = HWLM_CONTINUE_MATCHING; } else { - if (!t->activeArrayCount) { - rv = roseCatchUpAnchoredOnly(loc, scratch); - } else { - rv = roseCatchUpAll(loc, scratch); - } + rv = roseCatchUpAll(loc, scratch); } assert(rv != HWLM_CONTINUE_MATCHING @@ -185,13 +154,16 @@ hwlmcb_rv_t roseCatchUpTo(const struct RoseEngine *t, return rv; } -/* Catches up anything which may add triggers on the mpv: anchored matches - * and suf/outfixes. The MPV will be run only to intersperse matches in - * the output match stream if external matches are raised. */ +/** + * \brief Catches up anything which may add triggers on the MPV (suffixes and + * outfixes). + * + * The MPV will be run only to intersperse matches in the output match stream + * if external matches are raised. + */ static rose_inline hwlmcb_rv_t roseCatchUpMpvFeeders(const struct RoseEngine *t, - struct hs_scratch *scratch, u64a end, - char in_anchored) { + struct hs_scratch *scratch, u64a end) { /* no need to catch up if we are at the same offset as last time */ if (end <= scratch->tctxt.minNonMpvMatchOffset) { /* we must already be up to date */ @@ -203,27 +175,21 @@ hwlmcb_rv_t roseCatchUpMpvFeeders(const struct RoseEngine *t, assert(t->activeArrayCount); /* mpv is in active array */ assert(scratch->tctxt.minMatchOffset >= scratch->core_info.buf_offset); - u64a curr_anchored_end = currentAnchoredEnd(t, &scratch->tctxt); - if (in_anchored - || curr_anchored_end == ANCHORED_MATCH_SENTINEL - || curr_anchored_end > end) { - if (!t->mpvTriggeredByLeaf) { - /* no need to check as they never put triggers onto the mpv */ - return HWLM_CONTINUE_MATCHING; - } - /* sadly, this branch rarely gets taken as the mpv itself is usually - * alive. */ - char *state = scratch->core_info.state; - if (!mmbit_any(getActiveLeafArray(t, state), t->activeArrayCount)) { - scratch->tctxt.minNonMpvMatchOffset = end; - return HWLM_CONTINUE_MATCHING; - } - - return roseCatchUpSuf(loc, scratch); - } else { - return roseCatchUpAnchoredAndSuf(loc, scratch); + if (!t->mpvTriggeredByLeaf) { + /* no need to check as they never put triggers onto the mpv */ + return HWLM_CONTINUE_MATCHING; } + + /* sadly, this branch rarely gets taken as the mpv itself is usually + * alive. */ + char *state = scratch->core_info.state; + if (!mmbit_any(getActiveLeafArray(t, state), t->activeArrayCount)) { + scratch->tctxt.minNonMpvMatchOffset = end; + return HWLM_CONTINUE_MATCHING; + } + + return roseCatchUpSuf(loc, scratch); } #endif diff --git a/src/rose/eod.c b/src/rose/eod.c index c6f9e09e..24e9113d 100644 --- a/src/rose/eod.c +++ b/src/rose/eod.c @@ -50,8 +50,6 @@ void initContext(const struct RoseEngine *t, char *state, u64a offset, tctxt->minMatchOffset = offset; tctxt->minNonMpvMatchOffset = offset; tctxt->next_mpv_offset = offset; - tctxt->curr_anchored_loc = MMB_INVALID; - tctxt->curr_row_offset = 0; scratch->catchup_pq.qm_size = 0; scratch->al_log_sum = 0; /* clear the anchored logs */ @@ -332,6 +330,10 @@ void roseEodExec(const struct RoseEngine *t, u64a offset, scratch->core_info.len, scratch->core_info.hbuf, scratch->core_info.hlen); + // We should not have been called if we've already been told to terminate + // matching. + assert(!told_to_stop_matching(scratch)); + if (t->maxBiAnchoredWidth != ROSE_BOUND_INF && offset > t->maxBiAnchoredWidth) { DEBUG_PRINTF("bailing, we are beyond max width\n"); diff --git a/src/rose/infix.h b/src/rose/infix.h index e3abc7fd..9cf9c0ad 100644 --- a/src/rose/infix.h +++ b/src/rose/infix.h @@ -32,6 +32,7 @@ #include "ue2common.h" #include "nfa/nfa_api.h" #include "nfa/nfa_api_queue.h" +#include "nfa/nfa_internal.h" static really_inline int infixTooOld(struct mq *q, s64a curr_loc) { diff --git a/src/rose/match.c b/src/rose/match.c index d626950b..f62a5824 100644 --- a/src/rose/match.c +++ b/src/rose/match.c @@ -90,19 +90,15 @@ hwlmcb_rv_t roseDelayRebuildCallback(size_t start, size_t end, u32 id, DEBUG_PRINTF("STATE groups=0x%016llx\n", tctx->groups); - if (isLiteralDR(id)) { - return tctx->groups; - } - - assert(id < t->literalCount); const u32 *delayRebuildPrograms = getByOffset(t, t->litDelayRebuildProgramOffset); - const u32 programOffset = delayRebuildPrograms[id]; + assert(id < t->literalCount); + const u32 program = delayRebuildPrograms[id]; - if (programOffset) { + if (program) { const size_t match_len = end - start + 1; UNUSED hwlmcb_rv_t rv = - roseRunProgram(t, scratch, programOffset, real_end, match_len, 0); + roseRunProgram(t, scratch, program, real_end, match_len, 0); assert(rv != HWLM_TERMINATE_MATCHING); } @@ -115,31 +111,8 @@ hwlmcb_rv_t roseDelayRebuildCallback(size_t start, size_t end, u32 id, static really_inline hwlmcb_rv_t ensureMpvQueueFlushed(const struct RoseEngine *t, struct hs_scratch *scratch, u32 qi, s64a loc, - char in_anchored, char in_chained) { - return ensureQueueFlushed_i(t, scratch, qi, loc, 1, in_anchored, - in_chained); -} - -static rose_inline -void recordAnchoredMatch(const struct RoseEngine *t, struct hs_scratch *scratch, - ReportID reportId, u64a end) { - struct fatbit **anchoredRows = getAnchoredLog(scratch); - - DEBUG_PRINTF("record %u @ %llu\n", reportId, end); - assert(end - t->maxSafeAnchoredDROffset >= 1); - u32 adj_end = end - t->maxSafeAnchoredDROffset - 1; - DEBUG_PRINTF("adjusted location %u/%u\n", adj_end, - scratch->anchored_region_len); - - if (!bf64_set(&scratch->am_log_sum, adj_end)) { - // first time, clear row - fatbit_clear(anchoredRows[adj_end]); - } - - u32 idx = getAnchoredInverseMap(t)[reportId]; - DEBUG_PRINTF("record %u @ %llu index %u\n", reportId, end, idx); - assert(idx < t->anchoredMatches); - fatbit_set(anchoredRows[adj_end], t->anchoredMatches, idx); + char in_chained) { + return ensureQueueFlushed_i(t, scratch, qi, loc, 1, in_chained); } static rose_inline @@ -166,7 +139,7 @@ void recordAnchoredLiteralMatch(const struct RoseEngine *t, hwlmcb_rv_t roseHandleChainMatch(const struct RoseEngine *t, struct hs_scratch *scratch, ReportID r, - u64a end, char in_anchored, char in_catchup) { + u64a end, char in_catchup) { struct core_info *ci = &scratch->core_info; u8 *aa = getActiveLeafArray(t, scratch->core_info.state); @@ -209,7 +182,7 @@ hwlmcb_rv_t roseHandleChainMatch(const struct RoseEngine *t, DEBUG_PRINTF("queue %u full -> catching up nfas\n", qi); /* we know it is a chained nfa and the suffixes/outfixes must already * be known to be consistent */ - if (ensureMpvQueueFlushed(t, scratch, qi, loc, in_anchored, in_catchup) + if (ensureMpvQueueFlushed(t, scratch, qi, loc, in_catchup) == HWLM_TERMINATE_MATCHING) { return HWLM_TERMINATE_MATCHING; } @@ -255,7 +228,7 @@ hwlmcb_rv_t roseHandleMatch(const struct RoseEngine *t, ReportID id, u64a end, struct hs_scratch *scratch) { struct RoseContext *tctxt = &scratch->tctxt; - assert(end == tctxt->minMatchOffset); + assert(!t->needsCatchup || end == tctxt->minMatchOffset); DEBUG_PRINTF("firing callback id=%u, end=%llu\n", id, end); updateLastMatchOffset(tctxt, end); @@ -272,38 +245,6 @@ hwlmcb_rv_t roseHandleMatch(const struct RoseEngine *t, ReportID id, u64a end, return roseHaltIfExhausted(t, scratch); } -/* handles catchup, som, cb, etc */ -static really_inline -hwlmcb_rv_t roseHandleDirectReport(const struct RoseEngine *t, - struct hs_scratch *scratch, ReportID id, - u64a offset, char in_anchored) { - // The direct report path is only used for external reports. - assert(isExternalReport(getInternalReport(t, id))); - - if (roseCatchUpTo(t, scratch, offset, in_anchored) == - HWLM_TERMINATE_MATCHING) { - return HWLM_TERMINATE_MATCHING; - } - - return roseHandleMatch(t, id, offset, scratch); -} - -static really_inline -hwlmcb_rv_t roseHandleAnchoredDirectReport(const struct RoseEngine *t, - struct hs_scratch *scratch, - u64a real_end, ReportID report) { - DEBUG_PRINTF("direct report %u, real_end=%llu\n", report, real_end); - - if (real_end > t->maxSafeAnchoredDROffset) { - DEBUG_PRINTF("match in overlapped anchored region --> stash\n"); - recordAnchoredMatch(t, scratch, report, real_end); - return HWLM_CONTINUE_MATCHING; - } - - return roseHandleDirectReport(t, scratch, report, real_end, - 1 /* in anchored */); -} - int roseAnchoredCallback(u64a end, u32 id, void *ctx) { struct RoseContext *tctxt = ctx; struct hs_scratch *scratch = tctxtToScratch(tctxt); @@ -320,7 +261,7 @@ int roseAnchoredCallback(u64a end, u32 id, void *ctx) { return MO_HALT_MATCHING; } - hwlmcb_rv_t rv = HWLM_CONTINUE_MATCHING; + const size_t match_len = 0; /* delayed literals need to be delivered before real literals; however * delayed literals only come from the floating table so if we are going @@ -329,46 +270,14 @@ int roseAnchoredCallback(u64a end, u32 id, void *ctx) { /* no history checks from anchored region and we are before the flush * boundary */ - if (isLiteralMDR(id)) { - // Multi-direct report, list of reports indexed by the ID. - u32 mdr_offset = id & ~LITERAL_MDR_FLAG; - const ReportID *report = - (const ReportID *)((const char *)t + t->multidirectOffset) + - mdr_offset; - for (; *report != MO_INVALID_IDX; report++) { - rv = roseHandleAnchoredDirectReport(t, scratch, real_end, *report); - if (rv == HWLM_TERMINATE_MATCHING) { - return MO_HALT_MATCHING; - } - } - return MO_CONTINUE_MATCHING; - } else if (isLiteralDR(id)) { - // Single direct report. - ReportID report = literalToReport(id); - rv = roseHandleAnchoredDirectReport(t, scratch, real_end, report); - if (rv == HWLM_TERMINATE_MATCHING) { - return MO_HALT_MATCHING; - } - return MO_CONTINUE_MATCHING; - } - - assert(id < t->literalCount); - const u32 *programs = getByOffset(t, t->litProgramOffset); - const u32 programOffset = programs[id]; - assert(programOffset); - - // Anchored literals are never delayed. - assert(!((const u32 *)getByOffset(t, t->litDelayRebuildProgramOffset))[id]); - - DEBUG_PRINTF("literal id=%u\n", id); - if (real_end <= t->floatingMinLiteralMatchOffset) { roseFlushLastByteHistory(t, scratch, real_end); tctxt->lastEndOffset = real_end; } - const size_t match_len = 0; - if (roseRunProgram(t, scratch, programOffset, real_end, match_len, 1) == + const u32 *programs = getByOffset(t, t->litProgramOffset); + assert(id < t->literalCount); + if (roseRunProgram(t, scratch, programs[id], real_end, match_len, 1) == HWLM_TERMINATE_MATCHING) { assert(can_stop_matching(scratch)); DEBUG_PRINTF("caller requested termination\n"); @@ -387,65 +296,15 @@ int roseAnchoredCallback(u64a end, u32 id, void *ctx) { // Rose match-processing workhorse /* assumes not in_anchored */ static really_inline -hwlmcb_rv_t roseProcessMatch_i(const struct RoseEngine *t, - struct hs_scratch *scratch, u64a end, - size_t match_len, u32 id, char in_delay_play, - char in_anch_playback) { +hwlmcb_rv_t roseProcessMatch(const struct RoseEngine *t, + struct hs_scratch *scratch, u64a end, + size_t match_len, u32 id) { DEBUG_PRINTF("id=%u\n", id); - - if (!in_anch_playback && !in_delay_play) { - if (isLiteralMDR(id)) { - // Multi-direct report, list of reports indexed by the ID. - u32 mdr_offset = id & ~LITERAL_MDR_FLAG; - const ReportID *report = - (const ReportID *)((const char *)t + t->multidirectOffset) + - mdr_offset; - for (; *report != MO_INVALID_IDX; report++) { - DEBUG_PRINTF("handle multi-direct report %u\n", *report); - hwlmcb_rv_t rv = roseHandleDirectReport(t, scratch, *report, - end, 0 /* in anchored */); - if (rv == HWLM_TERMINATE_MATCHING) { - return HWLM_TERMINATE_MATCHING; - } - } - return HWLM_CONTINUE_MATCHING; - } else if (isLiteralDR(id)) { - // Single direct report. - ReportID report = literalToReport(id); - DEBUG_PRINTF("handle direct report %u\n", report); - return roseHandleDirectReport(t, scratch, report, end, - 0 /* in anchored */); - } - } - - assert(id < t->literalCount); const u32 *programs = getByOffset(t, t->litProgramOffset); + assert(id < t->literalCount); return roseRunProgram(t, scratch, programs[id], end, match_len, 0); } -static never_inline -hwlmcb_rv_t roseProcessDelayedMatch(const struct RoseEngine *t, - struct hs_scratch *scratch, u64a end, - u32 id) { - size_t match_len = 0; - return roseProcessMatch_i(t, scratch, end, match_len, id, 1, 0); -} - -static never_inline -hwlmcb_rv_t roseProcessDelayedAnchoredMatch(const struct RoseEngine *t, - struct hs_scratch *scratch, - u64a end, u32 id) { - size_t match_len = 0; - return roseProcessMatch_i(t, scratch, end, match_len, id, 0, 1); -} - -static really_inline -hwlmcb_rv_t roseProcessMainMatch(const struct RoseEngine *t, - struct hs_scratch *scratch, u64a end, - size_t match_len, u32 id) { - return roseProcessMatch_i(t, scratch, end, match_len, id, 0, 0); -} - static rose_inline hwlmcb_rv_t playDelaySlot(const struct RoseEngine *t, struct hs_scratch *scratch, @@ -472,8 +331,7 @@ hwlmcb_rv_t playDelaySlot(const struct RoseEngine *t, UNUSED rose_group old_groups = tctxt->groups; DEBUG_PRINTF("DELAYED MATCH id=%u offset=%llu\n", literal_id, offset); - hwlmcb_rv_t rv = - roseProcessDelayedMatch(t, scratch, offset, literal_id); + hwlmcb_rv_t rv = roseProcessMatch(t, scratch, offset, 0, literal_id); DEBUG_PRINTF("DONE groups=0x%016llx\n", tctxt->groups); /* delayed literals can't safely set groups. @@ -507,8 +365,7 @@ hwlmcb_rv_t flushAnchoredLiteralAtLoc(const struct RoseEngine *t, rose_group old_groups = tctxt->groups; DEBUG_PRINTF("ANCH REPLAY MATCH id=%u offset=%u\n", literal_id, curr_loc); - hwlmcb_rv_t rv = - roseProcessDelayedAnchoredMatch(t, scratch, curr_loc, literal_id); + hwlmcb_rv_t rv = roseProcessMatch(t, scratch, curr_loc, 0, literal_id); DEBUG_PRINTF("DONE groups=0x%016llx\n", tctxt->groups); /* anchored literals can't safely set groups. @@ -707,7 +564,7 @@ hwlmcb_rv_t roseCallback(size_t start, size_t end, u32 id, void *ctxt) { } size_t match_len = end - start + 1; - rv = roseProcessMainMatch(t, scratch, real_end, match_len, id); + rv = roseProcessMatch(t, scratch, real_end, match_len, id); DEBUG_PRINTF("DONE groups=0x%016llx\n", tctx->groups); @@ -719,3 +576,23 @@ hwlmcb_rv_t roseCallback(size_t start, size_t end, u32 id, void *ctxt) { DEBUG_PRINTF("user requested halt\n"); return HWLM_TERMINATE_MATCHING; } + +/** + * \brief Match callback adaptor used for matches from pure-literal cases. + * + * Literal match IDs in this path run limited Rose programs that do not use + * Rose state (which is not initialised in the pure-literal path). They can + * still, for example, check lookarounds or literal masks. + */ +hwlmcb_rv_t rosePureLiteralCallback(size_t start, size_t end, u32 id, + void *context) { + DEBUG_PRINTF("start=%zu, end=%zu, id=%u\n", start, end, id); + struct hs_scratch *scratch = context; + struct core_info *ci = &scratch->core_info; + const u64a real_end = (u64a)end + ci->buf_offset + 1; + const size_t match_len = end - start + 1; + const struct RoseEngine *rose = ci->rose; + const u32 *programs = getByOffset(rose, rose->litProgramOffset); + assert(id < rose->literalCount); + return roseRunProgram(rose, scratch, programs[id], real_end, match_len, 0); +} diff --git a/src/rose/match.h b/src/rose/match.h index 0629d8d7..f9889139 100644 --- a/src/rose/match.h +++ b/src/rose/match.h @@ -59,27 +59,9 @@ int roseAnchoredCallback(u64a end, u32 id, void *ctx); /* Common code, used all over Rose runtime */ -static rose_inline -void resetAnchoredLog(const struct RoseEngine *t, struct hs_scratch *scratch) { - struct fatbit **anchoredRows = getAnchoredLog(scratch); - u32 region_width = t->anchoredMatches; - struct RoseContext *tctxt = &scratch->tctxt; - - tctxt->curr_anchored_loc = bf64_iterate(scratch->am_log_sum, MMB_INVALID); - if (tctxt->curr_anchored_loc != MMB_INVALID) { - assert(tctxt->curr_anchored_loc < scratch->anchored_region_len); - struct fatbit *curr_row = anchoredRows[tctxt->curr_anchored_loc]; - tctxt->curr_row_offset = fatbit_iterate(curr_row, region_width, - MMB_INVALID); - assert(tctxt->curr_row_offset != MMB_INVALID); - } - DEBUG_PRINTF("AL reset --> %u, %u\n", tctxt->curr_anchored_loc, - tctxt->curr_row_offset); -} - hwlmcb_rv_t roseHandleChainMatch(const struct RoseEngine *t, struct hs_scratch *scratch, ReportID r, - u64a end, char in_anchored, char in_catchup); + u64a end, char in_catchup); static really_inline void initQueue(struct mq *q, u32 qi, const struct RoseEngine *t, diff --git a/src/rose/program_runtime.h b/src/rose/program_runtime.h index 3d25d682..d816d62e 100644 --- a/src/rose/program_runtime.h +++ b/src/rose/program_runtime.h @@ -225,8 +225,7 @@ hwlmcb_rv_t roseHaltIfExhausted(const struct RoseEngine *t, static really_inline hwlmcb_rv_t ensureQueueFlushed_i(const struct RoseEngine *t, struct hs_scratch *scratch, u32 qi, s64a loc, - char is_mpv, char in_anchored, - char in_catchup) { + char is_mpv, char in_catchup) { struct RoseContext *tctxt = &scratch->tctxt; u8 *aa = getActiveLeafArray(t, scratch->core_info.state); struct fatbit *activeQueues = scratch->aqa; @@ -258,8 +257,8 @@ hwlmcb_rv_t ensureQueueFlushed_i(const struct RoseEngine *t, } } - if (roseCatchUpTo(t, scratch, loc + scratch->core_info.buf_offset, - in_anchored) == HWLM_TERMINATE_MATCHING) { + if (roseCatchUpTo(t, scratch, loc + scratch->core_info.buf_offset) == + HWLM_TERMINATE_MATCHING) { return HWLM_TERMINATE_MATCHING; } } else { @@ -286,15 +285,14 @@ done_queue_empty: static rose_inline hwlmcb_rv_t ensureQueueFlushed(const struct RoseEngine *t, - struct hs_scratch *scratch, u32 qi, s64a loc, - char in_anchored) { - return ensureQueueFlushed_i(t, scratch, qi, loc, 0, in_anchored, 0); + struct hs_scratch *scratch, u32 qi, s64a loc) { + return ensureQueueFlushed_i(t, scratch, qi, loc, 0, 0); } static rose_inline hwlmcb_rv_t roseTriggerSuffix(const struct RoseEngine *t, struct hs_scratch *scratch, u32 qi, u32 top, - u64a som, u64a end, char in_anchored) { + u64a som, u64a end) { DEBUG_PRINTF("suffix qi=%u, top event=%u\n", qi, top); struct core_info *ci = &scratch->core_info; @@ -330,7 +328,7 @@ hwlmcb_rv_t roseTriggerSuffix(const struct RoseEngine *t, nfaQueueExecRose(q->nfa, q, MO_INVALID_IDX); q->cur = q->end = 0; pushQueueAt(q, 0, MQE_START, loc); - } else if (ensureQueueFlushed(t, scratch, qi, loc, in_anchored) + } else if (ensureQueueFlushed(t, scratch, qi, loc) == HWLM_TERMINATE_MATCHING) { return HWLM_TERMINATE_MATCHING; } @@ -575,18 +573,20 @@ void roseTriggerInfix(const struct RoseEngine *t, struct hs_scratch *scratch, static rose_inline hwlmcb_rv_t roseReport(const struct RoseEngine *t, struct hs_scratch *scratch, - ReportID id, u64a end, char is_exhaustible) { - assert(end == scratch->tctxt.minMatchOffset); + u64a end, ReportID id, ReportID onmatch, + s32 offset_adjust, u32 ekey) { + assert(!t->needsCatchup || end == scratch->tctxt.minMatchOffset); DEBUG_PRINTF("firing callback id=%u, end=%llu\n", id, end); updateLastMatchOffset(&scratch->tctxt, end); - int cb_rv = roseDeliverReport(end, id, scratch, is_exhaustible); + int cb_rv = roseDeliverReport(end, id, onmatch, offset_adjust, scratch, + ekey); if (cb_rv == MO_HALT_MATCHING) { DEBUG_PRINTF("termination requested\n"); return HWLM_TERMINATE_MATCHING; } - if (!is_exhaustible || cb_rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) { + if (ekey == INVALID_EKEY || cb_rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) { return HWLM_CONTINUE_MATCHING; } @@ -599,14 +599,12 @@ hwlmcb_rv_t roseReport(const struct RoseEngine *t, struct hs_scratch *scratch, static rose_inline hwlmcb_rv_t roseCatchUpAndHandleChainMatch(const struct RoseEngine *t, struct hs_scratch *scratch, - ReportID r, u64a end, - char in_anchored) { - if (roseCatchUpMpvFeeders(t, scratch, end, in_anchored) == - HWLM_TERMINATE_MATCHING) { + ReportID r, u64a end) { + if (roseCatchUpMpvFeeders(t, scratch, end) == HWLM_TERMINATE_MATCHING) { return HWLM_TERMINATE_MATCHING; } - return roseHandleChainMatch(t, scratch, r, end, in_anchored, 0); + return roseHandleChainMatch(t, scratch, r, end, 0); } static rose_inline @@ -618,7 +616,7 @@ void roseHandleSom(const struct RoseEngine *t, struct hs_scratch *scratch, // Reach into reports and handle internal reports that just manipulate SOM // slots ourselves, rather than going through the callback. - assert(end == scratch->tctxt.minMatchOffset); + assert(!t->needsCatchup || end == scratch->tctxt.minMatchOffset); DEBUG_PRINTF("firing som callback id=%u, end=%llu\n", id, end); updateLastMatchOffset(&scratch->tctxt, end); @@ -630,11 +628,12 @@ static rose_inline hwlmcb_rv_t roseReportSom(const struct RoseEngine *t, struct hs_scratch *scratch, ReportID id, u64a start, u64a end, char is_exhaustible) { - assert(end == scratch->tctxt.minMatchOffset); + assert(!t->needsCatchup || end == scratch->tctxt.minMatchOffset); DEBUG_PRINTF("firing som callback id=%u, end=%llu\n", id, end); updateLastMatchOffset(&scratch->tctxt, end); - int cb_rv = roseDeliverSomReport(start, end, id, scratch, is_exhaustible); + const struct internal_report *ir = getInternalReport(t, id); + int cb_rv = roseDeliverSomReport(start, end, ir, scratch, is_exhaustible); if (cb_rv == MO_HALT_MATCHING) { DEBUG_PRINTF("termination requested\n"); return HWLM_TERMINATE_MATCHING; @@ -656,7 +655,7 @@ void roseHandleSomSom(const struct RoseEngine *t, ReportID id, u64a start, // Reach into reports and handle internal reports that just manipulate SOM // slots ourselves, rather than going through the callback. - assert(end == scratch->tctxt.minMatchOffset); + assert(!t->needsCatchup || end == scratch->tctxt.minMatchOffset); updateLastMatchOffset(&scratch->tctxt, end); const struct internal_report *ri = getInternalReport(t, id); @@ -967,8 +966,7 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t, PROGRAM_NEXT_INSTRUCTION PROGRAM_CASE(CATCH_UP) { - if (roseCatchUpTo(t, scratch, end, in_anchored) == - HWLM_TERMINATE_MATCHING) { + if (roseCatchUpTo(t, scratch, end) == HWLM_TERMINATE_MATCHING) { return HWLM_TERMINATE_MATCHING; } } @@ -1010,8 +1008,7 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t, PROGRAM_CASE(TRIGGER_SUFFIX) { if (roseTriggerSuffix(t, scratch, ri->queue, ri->event, som, - end, in_anchored) - == HWLM_TERMINATE_MATCHING) { + end) == HWLM_TERMINATE_MATCHING) { return HWLM_TERMINATE_MATCHING; } work_done = 1; @@ -1056,8 +1053,8 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t, PROGRAM_NEXT_INSTRUCTION PROGRAM_CASE(REPORT_CHAIN) { - if (roseCatchUpAndHandleChainMatch(t, scratch, ri->report, end, - in_anchored) == + if (roseCatchUpAndHandleChainMatch(t, scratch, ri->report, + end) == HWLM_TERMINATE_MATCHING) { return HWLM_TERMINATE_MATCHING; } @@ -1078,9 +1075,9 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t, PROGRAM_NEXT_INSTRUCTION PROGRAM_CASE(REPORT) { - const char is_exhaustible = 0; - if (roseReport(t, scratch, ri->report, end, is_exhaustible) == - HWLM_TERMINATE_MATCHING) { + if (roseReport(t, scratch, end, ri->report, ri->onmatch, + ri->offset_adjust, + INVALID_EKEY) == HWLM_TERMINATE_MATCHING) { return HWLM_TERMINATE_MATCHING; } work_done = 1; @@ -1088,9 +1085,9 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t, PROGRAM_NEXT_INSTRUCTION PROGRAM_CASE(REPORT_EXHAUST) { - const char is_exhaustible = 1; - if (roseReport(t, scratch, ri->report, end, is_exhaustible) == - HWLM_TERMINATE_MATCHING) { + if (roseReport(t, scratch, end, ri->report, ri->onmatch, + ri->offset_adjust, + ri->ekey) == HWLM_TERMINATE_MATCHING) { return HWLM_TERMINATE_MATCHING; } work_done = 1; @@ -1117,6 +1114,33 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t, } PROGRAM_NEXT_INSTRUCTION + PROGRAM_CASE(DEDUPE_AND_REPORT) { + const struct internal_report *ir = + getInternalReport(t, ri->report); + const char do_som = t->hasSom; // FIXME: constant propagate + enum DedupeResult rv = dedupeCatchup( + t, ir, scratch, end, som, end + ir->offsetAdjust, do_som); + switch (rv) { + case DEDUPE_HALT: + return HWLM_TERMINATE_MATCHING; + case DEDUPE_SKIP: + assert(ri->fail_jump); // must progress + pc += ri->fail_jump; + continue; + case DEDUPE_CONTINUE: + break; + } + + const u32 ekey = INVALID_EKEY; + if (roseReport(t, scratch, end, ri->report, ir->onmatch, + ir->offsetAdjust, + ekey) == HWLM_TERMINATE_MATCHING) { + return HWLM_TERMINATE_MATCHING; + } + work_done = 1; + } + PROGRAM_NEXT_INSTRUCTION + PROGRAM_CASE(CHECK_EXHAUSTED) { DEBUG_PRINTF("check ekey %u\n", ri->ekey); assert(ri->ekey != INVALID_EKEY); diff --git a/src/rose/rose.h b/src/rose/rose.h index 95750363..5dc57bc2 100644 --- a/src/rose/rose.h +++ b/src/rose/rose.h @@ -85,6 +85,10 @@ void roseBlockExec(const struct RoseEngine *t, struct hs_scratch *scratch, assert(scratch); assert(scratch->core_info.buf); + // We should not have been called if we've already been told to terminate + // matching. + assert(!told_to_stop_matching(scratch)); + // If this block is shorter than our minimum width, then no pattern in this // RoseEngine could match. /* minWidth checks should have already been performed by the caller */ @@ -124,4 +128,7 @@ void roseEodExec(const struct RoseEngine *t, u64a offset, struct hs_scratch *scratch, RoseCallback callback, RoseCallbackSom som_callback); +hwlmcb_rv_t rosePureLiteralCallback(size_t start, size_t end, u32 id, + void *context); + #endif // ROSE_H diff --git a/src/rose/rose_build_anchored.cpp b/src/rose/rose_build_anchored.cpp index 57faa46c..805dc920 100644 --- a/src/rose/rose_build_anchored.cpp +++ b/src/rose/rose_build_anchored.cpp @@ -231,10 +231,6 @@ u32 anchoredStateSize(const anchored_matcher_info &atable) { return curr->state_offset + nfa->streamStateSize; } -bool anchoredIsMulti(const anchored_matcher_info &atable) { - return atable.next_offset; -} - namespace { typedef bitfield nfa_state_set; @@ -742,21 +738,24 @@ void buildSimpleDfas(const RoseBuildImpl &tbi, * from RoseBuildImpl. */ static -void getAnchoredDfas(RoseBuildImpl &tbi, - vector> *anchored_dfas) { +vector> getAnchoredDfas(RoseBuildImpl &build) { + vector> dfas; + // DFAs that already exist as raw_dfas. - for (auto &anch_dfas : tbi.anchored_nfas) { + for (auto &anch_dfas : build.anchored_nfas) { for (auto &rdfa : anch_dfas.second) { - anchored_dfas->push_back(move(rdfa)); + dfas.push_back(move(rdfa)); } } - tbi.anchored_nfas.clear(); + build.anchored_nfas.clear(); // DFAs we currently have as simple literals. - if (!tbi.anchored_simple.empty()) { - buildSimpleDfas(tbi, anchored_dfas); - tbi.anchored_simple.clear(); + if (!build.anchored_simple.empty()) { + buildSimpleDfas(build, &dfas); + build.anchored_simple.clear(); } + + return dfas; } /** @@ -770,9 +769,9 @@ void getAnchoredDfas(RoseBuildImpl &tbi, * \return Total bytes required for the complete anchored matcher. */ static -size_t buildNfas(vector> &anchored_dfas, - vector> *nfas, vector *start_offset, - const CompileContext &cc) { +size_t buildNfas(vector &anchored_dfas, + vector> *nfas, + vector *start_offset, const CompileContext &cc) { const size_t num_dfas = anchored_dfas.size(); nfas->reserve(num_dfas); @@ -781,12 +780,12 @@ size_t buildNfas(vector> &anchored_dfas, size_t total_size = 0; for (auto &rdfa : anchored_dfas) { - u32 removed_dots = remove_leading_dots(*rdfa); + u32 removed_dots = remove_leading_dots(rdfa); start_offset->push_back(removed_dots); - minimize_hopcroft(*rdfa, cc.grey); + minimize_hopcroft(rdfa, cc.grey); - aligned_unique_ptr nfa = mcclellanCompile(*rdfa, cc); + auto nfa = mcclellanCompile(rdfa, cc); if (!nfa) { assert(0); throw std::bad_alloc(); @@ -803,25 +802,41 @@ size_t buildNfas(vector> &anchored_dfas, return total_size; } -aligned_unique_ptr -buildAnchoredAutomataMatcher(RoseBuildImpl &build, size_t *asize) { - const CompileContext &cc = build.cc; - remapAnchoredReports(build); +vector buildAnchoredDfas(RoseBuildImpl &build) { + vector dfas; if (build.anchored_nfas.empty() && build.anchored_simple.empty()) { + DEBUG_PRINTF("empty\n"); + return dfas; + } + + remapAnchoredReports(build); + + auto anch_dfas = getAnchoredDfas(build); + mergeAnchoredDfas(anch_dfas, build); + + dfas.reserve(anch_dfas.size()); + for (auto &rdfa : anch_dfas) { + assert(rdfa); + dfas.push_back(move(*rdfa)); + } + return dfas; +} + +aligned_unique_ptr +buildAnchoredMatcher(RoseBuildImpl &build, vector &dfas, + size_t *asize) { + const CompileContext &cc = build.cc; + + if (dfas.empty()) { DEBUG_PRINTF("empty\n"); *asize = 0; return nullptr; } - vector> anchored_dfas; - getAnchoredDfas(build, &anchored_dfas); - - mergeAnchoredDfas(anchored_dfas, build); - vector> nfas; vector start_offset; // start offset for each dfa (dots removed) - size_t total_size = buildNfas(anchored_dfas, &nfas, &start_offset, cc); + size_t total_size = buildNfas(dfas, &nfas, &start_offset, cc); if (total_size > cc.grey.limitRoseAnchoredSize) { throw ResourceLimitError(); diff --git a/src/rose/rose_build_anchored.h b/src/rose/rose_build_anchored.h index d399907b..a5317f89 100644 --- a/src/rose/rose_build_anchored.h +++ b/src/rose/rose_build_anchored.h @@ -46,17 +46,22 @@ namespace ue2 { class NGHolder; class RoseBuildImpl; struct Grey; - -aligned_unique_ptr -buildAnchoredAutomataMatcher(RoseBuildImpl &build, size_t *asize); - -u32 anchoredStateSize(const anchored_matcher_info &atable); +struct raw_dfa; /** - * \brief True if there is an anchored matcher and it consists of multiple - * DFAs. + * \brief Construct a set of anchored DFAs from our anchored literals/engines. */ -bool anchoredIsMulti(const anchored_matcher_info &atable); +std::vector buildAnchoredDfas(RoseBuildImpl &build); + +/** + * \brief Construct an anchored_matcher_info runtime structure from the given + * set of DFAs. + */ +aligned_unique_ptr +buildAnchoredMatcher(RoseBuildImpl &build, std::vector &dfas, + size_t *asize); + +u32 anchoredStateSize(const anchored_matcher_info &atable); #define ANCHORED_FAIL 0 #define ANCHORED_SUCCESS 1 diff --git a/src/rose/rose_build_bytecode.cpp b/src/rose/rose_build_bytecode.cpp index a6ab726d..401c15c1 100644 --- a/src/rose/rose_build_bytecode.cpp +++ b/src/rose/rose_build_bytecode.cpp @@ -214,6 +214,7 @@ public: case ROSE_INSTR_REPORT_EXHAUST: return &u.reportExhaust; case ROSE_INSTR_REPORT_SOM: return &u.reportSom; case ROSE_INSTR_REPORT_SOM_EXHAUST: return &u.reportSomExhaust; + case ROSE_INSTR_DEDUPE_AND_REPORT: return &u.dedupeAndReport; case ROSE_INSTR_CHECK_EXHAUSTED: return &u.checkExhausted; case ROSE_INSTR_CHECK_MIN_LENGTH: return &u.checkMinLength; case ROSE_INSTR_SET_STATE: return &u.setState; @@ -257,6 +258,7 @@ public: case ROSE_INSTR_REPORT_EXHAUST: return sizeof(u.reportExhaust); case ROSE_INSTR_REPORT_SOM: return sizeof(u.reportSom); case ROSE_INSTR_REPORT_SOM_EXHAUST: return sizeof(u.reportSomExhaust); + case ROSE_INSTR_DEDUPE_AND_REPORT: return sizeof(u.dedupeAndReport); case ROSE_INSTR_CHECK_EXHAUSTED: return sizeof(u.checkExhausted); case ROSE_INSTR_CHECK_MIN_LENGTH: return sizeof(u.checkMinLength); case ROSE_INSTR_SET_STATE: return sizeof(u.setState); @@ -299,6 +301,7 @@ public: ROSE_STRUCT_REPORT_EXHAUST reportExhaust; ROSE_STRUCT_REPORT_SOM reportSom; ROSE_STRUCT_REPORT_SOM_EXHAUST reportSomExhaust; + ROSE_STRUCT_DEDUPE_AND_REPORT dedupeAndReport; ROSE_STRUCT_CHECK_EXHAUSTED checkExhausted; ROSE_STRUCT_CHECK_MIN_LENGTH checkMinLength; ROSE_STRUCT_SET_STATE setState; @@ -326,6 +329,25 @@ size_t hash_value(const RoseInstruction &ri) { return val; } +/** + * \brief Structure tracking which resources are used by this Rose instance at + * runtime. + * + * We use this to control how much initialisation we need to do at the + * beginning of a stream/block at runtime. + */ +struct RoseResources { + bool has_outfixes = false; + bool has_suffixes = false; + bool has_leftfixes = false; + bool has_literals = false; + bool has_states = false; + bool checks_groups = false; + bool has_lit_delay = false; + bool has_lit_mask = false; + bool has_anchored = false; +}; + struct build_context : boost::noncopyable { /** \brief information about engines to the left of a vertex */ map leftfix_info; @@ -373,6 +395,13 @@ struct build_context : boost::noncopyable { * RoseEngine. */ vector> engine_blob; + /** \brief True if reports need CATCH_UP instructions, to catch up anchored + * matches, suffixes, outfixes etc. */ + bool needs_catchup = false; + + /** \brief Resources in use (tracked as programs are added). */ + RoseResources resources; + /** \brief Base offset of engine_blob in the Rose engine bytecode. */ static constexpr u32 engine_blob_base = ROUNDUP_CL(sizeof(RoseEngine)); }; @@ -477,42 +506,74 @@ u32 countRosePrefixes(const vector &roses) { return num; } +/** + * \brief True if this Rose engine needs to run a catch up whenever a report is + * generated. + * + * This is only the case if there are no anchored literals, suffixes, outfixes + * etc. + */ static -bool isPureFloating(const RoseBuildImpl &tbi) { - if (!tbi.outfixes.empty()) { +bool needsCatchup(const RoseBuildImpl &build) { + if (!build.outfixes.empty()) { DEBUG_PRINTF("has outfixes\n"); - return false; + return true; } - const RoseGraph &g = tbi.g; + const RoseGraph &g = build.g; - if (!isLeafNode(tbi.anchored_root, g)) { + if (!isLeafNode(build.anchored_root, g)) { DEBUG_PRINTF("has anchored vertices\n"); - return false; + return true; } for (auto v : vertices_range(g)) { - if (tbi.root == v) { + if (build.root == v) { continue; } - if (tbi.anchored_root == v) { + if (build.anchored_root == v) { assert(isLeafNode(v, g)); continue; } - if (!tbi.allDirectFinalIds(v) || !tbi.isFloating(v)) { - DEBUG_PRINTF("vertex %zu isn't floating and direct\n", g[v].idx); - return false; + if (g[v].suffix) { + DEBUG_PRINTF("vertex %zu has suffix\n", g[v].idx); + return true; } - for (ReportID r : g[v].reports) { - const Report &ri = tbi.rm.getReport(r); - if (!isExternalReport(ri)) { - DEBUG_PRINTF("vertex %zu has non-external report\n", g[v].idx); - return false; - } - } + } + + DEBUG_PRINTF("no need for catch-up on report\n"); + return false; +} + +static +bool isPureFloating(const RoseResources &resources) { + if (resources.has_outfixes || resources.has_suffixes || + resources.has_leftfixes) { + DEBUG_PRINTF("has engines\n"); + return false; + } + + if (resources.has_anchored) { + DEBUG_PRINTF("has anchored matcher\n"); + return false; + } + + if (resources.has_states) { + DEBUG_PRINTF("has states\n"); + return false; + } + + if (resources.has_lit_delay) { + DEBUG_PRINTF("has delayed literals\n"); + return false; + } + + if (resources.checks_groups) { + DEBUG_PRINTF("has group checks\n"); + return false; } DEBUG_PRINTF("pure floating literals\n"); @@ -544,12 +605,23 @@ bool isSingleOutfix(const RoseBuildImpl &tbi, u32 outfixEndQueue) { } static -u8 pickRuntimeImpl(const RoseBuildImpl &tbi, u32 outfixEndQueue) { - if (isPureFloating(tbi)) { +u8 pickRuntimeImpl(const RoseBuildImpl &build, const build_context &bc, + u32 outfixEndQueue) { + DEBUG_PRINTF("has_outfixes=%d\n", bc.resources.has_outfixes); + DEBUG_PRINTF("has_suffixes=%d\n", bc.resources.has_suffixes); + DEBUG_PRINTF("has_leftfixes=%d\n", bc.resources.has_leftfixes); + DEBUG_PRINTF("has_literals=%d\n", bc.resources.has_literals); + DEBUG_PRINTF("has_states=%d\n", bc.resources.has_states); + DEBUG_PRINTF("checks_groups=%d\n", bc.resources.checks_groups); + DEBUG_PRINTF("has_lit_delay=%d\n", bc.resources.has_lit_delay); + DEBUG_PRINTF("has_lit_mask=%d\n", bc.resources.has_lit_mask); + DEBUG_PRINTF("has_anchored=%d\n", bc.resources.has_anchored); + + if (isPureFloating(bc.resources)) { return ROSE_RUNTIME_PURE_LITERAL; } - if (isSingleOutfix(tbi, outfixEndQueue)) { + if (isSingleOutfix(build, outfixEndQueue)) { return ROSE_RUNTIME_SINGLE_OUTFIX; } @@ -1880,31 +1952,27 @@ bool findHamsterMask(const RoseBuildImpl &tbi, const rose_literal_id &id, } static -bool isDirectHighlander(const RoseBuildImpl &tbi, +bool isDirectHighlander(const RoseBuildImpl &build, const u32 id, const rose_literal_info &info) { - u32 final_id = info.final_id; - assert(final_id != MO_INVALID_IDX); - - if ((final_id & LITERAL_MDR_FLAG) == LITERAL_MDR_FLAG) { - u32 i = final_id & ~LITERAL_MDR_FLAG; - assert(i < tbi.mdr_reports.size()); - for (ReportID report = tbi.mdr_reports[i]; report != MO_INVALID_IDX; - report = tbi.mdr_reports[++i]) { - const Report &ir = tbi.rm.getReport(report); - if (!isSimpleExhaustible(ir)) { - return false; - } - } - return true; - } else if (final_id & LITERAL_DR_FLAG) { - ReportID report = final_id & ~LITERAL_DR_FLAG; - const Report &ir = tbi.rm.getReport(report); - if (isSimpleExhaustible(ir)) { - return true; - } + if (!build.isDirectReport(id)) { + return false; } - return false; + auto is_simple_exhaustible = [&build](ReportID id) { + const Report &report = build.rm.getReport(id); + return isSimpleExhaustible(report); + }; + + assert(!info.vertices.empty()); + for (const auto &v : info.vertices) { + const auto &reports = build.g[v].reports; + assert(!reports.empty()); + if (!all_of(begin(reports), end(reports), + is_simple_exhaustible)) { + return false; + } + } + return true; } // Called by isNoRunsLiteral below. @@ -1958,7 +2026,7 @@ bool isNoRunsVertex(const RoseBuildImpl &tbi, NFAVertex u) { } static -bool isNoRunsLiteral(const RoseBuildImpl &tbi, UNUSED const u32 id, +bool isNoRunsLiteral(const RoseBuildImpl &tbi, const u32 id, const rose_literal_info &info) { DEBUG_PRINTF("lit id %u\n", id); @@ -1967,7 +2035,7 @@ bool isNoRunsLiteral(const RoseBuildImpl &tbi, UNUSED const u32 id, return false; } - if (isDirectHighlander(tbi, info)) { + if (isDirectHighlander(tbi, id, info)) { DEBUG_PRINTF("highlander direct report\n"); return true; } @@ -2293,8 +2361,8 @@ void enforceEngineSizeLimit(const NFA *n, const size_t nfa_size, const Grey &gre static u32 findMinFloatingLiteralMatch(const RoseBuildImpl &build, - const anchored_matcher_info *atable) { - if (atable && anchoredIsMulti(*atable)) { + const vector &anchored_dfas) { + if (anchored_dfas.size() > 1) { DEBUG_PRINTF("multiple anchored dfas\n"); /* We must regard matches from other anchored tables as unordered, as * we do for floating matches. */ @@ -2739,6 +2807,9 @@ flattenProgram(const vector> &programs) { case ROSE_INSTR_DEDUPE_SOM: ri.u.dedupeSom.fail_jump = jump_val; break; + case ROSE_INSTR_DEDUPE_AND_REPORT: + ri.u.dedupeAndReport.fail_jump = jump_val; + break; case ROSE_INSTR_CHECK_EXHAUSTED: ri.u.checkExhausted.fail_jump = jump_val; break; @@ -2765,6 +2836,55 @@ flattenProgram(const vector> &programs) { return out; } +static +void recordResources(RoseResources &resources, + const vector &program) { + for (const auto &ri : program) { + switch (ri.code()) { + case ROSE_INSTR_TRIGGER_SUFFIX: + resources.has_suffixes = true; + break; + case ROSE_INSTR_TRIGGER_INFIX: + case ROSE_INSTR_CHECK_INFIX: + case ROSE_INSTR_CHECK_PREFIX: + case ROSE_INSTR_SOM_LEFTFIX: + resources.has_leftfixes = true; + break; + case ROSE_INSTR_SET_STATE: + case ROSE_INSTR_CHECK_STATE: + case ROSE_INSTR_SPARSE_ITER_BEGIN: + case ROSE_INSTR_SPARSE_ITER_NEXT: + resources.has_states = true; + break; + case ROSE_INSTR_CHECK_GROUPS: + resources.checks_groups = true; + break; + case ROSE_INSTR_PUSH_DELAYED: + resources.has_lit_delay = true; + break; + case ROSE_INSTR_CHECK_LIT_MASK: + resources.has_lit_mask = true; + break; + default: + break; + } + } +} + +static +void recordResources(RoseResources &resources, + const RoseBuildImpl &build) { + if (!build.outfixes.empty()) { + resources.has_outfixes = true; + } + for (u32 i = 0; i < build.literal_info.size(); i++) { + if (build.hasFinalId(i)) { + resources.has_literals = true; + break; + } + } +} + static u32 writeProgram(build_context &bc, const vector &program) { if (program.empty()) { @@ -2788,6 +2908,8 @@ u32 writeProgram(build_context &bc, const vector &program) { return it->second; } + recordResources(bc.resources, program); + DEBUG_PRINTF("writing %zu instructions\n", program.size()); u32 programOffset = 0; for (const auto &ri : program) { @@ -3045,14 +3167,14 @@ void makeDedupeSom(const ReportID id, vector &report_block) { } static -void makeReport(RoseBuildImpl &build, const ReportID id, const bool has_som, - vector &program) { +void makeReport(RoseBuildImpl &build, build_context &bc, const ReportID id, + const bool has_som, vector &program) { assert(id < build.rm.numReports()); const Report &report = build.rm.getReport(id); vector report_block; - // Similarly, we can handle min/max offset checks. + // Handle min/max offset checks. if (report.minOffset > 0 || report.maxOffset < MAX_OFFSET) { auto ri = RoseInstruction(ROSE_INSTR_CHECK_BOUNDS, JumpTarget::NEXT_BLOCK); @@ -3064,7 +3186,7 @@ void makeReport(RoseBuildImpl &build, const ReportID id, const bool has_som, // Catch up -- everything except the INTERNAL_ROSE_CHAIN report needs this. // TODO: this could be floated in front of all the reports and only done // once. - if (report.type != INTERNAL_ROSE_CHAIN) { + if (bc.needs_catchup && report.type != INTERNAL_ROSE_CHAIN) { report_block.emplace_back(ROSE_INSTR_CATCH_UP); } @@ -3103,15 +3225,29 @@ void makeReport(RoseBuildImpl &build, const ReportID id, const bool has_som, if (!has_som) { // Dedupe is only necessary if this report has a dkey, or if there // are SOM reports to catch up. - if (build.rm.getDkey(report) != ~0U || build.hasSom) { - makeDedupe(id, report_block); - } + bool needs_dedupe = build.rm.getDkey(report) != ~0U || build.hasSom; if (report.ekey == INVALID_EKEY) { - report_block.emplace_back(ROSE_INSTR_REPORT); - report_block.back().u.report.report = id; + if (needs_dedupe) { + report_block.emplace_back(ROSE_INSTR_DEDUPE_AND_REPORT, + JumpTarget::NEXT_BLOCK); + report_block.back().u.dedupeAndReport.report = id; + } else { + report_block.emplace_back(ROSE_INSTR_REPORT); + auto &ri = report_block.back(); + ri.u.report.report = id; + ri.u.report.onmatch = report.onmatch; + ri.u.report.offset_adjust = report.offsetAdjust; + } } else { + if (needs_dedupe) { + makeDedupe(id, report_block); + } report_block.emplace_back(ROSE_INSTR_REPORT_EXHAUST); - report_block.back().u.reportExhaust.report = id; + auto &ri = report_block.back(); + ri.u.reportExhaust.report = id; + ri.u.reportExhaust.onmatch = report.onmatch; + ri.u.reportExhaust.offset_adjust = report.offsetAdjust; + ri.u.reportExhaust.ekey = report.ekey; } } else { // has_som makeDedupeSom(id, report_block); @@ -3196,7 +3332,7 @@ void makeRoleReports(RoseBuildImpl &build, build_context &bc, RoseVertex v, } for (ReportID id : g[v].reports) { - makeReport(build, id, has_som, program); + makeReport(build, bc, id, has_som, program); } } @@ -3869,6 +4005,20 @@ void makeCheckLitEarlyInstruction(const RoseBuildImpl &build, build_context &bc, program.push_back(RoseInstruction(ROSE_INSTR_CHECK_LIT_EARLY)); } +static +bool hasDelayedLiteral(RoseBuildImpl &build, + const vector &lit_edges) { + auto is_delayed = bind(&RoseBuildImpl::isDelayed, &build, _1); + for (const auto &e : lit_edges) { + auto v = target(e, build.g); + const auto &lits = build.g[v].literals; + if (any_of(begin(lits), end(lits), is_delayed)) { + return true; + } + } + return false; +} + static vector buildLitInitialProgram(RoseBuildImpl &build, build_context &bc, u32 final_id, @@ -3885,8 +4035,12 @@ vector buildLitInitialProgram(RoseBuildImpl &build, // Check lit mask. makeCheckLitMaskInstruction(build, final_id, pre_program); - // Check literal groups. - makeGroupCheckInstruction(build, final_id, pre_program); + // Check literal groups. This is an optimisation that we only perform for + // delayed literals, as their groups may be switched off; ordinarily, we + // can trust the HWLM matcher. + if (hasDelayedLiteral(build, lit_edges)) { + makeGroupCheckInstruction(build, final_id, pre_program); + } // Add instructions for pushing delayed matches, if there are any. makePushDelayedInstructions(build, final_id, pre_program); @@ -3982,8 +4136,8 @@ map> findEdgesByLiteral(const RoseBuildImpl &build) { for (const auto &lit_id : g[v].literals) { assert(lit_id < build.literal_info.size()); u32 final_id = build.literal_info.at(lit_id).final_id; - if (final_id == MO_INVALID_IDX || final_id & LITERAL_MDR_FLAG) { - // Unused, special or direct report IDs are handled elsewhere. + if (final_id == MO_INVALID_IDX) { + // Unused, special report IDs are handled elsewhere. continue; } unique_lit_edge_map[final_id].insert(e); @@ -4054,8 +4208,9 @@ vector makeEodAnchorProgram(RoseBuildImpl &build, makeRoleCheckNotHandled(bc, v, program); } + const bool has_som = false; for (const auto &id : g[v].reports) { - makeReport(build, id, false, program); + makeReport(build, bc, id, has_som, program); } return program; @@ -4135,34 +4290,6 @@ u32 writeEodProgram(RoseBuildImpl &build, build_context &bc) { return buildLiteralProgram(build, bc, MO_INVALID_IDX, edge_list); } -static -void calcAnchoredMatches(const RoseBuildImpl &build, vector &art, - vector &arit) { - const RoseGraph &g = build.g; - - u32 max_report = 0; - - for (RoseVertex v : vertices_range(g)) { - if (!build.isAnchored(v)) { - continue; - } - - for (ReportID r : g[v].reports) { - art.push_back(r); - max_report = max(max_report, r); - } - } - - assert(max_report < MO_INVALID_IDX); - - arit.resize(max_report + 1, MO_INVALID_IDX); - for (u32 i = 0; i < art.size(); i++) { - DEBUG_PRINTF("art[%u] = %u\n", i, art[i]); - arit[art[i]] = i; - DEBUG_PRINTF("arit[%u] = %u\n", art[i], arit[art[i]]); - } -} - static u32 history_required(const rose_literal_id &key) { if (key.msk.size() < key.s.length()) { @@ -4264,22 +4391,18 @@ void fillMatcherDistances(const RoseBuildImpl &build, RoseEngine *engine) { aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { DerivedBoundaryReports dboundary(boundary); - // Build literal matchers - size_t asize = 0, fsize = 0, esize = 0, sbsize = 0; - - size_t floatingStreamStateRequired = 0; size_t historyRequired = calcHistoryRequired(); // Updated by HWLM. - aligned_unique_ptr atable = - buildAnchoredAutomataMatcher(*this, &asize); - aligned_unique_ptr ftable = buildFloatingMatcher( - *this, &fsize, &historyRequired, &floatingStreamStateRequired); - aligned_unique_ptr etable = buildEodAnchoredMatcher(*this, &esize); - aligned_unique_ptr sbtable = buildSmallBlockMatcher(*this, &sbsize); + auto anchored_dfas = buildAnchoredDfas(*this); build_context bc; bc.floatingMinLiteralMatchOffset = - findMinFloatingLiteralMatch(*this, atable.get()); + findMinFloatingLiteralMatch(*this, anchored_dfas); + bc.needs_catchup = needsCatchup(*this); + recordResources(bc.resources, *this); + if (!anchored_dfas.empty()) { + bc.resources.has_anchored = true; + } // Build NFAs set no_retrigger_queues; @@ -4336,11 +4459,6 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { throw ResourceLimitError(); } - u32 amatcherOffset = 0; - u32 fmatcherOffset = 0; - u32 ematcherOffset = 0; - u32 sbmatcherOffset = 0; - u32 currOffset; /* relative to base of RoseEngine */ if (!bc.engine_blob.empty()) { currOffset = bc.engine_blob_base + byte_length(bc.engine_blob); @@ -4354,28 +4472,46 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { currOffset = ROUNDUP_CL(currOffset); DEBUG_PRINTF("currOffset %u\n", currOffset); + // Build anchored matcher. + size_t asize = 0; + u32 amatcherOffset = 0; + auto atable = buildAnchoredMatcher(*this, anchored_dfas, &asize); if (atable) { currOffset = ROUNDUP_CL(currOffset); amatcherOffset = currOffset; - currOffset += (u32)asize; + currOffset += verify_u32(asize); } + // Build floating HWLM matcher. + size_t fsize = 0; + size_t floatingStreamStateRequired = 0; + auto ftable = buildFloatingMatcher(*this, &fsize, &historyRequired, + &floatingStreamStateRequired); + u32 fmatcherOffset = 0; if (ftable) { currOffset = ROUNDUP_CL(currOffset); fmatcherOffset = currOffset; - currOffset += (u32)fsize; + currOffset += verify_u32(fsize); } + // Build EOD-anchored HWLM matcher. + size_t esize = 0; + auto etable = buildEodAnchoredMatcher(*this, &esize); + u32 ematcherOffset = 0; if (etable) { currOffset = ROUNDUP_CL(currOffset); ematcherOffset = currOffset; - currOffset += (u32)esize; + currOffset += verify_u32(esize); } + // Build small-block HWLM matcher. + size_t sbsize = 0; + auto sbtable = buildSmallBlockMatcher(*this, &sbsize); + u32 sbmatcherOffset = 0; if (sbtable) { currOffset = ROUNDUP_CL(currOffset); sbmatcherOffset = currOffset; - currOffset += (u32)sbsize; + currOffset += verify_u32(sbsize); } const vector &int_reports = rm.reports(); @@ -4400,22 +4536,6 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { u32 nfaInfoLen = sizeof(NfaInfo) * queue_count; currOffset = nfaInfoOffset + nfaInfoLen; - vector art; // Reports raised by anchored roles - vector arit; // inverse reportID -> position in art - calcAnchoredMatches(*this, art, arit); - - currOffset = ROUNDUP_N(currOffset, sizeof(ReportID)); - u32 anchoredReportMapOffset = currOffset; - currOffset += art.size() * sizeof(ReportID); - - currOffset = ROUNDUP_N(currOffset, sizeof(u32)); - u32 anchoredReportInverseMapOffset = currOffset; - currOffset += arit.size() * sizeof(u32); - - currOffset = ROUNDUP_N(currOffset, alignof(ReportID)); - u32 multidirectOffset = currOffset; - currOffset += mdr_reports.size() * sizeof(ReportID); - currOffset = ROUNDUP_N(currOffset, alignof(mmbit_sparse_iter)); u32 activeLeftIterOffset = currOffset; currOffset += activeLeftIter.size() * sizeof(mmbit_sparse_iter); @@ -4502,13 +4622,14 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { engine->somLocationCount = ssm.numSomSlots(); engine->simpleCallback = !rm.numEkeys() && hasSimpleReports(rm.reports()); + engine->needsCatchup = bc.needs_catchup ? 1 : 0; fillInReportInfo(engine.get(), intReportOffset, rm, int_reports); engine->literalCount = verify_u32(final_id_to_literal.size()); engine->litProgramOffset = litProgramOffset; engine->litDelayRebuildProgramOffset = litDelayRebuildProgramOffset; - engine->runtimeImpl = pickRuntimeImpl(*this, outfixEndQueue); + engine->runtimeImpl = pickRuntimeImpl(*this, bc, outfixEndQueue); engine->mpvTriggeredByLeaf = anyEndfixMpvTriggers(*this); engine->activeArrayCount = activeArrayCount; @@ -4531,10 +4652,6 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { engine->stateSize = mmbit_size(bc.numStates); engine->anchorStateSize = anchorStateSize; engine->nfaInfoOffset = nfaInfoOffset; - engine->anchoredReportMapOffset = anchoredReportMapOffset; - engine->anchoredReportInverseMapOffset - = anchoredReportInverseMapOffset; - engine->multidirectOffset = multidirectOffset; engine->eodProgramOffset = eodProgramOffset; engine->eodIterProgramOffset = eodIterProgramOffset; @@ -4580,17 +4697,14 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { engine->size = currOffset; engine->minWidth = hasBoundaryReports(boundary) ? 0 : minWidth; engine->minWidthExcludingBoundaries = minWidth; - engine->maxSafeAnchoredDROffset = findMinWidth(*this, ROSE_FLOATING); engine->floatingMinLiteralMatchOffset = bc.floatingMinLiteralMatchOffset; engine->maxBiAnchoredWidth = findMaxBAWidth(*this); engine->noFloatingRoots = hasNoFloatingRoots(); - engine->hasFloatingDirectReports = floating_direct_report; engine->requiresEodCheck = hasEodAnchors(*this, bc, outfixEndQueue); engine->hasOutfixesInSmallBlock = hasNonSmallBlockOutfix(outfixes); engine->canExhaust = rm.patternSetCanExhaust(); engine->hasSom = hasSom; - engine->anchoredMatches = verify_u32(art.size()); /* populate anchoredDistance, floatingDistance, floatingMinDistance, etc */ fillMatcherDistances(*this, engine.get()); @@ -4605,15 +4719,6 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { write_out(&engine->state_init, (char *)engine.get(), state_scatter, state_scatter_aux_offset); - if (atable && anchoredIsMulti(*atable)) { - engine->maxSafeAnchoredDROffset = 1; - } else { - /* overly conservative, really need the min offset of non dr anchored - matches */ - engine->maxSafeAnchoredDROffset = MIN(engine->maxSafeAnchoredDROffset, - engine->floatingMinLiteralMatchOffset); - } - NfaInfo *nfa_infos = (NfaInfo *)(ptr + nfaInfoOffset); populateNfaInfoBasics(*this, bc, outfixes, suffixEkeyLists, no_retrigger_queues, nfa_infos); @@ -4629,9 +4734,6 @@ aligned_unique_ptr RoseBuildImpl::buildFinalEngine(u32 minWidth) { ptr + lookaroundReachOffset, bc.lookaround); fillInSomRevNfas(engine.get(), ssm, rev_nfa_table_offset, rev_nfa_offsets); - copy_bytes(ptr + engine->anchoredReportMapOffset, art); - copy_bytes(ptr + engine->anchoredReportInverseMapOffset, arit); - copy_bytes(ptr + engine->multidirectOffset, mdr_reports); copy_bytes(ptr + engine->activeLeftIterOffset, activeLeftIter); // Safety check: we shouldn't have written anything to the engine blob diff --git a/src/rose/rose_build_compile.cpp b/src/rose/rose_build_compile.cpp index db1e49ee..477335ca 100644 --- a/src/rose/rose_build_compile.cpp +++ b/src/rose/rose_build_compile.cpp @@ -247,51 +247,12 @@ bool isUsedLiteral(const RoseBuildImpl &build, u32 lit_id) { return false; } -static -void makeDirectReport(RoseBuildImpl &build, u32 i) { - if (build.literals.right.at(i).table == ROSE_FLOATING) { - build.floating_direct_report = true; - } - - rose_literal_info &info = build.literal_info[i]; - assert(!info.vertices.empty()); - - vector reports; - for (const auto &v : info.vertices) { - const auto &r = build.g[v].reports; - reports.insert(end(reports), begin(r), end(r)); - } - sort(begin(reports), end(reports)); - reports.erase(unique(begin(reports), end(reports)), end(reports)); - - if (reports.size() == 1) { - // A single direct report. We set the high bit to indicate it's a - // direct report and encode the ReportID itself in the final_id - // field. - ReportID report = reports.front(); - assert(!(report & LITERAL_DR_FLAG)); - info.final_id = LITERAL_DR_FLAG | report; - DEBUG_PRINTF("direct report %u -> %u\n", info.final_id, report); - } else { - // A multi-direct report. Here we write the report set into a list - // to be triggered when we see this literal. - u32 mdr_index = verify_u32(build.mdr_reports.size()); - info.final_id = LITERAL_MDR_FLAG | mdr_index; - DEBUG_PRINTF("multi direct report %u -> [%s]\n", info.final_id, - as_string_list(reports).c_str()); - build.mdr_reports.insert(end(build.mdr_reports), begin(reports), - end(reports)); - build.mdr_reports.push_back(MO_INVALID_IDX); - } -} - +/** \brief Allocate final literal IDs for all literals. + * + * These are the literal ids used in the bytecode. + */ static void allocateFinalLiteralId(RoseBuildImpl &tbi) { - /* allocate final literal ids - these are the literal ids used in the - * bytecode. - * DRs already have special final ids allocated - */ - RoseGraph &g = tbi.g; set anch; @@ -309,11 +270,6 @@ void allocateFinalLiteralId(RoseBuildImpl &tbi) { continue; } - if (tbi.isDirectReport(i)) { - makeDirectReport(tbi, i); - continue; - } - // The special EOD event literal has its own program and does not need // a real literal ID. if (i == tbi.eod_event_literal_id) { @@ -902,23 +858,6 @@ bool RoseBuildImpl::isDelayed(u32 id) const { return literal_info.at(id).undelayed_id != id; } -bool RoseBuildImpl::hasDirectFinalId(u32 id) const { - return literal_info.at(id).final_id & LITERAL_MDR_FLAG; -} - -bool RoseBuildImpl::allDirectFinalIds(RoseVertex v) const { - const auto &lits = g[v].literals; - if (lits.empty()) { - return false; - } - for (const auto &lit : lits) { - if (!hasDirectFinalId(lit)) { - return false; - } - } - return true; -} - bool RoseBuildImpl::hasFinalId(u32 id) const { return literal_info.at(id).final_id != MO_INVALID_IDX; } diff --git a/src/rose/rose_build_dump.cpp b/src/rose/rose_build_dump.cpp index e73d81c3..cd32749e 100644 --- a/src/rose/rose_build_dump.cpp +++ b/src/rose/rose_build_dump.cpp @@ -146,15 +146,6 @@ public: os << ")"; } - if (!g[v].literals.empty()) { - u32 id = *g[v].literals.begin(); - if (id < build.literal_info.size() - && build.literal_info[id].final_id != MO_INVALID_IDX - && (build.literal_info[id].final_id & LITERAL_DR_FLAG)) { - os << "\\nDIRECT REPORT"; - } - } - if (ghost.find(v) != ghost.end()) { os << "\\nGHOST"; } diff --git a/src/rose/rose_build_impl.h b/src/rose/rose_build_impl.h index 1a5e77aa..59247645 100644 --- a/src/rose/rose_build_impl.h +++ b/src/rose/rose_build_impl.h @@ -415,7 +415,6 @@ public: // Is the Rose anchored? bool hasNoFloatingRoots() const; - bool hasDirectReports() const; RoseVertex cloneVertex(RoseVertex v); @@ -441,17 +440,6 @@ public: bool isDirectReport(u32 id) const; bool isDelayed(u32 id) const; - /** - * \brief True if the given literal ID is a direct or multi-direct report. - */ - bool hasDirectFinalId(u32 id) const; - - /** - * \brief True if all the literals associated with the given vertex are - * direct or multi-direct reports. - */ - bool allDirectFinalIds(RoseVertex v) const; - bool hasFinalId(u32 id) const; bool isAnchored(RoseVertex v) const; /* true iff has literal in anchored @@ -526,16 +514,10 @@ public: * null again). */ std::unique_ptr mpv_outfix = nullptr; - bool floating_direct_report; - u32 eod_event_literal_id; // ID of EOD event literal, or MO_INVALID_IDX. u32 max_rose_anchored_floating_overlap; - /** \brief Flattened list of report IDs for multi-direct reports, indexed - * by MDR final_id. */ - std::vector mdr_reports; - QueueIndexFactory qif; ReportManager &rm; SomSlotManager &ssm; diff --git a/src/rose/rose_build_misc.cpp b/src/rose/rose_build_misc.cpp index 1df1b2d9..e5c5b4e6 100644 --- a/src/rose/rose_build_misc.cpp +++ b/src/rose/rose_build_misc.cpp @@ -79,7 +79,6 @@ RoseBuildImpl::RoseBuildImpl(ReportManager &rm_in, SomSlotManager &ssm_in, group_end(0), anchored_base_id(MO_INVALID_IDX), ematcher_region_size(0), - floating_direct_report(false), eod_event_literal_id(MO_INVALID_IDX), max_rose_anchored_floating_overlap(0), rm(rm_in), diff --git a/src/rose/rose_dump.cpp b/src/rose/rose_dump.cpp index 4eaab326..89dce981 100644 --- a/src/rose/rose_dump.cpp +++ b/src/rose/rose_dump.cpp @@ -403,6 +403,13 @@ void dumpProgram(ofstream &os, const RoseEngine *t, const char *pc) { } PROGRAM_NEXT_INSTRUCTION + PROGRAM_CASE(DEDUPE_AND_REPORT) { + os << " report " << ri->report << endl; + dumpReport(os, t, ri->report); + os << " fail_jump " << offset + ri->fail_jump << endl; + } + PROGRAM_NEXT_INSTRUCTION + PROGRAM_CASE(CHECK_EXHAUSTED) { os << " ekey " << ri->ekey << endl; os << " fail_jump " << offset + ri->fail_jump << endl; @@ -858,8 +865,8 @@ void roseDumpText(const RoseEngine *t, FILE *f) { sbtable ? hwlmSize(sbtable) : 0, t->smallBlockDistance); fprintf(f, " - role state table : %zu bytes\n", t->rolesWithStateCount * sizeof(u32)); - fprintf(f, " - nfa info table : %u bytes\n", - t->anchoredReportMapOffset - t->nfaInfoOffset); + fprintf(f, " - nfa info table : %zu bytes\n", + t->queueCount * sizeof(NfaInfo)); fprintf(f, " - lookaround table : %u bytes\n", t->nfaInfoOffset - t->lookaroundTableOffset); fprintf(f, " - lookaround reach : %u bytes\n", @@ -898,8 +905,6 @@ void roseDumpText(const RoseEngine *t, FILE *f) { t->minWidthExcludingBoundaries); fprintf(f, " maxBiAnchoredWidth : %s\n", rose_off(t->maxBiAnchoredWidth).str().c_str()); - fprintf(f, " maxSafeAnchoredDROffset : %s\n", - rose_off(t->maxSafeAnchoredDROffset).str().c_str()); fprintf(f, " minFloatLitMatchOffset : %s\n", rose_off(t->floatingMinLiteralMatchOffset).str().c_str()); fprintf(f, " delay_base_id : %u\n", t->delay_base_id); @@ -936,7 +941,6 @@ void roseDumpText(const RoseEngine *t, FILE *f) { void roseDumpStructRaw(const RoseEngine *t, FILE *f) { fprintf(f, "struct RoseEngine {\n"); - DUMP_U8(t, hasFloatingDirectReports); DUMP_U8(t, noFloatingRoots); DUMP_U8(t, requiresEodCheck); DUMP_U8(t, hasOutfixesInSmallBlock); @@ -946,6 +950,7 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) { DUMP_U8(t, hasSom); DUMP_U8(t, somHorizon); DUMP_U8(t, simpleCallback); + DUMP_U8(t, needsCatchup); DUMP_U32(t, mode); DUMP_U32(t, historyRequired); DUMP_U32(t, ekeyCount); @@ -972,7 +977,6 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) { DUMP_U32(t, litProgramOffset); DUMP_U32(t, litDelayRebuildProgramOffset); DUMP_U32(t, literalCount); - DUMP_U32(t, multidirectOffset); DUMP_U32(t, activeArrayCount); DUMP_U32(t, activeLeftCount); DUMP_U32(t, queueCount); @@ -994,14 +998,10 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) { DUMP_U32(t, floatingDistance); DUMP_U32(t, floatingMinDistance); DUMP_U32(t, smallBlockDistance); - DUMP_U32(t, maxSafeAnchoredDROffset); DUMP_U32(t, floatingMinLiteralMatchOffset); DUMP_U32(t, nfaInfoOffset); - DUMP_U32(t, anchoredReportMapOffset); - DUMP_U32(t, anchoredReportInverseMapOffset); DUMP_U64(t, initialGroups); DUMP_U32(t, size); - DUMP_U32(t, anchoredMatches); DUMP_U32(t, delay_count); DUMP_U32(t, delay_base_id); DUMP_U32(t, anchored_count); diff --git a/src/rose/rose_internal.h b/src/rose/rose_internal.h index 0d6c96e9..e9edbc15 100644 --- a/src/rose/rose_internal.h +++ b/src/rose/rose_internal.h @@ -48,29 +48,6 @@ typedef u64a rose_group; #define MAX_DELAY (DELAY_SLOT_COUNT - 1) #define DELAY_MASK (DELAY_SLOT_COUNT - 1) -// Direct report stuff -#define LITERAL_DR_FLAG (1U << 31) -#define LITERAL_MDR_FLAG ((1U << 30) | (1U << 31)) - -/** \brief True if literal is either a direct report or a multi-direct report. - * */ -static really_inline -u32 isLiteralDR(u32 id) { - return id & LITERAL_DR_FLAG; -} - -static really_inline -u32 isLiteralMDR(u32 id) { - return (id & LITERAL_MDR_FLAG) == LITERAL_MDR_FLAG; -} - -static really_inline -ReportID literalToReport(u32 id) { - assert(id & LITERAL_DR_FLAG); - assert(!(id & (LITERAL_MDR_FLAG ^ LITERAL_DR_FLAG))); - return id & ~LITERAL_DR_FLAG; -} - /* Allocation of Rose literal ids * * The rose literal id space is segmented: @@ -87,16 +64,6 @@ ReportID literalToReport(u32 id) { * | | Delayed version of normal literals * | | * ---- literalCount - * ... - * ... - * ... - * ---- LITERAL_DR_FLAG - * | | Direct Report literals: immediately raise an internal report with id - * | | given by (lit_id & ~LITERAL_DR_FLAG). Raised by a or f tables (or e??). - * | | No literal programs. - * | | - * | | - * ---- */ /* Rose Literal Sources @@ -317,14 +284,12 @@ struct RoseBoundaryReports { * -# small block table * -# array of NFA offsets, one per queue * -# array of state offsets, one per queue (+) - * -# multi-direct report array * * (+) stateOffset array note: Offsets in the array are either into the stream * state (normal case) or into the tstate region of scratch (for transient rose * nfas). Rose nfa info table can distinguish the cases. */ struct RoseEngine { - u8 hasFloatingDirectReports; // has at least one floating direct report literal u8 noFloatingRoots; /* only need to run the anchored table if something * matched in the anchored table */ u8 requiresEodCheck; /* stuff happens at eod time */ @@ -339,6 +304,7 @@ struct RoseEngine { SOM precision) */ u8 simpleCallback; /**< has only external reports with no bounds checks, plus no exhaustion keys */ + u8 needsCatchup; /** catch up needs to be run on every report. */ u32 mode; /**< scanning mode, one of HS_MODE_{BLOCK,STREAM,VECTORED} */ u32 historyRequired; /**< max amount of history required for streaming */ u32 ekeyCount; /**< number of exhaustion keys */ @@ -392,7 +358,6 @@ struct RoseEngine { */ u32 literalCount; - u32 multidirectOffset; /**< offset of multi-direct report list. */ u32 activeArrayCount; //number of nfas tracked in the active array u32 activeLeftCount; //number of nfas tracked in the active rose array u32 queueCount; /**< number of nfa queues */ @@ -432,18 +397,12 @@ struct RoseEngine { u32 floatingMinDistance; /* start of region to run floating table over */ u32 smallBlockDistance; /* end of region to run the floating table over ROSE_BOUND_INF if not bounded */ - u32 maxSafeAnchoredDROffset; /* the maximum offset that we can safely raise - * a direct report from the anchored table - * without delaying it */ u32 floatingMinLiteralMatchOffset; /* the minimum offset that we can get a * 'valid' match from the floating * table */ u32 nfaInfoOffset; /* offset to the nfa info offset array */ - u32 anchoredReportMapOffset; /* am_log index --> reportid */ - u32 anchoredReportInverseMapOffset; /* reportid --> am_log index */ rose_group initialGroups; u32 size; // (bytes) - u32 anchoredMatches; /* number of anchored roles generating matches */ u32 delay_count; /* number of delayed literal ids. */ u32 delay_base_id; /* literal id of the first delayed literal. * delayed literal ids are contiguous */ diff --git a/src/rose/rose_program.h b/src/rose/rose_program.h index 36a9aabd..7ac0360b 100644 --- a/src/rose/rose_program.h +++ b/src/rose/rose_program.h @@ -77,6 +77,9 @@ enum RoseInstructionCode { /** \brief Fire an exhaustible SOM report. */ ROSE_INSTR_REPORT_SOM_EXHAUST, + /** \brief Super-instruction combining DEDUPE and REPORT. */ + ROSE_INSTR_DEDUPE_AND_REPORT, + ROSE_INSTR_CHECK_EXHAUSTED, //!< Check if an ekey has already been set. ROSE_INSTR_CHECK_MIN_LENGTH, //!< Check (EOM - SOM) against min length. ROSE_INSTR_SET_STATE, //!< Switch a state index on. @@ -230,12 +233,17 @@ struct ROSE_STRUCT_REPORT_SOM_AWARE { struct ROSE_STRUCT_REPORT { u8 code; //!< From enum RoseInstructionCode. - ReportID report; + ReportID report; //!< Internal report ID (used for assertions). + ReportID onmatch; //!< Report ID to deliver to user. + s32 offset_adjust; //!< Offset adjustment to apply to end offset. }; struct ROSE_STRUCT_REPORT_EXHAUST { u8 code; //!< From enum RoseInstructionCode. - ReportID report; + ReportID report; //!< Internal report ID (used for assertions). + ReportID onmatch; //!< Report ID to deliver to user. + s32 offset_adjust; //!< Offset adjustment to apply to end offset. + u32 ekey; //!< Exhaustion key. }; struct ROSE_STRUCT_REPORT_SOM { @@ -253,6 +261,12 @@ struct ROSE_STRUCT_REPORT_SOM_EXT { ReportID report; }; +struct ROSE_STRUCT_DEDUPE_AND_REPORT { + u8 code; //!< From enum RoseInstructionCode. + ReportID report; + u32 fail_jump; //!< Jump forward this many bytes on failure. +}; + struct ROSE_STRUCT_CHECK_EXHAUSTED { u8 code; //!< From enum RoseInstructionCode. u32 ekey; //!< Exhaustion key to check. diff --git a/src/rose/runtime.h b/src/rose/runtime.h index e7e95377..275adfb4 100644 --- a/src/rose/runtime.h +++ b/src/rose/runtime.h @@ -80,16 +80,6 @@ u8 *getActiveLeftArray(const struct RoseEngine *t, char *state) { return (u8 *)(state + t->stateOffsets.activeLeftArray); } -static really_inline -const u32 *getAnchoredInverseMap(const struct RoseEngine *t) { - return (const u32 *)(((const u8 *)t) + t->anchoredReportInverseMapOffset); -} - -static really_inline -const u32 *getAnchoredMap(const struct RoseEngine *t) { - return (const u32 *)(((const u8 *)t) + t->anchoredReportMapOffset); -} - static really_inline rose_group loadGroups(const struct RoseEngine *t, const char *state) { return partial_load_u64a(state + t->stateOffsets.groups, @@ -167,8 +157,6 @@ const struct internal_report *getInternalReport(const struct RoseEngine *t, return reports + intId; } -#define ANCHORED_MATCH_SENTINEL (~0U) - static really_inline void updateLastMatchOffset(struct RoseContext *tctxt, u64a offset) { DEBUG_PRINTF("match @%llu, last match @%llu\n", offset, diff --git a/src/rose/stream.c b/src/rose/stream.c index 161e0059..759f7553 100644 --- a/src/rose/stream.c +++ b/src/rose/stream.c @@ -396,7 +396,7 @@ void ensureStreamNeatAndTidy(const struct RoseEngine *t, char *state, u64a offset) { struct RoseContext *tctxt = &scratch->tctxt; - if (roseCatchUpTo(t, scratch, length + scratch->core_info.buf_offset, 0) == + if (roseCatchUpTo(t, scratch, length + scratch->core_info.buf_offset) == HWLM_TERMINATE_MATCHING) { return; /* dead; no need to clean up state. */ } @@ -429,6 +429,10 @@ void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch, assert(scratch->core_info.hbuf); assert(scratch->core_info.buf); + // We should not have been called if we've already been told to terminate + // matching. + assert(!told_to_stop_matching(scratch)); + assert(mmbit_sparse_iter_state_size(t->rolesWithStateCount) < MAX_SPARSE_ITER_STATES); @@ -459,13 +463,10 @@ void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch, tctxt->minMatchOffset = offset; tctxt->minNonMpvMatchOffset = offset; tctxt->next_mpv_offset = 0; - tctxt->curr_anchored_loc = MMB_INVALID; - tctxt->curr_row_offset = 0; DEBUG_PRINTF("BEGIN: history len=%zu, buffer len=%zu\n", scratch->core_info.hlen, scratch->core_info.len); fatbit_clear(scratch->aqa); - scratch->am_log_sum = 0; /* clear the anchored logs */ scratch->al_log_sum = 0; scratch->catchup_pq.qm_size = 0; @@ -484,8 +485,6 @@ void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch, if (can_stop_matching(scratch)) { goto exit; } - - resetAnchoredLog(t, scratch); } const struct HWLM *ftable = getFLiteralMatcher(t); diff --git a/src/runtime.c b/src/runtime.c index 24ee90f0..5f357918 100644 --- a/src/runtime.c +++ b/src/runtime.c @@ -159,126 +159,26 @@ void setStreamStatus(char *state, u8 status) { *(u8 *)state = status; } -static really_inline -hwlmcb_rv_t multiDirectAdaptor(u64a real_end, ReportID direct_id, void *context, - struct core_info *ci, char is_simple, - char do_som) { - // Multi-direct report, list of reports indexed by the ID. - u32 mdr_offset = direct_id & ~LITERAL_MDR_FLAG; - const struct RoseEngine *t = ci->rose; - const ReportID *id - = (const ReportID *)((const char *)t + t->multidirectOffset) - + mdr_offset; - for (; *id != MO_INVALID_IDX; id++) { - int rv = roseAdaptor_i(real_end, *id, context, is_simple, do_som); - if (rv == MO_HALT_MATCHING) { - return HWLM_TERMINATE_MATCHING; - } - } - return HWLM_CONTINUE_MATCHING; -} - static int roseAdaptor(u64a offset, ReportID id, struct hs_scratch *scratch) { return roseAdaptor_i(offset, id, scratch, 0, 0); } -static -hwlmcb_rv_t hwlmAdaptor(UNUSED size_t start, size_t end, u32 direct_id, - void *context) { - struct hs_scratch *scratch = (struct hs_scratch *)context; - struct core_info *ci = &scratch->core_info; - u64a real_end = (u64a)end + ci->buf_offset + 1; - - if (isLiteralMDR(direct_id)) { - return multiDirectAdaptor(real_end, direct_id, context, ci, 0, 0); - } - - ReportID id = literalToReport(direct_id); - int rv = roseAdaptor_i(real_end, id, context, 0, 0); - if (rv == MO_CONTINUE_MATCHING || rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) { - return HWLM_CONTINUE_MATCHING; - } else { - return HWLM_TERMINATE_MATCHING; - } -} - static int roseSimpleAdaptor(u64a offset, ReportID id, struct hs_scratch *scratch) { return roseAdaptor_i(offset, id, scratch, 1, 0); } -static -hwlmcb_rv_t hwlmSimpleAdaptor(UNUSED size_t start, size_t end, u32 direct_id, - void *context) { - struct hs_scratch *scratch = (struct hs_scratch *)context; - struct core_info *ci = &scratch->core_info; - u64a real_end = (u64a)end + ci->buf_offset + 1; - - if (isLiteralMDR(direct_id)) { - return multiDirectAdaptor(real_end, direct_id, context, ci, 1, 0); - } - - // Single direct report. - ReportID id = literalToReport(direct_id); - int rv = roseAdaptor_i(real_end, id, context, 1, 0); - if (rv == MO_CONTINUE_MATCHING || rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) { - return HWLM_CONTINUE_MATCHING; - } else { - return HWLM_TERMINATE_MATCHING; - } -} - static int roseSomAdaptor(u64a offset, ReportID id, struct hs_scratch *scratch) { return roseAdaptor_i(offset, id, scratch, 0, 1); } -static -hwlmcb_rv_t hwlmSomAdaptor(UNUSED size_t start, size_t end, u32 direct_id, - void *context) { - struct hs_scratch *scratch = (struct hs_scratch *)context; - struct core_info *ci = &scratch->core_info; - u64a real_end = (u64a)end + ci->buf_offset + 1; - - if (isLiteralMDR(direct_id)) { - return multiDirectAdaptor(real_end, direct_id, context, ci, 0, 1); - } - - ReportID id = literalToReport(direct_id); - int rv = roseAdaptor_i(real_end, id, context, 0, 1); - if (rv == MO_CONTINUE_MATCHING || rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) { - return HWLM_CONTINUE_MATCHING; - } else { - return HWLM_TERMINATE_MATCHING; - } -} - static int roseSimpleSomAdaptor(u64a offset, ReportID id, struct hs_scratch *scratch) { return roseAdaptor_i(offset, id, scratch, 1, 1); } -static -hwlmcb_rv_t hwlmSimpleSomAdaptor(UNUSED size_t start, size_t end, u32 direct_id, - void *context) { - struct hs_scratch *scratch = (struct hs_scratch *)context; - struct core_info *ci = &scratch->core_info; - u64a real_end = (u64a)end + ci->buf_offset + 1; - - if (isLiteralMDR(direct_id)) { - return multiDirectAdaptor(real_end, direct_id, context, ci, 1, 1); - } - - ReportID id = literalToReport(direct_id); - int rv = roseAdaptor_i(real_end, id, context, 1, 1); - if (rv == MO_CONTINUE_MATCHING || rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) { - return HWLM_CONTINUE_MATCHING; - } else { - return HWLM_TERMINATE_MATCHING; - } -} - static really_inline RoseCallback selectAdaptor(const struct RoseEngine *rose) { const char is_simple = rose->simpleCallback; @@ -291,18 +191,6 @@ RoseCallback selectAdaptor(const struct RoseEngine *rose) { } } -static really_inline -HWLMCallback selectHwlmAdaptor(const struct RoseEngine *rose) { - const char is_simple = rose->simpleCallback; - const char do_som = rose->hasSom; - - if (do_som) { - return is_simple ? hwlmSimpleSomAdaptor : hwlmSomAdaptor; - } else { - return is_simple ? hwlmSimpleAdaptor : hwlmAdaptor; - } -} - static int roseSomSomAdaptor(u64a from_offset, u64a to_offset, ReportID id, struct hs_scratch *scratch) { @@ -372,14 +260,21 @@ SomNfaCallback selectOutfixSomAdaptor(const struct RoseEngine *rose) { return is_simple ? outfixSimpleSomSomAdaptor : outfixSomSomAdaptor; } +/** + * \brief Fire callbacks for a boundary report list. + * + * Returns MO_HALT_MATCHING if the user has instructed us to halt, and + * MO_CONTINUE_MATCHING otherwise. + */ + static never_inline -void processReportList(const struct RoseEngine *rose, u32 base_offset, - u64a stream_offset, hs_scratch_t *scratch) { +int processReportList(const struct RoseEngine *rose, u32 base_offset, + u64a stream_offset, hs_scratch_t *scratch) { DEBUG_PRINTF("running report list at offset %u\n", base_offset); if (told_to_stop_matching(scratch)) { DEBUG_PRINTF("matching has been terminated\n"); - return; + return MO_HALT_MATCHING; } if (rose->hasSom && scratch->deduper.current_report_offset == ~0ULL) { @@ -393,20 +288,27 @@ void processReportList(const struct RoseEngine *rose, u32 base_offset, scratch->deduper.som_log_dirty = 0; } - const ReportID *report = - (const ReportID *)((const char *)rose + base_offset); + const ReportID *report = getByOffset(rose, base_offset); /* never required to do som as vacuous reports are always external */ if (rose->simpleCallback) { for (; *report != MO_INVALID_IDX; report++) { - roseSimpleAdaptor(stream_offset, *report, scratch); + int rv = roseSimpleAdaptor(stream_offset, *report, scratch); + if (rv == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } } } else { for (; *report != MO_INVALID_IDX; report++) { - roseAdaptor(stream_offset, *report, scratch); + int rv = roseAdaptor(stream_offset, *report, scratch); + if (rv == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } } } + + return MO_CONTINUE_MATCHING; } /** \brief Initialise SOM state. Used in both block and streaming mode. */ @@ -443,13 +345,13 @@ void pureLiteralBlockExec(const struct RoseEngine *rose, size_t length = scratch->core_info.len; DEBUG_PRINTF("rose engine %d\n", rose->runtimeImpl); - hwlmExec(ftable, buffer, length, 0, selectHwlmAdaptor(rose), scratch, + hwlmExec(ftable, buffer, length, 0, rosePureLiteralCallback, scratch, rose->initialGroups); } static really_inline -void initQueue(struct mq *q, u32 qi, const struct RoseEngine *t, - struct hs_scratch *scratch) { +void initOutfixQueue(struct mq *q, u32 qi, const struct RoseEngine *t, + struct hs_scratch *scratch) { const struct NfaInfo *info = getNfaInfoByQueue(t, qi); q->nfa = getNfaByInfo(t, info); q->end = 0; @@ -492,7 +394,7 @@ void soleOutfixBlockExec(const struct RoseEngine *t, } struct mq *q = scratch->queues; - initQueue(q, 0, t, scratch); + initOutfixQueue(q, 0, t, scratch); q->length = len; /* adjust for rev_accel */ nfaQueueInitState(nfa, q); pushQueueAt(q, 0, MQE_START, 0); @@ -579,6 +481,11 @@ hs_error_t hs_scan(const hs_database_t *db, const char *data, unsigned length, clearEvec(scratch->core_info.exhaustionVector, rose); + // Rose program execution (used for some report paths) depends on these + // values being initialised. + scratch->tctxt.lastMatchOffset = 0; + scratch->tctxt.minMatchOffset = 0; + if (!length) { if (rose->boundary.reportZeroEodOffset) { processReportList(rose, rose->boundary.reportZeroEodOffset, 0, @@ -588,7 +495,11 @@ hs_error_t hs_scan(const hs_database_t *db, const char *data, unsigned length, } if (rose->boundary.reportZeroOffset) { - processReportList(rose, rose->boundary.reportZeroOffset, 0, scratch); + int rv = processReportList(rose, rose->boundary.reportZeroOffset, 0, + scratch); + if (rv == MO_HALT_MATCHING) { + goto set_retval; + } } if (rose->minWidthExcludingBoundaries > length) { @@ -648,7 +559,8 @@ done_scan: } if (rose->boundary.reportEodOffset) { - processReportList(rose, rose->boundary.reportEodOffset, length, scratch); + processReportList(rose, rose->boundary.reportEodOffset, length, + scratch); } set_retval: @@ -782,7 +694,7 @@ void soleOutfixEodExec(hs_stream_t *id, hs_scratch_t *scratch) { const struct NFA *nfa = getNfaByQueue(t, 0); struct mq *q = scratch->queues; - initQueue(q, 0, t, scratch); + initOutfixQueue(q, 0, t, scratch); if (!scratch->core_info.buf_offset) { DEBUG_PRINTF("buf_offset is zero\n"); return; /* no vacuous engines */ @@ -821,13 +733,21 @@ void report_eod_matches(hs_stream_t *id, hs_scratch_t *scratch, if (!id->offset) { if (rose->boundary.reportZeroEodOffset) { - processReportList(rose, rose->boundary.reportZeroEodOffset, 0, - scratch); + int rv = processReportList(rose, rose->boundary.reportZeroEodOffset, + 0, scratch); + if (rv == MO_HALT_MATCHING) { + scratch->core_info.status |= STATUS_TERMINATED; + return; + } } } else { if (rose->boundary.reportEodOffset) { - processReportList(rose, rose->boundary.reportEodOffset, + int rv = processReportList(rose, rose->boundary.reportEodOffset, id->offset, scratch); + if (rv == MO_HALT_MATCHING) { + scratch->core_info.status |= STATUS_TERMINATED; + return; + } } if (rose->requiresEodCheck) { @@ -962,7 +882,7 @@ void pureLiteralStreamExec(struct hs_stream *stream_state, // start the match region at zero. const size_t start = 0; - hwlmExecStreaming(ftable, scratch, len2, start, selectHwlmAdaptor(rose), + hwlmExecStreaming(ftable, scratch, len2, start, rosePureLiteralCallback, scratch, rose->initialGroups, hwlm_stream_state); if (!told_to_stop_matching(scratch) && @@ -988,7 +908,7 @@ void soleOutfixStreamExec(struct hs_stream *stream_state, const struct NFA *nfa = getNfaByQueue(t, 0); struct mq *q = scratch->queues; - initQueue(q, 0, t, scratch); + initOutfixQueue(q, 0, t, scratch); if (!scratch->core_info.buf_offset) { nfaQueueInitState(nfa, q); pushQueueAt(q, 0, MQE_START, 0); @@ -1045,6 +965,11 @@ hs_error_t hs_scan_stream_internal(hs_stream_t *id, const char *data, assert(scratch->core_info.hlen <= id->offset && scratch->core_info.hlen <= rose->historyRequired); + // Rose program execution (used for some report paths) depends on these + // values being initialised. + scratch->tctxt.lastMatchOffset = 0; + scratch->tctxt.minMatchOffset = id->offset; + prefetch_data(data, length); if (rose->somLocationCount) { diff --git a/src/scratch.c b/src/scratch.c index 854fc312..42db42ac 100644 --- a/src/scratch.c +++ b/src/scratch.c @@ -75,8 +75,6 @@ hs_error_t alloc_scratch(const hs_scratch_t *proto, hs_scratch_t **scratch) { u32 bStateSize = proto->bStateSize; u32 tStateSize = proto->tStateSize; u32 fullStateSize = proto->fullStateSize; - u32 anchored_region_len = proto->anchored_region_len; - u32 anchored_region_width = proto->anchored_region_width; u32 anchored_literal_region_len = proto->anchored_literal_region_len; u32 anchored_literal_region_width = proto->anchored_literal_count; @@ -90,11 +88,8 @@ hs_error_t alloc_scratch(const hs_scratch_t *proto, hs_scratch_t **scratch) { size_t queue_size = queueCount * sizeof(struct mq); size_t qmpq_size = queueCount * sizeof(struct queue_match); - assert(anchored_region_len < 8 * sizeof(s->am_log_sum)); - assert(anchored_literal_region_len < 8 * sizeof(s->am_log_sum)); + assert(anchored_literal_region_len < 8 * sizeof(s->al_log_sum)); - size_t anchored_region_size = - fatbit_array_size(anchored_region_len, anchored_region_width); size_t anchored_literal_region_size = fatbit_array_size( anchored_literal_region_len, anchored_literal_region_width); size_t delay_region_size = @@ -109,7 +104,6 @@ hs_error_t alloc_scratch(const hs_scratch_t *proto, hs_scratch_t **scratch) { + 2 * fatbit_size(deduperCount) /* need odd and even logs */ + 2 * fatbit_size(deduperCount) /* ditto som logs */ + 2 * sizeof(u64a) * deduperCount /* start offsets for som */ - + anchored_region_size + anchored_literal_region_size + qmpq_size + delay_region_size + som_store_size @@ -165,16 +159,6 @@ hs_error_t alloc_scratch(const hs_scratch_t *proto, hs_scratch_t **scratch) { current += fatbit_size(proto->delay_count); } - current = ROUNDUP_PTR(current, alignof(struct fatbit *)); - s->am_log = (struct fatbit **)current; - current += sizeof(struct fatbit *) * anchored_region_len; - current = ROUNDUP_PTR(current, alignof(struct fatbit)); - for (u32 i = 0; i < anchored_region_len; i++) { - s->am_log[i] = (struct fatbit *)current; - assert(ISALIGNED(s->am_log[i])); - current += fatbit_size(anchored_region_width); - } - current = ROUNDUP_PTR(current, alignof(struct fatbit *)); s->al_log = (struct fatbit **)current; current += sizeof(struct fatbit *) * anchored_literal_region_len; @@ -295,22 +279,6 @@ hs_error_t hs_alloc_scratch(const hs_database_t *db, hs_scratch_t **scratch) { } proto->scratch_alloc = (char *)proto_tmp; - u32 max_anchored_match = rose->anchoredDistance; - if (max_anchored_match > rose->maxSafeAnchoredDROffset) { - u32 anchored_region_len = max_anchored_match - - rose->maxSafeAnchoredDROffset; - if (anchored_region_len > proto->anchored_region_len) { - resize = 1; - proto->anchored_region_len = anchored_region_len; - } - } - - u32 anchored_region_width = rose->anchoredMatches; - if (anchored_region_width > proto->anchored_region_width) { - resize = 1; - proto->anchored_region_width = anchored_region_width; - } - if (rose->anchoredDistance > proto->anchored_literal_region_len) { resize = 1; proto->anchored_literal_region_len = rose->anchoredDistance; diff --git a/src/scratch.h b/src/scratch.h index 48b3de7d..99cdb370 100644 --- a/src/scratch.h +++ b/src/scratch.h @@ -122,8 +122,6 @@ struct RoseContext { RoseCallback cb; RoseCallbackSom cb_som; u32 filledDelayedSlots; - u32 curr_anchored_loc; /**< last read/written row */ - u32 curr_row_offset; /**< last read/written entry */ u32 curr_qi; /**< currently executing main queue index during * \ref nfaQueueExec */ }; @@ -158,15 +156,11 @@ struct ALIGN_CL_DIRECTIVE hs_scratch { struct fatbit *aqa; /**< active queue array; fatbit of queues that are valid * & active */ struct fatbit **delay_slots; - struct fatbit **am_log; struct fatbit **al_log; - u64a am_log_sum; u64a al_log_sum; struct catchup_pq catchup_pq; struct core_info core_info; struct match_deduper deduper; - u32 anchored_region_len; - u32 anchored_region_width; u32 anchored_literal_region_len; u32 anchored_literal_count; u32 delay_count; @@ -192,11 +186,6 @@ struct hs_scratch *tctxtToScratch(struct RoseContext *tctxt) { ((char *)tctxt - offsetof(struct hs_scratch, tctxt)); } -static really_inline -struct fatbit **getAnchoredLog(struct hs_scratch *scratch) { - return scratch->am_log; -} - /* array of fatbit ptr; TODO: why not an array of fatbits? */ static really_inline struct fatbit **getAnchoredLiteralLog(struct hs_scratch *scratch) { diff --git a/src/scratch_dump.cpp b/src/scratch_dump.cpp index 00e09a83..78a854bb 100644 --- a/src/scratch_dump.cpp +++ b/src/scratch_dump.cpp @@ -56,12 +56,6 @@ void dumpScratch(const struct hs_scratch *s, FILE *f) { fprintf(f, " bStateSize : %u bytes\n", s->bStateSize); fprintf(f, " active queue array : %u bytes\n", mmbit_size(s->queueCount)); - - size_t anchored_region_size = - s->anchored_region_len * mmbit_size(s->anchored_region_width) + - sizeof(u8 *) + mmbit_size(s->anchored_region_len); - - fprintf(f, " anchored region : %zu bytes\n", anchored_region_size); fprintf(f, " qmpq : %zu bytes\n", s->queueCount * sizeof(struct queue_match)); fprintf(f, " delay info : %u bytes\n",