Rose: replace internal_report with program

Replace the use of the internal_report structure (for reports from
engines, MPV etc) with the Rose program interpreter.

SOM processing was reworked to use a new som_operation structure that is
embedded in the appropriate instructions.

CMakeLists.txt

@@ -479,6 +479,7 @@ set (hs_exec_SRCS
     src/nfa/vermicelli_run.h
     src/nfa/vermicelli_sse.h
     src/som/som.h
+    src/som/som_operation.h
     src/som/som_runtime.h
     src/som/som_runtime.c
     src/som/som_stream.c
@@ -795,6 +796,7 @@ SET (hs_SRCS
     src/som/slot_manager.h
     src/som/slot_manager_internal.h
     src/som/som.h
+    src/som/som_operation.h
     src/rose/rose_build.h
     src/rose/rose_build_add.cpp
     src/rose/rose_build_add_internal.h
@@ -849,7 +851,6 @@ SET (hs_SRCS
     src/util/dump_mask.cpp
     src/util/dump_mask.h
     src/util/graph.h
-    src/util/internal_report.h
     src/util/multibit_build.cpp
     src/util/multibit_build.h
     src/util/order_check.h
@@ -857,7 +858,6 @@ SET (hs_SRCS
     src/util/partitioned_set.h
     src/util/popcount.h
     src/util/queue_index_factory.h
-    src/util/report.cpp
     src/util/report.h
     src/util/report_manager.cpp
     src/util/report_manager.h

src/nfagraph/ng.cpp

@@ -130,6 +130,7 @@ bool addComponentSom(NG &ng, NGHolder &g, const NGWrapper &w,
 
     assert(g.kind == NFA_OUTFIX);
     dumpComponent(g, "haig", w.expressionIndex, comp_id, ng.cc.grey);
+    makeReportsSomPass(ng.rm, g);
     auto haig = attemptToBuildHaig(g, som, ng.ssm.somPrecision(), triggers,
                                    ng.cc.grey);
     if (haig) {

src/nfagraph/ng_dump.cpp

@@ -358,35 +358,38 @@ void dumpSmallWrite(const RoseEngine *rose, const Grey &grey) {
     smwrDumpNFA(smwr, false, grey.dumpPath);
 }
 
-static UNUSED
+static
-const char *irTypeToString(u8 type) {
+const char *reportTypeToString(ReportType type) {
-#define IR_TYPE_CASE(x) case x: return #x
+#define REPORT_TYPE_CASE(x) case x: return #x
     switch (type) {
-        IR_TYPE_CASE(EXTERNAL_CALLBACK);
+        REPORT_TYPE_CASE(EXTERNAL_CALLBACK);
-        IR_TYPE_CASE(EXTERNAL_CALLBACK_SOM_REL);
+        REPORT_TYPE_CASE(EXTERNAL_CALLBACK_SOM_REL);
-        IR_TYPE_CASE(INTERNAL_SOM_LOC_SET);
+        REPORT_TYPE_CASE(INTERNAL_SOM_LOC_SET);
-        IR_TYPE_CASE(INTERNAL_SOM_LOC_SET_IF_UNSET);
+        REPORT_TYPE_CASE(INTERNAL_SOM_LOC_SET_IF_UNSET);
-        IR_TYPE_CASE(INTERNAL_SOM_LOC_SET_IF_WRITABLE);
+        REPORT_TYPE_CASE(INTERNAL_SOM_LOC_SET_IF_WRITABLE);
-        IR_TYPE_CASE(INTERNAL_SOM_LOC_SET_SOM_REV_NFA);
+        REPORT_TYPE_CASE(INTERNAL_SOM_LOC_SET_SOM_REV_NFA);
-        IR_TYPE_CASE(INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET);
+        REPORT_TYPE_CASE(INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET);
-        IR_TYPE_CASE(INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE);
+        REPORT_TYPE_CASE(INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE);
-        IR_TYPE_CASE(INTERNAL_SOM_LOC_COPY);
+        REPORT_TYPE_CASE(INTERNAL_SOM_LOC_COPY);
-        IR_TYPE_CASE(INTERNAL_SOM_LOC_COPY_IF_WRITABLE);
+        REPORT_TYPE_CASE(INTERNAL_SOM_LOC_COPY_IF_WRITABLE);
-        IR_TYPE_CASE(INTERNAL_SOM_LOC_MAKE_WRITABLE);
+        REPORT_TYPE_CASE(INTERNAL_SOM_LOC_MAKE_WRITABLE);
-        IR_TYPE_CASE(EXTERNAL_CALLBACK_SOM_STORED);
+        REPORT_TYPE_CASE(EXTERNAL_CALLBACK_SOM_STORED);
-        IR_TYPE_CASE(EXTERNAL_CALLBACK_SOM_ABS);
+        REPORT_TYPE_CASE(EXTERNAL_CALLBACK_SOM_ABS);
-        IR_TYPE_CASE(EXTERNAL_CALLBACK_SOM_REV_NFA);
+        REPORT_TYPE_CASE(EXTERNAL_CALLBACK_SOM_REV_NFA);
-        IR_TYPE_CASE(INTERNAL_SOM_LOC_SET_FROM);
+        REPORT_TYPE_CASE(INTERNAL_SOM_LOC_SET_FROM);
-        IR_TYPE_CASE(INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE);
+        REPORT_TYPE_CASE(INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE);
-        IR_TYPE_CASE(INTERNAL_ROSE_CHAIN);
+        REPORT_TYPE_CASE(INTERNAL_ROSE_CHAIN);
-    default: return "<unknown>";
+        REPORT_TYPE_CASE(EXTERNAL_CALLBACK_SOM_PASS);
     }
-#undef IR_TYPE_CASE
+#undef REPORT_TYPE_CASE
+
+    assert(0);
+    return "<unknown>";
 }
 
-static really_inline
+static
-int isReverseNfaReport(const Report &ri) {
+int isReverseNfaReport(const Report &report) {
-    switch (ri.type) {
+    switch (report.type) {
     case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
     case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
     case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
@@ -398,9 +401,9 @@ int isReverseNfaReport(const Report &ri) {
     return 0;
 }
 
-static really_inline
+static
-int isSomRelSetReport(const Report &ri) {
+int isSomRelSetReport(const Report &report) {
-    switch (ri.type) {
+    switch (report.type) {
     case INTERNAL_SOM_LOC_SET:
     case INTERNAL_SOM_LOC_SET_IF_UNSET:
     case INTERNAL_SOM_LOC_SET_IF_WRITABLE:
@@ -420,31 +423,34 @@ void dumpReportManager(const ReportManager &rm, const Grey &grey) {
     ss << grey.dumpPath << "internal_reports.txt";
     FILE *f = fopen(ss.str().c_str(), "w");
     const vector<Report> &reports = rm.reports();
-    for (u32 i = 0; i < reports.size(); i++) {
+    for (size_t i = 0; i < reports.size(); i++) {
-        const Report &ir = reports[i];
+        const Report &report = reports[i];
-        fprintf(f, "int %u: %s onmatch: %u", i, irTypeToString(ir.type),
+        fprintf(f, "%zu: %s onmatch: %u", i, reportTypeToString(report.type),
-                ir.onmatch);
+                report.onmatch);
 
-        u32 dkey = rm.getDkey(ir);
+        u32 dkey = rm.getDkey(report);
         if (dkey != MO_INVALID_IDX) {
             fprintf(f, " dkey %u", dkey);
         }
-        if (ir.ekey != MO_INVALID_IDX) {
+        if (report.ekey != INVALID_EKEY) {
-            fprintf(f, " ekey %u", ir.ekey);
+            fprintf(f, " ekey %u", report.ekey);
         }
-        if (ir.hasBounds()) {
+        if (report.hasBounds()) {
             fprintf(f, " hasBounds (minOffset=%llu, maxOffset=%llu, "
                        "minLength=%llu)",
-                    ir.minOffset, ir.maxOffset, ir.minLength);
+                    report.minOffset, report.maxOffset, report.minLength);
         }
-        if (ir.offsetAdjust != 0) {
+        if (report.quashSom) {
-            fprintf(f, " offsetAdjust: %d", ir.offsetAdjust);
+            fprintf(f, " quashSom");
         }
-        if (isReverseNfaReport(ir)) {
+        if (report.offsetAdjust != 0) {
-            fprintf(f, " reverse nfa: %u", ir.revNfaIndex);
+            fprintf(f, " offsetAdjust: %d", report.offsetAdjust);
         }
-        if (isSomRelSetReport(ir)) {
+        if (isReverseNfaReport(report)) {
-            fprintf(f, " set, adjust: %lld", ir.somDistance);
+            fprintf(f, " reverse nfa: %u", report.revNfaIndex);
+        }
+        if (isSomRelSetReport(report)) {
+            fprintf(f, " set, adjust: %lld", report.somDistance);
         }
         fprintf(f, "\n");
     }

src/nfagraph/ng_puff.cpp

@@ -293,7 +293,7 @@ void constructPuff(NGHolder &g, const NFAVertex a, const NFAVertex puffv,
         u32 squashDistance = allowedSquashDistance(cr, width, g, puffv,
                                                    prefilter);
 
-        Report ir = makeRoseTrigger(event, squashDistance);
+        Report ir = makeMpvTrigger(event, squashDistance);
         /* only need to trigger once if floatingUnboundedDot */
         bool floatingUnboundedDot = unbounded && cr.all() && !fixed_depth;
         if (floatingUnboundedDot) {

src/nfagraph/ng_som.cpp

@@ -445,8 +445,9 @@ void replaceTempSomSlot(ReportManager &rm, NGHolder &g, u32 real_slot) {
 }
 
 static
-void setPrefixReports(ReportManager &rm, NGHolder &g, u8 ir_type, u32 som_loc,
+void setPrefixReports(ReportManager &rm, NGHolder &g, ReportType ir_type,
-                      const vector<DepthMinMax> &depths, bool prefix_by_rev) {
+                      u32 som_loc, const vector<DepthMinMax> &depths,
+                      bool prefix_by_rev) {
     Report ir = makeCallback(0U, 0);
     ir.type = ir_type;
     ir.onmatch = som_loc;
@@ -470,7 +471,7 @@ void setPrefixReports(ReportManager &rm, NGHolder &g, u8 ir_type, u32 som_loc,
 }
 
 static
-void updatePrefixReports(ReportManager &rm, NGHolder &g, u8 ir_type) {
+void updatePrefixReports(ReportManager &rm, NGHolder &g, ReportType ir_type) {
     /* update the som action on the prefix report */
     for (auto v : inv_adjacent_vertices_range(g.accept, g)) {
         auto &reports = g[v].reports;
@@ -555,7 +556,8 @@ bool finalRegion(const NGHolder &g,
 
 static
 void replaceExternalReportsWithSomRep(ReportManager &rm, NGHolder &g,
-                                      NFAVertex v, u8 ir_type, u64a param) {
+                                      NFAVertex v, ReportType ir_type,
+                                      u64a param) {
     assert(!g[v].reports.empty());
 
     flat_set<ReportID> r_new;
@@ -2409,6 +2411,33 @@ bool splitOffBestLiteral(const NGHolder &g,
     return true;
 }
 
+/**
+ * Replace the given graph's EXTERNAL_CALLBACK reports with
+ * EXTERNAL_CALLBACK_SOM_PASS reports.
+ */
+void makeReportsSomPass(ReportManager &rm, NGHolder &g) {
+    for (const auto &v : vertices_range(g)) {
+        const auto &reports = g[v].reports;
+        if (reports.empty()) {
+            continue;
+        }
+
+        flat_set<ReportID> new_reports;
+        for (const ReportID &id : reports) {
+            const Report &report = rm.getReport(id);
+            if (report.type != EXTERNAL_CALLBACK) {
+                new_reports.insert(id);
+                continue;
+            }
+            Report report2 = report;
+            report2.type = EXTERNAL_CALLBACK_SOM_PASS;
+            new_reports.insert(rm.getInternalId(report2));
+        }
+
+        g[v].reports = new_reports;
+    }
+}
+
 static
 bool doLitHaigSom(NG &ng, NGHolder &g, som_type som) {
     ue2_literal lit;
@@ -2431,6 +2460,8 @@ bool doLitHaigSom(NG &ng, NGHolder &g, som_type som) {
 
     assert(lit.length() <= MAX_MASK2_WIDTH || !mixed_sensitivity(lit));
 
+    makeReportsSomPass(ng.rm, *rhs);
+
     dumpHolder(*rhs, 91, "lithaig_rhs", ng.cc.grey);
 
     vector<vector<CharReach> > triggers;
@@ -2493,6 +2524,8 @@ bool doHaigLitHaigSom(NG &ng, NGHolder &g,
         return false; /* TODO: handle */
     }
 
+    makeReportsSomPass(ng.rm, *rhs);
+
     dumpHolder(*lhs, 92, "haiglithaig_lhs", ng.cc.grey);
     dumpHolder(*rhs, 93, "haiglithaig_rhs", ng.cc.grey);
 
@@ -2628,6 +2661,8 @@ bool doMultiLitHaigSom(NG &ng, NGHolder &g, som_type som) {
         return false;
     }
 
+    makeReportsSomPass(ng.rm, *rhs);
+
     dumpHolder(*rhs, 91, "lithaig_rhs", ng.cc.grey);
 
     vector<vector<CharReach>> triggers;

src/nfagraph/ng_som.h

@@ -72,6 +72,8 @@ sombe_rv doSom(NG &ng, NGHolder &h, const NGWrapper &w, u32 comp_id,
 sombe_rv doSomWithHaig(NG &ng, NGHolder &h, const NGWrapper &w, u32 comp_id,
                        som_type som);
 
+void makeReportsSomPass(ReportManager &rm, NGHolder &g);
+
 } // namespace ue2
 
 #endif // NG_SOM_H

src/report.h

@@ -43,7 +43,6 @@
 #include "som/som_runtime.h"
 #include "util/exhaust.h"
 #include "util/fatbit.h"
-#include "util/internal_report.h"
 
 static really_inline
 int satisfiesMinLength(u64a min_len, u64a from_offset,
@@ -68,14 +67,12 @@ enum DedupeResult {
 
 static really_inline
 enum DedupeResult dedupeCatchup(const struct RoseEngine *rose,
-                                const struct internal_report *ir,
                                 struct hs_scratch *scratch, u64a offset,
-                                u64a from_offset, u64a to_offset,
+                                u64a from_offset, u64a to_offset, u32 dkey,
-                                const char do_som) {
+                                s32 offset_adjust, char is_external_report,
+                                char quash_som, const char do_som) {
     DEBUG_PRINTF("offset=%llu, match=[%llu,%llu], dkey=%u, do_som=%d\n", offset,
-                 from_offset, to_offset, ir->dkey, do_som);
+                 from_offset, to_offset, dkey, do_som);
-    DEBUG_PRINTF("report type=%u, quashSom=%d\n", ir->type, ir->quashSom);
-    const u32 dkey = ir->dkey;
 
     // We should not have been called if there's no dedupe work to do.
     assert(do_som || dkey != MO_INVALID_IDX);
@@ -99,10 +96,9 @@ enum DedupeResult dedupeCatchup(const struct RoseEngine *rose,
 
     if (dkey != MO_INVALID_IDX) {
         const u32 dkeyCount = rose->dkeyCount;
-        const s32 offset_adj = ir->offsetAdjust;
+        if (is_external_report || quash_som) {
-        if (ir->type == EXTERNAL_CALLBACK || ir->quashSom) {
             DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset);
-            assert(offset_adj == 0 || offset_adj == -1);
+            assert(offset_adjust == 0 || offset_adjust == -1);
             if (fatbit_set(deduper->log[to_offset % 2], dkeyCount, dkey)) {
                 /* we have already raised this report at this offset, squash
                  * dupe match. */
@@ -112,7 +108,7 @@ enum DedupeResult dedupeCatchup(const struct RoseEngine *rose,
         } else if (do_som) {
             /* SOM external event */
             DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset);
-            assert(offset_adj == 0 || offset_adj == -1);
+            assert(offset_adjust == 0 || offset_adjust == -1);
             u64a *starts = deduper->som_start_log[to_offset % 2];
             if (fatbit_set(deduper->som_log[to_offset % 2], dkeyCount, dkey)) {
                 starts[dkey] = MIN(starts[dkey], from_offset);
@@ -121,7 +117,7 @@ enum DedupeResult dedupeCatchup(const struct RoseEngine *rose,
             }
             DEBUG_PRINTF("starts[%u]=%llu\n", dkey, starts[dkey]);
 
-            if (offset_adj) {
+            if (offset_adjust) {
                 deduper->som_log_dirty |= 1;
             } else {
                 deduper->som_log_dirty |= 2;
@@ -134,176 +130,6 @@ enum DedupeResult dedupeCatchup(const struct RoseEngine *rose,
     return DEDUPE_CONTINUE;
 }
 
-static really_inline
-enum DedupeResult dedupeCatchupSom(const struct RoseEngine *rose,
-                                   const struct internal_report *ir,
-                                   struct hs_scratch *scratch, u64a offset,
-                                   u64a from_offset, u64a to_offset) {
-    DEBUG_PRINTF("offset=%llu, match=[%llu,%llu], dkey=%u\n", offset,
-                 from_offset, to_offset, ir->dkey);
-    DEBUG_PRINTF("report type=%u, quashSom=%d\n", ir->type, ir->quashSom);
-
-    struct match_deduper *deduper = &scratch->deduper;
-    if (offset != deduper->current_report_offset) {
-        assert(deduper->current_report_offset == ~0ULL ||
-               deduper->current_report_offset < offset);
-        if (offset == deduper->current_report_offset + 1) {
-            fatbit_clear(deduper->log[offset % 2]);
-        } else {
-            fatbit_clear(deduper->log[0]);
-            fatbit_clear(deduper->log[1]);
-        }
-
-        if (flushStoredSomMatches(scratch, offset)) {
-            return DEDUPE_HALT;
-        }
-        deduper->current_report_offset = offset;
-    }
-
-    const u32 dkey = ir->dkey;
-    if (dkey != MO_INVALID_IDX) {
-        const u32 dkeyCount = rose->dkeyCount;
-        const s32 offset_adj = ir->offsetAdjust;
-        if (ir->quashSom) {
-            DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset);
-            assert(offset_adj == 0 || offset_adj == -1);
-            if (fatbit_set(deduper->log[to_offset % 2], dkeyCount, dkey)) {
-                /* we have already raised this report at this offset, squash
-                 * dupe match. */
-                DEBUG_PRINTF("dedupe\n");
-                return DEDUPE_SKIP;
-            }
-        } else {
-            /* SOM external event */
-            DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset);
-            assert(offset_adj == 0 || offset_adj == -1);
-            u64a *starts = deduper->som_start_log[to_offset % 2];
-            if (fatbit_set(deduper->som_log[to_offset % 2], dkeyCount, dkey)) {
-                starts[dkey] = MIN(starts[dkey], from_offset);
-            } else {
-                starts[dkey] = from_offset;
-            }
-            DEBUG_PRINTF("starts[%u]=%llu\n", dkey, starts[dkey]);
-
-            if (offset_adj) {
-                deduper->som_log_dirty |= 1;
-            } else {
-                deduper->som_log_dirty |= 2;
-            }
-
-            return DEDUPE_SKIP;
-        }
-    }
-
-    return DEDUPE_CONTINUE;
-}
-
-static really_inline
-int roseAdaptor_i(u64a offset, ReportID id, struct hs_scratch *scratch,
-                  char is_simple, char do_som) {
-    assert(id != MO_INVALID_IDX); // Should never get an invalid ID.
-    assert(scratch);
-    assert(scratch->magic == SCRATCH_MAGIC);
-
-    struct core_info *ci = &scratch->core_info;
-    const struct RoseEngine *rose = ci->rose;
-    DEBUG_PRINTF("internal report %u\n", id);
-    const struct internal_report *ir = getInternalReport(rose, id);
-
-    assert(isExternalReport(ir)); /* only external reports should reach here */
-
-    s32 offset_adj = ir->offsetAdjust;
-    u64a to_offset = offset;
-    u64a from_offset = 0;
-
-    u32 flags = 0;
-#ifndef RELEASE_BUILD
-    if (offset_adj) {
-        // alert testing tools that we've got adjusted matches
-        flags |= HS_MATCH_FLAG_ADJUSTED;
-    }
-#endif
-
-    DEBUG_PRINTF("internal match at %llu: IID=%u type=%hhu RID=%u "
-                 "offsetAdj=%d\n", offset, id, ir->type, ir->onmatch,
-                 offset_adj);
-
-    if (unlikely(can_stop_matching(scratch))) { /* ok - we are from rose */
-        DEBUG_PRINTF("pre broken - halting\n");
-        return MO_HALT_MATCHING;
-    }
-
-    if (!is_simple && ir->hasBounds) {
-        assert(ir->minOffset || ir->minLength || ir->maxOffset < MAX_OFFSET);
-        assert(ir->minOffset <= ir->maxOffset);
-        if (offset < ir->minOffset || offset > ir->maxOffset) {
-            DEBUG_PRINTF("match fell outside valid range %llu !: [%llu,%llu]\n",
-                         offset, ir->minOffset, ir->maxOffset);
-            return ROSE_CONTINUE_MATCHING_NO_EXHAUST;
-        }
-    }
-
-    if (!is_simple && ir->ekey != INVALID_EKEY &&
-        unlikely(isExhausted(ci->rose, ci->exhaustionVector, ir->ekey))) {
-        DEBUG_PRINTF("ate exhausted match\n");
-        return MO_CONTINUE_MATCHING;
-    }
-
-    if (ir->type == EXTERNAL_CALLBACK) {
-        from_offset = 0;
-    } else if (do_som) {
-        from_offset = handleSomExternal(scratch, ir, to_offset);
-    }
-
-    to_offset += offset_adj;
-    assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset);
-
-    if (do_som && ir->minLength) {
-        if (!satisfiesMinLength(ir->minLength, from_offset, to_offset)) {
-            return ROSE_CONTINUE_MATCHING_NO_EXHAUST;
-        }
-        if (ir->quashSom) {
-            from_offset = 0;
-        }
-    }
-
-    DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n",
-                 from_offset, to_offset, ir->onmatch, ci->userContext);
-
-    int halt = 0;
-
-    if (do_som || ir->dkey != MO_INVALID_IDX) {
-        enum DedupeResult dedupe_rv = dedupeCatchup(rose, ir, scratch, offset,
-                                                from_offset, to_offset, do_som);
-        switch (dedupe_rv) {
-        case DEDUPE_HALT:
-            halt = 1;
-            goto exit;
-        case DEDUPE_SKIP:
-            halt = 0;
-            goto exit;
-        case DEDUPE_CONTINUE:
-            break;
-        }
-    }
-
-    halt = ci->userCallback((unsigned int)ir->onmatch, from_offset, to_offset,
-                            flags, ci->userContext);
-exit:
-    if (halt) {
-        DEBUG_PRINTF("callback requested to terminate matches\n");
-        ci->status |= STATUS_TERMINATED;
-        return MO_HALT_MATCHING;
-    }
-
-    if (!is_simple && ir->ekey != INVALID_EKEY) {
-        markAsMatched(ci->rose, ci->exhaustionVector, ir->ekey);
-        return MO_CONTINUE_MATCHING;
-    } else {
-        return ROSE_CONTINUE_MATCHING_NO_EXHAUST;
-    }
-}
-
 /**
  * \brief Deliver the given report to the user callback.
  *
@@ -311,8 +137,8 @@ exit:
  * that dedupe catchup has been done.
  */
 static really_inline
-int roseDeliverReport(u64a offset, UNUSED ReportID id, ReportID onmatch,
+int roseDeliverReport(u64a offset, ReportID onmatch, s32 offset_adjust,
-                      s32 offset_adjust, struct hs_scratch *scratch, u32 ekey) {
+                      struct hs_scratch *scratch, u32 ekey) {
     assert(scratch);
     assert(scratch->magic == SCRATCH_MAGIC);
 
@@ -326,19 +152,7 @@ int roseDeliverReport(u64a offset, UNUSED ReportID id, ReportID onmatch,
     }
 #endif
 
-#ifndef NDEBUG
-    // Assertions for development builds.
-    UNUSED const struct internal_report *ir = getInternalReport(ci->rose, id);
-    assert(isExternalReport(ir)); /* only external reports should reach here */
-
     assert(!can_stop_matching(scratch));
-    assert(!ir->hasBounds ||
-           (offset >= ir->minOffset && offset <= ir->maxOffset));
-    assert(ir->type == EXTERNAL_CALLBACK);
-    assert(!ir->minLength);
-    assert(!ir->quashSom);
-#endif
-
     assert(ekey == INVALID_EKEY ||
            !isExhausted(ci->rose, ci->exhaustionVector, ekey));
 
@@ -364,102 +178,6 @@ int roseDeliverReport(u64a offset, UNUSED ReportID id, ReportID onmatch,
     }
 }
 
-static really_inline
-int roseSomAdaptor_i(u64a from_offset, u64a to_offset, ReportID id,
-                     struct hs_scratch *scratch, char is_simple) {
-    assert(id != MO_INVALID_IDX); // Should never get an invalid ID.
-    assert(scratch);
-    assert(scratch->magic == SCRATCH_MAGIC);
-
-    u32 flags = 0;
-
-    struct core_info *ci = &scratch->core_info;
-    const struct RoseEngine *rose = ci->rose;
-    const struct internal_report *ir = getInternalReport(rose, id);
-
-    /* internal events should be handled by rose directly */
-    assert(ir->type == EXTERNAL_CALLBACK);
-
-    DEBUG_PRINTF("internal match at %llu: IID=%u type=%hhu RID=%u "
-                 "offsetAdj=%d\n", to_offset, id, ir->type, ir->onmatch,
-                 ir->offsetAdjust);
-
-    if (unlikely(can_stop_matching(scratch))) {
-        DEBUG_PRINTF("pre broken - halting\n");
-        return MO_HALT_MATCHING;
-    }
-
-    if (!is_simple && ir->hasBounds) {
-        assert(ir->minOffset || ir->minLength || ir->maxOffset < MAX_OFFSET);
-        if (to_offset < ir->minOffset || to_offset > ir->maxOffset) {
-            DEBUG_PRINTF("match fell outside valid range %llu !: [%llu,%llu]\n",
-                         to_offset, ir->minOffset, ir->maxOffset);
-            return MO_CONTINUE_MATCHING;
-        }
-    }
-
-    int halt = 0;
-
-    if (!is_simple && ir->ekey != INVALID_EKEY &&
-        unlikely(isExhausted(ci->rose, ci->exhaustionVector, ir->ekey))) {
-        DEBUG_PRINTF("ate exhausted match\n");
-        goto exit;
-    }
-
-    u64a offset = to_offset;
-
-    to_offset += ir->offsetAdjust;
-    assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset);
-
-    if (!is_simple && ir->minLength) {
-        if (!satisfiesMinLength(ir->minLength, from_offset, to_offset)) {
-            return MO_CONTINUE_MATCHING;
-        }
-        if (ir->quashSom) {
-            from_offset = 0;
-        }
-    }
-
-    DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n",
-                 from_offset, to_offset, ir->onmatch, ci->userContext);
-
-#ifndef RELEASE_BUILD
-    if (ir->offsetAdjust != 0) {
-        // alert testing tools that we've got adjusted matches
-        flags |= HS_MATCH_FLAG_ADJUSTED;
-    }
-#endif
-
-    enum DedupeResult dedupe_rv =
-        dedupeCatchupSom(rose, ir, scratch, offset, from_offset, to_offset);
-    switch (dedupe_rv) {
-    case DEDUPE_HALT:
-        halt = 1;
-        goto exit;
-    case DEDUPE_SKIP:
-        halt = 0;
-        goto exit;
-    case DEDUPE_CONTINUE:
-        break;
-    }
-
-    halt = ci->userCallback((unsigned int)ir->onmatch, from_offset, to_offset,
-                            flags, ci->userContext);
-
-    if (!is_simple && ir->ekey != INVALID_EKEY) {
-        markAsMatched(ci->rose, ci->exhaustionVector, ir->ekey);
-    }
-
-exit:
-    if (halt) {
-        DEBUG_PRINTF("callback requested to terminate matches\n");
-        ci->status |= STATUS_TERMINATED;
-        return MO_HALT_MATCHING;
-    }
-
-    return MO_CONTINUE_MATCHING;
-}
-
 /**
  * \brief Deliver the given SOM report to the user callback.
  *
@@ -467,42 +185,34 @@ exit:
  * that dedupe catchup has been done.
  */
 static really_inline
-int roseDeliverSomReport(u64a from_offset, u64a to_offset,
+int roseDeliverSomReport(u64a from_offset, u64a to_offset, ReportID onmatch,
-                         const struct internal_report *ir,
+                         s32 offset_adjust, struct hs_scratch *scratch,
-                         struct hs_scratch *scratch, char is_exhaustible) {
+                         u32 ekey) {
     assert(scratch);
     assert(scratch->magic == SCRATCH_MAGIC);
-    assert(isExternalReport(ir)); /* only external reports should reach here */
 
     struct core_info *ci = &scratch->core_info;
 
     u32 flags = 0;
 #ifndef RELEASE_BUILD
-    if (ir->offsetAdjust != 0) {
+    if (offset_adjust) {
         // alert testing tools that we've got adjusted matches
         flags |= HS_MATCH_FLAG_ADJUSTED;
     }
 #endif
 
     assert(!can_stop_matching(scratch));
-    assert(!ir->hasBounds ||
+    assert(ekey == INVALID_EKEY ||
-           (to_offset >= ir->minOffset && to_offset <= ir->maxOffset));
+           !isExhausted(ci->rose, ci->exhaustionVector, ekey));
-    assert(ir->ekey == INVALID_EKEY ||
-           !isExhausted(ci->rose, ci->exhaustionVector, ir->ekey));
 
-    to_offset += ir->offsetAdjust;
+    to_offset += offset_adjust;
     assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset);
 
-    assert(!ir->minLength ||
-           satisfiesMinLength(ir->minLength, from_offset, to_offset));
-    assert(!ir->quashSom || from_offset == 0);
-
     DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n",
-                 from_offset, to_offset, ir->onmatch, ci->userContext);
+                 from_offset, to_offset, onmatch, ci->userContext);
 
-
+    int halt = ci->userCallback(onmatch, from_offset, to_offset, flags,
-    int halt = ci->userCallback((unsigned int)ir->onmatch, from_offset,
+                                ci->userContext);
-                                to_offset, flags, ci->userContext);
 
     if (halt) {
         DEBUG_PRINTF("callback requested to terminate matches\n");
@@ -510,9 +220,8 @@ int roseDeliverSomReport(u64a from_offset, u64a to_offset,
         return MO_HALT_MATCHING;
     }
 
-    if (is_exhaustible) {
+    if (ekey != INVALID_EKEY) {
-        assert(ir->ekey != INVALID_EKEY);
+        markAsMatched(ci->rose, ci->exhaustionVector, ekey);
-        markAsMatched(ci->rose, ci->exhaustionVector, ir->ekey);
         return MO_CONTINUE_MATCHING;
     } else {
         return ROSE_CONTINUE_MATCHING_NO_EXHAUST;

src/rose/block.c

@@ -133,7 +133,6 @@ void init_outfixes_for_block(const struct RoseEngine *t,
 
 static really_inline
 void init_for_block(const struct RoseEngine *t, struct hs_scratch *scratch,
-                    RoseCallback callback, RoseCallbackSom som_callback,
                     char *state, char is_small_block) {
     init_state_for_block(t, state);
 
@@ -144,8 +143,6 @@ void init_for_block(const struct RoseEngine *t, struct hs_scratch *scratch,
     tctxt->delayLastEndOffset = 0;
     tctxt->lastEndOffset = 0;
     tctxt->filledDelayedSlots = 0;
-    tctxt->cb = callback;
-    tctxt->cb_som = som_callback;
     tctxt->lastMatchOffset = 0;
     tctxt->minMatchOffset = 0;
     tctxt->minNonMpvMatchOffset = 0;
@@ -160,8 +157,7 @@ void init_for_block(const struct RoseEngine *t, struct hs_scratch *scratch,
     init_outfixes_for_block(t, scratch, state, is_small_block);
 }
 
-void roseBlockExec_i(const struct RoseEngine *t, struct hs_scratch *scratch,
+void roseBlockExec_i(const struct RoseEngine *t, struct hs_scratch *scratch) {
-                     RoseCallback callback, RoseCallbackSom som_callback) {
     assert(t);
     assert(scratch);
     assert(scratch->core_info.buf);
@@ -179,8 +175,7 @@ void roseBlockExec_i(const struct RoseEngine *t, struct hs_scratch *scratch,
 
     char *state = scratch->core_info.state;
 
-    init_for_block(t, scratch, callback, som_callback, state,
+    init_for_block(t, scratch, state, is_small_block);
-                   is_small_block);
 
     struct RoseContext *tctxt = &scratch->tctxt;
 

src/rose/catchup.c

@@ -33,6 +33,7 @@
 
 #include "catchup.h"
 #include "match.h"
+#include "program_runtime.h"
 #include "rose.h"
 #include "nfa/nfa_rev_api.h"
 #include "nfa/mpv.h"
@@ -46,43 +47,18 @@ typedef struct queue_match PQ_T;
 #include "util/pqueue.h"
 
 static really_inline
-int handleReportInternally(const struct RoseEngine *t,
+int roseNfaRunProgram(const struct RoseEngine *rose, struct hs_scratch *scratch,
-                           struct hs_scratch *scratch, ReportID id,
+                      u64a som, u64a offset, ReportID id, const char from_mpv) {
-                           u64a offset) {
+    assert(id < rose->reportProgramCount);
-    const struct internal_report *ri = getInternalReport(t, id);
+    const u32 *programs = getByOffset(rose, rose->reportProgramOffset);
-    if (ri->type == EXTERNAL_CALLBACK) {
-        return 0;
-    }
-    if (isInternalSomReport(ri)) {
-        handleSomInternal(scratch, ri, offset);
-        return 1;
-    }
-    if (ri->type == INTERNAL_ROSE_CHAIN) {
-        roseHandleChainMatch(t, scratch, id, offset, 1);
-        return 1;
-    }
 
-    return 0;
+    const size_t match_len = 0; // Unused in this path.
-}
+    const char in_anchored = 0;
+    const char in_catchup = 1;
+    roseRunProgram(rose, scratch, programs[id], som, offset, match_len,
+                   in_anchored, in_catchup, from_mpv, 0);
 
-static really_inline
+    return can_stop_matching(scratch) ? MO_HALT_MATCHING : MO_CONTINUE_MATCHING;
-int handleReportInternallyNoChain(const struct RoseEngine *t,
-                                  struct hs_scratch *scratch, ReportID id,
-                                  u64a offset) {
-    const struct internal_report *ri = getInternalReport(t, id);
-    if (ri->type == EXTERNAL_CALLBACK) {
-        return 0;
-    }
-    if (isInternalSomReport(ri)) {
-        handleSomInternal(scratch, ri, offset);
-        return 1;
-    }
-    if (ri->type == INTERNAL_ROSE_CHAIN) {
-        assert(0); /* chained engines cannot trigger other engines */
-        return 1;
-    }
-
-    return 0;
 }
 
 static really_inline
@@ -270,50 +246,15 @@ restart:
 }
 
 /* for use by mpv (chained) only */
-static UNUSED
+static
 int roseNfaFinalBlastAdaptor(u64a offset, ReportID id, void *context) {
     struct hs_scratch *scratch = context;
-    struct RoseContext *tctxt = &scratch->tctxt;
     const struct RoseEngine *t = scratch->core_info.rose;
 
-    DEBUG_PRINTF("called\n");
-
     DEBUG_PRINTF("masky got himself a blasted match @%llu id %u !woot!\n",
                  offset, id);
-    updateLastMatchOffset(tctxt, offset);
 
-    if (handleReportInternallyNoChain(t, scratch, id, offset)) {
+    int cb_rv = roseNfaRunProgram(t, scratch, 0, offset, id, 1);
-        return MO_CONTINUE_MATCHING;
-    }
-
-    int cb_rv = tctxt->cb(offset, id, scratch);
-    if (cb_rv == MO_HALT_MATCHING) {
-        return MO_HALT_MATCHING;
-    } else if (cb_rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) {
-        return MO_CONTINUE_MATCHING;
-    } else {
-        assert(cb_rv == MO_CONTINUE_MATCHING);
-        return !roseSuffixIsExhausted(t, 0,
-                                      scratch->core_info.exhaustionVector);
-    }
-}
-
-/* for use by mpv (chained) only */
-static UNUSED
-int roseNfaFinalBlastAdaptorNoInternal(u64a offset, ReportID id,
-                                       void *context) {
-    struct hs_scratch *scratch = context;
-    struct RoseContext *tctxt = &scratch->tctxt;
-    const struct RoseEngine *t = scratch->core_info.rose;
-
-    DEBUG_PRINTF("called\n");
-    /* chained nfas are run under the control of the anchored catchup */
-
-    DEBUG_PRINTF("masky got himself a blasted match @%llu id %u !woot!\n",
-                 offset, id);
-    updateLastMatchOffset(tctxt, offset);
-
-    int cb_rv = tctxt->cb(offset, id, scratch);
     if (cb_rv == MO_HALT_MATCHING) {
         return MO_HALT_MATCHING;
     } else if (cb_rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) {
@@ -418,11 +359,7 @@ hwlmcb_rv_t roseCatchUpMPV_i(const struct RoseEngine *t, s64a loc,
 
     assert(!q->report_current);
 
-    if (info->only_external) {
+    q->cb = roseNfaFinalBlastAdaptor;
-        q->cb = roseNfaFinalBlastAdaptorNoInternal;
-    } else {
-        q->cb = roseNfaFinalBlastAdaptor;
-    }
     q->som_cb = NULL;
 
     DEBUG_PRINTF("queue %u blasting, %u/%u [%lld/%lld]\n",
@@ -466,35 +403,28 @@ done:
                                       : HWLM_CONTINUE_MATCHING;
 }
 
-static UNUSED
+static really_inline
+char in_mpv(const struct RoseEngine *rose, const struct hs_scratch *scratch) {
+    const struct RoseContext *tctxt = &scratch->tctxt;
+    assert(tctxt->curr_qi < rose->queueCount);
+    if (tctxt->curr_qi < rose->outfixBeginQueue) {
+        assert(getNfaByQueue(rose, tctxt->curr_qi)->type == MPV_NFA_0);
+        return 1;
+    }
+    return 0;
+}
+
+static
 int roseNfaBlastAdaptor(u64a offset, ReportID id, void *context) {
     struct hs_scratch *scratch = context;
     struct RoseContext *tctxt = &scratch->tctxt;
     const struct RoseEngine *t = scratch->core_info.rose;
 
-    const struct internal_report *ri = getInternalReport(t, id);
-
-    DEBUG_PRINTF("called\n");
-    if (ri->type != INTERNAL_ROSE_CHAIN) {
-        /* INTERNAL_ROSE_CHAIN are not visible externally */
-        if (roseCatchUpMPV(t, offset - scratch->core_info.buf_offset,
-                           scratch) == HWLM_TERMINATE_MATCHING) {
-            DEBUG_PRINTF("done\n");
-            return MO_HALT_MATCHING;
-        }
-    }
-
     DEBUG_PRINTF("masky got himself a blasted match @%llu id %u !woot!\n",
                  offset, id);
 
-    if (handleReportInternally(t, scratch, id, offset)) {
+    const char from_mpv = in_mpv(t, scratch);
-        return can_stop_matching(scratch) ? MO_HALT_MATCHING
+    int cb_rv = roseNfaRunProgram(t, scratch, 0, offset, id, from_mpv);
-                                          : MO_CONTINUE_MATCHING;
-    }
-
-    updateLastMatchOffset(tctxt, offset);
-
-    int cb_rv = tctxt->cb(offset, id, scratch);
     if (cb_rv == MO_HALT_MATCHING) {
         return MO_HALT_MATCHING;
     } else if (cb_rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) {
@@ -506,24 +436,18 @@ int roseNfaBlastAdaptor(u64a offset, ReportID id, void *context) {
     }
 }
 
-static UNUSED
+static
 int roseNfaBlastAdaptorNoInternal(u64a offset, ReportID id, void *context) {
     struct hs_scratch *scratch = context;
     struct RoseContext *tctxt = &scratch->tctxt;
     const struct RoseEngine *t = scratch->core_info.rose;
 
-    DEBUG_PRINTF("called\n");
-    if (roseCatchUpMPV(t, offset - scratch->core_info.buf_offset, scratch) ==
-        HWLM_TERMINATE_MATCHING) {
-        DEBUG_PRINTF("done\n");
-        return MO_HALT_MATCHING;
-    }
-
     DEBUG_PRINTF("masky got himself a blasted match @%llu id %u !woot!\n",
                  offset, id);
-    updateLastMatchOffset(tctxt, offset);
 
-    int cb_rv = tctxt->cb(offset, id, scratch);
+    assert(!in_mpv(t, scratch));
+
+    int cb_rv = roseNfaRunProgram(t, scratch, 0, offset, id, 0);
     if (cb_rv == MO_HALT_MATCHING) {
         return MO_HALT_MATCHING;
     } else if (cb_rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) {
@@ -535,78 +459,21 @@ int roseNfaBlastAdaptorNoInternal(u64a offset, ReportID id, void *context) {
     }
 }
 
-static UNUSED
+static
-int roseNfaBlastAdaptorNoChain(u64a offset, ReportID id, void *context) {
-    struct hs_scratch *scratch = context;
-    struct RoseContext *tctxt = &scratch->tctxt;
-    const struct RoseEngine *t = scratch->core_info.rose;
-
-    DEBUG_PRINTF("masky got himself a blasted match @%llu id %u !woot!\n",
-                 offset, id);
-
-    updateLastMatchOffset(tctxt, offset);
-
-    if (handleReportInternallyNoChain(t, scratch, id, offset)) {
-        return MO_CONTINUE_MATCHING;
-    }
-
-    int cb_rv = tctxt->cb(offset, id, scratch);
-    if (cb_rv == MO_HALT_MATCHING) {
-        return MO_HALT_MATCHING;
-    } else if (cb_rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) {
-        return MO_CONTINUE_MATCHING;
-    } else {
-        assert(cb_rv == MO_CONTINUE_MATCHING);
-        return !roseSuffixIsExhausted(t, tctxt->curr_qi,
-                                      scratch->core_info.exhaustionVector);
-    }
-}
-
-static UNUSED
-int roseNfaBlastAdaptorNoInternalNoChain(u64a offset, ReportID id,
-                                         void *context) {
-    struct hs_scratch *scratch = context;
-    struct RoseContext *tctxt = &scratch->tctxt;
-    const struct RoseEngine *t = scratch->core_info.rose;
-
-    /* chained nfas are run under the control of the anchored catchup */
-
-    DEBUG_PRINTF("masky got himself a blasted match @%llu id %u !woot!\n",
-                 offset, id);
-    updateLastMatchOffset(tctxt, offset);
-
-    int cb_rv = tctxt->cb(offset, id, scratch);
-    if (cb_rv == MO_HALT_MATCHING) {
-        return MO_HALT_MATCHING;
-    } else if (cb_rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) {
-        return MO_CONTINUE_MATCHING;
-    } else {
-        assert(cb_rv == MO_CONTINUE_MATCHING);
-        return !roseSuffixIsExhausted(t, tctxt->curr_qi,
-                                      scratch->core_info.exhaustionVector);
-    }
-}
-
-static UNUSED
 int roseNfaBlastSomAdaptor(u64a from_offset, u64a offset, ReportID id,
                            void *context) {
     struct hs_scratch *scratch = context;
     struct RoseContext *tctxt = &scratch->tctxt;
     const struct RoseEngine *t = scratch->core_info.rose;
 
-    DEBUG_PRINTF("called\n");
-    if (roseCatchUpMPV(t, offset - scratch->core_info.buf_offset, scratch) ==
-        HWLM_TERMINATE_MATCHING) {
-        DEBUG_PRINTF("roseCatchUpNfas done\n");
-        return MO_HALT_MATCHING;
-    }
-
     DEBUG_PRINTF("masky got himself a blasted match @%llu id %u !woot!\n",
                  offset, id);
-    updateLastMatchOffset(tctxt, offset);
+
+    assert(!in_mpv(t, scratch));
 
     /* must be a external report as haig cannot directly participate in chain */
-    int cb_rv = tctxt->cb_som(from_offset, offset, id, scratch);
+    int cb_rv = roseNfaRunProgram(scratch->core_info.rose, scratch, from_offset,
+                                  offset, id, 0);
     if (cb_rv == MO_HALT_MATCHING) {
         return MO_HALT_MATCHING;
     } else if (cb_rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) {
@@ -620,38 +487,20 @@ int roseNfaBlastSomAdaptor(u64a from_offset, u64a offset, ReportID id,
 
 int roseNfaAdaptor(u64a offset, ReportID id, void *context) {
     struct hs_scratch *scratch = context;
-    struct RoseContext *tctxt = &scratch->tctxt;
     DEBUG_PRINTF("masky got himself a match @%llu id %u !woot!\n", offset, id);
 
-    updateLastMatchOffset(tctxt, offset);
+    return roseNfaRunProgram(scratch->core_info.rose, scratch, 0, offset, id,
-
+                             0);
-    const struct RoseEngine *t = scratch->core_info.rose;
-    if (handleReportInternally(t, scratch, id, offset)) {
-        return can_stop_matching(scratch) ? MO_HALT_MATCHING
-                                          : MO_CONTINUE_MATCHING;
-    }
-
-    return tctxt->cb(offset, id, scratch);
-}
-
-int roseNfaAdaptorNoInternal(u64a offset, ReportID id, void *context) {
-    struct hs_scratch *scratch = context;
-    struct RoseContext *tctxt = &scratch->tctxt;
-    DEBUG_PRINTF("masky got himself a match @%llu id %u !woot!\n", offset, id);
-    updateLastMatchOffset(tctxt, offset);
-
-    return tctxt->cb(offset, id, scratch);
 }
 
 int roseNfaSomAdaptor(u64a from_offset, u64a offset, ReportID id,
                       void *context) {
     struct hs_scratch *scratch = context;
-    struct RoseContext *tctxt = &scratch->tctxt;
     DEBUG_PRINTF("masky got himself a match @%llu id %u !woot!\n", offset, id);
-    updateLastMatchOffset(tctxt, offset);
 
     /* must be a external report as haig cannot directly participate in chain */
-    return tctxt->cb_som(from_offset, offset, id, scratch);
+    return roseNfaRunProgram(scratch->core_info.rose, scratch, from_offset,
+                             offset, id, 0);
 }
 
 static really_inline
@@ -661,29 +510,17 @@ char blast_queue(const struct RoseEngine *t, struct hs_scratch *scratch,
     const struct NfaInfo *info = getNfaInfoByQueue(t, qi);
 
     tctxt->curr_qi = qi;
-    if (has_chained_nfas(t)) {
+    if (info->only_external) {
-        if (info->only_external) {
+        q->cb = roseNfaBlastAdaptorNoInternal;
-            q->cb = roseNfaBlastAdaptorNoInternal;
-        } else {
-            q->cb = roseNfaBlastAdaptor;
-        }
     } else {
-        if (info->only_external) {
+        q->cb = roseNfaBlastAdaptor;
-            q->cb = roseNfaBlastAdaptorNoInternalNoChain;
-        } else {
-            q->cb = roseNfaBlastAdaptorNoChain;
-        }
     }
     q->report_current = report_current;
     q->som_cb = roseNfaBlastSomAdaptor;
     DEBUG_PRINTF("queue %u blasting, %u/%u [%lld/%lld]\n", qi, q->cur, q->end,
                  q_cur_loc(q), to_loc);
     char alive = nfaQueueExec(q->nfa, q, to_loc);
-    if (info->only_external) {
+    q->cb = roseNfaAdaptor;
-        q->cb = roseNfaAdaptorNoInternal;
-    } else {
-        q->cb = roseNfaAdaptor;
-    }
     q->som_cb = roseNfaSomAdaptor;
     assert(!q->report_current);
 
@@ -945,14 +782,16 @@ hwlmcb_rv_t buildSufPQ(const struct RoseEngine *t, char *state, s64a safe_loc,
 static never_inline
 hwlmcb_rv_t roseCatchUpNfas(const struct RoseEngine *t, s64a loc,
                             s64a final_loc, struct hs_scratch *scratch) {
-    struct RoseContext *tctxt = &scratch->tctxt;
     assert(t->activeArrayCount);
 
-    assert(scratch->core_info.buf_offset + loc >= tctxt->minNonMpvMatchOffset);
+    DEBUG_PRINTF("roseCatchUpNfas offset=%llu + %lld/%lld\n",
-    DEBUG_PRINTF("roseCatchUpNfas %lld/%lld\n", loc, final_loc);
+                 scratch->core_info.buf_offset, loc, final_loc);
     DEBUG_PRINTF("min non mpv match offset %llu\n",
                  scratch->tctxt.minNonMpvMatchOffset);
 
+    struct RoseContext *tctxt = &scratch->tctxt;
+    assert(scratch->core_info.buf_offset + loc >= tctxt->minNonMpvMatchOffset);
+
     char *state = scratch->core_info.state;
     struct mq *queues = scratch->queues;
     u8 *aa = getActiveLeafArray(t, state);

src/rose/catchup.h

@@ -56,9 +56,6 @@
 #include "ue2common.h"
 #include "util/multibit.h"
 
-
-/* Callbacks, defined in catchup.c */
-
 hwlmcb_rv_t roseCatchUpAll(s64a loc, struct hs_scratch *scratch);
 
 /* will only catch mpv up to last reported external match */

src/rose/eod.c

@@ -34,8 +34,7 @@
 
 static really_inline
 void initContext(const struct RoseEngine *t, char *state, u64a offset,
-                 struct hs_scratch *scratch, RoseCallback callback,
+                 struct hs_scratch *scratch) {
-                 RoseCallbackSom som_callback) {
     struct RoseContext *tctxt = &scratch->tctxt;
     tctxt->groups = loadGroups(t, state); /* TODO: diff groups for eod */
     tctxt->lit_offset_adjust = scratch->core_info.buf_offset
@@ -44,8 +43,6 @@ void initContext(const struct RoseEngine *t, char *state, u64a offset,
     tctxt->delayLastEndOffset = offset;
     tctxt->lastEndOffset = offset;
     tctxt->filledDelayedSlots = 0;
-    tctxt->cb = callback;
-    tctxt->cb_som = som_callback;
     tctxt->lastMatchOffset = 0;
     tctxt->minMatchOffset = offset;
     tctxt->minNonMpvMatchOffset = offset;
@@ -109,38 +106,21 @@ int roseEodRunIterator(const struct RoseEngine *t, u64a offset,
 
     DEBUG_PRINTF("running eod program at offset %u\n", t->eodIterProgramOffset);
 
+    const u64a som = 0;
     const size_t match_len = 0;
-    if (roseRunProgram(t, scratch, t->eodIterProgramOffset, offset, match_len,
+    const char in_anchored = 0;
-                       0) == HWLM_TERMINATE_MATCHING) {
+    const char in_catchup = 0;
+    const char from_mpv = 0;
+    const char skip_mpv_catchup = 1;
+    if (roseRunProgram(t, scratch, t->eodIterProgramOffset, som, offset,
+                       match_len, in_anchored, in_catchup,
+                       from_mpv, skip_mpv_catchup) == HWLM_TERMINATE_MATCHING) {
         return MO_HALT_MATCHING;
     }
 
     return MO_CONTINUE_MATCHING;
 }
 
-/**
- * \brief Adapts an NfaCallback to the rose callback specified in the
- * RoseContext.
- */
-static
-int eodNfaCallback(u64a offset, ReportID report, void *context) {
-    struct hs_scratch *scratch = context;
-    assert(scratch->magic == SCRATCH_MAGIC);
-    return scratch->tctxt.cb(offset, report, scratch);
-}
-
-/**
- * \brief Adapts a SomNfaCallback to the rose SOM callback specified in the
- * RoseContext.
- */
-static
-int eodNfaSomCallback(u64a from_offset, u64a to_offset, ReportID report,
-                      void *context) {
-    struct hs_scratch *scratch = context;
-    assert(scratch->magic == SCRATCH_MAGIC);
-    return scratch->tctxt.cb_som(from_offset, to_offset, report, scratch);
-}
-
 /**
  * \brief Check for (and deliver) reports from active output-exposed (suffix
  * or outfix) NFAs.
@@ -190,8 +170,8 @@ int roseCheckNfaEod(const struct RoseEngine *t, char *state,
             nfaExpandState(nfa, fstate, sstate, offset, key);
         }
 
-        if (nfaCheckFinalState(nfa, fstate, sstate, offset, eodNfaCallback,
+        if (nfaCheckFinalState(nfa, fstate, sstate, offset, roseReportAdaptor,
-                               eodNfaSomCallback,
+                               roseReportSomAdaptor,
                                scratch) == MO_HALT_MATCHING) {
             DEBUG_PRINTF("user instructed us to stop\n");
             return MO_HALT_MATCHING;
@@ -239,8 +219,8 @@ void roseCheckEodSuffixes(const struct RoseEngine *t, char *state, u64a offset,
          * history buffer. */
         char rv = nfaQueueExecRose(q->nfa, q, MO_INVALID_IDX);
         if (rv) { /* nfa is still alive */
-            if (nfaCheckFinalState(nfa, fstate, sstate, offset, eodNfaCallback,
+            if (nfaCheckFinalState(nfa, fstate, sstate, offset,
-                                   eodNfaSomCallback,
+                                   roseReportAdaptor, roseReportSomAdaptor,
                                    scratch) == MO_HALT_MATCHING) {
                 DEBUG_PRINTF("user instructed us to stop\n");
                 return;
@@ -261,9 +241,15 @@ int roseRunEodProgram(const struct RoseEngine *t, u64a offset,
     // There should be no pending delayed literals.
     assert(!scratch->tctxt.filledDelayedSlots);
 
+    const u64a som = 0;
     const size_t match_len = 0;
-    if (roseRunProgram(t, scratch, t->eodProgramOffset, offset, match_len, 0) ==
+    const char in_anchored = 0;
-        HWLM_TERMINATE_MATCHING) {
+    const char in_catchup = 0;
+    const char from_mpv = 0;
+    const char skip_mpv_catchup = 1;
+    if (roseRunProgram(t, scratch, t->eodProgramOffset, som, offset, match_len,
+                       in_anchored, in_catchup, from_mpv,
+                       skip_mpv_catchup) == HWLM_TERMINATE_MATCHING) {
         return MO_HALT_MATCHING;
     }
 
@@ -322,10 +308,8 @@ void roseEodExec_i(const struct RoseEngine *t, char *state, u64a offset,
 }
 
 void roseEodExec(const struct RoseEngine *t, u64a offset,
-                 struct hs_scratch *scratch, RoseCallback callback,
+                 struct hs_scratch *scratch) {
-                 RoseCallbackSom som_callback) {
     assert(scratch);
-    assert(callback);
     assert(t->requiresEodCheck);
     DEBUG_PRINTF("ci buf %p/%zu his %p/%zu\n", scratch->core_info.buf,
                  scratch->core_info.len, scratch->core_info.hbuf,
@@ -345,7 +329,7 @@ void roseEodExec(const struct RoseEngine *t, u64a offset,
     char *state = scratch->core_info.state;
     assert(state);
 
-    initContext(t, state, offset, scratch, callback, som_callback);
+    initContext(t, state, offset, scratch);
 
     roseEodExec_i(t, state, offset, scratch, 1);
 }

src/rose/match.c

@@ -96,9 +96,15 @@ hwlmcb_rv_t roseDelayRebuildCallback(size_t start, size_t end, u32 id,
     const u32 program = delayRebuildPrograms[id];
 
     if (program) {
+        const u64a som = 0;
         const size_t match_len = end - start + 1;
+        const char in_anchored = 0;
+        const char in_catchup = 0;
+        const char from_mpv = 0;
+        const char skip_mpv_catchup = 0;
         UNUSED hwlmcb_rv_t rv =
-            roseRunProgram(t, scratch, program, real_end, match_len, 0);
+            roseRunProgram(t, scratch, program, som, real_end, match_len,
+                           in_anchored, in_catchup, from_mpv, skip_mpv_catchup);
         assert(rv != HWLM_TERMINATE_MATCHING);
     }
 
@@ -138,8 +144,10 @@ void recordAnchoredLiteralMatch(const struct RoseEngine *t,
 }
 
 hwlmcb_rv_t roseHandleChainMatch(const struct RoseEngine *t,
-                                 struct hs_scratch *scratch, ReportID r,
+                                 struct hs_scratch *scratch, u32 event,
-                                 u64a end, char in_catchup) {
+                                 u64a top_squash_distance, u64a end,
+                                 char in_catchup) {
+    assert(event == MQE_TOP || event >= MQE_TOP_FIRST);
     struct core_info *ci = &scratch->core_info;
 
     u8 *aa = getActiveLeafArray(t, scratch->core_info.state);
@@ -147,18 +155,7 @@ hwlmcb_rv_t roseHandleChainMatch(const struct RoseEngine *t,
     struct fatbit *activeQueues = scratch->aqa;
     u32 qCount = t->queueCount;
 
-    const struct internal_report *ri = getInternalReport(t, r);
+    const u32 qi = 0; /* MPV is always queue 0 if it exists */
-    assert(ri->type == INTERNAL_ROSE_CHAIN);
-
-    u32 qi = 0; /* MPV is always queue 0 if it exists */
-    u32 event = ri->onmatch;
-    assert(event == MQE_TOP || event >= MQE_TOP_FIRST);
-
-    /* TODO: populate INTERNAL_ROSE_CHAIN internal reports with offset where
-     * possible */
-    if (end < ri->minOffset || (ri->maxOffset && end > ri->maxOffset)) {
-        return HWLM_CONTINUE_MATCHING;
-    }
     struct mq *q = &scratch->queues[qi];
     const struct NfaInfo *info = getNfaInfoByQueue(t, qi);
 
@@ -189,11 +186,11 @@ hwlmcb_rv_t roseHandleChainMatch(const struct RoseEngine *t,
         }
     }
 
-    if (ri->aux.topSquashDistance) {
+    if (top_squash_distance) {
         assert(q->cur != q->end);
         struct mq_item *last = &q->items[q->end - 1];
         if (last->type == event
-            && last->location >= loc - (s64a)ri->aux.topSquashDistance) {
+            && last->location >= loc - (s64a)top_squash_distance) {
             last->location = loc;
             goto event_enqueued;
         }
@@ -255,8 +252,14 @@ int roseAnchoredCallback(u64a end, u32 id, void *ctx) {
 
     const u32 *programs = getByOffset(t, t->litProgramOffset);
     assert(id < t->literalCount);
-    if (roseRunProgram(t, scratch, programs[id], real_end, match_len, 1) ==
+    const u64a som = 0;
-        HWLM_TERMINATE_MATCHING) {
+    const char in_anchored = 1;
+    const char in_catchup = 0;
+    const char from_mpv = 0;
+    const char skip_mpv_catchup = 0;
+    if (roseRunProgram(t, scratch, programs[id], som, real_end, match_len,
+                       in_anchored, in_catchup, from_mpv,
+                       skip_mpv_catchup) == HWLM_TERMINATE_MATCHING) {
         assert(can_stop_matching(scratch));
         DEBUG_PRINTF("caller requested termination\n");
         return MO_HALT_MATCHING;
@@ -280,7 +283,13 @@ hwlmcb_rv_t roseProcessMatch(const struct RoseEngine *t,
     DEBUG_PRINTF("id=%u\n", id);
     const u32 *programs = getByOffset(t, t->litProgramOffset);
     assert(id < t->literalCount);
-    return roseRunProgram(t, scratch, programs[id], end, match_len, 0);
+    const u64a som = 0;
+    const char in_anchored = 0;
+    const char in_catchup = 0;
+    const char from_mpv = 0;
+    const char skip_mpv_catchup = 0;
+    return roseRunProgram(t, scratch, programs[id], som, end, match_len,
+                          in_anchored, in_catchup, from_mpv, skip_mpv_catchup);
 }
 
 static rose_inline
@@ -568,11 +577,17 @@ hwlmcb_rv_t rosePureLiteralCallback(size_t start, size_t end, u32 id,
     struct hs_scratch *scratch = context;
     struct core_info *ci = &scratch->core_info;
     const u64a real_end = (u64a)end + ci->buf_offset + 1;
+    const u64a som = 0;
     const size_t match_len = end - start + 1;
     const struct RoseEngine *rose = ci->rose;
     const u32 *programs = getByOffset(rose, rose->litProgramOffset);
     assert(id < rose->literalCount);
-    return roseRunProgram(rose, scratch, programs[id], real_end, match_len, 0);
+    const char in_anchored = 0;
+    const char in_catchup = 0;
+    const char from_mpv = 0;
+    const char skip_mpv_catchup = 0;
+    return roseRunProgram(rose, scratch, programs[id], som, real_end, match_len,
+                          in_anchored, in_catchup, from_mpv, skip_mpv_catchup);
 }
 
 /**
@@ -606,13 +621,53 @@ int roseRunBoundaryProgram(const struct RoseEngine *rose, u32 program,
     // time we are running boundary report programs.
     scratch->tctxt.minMatchOffset = stream_offset;
 
+    const u64a som = 0;
     const size_t match_len = 0;
     const char in_anchored = 0;
-    hwlmcb_rv_t rv = roseRunProgram(rose, scratch, program, stream_offset,
+    const char in_catchup = 0;
-                                    match_len, in_anchored);
+    const char from_mpv = 0;
+    const char skip_mpv_catchup = 0;
+    hwlmcb_rv_t rv =
+        roseRunProgram(rose, scratch, program, som, stream_offset, match_len,
+                       in_anchored, in_catchup, from_mpv, skip_mpv_catchup);
     if (rv == HWLM_TERMINATE_MATCHING) {
         return MO_HALT_MATCHING;
     }
 
     return MO_CONTINUE_MATCHING;
 }
+
+static really_inline
+int roseReportAdaptor_i(u64a som, u64a offset, ReportID id, void *context) {
+    struct hs_scratch *scratch = context;
+    assert(scratch && scratch->magic == SCRATCH_MAGIC);
+
+    const struct RoseEngine *rose = scratch->core_info.rose;
+
+    assert(id < rose->reportProgramCount);
+    const u32 *programs = getByOffset(rose, rose->reportProgramOffset);
+
+    const size_t match_len = 0; // Unused in this path.
+    const char in_anchored = 0;
+    const char in_catchup = 0;
+    const char from_mpv = 0;
+    const char skip_mpv_catchup = 1;
+    hwlmcb_rv_t rv =
+        roseRunProgram(rose, scratch, programs[id], som, offset, match_len,
+                       in_anchored, in_catchup, from_mpv, skip_mpv_catchup);
+    if (rv == HWLM_TERMINATE_MATCHING) {
+        return MO_HALT_MATCHING;
+    }
+
+    return can_stop_matching(scratch) ? MO_HALT_MATCHING : MO_CONTINUE_MATCHING;
+}
+
+int roseReportAdaptor(u64a offset, ReportID id, void *context) {
+    DEBUG_PRINTF("offset=%llu, id=%u\n", offset, id);
+    return roseReportAdaptor_i(0, offset, id, context);
+}
+
+int roseReportSomAdaptor(u64a som, u64a offset, ReportID id, void *context) {
+    DEBUG_PRINTF("som=%llu, offset=%llu, id=%u\n", som, offset, id);
+    return roseReportAdaptor_i(som, offset, id, context);
+}

src/rose/match.h

@@ -41,13 +41,11 @@
 #include "som/som_runtime.h"
 #include "util/bitutils.h"
 #include "util/fatbit.h"
-#include "util/internal_report.h"
 #include "util/multibit.h"
 
 /* Callbacks, defined in catchup.c */
 
 int roseNfaAdaptor(u64a offset, ReportID id, void *context);
-int roseNfaAdaptorNoInternal(u64a offset, ReportID id, void *context);
 int roseNfaSomAdaptor(u64a from_offset, u64a offset, ReportID id, void *context);
 
 /* Callbacks, defined in match.c */
@@ -60,8 +58,9 @@ int roseAnchoredCallback(u64a end, u32 id, void *ctx);
 /* Common code, used all over Rose runtime */
 
 hwlmcb_rv_t roseHandleChainMatch(const struct RoseEngine *t,
-                                 struct hs_scratch *scratch, ReportID r,
+                                 struct hs_scratch *scratch, u32 event,
-                                 u64a end, char in_catchup);
+                                 u64a top_squash_distance, u64a end,
+                                 char in_catchup);
 
 static really_inline
 void initQueue(struct mq *q, u32 qi, const struct RoseEngine *t,
@@ -78,11 +77,7 @@ void initQueue(struct mq *q, u32 qi, const struct RoseEngine *t,
     q->length = scratch->core_info.len;
     q->history = scratch->core_info.hbuf;
     q->hlength = scratch->core_info.hlen;
-    if (info->only_external) {
+    q->cb = roseNfaAdaptor;
-        q->cb = roseNfaAdaptorNoInternal;
-    } else {
-        q->cb = roseNfaAdaptor;
-    }
     q->som_cb = roseNfaSomAdaptor;
     q->context = scratch;
     q->report_current = 0;

src/rose/program_runtime.h

@@ -573,14 +573,13 @@ void roseTriggerInfix(const struct RoseEngine *t, struct hs_scratch *scratch,
 
 static rose_inline
 hwlmcb_rv_t roseReport(const struct RoseEngine *t, struct hs_scratch *scratch,
-                       u64a end, ReportID id, ReportID onmatch,
+                       u64a end, ReportID onmatch, s32 offset_adjust,
-                       s32 offset_adjust, u32 ekey) {
+                       u32 ekey) {
     assert(!t->needsCatchup || end == scratch->tctxt.minMatchOffset);
-    DEBUG_PRINTF("firing callback id=%u, end=%llu\n", id, end);
+    DEBUG_PRINTF("firing callback onmatch=%u, end=%llu\n", onmatch, end);
     updateLastMatchOffset(&scratch->tctxt, end);
 
-    int cb_rv = roseDeliverReport(end, id, onmatch, offset_adjust, scratch,
+    int cb_rv = roseDeliverReport(end, onmatch, offset_adjust, scratch, ekey);
-                                  ekey);
     if (cb_rv == MO_HALT_MATCHING) {
         DEBUG_PRINTF("termination requested\n");
         return HWLM_TERMINATE_MATCHING;
@@ -599,47 +598,45 @@ hwlmcb_rv_t roseReport(const struct RoseEngine *t, struct hs_scratch *scratch,
 static rose_inline
 hwlmcb_rv_t roseCatchUpAndHandleChainMatch(const struct RoseEngine *t,
                                            struct hs_scratch *scratch,
-                                           ReportID r, u64a end) {
+                                           u32 event, u64a top_squash_distance,
-    if (roseCatchUpMpvFeeders(t, scratch, end) == HWLM_TERMINATE_MATCHING) {
+                                           u64a end, const char in_catchup) {
+    if (!in_catchup &&
+        roseCatchUpMpvFeeders(t, scratch, end) == HWLM_TERMINATE_MATCHING) {
         return HWLM_TERMINATE_MATCHING;
     }
-
+    return roseHandleChainMatch(t, scratch, event, top_squash_distance, end,
-    return roseHandleChainMatch(t, scratch, r, end, 0);
+                                in_catchup);
 }
 
 static rose_inline
-void roseHandleSom(const struct RoseEngine *t, struct hs_scratch *scratch,
+void roseHandleSom(UNUSED const struct RoseEngine *t,
-                   ReportID id, u64a end) {
+                   struct hs_scratch *scratch, const struct som_operation *sr,
-    DEBUG_PRINTF("id=%u, end=%llu, minMatchOffset=%llu\n", id, end,
+                   u64a end) {
-                  scratch->tctxt.minMatchOffset);
+    DEBUG_PRINTF("end=%llu, minMatchOffset=%llu\n", end,
-
+                 scratch->tctxt.minMatchOffset);
-    // Reach into reports and handle internal reports that just manipulate SOM
-    // slots ourselves, rather than going through the callback.
 
     assert(!t->needsCatchup || end == scratch->tctxt.minMatchOffset);
-    DEBUG_PRINTF("firing som callback id=%u, end=%llu\n", id, end);
     updateLastMatchOffset(&scratch->tctxt, end);
-
+    handleSomInternal(scratch, sr, end);
-    const struct internal_report *ri = getInternalReport(t, id);
-    handleSomInternal(scratch, ri, end);
 }
 
 static rose_inline
 hwlmcb_rv_t roseReportSom(const struct RoseEngine *t,
-                          struct hs_scratch *scratch, ReportID id, u64a start,
+                          struct hs_scratch *scratch, u64a start, u64a end,
-                          u64a end, char is_exhaustible) {
+                          ReportID onmatch, s32 offset_adjust, u32 ekey) {
     assert(!t->needsCatchup || end == scratch->tctxt.minMatchOffset);
-    DEBUG_PRINTF("firing som callback id=%u, end=%llu\n", id, end);
+    DEBUG_PRINTF("firing som callback onmatch=%u, start=%llu, end=%llu\n",
+                 onmatch, start, end);
     updateLastMatchOffset(&scratch->tctxt, end);
 
-    const struct internal_report *ir = getInternalReport(t, id);
+    int cb_rv = roseDeliverSomReport(start, end, onmatch, offset_adjust,
-    int cb_rv = roseDeliverSomReport(start, end, ir, scratch, is_exhaustible);
+                                     scratch, ekey);
     if (cb_rv == MO_HALT_MATCHING) {
         DEBUG_PRINTF("termination requested\n");
         return HWLM_TERMINATE_MATCHING;
     }
 
-    if (!is_exhaustible || cb_rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) {
+    if (ekey == INVALID_EKEY || cb_rv == ROSE_CONTINUE_MATCHING_NO_EXHAUST) {
         return HWLM_CONTINUE_MATCHING;
     }
 
@@ -647,19 +644,15 @@ hwlmcb_rv_t roseReportSom(const struct RoseEngine *t,
 }
 
 static rose_inline
-void roseHandleSomSom(const struct RoseEngine *t, ReportID id, u64a start,
+void roseHandleSomSom(UNUSED const struct RoseEngine *t,
-                      u64a end, struct hs_scratch *scratch) {
+                      struct hs_scratch *scratch,
-    DEBUG_PRINTF("id=%u, start=%llu, end=%llu, minMatchOffset=%llu\n",
+                      const struct som_operation *sr, u64a start, u64a end) {
-                  id, start, end, scratch->tctxt.minMatchOffset);
+    DEBUG_PRINTF("start=%llu, end=%llu, minMatchOffset=%llu\n", start, end,
-
+                 scratch->tctxt.minMatchOffset);
-    // Reach into reports and handle internal reports that just manipulate SOM
-    // slots ourselves, rather than going through the callback.
 
     assert(!t->needsCatchup || end == scratch->tctxt.minMatchOffset);
     updateLastMatchOffset(&scratch->tctxt, end);
-
+    setSomFromSomAware(scratch, sr, start, end);
-    const struct internal_report *ri = getInternalReport(t, id);
-    setSomFromSomAware(scratch, ri, start, end);
 }
 
 static really_inline
@@ -807,6 +800,16 @@ char roseCheckBounds(u64a end, u64a min_bound, u64a max_bound) {
     return end >= min_bound && end <= max_bound;
 }
 
+static
+void updateSeqPoint(struct RoseContext *tctxt, u64a offset,
+                    const char from_mpv) {
+    if (from_mpv) {
+        updateMinMatchOffsetFromMpv(tctxt, offset);
+    } else {
+        updateMinMatchOffset(tctxt, offset);
+    }
+}
+
 #define PROGRAM_CASE(name)                                                     \
     case ROSE_INSTR_##name: {                                                  \
         DEBUG_PRINTF("instruction: " #name " (pc=%u)\n",                       \
@@ -822,8 +825,10 @@ char roseCheckBounds(u64a end, u64a min_bound, u64a max_bound) {
 static rose_inline
 hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
                            struct hs_scratch *scratch, u32 programOffset,
-                           u64a end, size_t match_len, char in_anchored) {
+                           u64a som, u64a end, size_t match_len,
-    DEBUG_PRINTF("program begins at offset %u\n", programOffset);
+                           char in_anchored, char in_catchup, char from_mpv,
+                           char skip_mpv_catchup) {
+    DEBUG_PRINTF("program=%u, offsets [%llu,%llu]\n", programOffset, som, end);
 
     assert(programOffset);
     assert(programOffset < t->size);
@@ -831,8 +836,6 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
     const char *pc_base = getByOffset(t, programOffset);
     const char *pc = pc_base;
 
-    u64a som = 0;
-
     // Local sparse iterator state for programs that use the SPARSE_ITER_BEGIN
     // and SPARSE_ITER_NEXT instructions.
     struct mmbit_sparse_state si_state[MAX_SPARSE_ITER_STATES];
@@ -972,6 +975,17 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
             }
             PROGRAM_NEXT_INSTRUCTION
 
+            PROGRAM_CASE(CATCH_UP_MPV) {
+                if (from_mpv || skip_mpv_catchup) {
+                    DEBUG_PRINTF("skipping mpv catchup\n");
+                } else if (roseCatchUpMPV(t,
+                                          end - scratch->core_info.buf_offset,
+                                          scratch) == HWLM_TERMINATE_MATCHING) {
+                    return HWLM_TERMINATE_MATCHING;
+                }
+            }
+            PROGRAM_NEXT_INSTRUCTION
+
             PROGRAM_CASE(SOM_ADJUST) {
                 assert(ri->distance <= end);
                 som = end - ri->distance;
@@ -986,10 +1000,9 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(SOM_FROM_REPORT) {
-                const struct internal_report *ir =
+                som = handleSomExternal(scratch, &ri->som, end);
-                    getInternalReport(t, ri->report);
+                DEBUG_PRINTF("som from report %u is %llu\n", ri->som.onmatch,
-                som = handleSomExternal(scratch, ir, end);
+                             som);
-                DEBUG_PRINTF("som from report %u is %llu\n", ri->report, som);
             }
             PROGRAM_NEXT_INSTRUCTION
 
@@ -1016,11 +1029,13 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(DEDUPE) {
-                const struct internal_report *ir =
+                updateSeqPoint(tctxt, end, from_mpv);
-                    getInternalReport(t, ri->report);
+                const char do_som = t->hasSom; // TODO: constant propagate
-                const char do_som = t->hasSom; // FIXME: constant propagate
+                const char is_external_report = 1;
-                enum DedupeResult rv = dedupeCatchup(
+                enum DedupeResult rv =
-                    t, ir, scratch, end, som, end + ir->offsetAdjust, do_som);
+                    dedupeCatchup(t, scratch, end, som, end + ri->offset_adjust,
+                                  ri->dkey, ri->offset_adjust,
+                                  is_external_report, ri->quash_som, do_som);
                 switch (rv) {
                 case DEDUPE_HALT:
                     return HWLM_TERMINATE_MATCHING;
@@ -1035,10 +1050,13 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(DEDUPE_SOM) {
-                const struct internal_report *ir =
+                updateSeqPoint(tctxt, end, from_mpv);
-                    getInternalReport(t, ri->report);
+                const char is_external_report = 0;
-                enum DedupeResult rv = dedupeCatchupSom(
+                const char do_som = 1;
-                    t, ir, scratch, end, som, end + ir->offsetAdjust);
+                enum DedupeResult rv =
+                    dedupeCatchup(t, scratch, end, som, end + ri->offset_adjust,
+                                  ri->dkey, ri->offset_adjust,
+                                  is_external_report, ri->quash_som, do_som);
                 switch (rv) {
                 case DEDUPE_HALT:
                     return HWLM_TERMINATE_MATCHING;
@@ -1053,9 +1071,10 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT_CHAIN) {
-                if (roseCatchUpAndHandleChainMatch(t, scratch, ri->report,
+                // Note: sequence points updated inside this function.
-                                                   end) ==
+                if (roseCatchUpAndHandleChainMatch(
-                    HWLM_TERMINATE_MATCHING) {
+                        t, scratch, ri->event, ri->top_squash_distance, end,
+                        in_catchup) == HWLM_TERMINATE_MATCHING) {
                     return HWLM_TERMINATE_MATCHING;
                 }
                 work_done = 1;
@@ -1063,20 +1082,22 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT_SOM_INT) {
-                roseHandleSom(t, scratch, ri->report, end);
+                updateSeqPoint(tctxt, end, from_mpv);
+                roseHandleSom(t, scratch, &ri->som, end);
                 work_done = 1;
             }
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT_SOM_AWARE) {
-                roseHandleSomSom(t, ri->report, som, end, scratch);
+                updateSeqPoint(tctxt, end, from_mpv);
+                roseHandleSomSom(t, scratch, &ri->som, som, end);
                 work_done = 1;
             }
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT) {
-                if (roseReport(t, scratch, end, ri->report, ri->onmatch,
+                updateSeqPoint(tctxt, end, from_mpv);
-                               ri->offset_adjust,
+                if (roseReport(t, scratch, end, ri->onmatch, ri->offset_adjust,
                                INVALID_EKEY) == HWLM_TERMINATE_MATCHING) {
                     return HWLM_TERMINATE_MATCHING;
                 }
@@ -1085,8 +1106,8 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT_EXHAUST) {
-                if (roseReport(t, scratch, end, ri->report, ri->onmatch,
+                updateSeqPoint(tctxt, end, from_mpv);
-                               ri->offset_adjust,
+                if (roseReport(t, scratch, end, ri->onmatch, ri->offset_adjust,
                                ri->ekey) == HWLM_TERMINATE_MATCHING) {
                     return HWLM_TERMINATE_MATCHING;
                 }
@@ -1095,9 +1116,10 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT_SOM) {
-                const char is_exhaustible = 0;
+                updateSeqPoint(tctxt, end, from_mpv);
-                if (roseReportSom(t, scratch, ri->report, som, end,
+                if (roseReportSom(t, scratch, som, end, ri->onmatch,
-                                  is_exhaustible) == HWLM_TERMINATE_MATCHING) {
+                                  ri->offset_adjust,
+                                  INVALID_EKEY) == HWLM_TERMINATE_MATCHING) {
                     return HWLM_TERMINATE_MATCHING;
                 }
                 work_done = 1;
@@ -1105,9 +1127,10 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT_SOM_EXHAUST) {
-                const char is_exhaustible = 1;
+                updateSeqPoint(tctxt, end, from_mpv);
-                if (roseReportSom(t, scratch, ri->report, som, end,
+                if (roseReportSom(t, scratch, som, end, ri->onmatch,
-                                  is_exhaustible) == HWLM_TERMINATE_MATCHING) {
+                                  ri->offset_adjust,
+                                  ri->ekey) == HWLM_TERMINATE_MATCHING) {
                     return HWLM_TERMINATE_MATCHING;
                 }
                 work_done = 1;
@@ -1115,11 +1138,13 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(DEDUPE_AND_REPORT) {
-                const struct internal_report *ir =
+                updateSeqPoint(tctxt, end, from_mpv);
-                    getInternalReport(t, ri->report);
+                const char do_som = t->hasSom; // TODO: constant propagate
-                const char do_som = t->hasSom; // FIXME: constant propagate
+                const char is_external_report = 1;
-                enum DedupeResult rv = dedupeCatchup(
+                enum DedupeResult rv =
-                    t, ir, scratch, end, som, end + ir->offsetAdjust, do_som);
+                    dedupeCatchup(t, scratch, end, som, end + ri->offset_adjust,
+                                  ri->dkey, ri->offset_adjust,
+                                  is_external_report, ri->quash_som, do_som);
                 switch (rv) {
                 case DEDUPE_HALT:
                     return HWLM_TERMINATE_MATCHING;
@@ -1132,8 +1157,7 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
                 }
 
                 const u32 ekey = INVALID_EKEY;
-                if (roseReport(t, scratch, end, ri->report, ir->onmatch,
+                if (roseReport(t, scratch, end, ri->onmatch, ri->offset_adjust,
-                               ir->offsetAdjust,
                                ekey) == HWLM_TERMINATE_MATCHING) {
                     return HWLM_TERMINATE_MATCHING;
                 }

src/rose/rose.h

@@ -41,8 +41,7 @@ void roseInitState(const struct RoseEngine *t, char *state);
 
 void roseBlockEodExec(const struct RoseEngine *t, u64a offset,
                       struct hs_scratch *scratch);
-void roseBlockExec_i(const struct RoseEngine *t, struct hs_scratch *scratch,
+void roseBlockExec_i(const struct RoseEngine *t, struct hs_scratch *scratch);
-                     RoseCallback callback, RoseCallbackSom som_callback);
 
 static really_inline
 int roseBlockHasEodWork(const struct RoseEngine *t,
@@ -79,8 +78,7 @@ int roseBlockHasEodWork(const struct RoseEngine *t,
 
 /* assumes core_info in scratch has been init to point to data */
 static really_inline
-void roseBlockExec(const struct RoseEngine *t, struct hs_scratch *scratch,
+void roseBlockExec(const struct RoseEngine *t, struct hs_scratch *scratch) {
-                   RoseCallback callback, RoseCallbackSom som_callback) {
     assert(t);
     assert(scratch);
     assert(scratch->core_info.buf);
@@ -101,7 +99,7 @@ void roseBlockExec(const struct RoseEngine *t, struct hs_scratch *scratch,
     assert(t->maxBiAnchoredWidth == ROSE_BOUND_INF
            || length <= t->maxBiAnchoredWidth);
 
-    roseBlockExec_i(t, scratch, callback, som_callback);
+    roseBlockExec_i(t, scratch);
 
     if (!t->requiresEodCheck) {
         return;
@@ -121,16 +119,17 @@ void roseBlockExec(const struct RoseEngine *t, struct hs_scratch *scratch,
 }
 
 /* assumes core_info in scratch has been init to point to data */
-void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch,
+void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch);
-                    RoseCallback callback, RoseCallbackSom som_callback);
 
 void roseEodExec(const struct RoseEngine *t, u64a offset,
-                 struct hs_scratch *scratch, RoseCallback callback,
+                 struct hs_scratch *scratch);
-                 RoseCallbackSom som_callback);
 
 hwlmcb_rv_t rosePureLiteralCallback(size_t start, size_t end, u32 id,
                                     void *context);
 
+int roseReportAdaptor(u64a offset, ReportID id, void *context);
+int roseReportSomAdaptor(u64a som, u64a offset, ReportID id, void *context);
+
 int roseRunBoundaryProgram(const struct RoseEngine *rose, u32 program,
                            u64a stream_offset, struct hs_scratch *scratch);
 

src/rose/rose_build_bytecode.cpp

@@ -68,7 +68,6 @@
 #include "util/compile_error.h"
 #include "util/container.h"
 #include "util/graph_range.h"
-#include "util/internal_report.h"
 #include "util/multibit_build.h"
 #include "util/order_check.h"
 #include "util/queue_index_factory.h"
@@ -196,6 +195,7 @@ public:
         case ROSE_INSTR_ANCHORED_DELAY: return &u.anchoredDelay;
         case ROSE_INSTR_PUSH_DELAYED: return &u.pushDelayed;
         case ROSE_INSTR_CATCH_UP: return &u.catchUp;
+        case ROSE_INSTR_CATCH_UP_MPV: return &u.catchUpMpv;
         case ROSE_INSTR_SOM_ADJUST: return &u.somAdjust;
         case ROSE_INSTR_SOM_LEFTFIX: return &u.somLeftfix;
         case ROSE_INSTR_SOM_FROM_REPORT: return &u.somFromReport;
@@ -206,7 +206,7 @@ public:
         case ROSE_INSTR_DEDUPE_SOM: return &u.dedupeSom;
         case ROSE_INSTR_REPORT_CHAIN: return &u.reportChain;
         case ROSE_INSTR_REPORT_SOM_INT: return &u.reportSomInt;
-        case ROSE_INSTR_REPORT_SOM_AWARE: return &u.reportSom;
+        case ROSE_INSTR_REPORT_SOM_AWARE: return &u.reportSomAware;
         case ROSE_INSTR_REPORT: return &u.report;
         case ROSE_INSTR_REPORT_EXHAUST: return &u.reportExhaust;
         case ROSE_INSTR_REPORT_SOM: return &u.reportSom;
@@ -240,6 +240,7 @@ public:
         case ROSE_INSTR_ANCHORED_DELAY: return sizeof(u.anchoredDelay);
         case ROSE_INSTR_PUSH_DELAYED: return sizeof(u.pushDelayed);
         case ROSE_INSTR_CATCH_UP: return sizeof(u.catchUp);
+        case ROSE_INSTR_CATCH_UP_MPV: return sizeof(u.catchUpMpv);
         case ROSE_INSTR_SOM_ADJUST: return sizeof(u.somAdjust);
         case ROSE_INSTR_SOM_LEFTFIX: return sizeof(u.somLeftfix);
         case ROSE_INSTR_SOM_FROM_REPORT: return sizeof(u.somFromReport);
@@ -250,7 +251,7 @@ public:
         case ROSE_INSTR_DEDUPE_SOM: return sizeof(u.dedupeSom);
         case ROSE_INSTR_REPORT_CHAIN: return sizeof(u.reportChain);
         case ROSE_INSTR_REPORT_SOM_INT: return sizeof(u.reportSomInt);
-        case ROSE_INSTR_REPORT_SOM_AWARE: return sizeof(u.reportSom);
+        case ROSE_INSTR_REPORT_SOM_AWARE: return sizeof(u.reportSomAware);
         case ROSE_INSTR_REPORT: return sizeof(u.report);
         case ROSE_INSTR_REPORT_EXHAUST: return sizeof(u.reportExhaust);
         case ROSE_INSTR_REPORT_SOM: return sizeof(u.reportSom);
@@ -283,6 +284,7 @@ public:
         ROSE_STRUCT_ANCHORED_DELAY anchoredDelay;
         ROSE_STRUCT_PUSH_DELAYED pushDelayed;
         ROSE_STRUCT_CATCH_UP catchUp;
+        ROSE_STRUCT_CATCH_UP_MPV catchUpMpv;
         ROSE_STRUCT_SOM_ADJUST somAdjust;
         ROSE_STRUCT_SOM_LEFTFIX somLeftfix;
         ROSE_STRUCT_SOM_FROM_REPORT somFromReport;
@@ -396,6 +398,9 @@ struct build_context : boost::noncopyable {
      * matches, suffixes, outfixes etc. */
     bool needs_catchup = false;
 
+    /** \brief True if this Rose engine has an MPV engine. */
+    bool needs_mpv_catchup = false;
+
     /** \brief Resources in use (tracked as programs are added). */
     RoseResources resources;
 
@@ -578,7 +583,7 @@ bool isPureFloating(const RoseResources &resources) {
 }
 
 static
-bool isSingleOutfix(const RoseBuildImpl &tbi, u32 outfixEndQueue) {
+bool isSingleOutfix(const RoseBuildImpl &tbi) {
     for (auto v : vertices_range(tbi.g)) {
         if (tbi.isAnyStart(v)) {
             continue;
@@ -598,12 +603,12 @@ bool isSingleOutfix(const RoseBuildImpl &tbi, u32 outfixEndQueue) {
         return false; /* streaming runtime makes liberal use of broken flag */
     }
 
-    return outfixEndQueue == 1;
+    return tbi.outfixes.size() == 1;
 }
 
 static
 u8 pickRuntimeImpl(const RoseBuildImpl &build, const build_context &bc,
-                   u32 outfixEndQueue) {
+                   UNUSED u32 outfixEndQueue) {
     DEBUG_PRINTF("has_outfixes=%d\n", bc.resources.has_outfixes);
     DEBUG_PRINTF("has_suffixes=%d\n", bc.resources.has_suffixes);
     DEBUG_PRINTF("has_leftfixes=%d\n", bc.resources.has_leftfixes);
@@ -618,13 +623,38 @@ u8 pickRuntimeImpl(const RoseBuildImpl &build, const build_context &bc,
         return ROSE_RUNTIME_PURE_LITERAL;
     }
 
-    if (isSingleOutfix(build, outfixEndQueue)) {
+    if (isSingleOutfix(build)) {
         return ROSE_RUNTIME_SINGLE_OUTFIX;
     }
 
     return ROSE_RUNTIME_FULL_ROSE;
 }
 
+/**
+ * \brief True if this Rose engine needs to run MPV catch up in front of
+ * non-MPV reports.
+ */
+static
+bool needsMpvCatchup(const RoseBuildImpl &build) {
+    const auto &outfixes = build.outfixes;
+    bool has_mpv =
+        any_of(begin(outfixes), end(outfixes), [](const OutfixInfo &outfix) {
+            return outfix.is_nonempty_mpv();
+        });
+
+    if (!has_mpv) {
+        DEBUG_PRINTF("no mpv\n");
+        return false;
+    }
+
+    if (isSingleOutfix(build)) {
+        DEBUG_PRINTF("single outfix\n");
+        return false;
+    }
+
+    return true;
+}
+
 static
 void fillStateOffsets(const RoseBuildImpl &tbi, u32 rolesWithStateCount,
                       u32 anchorStateSize, u32 activeArrayCount,
@@ -1941,32 +1971,6 @@ struct DerivedBoundaryReports {
     set<ReportID> report_at_0_eod_full;
 };
 
-static
-void fillInReportInfo(RoseEngine *engine, u32 reportOffset,
-                      const ReportManager &rm, const vector<Report> &reports) {
-    internal_report *dest = (internal_report *)((char *)engine + reportOffset);
-    engine->intReportOffset = reportOffset;
-    engine->intReportCount = (u32)reports.size();
-
-    assert(ISALIGNED(dest));
-
-    for (const auto &report : reports) {
-        writeInternalReport(report, rm, dest++);
-    }
-
-    DEBUG_PRINTF("%zu reports of size %zu\n", reports.size(),
-                 sizeof(internal_report));
-}
-
-static
-bool hasSimpleReports(const vector<Report> &reports) {
-    auto it = find_if(reports.begin(), reports.end(), isComplexReport);
-
-    DEBUG_PRINTF("runtime has %scomplex reports\n",
-                 it == reports.end() ? "no " : "");
-    return it == reports.end();
-}
-
 static
 void prepSomRevNfas(const SomSlotManager &ssm, u32 *rev_nfa_table_offset,
                     vector<u32> *nfa_offsets, u32 *currOffset) {
@@ -2473,16 +2477,22 @@ void makeRoleAnchoredDelay(RoseBuildImpl &build, build_context &bc,
 }
 
 static
-void makeDedupe(const ReportID id, vector<RoseInstruction> &report_block) {
+void makeDedupe(const RoseBuildImpl &build, const Report &report,
+                vector<RoseInstruction> &report_block) {
     auto ri = RoseInstruction(ROSE_INSTR_DEDUPE, JumpTarget::NEXT_BLOCK);
-    ri.u.dedupe.report = id;
+    ri.u.dedupe.quash_som = report.quashSom;
+    ri.u.dedupe.dkey = build.rm.getDkey(report);
+    ri.u.dedupe.offset_adjust = report.offsetAdjust;
     report_block.push_back(move(ri));
 }
 
 static
-void makeDedupeSom(const ReportID id, vector<RoseInstruction> &report_block) {
+void makeDedupeSom(const RoseBuildImpl &build, const Report &report,
+                   vector<RoseInstruction> &report_block) {
     auto ri = RoseInstruction(ROSE_INSTR_DEDUPE_SOM, JumpTarget::NEXT_BLOCK);
-    ri.u.dedupeSom.report = id;
+    ri.u.dedupeSom.quash_som = report.quashSom;
+    ri.u.dedupeSom.dkey = build.rm.getDkey(report);
+    ri.u.dedupeSom.offset_adjust = report.offsetAdjust;
     report_block.push_back(move(ri));
 }
 
@@ -2510,6 +2520,92 @@ void makeCatchup(RoseBuildImpl &build, build_context &bc,
     program.emplace_back(ROSE_INSTR_CATCH_UP);
 }
 
+static
+void makeCatchupMpv(RoseBuildImpl &build, build_context &bc, ReportID id,
+                    vector<RoseInstruction> &program) {
+    if (!bc.needs_mpv_catchup) {
+        return;
+    }
+
+    const Report &report = build.rm.getReport(id);
+    if (report.type == INTERNAL_ROSE_CHAIN) {
+        return;
+    }
+
+    program.emplace_back(ROSE_INSTR_CATCH_UP_MPV);
+}
+
+static
+void writeSomOperation(const Report &report, som_operation *op) {
+    assert(op);
+
+    switch (report.type) {
+    case EXTERNAL_CALLBACK_SOM_REL:
+        op->type = SOM_EXTERNAL_CALLBACK_REL;
+        break;
+    case INTERNAL_SOM_LOC_SET:
+        op->type = SOM_INTERNAL_LOC_SET;
+        break;
+    case INTERNAL_SOM_LOC_SET_IF_UNSET:
+        op->type = SOM_INTERNAL_LOC_SET_IF_UNSET;
+        break;
+    case INTERNAL_SOM_LOC_SET_IF_WRITABLE:
+        op->type = SOM_INTERNAL_LOC_SET_IF_WRITABLE;
+        break;
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
+        op->type = SOM_INTERNAL_LOC_SET_REV_NFA;
+        break;
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
+        op->type = SOM_INTERNAL_LOC_SET_REV_NFA_IF_UNSET;
+        break;
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
+        op->type = SOM_INTERNAL_LOC_SET_REV_NFA_IF_WRITABLE;
+        break;
+    case INTERNAL_SOM_LOC_COPY:
+        op->type = SOM_INTERNAL_LOC_COPY;
+        break;
+    case INTERNAL_SOM_LOC_COPY_IF_WRITABLE:
+        op->type = SOM_INTERNAL_LOC_COPY_IF_WRITABLE;
+        break;
+    case INTERNAL_SOM_LOC_MAKE_WRITABLE:
+        op->type = SOM_INTERNAL_LOC_MAKE_WRITABLE;
+        break;
+    case EXTERNAL_CALLBACK_SOM_STORED:
+        op->type = SOM_EXTERNAL_CALLBACK_STORED;
+        break;
+    case EXTERNAL_CALLBACK_SOM_ABS:
+        op->type = SOM_EXTERNAL_CALLBACK_ABS;
+        break;
+    case EXTERNAL_CALLBACK_SOM_REV_NFA:
+        op->type = SOM_EXTERNAL_CALLBACK_REV_NFA;
+        break;
+    case INTERNAL_SOM_LOC_SET_FROM:
+        op->type = SOM_INTERNAL_LOC_SET_FROM;
+        break;
+    case INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE:
+        op->type = SOM_INTERNAL_LOC_SET_FROM_IF_WRITABLE;
+        break;
+    default:
+        // This report doesn't correspond to a SOM operation.
+        assert(0);
+        throw CompileError("Unable to generate bytecode.");
+    }
+
+    op->onmatch = report.onmatch;
+
+    switch (report.type) {
+    case EXTERNAL_CALLBACK_SOM_REV_NFA:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
+        op->aux.revNfaIndex = report.revNfaIndex;
+        break;
+    default:
+        op->aux.somDistance = report.somDistance;
+        break;
+    }
+}
+
 static
 void makeReport(RoseBuildImpl &build, const ReportID id,
                 const bool has_som, vector<RoseInstruction> &program) {
@@ -2536,10 +2632,12 @@ void makeReport(RoseBuildImpl &build, const ReportID id,
         report_block.push_back(move(ri));
     }
 
-    // External SOM reports need their SOM value calculated.
+    // External SOM reports that aren't passthrough need their SOM value
-    if (isExternalSomReport(report)) {
+    // calculated.
+    if (isExternalSomReport(report) &&
+        report.type != EXTERNAL_CALLBACK_SOM_PASS) {
         auto ri = RoseInstruction(ROSE_INSTR_SOM_FROM_REPORT);
-        ri.u.somFromReport.report = id;
+        writeSomOperation(report, &ri.u.somFromReport.som);
         report_block.push_back(move(ri));
     }
 
@@ -2567,33 +2665,40 @@ void makeReport(RoseBuildImpl &build, const ReportID id,
                 if (needs_dedupe) {
                     report_block.emplace_back(ROSE_INSTR_DEDUPE_AND_REPORT,
                                               JumpTarget::NEXT_BLOCK);
-                    report_block.back().u.dedupeAndReport.report = id;
+                    auto &ri = report_block.back();
+                    ri.u.dedupeAndReport.quash_som = report.quashSom;
+                    ri.u.dedupeAndReport.dkey = build.rm.getDkey(report);
+                    ri.u.dedupeAndReport.onmatch = report.onmatch;
+                    ri.u.dedupeAndReport.offset_adjust = report.offsetAdjust;
                 } else {
                     report_block.emplace_back(ROSE_INSTR_REPORT);
                     auto &ri = report_block.back();
-                    ri.u.report.report = id;
                     ri.u.report.onmatch = report.onmatch;
                     ri.u.report.offset_adjust = report.offsetAdjust;
                 }
             } else {
                 if (needs_dedupe) {
-                    makeDedupe(id, report_block);
+                    makeDedupe(build, report, report_block);
                 }
                 report_block.emplace_back(ROSE_INSTR_REPORT_EXHAUST);
                 auto &ri = report_block.back();
-                ri.u.reportExhaust.report = id;
                 ri.u.reportExhaust.onmatch = report.onmatch;
                 ri.u.reportExhaust.offset_adjust = report.offsetAdjust;
                 ri.u.reportExhaust.ekey = report.ekey;
             }
         } else { // has_som
-            makeDedupeSom(id, report_block);
+            makeDedupeSom(build, report, report_block);
             if (report.ekey == INVALID_EKEY) {
                 report_block.emplace_back(ROSE_INSTR_REPORT_SOM);
-                report_block.back().u.reportSom.report = id;
+                auto &ri = report_block.back();
+                ri.u.reportSom.onmatch = report.onmatch;
+                ri.u.reportSom.offset_adjust = report.offsetAdjust;
             } else {
                 report_block.emplace_back(ROSE_INSTR_REPORT_SOM_EXHAUST);
-                report_block.back().u.reportSomExhaust.report = id;
+                auto &ri = report_block.back();
+                ri.u.reportSomExhaust.onmatch = report.onmatch;
+                ri.u.reportSomExhaust.offset_adjust = report.offsetAdjust;
+                ri.u.reportSomExhaust.ekey = report.ekey;
             }
         }
         break;
@@ -2610,29 +2715,55 @@ void makeReport(RoseBuildImpl &build, const ReportID id,
     case INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE:
         if (has_som) {
             report_block.emplace_back(ROSE_INSTR_REPORT_SOM_AWARE);
-            report_block.back().u.reportSomAware.report = id;
+            auto &ri = report_block.back();
+            writeSomOperation(report, &ri.u.reportSomAware.som);
         } else {
             report_block.emplace_back(ROSE_INSTR_REPORT_SOM_INT);
-            report_block.back().u.reportSomInt.report = id;
+            auto &ri = report_block.back();
+            writeSomOperation(report, &ri.u.reportSomInt.som);
         }
         break;
-    case INTERNAL_ROSE_CHAIN:
+    case INTERNAL_ROSE_CHAIN: {
         report_block.emplace_back(ROSE_INSTR_REPORT_CHAIN);
-        report_block.back().u.reportChain.report = id;
+        auto &ri = report_block.back();
+        ri.u.reportChain.event = report.onmatch;
+        ri.u.reportChain.top_squash_distance = report.topSquashDistance;
         break;
+    }
     case EXTERNAL_CALLBACK_SOM_REL:
     case EXTERNAL_CALLBACK_SOM_STORED:
     case EXTERNAL_CALLBACK_SOM_ABS:
     case EXTERNAL_CALLBACK_SOM_REV_NFA:
-        makeDedupeSom(id, report_block);
+        makeDedupeSom(build, report, report_block);
         if (report.ekey == INVALID_EKEY) {
             report_block.emplace_back(ROSE_INSTR_REPORT_SOM);
-            report_block.back().u.reportSom.report = id;
+            auto &ri = report_block.back();
+            ri.u.reportSom.onmatch = report.onmatch;
+            ri.u.reportSom.offset_adjust = report.offsetAdjust;
         } else {
             report_block.emplace_back(ROSE_INSTR_REPORT_SOM_EXHAUST);
-            report_block.back().u.reportSomExhaust.report = id;
+            auto &ri = report_block.back();
+            ri.u.reportSomExhaust.onmatch = report.onmatch;
+            ri.u.reportSomExhaust.offset_adjust = report.offsetAdjust;
+            ri.u.reportSomExhaust.ekey = report.ekey;
         }
         break;
+    case EXTERNAL_CALLBACK_SOM_PASS:
+        makeDedupeSom(build, report, report_block);
+        if (report.ekey == INVALID_EKEY) {
+            report_block.emplace_back(ROSE_INSTR_REPORT_SOM);
+            auto &ri = report_block.back();
+            ri.u.reportSom.onmatch = report.onmatch;
+            ri.u.reportSom.offset_adjust = report.offsetAdjust;
+        } else {
+            report_block.emplace_back(ROSE_INSTR_REPORT_SOM_EXHAUST);
+            auto &ri = report_block.back();
+            ri.u.reportSomExhaust.onmatch = report.onmatch;
+            ri.u.reportSomExhaust.offset_adjust = report.offsetAdjust;
+            ri.u.reportSomExhaust.ekey = report.ekey;
+        }
+        break;
+
     default:
         assert(0);
         throw CompileError("Unable to generate bytecode.");
@@ -3571,6 +3702,26 @@ pair<u32, u32> buildLiteralPrograms(RoseBuildImpl &build, build_context &bc) {
     return {litProgramsOffset, delayRebuildProgramsOffset};
 }
 
+static
+u32 buildReportPrograms(RoseBuildImpl &build, build_context &bc) {
+    const auto &rm = build.rm;
+    const u32 numReports = verify_u32(rm.numReports());
+    vector<u32> programs(numReports);
+
+    vector<RoseInstruction> program;
+    for (ReportID id = 0; id < numReports; id++) {
+        program.clear();
+        const bool has_som = false;
+        makeCatchupMpv(build, bc, id, program);
+        makeReport(build, id, has_som, program);
+        programs[id] = writeProgram(bc, flattenProgram({program}));
+        DEBUG_PRINTF("program for report %u @ %u (%zu instructions)\n", id,
+                     programs.back(), program.size());
+    }
+
+    return add_to_engine_blob(bc, begin(programs), end(programs));
+}
+
 static
 vector<RoseInstruction> makeEodAnchorProgram(RoseBuildImpl &build,
                                              build_context &bc,
@@ -3787,6 +3938,7 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
     if (!anchored_dfas.empty()) {
         bc.resources.has_anchored = true;
     }
+    bc.needs_mpv_catchup = needsMpvCatchup(*this);
 
     auto boundary_out = makeBoundaryPrograms(*this, bc, boundary, dboundary);
 
@@ -3835,6 +3987,8 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
     u32 eodIterOffset;
     tie(eodIterProgramOffset, eodIterOffset) = buildEodAnchorProgram(*this, bc);
 
+    u32 reportProgramOffset = buildReportPrograms(*this, bc);
+
     vector<mmbit_sparse_iter> activeLeftIter;
     buildActiveLeftIter(leftInfoTable, activeLeftIter);
 
@@ -3900,12 +4054,6 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
         currOffset += verify_u32(sbsize);
     }
 
-    const vector<Report> &int_reports = rm.reports();
-
-    currOffset = ROUNDUP_CL(currOffset);
-    u32 intReportOffset = currOffset;
-    currOffset += sizeof(internal_report) * int_reports.size();
-
     u32 leftOffset = ROUNDUP_N(currOffset, alignof(LeftNfaInfo));
     u32 roseLen = sizeof(LeftNfaInfo) * leftInfoTable.size();
     currOffset = leftOffset + roseLen;
@@ -4004,14 +4152,13 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
     engine->somHorizon = ssm.somPrecision();
     engine->somLocationCount = ssm.numSomSlots();
 
-    engine->simpleCallback = !rm.numEkeys() && hasSimpleReports(rm.reports());
     engine->needsCatchup = bc.needs_catchup ? 1 : 0;
 
-    fillInReportInfo(engine.get(), intReportOffset, rm, int_reports);
-
     engine->literalCount = verify_u32(final_id_to_literal.size());
     engine->litProgramOffset = litProgramOffset;
     engine->litDelayRebuildProgramOffset = litDelayRebuildProgramOffset;
+    engine->reportProgramOffset = reportProgramOffset;
+    engine->reportProgramCount = verify_u32(rm.reports().size());
     engine->runtimeImpl = pickRuntimeImpl(*this, bc, outfixEndQueue);
     engine->mpvTriggeredByLeaf = anyEndfixMpvTriggers(*this);
 

src/rose/rose_build_merge.cpp

@@ -2833,7 +2833,7 @@ void mergePuffixes(RoseBuildImpl &tbi) {
         u32 squashDistance =
             allowedSquashDistance(repeat.reach, repeat.bounds.min, tbi, v);
 
-        Report ir = makeRoseTrigger(event, squashDistance);
+        Report ir = makeMpvTrigger(event, squashDistance);
         ReportID id = tbi.rm.getInternalId(ir);
 
         DEBUG_PRINTF("puffette event q%u t%u\n", queue, event);

src/rose/rose_dump.cpp

@@ -41,7 +41,6 @@
 #include "nfa/nfa_dump_api.h"
 #include "nfa/nfa_internal.h"
 #include "util/dump_charclass.h"
-#include "util/internal_report.h"
 #include "util/multibit_internal.h"
 #include "util/multibit.h"
 
@@ -192,20 +191,20 @@ void dumpJumpTable(ofstream &os, const RoseEngine *t,
 }
 
 static
-void dumpReport(ofstream &os, const RoseEngine *t, ReportID report) {
+void dumpSomOperation(ofstream &os, const som_operation &op) {
-    const auto *ir =
+    os << "    som (type=" << u32{op.type} << ", onmatch=" << op.onmatch;
-        (const internal_report *)loadFromByteCodeOffset(t, t->intReportOffset) +
+    switch (op.type) {
-        report;
+    case SOM_EXTERNAL_CALLBACK_REV_NFA:
-    os << "      type=" << u32{ir->type};
+    case SOM_INTERNAL_LOC_SET_REV_NFA:
-    os << ", onmatch=" << ir->onmatch;
+    case SOM_INTERNAL_LOC_SET_REV_NFA_IF_UNSET:
-    if (ir->ekey != INVALID_EKEY) {
+    case SOM_INTERNAL_LOC_SET_REV_NFA_IF_WRITABLE:
-        os << ", ekey=" << ir->ekey;
+        os << ", revNfaIndex=" << op.aux.revNfaIndex;
+        break;
+    default:
+        os << ", somDistance=" << op.aux.somDistance;
+        break;
     }
-    if (ir->dkey != MO_INVALID_IDX) {
+    os << ")" << endl;
-        os << ", dkey=" << ir->dkey;
-    }
-
-    os << endl;
 }
 
 static
@@ -314,6 +313,9 @@ void dumpProgram(ofstream &os, const RoseEngine *t, const char *pc) {
             PROGRAM_CASE(CATCH_UP) {}
             PROGRAM_NEXT_INSTRUCTION
 
+            PROGRAM_CASE(CATCH_UP_MPV) {}
+            PROGRAM_NEXT_INSTRUCTION
+
             PROGRAM_CASE(SOM_ADJUST) {
                 os << "    distance " << ri->distance << endl;
             }
@@ -326,8 +328,7 @@ void dumpProgram(ofstream &os, const RoseEngine *t, const char *pc) {
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(SOM_FROM_REPORT) {
-                os << "    report " << ri->report << endl;
+                dumpSomOperation(os, ri->som);
-                dumpReport(os, t, ri->report);
             }
             PROGRAM_NEXT_INSTRUCTION
 
@@ -348,64 +349,69 @@ void dumpProgram(ofstream &os, const RoseEngine *t, const char *pc) {
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(DEDUPE) {
-                os << "    report " << ri->report << endl;
+                os << "    quash_som " << u32{ri->quash_som} << endl;
-                dumpReport(os, t, ri->report);
+                os << "    dkey " << ri->dkey << endl;
+                os << "    offset_adjust " << ri->offset_adjust << endl;
                 os << "    fail_jump " << offset + ri->fail_jump << endl;
             }
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(DEDUPE_SOM) {
-                os << "    report " << ri->report << endl;
+                os << "    quash_som " << u32{ri->quash_som} << endl;
-                dumpReport(os, t, ri->report);
+                os << "    dkey " << ri->dkey << endl;
+                os << "    offset_adjust " << ri->offset_adjust << endl;
                 os << "    fail_jump " << offset + ri->fail_jump << endl;
             }
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT_CHAIN) {
-                os << "    report " << ri->report << endl;
+                os << "    event " << ri->event << endl;
-                dumpReport(os, t, ri->report);
+                os << "    top_squash_distance " << ri->top_squash_distance
+                   << endl;
             }
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT_SOM_INT) {
-                os << "    report " << ri->report << endl;
+                dumpSomOperation(os, ri->som);
-                dumpReport(os, t, ri->report);
             }
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT_SOM_AWARE) {
-                os << "    report " << ri->report << endl;
+                dumpSomOperation(os, ri->som);
-                dumpReport(os, t, ri->report);
             }
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT) {
-                os << "    report " << ri->report << endl;
+                os << "    onmatch " << ri->onmatch << endl;
-                dumpReport(os, t, ri->report);
+                os << "    offset_adjust " << ri->offset_adjust << endl;
             }
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT_EXHAUST) {
-                os << "    report " << ri->report << endl;
+                os << "    onmatch " << ri->onmatch << endl;
-                dumpReport(os, t, ri->report);
+                os << "    offset_adjust " << ri->offset_adjust << endl;
+                os << "    ekey " << ri->ekey << endl;
             }
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT_SOM) {
-                os << "    report " << ri->report << endl;
+                os << "    onmatch " << ri->onmatch << endl;
-                dumpReport(os, t, ri->report);
+                os << "    offset_adjust " << ri->offset_adjust << endl;
             }
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(REPORT_SOM_EXHAUST) {
-                os << "    report " << ri->report << endl;
+                os << "    onmatch " << ri->onmatch << endl;
-                dumpReport(os, t, ri->report);
+                os << "    offset_adjust " << ri->offset_adjust << endl;
+                os << "    ekey " << ri->ekey << endl;
             }
             PROGRAM_NEXT_INSTRUCTION
 
             PROGRAM_CASE(DEDUPE_AND_REPORT) {
-                os << "    report " << ri->report << endl;
+                os << "    quash_som " << u32{ri->quash_som} << endl;
-                dumpReport(os, t, ri->report);
+                os << "    dkey " << ri->dkey << endl;
+                os << "    onmatch " << ri->onmatch << endl;
+                os << "    offset_adjust " << ri->offset_adjust << endl;
                 os << "    fail_jump " << offset + ri->fail_jump << endl;
             }
             PROGRAM_NEXT_INSTRUCTION
@@ -537,6 +543,30 @@ void dumpRoseEodPrograms(const RoseEngine *t, const string &filename) {
     os.close();
 }
 
+static
+void dumpRoseReportPrograms(const RoseEngine *t, const string &filename) {
+    ofstream os(filename);
+
+    const u32 *programs =
+        (const u32 *)loadFromByteCodeOffset(t, t->reportProgramOffset);
+
+    for (u32 i = 0; i < t->reportProgramCount; i++) {
+        os << "Report " << i << endl;
+        os << "---------------" << endl;
+
+        if (programs[i]) {
+            os << "Program @ " << programs[i] << ":" << endl;
+            const char *prog =
+                (const char *)loadFromByteCodeOffset(t, programs[i]);
+            dumpProgram(os, t, prog);
+        } else {
+            os << "<No Program>" << endl;
+        }
+    }
+
+    os.close();
+}
+
 static
 void dumpNfaNotes(ofstream &fout, const RoseEngine *t, const NFA *n) {
     const u32 qindex = n->queueIndex;
@@ -834,9 +864,6 @@ void roseDumpText(const RoseEngine *t, FILE *f) {
     if (t->hasSom) {
         fprintf(f, " hasSom");
     }
-    if (t->simpleCallback) {
-        fprintf(f, " simpleCallback");
-    }
     fprintf(f, "\n");
 
     fprintf(f, "dkey count           : %u\n", t->dkeyCount);
@@ -949,7 +976,6 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
     DUMP_U8(t, canExhaust);
     DUMP_U8(t, hasSom);
     DUMP_U8(t, somHorizon);
-    DUMP_U8(t, simpleCallback);
     DUMP_U8(t, needsCatchup);
     DUMP_U32(t, mode);
     DUMP_U32(t, historyRequired);
@@ -972,10 +998,10 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
     DUMP_U32(t, eodmatcherMinWidth);
     DUMP_U32(t, amatcherMaxBiAnchoredWidth);
     DUMP_U32(t, fmatcherMaxBiAnchoredWidth);
-    DUMP_U32(t, intReportOffset);
-    DUMP_U32(t, intReportCount);
     DUMP_U32(t, litProgramOffset);
     DUMP_U32(t, litDelayRebuildProgramOffset);
+    DUMP_U32(t, reportProgramOffset);
+    DUMP_U32(t, reportProgramCount);
     DUMP_U32(t, literalCount);
     DUMP_U32(t, activeArrayCount);
     DUMP_U32(t, activeLeftCount);
@@ -1051,6 +1077,7 @@ void roseDumpComponents(const RoseEngine *t, bool dump_raw,
     dumpRevNfas(t, dump_raw, base);
     dumpRoseLitPrograms(t, base + "/rose_lit_programs.txt");
     dumpRoseEodPrograms(t, base + "/rose_eod_programs.txt");
+    dumpRoseReportPrograms(t, base + "/rose_report_programs.txt");
 }
 
 void roseDumpInternals(const RoseEngine *t, const string &base) {

src/rose/rose_internal.h

@@ -305,8 +305,6 @@ struct RoseEngine {
     u8  hasSom; /**< has at least one pattern which tracks SOM. */
     u8  somHorizon; /**< width in bytes of SOM offset storage (governed by
                         SOM precision) */
-    u8  simpleCallback; /**< has only external reports with no bounds checks,
-                             plus no exhaustion keys */
     u8 needsCatchup; /** catch up needs to be run on every report. */
     u32 mode; /**< scanning mode, one of HS_MODE_{BLOCK,STREAM,VECTORED} */
     u32 historyRequired; /**< max amount of history required for streaming */
@@ -343,9 +341,6 @@ struct RoseEngine {
     u32 fmatcherMaxBiAnchoredWidth; /**< maximum number of bytes that can still
                                      * produce a match for a pattern involved
                                      * with the anchored table. */
-    u32 intReportOffset; /**< offset of array of internal_report structures */
-    u32 intReportCount; /**< number of internal_report structures */
-
     /** \brief Offset of u32 array of program offsets for literals. */
     u32 litProgramOffset;
 
@@ -353,6 +348,12 @@ struct RoseEngine {
      * literals. */
     u32 litDelayRebuildProgramOffset;
 
+    /** \brief Offset of u32 array of program offsets for internal reports. */
+    u32 reportProgramOffset;
+
+    /** \brief Number of programs for internal reports. */
+    u32 reportProgramCount;
+
     /**
      * \brief Number of entries in the arrays pointed to by litProgramOffset,
      * litDelayRebuildProgramOffset.

src/rose/rose_program.h

@@ -33,6 +33,7 @@
 #ifndef ROSE_ROSE_PROGRAM_H
 #define ROSE_ROSE_PROGRAM_H
 
+#include "som/som_operation.h"
 #include "rose_internal.h"
 #include "ue2common.h"
 
@@ -53,9 +54,10 @@ enum RoseInstructionCode {
     ROSE_INSTR_CHECK_PREFIX,      //!< Prefix engine must be in accept state.
     ROSE_INSTR_PUSH_DELAYED,      //!< Push delayed literal matches.
     ROSE_INSTR_CATCH_UP,          //!< Catch up engines, anchored matches.
+    ROSE_INSTR_CATCH_UP_MPV,      //!< Catch up the MPV.
     ROSE_INSTR_SOM_ADJUST,        //!< Set SOM from a distance to EOM.
     ROSE_INSTR_SOM_LEFTFIX,       //!< Acquire SOM from a leftfix engine.
-    ROSE_INSTR_SOM_FROM_REPORT,   //!< Acquire SOM from an internal_report.
+    ROSE_INSTR_SOM_FROM_REPORT,   //!< Acquire SOM from a som_operation.
     ROSE_INSTR_SOM_ZERO,          //!< Set SOM to zero.
     ROSE_INSTR_TRIGGER_INFIX,     //!< Trigger an infix engine.
     ROSE_INSTR_TRIGGER_SUFFIX,    //!< Trigger a suffix engine.
@@ -171,6 +173,10 @@ struct ROSE_STRUCT_CATCH_UP {
     u8 code; //!< From enum RoseInstructionCode.
 };
 
+struct ROSE_STRUCT_CATCH_UP_MPV {
+    u8 code; //!< From enum RoseInstructionCode.
+};
+
 struct ROSE_STRUCT_SOM_ADJUST {
     u8 code; //!< From enum RoseInstructionCode.
     u32 distance; //!< Distance to EOM.
@@ -184,7 +190,7 @@ struct ROSE_STRUCT_SOM_LEFTFIX {
 
 struct ROSE_STRUCT_SOM_FROM_REPORT {
     u8 code; //!< From enum RoseInstructionCode.
-    ReportID report; //!< EXTERNAL_CALLBACK_SOM_* report to use.
+    struct som_operation som;
 };
 
 struct ROSE_STRUCT_SOM_ZERO {
@@ -206,41 +212,49 @@ struct ROSE_STRUCT_TRIGGER_SUFFIX {
 
 struct ROSE_STRUCT_DEDUPE {
     u8 code; //!< From enum RoseInstructionCode.
-    ReportID report;
+    u8 quash_som; //!< Force SOM to zero for this report.
+    u32 dkey; //!< Dedupe key.
+    s32 offset_adjust; //!< Offset adjustment to apply to end offset.
     u32 fail_jump; //!< Jump forward this many bytes on failure.
 };
 
 struct ROSE_STRUCT_DEDUPE_SOM {
     u8 code; //!< From enum RoseInstructionCode.
-    ReportID report;
+    u8 quash_som; //!< Force SOM to zero for this report.
+    u32 dkey; //!< Dedupe key.
+    s32 offset_adjust; //!< Offset adjustment to apply to end offset.
     u32 fail_jump; //!< Jump forward this many bytes on failure.
 };
 
 struct ROSE_STRUCT_REPORT_CHAIN {
     u8 code; //!< From enum RoseInstructionCode.
-    ReportID report;
+    u32 event; //!< Queue event, from MQE_*. Must be a top.
+
+    /**
+     * \brief Number of bytes behind us that we are allowed to squash
+     * identical top events on the queue.
+     */
+    u64a top_squash_distance;
 };
 
 struct ROSE_STRUCT_REPORT_SOM_INT {
     u8 code; //!< From enum RoseInstructionCode.
-    ReportID report;
+    struct som_operation som;
 };
 
 struct ROSE_STRUCT_REPORT_SOM_AWARE {
     u8 code; //!< From enum RoseInstructionCode.
-    ReportID report;
+    struct som_operation som;
 };
 
 struct ROSE_STRUCT_REPORT {
     u8 code; //!< From enum RoseInstructionCode.
-    ReportID report; //!< Internal report ID (used for assertions).
     ReportID onmatch; //!< Report ID to deliver to user.
     s32 offset_adjust; //!< Offset adjustment to apply to end offset.
 };
 
 struct ROSE_STRUCT_REPORT_EXHAUST {
     u8 code; //!< From enum RoseInstructionCode.
-    ReportID report; //!< Internal report ID (used for assertions).
     ReportID onmatch; //!< Report ID to deliver to user.
     s32 offset_adjust; //!< Offset adjustment to apply to end offset.
     u32 ekey; //!< Exhaustion key.
@@ -248,22 +262,23 @@ struct ROSE_STRUCT_REPORT_EXHAUST {
 
 struct ROSE_STRUCT_REPORT_SOM {
     u8 code; //!< From enum RoseInstructionCode.
-    ReportID report;
+    ReportID onmatch; //!< Report ID to deliver to user.
+    s32 offset_adjust; //!< Offset adjustment to apply to end offset.
 };
 
 struct ROSE_STRUCT_REPORT_SOM_EXHAUST {
     u8 code; //!< From enum RoseInstructionCode.
-    ReportID report;
+    ReportID onmatch; //!< Report ID to deliver to user.
-};
+    s32 offset_adjust; //!< Offset adjustment to apply to end offset.
-
+    u32 ekey; //!< Exhaustion key.
-struct ROSE_STRUCT_REPORT_SOM_EXT {
-    u8 code; //!< From enum RoseInstructionCode.
-    ReportID report;
 };
 
 struct ROSE_STRUCT_DEDUPE_AND_REPORT {
     u8 code; //!< From enum RoseInstructionCode.
-    ReportID report;
+    u8 quash_som; //!< Force SOM to zero for this report.
+    u32 dkey; //!< Dedupe key.
+    ReportID onmatch; //!< Report ID to deliver to user.
+    s32 offset_adjust; //!< Offset adjustment to apply to end offset.
     u32 fail_jump; //!< Jump forward this many bytes on failure.
 };
 

src/rose/runtime.h

@@ -33,10 +33,9 @@
 #ifndef ROSE_RUNTIME_H
 #define ROSE_RUNTIME_H
 
-#include "scratch.h"
 #include "rose_internal.h"
+#include "scratch.h"
 #include "util/exhaust.h" // for isExhausted
-#include "util/internal_report.h"
 #include "util/partial_store.h"
 
 /*
@@ -147,16 +146,6 @@ u32 has_chained_nfas(const struct RoseEngine *t) {
     return t->outfixBeginQueue;
 }
 
-/** \brief Fetch \ref internal_report structure for this internal ID. */
-static really_inline
-const struct internal_report *getInternalReport(const struct RoseEngine *t,
-                                                ReportID intId) {
-    const struct internal_report *reports =
-        (const struct internal_report *)((const u8 *)t + t->intReportOffset);
-    assert(intId < t->intReportCount);
-    return reports + intId;
-}
-
 static really_inline
 void updateLastMatchOffset(struct RoseContext *tctxt, u64a offset) {
     DEBUG_PRINTF("match @%llu, last match @%llu\n", offset,

src/rose/stream.c

@@ -422,8 +422,7 @@ void do_rebuild(const struct RoseEngine *t, const struct HWLM *ftable,
     assert(!can_stop_matching(scratch));
 }
 
-void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch,
+void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch) {
-                    RoseCallback callback, RoseCallbackSom som_callback) {
     DEBUG_PRINTF("OH HAI\n");
     assert(t);
     assert(scratch->core_info.hbuf);
@@ -457,8 +456,6 @@ void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch,
     tctxt->delayLastEndOffset = offset;
     tctxt->lastEndOffset = offset;
     tctxt->filledDelayedSlots = 0;
-    tctxt->cb = callback;
-    tctxt->cb_som = som_callback;
     tctxt->lastMatchOffset = 0;
     tctxt->minMatchOffset = offset;
     tctxt->minNonMpvMatchOffset = offset;

src/runtime.c

@@ -139,6 +139,12 @@ void populateCoreInfo(struct hs_scratch *s, const struct RoseEngine *rose,
     s->som_set_now_offset = ~0ULL;
     s->deduper.current_report_offset = ~0ULL;
     s->deduper.som_log_dirty = 1; /* som logs have not been cleared */
+
+    // Rose program execution (used for some report paths) depends on these
+    // values being initialised.
+    s->tctxt.lastMatchOffset = 0;
+    s->tctxt.minMatchOffset = offset;
+    s->tctxt.minNonMpvMatchOffset = offset;
 }
 
 #define STATUS_VALID_BITS                                                      \
@@ -159,107 +165,6 @@ void setStreamStatus(char *state, u8 status) {
     *(u8 *)state = status;
 }
 
-static
-int roseAdaptor(u64a offset, ReportID id, struct hs_scratch *scratch) {
-    return roseAdaptor_i(offset, id, scratch, 0, 0);
-}
-
-static
-int roseSimpleAdaptor(u64a offset, ReportID id, struct hs_scratch *scratch) {
-    return roseAdaptor_i(offset, id, scratch, 1, 0);
-}
-
-static
-int roseSomAdaptor(u64a offset, ReportID id, struct hs_scratch *scratch) {
-    return roseAdaptor_i(offset, id, scratch, 0, 1);
-}
-
-static
-int roseSimpleSomAdaptor(u64a offset, ReportID id, struct hs_scratch *scratch) {
-    return roseAdaptor_i(offset, id, scratch, 1, 1);
-}
-
-static really_inline
-RoseCallback selectAdaptor(const struct RoseEngine *rose) {
-    const char is_simple = rose->simpleCallback;
-    const char do_som = rose->hasSom;
-
-    if (do_som) {
-        return is_simple ? roseSimpleSomAdaptor : roseSomAdaptor;
-    } else {
-        return is_simple ? roseSimpleAdaptor : roseAdaptor;
-    }
-}
-
-static
-int roseSomSomAdaptor(u64a from_offset, u64a to_offset, ReportID id,
-                      struct hs_scratch *scratch) {
-    return roseSomAdaptor_i(from_offset, to_offset, id, scratch, 0);
-}
-
-static
-int roseSimpleSomSomAdaptor(u64a from_offset, u64a to_offset, ReportID id,
-                            struct hs_scratch *scratch) {
-    return roseSomAdaptor_i(from_offset, to_offset, id, scratch, 1);
-}
-
-static really_inline
-RoseCallbackSom selectSomAdaptor(const struct RoseEngine *rose) {
-    const char is_simple = rose->simpleCallback;
-
-    return is_simple ? roseSimpleSomSomAdaptor : roseSomSomAdaptor;
-}
-
-static
-int outfixSimpleSomAdaptor(u64a offset, ReportID id, void *context) {
-    return roseAdaptor_i(offset, id, context, 1, 1);
-}
-
-static
-int outfixSimpleAdaptor(u64a offset, ReportID id, void *context) {
-    return roseAdaptor_i(offset, id, context, 1, 0);
-}
-
-static
-int outfixSomAdaptor(u64a offset, ReportID id, void *context) {
-    return roseAdaptor_i(offset, id, context, 0, 1);
-}
-
-static
-int outfixAdaptor(u64a offset, ReportID id, void *context) {
-    return roseAdaptor_i(offset, id, context, 0, 0);
-}
-
-static really_inline
-NfaCallback selectOutfixAdaptor(const struct RoseEngine *rose) {
-    const char is_simple = rose->simpleCallback;
-    const char do_som = rose->hasSom;
-
-    if (do_som) {
-        return is_simple ? outfixSimpleSomAdaptor : outfixSomAdaptor;
-    } else {
-        return is_simple ? outfixSimpleAdaptor : outfixAdaptor;
-    }
-}
-
-static
-int outfixSimpleSomSomAdaptor(u64a from_offset, u64a to_offset, ReportID id,
-                              void *context) {
-    return roseSomAdaptor_i(from_offset, to_offset, id, context, 1);
-}
-
-static
-int outfixSomSomAdaptor(u64a from_offset, u64a to_offset, ReportID id,
-                        void *context) {
-    return roseSomAdaptor_i(from_offset, to_offset, id, context, 0);
-}
-
-static really_inline
-SomNfaCallback selectOutfixSomAdaptor(const struct RoseEngine *rose) {
-    const char is_simple = rose->simpleCallback;
-    return is_simple ? outfixSimpleSomSomAdaptor : outfixSomSomAdaptor;
-}
-
 /** \brief Initialise SOM state. Used in both block and streaming mode. */
 static really_inline
 void initSomState(const struct RoseEngine *rose, char *state) {
@@ -278,8 +183,7 @@ void rawBlockExec(const struct RoseEngine *rose, struct hs_scratch *scratch) {
 
     DEBUG_PRINTF("blockmode scan len=%zu\n", scratch->core_info.len);
 
-    roseBlockExec(rose, scratch, selectAdaptor(rose),
+    roseBlockExec(rose, scratch);
-                  selectSomAdaptor(rose));
 }
 
 static really_inline
@@ -312,8 +216,8 @@ void initOutfixQueue(struct mq *q, u32 qi, const struct RoseEngine *t,
     q->length = scratch->core_info.len;
     q->history = scratch->core_info.hbuf;
     q->hlength = scratch->core_info.hlen;
-    q->cb = selectOutfixAdaptor(t);
+    q->cb = roseReportAdaptor;
-    q->som_cb = selectOutfixSomAdaptor(t);
+    q->som_cb = roseReportSomAdaptor;
     q->context = scratch;
     q->report_current = 0;
 
@@ -376,18 +280,16 @@ void runSmallWriteEngine(const struct SmallWriteEngine *smwr,
 
     const struct NFA *nfa = getSmwrNfa(smwr);
 
-    const struct RoseEngine *rose = scratch->core_info.rose;
-
     size_t local_alen = length - smwr->start_offset;
     const u8 *local_buffer = buffer + smwr->start_offset;
 
     assert(isMcClellanType(nfa->type));
     if (nfa->type == MCCLELLAN_NFA_8) {
         nfaExecMcClellan8_B(nfa, smwr->start_offset, local_buffer,
-                            local_alen, selectOutfixAdaptor(rose), scratch);
+                            local_alen, roseReportAdaptor, scratch);
     } else {
         nfaExecMcClellan16_B(nfa, smwr->start_offset, local_buffer,
-                             local_alen, selectOutfixAdaptor(rose), scratch);
+                             local_alen, roseReportAdaptor, scratch);
     }
 }
 
@@ -430,11 +332,6 @@ hs_error_t hs_scan(const hs_database_t *db, const char *data, unsigned length,
 
     clearEvec(rose, scratch->core_info.exhaustionVector);
 
-    // Rose program execution (used for some report paths) depends on these
-    // values being initialised.
-    scratch->tctxt.lastMatchOffset = 0;
-    scratch->tctxt.minMatchOffset = 0;
-
     if (!length) {
         if (rose->boundary.reportZeroEodOffset) {
             roseRunBoundaryProgram(rose, rose->boundary.reportZeroEodOffset, 0,
@@ -617,8 +514,7 @@ void rawEodExec(hs_stream_t *id, hs_scratch_t *scratch) {
         return;
     }
 
-    roseEodExec(rose, id->offset, scratch, selectAdaptor(rose),
+    roseEodExec(rose, id->offset, scratch);
-                selectSomAdaptor(rose));
 }
 
 static never_inline
@@ -676,11 +572,6 @@ void report_eod_matches(hs_stream_t *id, hs_scratch_t *scratch,
                      getHistory(state, rose, id->offset),
                      getHistoryAmount(rose, id->offset), id->offset, status, 0);
 
-    // Rose program execution (used for some report paths) depends on these
-    // values being initialised.
-    scratch->tctxt.lastMatchOffset = 0;
-    scratch->tctxt.minMatchOffset = id->offset;
-
     if (rose->somLocationCount) {
         loadSomFromStream(scratch, id->offset);
     }
@@ -797,7 +688,7 @@ void rawStreamExec(struct hs_stream *stream_state, struct hs_scratch *scratch) {
 
     const struct RoseEngine *rose = stream_state->rose;
     assert(rose);
-    roseStreamExec(rose, scratch, selectAdaptor(rose), selectSomAdaptor(rose));
+    roseStreamExec(rose, scratch);
 
     if (!told_to_stop_matching(scratch) &&
         isAllExhausted(rose, scratch->core_info.exhaustionVector)) {
@@ -917,11 +808,6 @@ hs_error_t hs_scan_stream_internal(hs_stream_t *id, const char *data,
     assert(scratch->core_info.hlen <= id->offset
            && scratch->core_info.hlen <= rose->historyRequired);
 
-    // Rose program execution (used for some report paths) depends on these
-    // values being initialised.
-    scratch->tctxt.lastMatchOffset = 0;
-    scratch->tctxt.minMatchOffset = id->offset;
-
     prefetch_data(data, length);
 
     if (rose->somLocationCount) {

src/scratch.h

@@ -119,8 +119,6 @@ struct RoseContext {
                                 * still allowed to report */
     u64a next_mpv_offset; /**< earliest offset that the MPV can next report a
                            * match, cleared if top events arrive */
-    RoseCallback cb;
-    RoseCallbackSom cb_som;
     u32 filledDelayedSlots;
     u32 curr_qi;    /**< currently executing main queue index during
                      * \ref nfaQueueExec */

src/som/som_operation.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2016, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ * \brief SOM runtime: data structures.
+ *
+ * Data structures used for SOM operations.
+ */
+
+#ifndef SOM_OPERATION_H
+#define SOM_OPERATION_H
+
+#include "ue2common.h"
+
+#define SOM_EXTERNAL_CALLBACK_REL                     1
+#define SOM_INTERNAL_LOC_SET                          2
+#define SOM_INTERNAL_LOC_SET_IF_UNSET                 3
+#define SOM_INTERNAL_LOC_SET_IF_WRITABLE              4
+#define SOM_INTERNAL_LOC_SET_REV_NFA                  5
+#define SOM_INTERNAL_LOC_SET_REV_NFA_IF_UNSET         6
+#define SOM_INTERNAL_LOC_SET_REV_NFA_IF_WRITABLE      7
+#define SOM_INTERNAL_LOC_COPY                         8
+#define SOM_INTERNAL_LOC_COPY_IF_WRITABLE             9
+#define SOM_INTERNAL_LOC_MAKE_WRITABLE               10
+#define SOM_EXTERNAL_CALLBACK_STORED                 11
+#define SOM_EXTERNAL_CALLBACK_ABS                    12
+#define SOM_EXTERNAL_CALLBACK_REV_NFA                13
+#define SOM_INTERNAL_LOC_SET_FROM                    14
+#define SOM_INTERNAL_LOC_SET_FROM_IF_WRITABLE        15
+
+struct som_operation {
+    /** \brief Report type, from the definitions above. */
+    u8 type;
+
+    /* \brief SOM loc to modify. */
+    u32 onmatch;
+
+    union {
+        /** \brief SOM distance value, use varies according to type.
+         *
+         *  - for SOM_EXTERNAL_CALLBACK_REL, from-offset is this many bytes
+         *    before the to-offset.
+         *  - for SOM_EXTERNAL_CALLBACK_ABS, set from-offset to this value.
+         *  - for SOM_INTERNAL_LOC_COPY*, som location read_from.
+         */
+        u64a somDistance;
+
+        /** \brief Index of the reverse nfa.
+         *
+         * Used by SOM_EXTERNAL_CALLBACK_REV_NFA and
+         * SOM_INTERNAL_LOC_SET_REV_NFA*.
+         */
+        u64a revNfaIndex;
+    } aux;
+};
+
+#endif // SOM_OPERATION_H
+

src/som/som_runtime.c

@@ -39,6 +39,7 @@
  */
 
 #include "hs_internal.h"
+#include "som_operation.h"
 #include "som_runtime.h"
 #include "scratch.h"
 #include "ue2common.h"
@@ -47,11 +48,10 @@
 #include "nfa/nfa_internal.h"
 #include "util/fatbit.h"
 #include "util/multibit.h"
-#include "util/internal_report.h"
 
 static really_inline
 void setSomLoc(struct fatbit *som_set_now, u64a *som_store, u32 som_store_count,
-               const struct internal_report *ri, u64a to_offset) {
+               const struct som_operation *ri, u64a to_offset) {
     /* validity handled by callers */
     assert(to_offset >= ri->aux.somDistance);
     u64a start_offset = to_offset - ri->aux.somDistance;
@@ -112,7 +112,7 @@ const struct NFA *getSomRevNFA(const struct RoseEngine *t, u32 i) {
 }
 
 static
-void runRevNfa(struct hs_scratch *scratch, const struct internal_report *ri,
+void runRevNfa(struct hs_scratch *scratch, const struct som_operation *ri,
                const u64a to_offset, u64a *from_offset) {
     struct core_info *ci = &scratch->core_info;
 
@@ -159,7 +159,7 @@ void runRevNfa(struct hs_scratch *scratch, const struct internal_report *ri,
 static really_inline
 void setSomLocRevNfa(struct hs_scratch *scratch, struct fatbit *som_set_now,
                      u64a *som_store, u32 som_store_count,
-                     const struct internal_report *ri, u64a to_offset) {
+                     const struct som_operation *ri, u64a to_offset) {
     /* validity handled by callers */
     u64a from_offset = 0;
     runRevNfa(scratch, ri, to_offset, &from_offset);
@@ -178,7 +178,7 @@ void setSomLocRevNfa(struct hs_scratch *scratch, struct fatbit *som_set_now,
 }
 
 void handleSomInternal(struct hs_scratch *scratch,
-                       const struct internal_report *ri, const u64a to_offset) {
+                       const struct som_operation *ri, const u64a to_offset) {
     assert(scratch);
     assert(ri);
     DEBUG_PRINTF("-->som action required at %llu\n", to_offset);
@@ -209,21 +209,21 @@ void handleSomInternal(struct hs_scratch *scratch,
     }
 
     switch (ri->type) {
-    case INTERNAL_SOM_LOC_SET:
+    case SOM_INTERNAL_LOC_SET:
-        DEBUG_PRINTF("INTERNAL_SOM_LOC_SET\n");
+        DEBUG_PRINTF("SOM_INTERNAL_LOC_SET\n");
         mmbit_set(som_store_valid, som_store_count, ri->onmatch);
         setSomLoc(som_set_now, som_store, som_store_count, ri, to_offset);
         return;
-    case INTERNAL_SOM_LOC_SET_IF_UNSET:
+    case SOM_INTERNAL_LOC_SET_IF_UNSET:
-        DEBUG_PRINTF("INTERNAL_SOM_LOC_SET_IF_UNSET\n");
+        DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_IF_UNSET\n");
         if (ok_and_mark_if_unset(som_store_valid, som_set_now, som_store_count,
                                  ri->onmatch)) {
             setSomLoc(som_set_now, som_store, som_store_count, ri, to_offset);
         }
         return;
-    case INTERNAL_SOM_LOC_SET_IF_WRITABLE: {
+    case SOM_INTERNAL_LOC_SET_IF_WRITABLE: {
         u32 slot = ri->onmatch;
-        DEBUG_PRINTF("INTERNAL_SOM_LOC_SET_IF_WRITABLE\n");
+        DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_IF_WRITABLE\n");
         if (ok_and_mark_if_write(som_store_valid, som_set_now,
                                  som_store_writable, som_store_count, slot)) {
             setSomLoc(som_set_now, som_store, som_store_count, ri, to_offset);
@@ -245,23 +245,23 @@ void handleSomInternal(struct hs_scratch *scratch,
         }
         return;
     }
-    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
+    case SOM_INTERNAL_LOC_SET_REV_NFA:
-        DEBUG_PRINTF("INTERNAL_SOM_LOC_SET_SOM_REV_NFA\n");
+        DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_REV_NFA\n");
         mmbit_set(som_store_valid, som_store_count, ri->onmatch);
         setSomLocRevNfa(scratch, som_set_now, som_store, som_store_count, ri,
                         to_offset);
         return;
-    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
+    case SOM_INTERNAL_LOC_SET_REV_NFA_IF_UNSET:
-        DEBUG_PRINTF("INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET\n");
+        DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_REV_NFA_IF_UNSET\n");
         if (ok_and_mark_if_unset(som_store_valid, som_set_now, som_store_count,
                                  ri->onmatch)) {
             setSomLocRevNfa(scratch, som_set_now, som_store, som_store_count,
                             ri, to_offset);
         }
         return;
-    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE: {
+    case SOM_INTERNAL_LOC_SET_REV_NFA_IF_WRITABLE: {
         u32 slot = ri->onmatch;
-        DEBUG_PRINTF("INTERNAL_SOM_LOC_SET_IF_WRITABLE\n");
+        DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_IF_WRITABLE\n");
         if (ok_and_mark_if_write(som_store_valid, som_set_now,
                                  som_store_writable, som_store_count, slot)) {
             setSomLocRevNfa(scratch, som_set_now, som_store, som_store_count,
@@ -285,10 +285,10 @@ void handleSomInternal(struct hs_scratch *scratch,
         }
         return;
     }
-    case INTERNAL_SOM_LOC_COPY: {
+    case SOM_INTERNAL_LOC_COPY: {
         u32 slot_in = ri->aux.somDistance;
         u32 slot_out = ri->onmatch;
-        DEBUG_PRINTF("INTERNAL_SOM_LOC_COPY S[%u] = S[%u]\n", slot_out,
+        DEBUG_PRINTF("SOM_INTERNAL_LOC_COPY S[%u] = S[%u]\n", slot_out,
                      slot_in);
         assert(mmbit_isset(som_store_valid, som_store_count, slot_in));
         mmbit_set(som_store_valid, som_store_count, slot_out);
@@ -297,10 +297,10 @@ void handleSomInternal(struct hs_scratch *scratch,
 
         return;
     }
-    case INTERNAL_SOM_LOC_COPY_IF_WRITABLE: {
+    case SOM_INTERNAL_LOC_COPY_IF_WRITABLE: {
         u32 slot_in = ri->aux.somDistance;
         u32 slot_out = ri->onmatch;
-        DEBUG_PRINTF("INTERNAL_SOM_LOC_COPY_IF_WRITABLE S[%u] = S[%u]\n",
+        DEBUG_PRINTF("SOM_INTERNAL_LOC_COPY_IF_WRITABLE S[%u] = S[%u]\n",
                      slot_out, slot_in);
         assert(mmbit_isset(som_store_valid, som_store_count, slot_in));
         if (ok_and_mark_if_write(som_store_valid, som_set_now,
@@ -322,9 +322,9 @@ void handleSomInternal(struct hs_scratch *scratch,
         }
         return;
     }
-    case INTERNAL_SOM_LOC_MAKE_WRITABLE: {
+    case SOM_INTERNAL_LOC_MAKE_WRITABLE: {
         u32 slot = ri->onmatch;
-        DEBUG_PRINTF("INTERNAL_SOM_LOC_MAKE_WRITABLE\n");
+        DEBUG_PRINTF("SOM_INTERNAL_LOC_MAKE_WRITABLE\n");
         /* if just written to the loc, ignore the racing escape */
         if (fatbit_isset(som_set_now, som_store_count, slot)) {
             DEBUG_PRINTF("just written\n");
@@ -347,14 +347,14 @@ void handleSomInternal(struct hs_scratch *scratch,
         break;
     }
 
-    // All valid internal_report types should be handled and returned above.
+    // All valid som_operation types should be handled and returned above.
     assert(0);
     return;
 }
 
 // Returns the SOM offset.
 u64a handleSomExternal(struct hs_scratch *scratch,
-                       const struct internal_report *ri,
+                       const struct som_operation *ri,
                        const u64a to_offset) {
     assert(scratch);
     assert(ri);
@@ -368,20 +368,20 @@ u64a handleSomExternal(struct hs_scratch *scratch,
     assert(rose->hasSom);
 
     switch (ri->type) {
-    case EXTERNAL_CALLBACK_SOM_REL:
+    case SOM_EXTERNAL_CALLBACK_REL:
-        DEBUG_PRINTF("EXTERNAL_CALLBACK_SOM_REL: som is %llu chars back\n",
+        DEBUG_PRINTF("SOM_EXTERNAL_CALLBACK_REL: som is %llu chars back\n",
                      ri->aux.somDistance);
         assert(to_offset >= ri->aux.somDistance);
         return to_offset - ri->aux.somDistance;
-    case EXTERNAL_CALLBACK_SOM_ABS:
+    case SOM_EXTERNAL_CALLBACK_ABS:
-        DEBUG_PRINTF("EXTERNAL_CALLBACK_SOM_ABS: som is at %llu\n",
+        DEBUG_PRINTF("SOM_EXTERNAL_CALLBACK_ABS: som is at %llu\n",
                      ri->aux.somDistance);
         assert(to_offset >= ri->aux.somDistance);
         return ri->aux.somDistance;
-    case EXTERNAL_CALLBACK_SOM_STORED: {
+    case SOM_EXTERNAL_CALLBACK_STORED: {
         const u64a *som_store = scratch->som_store;
         u32 slot = ri->aux.somDistance;
-        DEBUG_PRINTF("EXTERNAL_CALLBACK_SOM_STORED: <- som_store[%u]=%llu\n",
+        DEBUG_PRINTF("SOM_EXTERNAL_CALLBACK_STORED: <- som_store[%u]=%llu\n",
                      slot, som_store[slot]);
 
         UNUSED const u32 som_store_count = rose->somLocationCount;
@@ -391,8 +391,8 @@ u64a handleSomExternal(struct hs_scratch *scratch,
         assert(mmbit_isset(som_store_valid, som_store_count, slot));
         return som_store[slot];
     }
-    case EXTERNAL_CALLBACK_SOM_REV_NFA: {
+    case SOM_EXTERNAL_CALLBACK_REV_NFA: {
-        DEBUG_PRINTF("EXTERNAL_CALLBACK_REV_NFA\n");
+        DEBUG_PRINTF("SOM_EXTERNAL_CALLBACK_REV_NFA\n");
         u64a from_offset = 0;
         runRevNfa(scratch, ri, to_offset, &from_offset);
         return from_offset;
@@ -402,19 +402,19 @@ u64a handleSomExternal(struct hs_scratch *scratch,
         break;
     }
 
-    // All valid internal_report types should be handled and returned above.
+    // All valid som_operation types should be handled and returned above.
     assert(0);
     return 0;
 }
 
 void setSomFromSomAware(struct hs_scratch *scratch,
-                        const struct internal_report *ri, u64a from_offset,
+                        const struct som_operation *ri, u64a from_offset,
                         u64a to_offset) {
     assert(scratch);
     assert(ri);
     assert(to_offset);
-    assert(ri->type == INTERNAL_SOM_LOC_SET_FROM
+    assert(ri->type == SOM_INTERNAL_LOC_SET_FROM
-           || ri->type == INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE);
+           || ri->type == SOM_INTERNAL_LOC_SET_FROM_IF_WRITABLE);
 
     struct core_info *ci = &scratch->core_info;
     const struct RoseEngine *rose = ci->rose;
@@ -435,12 +435,12 @@ void setSomFromSomAware(struct hs_scratch *scratch,
         scratch->som_set_now_offset = to_offset;
     }
 
-    if (ri->type == INTERNAL_SOM_LOC_SET_FROM) {
+    if (ri->type == SOM_INTERNAL_LOC_SET_FROM) {
-        DEBUG_PRINTF("INTERNAL_SOM_LOC_SET_FROM\n");
+        DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_FROM\n");
         mmbit_set(som_store_valid, som_store_count, ri->onmatch);
         setSomLoc(som_set_now, som_store, som_store_count, ri, from_offset);
     } else {
-        DEBUG_PRINTF("INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE\n");
+        DEBUG_PRINTF("SOM_INTERNAL_LOC_SET_FROM_IF_WRITABLE\n");
         if (ok_and_mark_if_write(som_store_valid, som_set_now,
                                  som_store_writable, som_store_count,
                                  ri->onmatch)) {

src/som/som_runtime.h

@@ -26,10 +26,10 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-/** \file
+/**
+ * \file
  * \brief SOM runtime code.
  *
- *
  * Runtime code for SOM handling called by the Rose callback adaptors.
  */
 
@@ -39,17 +39,17 @@
 #include "scratch.h"
 #include "ue2common.h"
 
-struct internal_report;
+struct som_operation;
 
 void handleSomInternal(struct hs_scratch *scratch,
-                       const struct internal_report *ri, const u64a to_offset);
+                       const struct som_operation *ri, const u64a to_offset);
 
 // Returns the from_offset.
 u64a handleSomExternal(struct hs_scratch *scratch,
-                       const struct internal_report *ri, const u64a to_offset);
+                       const struct som_operation *ri, const u64a to_offset);
 
 void setSomFromSomAware(struct hs_scratch *scratch,
-                        const struct internal_report *ri, u64a from_offset,
+                        const struct som_operation *ri, u64a from_offset,
                         u64a to_offset);
 
 int flushStoredSomMatches_i(struct hs_scratch *scratch, u64a offset);

src/util/exhaust.h

@@ -34,10 +34,12 @@
 #define EXHAUST_H
 
 #include "rose/rose_internal.h"
-#include "util/internal_report.h"
 #include "util/multibit.h"
 #include "ue2common.h"
 
+/** Index meaning a given exhaustion key is invalid. */
+#define INVALID_EKEY    (~(u32)0)
+
 /** \brief Test whether the given key (\a ekey) is set in the exhaustion vector
  * \a evec. */
 static really_inline

src/util/internal_report.h

@@ -1,201 +0,0 @@
-/*
- * Copyright (c) 2015, Intel Corporation
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *  * Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
- *  * Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *  * Neither the name of Intel Corporation nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-/** \file
- * \brief Definition of an internal_report, along with convenience functions.
- */
-
-#ifndef INTERNAL_REPORT_H
-#define INTERNAL_REPORT_H
-
-#include "ue2common.h"
-
-/* internal_report::type values */
-
-#define EXTERNAL_CALLBACK                             0
-#define EXTERNAL_CALLBACK_SOM_REL                     1
-#define INTERNAL_SOM_LOC_SET                          2
-#define INTERNAL_SOM_LOC_SET_IF_UNSET                 3
-#define INTERNAL_SOM_LOC_SET_IF_WRITABLE              4
-#define INTERNAL_SOM_LOC_SET_SOM_REV_NFA              5
-#define INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET     6
-#define INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE  7
-#define INTERNAL_SOM_LOC_COPY                         8
-#define INTERNAL_SOM_LOC_COPY_IF_WRITABLE             9
-#define INTERNAL_SOM_LOC_MAKE_WRITABLE               10
-#define EXTERNAL_CALLBACK_SOM_STORED                 11
-#define EXTERNAL_CALLBACK_SOM_ABS                    12
-#define EXTERNAL_CALLBACK_SOM_REV_NFA                13
-
-/** set the som loc to the value in from_offset */
-#define INTERNAL_SOM_LOC_SET_FROM                    14
-
-/** set the som loc to the value in from_offset */
-#define INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE        15
-#define INTERNAL_ROSE_CHAIN                          16
-
-/** Index meaning a given exhaustion key is invalid. */
-#define INVALID_EKEY      0xffffffff
-
-/** \brief All the data we use for handling a match, bytecode representation.
- *
- * Includes extparam constraints and bounds, exhaustion/dedupe keys, offset
- * adjustment and SOM information.
- *
- * At compile time, this data is in the ue2::Report structure, which is
- * converted to internal_report for layout in the bytecode.
- */
-struct ALIGN_CL_DIRECTIVE internal_report {
-    /** \brief from EXTERNAL_ and INTERNAL_ defined above. */
-    u8 type;
-
-    /** \brief do we actually use minOffset, maxOffset */
-    u8 hasBounds;
-
-    /** \brief use SOM for minLength, but don't report it to user callback. */
-    u8 quashSom;
-
-    /** \brief min offset in the stream at which this report can match. */
-    u64a minOffset;
-
-    /** \brief max offset in the stream at which this report can match. */
-    u64a maxOffset;
-
-    /** \brief min match length (start of match to current offset) */
-    u64a minLength;
-
-    /** \brief Exhaustion key.
-     *
-     * If exhaustible, the ekey to check before reporting a match.
-     * Additionally after reporting a match the ekey will be set. If not
-     * exhaustible, this will be INVALID_EKEY. */
-    u32 ekey;
-
-    /** \brief Dedupe key. */
-    u32 dkey;
-
-    /** \brief Adjustment to add to the match offset when we report a match.
-     *
-     * This is usually used for reports attached to states that form part of a
-     * zero-width assertion, like '$'. */
-    s32 offsetAdjust;
-
-    /** \brief Match report ID, for external reports.
-     *
-     * - external callback -> external report id
-     * - internal_som_* -> som loc to modify,
-     * - INTERNAL_ROSE_CHAIN -> top event to push on
-     * - otherwise target subnfa. */
-    u32 onmatch;
-
-    union {
-        /** \brief SOM distance value, use varies according to type.
-         *
-         *  - for EXTERNAL_CALLBACK_SOM_REL, from-offset is this many bytes
-         *    before the to-offset.
-         *  - for EXTERNAL_CALLBACK_SOM_ABS, set from-offset to this value.
-         *  - for INTERNAL_SOM_LOC_COPY*, som location read_from.
-         */
-        u64a somDistance;
-
-        /** \brief Index of the reverse nfa.
-         * Used by EXTERNAL_CALLBACK_SOM_REV_NFA and
-         *  INTERNAL_SOM_LOC_SET_SOM_REV_NFA*
-         */
-        u64a revNfaIndex;
-
-        /**
-         * Used by INTERNAL_ROSE_CHAIN, Number of bytes behind us that we are
-         * allowed to squash identical top events on the queue.
-         */
-        u64a topSquashDistance;
-    } aux;
-};
-
-static really_inline
-int isInternalSomReport(const struct internal_report *ri) {
-    switch (ri->type) {
-    case INTERNAL_SOM_LOC_SET:
-    case INTERNAL_SOM_LOC_SET_IF_UNSET:
-    case INTERNAL_SOM_LOC_SET_IF_WRITABLE:
-    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
-    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
-    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
-    case INTERNAL_SOM_LOC_COPY:
-    case INTERNAL_SOM_LOC_COPY_IF_WRITABLE:
-    case INTERNAL_SOM_LOC_MAKE_WRITABLE:
-    case INTERNAL_SOM_LOC_SET_FROM:
-    case INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE:
-        return 1;
-    case EXTERNAL_CALLBACK:
-    case EXTERNAL_CALLBACK_SOM_REL:
-    case EXTERNAL_CALLBACK_SOM_STORED:
-    case EXTERNAL_CALLBACK_SOM_ABS:
-    case EXTERNAL_CALLBACK_SOM_REV_NFA:
-    case INTERNAL_ROSE_CHAIN:
-        return 0;
-    default:
-        break; // fall through
-    }
-    assert(0); // unknown?
-    return 0;
-}
-
-#ifndef NDEBUG
-/* used in asserts */
-static UNUSED
-char isExternalReport(const struct internal_report *ir) {
-    switch (ir->type) {
-    case INTERNAL_SOM_LOC_SET:
-    case INTERNAL_SOM_LOC_SET_IF_UNSET:
-    case INTERNAL_SOM_LOC_SET_IF_WRITABLE:
-    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
-    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
-    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
-    case INTERNAL_SOM_LOC_COPY:
-    case INTERNAL_SOM_LOC_COPY_IF_WRITABLE:
-    case INTERNAL_SOM_LOC_MAKE_WRITABLE:
-    case INTERNAL_SOM_LOC_SET_FROM:
-    case INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE:
-    case INTERNAL_ROSE_CHAIN:
-        return 0;
-    case EXTERNAL_CALLBACK:
-    case EXTERNAL_CALLBACK_SOM_REL:
-    case EXTERNAL_CALLBACK_SOM_STORED:
-    case EXTERNAL_CALLBACK_SOM_ABS:
-    case EXTERNAL_CALLBACK_SOM_REV_NFA:
-        return 1;
-    default:
-        break; // fall through
-    }
-    assert(0); // unknown?
-    return 1;
-}
-#endif
-
-#endif // INTERNAL_REPORT_H

src/util/report.cpp

@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2015, Intel Corporation
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *  * Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
- *  * Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *  * Neither the name of Intel Corporation nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "internal_report.h"
-#include "report.h"
-#include "report_manager.h"
-
-namespace ue2 {
-
-void writeInternalReport(const Report &report, const ReportManager &rm,
-                         internal_report *ir) {
-    assert(ir);
-    assert(ISALIGNED(ir));
-
-    ir->type = report.type;
-    ir->hasBounds = report.hasBounds() ? 1 : 0;
-    ir->quashSom = report.quashSom ? 1 : 0;
-    ir->minOffset = report.minOffset;
-    ir->maxOffset = report.maxOffset;
-    ir->minLength = report.minLength;
-    ir->ekey = report.ekey;
-    ir->offsetAdjust = report.offsetAdjust;
-    ir->onmatch = report.onmatch;
-
-    switch (report.type) {
-    case INTERNAL_ROSE_CHAIN:
-        ir->aux.topSquashDistance = report.topSquashDistance;
-        break;
-    case EXTERNAL_CALLBACK_SOM_REV_NFA:
-    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
-    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
-    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
-        ir->aux.revNfaIndex = report.revNfaIndex;
-        break;
-    default:
-        ir->aux.somDistance = report.somDistance;
-        break;
-    }
-
-    // Dedupe keys are managed by ReportManager.
-    ir->dkey = rm.getDkey(report);
-}
-
-} // namespace ue2

src/util/report.h

@@ -34,31 +34,49 @@
 #ifndef UTIL_REPORT_H
 #define UTIL_REPORT_H
 
-#include "internal_report.h"
+#include "util/exhaust.h" // for INVALID_EKEY
 #include "order_check.h"
 #include "ue2common.h"
 
 #include <cassert>
 
-struct internal_report;
-
 namespace ue2 {
 
 class ReportManager;
 
+enum ReportType {
+    EXTERNAL_CALLBACK,
+    EXTERNAL_CALLBACK_SOM_REL,
+    INTERNAL_SOM_LOC_SET,
+    INTERNAL_SOM_LOC_SET_IF_UNSET,
+    INTERNAL_SOM_LOC_SET_IF_WRITABLE,
+    INTERNAL_SOM_LOC_SET_SOM_REV_NFA,
+    INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET,
+    INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE,
+    INTERNAL_SOM_LOC_COPY,
+    INTERNAL_SOM_LOC_COPY_IF_WRITABLE,
+    INTERNAL_SOM_LOC_MAKE_WRITABLE,
+    EXTERNAL_CALLBACK_SOM_STORED,
+    EXTERNAL_CALLBACK_SOM_ABS,
+    EXTERNAL_CALLBACK_SOM_REV_NFA,
+    INTERNAL_SOM_LOC_SET_FROM,
+    INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE,
+    INTERNAL_ROSE_CHAIN,
+    EXTERNAL_CALLBACK_SOM_PASS
+};
+
 /**
  * \brief All the data we use for handling a match.
  *
  * Includes extparam constraints and bounds, exhaustion/dedupe keys, offset
  * adjustment and SOM information.
  *
- * The data in this structure is converted into an \ref internal_report in the
+ * The data in this structure eventually becomes a list of Rose programs
- * bytecode.
+ * instructions.
  */
 struct Report {
-    Report(u8 type_in, u32 onmatch_in) : type(type_in), onmatch(onmatch_in) {
+    Report(ReportType type_in, u32 onmatch_in)
-        assert(type <= INTERNAL_ROSE_CHAIN);
+        : type(type_in), onmatch(onmatch_in) {}
-    }
 
     /** \brief True if this report has bounds from extended parameters, i.e.
      * min offset, max offset, min length. */
@@ -66,8 +84,8 @@ struct Report {
         return minOffset > 0 || maxOffset < MAX_OFFSET || minLength > 0;
     }
 
-    /** \brief from EXTERNAL_ and INTERNAL_ defined in internal_report.h. */
+    /** \brief Type of this report. */
-    u8 type;
+    ReportType type;
 
     /** \brief use SOM for minLength, but don't report it to user callback. */
     bool quashSom = false;
@@ -147,6 +165,7 @@ bool isInternalSomReport(const Report &r) {
     case EXTERNAL_CALLBACK_SOM_ABS:
     case EXTERNAL_CALLBACK_SOM_REV_NFA:
     case INTERNAL_ROSE_CHAIN:
+    case EXTERNAL_CALLBACK_SOM_PASS:
         return false;
     default:
         break; // fall through
@@ -176,6 +195,7 @@ bool isExternalReport(const Report &r) {
     case EXTERNAL_CALLBACK_SOM_STORED:
     case EXTERNAL_CALLBACK_SOM_ABS:
     case EXTERNAL_CALLBACK_SOM_REV_NFA:
+    case EXTERNAL_CALLBACK_SOM_PASS:
         return true;
     default:
         break; // fall through
@@ -228,7 +248,7 @@ Report makeSomRelativeCallback(u32 report, s32 offsetAdjust, u64a distance) {
 }
 
 static inline
-Report makeRoseTrigger(u32 event, u64a squashDistance) {
+Report makeMpvTrigger(u32 event, u64a squashDistance) {
     Report ir(INTERNAL_ROSE_CHAIN, event);
     ir.ekey = INVALID_EKEY;
     ir.topSquashDistance = squashDistance;
@@ -254,22 +274,6 @@ bool isSimpleExhaustible(const Report &ir) {
     return true;
 }
 
-/** True if this report requires some of the more esoteric processing in the
- * rose adaptor, rather than just firing a callback or doing SOM handling.
- */
-static inline
-bool isComplexReport(const Report &ir) {
-    if (ir.hasBounds() || ir.ekey != INVALID_EKEY) {
-        return true;
-    }
-
-    return false;
-}
-
-/** \brief Write the given Report into an internal_report structure. */
-void writeInternalReport(const Report &report, const ReportManager &rm,
-                         internal_report *ir);
-
 } // namespace
 
 #endif // UTIL_REPORT_H