diff --git a/src/nfa/limex_common_impl.h b/src/nfa/limex_common_impl.h index ae0d1da8..5bd5187b 100644 --- a/src/nfa/limex_common_impl.h +++ b/src/nfa/limex_common_impl.h @@ -46,6 +46,7 @@ #define INITIAL_FN JOIN(moNfaInitial, SIZE) #define TOP_FN JOIN(moNfaTop, SIZE) #define TOPN_FN JOIN(moNfaTopN, SIZE) +#define PROCESS_ACCEPTS_IMPL_FN JOIN(moProcessAcceptsImpl, SIZE) #define PROCESS_ACCEPTS_FN JOIN(moProcessAccepts, SIZE) #define PROCESS_ACCEPTS_NOSQUASH_FN JOIN(moProcessAcceptsNoSquash, SIZE) #define CONTEXT_T JOIN(NFAContext, SIZE) @@ -60,6 +61,20 @@ #define SQUASH_UNTUG_BR_FN JOIN(lazyTug, SIZE) #define GET_NFA_REPEAT_INFO_FN JOIN(getNfaRepeatInfo, SIZE) +#if defined(ARCH_64_BIT) && (SIZE >= 64) +#define CHUNK_T u64a +#define FIND_AND_CLEAR_FN findAndClearLSB_64 +#define POPCOUNT_FN popcount64 +#define RANK_IN_MASK_FN rank_in_mask64 +#else +#define CHUNK_T u32 +#define FIND_AND_CLEAR_FN findAndClearLSB_32 +#define POPCOUNT_FN popcount32 +#define RANK_IN_MASK_FN rank_in_mask32 +#endif + +#define NUM_STATE_CHUNKS (sizeof(STATE_T) / sizeof(CHUNK_T)) + static really_inline void SQUASH_UNTUG_BR_FN(const IMPL_NFA_T *limex, const union RepeatControl *repeat_ctrl, @@ -98,63 +113,84 @@ void SQUASH_UNTUG_BR_FN(const IMPL_NFA_T *limex, } } -static never_inline -char PROCESS_ACCEPTS_FN(const IMPL_NFA_T *limex, STATE_T *s, - const struct NFAAccept *acceptTable, u32 acceptCount, - u64a offset, NfaCallback callback, void *context) { +static really_inline +char PROCESS_ACCEPTS_IMPL_FN(const IMPL_NFA_T *limex, const STATE_T *s, + STATE_T *squash, const ENG_STATE_T *squashMasks, + const STATE_T *acceptMask, + const struct NFAAccept *acceptTable, u64a offset, + NfaCallback callback, void *context) { assert(s); assert(limex); assert(callback); - assert(acceptCount); + const STATE_T accept_mask = *acceptMask; + STATE_T accepts = AND_STATE(*s, accept_mask); + + // Caller must ensure that we have at least one accept state on. + assert(ISNONZERO_STATE(accepts)); + + CHUNK_T chunks[NUM_STATE_CHUNKS]; + memcpy(chunks, &accepts, sizeof(accepts)); + + CHUNK_T mask_chunks[NUM_STATE_CHUNKS]; + memcpy(mask_chunks, &accept_mask, sizeof(accept_mask)); + + u32 base_index = 0; // Cumulative sum of mask popcount up to current chunk. + for (u32 i = 0; i < NUM_STATE_CHUNKS; i++) { + CHUNK_T chunk = chunks[i]; + while (chunk != 0) { + u32 bit = FIND_AND_CLEAR_FN(&chunk); + u32 local_idx = RANK_IN_MASK_FN(mask_chunks[i], bit); + u32 idx = local_idx + base_index; + const struct NFAAccept *a = &acceptTable[idx]; + DEBUG_PRINTF("state %u: firing report list=%u, offset=%llu\n", + bit + i * (u32)sizeof(chunk) * 8, a->reports, offset); + int rv = limexRunAccept((const char *)limex, a, callback, context, + offset); + if (unlikely(rv == MO_HALT_MATCHING)) { + return 1; + } + if (squash != NULL && a->squash != MO_INVALID_IDX) { + assert(squashMasks); + assert(a->squash < limex->squashCount); + const ENG_STATE_T *sq = &squashMasks[a->squash]; + DEBUG_PRINTF("squash mask %u @ %p\n", a->squash, sq); + *squash = AND_STATE(*squash, LOAD_FROM_ENG(sq)); + } + } + base_index += POPCOUNT_FN(mask_chunks[i]); + } + + return 0; +} + +static never_inline +char PROCESS_ACCEPTS_FN(const IMPL_NFA_T *limex, STATE_T *s, + const STATE_T *acceptMask, + const struct NFAAccept *acceptTable, u64a offset, + NfaCallback callback, void *context) { // We have squash masks we might have to apply after firing reports. STATE_T squash = ONES_STATE; const ENG_STATE_T *squashMasks = (const ENG_STATE_T *) ((const char *)limex + limex->squashOffset); - for (u32 i = 0; i < acceptCount; i++) { - const struct NFAAccept *a = &acceptTable[i]; - if (TESTBIT_STATE(*s, a->state)) { - DEBUG_PRINTF("state %u is on, firing report id=%u, offset=%llu\n", - a->state, a->externalId, offset); - int rv = callback(0, offset, a->externalId, context); - if (unlikely(rv == MO_HALT_MATCHING)) { - return 1; - } - if (a->squash != MO_INVALID_IDX) { - assert(a->squash < limex->squashCount); - const ENG_STATE_T *sq = &squashMasks[a->squash]; - DEBUG_PRINTF("squash mask %u @ %p\n", a->squash, sq); - squash = AND_STATE(squash, LOAD_FROM_ENG(sq)); - } - } - } + return PROCESS_ACCEPTS_IMPL_FN(limex, s, &squash, squashMasks, acceptMask, + acceptTable, offset, callback, context); *s = AND_STATE(*s, squash); - return 0; } static never_inline -char PROCESS_ACCEPTS_NOSQUASH_FN(const STATE_T *s, +char PROCESS_ACCEPTS_NOSQUASH_FN(const IMPL_NFA_T *limex, const STATE_T *s, + const STATE_T *acceptMask, const struct NFAAccept *acceptTable, - u32 acceptCount, u64a offset, - NfaCallback callback, void *context) { - assert(s); - assert(callback); - assert(acceptCount); + u64a offset, NfaCallback callback, + void *context) { + STATE_T *squash = NULL; + const ENG_STATE_T *squashMasks = NULL; - for (u32 i = 0; i < acceptCount; i++) { - const struct NFAAccept *a = &acceptTable[i]; - if (TESTBIT_STATE(*s, a->state)) { - DEBUG_PRINTF("state %u is on, firing report id=%u, offset=%llu\n", - a->state, a->externalId, offset); - int rv = callback(0, offset, a->externalId, context); - if (unlikely(rv == MO_HALT_MATCHING)) { - return 1; - } - } - } - return 0; + return PROCESS_ACCEPTS_IMPL_FN(limex, s, squash, squashMasks, acceptMask, + acceptTable, offset, callback, context); } // Run EOD accepts. Note that repeat_ctrl and repeat_state may be NULL if this @@ -179,8 +215,8 @@ char TESTEOD_FN(const IMPL_NFA_T *limex, const STATE_T *s, if (unlikely(ISNONZERO_STATE(foundAccepts))) { const struct NFAAccept *acceptEodTable = getAcceptEodTable(limex); - if (PROCESS_ACCEPTS_NOSQUASH_FN(&foundAccepts, acceptEodTable, - limex->acceptEodCount, offset, callback, + if (PROCESS_ACCEPTS_NOSQUASH_FN(limex, &foundAccepts, &acceptEodMask, + acceptEodTable, offset, callback, context)) { return MO_HALT_MATCHING; } @@ -206,8 +242,8 @@ char REPORTCURRENT_FN(const IMPL_NFA_T *limex, const struct mq *q) { const struct NFAAccept *acceptTable = getAcceptTable(limex); u64a offset = q_cur_offset(q); - if (PROCESS_ACCEPTS_NOSQUASH_FN(&foundAccepts, acceptTable, - limex->acceptCount, offset, q->cb, + if (PROCESS_ACCEPTS_NOSQUASH_FN(limex, &foundAccepts, &acceptMask, + acceptTable, offset, q->cb, q->context)) { return MO_HALT_MATCHING; } @@ -307,37 +343,45 @@ char LIMEX_INACCEPT_FN(const IMPL_NFA_T *limex, STATE_T state, u64a offset, ReportID report) { assert(limex); - const STATE_T acceptMask = LOAD_FROM_ENG(&limex->accept); - STATE_T accstate = AND_STATE(state, acceptMask); + const STATE_T accept_mask = LOAD_FROM_ENG(&limex->accept); + STATE_T accepts = AND_STATE(state, accept_mask); // Are we in an accept state? - if (ISZERO_STATE(accstate)) { + if (ISZERO_STATE(accepts)) { DEBUG_PRINTF("no accept states are on\n"); return 0; } - SQUASH_UNTUG_BR_FN(limex, repeat_ctrl, repeat_state, offset, &accstate); + SQUASH_UNTUG_BR_FN(limex, repeat_ctrl, repeat_state, offset, &accepts); DEBUG_PRINTF("looking for report %u\n", report); -#ifdef DEBUG - DEBUG_PRINTF("accept states that are on: "); - for (u32 i = 0; i < sizeof(STATE_T) * 8; i++) { - if (TESTBIT_STATE(accstate, i)) printf("%u ", i); - } - printf("\n"); -#endif - - // Does one of our states match the given report ID? const struct NFAAccept *acceptTable = getAcceptTable(limex); - for (u32 i = 0; i < limex->acceptCount; i++) { - const struct NFAAccept *a = &acceptTable[i]; - DEBUG_PRINTF("checking idx=%u, externalId=%u\n", a->state, - a->externalId); - if (a->externalId == report && TESTBIT_STATE(accstate, a->state)) { - DEBUG_PRINTF("report is on!\n"); - return 1; + + CHUNK_T chunks[NUM_STATE_CHUNKS]; + memcpy(chunks, &accepts, sizeof(accepts)); + + CHUNK_T mask_chunks[NUM_STATE_CHUNKS]; + memcpy(mask_chunks, &accept_mask, sizeof(accept_mask)); + + u32 base_index = 0; // Cumulative sum of mask popcount up to current chunk. + for (u32 i = 0; i < NUM_STATE_CHUNKS; i++) { + CHUNK_T chunk = chunks[i]; + while (chunk != 0) { + u32 bit = FIND_AND_CLEAR_FN(&chunk); + u32 local_idx = RANK_IN_MASK_FN(mask_chunks[i], bit); + u32 idx = local_idx + base_index; + assert(idx < limex->acceptCount); + const struct NFAAccept *a = &acceptTable[idx]; + DEBUG_PRINTF("state %u is on, report list at %u\n", + bit + i * (u32)sizeof(chunk) * 8, a->reports); + + if (limexAcceptHasReport((const char *)limex, a, report)) { + DEBUG_PRINTF("report %u is on\n", report); + return 1; + } } + base_index += POPCOUNT_FN(mask_chunks[i]); } return 0; @@ -381,7 +425,14 @@ char LIMEX_INANYACCEPT_FN(const IMPL_NFA_T *limex, STATE_T state, #undef TESTBIT_STATE #undef ISNONZERO_STATE #undef ISZERO_STATE +#undef PROCESS_ACCEPTS_IMPL_FN #undef PROCESS_ACCEPTS_FN #undef PROCESS_ACCEPTS_NOSQUASH_FN #undef SQUASH_UNTUG_BR_FN #undef GET_NFA_REPEAT_INFO_FN + +#undef CHUNK_T +#undef FIND_AND_CLEAR_FN +#undef POPCOUNT_FN +#undef RANK_IN_MASK_FN +#undef NUM_STATE_CHUNKS diff --git a/src/nfa/limex_compile.cpp b/src/nfa/limex_compile.cpp index 78b9729f..89eaf10a 100644 --- a/src/nfa/limex_compile.cpp +++ b/src/nfa/limex_compile.cpp @@ -992,14 +992,105 @@ void buildAccel(const build_info &args, NFAStateSet &accelMask, } static -void buildAccepts(const build_info &args, NFAStateSet &acceptMask, - NFAStateSet &acceptEodMask, vector &accepts, - vector &acceptsEod, vector &squash) { +u32 addSquashMask(const build_info &args, const NFAVertex &v, + vector &squash) { + auto sit = args.reportSquashMap.find(v); + if (sit == args.reportSquashMap.end()) { + return MO_INVALID_IDX; + } + + // This state has a squash mask. Paw through the existing vector to + // see if we've already seen it, otherwise add a new one. + auto it = find(squash.begin(), squash.end(), sit->second); + if (it != squash.end()) { + return verify_u32(distance(squash.begin(), it)); + } + u32 idx = verify_u32(squash.size()); + squash.push_back(sit->second); + return idx; +} + +static +u32 addReports(const flat_set &r, vector &reports, + unordered_map, u32> &reportListCache) { + assert(!r.empty()); + + vector my_reports(begin(r), end(r)); + my_reports.push_back(MO_INVALID_IDX); // sentinel + + auto cache_it = reportListCache.find(my_reports); + if (cache_it != end(reportListCache)) { + u32 offset = cache_it->second; + DEBUG_PRINTF("reusing cached report list at %u\n", offset); + return offset; + } + + auto it = search(begin(reports), end(reports), begin(my_reports), + end(my_reports)); + if (it != end(reports)) { + u32 offset = verify_u32(distance(begin(reports), it)); + DEBUG_PRINTF("reusing found report list at %u\n", offset); + return offset; + } + + u32 offset = verify_u32(reports.size()); + insert(&reports, reports.end(), my_reports); + reportListCache.emplace(move(my_reports), offset); + return offset; +} + +static +void buildAcceptsList(const build_info &args, + unordered_map, u32> &reports_cache, + vector &verts, vector &accepts, + vector &reports, vector &squash) { + if (verts.empty()) { + return; + } + + DEBUG_PRINTF("building accept lists for %zu states\n", verts.size()); + + auto cmp_state_id = [&args](NFAVertex a, NFAVertex b) { + u32 a_state = args.state_ids.at(a); + u32 b_state = args.state_ids.at(b); + assert(a_state != b_state || a == b); + return a_state < b_state; + }; + + sort(begin(verts), end(verts), cmp_state_id); + + const NGHolder &h = args.h; + for (const auto &v : verts) { + DEBUG_PRINTF("state=%u, reports: [%s]\n", args.state_ids.at(v), + as_string_list(h[v].reports).c_str()); + NFAAccept a; + memset(&a, 0, sizeof(a)); + assert(!h[v].reports.empty()); + if (h[v].reports.size() == 1) { + a.single_report = 1; + a.reports = *h[v].reports.begin(); + } else { + a.single_report = 0; + a.reports = addReports(h[v].reports, reports, reports_cache); + } + a.squash = addSquashMask(args, v, squash); + accepts.push_back(move(a)); + } +} + +static +void buildAccepts(const build_info &args, + unordered_map, u32> &reports_cache, + NFAStateSet &acceptMask, NFAStateSet &acceptEodMask, + vector &accepts, vector &acceptsEod, + vector &reports, vector &squash) { const NGHolder &h = args.h; acceptMask.resize(args.num_states); acceptEodMask.resize(args.num_states); + vector verts_accept, verts_accept_eod; + for (auto v : vertices_range(h)) { u32 state_id = args.state_ids.at(v); @@ -1007,41 +1098,20 @@ void buildAccepts(const build_info &args, NFAStateSet &acceptMask, continue; } - u32 squashMaskOffset = MO_INVALID_IDX; - auto sit = args.reportSquashMap.find(v); - if (sit != args.reportSquashMap.end()) { - // This state has a squash mask. Paw through the existing vector to - // see if we've already seen it, otherwise add a new one. - auto it = find(squash.begin(), squash.end(), sit->second); - if (it != squash.end()) { - squashMaskOffset = verify_u32(distance(squash.begin(), it)); - } else { - squashMaskOffset = verify_u32(squash.size()); - squash.push_back(sit->second); - } - } - - // Add an accept (or acceptEod) per report ID. - - vector *accepts_out; if (edge(v, h.accept, h).second) { acceptMask.set(state_id); - accepts_out = &accepts; + verts_accept.push_back(v); } else { assert(edge(v, h.acceptEod, h).second); acceptEodMask.set(state_id); - accepts_out = &acceptsEod; - } - - for (auto report : h[v].reports) { - accepts_out->push_back(NFAAccept()); - NFAAccept &a = accepts_out->back(); - a.state = state_id; - a.externalId = report; - a.squash = squashMaskOffset; - DEBUG_PRINTF("Accept: state=%u, externalId=%u\n", state_id, report); + verts_accept_eod.push_back(v); } } + + buildAcceptsList(args, reports_cache, verts_accept, accepts, reports, + squash); + buildAcceptsList(args, reports_cache, verts_accept_eod, acceptsEod, reports, + squash); } static @@ -1314,36 +1384,12 @@ struct ExceptionProto { } }; -static -u32 getReportListIndex(const flat_set &reports, - vector &exceptionReports, - map, u32> &reportListCache) { - if (reports.empty()) { - return MO_INVALID_IDX; - } - - const vector r(reports.begin(), reports.end()); - - auto it = reportListCache.find(r); - if (it != reportListCache.end()) { - u32 idx = it->second; - assert(idx < exceptionReports.size()); - assert(equal(r.begin(), r.end(), exceptionReports.begin() + idx)); - return idx; - } - - u32 idx = verify_u32(exceptionReports.size()); - reportListCache[r] = idx; - exceptionReports.insert(exceptionReports.end(), r.begin(), r.end()); - exceptionReports.push_back(MO_INVALID_IDX); // terminator - return idx; -} - static u32 buildExceptionMap(const build_info &args, + unordered_map, u32> &reports_cache, const ue2::unordered_set &exceptional, - map > &exceptionMap, - vector &exceptionReports) { + map> &exceptionMap, + vector &reportList) { const NGHolder &h = args.h; const u32 num_states = args.num_states; u32 exceptionCount = 0; @@ -1361,10 +1407,6 @@ u32 buildExceptionMap(const build_info &args, } } - // We track report lists that have already been written into the global - // list in case we can reuse them. - map, u32> reportListCache; - for (auto v : vertices_range(h)) { const u32 i = args.state_ids.at(v); @@ -1383,8 +1425,12 @@ u32 buildExceptionMap(const build_info &args, DEBUG_PRINTF("state %u is exceptional due to accept " "(%zu reports)\n", i, reports.size()); - e.reports_index = - getReportListIndex(reports, exceptionReports, reportListCache); + if (reports.empty()) { + e.reports_index = MO_INVALID_IDX; + } else { + e.reports_index = + addReports(reports, reportList, reports_cache); + } // We may be applying a report squash too. auto mi = args.reportSquashMap.find(v); @@ -1810,9 +1856,10 @@ struct Factory { } static - void writeExceptions(const map > &exceptionMap, - const vector &repeatOffsets, - implNFA_t *limex, const u32 exceptionsOffset) { + void writeExceptions(const map> &exceptionMap, + const vector &repeatOffsets, implNFA_t *limex, + const u32 exceptionsOffset, + const u32 reportListOffset) { DEBUG_PRINTF("exceptionsOffset=%u\n", exceptionsOffset); exception_t *etable = (exception_t *)((char *)limex + exceptionsOffset); @@ -1839,7 +1886,12 @@ struct Factory { exception_t &e = etable[ecount]; maskSetBits(e.squash, proto.squash_states); maskSetBits(e.successors, proto.succ_states); - e.reports = proto.reports_index; + if (proto.reports_index == MO_INVALID_IDX) { + e.reports = MO_INVALID_IDX; + } else { + e.reports = reportListOffset + + proto.reports_index * sizeof(ReportID); + } e.hasSquash = verify_u8(proto.squash); e.trigger = verify_u8(proto.trigger); u32 repeat_offset = proto.repeat_index == MO_INVALID_IDX @@ -1958,7 +2010,9 @@ struct Factory { const vector &acceptsEod, const vector &squash, implNFA_t *limex, const u32 acceptsOffset, const u32 acceptsEodOffset, - const u32 squashOffset) { + const u32 squashOffset, const u32 reportListOffset) { + char *limex_base = (char *)limex; + DEBUG_PRINTF("acceptsOffset=%u, acceptsEodOffset=%u, squashOffset=%u\n", acceptsOffset, acceptsEodOffset, squashOffset); @@ -1966,27 +2020,38 @@ struct Factory { maskSetBits(limex->accept, acceptMask); maskSetBits(limex->acceptAtEOD, acceptEodMask); + // Transforms the index into the report list into an offset relative to + // the base of the limex. + auto report_offset_fn = [&](NFAAccept a) { + if (!a.single_report) { + a.reports = reportListOffset + a.reports * sizeof(ReportID); + } + return a; + }; + // Write accept table. limex->acceptOffset = acceptsOffset; limex->acceptCount = verify_u32(accepts.size()); DEBUG_PRINTF("NFA has %zu accepts\n", accepts.size()); - NFAAccept *acceptsTable = (NFAAccept *)((char *)limex + acceptsOffset); + NFAAccept *acceptsTable = (NFAAccept *)(limex_base + acceptsOffset); assert(ISALIGNED(acceptsTable)); - copy(accepts.begin(), accepts.end(), acceptsTable); + transform(accepts.begin(), accepts.end(), acceptsTable, + report_offset_fn); // Write eod accept table. limex->acceptEodOffset = acceptsEodOffset; limex->acceptEodCount = verify_u32(acceptsEod.size()); DEBUG_PRINTF("NFA has %zu EOD accepts\n", acceptsEod.size()); - NFAAccept *acceptsEodTable = (NFAAccept *)((char *)limex + acceptsEodOffset); + NFAAccept *acceptsEodTable = (NFAAccept *)(limex_base + acceptsEodOffset); assert(ISALIGNED(acceptsEodTable)); - copy(acceptsEod.begin(), acceptsEod.end(), acceptsEodTable); + transform(acceptsEod.begin(), acceptsEod.end(), acceptsEodTable, + report_offset_fn); // Write squash mask table. limex->squashCount = verify_u32(squash.size()); limex->squashOffset = squashOffset; DEBUG_PRINTF("NFA has %zu report squash masks\n", squash.size()); - tableRow_t *mask = (tableRow_t *)((char *)limex + squashOffset); + tableRow_t *mask = (tableRow_t *)(limex_base + squashOffset); assert(ISALIGNED(mask)); for (size_t i = 0, end = squash.size(); i < end; i++) { maskSetBits(mask[i], squash[i]); @@ -2023,15 +2088,12 @@ struct Factory { } static - void writeExceptionReports(const vector &reports, - implNFA_t *limex, - const u32 exceptionReportsOffset) { - DEBUG_PRINTF("exceptionReportsOffset=%u\n", exceptionReportsOffset); - - limex->exReportOffset = exceptionReportsOffset; - assert(ISALIGNED_N((char *)limex + exceptionReportsOffset, + void writeReportList(const vector &reports, implNFA_t *limex, + const u32 reportListOffset) { + DEBUG_PRINTF("reportListOffset=%u\n", reportListOffset); + assert(ISALIGNED_N((char *)limex + reportListOffset, alignof(ReportID))); - copy_bytes((char *)limex + exceptionReportsOffset, reports); + copy_bytes((char *)limex + reportListOffset, reports); } static @@ -2050,6 +2112,10 @@ struct Factory { repeatSize += repeats[i].second; } + // We track report lists that have already been written into the global + // list in case we can reuse them. + unordered_map, u32> reports_cache; + ue2::unordered_set exceptional; u32 shiftCount = findBestNumOfVarShifts(args); assert(shiftCount); @@ -2057,9 +2123,10 @@ struct Factory { findExceptionalTransitions(args, exceptional, maxShift); map > exceptionMap; - vector exceptionReports; - u32 exceptionCount = buildExceptionMap(args, exceptional, exceptionMap, - exceptionReports); + vector reportList; + + u32 exceptionCount = buildExceptionMap(args, reports_cache, exceptional, + exceptionMap, reportList); assert(exceptionCount <= args.num_states); @@ -2076,8 +2143,8 @@ struct Factory { NFAStateSet acceptMask, acceptEodMask; vector accepts, acceptsEod; vector squash; - buildAccepts(args, acceptMask, acceptEodMask, accepts, acceptsEod, - squash); + buildAccepts(args, reports_cache, acceptMask, acceptEodMask, accepts, + acceptsEod, reportList, squash); // Build all our accel info. NFAStateSet accelMask, accelFriendsMask; @@ -2118,8 +2185,8 @@ struct Factory { const u32 exceptionsOffset = offset; offset += sizeof(exception_t) * exceptionCount; - const u32 exceptionReportsOffset = offset; - offset += sizeof(ReportID) * exceptionReports.size(); + const u32 reportListOffset = offset; + offset += sizeof(ReportID) * reportList.size(); const u32 repeatOffsetsOffset = offset; offset += sizeof(u32) * args.repeats.size(); @@ -2146,7 +2213,8 @@ struct Factory { limex, accelTableOffset, accelAuxOffset); writeAccepts(acceptMask, acceptEodMask, accepts, acceptsEod, squash, - limex, acceptsOffset, acceptsEodOffset, squashOffset); + limex, acceptsOffset, acceptsEodOffset, squashOffset, + reportListOffset); limex->shiftCount = shiftCount; writeShiftMasks(args, limex); @@ -2154,14 +2222,15 @@ struct Factory { // Determine the state required for our state vector. findStateSize(args, limex); - writeExceptionReports(exceptionReports, limex, exceptionReportsOffset); + writeReportList(reportList, limex, reportListOffset); // Repeat structures and offset table. vector repeatOffsets; writeRepeats(repeats, repeatOffsets, limex, repeatOffsetsOffset, repeatsOffset); - writeExceptions(exceptionMap, repeatOffsets, limex, exceptionsOffset); + writeExceptions(exceptionMap, repeatOffsets, limex, exceptionsOffset, + reportListOffset); writeLimexMasks(args, limex); diff --git a/src/nfa/limex_dump.cpp b/src/nfa/limex_dump.cpp index 181951dc..149e8107 100644 --- a/src/nfa/limex_dump.cpp +++ b/src/nfa/limex_dump.cpp @@ -180,26 +180,40 @@ void dumpAccel(const limex_type *limex, FILE *f) { } } +static +void dumpAcceptList(const char *limex_base, const struct NFAAccept *accepts, + u32 acceptCount, FILE *f) { + for (u32 i = 0; i < acceptCount; i++) { + const NFAAccept &a = accepts[i]; + if (a.single_report) { + fprintf(f, " idx %u fires single report %u\n", i, a.reports); + continue; + } + fprintf(f, " idx %u fires report list %u:", i, a.reports); + const ReportID *report = (const ReportID *)(limex_base + a.reports); + for (; *report != MO_INVALID_IDX; report++) { + fprintf(f, " %u", *report); + } + fprintf(f, "\n"); + } +} + template static void dumpAccepts(const limex_type *limex, FILE *f) { - u32 acceptCount = limex->acceptCount; - u32 acceptEodCount = limex->acceptEodCount; + const char *limex_base = (const char *)limex; + + const u32 acceptCount = limex->acceptCount; + const u32 acceptEodCount = limex->acceptEodCount; fprintf(f, "\n%u accepts.\n", acceptCount); - const struct NFAAccept *accepts - = (const struct NFAAccept *)((const char *)limex + limex->acceptOffset); - for (u32 i = 0; i < acceptCount; i++) { - fprintf(f, " state %u fires report %u\n", accepts[i].state, - accepts[i].externalId); - } + const auto *accepts = + (const struct NFAAccept *)(limex_base + limex->acceptOffset); + dumpAcceptList(limex_base, accepts, acceptCount, f); fprintf(f, "\n%u accepts at EOD.\n", acceptEodCount); - accepts = (const struct NFAAccept *)((const char *)limex - + limex->acceptEodOffset); - for (u32 i = 0; i < acceptEodCount; i++) { - fprintf(f, " state %u fires report %u\n", accepts[i].state, - accepts[i].externalId); - } + const auto *accepts_eod = + (const struct NFAAccept *)(limex_base + limex->acceptEodOffset); + dumpAcceptList(limex_base, accepts_eod, acceptEodCount, f); fprintf(f, "\n"); } @@ -226,20 +240,15 @@ getExceptionTable(const limex_type *limex) { ((const char *)limex + limex->exceptionOffset); } -template -static -const ReportID *getReportList(const limex_type *limex) { - return (const ReportID *)((const char *)limex + limex->exReportOffset); -} - template static void dumpLimexExceptions(const limex_type *limex, FILE *f) { const typename limex_traits::exception_type *e = getExceptionTable(limex); - const ReportID *reports = getReportList(limex); const u32 size = limex_traits::size; + const char *limex_base = (const char *)limex; + fprintf(f, "\n"); for (u32 i = 0; i < limex->exceptionCount; i++) { fprintf(f, "exception %u: hasSquash=%u, reports offset=%u\n", @@ -255,7 +264,7 @@ void dumpLimexExceptions(const limex_type *limex, FILE *f) { if (e[i].reports == MO_INVALID_IDX) { fprintf(f, " \n"); } else { - const ReportID *r = reports + e[i].reports; + const ReportID *r = (const ReportID *)(limex_base + e[i].reports); while (*r != MO_INVALID_IDX) { fprintf(f, " %u", *r++); } diff --git a/src/nfa/limex_exceptional.h b/src/nfa/limex_exceptional.h index c8296f91..e770c327 100644 --- a/src/nfa/limex_exceptional.h +++ b/src/nfa/limex_exceptional.h @@ -95,7 +95,6 @@ int RUN_EXCEPTION_FN(const EXCEPTION_T *e, STATE_ARG, STATE_T *local_succ, #endif const struct IMPL_NFA_T *limex, - const ReportID *exReports, u64a offset, struct CONTEXT_T *ctx, struct proto_cache *new_cache, @@ -161,7 +160,8 @@ int RUN_EXCEPTION_FN(const EXCEPTION_T *e, STATE_ARG, // Some exceptions fire accepts. if (e->reports != MO_INVALID_IDX) { if (flags & CALLBACK_OUTPUT) { - const ReportID *reports = exReports + e->reports; + const ReportID *reports = + (const ReportID *)((const char *)limex + e->reports); if (unlikely(limexRunReports(reports, ctx->callback, ctx->context, offset) == MO_HALT_MATCHING)) { @@ -210,8 +210,7 @@ int RUN_EXCEPTION_FN(const EXCEPTION_T *e, STATE_ARG, static really_inline int PE_FN(STATE_ARG, ESTATE_ARG, u32 diffmask, STATE_T *succ, const struct IMPL_NFA_T *limex, const EXCEPTION_T *exceptions, - const ReportID *exReports, u64a offset, struct CONTEXT_T *ctx, - char in_rev, char flags) { + u64a offset, struct CONTEXT_T *ctx, char in_rev, char flags) { assert(diffmask > 0); // guaranteed by caller macro if (EQ_STATE(estate, ctx->cached_estate)) { @@ -271,8 +270,8 @@ int PE_FN(STATE_ARG, ESTATE_ARG, u32 diffmask, STATE_T *succ, #ifndef BIG_MODEL &local_succ, #endif - limex, exReports, offset, ctx, &new_cache, - &cacheable, in_rev, flags)) { + limex, offset, ctx, &new_cache, &cacheable, + in_rev, flags)) { return PE_RV_HALT; } } while (word); @@ -326,7 +325,9 @@ int PE_FN(STATE_ARG, ESTATE_ARG, u32 diffmask, STATE_T *succ, #undef STATE_ARG_NAME #undef STATE_ARG_P +#undef IMPL_NFA_T + #undef CHUNK_T #undef FIND_AND_CLEAR_FN -#undef IMPL_NFA_T -#undef GET_NFA_REPEAT_INFO_FN +#undef POPCOUNT_FN +#undef RANK_IN_MASK_FN diff --git a/src/nfa/limex_internal.h b/src/nfa/limex_internal.h index 03ebb384..0d46732f 100644 --- a/src/nfa/limex_internal.h +++ b/src/nfa/limex_internal.h @@ -132,7 +132,6 @@ struct LimExNFA##size { \ u32 acceptEodOffset; /* rel. to start of LimExNFA */ \ u32 exceptionCount; \ u32 exceptionOffset; /* rel. to start of LimExNFA */ \ - u32 exReportOffset; /* rel. to start of LimExNFA */ \ u32 repeatCount; \ u32 repeatOffset; \ u32 squashOffset; /* rel. to start of LimExNFA; for accept squashing */ \ @@ -184,9 +183,16 @@ struct NFARepeatInfo { }; struct NFAAccept { - u32 state; //!< state ID of triggering state - ReportID externalId; //!< report ID to raise - u32 squash; //!< offset into masks, or MO_INVALID_IDX + u8 single_report; //!< If true, 'reports' is report id. + + /** + * \brief If single report is true, this is the report id to fire. + * Otherwise, it is the offset (relative to the start of the LimExNFA + * structure) of a list of reports, terminated with MO_INVALID_IDX. + */ + u32 reports; + + u32 squash; //!< Offset into squash masks, or MO_INVALID_IDX. }; #endif diff --git a/src/nfa/limex_native.c b/src/nfa/limex_native.c index c9949836..f6f5809c 100644 --- a/src/nfa/limex_native.c +++ b/src/nfa/limex_native.c @@ -73,8 +73,7 @@ static really_inline int processExceptional32(u32 s, u32 estate, UNUSED u32 diffmask, u32 *succ, const struct LimExNFA32 *limex, - const struct NFAException32 *exceptions, - const ReportID *exReports, u64a offset, + const struct NFAException32 *exceptions, u64a offset, struct NFAContext32 *ctx, char in_rev, char flags) { assert(estate != 0); // guaranteed by calling macro @@ -104,8 +103,8 @@ int processExceptional32(u32 s, u32 estate, UNUSED u32 diffmask, u32 *succ, u32 bit = findAndClearLSB_32(&estate); u32 idx = rank_in_mask32(limex->exceptionMask, bit); const struct NFAException32 *e = &exceptions[idx]; - if (!runException32(e, s, succ, &local_succ, limex, exReports, offset, - ctx, &new_cache, &cacheable, in_rev, flags)) { + if (!runException32(e, s, succ, &local_succ, limex, offset, ctx, + &new_cache, &cacheable, in_rev, flags)) { return PE_RV_HALT; } } while (estate != 0); diff --git a/src/nfa/limex_runtime.h b/src/nfa/limex_runtime.h index 75094ef6..6109d382 100644 --- a/src/nfa/limex_runtime.h +++ b/src/nfa/limex_runtime.h @@ -103,14 +103,42 @@ int limexRunReports(const ReportID *reports, NfaCallback callback, return MO_CONTINUE_MATCHING; // continue } +static really_inline +int limexRunAccept(const char *limex_base, const struct NFAAccept *accept, + NfaCallback callback, void *context, u64a offset) { + if (accept->single_report) { + const ReportID report = accept->reports; + DEBUG_PRINTF("firing single report for id %u at offset %llu\n", report, + offset); + return callback(0, offset, report, context); + } + const ReportID *reports = (const ReportID *)(limex_base + accept->reports); + return limexRunReports(reports, callback, context, offset); +} + +static really_inline +int limexAcceptHasReport(const char *limex_base, const struct NFAAccept *accept, + ReportID report) { + if (accept->single_report) { + return accept->reports == report; + } + + const ReportID *reports = (const ReportID *)(limex_base + accept->reports); + assert(*reports != MO_INVALID_IDX); + do { + if (*reports == report) { + return 1; + } + reports++; + } while (*reports != MO_INVALID_IDX); + + return 0; +} + /** \brief Return a (correctly typed) pointer to the exception table. */ #define getExceptionTable(exc_type, lim) \ ((const exc_type *)((const char *)(lim) + (lim)->exceptionOffset)) -/** \brief Return a pointer to the exceptional reports list. */ -#define getExReports(lim) \ - ((const ReportID *)((const char *)(lim) + (lim)->exReportOffset)) - /** \brief Return a pointer to the ordinary accepts table. */ #define getAcceptTable(lim) \ ((const struct NFAAccept *)((const char *)(lim) + (lim)->acceptOffset)) diff --git a/src/nfa/limex_runtime_impl.h b/src/nfa/limex_runtime_impl.h index 5bc79c24..45ceb2b5 100644 --- a/src/nfa/limex_runtime_impl.h +++ b/src/nfa/limex_runtime_impl.h @@ -103,8 +103,7 @@ // continue, 1 if an accept was fired and the user instructed us to halt. static really_inline char RUN_EXCEPTIONS_FN(const IMPL_NFA_T *limex, const EXCEPTION_T *exceptions, - const ReportID *exReports, STATE_T s, - const STATE_T emask, size_t i, u64a offset, + STATE_T s, const STATE_T emask, size_t i, u64a offset, STATE_T *succ, u64a *final_loc, struct CONTEXT_T *ctx, const char flags, const char in_rev, const char first_match) { @@ -131,7 +130,7 @@ char RUN_EXCEPTIONS_FN(const IMPL_NFA_T *limex, const EXCEPTION_T *exceptions, char localflags = (!i && !in_rev) ? NO_OUTPUT | FIRST_BYTE : flags; int rv = JOIN(processExceptional, SIZE)( - pass_state, pass_estate, diffmask, succ, limex, exceptions, exReports, + pass_state, pass_estate, diffmask, succ, limex, exceptions, callback_offset, ctx, in_rev, localflags); if (rv == PE_RV_HALT) { return 1; // Halt matching. @@ -207,7 +206,6 @@ char STREAM_FN(const IMPL_NFA_T *limex, const u8 *input, size_t length, const union AccelAux *accelAux = (const union AccelAux *)((const char *)limex + limex->accelAuxOffset); const EXCEPTION_T *exceptions = getExceptionTable(EXCEPTION_T, limex); - const ReportID *exReports = getExReports(limex); STATE_T s = ctx->s; /* assert(ISALIGNED_16(exceptions)); */ @@ -235,9 +233,8 @@ without_accel: STATE_T succ; NFA_EXEC_GET_LIM_SUCC(limex, s, succ); - if (RUN_EXCEPTIONS_FN(limex, exceptions, exReports, s, EXCEPTION_MASK, - i, offset, &succ, final_loc, ctx, flags, 0, - first_match)) { + if (RUN_EXCEPTIONS_FN(limex, exceptions, s, EXCEPTION_MASK, i, offset, + &succ, final_loc, ctx, flags, 0, first_match)) { return MO_HALT_MATCHING; } @@ -286,9 +283,8 @@ with_accel: STATE_T succ; NFA_EXEC_GET_LIM_SUCC(limex, s, succ); - if (RUN_EXCEPTIONS_FN(limex, exceptions, exReports, s, EXCEPTION_MASK, - i, offset, &succ, final_loc, ctx, flags, 0, - first_match)) { + if (RUN_EXCEPTIONS_FN(limex, exceptions, s, EXCEPTION_MASK, i, offset, + &succ, final_loc, ctx, flags, 0, first_match)) { return MO_HALT_MATCHING; } @@ -300,8 +296,6 @@ with_accel: if ((first_match || (flags & CALLBACK_OUTPUT)) && limex->acceptCount) { STATE_T acceptMask = LOAD_FROM_ENG(&limex->accept); const struct NFAAccept *acceptTable = getAcceptTable(limex); - const u32 acceptCount = limex->acceptCount; - STATE_T foundAccepts = AND_STATE(s, acceptMask); if (unlikely(ISNONZERO_STATE(foundAccepts))) { if (first_match) { @@ -309,8 +303,8 @@ with_accel: assert(final_loc); *final_loc = length; return MO_HALT_MATCHING; - } else if (PROCESS_ACCEPTS_FN(limex, &ctx->s, acceptTable, - acceptCount, offset + length, + } else if (PROCESS_ACCEPTS_FN(limex, &ctx->s, &acceptMask, + acceptTable, offset + length, ctx->callback, ctx->context)) { return MO_HALT_MATCHING; } @@ -331,7 +325,6 @@ char REV_STREAM_FN(const IMPL_NFA_T *limex, const u8 *input, size_t length, const STATE_T exceptionMask = LOAD_FROM_ENG(&limex->exceptionMask); #endif const EXCEPTION_T *exceptions = getExceptionTable(EXCEPTION_T, limex); - const ReportID *exReports = getExReports(limex); STATE_T s = ctx->s; /* assert(ISALIGNED_16(exceptions)); */ @@ -351,9 +344,8 @@ char REV_STREAM_FN(const IMPL_NFA_T *limex, const u8 *input, size_t length, STATE_T succ; NFA_EXEC_GET_LIM_SUCC(limex, s, succ); - if (RUN_EXCEPTIONS_FN(limex, exceptions, exReports, s, - EXCEPTION_MASK, i, offset, &succ, final_loc, ctx, - flags, 1, 0)) { + if (RUN_EXCEPTIONS_FN(limex, exceptions, s, EXCEPTION_MASK, i, offset, + &succ, final_loc, ctx, flags, 1, 0)) { return MO_HALT_MATCHING; } @@ -369,8 +361,8 @@ char REV_STREAM_FN(const IMPL_NFA_T *limex, const u8 *input, size_t length, if (acceptCount) { STATE_T foundAccepts = AND_STATE(s, acceptMask); if (unlikely(ISNONZERO_STATE(foundAccepts))) { - if (PROCESS_ACCEPTS_NOSQUASH_FN(&ctx->s, acceptTable, acceptCount, - offset, ctx->callback, + if (PROCESS_ACCEPTS_NOSQUASH_FN(limex, &ctx->s, &acceptMask, + acceptTable, offset, ctx->callback, ctx->context)) { return MO_HALT_MATCHING; }