multibit, fatbit: make _size build-time only

This commit makes mmbit_size() and fatbit_size compile-time only, and
adds a resource limit for very large multibits.

src/rose/rose_build_bytecode.cpp

@@ -81,6 +81,7 @@
 #include "util/compile_context.h"
 #include "util/compile_error.h"
 #include "util/container.h"
+#include "util/fatbit_build.h"
 #include "util/graph_range.h"
 #include "util/make_unique.h"
 #include "util/multibit_build.h"
@@ -5435,11 +5436,13 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
 
     engine->ekeyCount = rm.numEkeys();
     engine->dkeyCount = rm.numDkeys();
+    engine->dkeyLogSize = fatbit_size(engine->dkeyCount);
     engine->invDkeyOffset = dkeyOffset;
     copy_bytes(ptr + dkeyOffset, rm.getDkeyToReportTable());
 
     engine->somHorizon = ssm.somPrecision();
     engine->somLocationCount = ssm.numSomSlots();
+    engine->somLocationFatbitSize = fatbit_size(engine->somLocationCount);
 
     engine->needsCatchup = bc.needs_catchup ? 1 : 0;
 
@@ -5454,8 +5457,10 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
     engine->activeArrayCount = activeArrayCount;
     engine->activeLeftCount = activeLeftCount;
     engine->queueCount = queue_count;
+    engine->activeQueueArraySize = fatbit_size(queue_count);
     engine->eagerIterOffset = eagerIterOffset;
     engine->handledKeyCount = bc.handledKeys.size();
+    engine->handledKeyFatbitSize = fatbit_size(engine->handledKeyCount);
 
     engine->rolesWithStateCount = bc.numStates;
 
@@ -5475,11 +5480,13 @@ aligned_unique_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
 
     engine->lastByteHistoryIterOffset = lastByteOffset;
 
-    u32 delay_count = verify_u32(final_id_to_literal.size() - delay_base_id);
-    engine->delay_count = delay_count;
+    engine->delay_count =
+        verify_u32(final_id_to_literal.size() - delay_base_id);
+    engine->delay_fatbit_size = fatbit_size(engine->delay_count);
     engine->delay_base_id = delay_base_id;
     engine->anchored_base_id = anchored_base_id;
     engine->anchored_count = delay_base_id - anchored_base_id;
+    engine->anchored_fatbit_size = fatbit_size(engine->anchored_count);
 
     engine->rosePrefixCount = rosePrefixCount;
 

src/rose/rose_dump.cpp

@@ -42,7 +42,7 @@
 #include "nfa/nfa_internal.h"
 #include "nfa/nfa_kind.h"
 #include "util/dump_charclass.h"
-#include "util/multibit_internal.h"
+#include "util/multibit_build.h"
 #include "util/multibit.h"
 
 #include <algorithm>
@@ -1232,8 +1232,10 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
     DUMP_U32(t, historyRequired);
     DUMP_U32(t, ekeyCount);
     DUMP_U32(t, dkeyCount);
+    DUMP_U32(t, dkeyLogSize);
     DUMP_U32(t, invDkeyOffset);
     DUMP_U32(t, somLocationCount);
+    DUMP_U32(t, somLocationFatbitSize);
     DUMP_U32(t, rolesWithStateCount);
     DUMP_U32(t, stateSize);
     DUMP_U32(t, anchorStateSize);
@@ -1258,8 +1260,10 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
     DUMP_U32(t, activeArrayCount);
     DUMP_U32(t, activeLeftCount);
     DUMP_U32(t, queueCount);
+    DUMP_U32(t, activeQueueArraySize);
     DUMP_U32(t, eagerIterOffset);
     DUMP_U32(t, handledKeyCount);
+    DUMP_U32(t, handledKeyFatbitSize);
     DUMP_U32(t, leftOffset);
     DUMP_U32(t, roseCount);
     DUMP_U32(t, lookaroundTableOffset);
@@ -1280,8 +1284,10 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
     DUMP_U64(t, floating_group_mask);
     DUMP_U32(t, size);
     DUMP_U32(t, delay_count);
+    DUMP_U32(t, delay_fatbit_size);
     DUMP_U32(t, delay_base_id);
     DUMP_U32(t, anchored_count);
+    DUMP_U32(t, anchored_fatbit_size);
     DUMP_U32(t, anchored_base_id);
     DUMP_U32(t, maxFloatingDelayedMatch);
     DUMP_U32(t, delayRebuildLength);

src/rose/rose_internal.h

@@ -309,9 +309,11 @@ struct RoseEngine {
     u32 historyRequired; /**< max amount of history required for streaming */
     u32 ekeyCount; /**< number of exhaustion keys */
     u32 dkeyCount; /**< number of dedupe keys */
+    u32 dkeyLogSize; /**< size of fatbit for storing dkey log (bytes) */
     u32 invDkeyOffset; /**< offset to table mapping from dkeys to the external
                          *  report ids */
     u32 somLocationCount; /**< number of som locations required */
+    u32 somLocationFatbitSize; /**< size of SOM location fatbit (bytes) */
     u32 rolesWithStateCount; // number of roles with entries in state bitset
     u32 stateSize; /* size of the state bitset
                     * WARNING: not the size of the rose state */
@@ -370,14 +372,18 @@ struct RoseEngine {
     u32 activeArrayCount; //number of nfas tracked in the active array
     u32 activeLeftCount; //number of nfas tracked in the active rose array
     u32 queueCount;      /**< number of nfa queues */
+    u32 activeQueueArraySize; //!< size of fatbit for active queues (bytes)
 
     u32 eagerIterOffset; /**< offset to sparse iter for eager prefixes or 0 if
                           * none */
 
     /** \brief Number of keys used by CHECK_SET_HANDLED instructions in role
-     * programs. Used to size the handled_roles fatbit in scratch. */
+     * programs. */
     u32 handledKeyCount;
 
+    /** \brief Size of the handled keys fatbit in scratch (bytes). */
+    u32 handledKeyFatbitSize;
+
     u32 leftOffset;
     u32 roseCount;
     u32 lookaroundTableOffset; //!< base of lookaround offset list (of s8 values)
@@ -412,9 +418,11 @@ struct RoseEngine {
     rose_group floating_group_mask; /* groups that are used by the ftable */
     u32 size; // (bytes)
     u32 delay_count; /* number of delayed literal ids. */
+    u32 delay_fatbit_size; //!< size of each delay fatbit in scratch (bytes)
     u32 delay_base_id; /* literal id of the first delayed literal.
                         * delayed literal ids are contiguous */
     u32 anchored_count; /* number of anchored literal ids */
+    u32 anchored_fatbit_size; //!< size of each anch fatbit in scratch (bytes)
     u32 anchored_base_id; /* literal id of the first literal in the A table.
                            * anchored literal ids are contiguous */
     u32 maxFloatingDelayedMatch; /* max offset that a delayed literal can