FDR: Squash buckets of included literals in FDR confirm

- Change the compile of literal matchers to two passes.
 - Reverse the bucket assignment in FDR, bucket with longer literals has
   smaller bucket id.
 - Squash the buckets of included literals and jump to the the program of
   included literals directly from parent literal program without going
   through FDR confirm for included iterals.

src/hwlm/hwlm_build.cpp

@@ -41,8 +41,12 @@
 #include "scratch.h"
 #include "ue2common.h"
 #include "fdr/fdr_compile.h"
+#include "fdr/fdr_compile_internal.h"
+#include "fdr/fdr_engine_description.h"
+#include "fdr/teddy_engine_description.h"
 #include "util/compile_context.h"
 #include "util/compile_error.h"
+#include "util/make_unique.h"
 #include "util/ue2string.h"
 
 #include <cassert>
@@ -53,6 +57,28 @@ using namespace std;
 
 namespace ue2 {
 
+HWLMProto::HWLMProto(u8 engType_in, vector<hwlmLiteral> lits_in)
+    : engType(engType_in), lits(move(lits_in)) {}
+
+HWLMProto::HWLMProto(u8 engType_in,
+                     unique_ptr<FDREngineDescription> eng_in,
+                     vector<hwlmLiteral> lits_in,
+                     map<u32, vector<u32>> bucketToLits_in,
+                     bool make_small_in)
+    : engType(engType_in), fdrEng(move(eng_in)), lits(move(lits_in)),
+      bucketToLits(move(bucketToLits_in)), make_small(make_small_in) {}
+
+HWLMProto::HWLMProto(u8 engType_in,
+                     unique_ptr<TeddyEngineDescription> eng_in,
+                     vector<hwlmLiteral> lits_in,
+                     map<u32, vector<u32>> bucketToLits_in,
+                     bool make_small_in)
+    : engType(engType_in), teddyEng(move(eng_in)),
+      lits(move(lits_in)),
+      bucketToLits(move(bucketToLits_in)), make_small(make_small_in) {}
+
+HWLMProto::~HWLMProto() {}
+
 static
 void dumpLits(UNUSED const vector<hwlmLiteral> &lits) {
 #ifdef DEBUG
@@ -92,9 +118,52 @@ bool isNoodleable(const vector<hwlmLiteral> &lits,
     return true;
 }
 
-bytecode_ptr<HWLM> hwlmBuild(const vector<hwlmLiteral> &lits, bool make_small,
-                             const CompileContext &cc,
+bytecode_ptr<HWLM> hwlmBuild(const HWLMProto &proto, const CompileContext &cc,
                              UNUSED hwlm_group_t expected_groups) {
+    size_t engSize = 0;
+    shared_ptr<void> eng;
+
+    const auto &lits = proto.lits;
+    DEBUG_PRINTF("building table with %zu strings\n", lits.size());
+
+    if (proto.engType == HWLM_ENGINE_NOOD) {
+        DEBUG_PRINTF("build noodle table\n");
+        const hwlmLiteral &lit = lits.front();
+        auto noodle = noodBuildTable(lit);
+        if (noodle) {
+            engSize = noodle.size();
+        }
+        eng = move(noodle);
+    } else {
+        DEBUG_PRINTF("building a new deal\n");
+        auto fdr = fdrBuildTable(proto, cc.grey);
+        if (fdr) {
+            engSize = fdr.size();
+        }
+        eng = move(fdr);
+    }
+
+    if (!eng) {
+        return nullptr;
+    }
+
+    assert(engSize);
+    if (engSize > cc.grey.limitLiteralMatcherSize) {
+        throw ResourceLimitError();
+    }
+
+    const size_t hwlm_len = ROUNDUP_CL(sizeof(HWLM)) + engSize;
+    auto h = make_zeroed_bytecode_ptr<HWLM>(hwlm_len, 64);
+
+    h->type = proto.engType;
+    memcpy(HWLM_DATA(h.get()), eng.get(), engSize);
+
+    return h;
+}
+
+unique_ptr<HWLMProto>
+hwlmBuildProto(vector<hwlmLiteral> &lits, bool make_small,
+               const CompileContext &cc) {
     assert(!lits.empty());
     dumpLits(lits);
 
@@ -124,9 +193,7 @@ bytecode_ptr<HWLM> hwlmBuild(const vector<hwlmLiteral> &lits, bool make_small,
         }
     }
 
-    u8 engType = 0;
-    size_t engSize = 0;
-    shared_ptr<void> eng;
+    unique_ptr<HWLMProto> proto;
 
     DEBUG_PRINTF("building table with %zu strings\n", lits.size());
 
@@ -134,39 +201,17 @@ bytecode_ptr<HWLM> hwlmBuild(const vector<hwlmLiteral> &lits, bool make_small,
 
     if (isNoodleable(lits, cc)) {
         DEBUG_PRINTF("build noodle table\n");
-        engType = HWLM_ENGINE_NOOD;
-        const hwlmLiteral &lit = lits.front();
-        auto noodle = noodBuildTable(lit);
-        if (noodle) {
-            engSize = noodle.size();
-        }
-        eng = move(noodle);
+        proto = ue2::make_unique<HWLMProto>(HWLM_ENGINE_NOOD, lits);
     } else {
         DEBUG_PRINTF("building a new deal\n");
-        engType = HWLM_ENGINE_FDR;
-        auto fdr = fdrBuildTable(lits, make_small, cc.target_info, cc.grey);
-        if (fdr) {
-            engSize = fdr.size();
+        proto = fdrBuildProto(HWLM_ENGINE_FDR, lits, make_small,
+                              cc.target_info, cc.grey);
+        if (!proto) {
+            return nullptr;
         }
-        eng = move(fdr);
     }
 
-    if (!eng) {
-        return nullptr;
-    }
-
-    assert(engSize);
-    if (engSize > cc.grey.limitLiteralMatcherSize) {
-        throw ResourceLimitError();
-    }
-
-    const size_t hwlm_len = ROUNDUP_CL(sizeof(HWLM)) + engSize;
-    auto h = make_zeroed_bytecode_ptr<HWLM>(hwlm_len, 64);
-
-    h->type = engType;
-    memcpy(HWLM_DATA(h.get()), eng.get(), engSize);
-
-    return h;
+    return proto;
 }
 
 size_t hwlmSize(const HWLM *h) {