fdr: add grey box control for flood detection

src/fdr/fdr_compile.cpp

@@ -74,6 +74,7 @@ namespace {
 class FDRCompiler : boost::noncopyable {
 private:
     const FDREngineDescription ŋ
+    const Grey &grey;
     vector<u8> tab;
     vector<hwlmLiteral> lits;
     map<BucketIndex, std::vector<LiteralIndex> > bucketToLits;
@@ -90,9 +91,9 @@ private:
 
 public:
     FDRCompiler(vector<hwlmLiteral> lits_in, const FDREngineDescription &eng_in,
-                bool make_small_in)
+                bool make_small_in, const Grey &grey_in)
-        : eng(eng_in), tab(eng_in.getTabSizeBytes()), lits(move(lits_in)),
+        : eng(eng_in), grey(grey_in), tab(eng_in.getTabSizeBytes()),
-          make_small(make_small_in) {}
+          lits(move(lits_in)), make_small(make_small_in) {}
 
     aligned_unique_ptr<FDR> build();
 };
@@ -146,7 +147,7 @@ void FDRCompiler::createInitialState(FDR *fdr) {
 aligned_unique_ptr<FDR> FDRCompiler::setupFDR() {
     size_t tabSize = eng.getTabSizeBytes();
 
-    auto floodControlTmp = setupFDRFloodControl(lits, eng);
+    auto floodControlTmp = setupFDRFloodControl(lits, eng, grey);
     auto confirmTmp = setupFullConfs(lits, eng, bucketToLits, make_small);
 
     assert(ISALIGNED_16(tabSize));
@@ -543,7 +544,7 @@ aligned_unique_ptr<FDR> fdrBuildTableInternal(const vector<hwlmLiteral> &lits,
     DEBUG_PRINTF("cpu has %s\n", target.has_avx2() ? "avx2" : "no-avx2");
 
     if (grey.fdrAllowTeddy) {
-        auto fdr = teddyBuildTableHinted(lits, make_small, hint, target);
+        auto fdr = teddyBuildTableHinted(lits, make_small, hint, target, grey);
         if (fdr) {
             DEBUG_PRINTF("build with teddy succeeded\n");
             return fdr;
@@ -566,7 +567,7 @@ aligned_unique_ptr<FDR> fdrBuildTableInternal(const vector<hwlmLiteral> &lits,
         des->stride = 1;
     }
 
-    FDRCompiler fc(lits, *des, make_small);
+    FDRCompiler fc(lits, *des, make_small, grey);
     return fc.build();
 }
 

src/fdr/fdr_compile_internal.h

@@ -55,6 +55,7 @@ typedef u32 PositionInBucket;  // zero is 'we are matching right now!",
 class EngineDescription;
 class FDREngineDescription;
 struct hwlmStreamingControl;
+struct Grey;
 
 std::pair<aligned_unique_ptr<u8>, size_t> setupFullConfs(
     const std::vector<hwlmLiteral> &lits, const EngineDescription &eng,
@@ -67,7 +68,7 @@ std::pair<aligned_unique_ptr<u8>, size_t> setupFullConfs(
 // right state yet to allow blindly advancing
 std::pair<aligned_unique_ptr<u8>, size_t>
 setupFDRFloodControl(const std::vector<hwlmLiteral> &lits,
-                     const EngineDescription &eng);
+                     const EngineDescription &eng, const Grey &grey);
 
 std::pair<aligned_unique_ptr<u8>, size_t>
 fdrBuildTableStreaming(const std::vector<hwlmLiteral> &lits,

src/fdr/flood_compile.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2016, Intel Corporation
+ * Copyright (c) 2015-2017, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -30,6 +30,7 @@
 #include "fdr_confirm.h"
 #include "fdr_compile_internal.h"
 #include "fdr_engine_description.h"
+#include "grey.h"
 #include "ue2common.h"
 #include "util/alloc.h"
 #include "util/bitutils.h"
@@ -92,7 +93,7 @@ void addFlood(vector<FDRFlood> &tmpFlood, u8 c, const hwlmLiteral &lit,
 
 pair<aligned_unique_ptr<u8>, size_t>
 setupFDRFloodControl(const vector<hwlmLiteral> &lits,
-                     const EngineDescription &eng) {
+                     const EngineDescription &eng, const Grey &grey) {
     vector<FDRFlood> tmpFlood(N_CHARS);
     u32 default_suffix = eng.getDefaultFloodSuffixLength();
 
@@ -187,6 +188,14 @@ setupFDRFloodControl(const vector<hwlmLiteral> &lits,
     }
 #endif
 
+    // If flood detection has been switched off in the grey box, we comply by
+    // setting idCount too high for all floods.
+    if (!grey.fdrAllowFlood) {
+        for (auto &fl : tmpFlood) {
+            fl.idCount = FDR_FLOOD_MAX_IDS;
+        }
+    }
+
     map<FDRFlood, CharReach, FloodComparator> flood2chars;
     for (u32 i = 0; i < N_CHARS; i++) {
         FDRFlood fl = tmpFlood[i];

src/fdr/teddy_compile.cpp

@@ -31,6 +31,7 @@
 #include "fdr_compile_internal.h"
 #include "fdr_confirm.h"
 #include "fdr_engine_description.h"
+#include "grey.h"
 #include "ue2common.h"
 #include "util/alloc.h"
 #include "util/compare.h"
@@ -66,13 +67,16 @@ namespace {
 
 class TeddyCompiler : boost::noncopyable {
     const TeddyEngineDescription ŋ
+    const Grey &grey;
     const vector<hwlmLiteral> &lits;
     bool make_small;
 
 public:
     TeddyCompiler(const vector<hwlmLiteral> &lits_in,
-                  const TeddyEngineDescription &eng_in, bool make_small_in)
+                  const TeddyEngineDescription &eng_in, bool make_small_in,
-        : eng(eng_in), lits(lits_in), make_small(make_small_in) {}
+                  const Grey &grey_in)
+        : eng(eng_in), grey(grey_in), lits(lits_in), make_small(make_small_in) {
+    }
 
     aligned_unique_ptr<FDR> build();
     bool pack(map<BucketIndex, std::vector<LiteralIndex> > &bucketToLits);
@@ -307,7 +311,7 @@ aligned_unique_ptr<FDR> TeddyCompiler::build() {
 
     size_t maskLen = eng.numMasks * 16 * 2 * maskWidth;
 
-    auto floodControlTmp = setupFDRFloodControl(lits, eng);
+    auto floodControlTmp = setupFDRFloodControl(lits, eng, grey);
     auto confirmTmp = setupFullConfs(lits, eng, bucketToLits, make_small);
 
     size_t size = ROUNDUP_N(sizeof(Teddy) +
@@ -417,7 +421,8 @@ aligned_unique_ptr<FDR> TeddyCompiler::build() {
 
 aligned_unique_ptr<FDR> teddyBuildTableHinted(const vector<hwlmLiteral> &lits,
                                               bool make_small, u32 hint,
-                                              const target_t &target) {
+                                              const target_t &target,
+                                              const Grey &grey) {
     unique_ptr<TeddyEngineDescription> des;
     if (hint == HINT_INVALID) {
         des = chooseTeddyEngine(target, lits);
@@ -427,7 +432,7 @@ aligned_unique_ptr<FDR> teddyBuildTableHinted(const vector<hwlmLiteral> &lits,
     if (!des) {
         return nullptr;
     }
-    TeddyCompiler tc(lits, *des, make_small);
+    TeddyCompiler tc(lits, *des, make_small, grey);
     return tc.build();
 }
 

src/fdr/teddy_compile.h

@@ -43,11 +43,12 @@ struct target_t;
 
 namespace ue2 {
 
+struct Grey;
 struct hwlmLiteral;
 
 ue2::aligned_unique_ptr<FDR>
 teddyBuildTableHinted(const std::vector<hwlmLiteral> &lits, bool make_small,
-                      u32 hint, const target_t &target);
+                      u32 hint, const target_t &target, const Grey &grey);
 
 } // namespace ue2
 

src/grey.cpp

@@ -63,6 +63,7 @@ Grey::Grey(void) :
                    allowDecoratedLiteral(true),
                    allowNoodle(true),
                    fdrAllowTeddy(true),
+                   fdrAllowFlood(true),
                    violetAvoidSuffixes(true),
                    violetAvoidWeakInfixes(true),
                    violetDoubleCut(true),
@@ -226,6 +227,7 @@ void applyGreyOverrides(Grey *g, const string &s) {
         G_UPDATE(allowDecoratedLiteral);
         G_UPDATE(allowNoodle);
         G_UPDATE(fdrAllowTeddy);
+        G_UPDATE(fdrAllowFlood);
         G_UPDATE(violetAvoidSuffixes);
         G_UPDATE(violetAvoidWeakInfixes);
         G_UPDATE(violetDoubleCut);

src/grey.h

@@ -64,6 +64,7 @@ struct Grey {
 
     bool allowNoodle;
     bool fdrAllowTeddy;
+    bool fdrAllowFlood;
 
     u32  violetAvoidSuffixes; /* 0=never, 1=sometimes, 2=always */
     bool violetAvoidWeakInfixes;