Initial commit of Hyperscan

src/fdr/fdr_engine_description.cpp

@@ -0,0 +1,216 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fdr_compile_internal.h"
+#include "fdr_engine_description.h"
+#include "hs_compile.h"
+#include "util/target_info.h"
+#include "util/compare.h" // for ourisalpha()
+#include "util/make_unique.h"
+
+#include <cassert>
+#include <cstdlib>
+#include <map>
+#include <string>
+
+using namespace std;
+
+namespace ue2 {
+
+#include "fdr_autogen_compiler.cpp"
+
+FDREngineDescription::FDREngineDescription(const FDREngineDef &def)
+    : EngineDescription(def.id, targetByArchFeatures(def.cpu_features),
+                        def.numBuckets, def.confirmPullBackDistance,
+                        def.confirmTopLevelSplit),
+      schemeWidth(def.schemeWidth), stride(def.stride), bits(def.bits) {}
+
+u32 FDREngineDescription::getDefaultFloodSuffixLength() const {
+    // rounding up, so that scheme width 32 and 6 buckets is 6 not 5!
+    // the +1 avoids pain due to various reach choices
+    return ((getSchemeWidth() + getNumBuckets() - 1) / getNumBuckets()) + 1;
+}
+
+static
+u32 findDesiredStride(size_t num_lits, size_t min_len, size_t min_len_count) {
+    u32 desiredStride = 1; // always our safe fallback
+    if (min_len > 1) {
+        if (num_lits < 250) {
+            // small cases we just go for it
+            desiredStride = min_len;
+        } else if (num_lits < 800) {
+            // intermediate cases
+            desiredStride = min_len - 1;
+        } else if (num_lits < 5000) {
+            // for larger but not huge sizes, go to stride 2 only if we have at
+            // least minlen 3
+            desiredStride = MIN(min_len - 1, 2);
+        }
+    }
+
+    // patch if count is quite large - a ton of length 2 literals can
+    // break things
+#ifdef TRY_THIS_LATER
+    if ((min_len == 2) && (desiredStride == 2) && (min_len_count > 20)) {
+        desiredStride = 1;
+    }
+#endif
+
+    // patch stuff just for the stride 4 case; don't let min_len=4,
+    // desiredStride=4 through as even a few length 4 literals can break things
+    // (far more fragile)
+    if ((min_len == 4) && (desiredStride == 4) && (min_len_count > 2)) {
+        desiredStride = 2;
+    }
+
+    return desiredStride;
+}
+
+unique_ptr<FDREngineDescription> chooseEngine(const target_t &target,
+                                              const vector<hwlmLiteral> &vl,
+                                              bool make_small) {
+    vector<FDREngineDescription> allDescs;
+    getFdrDescriptions(&allDescs);
+
+    // find desired stride
+    size_t count;
+    size_t msl = minLenCount(vl, &count);
+    u32 desiredStride = findDesiredStride(vl.size(), msl, count);
+
+    DEBUG_PRINTF("%zu lits, msl=%zu, desiredStride=%u\n", vl.size(), msl,
+                 desiredStride);
+
+    const FDREngineDescription *best = nullptr;
+    u32 best_score = 0;
+
+    for (size_t engineID = 0; engineID < allDescs.size(); engineID++) {
+        const FDREngineDescription &eng = allDescs[engineID];
+        if (!eng.isValidOnTarget(target)) {
+            continue;
+        }
+        if (msl < eng.stride) {
+            continue;
+        }
+
+        u32 score = 100;
+
+        score -= absdiff(desiredStride, eng.stride);
+
+        if (eng.stride <= desiredStride) {
+            score += eng.stride;
+        }
+
+        u32 effLits = vl.size(); /* * desiredStride;*/
+        u32 ideal;
+        if (effLits < eng.getNumBuckets()) {
+            if (eng.stride == 1) {
+                ideal = 8;
+            } else {
+                ideal = 10;
+            }
+        } else if (effLits < 20) {
+            ideal = 10;
+        } else if (effLits < 100) {
+            ideal = 11;
+        } else if (effLits < 1000) {
+            ideal = 12;
+        } else if (effLits < 10000) {
+            ideal = 13;
+        } else {
+            ideal = 15;
+        }
+
+        if (ideal != 8 && eng.schemeWidth == 32) {
+            ideal += 1;
+        }
+
+        if (make_small) {
+            ideal -= 2;
+        }
+
+        if (eng.stride > 1) {
+            ideal++;
+        }
+
+        DEBUG_PRINTF("effLits %u\n", effLits);
+
+        if (target.is_atom_class() && !make_small && effLits < 4000) {
+            /* Unless it is a very heavy case, we want to build smaller tables
+             * on lightweight machines due to their small caches. */
+            ideal -= 2;
+        }
+
+        score -= absdiff(ideal, eng.bits);
+
+        DEBUG_PRINTF("fdr %u: width=%u, bits=%u, buckets=%u, stride=%u "
+                     "-> score=%u\n",
+                     eng.getID(), eng.schemeWidth, eng.bits,
+                     eng.getNumBuckets(), eng.stride, score);
+
+        if (!best || score > best_score) {
+            best = &eng;
+            best_score = score;
+        }
+    }
+
+    if (!best) {
+        DEBUG_PRINTF("failed to find engine\n");
+        return nullptr;
+    }
+
+    DEBUG_PRINTF("using engine %u\n", best->getID());
+    return ue2::make_unique<FDREngineDescription>(*best);
+}
+
+SchemeBitIndex FDREngineDescription::getSchemeBit(BucketIndex b,
+                                                  PositionInBucket p) const {
+    assert(p < getBucketWidth(b));
+    SchemeBitIndex sbi = p * getNumBuckets() + b;
+    assert(sbi < getSchemeWidth());
+    return sbi;
+}
+
+u32 FDREngineDescription::getBucketWidth(BucketIndex) const {
+    u32 sw = getSchemeWidth();
+    u32 nm = getNumBuckets();
+    assert(sw % nm == 0);
+    return sw/nm;
+}
+
+unique_ptr<FDREngineDescription> getFdrDescription(u32 engineID) {
+    vector<FDREngineDescription> allDescs;
+    getFdrDescriptions(&allDescs);
+
+    if (engineID >= allDescs.size()) {
+        return nullptr;
+    }
+
+    return ue2::make_unique<FDREngineDescription>(allDescs[engineID]);
+}
+
+} // namespace ue2