Initial commit of Hyperscan

src/hwlm/hwlm.c

@@ -0,0 +1,240 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Hamster Wheel Literal Matcher: runtime.
+ */
+#include "hwlm.h"
+#include "hwlm_internal.h"
+#include "noodle_engine.h"
+#include "scratch.h"
+#include "ue2common.h"
+#include "fdr/fdr.h"
+#include "nfa/accel.h"
+#include "nfa/shufti.h"
+#include "nfa/vermicelli.h"
+#include <string.h>
+
+#define MIN_ACCEL_LEN_BLOCK  16
+#define MIN_ACCEL_LEN_STREAM 16
+
+static really_inline
+const u8 *run_hwlm_accel(const union AccelAux *aux, const u8 *ptr,
+                         const u8 *end) {
+    switch (aux->accel_type) {
+    case ACCEL_VERM:
+        DEBUG_PRINTF("single vermicelli for 0x%02hhx\n", aux->verm.c);
+        return vermicelliExec(aux->verm.c, 0, ptr, end);
+    case ACCEL_VERM_NOCASE:
+        DEBUG_PRINTF("single vermicelli-nocase for 0x%02hhx\n", aux->verm.c);
+        return vermicelliExec(aux->verm.c, 1, ptr, end);
+    case ACCEL_DVERM:
+        DEBUG_PRINTF("double vermicelli for 0x%02hhx%02hhx\n", aux->dverm.c1,
+                     aux->dverm.c2);
+        return vermicelliDoubleExec(aux->dverm.c1, aux->dverm.c2, 0, ptr, end);
+    case ACCEL_DVERM_NOCASE:
+        DEBUG_PRINTF("double vermicelli-nocase for 0x%02hhx%02hhx\n",
+                     aux->dverm.c1, aux->dverm.c2);
+        return vermicelliDoubleExec(aux->dverm.c1, aux->dverm.c2, 1, ptr, end);
+    case ACCEL_SHUFTI:
+        DEBUG_PRINTF("single shufti\n");
+        return shuftiExec(aux->shufti.lo, aux->shufti.hi, ptr, end);
+    default:
+        /* no acceleration, fall through and return current ptr */
+        return ptr;
+    }
+}
+
+static really_inline
+void do_accel_block(const union AccelAux *aux, const u8 *buf, size_t len,
+                    size_t *start) {
+    if (len - *start < MIN_ACCEL_LEN_BLOCK) {
+        return;
+    }
+
+    const u8 *ptr = buf + *start;
+    const u8 *end = buf + len;
+    const u8 offset = aux->generic.offset;
+    ptr = run_hwlm_accel(aux, ptr, end);
+
+    if (offset) {
+        ptr -= offset;
+        if (ptr < buf) {
+            ptr = buf;
+        }
+    }
+    assert(ptr >= buf);
+    *start = ptr - buf;
+}
+
+static really_inline
+int inaccurate_accel(u8 type) {
+    /* accels which don't always catch up to the boundary
+     * DSHUFTI is also inaccurate but it is not used by the hamsters */
+    return type == ACCEL_DVERM_NOCASE || type == ACCEL_DVERM;
+}
+
+static never_inline
+void do_accel_streaming(const union AccelAux *aux, const u8 *hbuf, size_t hlen,
+                        const u8 *buf, size_t len, size_t *start) {
+    if (aux->accel_type == ACCEL_NONE || len - *start < MIN_ACCEL_LEN_STREAM) {
+        return;
+    }
+
+    const u8 offset = aux->generic.offset;
+
+    DEBUG_PRINTF("using accel %hhu offset %hhu\n", aux->accel_type, offset);
+
+    // Scan history buffer, but only if the start offset (which always refers to
+    // buf) is zero.
+
+    if (!*start && hlen) {
+        const u8 *ptr1 = hbuf;
+        const u8 *end1 = hbuf + hlen;
+        if (hlen >= 16) {
+            ptr1 = run_hwlm_accel(aux, ptr1, end1);
+        }
+
+        if ((hlen <= 16 || inaccurate_accel(aux->accel_type))
+            && end1 != ptr1 && end1 - ptr1 <= 16) {
+            DEBUG_PRINTF("already scanned %zu/%zu\n", ptr1 - hbuf, hlen);
+            /* see if we can finish off the history buffer completely */
+            u8 ALIGN_DIRECTIVE temp[17];
+            ptrdiff_t tlen = end1 - ptr1;
+            memcpy(temp, ptr1, tlen);
+            memset(temp + tlen, 0, 17 - tlen);
+            if (len) { /* for dverm */
+                temp[end1 - ptr1] = *buf;
+            }
+
+            const u8 *tempp = run_hwlm_accel(aux, temp, temp + 17);
+
+            if (tempp - temp >= tlen) {
+                ptr1 = end1;
+            }
+            DEBUG_PRINTF("got %zu\n", tempp - temp);
+        }
+
+        if (ptr1 != end1) {
+            DEBUG_PRINTF("bailing in history\n");
+            return;
+        }
+    }
+
+    DEBUG_PRINTF("scanning main buffer, start=%zu, len=%zu\n", *start, len);
+
+    const u8 *ptr2 = buf + *start;
+    const u8 *end2 = buf + len;
+
+    const u8 *found = run_hwlm_accel(aux, ptr2, end2);
+
+    if (found >= ptr2 + offset) {
+        size_t delta = found - offset - ptr2;
+        DEBUG_PRINTF("got %zu/%zu in 2nd buffer\n", delta, len);
+        *start += delta;
+    } else if (hlen) {
+        UNUSED size_t remaining = offset + ptr2 - found;
+        DEBUG_PRINTF("got %zu/%zu remaining in 1st buffer\n", remaining, hlen);
+    }
+}
+
+hwlm_error_t hwlmExec(const struct HWLM *t, const u8 *buf, size_t len,
+                      size_t start, HWLMCallback cb, void *ctxt,
+                      hwlm_group_t groups) {
+    DEBUG_PRINTF("buf len=%zu, start=%zu, groups=%llx\n", len, start, groups);
+    if (!groups) {
+        DEBUG_PRINTF("groups all off\n");
+        return HWLM_SUCCESS;
+    }
+
+    assert(start < len);
+
+    if (t->type == HWLM_ENGINE_NOOD) {
+        DEBUG_PRINTF("calling noodExec\n");
+        return noodExec(HWLM_C_DATA(t), buf + start, len - start, start, cb,
+                        ctxt);
+    } else {
+        assert(t->type == HWLM_ENGINE_FDR);
+        const union AccelAux *aa = &t->accel0;
+        if ((groups & ~t->accel1_groups) == 0) {
+            DEBUG_PRINTF("using hq accel %hhu\n", t->accel1.accel_type);
+            aa = &t->accel1;
+        }
+        do_accel_block(aa, buf, len, &start);
+        DEBUG_PRINTF("calling frankie (groups=%08llx, start=%zu)\n", groups,
+                     start);
+        return fdrExec(HWLM_C_DATA(t), buf, len, start, cb, ctxt, groups);
+    }
+}
+
+hwlm_error_t hwlmExecStreaming(const struct HWLM *t, struct hs_scratch *scratch,
+                               size_t len, size_t start, HWLMCallback cb,
+                               void *ctxt, hwlm_group_t groups,
+                               u8 *stream_state) {
+    const u8 *hbuf = scratch->core_info.hbuf;
+    const size_t hlen = scratch->core_info.hlen;
+    const u8 *buf = scratch->core_info.buf;
+
+    DEBUG_PRINTF("hbuf len=%zu, buf len=%zu, start=%zu, groups=%llx\n", hlen,
+                 len, start, groups);
+
+    if (!groups) {
+        return HWLM_SUCCESS;
+    }
+
+    assert(start < len);
+
+    if (t->type == HWLM_ENGINE_NOOD) {
+        DEBUG_PRINTF("calling noodExec\n");
+        // If we've been handed a start offset, we can use a block mode scan at
+        // that offset.
+        if (start) {
+            return noodExec(HWLM_C_DATA(t), buf + start, len - start, start,
+                            cb, ctxt);
+        } else {
+            return noodExecStreaming(HWLM_C_DATA(t), hbuf, hlen, buf, len, cb,
+                                     ctxt, scratch->fdr_temp_buf,
+                                     FDR_TEMP_BUF_SIZE);
+        }
+    } else {
+        // t->type == HWLM_ENGINE_FDR
+        const union AccelAux *aa = &t->accel0;
+        if ((groups & ~t->accel1_groups) == 0) {
+            DEBUG_PRINTF("using hq accel %hhu\n", t->accel1.accel_type);
+            aa = &t->accel1;
+        }
+        // if no active stream state, use acceleration
+        if (!fdrStreamStateActive(HWLM_C_DATA(t), stream_state)) {
+            do_accel_streaming(aa, hbuf, hlen, buf, len, &start);
+        }
+        DEBUG_PRINTF("calling frankie (groups=%08llx, start=%zu)\n", groups,
+                     start);
+        return fdrExecStreaming(HWLM_C_DATA(t), hbuf, hlen, buf, len,
+                                start, cb, ctxt, groups, stream_state);
+    }
+}

src/hwlm/hwlm.h

@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Hamster Wheel Literal Matcher: runtime API.
+ */
+
+#ifndef HWLM_H
+#define HWLM_H
+
+#include "ue2common.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/** \brief Error return type for exec functions. */
+typedef int hwlm_error_t;
+
+/** \brief Type representing a set of groups as a bitmap. */
+typedef u64a hwlm_group_t;
+
+/** \brief HWLM callback return type. */
+typedef hwlm_group_t hwlmcb_rv_t;
+
+/** \brief Value representing all possible literal groups. */
+#define HWLM_ALL_GROUPS         ((hwlm_group_t)~0ULL)
+
+/** \brief Callback return value indicating that we should continue matching. */
+#define HWLM_CONTINUE_MATCHING  HWLM_ALL_GROUPS
+
+/** \brief Callback return value indicating that we should halt matching. */
+#define HWLM_TERMINATE_MATCHING 0
+
+/** \brief Matching finished without being terminated by the user. */
+#define HWLM_SUCCESS       0
+
+/** \brief The user terminated matching by returning HWLM_TERMINATE_MATCHING
+ * from the match callback. */
+#define HWLM_TERMINATED    1
+
+/** \brief An error occurred during matching.
+ *
+ * This should only be used if an unsupported engine was called (like one
+ * designed for a different architecture). */
+#define HWLM_ERROR_UNKNOWN 2
+
+struct hs_scratch;
+struct HWLM;
+
+/** \brief The type for an HWLM callback.
+ *
+ * This callback receives a start-of-match offset, an end-of-match offset, the
+ * ID of the match and the context pointer that was passed into \ref
+ * hwlmExec or \ref hwlmExecStreaming.
+ *
+ * A callback return of \ref HWLM_TERMINATE_MATCHING will stop matching.
+ *
+ * A callback return of \ref HWLM_CONTINUE_MATCHING continues matching.
+ *
+ * An arbitrary group mask may be given as the return value. This will be taken
+ * as a hint by the underlying engine that only literals with groups
+ * overlapping the provided mask need to be reported.
+ *
+ * The underlying engine may choose not to report a match if there is no group
+ * belonging to the literal which was active at the when the end match location
+ * was first reached.
+ */
+typedef hwlmcb_rv_t (*HWLMCallback)(size_t start, size_t end, u32 id,
+                                    void *context);
+
+/** \brief Match strings in table.
+ *
+ * If a match occurs, the callback function given will be called with the index
+ * of the last character in the string and the \p context (passed through
+ * without interpretation).
+ *
+ * Returns \ref HWLM_TERMINATED if scanning is cancelled due to the callback
+ * returning \ref HWLM_TERMINATE_MATCHING.
+ *
+ * \p start is the first offset at which a match may start.
+ *
+ * The underlying engine may choose not to report any match which starts before
+ * the first possible match of a literal which is in the initial group mask.
+ */
+hwlm_error_t hwlmExec(const struct HWLM *tab, const u8 *buf, size_t len,
+                      size_t start, HWLMCallback callback, void *context,
+                      hwlm_group_t groups);
+
+/** \brief As for \ref hwlmExec, but a streaming case across two buffers.
+ *
+ * \p scratch is used to access fdr_temp_buf and to access the history buffer,
+ * history length and the main buffer.
+ *
+ * \p len is the length of the main buffer to be scanned.
+ *
+ * \p start is an advisory hint representing the first offset at which a match
+ * may start. Some underlying literal matches may not respect it.
+ *
+ * Two buffers/lengths are provided. Matches that occur entirely within
+ * the history buffer will not be reported by this function. The offsets
+ * reported for the main buffer are relative to the start of that buffer (a
+ * match at byte 10 of the main buffer is reported as 10). Matches that start
+ * in the history buffer will have starts reported with 'negative' values.
+ */
+hwlm_error_t hwlmExecStreaming(const struct HWLM *tab,
+                               struct hs_scratch *scratch, size_t len,
+                               size_t start, HWLMCallback callback,
+                               void *context, hwlm_group_t groups,
+                               u8 *stream_state);
+
+#ifdef __cplusplus
+}       /* extern "C" */
+#endif
+
+#endif

src/hwlm/hwlm_build.cpp

@@ -0,0 +1,635 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Hamster Wheel Literal Matcher: build code.
+ */
+#include "grey.h"
+#include "hwlm.h"
+#include "hwlm_build.h"
+#include "hwlm_internal.h"
+#include "noodle_engine.h"
+#include "noodle_build.h"
+#include "ue2common.h"
+#include "fdr/fdr_compile.h"
+#include "fdr/fdr.h"
+#include "nfa/shufticompile.h"
+#include "util/alloc.h"
+#include "util/bitutils.h"
+#include "util/charreach.h"
+#include "util/compare.h"
+#include "util/compile_context.h"
+#include "util/compile_error.h"
+#include "util/dump_charclass.h"
+#include "util/target_info.h"
+#include "util/ue2string.h"
+#include "util/verify_types.h"
+
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <vector>
+
+using namespace std;
+
+namespace ue2 {
+
+static const unsigned int MAX_ACCEL_OFFSET = 16;
+static const unsigned int MAX_SHUFTI_WIDTH = 240;
+
+static
+bool findDVerm(const vector<const hwlmLiteral *> &lits, AccelAux *aux) {
+    const hwlmLiteral &first = *lits.front();
+
+    struct candidate {
+        candidate(void)
+            : c1(0), c2(0), max_offset(0), b5insens(false), valid(false) {}
+        candidate(const hwlmLiteral &base, u32 offset)
+            : c1(base.s[offset]), c2(base.s[offset + 1]), max_offset(0),
+              b5insens(false), valid(true) {}
+        char c1;
+        char c2;
+        u32 max_offset;
+        bool b5insens;
+        bool valid;
+
+        bool operator>(const candidate &other) const {
+            if (!valid) {
+                return false;
+            }
+
+            if (!other.valid) {
+                return true;
+            }
+
+            if (other.cdiffers() && !cdiffers()) {
+                return false;
+            }
+
+            if (!other.cdiffers() && cdiffers()) {
+                return true;
+            }
+
+            if (!other.b5insens && b5insens) {
+                return false;
+            }
+
+            if (other.b5insens && !b5insens) {
+                return true;
+            }
+
+            if (max_offset > other.max_offset) {
+                return false;
+            }
+
+            return true;
+        }
+
+        bool cdiffers(void) const {
+            if (!b5insens) {
+                return c1 != c2;
+            }
+            return (c1 & CASE_CLEAR) != (c2 & CASE_CLEAR);
+        }
+    };
+
+    candidate best;
+
+    for (u32 i = 0; i < MIN(MAX_ACCEL_OFFSET, first.s.length()) - 1; i++) {
+        candidate curr(first, i);
+
+        /* check to see if this pair appears in each string */
+        for (const auto &lit_ptr : lits) {
+            const hwlmLiteral &lit = *lit_ptr;
+            if (lit.nocase && (ourisalpha(curr.c1) || ourisalpha(curr.c2))) {
+                curr.b5insens = true; /* no choice but to be case insensitive */
+            }
+
+            bool found = false;
+            bool found_nc = false;
+            for (u32 j = 0;
+                 !found && j < MIN(MAX_ACCEL_OFFSET, lit.s.length()) - 1; j++) {
+                found |= curr.c1 == lit.s[j] && curr.c2 == lit.s[j + 1];
+                found_nc |= (curr.c1 & CASE_CLEAR) == (lit.s[j] & CASE_CLEAR)
+                    && (curr.c2 & CASE_CLEAR) == (lit.s[j + 1] & CASE_CLEAR);
+
+                if (curr.b5insens) {
+                    found = found_nc;
+                }
+            }
+
+            if (!curr.b5insens && !found && found_nc) {
+                curr.b5insens = true;
+                found = true;
+            }
+
+            if (!found) {
+                goto next_candidate;
+            }
+        }
+
+        /* check to find the max offset where this appears */
+        for (const auto &lit_ptr : lits) {
+            const hwlmLiteral &lit = *lit_ptr;
+            for (u32 j = 0; j < MIN(MAX_ACCEL_OFFSET, lit.s.length()) - 1;
+                 j++) {
+                bool found = false;
+                if (curr.b5insens) {
+                    found = (curr.c1 & CASE_CLEAR) == (lit.s[j] & CASE_CLEAR)
+                     && (curr.c2 & CASE_CLEAR) == (lit.s[j + 1] & CASE_CLEAR);
+                } else {
+                    found = curr.c1 == lit.s[j] && curr.c2 == lit.s[j + 1];
+                }
+
+                if (found) {
+                    curr.max_offset = MAX(curr.max_offset, j);
+                    break;
+                }
+            }
+        }
+
+        if (curr > best) {
+            best = curr;
+        }
+
+    next_candidate:;
+    }
+
+    if (!best.valid) {
+        return false;
+    }
+
+    aux->dverm.offset = verify_u8(best.max_offset);
+
+    if (!best.b5insens) {
+        aux->dverm.accel_type = ACCEL_DVERM;
+        aux->dverm.c1 = best.c1;
+        aux->dverm.c2 = best.c2;
+        DEBUG_PRINTF("built dverm for %02hhx%02hhx\n",
+                     aux->dverm.c1, aux->dverm.c2);
+    } else {
+        aux->dverm.accel_type = ACCEL_DVERM_NOCASE;
+        aux->dverm.c1 = best.c1 & CASE_CLEAR;
+        aux->dverm.c2 = best.c2 & CASE_CLEAR;
+        DEBUG_PRINTF("built dverm nc for %02hhx%02hhx\n",
+                     aux->dverm.c1, aux->dverm.c2);
+    }
+    return true;
+}
+
+static
+bool findSVerm(const vector<const hwlmLiteral *> &lits, AccelAux *aux) {
+    const hwlmLiteral &first = *lits.front();
+
+    struct candidate {
+        candidate(void)
+            : c(0), max_offset(0), b5insens(false), valid(false) {}
+        candidate(const hwlmLiteral &base, u32 offset)
+            : c(base.s[offset]), max_offset(0),
+              b5insens(false), valid(true) {}
+        char c;
+        u32 max_offset;
+        bool b5insens;
+        bool valid;
+
+        bool operator>(const candidate &other) const {
+            if (!valid) {
+                return false;
+            }
+
+            if (!other.valid) {
+                return true;
+            }
+
+            if (!other.b5insens && b5insens) {
+                return false;
+            }
+
+            if (other.b5insens && !b5insens) {
+                return true;
+            }
+
+            if (max_offset > other.max_offset) {
+                return false;
+            }
+
+            return true;
+        }
+    };
+
+    candidate best;
+
+    for (u32 i = 0; i < MIN(MAX_ACCEL_OFFSET, first.s.length()); i++) {
+        candidate curr(first, i);
+
+        /* check to see if this pair appears in each string */
+        for (const auto &lit_ptr : lits) {
+            const hwlmLiteral &lit = *lit_ptr;
+            if (lit.nocase && ourisalpha(curr.c)) {
+                curr.b5insens = true; /* no choice but to be case insensitive */
+            }
+
+            bool found = false;
+            bool found_nc = false;
+            for (u32 j = 0;
+                 !found && j < MIN(MAX_ACCEL_OFFSET, lit.s.length()); j++) {
+                found |= curr.c == lit.s[j];
+                found_nc |= (curr.c & CASE_CLEAR) == (lit.s[j] & CASE_CLEAR);
+
+                if (curr.b5insens) {
+                    found = found_nc;
+                }
+            }
+
+            if (!curr.b5insens && !found && found_nc) {
+                curr.b5insens = true;
+                found = true;
+            }
+
+            if (!found) {
+                goto next_candidate;
+            }
+        }
+
+        /* check to find the max offset where this appears */
+        for (const auto &lit_ptr : lits) {
+            const hwlmLiteral &lit = *lit_ptr;
+            for (u32 j = 0; j < MIN(MAX_ACCEL_OFFSET, lit.s.length()); j++) {
+                bool found = false;
+                if (curr.b5insens) {
+                    found = (curr.c & CASE_CLEAR) == (lit.s[j] & CASE_CLEAR);
+                } else {
+                    found = curr.c == lit.s[j];
+                }
+
+                if (found) {
+                    curr.max_offset = MAX(curr.max_offset, j);
+                    break;
+                }
+            }
+        }
+
+        if (curr > best) {
+            best = curr;
+        }
+
+    next_candidate:;
+    }
+
+    if (!best.valid) {
+        return false;
+    }
+
+    if (!best.b5insens) {
+        aux->verm.accel_type = ACCEL_VERM;
+        aux->verm.c = best.c;
+        DEBUG_PRINTF("built verm for %02hhx\n", aux->verm.c);
+    } else {
+        aux->verm.accel_type = ACCEL_VERM_NOCASE;
+        aux->verm.c = best.c & CASE_CLEAR;
+        DEBUG_PRINTF("built verm nc for %02hhx\n", aux->verm.c);
+    }
+    aux->verm.offset = verify_u8(best.max_offset);
+
+    return true;
+}
+
+static
+void filterLits(const vector<hwlmLiteral> &lits, hwlm_group_t expected_groups,
+                vector<const hwlmLiteral *> *filtered_lits, u32 *min_len) {
+    *min_len = MAX_ACCEL_OFFSET;
+
+    for (const auto &lit : lits) {
+        if (!(lit.groups & expected_groups)) {
+            continue;
+        }
+
+        const size_t lit_len = lit.s.length();
+        if (lit_len < *min_len) {
+            *min_len = verify_u32(lit_len);
+        }
+
+        filtered_lits->push_back(&lit);
+
+#ifdef DEBUG
+        DEBUG_PRINTF("lit:");
+        for (u32 i = 0; i < lit.s.length(); i++) {
+            printf("%02hhx", lit.s[i]);
+        }
+        printf("\n");
+#endif
+    }
+}
+
+static
+void findForwardAccelScheme(const vector<hwlmLiteral> &lits,
+                            hwlm_group_t expected_groups, AccelAux *aux) {
+    DEBUG_PRINTF("building accel expected=%016llx\n", expected_groups);
+    u32 min_len = MAX_ACCEL_OFFSET;
+    vector<const hwlmLiteral *> filtered_lits;
+
+    filterLits(lits, expected_groups, &filtered_lits, &min_len);
+    if (filtered_lits.empty()) {
+        return;
+    }
+
+    if (findDVerm(filtered_lits, aux)
+        || findSVerm(filtered_lits, aux)) {
+        return;
+    }
+
+    vector<CharReach> reach(MAX_ACCEL_OFFSET, CharReach());
+    for (const auto &lit : lits) {
+        if (!(lit.groups & expected_groups)) {
+            continue;
+        }
+
+        for (u32 i = 0; i < MAX_ACCEL_OFFSET && i < lit.s.length(); i++) {
+            unsigned char c = lit.s[i];
+            if (lit.nocase) {
+                DEBUG_PRINTF("adding %02hhx to %u\n", mytoupper(c), i);
+                DEBUG_PRINTF("adding %02hhx to %u\n", mytolower(c), i);
+                reach[i].set(mytoupper(c));
+                reach[i].set(mytolower(c));
+            } else {
+                DEBUG_PRINTF("adding %02hhx to %u\n", c, i);
+                reach[i].set(c);
+            }
+        }
+    }
+
+    u32 min_count = ~0U;
+    u32 min_offset = ~0U;
+    for (u32 i = 0; i < min_len; i++) {
+        size_t count = reach[i].count();
+        DEBUG_PRINTF("offset %u is %s (reach %zu)\n", i,
+                     describeClass(reach[i]).c_str(), count);
+        if (count < min_count) {
+            min_count = (u32)count;
+            min_offset = i;
+        }
+    }
+    assert(min_offset <= min_len);
+
+    if (min_count > MAX_SHUFTI_WIDTH) {
+        DEBUG_PRINTF("min shufti with %u chars is too wide\n", min_count);
+        return;
+    }
+
+    const CharReach &cr = reach[min_offset];
+    if (shuftiBuildMasks(cr, &aux->shufti.lo, &aux->shufti.hi) != -1) {
+        DEBUG_PRINTF("built shufti for %s (%zu chars, offset %u)\n",
+                     describeClass(cr).c_str(), cr.count(), min_offset);
+        aux->shufti.accel_type = ACCEL_SHUFTI;
+        aux->shufti.offset = verify_u8(min_offset);
+        return;
+    }
+
+    DEBUG_PRINTF("fail\n");
+}
+
+static
+void buildForwardAccel(HWLM *h, const vector<hwlmLiteral> &lits,
+                       hwlm_group_t expected_groups) {
+    findForwardAccelScheme(lits, expected_groups, &h->accel1);
+    findForwardAccelScheme(lits, HWLM_ALL_GROUPS, &h->accel0);
+
+    h->accel1_groups = expected_groups;
+}
+
+static
+void dumpLits(UNUSED const vector<hwlmLiteral> &lits) {
+#ifdef DEBUG
+    DEBUG_PRINTF("building lit table for:\n");
+    for (const auto &lit : lits) {
+        printf("\t%u:%016llx %s%s\n", lit.id, lit.groups,
+               escapeString(lit.s).c_str(), lit.nocase ? " (nc)" : "");
+    }
+#endif
+}
+
+#ifndef NDEBUG
+// Called by an assertion.
+static
+bool everyoneHasGroups(const vector<hwlmLiteral> &lits) {
+    for (const auto &lit : lits) {
+        if (!lit.groups) {
+            return false;
+        }
+    }
+    return true;
+}
+#endif
+
+static
+bool isNoodleable(const vector<hwlmLiteral> &lits,
+                  const hwlmStreamingControl *stream_control,
+                  const CompileContext &cc) {
+    if (!cc.grey.allowNoodle) {
+        return false;
+    }
+
+    if (lits.size() != 1) {
+        DEBUG_PRINTF("too many literals for noodle\n");
+        return false;
+    }
+
+    if (stream_control) { // nullptr if in block mode
+        if (lits.front().s.length() + 1 > stream_control->history_max) {
+            DEBUG_PRINTF("length of %zu too long for history max %zu\n",
+                         lits.front().s.length(),
+                         stream_control->history_max);
+            return false;
+        }
+    }
+
+    if (!lits.front().msk.empty()) {
+        DEBUG_PRINTF("noodle can't handle supplementary masks\n");
+        return false;
+    }
+
+    return true;
+}
+
+aligned_unique_ptr<HWLM> hwlmBuild(const vector<hwlmLiteral> &lits,
+                                   hwlmStreamingControl *stream_control,
+                                   bool make_small, const CompileContext &cc,
+                                   hwlm_group_t expected_groups) {
+    assert(!lits.empty());
+    dumpLits(lits);
+
+    if (stream_control) {
+        assert(stream_control->history_min <= stream_control->history_max);
+    }
+
+    // Check that we haven't exceeded the maximum number of literals.
+    if (lits.size() > cc.grey.limitLiteralCount) {
+        throw ResourceLimitError();
+    }
+
+    // Safety and resource limit checks.
+    u64a total_chars = 0;
+    for (const auto &lit : lits) {
+        assert(!lit.s.empty());
+
+        if (lit.s.length() > cc.grey.limitLiteralLength) {
+            throw ResourceLimitError();
+        }
+        total_chars += lit.s.length();
+        if (total_chars > cc.grey.limitLiteralMatcherChars) {
+            throw ResourceLimitError();
+        }
+
+        // We do not allow the all-ones ID, as we reserve that for internal use
+        // within literal matchers.
+        if (lit.id == 0xffffffffu) {
+            assert(!"reserved id 0xffffffff used");
+            throw CompileError("Internal error.");
+        }
+    }
+
+    u8 engType = 0;
+    size_t engSize = 0;
+    shared_ptr<void> eng;
+
+    DEBUG_PRINTF("building table with %zu strings\n", lits.size());
+
+    assert(everyoneHasGroups(lits));
+
+    if (isNoodleable(lits, stream_control, cc)) {
+        DEBUG_PRINTF("build noodle table\n");
+        engType = HWLM_ENGINE_NOOD;
+        const hwlmLiteral &lit = lits.front();
+        auto noodle = noodBuildTable((const u8 *)lit.s.c_str(), lit.s.length(),
+                                     lit.nocase, lit.id);
+        if (noodle) {
+            engSize = noodSize(noodle.get());
+        }
+        if (stream_control) {
+            // For now, a single literal still goes to noodle and asks
+            // for a great big history
+            stream_control->literal_history_required = lit.s.length() - 1;
+            assert(stream_control->literal_history_required
+                   <= stream_control->history_max);
+            stream_control->literal_stream_state_required = 0;
+        }
+        eng = move(noodle);
+    } else {
+        DEBUG_PRINTF("building a new deal\n");
+        engType = HWLM_ENGINE_FDR;
+        auto fdr = fdrBuildTable(lits, make_small, cc.target_info, cc.grey,
+                            stream_control);
+        if (fdr) {
+            engSize = fdrSize(fdr.get());
+        }
+        eng = move(fdr);
+    }
+
+    if (!eng) {
+        return nullptr;
+    }
+
+    assert(engSize);
+    if (engSize > cc.grey.limitLiteralMatcherSize) {
+        throw ResourceLimitError();
+    }
+
+    auto h = aligned_zmalloc_unique<HWLM>(ROUNDUP_CL(sizeof(HWLM)) + engSize);
+
+    h->type = engType;
+    memcpy(HWLM_DATA(h.get()), eng.get(), engSize);
+
+    if (engType == HWLM_ENGINE_FDR && cc.grey.hamsterAccelForward) {
+        buildForwardAccel(h.get(), lits, expected_groups);
+    }
+
+    if (stream_control) {
+        DEBUG_PRINTF("requires %zu (of max %zu) bytes of history\n",
+                     stream_control->literal_history_required,
+                     stream_control->history_max);
+        assert(stream_control->literal_history_required
+                    <= stream_control->history_max);
+    }
+
+    return h;
+}
+
+size_t hwlmSize(const HWLM *h) {
+    size_t engSize = 0;
+
+    switch (h->type) {
+    case HWLM_ENGINE_NOOD:
+        engSize = noodSize((const noodTable *)HWLM_C_DATA(h));
+        break;
+    case HWLM_ENGINE_FDR:
+        engSize = fdrSize((const FDR *)HWLM_C_DATA(h));
+        break;
+    }
+
+    if (!engSize) {
+        return 0;
+    }
+
+    return engSize + ROUNDUP_CL(sizeof(*h));
+}
+
+size_t hwlmFloodProneSuffixLen(size_t numLiterals, const CompileContext &cc) {
+    const size_t NO_LIMIT = ~(size_t)0;
+
+    // NOTE: this function contains a number of magic numbers which are
+    // conservative estimates of flood-proneness based on internal details of
+    // the various literal engines that fall under the HWLM aegis. If you
+    // change those engines, you might need to change this function too.
+
+    DEBUG_PRINTF("%zu literals\n", numLiterals);
+
+    if (cc.grey.allowNoodle && numLiterals <= 1) {
+        DEBUG_PRINTF("noodle\n");
+        return NO_LIMIT;
+    }
+
+    if (cc.grey.fdrAllowTeddy) {
+        if (numLiterals <= 48) {
+            DEBUG_PRINTF("teddy\n");
+            return 3;
+        }
+        if (cc.target_info.has_avx2() && numLiterals <= 96) {
+            DEBUG_PRINTF("avx2 teddy\n");
+            return 3;
+        }
+    }
+
+    // TODO: we had thought we could push this value up to 9, but it seems that
+    // hurts performance on floods in some FDR models. Super-conservative for
+    // now.
+    DEBUG_PRINTF("fdr\n");
+    return 3;
+}
+
+} // namespace ue2

src/hwlm/hwlm_build.h

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Hamster Wheel Literal Matcher: build API.
+ */
+
+#ifndef HWLM_BUILD_H
+#define HWLM_BUILD_H
+
+#include "hwlm.h"
+#include "hwlm_literal.h"
+#include "ue2common.h"
+#include "util/alloc.h"
+
+#include <memory>
+#include <vector>
+
+struct HWLM;
+
+namespace ue2 {
+
+struct CompileContext;
+struct Grey;
+struct target_t;
+
+/** \brief Structure gathering together the input/output parameters related to
+ * streaming mode operation. */
+struct hwlmStreamingControl {
+    /** \brief IN parameter: Upper limit on the amount of history that can be
+     * requested. */
+    size_t history_max;
+
+    /** \brief IN parameter: History already known to be used before literal
+     * analysis. */
+    size_t history_min;
+
+    /** \brief OUT parameter: History required by the literal matcher to
+     * correctly match all literals. */
+    size_t literal_history_required;
+
+    /** OUT parameter: Stream state required by literal matcher in bytes. Can
+     * be zero, and generally will be small (0-8 bytes). */
+    size_t literal_stream_state_required;
+};
+
+/** \brief Build an \ref HWLM literal matcher runtime structure for a group of
+ * literals.
+ *
+ * \param lits The group of literals.
+ * \param stream_control Streaming control parameters. If the matcher will
+ *        operate in non-streaming (block) mode, this pointer should be NULL.
+ * \param make_small Optimise matcher for small size.
+ * \param cc Compile context.
+ * \param expected_groups FIXME: document me!
+ *
+ * Build failures are generally a result of memory allocation failure. These
+ * may result in a nullptr return value, or a std::bad_alloc exception being
+ * thrown.
+ */
+aligned_unique_ptr<HWLM>
+hwlmBuild(const std::vector<hwlmLiteral> &lits,
+          hwlmStreamingControl *stream_control, bool make_small,
+          const CompileContext &cc,
+          hwlm_group_t expected_groups = HWLM_ALL_GROUPS);
+
+/**
+ * Returns an estimate of the number of repeated characters on the end of a
+ * literal that will make a literal set of size \a numLiterals suffer
+ * performance degradation.
+ */
+size_t hwlmFloodProneSuffixLen(size_t numLiterals, const CompileContext &cc);
+
+/** \brief Return the size in bytes of an HWLM structure. */
+size_t hwlmSize(const HWLM *h);
+
+} // namespace
+
+#endif // HWLM_BUILD_H

src/hwlm/hwlm_dump.cpp

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Hamster Wheel Literal Matcher: dump code.
+ */
+
+#include "config.h"
+
+#include "hwlm_dump.h"
+#include "hwlm_internal.h"
+#include "noodle_build.h"
+#include "ue2common.h"
+#include "fdr/fdr_dump.h"
+#include "nfa/accel_dump.h"
+
+#include <cstdio>
+
+#ifndef DUMP_SUPPORT
+#error No dump support!
+#endif
+
+namespace ue2 {
+
+void hwlmPrintStats(const HWLM *h, FILE *f) {
+    switch (h->type) {
+    case HWLM_ENGINE_NOOD:
+        noodPrintStats((const noodTable *)HWLM_C_DATA(h), f);
+        break;
+    case HWLM_ENGINE_FDR:
+        fdrPrintStats((const FDR *)HWLM_C_DATA(h), f);
+        break;
+    default:
+        fprintf(f, "<unknown hwlm subengine>\n");
+    }
+
+    fprintf(f, "accel1_groups: %016llx\n", h->accel1_groups);
+
+    fprintf(f, "accel1:");
+    dumpAccelInfo(f, h->accel1);
+    fprintf(f, "accel0:");
+    dumpAccelInfo(f, h->accel0);
+}
+
+} // namespace ue2

src/hwlm/hwlm_dump.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Hamster Wheel Literal Matcher: dump API.
+ */
+
+#ifndef HWLM_DUMP_H
+#define HWLM_DUMP_H
+
+#ifdef DUMP_SUPPORT
+
+#include <cstdio>
+
+struct HWLM;
+
+namespace ue2 {
+
+/** \brief Dump some information about the give HWLM structure. */
+void hwlmPrintStats(const HWLM *h, FILE *f);
+
+} // namespace ue2
+
+#endif
+#endif

src/hwlm/hwlm_internal.h

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Hamster Wheel Literal Matcher: data structures.
+ */
+
+#ifndef HWLM_INTERNAL_H
+#define HWLM_INTERNAL_H
+
+#include "hwlm.h"
+#include "ue2common.h"
+#include "nfa/accel.h"
+
+/** \brief Underlying engine is FDR. */
+#define HWLM_ENGINE_FDR     12
+
+/** \brief Underlying engine is Noodle. */
+#define HWLM_ENGINE_NOOD    16
+
+/** \brief Main Hamster Wheel Literal Matcher header. Followed by
+ * engine-specific structure. */
+struct HWLM {
+    u8 type; /**< HWLM_ENGINE_NOOD or HWLM_ENGINE_FDR */
+    hwlm_group_t accel1_groups; /**< accelerable groups. */
+    union AccelAux accel1; /**< used if group mask is subset of accel1_groups */
+    union AccelAux accel0; /**< fallback accel scheme */
+};
+
+/** \brief Fetch a const pointer to the underlying engine. */
+#define HWLM_C_DATA(p) ((const void *)((const char *)(p)                  \
+                                       + ROUNDUP_CL(sizeof(struct HWLM))))
+
+/** \brief Fetch a pointer to the underlying engine. */
+#define HWLM_DATA(p) ((void *)((char *)(p) + ROUNDUP_CL(sizeof(struct HWLM))))
+
+#endif

src/hwlm/hwlm_literal.cpp

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Hamster Wheel Literal Matcher: literal representation at build time.
+ */
+#include "hwlm_literal.h"
+#include "util/bitutils.h" // for CASE_BIT
+#include "util/compare.h" // for ourisalpha
+#include "util/ue2string.h" // for escapeString
+
+#include <iomanip>
+#include <sstream>
+
+#include <boost/algorithm/cxx11/all_of.hpp>
+
+using namespace std;
+using namespace boost::algorithm;
+
+namespace ue2 {
+
+#ifdef DEBUG
+static UNUSED
+std::string dumpMask(const vector<u8> &v) {
+    ostringstream oss;
+    vector<u8>::const_iterator it, ite;
+    for (it = v.begin(), ite = v.end(); it != ite; ++it) {
+        oss << setfill('0') << setw(2) << hex << (unsigned int)*it;
+    }
+    return oss.str();
+}
+#endif
+
+bool maskIsConsistent(const std::string &s, bool nocase, const vector<u8> &msk,
+                      const vector<u8> &cmp) {
+    string::const_reverse_iterator si = s.rbegin();
+    vector<u8>::const_reverse_iterator mi = msk.rbegin(), ci = cmp.rbegin();
+
+    for (; si != s.rend() && mi != msk.rend(); ++si, ++mi, ++ci) {
+        u8 c = *si, m = *mi, v = *ci;
+        if (nocase && ourisalpha(c)) {
+            m &= ~CASE_BIT;
+            v &= ~CASE_BIT;
+        }
+
+        assert(ci != cmp.rend());
+        if ((c & m) != v) {
+            DEBUG_PRINTF("c = %02hhx; *ci = %02hhx m =%02hhx\n", c, *ci, m);
+            DEBUG_PRINTF("s = %s; dist = %zd\n", s.c_str(), si - s.rbegin());
+            return false;
+        }
+    }
+
+    return true;
+}
+
+/** \brief Complete constructor, takes group information and msk/cmp.
+ *
+ * This constructor takes a msk/cmp pair. Both must be vectors of length <=
+ * \ref HWLM_MASKLEN. */
+hwlmLiteral::hwlmLiteral(const std::string &s_in, bool nocase_in,
+                         bool noruns_in, u32 id_in, hwlm_group_t groups_in,
+                         const vector<u8> &msk_in, const vector<u8> &cmp_in)
+    : s(s_in), id(id_in), nocase(nocase_in), noruns(noruns_in),
+      groups(groups_in), msk(msk_in), cmp(cmp_in) {
+    assert(msk.size() <= HWLM_MASKLEN);
+    assert(msk.size() == cmp.size());
+
+    DEBUG_PRINTF("literal '%s', msk=%s, cmp=%s\n",
+                 escapeString(s).c_str(), dumpMask(msk).c_str(),
+                 dumpMask(cmp).c_str());
+
+    // Mask and compare vectors MUST be the same size.
+    assert(msk.size() == cmp.size());
+
+    // We must have been passed a msk/cmp that can be applied to s.
+    assert(maskIsConsistent(s, nocase, msk, cmp));
+
+    // In the name of good hygiene, zap msk/cmp if msk is all zeroes.
+    if (all_of_equal(msk.begin(), msk.end(), 0)) {
+        msk.clear();
+        cmp.clear();
+    }
+}
+
+} // namespace ue2

src/hwlm/hwlm_literal.h

@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Hamster Wheel Literal Matcher: literal representation at build time.
+ */
+
+#ifndef HWLM_LITERAL_H
+#define HWLM_LITERAL_H
+
+#include "hwlm.h"
+#include "ue2common.h"
+
+#include <string>
+#include <vector>
+
+namespace ue2 {
+
+/** \brief Max length of the hwlmLiteral::msk and hwlmLiteral::cmp vectors. */
+#define HWLM_MASKLEN 8
+
+/** \brief Class representing a literal, fed to \ref hwlmBuild. */
+struct hwlmLiteral {
+    std::string s; //!< \brief The literal itself.
+
+    /** \brief The ID to pass to the callback if this literal matches.
+     *
+     * Note that the special value 0xFFFFFFFF is reserved for internal use and
+     * should not be used. */
+    u32 id;
+
+    bool nocase; //!< \brief True if literal is case-insensitive.
+
+    /** \brief Matches for runs of this literal can be quashed.
+     *
+     * Advisory flag meaning that there is no value in returning runs of
+     * additional matches for a literal after the first one, so such matches
+     * can be quashed by the literal matcher. */
+    bool noruns;
+
+    /** \brief Set of groups that literal belongs to.
+     *
+     * Use \ref HWLM_ALL_GROUPS for a literal that could match regardless of
+     * the groups that are switched on. */
+    hwlm_group_t groups;
+
+    /** \brief Supplementary comparison mask.
+     *
+     * These two values add a supplementary comparison that is done over the
+     * final 8 bytes of the string -- if v is those bytes, then the string must
+     * match as well as (v & msk) == cmp.
+     *
+     * An empty msk is the safe way of not adding any comparison to the string
+     * unnecessarily filling in msk may turn off optimizations.
+     *
+     * The msk/cmp mechanism must NOT place a value into the literal that
+     * conflicts with the contents of the string, but can be allowed to add
+     * additional power within the string -- for example, to allow some case
+     * sensitivity within a case-insensitive string.
+
+     * Values are stored in memory order -- i.e. the last byte of the mask
+     * corresponds to the last byte of the string. Both vectors must be the
+     * same size, and must not exceed \ref HWLM_MASKLEN in length.
+     */
+    std::vector<u8> msk;
+
+    /** \brief Supplementary comparison value.
+     *
+     * See documentation for \ref msk.
+     */
+    std::vector<u8> cmp;
+
+    /** \brief Simple constructor: no group information, no msk/cmp. */
+    hwlmLiteral(const std::string &s_in, bool nocase_in, u32 id_in)
+        : s(s_in), id(id_in), nocase(nocase_in), noruns(false),
+          groups(HWLM_ALL_GROUPS), msk(0), cmp(0) {}
+
+    /** \brief Complete constructor, takes group information and msk/cmp.
+     *
+     * This constructor takes a msk/cmp pair. Both must be vectors of length <=
+     * \ref HWLM_MASKLEN. */
+    hwlmLiteral(const std::string &s_in, bool nocase_in, bool noruns_in,
+                u32 id_in, hwlm_group_t groups_in,
+                const std::vector<u8> &msk_in, const std::vector<u8> &cmp_in);
+};
+
+/**
+ * Consistency test; returns false if the given msk/cmp test can never match
+ * the literal string s.
+ */
+bool maskIsConsistent(const std::string &s, bool nocase,
+                      const std::vector<u8> &msk, const std::vector<u8> &cmp);
+
+} // namespace ue2
+
+#endif // HWLM_LITERAL_H

src/hwlm/noodle_build.cpp

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Noodle literal matcher: build code.
+ */
+#include <cstring> // for memcpy
+
+#include "noodle_build.h"
+#include "noodle_internal.h"
+#include "ue2common.h"
+#include "util/alloc.h"
+#include "util/compare.h"
+#include "util/verify_types.h"
+
+namespace ue2 {
+
+static
+size_t findNoodFragOffset(const u8 *lit, size_t len, bool nocase) {
+    size_t offset = 0;
+    for (size_t i = 0; i + 1 < len; i++) {
+        int diff = 0;
+        const char c = lit[i];
+        const char d = lit[i + 1];
+        if (nocase && ourisalpha(c)) {
+            diff = (mytoupper(c) != mytoupper(d));
+        } else {
+            diff = (c != d);
+        }
+        offset = i;
+        if (diff) {
+            break;
+        }
+    }
+    return offset;
+}
+
+/** \brief Construct a Noodle matcher for the given literal. */
+aligned_unique_ptr<noodTable> noodBuildTable(const u8 *lit, size_t len,
+                                             bool nocase, u32 id) {
+    size_t noodle_len = sizeof(noodTable) + len;
+    aligned_unique_ptr<noodTable> n =
+        aligned_zmalloc_unique<noodTable>(noodle_len);
+    assert(n);
+
+    size_t key_offset = findNoodFragOffset(lit, len, nocase);
+
+    n->id = id;
+    n->len = verify_u32(len);
+    n->key_offset = verify_u32(key_offset);
+    n->nocase = nocase ? 1 : 0;
+    memcpy(n->str, lit, len);
+
+    return n;
+}
+
+size_t noodSize(const noodTable *n) {
+    assert(n); // shouldn't call with null
+    return sizeof(*n) + n->len;
+}
+
+} // namespace ue2
+
+#ifdef DUMP_SUPPORT
+#include <cctype>
+
+namespace ue2 {
+
+void noodPrintStats(const noodTable *n, FILE *f) {
+    fprintf(f, "Noodle table\n");
+    fprintf(f, "Len: %u Key Offset: %u\n", n->len, n->key_offset);
+    fprintf(f, "String: ");
+    for (u32 i = 0; i < n->len; i++) {
+        if (isgraph(n->str[i]) && n->str[i] != '\\') {
+            fprintf(f, "%c", n->str[i]);
+        } else {
+            fprintf(f, "\\x%02hhx", n->str[i]);
+        }
+    }
+    fprintf(f, "\n");
+}
+
+} // namespace ue2
+
+#endif

src/hwlm/noodle_build.h

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Noodle literal matcher: build code.
+ */
+
+#ifndef NOODLE_BUILD_H_048A1A6D585A9A
+#define NOODLE_BUILD_H_048A1A6D585A9A
+
+#include "ue2common.h"
+#include "util/alloc.h"
+
+struct noodTable;
+
+namespace ue2 {
+
+/** \brief Construct a Noodle matcher for the given literal. */
+ue2::aligned_unique_ptr<noodTable> noodBuildTable(const u8 *lit, size_t len,
+                                                  bool nocase, u32 id);
+
+size_t noodSize(const noodTable *n);
+
+} // namespace ue2
+
+#ifdef DUMP_SUPPORT
+
+#include <cstdio>
+
+namespace ue2 {
+
+void noodPrintStats(const noodTable *n, FILE *f);
+
+} // namespace ue2
+
+#endif // DUMP_SUPPORT
+
+#endif /* NOODLE_BUILD_H_048A1A6D585A9A */
+

src/hwlm/noodle_engine.c

@@ -0,0 +1,364 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Noodle literal matcher: runtime.
+ */
+#include "hwlm.h"
+#include "noodle_engine.h"
+#include "noodle_internal.h"
+#include "ue2common.h"
+#include "util/bitutils.h"
+#include "util/compare.h"
+#include "util/masked_move.h"
+#include "util/simd_utils.h"
+
+#include <ctype.h>
+#include <stdbool.h>
+#include <string.h>
+
+/** \brief Noodle runtime context. */
+struct cb_info {
+    HWLMCallback cb; //!< callback function called on match
+    u32 id; //!< ID to pass to callback on match
+    void *ctx; //!< caller-supplied context to pass to callback
+    size_t offsetAdj; //!< used in streaming mode
+};
+
+#define RETURN_IF_TERMINATED(x)                                                \
+    {                                                                          \
+        if ((x) == HWLM_TERMINATED) {                                          \
+            return HWLM_TERMINATED;                                            \
+        }                                                                      \
+    }
+
+#define SINGLE_ZSCAN()                                                         \
+    do {                                                                       \
+        while (unlikely(z)) {                                                  \
+            u32 pos = findAndClearLSB_32(&z);                                  \
+            size_t matchPos = d - buf + pos;                                   \
+            hwlmcb_rv_t rv = final(buf, len, key, 1, 0, 0, noCase, cbi,        \
+                                   matchPos);                                  \
+            RETURN_IF_TERMINATED(rv);                                          \
+        }                                                                      \
+    } while (0)
+
+#define DOUBLE_ZSCAN()                                                         \
+    do {                                                                       \
+        while (unlikely(z)) {                                                  \
+            u32 pos = findAndClearLSB_32(&z);                                  \
+            size_t matchPos = d - buf + pos - 1;                               \
+            hwlmcb_rv_t rv = final(buf, len, key, keyLen, keyOffset, 1,        \
+                                   noCase, cbi, matchPos);                     \
+            RETURN_IF_TERMINATED(rv);                                          \
+        }                                                                      \
+    } while (0)
+
+static really_inline
+u8 caseClear8(u8 x, bool noCase) {
+    return (u8)(noCase ? (x & (u8)0xdf) : x);
+}
+
+// Make sure the rest of the string is there. The single character scanner
+// is used only for single chars with case insensitivity used correctly,
+// so it can go straight to the callback if we get this far.
+static really_inline
+hwlm_error_t final(const u8 *buf, size_t len, const u8 *key, size_t keyLen,
+                   size_t keyOffset, bool is_double, bool noCase,
+                   const struct cb_info *cbi, size_t pos) {
+    pos -= keyOffset;
+    if (is_double) {
+        if (pos + keyLen > len) {
+            return HWLM_SUCCESS;
+        }
+        if (cmpForward(buf + pos, key, keyLen, noCase)) { // ret 1 on mismatch
+            return HWLM_SUCCESS;
+        }
+    }
+    pos += cbi->offsetAdj;
+    DEBUG_PRINTF("match @ %zu->%zu\n", pos, (pos + keyLen - 1));
+    hwlmcb_rv_t rv = cbi->cb(pos, (pos + keyLen - 1), cbi->id, cbi->ctx);
+    if (rv == HWLM_TERMINATE_MATCHING) {
+        return HWLM_TERMINATED;
+    }
+    return HWLM_SUCCESS;
+}
+
+#if defined(__AVX2__)
+#define CHUNKSIZE 32
+#define MASK_TYPE m256
+#include "noodle_engine_avx2.c"
+#else
+#define CHUNKSIZE 16
+#define MASK_TYPE m128
+#include "noodle_engine_sse.c"
+#endif
+
+static really_inline
+hwlm_error_t scanSingleMain(const u8 *buf, size_t len, const u8 *key,
+                            bool noCase, const struct cb_info *cbi) {
+    hwlm_error_t rv;
+    size_t end = len;
+
+    const MASK_TYPE mask1 = getMask(key[0], noCase);
+    const MASK_TYPE caseMask = getCaseMask();
+
+    if (len < CHUNKSIZE) {
+        rv = scanSingleShort(buf, len, key, noCase, caseMask, mask1, cbi, 0, len);
+        return rv;
+    }
+
+    if (len == CHUNKSIZE) {
+        rv = scanSingleUnaligned(buf, len, 0, key, noCase, caseMask, mask1, cbi,
+                                 0, len);
+        return rv;
+    }
+
+    uintptr_t data = (uintptr_t)buf;
+    uintptr_t s2Start = ROUNDUP_N(data, CHUNKSIZE) - data;
+    uintptr_t last = data + end;
+    uintptr_t s2End = ROUNDDOWN_N(last, CHUNKSIZE) - data;
+    uintptr_t s3Start = len - CHUNKSIZE;
+
+    if (s2Start) {
+        // first scan out to the fast scan starting point
+        DEBUG_PRINTF("stage 1: -> %zu\n", s2Start);
+        rv = scanSingleUnaligned(buf, len, 0, key, noCase, caseMask, mask1, cbi,
+                                 0, s2Start);
+        RETURN_IF_TERMINATED(rv);
+    }
+
+    if (likely(s2Start != s2End)) {
+        // scan as far as we can, bounded by the last point this key can
+        // possibly match
+        DEBUG_PRINTF("fast: ~ %zu -> %zu\n", s2Start, s2End);
+        rv = scanSingleFast(buf, len, key, noCase, caseMask, mask1, cbi,
+                            s2Start, s2End);
+        RETURN_IF_TERMINATED(rv);
+    }
+
+    // if we are done bail out
+    if (s2End == end) {
+        return HWLM_SUCCESS;
+    }
+
+    DEBUG_PRINTF("stage 3: %zu -> %zu\n", s2End, end);
+    rv = scanSingleUnaligned(buf, len, s3Start, key, noCase, caseMask, mask1,
+                             cbi, s2End, end);
+
+    return rv;
+}
+
+static really_inline
+hwlm_error_t scanDoubleMain(const u8 *buf, size_t len, const u8 *key,
+                            size_t keyLen, size_t keyOffset, bool noCase,
+                            const struct cb_info *cbi) {
+    hwlm_error_t rv;
+    // we stop scanning for the key-fragment when the rest of the key can't
+    // possibly fit in the remaining buffer
+    size_t end = len - keyLen + keyOffset + 2;
+
+    const MASK_TYPE caseMask = getCaseMask();
+    const MASK_TYPE mask1 = getMask(key[keyOffset + 0], noCase);
+    const MASK_TYPE mask2 = getMask(key[keyOffset + 1], noCase);
+
+    if (end - keyOffset < CHUNKSIZE) {
+        rv = scanDoubleShort(buf, len, key, keyLen, keyOffset, noCase, caseMask,
+                             mask1, mask2, cbi, keyOffset, end);
+        return rv;
+    }
+    if (end - keyOffset == CHUNKSIZE) {
+        rv = scanDoubleUnaligned(buf, len, keyOffset, key, keyLen, keyOffset,
+                                 noCase, caseMask, mask1, mask2, cbi, keyOffset,
+                                 end);
+        return rv;
+    }
+
+    uintptr_t data = (uintptr_t)buf;
+    uintptr_t s2Start = ROUNDUP_N(data + keyOffset, CHUNKSIZE) - data;
+    uintptr_t s1End = s2Start + 1;
+    uintptr_t last = data + end;
+    uintptr_t s2End = ROUNDDOWN_N(last, CHUNKSIZE) - data;
+    uintptr_t s3Start = end - CHUNKSIZE;
+    uintptr_t off = keyOffset;
+
+    if (s2Start != keyOffset) {
+        // first scan out to the fast scan starting point plus one char past to
+        // catch the key on the overlap
+        DEBUG_PRINTF("stage 1: -> %zu\n", s2Start);
+        rv = scanDoubleUnaligned(buf, len, keyOffset, key, keyLen, keyOffset,
+                                 noCase, caseMask, mask1, mask2, cbi, off,
+                                 s1End);
+        RETURN_IF_TERMINATED(rv);
+    }
+    off = s1End;
+
+    if (s2Start >= end) {
+        DEBUG_PRINTF("s2 == mL %zu\n", end);
+        return HWLM_SUCCESS;
+    }
+
+    if (likely(s2Start != s2End)) {
+        // scan as far as we can, bounded by the last point this key can
+        // possibly match
+        DEBUG_PRINTF("fast: ~ %zu -> %zu\n", s2Start, s3Start);
+        rv = scanDoubleFast(buf, len, key, keyLen, keyOffset, noCase, caseMask,
+                            mask1, mask2, cbi, s2Start, s2End);
+        RETURN_IF_TERMINATED(rv);
+        off = s2End;
+    }
+
+    // if there isn't enough data left to match the key, bail out
+    if (s2End == end) {
+        return HWLM_SUCCESS;
+    }
+
+    DEBUG_PRINTF("stage 3: %zu -> %zu\n", s3Start, end);
+    rv = scanDoubleUnaligned(buf, len, s3Start, key, keyLen, keyOffset, noCase,
+                             caseMask, mask1, mask2, cbi, off, end);
+
+    return rv;
+}
+
+
+static really_inline
+hwlm_error_t scanSingleNoCase(const u8 *buf, size_t len, const u8 *key,
+                              const struct cb_info *cbi) {
+    return scanSingleMain(buf, len, key, 1, cbi);
+}
+
+static really_inline
+hwlm_error_t scanSingleCase(const u8 *buf, size_t len, const u8 *key,
+                            const struct cb_info *cbi) {
+    return scanSingleMain(buf, len, key, 0, cbi);
+}
+
+// Single-character specialisation, used when keyLen = 1
+static really_inline
+hwlm_error_t scanSingle(const u8 *buf, size_t len, const u8 *key, bool noCase,
+                        const struct cb_info *cbi) {
+    if (!ourisalpha(key[0])) {
+        noCase = 0; // force noCase off if we don't have an alphabetic char
+    }
+
+    // kinda ugly, but this forces constant propagation
+    if (noCase) {
+        return scanSingleNoCase(buf, len, key, cbi);
+    } else {
+        return scanSingleCase(buf, len, key, cbi);
+    }
+}
+
+
+static really_inline
+hwlm_error_t scanDoubleNoCase(const u8 *buf, size_t len, const u8 *key,
+                              size_t keyLen, size_t keyOffset,
+                              const struct cb_info *cbi) {
+    return scanDoubleMain(buf, len, key, keyLen, keyOffset, 1, cbi);
+}
+
+static really_inline
+hwlm_error_t scanDoubleCase(const u8 *buf, size_t len, const u8 *key,
+                            size_t keyLen, size_t keyOffset,
+                            const struct cb_info *cbi) {
+    return scanDoubleMain(buf, len, key, keyLen, keyOffset, 0, cbi);
+}
+
+
+static really_inline
+hwlm_error_t scanDouble(const u8 *buf, size_t len, const u8 *key, size_t keyLen,
+                        size_t keyOffset, bool noCase,
+                        const struct cb_info *cbi) {
+    // kinda ugly, but this forces constant propagation
+    if (noCase) {
+        return scanDoubleNoCase(buf, len, key, keyLen, keyOffset, cbi);
+    } else {
+        return scanDoubleCase(buf, len, key, keyLen, keyOffset, cbi);
+    }
+}
+
+// main entry point for the scan code
+static really_inline
+hwlm_error_t scan(const u8 *buf, size_t len, const u8 *key, size_t keyLen,
+                  size_t keyOffset, bool noCase, const struct cb_info *cbi) {
+    if (len < keyLen) {
+        // can't find string of length keyLen in a shorter buffer
+        return HWLM_SUCCESS;
+    }
+
+    if (keyLen == 1) {
+        assert(keyOffset == 0);
+        return scanSingle(buf, len, key, noCase, cbi);
+    } else {
+        return scanDouble(buf, len, key, keyLen, keyOffset, noCase, cbi);
+    }
+}
+
+/** \brief Block-mode scanner. */
+hwlm_error_t noodExec(const struct noodTable *n, const u8 *buf, size_t len,
+                      size_t offset_adj, HWLMCallback cb, void *ctxt) {
+    assert(n && buf);
+
+    struct cb_info cbi = { cb, n->id, ctxt, offset_adj };
+    DEBUG_PRINTF("nood scan of %zu bytes for %*s\n", len, n->len, n->str);
+    return scan(buf, len, n->str, n->len, n->key_offset, n->nocase, &cbi);
+}
+
+/** \brief Streaming-mode scanner. */
+hwlm_error_t noodExecStreaming(const struct noodTable *n, const u8 *hbuf,
+                               size_t hlen, const u8 *buf, size_t len,
+                               HWLMCallback cb, void *ctxt, u8 *temp_buf,
+                               UNUSED size_t temp_buffer_size) {
+    assert(n);
+
+    struct cb_info cbi = {cb, n->id, ctxt, 0};
+    hwlm_error_t rv;
+
+    if (hlen) {
+        assert(hbuf);
+
+        size_t tl1 = MIN(n->len - 1, hlen);
+        size_t tl2 = MIN(n->len - 1, len);
+        size_t temp_len = tl1 + tl2;
+        assert(temp_len < temp_buffer_size);
+        memcpy(temp_buf, hbuf + hlen - tl1, tl1);
+        memcpy(temp_buf + tl1, buf, tl2);
+
+        cbi.offsetAdj = -tl1;
+        rv = scan(temp_buf, temp_len, n->str, n->len, n->key_offset, n->nocase,
+                  &cbi);
+        if (rv == HWLM_TERMINATED) {
+            return HWLM_TERMINATED;
+        }
+    }
+
+    assert(buf);
+
+    cbi.offsetAdj = 0;
+    return scan(buf, len, n->str, n->len, n->key_offset, n->nocase, &cbi);
+}

src/hwlm/noodle_engine.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Noodle literal matcher: runtime API.
+ */
+
+#ifndef NOODLE_ENGINE_H
+#define NOODLE_ENGINE_H
+
+#include "hwlm.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+struct noodTable;
+
+/** \brief Block-mode scanner. */
+hwlm_error_t noodExec(const struct noodTable *n, const u8 *buf, size_t len,
+                      size_t offset_adj, HWLMCallback cb, void *ctxt);
+
+/** \brief Streaming-mode scanner. */
+hwlm_error_t noodExecStreaming(const struct noodTable *n, const u8 *hbuf,
+                               size_t hlen, const u8 *buf, size_t len,
+                               HWLMCallback cb, void *ctxt, u8 *temp_buf,
+                               size_t temp_buffer_size);
+
+#ifdef __cplusplus
+}       /* extern "C" */
+#endif
+
+#endif

src/hwlm/noodle_engine_avx2.c

@@ -0,0 +1,234 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* noodle scan parts for AVX */
+
+static really_inline m256 getMask(u8 c, bool noCase) {
+    u8 k = caseClear8(c, noCase);
+    return set32x8(k);
+}
+
+static really_inline m256 getCaseMask(void) {
+    return set32x8(0xdf);
+}
+
+static really_inline
+hwlm_error_t scanSingleUnaligned(const u8 *buf, size_t len, size_t offset,
+                                 const u8 *key, bool noCase, m256 caseMask,
+                                 m256 mask1, const struct cb_info *cbi,
+                                 size_t start, size_t end) {
+    const u8 *d = buf + offset;
+    DEBUG_PRINTF("start %zu end %zu offset %zu\n", start, end, offset);
+    const size_t l = end - start;
+
+    m256 v = loadu256(d);
+
+    if (noCase) {
+        v = and256(v, caseMask);
+    }
+
+    u32 z = movemask256(eq256(mask1, v));
+
+    u32 buf_off = start - offset;
+    u32 mask = (u32)((u64a)(1ULL << l) - 1) << buf_off;
+    DEBUG_PRINTF("mask 0x%08x z 0x%08x\n", mask, z);
+
+    z &= mask;
+
+    SINGLE_ZSCAN();
+
+    return HWLM_SUCCESS;
+}
+
+static really_inline
+hwlm_error_t scanDoubleUnaligned(const u8 *buf, size_t len, size_t offset,
+                                 const u8 *key, size_t keyLen, size_t keyOffset,
+                                 bool noCase, m256 caseMask, m256 mask1,
+                                 m256 mask2, const struct cb_info *cbi,
+                                 size_t start, size_t end) {
+    const u8 *d = buf + offset;
+    DEBUG_PRINTF("start %zu end %zu offset %zu\n", start, end, offset);
+    size_t l = end - start;
+
+    m256 v = loadu256(d);
+
+    if (noCase) {
+        v = and256(v, caseMask);
+    }
+
+    u32 z0 = movemask256(eq256(mask1, v));
+    u32 z1 = movemask256(eq256(mask2, v));
+    u32 z = (z0 << 1) & z1;
+
+    // mask out where we can't match
+    u32 buf_off = start - offset;
+    u32 mask = (u32)((u64a)(1ULL << l) - 1) << buf_off;
+    DEBUG_PRINTF("mask 0x%08x z 0x%08x\n", mask, z);
+    z &= mask;
+
+    DOUBLE_ZSCAN();
+
+    return HWLM_SUCCESS;
+}
+
+// The short scan routine. It is used both to scan data up to an
+// alignment boundary if needed and to finish off data that the aligned scan
+// function can't handle (due to small/unaligned chunk at end)
+static really_inline
+hwlm_error_t scanSingleShort(const u8 *buf, size_t len, const u8 *key,
+                             bool noCase, m256 caseMask, m256 mask1,
+                             const struct cb_info *cbi, size_t start,
+                             size_t end) {
+    const u8 *d = buf + start;
+    size_t l = end - start;
+    DEBUG_PRINTF("l %zu\n", l);
+    assert(l <= 32);
+    if (!l) {
+        return HWLM_SUCCESS;
+    }
+
+    m256 v;
+
+    if (l < 4) {
+        u8 *vp = (u8*)&v;
+        switch (l) {
+            case 3: vp[2] = d[2];
+            case 2: vp[1] = d[1];
+            case 1: vp[0] = d[0];
+        }
+    } else {
+        v = masked_move256_len(d, l);
+    }
+
+    if (noCase) {
+        v = and256(v, caseMask);
+    }
+
+    // mask out where we can't match
+    u32 mask = (0xFFFFFFFF >> (32 - l));
+
+    u32 z = mask & movemask256(eq256(mask1, v));
+
+    SINGLE_ZSCAN();
+
+    return HWLM_SUCCESS;
+}
+
+static really_inline
+hwlm_error_t scanDoubleShort(const u8 *buf, size_t len, const u8 *key,
+                             size_t keyLen, size_t keyOffset, bool noCase,
+                             m256 caseMask, m256 mask1, m256 mask2,
+                             const struct cb_info *cbi, size_t start,
+                             size_t end) {
+    const u8 *d = buf + start;
+    size_t l = end - start;
+    if (!l) {
+        return HWLM_SUCCESS;
+    }
+    assert(l <= 32);
+    m256 v;
+
+    DEBUG_PRINTF("d %zu\n", d - buf);
+    if (l < 4) {
+        u8 *vp = (u8*)&v;
+        switch (l) {
+            case 3: vp[2] = d[2];
+            case 2: vp[1] = d[1];
+            case 1: vp[0] = d[0];
+        }
+    } else {
+        v = masked_move256_len(d, l);
+    }
+    if (noCase) {
+        v = and256(v, caseMask);
+    }
+
+    u32 z0 = movemask256(eq256(mask1, v));
+    u32 z1 = movemask256(eq256(mask2, v));
+    u32 z = (z0 << 1) & z1;
+
+    // mask out where we can't match
+    u32 mask = (0xFFFFFFFF >> (32 - l));
+    z &= mask;
+
+    DOUBLE_ZSCAN();
+
+    return HWLM_SUCCESS;
+}
+
+static really_inline
+hwlm_error_t scanSingleFast(const u8 *buf, size_t len, const u8 *key,
+                            bool noCase, m256 caseMask, m256 mask1,
+                            const struct cb_info *cbi, size_t start,
+                            size_t end) {
+    const u8 *d = buf + start, *e = buf + end;
+    assert(d < e);
+
+    for (; d < e; d += 32) {
+        m256 v = noCase ? and256(load256(d), caseMask) : load256(d);
+
+        u32 z = movemask256(eq256(mask1, v));
+
+        // On large packet buffers, this prefetch appears to get us about 2%.
+        __builtin_prefetch(d + 128);
+
+        SINGLE_ZSCAN();
+    }
+    return HWLM_SUCCESS;
+}
+
+static really_inline
+hwlm_error_t scanDoubleFast(const u8 *buf, size_t len, const u8 *key,
+                            size_t keyLen, size_t keyOffset, bool noCase,
+                            m256 caseMask, m256 mask1, m256 mask2,
+                            const struct cb_info *cbi, size_t start,
+                            size_t end) {
+    const u8 *d = buf + start, *e = buf + end;
+    DEBUG_PRINTF("start %zu end %zu \n", start, end);
+    assert(d < e);
+    u8 lastz0 = 0;
+
+    for (; d < e; d += 32) {
+        m256 v = noCase ? and256(load256(d), caseMask) : load256(d);
+
+        // we have to pull the masks out of the AVX registers because we can't
+        // byte shift between the lanes
+        u32 z0 = movemask256(eq256(mask1, v));
+        u32 z1 = movemask256(eq256(mask2, v));
+        u32 z = (lastz0 | (z0 << 1)) & z1;
+        lastz0 = (z0 & 0x80000000) >> 31;
+
+        // On large packet buffers, this prefetch appears to get us about 2%.
+        __builtin_prefetch(d + 128);
+
+        DOUBLE_ZSCAN();
+
+    }
+    return HWLM_SUCCESS;
+}
+

src/hwlm/noodle_engine_sse.c

@@ -0,0 +1,202 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* noodle scan parts for SSE */
+
+static really_inline m128 getMask(u8 c, bool noCase) {
+    u8 k = caseClear8(c, noCase);
+    return set16x8(k);
+}
+
+static really_inline m128 getCaseMask(void) {
+    return set16x8(0xdf);
+}
+
+static really_inline
+hwlm_error_t scanSingleShort(const u8 *buf, size_t len, const u8 *key,
+                             bool noCase, m128 caseMask, m128 mask1,
+                             const struct cb_info *cbi, size_t start,
+                             size_t end) {
+    const u8 *d = buf + start;
+    size_t l = end - start;
+    DEBUG_PRINTF("l %zu\n", l);
+    assert(l <= 16);
+    if (!l) {
+        return HWLM_SUCCESS;
+    }
+    m128 v = zeroes128();
+    // we don't have a clever way of doing this move yet
+    memcpy(&v, d, l);
+    if (noCase) {
+        v = and128(v, caseMask);
+    }
+
+    // mask out where we can't match
+    u32 mask = (0xFFFF >> (16 - l));
+
+    u32 z = mask & movemask128(eq128(mask1, v));
+
+    SINGLE_ZSCAN();
+
+    return HWLM_SUCCESS;
+}
+
+static really_inline
+hwlm_error_t scanSingleUnaligned(const u8 *buf, size_t len, size_t offset,
+                                 const u8 *key, bool noCase, m128 caseMask,
+                                 m128 mask1, const struct cb_info *cbi,
+                                 size_t start, size_t end) {
+    const u8 *d = buf + offset;
+    DEBUG_PRINTF("start %zu end %zu offset %zu\n", start, end, offset);
+    const size_t l = end - start;
+
+    m128 v = loadu128(d);
+
+    if (noCase) {
+        v = and128(v, caseMask);
+    }
+
+    u32 buf_off = start - offset;
+    u32 mask = ((1 << l) - 1) << buf_off;
+
+    u32 z = mask & movemask128(eq128(mask1, v));
+
+    DEBUG_PRINTF("mask 0x%08x z 0x%08x\n", mask, z);
+
+    z &= mask;
+
+    SINGLE_ZSCAN();
+
+    return HWLM_SUCCESS;
+}
+
+static really_inline
+hwlm_error_t scanDoubleShort(const u8 *buf, size_t len, const u8 *key,
+                             size_t keyLen, size_t keyOffset, bool noCase,
+                             m128 caseMask, m128 mask1, m128 mask2,
+                             const struct cb_info *cbi, size_t start,
+                             size_t end) {
+    const u8 *d = buf + start;
+    size_t l = end - start;
+    if (!l) {
+        return HWLM_SUCCESS;
+    }
+    assert(l <= 32);
+
+    DEBUG_PRINTF("d %zu\n", d - buf);
+    m128 v = zeroes128();
+    memcpy(&v, d, l);
+    if (noCase) {
+        v = and128(v, caseMask);
+    }
+
+    u32 z = movemask128(and128(shiftLeft8Bits(eq128(mask1, v)), eq128(mask2, v)));
+
+    // mask out where we can't match
+    u32 mask = (0xFFFF >> (16 - l));
+    z &= mask;
+
+    DOUBLE_ZSCAN();
+
+    return HWLM_SUCCESS;
+}
+
+static really_inline
+hwlm_error_t scanDoubleUnaligned(const u8 *buf, size_t len, size_t offset,
+                                 const u8 *key, size_t keyLen, size_t keyOffset,
+                                 bool noCase, m128 caseMask, m128 mask1,
+                                 m128 mask2, const struct cb_info *cbi,
+                                 size_t start, size_t end) {
+    const u8 *d = buf + offset;
+    DEBUG_PRINTF("start %zu end %zu offset %zu\n", start, end, offset);
+    size_t l = end - start;
+
+    m128 v = loadu128(d);
+
+    if (noCase) {
+        v = and128(v, caseMask);
+    }
+
+    u32 z = movemask128(and128(shiftLeft8Bits(eq128(mask1, v)), eq128(mask2, v)));
+
+    // mask out where we can't match
+    u32 buf_off = start - offset;
+    u32 mask = ((1 << l) - 1) << buf_off;
+    DEBUG_PRINTF("mask 0x%08x z 0x%08x\n", mask, z);
+    z &= mask;
+
+    DOUBLE_ZSCAN();
+
+    return HWLM_SUCCESS;
+}
+
+static really_inline
+hwlm_error_t scanSingleFast(const u8 *buf, size_t len, const u8 *key,
+                            bool noCase, m128 caseMask, m128 mask1,
+                            const struct cb_info *cbi, size_t start,
+                            size_t end) {
+    const u8 *d = buf + start, *e = buf + end;
+    assert(d < e);
+
+    for (; d < e; d += 16) {
+        m128 v = noCase ? and128(load128(d), caseMask) : load128(d);
+
+        u32 z = movemask128(eq128(mask1, v));
+
+        // On large packet buffers, this prefetch appears to get us about 2%.
+        __builtin_prefetch(d + 128);
+
+        SINGLE_ZSCAN();
+    }
+    return HWLM_SUCCESS;
+}
+
+static really_inline
+hwlm_error_t scanDoubleFast(const u8 *buf, size_t len, const u8 *key,
+                            size_t keyLen, size_t keyOffset, bool noCase,
+                            m128 caseMask, m128 mask1, m128 mask2,
+                            const struct cb_info *cbi, size_t start,
+                            size_t end) {
+    const u8 *d = buf + start, *e = buf + end;
+    assert(d < e);
+    m128 lastz1 = zeroes128();
+
+    for (; d < e; d += 16) {
+        m128 v = noCase ? and128(load128(d), caseMask) : load128(d);
+        m128 z1 = eq128(mask1, v);
+        m128 z2 = eq128(mask2, v);
+        u32 z = movemask128(and128(or128(lastz1, shiftLeft8Bits(z1)), z2));
+        lastz1 = _mm_srli_si128(z1, 15);
+
+        // On large packet buffers, this prefetch appears to get us about 2%.
+        __builtin_prefetch(d + 128);
+        DEBUG_PRINTF("z 0x%08x\n", z);
+        DOUBLE_ZSCAN();
+    }
+    return HWLM_SUCCESS;
+}

src/hwlm/noodle_internal.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Data structures for Noodle literal matcher engine.
+ */
+
+#ifndef NOODLE_INTERNAL_H_25D751C42E34A6
+#define NOODLE_INTERNAL_H_25D751C42E34A6
+
+#include "ue2common.h"
+
+struct noodTable {
+    u32 id;
+    u32 len;
+    u32 key_offset;
+    u8  nocase;
+    u8  str[];
+};
+
+#endif /* NOODLE_INTERNAL_H_25D751C42E34A6 */
+