Merge pull request #78 from VectorCamp/feature/refactor-vermicelli

Feature/refactor vermicelli

CMakeLists.txt

@@ -616,9 +616,9 @@ set (hs_exec_SRCS
     src/nfa/tamarama_internal.h
     src/nfa/truffle.cpp
     src/nfa/truffle.h
-    src/nfa/vermicelli.h
+    src/nfa/vermicelli.hpp
     src/nfa/vermicelli_run.h
-    src/nfa/vermicelli_sse.h
+    src/nfa/vermicelli_simd.cpp
     src/som/som.h
     src/som/som_operation.h
     src/som/som_runtime.h

benchmarks/benchmarks.hpp

@@ -30,7 +30,7 @@
 #include "nfa/shufticompile.h"
 #include "nfa/truffle.h"
 #include "nfa/trufflecompile.h"
-#include "nfa/vermicelli.h"
+#include "nfa/vermicelli.hpp"
 #include "hwlm/noodle_build.h"
 #include "hwlm/noodle_engine.h"
 #include "hwlm/noodle_internal.h"

src/hwlm/hwlm.c

@@ -39,7 +39,7 @@
 #include "nfa/accel.h"
 #include "nfa/shufti.h"
 #include "nfa/truffle.h"
-#include "nfa/vermicelli.h"
+#include "nfa/vermicelli.hpp"
 #include <string.h>
 
 #define MIN_ACCEL_LEN_BLOCK  16

src/hwlm/noodle_engine_simd.hpp

@@ -30,26 +30,7 @@
 /* SIMD engine agnostic noodle scan parts */
 
 #include "util/supervector/supervector.hpp"
-
+#include "util/supervector/casemask.hpp"
-static u8 CASEMASK[] = { 0xff, 0xdf };
-
-static really_inline
-u8 caseClear8(u8 x, bool noCase)
-{
-    return static_cast<u8>(x & CASEMASK[(u8)noCase]);
-}
-
-template<uint16_t S>
-static really_inline SuperVector<S> getMask(u8 c, bool noCase) {
-    u8 k = caseClear8(c, noCase);
-    return SuperVector<S>(k);
-}
-
-template<uint16_t S>
-static really_inline SuperVector<S> getCaseMask(void) {
-    return SuperVector<S>(CASEMASK[1]);
-}
-
 
 static really_really_inline
 hwlm_error_t single_zscan(const struct noodTable *n,const u8 *d, const u8 *buf,

src/nfa/accel.c

@@ -30,7 +30,7 @@
 #include "accel.h"
 #include "shufti.h"
 #include "truffle.h"
-#include "vermicelli.h"
+#include "vermicelli.hpp"
 #include "ue2common.h"
 
 const u8 *run_accel(const union AccelAux *accel, const u8 *c, const u8 *c_end) {

src/nfa/arm/shufti.hpp

@@ -1,7 +1,6 @@
 /*
  * Copyright (c) 2015-2017, Intel Corporation
  * Copyright (c) 2020-2021, VectorCamp PC
- * Copyright (c) 2021, Arm Limited
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -30,7 +29,6 @@
 
 /** \file
  * \brief Shufti: character class acceleration.
- *
  */
 
 template <uint16_t S>

src/nfa/arm/vermicelli.hpp

@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2015-2020, Intel Corporation
+ * Copyright (c) 2020-2021, VectorCamp PC
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Vermicelli: single-byte and double-byte acceleration.
+ */
+
+template <uint16_t S>
+static really_inline
+const u8 *vermicelliBlock(SuperVector<S> const data, SuperVector<S> const chars, SuperVector<S> const casemask, u8 const *buf, u16 const len) {
+
+    SuperVector<S> mask = chars.eq(casemask & data);
+    return first_non_zero_match<S>(buf, mask, len);
+}
+
+template <uint16_t S>
+static really_inline
+const u8 *vermicelliBlockNeg(SuperVector<S> const data, SuperVector<S> const chars, SuperVector<S> const casemask, u8 const *buf, u16 const len) {
+
+    SuperVector<S> mask = !chars.eq(casemask & data);
+    return first_zero_match_inverted<S>(buf, mask, len);
+}
+
+template <uint16_t S>
+static really_inline
+const u8 *rvermicelliBlock(SuperVector<S> const data, SuperVector<S> const chars, SuperVector<S> const casemask, u8 const *buf, u16 const len) {
+
+    SuperVector<S> mask = chars.eq(casemask & data);
+    return last_non_zero_match<S>(buf, mask, len);
+}
+
+template <uint16_t S>
+static really_inline
+const u8 *rvermicelliBlockNeg(SuperVector<S> const data, SuperVector<S> const chars, SuperVector<S> const casemask, const u8 *buf, u16 const len) {
+
+    data.print8("data");
+    chars.print8("chars");
+    casemask.print8("casemask");
+    SuperVector<S> mask = !chars.eq(casemask & data);
+    mask.print8("mask");
+    return last_zero_match_inverted<S>(buf, mask, len);
+}
+
+template <uint16_t S>
+static really_inline
+const u8 *vermicelliDoubleBlock(SuperVector<S> const data, SuperVector<S> const chars1, SuperVector<S> const chars2, SuperVector<S> const casemask,
+                                u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len) {
+
+    SuperVector<S> v = casemask & data;
+    SuperVector<S> mask1 = chars1.eq(v);
+    SuperVector<S> mask2 = chars2.eq(v);
+    SuperVector<S> mask = mask1 & (mask2 >> 1);
+
+    DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]);
+    bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1));
+    DEBUG_PRINTF("partial = %d\n", partial_match);
+    if (partial_match) return buf - 1;
+
+    return first_non_zero_match<S>(buf, mask, len);
+}
+
+template <uint16_t S>
+static really_inline
+const u8 *rvermicelliDoubleBlock(SuperVector<S> const data, SuperVector<S> const chars1, SuperVector<S> const chars2, SuperVector<S> const casemask,
+                                 u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len) {
+
+    SuperVector<S> v = casemask & data;
+    SuperVector<S> mask1 = chars1.eq(v);
+    SuperVector<S> mask2 = chars2.eq(v);
+    SuperVector<S> mask = (mask1 << 1)& mask2;
+
+    DEBUG_PRINTF("buf[0] = %02hhx, buf[-1] = %02hhx\n", buf[0], buf[-1]);
+    bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1));
+    DEBUG_PRINTF("partial = %d\n", partial_match);
+    if (partial_match) {
+        mask = mask | (SuperVector<S>::Ones() >> (S-1));
+    }
+
+    return last_non_zero_match<S>(buf, mask, len);
+}
+
+template <uint16_t S>
+static really_inline
+const u8 *vermicelliDoubleMaskedBlock(SuperVector<S> const data, SuperVector<S> const chars1, SuperVector<S> const chars2,
+                                      SuperVector<S> const mask1, SuperVector<S> const mask2,
+                                      u8 const c1, u8 const c2, u8 const m1, u8 const m2, u8 const *buf, u16 const len) {
+
+    SuperVector<S> v1 = chars1.eq(data & mask1);
+    SuperVector<S> v2 = chars2.eq(data & mask2);
+    SuperVector<S> mask = v1 & (v2 >> 1);
+
+    DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]);
+    bool partial_match = (((buf[0] & m1) == c2) && ((buf[-1] & m2) == c1));
+    DEBUG_PRINTF("partial = %d\n", partial_match);
+    if (partial_match) return buf - 1;
+
+    return first_non_zero_match<S>(buf, mask, len);
+}
+

src/nfa/castle.c

@@ -40,7 +40,7 @@
 #include "repeat.h"
 #include "shufti.h"
 #include "truffle.h"
-#include "vermicelli.h"
+#include "vermicelli.hpp"
 #include "util/bitutils.h"
 #include "util/multibit.h"
 #include "util/partial_store.h"

src/nfa/lbr.c

@@ -40,7 +40,7 @@
 #include "repeat_internal.h"
 #include "shufti.h"
 #include "truffle.h"
-#include "vermicelli.h"
+#include "vermicelli.hpp"
 #include "util/partial_store.h"
 #include "util/unaligned.h"
 

src/nfa/limex_accel.c

@@ -40,7 +40,7 @@
 #include "shufti.h"
 #include "truffle.h"
 #include "ue2common.h"
-#include "vermicelli.h"
+#include "vermicelli.hpp"
 #include "util/arch.h"
 #include "util/bitutils.h"
 #include "util/simd_utils.h"

src/nfa/mpv.c

@@ -36,7 +36,7 @@
 #include "shufti.h"
 #include "truffle.h"
 #include "ue2common.h"
-#include "vermicelli.h"
+#include "vermicelli.hpp"
 #include "vermicelli_run.h"
 #include "util/multibit.h"
 #include "util/partial_store.h"

src/nfa/nfa_rev_api.h

@@ -35,7 +35,7 @@
 
 #include "accel.h"
 #include "nfa_internal.h"
-#include "vermicelli.h"
+#include "vermicelli.hpp"
 #include "util/unaligned.h"
 
 static really_inline

src/nfa/shufti_simd.hpp

@@ -63,7 +63,7 @@ static really_inline
 const u8 *fwdBlock(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars, const u8 *buf) {
     SuperVector<S> v = blockSingleMask(mask_lo, mask_hi, chars);
 
-    return firstMatch<S>(buf, v);
+    return first_zero_match_inverted<S>(buf, v);
 }
 
 template <uint16_t S>
@@ -71,7 +71,7 @@ static really_inline
 const u8 *revBlock(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars, const u8 *buf) {
     SuperVector<S> v = blockSingleMask(mask_lo, mask_hi, chars);
 
-    return lastMatch<S>(buf, v);
+    return last_zero_match_inverted<S>(buf, v);
 }
 
 template <uint16_t S>
@@ -80,7 +80,7 @@ const u8 *fwdBlockDouble(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi, Super
 
     SuperVector<S> mask = blockDoubleMask(mask1_lo, mask1_hi, mask2_lo, mask2_hi, chars);
 
-    return firstMatch<S>(buf, mask);
+    return first_zero_match_inverted<S>(buf, mask);
 }
 
 template <uint16_t S>

src/nfa/truffle_simd.hpp

@@ -56,7 +56,7 @@ static really_inline
 const u8 *fwdBlock(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars, const u8 *buf) {
     SuperVector<S> res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars);
 
-    return firstMatch<S>(buf, res);
+    return first_zero_match_inverted<S>(buf, res);
 }
 
 template <uint16_t S>
@@ -120,7 +120,7 @@ static really_inline
 const u8 *revBlock(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> v, 
                     const u8 *buf) {
     SuperVector<S> res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, v);
-    return lastMatch<S>(buf, res);
+    return last_zero_match_inverted<S>(buf, res);
 }
 
 template <uint16_t S>

src/nfa/vermicelli.h

@@ -1,121 +0,0 @@
-/*
- * Copyright (c) 2015-2020, Intel Corporation
- * Copyright (c) 2021, Arm Limited
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *  * Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
- *  * Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *  * Neither the name of Intel Corporation nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-/** \file
- * \brief Vermicelli: single-byte and double-byte acceleration.
- */
-
-#ifndef VERMICELLI_H
-#define VERMICELLI_H
-
-#include "util/bitutils.h"
-#include "util/simd_utils.h"
-#include "util/unaligned.h"
-
-#if !defined(HAVE_AVX512)
-#include "vermicelli_common.h"
-#endif
-
-#ifdef HAVE_SVE2
-#include "vermicelli_sve.h"
-#else
-#include "vermicelli_sse.h"
-#endif
-
-static really_inline
-const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char m2,
-                                     const u8 *buf, const u8 *buf_end) {
-    DEBUG_PRINTF("double verm scan (\\x%02hhx&\\x%02hhx)(\\x%02hhx&\\x%02hhx) "
-                 "over %zu bytes\n", c1, m1, c2, m2, (size_t)(buf_end - buf));
-    assert(buf < buf_end);
-
-    VERM_TYPE chars1 = VERM_SET_FN(c1);
-    VERM_TYPE chars2 = VERM_SET_FN(c2);
-    VERM_TYPE mask1 = VERM_SET_FN(m1);
-    VERM_TYPE mask2 = VERM_SET_FN(m2);
-
-#ifdef HAVE_AVX512
-    if (buf_end - buf <= VERM_BOUNDARY) {
-        const u8 *ptr = dvermMiniMasked(chars1, chars2, mask1, mask2, buf,
-                                        buf_end);
-        if (ptr) {
-            return ptr;
-        }
-
-        /* check for partial match at end */
-        if ((buf_end[-1] & m1) == (u8)c1) {
-            DEBUG_PRINTF("partial!!!\n");
-            return buf_end - 1;
-        }
-
-        return buf_end;
-    }
-#endif
-
-    assert((buf_end - buf) >= VERM_BOUNDARY);
-    uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY;
-    if (min) {
-        // Input isn't aligned, so we need to run one iteration with an
-        // unaligned load, then skip buf forward to the next aligned address.
-        // There's some small overlap here, but we don't mind scanning it twice
-        // if we can do it quickly, do we?
-        const u8 *p = dvermPreconditionMasked(chars1, chars2, mask1, mask2, buf);
-        if (p) {
-            return p;
-        }
-
-        buf += VERM_BOUNDARY - min;
-        assert(buf < buf_end);
-    }
-
-    // Aligned loops from here on in
-    const u8 *ptr = dvermSearchAlignedMasked(chars1, chars2, mask1, mask2, c1,
-                                             c2, m1, m2, buf, buf_end);
-    if (ptr) {
-        return ptr;
-    }
-
-    // Tidy up the mess at the end
-    ptr = dvermPreconditionMasked(chars1, chars2, mask1, mask2,
-                                  buf_end - VERM_BOUNDARY);
-
-    if (ptr) {
-        return ptr;
-    }
-
-    /* check for partial match at end */
-    if ((buf_end[-1] & m1) == (u8)c1) {
-        DEBUG_PRINTF("partial!!!\n");
-        return buf_end - 1;
-    }
-
-    return buf_end;
-}
-
-#endif /* VERMICELLI_H */

src/nfa/vermicelli.hpp

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2015-2020, Intel Corporation
+ * Copyright (c) 2020-2021, VectorCamp PC
  * Copyright (c) 2021, Arm Limited
  *
  * Redistribution and use in source and binary forms, with or without
@@ -28,52 +29,72 @@
  */
 
 /** \file
- * \brief Vermicelli: Implementation shared between architectures.
+ * \brief Vermicelli: single-byte and double-byte acceleration.
- *
- * (users should include vermicelli.h instead of this)
  */
 
-#define VERM_BOUNDARY 16
+#ifndef VERMICELLI_HPP
-#define VERM_TYPE m128
+#define VERMICELLI_HPP
-#define VERM_SET_FN set1_16x8
 
-// returns NULL if not found
+#include "util/bitutils.h"
-static really_inline
-const u8 *dvermPreconditionMasked(m128 chars1, m128 chars2,
-                                  m128 mask1, m128 mask2, const u8 *buf) {
-    m128 data = loadu128(buf); // unaligned
-    m128 v1 = eq128(chars1, and128(data, mask1));
-    m128 v2 = eq128(chars2, and128(data, mask2));
-    u32 z = movemask128(and128(v1, rshiftbyte_m128(v2, 1)));
 
-    /* no fixup of the boundary required - the aligned run will pick it up */
+#ifdef HAVE_SVE2
-    if (unlikely(z)) {
+#include "vermicelli_sve.h"
-        u32 pos = ctz32(z);
+#endif
-        return buf + pos;
+
-    }
+#ifdef __cplusplus
-    return NULL;
+extern "C" {
+#endif
+const u8 *vermicelliExec(char c, char noCase, const u8 *buf, const u8 *buf_end);
+#ifdef __cplusplus
 }
+#endif
 
-static really_inline
+#ifdef __cplusplus
-const u8 *dvermSearchAlignedMasked(m128 chars1, m128 chars2,
+extern "C" {
-                                   m128 mask1, m128 mask2, u8 c1, u8 c2, u8 m1,
+#endif
-                                   u8 m2, const u8 *buf, const u8 *buf_end) {
+const u8 *nvermicelliExec(char c, char noCase, const u8 *buf, const u8 *buf_end);
-    assert((size_t)buf % 16 == 0);
+#ifdef __cplusplus
-
-    for (; buf + 16 < buf_end; buf += 16) {
-        m128 data = load128(buf);
-        m128 v1 = eq128(chars1, and128(data, mask1));
-        m128 v2 = eq128(chars2, and128(data, mask2));
-        u32 z = movemask128(and128(v1, rshiftbyte_m128(v2, 1)));
-
-        if ((buf[15] & m1) == c1 && (buf[16] & m2) == c2) {
-            z |= (1 << 15);
-        }
-        if (unlikely(z)) {
-            u32 pos = ctz32(z);
-            return buf + pos;
-        }
-    }
-
-    return NULL;
 }
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+const u8 *rvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end);
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+const u8 *rnvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end);
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end);
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end);
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char m2, const u8 *buf, const u8 *buf_end);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* VERMICELLI_HPP */

src/nfa/vermicelli_run.h

@@ -26,7 +26,10 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "vermicelli.h"
+#include "vermicelli.hpp"
+
+#define VERM_BOUNDARY 16
+#define VERM_TYPE m128
 
 static really_inline
 const u8 *find_xverm_run(char c, char nocase, u32 repeat, UNUSED const u8 *buf,

src/nfa/vermicelli_simd.cpp

@@ -0,0 +1,549 @@
+/*
+ * Copyright (c) 2015-2020, Intel Corporation
+ * Copyright (c) 2020-2021, VectorCamp PC
+ * Copyright (c) 2021, Arm Limited
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Vermicelli: single-byte and double-byte acceleration.
+ */
+
+#include "util/bitutils.h"
+#include "util/simd_utils.h"
+
+#include "vermicelli.hpp"
+#include "util/supervector/casemask.hpp"
+#include "util/match.hpp"
+
+template <uint16_t S>
+static really_inline
+const u8 *vermicelliBlock(SuperVector<S> const data, SuperVector<S> const chars, SuperVector<S> const casemask, u8 const *buf, u16 const len);
+
+template <uint16_t S>
+static really_inline
+const u8 *vermicelliBlockNeg(SuperVector<S> const data, SuperVector<S> const chars, SuperVector<S> const casemask, u8 const *buf, u16 const len);
+
+template <uint16_t S>
+static really_inline
+const u8 *rvermicelliBlock(SuperVector<S> const data, SuperVector<S> const chars, SuperVector<S> const casemask, u8 const *buf, u16 const len);
+
+template <uint16_t S>
+static really_inline
+const u8 *rvermicelliBlockNeg(SuperVector<S> const data, SuperVector<S> const chars, SuperVector<S> const casemask, const u8 *buf, u16 const len);
+
+template <uint16_t S>
+static really_inline
+const u8 *vermicelliDoubleBlock(SuperVector<S> const data, SuperVector<S> const chars1, SuperVector<S> const chars2, SuperVector<S> const casemask,
+                                u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len);
+
+template <uint16_t S>
+static really_inline
+const u8 *rvermicelliDoubleBlock(SuperVector<S> const data, SuperVector<S> const chars1, SuperVector<S> const chars2, SuperVector<S> const casemask,
+                                 u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len);
+
+template <uint16_t S>
+static really_inline
+const u8 *vermicelliDoubleMaskedBlock(SuperVector<S> const data, SuperVector<S> const chars1, SuperVector<S> const chars2,
+                                      SuperVector<S> const mask1, SuperVector<S> const mask2,
+                                      u8 const c1, u8 const c2, u8 const m1, u8 const m2, u8 const *buf, u16 const len);
+
+#if defined(ARCH_IA32) || defined(ARCH_X86_64)
+#include "x86/vermicelli.hpp"
+#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64)
+#include "arm/vermicelli.hpp"
+#endif
+
+template <uint16_t S>
+static const u8 *vermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const casemask, u8 const *buf, u8 const *buf_end) {
+    assert(buf && buf_end);
+    assert(buf < buf_end);
+    DEBUG_PRINTF("verm %p len %zu\n", buf, buf_end - buf);
+    DEBUG_PRINTF("b %s\n", buf);
+
+    const u8 *d = buf;
+    const u8 *rv;
+
+    __builtin_prefetch(d +   64);
+    __builtin_prefetch(d + 2*64);
+    __builtin_prefetch(d + 3*64);
+    __builtin_prefetch(d + 4*64);
+    DEBUG_PRINTF("start %p end %p \n", d, buf_end);
+    assert(d < buf_end);
+    if (d + S <= buf_end) {
+        // Reach vector aligned boundaries
+        DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S));
+        if (!ISALIGNED_N(d, S)) {
+            u8 const *d1 = ROUNDUP_PTR(d, S);
+            SuperVector<S> data = SuperVector<S>::loadu(d);
+            rv = vermicelliBlock(data, chars, casemask, d, S);
+            if (rv) return rv;
+            d = d1;
+        }
+
+        while(d + S <= buf_end) {
+            __builtin_prefetch(d + 64);
+            DEBUG_PRINTF("d %p \n", d);
+            SuperVector<S> data = SuperVector<S>::load(d);
+            rv = vermicelliBlock(data, chars, casemask, d, S);
+            if (rv) return rv;
+            d += S;
+        }
+    }
+
+    DEBUG_PRINTF("d %p e %p \n", d, buf_end);
+    // finish off tail
+
+    if (d != buf_end) {
+        SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d);
+        rv = vermicelliBlock(data, chars, casemask, d, buf_end - d);
+        DEBUG_PRINTF("rv %p \n", rv);
+        if (rv && rv < buf_end) return rv;
+    }
+
+    return buf_end;
+}
+
+template <uint16_t S>
+static const u8 *nvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const casemask, const u8 *buf, const u8 *buf_end) {
+    assert(buf && buf_end);
+    assert(buf < buf_end);
+    DEBUG_PRINTF("verm %p len %zu\n", buf, buf_end - buf);
+    DEBUG_PRINTF("b %s\n", buf);
+
+    const u8 *d = buf;
+    const u8 *rv;
+
+    __builtin_prefetch(d +   64);
+    __builtin_prefetch(d + 2*64);
+    __builtin_prefetch(d + 3*64);
+    __builtin_prefetch(d + 4*64);
+    DEBUG_PRINTF("start %p end %p \n", d, buf_end);
+    assert(d < buf_end);
+    if (d + S <= buf_end) {
+        // Reach vector aligned boundaries
+        DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S));
+        if (!ISALIGNED_N(d, S)) {
+            u8 const *d1 = ROUNDUP_PTR(d, S);
+            SuperVector<S> data = SuperVector<S>::loadu(d);
+            rv = vermicelliBlockNeg(data, chars, casemask, d, S);
+            if (rv) return rv;
+            d = d1;
+        }
+
+        while(d + S <= buf_end) {
+            __builtin_prefetch(d + 64);
+            DEBUG_PRINTF("d %p \n", d);
+            SuperVector<S> data = SuperVector<S>::load(d);
+            rv = vermicelliBlockNeg(data, chars, casemask, d, S);
+            if (rv) return rv;
+            d += S;
+        }
+    }
+
+    DEBUG_PRINTF("d %p e %p \n", d, buf_end);
+    // finish off tail
+
+    if (d != buf_end) {
+        SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d);
+        rv = vermicelliBlockNeg(data, chars, casemask, d, buf_end - d);
+        DEBUG_PRINTF("rv %p \n", rv);
+        if (rv && rv < buf_end) return rv;
+    }
+
+    return buf_end;
+}
+
+// Reverse vermicelli scan. Provides exact semantics and returns (buf - 1) if
+// character not found.
+template <uint16_t S>
+const u8 *rvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const casemask, const u8 *buf, const u8 *buf_end) {
+    assert(buf && buf_end);
+    assert(buf < buf_end);
+    DEBUG_PRINTF("rverm %p len %zu\n", buf, buf_end - buf);
+    DEBUG_PRINTF("b %s\n", buf);
+
+    const u8 *d = buf_end;
+    const u8 *rv;
+
+    __builtin_prefetch(d -   64);
+    __builtin_prefetch(d - 2*64);
+    __builtin_prefetch(d - 3*64);
+    __builtin_prefetch(d - 4*64);
+    DEBUG_PRINTF("start %p end %p \n", buf, d);
+    assert(d > buf);
+    if (d - S >= buf) {
+        // Reach vector aligned boundaries
+        DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S));
+        if (!ISALIGNED_N(d, S)) {
+            u8 const *d1 = ROUNDDOWN_PTR(d, S);
+            SuperVector<S> data = SuperVector<S>::loadu(d - S);
+            rv = rvermicelliBlock(data, chars, casemask, d - S, S);
+            DEBUG_PRINTF("rv %p \n", rv);
+            if (rv) return rv;
+            d = d1;
+        }
+
+        while (d - S >= buf) {
+            DEBUG_PRINTF("aligned %p \n", d);
+            // On large packet buffers, this prefetch appears to get us about 2%.
+            __builtin_prefetch(d - 64);
+
+            d -= S;
+            SuperVector<S> data = SuperVector<S>::load(d);
+            rv = rvermicelliBlock(data, chars, casemask, d, S);
+            if (rv) return rv;
+        }
+    }
+
+    DEBUG_PRINTF("tail d %p e %p \n", buf, d);
+    // finish off head
+
+    if (d != buf) {
+        SuperVector<S> data = SuperVector<S>::loadu(buf);
+        rv = rvermicelliBlock(data, chars, casemask, buf, d - buf);
+        DEBUG_PRINTF("rv %p \n", rv);
+        if (rv && rv < buf_end) return rv;
+    }
+
+    return buf - 1;
+}
+
+// Reverse vermicelli scan. Provides exact semantics and returns (buf - 1) if
+// character not found.
+template <uint16_t S>
+const u8 *rnvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const casemask, const u8 *buf, const u8 *buf_end) {
+    assert(buf && buf_end);
+    assert(buf < buf_end);
+    DEBUG_PRINTF("rverm %p len %zu\n", buf, buf_end - buf);
+    DEBUG_PRINTF("b %s\n", buf);
+
+    const u8 *d = buf_end;
+    const u8 *rv;
+
+    __builtin_prefetch(d -   64);
+    __builtin_prefetch(d - 2*64);
+    __builtin_prefetch(d - 3*64);
+    __builtin_prefetch(d - 4*64);
+    DEBUG_PRINTF("start %p end %p \n", buf, d);
+    assert(d > buf);
+    if (d - S >= buf) {
+        // Reach vector aligned boundaries
+        DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S));
+        if (!ISALIGNED_N(d, S)) {
+            u8 const *d1 = ROUNDDOWN_PTR(d, S);
+            SuperVector<S> data = SuperVector<S>::loadu(d - S);
+            rv = rvermicelliBlockNeg(data, chars, casemask, d - S, S);
+            DEBUG_PRINTF("rv %p \n", rv);
+            if (rv) return rv;
+            d = d1;
+        }
+
+        while (d - S >= buf) {
+            DEBUG_PRINTF("aligned %p \n", d);
+            // On large packet buffers, this prefetch appears to get us about 2%.
+            __builtin_prefetch(d - 64);
+
+            d -= S;
+            SuperVector<S> data = SuperVector<S>::load(d);
+            rv = rvermicelliBlockNeg(data, chars, casemask, d, S);
+            if (rv) return rv;
+        }
+    }
+
+    DEBUG_PRINTF("tail d %p e %p \n", buf, d);
+    // finish off head
+
+    if (d != buf) {
+        SuperVector<S> data = SuperVector<S>::loadu(buf);
+        rv = rvermicelliBlockNeg(data, chars, casemask, buf, d - buf);
+        DEBUG_PRINTF("rv %p \n", rv);
+        if (rv && rv < buf_end) return rv;
+    }
+
+    return buf - 1;
+}
+
+template <uint16_t S>
+static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector<S> const casemask,
+                                          const u8 *buf, const u8 *buf_end) {
+    assert(buf && buf_end);
+    assert(buf < buf_end);
+    DEBUG_PRINTF("verm %p len %zu\n", buf, buf_end - buf);
+    DEBUG_PRINTF("b %s\n", buf);
+
+    const u8 *d = buf;
+    const u8 *rv;
+    // SuperVector<S> lastmask1{0};
+    const SuperVector<VECTORSIZE> chars1 = SuperVector<VECTORSIZE>::dup_u8(c1);
+    const SuperVector<VECTORSIZE> chars2 = SuperVector<VECTORSIZE>::dup_u8(c2);
+    const u8 casechar = casemask.u.u8[0];
+
+    __builtin_prefetch(d +   64);
+    __builtin_prefetch(d + 2*64);
+    __builtin_prefetch(d + 3*64);
+    __builtin_prefetch(d + 4*64);
+    DEBUG_PRINTF("start %p end %p \n", d, buf_end);
+    assert(d < buf_end);
+    if (d + S <= buf_end) {
+        // Reach vector aligned boundaries
+        DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S));
+        if (!ISALIGNED_N(d, S)) {
+            u8 const *d1 = ROUNDUP_PTR(d, S);
+            SuperVector<S> data = SuperVector<S>::loadu(d);
+            rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d, S);
+            if (rv) return rv;
+            d = d1;
+        }
+
+        while(d + S <= buf_end) {
+            __builtin_prefetch(d + 64);
+            DEBUG_PRINTF("d %p \n", d);
+            SuperVector<S> data = SuperVector<S>::load(d);
+            rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d, S);
+            if (rv) return rv;
+            d += S;
+        }
+    }
+
+    DEBUG_PRINTF("tail d %p e %p \n", d, buf_end);
+    // finish off tail
+
+    if (d != buf_end) {
+        SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d);
+        rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d, buf_end - d);
+        DEBUG_PRINTF("rv %p \n", rv);
+        if (rv && rv < buf_end) return rv;
+    }
+
+    DEBUG_PRINTF("real tail d %p e %p \n", d, buf_end);
+    /* check for partial match at end */
+    u8 mask = casemask.u.u8[0];
+    if ((buf_end[-1] & mask) == (u8)c1) {
+        DEBUG_PRINTF("partial!!!\n");
+        return buf_end - 1;
+    }
+
+    return buf_end;
+}
+
+// /* returns highest offset of c2 (NOTE: not c1) */
+template <uint16_t S>
+const u8 *rvermicelliDoubleExecReal(char c1, char c2, SuperVector<S> const casemask, const u8 *buf, const u8 *buf_end) {
+    assert(buf && buf_end);
+    assert(buf < buf_end);
+    DEBUG_PRINTF("rverm %p len %zu\n", buf, buf_end - buf);
+    DEBUG_PRINTF("b %s\n", buf);
+    char s[255];
+    snprintf(s, buf_end - buf + 1, "%s", buf);
+    DEBUG_PRINTF("b %s\n", s);
+
+    const u8 *d = buf_end;
+    const u8 *rv;
+    const SuperVector<VECTORSIZE> chars1 = SuperVector<VECTORSIZE>::dup_u8(c1);
+    const SuperVector<VECTORSIZE> chars2 = SuperVector<VECTORSIZE>::dup_u8(c2);
+    const u8 casechar = casemask.u.u8[0];
+
+    __builtin_prefetch(d -   64);
+    __builtin_prefetch(d - 2*64);
+    __builtin_prefetch(d - 3*64);
+    __builtin_prefetch(d - 4*64);
+    DEBUG_PRINTF("start %p end %p \n", buf, d);
+    assert(d > buf);
+    if (d - S >= buf) {
+        // Reach vector aligned boundaries
+        DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S));
+        if (!ISALIGNED_N(d, S)) {
+            u8 const *d1 = ROUNDDOWN_PTR(d, S);
+            SuperVector<S> data = SuperVector<S>::loadu(d - S);
+            rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d - S, S);
+            DEBUG_PRINTF("rv %p \n", rv);
+            if (rv && rv < buf_end) return rv;
+            d = d1;
+        }
+
+        while (d - S >= buf) {
+            DEBUG_PRINTF("aligned %p \n", d);
+            // On large packet buffers, this prefetch appears to get us about 2%.
+            __builtin_prefetch(d - 64);
+
+            d -= S;
+            SuperVector<S> data = SuperVector<S>::load(d);
+            rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d, S);
+            if (rv) return rv;
+        }
+    }
+
+    DEBUG_PRINTF("tail d %p e %p \n", buf, d);
+    // finish off head
+
+    if (d != buf) {
+        SuperVector<S> data = SuperVector<S>::loadu(buf);
+        rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, buf, d - buf);
+        DEBUG_PRINTF("rv %p \n", rv);
+        if (rv && rv < buf_end) return rv;
+    }
+
+    return buf - 1;
+}
+
+template <uint16_t S>
+static const u8 *vermicelliDoubleMaskedExecReal(u8 const c1, u8 const c2, u8 const m1, u8 const m2,
+                                                const u8 *buf, const u8 *buf_end) {
+    assert(buf && buf_end);
+    assert(buf < buf_end);
+    DEBUG_PRINTF("verm %p len %zu\n", buf, buf_end - buf);
+    DEBUG_PRINTF("b %s\n", buf);
+
+    const u8 *d = buf;
+    const u8 *rv;
+    // SuperVector<S> lastmask1{0};
+    const SuperVector<VECTORSIZE> chars1 = SuperVector<VECTORSIZE>::dup_u8(c1);
+    const SuperVector<VECTORSIZE> chars2 = SuperVector<VECTORSIZE>::dup_u8(c2);
+    const SuperVector<VECTORSIZE> mask1 = SuperVector<VECTORSIZE>::dup_u8(m1);
+    const SuperVector<VECTORSIZE> mask2 = SuperVector<VECTORSIZE>::dup_u8(m2);
+
+    __builtin_prefetch(d +   64);
+    __builtin_prefetch(d + 2*64);
+    __builtin_prefetch(d + 3*64);
+    __builtin_prefetch(d + 4*64);
+    DEBUG_PRINTF("start %p end %p \n", d, buf_end);
+    assert(d < buf_end);
+    if (d + S <= buf_end) {
+        // Reach vector aligned boundaries
+        DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S));
+        if (!ISALIGNED_N(d, S)) {
+            u8 const *d1 = ROUNDUP_PTR(d, S);
+            SuperVector<S> data = SuperVector<S>::loadu(d);
+            rv = vermicelliDoubleMaskedBlock(data, chars1, chars2, mask1, mask2, c1, c2, m1, m2, d, S);
+            if (rv) return rv;
+            d = d1;
+        }
+
+        while(d + S <= buf_end) {
+            __builtin_prefetch(d + 64);
+            DEBUG_PRINTF("d %p \n", d);
+            SuperVector<S> data = SuperVector<S>::load(d);
+            rv = vermicelliDoubleMaskedBlock(data, chars1, chars2, mask1, mask2, c1, c2, m1, m2, d, S);
+            if (rv) return rv;
+            d += S;
+        }
+    }
+
+    DEBUG_PRINTF("tail d %p e %p \n", d, buf_end);
+    // finish off tail
+
+    if (d != buf_end) {
+        SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d);
+        rv = vermicelliDoubleMaskedBlock(data, chars1, chars2, mask1, mask2, c1, c2, m1, m2, d, buf_end - d);
+        DEBUG_PRINTF("rv %p \n", rv);
+        if (rv && rv < buf_end) return rv;
+    }
+
+    DEBUG_PRINTF("real tail d %p e %p \n", d, buf_end);
+    /* check for partial match at end */
+    if ((buf_end[-1] & m1) == (u8)c1) {
+        DEBUG_PRINTF("partial!!!\n");
+        return buf_end - 1;
+    }
+
+    return buf_end;
+}
+
+extern "C" const u8 *vermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) {
+    DEBUG_PRINTF("verm scan %s\\x%02hhx over %zu bytes\n",
+                 nocase ? "nocase " : "", c, (size_t)(buf_end - buf));
+    assert(buf < buf_end);
+
+    const SuperVector<VECTORSIZE> chars = SuperVector<VECTORSIZE>::dup_u8(c);
+    const SuperVector<VECTORSIZE> casemask{nocase ? getCaseMask<VECTORSIZE>() : SuperVector<VECTORSIZE>::Ones()};
+
+    return vermicelliExecReal<VECTORSIZE>(chars, casemask, buf, buf_end);
+}
+
+/* like vermicelliExec except returns the address of the first character which
+ * is not c */
+extern "C" const u8 *nvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) {
+    DEBUG_PRINTF("nverm scan %s\\x%02hhx over %zu bytes\n",
+                 nocase ? "nocase " : "", c, (size_t)(buf_end - buf));
+    assert(buf < buf_end);
+
+    const SuperVector<VECTORSIZE> chars = SuperVector<VECTORSIZE>::dup_u8(c);
+    const SuperVector<VECTORSIZE> casemask{nocase ? getCaseMask<VECTORSIZE>() : SuperVector<VECTORSIZE>::Ones()};
+
+    return nvermicelliExecReal<VECTORSIZE>(chars, casemask, buf, buf_end);
+}
+
+extern "C" const u8 *rvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) {
+    DEBUG_PRINTF("rev verm scan %s\\x%02hhx over %zu bytes\n",
+                 nocase ? "nocase " : "", c, (size_t)(buf_end - buf));
+    assert(buf < buf_end);
+
+    const SuperVector<VECTORSIZE> chars = SuperVector<VECTORSIZE>::dup_u8(c);
+    const SuperVector<VECTORSIZE> casemask{nocase ? getCaseMask<VECTORSIZE>() : SuperVector<VECTORSIZE>::Ones()};
+
+    return rvermicelliExecReal<VECTORSIZE>(chars, casemask, buf, buf_end);
+}
+
+extern "C" const u8 *rnvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) {
+     DEBUG_PRINTF("rev verm scan %s\\x%02hhx over %zu bytes\n",
+                  nocase ? "nocase " : "", c, (size_t)(buf_end - buf));
+    assert(buf < buf_end);
+
+    const SuperVector<VECTORSIZE> chars = SuperVector<VECTORSIZE>::dup_u8(c);
+    const SuperVector<VECTORSIZE> casemask{nocase ? getCaseMask<VECTORSIZE>() : SuperVector<VECTORSIZE>::Ones()};
+
+    return rnvermicelliExecReal<VECTORSIZE>(chars, casemask, buf, buf_end);
+}
+
+extern "C" const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end) {
+    DEBUG_PRINTF("double verm scan %s\\x%02hhx%02hhx over %zu bytes\n",
+                 nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf));
+    assert(buf < buf_end);
+
+    const SuperVector<VECTORSIZE> casemask{nocase ? getCaseMask<VECTORSIZE>() : SuperVector<VECTORSIZE>::Ones()};
+
+    return vermicelliDoubleExecReal<VECTORSIZE>(c1, c2, casemask, buf, buf_end);
+}
+
+extern "C" const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end) {
+    DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n",
+                 nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf));
+    assert(buf < buf_end);
+
+    const SuperVector<VECTORSIZE> casemask{nocase ? getCaseMask<VECTORSIZE>() : SuperVector<VECTORSIZE>::Ones()};
+
+    return rvermicelliDoubleExecReal<VECTORSIZE>(c1, c2, casemask, buf, buf_end);
+}
+
+extern "C" const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char m2,
+                                     const u8 *buf, const u8 *buf_end) {
+    DEBUG_PRINTF("double verm scan (\\x%02hhx&\\x%02hhx)(\\x%02hhx&\\x%02hhx) "
+                 "over %zu bytes\n", c1, m1, c2, m2, (size_t)(buf_end - buf));
+    assert(buf < buf_end);
+
+    return vermicelliDoubleMaskedExecReal<VECTORSIZE>(c1, c2, m1, m2, buf, buf_end);
+}

src/nfa/x86/shufti.hpp

@@ -31,12 +31,6 @@
  * \brief Shufti: character class acceleration.
  */
 
-#ifndef SHUFTI_SIMD_X86_HPP
-#define SHUFTI_SIMD_X86_HPP
-
-#include "util/supervector/supervector.hpp"
-#include "util/match.hpp"
-
 template <uint16_t S>
 static really_inline
 const SuperVector<S> blockSingleMask(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars) {
@@ -44,12 +38,10 @@ const SuperVector<S> blockSingleMask(SuperVector<S> mask_lo, SuperVector<S> mask
 
     SuperVector<S> c_lo = chars & low4bits;
     SuperVector<S> c_hi = chars.template vshr_64_imm<4>() & low4bits;
-    c_lo = mask_lo.template pshufb(c_lo);
+    c_lo = mask_lo.pshufb(c_lo);
-    c_hi = mask_hi.template pshufb(c_hi);
+    c_hi = mask_hi.pshufb(c_hi);
 
-    SuperVector c = c_lo & c_hi;
+    return (c_lo & c_hi).eq(SuperVector<S>::Zeroes());
-
-    return c.eq(SuperVector<S>::Zeroes());
 }
 
 template <uint16_t S>
@@ -80,5 +72,3 @@ SuperVector<S> blockDoubleMask(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi,
 
     return c.eq(SuperVector<S>::Ones());
 }
-
-#endif // SHUFTI_SIMD_X86_HPP

src/nfa/x86/vermicelli.hpp

@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2015-2020, Intel Corporation
+ * Copyright (c) 2020-2021, VectorCamp PC
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Vermicelli: single-byte and double-byte acceleration.
+ */
+
+template <uint16_t S>
+static really_inline
+const u8 *vermicelliBlock(SuperVector<S> const data, SuperVector<S> const chars, SuperVector<S> const casemask, u8 const *buf, u16 const len) {
+
+    SuperVector<S> mask = chars.eq(casemask & data);
+    return first_non_zero_match<S>(buf, mask, len);
+}
+
+template <uint16_t S>
+static really_inline
+const u8 *vermicelliBlockNeg(SuperVector<S> const data, SuperVector<S> const chars, SuperVector<S> const casemask, u8 const *buf, u16 const len) {
+
+    SuperVector<S> mask = chars.eq(casemask & data);
+    return first_zero_match_inverted<S>(buf, mask, len);
+}
+
+template <uint16_t S>
+static really_inline
+const u8 *rvermicelliBlock(SuperVector<S> const data, SuperVector<S> const chars, SuperVector<S> const casemask, u8 const *buf, u16 const len) {
+
+    SuperVector<S> mask = chars.eq(casemask & data);
+    return last_non_zero_match<S>(buf, mask, len);
+}
+
+template <uint16_t S>
+static really_inline
+const u8 *rvermicelliBlockNeg(SuperVector<S> const data, SuperVector<S> const chars, SuperVector<S> const casemask, const u8 *buf, u16 const len) {
+
+    data.print8("data");
+    chars.print8("chars");
+    casemask.print8("casemask");
+    SuperVector<S> mask = chars.eq(casemask & data);
+    mask.print8("mask");
+    return last_zero_match_inverted<S>(buf, mask, len);
+}
+
+template <uint16_t S>
+static really_inline
+const u8 *vermicelliDoubleBlock(SuperVector<S> const data, SuperVector<S> const chars1, SuperVector<S> const chars2, SuperVector<S> const casemask,
+                                u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len) {
+
+    SuperVector<S> v = casemask & data;
+    SuperVector<S> mask1 = chars1.eq(v);
+    SuperVector<S> mask2 = chars2.eq(v);
+    SuperVector<S> mask = mask1 & (mask2 >> 1);
+
+    DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]);
+    bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1));
+    DEBUG_PRINTF("partial = %d\n", partial_match);
+    if (partial_match) return buf - 1;
+
+    return first_non_zero_match<S>(buf, mask, len);
+}
+
+template <uint16_t S>
+static really_inline
+const u8 *rvermicelliDoubleBlock(SuperVector<S> const data, SuperVector<S> const chars1, SuperVector<S> const chars2, SuperVector<S> const casemask,
+                                 u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len) {
+
+    SuperVector<S> v = casemask & data;
+    SuperVector<S> mask1 = chars1.eq(v);
+    SuperVector<S> mask2 = chars2.eq(v);
+    SuperVector<S> mask = (mask1 << 1)& mask2;
+
+    DEBUG_PRINTF("buf[0] = %02hhx, buf[-1] = %02hhx\n", buf[0], buf[-1]);
+    bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1));
+    DEBUG_PRINTF("partial = %d\n", partial_match);
+    if (partial_match) {
+        mask = mask | (SuperVector<S>::Ones() >> (S-1));
+    }
+
+    return last_non_zero_match<S>(buf, mask, len);
+}
+
+template <uint16_t S>
+static really_inline
+const u8 *vermicelliDoubleMaskedBlock(SuperVector<S> const data, SuperVector<S> const chars1, SuperVector<S> const chars2,
+                                      SuperVector<S> const mask1, SuperVector<S> const mask2,
+                                      u8 const c1, u8 const c2, u8 const m1, u8 const m2, u8 const *buf, u16 const len) {
+
+    SuperVector<S> v1 = chars1.eq(data & mask1);
+    SuperVector<S> v2 = chars2.eq(data & mask2);
+    SuperVector<S> mask = v1 & (v2 >> 1);
+
+    DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]);
+    bool partial_match = (((buf[0] & m1) == c2) && ((buf[-1] & m2) == c1));
+    DEBUG_PRINTF("partial = %d\n", partial_match);
+    if (partial_match) return buf - 1;
+
+    return first_non_zero_match<S>(buf, mask, len);
+}
+

src/util/arch/arm/match.hpp

@@ -29,7 +29,7 @@
 
 template <>
 really_really_inline
-const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> mask) {
+const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> mask, u16 const UNUSED len) {
     uint32x4_t m = mask.u.u32x4[0];
     uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0);
     if (vmax != 0) {
@@ -48,7 +48,44 @@ const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> mask) {
 
 template <>
 really_really_inline
-const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> mask) {
+const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> mask, u16 const UNUSED len) {
+    uint32x4_t m = mask.u.u32x4[0];
+    uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0);
+    if (vmax != 0) {
+    typename SuperVector<16>::movemask_type z = mask.movemask();
+        DEBUG_PRINTF("buf %p z %08x \n", buf, z);
+        DEBUG_PRINTF("z %08x\n", z);
+        u32 pos = clz32(z & 0xffff);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        assert(pos >= 16 && pos < 32);
+        return buf + (31 - pos);
+    } else {
+        return NULL; // no match
+    }
+}
+
+template <>
+really_really_inline
+const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask, u16 const UNUSED len) {
+    uint32x4_t m = mask.u.u32x4[0];
+    uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0);
+    if (vmax != 0) {
+	typename SuperVector<16>::movemask_type z = mask.movemask();
+        DEBUG_PRINTF("z %08x\n", z);
+        DEBUG_PRINTF("buf %p z %08x \n", buf, z);
+        u32 pos = ctz32(z & 0xffff);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        assert(pos < 16);
+        DEBUG_PRINTF("buf + pos %p\n", buf + pos);
+        return buf + pos;
+    } else {
+        return NULL; // no match
+    }
+}
+
+template <>
+really_really_inline
+const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask, u16 const UNUSED len) {
     uint32x4_t m = mask.u.u32x4[0];
     uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0);
     if (vmax != 0) {

src/util/arch/x86/match.hpp

@@ -29,7 +29,106 @@
 
 template <>
 really_really_inline
-const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> v) {
+const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> v, u16 const UNUSED len) {
+    SuperVector<16>::movemask_type z = v.movemask();
+    DEBUG_PRINTF("buf %p z %08x \n", buf, z);
+    DEBUG_PRINTF("z %08x\n", z);
+    if (unlikely(z)) {
+        u32 pos = ctz32(z);
+        DEBUG_PRINTF("~z %08x\n", ~z);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        assert(pos < 16);
+        return buf + pos;
+    } else {
+        return NULL; // no match
+    }
+}
+
+template <>
+really_really_inline
+const u8 *first_non_zero_match<32>(const u8 *buf, SuperVector<32> v, u16 const UNUSED len) {
+    SuperVector<32>::movemask_type z = v.movemask();
+    DEBUG_PRINTF("z 0x%08x\n", z);
+    if (unlikely(z)) {
+        u32 pos = ctz32(z);
+        assert(pos < 32);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        return buf + pos;
+    } else {
+        return NULL; // no match
+    }
+}
+template <>
+really_really_inline
+const u8 *first_non_zero_match<64>(const u8 *buf, SuperVector<64>v, u16 const len) {
+    SuperVector<64>::movemask_type z = v.movemask();
+    DEBUG_PRINTF("z 0x%016llx\n", z);
+    u64a mask = (~0ULL) >> (64 - len);
+    DEBUG_PRINTF("mask %016llx\n", mask);
+    z &= mask;
+    DEBUG_PRINTF("z 0x%016llx\n", z);
+    if (unlikely(z)) {
+        u32 pos = ctz64(z);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        assert(pos < 64);
+        return buf + pos;
+    } else {
+        return NULL; // no match
+    }
+}
+
+template <>
+really_really_inline
+const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> v, u16 const UNUSED len) {
+    SuperVector<16>::movemask_type z = v.movemask();
+    DEBUG_PRINTF("buf %p z %08x \n", buf, z);
+    DEBUG_PRINTF("z %08x\n", z);
+    if (unlikely(z)) {
+        u32 pos = clz32(z);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        assert(pos >= 16 && pos < 32);
+        return buf + (31 - pos);
+    } else {
+        return NULL; // no match
+    }
+}
+
+template <>
+really_really_inline
+const u8 *last_non_zero_match<32>(const u8 *buf, SuperVector<32> v, u16 const UNUSED len) {
+    SuperVector<32>::movemask_type z = v.movemask();
+    DEBUG_PRINTF("z 0x%08x\n", z);
+    if (unlikely(z)) {
+        u32 pos = clz32(z);
+        assert(pos < 32);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        return buf + (31 - pos);
+    } else {
+        return NULL; // no match
+    }
+}
+template <>
+really_really_inline
+const u8 *last_non_zero_match<64>(const u8 *buf, SuperVector<64>v, u16 const len) {
+    SuperVector<64>::movemask_type z = v.movemask();
+    DEBUG_PRINTF("z 0x%016llx\n", z);
+    u64a mask = (~0ULL) >> (64 - len);
+    DEBUG_PRINTF("mask %016llx\n", mask);
+    z &= mask;
+    DEBUG_PRINTF("z 0x%016llx\n", z);
+    if (unlikely(z)) {
+        u32 pos = clz64(z);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        assert(pos < 64);
+        return buf + (63 - pos);
+    } else {
+        return NULL; // no match
+    }
+}
+
+template <>
+really_really_inline
+const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v, u16 const UNUSED len) {
     SuperVector<16>::movemask_type z = v.movemask();
     DEBUG_PRINTF("buf %p z %08x \n", buf, z);
     DEBUG_PRINTF("z %08x\n", z);
@@ -46,7 +145,7 @@ const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> v) {
 
 template <>
 really_really_inline
-const u8 *firstMatch<32>(const u8 *buf, SuperVector<32> v) {
+const u8 *first_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v, u16 const UNUSED len) {
     SuperVector<32>::movemask_type z = v.movemask();
     DEBUG_PRINTF("z 0x%08x\n", z);
     if (unlikely(z != 0xffffffff)) {
@@ -60,11 +159,15 @@ const u8 *firstMatch<32>(const u8 *buf, SuperVector<32> v) {
 }
 template <>
 really_really_inline
-const u8 *firstMatch<64>(const u8 *buf, SuperVector<64>v) {
+const u8 *first_zero_match_inverted<64>(const u8 *buf, SuperVector<64>v, u16 const len) {
     SuperVector<64>::movemask_type z = v.movemask();
     DEBUG_PRINTF("z 0x%016llx\n", z);
-    if (unlikely(z != ~0ULL)) {
+    u64a mask = (~0ULL) >> (64 - len);
-        u32 pos = ctz64(~z);
+    DEBUG_PRINTF("mask %016llx\n", mask);
+    z = ~z & mask;
+    DEBUG_PRINTF("z 0x%016llx\n", z);
+    if (unlikely(z)) {
+        u32 pos = ctz64(z);
         DEBUG_PRINTF("match @ pos %u\n", pos);
         assert(pos < 64);
         return buf + pos;
@@ -75,7 +178,7 @@ const u8 *firstMatch<64>(const u8 *buf, SuperVector<64>v) {
 
 template <>
 really_really_inline
-const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> v) {
+const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v, uint16_t UNUSED len ) {
     SuperVector<16>::movemask_type z = v.movemask();
     DEBUG_PRINTF("buf %p z %08x \n", buf, z);
     DEBUG_PRINTF("z %08x\n", z);
@@ -92,10 +195,10 @@ const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> v) {
 
 template<>
 really_really_inline
-const u8 *lastMatch<32>(const u8 *buf, SuperVector<32> v) {
+const u8 *last_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v, uint16_t UNUSED len) {
     SuperVector<32>::movemask_type z = v.movemask();
     if (unlikely(z != 0xffffffff)) {
-        u32 pos = clz32(~z);
+        u32 pos = clz32(~z & 0xffffffff);
         DEBUG_PRINTF("buf=%p, pos=%u\n", buf, pos);
         assert(pos < 32);
         return buf + (31 - pos);
@@ -106,11 +209,17 @@ const u8 *lastMatch<32>(const u8 *buf, SuperVector<32> v) {
 
 template <>
 really_really_inline
-const u8 *lastMatch<64>(const u8 *buf, SuperVector<64> v) {
+const u8 *last_zero_match_inverted<64>(const u8 *buf, SuperVector<64> v, uint16_t len) {
+    v.print8("v");
     SuperVector<64>::movemask_type z = v.movemask();
     DEBUG_PRINTF("z 0x%016llx\n", z);
-    if (unlikely(z != ~0ULL)) {
+    u64a mask = (~0ULL) >> (64 - len);
-        u32 pos = clz64(~z);
+    DEBUG_PRINTF("mask %016llx\n", mask);
+    z = ~z & mask;
+    DEBUG_PRINTF("z 0x%016llx\n", z);
+    if (unlikely(z)) {
+        u32 pos = clz64(z);
+        DEBUG_PRINTF("~z 0x%016llx\n", ~z);
         DEBUG_PRINTF("match @ pos %u\n", pos);
         assert(pos < 64);
         return buf + (63 - pos);

src/util/match.hpp

@@ -38,10 +38,16 @@
 #include "util/supervector/supervector.hpp"
 
 template <u16 S>
-const u8 *firstMatch(const u8 *buf, SuperVector<S> v);
+const u8 *first_non_zero_match(const u8 *buf, SuperVector<S> v, u16 const len = S);
 
 template <u16 S>
-const u8 *lastMatch(const u8 *buf, SuperVector<S> v);
+const u8 *last_non_zero_match(const u8 *buf, SuperVector<S> v, u16 const len = S);
+
+template <u16 S>
+const u8 *first_zero_match_inverted(const u8 *buf, SuperVector<S> v, u16 const len = S);
+
+template <u16 S>
+const u8 *last_zero_match_inverted(const u8 *buf, SuperVector<S> v, u16 len = S);
 
 #if defined(ARCH_IA32) || defined(ARCH_X86_64)
 #include "util/arch/x86/match.hpp"

src/util/supervector/casemask.hpp

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2017, Intel Corporation
+ * Copyright (c) 2020-2021, VectorCamp PC
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef CASEMASK_HPP
+#define CASEMASK_HPP
+
+#include "util/supervector/supervector.hpp"
+
+static u8 CASEMASK[] = { 0xff, 0xdf };
+
+static really_inline
+u8 caseClear8(u8 x, bool noCase)
+{
+    return static_cast<u8>(x & CASEMASK[(u8)noCase]);
+}
+
+template<uint16_t S>
+static really_inline SuperVector<S> getMask(u8 c, bool noCase) {
+    u8 k = caseClear8(c, noCase);
+    return SuperVector<S>(k);
+}
+
+template<uint16_t S>
+static really_inline SuperVector<S> getCaseMask(void) {
+    return SuperVector<S>(CASEMASK[1]);
+}
+
+#endif // CASEMASK_HPP

unit/internal/rvermicelli.cpp

@@ -30,7 +30,7 @@
 #include "config.h"
 
 #include "gtest/gtest.h"
-#include "nfa/vermicelli.h"
+#include "nfa/vermicelli.hpp"
 
 #define BOUND (~(VERM_BOUNDARY - 1))
 

unit/internal/vermicelli.cpp

@@ -30,7 +30,7 @@
 #include "config.h"
 
 #include "gtest/gtest.h"
-#include "nfa/vermicelli.h"
+#include "nfa/vermicelli.hpp"
 
 TEST(Vermicelli, ExecNoMatch1) {
     char t1[] = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";