diff --git a/CMakeLists.txt b/CMakeLists.txt index 410d4214..0875b105 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -616,7 +616,7 @@ set (hs_exec_SRCS src/nfa/tamarama_internal.h src/nfa/truffle.cpp src/nfa/truffle.h - src/nfa/vermicelli.h + src/nfa/vermicelli.hpp src/nfa/vermicelli_run.h src/nfa/vermicelli_simd.cpp src/som/som.h diff --git a/src/hwlm/hwlm.c b/src/hwlm/hwlm.c index 5d69e3c4..e50deff7 100644 --- a/src/hwlm/hwlm.c +++ b/src/hwlm/hwlm.c @@ -39,7 +39,6 @@ #include "nfa/accel.h" #include "nfa/shufti.h" #include "nfa/truffle.h" -#include "nfa/vermicelli.h" #include "nfa/vermicelli.hpp" #include diff --git a/src/nfa/accel.c b/src/nfa/accel.c index b35e0633..7661b7a7 100644 --- a/src/nfa/accel.c +++ b/src/nfa/accel.c @@ -30,7 +30,6 @@ #include "accel.h" #include "shufti.h" #include "truffle.h" -#include "vermicelli.h" #include "vermicelli.hpp" #include "ue2common.h" diff --git a/src/nfa/castle.c b/src/nfa/castle.c index be29ca29..29208f8d 100644 --- a/src/nfa/castle.c +++ b/src/nfa/castle.c @@ -40,7 +40,6 @@ #include "repeat.h" #include "shufti.h" #include "truffle.h" -#include "vermicelli.h" #include "vermicelli.hpp" #include "util/bitutils.h" #include "util/multibit.h" diff --git a/src/nfa/lbr.c b/src/nfa/lbr.c index 8fc83988..52e81ad6 100644 --- a/src/nfa/lbr.c +++ b/src/nfa/lbr.c @@ -40,7 +40,6 @@ #include "repeat_internal.h" #include "shufti.h" #include "truffle.h" -#include "vermicelli.h" #include "vermicelli.hpp" #include "util/partial_store.h" #include "util/unaligned.h" @@ -534,4 +533,4 @@ char lbrFwdScanTruf(const struct NFA *nfa, const u8 *buf, #ifdef HAVE_SVE2 #include "lbr_sve.h" -#endif \ No newline at end of file +#endif diff --git a/src/nfa/limex_accel.c b/src/nfa/limex_accel.c index 4834b6a5..a85d5a07 100644 --- a/src/nfa/limex_accel.c +++ b/src/nfa/limex_accel.c @@ -40,7 +40,7 @@ #include "shufti.h" #include "truffle.h" #include "ue2common.h" -#include "vermicelli.h" +#include "vermicelli.hpp" #include "util/arch.h" #include "util/bitutils.h" #include "util/simd_utils.h" diff --git a/src/nfa/mpv.c b/src/nfa/mpv.c index 5829d43d..cba3d159 100644 --- a/src/nfa/mpv.c +++ b/src/nfa/mpv.c @@ -36,7 +36,7 @@ #include "shufti.h" #include "truffle.h" #include "ue2common.h" -#include "vermicelli.h" +#include "vermicelli.hpp" #include "vermicelli_run.h" #include "util/multibit.h" #include "util/partial_store.h" diff --git a/src/nfa/nfa_rev_api.h b/src/nfa/nfa_rev_api.h index 72224c3b..d82c52a4 100644 --- a/src/nfa/nfa_rev_api.h +++ b/src/nfa/nfa_rev_api.h @@ -35,7 +35,6 @@ #include "accel.h" #include "nfa_internal.h" -#include "vermicelli.h" #include "vermicelli.hpp" #include "util/unaligned.h" diff --git a/src/nfa/vermicelli.h b/src/nfa/vermicelli.h deleted file mode 100644 index 39e9555e..00000000 --- a/src/nfa/vermicelli.h +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (c) 2015-2020, Intel Corporation - * Copyright (c) 2021, Arm Limited - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** \file - * \brief Vermicelli: single-byte and double-byte acceleration. - */ - -#ifndef VERMICELLI_H -#define VERMICELLI_H - -#include "util/bitutils.h" -#include "util/simd_utils.h" -#include "util/unaligned.h" - -#if !defined(HAVE_AVX512) -#include "vermicelli_common.h" -#endif - -#ifdef HAVE_SVE2 -#include "vermicelli_sve.h" -#endif - -static really_inline -const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char m2, - const u8 *buf, const u8 *buf_end) { - DEBUG_PRINTF("double verm scan (\\x%02hhx&\\x%02hhx)(\\x%02hhx&\\x%02hhx) " - "over %zu bytes\n", c1, m1, c2, m2, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars1 = VERM_SET_FN(c1); - VERM_TYPE chars2 = VERM_SET_FN(c2); - VERM_TYPE mask1 = VERM_SET_FN(m1); - VERM_TYPE mask2 = VERM_SET_FN(m2); - -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = dvermMiniMasked(chars1, chars2, mask1, mask2, buf, - buf_end); - if (ptr) { - return ptr; - } - - /* check for partial match at end */ - if ((buf_end[-1] & m1) == (u8)c1) { - DEBUG_PRINTF("partial!!!\n"); - return buf_end - 1; - } - - return buf_end; - } -#endif - - assert((buf_end - buf) >= VERM_BOUNDARY); - uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY; - if (min) { - // Input isn't aligned, so we need to run one iteration with an - // unaligned load, then skip buf forward to the next aligned address. - // There's some small overlap here, but we don't mind scanning it twice - // if we can do it quickly, do we? - const u8 *p = dvermPreconditionMasked(chars1, chars2, mask1, mask2, buf); - if (p) { - return p; - } - - buf += VERM_BOUNDARY - min; - assert(buf < buf_end); - } - - // Aligned loops from here on in - const u8 *ptr = dvermSearchAlignedMasked(chars1, chars2, mask1, mask2, c1, - c2, m1, m2, buf, buf_end); - if (ptr) { - return ptr; - } - - // Tidy up the mess at the end - ptr = dvermPreconditionMasked(chars1, chars2, mask1, mask2, - buf_end - VERM_BOUNDARY); - - if (ptr) { - return ptr; - } - - /* check for partial match at end */ - if ((buf_end[-1] & m1) == (u8)c1) { - DEBUG_PRINTF("partial!!!\n"); - return buf_end - 1; - } - - return buf_end; -} - -#endif /* VERMICELLI_H */ diff --git a/src/nfa/vermicelli_common.h b/src/nfa/vermicelli_common.h deleted file mode 100644 index aca58dcb..00000000 --- a/src/nfa/vermicelli_common.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2015-2020, Intel Corporation - * Copyright (c) 2021, Arm Limited - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** \file - * \brief Vermicelli: Implementation shared between architectures. - * - * (users should include vermicelli.h instead of this) - */ - -#define VERM_BOUNDARY 16 -#define VERM_TYPE m128 -#define VERM_SET_FN set1_16x8 - -// returns NULL if not found -static really_inline -const u8 *dvermPreconditionMasked(m128 chars1, m128 chars2, - m128 mask1, m128 mask2, const u8 *buf) { - m128 data = loadu128(buf); // unaligned - m128 v1 = eq128(chars1, and128(data, mask1)); - m128 v2 = eq128(chars2, and128(data, mask2)); - u32 z = movemask128(and128(v1, rshiftbyte_m128(v2, 1))); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - u32 pos = ctz32(z); - return buf + pos; - } - return NULL; -} - -static really_inline -const u8 *dvermSearchAlignedMasked(m128 chars1, m128 chars2, - m128 mask1, m128 mask2, u8 c1, u8 c2, u8 m1, - u8 m2, const u8 *buf, const u8 *buf_end) { - assert((size_t)buf % 16 == 0); - - for (; buf + 16 < buf_end; buf += 16) { - m128 data = load128(buf); - m128 v1 = eq128(chars1, and128(data, mask1)); - m128 v2 = eq128(chars2, and128(data, mask2)); - u32 z = movemask128(and128(v1, rshiftbyte_m128(v2, 1))); - - if ((buf[15] & m1) == c1 && (buf[16] & m2) == c2) { - z |= (1 << 15); - } - if (unlikely(z)) { - u32 pos = ctz32(z); - return buf + pos; - } - } - - return NULL; -} \ No newline at end of file diff --git a/src/nfa/vermicelli_run.h b/src/nfa/vermicelli_run.h index b75f1414..1deda48a 100644 --- a/src/nfa/vermicelli_run.h +++ b/src/nfa/vermicelli_run.h @@ -26,9 +26,11 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#include "vermicelli.h" #include "vermicelli.hpp" +#define VERM_BOUNDARY 16 +#define VERM_TYPE m128 + static really_inline const u8 *find_xverm_run(char c, char nocase, u32 repeat, UNUSED const u8 *buf, const u8 *buf_start, const u8 *buf_end, char negate) { diff --git a/unit/internal/rvermicelli.cpp b/unit/internal/rvermicelli.cpp index d29b1133..5cd52e4d 100644 --- a/unit/internal/rvermicelli.cpp +++ b/unit/internal/rvermicelli.cpp @@ -30,7 +30,6 @@ #include "config.h" #include "gtest/gtest.h" -#include "nfa/vermicelli.h" #include "nfa/vermicelli.hpp" #define BOUND (~(VERM_BOUNDARY - 1)) diff --git a/unit/internal/vermicelli.cpp b/unit/internal/vermicelli.cpp index 3319b87c..e6d976ad 100644 --- a/unit/internal/vermicelli.cpp +++ b/unit/internal/vermicelli.cpp @@ -30,7 +30,6 @@ #include "config.h" #include "gtest/gtest.h" -#include "nfa/vermicelli.h" #include "nfa/vermicelli.hpp" TEST(Vermicelli, ExecNoMatch1) {