From 8be8ed309f5f8796b9ac941a992dff471094454c Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 27 Oct 2021 12:29:39 +0300 Subject: [PATCH 01/25] added refactored vermicelli_simd.cpp implementation --- src/nfa/vermicelli.hpp | 78 ++++++ src/nfa/vermicelli_simd.cpp | 508 ++++++++++++++++++++++++++++++++++++ 2 files changed, 586 insertions(+) create mode 100644 src/nfa/vermicelli.hpp create mode 100644 src/nfa/vermicelli_simd.cpp diff --git a/src/nfa/vermicelli.hpp b/src/nfa/vermicelli.hpp new file mode 100644 index 00000000..0b4686e1 --- /dev/null +++ b/src/nfa/vermicelli.hpp @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2015-2020, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2021, Arm Limited + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * \brief Vermicelli: single-byte and double-byte acceleration. + */ + +#ifndef VERMICELLI_HPP +#define VERMICELLI_HPP + +#ifdef __cplusplus +extern "C" { +#endif +const u8 *vermicelliExec(char c, char noCase, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + +#ifdef __cplusplus +extern "C" { +#endif +const u8 *nvermicelliExec(char c, char noCase, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + +#ifdef __cplusplus +extern "C" { +#endif +const u8 *rvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + +#ifdef __cplusplus +extern "C" { +#endif +const u8 *rnvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + +#ifdef __cplusplus +extern "C" { +#endif +const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + +#endif /* VERMICELLI_HPP */ \ No newline at end of file diff --git a/src/nfa/vermicelli_simd.cpp b/src/nfa/vermicelli_simd.cpp new file mode 100644 index 00000000..6348e6f3 --- /dev/null +++ b/src/nfa/vermicelli_simd.cpp @@ -0,0 +1,508 @@ +/* + * Copyright (c) 2015-2020, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2021, Arm Limited + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * \brief Vermicelli: single-byte and double-byte acceleration. + */ + +#include "util/bitutils.h" +#include "util/simd_utils.h" + +#include "vermicelli.hpp" +#include "util/supervector/casemask.hpp" +#include "util/match.hpp" + +template +static really_inline +const u8 *vermicelliSingleBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { + + SuperVector mask = chars.eq(casemask & data); + return first_non_zero_match(buf, mask); +} + +template +static really_inline +const u8 *rvermicelliSingleBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { + + SuperVector mask = chars.eq(casemask & data); + return last_non_zero_match(buf, mask); +} + +template +static really_inline +const u8 *vermicelliDoubleBlock(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, + const u8 *buf/*, SuperVector *lastmask1, size_t len = S*/) { + + // lastmask1->print8("lastmask1"); + data.print8("data"); + chars1.print8("chars1"); + chars2.print8("chars2"); + casemask.print8("casemask"); + SuperVector v = casemask & data; + v.print8("v"); + SuperVector mask1 = chars1.eq(v); + mask1.print8("mask1"); + SuperVector mask2 = chars2.eq(v); + mask2.print8("mask2"); + SuperVector mask = (mask1 & (mask2 >> 1)); + mask.print8("mask"); + DEBUG_PRINTF("len = %ld\n", len); + // *lastmask1 = mask1 >> (len -1); + // lastmask1->print8("lastmask1"); + + return first_non_zero_match(buf, mask); +} + +template +static really_inline +const u8 *vermicelliSingleBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { + + SuperVector mask = chars.eq(casemask & data); + return first_zero_match_inverted(buf, mask); +} + +template +static really_inline +const u8 *rvermicelliSingleBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { + + SuperVector mask = chars.eq(casemask & data); + return last_zero_match_inverted(buf, mask); +} +/* +template +static really_inline +const u8 *vermicelliDoubleBlockNeg(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, + const u8 *buf, size_t len = S) { + + // lastmask1.print8("lastmask1"); + data.print8("data"); + chars1.print8("chars1"); + chars2.print8("chars2"); + casemask.print8("casemask"); + SuperVector v = casemask & data; + v.print8("v"); + SuperVector mask1 = chars1.eq(v); + mask1.print8("mask1"); + SuperVector mask2 = chars2.eq(v); + mask2.print8("mask2"); + SuperVector mask = (mask1 & (mask2 >> 1));// | lastmask1; + mask.print8("mask"); + DEBUG_PRINTF("len = %ld\n", len); + // lastmask1 = mask << (len -1); + // lastmask1.print8("lastmask1"); + + return last_zero_match_inverted(buf, mask); +}*/ + +template +static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("verm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + + const u8 *d = buf; + const u8 *rv; + + __builtin_prefetch(d + 64); + __builtin_prefetch(d + 2*64); + __builtin_prefetch(d + 3*64); + __builtin_prefetch(d + 4*64); + DEBUG_PRINTF("start %p end %p \n", d, buf_end); + assert(d < buf_end); + if (d + S <= buf_end) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d); + rv = vermicelliSingleBlock(data, chars, casemask, d); + if (rv) return rv; + d = ROUNDUP_PTR(d, S); + } + + while(d + S <= buf_end) { + __builtin_prefetch(d + 64); + DEBUG_PRINTF("d %p \n", d); + SuperVector data = SuperVector::load(d); + rv = vermicelliSingleBlock(data, chars, casemask, d); + if (rv) return rv; + d += S; + } + } + + DEBUG_PRINTF("d %p e %p \n", d, buf_end); + // finish off tail + + if (d != buf_end) { + SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); + rv = vermicelliSingleBlock(data, chars, casemask, d); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } + + return buf_end; +} + +template +static const u8 *nvermicelliExecReal(SuperVector const chars, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("verm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + + const u8 *d = buf; + const u8 *rv; + + + + __builtin_prefetch(d + 64); + __builtin_prefetch(d + 2*64); + __builtin_prefetch(d + 3*64); + __builtin_prefetch(d + 4*64); + DEBUG_PRINTF("start %p end %p \n", d, buf_end); + assert(d < buf_end); + if (d + S <= buf_end) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d); + rv = vermicelliSingleBlockNeg(data, chars, casemask, d); + if (rv) return rv; + d = ROUNDUP_PTR(d, S); + } + + while(d + S <= buf_end) { + __builtin_prefetch(d + 64); + DEBUG_PRINTF("d %p \n", d); + SuperVector data = SuperVector::load(d); + rv = vermicelliSingleBlockNeg(data, chars, casemask, d); + if (rv) return rv; + d += S; + } + } + + DEBUG_PRINTF("d %p e %p \n", d, buf_end); + // finish off tail + + if (d != buf_end) { + SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); + rv = vermicelliSingleBlockNeg(data, chars, casemask, d); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } + + return buf_end; +} + +// Reverse vermicelli scan. Provides exact semantics and returns (buf - 1) if +// character not found. +template +const u8 *rvermicelliExecReal(SuperVector const chars, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("rverm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + + const u8 *d = buf_end; + const u8 *rv; + + __builtin_prefetch(d - 64); + __builtin_prefetch(d - 2*64); + __builtin_prefetch(d - 3*64); + __builtin_prefetch(d - 4*64); + DEBUG_PRINTF("start %p end %p \n", buf, d); + assert(d > buf); + if (d - S >= buf) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d - S); + rv = rvermicelliSingleBlock(data, chars, casemask, d - S); + DEBUG_PRINTF("rv %p \n", rv); + if (rv) return rv; + d = ROUNDDOWN_PTR(d, S); + } + + while (d - S >= buf) { + DEBUG_PRINTF("aligned %p \n", d); + // On large packet buffers, this prefetch appears to get us about 2%. + __builtin_prefetch(d - 64); + + d -= S; + SuperVector data = SuperVector::load(d); + rv = rvermicelliSingleBlock(data, chars, casemask, d); + if (rv) return rv; + } + } + + DEBUG_PRINTF("tail d %p e %p \n", buf, d); + // finish off head + + if (d != buf) { + SuperVector data = SuperVector::loadu(buf); + rv = rvermicelliSingleBlock(data, chars, casemask, buf); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } + + return buf - 1; +} + +// Reverse vermicelli scan. Provides exact semantics and returns (buf - 1) if +// character not found. +template +const u8 *rnvermicelliExecReal(SuperVector const chars, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("rverm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + + const u8 *d = buf_end; + const u8 *rv; + + __builtin_prefetch(d - 64); + __builtin_prefetch(d - 2*64); + __builtin_prefetch(d - 3*64); + __builtin_prefetch(d - 4*64); + DEBUG_PRINTF("start %p end %p \n", buf, d); + assert(d > buf); + if (d - S >= buf) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d - S); + rv = rvermicelliSingleBlockNeg(data, chars, casemask, d - S); + DEBUG_PRINTF("rv %p \n", rv); + if (rv) return rv; + d = ROUNDDOWN_PTR(d, S); + } + + while (d - S >= buf) { + DEBUG_PRINTF("aligned %p \n", d); + // On large packet buffers, this prefetch appears to get us about 2%. + __builtin_prefetch(d - 64); + + d -= S; + SuperVector data = SuperVector::load(d); + rv = rvermicelliSingleBlockNeg(data, chars, casemask, d); + if (rv) return rv; + } + } + + DEBUG_PRINTF("tail d %p e %p \n", buf, d); + // finish off head + + if (d != buf) { + SuperVector data = SuperVector::loadu(buf); + rv = rvermicelliSingleBlockNeg(data, chars, casemask, buf); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } + + return buf - 1; +} + +template +static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector const casemask, + const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("verm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + + const u8 *d = buf; + const u8 *rv; + // SuperVector lastmask1{0}; + const SuperVector chars1 = SuperVector::dup_u8(c1); + const SuperVector chars2 = SuperVector::dup_u8(c2); + const u8 casechar = casemask.u.u8[0]; + + __builtin_prefetch(d + 64); + __builtin_prefetch(d + 2*64); + __builtin_prefetch(d + 3*64); + __builtin_prefetch(d + 4*64); + DEBUG_PRINTF("start %p end %p \n", d, buf_end); + assert(d < buf_end); + if (d + S <= buf_end) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, &lastmask1); + if (rv) return rv; + d = ROUNDUP_PTR(d, S); + } + + while(d + S <= buf_end) { + __builtin_prefetch(d + 64); + DEBUG_PRINTF("d %p \n", d); + SuperVector data = SuperVector::load(d); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, &lastmask1); + if (rv) { + bool partial_match = (((rv[0] & casechar) == c2) && ((rv[-1] & casechar) == c1)); + return rv - partial_match; + } + d += S; + } + } + + DEBUG_PRINTF("tail d %p e %p \n", d, buf_end); + // finish off tail + + if (d != buf_end) { + SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, buf_end - d); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } + + DEBUG_PRINTF("real tail d %p e %p \n", d, buf_end); + /* check for partial match at end */ + u8 mask = casemask.u.u8[0]; + // u8 c1 = chars1.u.u8[0]; + if ((buf_end[-1] & mask) == (u8)c1) { + DEBUG_PRINTF("partial!!!\n"); + return buf_end - 1; + } + + return buf_end; +} + +// /* returns highest offset of c2 (NOTE: not c1) */ +// static really_inline +// const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, +// const u8 *buf_end) { +// DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", +// nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); +// assert(buf < buf_end); + +// VERM_TYPE chars1 = VERM_SET_FN(c1); /* nocase already uppercase */ +// VERM_TYPE chars2 = VERM_SET_FN(c2); /* nocase already uppercase */ + +// #ifdef HAVE_AVX512 +// if (buf_end - buf <= VERM_BOUNDARY) { +// const u8 *ptr = nocase +// ? rdvermMiniNocase(chars1, chars2, buf, buf_end) +// : rdvermMini(chars1, chars2, buf, buf_end); + +// if (ptr) { +// return ptr; +// } + +// // check for partial match at end ??? +// return buf - 1; +// } +// #endif + +// assert((buf_end - buf) >= VERM_BOUNDARY); +// size_t min = (size_t)buf_end % VERM_BOUNDARY; +// if (min) { +// // input not aligned, so we need to run one iteration with an unaligned +// // load, then skip buf forward to the next aligned address. There's +// // some small overlap here, but we don't mind scanning it twice if we +// // can do it quickly, do we? +// const u8 *ptr = nocase ? rdvermPreconditionNocase(chars1, chars2, +// buf_end - VERM_BOUNDARY) +// : rdvermPrecondition(chars1, chars2, +// buf_end - VERM_BOUNDARY); + +// if (ptr) { +// return ptr; +// } + +// buf_end -= min; +// if (buf >= buf_end) { +// return buf_end; +// } +// } + +// // Aligned loops from here on in +// if (nocase) { +// return rdvermSearchAlignedNocase(chars1, chars2, c1, c2, buf, buf_end); +// } else { +// return rdvermSearchAligned(chars1, chars2, c1, c2, buf, buf_end); +// } +// } + +extern "C" const u8 *vermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("verm scan %s\\x%02hhx over %zu bytes\n", + nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + const SuperVector chars = SuperVector::dup_u8(c); + const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; + + return vermicelliExecReal(chars, casemask, buf, buf_end); +} + +/* like vermicelliExec except returns the address of the first character which + * is not c */ +extern "C" const u8 *nvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("nverm scan %s\\x%02hhx over %zu bytes\n", + nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + const SuperVector chars = SuperVector::dup_u8(c); + const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; + + return nvermicelliExecReal(chars, casemask, buf, buf_end); +} + +extern "C" const u8 *rvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("rev verm scan %s\\x%02hhx over %zu bytes\n", + nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + const SuperVector chars = SuperVector::dup_u8(c); + const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; + + return rvermicelliExecReal(chars, casemask, buf, buf_end); +} + +extern "C" const u8 *rnvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("rev verm scan %s\\x%02hhx over %zu bytes\n", + nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + const SuperVector chars = SuperVector::dup_u8(c); + const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; + + return rnvermicelliExecReal(chars, casemask, buf, buf_end); +} + +extern "C" const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", + nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; + + return vermicelliDoubleExecReal(c1, c2, casemask, buf, buf_end); +} \ No newline at end of file From 70ddb11a72cc39e08d4ace7a74210fe2de4da28a Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 27 Oct 2021 12:29:59 +0300 Subject: [PATCH 02/25] add to CMake --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 05e6a5c7..c0c8666c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -621,6 +621,7 @@ set (hs_exec_SRCS src/nfa/vermicelli.h src/nfa/vermicelli_run.h src/nfa/vermicelli_sse.h + src/nfa/vermicelli_simd.cpp src/som/som.h src/som/som_operation.h src/som/som_runtime.h From 6e5a8353c5775cd1046d97c010e5470262a4dbbd Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 27 Oct 2021 12:30:42 +0300 Subject: [PATCH 03/25] move casemask helper functions to separate header --- src/hwlm/noodle_engine_simd.hpp | 21 +----------- src/util/supervector/casemask.hpp | 54 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 20 deletions(-) create mode 100644 src/util/supervector/casemask.hpp diff --git a/src/hwlm/noodle_engine_simd.hpp b/src/hwlm/noodle_engine_simd.hpp index d5f6a8d0..dfe7eea1 100644 --- a/src/hwlm/noodle_engine_simd.hpp +++ b/src/hwlm/noodle_engine_simd.hpp @@ -30,26 +30,7 @@ /* SIMD engine agnostic noodle scan parts */ #include "util/supervector/supervector.hpp" - -static u8 CASEMASK[] = { 0xff, 0xdf }; - -static really_inline -u8 caseClear8(u8 x, bool noCase) -{ - return static_cast(x & CASEMASK[(u8)noCase]); -} - -template -static really_inline SuperVector getMask(u8 c, bool noCase) { - u8 k = caseClear8(c, noCase); - return SuperVector(k); -} - -template -static really_inline SuperVector getCaseMask(void) { - return SuperVector(CASEMASK[1]); -} - +#include "util/supervector/casemask.hpp" static really_really_inline hwlm_error_t single_zscan(const struct noodTable *n,const u8 *d, const u8 *buf, diff --git a/src/util/supervector/casemask.hpp b/src/util/supervector/casemask.hpp new file mode 100644 index 00000000..10fa5f1a --- /dev/null +++ b/src/util/supervector/casemask.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef CASEMASK_HPP +#define CASEMASK_HPP + +#include "util/supervector/supervector.hpp" + +static u8 CASEMASK[] = { 0xff, 0xdf }; + +static really_inline +u8 caseClear8(u8 x, bool noCase) +{ + return static_cast(x & CASEMASK[(u8)noCase]); +} + +template +static really_inline SuperVector getMask(u8 c, bool noCase) { + u8 k = caseClear8(c, noCase); + return SuperVector(k); +} + +template +static really_inline SuperVector getCaseMask(void) { + return SuperVector(CASEMASK[1]); +} + +#endif // CASEMASK_HPP \ No newline at end of file From 70414574eef41b411f87a7d286a314f6724b797c Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 27 Oct 2021 12:31:04 +0300 Subject: [PATCH 04/25] nits --- src/nfa/arm/shufti.hpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/nfa/arm/shufti.hpp b/src/nfa/arm/shufti.hpp index 76461175..e710fd16 100644 --- a/src/nfa/arm/shufti.hpp +++ b/src/nfa/arm/shufti.hpp @@ -1,7 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation * Copyright (c) 2020-2021, VectorCamp PC - * Copyright (c) 2021, Arm Limited * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -30,7 +29,6 @@ /** \file * \brief Shufti: character class acceleration. - * */ template @@ -73,4 +71,4 @@ SuperVector blockDoubleMask(SuperVector mask1_lo, SuperVector mask1_hi, t.print8("t"); return !t.eq(SuperVector::Ones()); -} +} \ No newline at end of file From 8ae6e613cb3a74ea3b7210a6090fd5216f4e3369 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 27 Oct 2021 12:32:03 +0300 Subject: [PATCH 05/25] renamed matcher functions, added new ones for Vermicelli --- src/nfa/shufti_simd.hpp | 6 +-- src/nfa/truffle_simd.hpp | 4 +- src/nfa/x86/shufti.hpp | 16 ++---- src/util/arch/arm/match.hpp | 41 +++++++++++++- src/util/arch/x86/match.hpp | 103 +++++++++++++++++++++++++++++++++--- src/util/match.hpp | 10 +++- 6 files changed, 152 insertions(+), 28 deletions(-) diff --git a/src/nfa/shufti_simd.hpp b/src/nfa/shufti_simd.hpp index e7f3f6c9..09850c00 100644 --- a/src/nfa/shufti_simd.hpp +++ b/src/nfa/shufti_simd.hpp @@ -63,7 +63,7 @@ static really_inline const u8 *fwdBlock(SuperVector mask_lo, SuperVector mask_hi, SuperVector chars, const u8 *buf) { SuperVector v = blockSingleMask(mask_lo, mask_hi, chars); - return firstMatch(buf, v); + return first_zero_match_inverted(buf, v); } template @@ -71,7 +71,7 @@ static really_inline const u8 *revBlock(SuperVector mask_lo, SuperVector mask_hi, SuperVector chars, const u8 *buf) { SuperVector v = blockSingleMask(mask_lo, mask_hi, chars); - return lastMatch(buf, v); + return last_zero_match_inverted(buf, v); } template @@ -80,7 +80,7 @@ const u8 *fwdBlockDouble(SuperVector mask1_lo, SuperVector mask1_hi, Super SuperVector mask = blockDoubleMask(mask1_lo, mask1_hi, mask2_lo, mask2_hi, chars); - return firstMatch(buf, mask); + return first_zero_match_inverted(buf, mask); } template diff --git a/src/nfa/truffle_simd.hpp b/src/nfa/truffle_simd.hpp index 8d61722b..13a5e787 100644 --- a/src/nfa/truffle_simd.hpp +++ b/src/nfa/truffle_simd.hpp @@ -56,7 +56,7 @@ static really_inline const u8 *fwdBlock(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector chars, const u8 *buf) { SuperVector res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars); - return firstMatch(buf, res); + return first_zero_match_inverted(buf, res); } template @@ -120,7 +120,7 @@ static really_inline const u8 *revBlock(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector v, const u8 *buf) { SuperVector res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, v); - return lastMatch(buf, res); + return last_zero_match_inverted(buf, res); } template diff --git a/src/nfa/x86/shufti.hpp b/src/nfa/x86/shufti.hpp index 79ef7481..6fb34b2f 100644 --- a/src/nfa/x86/shufti.hpp +++ b/src/nfa/x86/shufti.hpp @@ -31,12 +31,6 @@ * \brief Shufti: character class acceleration. */ -#ifndef SHUFTI_SIMD_X86_HPP -#define SHUFTI_SIMD_X86_HPP - -#include "util/supervector/supervector.hpp" -#include "util/match.hpp" - template static really_inline const SuperVector blockSingleMask(SuperVector mask_lo, SuperVector mask_hi, SuperVector chars) { @@ -44,12 +38,10 @@ const SuperVector blockSingleMask(SuperVector mask_lo, SuperVector mask SuperVector c_lo = chars & low4bits; SuperVector c_hi = chars.template vshr_64_imm<4>() & low4bits; - c_lo = mask_lo.template pshufb(c_lo); - c_hi = mask_hi.template pshufb(c_hi); + c_lo = mask_lo.pshufb(c_lo); + c_hi = mask_hi.pshufb(c_hi); - SuperVector c = c_lo & c_hi; - - return c.eq(SuperVector::Zeroes()); + return (c_lo & c_hi).eq(SuperVector::Zeroes()); } template @@ -80,5 +72,3 @@ SuperVector blockDoubleMask(SuperVector mask1_lo, SuperVector mask1_hi, return c.eq(SuperVector::Ones()); } - -#endif // SHUFTI_SIMD_X86_HPP diff --git a/src/util/arch/arm/match.hpp b/src/util/arch/arm/match.hpp index 46d84d06..c74454ea 100644 --- a/src/util/arch/arm/match.hpp +++ b/src/util/arch/arm/match.hpp @@ -29,7 +29,44 @@ template <> really_really_inline -const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> mask) { +const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> mask) { + uint32x4_t res_t = vreinterpretq_u32_u8(mask.u.v128[0]); + uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(res_t, res_t)), 0); + if (vmax != 0) { + typename SuperVector<16>::movemask_type z = mask.movemask(); + DEBUG_PRINTF("z %08x\n", z); + DEBUG_PRINTF("buf %p z %08x \n", buf, z); + u32 pos = ctz32(z & 0xffff); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos < 16); + DEBUG_PRINTF("buf + pos %p\n", buf + pos); + return buf + pos; + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> mask) { + uint32x4_t res_t = vreinterpretq_u32_u8(mask.u.v128[0]); + uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(res_t, res_t)), 0); + if (vmax != 0) { + typename SuperVector<16>::movemask_type z = mask.movemask(); + DEBUG_PRINTF("buf %p z %08x \n", buf, z); + DEBUG_PRINTF("z %08x\n", z); + u32 pos = clz32(z & 0xffff); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos >= 16 && pos < 32); + return buf + (31 - pos); + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask) { uint32x4_t res_t = vreinterpretq_u32_u8(mask.u.v128[0]); uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(res_t, res_t)), 0); if (vmax != 0) { @@ -48,7 +85,7 @@ const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> mask) { template <> really_really_inline -const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> mask) { +const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask) { uint32x4_t res_t = vreinterpretq_u32_u8(mask.u.v128[0]); uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(res_t, res_t)), 0); if (vmax != 0) { diff --git a/src/util/arch/x86/match.hpp b/src/util/arch/x86/match.hpp index 159f7355..26283ca7 100644 --- a/src/util/arch/x86/match.hpp +++ b/src/util/arch/x86/match.hpp @@ -29,7 +29,98 @@ template <> really_really_inline -const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> v) { +const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> v) { + SuperVector<16>::movemask_type z = v.movemask(); + DEBUG_PRINTF("buf %p z %08x \n", buf, z); + DEBUG_PRINTF("z %08x\n", z); + if (unlikely(z)) { + u32 pos = ctz32(z); + DEBUG_PRINTF("~z %08x\n", ~z); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos < 16); + return buf + pos; + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *first_non_zero_match<32>(const u8 *buf, SuperVector<32> v) { + SuperVector<32>::movemask_type z = v.movemask(); + DEBUG_PRINTF("z 0x%08x\n", z); + if (unlikely(z)) { + u32 pos = ctz32(z); + assert(pos < 32); + DEBUG_PRINTF("match @ pos %u\n", pos); + return buf + pos; + } else { + return NULL; // no match + } +} +template <> +really_really_inline +const u8 *first_non_zero_match<64>(const u8 *buf, SuperVector<64>v) { + SuperVector<64>::movemask_type z = v.movemask(); + DEBUG_PRINTF("z 0x%016llx\n", z); + if (unlikely(z)) { + u32 pos = ctz64(z); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos < 64); + return buf + pos; + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> v) { + SuperVector<16>::movemask_type z = v.movemask(); + DEBUG_PRINTF("buf %p z %08x \n", buf, z); + DEBUG_PRINTF("z %08x\n", z); + if (unlikely(z)) { + u32 pos = clz32(z); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos >= 16 && pos < 32); + return buf + (31 - pos); + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *last_non_zero_match<32>(const u8 *buf, SuperVector<32> v) { + SuperVector<32>::movemask_type z = v.movemask(); + DEBUG_PRINTF("z 0x%08x\n", z); + if (unlikely(z)) { + u32 pos = clz32(z); + assert(pos < 32); + DEBUG_PRINTF("match @ pos %u\n", pos); + return buf + (31 - pos); + } else { + return NULL; // no match + } +} +template <> +really_really_inline +const u8 *last_non_zero_match<64>(const u8 *buf, SuperVector<64>v) { + SuperVector<64>::movemask_type z = v.movemask(); + DEBUG_PRINTF("z 0x%016llx\n", z); + if (unlikely(z)) { + u32 pos = clz64(z); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos < 64); + return buf + (31 - pos); + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) { SuperVector<16>::movemask_type z = v.movemask(); DEBUG_PRINTF("buf %p z %08x \n", buf, z); DEBUG_PRINTF("z %08x\n", z); @@ -46,7 +137,7 @@ const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> v) { template <> really_really_inline -const u8 *firstMatch<32>(const u8 *buf, SuperVector<32> v) { +const u8 *first_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) { SuperVector<32>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%08x\n", z); if (unlikely(z != 0xffffffff)) { @@ -60,7 +151,7 @@ const u8 *firstMatch<32>(const u8 *buf, SuperVector<32> v) { } template <> really_really_inline -const u8 *firstMatch<64>(const u8 *buf, SuperVector<64>v) { +const u8 *first_zero_match_inverted<64>(const u8 *buf, SuperVector<64>v) { SuperVector<64>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%016llx\n", z); if (unlikely(z != ~0ULL)) { @@ -75,7 +166,7 @@ const u8 *firstMatch<64>(const u8 *buf, SuperVector<64>v) { template <> really_really_inline -const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> v) { +const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) { SuperVector<16>::movemask_type z = v.movemask(); DEBUG_PRINTF("buf %p z %08x \n", buf, z); DEBUG_PRINTF("z %08x\n", z); @@ -92,7 +183,7 @@ const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> v) { template<> really_really_inline -const u8 *lastMatch<32>(const u8 *buf, SuperVector<32> v) { +const u8 *last_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) { SuperVector<32>::movemask_type z = v.movemask(); if (unlikely(z != 0xffffffff)) { u32 pos = clz32(~z); @@ -106,7 +197,7 @@ const u8 *lastMatch<32>(const u8 *buf, SuperVector<32> v) { template <> really_really_inline -const u8 *lastMatch<64>(const u8 *buf, SuperVector<64> v) { +const u8 *last_zero_match_inverted<64>(const u8 *buf, SuperVector<64> v) { SuperVector<64>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%016llx\n", z); if (unlikely(z != ~0ULL)) { diff --git a/src/util/match.hpp b/src/util/match.hpp index 9331d1f8..9b3c8fb9 100644 --- a/src/util/match.hpp +++ b/src/util/match.hpp @@ -38,10 +38,16 @@ #include "util/supervector/supervector.hpp" template -const u8 *firstMatch(const u8 *buf, SuperVector v); +const u8 *first_non_zero_match(const u8 *buf, SuperVector v); template -const u8 *lastMatch(const u8 *buf, SuperVector v); +const u8 *last_non_zero_match(const u8 *buf, SuperVector v); + +template +const u8 *first_zero_match_inverted(const u8 *buf, SuperVector v); + +template +const u8 *last_zero_match_inverted(const u8 *buf, SuperVector v); #if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/arch/x86/match.hpp" From dd45bf0d3502543bd57ad1ad6f55ba4731854fae Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 27 Oct 2021 12:32:54 +0300 Subject: [PATCH 06/25] add new include file --- src/hwlm/hwlm.c | 1 + src/nfa/accel.c | 1 + src/nfa/castle.c | 1 + src/nfa/lbr.c | 1 + src/nfa/nfa_rev_api.h | 1 + src/nfa/vermicelli_run.h | 1 + unit/internal/rvermicelli.cpp | 3 ++- unit/internal/vermicelli.cpp | 3 ++- 8 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/hwlm/hwlm.c b/src/hwlm/hwlm.c index c1c2837f..5d69e3c4 100644 --- a/src/hwlm/hwlm.c +++ b/src/hwlm/hwlm.c @@ -40,6 +40,7 @@ #include "nfa/shufti.h" #include "nfa/truffle.h" #include "nfa/vermicelli.h" +#include "nfa/vermicelli.hpp" #include #define MIN_ACCEL_LEN_BLOCK 16 diff --git a/src/nfa/accel.c b/src/nfa/accel.c index 34bd24a9..b35e0633 100644 --- a/src/nfa/accel.c +++ b/src/nfa/accel.c @@ -31,6 +31,7 @@ #include "shufti.h" #include "truffle.h" #include "vermicelli.h" +#include "vermicelli.hpp" #include "ue2common.h" const u8 *run_accel(const union AccelAux *accel, const u8 *c, const u8 *c_end) { diff --git a/src/nfa/castle.c b/src/nfa/castle.c index c7dd6d50..be29ca29 100644 --- a/src/nfa/castle.c +++ b/src/nfa/castle.c @@ -41,6 +41,7 @@ #include "shufti.h" #include "truffle.h" #include "vermicelli.h" +#include "vermicelli.hpp" #include "util/bitutils.h" #include "util/multibit.h" #include "util/partial_store.h" diff --git a/src/nfa/lbr.c b/src/nfa/lbr.c index 68e8e3f4..8fc83988 100644 --- a/src/nfa/lbr.c +++ b/src/nfa/lbr.c @@ -41,6 +41,7 @@ #include "shufti.h" #include "truffle.h" #include "vermicelli.h" +#include "vermicelli.hpp" #include "util/partial_store.h" #include "util/unaligned.h" diff --git a/src/nfa/nfa_rev_api.h b/src/nfa/nfa_rev_api.h index 370f96ef..72224c3b 100644 --- a/src/nfa/nfa_rev_api.h +++ b/src/nfa/nfa_rev_api.h @@ -36,6 +36,7 @@ #include "accel.h" #include "nfa_internal.h" #include "vermicelli.h" +#include "vermicelli.hpp" #include "util/unaligned.h" static really_inline diff --git a/src/nfa/vermicelli_run.h b/src/nfa/vermicelli_run.h index d6fe7ec7..b75f1414 100644 --- a/src/nfa/vermicelli_run.h +++ b/src/nfa/vermicelli_run.h @@ -27,6 +27,7 @@ */ #include "vermicelli.h" +#include "vermicelli.hpp" static really_inline const u8 *find_xverm_run(char c, char nocase, u32 repeat, UNUSED const u8 *buf, diff --git a/unit/internal/rvermicelli.cpp b/unit/internal/rvermicelli.cpp index d89067d0..d29b1133 100644 --- a/unit/internal/rvermicelli.cpp +++ b/unit/internal/rvermicelli.cpp @@ -31,6 +31,7 @@ #include "gtest/gtest.h" #include "nfa/vermicelli.h" +#include "nfa/vermicelli.hpp" #define BOUND (~(VERM_BOUNDARY - 1)) @@ -563,4 +564,4 @@ TEST(RNVermicelli16, Exec5) { } } -#endif // HAVE_SVE2 \ No newline at end of file +#endif // HAVE_SVE2 diff --git a/unit/internal/vermicelli.cpp b/unit/internal/vermicelli.cpp index dc458cb9..3319b87c 100644 --- a/unit/internal/vermicelli.cpp +++ b/unit/internal/vermicelli.cpp @@ -31,6 +31,7 @@ #include "gtest/gtest.h" #include "nfa/vermicelli.h" +#include "nfa/vermicelli.hpp" TEST(Vermicelli, ExecNoMatch1) { char t1[] = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; @@ -1150,4 +1151,4 @@ TEST(DoubleVermicelliMasked16, Exec5) { } } -#endif // HAVE_SVE2 \ No newline at end of file +#endif // HAVE_SVE2 From f4a490ac003e0ff7282ba87f4742fe5d52326b24 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Mon, 1 Nov 2021 16:50:38 +0200 Subject: [PATCH 07/25] remove unneeded header --- src/nfa/vermicelli.h | 2 - src/nfa/vermicelli_sse.h | 1284 -------------------------------------- 2 files changed, 1286 deletions(-) delete mode 100644 src/nfa/vermicelli_sse.h diff --git a/src/nfa/vermicelli.h b/src/nfa/vermicelli.h index 9defd899..39e9555e 100644 --- a/src/nfa/vermicelli.h +++ b/src/nfa/vermicelli.h @@ -44,8 +44,6 @@ #ifdef HAVE_SVE2 #include "vermicelli_sve.h" -#else -#include "vermicelli_sse.h" #endif static really_inline diff --git a/src/nfa/vermicelli_sse.h b/src/nfa/vermicelli_sse.h deleted file mode 100644 index 268e9e08..00000000 --- a/src/nfa/vermicelli_sse.h +++ /dev/null @@ -1,1284 +0,0 @@ -/* - * Copyright (c) 2015-2020, Intel Corporation - * Copyright (c) 2021, Arm Limited - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** \file - * \brief Vermicelli: Intel SSE implementation. - * - * (users should include vermicelli.h instead of this) - */ - -#if !defined(HAVE_AVX512) - -#define VERM_BOUNDARY 16 -#define VERM_TYPE m128 -#define VERM_SET_FN set1_16x8 - -static really_inline -const u8 *vermSearchAligned(m128 chars, const u8 *buf, const u8 *buf_end, - char negate) { - assert((size_t)buf % 16 == 0); - for (; buf + 31 < buf_end; buf += 32) { - m128 data = load128(buf); - u32 z1 = movemask128(eq128(chars, data)); - m128 data2 = load128(buf + 16); - u32 z2 = movemask128(eq128(chars, data2)); - u32 z = z1 | (z2 << 16); - if (negate) { - z = ~z; - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - for (; buf + 15 < buf_end; buf += 16) { - m128 data = load128(buf); - u32 z = movemask128(eq128(chars, data)); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - return NULL; -} - -static really_inline -const u8 *vermSearchAlignedNocase(m128 chars, const u8 *buf, - const u8 *buf_end, char negate) { - assert((size_t)buf % 16 == 0); - m128 casemask = set1_16x8(CASE_CLEAR); - - for (; buf + 31 < buf_end; buf += 32) { - m128 data = load128(buf); - u32 z1 = movemask128(eq128(chars, and128(casemask, data))); - m128 data2 = load128(buf + 16); - u32 z2 = movemask128(eq128(chars, and128(casemask, data2))); - u32 z = z1 | (z2 << 16); - if (negate) { - z = ~z; - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - - for (; buf + 15 < buf_end; buf += 16) { - m128 data = load128(buf); - u32 z = movemask128(eq128(chars, and128(casemask, data))); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *vermUnalign(m128 chars, const u8 *buf, char negate) { - m128 data = loadu128(buf); // unaligned - u32 z = movemask128(eq128(chars, data)); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *vermUnalignNocase(m128 chars, const u8 *buf, char negate) { - m128 casemask = set1_16x8(CASE_CLEAR); - m128 data = loadu128(buf); // unaligned - u32 z = movemask128(eq128(chars, and128(casemask, data))); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - -static really_inline -const u8 *lastMatchOffset(const u8 *buf_end, u32 z) { - assert(z); - return buf_end - 16 + 31 - clz32(z); -} - -static really_inline -const u8 *rvermSearchAligned(m128 chars, const u8 *buf, const u8 *buf_end, - char negate) { - assert((size_t)buf_end % 16 == 0); - for (; buf + 15 < buf_end; buf_end -= 16) { - m128 data = load128(buf_end - 16); - u32 z = movemask128(eq128(chars, data)); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf_end, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - return NULL; -} - -static really_inline -const u8 *rvermSearchAlignedNocase(m128 chars, const u8 *buf, - const u8 *buf_end, char negate) { - assert((size_t)buf_end % 16 == 0); - m128 casemask = set1_16x8(CASE_CLEAR); - - for (; buf + 15 < buf_end; buf_end -= 16) { - m128 data = load128(buf_end - 16); - u32 z = movemask128(eq128(chars, and128(casemask, data))); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf_end, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *rvermUnalign(m128 chars, const u8 *buf, char negate) { - m128 data = loadu128(buf); // unaligned - u32 z = movemask128(eq128(chars, data)); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf + 16, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *rvermUnalignNocase(m128 chars, const u8 *buf, char negate) { - m128 casemask = set1_16x8(CASE_CLEAR); - m128 data = loadu128(buf); // unaligned - u32 z = movemask128(eq128(chars, and128(casemask, data))); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf + 16, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - -static really_inline -const u8 *dvermSearchAligned(m128 chars1, m128 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - for (; buf + 16 < buf_end; buf += 16) { - m128 data = load128(buf); - u32 z = movemask128(and128(eq128(chars1, data), - rshiftbyte_m128(eq128(chars2, data), 1))); - if (buf[15] == c1 && buf[16] == c2) { - z |= (1 << 15); - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - - return NULL; -} - -static really_inline -const u8 *dvermSearchAlignedNocase(m128 chars1, m128 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - assert((size_t)buf % 16 == 0); - m128 casemask = set1_16x8(CASE_CLEAR); - - for (; buf + 16 < buf_end; buf += 16) { - m128 data = load128(buf); - m128 v = and128(casemask, data); - u32 z = movemask128(and128(eq128(chars1, v), - rshiftbyte_m128(eq128(chars2, v), 1))); - if ((buf[15] & CASE_CLEAR) == c1 && (buf[16] & CASE_CLEAR) == c2) { - z |= (1 << 15); - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *dvermPrecondition(m128 chars1, m128 chars2, const u8 *buf) { - m128 data = loadu128(buf); // unaligned - u32 z = movemask128(and128(eq128(chars1, data), - rshiftbyte_m128(eq128(chars2, data), 1))); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *dvermPreconditionNocase(m128 chars1, m128 chars2, const u8 *buf) { - /* due to laziness, nonalphas and nocase having interesting behaviour */ - m128 casemask = set1_16x8(CASE_CLEAR); - m128 data = loadu128(buf); // unaligned - m128 v = and128(casemask, data); - u32 z = movemask128(and128(eq128(chars1, v), - rshiftbyte_m128(eq128(chars2, v), 1))); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - - -static really_inline -const u8 *rdvermSearchAligned(m128 chars1, m128 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - assert((size_t)buf_end % 16 == 0); - - for (; buf + 16 < buf_end; buf_end -= 16) { - m128 data = load128(buf_end - 16); - u32 z = movemask128(and128(eq128(chars2, data), - lshiftbyte_m128(eq128(chars1, data), 1))); - if (buf_end[-17] == c1 && buf_end[-16] == c2) { - z |= 1; - } - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf_end, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - return buf_end; -} - -static really_inline -const u8 *rdvermSearchAlignedNocase(m128 chars1, m128 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - assert((size_t)buf_end % 16 == 0); - m128 casemask = set1_16x8(CASE_CLEAR); - - for (; buf + 16 < buf_end; buf_end -= 16) { - m128 data = load128(buf_end - 16); - m128 v = and128(casemask, data); - u32 z = movemask128(and128(eq128(chars2, v), - lshiftbyte_m128(eq128(chars1, v), 1))); - if ((buf_end[-17] & CASE_CLEAR) == c1 - && (buf_end[-16] & CASE_CLEAR) == c2) { - z |= 1; - } - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf_end, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - return buf_end; -} - -// returns NULL if not found -static really_inline -const u8 *rdvermPrecondition(m128 chars1, m128 chars2, const u8 *buf) { - m128 data = loadu128(buf); - u32 z = movemask128(and128(eq128(chars2, data), - lshiftbyte_m128(eq128(chars1, data), 1))); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf + 16, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *rdvermPreconditionNocase(m128 chars1, m128 chars2, const u8 *buf) { - /* due to laziness, nonalphas and nocase having interesting behaviour */ - m128 casemask = set1_16x8(CASE_CLEAR); - m128 data = loadu128(buf); - m128 v = and128(casemask, data); - u32 z = movemask128(and128(eq128(chars2, v), - lshiftbyte_m128(eq128(chars1, v), 1))); - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf + 16, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - - return NULL; -} - -#else // HAVE_AVX512 - -#define VERM_BOUNDARY 64 -#define VERM_TYPE m512 -#define VERM_SET_FN set1_64x8 - -static really_inline -const u8 *vermMini(m512 chars, const u8 *buf, const u8 *buf_end, char negate) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - - u64a z = eq512mask(chars, data); - - if (negate) { - z = ~z & mask; - } - z &= mask; - if (unlikely(z)) { - return buf + ctz64(z); - } - return NULL; -} - -static really_inline -const u8 *vermMiniNocase(m512 chars, const u8 *buf, const u8 *buf_end, - char negate) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - m512 casemask = set1_64x8(CASE_CLEAR); - m512 v = and512(casemask, data); - - u64a z = eq512mask(chars, v); - - if (negate) { - z = ~z & mask; - } - z &= mask; - if (unlikely(z)) { - return buf + ctz64(z); - } - return NULL; -} - -static really_inline -const u8 *vermSearchAligned(m512 chars, const u8 *buf, const u8 *buf_end, - char negate) { - assert((size_t)buf % 64 == 0); - for (; buf + 63 < buf_end; buf += 64) { - m512 data = load512(buf); - u64a z = eq512mask(chars, data); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - } - return NULL; -} - -static really_inline -const u8 *vermSearchAlignedNocase(m512 chars, const u8 *buf, - const u8 *buf_end, char negate) { - assert((size_t)buf % 64 == 0); - m512 casemask = set1_64x8(CASE_CLEAR); - - for (; buf + 63 < buf_end; buf += 64) { - m512 data = load512(buf); - u64a z = eq512mask(chars, and512(casemask, data)); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *vermUnalign(m512 chars, const u8 *buf, char negate) { - m512 data = loadu512(buf); // unaligned - u64a z = eq512mask(chars, data); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - return buf + ctz64(z); - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *vermUnalignNocase(m512 chars, const u8 *buf, char negate) { - m512 casemask = set1_64x8(CASE_CLEAR); - m512 data = loadu512(buf); // unaligned - u64a z = eq512mask(chars, and512(casemask, data)); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - return buf + ctz64(z); - } - return NULL; -} - -static really_inline -const u8 *dvermMini(m512 chars1, m512 chars2, const u8 *buf, - const u8 *buf_end) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - - u64a z = eq512mask(chars1, data) & (eq512mask(chars2, data) >> 1); - - z &= mask; - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - return NULL; -} - -static really_inline -const u8 *dvermMiniNocase(m512 chars1, m512 chars2, const u8 *buf, - const u8 *buf_end) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - m512 casemask = set1_64x8(CASE_CLEAR); - m512 v = and512(casemask, data); - - u64a z = eq512mask(chars1, v) & (eq512mask(chars2, v) >> 1); - - z &= mask; - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - return NULL; -} - -static really_inline -const u8 *dvermMiniMasked(m512 chars1, m512 chars2, m512 mask1, m512 mask2, - const u8 *buf, const u8 *buf_end) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - m512 v1 = and512(data, mask1); - m512 v2 = and512(data, mask2); - - u64a z = eq512mask(chars1, v1) & (eq512mask(chars2, v2) >> 1); - - z &= mask; - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - return NULL; -} - -static really_inline -const u8 *dvermSearchAligned(m512 chars1, m512 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - for (; buf + 64 < buf_end; buf += 64) { - m512 data = load512(buf); - u64a z = eq512mask(chars1, data) & (eq512mask(chars2, data) >> 1); - if (buf[63] == c1 && buf[64] == c2) { - z |= (1ULL << 63); - } - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - } - - return NULL; -} - -static really_inline -const u8 *dvermSearchAlignedNocase(m512 chars1, m512 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - assert((size_t)buf % 64 == 0); - m512 casemask = set1_64x8(CASE_CLEAR); - - for (; buf + 64 < buf_end; buf += 64) { - m512 data = load512(buf); - m512 v = and512(casemask, data); - u64a z = eq512mask(chars1, v) & (eq512mask(chars2, v) >> 1); - if ((buf[63] & CASE_CLEAR) == c1 && (buf[64] & CASE_CLEAR) == c2) { - z |= (1ULL << 63); - } - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - } - - return NULL; -} - -static really_inline -const u8 *dvermSearchAlignedMasked(m512 chars1, m512 chars2, - m512 mask1, m512 mask2, u8 c1, u8 c2, u8 m1, - u8 m2, const u8 *buf, const u8 *buf_end) { - assert((size_t)buf % 64 == 0); - - for (; buf + 64 < buf_end; buf += 64) { - m512 data = load512(buf); - m512 v1 = and512(data, mask1); - m512 v2 = and512(data, mask2); - u64a z = eq512mask(chars1, v1) & (eq512mask(chars2, v2) >> 1); - - if ((buf[63] & m1) == c1 && (buf[64] & m2) == c2) { - z |= (1ULL << 63); - } - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - } - - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *dvermPrecondition(m512 chars1, m512 chars2, const u8 *buf) { - m512 data = loadu512(buf); // unaligned - u64a z = eq512mask(chars1, data) & (eq512mask(chars2, data) >> 1); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *dvermPreconditionNocase(m512 chars1, m512 chars2, const u8 *buf) { - /* due to laziness, nonalphas and nocase having interesting behaviour */ - m512 casemask = set1_64x8(CASE_CLEAR); - m512 data = loadu512(buf); // unaligned - m512 v = and512(casemask, data); - u64a z = eq512mask(chars1, v) & (eq512mask(chars2, v) >> 1); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *dvermPreconditionMasked(m512 chars1, m512 chars2, - m512 mask1, m512 mask2, const u8 *buf) { - m512 data = loadu512(buf); // unaligned - m512 v1 = and512(data, mask1); - m512 v2 = and512(data, mask2); - u64a z = eq512mask(chars1, v1) & (eq512mask(chars2, v2) >> 1); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - return NULL; -} - -static really_inline -const u8 *lastMatchOffset(const u8 *buf_end, u64a z) { - assert(z); - return buf_end - 64 + 63 - clz64(z); -} - -static really_inline -const u8 *rvermMini(m512 chars, const u8 *buf, const u8 *buf_end, char negate) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - - u64a z = eq512mask(chars, data); - - if (negate) { - z = ~z & mask; - } - z &= mask; - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - return NULL; -} - -static really_inline -const u8 *rvermMiniNocase(m512 chars, const u8 *buf, const u8 *buf_end, - char negate) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - m512 casemask = set1_64x8(CASE_CLEAR); - m512 v = and512(casemask, data); - - u64a z = eq512mask(chars, v); - - if (negate) { - z = ~z & mask; - } - z &= mask; - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - return NULL; -} - -static really_inline -const u8 *rvermSearchAligned(m512 chars, const u8 *buf, const u8 *buf_end, - char negate) { - assert((size_t)buf_end % 64 == 0); - for (; buf + 63 < buf_end; buf_end -= 64) { - m512 data = load512(buf_end - 64); - u64a z = eq512mask(chars, data); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - return lastMatchOffset(buf_end, z); - } - } - return NULL; -} - -static really_inline -const u8 *rvermSearchAlignedNocase(m512 chars, const u8 *buf, - const u8 *buf_end, char negate) { - assert((size_t)buf_end % 64 == 0); - m512 casemask = set1_64x8(CASE_CLEAR); - - for (; buf + 63 < buf_end; buf_end -= 64) { - m512 data = load512(buf_end - 64); - u64a z = eq512mask(chars, and512(casemask, data)); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - return lastMatchOffset(buf_end, z); - } - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *rvermUnalign(m512 chars, const u8 *buf, char negate) { - m512 data = loadu512(buf); // unaligned - u64a z = eq512mask(chars, data); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *rvermUnalignNocase(m512 chars, const u8 *buf, char negate) { - m512 casemask = set1_64x8(CASE_CLEAR); - m512 data = loadu512(buf); // unaligned - u64a z = eq512mask(chars, and512(casemask, data)); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - return NULL; -} - -static really_inline -const u8 *rdvermMini(m512 chars1, m512 chars2, const u8 *buf, - const u8 *buf_end) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - - u64a z = eq512mask(chars2, data) & (eq512mask(chars1, data) << 1); - - z &= mask; - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - return NULL; -} - -static really_inline -const u8 *rdvermMiniNocase(m512 chars1, m512 chars2, const u8 *buf, - const u8 *buf_end) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - m512 casemask = set1_64x8(CASE_CLEAR); - m512 v = and512(casemask, data); - - u64a z = eq512mask(chars2, v) & (eq512mask(chars1, v) << 1); - - z &= mask; - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - return NULL; -} - -static really_inline -const u8 *rdvermSearchAligned(m512 chars1, m512 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - assert((size_t)buf_end % 64 == 0); - - for (; buf + 64 < buf_end; buf_end -= 64) { - m512 data = load512(buf_end - 64); - u64a z = eq512mask(chars2, data) & (eq512mask(chars1, data) << 1); - if (buf_end[-65] == c1 && buf_end[-64] == c2) { - z |= 1; - } - if (unlikely(z)) { - return lastMatchOffset(buf_end, z); - } - } - return buf_end; -} - -static really_inline -const u8 *rdvermSearchAlignedNocase(m512 chars1, m512 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - assert((size_t)buf_end % 64 == 0); - m512 casemask = set1_64x8(CASE_CLEAR); - - for (; buf + 64 < buf_end; buf_end -= 64) { - m512 data = load512(buf_end - 64); - m512 v = and512(casemask, data); - u64a z = eq512mask(chars2, v) & (eq512mask(chars1, v) << 1); - if ((buf_end[-65] & CASE_CLEAR) == c1 - && (buf_end[-64] & CASE_CLEAR) == c2) { - z |= 1; - } - if (unlikely(z)) { - return lastMatchOffset(buf_end, z); - } - } - return buf_end; -} - -// returns NULL if not found -static really_inline -const u8 *rdvermPrecondition(m512 chars1, m512 chars2, const u8 *buf) { - m512 data = loadu512(buf); - u64a z = eq512mask(chars2, data) & (eq512mask(chars1, data) << 1); - - // no fixup of the boundary required - the aligned run will pick it up - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *rdvermPreconditionNocase(m512 chars1, m512 chars2, const u8 *buf) { - // due to laziness, nonalphas and nocase having interesting behaviour - m512 casemask = set1_64x8(CASE_CLEAR); - m512 data = loadu512(buf); - m512 v = and512(casemask, data); - u64a z = eq512mask(chars2, v) & (eq512mask(chars1, v) << 1); - // no fixup of the boundary required - the aligned run will pick it up - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - - return NULL; -} - -#endif // HAVE_AVX512 - -static really_inline -const u8 *vermicelliExec(char c, char nocase, const u8 *buf, - const u8 *buf_end) { - DEBUG_PRINTF("verm scan %s\\x%02hhx over %zu bytes\n", - nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */ - - // Handle small scans. -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = nocase - ? vermMiniNocase(chars, buf, buf_end, 0) - : vermMini(chars, buf, buf_end, 0); - if (ptr) { - return ptr; - } - return buf_end; - } -#else - if (buf_end - buf < VERM_BOUNDARY) { - for (; buf < buf_end; buf++) { - char cur = (char)*buf; - if (nocase) { - cur &= CASE_CLEAR; - } - if (cur == c) { - break; - } - } - return buf; - } -#endif - - uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY; - if (min) { - // Input isn't aligned, so we need to run one iteration with an - // unaligned load, then skip buf forward to the next aligned address. - // There's some small overlap here, but we don't mind scanning it twice - // if we can do it quickly, do we? - const u8 *ptr = nocase ? vermUnalignNocase(chars, buf, 0) - : vermUnalign(chars, buf, 0); - if (ptr) { - return ptr; - } - - buf += VERM_BOUNDARY - min; - assert(buf < buf_end); - } - - // Aligned loops from here on in - const u8 *ptr = nocase ? vermSearchAlignedNocase(chars, buf, buf_end - 1, 0) - : vermSearchAligned(chars, buf, buf_end - 1, 0); - if (ptr) { - return ptr; - } - - // Tidy up the mess at the end - ptr = nocase ? vermUnalignNocase(chars, buf_end - VERM_BOUNDARY, 0) - : vermUnalign(chars, buf_end - VERM_BOUNDARY, 0); - return ptr ? ptr : buf_end; -} - -/* like vermicelliExec except returns the address of the first character which - * is not c */ -static really_inline -const u8 *nvermicelliExec(char c, char nocase, const u8 *buf, - const u8 *buf_end) { - DEBUG_PRINTF("nverm scan %s\\x%02hhx over %zu bytes\n", - nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */ - - // Handle small scans. -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = nocase - ? vermMiniNocase(chars, buf, buf_end, 1) - : vermMini(chars, buf, buf_end, 1); - if (ptr) { - return ptr; - } - return buf_end; - } -#else - if (buf_end - buf < VERM_BOUNDARY) { - for (; buf < buf_end; buf++) { - char cur = (char)*buf; - if (nocase) { - cur &= CASE_CLEAR; - } - if (cur != c) { - break; - } - } - return buf; - } -#endif - - size_t min = (size_t)buf % VERM_BOUNDARY; - if (min) { - // Input isn't aligned, so we need to run one iteration with an - // unaligned load, then skip buf forward to the next aligned address. - // There's some small overlap here, but we don't mind scanning it twice - // if we can do it quickly, do we? - const u8 *ptr = nocase ? vermUnalignNocase(chars, buf, 1) - : vermUnalign(chars, buf, 1); - if (ptr) { - return ptr; - } - - buf += VERM_BOUNDARY - min; - assert(buf < buf_end); - } - - // Aligned loops from here on in - const u8 *ptr = nocase ? vermSearchAlignedNocase(chars, buf, buf_end - 1, 1) - : vermSearchAligned(chars, buf, buf_end - 1, 1); - if (ptr) { - return ptr; - } - - // Tidy up the mess at the end - ptr = nocase ? vermUnalignNocase(chars, buf_end - VERM_BOUNDARY, 1) - : vermUnalign(chars, buf_end - VERM_BOUNDARY, 1); - return ptr ? ptr : buf_end; -} - -// Reverse vermicelli scan. Provides exact semantics and returns (buf - 1) if -// character not found. -static really_inline -const u8 *rvermicelliExec(char c, char nocase, const u8 *buf, - const u8 *buf_end) { - DEBUG_PRINTF("rev verm scan %s\\x%02hhx over %zu bytes\n", - nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */ - - // Handle small scans. -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = nocase - ? rvermMiniNocase(chars, buf, buf_end, 0) - : rvermMini(chars, buf, buf_end, 0); - if (ptr) { - return ptr; - } - return buf - 1; - } -#else - if (buf_end - buf < VERM_BOUNDARY) { - for (buf_end--; buf_end >= buf; buf_end--) { - char cur = (char)*buf_end; - if (nocase) { - cur &= CASE_CLEAR; - } - if (cur == c) { - break; - } - } - return buf_end; - } -#endif - - size_t min = (size_t)buf_end % VERM_BOUNDARY; - if (min) { - // Input isn't aligned, so we need to run one iteration with an - // unaligned load, then skip buf backward to the next aligned address. - // There's some small overlap here, but we don't mind scanning it twice - // if we can do it quickly, do we? - const u8 *ptr = nocase ? rvermUnalignNocase(chars, - buf_end - VERM_BOUNDARY, - 0) - : rvermUnalign(chars, buf_end - VERM_BOUNDARY, - 0); - - if (ptr) { - return ptr; - } - - buf_end -= min; - if (buf >= buf_end) { - return buf_end; - } - } - - // Aligned loops from here on in. - const u8 *ptr = nocase ? rvermSearchAlignedNocase(chars, buf, buf_end, 0) - : rvermSearchAligned(chars, buf, buf_end, 0); - if (ptr) { - return ptr; - } - - // Tidy up the mess at the end, return buf - 1 if not found. - ptr = nocase ? rvermUnalignNocase(chars, buf, 0) - : rvermUnalign(chars, buf, 0); - return ptr ? ptr : buf - 1; -} - -/* like rvermicelliExec except returns the address of the last character which - * is not c */ -static really_inline -const u8 *rnvermicelliExec(char c, char nocase, const u8 *buf, - const u8 *buf_end) { - DEBUG_PRINTF("rev verm scan %s\\x%02hhx over %zu bytes\n", - nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */ - - // Handle small scans. -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = nocase - ? rvermMiniNocase(chars, buf, buf_end, 1) - : rvermMini(chars, buf, buf_end, 1); - if (ptr) { - return ptr; - } - return buf - 1; - } -#else - if (buf_end - buf < VERM_BOUNDARY) { - for (buf_end--; buf_end >= buf; buf_end--) { - char cur = (char)*buf_end; - if (nocase) { - cur &= CASE_CLEAR; - } - if (cur != c) { - break; - } - } - return buf_end; - } -#endif - - size_t min = (size_t)buf_end % VERM_BOUNDARY; - if (min) { - // Input isn't aligned, so we need to run one iteration with an - // unaligned load, then skip buf backward to the next aligned address. - // There's some small overlap here, but we don't mind scanning it twice - // if we can do it quickly, do we? - const u8 *ptr = nocase ? rvermUnalignNocase(chars, - buf_end - VERM_BOUNDARY, - 1) - : rvermUnalign(chars, buf_end - VERM_BOUNDARY, - 1); - - if (ptr) { - return ptr; - } - - buf_end -= min; - if (buf >= buf_end) { - return buf_end; - } - } - - // Aligned loops from here on in. - const u8 *ptr = nocase ? rvermSearchAlignedNocase(chars, buf, buf_end, 1) - : rvermSearchAligned(chars, buf, buf_end, 1); - if (ptr) { - return ptr; - } - - // Tidy up the mess at the end, return buf - 1 if not found. - ptr = nocase ? rvermUnalignNocase(chars, buf, 1) - : rvermUnalign(chars, buf, 1); - return ptr ? ptr : buf - 1; -} - -static really_inline -const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, - const u8 *buf_end) { - DEBUG_PRINTF("double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", - nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars1 = VERM_SET_FN(c1); /* nocase already uppercase */ - VERM_TYPE chars2 = VERM_SET_FN(c2); /* nocase already uppercase */ - -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = nocase - ? dvermMiniNocase(chars1, chars2, buf, buf_end) - : dvermMini(chars1, chars2, buf, buf_end); - if (ptr) { - return ptr; - } - - /* check for partial match at end */ - u8 mask = nocase ? CASE_CLEAR : 0xff; - if ((buf_end[-1] & mask) == (u8)c1) { - DEBUG_PRINTF("partial!!!\n"); - return buf_end - 1; - } - - return buf_end; - } -#endif - - assert((buf_end - buf) >= VERM_BOUNDARY); - uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY; - if (min) { - // Input isn't aligned, so we need to run one iteration with an - // unaligned load, then skip buf forward to the next aligned address. - // There's some small overlap here, but we don't mind scanning it twice - // if we can do it quickly, do we? - const u8 *ptr = nocase - ? dvermPreconditionNocase(chars1, chars2, buf) - : dvermPrecondition(chars1, chars2, buf); - if (ptr) { - return ptr; - } - - buf += VERM_BOUNDARY - min; - assert(buf < buf_end); - } - - // Aligned loops from here on in - const u8 *ptr = nocase ? dvermSearchAlignedNocase(chars1, chars2, c1, c2, - buf, buf_end) - : dvermSearchAligned(chars1, chars2, c1, c2, buf, - buf_end); - if (ptr) { - return ptr; - } - - // Tidy up the mess at the end - ptr = nocase ? dvermPreconditionNocase(chars1, chars2, - buf_end - VERM_BOUNDARY) - : dvermPrecondition(chars1, chars2, buf_end - VERM_BOUNDARY); - - if (ptr) { - return ptr; - } - - /* check for partial match at end */ - u8 mask = nocase ? CASE_CLEAR : 0xff; - if ((buf_end[-1] & mask) == (u8)c1) { - DEBUG_PRINTF("partial!!!\n"); - return buf_end - 1; - } - - return buf_end; -} - -/* returns highest offset of c2 (NOTE: not c1) */ -static really_inline -const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, - const u8 *buf_end) { - DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", - nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars1 = VERM_SET_FN(c1); /* nocase already uppercase */ - VERM_TYPE chars2 = VERM_SET_FN(c2); /* nocase already uppercase */ - -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = nocase - ? rdvermMiniNocase(chars1, chars2, buf, buf_end) - : rdvermMini(chars1, chars2, buf, buf_end); - - if (ptr) { - return ptr; - } - - // check for partial match at end ??? - return buf - 1; - } -#endif - - assert((buf_end - buf) >= VERM_BOUNDARY); - size_t min = (size_t)buf_end % VERM_BOUNDARY; - if (min) { - // input not aligned, so we need to run one iteration with an unaligned - // load, then skip buf forward to the next aligned address. There's - // some small overlap here, but we don't mind scanning it twice if we - // can do it quickly, do we? - const u8 *ptr = nocase ? rdvermPreconditionNocase(chars1, chars2, - buf_end - VERM_BOUNDARY) - : rdvermPrecondition(chars1, chars2, - buf_end - VERM_BOUNDARY); - - if (ptr) { - return ptr; - } - - buf_end -= min; - if (buf >= buf_end) { - return buf_end; - } - } - - // Aligned loops from here on in - if (nocase) { - return rdvermSearchAlignedNocase(chars1, chars2, c1, c2, buf, buf_end); - } else { - return rdvermSearchAligned(chars1, chars2, c1, c2, buf, buf_end); - } -} \ No newline at end of file From 44dc75a3ea5ea787515606e257d337821d47eb5c Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Mon, 1 Nov 2021 16:51:18 +0200 Subject: [PATCH 08/25] complete refactoring and unification of Vermicelli functions --- src/nfa/vermicelli.hpp | 8 ++ src/nfa/vermicelli_simd.cpp | 240 ++++++++++++++++++------------------ 2 files changed, 127 insertions(+), 121 deletions(-) diff --git a/src/nfa/vermicelli.hpp b/src/nfa/vermicelli.hpp index 0b4686e1..83eb2335 100644 --- a/src/nfa/vermicelli.hpp +++ b/src/nfa/vermicelli.hpp @@ -75,4 +75,12 @@ const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, con } #endif +#ifdef __cplusplus +extern "C" { +#endif +const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + #endif /* VERMICELLI_HPP */ \ No newline at end of file diff --git a/src/nfa/vermicelli_simd.cpp b/src/nfa/vermicelli_simd.cpp index 6348e6f3..cd818dfb 100644 --- a/src/nfa/vermicelli_simd.cpp +++ b/src/nfa/vermicelli_simd.cpp @@ -41,48 +41,16 @@ template static really_inline -const u8 *vermicelliSingleBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { +const u8 *vermicelliBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { SuperVector mask = chars.eq(casemask & data); return first_non_zero_match(buf, mask); } -template -static really_inline -const u8 *rvermicelliSingleBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { - - SuperVector mask = chars.eq(casemask & data); - return last_non_zero_match(buf, mask); -} template static really_inline -const u8 *vermicelliDoubleBlock(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, - const u8 *buf/*, SuperVector *lastmask1, size_t len = S*/) { - - // lastmask1->print8("lastmask1"); - data.print8("data"); - chars1.print8("chars1"); - chars2.print8("chars2"); - casemask.print8("casemask"); - SuperVector v = casemask & data; - v.print8("v"); - SuperVector mask1 = chars1.eq(v); - mask1.print8("mask1"); - SuperVector mask2 = chars2.eq(v); - mask2.print8("mask2"); - SuperVector mask = (mask1 & (mask2 >> 1)); - mask.print8("mask"); - DEBUG_PRINTF("len = %ld\n", len); - // *lastmask1 = mask1 >> (len -1); - // lastmask1->print8("lastmask1"); - - return first_non_zero_match(buf, mask); -} - -template -static really_inline -const u8 *vermicelliSingleBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { +const u8 *vermicelliBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { SuperVector mask = chars.eq(casemask & data); return first_zero_match_inverted(buf, mask); @@ -90,36 +58,58 @@ const u8 *vermicelliSingleBlockNeg(SuperVector data, SuperVector chars, Su template static really_inline -const u8 *rvermicelliSingleBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { +const u8 *rvermicelliBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { + + SuperVector mask = chars.eq(casemask & data); + return last_non_zero_match(buf, mask); +} + + +template +static really_inline +const u8 *rvermicelliBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { SuperVector mask = chars.eq(casemask & data); return last_zero_match_inverted(buf, mask); } -/* + template static really_inline -const u8 *vermicelliDoubleBlockNeg(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, - const u8 *buf, size_t len = S) { +const u8 *vermicelliDoubleBlock(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, + u8 const c1, u8 const c2, u8 const casechar, const u8 *buf) { - // lastmask1.print8("lastmask1"); - data.print8("data"); - chars1.print8("chars1"); - chars2.print8("chars2"); - casemask.print8("casemask"); SuperVector v = casemask & data; - v.print8("v"); SuperVector mask1 = chars1.eq(v); - mask1.print8("mask1"); SuperVector mask2 = chars2.eq(v); - mask2.print8("mask2"); - SuperVector mask = (mask1 & (mask2 >> 1));// | lastmask1; - mask.print8("mask"); - DEBUG_PRINTF("len = %ld\n", len); - // lastmask1 = mask << (len -1); - // lastmask1.print8("lastmask1"); + SuperVector mask = mask1 & (mask2 >> 1); - return last_zero_match_inverted(buf, mask); -}*/ + DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]); + bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1)); + DEBUG_PRINTF("partial = %d\n", partial_match); + if (partial_match) return buf - 1; + + return first_non_zero_match(buf, mask); +} + +template +static really_inline +const u8 *rvermicelliDoubleBlock(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, + u8 const c1, u8 const c2, u8 const casechar, const u8 *buf) { + + SuperVector v = casemask & data; + SuperVector mask1 = chars1.eq(v); + SuperVector mask2 = chars2.eq(v); + SuperVector mask = (mask1 << 1)& mask2; + + DEBUG_PRINTF("buf[0] = %02hhx, buf[-1] = %02hhx\n", buf[0], buf[-1]); + bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1)); + DEBUG_PRINTF("partial = %d\n", partial_match); + if (partial_match) { + mask = mask | (SuperVector::Ones() >> (S-1)); + } + + return last_non_zero_match(buf, mask); +} template static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { @@ -142,7 +132,7 @@ static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector c DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); if (!ISALIGNED_N(d, S)) { SuperVector data = SuperVector::loadu(d); - rv = vermicelliSingleBlock(data, chars, casemask, d); + rv = vermicelliBlock(data, chars, casemask, d); if (rv) return rv; d = ROUNDUP_PTR(d, S); } @@ -151,7 +141,7 @@ static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector c __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); SuperVector data = SuperVector::load(d); - rv = vermicelliSingleBlock(data, chars, casemask, d); + rv = vermicelliBlock(data, chars, casemask, d); if (rv) return rv; d += S; } @@ -162,7 +152,7 @@ static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector c if (d != buf_end) { SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); - rv = vermicelliSingleBlock(data, chars, casemask, d); + rv = vermicelliBlock(data, chars, casemask, d); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -180,8 +170,6 @@ static const u8 *nvermicelliExecReal(SuperVector const chars, SuperVector const u8 *d = buf; const u8 *rv; - - __builtin_prefetch(d + 64); __builtin_prefetch(d + 2*64); __builtin_prefetch(d + 3*64); @@ -193,7 +181,7 @@ static const u8 *nvermicelliExecReal(SuperVector const chars, SuperVector DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); if (!ISALIGNED_N(d, S)) { SuperVector data = SuperVector::loadu(d); - rv = vermicelliSingleBlockNeg(data, chars, casemask, d); + rv = vermicelliBlockNeg(data, chars, casemask, d); if (rv) return rv; d = ROUNDUP_PTR(d, S); } @@ -202,7 +190,7 @@ static const u8 *nvermicelliExecReal(SuperVector const chars, SuperVector __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); SuperVector data = SuperVector::load(d); - rv = vermicelliSingleBlockNeg(data, chars, casemask, d); + rv = vermicelliBlockNeg(data, chars, casemask, d); if (rv) return rv; d += S; } @@ -213,7 +201,7 @@ static const u8 *nvermicelliExecReal(SuperVector const chars, SuperVector if (d != buf_end) { SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); - rv = vermicelliSingleBlockNeg(data, chars, casemask, d); + rv = vermicelliBlockNeg(data, chars, casemask, d); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -244,7 +232,7 @@ const u8 *rvermicelliExecReal(SuperVector const chars, SuperVector const c DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); if (!ISALIGNED_N(d, S)) { SuperVector data = SuperVector::loadu(d - S); - rv = rvermicelliSingleBlock(data, chars, casemask, d - S); + rv = rvermicelliBlock(data, chars, casemask, d - S); DEBUG_PRINTF("rv %p \n", rv); if (rv) return rv; d = ROUNDDOWN_PTR(d, S); @@ -257,7 +245,7 @@ const u8 *rvermicelliExecReal(SuperVector const chars, SuperVector const c d -= S; SuperVector data = SuperVector::load(d); - rv = rvermicelliSingleBlock(data, chars, casemask, d); + rv = rvermicelliBlock(data, chars, casemask, d); if (rv) return rv; } } @@ -267,7 +255,7 @@ const u8 *rvermicelliExecReal(SuperVector const chars, SuperVector const c if (d != buf) { SuperVector data = SuperVector::loadu(buf); - rv = rvermicelliSingleBlock(data, chars, casemask, buf); + rv = rvermicelliBlock(data, chars, casemask, buf); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -298,7 +286,7 @@ const u8 *rnvermicelliExecReal(SuperVector const chars, SuperVector const DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); if (!ISALIGNED_N(d, S)) { SuperVector data = SuperVector::loadu(d - S); - rv = rvermicelliSingleBlockNeg(data, chars, casemask, d - S); + rv = rvermicelliBlockNeg(data, chars, casemask, d - S); DEBUG_PRINTF("rv %p \n", rv); if (rv) return rv; d = ROUNDDOWN_PTR(d, S); @@ -311,7 +299,7 @@ const u8 *rnvermicelliExecReal(SuperVector const chars, SuperVector const d -= S; SuperVector data = SuperVector::load(d); - rv = rvermicelliSingleBlockNeg(data, chars, casemask, d); + rv = rvermicelliBlockNeg(data, chars, casemask, d); if (rv) return rv; } } @@ -321,7 +309,7 @@ const u8 *rnvermicelliExecReal(SuperVector const chars, SuperVector const if (d != buf) { SuperVector data = SuperVector::loadu(buf); - rv = rvermicelliSingleBlockNeg(data, chars, casemask, buf); + rv = rvermicelliBlockNeg(data, chars, casemask, buf); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -355,7 +343,7 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); if (!ISALIGNED_N(d, S)) { SuperVector data = SuperVector::loadu(d); - rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, &lastmask1); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, &lastmask1); if (rv) return rv; d = ROUNDUP_PTR(d, S); } @@ -364,11 +352,8 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); SuperVector data = SuperVector::load(d); - rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, &lastmask1); - if (rv) { - bool partial_match = (((rv[0] & casechar) == c2) && ((rv[-1] & casechar) == c1)); - return rv - partial_match; - } + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, &lastmask1); + if (rv) return rv; d += S; } } @@ -378,7 +363,7 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< if (d != buf_end) { SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); - rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, buf_end - d); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, buf_end - d); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -396,60 +381,63 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< } // /* returns highest offset of c2 (NOTE: not c1) */ -// static really_inline -// const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, -// const u8 *buf_end) { -// DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", -// nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); -// assert(buf < buf_end); +template +const u8 *rvermicelliDoubleExecReal(char c1, char c2, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("rverm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + char s[255]; + snprintf(s, buf_end - buf + 1, "%s", buf); + DEBUG_PRINTF("b %s\n", s); -// VERM_TYPE chars1 = VERM_SET_FN(c1); /* nocase already uppercase */ -// VERM_TYPE chars2 = VERM_SET_FN(c2); /* nocase already uppercase */ + const u8 *d = buf_end; + const u8 *rv; + const SuperVector chars1 = SuperVector::dup_u8(c1); + const SuperVector chars2 = SuperVector::dup_u8(c2); + const u8 casechar = casemask.u.u8[0]; -// #ifdef HAVE_AVX512 -// if (buf_end - buf <= VERM_BOUNDARY) { -// const u8 *ptr = nocase -// ? rdvermMiniNocase(chars1, chars2, buf, buf_end) -// : rdvermMini(chars1, chars2, buf, buf_end); + __builtin_prefetch(d - 64); + __builtin_prefetch(d - 2*64); + __builtin_prefetch(d - 3*64); + __builtin_prefetch(d - 4*64); + DEBUG_PRINTF("start %p end %p \n", buf, d); + assert(d > buf); + if (d - S >= buf) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d - S); + rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d - S); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + d = ROUNDDOWN_PTR(d, S); + } -// if (ptr) { -// return ptr; -// } + while (d - S >= buf) { + DEBUG_PRINTF("aligned %p \n", d); + // On large packet buffers, this prefetch appears to get us about 2%. + __builtin_prefetch(d - 64); -// // check for partial match at end ??? -// return buf - 1; -// } -// #endif + d -= S; + SuperVector data = SuperVector::load(d); + rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d); + if (rv) return rv; + } + } -// assert((buf_end - buf) >= VERM_BOUNDARY); -// size_t min = (size_t)buf_end % VERM_BOUNDARY; -// if (min) { -// // input not aligned, so we need to run one iteration with an unaligned -// // load, then skip buf forward to the next aligned address. There's -// // some small overlap here, but we don't mind scanning it twice if we -// // can do it quickly, do we? -// const u8 *ptr = nocase ? rdvermPreconditionNocase(chars1, chars2, -// buf_end - VERM_BOUNDARY) -// : rdvermPrecondition(chars1, chars2, -// buf_end - VERM_BOUNDARY); + DEBUG_PRINTF("tail d %p e %p \n", buf, d); + // finish off head -// if (ptr) { -// return ptr; -// } + if (d != buf) { + SuperVector data = SuperVector::loadu(buf); + rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, buf); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } -// buf_end -= min; -// if (buf >= buf_end) { -// return buf_end; -// } -// } - -// // Aligned loops from here on in -// if (nocase) { -// return rdvermSearchAlignedNocase(chars1, chars2, c1, c2, buf, buf_end); -// } else { -// return rdvermSearchAligned(chars1, chars2, c1, c2, buf, buf_end); -// } -// } + return buf - 1; +} extern "C" const u8 *vermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) { DEBUG_PRINTF("verm scan %s\\x%02hhx over %zu bytes\n", @@ -505,4 +493,14 @@ extern "C" const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; return vermicelliDoubleExecReal(c1, c2, casemask, buf, buf_end); +} + +extern "C" const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", + nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; + + return rvermicelliDoubleExecReal(c1, c2, casemask, buf, buf_end); } \ No newline at end of file From 7b65b298c1363b6d18c7b9900828a64be1527f4b Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Tue, 19 Oct 2021 18:23:13 +0000 Subject: [PATCH 09/25] add arm vector types in union, avoid -flax-conversions, fix castings --- CMakeLists.txt | 2 - src/util/arch/arm/match.hpp | 8 +- src/util/arch/arm/simd_utils.h | 40 +++--- src/util/supervector/arch/arm/impl.cpp | 172 +++++++++++++++---------- src/util/supervector/supervector.hpp | 14 +- unit/internal/simd_utils.cpp | 2 +- 6 files changed, 145 insertions(+), 93 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 05e6a5c7..92abf6dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -277,8 +277,6 @@ elseif (ARCH_ARM32 OR ARCH_AARCH64) message(FATAL_ERROR "arm_sve.h is required to build for SVE.") endif() endif() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -flax-vector-conversions") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flax-vector-conversions") endif() CHECK_FUNCTION_EXISTS(posix_memalign HAVE_POSIX_MEMALIGN) diff --git a/src/util/arch/arm/match.hpp b/src/util/arch/arm/match.hpp index 46d84d06..e7f757bd 100644 --- a/src/util/arch/arm/match.hpp +++ b/src/util/arch/arm/match.hpp @@ -30,8 +30,8 @@ template <> really_really_inline const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> mask) { - uint32x4_t res_t = vreinterpretq_u32_u8(mask.u.v128[0]); - uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(res_t, res_t)), 0); + uint32x4_t m = mask.u.u32x4[0]; + uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0); if (vmax != 0) { typename SuperVector<16>::movemask_type z = mask.movemask(); DEBUG_PRINTF("z %08x\n", z); @@ -49,8 +49,8 @@ const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> mask) { template <> really_really_inline const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> mask) { - uint32x4_t res_t = vreinterpretq_u32_u8(mask.u.v128[0]); - uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(res_t, res_t)), 0); + uint32x4_t m = mask.u.u32x4[0]; + uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0); if (vmax != 0) { typename SuperVector<16>::movemask_type z = mask.movemask(); DEBUG_PRINTF("buf %p z %08x \n", buf, z); diff --git a/src/util/arch/arm/simd_utils.h b/src/util/arch/arm/simd_utils.h index 24851773..630cac93 100644 --- a/src/util/arch/arm/simd_utils.h +++ b/src/util/arch/arm/simd_utils.h @@ -100,7 +100,7 @@ static really_inline int isnonzero128(m128 a) { */ static really_inline u32 diffrich128(m128 a, m128 b) { static const uint32x4_t movemask = { 1, 2, 4, 8 }; - return vaddvq_u32(vandq_u32(vmvnq_s32(vceqq_s32((int32x4_t)a, (int32x4_t)b)), movemask)); + return vaddvq_u32(vandq_u32(vmvnq_u32(vceqq_u32((uint32x4_t)a, (uint32x4_t)b)), movemask)); } /** @@ -109,53 +109,53 @@ static really_inline u32 diffrich128(m128 a, m128 b) { */ static really_inline u32 diffrich64_128(m128 a, m128 b) { static const uint64x2_t movemask = { 1, 4 }; - return vaddvq_u64(vandq_u64(vmvnq_s32(vceqq_s64((int64x2_t)a, (int64x2_t)b)), movemask)); + return (u32) vaddvq_u64(vandq_u64((uint64x2_t)vmvnq_u32((uint32x4_t)vceqq_u64((uint64x2_t)a, (uint64x2_t)b)), movemask)); } static really_really_inline m128 add_2x64(m128 a, m128 b) { - return (m128) vaddq_u64((int64x2_t)a, (int64x2_t)b); + return (m128) vaddq_u64((uint64x2_t)a, (uint64x2_t)b); } static really_really_inline m128 sub_2x64(m128 a, m128 b) { - return (m128) vsubq_u64((int64x2_t)a, (int64x2_t)b); + return (m128) vsubq_u64((uint64x2_t)a, (uint64x2_t)b); } static really_really_inline m128 lshift_m128(m128 a, unsigned b) { - return (m128) vshlq_n_s32((int64x2_t)a, b); + return (m128) vshlq_n_u32((uint32x4_t)a, b); } static really_really_inline m128 rshift_m128(m128 a, unsigned b) { - return (m128) vshrq_n_s32((int64x2_t)a, b); + return (m128) vshrq_n_u32((uint32x4_t)a, b); } static really_really_inline m128 lshift64_m128(m128 a, unsigned b) { - return (m128) vshlq_n_s64((int64x2_t)a, b); + return (m128) vshlq_n_u64((uint64x2_t)a, b); } static really_really_inline m128 rshift64_m128(m128 a, unsigned b) { - return (m128) vshrq_n_s64((int64x2_t)a, b); + return (m128) vshrq_n_u64((uint64x2_t)a, b); } static really_inline m128 eq128(m128 a, m128 b) { - return (m128) vceqq_s8((int8x16_t)a, (int8x16_t)b); + return (m128) vceqq_u8((uint8x16_t)a, (uint8x16_t)b); } static really_inline m128 eq64_m128(m128 a, m128 b) { - return (m128) vceqq_u64((int64x2_t)a, (int64x2_t)b); + return (m128) vceqq_u64((uint64x2_t)a, (uint64x2_t)b); } static really_inline u32 movemask128(m128 a) { static const uint8x16_t powers = { 1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128 }; // Compute the mask from the input - uint64x2_t mask = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vandq_u8((uint8x16_t)a, powers)))); - uint64x2_t mask1 = (m128)vextq_s8(mask, zeroes128(), 7); + uint8x16_t mask = (uint8x16_t) vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vandq_u8((uint8x16_t)a, powers)))); + uint8x16_t mask1 = vextq_u8(mask, (uint8x16_t)zeroes128(), 7); mask = vorrq_u8(mask, mask1); // Get the resulting bytes @@ -187,7 +187,7 @@ static really_inline u64a movq(const m128 in) { /* another form of movq */ static really_inline m128 load_m128_from_u64a(const u64a *p) { - return (m128) vsetq_lane_u64(*p, zeroes128(), 0); + return (m128) vsetq_lane_u64(*p, (uint64x2_t) zeroes128(), 0); } static really_inline u32 extract32from128(const m128 in, unsigned imm) { @@ -220,10 +220,10 @@ static really_inline u64a extract64from128(const m128 in, unsigned imm) { #else switch (imm) { case 0: - return vgetq_lane_u64((uint32x4_t) in, 0); + return vgetq_lane_u64((uint64x2_t) in, 0); break; case 1: - return vgetq_lane_u64((uint32x4_t) in, 1); + return vgetq_lane_u64((uint64x2_t) in, 1); break; default: return 0; @@ -233,11 +233,11 @@ static really_inline u64a extract64from128(const m128 in, unsigned imm) { } static really_inline m128 low64from128(const m128 in) { - return vcombine_u64(vget_low_u64(in), vdup_n_u64(0)); + return (m128) vcombine_u64(vget_low_u64((uint64x2_t)in), vdup_n_u64(0)); } static really_inline m128 high64from128(const m128 in) { - return vcombine_u64(vget_high_u64(in), vdup_n_u64(0)); + return (m128) vcombine_u64(vget_high_u64((uint64x2_t)in), vdup_n_u64(0)); } static really_inline m128 add128(m128 a, m128 b) { @@ -257,7 +257,7 @@ static really_inline m128 or128(m128 a, m128 b) { } static really_inline m128 andnot128(m128 a, m128 b) { - return (m128) (m128) vandq_s8( vmvnq_s8(a), b); + return (m128) vandq_s8( vmvnq_s8((int8x16_t) a), (int8x16_t) b); } // aligned load @@ -401,12 +401,12 @@ m128 pshufb_m128(m128 a, m128 b) { static really_inline m128 max_u8_m128(m128 a, m128 b) { - return (m128) vmaxq_u8((int8x16_t)a, (int8x16_t)b); + return (m128) vmaxq_u8((uint8x16_t)a, (uint8x16_t)b); } static really_inline m128 min_u8_m128(m128 a, m128 b) { - return (m128) vminq_u8((int8x16_t)a, (int8x16_t)b); + return (m128) vminq_u8((uint8x16_t)a, (uint8x16_t)b); } static really_inline diff --git a/src/util/supervector/arch/arm/impl.cpp b/src/util/supervector/arch/arm/impl.cpp index 34e5486d..f804abeb 100644 --- a/src/util/supervector/arch/arm/impl.cpp +++ b/src/util/supervector/arch/arm/impl.cpp @@ -45,72 +45,114 @@ really_inline SuperVector<16>::SuperVector(typename base_type::type const v) template<> template<> -really_inline SuperVector<16>::SuperVector(int8x16_t const other) +really_inline SuperVector<16>::SuperVector(int8x16_t other) { - u.v128[0] = static_cast(other); + u.s8x16[0] = other; } template<> template<> -really_inline SuperVector<16>::SuperVector(uint8x16_t const other) +really_inline SuperVector<16>::SuperVector(uint8x16_t other) { - u.v128[0] = static_cast(other); + u.u8x16[0] = other; +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(int16x8_t other) +{ + u.s16x8[0] = other; +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(uint16x8_t other) +{ + u.u16x8[0] = other; +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(int32x4_t other) +{ + u.s32x4[0] = other; +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(uint32x4_t other) +{ + u.u32x4[0] = other; +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(int64x2_t other) +{ + u.s64x2[0] = other; +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(uint64x2_t other) +{ + u.u64x2[0] = other; } template<> template<> really_inline SuperVector<16>::SuperVector(int8_t const other) { - u.v128[0] = vdupq_n_s8(other); + u.s8x16[0] = vdupq_n_s8(other); } template<> template<> really_inline SuperVector<16>::SuperVector(uint8_t const other) { - u.v128[0] = vdupq_n_u8(other); + u.u8x16[0] = vdupq_n_u8(other); } template<> template<> really_inline SuperVector<16>::SuperVector(int16_t const other) { - u.v128[0] = vdupq_n_s16(other); + u.s16x8[0] = vdupq_n_s16(other); } template<> template<> really_inline SuperVector<16>::SuperVector(uint16_t const other) { - u.v128[0] = vdupq_n_u16(other); + u.u16x8[0] = vdupq_n_u16(other); } template<> template<> really_inline SuperVector<16>::SuperVector(int32_t const other) { - u.v128[0] = vdupq_n_s32(other); + u.s32x4[0] = vdupq_n_s32(other); } template<> template<> really_inline SuperVector<16>::SuperVector(uint32_t const other) { - u.v128[0] = vdupq_n_u32(other); + u.u32x4[0] = vdupq_n_u32(other); } template<> template<> really_inline SuperVector<16>::SuperVector(int64_t const other) { - u.v128[0] = vdupq_n_s64(other); + u.s64x2[0] = vdupq_n_s64(other); } template<> template<> really_inline SuperVector<16>::SuperVector(uint64_t const other) { - u.v128[0] = vdupq_n_u64(other); + u.u64x2[0] = vdupq_n_u64(other); } // Constants @@ -137,37 +179,37 @@ really_inline void SuperVector<16>::operator=(SuperVector<16> const &other) template <> really_inline SuperVector<16> SuperVector<16>::operator&(SuperVector<16> const &b) const { - return {vandq_s8(u.v128[0], b.u.v128[0])}; + return {vandq_u8(u.u8x16[0], b.u.u8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator|(SuperVector<16> const &b) const { - return {vorrq_s8(u.v128[0], b.u.v128[0])}; + return {vorrq_u8(u.u8x16[0], b.u.u8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator^(SuperVector<16> const &b) const { - return {veorq_s8(u.v128[0], b.u.v128[0])}; + return {veorq_u8(u.u8x16[0], b.u.u8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator!() const { - return {vmvnq_s8(u.v128[0])}; + return {vmvnq_u8(u.u8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::opandnot(SuperVector<16> const &b) const { - return {vandq_s8(vmvnq_s8(u.v128[0]), b.u.v128[0])}; + return {vandq_u8(vmvnq_u8(u.u8x16[0]), b.u.u8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator==(SuperVector<16> const &b) const { - return {vceqq_s8((int16x8_t)u.v128[0], (int16x8_t)b.u.v128[0])}; + return {vceqq_u8(u.u8x16[0], b.u.u8x16[0])}; } template <> @@ -179,25 +221,25 @@ really_inline SuperVector<16> SuperVector<16>::operator!=(SuperVector<16> const template <> really_inline SuperVector<16> SuperVector<16>::operator>(SuperVector<16> const &b) const { - return {vcgtq_s8((int16x8_t)u.v128[0], (int16x8_t)b.u.v128[0])}; + return {vcgtq_s8(u.s8x16[0], b.u.s8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator>=(SuperVector<16> const &b) const { - return {vcgeq_s8((int16x8_t)u.v128[0], (int16x8_t)b.u.v128[0])}; + return {vcgeq_u8(u.u8x16[0], b.u.u8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator<(SuperVector<16> const &b) const { - return {vcltq_s8((int16x8_t)u.v128[0], (int16x8_t)b.u.v128[0])}; + return {vcltq_s8(u.s8x16[0], b.u.s8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator<=(SuperVector<16> const &b) const { - return {vcgeq_s8((int16x8_t)u.v128[0], (int16x8_t)b.u.v128[0])}; + return {vcgeq_s8(u.s8x16[0], b.u.s8x16[0])}; } template <> @@ -212,9 +254,9 @@ really_inline typename SuperVector<16>::movemask_type SuperVector<16>::movemask( SuperVector powers{0x8040201008040201UL}; // Compute the mask from the input - uint64x2_t mask = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vandq_u8((uint16x8_t)u.v128[0], powers.u.v128[0])))); - uint64x2_t mask1 = (m128)vextq_s8(mask, vdupq_n_u8(0), 7); - mask = vorrq_u8(mask, mask1); + uint8x16_t mask = (uint8x16_t) vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vandq_u8(u.u8x16[0], powers.u.u8x16[0])))); + uint64x2_t mask1 = (uint64x2_t) vextq_u8(mask, vdupq_n_u8(0), 7); + mask = vorrq_u8(mask, (uint8x16_t) mask1); // Get the resulting bytes uint16_t output; @@ -232,35 +274,35 @@ template <> template really_inline SuperVector<16> SuperVector<16>::vshl_8_imm() const { - return {(m128)vshlq_n_s8(u.v128[0], N)}; + return {vshlq_n_u8(u.u8x16[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshl_16_imm() const { - return {(m128)vshlq_n_s16(u.v128[0], N)}; + return {vshlq_n_u16(u.u16x8[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshl_32_imm() const { - return {(m128)vshlq_n_s32(u.v128[0], N)}; + return {vshlq_n_u32(u.u32x4[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshl_64_imm() const { - return {(m128)vshlq_n_s64(u.v128[0], N)}; + return {vshlq_n_u64(u.u64x2[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshl_128_imm() const { - return {vextq_s8(vdupq_n_u8(0), (int16x8_t)u.v128[0], 16 - N)}; + return {vextq_u8(vdupq_n_u8(0), u.u8x16[0], 16 - N)}; } template <> @@ -274,35 +316,35 @@ template <> template really_inline SuperVector<16> SuperVector<16>::vshr_8_imm() const { - return {(m128)vshrq_n_s8(u.v128[0], N)}; + return {vshrq_n_u8(u.u8x16[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshr_16_imm() const { - return {(m128)vshrq_n_s16(u.v128[0], N)}; + return {vshrq_n_u16(u.u16x8[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshr_32_imm() const { - return {(m128)vshrq_n_s32(u.v128[0], N)}; + return {vshrq_n_u32(u.u32x4[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshr_64_imm() const { - return {(m128)vshrq_n_s64(u.v128[0], N)}; + return {vshrq_n_u64(u.u64x2[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshr_128_imm() const { - return {vextq_s8((int16x8_t)u.v128[0], vdupq_n_u8(0), N)}; + return {vextq_u8(u.u8x16[0], vdupq_n_u8(0), N)}; } template <> @@ -334,7 +376,7 @@ really_inline SuperVector<16> SuperVector<16>::vshl_8 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshlq_n_s8(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshlq_n_u8(u.u8x16[0], n)}; }); return result; } @@ -344,7 +386,7 @@ really_inline SuperVector<16> SuperVector<16>::vshl_16 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshlq_n_s16(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshlq_n_u16(u.u16x8[0], n)}; }); return result; } @@ -354,7 +396,7 @@ really_inline SuperVector<16> SuperVector<16>::vshl_32 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshlq_n_s32(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshlq_n_u32(u.u32x4[0], n)}; }); return result; } @@ -364,7 +406,7 @@ really_inline SuperVector<16> SuperVector<16>::vshl_64 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshlq_n_s64(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshlq_n_u64(u.u64x2[0], n)}; }); return result; } @@ -374,7 +416,7 @@ really_inline SuperVector<16> SuperVector<16>::vshl_128(uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vextq_s8(vdupq_n_u8(0), (int16x8_t)u.v128[0], 16 - n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vextq_u8(vdupq_n_u8(0), u.u8x16[0], 16 - n)}; }); return result; } @@ -390,7 +432,7 @@ really_inline SuperVector<16> SuperVector<16>::vshr_8 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshrq_n_s8(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshrq_n_u8(u.u8x16[0], n)}; }); return result; } @@ -400,7 +442,7 @@ really_inline SuperVector<16> SuperVector<16>::vshr_16 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshrq_n_s16(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshrq_n_u16(u.u16x8[0], n)}; }); return result; } @@ -410,7 +452,7 @@ really_inline SuperVector<16> SuperVector<16>::vshr_32 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshrq_n_s32(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshrq_n_u32(u.u32x4[0], n)}; }); return result; } @@ -420,7 +462,7 @@ really_inline SuperVector<16> SuperVector<16>::vshr_64 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshrq_n_s64(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshrq_n_u64(u.u64x2[0], n)}; }); return result; } @@ -430,7 +472,7 @@ really_inline SuperVector<16> SuperVector<16>::vshr_128(uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vextq_s8((int16x8_t)u.v128[0], vdupq_n_u8(0), n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vextq_u8(u.u8x16[0], vdupq_n_u8(0), n)}; }); return result; } @@ -444,7 +486,7 @@ really_inline SuperVector<16> SuperVector<16>::vshr(uint8_t const N) const template <> really_inline SuperVector<16> SuperVector<16>::operator>>(uint8_t const N) const { - return {vextq_s8((int16x8_t)u.v128[0], vdupq_n_u8(0), N)}; + return {vextq_u8(u.u8x16[0], vdupq_n_u8(0), N)}; } #else template <> @@ -458,7 +500,7 @@ really_inline SuperVector<16> SuperVector<16>::operator>>(uint8_t const N) const template <> really_inline SuperVector<16> SuperVector<16>::operator<<(uint8_t const N) const { - return {vextq_s8(vdupq_n_u8(0), (int16x8_t)u.v128[0], 16 - N)}; + return {vextq_u8(vdupq_n_u8(0), u.u8x16[0], 16 - N)}; } #else template <> @@ -512,7 +554,7 @@ really_inline SuperVector<16> SuperVector<16>::alignr(SuperVector<16> &other, in if (offset == 16) { return *this; } else { - return {vextq_s8((int16x8_t)other.u.v128[0], (int16x8_t)u.v128[0], offset)}; + return {vextq_u8(other.u.u8x16[0], u.u8x16[0], offset)}; } } #else @@ -521,21 +563,21 @@ really_inline SuperVector<16> SuperVector<16>::alignr(SuperVector<16> &other, in { switch(offset) { case 0: return other; break; - case 1: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 1)}; break; - case 2: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 2)}; break; - case 3: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 3)}; break; - case 4: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 4)}; break; - case 5: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 5)}; break; - case 6: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 6)}; break; - case 7: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 7)}; break; - case 8: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 8)}; break; - case 9: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 9)}; break; - case 10: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 10)}; break; - case 11: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 11)}; break; - case 12: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 12)}; break; - case 13: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 13)}; break; - case 14: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 14)}; break; - case 15: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 15)}; break; + case 1: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 1)}; break; + case 2: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 2)}; break; + case 3: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 3)}; break; + case 4: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 4)}; break; + case 5: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 5)}; break; + case 6: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 6)}; break; + case 7: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 7)}; break; + case 8: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 8)}; break; + case 9: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 9)}; break; + case 10: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 10)}; break; + case 11: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 11)}; break; + case 12: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 12)}; break; + case 13: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 13)}; break; + case 14: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 14)}; break; + case 15: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 15)}; break; case 16: return *this; break; default: break; } @@ -547,7 +589,7 @@ template<> template<> really_inline SuperVector<16> SuperVector<16>::pshufb(SuperVector<16> b) { - return {vqtbl1q_s8((int8x16_t)u.v128[0], (uint8x16_t)b.u.v128[0])}; + return {vqtbl1q_u8(u.u8x16[0], b.u.u8x16[0])}; } template<> @@ -565,7 +607,7 @@ template<> really_inline SuperVector<16> SuperVector<16>::pshufb_maskz(SuperVector<16> b, uint8_t const len) { SuperVector mask = Ones_vshr(16 -len); - return mask & pshufb(b); + return mask & pshufb(b); } #endif // SIMD_IMPL_HPP diff --git a/src/util/supervector/supervector.hpp b/src/util/supervector/supervector.hpp index 76e167ce..e69e4b42 100644 --- a/src/util/supervector/supervector.hpp +++ b/src/util/supervector/supervector.hpp @@ -162,6 +162,18 @@ public: typename BaseVector<16>::type ALIGN_ATTR(BaseVector<16>::size) v128[SIZE / BaseVector<16>::size]; typename BaseVector<32>::type ALIGN_ATTR(BaseVector<32>::size) v256[SIZE / BaseVector<32>::size]; typename BaseVector<64>::type ALIGN_ATTR(BaseVector<64>::size) v512[SIZE / BaseVector<64>::size]; + +#if defined(ARCH_ARM32) || defined(ARCH_AARCH64) + uint64x2_t ALIGN_ATTR(BaseVector<16>::size) u64x2[SIZE / BaseVector<16>::size]; + int64x2_t ALIGN_ATTR(BaseVector<16>::size) s64x2[SIZE / BaseVector<16>::size]; + uint32x4_t ALIGN_ATTR(BaseVector<16>::size) u32x4[SIZE / BaseVector<16>::size]; + int32x4_t ALIGN_ATTR(BaseVector<16>::size) s32x4[SIZE / BaseVector<16>::size]; + uint16x8_t ALIGN_ATTR(BaseVector<16>::size) u16x8[SIZE / BaseVector<16>::size]; + int16x8_t ALIGN_ATTR(BaseVector<16>::size) s16x8[SIZE / BaseVector<16>::size]; + uint8x16_t ALIGN_ATTR(BaseVector<16>::size) u8x16[SIZE / BaseVector<16>::size]; + int8x16_t ALIGN_ATTR(BaseVector<16>::size) s8x16[SIZE / BaseVector<16>::size]; +#endif + uint64_t u64[SIZE / sizeof(uint64_t)]; int64_t s64[SIZE / sizeof(int64_t)]; uint32_t u32[SIZE / sizeof(uint32_t)]; @@ -180,7 +192,7 @@ public: SuperVector(typename base_type::type const v); template - SuperVector(T const other); + SuperVector(T other); SuperVector(SuperVector const lo, SuperVector const hi); SuperVector(previous_type const lo, previous_type const hi); diff --git a/unit/internal/simd_utils.cpp b/unit/internal/simd_utils.cpp index 2a9accae..9b206e1b 100644 --- a/unit/internal/simd_utils.cpp +++ b/unit/internal/simd_utils.cpp @@ -667,7 +667,7 @@ TEST(SimdUtilsTest, movq) { simd = _mm_set_epi64x(~0LL, 0x123456789abcdef); #elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) int64x2_t a = { 0x123456789abcdefLL, ~0LL }; - simd = vreinterpretq_s64_s8(a); + simd = vreinterpretq_s32_s64(a); #endif #endif r = movq(simd); From 9abfdcaa8425ad12f42b0d3e11f321e0a2d74a28 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Mon, 1 Nov 2021 14:48:17 +0200 Subject: [PATCH 10/25] add Vermicelli/RVermicelli to microbenchmark utility --- benchmarks/benchmarks.cpp | 28 ++++++++++++++++++++++++++++ benchmarks/benchmarks.hpp | 1 + 2 files changed, 29 insertions(+) diff --git a/benchmarks/benchmarks.cpp b/benchmarks/benchmarks.cpp index 49990bd7..91cab3f8 100644 --- a/benchmarks/benchmarks.cpp +++ b/benchmarks/benchmarks.cpp @@ -191,6 +191,34 @@ int main(){ ); } + for (size_t i = 0; i < std::size(sizes); i++) { + MicroBenchmark bench("Vermicelli", sizes[i]); + run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, + [&](MicroBenchmark &b) { + b.chars.set('a'); + ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); + memset(b.buf.data(), 'b', b.size); + }, + [&](MicroBenchmark &b) { + return vermicelliExec('a', 'b', b.buf.data(), b.buf.data() + b.size); + } + ); + } + + for (size_t i = 0; i < std::size(sizes); i++) { + MicroBenchmark bench("Reverse Vermicelli", sizes[i]); + run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench, + [&](MicroBenchmark &b) { + b.chars.set('a'); + ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); + memset(b.buf.data(), 'b', b.size); + }, + [&](MicroBenchmark &b) { + return rvermicelliExec('a', 'b', b.buf.data(), b.buf.data() + b.size); + } + ); + } + for (size_t i = 0; i < std::size(sizes); i++) { //we imitate the noodle unit tests std::string str; diff --git a/benchmarks/benchmarks.hpp b/benchmarks/benchmarks.hpp index 37326523..eb892e51 100644 --- a/benchmarks/benchmarks.hpp +++ b/benchmarks/benchmarks.hpp @@ -30,6 +30,7 @@ #include "nfa/shufticompile.h" #include "nfa/truffle.h" #include "nfa/trufflecompile.h" +#include "nfa/vermicelli.h" #include "hwlm/noodle_build.h" #include "hwlm/noodle_engine.h" #include "hwlm/noodle_internal.h" From 2fa947af9c3f070ce8d83f2acb4caa380dac6597 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 27 Oct 2021 12:29:39 +0300 Subject: [PATCH 11/25] added refactored vermicelli_simd.cpp implementation --- src/nfa/vermicelli.hpp | 78 ++++++ src/nfa/vermicelli_simd.cpp | 508 ++++++++++++++++++++++++++++++++++++ 2 files changed, 586 insertions(+) create mode 100644 src/nfa/vermicelli.hpp create mode 100644 src/nfa/vermicelli_simd.cpp diff --git a/src/nfa/vermicelli.hpp b/src/nfa/vermicelli.hpp new file mode 100644 index 00000000..0b4686e1 --- /dev/null +++ b/src/nfa/vermicelli.hpp @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2015-2020, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2021, Arm Limited + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * \brief Vermicelli: single-byte and double-byte acceleration. + */ + +#ifndef VERMICELLI_HPP +#define VERMICELLI_HPP + +#ifdef __cplusplus +extern "C" { +#endif +const u8 *vermicelliExec(char c, char noCase, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + +#ifdef __cplusplus +extern "C" { +#endif +const u8 *nvermicelliExec(char c, char noCase, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + +#ifdef __cplusplus +extern "C" { +#endif +const u8 *rvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + +#ifdef __cplusplus +extern "C" { +#endif +const u8 *rnvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + +#ifdef __cplusplus +extern "C" { +#endif +const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + +#endif /* VERMICELLI_HPP */ \ No newline at end of file diff --git a/src/nfa/vermicelli_simd.cpp b/src/nfa/vermicelli_simd.cpp new file mode 100644 index 00000000..6348e6f3 --- /dev/null +++ b/src/nfa/vermicelli_simd.cpp @@ -0,0 +1,508 @@ +/* + * Copyright (c) 2015-2020, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2021, Arm Limited + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * \brief Vermicelli: single-byte and double-byte acceleration. + */ + +#include "util/bitutils.h" +#include "util/simd_utils.h" + +#include "vermicelli.hpp" +#include "util/supervector/casemask.hpp" +#include "util/match.hpp" + +template +static really_inline +const u8 *vermicelliSingleBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { + + SuperVector mask = chars.eq(casemask & data); + return first_non_zero_match(buf, mask); +} + +template +static really_inline +const u8 *rvermicelliSingleBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { + + SuperVector mask = chars.eq(casemask & data); + return last_non_zero_match(buf, mask); +} + +template +static really_inline +const u8 *vermicelliDoubleBlock(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, + const u8 *buf/*, SuperVector *lastmask1, size_t len = S*/) { + + // lastmask1->print8("lastmask1"); + data.print8("data"); + chars1.print8("chars1"); + chars2.print8("chars2"); + casemask.print8("casemask"); + SuperVector v = casemask & data; + v.print8("v"); + SuperVector mask1 = chars1.eq(v); + mask1.print8("mask1"); + SuperVector mask2 = chars2.eq(v); + mask2.print8("mask2"); + SuperVector mask = (mask1 & (mask2 >> 1)); + mask.print8("mask"); + DEBUG_PRINTF("len = %ld\n", len); + // *lastmask1 = mask1 >> (len -1); + // lastmask1->print8("lastmask1"); + + return first_non_zero_match(buf, mask); +} + +template +static really_inline +const u8 *vermicelliSingleBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { + + SuperVector mask = chars.eq(casemask & data); + return first_zero_match_inverted(buf, mask); +} + +template +static really_inline +const u8 *rvermicelliSingleBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { + + SuperVector mask = chars.eq(casemask & data); + return last_zero_match_inverted(buf, mask); +} +/* +template +static really_inline +const u8 *vermicelliDoubleBlockNeg(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, + const u8 *buf, size_t len = S) { + + // lastmask1.print8("lastmask1"); + data.print8("data"); + chars1.print8("chars1"); + chars2.print8("chars2"); + casemask.print8("casemask"); + SuperVector v = casemask & data; + v.print8("v"); + SuperVector mask1 = chars1.eq(v); + mask1.print8("mask1"); + SuperVector mask2 = chars2.eq(v); + mask2.print8("mask2"); + SuperVector mask = (mask1 & (mask2 >> 1));// | lastmask1; + mask.print8("mask"); + DEBUG_PRINTF("len = %ld\n", len); + // lastmask1 = mask << (len -1); + // lastmask1.print8("lastmask1"); + + return last_zero_match_inverted(buf, mask); +}*/ + +template +static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("verm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + + const u8 *d = buf; + const u8 *rv; + + __builtin_prefetch(d + 64); + __builtin_prefetch(d + 2*64); + __builtin_prefetch(d + 3*64); + __builtin_prefetch(d + 4*64); + DEBUG_PRINTF("start %p end %p \n", d, buf_end); + assert(d < buf_end); + if (d + S <= buf_end) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d); + rv = vermicelliSingleBlock(data, chars, casemask, d); + if (rv) return rv; + d = ROUNDUP_PTR(d, S); + } + + while(d + S <= buf_end) { + __builtin_prefetch(d + 64); + DEBUG_PRINTF("d %p \n", d); + SuperVector data = SuperVector::load(d); + rv = vermicelliSingleBlock(data, chars, casemask, d); + if (rv) return rv; + d += S; + } + } + + DEBUG_PRINTF("d %p e %p \n", d, buf_end); + // finish off tail + + if (d != buf_end) { + SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); + rv = vermicelliSingleBlock(data, chars, casemask, d); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } + + return buf_end; +} + +template +static const u8 *nvermicelliExecReal(SuperVector const chars, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("verm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + + const u8 *d = buf; + const u8 *rv; + + + + __builtin_prefetch(d + 64); + __builtin_prefetch(d + 2*64); + __builtin_prefetch(d + 3*64); + __builtin_prefetch(d + 4*64); + DEBUG_PRINTF("start %p end %p \n", d, buf_end); + assert(d < buf_end); + if (d + S <= buf_end) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d); + rv = vermicelliSingleBlockNeg(data, chars, casemask, d); + if (rv) return rv; + d = ROUNDUP_PTR(d, S); + } + + while(d + S <= buf_end) { + __builtin_prefetch(d + 64); + DEBUG_PRINTF("d %p \n", d); + SuperVector data = SuperVector::load(d); + rv = vermicelliSingleBlockNeg(data, chars, casemask, d); + if (rv) return rv; + d += S; + } + } + + DEBUG_PRINTF("d %p e %p \n", d, buf_end); + // finish off tail + + if (d != buf_end) { + SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); + rv = vermicelliSingleBlockNeg(data, chars, casemask, d); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } + + return buf_end; +} + +// Reverse vermicelli scan. Provides exact semantics and returns (buf - 1) if +// character not found. +template +const u8 *rvermicelliExecReal(SuperVector const chars, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("rverm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + + const u8 *d = buf_end; + const u8 *rv; + + __builtin_prefetch(d - 64); + __builtin_prefetch(d - 2*64); + __builtin_prefetch(d - 3*64); + __builtin_prefetch(d - 4*64); + DEBUG_PRINTF("start %p end %p \n", buf, d); + assert(d > buf); + if (d - S >= buf) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d - S); + rv = rvermicelliSingleBlock(data, chars, casemask, d - S); + DEBUG_PRINTF("rv %p \n", rv); + if (rv) return rv; + d = ROUNDDOWN_PTR(d, S); + } + + while (d - S >= buf) { + DEBUG_PRINTF("aligned %p \n", d); + // On large packet buffers, this prefetch appears to get us about 2%. + __builtin_prefetch(d - 64); + + d -= S; + SuperVector data = SuperVector::load(d); + rv = rvermicelliSingleBlock(data, chars, casemask, d); + if (rv) return rv; + } + } + + DEBUG_PRINTF("tail d %p e %p \n", buf, d); + // finish off head + + if (d != buf) { + SuperVector data = SuperVector::loadu(buf); + rv = rvermicelliSingleBlock(data, chars, casemask, buf); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } + + return buf - 1; +} + +// Reverse vermicelli scan. Provides exact semantics and returns (buf - 1) if +// character not found. +template +const u8 *rnvermicelliExecReal(SuperVector const chars, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("rverm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + + const u8 *d = buf_end; + const u8 *rv; + + __builtin_prefetch(d - 64); + __builtin_prefetch(d - 2*64); + __builtin_prefetch(d - 3*64); + __builtin_prefetch(d - 4*64); + DEBUG_PRINTF("start %p end %p \n", buf, d); + assert(d > buf); + if (d - S >= buf) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d - S); + rv = rvermicelliSingleBlockNeg(data, chars, casemask, d - S); + DEBUG_PRINTF("rv %p \n", rv); + if (rv) return rv; + d = ROUNDDOWN_PTR(d, S); + } + + while (d - S >= buf) { + DEBUG_PRINTF("aligned %p \n", d); + // On large packet buffers, this prefetch appears to get us about 2%. + __builtin_prefetch(d - 64); + + d -= S; + SuperVector data = SuperVector::load(d); + rv = rvermicelliSingleBlockNeg(data, chars, casemask, d); + if (rv) return rv; + } + } + + DEBUG_PRINTF("tail d %p e %p \n", buf, d); + // finish off head + + if (d != buf) { + SuperVector data = SuperVector::loadu(buf); + rv = rvermicelliSingleBlockNeg(data, chars, casemask, buf); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } + + return buf - 1; +} + +template +static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector const casemask, + const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("verm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + + const u8 *d = buf; + const u8 *rv; + // SuperVector lastmask1{0}; + const SuperVector chars1 = SuperVector::dup_u8(c1); + const SuperVector chars2 = SuperVector::dup_u8(c2); + const u8 casechar = casemask.u.u8[0]; + + __builtin_prefetch(d + 64); + __builtin_prefetch(d + 2*64); + __builtin_prefetch(d + 3*64); + __builtin_prefetch(d + 4*64); + DEBUG_PRINTF("start %p end %p \n", d, buf_end); + assert(d < buf_end); + if (d + S <= buf_end) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, &lastmask1); + if (rv) return rv; + d = ROUNDUP_PTR(d, S); + } + + while(d + S <= buf_end) { + __builtin_prefetch(d + 64); + DEBUG_PRINTF("d %p \n", d); + SuperVector data = SuperVector::load(d); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, &lastmask1); + if (rv) { + bool partial_match = (((rv[0] & casechar) == c2) && ((rv[-1] & casechar) == c1)); + return rv - partial_match; + } + d += S; + } + } + + DEBUG_PRINTF("tail d %p e %p \n", d, buf_end); + // finish off tail + + if (d != buf_end) { + SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, buf_end - d); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } + + DEBUG_PRINTF("real tail d %p e %p \n", d, buf_end); + /* check for partial match at end */ + u8 mask = casemask.u.u8[0]; + // u8 c1 = chars1.u.u8[0]; + if ((buf_end[-1] & mask) == (u8)c1) { + DEBUG_PRINTF("partial!!!\n"); + return buf_end - 1; + } + + return buf_end; +} + +// /* returns highest offset of c2 (NOTE: not c1) */ +// static really_inline +// const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, +// const u8 *buf_end) { +// DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", +// nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); +// assert(buf < buf_end); + +// VERM_TYPE chars1 = VERM_SET_FN(c1); /* nocase already uppercase */ +// VERM_TYPE chars2 = VERM_SET_FN(c2); /* nocase already uppercase */ + +// #ifdef HAVE_AVX512 +// if (buf_end - buf <= VERM_BOUNDARY) { +// const u8 *ptr = nocase +// ? rdvermMiniNocase(chars1, chars2, buf, buf_end) +// : rdvermMini(chars1, chars2, buf, buf_end); + +// if (ptr) { +// return ptr; +// } + +// // check for partial match at end ??? +// return buf - 1; +// } +// #endif + +// assert((buf_end - buf) >= VERM_BOUNDARY); +// size_t min = (size_t)buf_end % VERM_BOUNDARY; +// if (min) { +// // input not aligned, so we need to run one iteration with an unaligned +// // load, then skip buf forward to the next aligned address. There's +// // some small overlap here, but we don't mind scanning it twice if we +// // can do it quickly, do we? +// const u8 *ptr = nocase ? rdvermPreconditionNocase(chars1, chars2, +// buf_end - VERM_BOUNDARY) +// : rdvermPrecondition(chars1, chars2, +// buf_end - VERM_BOUNDARY); + +// if (ptr) { +// return ptr; +// } + +// buf_end -= min; +// if (buf >= buf_end) { +// return buf_end; +// } +// } + +// // Aligned loops from here on in +// if (nocase) { +// return rdvermSearchAlignedNocase(chars1, chars2, c1, c2, buf, buf_end); +// } else { +// return rdvermSearchAligned(chars1, chars2, c1, c2, buf, buf_end); +// } +// } + +extern "C" const u8 *vermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("verm scan %s\\x%02hhx over %zu bytes\n", + nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + const SuperVector chars = SuperVector::dup_u8(c); + const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; + + return vermicelliExecReal(chars, casemask, buf, buf_end); +} + +/* like vermicelliExec except returns the address of the first character which + * is not c */ +extern "C" const u8 *nvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("nverm scan %s\\x%02hhx over %zu bytes\n", + nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + const SuperVector chars = SuperVector::dup_u8(c); + const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; + + return nvermicelliExecReal(chars, casemask, buf, buf_end); +} + +extern "C" const u8 *rvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("rev verm scan %s\\x%02hhx over %zu bytes\n", + nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + const SuperVector chars = SuperVector::dup_u8(c); + const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; + + return rvermicelliExecReal(chars, casemask, buf, buf_end); +} + +extern "C" const u8 *rnvermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("rev verm scan %s\\x%02hhx over %zu bytes\n", + nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + const SuperVector chars = SuperVector::dup_u8(c); + const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; + + return rnvermicelliExecReal(chars, casemask, buf, buf_end); +} + +extern "C" const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", + nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; + + return vermicelliDoubleExecReal(c1, c2, casemask, buf, buf_end); +} \ No newline at end of file From 4a569affbccd4d2f9b90fd917d422d2141054fbe Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 27 Oct 2021 12:29:59 +0300 Subject: [PATCH 12/25] add to CMake --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 92abf6dc..8d4af1fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -619,6 +619,7 @@ set (hs_exec_SRCS src/nfa/vermicelli.h src/nfa/vermicelli_run.h src/nfa/vermicelli_sse.h + src/nfa/vermicelli_simd.cpp src/som/som.h src/som/som_operation.h src/som/som_runtime.h From 713aaef799af3f4a9b8f50fa1f3a435540d94eff Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 27 Oct 2021 12:30:42 +0300 Subject: [PATCH 13/25] move casemask helper functions to separate header --- src/hwlm/noodle_engine_simd.hpp | 21 +----------- src/util/supervector/casemask.hpp | 54 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 20 deletions(-) create mode 100644 src/util/supervector/casemask.hpp diff --git a/src/hwlm/noodle_engine_simd.hpp b/src/hwlm/noodle_engine_simd.hpp index d5f6a8d0..dfe7eea1 100644 --- a/src/hwlm/noodle_engine_simd.hpp +++ b/src/hwlm/noodle_engine_simd.hpp @@ -30,26 +30,7 @@ /* SIMD engine agnostic noodle scan parts */ #include "util/supervector/supervector.hpp" - -static u8 CASEMASK[] = { 0xff, 0xdf }; - -static really_inline -u8 caseClear8(u8 x, bool noCase) -{ - return static_cast(x & CASEMASK[(u8)noCase]); -} - -template -static really_inline SuperVector getMask(u8 c, bool noCase) { - u8 k = caseClear8(c, noCase); - return SuperVector(k); -} - -template -static really_inline SuperVector getCaseMask(void) { - return SuperVector(CASEMASK[1]); -} - +#include "util/supervector/casemask.hpp" static really_really_inline hwlm_error_t single_zscan(const struct noodTable *n,const u8 *d, const u8 *buf, diff --git a/src/util/supervector/casemask.hpp b/src/util/supervector/casemask.hpp new file mode 100644 index 00000000..10fa5f1a --- /dev/null +++ b/src/util/supervector/casemask.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef CASEMASK_HPP +#define CASEMASK_HPP + +#include "util/supervector/supervector.hpp" + +static u8 CASEMASK[] = { 0xff, 0xdf }; + +static really_inline +u8 caseClear8(u8 x, bool noCase) +{ + return static_cast(x & CASEMASK[(u8)noCase]); +} + +template +static really_inline SuperVector getMask(u8 c, bool noCase) { + u8 k = caseClear8(c, noCase); + return SuperVector(k); +} + +template +static really_inline SuperVector getCaseMask(void) { + return SuperVector(CASEMASK[1]); +} + +#endif // CASEMASK_HPP \ No newline at end of file From 16e5e2ae646f875b60272775286b93c30249d3f5 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 27 Oct 2021 12:31:04 +0300 Subject: [PATCH 14/25] nits --- src/nfa/arm/shufti.hpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/nfa/arm/shufti.hpp b/src/nfa/arm/shufti.hpp index 76461175..e710fd16 100644 --- a/src/nfa/arm/shufti.hpp +++ b/src/nfa/arm/shufti.hpp @@ -1,7 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation * Copyright (c) 2020-2021, VectorCamp PC - * Copyright (c) 2021, Arm Limited * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -30,7 +29,6 @@ /** \file * \brief Shufti: character class acceleration. - * */ template @@ -73,4 +71,4 @@ SuperVector blockDoubleMask(SuperVector mask1_lo, SuperVector mask1_hi, t.print8("t"); return !t.eq(SuperVector::Ones()); -} +} \ No newline at end of file From 5eabceddcfffcf5e312721e3d9a98c1f8b130c8e Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 27 Oct 2021 12:32:03 +0300 Subject: [PATCH 15/25] renamed matcher functions, added new ones for Vermicelli --- src/nfa/shufti_simd.hpp | 6 +-- src/nfa/truffle_simd.hpp | 4 +- src/nfa/x86/shufti.hpp | 16 ++---- src/util/arch/arm/match.hpp | 41 +++++++++++++- src/util/arch/x86/match.hpp | 103 +++++++++++++++++++++++++++++++++--- src/util/match.hpp | 10 +++- 6 files changed, 152 insertions(+), 28 deletions(-) diff --git a/src/nfa/shufti_simd.hpp b/src/nfa/shufti_simd.hpp index e7f3f6c9..09850c00 100644 --- a/src/nfa/shufti_simd.hpp +++ b/src/nfa/shufti_simd.hpp @@ -63,7 +63,7 @@ static really_inline const u8 *fwdBlock(SuperVector mask_lo, SuperVector mask_hi, SuperVector chars, const u8 *buf) { SuperVector v = blockSingleMask(mask_lo, mask_hi, chars); - return firstMatch(buf, v); + return first_zero_match_inverted(buf, v); } template @@ -71,7 +71,7 @@ static really_inline const u8 *revBlock(SuperVector mask_lo, SuperVector mask_hi, SuperVector chars, const u8 *buf) { SuperVector v = blockSingleMask(mask_lo, mask_hi, chars); - return lastMatch(buf, v); + return last_zero_match_inverted(buf, v); } template @@ -80,7 +80,7 @@ const u8 *fwdBlockDouble(SuperVector mask1_lo, SuperVector mask1_hi, Super SuperVector mask = blockDoubleMask(mask1_lo, mask1_hi, mask2_lo, mask2_hi, chars); - return firstMatch(buf, mask); + return first_zero_match_inverted(buf, mask); } template diff --git a/src/nfa/truffle_simd.hpp b/src/nfa/truffle_simd.hpp index 8d61722b..13a5e787 100644 --- a/src/nfa/truffle_simd.hpp +++ b/src/nfa/truffle_simd.hpp @@ -56,7 +56,7 @@ static really_inline const u8 *fwdBlock(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector chars, const u8 *buf) { SuperVector res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars); - return firstMatch(buf, res); + return first_zero_match_inverted(buf, res); } template @@ -120,7 +120,7 @@ static really_inline const u8 *revBlock(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector v, const u8 *buf) { SuperVector res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, v); - return lastMatch(buf, res); + return last_zero_match_inverted(buf, res); } template diff --git a/src/nfa/x86/shufti.hpp b/src/nfa/x86/shufti.hpp index 79ef7481..6fb34b2f 100644 --- a/src/nfa/x86/shufti.hpp +++ b/src/nfa/x86/shufti.hpp @@ -31,12 +31,6 @@ * \brief Shufti: character class acceleration. */ -#ifndef SHUFTI_SIMD_X86_HPP -#define SHUFTI_SIMD_X86_HPP - -#include "util/supervector/supervector.hpp" -#include "util/match.hpp" - template static really_inline const SuperVector blockSingleMask(SuperVector mask_lo, SuperVector mask_hi, SuperVector chars) { @@ -44,12 +38,10 @@ const SuperVector blockSingleMask(SuperVector mask_lo, SuperVector mask SuperVector c_lo = chars & low4bits; SuperVector c_hi = chars.template vshr_64_imm<4>() & low4bits; - c_lo = mask_lo.template pshufb(c_lo); - c_hi = mask_hi.template pshufb(c_hi); + c_lo = mask_lo.pshufb(c_lo); + c_hi = mask_hi.pshufb(c_hi); - SuperVector c = c_lo & c_hi; - - return c.eq(SuperVector::Zeroes()); + return (c_lo & c_hi).eq(SuperVector::Zeroes()); } template @@ -80,5 +72,3 @@ SuperVector blockDoubleMask(SuperVector mask1_lo, SuperVector mask1_hi, return c.eq(SuperVector::Ones()); } - -#endif // SHUFTI_SIMD_X86_HPP diff --git a/src/util/arch/arm/match.hpp b/src/util/arch/arm/match.hpp index e7f757bd..ba5f797f 100644 --- a/src/util/arch/arm/match.hpp +++ b/src/util/arch/arm/match.hpp @@ -29,7 +29,44 @@ template <> really_really_inline -const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> mask) { +const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> mask) { + uint32x4_t m = mask.u.u32x4[0]; + uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0); + if (vmax != 0) { + typename SuperVector<16>::movemask_type z = mask.movemask(); + DEBUG_PRINTF("z %08x\n", z); + DEBUG_PRINTF("buf %p z %08x \n", buf, z); + u32 pos = ctz32(z & 0xffff); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos < 16); + DEBUG_PRINTF("buf + pos %p\n", buf + pos); + return buf + pos; + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> mask) { + uint32x4_t m = mask.u.u32x4[0]; + uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0); + if (vmax != 0) { + typename SuperVector<16>::movemask_type z = mask.movemask(); + DEBUG_PRINTF("buf %p z %08x \n", buf, z); + DEBUG_PRINTF("z %08x\n", z); + u32 pos = clz32(z & 0xffff); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos >= 16 && pos < 32); + return buf + (31 - pos); + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask) { uint32x4_t m = mask.u.u32x4[0]; uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0); if (vmax != 0) { @@ -48,7 +85,7 @@ const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> mask) { template <> really_really_inline -const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> mask) { +const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask) { uint32x4_t m = mask.u.u32x4[0]; uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0); if (vmax != 0) { diff --git a/src/util/arch/x86/match.hpp b/src/util/arch/x86/match.hpp index 159f7355..26283ca7 100644 --- a/src/util/arch/x86/match.hpp +++ b/src/util/arch/x86/match.hpp @@ -29,7 +29,98 @@ template <> really_really_inline -const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> v) { +const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> v) { + SuperVector<16>::movemask_type z = v.movemask(); + DEBUG_PRINTF("buf %p z %08x \n", buf, z); + DEBUG_PRINTF("z %08x\n", z); + if (unlikely(z)) { + u32 pos = ctz32(z); + DEBUG_PRINTF("~z %08x\n", ~z); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos < 16); + return buf + pos; + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *first_non_zero_match<32>(const u8 *buf, SuperVector<32> v) { + SuperVector<32>::movemask_type z = v.movemask(); + DEBUG_PRINTF("z 0x%08x\n", z); + if (unlikely(z)) { + u32 pos = ctz32(z); + assert(pos < 32); + DEBUG_PRINTF("match @ pos %u\n", pos); + return buf + pos; + } else { + return NULL; // no match + } +} +template <> +really_really_inline +const u8 *first_non_zero_match<64>(const u8 *buf, SuperVector<64>v) { + SuperVector<64>::movemask_type z = v.movemask(); + DEBUG_PRINTF("z 0x%016llx\n", z); + if (unlikely(z)) { + u32 pos = ctz64(z); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos < 64); + return buf + pos; + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> v) { + SuperVector<16>::movemask_type z = v.movemask(); + DEBUG_PRINTF("buf %p z %08x \n", buf, z); + DEBUG_PRINTF("z %08x\n", z); + if (unlikely(z)) { + u32 pos = clz32(z); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos >= 16 && pos < 32); + return buf + (31 - pos); + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *last_non_zero_match<32>(const u8 *buf, SuperVector<32> v) { + SuperVector<32>::movemask_type z = v.movemask(); + DEBUG_PRINTF("z 0x%08x\n", z); + if (unlikely(z)) { + u32 pos = clz32(z); + assert(pos < 32); + DEBUG_PRINTF("match @ pos %u\n", pos); + return buf + (31 - pos); + } else { + return NULL; // no match + } +} +template <> +really_really_inline +const u8 *last_non_zero_match<64>(const u8 *buf, SuperVector<64>v) { + SuperVector<64>::movemask_type z = v.movemask(); + DEBUG_PRINTF("z 0x%016llx\n", z); + if (unlikely(z)) { + u32 pos = clz64(z); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos < 64); + return buf + (31 - pos); + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) { SuperVector<16>::movemask_type z = v.movemask(); DEBUG_PRINTF("buf %p z %08x \n", buf, z); DEBUG_PRINTF("z %08x\n", z); @@ -46,7 +137,7 @@ const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> v) { template <> really_really_inline -const u8 *firstMatch<32>(const u8 *buf, SuperVector<32> v) { +const u8 *first_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) { SuperVector<32>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%08x\n", z); if (unlikely(z != 0xffffffff)) { @@ -60,7 +151,7 @@ const u8 *firstMatch<32>(const u8 *buf, SuperVector<32> v) { } template <> really_really_inline -const u8 *firstMatch<64>(const u8 *buf, SuperVector<64>v) { +const u8 *first_zero_match_inverted<64>(const u8 *buf, SuperVector<64>v) { SuperVector<64>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%016llx\n", z); if (unlikely(z != ~0ULL)) { @@ -75,7 +166,7 @@ const u8 *firstMatch<64>(const u8 *buf, SuperVector<64>v) { template <> really_really_inline -const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> v) { +const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) { SuperVector<16>::movemask_type z = v.movemask(); DEBUG_PRINTF("buf %p z %08x \n", buf, z); DEBUG_PRINTF("z %08x\n", z); @@ -92,7 +183,7 @@ const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> v) { template<> really_really_inline -const u8 *lastMatch<32>(const u8 *buf, SuperVector<32> v) { +const u8 *last_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) { SuperVector<32>::movemask_type z = v.movemask(); if (unlikely(z != 0xffffffff)) { u32 pos = clz32(~z); @@ -106,7 +197,7 @@ const u8 *lastMatch<32>(const u8 *buf, SuperVector<32> v) { template <> really_really_inline -const u8 *lastMatch<64>(const u8 *buf, SuperVector<64> v) { +const u8 *last_zero_match_inverted<64>(const u8 *buf, SuperVector<64> v) { SuperVector<64>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%016llx\n", z); if (unlikely(z != ~0ULL)) { diff --git a/src/util/match.hpp b/src/util/match.hpp index 9331d1f8..9b3c8fb9 100644 --- a/src/util/match.hpp +++ b/src/util/match.hpp @@ -38,10 +38,16 @@ #include "util/supervector/supervector.hpp" template -const u8 *firstMatch(const u8 *buf, SuperVector v); +const u8 *first_non_zero_match(const u8 *buf, SuperVector v); template -const u8 *lastMatch(const u8 *buf, SuperVector v); +const u8 *last_non_zero_match(const u8 *buf, SuperVector v); + +template +const u8 *first_zero_match_inverted(const u8 *buf, SuperVector v); + +template +const u8 *last_zero_match_inverted(const u8 *buf, SuperVector v); #if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/arch/x86/match.hpp" From bc1a1127cff619442b5ea3f1a61a8daabafcb049 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 27 Oct 2021 12:32:54 +0300 Subject: [PATCH 16/25] add new include file --- src/hwlm/hwlm.c | 1 + src/nfa/accel.c | 1 + src/nfa/castle.c | 1 + src/nfa/lbr.c | 1 + src/nfa/nfa_rev_api.h | 1 + src/nfa/vermicelli_run.h | 1 + unit/internal/rvermicelli.cpp | 3 ++- unit/internal/vermicelli.cpp | 3 ++- 8 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/hwlm/hwlm.c b/src/hwlm/hwlm.c index c1c2837f..5d69e3c4 100644 --- a/src/hwlm/hwlm.c +++ b/src/hwlm/hwlm.c @@ -40,6 +40,7 @@ #include "nfa/shufti.h" #include "nfa/truffle.h" #include "nfa/vermicelli.h" +#include "nfa/vermicelli.hpp" #include #define MIN_ACCEL_LEN_BLOCK 16 diff --git a/src/nfa/accel.c b/src/nfa/accel.c index 34bd24a9..b35e0633 100644 --- a/src/nfa/accel.c +++ b/src/nfa/accel.c @@ -31,6 +31,7 @@ #include "shufti.h" #include "truffle.h" #include "vermicelli.h" +#include "vermicelli.hpp" #include "ue2common.h" const u8 *run_accel(const union AccelAux *accel, const u8 *c, const u8 *c_end) { diff --git a/src/nfa/castle.c b/src/nfa/castle.c index c7dd6d50..be29ca29 100644 --- a/src/nfa/castle.c +++ b/src/nfa/castle.c @@ -41,6 +41,7 @@ #include "shufti.h" #include "truffle.h" #include "vermicelli.h" +#include "vermicelli.hpp" #include "util/bitutils.h" #include "util/multibit.h" #include "util/partial_store.h" diff --git a/src/nfa/lbr.c b/src/nfa/lbr.c index 68e8e3f4..8fc83988 100644 --- a/src/nfa/lbr.c +++ b/src/nfa/lbr.c @@ -41,6 +41,7 @@ #include "shufti.h" #include "truffle.h" #include "vermicelli.h" +#include "vermicelli.hpp" #include "util/partial_store.h" #include "util/unaligned.h" diff --git a/src/nfa/nfa_rev_api.h b/src/nfa/nfa_rev_api.h index 370f96ef..72224c3b 100644 --- a/src/nfa/nfa_rev_api.h +++ b/src/nfa/nfa_rev_api.h @@ -36,6 +36,7 @@ #include "accel.h" #include "nfa_internal.h" #include "vermicelli.h" +#include "vermicelli.hpp" #include "util/unaligned.h" static really_inline diff --git a/src/nfa/vermicelli_run.h b/src/nfa/vermicelli_run.h index d6fe7ec7..b75f1414 100644 --- a/src/nfa/vermicelli_run.h +++ b/src/nfa/vermicelli_run.h @@ -27,6 +27,7 @@ */ #include "vermicelli.h" +#include "vermicelli.hpp" static really_inline const u8 *find_xverm_run(char c, char nocase, u32 repeat, UNUSED const u8 *buf, diff --git a/unit/internal/rvermicelli.cpp b/unit/internal/rvermicelli.cpp index d89067d0..d29b1133 100644 --- a/unit/internal/rvermicelli.cpp +++ b/unit/internal/rvermicelli.cpp @@ -31,6 +31,7 @@ #include "gtest/gtest.h" #include "nfa/vermicelli.h" +#include "nfa/vermicelli.hpp" #define BOUND (~(VERM_BOUNDARY - 1)) @@ -563,4 +564,4 @@ TEST(RNVermicelli16, Exec5) { } } -#endif // HAVE_SVE2 \ No newline at end of file +#endif // HAVE_SVE2 diff --git a/unit/internal/vermicelli.cpp b/unit/internal/vermicelli.cpp index dc458cb9..3319b87c 100644 --- a/unit/internal/vermicelli.cpp +++ b/unit/internal/vermicelli.cpp @@ -31,6 +31,7 @@ #include "gtest/gtest.h" #include "nfa/vermicelli.h" +#include "nfa/vermicelli.hpp" TEST(Vermicelli, ExecNoMatch1) { char t1[] = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; @@ -1150,4 +1151,4 @@ TEST(DoubleVermicelliMasked16, Exec5) { } } -#endif // HAVE_SVE2 \ No newline at end of file +#endif // HAVE_SVE2 From d47641c2fc688b539b4b1bb657e1bca0ad8a2f56 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Mon, 1 Nov 2021 16:50:38 +0200 Subject: [PATCH 17/25] remove unneeded header --- src/nfa/vermicelli.h | 2 - src/nfa/vermicelli_sse.h | 1284 -------------------------------------- 2 files changed, 1286 deletions(-) delete mode 100644 src/nfa/vermicelli_sse.h diff --git a/src/nfa/vermicelli.h b/src/nfa/vermicelli.h index 9defd899..39e9555e 100644 --- a/src/nfa/vermicelli.h +++ b/src/nfa/vermicelli.h @@ -44,8 +44,6 @@ #ifdef HAVE_SVE2 #include "vermicelli_sve.h" -#else -#include "vermicelli_sse.h" #endif static really_inline diff --git a/src/nfa/vermicelli_sse.h b/src/nfa/vermicelli_sse.h deleted file mode 100644 index 268e9e08..00000000 --- a/src/nfa/vermicelli_sse.h +++ /dev/null @@ -1,1284 +0,0 @@ -/* - * Copyright (c) 2015-2020, Intel Corporation - * Copyright (c) 2021, Arm Limited - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** \file - * \brief Vermicelli: Intel SSE implementation. - * - * (users should include vermicelli.h instead of this) - */ - -#if !defined(HAVE_AVX512) - -#define VERM_BOUNDARY 16 -#define VERM_TYPE m128 -#define VERM_SET_FN set1_16x8 - -static really_inline -const u8 *vermSearchAligned(m128 chars, const u8 *buf, const u8 *buf_end, - char negate) { - assert((size_t)buf % 16 == 0); - for (; buf + 31 < buf_end; buf += 32) { - m128 data = load128(buf); - u32 z1 = movemask128(eq128(chars, data)); - m128 data2 = load128(buf + 16); - u32 z2 = movemask128(eq128(chars, data2)); - u32 z = z1 | (z2 << 16); - if (negate) { - z = ~z; - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - for (; buf + 15 < buf_end; buf += 16) { - m128 data = load128(buf); - u32 z = movemask128(eq128(chars, data)); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - return NULL; -} - -static really_inline -const u8 *vermSearchAlignedNocase(m128 chars, const u8 *buf, - const u8 *buf_end, char negate) { - assert((size_t)buf % 16 == 0); - m128 casemask = set1_16x8(CASE_CLEAR); - - for (; buf + 31 < buf_end; buf += 32) { - m128 data = load128(buf); - u32 z1 = movemask128(eq128(chars, and128(casemask, data))); - m128 data2 = load128(buf + 16); - u32 z2 = movemask128(eq128(chars, and128(casemask, data2))); - u32 z = z1 | (z2 << 16); - if (negate) { - z = ~z; - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - - for (; buf + 15 < buf_end; buf += 16) { - m128 data = load128(buf); - u32 z = movemask128(eq128(chars, and128(casemask, data))); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *vermUnalign(m128 chars, const u8 *buf, char negate) { - m128 data = loadu128(buf); // unaligned - u32 z = movemask128(eq128(chars, data)); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *vermUnalignNocase(m128 chars, const u8 *buf, char negate) { - m128 casemask = set1_16x8(CASE_CLEAR); - m128 data = loadu128(buf); // unaligned - u32 z = movemask128(eq128(chars, and128(casemask, data))); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - -static really_inline -const u8 *lastMatchOffset(const u8 *buf_end, u32 z) { - assert(z); - return buf_end - 16 + 31 - clz32(z); -} - -static really_inline -const u8 *rvermSearchAligned(m128 chars, const u8 *buf, const u8 *buf_end, - char negate) { - assert((size_t)buf_end % 16 == 0); - for (; buf + 15 < buf_end; buf_end -= 16) { - m128 data = load128(buf_end - 16); - u32 z = movemask128(eq128(chars, data)); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf_end, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - return NULL; -} - -static really_inline -const u8 *rvermSearchAlignedNocase(m128 chars, const u8 *buf, - const u8 *buf_end, char negate) { - assert((size_t)buf_end % 16 == 0); - m128 casemask = set1_16x8(CASE_CLEAR); - - for (; buf + 15 < buf_end; buf_end -= 16) { - m128 data = load128(buf_end - 16); - u32 z = movemask128(eq128(chars, and128(casemask, data))); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf_end, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *rvermUnalign(m128 chars, const u8 *buf, char negate) { - m128 data = loadu128(buf); // unaligned - u32 z = movemask128(eq128(chars, data)); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf + 16, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *rvermUnalignNocase(m128 chars, const u8 *buf, char negate) { - m128 casemask = set1_16x8(CASE_CLEAR); - m128 data = loadu128(buf); // unaligned - u32 z = movemask128(eq128(chars, and128(casemask, data))); - if (negate) { - z = ~z & 0xffff; - } - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf + 16, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - -static really_inline -const u8 *dvermSearchAligned(m128 chars1, m128 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - for (; buf + 16 < buf_end; buf += 16) { - m128 data = load128(buf); - u32 z = movemask128(and128(eq128(chars1, data), - rshiftbyte_m128(eq128(chars2, data), 1))); - if (buf[15] == c1 && buf[16] == c2) { - z |= (1 << 15); - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - - return NULL; -} - -static really_inline -const u8 *dvermSearchAlignedNocase(m128 chars1, m128 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - assert((size_t)buf % 16 == 0); - m128 casemask = set1_16x8(CASE_CLEAR); - - for (; buf + 16 < buf_end; buf += 16) { - m128 data = load128(buf); - m128 v = and128(casemask, data); - u32 z = movemask128(and128(eq128(chars1, v), - rshiftbyte_m128(eq128(chars2, v), 1))); - if ((buf[15] & CASE_CLEAR) == c1 && (buf[16] & CASE_CLEAR) == c2) { - z |= (1 << 15); - } - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *dvermPrecondition(m128 chars1, m128 chars2, const u8 *buf) { - m128 data = loadu128(buf); // unaligned - u32 z = movemask128(and128(eq128(chars1, data), - rshiftbyte_m128(eq128(chars2, data), 1))); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *dvermPreconditionNocase(m128 chars1, m128 chars2, const u8 *buf) { - /* due to laziness, nonalphas and nocase having interesting behaviour */ - m128 casemask = set1_16x8(CASE_CLEAR); - m128 data = loadu128(buf); // unaligned - m128 v = and128(casemask, data); - u32 z = movemask128(and128(eq128(chars1, v), - rshiftbyte_m128(eq128(chars2, v), 1))); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - const u8 *matchPos = buf + ctz32(z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - - -static really_inline -const u8 *rdvermSearchAligned(m128 chars1, m128 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - assert((size_t)buf_end % 16 == 0); - - for (; buf + 16 < buf_end; buf_end -= 16) { - m128 data = load128(buf_end - 16); - u32 z = movemask128(and128(eq128(chars2, data), - lshiftbyte_m128(eq128(chars1, data), 1))); - if (buf_end[-17] == c1 && buf_end[-16] == c2) { - z |= 1; - } - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf_end, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - return buf_end; -} - -static really_inline -const u8 *rdvermSearchAlignedNocase(m128 chars1, m128 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - assert((size_t)buf_end % 16 == 0); - m128 casemask = set1_16x8(CASE_CLEAR); - - for (; buf + 16 < buf_end; buf_end -= 16) { - m128 data = load128(buf_end - 16); - m128 v = and128(casemask, data); - u32 z = movemask128(and128(eq128(chars2, v), - lshiftbyte_m128(eq128(chars1, v), 1))); - if ((buf_end[-17] & CASE_CLEAR) == c1 - && (buf_end[-16] & CASE_CLEAR) == c2) { - z |= 1; - } - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf_end, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - } - return buf_end; -} - -// returns NULL if not found -static really_inline -const u8 *rdvermPrecondition(m128 chars1, m128 chars2, const u8 *buf) { - m128 data = loadu128(buf); - u32 z = movemask128(and128(eq128(chars2, data), - lshiftbyte_m128(eq128(chars1, data), 1))); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf + 16, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *rdvermPreconditionNocase(m128 chars1, m128 chars2, const u8 *buf) { - /* due to laziness, nonalphas and nocase having interesting behaviour */ - m128 casemask = set1_16x8(CASE_CLEAR); - m128 data = loadu128(buf); - m128 v = and128(casemask, data); - u32 z = movemask128(and128(eq128(chars2, v), - lshiftbyte_m128(eq128(chars1, v), 1))); - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - const u8 *matchPos = lastMatchOffset(buf + 16, z); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - - return NULL; -} - -#else // HAVE_AVX512 - -#define VERM_BOUNDARY 64 -#define VERM_TYPE m512 -#define VERM_SET_FN set1_64x8 - -static really_inline -const u8 *vermMini(m512 chars, const u8 *buf, const u8 *buf_end, char negate) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - - u64a z = eq512mask(chars, data); - - if (negate) { - z = ~z & mask; - } - z &= mask; - if (unlikely(z)) { - return buf + ctz64(z); - } - return NULL; -} - -static really_inline -const u8 *vermMiniNocase(m512 chars, const u8 *buf, const u8 *buf_end, - char negate) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - m512 casemask = set1_64x8(CASE_CLEAR); - m512 v = and512(casemask, data); - - u64a z = eq512mask(chars, v); - - if (negate) { - z = ~z & mask; - } - z &= mask; - if (unlikely(z)) { - return buf + ctz64(z); - } - return NULL; -} - -static really_inline -const u8 *vermSearchAligned(m512 chars, const u8 *buf, const u8 *buf_end, - char negate) { - assert((size_t)buf % 64 == 0); - for (; buf + 63 < buf_end; buf += 64) { - m512 data = load512(buf); - u64a z = eq512mask(chars, data); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - } - return NULL; -} - -static really_inline -const u8 *vermSearchAlignedNocase(m512 chars, const u8 *buf, - const u8 *buf_end, char negate) { - assert((size_t)buf % 64 == 0); - m512 casemask = set1_64x8(CASE_CLEAR); - - for (; buf + 63 < buf_end; buf += 64) { - m512 data = load512(buf); - u64a z = eq512mask(chars, and512(casemask, data)); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *vermUnalign(m512 chars, const u8 *buf, char negate) { - m512 data = loadu512(buf); // unaligned - u64a z = eq512mask(chars, data); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - return buf + ctz64(z); - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *vermUnalignNocase(m512 chars, const u8 *buf, char negate) { - m512 casemask = set1_64x8(CASE_CLEAR); - m512 data = loadu512(buf); // unaligned - u64a z = eq512mask(chars, and512(casemask, data)); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - return buf + ctz64(z); - } - return NULL; -} - -static really_inline -const u8 *dvermMini(m512 chars1, m512 chars2, const u8 *buf, - const u8 *buf_end) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - - u64a z = eq512mask(chars1, data) & (eq512mask(chars2, data) >> 1); - - z &= mask; - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - return NULL; -} - -static really_inline -const u8 *dvermMiniNocase(m512 chars1, m512 chars2, const u8 *buf, - const u8 *buf_end) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - m512 casemask = set1_64x8(CASE_CLEAR); - m512 v = and512(casemask, data); - - u64a z = eq512mask(chars1, v) & (eq512mask(chars2, v) >> 1); - - z &= mask; - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - return NULL; -} - -static really_inline -const u8 *dvermMiniMasked(m512 chars1, m512 chars2, m512 mask1, m512 mask2, - const u8 *buf, const u8 *buf_end) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - m512 v1 = and512(data, mask1); - m512 v2 = and512(data, mask2); - - u64a z = eq512mask(chars1, v1) & (eq512mask(chars2, v2) >> 1); - - z &= mask; - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - return NULL; -} - -static really_inline -const u8 *dvermSearchAligned(m512 chars1, m512 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - for (; buf + 64 < buf_end; buf += 64) { - m512 data = load512(buf); - u64a z = eq512mask(chars1, data) & (eq512mask(chars2, data) >> 1); - if (buf[63] == c1 && buf[64] == c2) { - z |= (1ULL << 63); - } - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - } - - return NULL; -} - -static really_inline -const u8 *dvermSearchAlignedNocase(m512 chars1, m512 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - assert((size_t)buf % 64 == 0); - m512 casemask = set1_64x8(CASE_CLEAR); - - for (; buf + 64 < buf_end; buf += 64) { - m512 data = load512(buf); - m512 v = and512(casemask, data); - u64a z = eq512mask(chars1, v) & (eq512mask(chars2, v) >> 1); - if ((buf[63] & CASE_CLEAR) == c1 && (buf[64] & CASE_CLEAR) == c2) { - z |= (1ULL << 63); - } - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - } - - return NULL; -} - -static really_inline -const u8 *dvermSearchAlignedMasked(m512 chars1, m512 chars2, - m512 mask1, m512 mask2, u8 c1, u8 c2, u8 m1, - u8 m2, const u8 *buf, const u8 *buf_end) { - assert((size_t)buf % 64 == 0); - - for (; buf + 64 < buf_end; buf += 64) { - m512 data = load512(buf); - m512 v1 = and512(data, mask1); - m512 v2 = and512(data, mask2); - u64a z = eq512mask(chars1, v1) & (eq512mask(chars2, v2) >> 1); - - if ((buf[63] & m1) == c1 && (buf[64] & m2) == c2) { - z |= (1ULL << 63); - } - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - } - - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *dvermPrecondition(m512 chars1, m512 chars2, const u8 *buf) { - m512 data = loadu512(buf); // unaligned - u64a z = eq512mask(chars1, data) & (eq512mask(chars2, data) >> 1); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *dvermPreconditionNocase(m512 chars1, m512 chars2, const u8 *buf) { - /* due to laziness, nonalphas and nocase having interesting behaviour */ - m512 casemask = set1_64x8(CASE_CLEAR); - m512 data = loadu512(buf); // unaligned - m512 v = and512(casemask, data); - u64a z = eq512mask(chars1, v) & (eq512mask(chars2, v) >> 1); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *dvermPreconditionMasked(m512 chars1, m512 chars2, - m512 mask1, m512 mask2, const u8 *buf) { - m512 data = loadu512(buf); // unaligned - m512 v1 = and512(data, mask1); - m512 v2 = and512(data, mask2); - u64a z = eq512mask(chars1, v1) & (eq512mask(chars2, v2) >> 1); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - u64a pos = ctz64(z); - return buf + pos; - } - return NULL; -} - -static really_inline -const u8 *lastMatchOffset(const u8 *buf_end, u64a z) { - assert(z); - return buf_end - 64 + 63 - clz64(z); -} - -static really_inline -const u8 *rvermMini(m512 chars, const u8 *buf, const u8 *buf_end, char negate) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - - u64a z = eq512mask(chars, data); - - if (negate) { - z = ~z & mask; - } - z &= mask; - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - return NULL; -} - -static really_inline -const u8 *rvermMiniNocase(m512 chars, const u8 *buf, const u8 *buf_end, - char negate) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - m512 casemask = set1_64x8(CASE_CLEAR); - m512 v = and512(casemask, data); - - u64a z = eq512mask(chars, v); - - if (negate) { - z = ~z & mask; - } - z &= mask; - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - return NULL; -} - -static really_inline -const u8 *rvermSearchAligned(m512 chars, const u8 *buf, const u8 *buf_end, - char negate) { - assert((size_t)buf_end % 64 == 0); - for (; buf + 63 < buf_end; buf_end -= 64) { - m512 data = load512(buf_end - 64); - u64a z = eq512mask(chars, data); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - return lastMatchOffset(buf_end, z); - } - } - return NULL; -} - -static really_inline -const u8 *rvermSearchAlignedNocase(m512 chars, const u8 *buf, - const u8 *buf_end, char negate) { - assert((size_t)buf_end % 64 == 0); - m512 casemask = set1_64x8(CASE_CLEAR); - - for (; buf + 63 < buf_end; buf_end -= 64) { - m512 data = load512(buf_end - 64); - u64a z = eq512mask(chars, and512(casemask, data)); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - return lastMatchOffset(buf_end, z); - } - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *rvermUnalign(m512 chars, const u8 *buf, char negate) { - m512 data = loadu512(buf); // unaligned - u64a z = eq512mask(chars, data); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *rvermUnalignNocase(m512 chars, const u8 *buf, char negate) { - m512 casemask = set1_64x8(CASE_CLEAR); - m512 data = loadu512(buf); // unaligned - u64a z = eq512mask(chars, and512(casemask, data)); - if (negate) { - z = ~z & ~0ULL; - } - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - return NULL; -} - -static really_inline -const u8 *rdvermMini(m512 chars1, m512 chars2, const u8 *buf, - const u8 *buf_end) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - - u64a z = eq512mask(chars2, data) & (eq512mask(chars1, data) << 1); - - z &= mask; - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - return NULL; -} - -static really_inline -const u8 *rdvermMiniNocase(m512 chars1, m512 chars2, const u8 *buf, - const u8 *buf_end) { - uintptr_t len = buf_end - buf; - __mmask64 mask = (~0ULL) >> (64 - len); - m512 data = loadu_maskz_m512(mask, buf); - m512 casemask = set1_64x8(CASE_CLEAR); - m512 v = and512(casemask, data); - - u64a z = eq512mask(chars2, v) & (eq512mask(chars1, v) << 1); - - z &= mask; - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - return NULL; -} - -static really_inline -const u8 *rdvermSearchAligned(m512 chars1, m512 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - assert((size_t)buf_end % 64 == 0); - - for (; buf + 64 < buf_end; buf_end -= 64) { - m512 data = load512(buf_end - 64); - u64a z = eq512mask(chars2, data) & (eq512mask(chars1, data) << 1); - if (buf_end[-65] == c1 && buf_end[-64] == c2) { - z |= 1; - } - if (unlikely(z)) { - return lastMatchOffset(buf_end, z); - } - } - return buf_end; -} - -static really_inline -const u8 *rdvermSearchAlignedNocase(m512 chars1, m512 chars2, u8 c1, u8 c2, - const u8 *buf, const u8 *buf_end) { - assert((size_t)buf_end % 64 == 0); - m512 casemask = set1_64x8(CASE_CLEAR); - - for (; buf + 64 < buf_end; buf_end -= 64) { - m512 data = load512(buf_end - 64); - m512 v = and512(casemask, data); - u64a z = eq512mask(chars2, v) & (eq512mask(chars1, v) << 1); - if ((buf_end[-65] & CASE_CLEAR) == c1 - && (buf_end[-64] & CASE_CLEAR) == c2) { - z |= 1; - } - if (unlikely(z)) { - return lastMatchOffset(buf_end, z); - } - } - return buf_end; -} - -// returns NULL if not found -static really_inline -const u8 *rdvermPrecondition(m512 chars1, m512 chars2, const u8 *buf) { - m512 data = loadu512(buf); - u64a z = eq512mask(chars2, data) & (eq512mask(chars1, data) << 1); - - // no fixup of the boundary required - the aligned run will pick it up - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - - return NULL; -} - -// returns NULL if not found -static really_inline -const u8 *rdvermPreconditionNocase(m512 chars1, m512 chars2, const u8 *buf) { - // due to laziness, nonalphas and nocase having interesting behaviour - m512 casemask = set1_64x8(CASE_CLEAR); - m512 data = loadu512(buf); - m512 v = and512(casemask, data); - u64a z = eq512mask(chars2, v) & (eq512mask(chars1, v) << 1); - // no fixup of the boundary required - the aligned run will pick it up - if (unlikely(z)) { - return lastMatchOffset(buf + 64, z); - } - - return NULL; -} - -#endif // HAVE_AVX512 - -static really_inline -const u8 *vermicelliExec(char c, char nocase, const u8 *buf, - const u8 *buf_end) { - DEBUG_PRINTF("verm scan %s\\x%02hhx over %zu bytes\n", - nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */ - - // Handle small scans. -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = nocase - ? vermMiniNocase(chars, buf, buf_end, 0) - : vermMini(chars, buf, buf_end, 0); - if (ptr) { - return ptr; - } - return buf_end; - } -#else - if (buf_end - buf < VERM_BOUNDARY) { - for (; buf < buf_end; buf++) { - char cur = (char)*buf; - if (nocase) { - cur &= CASE_CLEAR; - } - if (cur == c) { - break; - } - } - return buf; - } -#endif - - uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY; - if (min) { - // Input isn't aligned, so we need to run one iteration with an - // unaligned load, then skip buf forward to the next aligned address. - // There's some small overlap here, but we don't mind scanning it twice - // if we can do it quickly, do we? - const u8 *ptr = nocase ? vermUnalignNocase(chars, buf, 0) - : vermUnalign(chars, buf, 0); - if (ptr) { - return ptr; - } - - buf += VERM_BOUNDARY - min; - assert(buf < buf_end); - } - - // Aligned loops from here on in - const u8 *ptr = nocase ? vermSearchAlignedNocase(chars, buf, buf_end - 1, 0) - : vermSearchAligned(chars, buf, buf_end - 1, 0); - if (ptr) { - return ptr; - } - - // Tidy up the mess at the end - ptr = nocase ? vermUnalignNocase(chars, buf_end - VERM_BOUNDARY, 0) - : vermUnalign(chars, buf_end - VERM_BOUNDARY, 0); - return ptr ? ptr : buf_end; -} - -/* like vermicelliExec except returns the address of the first character which - * is not c */ -static really_inline -const u8 *nvermicelliExec(char c, char nocase, const u8 *buf, - const u8 *buf_end) { - DEBUG_PRINTF("nverm scan %s\\x%02hhx over %zu bytes\n", - nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */ - - // Handle small scans. -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = nocase - ? vermMiniNocase(chars, buf, buf_end, 1) - : vermMini(chars, buf, buf_end, 1); - if (ptr) { - return ptr; - } - return buf_end; - } -#else - if (buf_end - buf < VERM_BOUNDARY) { - for (; buf < buf_end; buf++) { - char cur = (char)*buf; - if (nocase) { - cur &= CASE_CLEAR; - } - if (cur != c) { - break; - } - } - return buf; - } -#endif - - size_t min = (size_t)buf % VERM_BOUNDARY; - if (min) { - // Input isn't aligned, so we need to run one iteration with an - // unaligned load, then skip buf forward to the next aligned address. - // There's some small overlap here, but we don't mind scanning it twice - // if we can do it quickly, do we? - const u8 *ptr = nocase ? vermUnalignNocase(chars, buf, 1) - : vermUnalign(chars, buf, 1); - if (ptr) { - return ptr; - } - - buf += VERM_BOUNDARY - min; - assert(buf < buf_end); - } - - // Aligned loops from here on in - const u8 *ptr = nocase ? vermSearchAlignedNocase(chars, buf, buf_end - 1, 1) - : vermSearchAligned(chars, buf, buf_end - 1, 1); - if (ptr) { - return ptr; - } - - // Tidy up the mess at the end - ptr = nocase ? vermUnalignNocase(chars, buf_end - VERM_BOUNDARY, 1) - : vermUnalign(chars, buf_end - VERM_BOUNDARY, 1); - return ptr ? ptr : buf_end; -} - -// Reverse vermicelli scan. Provides exact semantics and returns (buf - 1) if -// character not found. -static really_inline -const u8 *rvermicelliExec(char c, char nocase, const u8 *buf, - const u8 *buf_end) { - DEBUG_PRINTF("rev verm scan %s\\x%02hhx over %zu bytes\n", - nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */ - - // Handle small scans. -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = nocase - ? rvermMiniNocase(chars, buf, buf_end, 0) - : rvermMini(chars, buf, buf_end, 0); - if (ptr) { - return ptr; - } - return buf - 1; - } -#else - if (buf_end - buf < VERM_BOUNDARY) { - for (buf_end--; buf_end >= buf; buf_end--) { - char cur = (char)*buf_end; - if (nocase) { - cur &= CASE_CLEAR; - } - if (cur == c) { - break; - } - } - return buf_end; - } -#endif - - size_t min = (size_t)buf_end % VERM_BOUNDARY; - if (min) { - // Input isn't aligned, so we need to run one iteration with an - // unaligned load, then skip buf backward to the next aligned address. - // There's some small overlap here, but we don't mind scanning it twice - // if we can do it quickly, do we? - const u8 *ptr = nocase ? rvermUnalignNocase(chars, - buf_end - VERM_BOUNDARY, - 0) - : rvermUnalign(chars, buf_end - VERM_BOUNDARY, - 0); - - if (ptr) { - return ptr; - } - - buf_end -= min; - if (buf >= buf_end) { - return buf_end; - } - } - - // Aligned loops from here on in. - const u8 *ptr = nocase ? rvermSearchAlignedNocase(chars, buf, buf_end, 0) - : rvermSearchAligned(chars, buf, buf_end, 0); - if (ptr) { - return ptr; - } - - // Tidy up the mess at the end, return buf - 1 if not found. - ptr = nocase ? rvermUnalignNocase(chars, buf, 0) - : rvermUnalign(chars, buf, 0); - return ptr ? ptr : buf - 1; -} - -/* like rvermicelliExec except returns the address of the last character which - * is not c */ -static really_inline -const u8 *rnvermicelliExec(char c, char nocase, const u8 *buf, - const u8 *buf_end) { - DEBUG_PRINTF("rev verm scan %s\\x%02hhx over %zu bytes\n", - nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */ - - // Handle small scans. -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = nocase - ? rvermMiniNocase(chars, buf, buf_end, 1) - : rvermMini(chars, buf, buf_end, 1); - if (ptr) { - return ptr; - } - return buf - 1; - } -#else - if (buf_end - buf < VERM_BOUNDARY) { - for (buf_end--; buf_end >= buf; buf_end--) { - char cur = (char)*buf_end; - if (nocase) { - cur &= CASE_CLEAR; - } - if (cur != c) { - break; - } - } - return buf_end; - } -#endif - - size_t min = (size_t)buf_end % VERM_BOUNDARY; - if (min) { - // Input isn't aligned, so we need to run one iteration with an - // unaligned load, then skip buf backward to the next aligned address. - // There's some small overlap here, but we don't mind scanning it twice - // if we can do it quickly, do we? - const u8 *ptr = nocase ? rvermUnalignNocase(chars, - buf_end - VERM_BOUNDARY, - 1) - : rvermUnalign(chars, buf_end - VERM_BOUNDARY, - 1); - - if (ptr) { - return ptr; - } - - buf_end -= min; - if (buf >= buf_end) { - return buf_end; - } - } - - // Aligned loops from here on in. - const u8 *ptr = nocase ? rvermSearchAlignedNocase(chars, buf, buf_end, 1) - : rvermSearchAligned(chars, buf, buf_end, 1); - if (ptr) { - return ptr; - } - - // Tidy up the mess at the end, return buf - 1 if not found. - ptr = nocase ? rvermUnalignNocase(chars, buf, 1) - : rvermUnalign(chars, buf, 1); - return ptr ? ptr : buf - 1; -} - -static really_inline -const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, - const u8 *buf_end) { - DEBUG_PRINTF("double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", - nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars1 = VERM_SET_FN(c1); /* nocase already uppercase */ - VERM_TYPE chars2 = VERM_SET_FN(c2); /* nocase already uppercase */ - -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = nocase - ? dvermMiniNocase(chars1, chars2, buf, buf_end) - : dvermMini(chars1, chars2, buf, buf_end); - if (ptr) { - return ptr; - } - - /* check for partial match at end */ - u8 mask = nocase ? CASE_CLEAR : 0xff; - if ((buf_end[-1] & mask) == (u8)c1) { - DEBUG_PRINTF("partial!!!\n"); - return buf_end - 1; - } - - return buf_end; - } -#endif - - assert((buf_end - buf) >= VERM_BOUNDARY); - uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY; - if (min) { - // Input isn't aligned, so we need to run one iteration with an - // unaligned load, then skip buf forward to the next aligned address. - // There's some small overlap here, but we don't mind scanning it twice - // if we can do it quickly, do we? - const u8 *ptr = nocase - ? dvermPreconditionNocase(chars1, chars2, buf) - : dvermPrecondition(chars1, chars2, buf); - if (ptr) { - return ptr; - } - - buf += VERM_BOUNDARY - min; - assert(buf < buf_end); - } - - // Aligned loops from here on in - const u8 *ptr = nocase ? dvermSearchAlignedNocase(chars1, chars2, c1, c2, - buf, buf_end) - : dvermSearchAligned(chars1, chars2, c1, c2, buf, - buf_end); - if (ptr) { - return ptr; - } - - // Tidy up the mess at the end - ptr = nocase ? dvermPreconditionNocase(chars1, chars2, - buf_end - VERM_BOUNDARY) - : dvermPrecondition(chars1, chars2, buf_end - VERM_BOUNDARY); - - if (ptr) { - return ptr; - } - - /* check for partial match at end */ - u8 mask = nocase ? CASE_CLEAR : 0xff; - if ((buf_end[-1] & mask) == (u8)c1) { - DEBUG_PRINTF("partial!!!\n"); - return buf_end - 1; - } - - return buf_end; -} - -/* returns highest offset of c2 (NOTE: not c1) */ -static really_inline -const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, - const u8 *buf_end) { - DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", - nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars1 = VERM_SET_FN(c1); /* nocase already uppercase */ - VERM_TYPE chars2 = VERM_SET_FN(c2); /* nocase already uppercase */ - -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = nocase - ? rdvermMiniNocase(chars1, chars2, buf, buf_end) - : rdvermMini(chars1, chars2, buf, buf_end); - - if (ptr) { - return ptr; - } - - // check for partial match at end ??? - return buf - 1; - } -#endif - - assert((buf_end - buf) >= VERM_BOUNDARY); - size_t min = (size_t)buf_end % VERM_BOUNDARY; - if (min) { - // input not aligned, so we need to run one iteration with an unaligned - // load, then skip buf forward to the next aligned address. There's - // some small overlap here, but we don't mind scanning it twice if we - // can do it quickly, do we? - const u8 *ptr = nocase ? rdvermPreconditionNocase(chars1, chars2, - buf_end - VERM_BOUNDARY) - : rdvermPrecondition(chars1, chars2, - buf_end - VERM_BOUNDARY); - - if (ptr) { - return ptr; - } - - buf_end -= min; - if (buf >= buf_end) { - return buf_end; - } - } - - // Aligned loops from here on in - if (nocase) { - return rdvermSearchAlignedNocase(chars1, chars2, c1, c2, buf, buf_end); - } else { - return rdvermSearchAligned(chars1, chars2, c1, c2, buf, buf_end); - } -} \ No newline at end of file From f6fd8454008e4464c2ddfd2de2dd827aa1209ea7 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Mon, 1 Nov 2021 16:51:18 +0200 Subject: [PATCH 18/25] complete refactoring and unification of Vermicelli functions --- src/nfa/vermicelli.hpp | 8 ++ src/nfa/vermicelli_simd.cpp | 240 ++++++++++++++++++------------------ 2 files changed, 127 insertions(+), 121 deletions(-) diff --git a/src/nfa/vermicelli.hpp b/src/nfa/vermicelli.hpp index 0b4686e1..83eb2335 100644 --- a/src/nfa/vermicelli.hpp +++ b/src/nfa/vermicelli.hpp @@ -75,4 +75,12 @@ const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, con } #endif +#ifdef __cplusplus +extern "C" { +#endif +const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + #endif /* VERMICELLI_HPP */ \ No newline at end of file diff --git a/src/nfa/vermicelli_simd.cpp b/src/nfa/vermicelli_simd.cpp index 6348e6f3..cd818dfb 100644 --- a/src/nfa/vermicelli_simd.cpp +++ b/src/nfa/vermicelli_simd.cpp @@ -41,48 +41,16 @@ template static really_inline -const u8 *vermicelliSingleBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { +const u8 *vermicelliBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { SuperVector mask = chars.eq(casemask & data); return first_non_zero_match(buf, mask); } -template -static really_inline -const u8 *rvermicelliSingleBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { - - SuperVector mask = chars.eq(casemask & data); - return last_non_zero_match(buf, mask); -} template static really_inline -const u8 *vermicelliDoubleBlock(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, - const u8 *buf/*, SuperVector *lastmask1, size_t len = S*/) { - - // lastmask1->print8("lastmask1"); - data.print8("data"); - chars1.print8("chars1"); - chars2.print8("chars2"); - casemask.print8("casemask"); - SuperVector v = casemask & data; - v.print8("v"); - SuperVector mask1 = chars1.eq(v); - mask1.print8("mask1"); - SuperVector mask2 = chars2.eq(v); - mask2.print8("mask2"); - SuperVector mask = (mask1 & (mask2 >> 1)); - mask.print8("mask"); - DEBUG_PRINTF("len = %ld\n", len); - // *lastmask1 = mask1 >> (len -1); - // lastmask1->print8("lastmask1"); - - return first_non_zero_match(buf, mask); -} - -template -static really_inline -const u8 *vermicelliSingleBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { +const u8 *vermicelliBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { SuperVector mask = chars.eq(casemask & data); return first_zero_match_inverted(buf, mask); @@ -90,36 +58,58 @@ const u8 *vermicelliSingleBlockNeg(SuperVector data, SuperVector chars, Su template static really_inline -const u8 *rvermicelliSingleBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { +const u8 *rvermicelliBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { + + SuperVector mask = chars.eq(casemask & data); + return last_non_zero_match(buf, mask); +} + + +template +static really_inline +const u8 *rvermicelliBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { SuperVector mask = chars.eq(casemask & data); return last_zero_match_inverted(buf, mask); } -/* + template static really_inline -const u8 *vermicelliDoubleBlockNeg(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, - const u8 *buf, size_t len = S) { +const u8 *vermicelliDoubleBlock(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, + u8 const c1, u8 const c2, u8 const casechar, const u8 *buf) { - // lastmask1.print8("lastmask1"); - data.print8("data"); - chars1.print8("chars1"); - chars2.print8("chars2"); - casemask.print8("casemask"); SuperVector v = casemask & data; - v.print8("v"); SuperVector mask1 = chars1.eq(v); - mask1.print8("mask1"); SuperVector mask2 = chars2.eq(v); - mask2.print8("mask2"); - SuperVector mask = (mask1 & (mask2 >> 1));// | lastmask1; - mask.print8("mask"); - DEBUG_PRINTF("len = %ld\n", len); - // lastmask1 = mask << (len -1); - // lastmask1.print8("lastmask1"); + SuperVector mask = mask1 & (mask2 >> 1); - return last_zero_match_inverted(buf, mask); -}*/ + DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]); + bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1)); + DEBUG_PRINTF("partial = %d\n", partial_match); + if (partial_match) return buf - 1; + + return first_non_zero_match(buf, mask); +} + +template +static really_inline +const u8 *rvermicelliDoubleBlock(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, + u8 const c1, u8 const c2, u8 const casechar, const u8 *buf) { + + SuperVector v = casemask & data; + SuperVector mask1 = chars1.eq(v); + SuperVector mask2 = chars2.eq(v); + SuperVector mask = (mask1 << 1)& mask2; + + DEBUG_PRINTF("buf[0] = %02hhx, buf[-1] = %02hhx\n", buf[0], buf[-1]); + bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1)); + DEBUG_PRINTF("partial = %d\n", partial_match); + if (partial_match) { + mask = mask | (SuperVector::Ones() >> (S-1)); + } + + return last_non_zero_match(buf, mask); +} template static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { @@ -142,7 +132,7 @@ static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector c DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); if (!ISALIGNED_N(d, S)) { SuperVector data = SuperVector::loadu(d); - rv = vermicelliSingleBlock(data, chars, casemask, d); + rv = vermicelliBlock(data, chars, casemask, d); if (rv) return rv; d = ROUNDUP_PTR(d, S); } @@ -151,7 +141,7 @@ static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector c __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); SuperVector data = SuperVector::load(d); - rv = vermicelliSingleBlock(data, chars, casemask, d); + rv = vermicelliBlock(data, chars, casemask, d); if (rv) return rv; d += S; } @@ -162,7 +152,7 @@ static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector c if (d != buf_end) { SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); - rv = vermicelliSingleBlock(data, chars, casemask, d); + rv = vermicelliBlock(data, chars, casemask, d); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -180,8 +170,6 @@ static const u8 *nvermicelliExecReal(SuperVector const chars, SuperVector const u8 *d = buf; const u8 *rv; - - __builtin_prefetch(d + 64); __builtin_prefetch(d + 2*64); __builtin_prefetch(d + 3*64); @@ -193,7 +181,7 @@ static const u8 *nvermicelliExecReal(SuperVector const chars, SuperVector DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); if (!ISALIGNED_N(d, S)) { SuperVector data = SuperVector::loadu(d); - rv = vermicelliSingleBlockNeg(data, chars, casemask, d); + rv = vermicelliBlockNeg(data, chars, casemask, d); if (rv) return rv; d = ROUNDUP_PTR(d, S); } @@ -202,7 +190,7 @@ static const u8 *nvermicelliExecReal(SuperVector const chars, SuperVector __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); SuperVector data = SuperVector::load(d); - rv = vermicelliSingleBlockNeg(data, chars, casemask, d); + rv = vermicelliBlockNeg(data, chars, casemask, d); if (rv) return rv; d += S; } @@ -213,7 +201,7 @@ static const u8 *nvermicelliExecReal(SuperVector const chars, SuperVector if (d != buf_end) { SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); - rv = vermicelliSingleBlockNeg(data, chars, casemask, d); + rv = vermicelliBlockNeg(data, chars, casemask, d); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -244,7 +232,7 @@ const u8 *rvermicelliExecReal(SuperVector const chars, SuperVector const c DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); if (!ISALIGNED_N(d, S)) { SuperVector data = SuperVector::loadu(d - S); - rv = rvermicelliSingleBlock(data, chars, casemask, d - S); + rv = rvermicelliBlock(data, chars, casemask, d - S); DEBUG_PRINTF("rv %p \n", rv); if (rv) return rv; d = ROUNDDOWN_PTR(d, S); @@ -257,7 +245,7 @@ const u8 *rvermicelliExecReal(SuperVector const chars, SuperVector const c d -= S; SuperVector data = SuperVector::load(d); - rv = rvermicelliSingleBlock(data, chars, casemask, d); + rv = rvermicelliBlock(data, chars, casemask, d); if (rv) return rv; } } @@ -267,7 +255,7 @@ const u8 *rvermicelliExecReal(SuperVector const chars, SuperVector const c if (d != buf) { SuperVector data = SuperVector::loadu(buf); - rv = rvermicelliSingleBlock(data, chars, casemask, buf); + rv = rvermicelliBlock(data, chars, casemask, buf); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -298,7 +286,7 @@ const u8 *rnvermicelliExecReal(SuperVector const chars, SuperVector const DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); if (!ISALIGNED_N(d, S)) { SuperVector data = SuperVector::loadu(d - S); - rv = rvermicelliSingleBlockNeg(data, chars, casemask, d - S); + rv = rvermicelliBlockNeg(data, chars, casemask, d - S); DEBUG_PRINTF("rv %p \n", rv); if (rv) return rv; d = ROUNDDOWN_PTR(d, S); @@ -311,7 +299,7 @@ const u8 *rnvermicelliExecReal(SuperVector const chars, SuperVector const d -= S; SuperVector data = SuperVector::load(d); - rv = rvermicelliSingleBlockNeg(data, chars, casemask, d); + rv = rvermicelliBlockNeg(data, chars, casemask, d); if (rv) return rv; } } @@ -321,7 +309,7 @@ const u8 *rnvermicelliExecReal(SuperVector const chars, SuperVector const if (d != buf) { SuperVector data = SuperVector::loadu(buf); - rv = rvermicelliSingleBlockNeg(data, chars, casemask, buf); + rv = rvermicelliBlockNeg(data, chars, casemask, buf); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -355,7 +343,7 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); if (!ISALIGNED_N(d, S)) { SuperVector data = SuperVector::loadu(d); - rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, &lastmask1); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, &lastmask1); if (rv) return rv; d = ROUNDUP_PTR(d, S); } @@ -364,11 +352,8 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); SuperVector data = SuperVector::load(d); - rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, &lastmask1); - if (rv) { - bool partial_match = (((rv[0] & casechar) == c2) && ((rv[-1] & casechar) == c1)); - return rv - partial_match; - } + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, &lastmask1); + if (rv) return rv; d += S; } } @@ -378,7 +363,7 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< if (d != buf_end) { SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); - rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, buf_end - d); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, buf_end - d); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -396,60 +381,63 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< } // /* returns highest offset of c2 (NOTE: not c1) */ -// static really_inline -// const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, -// const u8 *buf_end) { -// DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", -// nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); -// assert(buf < buf_end); +template +const u8 *rvermicelliDoubleExecReal(char c1, char c2, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("rverm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + char s[255]; + snprintf(s, buf_end - buf + 1, "%s", buf); + DEBUG_PRINTF("b %s\n", s); -// VERM_TYPE chars1 = VERM_SET_FN(c1); /* nocase already uppercase */ -// VERM_TYPE chars2 = VERM_SET_FN(c2); /* nocase already uppercase */ + const u8 *d = buf_end; + const u8 *rv; + const SuperVector chars1 = SuperVector::dup_u8(c1); + const SuperVector chars2 = SuperVector::dup_u8(c2); + const u8 casechar = casemask.u.u8[0]; -// #ifdef HAVE_AVX512 -// if (buf_end - buf <= VERM_BOUNDARY) { -// const u8 *ptr = nocase -// ? rdvermMiniNocase(chars1, chars2, buf, buf_end) -// : rdvermMini(chars1, chars2, buf, buf_end); + __builtin_prefetch(d - 64); + __builtin_prefetch(d - 2*64); + __builtin_prefetch(d - 3*64); + __builtin_prefetch(d - 4*64); + DEBUG_PRINTF("start %p end %p \n", buf, d); + assert(d > buf); + if (d - S >= buf) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d - S); + rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d - S); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + d = ROUNDDOWN_PTR(d, S); + } -// if (ptr) { -// return ptr; -// } + while (d - S >= buf) { + DEBUG_PRINTF("aligned %p \n", d); + // On large packet buffers, this prefetch appears to get us about 2%. + __builtin_prefetch(d - 64); -// // check for partial match at end ??? -// return buf - 1; -// } -// #endif + d -= S; + SuperVector data = SuperVector::load(d); + rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d); + if (rv) return rv; + } + } -// assert((buf_end - buf) >= VERM_BOUNDARY); -// size_t min = (size_t)buf_end % VERM_BOUNDARY; -// if (min) { -// // input not aligned, so we need to run one iteration with an unaligned -// // load, then skip buf forward to the next aligned address. There's -// // some small overlap here, but we don't mind scanning it twice if we -// // can do it quickly, do we? -// const u8 *ptr = nocase ? rdvermPreconditionNocase(chars1, chars2, -// buf_end - VERM_BOUNDARY) -// : rdvermPrecondition(chars1, chars2, -// buf_end - VERM_BOUNDARY); + DEBUG_PRINTF("tail d %p e %p \n", buf, d); + // finish off head -// if (ptr) { -// return ptr; -// } + if (d != buf) { + SuperVector data = SuperVector::loadu(buf); + rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, buf); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } -// buf_end -= min; -// if (buf >= buf_end) { -// return buf_end; -// } -// } - -// // Aligned loops from here on in -// if (nocase) { -// return rdvermSearchAlignedNocase(chars1, chars2, c1, c2, buf, buf_end); -// } else { -// return rdvermSearchAligned(chars1, chars2, c1, c2, buf, buf_end); -// } -// } + return buf - 1; +} extern "C" const u8 *vermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) { DEBUG_PRINTF("verm scan %s\\x%02hhx over %zu bytes\n", @@ -505,4 +493,14 @@ extern "C" const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; return vermicelliDoubleExecReal(c1, c2, casemask, buf, buf_end); +} + +extern "C" const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", + nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; + + return rvermicelliDoubleExecReal(c1, c2, casemask, buf, buf_end); } \ No newline at end of file From 59505f98ba1a4de6d6822b67961e94006f877d06 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Mon, 1 Nov 2021 16:40:01 +0000 Subject: [PATCH 19/25] remove vermicelli_sse.h --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d4af1fc..410d4214 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -618,7 +618,6 @@ set (hs_exec_SRCS src/nfa/truffle.h src/nfa/vermicelli.h src/nfa/vermicelli_run.h - src/nfa/vermicelli_sse.h src/nfa/vermicelli_simd.cpp src/som/som.h src/som/som_operation.h From 16f3cca98be32ed1027f9e9441415d14c2709c44 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Mon, 1 Nov 2021 16:40:17 +0000 Subject: [PATCH 20/25] add vermicelli.hpp to includes --- benchmarks/benchmarks.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/benchmarks.hpp b/benchmarks/benchmarks.hpp index eb892e51..974d2234 100644 --- a/benchmarks/benchmarks.hpp +++ b/benchmarks/benchmarks.hpp @@ -30,7 +30,7 @@ #include "nfa/shufticompile.h" #include "nfa/truffle.h" #include "nfa/trufflecompile.h" -#include "nfa/vermicelli.h" +#include "nfa/vermicelli.hpp" #include "hwlm/noodle_build.h" #include "hwlm/noodle_engine.h" #include "hwlm/noodle_internal.h" From 869d2bd53b8a31550ae0c689c5c43ce31eeeb4da Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Tue, 2 Nov 2021 22:30:21 +0200 Subject: [PATCH 21/25] refactor vermicelliDoubleMaskedExec() --- src/nfa/vermicelli.hpp | 14 ++++++ src/nfa/vermicelli_simd.cpp | 96 +++++++++++++++++++++++++++++++++++-- 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/src/nfa/vermicelli.hpp b/src/nfa/vermicelli.hpp index 83eb2335..105194b1 100644 --- a/src/nfa/vermicelli.hpp +++ b/src/nfa/vermicelli.hpp @@ -35,6 +35,12 @@ #ifndef VERMICELLI_HPP #define VERMICELLI_HPP +#include "util/bitutils.h" + +#ifdef HAVE_SVE2 +#include "vermicelli_sve.h" +#endif + #ifdef __cplusplus extern "C" { #endif @@ -83,4 +89,12 @@ const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, co } #endif +#ifdef __cplusplus +extern "C" { +#endif +const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char m2, const u8 *buf, const u8 *buf_end); +#ifdef __cplusplus +} +#endif + #endif /* VERMICELLI_HPP */ \ No newline at end of file diff --git a/src/nfa/vermicelli_simd.cpp b/src/nfa/vermicelli_simd.cpp index cd818dfb..c2215651 100644 --- a/src/nfa/vermicelli_simd.cpp +++ b/src/nfa/vermicelli_simd.cpp @@ -111,6 +111,24 @@ const u8 *rvermicelliDoubleBlock(SuperVector data, SuperVector chars1, Sup return last_non_zero_match(buf, mask); } +template +static really_inline +const u8 *vermicelliDoubleMaskedBlock(SuperVector data, SuperVector chars1, SuperVector chars2, + SuperVector mask1, SuperVector mask2, + u8 const c1, u8 const c2, u8 const m1, u8 const m2, const u8 *buf) { + + SuperVector v1 = chars1.eq(data & mask1); + SuperVector v2 = chars2.eq(data & mask2); + SuperVector mask = v1 & (v2 >> 1); + + DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]); + bool partial_match = (((buf[0] & m1) == c2) && ((buf[-1] & m2) == c1)); + DEBUG_PRINTF("partial = %d\n", partial_match); + if (partial_match) return buf - 1; + + return first_non_zero_match(buf, mask); +} + template static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { assert(buf && buf_end); @@ -343,7 +361,7 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); if (!ISALIGNED_N(d, S)) { SuperVector data = SuperVector::loadu(d); - rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, &lastmask1); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d); if (rv) return rv; d = ROUNDUP_PTR(d, S); } @@ -352,7 +370,7 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); SuperVector data = SuperVector::load(d); - rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, &lastmask1); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d); if (rv) return rv; d += S; } @@ -363,7 +381,7 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< if (d != buf_end) { SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); - rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, buf_end - d); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -371,7 +389,6 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< DEBUG_PRINTF("real tail d %p e %p \n", d, buf_end); /* check for partial match at end */ u8 mask = casemask.u.u8[0]; - // u8 c1 = chars1.u.u8[0]; if ((buf_end[-1] & mask) == (u8)c1) { DEBUG_PRINTF("partial!!!\n"); return buf_end - 1; @@ -439,6 +456,68 @@ const u8 *rvermicelliDoubleExecReal(char c1, char c2, SuperVector const casem return buf - 1; } +template +static const u8 *vermicelliDoubleMaskedExecReal(u8 const c1, u8 const c2, u8 const m1, u8 const m2, + const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("verm %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + + const u8 *d = buf; + const u8 *rv; + // SuperVector lastmask1{0}; + const SuperVector chars1 = SuperVector::dup_u8(c1); + const SuperVector chars2 = SuperVector::dup_u8(c2); + const SuperVector mask1 = SuperVector::dup_u8(m1); + const SuperVector mask2 = SuperVector::dup_u8(m2); + + __builtin_prefetch(d + 64); + __builtin_prefetch(d + 2*64); + __builtin_prefetch(d + 3*64); + __builtin_prefetch(d + 4*64); + DEBUG_PRINTF("start %p end %p \n", d, buf_end); + assert(d < buf_end); + if (d + S <= buf_end) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector data = SuperVector::loadu(d); + rv = vermicelliDoubleMaskedBlock(data, chars1, chars2, mask1, mask2, c1, c2, m1, m2, d); + if (rv) return rv; + d = ROUNDUP_PTR(d, S); + } + + while(d + S <= buf_end) { + __builtin_prefetch(d + 64); + DEBUG_PRINTF("d %p \n", d); + SuperVector data = SuperVector::load(d); + rv = vermicelliDoubleMaskedBlock(data, chars1, chars2, mask1, mask2, c1, c2, m1, m2, d); + if (rv) return rv; + d += S; + } + } + + DEBUG_PRINTF("tail d %p e %p \n", d, buf_end); + // finish off tail + + if (d != buf_end) { + SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); + rv = vermicelliDoubleMaskedBlock(data, chars1, chars2, mask1, mask2, c1, c2, m1, m2, d); + DEBUG_PRINTF("rv %p \n", rv); + if (rv && rv < buf_end) return rv; + } + + DEBUG_PRINTF("real tail d %p e %p \n", d, buf_end); + /* check for partial match at end */ + if ((buf_end[-1] & m1) == (u8)c1) { + DEBUG_PRINTF("partial!!!\n"); + return buf_end - 1; + } + + return buf_end; +} + extern "C" const u8 *vermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) { DEBUG_PRINTF("verm scan %s\\x%02hhx over %zu bytes\n", nocase ? "nocase " : "", c, (size_t)(buf_end - buf)); @@ -503,4 +582,13 @@ extern "C" const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const const SuperVector casemask{nocase ? getCaseMask() : SuperVector::Ones()}; return rvermicelliDoubleExecReal(c1, c2, casemask, buf, buf_end); +} + +extern "C" const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char m2, + const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("double verm scan (\\x%02hhx&\\x%02hhx)(\\x%02hhx&\\x%02hhx) " + "over %zu bytes\n", c1, m1, c2, m2, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + return vermicelliDoubleMaskedExecReal(c1, c2, m1, m2, buf, buf_end); } \ No newline at end of file From 210295a702d4d40a4c771337e06201ee6d8c8baf Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Tue, 2 Nov 2021 22:30:53 +0200 Subject: [PATCH 22/25] remove vermicelli.h and replace it with vermicelli.hpp --- CMakeLists.txt | 2 +- src/hwlm/hwlm.c | 1 - src/nfa/accel.c | 1 - src/nfa/castle.c | 1 - src/nfa/lbr.c | 3 +- src/nfa/limex_accel.c | 2 +- src/nfa/mpv.c | 2 +- src/nfa/nfa_rev_api.h | 1 - src/nfa/vermicelli.h | 119 ---------------------------------- src/nfa/vermicelli_common.h | 79 ---------------------- src/nfa/vermicelli_run.h | 4 +- unit/internal/rvermicelli.cpp | 1 - unit/internal/vermicelli.cpp | 1 - 13 files changed, 7 insertions(+), 210 deletions(-) delete mode 100644 src/nfa/vermicelli.h delete mode 100644 src/nfa/vermicelli_common.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 410d4214..0875b105 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -616,7 +616,7 @@ set (hs_exec_SRCS src/nfa/tamarama_internal.h src/nfa/truffle.cpp src/nfa/truffle.h - src/nfa/vermicelli.h + src/nfa/vermicelli.hpp src/nfa/vermicelli_run.h src/nfa/vermicelli_simd.cpp src/som/som.h diff --git a/src/hwlm/hwlm.c b/src/hwlm/hwlm.c index 5d69e3c4..e50deff7 100644 --- a/src/hwlm/hwlm.c +++ b/src/hwlm/hwlm.c @@ -39,7 +39,6 @@ #include "nfa/accel.h" #include "nfa/shufti.h" #include "nfa/truffle.h" -#include "nfa/vermicelli.h" #include "nfa/vermicelli.hpp" #include diff --git a/src/nfa/accel.c b/src/nfa/accel.c index b35e0633..7661b7a7 100644 --- a/src/nfa/accel.c +++ b/src/nfa/accel.c @@ -30,7 +30,6 @@ #include "accel.h" #include "shufti.h" #include "truffle.h" -#include "vermicelli.h" #include "vermicelli.hpp" #include "ue2common.h" diff --git a/src/nfa/castle.c b/src/nfa/castle.c index be29ca29..29208f8d 100644 --- a/src/nfa/castle.c +++ b/src/nfa/castle.c @@ -40,7 +40,6 @@ #include "repeat.h" #include "shufti.h" #include "truffle.h" -#include "vermicelli.h" #include "vermicelli.hpp" #include "util/bitutils.h" #include "util/multibit.h" diff --git a/src/nfa/lbr.c b/src/nfa/lbr.c index 8fc83988..52e81ad6 100644 --- a/src/nfa/lbr.c +++ b/src/nfa/lbr.c @@ -40,7 +40,6 @@ #include "repeat_internal.h" #include "shufti.h" #include "truffle.h" -#include "vermicelli.h" #include "vermicelli.hpp" #include "util/partial_store.h" #include "util/unaligned.h" @@ -534,4 +533,4 @@ char lbrFwdScanTruf(const struct NFA *nfa, const u8 *buf, #ifdef HAVE_SVE2 #include "lbr_sve.h" -#endif \ No newline at end of file +#endif diff --git a/src/nfa/limex_accel.c b/src/nfa/limex_accel.c index 4834b6a5..a85d5a07 100644 --- a/src/nfa/limex_accel.c +++ b/src/nfa/limex_accel.c @@ -40,7 +40,7 @@ #include "shufti.h" #include "truffle.h" #include "ue2common.h" -#include "vermicelli.h" +#include "vermicelli.hpp" #include "util/arch.h" #include "util/bitutils.h" #include "util/simd_utils.h" diff --git a/src/nfa/mpv.c b/src/nfa/mpv.c index 5829d43d..cba3d159 100644 --- a/src/nfa/mpv.c +++ b/src/nfa/mpv.c @@ -36,7 +36,7 @@ #include "shufti.h" #include "truffle.h" #include "ue2common.h" -#include "vermicelli.h" +#include "vermicelli.hpp" #include "vermicelli_run.h" #include "util/multibit.h" #include "util/partial_store.h" diff --git a/src/nfa/nfa_rev_api.h b/src/nfa/nfa_rev_api.h index 72224c3b..d82c52a4 100644 --- a/src/nfa/nfa_rev_api.h +++ b/src/nfa/nfa_rev_api.h @@ -35,7 +35,6 @@ #include "accel.h" #include "nfa_internal.h" -#include "vermicelli.h" #include "vermicelli.hpp" #include "util/unaligned.h" diff --git a/src/nfa/vermicelli.h b/src/nfa/vermicelli.h deleted file mode 100644 index 39e9555e..00000000 --- a/src/nfa/vermicelli.h +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (c) 2015-2020, Intel Corporation - * Copyright (c) 2021, Arm Limited - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** \file - * \brief Vermicelli: single-byte and double-byte acceleration. - */ - -#ifndef VERMICELLI_H -#define VERMICELLI_H - -#include "util/bitutils.h" -#include "util/simd_utils.h" -#include "util/unaligned.h" - -#if !defined(HAVE_AVX512) -#include "vermicelli_common.h" -#endif - -#ifdef HAVE_SVE2 -#include "vermicelli_sve.h" -#endif - -static really_inline -const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char m2, - const u8 *buf, const u8 *buf_end) { - DEBUG_PRINTF("double verm scan (\\x%02hhx&\\x%02hhx)(\\x%02hhx&\\x%02hhx) " - "over %zu bytes\n", c1, m1, c2, m2, (size_t)(buf_end - buf)); - assert(buf < buf_end); - - VERM_TYPE chars1 = VERM_SET_FN(c1); - VERM_TYPE chars2 = VERM_SET_FN(c2); - VERM_TYPE mask1 = VERM_SET_FN(m1); - VERM_TYPE mask2 = VERM_SET_FN(m2); - -#ifdef HAVE_AVX512 - if (buf_end - buf <= VERM_BOUNDARY) { - const u8 *ptr = dvermMiniMasked(chars1, chars2, mask1, mask2, buf, - buf_end); - if (ptr) { - return ptr; - } - - /* check for partial match at end */ - if ((buf_end[-1] & m1) == (u8)c1) { - DEBUG_PRINTF("partial!!!\n"); - return buf_end - 1; - } - - return buf_end; - } -#endif - - assert((buf_end - buf) >= VERM_BOUNDARY); - uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY; - if (min) { - // Input isn't aligned, so we need to run one iteration with an - // unaligned load, then skip buf forward to the next aligned address. - // There's some small overlap here, but we don't mind scanning it twice - // if we can do it quickly, do we? - const u8 *p = dvermPreconditionMasked(chars1, chars2, mask1, mask2, buf); - if (p) { - return p; - } - - buf += VERM_BOUNDARY - min; - assert(buf < buf_end); - } - - // Aligned loops from here on in - const u8 *ptr = dvermSearchAlignedMasked(chars1, chars2, mask1, mask2, c1, - c2, m1, m2, buf, buf_end); - if (ptr) { - return ptr; - } - - // Tidy up the mess at the end - ptr = dvermPreconditionMasked(chars1, chars2, mask1, mask2, - buf_end - VERM_BOUNDARY); - - if (ptr) { - return ptr; - } - - /* check for partial match at end */ - if ((buf_end[-1] & m1) == (u8)c1) { - DEBUG_PRINTF("partial!!!\n"); - return buf_end - 1; - } - - return buf_end; -} - -#endif /* VERMICELLI_H */ diff --git a/src/nfa/vermicelli_common.h b/src/nfa/vermicelli_common.h deleted file mode 100644 index aca58dcb..00000000 --- a/src/nfa/vermicelli_common.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2015-2020, Intel Corporation - * Copyright (c) 2021, Arm Limited - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** \file - * \brief Vermicelli: Implementation shared between architectures. - * - * (users should include vermicelli.h instead of this) - */ - -#define VERM_BOUNDARY 16 -#define VERM_TYPE m128 -#define VERM_SET_FN set1_16x8 - -// returns NULL if not found -static really_inline -const u8 *dvermPreconditionMasked(m128 chars1, m128 chars2, - m128 mask1, m128 mask2, const u8 *buf) { - m128 data = loadu128(buf); // unaligned - m128 v1 = eq128(chars1, and128(data, mask1)); - m128 v2 = eq128(chars2, and128(data, mask2)); - u32 z = movemask128(and128(v1, rshiftbyte_m128(v2, 1))); - - /* no fixup of the boundary required - the aligned run will pick it up */ - if (unlikely(z)) { - u32 pos = ctz32(z); - return buf + pos; - } - return NULL; -} - -static really_inline -const u8 *dvermSearchAlignedMasked(m128 chars1, m128 chars2, - m128 mask1, m128 mask2, u8 c1, u8 c2, u8 m1, - u8 m2, const u8 *buf, const u8 *buf_end) { - assert((size_t)buf % 16 == 0); - - for (; buf + 16 < buf_end; buf += 16) { - m128 data = load128(buf); - m128 v1 = eq128(chars1, and128(data, mask1)); - m128 v2 = eq128(chars2, and128(data, mask2)); - u32 z = movemask128(and128(v1, rshiftbyte_m128(v2, 1))); - - if ((buf[15] & m1) == c1 && (buf[16] & m2) == c2) { - z |= (1 << 15); - } - if (unlikely(z)) { - u32 pos = ctz32(z); - return buf + pos; - } - } - - return NULL; -} \ No newline at end of file diff --git a/src/nfa/vermicelli_run.h b/src/nfa/vermicelli_run.h index b75f1414..1deda48a 100644 --- a/src/nfa/vermicelli_run.h +++ b/src/nfa/vermicelli_run.h @@ -26,9 +26,11 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#include "vermicelli.h" #include "vermicelli.hpp" +#define VERM_BOUNDARY 16 +#define VERM_TYPE m128 + static really_inline const u8 *find_xverm_run(char c, char nocase, u32 repeat, UNUSED const u8 *buf, const u8 *buf_start, const u8 *buf_end, char negate) { diff --git a/unit/internal/rvermicelli.cpp b/unit/internal/rvermicelli.cpp index d29b1133..5cd52e4d 100644 --- a/unit/internal/rvermicelli.cpp +++ b/unit/internal/rvermicelli.cpp @@ -30,7 +30,6 @@ #include "config.h" #include "gtest/gtest.h" -#include "nfa/vermicelli.h" #include "nfa/vermicelli.hpp" #define BOUND (~(VERM_BOUNDARY - 1)) diff --git a/unit/internal/vermicelli.cpp b/unit/internal/vermicelli.cpp index 3319b87c..e6d976ad 100644 --- a/unit/internal/vermicelli.cpp +++ b/unit/internal/vermicelli.cpp @@ -30,7 +30,6 @@ #include "config.h" #include "gtest/gtest.h" -#include "nfa/vermicelli.h" #include "nfa/vermicelli.hpp" TEST(Vermicelli, ExecNoMatch1) { From 24fa54081b6227d8fab59d622717697aeb42aac7 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Fri, 5 Nov 2021 14:30:22 +0200 Subject: [PATCH 23/25] add len parameter and mask, fixes corner cases on AVX512 --- src/nfa/vermicelli_simd.cpp | 107 ++++++++++++++++++++---------------- src/util/arch/x86/match.hpp | 54 ++++++++++++------ src/util/match.hpp | 8 +-- 3 files changed, 99 insertions(+), 70 deletions(-) diff --git a/src/nfa/vermicelli_simd.cpp b/src/nfa/vermicelli_simd.cpp index c2215651..e8b7caaf 100644 --- a/src/nfa/vermicelli_simd.cpp +++ b/src/nfa/vermicelli_simd.cpp @@ -41,42 +41,46 @@ template static really_inline -const u8 *vermicelliBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { +const u8 *vermicelliBlock(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len) { SuperVector mask = chars.eq(casemask & data); - return first_non_zero_match(buf, mask); + return first_non_zero_match(buf, mask, len); } template static really_inline -const u8 *vermicelliBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { +const u8 *vermicelliBlockNeg(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len) { SuperVector mask = chars.eq(casemask & data); - return first_zero_match_inverted(buf, mask); + return first_zero_match_inverted(buf, mask, len); } template static really_inline -const u8 *rvermicelliBlock(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { +const u8 *rvermicelliBlock(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len) { SuperVector mask = chars.eq(casemask & data); - return last_non_zero_match(buf, mask); + return last_non_zero_match(buf, mask, len); } template static really_inline -const u8 *rvermicelliBlockNeg(SuperVector data, SuperVector chars, SuperVector casemask, const u8 *buf) { +const u8 *rvermicelliBlockNeg(SuperVector const data, SuperVector const chars, SuperVector const casemask, const u8 *buf, u16 const len) { + data.print8("data"); + chars.print8("chars"); + casemask.print8("casemask"); SuperVector mask = chars.eq(casemask & data); - return last_zero_match_inverted(buf, mask); + mask.print8("mask"); + return last_zero_match_inverted(buf, mask, len); } template static really_inline -const u8 *vermicelliDoubleBlock(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, - u8 const c1, u8 const c2, u8 const casechar, const u8 *buf) { +const u8 *vermicelliDoubleBlock(SuperVector const data, SuperVector const chars1, SuperVector const chars2, SuperVector const casemask, + u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len) { SuperVector v = casemask & data; SuperVector mask1 = chars1.eq(v); @@ -88,13 +92,13 @@ const u8 *vermicelliDoubleBlock(SuperVector data, SuperVector chars1, Supe DEBUG_PRINTF("partial = %d\n", partial_match); if (partial_match) return buf - 1; - return first_non_zero_match(buf, mask); + return first_non_zero_match(buf, mask, len); } template static really_inline -const u8 *rvermicelliDoubleBlock(SuperVector data, SuperVector chars1, SuperVector chars2, SuperVector casemask, - u8 const c1, u8 const c2, u8 const casechar, const u8 *buf) { +const u8 *rvermicelliDoubleBlock(SuperVector const data, SuperVector const chars1, SuperVector const chars2, SuperVector const casemask, + u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len) { SuperVector v = casemask & data; SuperVector mask1 = chars1.eq(v); @@ -108,14 +112,14 @@ const u8 *rvermicelliDoubleBlock(SuperVector data, SuperVector chars1, Sup mask = mask | (SuperVector::Ones() >> (S-1)); } - return last_non_zero_match(buf, mask); + return last_non_zero_match(buf, mask, len); } template static really_inline -const u8 *vermicelliDoubleMaskedBlock(SuperVector data, SuperVector chars1, SuperVector chars2, - SuperVector mask1, SuperVector mask2, - u8 const c1, u8 const c2, u8 const m1, u8 const m2, const u8 *buf) { +const u8 *vermicelliDoubleMaskedBlock(SuperVector const data, SuperVector const chars1, SuperVector const chars2, + SuperVector const mask1, SuperVector const mask2, + u8 const c1, u8 const c2, u8 const m1, u8 const m2, u8 const *buf, u16 const len) { SuperVector v1 = chars1.eq(data & mask1); SuperVector v2 = chars2.eq(data & mask2); @@ -126,11 +130,11 @@ const u8 *vermicelliDoubleMaskedBlock(SuperVector data, SuperVector chars1 DEBUG_PRINTF("partial = %d\n", partial_match); if (partial_match) return buf - 1; - return first_non_zero_match(buf, mask); + return first_non_zero_match(buf, mask, len); } template -static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector const casemask, const u8 *buf, const u8 *buf_end) { +static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector const casemask, u8 const *buf, u8 const *buf_end) { assert(buf && buf_end); assert(buf < buf_end); DEBUG_PRINTF("verm %p len %zu\n", buf, buf_end - buf); @@ -149,17 +153,18 @@ static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector c // Reach vector aligned boundaries DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); if (!ISALIGNED_N(d, S)) { + u8 const *d1 = ROUNDUP_PTR(d, S); SuperVector data = SuperVector::loadu(d); - rv = vermicelliBlock(data, chars, casemask, d); + rv = vermicelliBlock(data, chars, casemask, d, S); if (rv) return rv; - d = ROUNDUP_PTR(d, S); + d = d1; } while(d + S <= buf_end) { __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); SuperVector data = SuperVector::load(d); - rv = vermicelliBlock(data, chars, casemask, d); + rv = vermicelliBlock(data, chars, casemask, d, S); if (rv) return rv; d += S; } @@ -170,7 +175,7 @@ static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector c if (d != buf_end) { SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); - rv = vermicelliBlock(data, chars, casemask, d); + rv = vermicelliBlock(data, chars, casemask, d, buf_end - d); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -198,17 +203,18 @@ static const u8 *nvermicelliExecReal(SuperVector const chars, SuperVector // Reach vector aligned boundaries DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); if (!ISALIGNED_N(d, S)) { + u8 const *d1 = ROUNDUP_PTR(d, S); SuperVector data = SuperVector::loadu(d); - rv = vermicelliBlockNeg(data, chars, casemask, d); + rv = vermicelliBlockNeg(data, chars, casemask, d, S); if (rv) return rv; - d = ROUNDUP_PTR(d, S); + d = d1; } while(d + S <= buf_end) { __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); SuperVector data = SuperVector::load(d); - rv = vermicelliBlockNeg(data, chars, casemask, d); + rv = vermicelliBlockNeg(data, chars, casemask, d, S); if (rv) return rv; d += S; } @@ -219,7 +225,7 @@ static const u8 *nvermicelliExecReal(SuperVector const chars, SuperVector if (d != buf_end) { SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); - rv = vermicelliBlockNeg(data, chars, casemask, d); + rv = vermicelliBlockNeg(data, chars, casemask, d, buf_end - d); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -249,11 +255,12 @@ const u8 *rvermicelliExecReal(SuperVector const chars, SuperVector const c // Reach vector aligned boundaries DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); if (!ISALIGNED_N(d, S)) { + u8 const *d1 = ROUNDDOWN_PTR(d, S); SuperVector data = SuperVector::loadu(d - S); - rv = rvermicelliBlock(data, chars, casemask, d - S); + rv = rvermicelliBlock(data, chars, casemask, d - S, S); DEBUG_PRINTF("rv %p \n", rv); if (rv) return rv; - d = ROUNDDOWN_PTR(d, S); + d = d1; } while (d - S >= buf) { @@ -263,7 +270,7 @@ const u8 *rvermicelliExecReal(SuperVector const chars, SuperVector const c d -= S; SuperVector data = SuperVector::load(d); - rv = rvermicelliBlock(data, chars, casemask, d); + rv = rvermicelliBlock(data, chars, casemask, d, S); if (rv) return rv; } } @@ -273,7 +280,7 @@ const u8 *rvermicelliExecReal(SuperVector const chars, SuperVector const c if (d != buf) { SuperVector data = SuperVector::loadu(buf); - rv = rvermicelliBlock(data, chars, casemask, buf); + rv = rvermicelliBlock(data, chars, casemask, buf, d - buf); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -303,11 +310,12 @@ const u8 *rnvermicelliExecReal(SuperVector const chars, SuperVector const // Reach vector aligned boundaries DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); if (!ISALIGNED_N(d, S)) { + u8 const *d1 = ROUNDDOWN_PTR(d, S); SuperVector data = SuperVector::loadu(d - S); - rv = rvermicelliBlockNeg(data, chars, casemask, d - S); + rv = rvermicelliBlockNeg(data, chars, casemask, d - S, S); DEBUG_PRINTF("rv %p \n", rv); if (rv) return rv; - d = ROUNDDOWN_PTR(d, S); + d = d1; } while (d - S >= buf) { @@ -317,7 +325,7 @@ const u8 *rnvermicelliExecReal(SuperVector const chars, SuperVector const d -= S; SuperVector data = SuperVector::load(d); - rv = rvermicelliBlockNeg(data, chars, casemask, d); + rv = rvermicelliBlockNeg(data, chars, casemask, d, S); if (rv) return rv; } } @@ -327,7 +335,7 @@ const u8 *rnvermicelliExecReal(SuperVector const chars, SuperVector const if (d != buf) { SuperVector data = SuperVector::loadu(buf); - rv = rvermicelliBlockNeg(data, chars, casemask, buf); + rv = rvermicelliBlockNeg(data, chars, casemask, buf, d - buf); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -360,17 +368,18 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< // Reach vector aligned boundaries DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); if (!ISALIGNED_N(d, S)) { + u8 const *d1 = ROUNDUP_PTR(d, S); SuperVector data = SuperVector::loadu(d); - rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d, S); if (rv) return rv; - d = ROUNDUP_PTR(d, S); + d = d1; } while(d + S <= buf_end) { __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); SuperVector data = SuperVector::load(d); - rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d, S); if (rv) return rv; d += S; } @@ -381,7 +390,7 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector< if (d != buf_end) { SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); - rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d); + rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d, buf_end - d); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -424,11 +433,12 @@ const u8 *rvermicelliDoubleExecReal(char c1, char c2, SuperVector const casem // Reach vector aligned boundaries DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); if (!ISALIGNED_N(d, S)) { + u8 const *d1 = ROUNDDOWN_PTR(d, S); SuperVector data = SuperVector::loadu(d - S); - rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d - S); + rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d - S, S); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; - d = ROUNDDOWN_PTR(d, S); + d = d1; } while (d - S >= buf) { @@ -438,7 +448,7 @@ const u8 *rvermicelliDoubleExecReal(char c1, char c2, SuperVector const casem d -= S; SuperVector data = SuperVector::load(d); - rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d); + rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d, S); if (rv) return rv; } } @@ -448,7 +458,7 @@ const u8 *rvermicelliDoubleExecReal(char c1, char c2, SuperVector const casem if (d != buf) { SuperVector data = SuperVector::loadu(buf); - rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, buf); + rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, buf, d - buf); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -482,17 +492,18 @@ static const u8 *vermicelliDoubleMaskedExecReal(u8 const c1, u8 const c2, u8 con // Reach vector aligned boundaries DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); if (!ISALIGNED_N(d, S)) { + u8 const *d1 = ROUNDUP_PTR(d, S); SuperVector data = SuperVector::loadu(d); - rv = vermicelliDoubleMaskedBlock(data, chars1, chars2, mask1, mask2, c1, c2, m1, m2, d); + rv = vermicelliDoubleMaskedBlock(data, chars1, chars2, mask1, mask2, c1, c2, m1, m2, d, S); if (rv) return rv; - d = ROUNDUP_PTR(d, S); + d = d1; } while(d + S <= buf_end) { __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); SuperVector data = SuperVector::load(d); - rv = vermicelliDoubleMaskedBlock(data, chars1, chars2, mask1, mask2, c1, c2, m1, m2, d); + rv = vermicelliDoubleMaskedBlock(data, chars1, chars2, mask1, mask2, c1, c2, m1, m2, d, S); if (rv) return rv; d += S; } @@ -503,7 +514,7 @@ static const u8 *vermicelliDoubleMaskedExecReal(u8 const c1, u8 const c2, u8 con if (d != buf_end) { SuperVector data = SuperVector::loadu_maskz(d, buf_end - d); - rv = vermicelliDoubleMaskedBlock(data, chars1, chars2, mask1, mask2, c1, c2, m1, m2, d); + rv = vermicelliDoubleMaskedBlock(data, chars1, chars2, mask1, mask2, c1, c2, m1, m2, d, buf_end - d); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -591,4 +602,4 @@ extern "C" const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char assert(buf < buf_end); return vermicelliDoubleMaskedExecReal(c1, c2, m1, m2, buf, buf_end); -} \ No newline at end of file +} diff --git a/src/util/arch/x86/match.hpp b/src/util/arch/x86/match.hpp index 26283ca7..cbf4ab6b 100644 --- a/src/util/arch/x86/match.hpp +++ b/src/util/arch/x86/match.hpp @@ -29,7 +29,7 @@ template <> really_really_inline -const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> v) { +const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> v, u16 const UNUSED len) { SuperVector<16>::movemask_type z = v.movemask(); DEBUG_PRINTF("buf %p z %08x \n", buf, z); DEBUG_PRINTF("z %08x\n", z); @@ -46,7 +46,7 @@ const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> v) { template <> really_really_inline -const u8 *first_non_zero_match<32>(const u8 *buf, SuperVector<32> v) { +const u8 *first_non_zero_match<32>(const u8 *buf, SuperVector<32> v, u16 const UNUSED len) { SuperVector<32>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%08x\n", z); if (unlikely(z)) { @@ -60,9 +60,13 @@ const u8 *first_non_zero_match<32>(const u8 *buf, SuperVector<32> v) { } template <> really_really_inline -const u8 *first_non_zero_match<64>(const u8 *buf, SuperVector<64>v) { +const u8 *first_non_zero_match<64>(const u8 *buf, SuperVector<64>v, u16 const len) { SuperVector<64>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%016llx\n", z); + u64a mask = (~0ULL) >> (64 - len); + DEBUG_PRINTF("mask %016llx\n", mask); + z &= mask; + DEBUG_PRINTF("z 0x%016llx\n", z); if (unlikely(z)) { u32 pos = ctz64(z); DEBUG_PRINTF("match @ pos %u\n", pos); @@ -75,7 +79,7 @@ const u8 *first_non_zero_match<64>(const u8 *buf, SuperVector<64>v) { template <> really_really_inline -const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> v) { +const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> v, u16 const UNUSED len) { SuperVector<16>::movemask_type z = v.movemask(); DEBUG_PRINTF("buf %p z %08x \n", buf, z); DEBUG_PRINTF("z %08x\n", z); @@ -91,7 +95,7 @@ const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> v) { template <> really_really_inline -const u8 *last_non_zero_match<32>(const u8 *buf, SuperVector<32> v) { +const u8 *last_non_zero_match<32>(const u8 *buf, SuperVector<32> v, u16 const UNUSED len) { SuperVector<32>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%08x\n", z); if (unlikely(z)) { @@ -105,14 +109,18 @@ const u8 *last_non_zero_match<32>(const u8 *buf, SuperVector<32> v) { } template <> really_really_inline -const u8 *last_non_zero_match<64>(const u8 *buf, SuperVector<64>v) { +const u8 *last_non_zero_match<64>(const u8 *buf, SuperVector<64>v, u16 const len) { SuperVector<64>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%016llx\n", z); + u64a mask = (~0ULL) >> (64 - len); + DEBUG_PRINTF("mask %016llx\n", mask); + z &= mask; + DEBUG_PRINTF("z 0x%016llx\n", z); if (unlikely(z)) { u32 pos = clz64(z); DEBUG_PRINTF("match @ pos %u\n", pos); assert(pos < 64); - return buf + (31 - pos); + return buf + (63 - pos); } else { return NULL; // no match } @@ -120,7 +128,7 @@ const u8 *last_non_zero_match<64>(const u8 *buf, SuperVector<64>v) { template <> really_really_inline -const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) { +const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v, u16 const UNUSED len) { SuperVector<16>::movemask_type z = v.movemask(); DEBUG_PRINTF("buf %p z %08x \n", buf, z); DEBUG_PRINTF("z %08x\n", z); @@ -137,7 +145,7 @@ const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) { template <> really_really_inline -const u8 *first_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) { +const u8 *first_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v, u16 const UNUSED len) { SuperVector<32>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%08x\n", z); if (unlikely(z != 0xffffffff)) { @@ -151,11 +159,15 @@ const u8 *first_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) { } template <> really_really_inline -const u8 *first_zero_match_inverted<64>(const u8 *buf, SuperVector<64>v) { +const u8 *first_zero_match_inverted<64>(const u8 *buf, SuperVector<64>v, u16 const len) { SuperVector<64>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%016llx\n", z); - if (unlikely(z != ~0ULL)) { - u32 pos = ctz64(~z); + u64a mask = (~0ULL) >> (64 - len); + DEBUG_PRINTF("mask %016llx\n", mask); + z = ~z & mask; + DEBUG_PRINTF("z 0x%016llx\n", z); + if (unlikely(z)) { + u32 pos = ctz64(z); DEBUG_PRINTF("match @ pos %u\n", pos); assert(pos < 64); return buf + pos; @@ -166,7 +178,7 @@ const u8 *first_zero_match_inverted<64>(const u8 *buf, SuperVector<64>v) { template <> really_really_inline -const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) { +const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v, uint16_t UNUSED len ) { SuperVector<16>::movemask_type z = v.movemask(); DEBUG_PRINTF("buf %p z %08x \n", buf, z); DEBUG_PRINTF("z %08x\n", z); @@ -183,10 +195,10 @@ const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) { template<> really_really_inline -const u8 *last_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) { +const u8 *last_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v, uint16_t UNUSED len) { SuperVector<32>::movemask_type z = v.movemask(); if (unlikely(z != 0xffffffff)) { - u32 pos = clz32(~z); + u32 pos = clz32(~z & 0xffffffff); DEBUG_PRINTF("buf=%p, pos=%u\n", buf, pos); assert(pos < 32); return buf + (31 - pos); @@ -197,11 +209,17 @@ const u8 *last_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) { template <> really_really_inline -const u8 *last_zero_match_inverted<64>(const u8 *buf, SuperVector<64> v) { +const u8 *last_zero_match_inverted<64>(const u8 *buf, SuperVector<64> v, uint16_t len) { + v.print8("v"); SuperVector<64>::movemask_type z = v.movemask(); DEBUG_PRINTF("z 0x%016llx\n", z); - if (unlikely(z != ~0ULL)) { - u32 pos = clz64(~z); + u64a mask = (~0ULL) >> (64 - len); + DEBUG_PRINTF("mask %016llx\n", mask); + z = ~z & mask; + DEBUG_PRINTF("z 0x%016llx\n", z); + if (unlikely(z)) { + u32 pos = clz64(z); + DEBUG_PRINTF("~z 0x%016llx\n", ~z); DEBUG_PRINTF("match @ pos %u\n", pos); assert(pos < 64); return buf + (63 - pos); diff --git a/src/util/match.hpp b/src/util/match.hpp index 9b3c8fb9..030db9bb 100644 --- a/src/util/match.hpp +++ b/src/util/match.hpp @@ -38,16 +38,16 @@ #include "util/supervector/supervector.hpp" template -const u8 *first_non_zero_match(const u8 *buf, SuperVector v); +const u8 *first_non_zero_match(const u8 *buf, SuperVector v, u16 const len = S); template -const u8 *last_non_zero_match(const u8 *buf, SuperVector v); +const u8 *last_non_zero_match(const u8 *buf, SuperVector v, u16 const len = S); template -const u8 *first_zero_match_inverted(const u8 *buf, SuperVector v); +const u8 *first_zero_match_inverted(const u8 *buf, SuperVector v, u16 const len = S); template -const u8 *last_zero_match_inverted(const u8 *buf, SuperVector v); +const u8 *last_zero_match_inverted(const u8 *buf, SuperVector v, u16 len = S); #if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/arch/x86/match.hpp" From dcf6b59e8d05a5f9647ea90352b64a4c8840043f Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Mon, 8 Nov 2021 19:45:21 +0000 Subject: [PATCH 24/25] split vermicelli block implementations per arch --- src/nfa/arm/vermicelli.hpp | 125 ++++++++++++++++++++++++++++++++++++ src/nfa/vermicelli_simd.cpp | 80 ++++------------------- src/nfa/x86/vermicelli.hpp | 125 ++++++++++++++++++++++++++++++++++++ 3 files changed, 262 insertions(+), 68 deletions(-) create mode 100644 src/nfa/arm/vermicelli.hpp create mode 100644 src/nfa/x86/vermicelli.hpp diff --git a/src/nfa/arm/vermicelli.hpp b/src/nfa/arm/vermicelli.hpp new file mode 100644 index 00000000..d790fa1f --- /dev/null +++ b/src/nfa/arm/vermicelli.hpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2015-2020, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * \brief Vermicelli: single-byte and double-byte acceleration. + */ + +template +static really_inline +const u8 *vermicelliBlock(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len) { + + SuperVector mask = chars.eq(casemask & data); + return first_non_zero_match(buf, mask, len); +} + +template +static really_inline +const u8 *vermicelliBlockNeg(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len) { + + SuperVector mask = !chars.eq(casemask & data); + return first_zero_match_inverted(buf, mask, len); +} + +template +static really_inline +const u8 *rvermicelliBlock(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len) { + + SuperVector mask = chars.eq(casemask & data); + return last_non_zero_match(buf, mask, len); +} + +template +static really_inline +const u8 *rvermicelliBlockNeg(SuperVector const data, SuperVector const chars, SuperVector const casemask, const u8 *buf, u16 const len) { + + data.print8("data"); + chars.print8("chars"); + casemask.print8("casemask"); + SuperVector mask = !chars.eq(casemask & data); + mask.print8("mask"); + return last_zero_match_inverted(buf, mask, len); +} + +template +static really_inline +const u8 *vermicelliDoubleBlock(SuperVector const data, SuperVector const chars1, SuperVector const chars2, SuperVector const casemask, + u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len) { + + SuperVector v = casemask & data; + SuperVector mask1 = chars1.eq(v); + SuperVector mask2 = chars2.eq(v); + SuperVector mask = mask1 & (mask2 >> 1); + + DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]); + bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1)); + DEBUG_PRINTF("partial = %d\n", partial_match); + if (partial_match) return buf - 1; + + return first_non_zero_match(buf, mask, len); +} + +template +static really_inline +const u8 *rvermicelliDoubleBlock(SuperVector const data, SuperVector const chars1, SuperVector const chars2, SuperVector const casemask, + u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len) { + + SuperVector v = casemask & data; + SuperVector mask1 = chars1.eq(v); + SuperVector mask2 = chars2.eq(v); + SuperVector mask = (mask1 << 1)& mask2; + + DEBUG_PRINTF("buf[0] = %02hhx, buf[-1] = %02hhx\n", buf[0], buf[-1]); + bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1)); + DEBUG_PRINTF("partial = %d\n", partial_match); + if (partial_match) { + mask = mask | (SuperVector::Ones() >> (S-1)); + } + + return last_non_zero_match(buf, mask, len); +} + +template +static really_inline +const u8 *vermicelliDoubleMaskedBlock(SuperVector const data, SuperVector const chars1, SuperVector const chars2, + SuperVector const mask1, SuperVector const mask2, + u8 const c1, u8 const c2, u8 const m1, u8 const m2, u8 const *buf, u16 const len) { + + SuperVector v1 = chars1.eq(data & mask1); + SuperVector v2 = chars2.eq(data & mask2); + SuperVector mask = v1 & (v2 >> 1); + + DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]); + bool partial_match = (((buf[0] & m1) == c2) && ((buf[-1] & m2) == c1)); + DEBUG_PRINTF("partial = %d\n", partial_match); + if (partial_match) return buf - 1; + + return first_non_zero_match(buf, mask, len); +} + diff --git a/src/nfa/vermicelli_simd.cpp b/src/nfa/vermicelli_simd.cpp index e8b7caaf..dbce6dc4 100644 --- a/src/nfa/vermicelli_simd.cpp +++ b/src/nfa/vermicelli_simd.cpp @@ -41,97 +41,41 @@ template static really_inline -const u8 *vermicelliBlock(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len) { - - SuperVector mask = chars.eq(casemask & data); - return first_non_zero_match(buf, mask, len); -} - +const u8 *vermicelliBlock(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len); template static really_inline -const u8 *vermicelliBlockNeg(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len) { - - SuperVector mask = chars.eq(casemask & data); - return first_zero_match_inverted(buf, mask, len); -} +const u8 *vermicelliBlockNeg(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len); template static really_inline -const u8 *rvermicelliBlock(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len) { - - SuperVector mask = chars.eq(casemask & data); - return last_non_zero_match(buf, mask, len); -} - +const u8 *rvermicelliBlock(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len); template static really_inline -const u8 *rvermicelliBlockNeg(SuperVector const data, SuperVector const chars, SuperVector const casemask, const u8 *buf, u16 const len) { - - data.print8("data"); - chars.print8("chars"); - casemask.print8("casemask"); - SuperVector mask = chars.eq(casemask & data); - mask.print8("mask"); - return last_zero_match_inverted(buf, mask, len); -} +const u8 *rvermicelliBlockNeg(SuperVector const data, SuperVector const chars, SuperVector const casemask, const u8 *buf, u16 const len); template static really_inline const u8 *vermicelliDoubleBlock(SuperVector const data, SuperVector const chars1, SuperVector const chars2, SuperVector const casemask, - u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len) { - - SuperVector v = casemask & data; - SuperVector mask1 = chars1.eq(v); - SuperVector mask2 = chars2.eq(v); - SuperVector mask = mask1 & (mask2 >> 1); - - DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]); - bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1)); - DEBUG_PRINTF("partial = %d\n", partial_match); - if (partial_match) return buf - 1; - - return first_non_zero_match(buf, mask, len); -} + u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len); template static really_inline const u8 *rvermicelliDoubleBlock(SuperVector const data, SuperVector const chars1, SuperVector const chars2, SuperVector const casemask, - u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len) { - - SuperVector v = casemask & data; - SuperVector mask1 = chars1.eq(v); - SuperVector mask2 = chars2.eq(v); - SuperVector mask = (mask1 << 1)& mask2; - - DEBUG_PRINTF("buf[0] = %02hhx, buf[-1] = %02hhx\n", buf[0], buf[-1]); - bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1)); - DEBUG_PRINTF("partial = %d\n", partial_match); - if (partial_match) { - mask = mask | (SuperVector::Ones() >> (S-1)); - } - - return last_non_zero_match(buf, mask, len); -} + u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len); template static really_inline const u8 *vermicelliDoubleMaskedBlock(SuperVector const data, SuperVector const chars1, SuperVector const chars2, SuperVector const mask1, SuperVector const mask2, - u8 const c1, u8 const c2, u8 const m1, u8 const m2, u8 const *buf, u16 const len) { + u8 const c1, u8 const c2, u8 const m1, u8 const m2, u8 const *buf, u16 const len); - SuperVector v1 = chars1.eq(data & mask1); - SuperVector v2 = chars2.eq(data & mask2); - SuperVector mask = v1 & (v2 >> 1); - - DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]); - bool partial_match = (((buf[0] & m1) == c2) && ((buf[-1] & m2) == c1)); - DEBUG_PRINTF("partial = %d\n", partial_match); - if (partial_match) return buf - 1; - - return first_non_zero_match(buf, mask, len); -} +#if defined(ARCH_IA32) || defined(ARCH_X86_64) +#include "x86/vermicelli.hpp" +#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) +#include "arm/vermicelli.hpp" +#endif template static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector const casemask, u8 const *buf, u8 const *buf_end) { diff --git a/src/nfa/x86/vermicelli.hpp b/src/nfa/x86/vermicelli.hpp new file mode 100644 index 00000000..8b461dfe --- /dev/null +++ b/src/nfa/x86/vermicelli.hpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2015-2020, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * \brief Vermicelli: single-byte and double-byte acceleration. + */ + +template +static really_inline +const u8 *vermicelliBlock(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len) { + + SuperVector mask = chars.eq(casemask & data); + return first_non_zero_match(buf, mask, len); +} + +template +static really_inline +const u8 *vermicelliBlockNeg(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len) { + + SuperVector mask = chars.eq(casemask & data); + return first_zero_match_inverted(buf, mask, len); +} + +template +static really_inline +const u8 *rvermicelliBlock(SuperVector const data, SuperVector const chars, SuperVector const casemask, u8 const *buf, u16 const len) { + + SuperVector mask = chars.eq(casemask & data); + return last_non_zero_match(buf, mask, len); +} + +template +static really_inline +const u8 *rvermicelliBlockNeg(SuperVector const data, SuperVector const chars, SuperVector const casemask, const u8 *buf, u16 const len) { + + data.print8("data"); + chars.print8("chars"); + casemask.print8("casemask"); + SuperVector mask = chars.eq(casemask & data); + mask.print8("mask"); + return last_zero_match_inverted(buf, mask, len); +} + +template +static really_inline +const u8 *vermicelliDoubleBlock(SuperVector const data, SuperVector const chars1, SuperVector const chars2, SuperVector const casemask, + u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len) { + + SuperVector v = casemask & data; + SuperVector mask1 = chars1.eq(v); + SuperVector mask2 = chars2.eq(v); + SuperVector mask = mask1 & (mask2 >> 1); + + DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]); + bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1)); + DEBUG_PRINTF("partial = %d\n", partial_match); + if (partial_match) return buf - 1; + + return first_non_zero_match(buf, mask, len); +} + +template +static really_inline +const u8 *rvermicelliDoubleBlock(SuperVector const data, SuperVector const chars1, SuperVector const chars2, SuperVector const casemask, + u8 const c1, u8 const c2, u8 const casechar, u8 const *buf, u16 const len) { + + SuperVector v = casemask & data; + SuperVector mask1 = chars1.eq(v); + SuperVector mask2 = chars2.eq(v); + SuperVector mask = (mask1 << 1)& mask2; + + DEBUG_PRINTF("buf[0] = %02hhx, buf[-1] = %02hhx\n", buf[0], buf[-1]); + bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1)); + DEBUG_PRINTF("partial = %d\n", partial_match); + if (partial_match) { + mask = mask | (SuperVector::Ones() >> (S-1)); + } + + return last_non_zero_match(buf, mask, len); +} + +template +static really_inline +const u8 *vermicelliDoubleMaskedBlock(SuperVector const data, SuperVector const chars1, SuperVector const chars2, + SuperVector const mask1, SuperVector const mask2, + u8 const c1, u8 const c2, u8 const m1, u8 const m2, u8 const *buf, u16 const len) { + + SuperVector v1 = chars1.eq(data & mask1); + SuperVector v2 = chars2.eq(data & mask2); + SuperVector mask = v1 & (v2 >> 1); + + DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]); + bool partial_match = (((buf[0] & m1) == c2) && ((buf[-1] & m2) == c1)); + DEBUG_PRINTF("partial = %d\n", partial_match); + if (partial_match) return buf - 1; + + return first_non_zero_match(buf, mask, len); +} + From 41b98d7d8f5a53b8c1c67b5ca712851439c81ca1 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Mon, 8 Nov 2021 19:45:36 +0000 Subject: [PATCH 25/25] add len parameter to arm matchers as well --- src/util/arch/arm/match.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/util/arch/arm/match.hpp b/src/util/arch/arm/match.hpp index ba5f797f..892c3877 100644 --- a/src/util/arch/arm/match.hpp +++ b/src/util/arch/arm/match.hpp @@ -29,7 +29,7 @@ template <> really_really_inline -const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> mask) { +const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> mask, u16 const UNUSED len) { uint32x4_t m = mask.u.u32x4[0]; uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0); if (vmax != 0) { @@ -48,7 +48,7 @@ const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> mask) { template <> really_really_inline -const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> mask) { +const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> mask, u16 const UNUSED len) { uint32x4_t m = mask.u.u32x4[0]; uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0); if (vmax != 0) { @@ -66,7 +66,7 @@ const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> mask) { template <> really_really_inline -const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask) { +const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask, u16 const UNUSED len) { uint32x4_t m = mask.u.u32x4[0]; uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0); if (vmax != 0) { @@ -85,7 +85,7 @@ const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask) { template <> really_really_inline -const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask) { +const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask, u16 const UNUSED len) { uint32x4_t m = mask.u.u32x4[0]; uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0); if (vmax != 0) {