From 81fba99f3a11a276e85457c5982bd547d7e1c193 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Thu, 25 Nov 2021 18:48:24 +0200 Subject: [PATCH] fix SVE2 build after the changes --- CMakeLists.txt | 7 ++- src/hwlm/noodle_engine_sve.hpp | 8 +-- src/nfa/vermicelli.hpp | 6 +- src/nfa/vermicelli_sve.h | 108 ++++++++++++++++++++++++++++++--- 4 files changed, 114 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e3b5a2ee..a741961c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -634,7 +634,6 @@ set (hs_exec_SRCS src/nfa/truffle.h src/nfa/vermicelli.hpp src/nfa/vermicelli_run.h - src/nfa/vermicelli_simd.cpp src/som/som.h src/som/som_operation.h src/som/som_runtime.h @@ -702,6 +701,12 @@ set (hs_exec_SRCS endif () endif() +if (NOT BUILD_SVE2) +set (hs_exec_SRCS + ${hs_exec_SRCS} + src/nfa/vermicelli_simd.cpp) +endif() + set (hs_exec_avx2_SRCS src/fdr/teddy_avx2.c src/util/arch/x86/masked_move.c diff --git a/src/hwlm/noodle_engine_sve.hpp b/src/hwlm/noodle_engine_sve.hpp index aece9c82..cc2d7700 100644 --- a/src/hwlm/noodle_engine_sve.hpp +++ b/src/hwlm/noodle_engine_sve.hpp @@ -170,7 +170,7 @@ hwlm_error_t scanDoubleOnce(const struct noodTable *n, const u8 *buf, svbool_t pg = svwhilelt_b8_s64(0, e - d); svbool_t pg_rot = svwhilelt_b8_s64(0, e - d + 1); svbool_t matched, matched_rot; - svbool_t any = doubleMatched(chars, d, pg, pg_rot, &matched, &matched_rot); + svbool_t any = doubleMatched(svreinterpret_u16(chars), d, pg, pg_rot, &matched, &matched_rot); return doubleCheckMatched(n, buf, len, cbi, d, matched, matched_rot, any); } @@ -187,7 +187,7 @@ hwlm_error_t scanDoubleLoop(const struct noodTable *n, const u8 *buf, for (size_t i = 0; i < loops; i++, d += svcntb()) { DEBUG_PRINTF("d %p \n", d); svbool_t matched, matched_rot; - svbool_t any = doubleMatched(chars, d, svptrue_b8(), svptrue_b8(), + svbool_t any = doubleMatched(svreinterpret_u16(chars), d, svptrue_b8(), svptrue_b8(), &matched, &matched_rot); hwlm_error_t rv = doubleCheckMatched(n, buf, len, cbi, d, matched, matched_rot, any); @@ -220,7 +220,7 @@ hwlm_error_t scanDouble(const struct noodTable *n, const u8 *buf, size_t len, } ++d; - svuint16_t chars = getCharMaskDouble(n->key0, n->key1, noCase); + svuint8_t chars = svreinterpret_u8(getCharMaskDouble(n->key0, n->key1, noCase)); if (scan_len <= svcntb()) { return scanDoubleOnce(n, buf, len, cbi, chars, d, e); @@ -234,4 +234,4 @@ hwlm_error_t scanDouble(const struct noodTable *n, const u8 *buf, size_t len, RETURN_IF_TERMINATED(rv); } return scanDoubleLoop(n, buf, len, cbi, chars, d1, e); -} \ No newline at end of file +} diff --git a/src/nfa/vermicelli.hpp b/src/nfa/vermicelli.hpp index 105194b1..f4958ada 100644 --- a/src/nfa/vermicelli.hpp +++ b/src/nfa/vermicelli.hpp @@ -39,7 +39,7 @@ #ifdef HAVE_SVE2 #include "vermicelli_sve.h" -#endif +#else #ifdef __cplusplus extern "C" { @@ -97,4 +97,6 @@ const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char m2, const u } #endif -#endif /* VERMICELLI_HPP */ \ No newline at end of file +#endif + +#endif /* VERMICELLI_HPP */ diff --git a/src/nfa/vermicelli_sve.h b/src/nfa/vermicelli_sve.h index 42476a69..13f84341 100644 --- a/src/nfa/vermicelli_sve.h +++ b/src/nfa/vermicelli_sve.h @@ -270,25 +270,24 @@ static really_inline const u8 *dvermSearch(svuint8_t chars, const u8 *buf, const u8 *buf_end) { size_t len = buf_end - buf; if (len <= svcntb()) { - return dvermSearchOnce(chars, buf, buf_end); + return dvermSearchOnce(svreinterpret_u16(chars), buf, buf_end); } // peel off first part to align to the vector size const u8 *aligned_buf = ROUNDUP_PTR(buf, svcntb_pat(SV_POW2)); assert(aligned_buf < buf_end); if (buf != aligned_buf) { - const u8 *ptr = dvermSearchLoopBody(chars, buf); + const u8 *ptr = dvermSearchLoopBody(svreinterpret_u16(chars), buf); if (ptr) return ptr; } buf = aligned_buf; size_t loops = (buf_end - buf) / svcntb(); DEBUG_PRINTF("loops %zu \n", loops); for (size_t i = 0; i < loops; i++, buf += svcntb()) { - const u8 *ptr = dvermSearchLoopBody(chars, buf); + const u8 *ptr = dvermSearchLoopBody(svreinterpret_u16(chars), buf); if (ptr) return ptr; } DEBUG_PRINTF("buf %p buf_end %p \n", buf, buf_end); - return buf == buf_end ? NULL : dvermSearchLoopBody(chars, - buf_end - svcntb()); + return buf == buf_end ? NULL : dvermSearchLoopBody(svreinterpret_u16(chars), buf_end - svcntb()); } static really_inline @@ -372,7 +371,7 @@ const u8 *vermicelliDoubleExec(char c1, char c2, bool nocase, const u8 *buf, assert(buf < buf_end); if (buf_end - buf > 1) { ++buf; - svuint16_t chars = getCharMaskDouble(c1, c2, nocase); + svuint8_t chars = svreinterpret_u8(getCharMaskDouble(c1, c2, nocase)); const u8 *ptr = dvermSearch(chars, buf, buf_end); if (ptr) { return ptr; @@ -459,7 +458,7 @@ const u8 *vermicelliDouble16Exec(const m128 mask, const u64a firsts, DEBUG_PRINTF("double verm16 scan over %td bytes\n", buf_end - buf); if (buf_end - buf > 1) { ++buf; - svuint16_t chars = svreinterpret_u16(getDupSVEMaskFrom128(mask)); + svuint8_t chars = svreinterpret_u8(getDupSVEMaskFrom128(mask)); const u8 *ptr = dvermSearch(chars, buf, buf_end); if (ptr) { return ptr; @@ -480,7 +479,7 @@ const u8 *vermicelliDoubleMasked16Exec(const m128 mask, char c1, char m1, DEBUG_PRINTF("double verm16 masked scan over %td bytes\n", buf_end - buf); if (buf_end - buf > 1) { ++buf; - svuint16_t chars = svreinterpret_u16(getDupSVEMaskFrom128(mask)); + svuint8_t chars = getDupSVEMaskFrom128(mask); const u8 *ptr = dvermSearch(chars, buf, buf_end); if (ptr) { return ptr; @@ -494,3 +493,96 @@ const u8 *vermicelliDoubleMasked16Exec(const m128 mask, char c1, char m1, return buf_end; } + +// returns NULL if not found +static really_inline +const u8 *dvermPreconditionMasked(m128 chars1, m128 chars2, + m128 mask1, m128 mask2, const u8 *buf) { + m128 data = loadu128(buf); // unaligned + m128 v1 = eq128(chars1, and128(data, mask1)); + m128 v2 = eq128(chars2, and128(data, mask2)); + u32 z = movemask128(and128(v1, rshiftbyte_m128(v2, 1))); + + /* no fixup of the boundary required - the aligned run will pick it up */ + if (unlikely(z)) { + u32 pos = ctz32(z); + return buf + pos; + } + return NULL; +} + +static really_inline +const u8 *dvermSearchAlignedMasked(m128 chars1, m128 chars2, + m128 mask1, m128 mask2, u8 c1, u8 c2, u8 m1, + u8 m2, const u8 *buf, const u8 *buf_end) { + assert((size_t)buf % 16 == 0); + + for (; buf + 16 < buf_end; buf += 16) { + m128 data = load128(buf); + m128 v1 = eq128(chars1, and128(data, mask1)); + m128 v2 = eq128(chars2, and128(data, mask2)); + u32 z = movemask128(and128(v1, rshiftbyte_m128(v2, 1))); + + if ((buf[15] & m1) == c1 && (buf[16] & m2) == c2) { + z |= (1 << 15); + } + if (unlikely(z)) { + u32 pos = ctz32(z); + return buf + pos; + } + } + + return NULL; +} + +static really_inline +const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char m2, + const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("double verm scan (\\x%02hhx&\\x%02hhx)(\\x%02hhx&\\x%02hhx) " + "over %zu bytes\n", c1, m1, c2, m2, (size_t)(buf_end - buf)); + assert(buf < buf_end); + + m128 chars1 = set1_16x8(c1); + m128 chars2 = set1_16x8(c2); + m128 mask1 = set1_16x8(m1); + m128 mask2 = set1_16x8(m2); + + assert((buf_end - buf) >= 16); + uintptr_t min = (uintptr_t)buf % 16; + if (min) { + // Input isn't aligned, so we need to run one iteration with an + // unaligned load, then skip buf forward to the next aligned address. + // There's some small overlap here, but we don't mind scanning it twice + // if we can do it quickly, do we? + const u8 *p = dvermPreconditionMasked(chars1, chars2, mask1, mask2, buf); + if (p) { + return p; + } + + buf += 16 - min; + assert(buf < buf_end); + } + + // Aligned loops from here on in + const u8 *ptr = dvermSearchAlignedMasked(chars1, chars2, mask1, mask2, c1, + c2, m1, m2, buf, buf_end); + if (ptr) { + return ptr; + } + + // Tidy up the mess at the end + ptr = dvermPreconditionMasked(chars1, chars2, mask1, mask2, + buf_end - 16); + + if (ptr) { + return ptr; + } + + /* check for partial match at end */ + if ((buf_end[-1] & m1) == (u8)c1) { + DEBUG_PRINTF("partial!!!\n"); + return buf_end - 1; + } + + return buf_end; +}