From c95a4c3dd18e02163afb00e348bc2e439c977069 Mon Sep 17 00:00:00 2001 From: George Wort Date: Tue, 13 Jul 2021 15:09:38 +0100 Subject: [PATCH] Use SVE for single shufti. Change-Id: Ic76940c5bb9b81a1c45d39e9ca396a158c50a7dc --- src/nfa/shufti.cpp | 21 ++-- src/nfa/shufti_common.hpp | 177 +++++++++++++++++++++++++++++++++ src/nfa/shufti_simd.hpp | 45 +++------ src/nfa/shufti_sve.hpp | 151 ++++++++++++++++++++++++++++ src/nfa/vermicelli_sve.h | 42 ++------ src/util/arch/arm/simd_utils.h | 36 +++++++ 6 files changed, 390 insertions(+), 82 deletions(-) create mode 100644 src/nfa/shufti_common.hpp create mode 100644 src/nfa/shufti_sve.hpp diff --git a/src/nfa/shufti.cpp b/src/nfa/shufti.cpp index d78a7054..89352047 100644 --- a/src/nfa/shufti.cpp +++ b/src/nfa/shufti.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2015-2017, Intel Corporation * Copyright (c) 2020, 2021, VectorCamp PC + * Copyright (c) 2021, Arm Limited * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -69,20 +70,10 @@ const u8 *shuftiRevSlow(const u8 *lo, const u8 *hi, const u8 *buf, return buf_end; } +#ifdef HAVE_SVE +#include "shufti_sve.hpp" +#else #include "shufti_simd.hpp" +#endif -const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - const u8 *buf_end) { - return shuftiExecReal(mask_lo, mask_hi, buf, buf_end); -} - -const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - const u8 *buf_end) { - return rshuftiExecReal(mask_lo, mask_hi, buf, buf_end); -} - -const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, - m128 mask2_lo, m128 mask2_hi, - const u8 *buf, const u8 *buf_end) { - return shuftiDoubleExecReal(mask1_lo, mask1_hi, mask2_lo, mask2_hi, buf, buf_end); -} +#include "shufti_common.hpp" \ No newline at end of file diff --git a/src/nfa/shufti_common.hpp b/src/nfa/shufti_common.hpp new file mode 100644 index 00000000..ccd06923 --- /dev/null +++ b/src/nfa/shufti_common.hpp @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2021, Arm Limited + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * \brief Shufti: character class acceleration. + * + * Utilises the SSSE3 pshufb shuffle instruction + */ + +#include "shufti.h" +#include "ue2common.h" +#include "util/arch.h" +#include "util/bitutils.h" +#include "util/unaligned.h" + +#include "util/supervector/supervector.hpp" +#include "util/match.hpp" + +template +static really_inline +const u8 *fwdBlockDouble(SuperVector mask1_lo, SuperVector mask1_hi, SuperVector mask2_lo, SuperVector mask2_hi, + SuperVector chars, const u8 *buf) { + + const SuperVector low4bits = SuperVector::dup_u8(0xf); + SuperVector chars_lo = chars & low4bits; + chars_lo.print8("chars_lo"); + SuperVector chars_hi = chars.rshift64(4) & low4bits; + chars_hi.print8("chars_hi"); + SuperVector c1_lo = mask1_lo.pshufb(chars_lo); + c1_lo.print8("c1_lo"); + SuperVector c1_hi = mask1_hi.pshufb(chars_hi); + c1_hi.print8("c1_hi"); + SuperVector t1 = c1_lo | c1_hi; + t1.print8("t1"); + + SuperVector c2_lo = mask2_lo.pshufb(chars_lo); + c2_lo.print8("c2_lo"); + SuperVector c2_hi = mask2_hi.pshufb(chars_hi); + c2_hi.print8("c2_hi"); + SuperVector t2 = c2_lo | c2_hi; + t2.print8("t2"); + t2.rshift128(1).print8("t2.rshift128(1)"); + SuperVector t = t1 | (t2.rshift128(1)); + t.print8("t"); + + typename SuperVector::movemask_type z = t.eqmask(SuperVector::Ones()); + DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); + return firstMatch(buf, z); +} + +template +static really_inline const u8 *shuftiDoubleMini(SuperVector mask1_lo, SuperVector mask1_hi, SuperVector mask2_lo, SuperVector mask2_hi, + const u8 *buf, const u8 *buf_end){ + uintptr_t len = buf_end - buf; + assert(len < S); + + const SuperVector low4bits = SuperVector::dup_u8(0xf); + + DEBUG_PRINTF("buf %p buf_end %p \n", buf, buf_end); + SuperVector chars = SuperVector::loadu_maskz(buf, len); + chars.print8("chars"); + + SuperVector chars_lo = chars & low4bits; + chars_lo.print8("chars_lo"); + SuperVector chars_hi = chars.rshift64(4) & low4bits; + chars_hi.print8("chars_hi"); + SuperVector c1_lo = mask1_lo.pshufb_maskz(chars_lo, len); + c1_lo.print8("c1_lo"); + SuperVector c1_hi = mask1_hi.pshufb_maskz(chars_hi, len); + c1_hi.print8("c1_hi"); + SuperVector t1 = c1_lo | c1_hi; + t1.print8("t1"); + + SuperVector c2_lo = mask2_lo.pshufb_maskz(chars_lo, len); + c2_lo.print8("c2_lo"); + SuperVector c2_hi = mask2_hi.pshufb_maskz(chars_hi, len); + c2_hi.print8("c2_hi"); + SuperVector t2 = c2_lo | c2_hi; + t2.print8("t2"); + t2.rshift128(1).print8("t2.rshift128(1)"); + SuperVector t = t1 | (t2.rshift128(1)); + t.print8("t"); + + typename SuperVector::movemask_type z = t.eqmask(SuperVector::Ones()); + DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); + return firstMatch(buf, z); +} + +template +const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 mask2_hi, + const u8 *buf, const u8 *buf_end) { + assert(buf && buf_end); + assert(buf < buf_end); + DEBUG_PRINTF("shufti %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("b %s\n", buf); + + const SuperVector wide_mask1_lo(mask1_lo); + const SuperVector wide_mask1_hi(mask1_hi); + const SuperVector wide_mask2_lo(mask2_lo); + const SuperVector wide_mask2_hi(mask2_hi); + + const u8 *d = buf; + const u8 *rv; + + DEBUG_PRINTF("start %p end %p \n", d, buf_end); + assert(d < buf_end); + if (d + S <= buf_end) { + // peel off first part to cacheline boundary + const u8 *d1 = ROUNDUP_PTR(d, S); + DEBUG_PRINTF("until aligned %p \n", d1); + if (d1 != d) { + SuperVector chars = SuperVector::loadu(d); + rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, d); + DEBUG_PRINTF("rv %p \n", rv); + if (rv) return rv; + d = d1; + } + + size_t loops = (buf_end - d) / S; + DEBUG_PRINTF("loops %ld \n", loops); + + for (size_t i = 0; i < loops; i++, d+= S) { + DEBUG_PRINTF("it = %ld, d %p \n", i, d); + const u8 *base = ROUNDUP_PTR(d, S); + // On large packet buffers, this prefetch appears to get us about 2%. + __builtin_prefetch(base + 256); + + SuperVector chars = SuperVector::load(d); + rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, d); + if (rv) return rv; + } + } + + DEBUG_PRINTF("tail d %p e %p \n", d, buf_end); + // finish off tail + + if (d != buf_end) { + rv = shuftiDoubleMini(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, d, buf_end); + DEBUG_PRINTF("rv %p \n", rv); + if (rv >= buf && rv < buf_end) return rv; + } + + return buf_end; +} + +const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, + m128 mask2_lo, m128 mask2_hi, + const u8 *buf, const u8 *buf_end) { + return shuftiDoubleExecReal(mask1_lo, mask1_hi, mask2_lo, mask2_hi, buf, buf_end); +} \ No newline at end of file diff --git a/src/nfa/shufti_simd.hpp b/src/nfa/shufti_simd.hpp index 818798c4..a8b9352b 100644 --- a/src/nfa/shufti_simd.hpp +++ b/src/nfa/shufti_simd.hpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2015-2017, Intel Corporation * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2021, Arm Limited * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -82,7 +83,7 @@ const u8 *shortShufti(SuperVector mask_lo, SuperVector mask_hi, const u8 * DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); z &= maskb | maske; DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); - + return firstMatch(buf, z); }*/ @@ -146,7 +147,7 @@ const u8 *shuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *bu // rv = shortShufti(wide_mask_lo, wide_mask_hi, buf_end - S, buf_end); DEBUG_PRINTF("rv %p \n", rv); } - + return rv; } @@ -199,40 +200,13 @@ const u8 *rshuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *b DEBUG_PRINTF("rv %p \n", rv); if (rv) return rv; } - + return buf - 1; } -template -static really_inline -const u8 *fwdBlockDouble(SuperVector mask1_lo, SuperVector mask1_hi, SuperVector mask2_lo, SuperVector mask2_hi, - SuperVector chars, const u8 *buf) { - - const SuperVector low4bits = SuperVector::dup_u8(0xf); - SuperVector chars_lo = chars & low4bits; - chars_lo.print8("chars_lo"); - SuperVector chars_hi = chars.rshift64(4) & low4bits; - chars_hi.print8("chars_hi"); - SuperVector c1_lo = mask1_lo.pshufb(chars_lo); - c1_lo.print8("c1_lo"); - SuperVector c1_hi = mask1_hi.pshufb(chars_hi); - c1_hi.print8("c1_hi"); - SuperVector t1 = c1_lo | c1_hi; - t1.print8("t1"); - - SuperVector c2_lo = mask2_lo.pshufb(chars_lo); - c2_lo.print8("c2_lo"); - SuperVector c2_hi = mask2_hi.pshufb(chars_hi); - c2_hi.print8("c2_hi"); - SuperVector t2 = c2_lo | c2_hi; - t2.print8("t2"); - t2.rshift128(1).print8("t2.rshift128(1)"); - SuperVector t = t1 | (t2.rshift128(1)); - t.print8("t"); - - typename SuperVector::movemask_type z = t.eqmask(SuperVector::Ones()); - DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); - return firstMatch(buf, z); +const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, + const u8 *buf_end) { + return shuftiExecReal(mask_lo, mask_hi, buf, buf_end); } template @@ -326,3 +300,8 @@ const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 return buf_end; } + +const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, + const u8 *buf_end) { + return rshuftiExecReal(mask_lo, mask_hi, buf, buf_end); +} diff --git a/src/nfa/shufti_sve.hpp b/src/nfa/shufti_sve.hpp new file mode 100644 index 00000000..f2c5295e --- /dev/null +++ b/src/nfa/shufti_sve.hpp @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2021, Arm Limited + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * \brief Shufti: character class acceleration. + * + * Utilises the SVE tbl shuffle instruction + */ + +static really_inline +svbool_t singleMatched(svuint8_t mask_lo, svuint8_t mask_hi, + const u8 *buf, svbool_t pg) { + svuint8_t vec = svld1_u8(pg, buf); + svuint8_t c_lo = svtbl(mask_lo, svand_z(svptrue_b8(), vec, (uint8_t)0xf)); + svuint8_t c_hi = svtbl(mask_hi, svlsr_z(svptrue_b8(), vec, 4)); + svuint8_t t = svand_z(svptrue_b8(), c_lo, c_hi); + return svcmpne(pg, t, (uint8_t)0); +} + +static really_inline +const u8 *shuftiOnce(svuint8_t mask_lo, svuint8_t mask_hi, + const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("start %p end %p\n", buf, buf_end); + assert(buf <= buf_end); + svbool_t pg = svwhilelt_b8_s64(0, buf_end - buf); + svbool_t matched = singleMatched(mask_lo, mask_hi, buf, pg); + return accelSearchCheckMatched(buf, matched); +} + +static really_inline +const u8 *shuftiLoopBody(svuint8_t mask_lo, svuint8_t mask_hi, const u8 *buf) { + DEBUG_PRINTF("start %p end %p\n", buf, buf + svcntb()); + svbool_t matched = singleMatched(mask_lo, mask_hi, buf, svptrue_b8()); + return accelSearchCheckMatched(buf, matched); +} + +static really_inline +const u8 *rshuftiOnce(svuint8_t mask_lo, svuint8_t mask_hi, + const u8 *buf, const u8 *buf_end) { + DEBUG_PRINTF("start %p end %p\n", buf, buf_end); + assert(buf <= buf_end); + DEBUG_PRINTF("l = %td\n", buf_end - buf); + svbool_t pg = svwhilelt_b8_s64(0, buf_end - buf); + svbool_t matched = singleMatched(mask_lo, mask_hi, buf, pg); + return accelRevSearchCheckMatched(buf, matched); +} + +static really_inline +const u8 *rshuftiLoopBody(svuint8_t mask_lo, svuint8_t mask_hi, + const u8 *buf) { + svbool_t matched = singleMatched(mask_lo, mask_hi, buf, svptrue_b8()); + return accelRevSearchCheckMatched(buf, matched); +} + +static really_inline +const u8 *shuftiSearch(svuint8_t mask_lo, svuint8_t mask_hi, + const u8 *buf, const u8 *buf_end) { + assert(buf < buf_end); + size_t len = buf_end - buf; + if (len <= svcntb()) { + return shuftiOnce(mask_lo, mask_hi, buf, buf_end); + } + // peel off first part to align to the vector size + const u8 *aligned_buf = ROUNDUP_PTR(buf, svcntb_pat(SV_POW2)); + assert(aligned_buf < buf_end); + if (buf != aligned_buf) { + const u8 *ptr = shuftiLoopBody(mask_lo, mask_hi, buf); + if (ptr) return ptr; + } + buf = aligned_buf; + size_t loops = (buf_end - buf) / svcntb(); + DEBUG_PRINTF("loops %zu \n", loops); + for (size_t i = 0; i < loops; i++, buf += svcntb()) { + const u8 *ptr = shuftiLoopBody(mask_lo, mask_hi, buf); + if (ptr) return ptr; + } + DEBUG_PRINTF("buf %p buf_end %p \n", buf, buf_end); + return buf == buf_end ? NULL : shuftiLoopBody(mask_lo, mask_hi, + buf_end - svcntb()); +} + +static really_inline +const u8 *rshuftiSearch(svuint8_t mask_lo, svuint8_t mask_hi, + const u8 *buf, const u8 *buf_end) { + assert(buf < buf_end); + size_t len = buf_end - buf; + if (len <= svcntb()) { + return rshuftiOnce(mask_lo, mask_hi, buf, buf_end); + } + // peel off first part to align to the vector size + const u8 *aligned_buf_end = ROUNDDOWN_PTR(buf_end, svcntb_pat(SV_POW2)); + assert(buf < aligned_buf_end); + if (buf_end != aligned_buf_end) { + const u8 *ptr = rshuftiLoopBody(mask_lo, mask_hi, buf_end - svcntb()); + if (ptr) return ptr; + } + buf_end = aligned_buf_end; + size_t loops = (buf_end - buf) / svcntb(); + DEBUG_PRINTF("loops %zu \n", loops); + for (size_t i = 0; i < loops; i++) { + buf_end -= svcntb(); + const u8 *ptr = rshuftiLoopBody(mask_lo, mask_hi, buf_end); + if (ptr) return ptr; + } + DEBUG_PRINTF("buf %p buf_end %p \n", buf, buf_end); + return buf == buf_end ? NULL : rshuftiLoopBody(mask_lo, mask_hi, buf); +} + +const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, + const u8 *buf_end) { + DEBUG_PRINTF("shufti scan over %td bytes\n", buf_end - buf); + svuint8_t sve_mask_lo = getSVEMaskFrom128(mask_lo); + svuint8_t sve_mask_hi = getSVEMaskFrom128(mask_hi); + const u8 *ptr = shuftiSearch(sve_mask_lo, sve_mask_hi, buf, buf_end); + return ptr ? ptr : buf_end; +} + +const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, + const u8 *buf_end) { + DEBUG_PRINTF("rshufti scan over %td bytes\n", buf_end - buf); + svuint8_t sve_mask_lo = getSVEMaskFrom128(mask_lo); + svuint8_t sve_mask_hi = getSVEMaskFrom128(mask_hi); + const u8 *ptr = rshuftiSearch(sve_mask_lo, sve_mask_hi, buf, buf_end); + return ptr ? ptr : buf - 1; +} \ No newline at end of file diff --git a/src/nfa/vermicelli_sve.h b/src/nfa/vermicelli_sve.h index cadaac8e..88ed688c 100644 --- a/src/nfa/vermicelli_sve.h +++ b/src/nfa/vermicelli_sve.h @@ -32,15 +32,10 @@ * (users should include vermicelli.h instead of this) */ -static really_inline -uint64_t vermSearchGetOffset(svbool_t matched) { - return svcntp_b8(svptrue_b8(), svbrkb_z(svptrue_b8(), matched)); -} - static really_inline int dvermSearchGetOffset(svbool_t matched, svbool_t matched_rot) { - int offset = vermSearchGetOffset(matched); - int offset_rot = vermSearchGetOffset(matched_rot) - 1; + int offset = accelSearchGetOffset(matched); + int offset_rot = accelSearchGetOffset(matched_rot) - 1; return (offset_rot < offset) ? offset_rot : offset; } @@ -56,27 +51,6 @@ uint64_t rdvermSearchGetOffset(svbool_t matched, svbool_t matched_rot) { return (offset_rot < offset) ? offset_rot : offset; } -static really_inline -const u8 *vermSearchCheckMatched(const u8 *buf, svbool_t matched) { - if (unlikely(svptest_any(svptrue_b8(), matched))) { - const u8 *matchPos = buf + vermSearchGetOffset(matched); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - -static really_inline -const u8 *rvermSearchCheckMatched(const u8 *buf, svbool_t matched) { - if (unlikely(svptest_any(svptrue_b8(), matched))) { - const u8 *matchPos = buf + (svcntb() - - svcntp_b8(svptrue_b8(), svbrka_z(svptrue_b8(), svrev_b8(matched)))); - DEBUG_PRINTF("match pos %p\n", matchPos); - return matchPos; - } - return NULL; -} - static really_inline const u8 *dvermSearchCheckMatched(const u8 *buf, svbool_t matched, svbool_t matched_rot, svbool_t any) { @@ -130,14 +104,14 @@ const u8 *vermSearchOnce(svuint8_t chars, const u8 *buf, const u8 *buf_end, DEBUG_PRINTF("l = %td\n", buf_end - buf); svbool_t pg = svwhilelt_b8_s64(0, buf_end - buf); svbool_t matched = singleMatched(chars, buf, pg, negate, 0); - return vermSearchCheckMatched(buf, matched); + return accelSearchCheckMatched(buf, matched); } static really_inline const u8 *vermSearchLoopBody(svuint8_t chars, const u8 *buf, bool negate) { DEBUG_PRINTF("start %p end %p\n", buf, buf + svcntb()); svbool_t matched = singleMatched(chars, buf, svptrue_b8(), negate, 0); - return vermSearchCheckMatched(buf, matched); + return accelSearchCheckMatched(buf, matched); } static really_inline @@ -149,9 +123,9 @@ const u8 *vermSearchLoopBodyUnrolled(svuint8_t chars, const u8 *buf, svbool_t any = svorr_z(svptrue_b8(), matched0, matched1); if (unlikely(svptest_any(svptrue_b8(), any))) { if (svptest_any(svptrue_b8(), matched0)) { - return buf + vermSearchGetOffset(matched0); + return buf + accelSearchGetOffset(matched0); } else { - return buf + svcntb() + vermSearchGetOffset(matched1); + return buf + svcntb() + accelSearchGetOffset(matched1); } } return NULL; @@ -165,14 +139,14 @@ const u8 *rvermSearchOnce(svuint8_t chars, const u8 *buf, const u8 *buf_end, DEBUG_PRINTF("l = %td\n", buf_end - buf); svbool_t pg = svwhilelt_b8_s64(0, buf_end - buf); svbool_t matched = singleMatched(chars, buf, pg, negate, 0); - return rvermSearchCheckMatched(buf, matched); + return accelRevSearchCheckMatched(buf, matched); } static really_inline const u8 *rvermSearchLoopBody(svuint8_t chars, const u8 *buf, bool negate) { DEBUG_PRINTF("start %p end %p\n", buf, buf + svcntb()); svbool_t matched = singleMatched(chars, buf, svptrue_b8(), negate, 0); - return rvermSearchCheckMatched(buf, matched); + return accelRevSearchCheckMatched(buf, matched); } static really_inline diff --git a/src/util/arch/arm/simd_utils.h b/src/util/arch/arm/simd_utils.h index 9e73e931..e5bc2948 100644 --- a/src/util/arch/arm/simd_utils.h +++ b/src/util/arch/arm/simd_utils.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2020, Intel Corporation + * Copyright (c) 2021, Arm Limited * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,6 +42,41 @@ #include "util/unaligned.h" #include "util/intrinsics.h" +#ifdef HAVE_SVE + +really_really_inline +uint64_t accelSearchGetOffset(svbool_t matched) { + return svcntp_b8(svptrue_b8(), svbrkb_z(svptrue_b8(), matched)); +} + +really_really_inline +const u8 *accelSearchCheckMatched(const u8 *buf, svbool_t matched) { + if (unlikely(svptest_any(svptrue_b8(), matched))) { + const u8 *matchPos = buf + accelSearchGetOffset(matched); + DEBUG_PRINTF("match pos %p\n", matchPos); + return matchPos; + } + return NULL; +} + +really_really_inline +const u8 *accelRevSearchCheckMatched(const u8 *buf, svbool_t matched) { + if (unlikely(svptest_any(svptrue_b8(), matched))) { + const u8 *matchPos = buf + (svcntb() - + svcntp_b8(svptrue_b8(), svbrka_z(svptrue_b8(), svrev_b8(matched)))); + DEBUG_PRINTF("match pos %p\n", matchPos); + return matchPos; + } + return NULL; +} + +static really_inline +svuint8_t getSVEMaskFrom128(m128 mask) { + return svld1_u8(svptrue_pat_b8(SV_VL16), (const uint8_t *)&mask); +} + +#endif + #ifdef HAVE_SVE2 static really_inline