github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-06-28 16:41:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Use SVE for single shufti.

Browse Source 
Change-Id: Ic76940c5bb9b81a1c45d39e9ca396a158c50a7dc
This commit is contained in:
George Wort 2021-07-13 15:09:38 +01:00 committed by Konstantinos Margaritis
parent 56ef2d5f72
commit c95a4c3dd1
6 changed files with 390 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/nfa/shufti.cpp
View File

@ -1,6 +1,7 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2020, 2021, VectorCamp PC
* Copyright (c) 2021, Arm Limited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -69,20 +70,10 @@ const u8 *shuftiRevSlow(const u8 *lo, const u8 *hi, const u8 *buf,
return buf_end;
}
#ifdef HAVE_SVE
#include "shufti_sve.hpp"
#else
#include "shufti_simd.hpp"
#endif
const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
const u8 *buf_end) {
return shuftiExecReal<VECTORSIZE>(mask_lo, mask_hi, buf, buf_end);
}
const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
const u8 *buf_end) {
return rshuftiExecReal<VECTORSIZE>(mask_lo, mask_hi, buf, buf_end);
}
const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi,
m128 mask2_lo, m128 mask2_hi,
const u8 *buf, const u8 *buf_end) {
return shuftiDoubleExecReal<VECTORSIZE>(mask1_lo, mask1_hi, mask2_lo, mask2_hi, buf, buf_end);
}
#include "shufti_common.hpp"

177
src/nfa/shufti_common.hpp Normal file
View File

@ -0,0 +1,177 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2020-2021, VectorCamp PC
* Copyright (c) 2021, Arm Limited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief Shufti: character class acceleration.
*
* Utilises the SSSE3 pshufb shuffle instruction
*/
#include "shufti.h"
#include "ue2common.h"
#include "util/arch.h"
#include "util/bitutils.h"
#include "util/unaligned.h"
#include "util/supervector/supervector.hpp"
#include "util/match.hpp"
template <uint16_t S>
static really_inline
const u8 *fwdBlockDouble(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi, SuperVector<S> mask2_lo, SuperVector<S> mask2_hi,
SuperVector<S> chars, const u8 *buf) {
const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
SuperVector<S> chars_lo = chars & low4bits;
chars_lo.print8("chars_lo");
SuperVector<S> chars_hi = chars.rshift64(4) & low4bits;
chars_hi.print8("chars_hi");
SuperVector<S> c1_lo = mask1_lo.pshufb(chars_lo);
c1_lo.print8("c1_lo");
SuperVector<S> c1_hi = mask1_hi.pshufb(chars_hi);
c1_hi.print8("c1_hi");
SuperVector<S> t1 = c1_lo | c1_hi;
t1.print8("t1");
SuperVector<S> c2_lo = mask2_lo.pshufb(chars_lo);
c2_lo.print8("c2_lo");
SuperVector<S> c2_hi = mask2_hi.pshufb(chars_hi);
c2_hi.print8("c2_hi");
SuperVector<S> t2 = c2_lo | c2_hi;
t2.print8("t2");
t2.rshift128(1).print8("t2.rshift128(1)");
SuperVector<S> t = t1 | (t2.rshift128(1));
t.print8("t");
typename SuperVector<S>::movemask_type z = t.eqmask(SuperVector<S>::Ones());
DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
return firstMatch<S>(buf, z);
}
template <uint16_t S>
static really_inline const u8 *shuftiDoubleMini(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi, SuperVector<S> mask2_lo, SuperVector<S> mask2_hi,
const u8 *buf, const u8 *buf_end){
uintptr_t len = buf_end - buf;
assert(len < S);
const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
DEBUG_PRINTF("buf %p buf_end %p \n", buf, buf_end);
SuperVector<S> chars = SuperVector<S>::loadu_maskz(buf, len);
chars.print8("chars");
SuperVector<S> chars_lo = chars & low4bits;
chars_lo.print8("chars_lo");
SuperVector<S> chars_hi = chars.rshift64(4) & low4bits;
chars_hi.print8("chars_hi");
SuperVector<S> c1_lo = mask1_lo.pshufb_maskz(chars_lo, len);
c1_lo.print8("c1_lo");
SuperVector<S> c1_hi = mask1_hi.pshufb_maskz(chars_hi, len);
c1_hi.print8("c1_hi");
SuperVector<S> t1 = c1_lo | c1_hi;
t1.print8("t1");
SuperVector<S> c2_lo = mask2_lo.pshufb_maskz(chars_lo, len);
c2_lo.print8("c2_lo");
SuperVector<S> c2_hi = mask2_hi.pshufb_maskz(chars_hi, len);
c2_hi.print8("c2_hi");
SuperVector<S> t2 = c2_lo | c2_hi;
t2.print8("t2");
t2.rshift128(1).print8("t2.rshift128(1)");
SuperVector<S> t = t1 | (t2.rshift128(1));
t.print8("t");
typename SuperVector<S>::movemask_type z = t.eqmask(SuperVector<S>::Ones());
DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
return firstMatch<S>(buf, z);
}
template <uint16_t S>
const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 mask2_hi,
const u8 *buf, const u8 *buf_end) {
assert(buf && buf_end);
assert(buf < buf_end);
DEBUG_PRINTF("shufti %p len %zu\n", buf, buf_end - buf);
DEBUG_PRINTF("b %s\n", buf);
const SuperVector<S> wide_mask1_lo(mask1_lo);
const SuperVector<S> wide_mask1_hi(mask1_hi);
const SuperVector<S> wide_mask2_lo(mask2_lo);
const SuperVector<S> wide_mask2_hi(mask2_hi);
const u8 *d = buf;
const u8 *rv;
DEBUG_PRINTF("start %p end %p \n", d, buf_end);
assert(d < buf_end);
if (d + S <= buf_end) {
// peel off first part to cacheline boundary
const u8 *d1 = ROUNDUP_PTR(d, S);
DEBUG_PRINTF("until aligned %p \n", d1);
if (d1 != d) {
SuperVector<S> chars = SuperVector<S>::loadu(d);
rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, d);
DEBUG_PRINTF("rv %p \n", rv);
if (rv) return rv;
d = d1;
}
size_t loops = (buf_end - d) / S;
DEBUG_PRINTF("loops %ld \n", loops);
for (size_t i = 0; i < loops; i++, d+= S) {
DEBUG_PRINTF("it = %ld, d %p \n", i, d);
const u8 *base = ROUNDUP_PTR(d, S);
// On large packet buffers, this prefetch appears to get us about 2%.
__builtin_prefetch(base + 256);
SuperVector<S> chars = SuperVector<S>::load(d);
rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, d);
if (rv) return rv;
}
}
DEBUG_PRINTF("tail d %p e %p \n", d, buf_end);
// finish off tail
if (d != buf_end) {
rv = shuftiDoubleMini(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, d, buf_end);
DEBUG_PRINTF("rv %p \n", rv);
if (rv >= buf && rv < buf_end) return rv;
}
return buf_end;
}
const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi,
m128 mask2_lo, m128 mask2_hi,
const u8 *buf, const u8 *buf_end) {
return shuftiDoubleExecReal<VECTORSIZE>(mask1_lo, mask1_hi, mask2_lo, mask2_hi, buf, buf_end);
}

45
src/nfa/shufti_simd.hpp
View File

@ -1,6 +1,7 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2020-2021, VectorCamp PC
* Copyright (c) 2021, Arm Limited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -82,7 +83,7 @@ const u8 *shortShufti(SuperVector<S> mask_lo, SuperVector<S> mask_hi, const u8 *
DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
z &= maskb | maske;
DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
return firstMatch<S>(buf, z);
}*/
@ -146,7 +147,7 @@ const u8 *shuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *bu
// rv = shortShufti(wide_mask_lo, wide_mask_hi, buf_end - S, buf_end);
DEBUG_PRINTF("rv %p \n", rv);
}
return rv;
}
@ -199,40 +200,13 @@ const u8 *rshuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *b
DEBUG_PRINTF("rv %p \n", rv);
if (rv) return rv;
}
return buf - 1;
}
template <uint16_t S>
static really_inline
const u8 *fwdBlockDouble(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi, SuperVector<S> mask2_lo, SuperVector<S> mask2_hi,
SuperVector<S> chars, const u8 *buf) {
const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
SuperVector<S> chars_lo = chars & low4bits;
chars_lo.print8("chars_lo");
SuperVector<S> chars_hi = chars.rshift64(4) & low4bits;
chars_hi.print8("chars_hi");
SuperVector<S> c1_lo = mask1_lo.pshufb(chars_lo);
c1_lo.print8("c1_lo");
SuperVector<S> c1_hi = mask1_hi.pshufb(chars_hi);
c1_hi.print8("c1_hi");
SuperVector<S> t1 = c1_lo | c1_hi;
t1.print8("t1");
SuperVector<S> c2_lo = mask2_lo.pshufb(chars_lo);
c2_lo.print8("c2_lo");
SuperVector<S> c2_hi = mask2_hi.pshufb(chars_hi);
c2_hi.print8("c2_hi");
SuperVector<S> t2 = c2_lo | c2_hi;
t2.print8("t2");
t2.rshift128(1).print8("t2.rshift128(1)");
SuperVector<S> t = t1 | (t2.rshift128(1));
t.print8("t");
typename SuperVector<S>::movemask_type z = t.eqmask(SuperVector<S>::Ones());
DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
return firstMatch<S>(buf, z);
const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
const u8 *buf_end) {
return shuftiExecReal<VECTORSIZE>(mask_lo, mask_hi, buf, buf_end);
}
template <uint16_t S>
@ -326,3 +300,8 @@ const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128
return buf_end;
}
const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
const u8 *buf_end) {
return rshuftiExecReal<VECTORSIZE>(mask_lo, mask_hi, buf, buf_end);
}

151
src/nfa/shufti_sve.hpp Normal file
View File

@ -0,0 +1,151 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2021, Arm Limited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief Shufti: character class acceleration.
*
* Utilises the SVE tbl shuffle instruction
*/
static really_inline
svbool_t singleMatched(svuint8_t mask_lo, svuint8_t mask_hi,
const u8 *buf, svbool_t pg) {
svuint8_t vec = svld1_u8(pg, buf);
svuint8_t c_lo = svtbl(mask_lo, svand_z(svptrue_b8(), vec, (uint8_t)0xf));
svuint8_t c_hi = svtbl(mask_hi, svlsr_z(svptrue_b8(), vec, 4));
svuint8_t t = svand_z(svptrue_b8(), c_lo, c_hi);
return svcmpne(pg, t, (uint8_t)0);
}
static really_inline
const u8 *shuftiOnce(svuint8_t mask_lo, svuint8_t mask_hi,
const u8 *buf, const u8 *buf_end) {
DEBUG_PRINTF("start %p end %p\n", buf, buf_end);
assert(buf <= buf_end);
svbool_t pg = svwhilelt_b8_s64(0, buf_end - buf);
svbool_t matched = singleMatched(mask_lo, mask_hi, buf, pg);
return accelSearchCheckMatched(buf, matched);
}
static really_inline
const u8 *shuftiLoopBody(svuint8_t mask_lo, svuint8_t mask_hi, const u8 *buf) {
DEBUG_PRINTF("start %p end %p\n", buf, buf + svcntb());
svbool_t matched = singleMatched(mask_lo, mask_hi, buf, svptrue_b8());
return accelSearchCheckMatched(buf, matched);
}
static really_inline
const u8 *rshuftiOnce(svuint8_t mask_lo, svuint8_t mask_hi,
const u8 *buf, const u8 *buf_end) {
DEBUG_PRINTF("start %p end %p\n", buf, buf_end);
assert(buf <= buf_end);
DEBUG_PRINTF("l = %td\n", buf_end - buf);
svbool_t pg = svwhilelt_b8_s64(0, buf_end - buf);
svbool_t matched = singleMatched(mask_lo, mask_hi, buf, pg);
return accelRevSearchCheckMatched(buf, matched);
}
static really_inline
const u8 *rshuftiLoopBody(svuint8_t mask_lo, svuint8_t mask_hi,
const u8 *buf) {
svbool_t matched = singleMatched(mask_lo, mask_hi, buf, svptrue_b8());
return accelRevSearchCheckMatched(buf, matched);
}
static really_inline
const u8 *shuftiSearch(svuint8_t mask_lo, svuint8_t mask_hi,
const u8 *buf, const u8 *buf_end) {
assert(buf < buf_end);
size_t len = buf_end - buf;
if (len <= svcntb()) {
return shuftiOnce(mask_lo, mask_hi, buf, buf_end);
}
// peel off first part to align to the vector size
const u8 *aligned_buf = ROUNDUP_PTR(buf, svcntb_pat(SV_POW2));
assert(aligned_buf < buf_end);
if (buf != aligned_buf) {
const u8 *ptr = shuftiLoopBody(mask_lo, mask_hi, buf);
if (ptr) return ptr;
}
buf = aligned_buf;
size_t loops = (buf_end - buf) / svcntb();
DEBUG_PRINTF("loops %zu \n", loops);
for (size_t i = 0; i < loops; i++, buf += svcntb()) {
const u8 *ptr = shuftiLoopBody(mask_lo, mask_hi, buf);
if (ptr) return ptr;
}
DEBUG_PRINTF("buf %p buf_end %p \n", buf, buf_end);
return buf == buf_end ? NULL : shuftiLoopBody(mask_lo, mask_hi,
buf_end - svcntb());
}
static really_inline
const u8 *rshuftiSearch(svuint8_t mask_lo, svuint8_t mask_hi,
const u8 *buf, const u8 *buf_end) {
assert(buf < buf_end);
size_t len = buf_end - buf;
if (len <= svcntb()) {
return rshuftiOnce(mask_lo, mask_hi, buf, buf_end);
}
// peel off first part to align to the vector size
const u8 *aligned_buf_end = ROUNDDOWN_PTR(buf_end, svcntb_pat(SV_POW2));
assert(buf < aligned_buf_end);
if (buf_end != aligned_buf_end) {
const u8 *ptr = rshuftiLoopBody(mask_lo, mask_hi, buf_end - svcntb());
if (ptr) return ptr;
}
buf_end = aligned_buf_end;
size_t loops = (buf_end - buf) / svcntb();
DEBUG_PRINTF("loops %zu \n", loops);
for (size_t i = 0; i < loops; i++) {
buf_end -= svcntb();
const u8 *ptr = rshuftiLoopBody(mask_lo, mask_hi, buf_end);
if (ptr) return ptr;
}
DEBUG_PRINTF("buf %p buf_end %p \n", buf, buf_end);
return buf == buf_end ? NULL : rshuftiLoopBody(mask_lo, mask_hi, buf);
}
const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
const u8 *buf_end) {
DEBUG_PRINTF("shufti scan over %td bytes\n", buf_end - buf);
svuint8_t sve_mask_lo = getSVEMaskFrom128(mask_lo);
svuint8_t sve_mask_hi = getSVEMaskFrom128(mask_hi);
const u8 *ptr = shuftiSearch(sve_mask_lo, sve_mask_hi, buf, buf_end);
return ptr ? ptr : buf_end;
}
const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
const u8 *buf_end) {
DEBUG_PRINTF("rshufti scan over %td bytes\n", buf_end - buf);
svuint8_t sve_mask_lo = getSVEMaskFrom128(mask_lo);
svuint8_t sve_mask_hi = getSVEMaskFrom128(mask_hi);
const u8 *ptr = rshuftiSearch(sve_mask_lo, sve_mask_hi, buf, buf_end);
return ptr ? ptr : buf - 1;
}

42
src/nfa/vermicelli_sve.h
View File

@ -32,15 +32,10 @@
* (users should include vermicelli.h instead of this)
*/
static really_inline
uint64_t vermSearchGetOffset(svbool_t matched) {
return svcntp_b8(svptrue_b8(), svbrkb_z(svptrue_b8(), matched));
}
static really_inline
int dvermSearchGetOffset(svbool_t matched, svbool_t matched_rot) {
int offset = vermSearchGetOffset(matched);
int offset_rot = vermSearchGetOffset(matched_rot) - 1;
int offset = accelSearchGetOffset(matched);
int offset_rot = accelSearchGetOffset(matched_rot) - 1;
return (offset_rot < offset) ? offset_rot : offset;
}
@ -56,27 +51,6 @@ uint64_t rdvermSearchGetOffset(svbool_t matched, svbool_t matched_rot) {
return (offset_rot < offset) ? offset_rot : offset;
}
static really_inline
const u8 *vermSearchCheckMatched(const u8 *buf, svbool_t matched) {
if (unlikely(svptest_any(svptrue_b8(), matched))) {
const u8 *matchPos = buf + vermSearchGetOffset(matched);
DEBUG_PRINTF("match pos %p\n", matchPos);
return matchPos;
}
return NULL;
}
static really_inline
const u8 *rvermSearchCheckMatched(const u8 *buf, svbool_t matched) {
if (unlikely(svptest_any(svptrue_b8(), matched))) {
const u8 *matchPos = buf + (svcntb() -
svcntp_b8(svptrue_b8(), svbrka_z(svptrue_b8(), svrev_b8(matched))));
DEBUG_PRINTF("match pos %p\n", matchPos);
return matchPos;
}
return NULL;
}
static really_inline
const u8 *dvermSearchCheckMatched(const u8 *buf, svbool_t matched,
svbool_t matched_rot, svbool_t any) {
@ -130,14 +104,14 @@ const u8 *vermSearchOnce(svuint8_t chars, const u8 *buf, const u8 *buf_end,
DEBUG_PRINTF("l = %td\n", buf_end - buf);
svbool_t pg = svwhilelt_b8_s64(0, buf_end - buf);
svbool_t matched = singleMatched(chars, buf, pg, negate, 0);
return vermSearchCheckMatched(buf, matched);
return accelSearchCheckMatched(buf, matched);
}
static really_inline
const u8 *vermSearchLoopBody(svuint8_t chars, const u8 *buf, bool negate) {
DEBUG_PRINTF("start %p end %p\n", buf, buf + svcntb());
svbool_t matched = singleMatched(chars, buf, svptrue_b8(), negate, 0);
return vermSearchCheckMatched(buf, matched);
return accelSearchCheckMatched(buf, matched);
}
static really_inline
@ -149,9 +123,9 @@ const u8 *vermSearchLoopBodyUnrolled(svuint8_t chars, const u8 *buf,
svbool_t any = svorr_z(svptrue_b8(), matched0, matched1);
if (unlikely(svptest_any(svptrue_b8(), any))) {
if (svptest_any(svptrue_b8(), matched0)) {
return buf + vermSearchGetOffset(matched0);
return buf + accelSearchGetOffset(matched0);
} else {
return buf + svcntb() + vermSearchGetOffset(matched1);
return buf + svcntb() + accelSearchGetOffset(matched1);
}
}
return NULL;
@ -165,14 +139,14 @@ const u8 *rvermSearchOnce(svuint8_t chars, const u8 *buf, const u8 *buf_end,
DEBUG_PRINTF("l = %td\n", buf_end - buf);
svbool_t pg = svwhilelt_b8_s64(0, buf_end - buf);
svbool_t matched = singleMatched(chars, buf, pg, negate, 0);
return rvermSearchCheckMatched(buf, matched);
return accelRevSearchCheckMatched(buf, matched);
}
static really_inline
const u8 *rvermSearchLoopBody(svuint8_t chars, const u8 *buf, bool negate) {
DEBUG_PRINTF("start %p end %p\n", buf, buf + svcntb());
svbool_t matched = singleMatched(chars, buf, svptrue_b8(), negate, 0);
return rvermSearchCheckMatched(buf, matched);
return accelRevSearchCheckMatched(buf, matched);
}
static really_inline

36
src/util/arch/arm/simd_utils.h
View File

@ -1,5 +1,6 @@
/*
* Copyright (c) 2015-2020, Intel Corporation
* Copyright (c) 2021, Arm Limited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@ -41,6 +42,41 @@
#include "util/unaligned.h"
#include "util/intrinsics.h"
#ifdef HAVE_SVE
really_really_inline
uint64_t accelSearchGetOffset(svbool_t matched) {
return svcntp_b8(svptrue_b8(), svbrkb_z(svptrue_b8(), matched));
}
really_really_inline
const u8 *accelSearchCheckMatched(const u8 *buf, svbool_t matched) {
if (unlikely(svptest_any(svptrue_b8(), matched))) {
const u8 *matchPos = buf + accelSearchGetOffset(matched);
DEBUG_PRINTF("match pos %p\n", matchPos);
return matchPos;
}
return NULL;
}
really_really_inline
const u8 *accelRevSearchCheckMatched(const u8 *buf, svbool_t matched) {
if (unlikely(svptest_any(svptrue_b8(), matched))) {
const u8 *matchPos = buf + (svcntb() -
svcntp_b8(svptrue_b8(), svbrka_z(svptrue_b8(), svrev_b8(matched))));
DEBUG_PRINTF("match pos %p\n", matchPos);
return matchPos;
}
return NULL;
}
static really_inline
svuint8_t getSVEMaskFrom128(m128 mask) {
return svld1_u8(svptrue_pat_b8(SV_VL16), (const uint8_t *)&mask);
}
#endif
#ifdef HAVE_SVE2
static really_inline