diff --git a/src/nfa/shufti_simd.hpp b/src/nfa/shufti_simd.hpp index cbfd23ba..86b20deb 100644 --- a/src/nfa/shufti_simd.hpp +++ b/src/nfa/shufti_simd.hpp @@ -235,6 +235,44 @@ const u8 *fwdBlockDouble(SuperVector mask1_lo, SuperVector mask1_hi, Super return firstMatch(buf, z); } +template +static really_inline const u8 *shuftiDoubleMini(SuperVector mask1_lo, SuperVector mask1_hi, SuperVector mask2_lo, SuperVector mask2_hi, + const u8 *buf, const u8 *buf_end){ + uintptr_t len = buf_end - buf; + assert(len < S); + + const SuperVector low4bits = SuperVector::dup_u8(0xf); + + DEBUG_PRINTF("buf %p buf_end %p \n", buf, buf_end); + SuperVector chars = SuperVector::loadu_maskz(buf, len); + chars.print8("chars"); + + SuperVector chars_lo = chars & low4bits; + chars_lo.print8("chars_lo"); + SuperVector chars_hi = chars.rshift64(4) & low4bits; + chars_hi.print8("chars_hi"); + SuperVector c1_lo = mask1_lo.pshufb_maskz(chars_lo, len); + c1_lo.print8("c1_lo"); + SuperVector c1_hi = mask1_hi.pshufb_maskz(chars_hi, len); + c1_hi.print8("c1_hi"); + SuperVector t1 = c1_lo | c1_hi; + t1.print8("t1"); + + SuperVector c2_lo = mask2_lo.pshufb_maskz(chars_lo, len); + c2_lo.print8("c2_lo"); + SuperVector c2_hi = mask2_hi.pshufb_maskz(chars_hi, len); + c2_hi.print8("c2_hi"); + SuperVector t2 = c2_lo | c2_hi; + t2.print8("t2"); + t2.rshift128(1).print8("t2.rshift128(1)"); + SuperVector t = t1 | (t2.rshift128(1)); + t.print8("t"); + + typename SuperVector::movemask_type z = t.eqmask(SuperVector::Ones()); + DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); + return firstMatch(buf, z); +} + template const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 mask2_hi, const u8 *buf, const u8 *buf_end) { @@ -284,8 +322,7 @@ const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 // finish off tail if (d != buf_end) { - SuperVector chars = SuperVector::loadu(buf_end - S); - rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, buf_end - S); + rv = shuftiDoubleMini(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, d, buf_end); DEBUG_PRINTF("rv %p \n", rv); if (rv >= buf && rv < buf_end) return rv; } diff --git a/src/util/supervector/arch/arm/impl.cpp b/src/util/supervector/arch/arm/impl.cpp index e40b6a38..65d0faa5 100644 --- a/src/util/supervector/arch/arm/impl.cpp +++ b/src/util/supervector/arch/arm/impl.cpp @@ -348,6 +348,13 @@ really_inline SuperVector<16> SuperVector<16>::pshufb(SuperVector<16> b) return {vqtbl1q_s8((int8x16_t)u.v128[0], (uint8x16_t)btranslated)}; } +template<> +really_inline SuperVector<16> SuperVector<16>::pshufb_maskz(SuperVector<16> b, uint8_t const len) +{ + SuperVector<16> mask = Ones().rshift128_var(16 -len); + return mask & pshufb(b); +} + #ifdef HS_OPTIMIZE template<> really_inline SuperVector<16> SuperVector<16>::lshift64(uint8_t const N) diff --git a/src/util/supervector/arch/x86/impl.cpp b/src/util/supervector/arch/x86/impl.cpp index e64583e1..3c305d4b 100644 --- a/src/util/supervector/arch/x86/impl.cpp +++ b/src/util/supervector/arch/x86/impl.cpp @@ -312,6 +312,13 @@ really_inline SuperVector<16> SuperVector<16>::pshufb(SuperVector<16> b) return {_mm_shuffle_epi8(u.v128[0], b.u.v128[0])}; } +template<> +really_inline SuperVector<16> SuperVector<16>::pshufb_maskz(SuperVector<16> b, uint8_t const len) +{ + SuperVector<16> mask = Ones().rshift128_var(16 -len); + return mask & pshufb(b); +} + #ifdef HS_OPTIMIZE template<> really_inline SuperVector<16> SuperVector<16>::lshift64(uint8_t const N) @@ -733,6 +740,13 @@ really_inline SuperVector<32> SuperVector<32>::pshufb(SuperVector<32> b) return {_mm256_shuffle_epi8(u.v256[0], b.u.v256[0])}; } +template<> +really_inline SuperVector<32> SuperVector<32>::pshufb_maskz(SuperVector<32> b, uint8_t const len) +{ + SuperVector<32> mask = Ones().rshift128_var(32 -len); + return mask & pshufb(b); +} + #ifdef HS_OPTIMIZE template<> really_inline SuperVector<32> SuperVector<32>::lshift64(uint8_t const N) @@ -1176,6 +1190,13 @@ really_inline SuperVector<64> SuperVector<64>::pshufb(SuperVector<64> b) return {_mm512_shuffle_epi8(u.v512[0], b.u.v512[0])}; } +template<> +really_inline SuperVector<64> SuperVector<64>::pshufb_maskz(SuperVector<64> b, uint8_t const len) +{ + u64a mask = (~0ULL) >> (64 - len); + DEBUG_PRINTF("mask = %016llx\n", mask); + return {_mm512_maskz_shuffle_epi8(mask, u.v512[0], b.u.v512[0])}; +} #ifdef HS_OPTIMIZE template<> diff --git a/src/util/supervector/supervector.hpp b/src/util/supervector/supervector.hpp index bd7fd18a..e834fef0 100644 --- a/src/util/supervector/supervector.hpp +++ b/src/util/supervector/supervector.hpp @@ -216,6 +216,7 @@ public: SuperVector alignr(SuperVector &other, int8_t offset); SuperVector pshufb(SuperVector b); + SuperVector pshufb_maskz(SuperVector b, uint8_t const len); SuperVector lshift64(uint8_t const N); SuperVector rshift64(uint8_t const N); SuperVector lshift128(uint8_t const N);