diff --git a/src/nfa/shufti_simd.hpp b/src/nfa/shufti_simd.hpp index 3c5a1fbe..f8621afe 100644 --- a/src/nfa/shufti_simd.hpp +++ b/src/nfa/shufti_simd.hpp @@ -43,56 +43,82 @@ #include "util/supervector/supervector.hpp" #include "util/match.hpp" +#include +#include +#include +#include +#include +#include +#include + +#include +#include + template static really_inline -typename SuperVector::movemask_type block(SuperVector mask_lo, SuperVector mask_hi, - SuperVector chars) { +const SuperVector blockSingleMask(SuperVector mask_lo, SuperVector mask_hi, SuperVector chars) { const SuperVector low4bits = SuperVector::dup_u8(0xf); SuperVector c_lo = chars & low4bits; - c_lo = mask_lo.pshufb(c_lo); - SuperVector c_hi = mask_hi.pshufb(chars.template vshr_64_imm<4>() & low4bits); - SuperVector t = c_lo & c_hi; + SuperVector c_hi = chars.template vshr_8_imm<4>(); + c_lo = mask_lo.template pshufb(c_lo); + c_hi = mask_hi.template pshufb(c_hi); - return t.eqmask(SuperVector::Zeroes()); + return (c_lo & c_hi) > (SuperVector::Zeroes()); +} + +template +static really_inline +SuperVector blockDoubleMask(SuperVector mask1_lo, SuperVector mask1_hi, SuperVector mask2_lo, SuperVector mask2_hi, SuperVector chars) { + + const SuperVector low4bits = SuperVector::dup_u8(0xf); + SuperVector chars_lo = chars & low4bits; + chars_lo.print8("chars_lo"); + SuperVector chars_hi = chars.template vshr_64_imm<4>() & low4bits; + chars_hi.print8("chars_hi"); + SuperVector c1_lo = mask1_lo.template pshufb(chars_lo); + c1_lo.print8("c1_lo"); + SuperVector c1_hi = mask1_hi.template pshufb(chars_hi); + c1_hi.print8("c1_hi"); + SuperVector t1 = c1_lo | c1_hi; + t1.print8("t1"); + + SuperVector c2_lo = mask2_lo.template pshufb(chars_lo); + c2_lo.print8("c2_lo"); + SuperVector c2_hi = mask2_hi.template pshufb(chars_hi); + c2_hi.print8("c2_hi"); + SuperVector t2 = c2_lo | c2_hi; + t2.print8("t2"); + t2.template vshr_128_imm<1>().print8("t2.vshr_128(1)"); + SuperVector t = t1 | (t2.template vshr_128_imm<1>()); + t.print8("t"); + + return !t.eq(SuperVector::Ones()); } template static really_inline const u8 *fwdBlock(SuperVector mask_lo, SuperVector mask_hi, SuperVector chars, const u8 *buf) { - typename SuperVector::movemask_type z = block(mask_lo, mask_hi, chars); - DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); + SuperVector v = blockSingleMask(mask_lo, mask_hi, chars); - return firstMatch(buf, z); + return firstMatch(buf, v); } -/* -template -static really_inline -const u8 *shortShufti(SuperVector mask_lo, SuperVector mask_hi, const u8 *buf, const u8 *buf_end) { - DEBUG_PRINTF("short shufti %p len %zu\n", buf, buf_end - buf); - uintptr_t len = buf_end - buf; - assert(len <= S); - - SuperVector chars = SuperVector::loadu_maskz(buf, static_cast(len)); - //printv_u8("chars", chars); - uint8_t alignment = (uintptr_t)(buf) & 15; - typename SuperVector::movemask_type maskb = 1 << alignment; - typename SuperVector::movemask_type maske = SINGLE_LOAD_MASK(len - alignment); - typename SuperVector::movemask_type z = block(mask_lo, mask_hi, chars); - // reuse the load mask to indicate valid bytes - DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); - z &= maskb | maske; - DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); - - return firstMatch(buf, z); -}*/ template static really_inline const u8 *revBlock(SuperVector mask_lo, SuperVector mask_hi, SuperVector chars, const u8 *buf) { - typename SuperVector::movemask_type z = block(mask_lo, mask_hi, chars); - DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); - return lastMatch(buf, z); + SuperVector v = blockSingleMask(mask_lo, mask_hi, chars); + + return lastMatch(buf, v); +} + +template +static really_inline +const u8 *fwdBlockDouble(SuperVector mask1_lo, SuperVector mask1_hi, SuperVector mask2_lo, SuperVector mask2_hi, SuperVector chars, const u8 *buf) { + + SuperVector mask = blockDoubleMask(mask1_lo, mask1_hi, mask2_lo, mask2_hi, chars); + + return firstMatch(buf, mask); } template @@ -108,54 +134,50 @@ const u8 *shuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *bu const u8 *d = buf; const u8 *rv; + __builtin_prefetch(d + 64); + __builtin_prefetch(d + 2*64); + __builtin_prefetch(d + 3*64); + __builtin_prefetch(d + 4*64); DEBUG_PRINTF("start %p end %p \n", d, buf_end); assert(d < buf_end); if (d + S <= buf_end) { - // peel off first part to cacheline boundary - const u8 *d1 = ROUNDUP_PTR(d, S); - DEBUG_PRINTF("until aligned %p \n", d1); - if (d1 != d) { - rv = shuftiFwdSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, d, d1); - // rv = shortShufti(wide_mask_lo, wide_mask_hi, d, d1); - if (rv != d1) { - return rv; - } - d = d1; + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector chars = SuperVector::loadu(d); + rv = fwdBlock(wide_mask_lo, wide_mask_hi, chars, d); + if (rv) return rv; + d = ROUNDUP_PTR(d, S); } - size_t loops = (buf_end - d) / S; - DEBUG_PRINTF("loops %ld \n", loops); - - for (size_t i = 0; i < loops; i++, d+= S) { + while(d + S <= buf_end) { + __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); - const u8 *base = ROUNDUP_PTR(d, S); - // On large packet buffers, this prefetch appears to get us about 2%. - __builtin_prefetch(base + 256); - SuperVector chars = SuperVector::load(d); rv = fwdBlock(wide_mask_lo, wide_mask_hi, chars, d); if (rv) return rv; + d += S; } } DEBUG_PRINTF("d %p e %p \n", d, buf_end); // finish off tail - rv = buf_end; if (d != buf_end) { - rv = shuftiFwdSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, d, buf_end); - // rv = shortShufti(wide_mask_lo, wide_mask_hi, buf_end - S, buf_end); + SuperVector chars = SuperVector::loadu(d); + rv = fwdBlock(wide_mask_lo, wide_mask_hi, chars, d); DEBUG_PRINTF("rv %p \n", rv); + if (rv) return rv; } - return rv; + return buf_end; } template const u8 *rshuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *buf_end) { assert(buf && buf_end); assert(buf < buf_end); - DEBUG_PRINTF("shufti %p len %zu\n", buf, buf_end - buf); + DEBUG_PRINTF("rshufti %p len %zu\n", buf, buf_end - buf); DEBUG_PRINTF("b %s\n", buf); const SuperVector wide_mask_lo(mask_lo); @@ -164,27 +186,29 @@ const u8 *rshuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *b const u8 *d = buf_end; const u8 *rv; + __builtin_prefetch(d - 64); + __builtin_prefetch(d - 2*64); + __builtin_prefetch(d - 3*64); + __builtin_prefetch(d - 4*64); DEBUG_PRINTF("start %p end %p \n", buf, d); assert(d > buf); if (d - S >= buf) { - // peel off first part to cacheline boundary - const u8 *d1 = ROUNDDOWN_PTR(d, S); - DEBUG_PRINTF("until aligned %p \n", d1); - if (d1 != d) { - rv = shuftiRevSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, d1, d); + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { + SuperVector chars = SuperVector::loadu(d - S); + rv = revBlock(wide_mask_lo, wide_mask_hi, chars, d - S); DEBUG_PRINTF("rv %p \n", rv); - // rv = shortShufti(wide_mask_lo, wide_mask_hi, d, d1); - if (rv != d1 - 1) return rv; - d = d1; + if (rv) return rv; + d = ROUNDDOWN_PTR(d, S); } while (d - S >= buf) { DEBUG_PRINTF("aligned %p \n", d); - d -= S; - const u8 *base = ROUNDDOWN_PTR(buf, S); // On large packet buffers, this prefetch appears to get us about 2%. - __builtin_prefetch(base + 256); + __builtin_prefetch(d - 64); + d -= S; SuperVector chars = SuperVector::load(d); rv = revBlock(wide_mask_lo, wide_mask_hi, chars, d); if (rv) return rv; @@ -192,11 +216,11 @@ const u8 *rshuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *b } DEBUG_PRINTF("tail d %p e %p \n", buf, d); - // finish off tail + // finish off head if (d != buf) { - rv = shuftiRevSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, buf, d); - // rv = shortShufti(wide_mask_lo, wide_mask_hi, buf_end - S, buf_end); + SuperVector chars = SuperVector::loadu(buf); + rv = revBlock(wide_mask_lo, wide_mask_hi, chars, buf); DEBUG_PRINTF("rv %p \n", rv); if (rv) return rv; } @@ -204,80 +228,10 @@ const u8 *rshuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *b return buf - 1; } -template -static really_inline -const u8 *fwdBlockDouble(SuperVector mask1_lo, SuperVector mask1_hi, SuperVector mask2_lo, SuperVector mask2_hi, - SuperVector chars, const u8 *buf) { - - const SuperVector low4bits = SuperVector::dup_u8(0xf); - SuperVector chars_lo = chars & low4bits; - chars_lo.print8("chars_lo"); - SuperVector chars_hi = chars.template vshr_64_imm<4>() & low4bits; - chars_hi.print8("chars_hi"); - SuperVector c1_lo = mask1_lo.pshufb(chars_lo); - c1_lo.print8("c1_lo"); - SuperVector c1_hi = mask1_hi.pshufb(chars_hi); - c1_hi.print8("c1_hi"); - SuperVector t1 = c1_lo | c1_hi; - t1.print8("t1"); - - SuperVector c2_lo = mask2_lo.pshufb(chars_lo); - c2_lo.print8("c2_lo"); - SuperVector c2_hi = mask2_hi.pshufb(chars_hi); - c2_hi.print8("c2_hi"); - SuperVector t2 = c2_lo | c2_hi; - t2.print8("t2"); - t2.template vshr_128_imm<1>().print8("t2.rshift128(1)"); - SuperVector t = t1 | (t2.template vshr_128_imm<1>()); - t.print8("t"); - - typename SuperVector::movemask_type z = t.eqmask(SuperVector::Ones()); - DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); - return firstMatch(buf, z); -} - -template -static really_inline const u8 *shuftiDoubleMini(SuperVector mask1_lo, SuperVector mask1_hi, SuperVector mask2_lo, SuperVector mask2_hi, - const u8 *buf, const u8 *buf_end){ - uintptr_t len = buf_end - buf; - assert(len < S); - - const SuperVector low4bits = SuperVector::dup_u8(0xf); - - DEBUG_PRINTF("buf %p buf_end %p \n", buf, buf_end); - SuperVector chars = SuperVector::loadu_maskz(buf, len); - chars.print8("chars"); - - SuperVector chars_lo = chars & low4bits; - chars_lo.print8("chars_lo"); - SuperVector chars_hi = chars.template vshr_64_imm<4>() & low4bits; - chars_hi.print8("chars_hi"); - SuperVector c1_lo = mask1_lo.pshufb_maskz(chars_lo, len); - c1_lo.print8("c1_lo"); - SuperVector c1_hi = mask1_hi.pshufb_maskz(chars_hi, len); - c1_hi.print8("c1_hi"); - SuperVector t1 = c1_lo | c1_hi; - t1.print8("t1"); - - SuperVector c2_lo = mask2_lo.pshufb_maskz(chars_lo, len); - c2_lo.print8("c2_lo"); - SuperVector c2_hi = mask2_hi.pshufb_maskz(chars_hi, len); - c2_hi.print8("c2_hi"); - SuperVector t2 = c2_lo | c2_hi; - t2.print8("t2"); - t2.template vshr_128_imm<1>().print8("t2.rshift128(1)"); - SuperVector t = t1 | (t2.template vshr_128_imm<1>()); - t.print8("t"); - - typename SuperVector::movemask_type z = t.eqmask(SuperVector::Ones()); - DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); - return firstMatch(buf, z); -} - template const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 mask2_hi, const u8 *buf, const u8 *buf_end) { - assert(buf && buf_end); + assert(buf && buf_end); assert(buf < buf_end); DEBUG_PRINTF("shufti %p len %zu\n", buf, buf_end - buf); DEBUG_PRINTF("b %s\n", buf); @@ -290,32 +244,31 @@ const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 const u8 *d = buf; const u8 *rv; + __builtin_prefetch(d + 64); + __builtin_prefetch(d + 2*64); + __builtin_prefetch(d + 3*64); + __builtin_prefetch(d + 4*64); DEBUG_PRINTF("start %p end %p \n", d, buf_end); assert(d < buf_end); if (d + S <= buf_end) { // peel off first part to cacheline boundary - const u8 *d1 = ROUNDUP_PTR(d, S); - DEBUG_PRINTF("until aligned %p \n", d1); - if (d1 != d) { + DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); + if (!ISALIGNED_N(d, S)) { SuperVector chars = SuperVector::loadu(d); rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, d); DEBUG_PRINTF("rv %p \n", rv); if (rv) return rv; - d = d1; + d = ROUNDUP_PTR(d, S); } - size_t loops = (buf_end - d) / S; - DEBUG_PRINTF("loops %ld \n", loops); - - for (size_t i = 0; i < loops; i++, d+= S) { - DEBUG_PRINTF("it = %ld, d %p \n", i, d); - const u8 *base = ROUNDUP_PTR(d, S); - // On large packet buffers, this prefetch appears to get us about 2%. - __builtin_prefetch(base + 256); + while(d + S <= buf_end) { + __builtin_prefetch(d + 64); + DEBUG_PRINTF("d %p \n", d); SuperVector chars = SuperVector::load(d); rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, d); if (rv) return rv; + d += S; } } @@ -323,9 +276,10 @@ const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 // finish off tail if (d != buf_end) { - rv = shuftiDoubleMini(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, d, buf_end); + SuperVector chars = SuperVector::loadu(buf_end - S); + rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, buf_end - S); DEBUG_PRINTF("rv %p \n", rv); - if (rv >= buf && rv < buf_end) return rv; + if (rv) return rv; } return buf_end; diff --git a/src/nfa/truffle_simd.hpp b/src/nfa/truffle_simd.hpp index c5f85135..bfe976ce 100644 --- a/src/nfa/truffle_simd.hpp +++ b/src/nfa/truffle_simd.hpp @@ -28,9 +28,8 @@ */ /** \file - * \brief Shufti: character class acceleration. + * \brief Truffle: character class acceleration. * - * Utilises the SSSE3 pshufb shuffle instruction */ #include "truffle.h" @@ -44,64 +43,40 @@ template static really_inline -typename SuperVector::movemask_type block(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, - SuperVector v){ - - SuperVector highconst = SuperVector::dup_u8(0x80); - SuperVector shuf_mask_hi = SuperVector::dup_u64(0x8040201008040201); - - SuperVector shuf1 = shuf_mask_lo_highclear.pshufb(v); - SuperVector t1 = v ^ highconst; - SuperVector shuf2 = shuf_mask_lo_highset.pshufb(t1); - SuperVector t2 = highconst.opandnot(v.template vshr_64_imm<4>()); - SuperVector shuf3 = shuf_mask_hi.pshufb(t2); - SuperVector tmp = (shuf1 | shuf2) & shuf3; +SuperVector block(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector chars) { + chars.print8("chars"); shuf_mask_lo_highclear.print8("shuf_mask_lo_highclear"); shuf_mask_lo_highset.print8("shuf_mask_lo_highset"); - v.print8("v"); + + SuperVector highconst = SuperVector::dup_u8(0x80); highconst.print8("highconst"); + SuperVector shuf_mask_hi = SuperVector::dup_u64(0x8040201008040201); shuf_mask_hi.print8("shuf_mask_hi"); + + SuperVector shuf1 = shuf_mask_lo_highclear.template pshufb(chars); shuf1.print8("shuf1"); + SuperVector t1 = chars ^ highconst; t1.print8("t1"); + SuperVector shuf2 = shuf_mask_lo_highset.template pshufb(t1); shuf2.print8("shuf2"); + SuperVector t2 = highconst.opandnot(chars.template vshr_64_imm<4>()); t2.print8("t2"); + SuperVector shuf3 = shuf_mask_hi.template pshufb(t2); shuf3.print8("shuf3"); - tmp.print8("tmp"); - DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)tmp.eqmask(SuperVector::Zeroes())); + SuperVector res = (shuf1 | shuf2) & shuf3; + res.print8("(shuf1 | shuf2) & shuf3"); - return tmp.eqmask(SuperVector::Zeroes()); -} - -template -static really_inline const u8 *truffleMini(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, - const u8 *buf, const u8 *buf_end){ - uintptr_t len = buf_end - buf; - assert(len < S); - - DEBUG_PRINTF("buf %p buf_end %p \n", buf, buf_end); - SuperVector chars = SuperVector::loadu_maskz(buf, len); - chars.print8("chars"); - - typename SuperVector::movemask_type z = block(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars); - const u8 *rv = firstMatch(buf, z); - DEBUG_PRINTF("rv %p buf+len %p \n", rv, buf+len); - - if (rv && rv < buf+len) { - return rv; - } - return buf_end; + return !res.eq(SuperVector::Zeroes());//{(m128)vcgtq_u8((uint8x16_t)tmp.u.v128[0], vdupq_n_u8(0))}; } template static really_inline -const u8 *fwdBlock(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector v, - const u8 *buf) { - typename SuperVector::movemask_type z = block(shuf_mask_lo_highclear, shuf_mask_lo_highset, v); - DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); - return firstMatch(buf, z); -} +const u8 *fwdBlock(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector chars, const u8 *buf) { + SuperVector res = block(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars); + return firstMatch(buf, res); +} template const u8 *truffleExecReal(m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_highset, const u8 *buf, const u8 *buf_end) { @@ -119,45 +94,41 @@ const u8 *truffleExecReal(m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_highse DEBUG_PRINTF("start %p end %p \n", d, buf_end); assert(d < buf_end); + __builtin_prefetch(d + 64); + __builtin_prefetch(d + 2*64); + __builtin_prefetch(d + 3*64); + __builtin_prefetch(d + 4*64); if (d + S <= buf_end) { + // Reach vector aligned boundaries + DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); if (!ISALIGNED_N(d, S)) { - // peel off first part to cacheline boundary - const u8 *d1 = ROUNDUP_PTR(d, S); - DEBUG_PRINTF("until aligned %p \n", d1); - if (d1 != d) { - rv = truffleMini(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, d, d1); - if (rv != d1) { - return rv; - } - d = d1; - } + SuperVector chars = SuperVector::loadu(d); + rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d); + if (rv) return rv; + d = ROUNDUP_PTR(d, S); } - size_t loops = (buf_end - d) / S; - DEBUG_PRINTF("loops %ld \n", loops); - - for (size_t i = 0; i < loops; i++, d+= S) { + while(d + S <= buf_end) { + __builtin_prefetch(d + 64); DEBUG_PRINTF("d %p \n", d); - const u8 *base = ROUNDUP_PTR(d, S); - // On large packet buffers, this prefetch appears to get us about 2%. - __builtin_prefetch(base + 256); - SuperVector chars = SuperVector::load(d); rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d); if (rv) return rv; + d += S; } } - DEBUG_PRINTF("d %p e %p \n", d, buf_end); + DEBUG_PRINTF("d %p e %p \n", d, buf_end); // finish off tail - rv = buf_end; if (d != buf_end) { - rv = truffleMini(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, d, buf_end); + SuperVector chars = SuperVector::loadu(d); + rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d); DEBUG_PRINTF("rv %p \n", rv); + if (rv) return rv; } - - return rv; + + return buf_end; } @@ -170,8 +141,8 @@ static really_inline const u8 *truffleRevMini(SuperVector shuf_mask_lo_highcl SuperVector chars = SuperVector::loadu_maskz(buf, len); - typename SuperVector::movemask_type z = block(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars); - const u8 *rv = lastMatch(buf, z); + SuperVector v = block(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars); + const u8 *rv = lastMatch(buf, v); DEBUG_PRINTF("rv %p buf+len %p \n", rv, buf+len); if (rv && rv < buf+len) { @@ -184,9 +155,8 @@ template static really_inline const u8 *revBlock(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector v, const u8 *buf) { - typename SuperVector::movemask_type z = block(shuf_mask_lo_highclear, shuf_mask_lo_highset, v); - DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z); - return lastMatch(buf, z); + SuperVector res = block(shuf_mask_lo_highclear, shuf_mask_lo_highset, v); + return lastMatch(buf, res); } @@ -220,9 +190,8 @@ const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highse while (d - S >= buf) { d -= S; DEBUG_PRINTF("d %p \n", d); - const u8 *base = ROUNDDOWN_PTR(buf, S); // On large packet buffers, this prefetch appears to get us about 2%. - __builtin_prefetch(base + 256); + __builtin_prefetch(d - 64); SuperVector chars = SuperVector::load(d); rv = revBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d);