github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-06-28 16:41:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

remove low4bits from the arguments, fix cases that mostly affect loading large (64) vectors and falling out of bounds

Browse Source
This commit is contained in:
Konstantinos Margaritis 2021-07-23 11:45:58 +03:00
parent f2d9784979
commit 6f44a1aa26
2 changed files with 39 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
src/nfa/shufti_simd.hpp
View File

@ -45,40 +45,29 @@
template <uint16_t S>
static really_inline
typename SuperVector<S>::movemask_type block(SuperVector<S> mask_lo, SuperVector<S> mask_hi,
SuperVector<S> chars, const SuperVector<S> low4bits) {
SuperVector<S> chars) {
const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
SuperVector<S> c_lo = chars & low4bits;
//printv_u8("c_lo", c_lo);
c_lo = mask_lo.pshufb(c_lo);
//printv_u8("c_lo", c_lo);
SuperVector<S> c_hi = mask_hi.pshufb(chars.rshift64(4) & low4bits);
SuperVector<S> t = c_lo & c_hi;
/*printv_u8("low4bits", low4bits);
printv_u8("mask_lo", mask_lo);
printv_u8("mask_hi", mask_hi);
printv_u8("chars", chars);
printv_u8("c_lo", c_lo);
printv_u8("c_hi", c_hi);
printv_u8("t", t);*/
return t.eqmask(SuperVector<S>::Zeroes());
}
template <uint16_t S>
static really_inline
const u8 *fwdBlock(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars,
const SuperVector<S> low4bits, const u8 *buf) {
typename SuperVector<S>::movemask_type z = block(mask_lo, mask_hi, chars, low4bits);
const u8 *fwdBlock(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars, const u8 *buf) {
typename SuperVector<S>::movemask_type z = block(mask_lo, mask_hi, chars);
DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
return firstMatch<S>(buf, z);
}
/*
template <uint16_t S>
static really_inline
const u8 *shortShufti(SuperVector<S> mask_lo, SuperVector<S> mask_hi, const u8 *buf,
const u8 *buf_end, const SuperVector<S> low4bits) {
const u8 *shortShufti(SuperVector<S> mask_lo, SuperVector<S> mask_hi, const u8 *buf, const u8 *buf_end) {
DEBUG_PRINTF("short shufti %p len %zu\n", buf, buf_end - buf);
uintptr_t len = buf_end - buf;
assert(len <= S);
@ -88,20 +77,19 @@ const u8 *shortShufti(SuperVector<S> mask_lo, SuperVector<S> mask_hi, const u8 *
uint8_t alignment = (uintptr_t)(buf) & 15;
typename SuperVector<S>::movemask_type maskb = 1 << alignment;
typename SuperVector<S>::movemask_type maske = SINGLE_LOAD_MASK(len - alignment);
typename SuperVector<S>::movemask_type z = block(mask_lo, mask_hi, chars, low4bits);
typename SuperVector<S>::movemask_type z = block(mask_lo, mask_hi, chars);
// reuse the load mask to indicate valid bytes
DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
z &= maskb | maske;
DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
return firstMatch<S>(buf, z);
}
}*/
template <uint16_t S>
static really_inline
const u8 *revBlock(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars,
const SuperVector<S> low4bits, const u8 *buf) {
typename SuperVector<S>::movemask_type z = block(mask_lo, mask_hi, chars, low4bits);
const u8 *revBlock(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars, const u8 *buf) {
typename SuperVector<S>::movemask_type z = block(mask_lo, mask_hi, chars);
DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
return lastMatch<S>(buf, z);
}
@ -113,7 +101,6 @@ const u8 *shuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *bu
DEBUG_PRINTF("shufti %p len %zu\n", buf, buf_end - buf);
DEBUG_PRINTF("b %s\n", buf);
const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
const SuperVector<S> wide_mask_lo(mask_lo);
const SuperVector<S> wide_mask_hi(mask_hi);
@ -128,7 +115,7 @@ const u8 *shuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *bu
DEBUG_PRINTF("until aligned %p \n", d1);
if (d1 != d) {
rv = shuftiFwdSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, d, d1);
// rv = shortShufti(wide_mask_lo, wide_mask_hi, d, d1, low4bits);
// rv = shortShufti(wide_mask_lo, wide_mask_hi, d, d1);
if (rv != d1) {
return rv;
}
@ -145,7 +132,7 @@ const u8 *shuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *bu
__builtin_prefetch(base + 256);
SuperVector<S> chars = SuperVector<S>::load(d);
rv = fwdBlock(wide_mask_lo, wide_mask_hi, chars, low4bits, d);
rv = fwdBlock(wide_mask_lo, wide_mask_hi, chars, d);
if (rv) return rv;
}
}
@ -156,7 +143,7 @@ const u8 *shuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *bu
rv = buf_end;
if (d != buf_end) {
rv = shuftiFwdSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, d, buf_end);
// rv = shortShufti(wide_mask_lo, wide_mask_hi, buf_end - S, buf_end, low4bits);
// rv = shortShufti(wide_mask_lo, wide_mask_hi, buf_end - S, buf_end);
DEBUG_PRINTF("rv %p \n", rv);
}
@ -170,7 +157,6 @@ const u8 *rshuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *b
DEBUG_PRINTF("shufti %p len %zu\n", buf, buf_end - buf);
DEBUG_PRINTF("b %s\n", buf);
const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
const SuperVector<S> wide_mask_lo(mask_lo);
const SuperVector<S> wide_mask_hi(mask_hi);
@ -186,30 +172,30 @@ const u8 *rshuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *b
if (d1 != d) {
rv = shuftiRevSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, d1, d);
DEBUG_PRINTF("rv %p \n", rv);
// rv = shortShufti(wide_mask_lo, wide_mask_hi, d, d1, low4bits);
// rv = shortShufti(wide_mask_lo, wide_mask_hi, d, d1);
if (rv != d1 - 1) return rv;
d = d1;
}
while (d - S >= buf) {
DEBUG_PRINTF("aligned %p \n", d);
d -= S;
DEBUG_PRINTF("d %p \n", d);
const u8 *base = ROUNDDOWN_PTR(buf, S);
// On large packet buffers, this prefetch appears to get us about 2%.
__builtin_prefetch(base + 256);
SuperVector<S> chars = SuperVector<S>::load(d);
rv = revBlock(wide_mask_lo, wide_mask_hi, chars, low4bits, d);
rv = revBlock(wide_mask_lo, wide_mask_hi, chars, d);
if (rv) return rv;
}
}
DEBUG_PRINTF("d %p e %p \n", buf, d);
DEBUG_PRINTF("tail d %p e %p \n", buf, d);
// finish off tail
if (d != buf) {
rv = shuftiRevSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, buf, d);
// rv = shortShufti(wide_mask_lo, wide_mask_hi, buf_end - S, buf_end, low4bits);
// rv = shortShufti(wide_mask_lo, wide_mask_hi, buf_end - S, buf_end);
DEBUG_PRINTF("rv %p \n", rv);
if (rv) return rv;
}
@ -220,18 +206,29 @@ const u8 *rshuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *b
template <uint16_t S>
static really_inline
const u8 *fwdBlockDouble(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi, SuperVector<S> mask2_lo, SuperVector<S> mask2_hi,
SuperVector<S> chars, const SuperVector<S> low4bits, const u8 *buf) {
SuperVector<S> chars, const u8 *buf) {
const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
SuperVector<S> chars_lo = chars & low4bits;
chars_lo.print8("chars_lo");
SuperVector<S> chars_hi = chars.rshift64(4) & low4bits;
chars_hi.print8("chars_hi");
SuperVector<S> c1_lo = mask1_lo.pshufb(chars_lo);
c1_lo.print8("c1_lo");
SuperVector<S> c1_hi = mask1_hi.pshufb(chars_hi);
c1_hi.print8("c1_hi");
SuperVector<S> t1 = c1_lo | c1_hi;
t1.print8("t1");
SuperVector<S> c2_lo = mask2_lo.pshufb(chars_lo);
c2_lo.print8("c2_lo");
SuperVector<S> c2_hi = mask2_hi.pshufb(chars_hi);
c2_hi.print8("c2_hi");
SuperVector<S> t2 = c2_lo | c2_hi;
t2.print8("t2");
t2.rshift128(1).print8("t2.rshift128(1)");
SuperVector<S> t = t1 | (t2.rshift128(1));
t.print8("t");
typename SuperVector<S>::movemask_type z = t.eqmask(SuperVector<S>::Ones());
DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
@ -239,15 +236,13 @@ const u8 *fwdBlockDouble(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi, Super
}
template <uint16_t S>
const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi,
m128 mask2_lo, m128 mask2_hi,
const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 mask2_hi,
const u8 *buf, const u8 *buf_end) {
assert(buf && buf_end);
assert(buf < buf_end);
DEBUG_PRINTF("shufti %p len %zu\n", buf, buf_end - buf);
DEBUG_PRINTF("b %s\n", buf);
const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
const SuperVector<S> wide_mask1_lo(mask1_lo);
const SuperVector<S> wide_mask1_hi(mask1_hi);
const SuperVector<S> wide_mask2_lo(mask2_lo);
@ -264,7 +259,7 @@ const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi,
DEBUG_PRINTF("until aligned %p \n", d1);
if (d1 != d) {
SuperVector<S> chars = SuperVector<S>::loadu(d);
rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, low4bits, d);
rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, d);
DEBUG_PRINTF("rv %p \n", rv);
if (rv) return rv;
d = d1;
@ -274,25 +269,25 @@ const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi,
DEBUG_PRINTF("loops %ld \n", loops);
for (size_t i = 0; i < loops; i++, d+= S) {
DEBUG_PRINTF("d %p \n", d);
DEBUG_PRINTF("it = %ld, d %p \n", i, d);
const u8 *base = ROUNDUP_PTR(d, S);
// On large packet buffers, this prefetch appears to get us about 2%.
__builtin_prefetch(base + 256);
SuperVector<S> chars = SuperVector<S>::load(d);
rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, low4bits, d);
rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, d);
if (rv) return rv;
}
}
DEBUG_PRINTF("d %p e %p \n", d, buf_end);
DEBUG_PRINTF("tail d %p e %p \n", d, buf_end);
// finish off tail
if (d != buf_end) {
SuperVector<S> chars = SuperVector<S>::loadu(buf_end - S);
rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, low4bits, buf_end - S);
rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, buf_end - S);
DEBUG_PRINTF("rv %p \n", rv);
if (rv) return rv;
if (rv >= buf && rv < buf_end) return rv;
}
return buf_end;

2
src/nfa/truffle_simd.hpp
View File

@ -236,7 +236,7 @@ const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highse
if (d != buf) {
rv = truffleRevMini(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, buf, d);
DEBUG_PRINTF("rv %p \n", rv);
if (rv) return rv;
if (rv >= buf && rv < buf_end) return rv;
}
return buf - 1;