github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-11-17 09:45:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

add len parameter and mask, fixes corner cases on AVX512

Browse Source
This commit is contained in:
Konstantinos Margaritis
2021-11-05 14:30:22 +02:00
parent 694e2faf7f
commit 6317e24a82
3 changed files with 99 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
src/util/arch/x86/match.hpp
View File

@@ -29,7 +29,7 @@
template <>
really_really_inline
const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> v) {
const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> v, u16 const UNUSED len) {
SuperVector<16>::movemask_type z = v.movemask();
DEBUG_PRINTF("buf %p z %08x \n", buf, z);
DEBUG_PRINTF("z %08x\n", z);
@@ -46,7 +46,7 @@ const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> v) {
template <>
really_really_inline
const u8 *first_non_zero_match<32>(const u8 *buf, SuperVector<32> v) {
const u8 *first_non_zero_match<32>(const u8 *buf, SuperVector<32> v, u16 const UNUSED len) {
SuperVector<32>::movemask_type z = v.movemask();
DEBUG_PRINTF("z 0x%08x\n", z);
if (unlikely(z)) {
@@ -60,9 +60,13 @@ const u8 *first_non_zero_match<32>(const u8 *buf, SuperVector<32> v) {
}
template <>
really_really_inline
const u8 *first_non_zero_match<64>(const u8 *buf, SuperVector<64>v) {
const u8 *first_non_zero_match<64>(const u8 *buf, SuperVector<64>v, u16 const len) {
SuperVector<64>::movemask_type z = v.movemask();
DEBUG_PRINTF("z 0x%016llx\n", z);
u64a mask = (~0ULL) >> (64 - len);
DEBUG_PRINTF("mask %016llx\n", mask);
z &= mask;
DEBUG_PRINTF("z 0x%016llx\n", z);
if (unlikely(z)) {
u32 pos = ctz64(z);
DEBUG_PRINTF("match @ pos %u\n", pos);
@@ -75,7 +79,7 @@ const u8 *first_non_zero_match<64>(const u8 *buf, SuperVector<64>v) {
template <>
really_really_inline
const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> v) {
const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> v, u16 const UNUSED len) {
SuperVector<16>::movemask_type z = v.movemask();
DEBUG_PRINTF("buf %p z %08x \n", buf, z);
DEBUG_PRINTF("z %08x\n", z);
@@ -91,7 +95,7 @@ const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> v) {
template <>
really_really_inline
const u8 *last_non_zero_match<32>(const u8 *buf, SuperVector<32> v) {
const u8 *last_non_zero_match<32>(const u8 *buf, SuperVector<32> v, u16 const UNUSED len) {
SuperVector<32>::movemask_type z = v.movemask();
DEBUG_PRINTF("z 0x%08x\n", z);
if (unlikely(z)) {
@@ -105,14 +109,18 @@ const u8 *last_non_zero_match<32>(const u8 *buf, SuperVector<32> v) {
}
template <>
really_really_inline
const u8 *last_non_zero_match<64>(const u8 *buf, SuperVector<64>v) {
const u8 *last_non_zero_match<64>(const u8 *buf, SuperVector<64>v, u16 const len) {
SuperVector<64>::movemask_type z = v.movemask();
DEBUG_PRINTF("z 0x%016llx\n", z);
u64a mask = (~0ULL) >> (64 - len);
DEBUG_PRINTF("mask %016llx\n", mask);
z &= mask;
DEBUG_PRINTF("z 0x%016llx\n", z);
if (unlikely(z)) {
u32 pos = clz64(z);
DEBUG_PRINTF("match @ pos %u\n", pos);
assert(pos < 64);
return buf + (31 - pos);
return buf + (63 - pos);
} else {
return NULL; // no match
}
@@ -120,7 +128,7 @@ const u8 *last_non_zero_match<64>(const u8 *buf, SuperVector<64>v) {
template <>
really_really_inline
const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) {
const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v, u16 const UNUSED len) {
SuperVector<16>::movemask_type z = v.movemask();
DEBUG_PRINTF("buf %p z %08x \n", buf, z);
DEBUG_PRINTF("z %08x\n", z);
@@ -137,7 +145,7 @@ const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) {
template <>
really_really_inline
const u8 *first_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) {
const u8 *first_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v, u16 const UNUSED len) {
SuperVector<32>::movemask_type z = v.movemask();
DEBUG_PRINTF("z 0x%08x\n", z);
if (unlikely(z != 0xffffffff)) {
@@ -151,11 +159,15 @@ const u8 *first_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) {
}
template <>
really_really_inline
const u8 *first_zero_match_inverted<64>(const u8 *buf, SuperVector<64>v) {
const u8 *first_zero_match_inverted<64>(const u8 *buf, SuperVector<64>v, u16 const len) {
SuperVector<64>::movemask_type z = v.movemask();
DEBUG_PRINTF("z 0x%016llx\n", z);
if (unlikely(z != ~0ULL)) {
u32 pos = ctz64(~z);
u64a mask = (~0ULL) >> (64 - len);
DEBUG_PRINTF("mask %016llx\n", mask);
z = ~z & mask;
DEBUG_PRINTF("z 0x%016llx\n", z);
if (unlikely(z)) {
u32 pos = ctz64(z);
DEBUG_PRINTF("match @ pos %u\n", pos);
assert(pos < 64);
return buf + pos;
@@ -166,7 +178,7 @@ const u8 *first_zero_match_inverted<64>(const u8 *buf, SuperVector<64>v) {
template <>
really_really_inline
const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) {
const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v, uint16_t UNUSED len ) {
SuperVector<16>::movemask_type z = v.movemask();
DEBUG_PRINTF("buf %p z %08x \n", buf, z);
DEBUG_PRINTF("z %08x\n", z);
@@ -183,10 +195,10 @@ const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) {
template<>
really_really_inline
const u8 *last_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) {
const u8 *last_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v, uint16_t UNUSED len) {
SuperVector<32>::movemask_type z = v.movemask();
if (unlikely(z != 0xffffffff)) {
u32 pos = clz32(~z);
u32 pos = clz32(~z & 0xffffffff);
DEBUG_PRINTF("buf=%p, pos=%u\n", buf, pos);
assert(pos < 32);
return buf + (31 - pos);
@@ -197,11 +209,17 @@ const u8 *last_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) {
template <>
really_really_inline
const u8 *last_zero_match_inverted<64>(const u8 *buf, SuperVector<64> v) {
const u8 *last_zero_match_inverted<64>(const u8 *buf, SuperVector<64> v, uint16_t len) {
v.print8("v");
SuperVector<64>::movemask_type z = v.movemask();
DEBUG_PRINTF("z 0x%016llx\n", z);
if (unlikely(z != ~0ULL)) {
u32 pos = clz64(~z);
u64a mask = (~0ULL) >> (64 - len);
DEBUG_PRINTF("mask %016llx\n", mask);
z = ~z & mask;
DEBUG_PRINTF("z 0x%016llx\n", z);
if (unlikely(z)) {
u32 pos = clz64(z);
DEBUG_PRINTF("~z 0x%016llx\n", ~z);
DEBUG_PRINTF("match @ pos %u\n", pos);
assert(pos < 64);
return buf + (63 - pos);

8
src/util/match.hpp
View File

@@ -38,16 +38,16 @@
#include "util/supervector/supervector.hpp"
template <u16 S>
const u8 *first_non_zero_match(const u8 *buf, SuperVector<S> v);
const u8 *first_non_zero_match(const u8 *buf, SuperVector<S> v, u16 const len = S);
template <u16 S>
const u8 *last_non_zero_match(const u8 *buf, SuperVector<S> v);
const u8 *last_non_zero_match(const u8 *buf, SuperVector<S> v, u16 const len = S);
template <u16 S>
const u8 *first_zero_match_inverted(const u8 *buf, SuperVector<S> v);
const u8 *first_zero_match_inverted(const u8 *buf, SuperVector<S> v, u16 const len = S);
template <u16 S>
const u8 *last_zero_match_inverted(const u8 *buf, SuperVector<S> v);
const u8 *last_zero_match_inverted(const u8 *buf, SuperVector<S> v, u16 len = S);
#if defined(ARCH_IA32) || defined(ARCH_X86_64)
#include "util/arch/x86/match.hpp"