renamed matcher functions, added new ones for Vermicelli

src/nfa/shufti_simd.hpp

@@ -63,7 +63,7 @@ static really_inline
 const u8 *fwdBlock(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars, const u8 *buf) {
     SuperVector<S> v = blockSingleMask(mask_lo, mask_hi, chars);
 
-    return firstMatch<S>(buf, v);
+    return first_zero_match_inverted<S>(buf, v);
 }
 
 template <uint16_t S>
@@ -71,7 +71,7 @@ static really_inline
 const u8 *revBlock(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars, const u8 *buf) {
     SuperVector<S> v = blockSingleMask(mask_lo, mask_hi, chars);
 
-    return lastMatch<S>(buf, v);
+    return last_zero_match_inverted<S>(buf, v);
 }
 
 template <uint16_t S>
@@ -80,7 +80,7 @@ const u8 *fwdBlockDouble(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi, Super
 
     SuperVector<S> mask = blockDoubleMask(mask1_lo, mask1_hi, mask2_lo, mask2_hi, chars);
 
-    return firstMatch<S>(buf, mask);
+    return first_zero_match_inverted<S>(buf, mask);
 }
 
 template <uint16_t S>

src/nfa/truffle_simd.hpp

@@ -56,7 +56,7 @@ static really_inline
 const u8 *fwdBlock(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars, const u8 *buf) {
     SuperVector<S> res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars);
 
-    return firstMatch<S>(buf, res);
+    return first_zero_match_inverted<S>(buf, res);
 }
 
 template <uint16_t S>
@@ -120,7 +120,7 @@ static really_inline
 const u8 *revBlock(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> v, 
                     const u8 *buf) {
     SuperVector<S> res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, v);
-    return lastMatch<S>(buf, res);
+    return last_zero_match_inverted<S>(buf, res);
 }
 
 template <uint16_t S>

src/nfa/x86/shufti.hpp

@@ -31,12 +31,6 @@
  * \brief Shufti: character class acceleration.
  */
 
-#ifndef SHUFTI_SIMD_X86_HPP
-#define SHUFTI_SIMD_X86_HPP
-
-#include "util/supervector/supervector.hpp"
-#include "util/match.hpp"
-
 template <uint16_t S>
 static really_inline
 const SuperVector<S> blockSingleMask(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars) {
@@ -44,12 +38,10 @@ const SuperVector<S> blockSingleMask(SuperVector<S> mask_lo, SuperVector<S> mask
 
     SuperVector<S> c_lo = chars & low4bits;
     SuperVector<S> c_hi = chars.template vshr_64_imm<4>() & low4bits;
-    c_lo = mask_lo.template pshufb(c_lo);
+    c_lo = mask_lo.pshufb(c_lo);
-    c_hi = mask_hi.template pshufb(c_hi);
+    c_hi = mask_hi.pshufb(c_hi);
 
-    SuperVector c = c_lo & c_hi;
+    return (c_lo & c_hi).eq(SuperVector<S>::Zeroes());
-
-    return c.eq(SuperVector<S>::Zeroes());
 }
 
 template <uint16_t S>
@@ -80,5 +72,3 @@ SuperVector<S> blockDoubleMask(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi,
 
     return c.eq(SuperVector<S>::Ones());
 }
-
-#endif // SHUFTI_SIMD_X86_HPP

src/util/arch/arm/match.hpp

@@ -29,7 +29,44 @@
 
 template <>
 really_really_inline
-const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> mask) {
+const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> mask) {
+    uint32x4_t res_t = vreinterpretq_u32_u8(mask.u.v128[0]);
+    uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(res_t, res_t)), 0);
+    if (vmax != 0) {
+    typename SuperVector<16>::movemask_type z = mask.movemask();
+        DEBUG_PRINTF("z %08x\n", z);
+        DEBUG_PRINTF("buf %p z %08x \n", buf, z);
+        u32 pos = ctz32(z & 0xffff);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        assert(pos < 16);
+        DEBUG_PRINTF("buf + pos %p\n", buf + pos);
+        return buf + pos;
+    } else {
+        return NULL; // no match
+    }
+}
+
+template <>
+really_really_inline
+const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> mask) {
+    uint32x4_t res_t = vreinterpretq_u32_u8(mask.u.v128[0]);
+    uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(res_t, res_t)), 0);
+    if (vmax != 0) {
+    typename SuperVector<16>::movemask_type z = mask.movemask();
+        DEBUG_PRINTF("buf %p z %08x \n", buf, z);
+        DEBUG_PRINTF("z %08x\n", z);
+        u32 pos = clz32(z & 0xffff);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        assert(pos >= 16 && pos < 32);
+        return buf + (31 - pos);
+    } else {
+        return NULL; // no match
+    }
+}
+
+template <>
+really_really_inline
+const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask) {
     uint32x4_t res_t = vreinterpretq_u32_u8(mask.u.v128[0]);
     uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(res_t, res_t)), 0);
     if (vmax != 0) {
@@ -48,7 +85,7 @@ const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> mask) {
 
 template <>
 really_really_inline
-const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> mask) {
+const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask) {
     uint32x4_t res_t = vreinterpretq_u32_u8(mask.u.v128[0]);
     uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(res_t, res_t)), 0);
     if (vmax != 0) {

src/util/arch/x86/match.hpp

@@ -29,7 +29,98 @@
 
 template <>
 really_really_inline
-const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> v) {
+const u8 *first_non_zero_match<16>(const u8 *buf, SuperVector<16> v) {
+    SuperVector<16>::movemask_type z = v.movemask();
+    DEBUG_PRINTF("buf %p z %08x \n", buf, z);
+    DEBUG_PRINTF("z %08x\n", z);
+    if (unlikely(z)) {
+        u32 pos = ctz32(z);
+        DEBUG_PRINTF("~z %08x\n", ~z);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        assert(pos < 16);
+        return buf + pos;
+    } else {
+        return NULL; // no match
+    }
+}
+
+template <>
+really_really_inline
+const u8 *first_non_zero_match<32>(const u8 *buf, SuperVector<32> v) {
+    SuperVector<32>::movemask_type z = v.movemask();
+    DEBUG_PRINTF("z 0x%08x\n", z);
+    if (unlikely(z)) {
+        u32 pos = ctz32(z);
+        assert(pos < 32);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        return buf + pos;
+    } else {
+        return NULL; // no match
+    }
+}
+template <>
+really_really_inline
+const u8 *first_non_zero_match<64>(const u8 *buf, SuperVector<64>v) {
+    SuperVector<64>::movemask_type z = v.movemask();
+    DEBUG_PRINTF("z 0x%016llx\n", z);
+    if (unlikely(z)) {
+        u32 pos = ctz64(z);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        assert(pos < 64);
+        return buf + pos;
+    } else {
+        return NULL; // no match
+    }
+}
+
+template <>
+really_really_inline
+const u8 *last_non_zero_match<16>(const u8 *buf, SuperVector<16> v) {
+    SuperVector<16>::movemask_type z = v.movemask();
+    DEBUG_PRINTF("buf %p z %08x \n", buf, z);
+    DEBUG_PRINTF("z %08x\n", z);
+    if (unlikely(z)) {
+        u32 pos = clz32(z);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        assert(pos >= 16 && pos < 32);
+        return buf + (31 - pos);
+    } else {
+        return NULL; // no match
+    }
+}
+
+template <>
+really_really_inline
+const u8 *last_non_zero_match<32>(const u8 *buf, SuperVector<32> v) {
+    SuperVector<32>::movemask_type z = v.movemask();
+    DEBUG_PRINTF("z 0x%08x\n", z);
+    if (unlikely(z)) {
+        u32 pos = clz32(z);
+        assert(pos < 32);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        return buf + (31 - pos);
+    } else {
+        return NULL; // no match
+    }
+}
+template <>
+really_really_inline
+const u8 *last_non_zero_match<64>(const u8 *buf, SuperVector<64>v) {
+    SuperVector<64>::movemask_type z = v.movemask();
+    DEBUG_PRINTF("z 0x%016llx\n", z);
+    if (unlikely(z)) {
+        u32 pos = clz64(z);
+        DEBUG_PRINTF("match @ pos %u\n", pos);
+        assert(pos < 64);
+        return buf + (31 - pos);
+    } else {
+        return NULL; // no match
+    }
+}
+
+template <>
+really_really_inline
+const u8 *first_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) {
     SuperVector<16>::movemask_type z = v.movemask();
     DEBUG_PRINTF("buf %p z %08x \n", buf, z);
     DEBUG_PRINTF("z %08x\n", z);
@@ -46,7 +137,7 @@ const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> v) {
 
 template <>
 really_really_inline
-const u8 *firstMatch<32>(const u8 *buf, SuperVector<32> v) {
+const u8 *first_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) {
     SuperVector<32>::movemask_type z = v.movemask();
     DEBUG_PRINTF("z 0x%08x\n", z);
     if (unlikely(z != 0xffffffff)) {
@@ -60,7 +151,7 @@ const u8 *firstMatch<32>(const u8 *buf, SuperVector<32> v) {
 }
 template <>
 really_really_inline
-const u8 *firstMatch<64>(const u8 *buf, SuperVector<64>v) {
+const u8 *first_zero_match_inverted<64>(const u8 *buf, SuperVector<64>v) {
     SuperVector<64>::movemask_type z = v.movemask();
     DEBUG_PRINTF("z 0x%016llx\n", z);
     if (unlikely(z != ~0ULL)) {
@@ -75,7 +166,7 @@ const u8 *firstMatch<64>(const u8 *buf, SuperVector<64>v) {
 
 template <>
 really_really_inline
-const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> v) {
+const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> v) {
     SuperVector<16>::movemask_type z = v.movemask();
     DEBUG_PRINTF("buf %p z %08x \n", buf, z);
     DEBUG_PRINTF("z %08x\n", z);
@@ -92,7 +183,7 @@ const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> v) {
 
 template<>
 really_really_inline
-const u8 *lastMatch<32>(const u8 *buf, SuperVector<32> v) {
+const u8 *last_zero_match_inverted<32>(const u8 *buf, SuperVector<32> v) {
     SuperVector<32>::movemask_type z = v.movemask();
     if (unlikely(z != 0xffffffff)) {
         u32 pos = clz32(~z);
@@ -106,7 +197,7 @@ const u8 *lastMatch<32>(const u8 *buf, SuperVector<32> v) {
 
 template <>
 really_really_inline
-const u8 *lastMatch<64>(const u8 *buf, SuperVector<64> v) {
+const u8 *last_zero_match_inverted<64>(const u8 *buf, SuperVector<64> v) {
     SuperVector<64>::movemask_type z = v.movemask();
     DEBUG_PRINTF("z 0x%016llx\n", z);
     if (unlikely(z != ~0ULL)) {

src/util/match.hpp

@@ -38,10 +38,16 @@
 #include "util/supervector/supervector.hpp"
 
 template <u16 S>
-const u8 *firstMatch(const u8 *buf, SuperVector<S> v);
+const u8 *first_non_zero_match(const u8 *buf, SuperVector<S> v);
 
 template <u16 S>
-const u8 *lastMatch(const u8 *buf, SuperVector<S> v);
+const u8 *last_non_zero_match(const u8 *buf, SuperVector<S> v);
+
+template <u16 S>
+const u8 *first_zero_match_inverted(const u8 *buf, SuperVector<S> v);
+
+template <u16 S>
+const u8 *last_zero_match_inverted(const u8 *buf, SuperVector<S> v);
 
 #if defined(ARCH_IA32) || defined(ARCH_X86_64)
 #include "util/arch/x86/match.hpp"