Merge pull request #217 from ypicchi-arm/feature/Add-truffle-SVE-implementation

Add truffle SVE implementation

src/nfa/arm/truffle.hpp

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2015-2017, Intel Corporation
  * Copyright (c) 2020-2021, VectorCamp PC
+ * Copyright (c) 2023, Arm Limited
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -32,6 +33,76 @@
  *
  */
 
+#ifdef HAVE_SVE
+
+/*
+ * blockSingleMask takes in a character set (as masks) and a string and return for each character
+ * of the string weither or not it is part of the set.
+ *
+ * 'shuf_mask_lo_highclear' and 'shuf_mask_lo_highset' are 128-bit masks where each bit
+ * represents whether or not a character is in the character set. The 'highclear' and
+ * 'highset' in the name refers to the MSb of the byte of the character (allowing two
+ * 128-bit masks to cover all 256 values).
+ * 
+ * The masks are arrays of 16 bytes each and are encoded this way:
+ * Let C be a character in the set. The bit describing that character is at byte[C%16] and
+ * within that byte, it's at bit[C/16]
+ * As example, 'a' = 0x61, so the resulting mask will be: 0x00 0x40 0x00 0x00 0x00 ...
+ * 
+ * Assume both mask are 128b wide. If they are larger, the additional bits must be zero
+ */
+static really_inline
+svuint8_t blockSingleMask(svuint8_t shuf_mask_lo_highclear, svuint8_t shuf_mask_lo_highset, svuint8_t chars) {
+
+    const svuint8_t highconst = svdup_u8(0x80);
+    const svuint8_t pshub_mask = svdup_u8(0x8f);
+    const svuint8_t unique_bit_per_lane_mask = svreinterpret_u8(svdup_u64(0x8040201008040201));
+
+    /*
+     * svtbl does a table lookup. Each byte in the second argument indexes into the array of bytes
+     * in shuf_mask_lo_highclear and saves the result in the corresponding byte of byte_select_low.
+     * We mask the chars so that we are using the low nibble of char as the index but we keep the
+     * MSb so that high characters (not represented by the highclear mask) become an index out of
+     * bounds and result in a 0.
+     */
+    svuint8_t byte_select_low = svtbl(shuf_mask_lo_highclear, svand_x(svptrue_b8(), chars, pshub_mask));
+
+    /* 
+     * We flip the MSb of the chars and do the same table lookup with the highset mask.
+     * This way it's the characters with MSb cleared that will result in out of bands indexes.
+     * This allows us to cover the full range (0-127 and 128-255)
+     */
+    svuint8_t char_high_flipped = sveor_x(svptrue_b8(), chars, highconst);
+    svuint8_t byte_select_high = svtbl(shuf_mask_lo_highset, svand_x(svptrue_b8(), char_high_flipped, pshub_mask));
+
+    /*
+     * We now have selected the byte that contain the bit corresponding to the char. We need to
+     * further filter it, otherwise we'd get a match for any character % 16 to a searched character
+     * 
+     * The low nibble was used previously to select the byte out of the mask. The high nibble is
+     * used to select the bit out of the byte. So we shift everything right by 4.
+     * 
+     * Using svtbl, we can make an array where each element is a different bit. Using the high
+     * nibble we can get a mask selecting only the bit out of a byte that may have the relevant
+     * charset char.
+     */
+    svuint8_t char_high_nibble = svlsr_x(svptrue_b8(), chars, 4);
+    svuint8_t bit_select = svtbl(unique_bit_per_lane_mask, char_high_nibble);
+    /*
+     * For every lane, only one of the byte selected may have a value, so we can OR them. We
+     * then apply the bit_select mask. What is left is the bit in the charset encoding the
+     * character in char. A non zero value means the char was in the charset
+     * 
+     * The _x suffix only works if we process a full char vector. If we were to use a partial
+     * vector, then _z and a mask would be required on this svand only. Otherwise, the disabled
+     * lanes may have arbitrary values
+     */
+    svuint8_t res = svand_x(svptrue_b8(), svorr_x(svptrue_b8(), byte_select_low, byte_select_high), bit_select);
+
+    return res;
+}
+#else
+
 template <uint16_t S>
 static really_inline
 const SuperVector<S> blockSingleMask(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars) {
@@ -60,3 +131,4 @@ const SuperVector<S> blockSingleMask(SuperVector<S> shuf_mask_lo_highclear, Supe
 
     return !res.eq(SuperVector<S>::Zeroes());
 }
+#endif //HAVE_SVE

src/nfa/truffle.cpp

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2015-2017, Intel Corporation
  * Copyright (c) 2020, 2021, VectorCamp PC
+ * Copyright (c) 2023, Arm Limited
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -37,7 +38,17 @@
 #include "util/bitutils.h"
 
 #include "truffle_simd.hpp"
+#ifdef HAVE_SVE
+const u8 *truffleExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
+                      const u8 *buf_end) {
+    return truffleExecSVE(mask_lo, mask_hi, buf, buf_end);
+}
 
+const u8 *rtruffleExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
+                       const u8 *buf_end) {
+    return rtruffleExecSVE(mask_lo, mask_hi, buf, buf_end);
+}
+#else
 const u8 *truffleExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
                       const u8 *buf_end) {
     return truffleExecReal<VECTORSIZE>(mask_lo, mask_hi, buf, buf_end);
@@ -47,3 +58,4 @@ const u8 *rtruffleExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
                        const u8 *buf_end) {
     return rtruffleExecReal<VECTORSIZE>(mask_lo, mask_hi, buf, buf_end);
 }
+#endif //HAVE_SVE

src/nfa/truffle_simd.hpp

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2015-2017, Intel Corporation
  * Copyright (c) 2020-2023, VectorCamp PC
+ * Copyright (c) 2023, Arm Limited
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -41,9 +42,14 @@
 #include "util/supervector/supervector.hpp"
 #include "util/match.hpp"
 
+#ifdef HAVE_SVE
+static really_inline
+svuint8_t blockSingleMask(svuint8_t shuf_mask_lo_highclear, svuint8_t shuf_mask_lo_highset, svuint8_t chars);
+#else
 template <uint16_t S>
 static really_inline
 const SuperVector<S> blockSingleMask(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars);
+#endif //HAVE_SVE
 
 #if defined(VS_SIMDE_BACKEND)
 #include "x86/truffle.hpp"
@@ -57,6 +63,162 @@ const SuperVector<S> blockSingleMask(SuperVector<S> shuf_mask_lo_highclear, Supe
 #endif
 #endif
 
+#ifdef HAVE_SVE
+
+const u8 *truffleExecSVE(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highset,
+                      const u8 *buf, const u8 *buf_end);
+
+const u8 *rtruffleExecSVE(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highset,
+                       const u8 *buf, const u8 *buf_end);
+
+static really_inline
+const u8 *scanBlock(svuint8_t shuf_mask_lo_highclear, svuint8_t shuf_mask_lo_highset, svuint8_t chars, const u8 *buf, bool forward) {
+
+    const size_t vector_size_int_8 = svcntb();
+
+    const svuint8_t result_mask = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars);
+    uint64_t index;
+    if (forward) {
+        index = first_non_zero(vector_size_int_8, result_mask);
+    } else {
+        index = last_non_zero(vector_size_int_8, result_mask);
+    }
+
+    if(index < vector_size_int_8) {
+        return buf+index;
+    } else {
+        return NULL;
+    }
+}
+
+really_inline
+const u8 *truffleExecSVE(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highset, const u8 *buf, const u8 *buf_end) {
+    const int vect_size_int8 = svcntb();
+    // Activate only 16 lanes to read the m128 buffers
+    const svbool_t lane_pred_16 = svwhilelt_b8(0, 16);
+    assert(buf && buf_end);
+    assert(buf < buf_end);
+    DEBUG_PRINTF("truffle %p len %zu\n", buf, buf_end - buf);
+    DEBUG_PRINTF("b %s\n", buf);
+
+    svuint8_t wide_shuf_mask_lo_highclear = svld1(lane_pred_16, (uint8_t*) &shuf_mask_lo_highclear);
+    svuint8_t wide_shuf_mask_lo_highset = svld1(lane_pred_16, (uint8_t*) &shuf_mask_lo_highset);
+
+    const u8 *work_buffer = buf;
+    const u8 *ret_val;
+
+    DEBUG_PRINTF("start %p end %p \n", work_buffer, buf_end);
+    assert(work_buffer < buf_end);
+
+    __builtin_prefetch(work_buffer + 16*64);
+
+    if (work_buffer + vect_size_int8 <= buf_end) {
+        // Reach vector aligned boundaries
+        DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(work_buffer, vect_size_int8));
+        if (!ISALIGNED_N(work_buffer, vect_size_int8)) {
+            svuint8_t chars = svld1(svptrue_b8(), work_buffer);
+            const u8 *alligned_buffer = ROUNDUP_PTR(work_buffer, vect_size_int8);
+            ret_val = scanBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, work_buffer, true);
+            if (ret_val && ret_val < alligned_buffer) return ret_val;
+            work_buffer = alligned_buffer;
+        }
+
+        while(work_buffer + vect_size_int8 <= buf_end) {
+            __builtin_prefetch(work_buffer + 16*64);
+            DEBUG_PRINTF("work_buffer %p \n", work_buffer);
+            svuint8_t chars = svld1(svptrue_b8(), work_buffer);
+            ret_val = scanBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, work_buffer, true);
+            if (ret_val) return ret_val;
+            work_buffer += vect_size_int8;
+        }
+    }
+
+    DEBUG_PRINTF("work_buffer %p e %p \n", work_buffer, buf_end);
+    // finish off tail
+
+    if (work_buffer != buf_end) {
+        svuint8_t chars;
+        const u8* end_buf;
+        if (buf_end - buf < vect_size_int8) {
+            const svbool_t remaining_lanes = svwhilelt_b8(0ll, buf_end - buf);
+            chars = svld1(remaining_lanes, buf);
+            end_buf = buf;
+        } else {
+            chars = svld1(svptrue_b8(), buf_end - vect_size_int8);
+            end_buf = buf_end - vect_size_int8;
+        }
+        ret_val = scanBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, end_buf, true);
+        DEBUG_PRINTF("ret_val %p \n", ret_val);
+        if (ret_val && ret_val < buf_end) return ret_val;
+    }
+
+    return buf_end;
+}
+
+really_inline
+const u8 *rtruffleExecSVE(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highset, const u8 *buf, const u8 *buf_end){
+    const int vect_size_int8 = svcntb();
+    // Activate only 16 lanes to read the m128 buffers
+    const svbool_t lane_pred_16 = svwhilelt_b8(0, 16);
+    assert(buf && buf_end);
+    assert(buf < buf_end);
+    DEBUG_PRINTF("truffle %p len %zu\n", buf, buf_end - buf);
+    DEBUG_PRINTF("b %s\n", buf);
+
+    svuint8_t wide_shuf_mask_lo_highclear = svld1(lane_pred_16, (uint8_t*) &shuf_mask_lo_highclear);
+    svuint8_t wide_shuf_mask_lo_highset = svld1(lane_pred_16, (uint8_t*) &shuf_mask_lo_highset);
+
+    const u8 *work_buffer = buf_end;
+    const u8 *ret_val;
+
+    DEBUG_PRINTF("start %p end %p \n", buf, work_buffer);
+    assert(work_buffer > buf);
+
+    __builtin_prefetch(work_buffer - 16*64);
+
+    if (work_buffer - vect_size_int8 >= buf) {
+        // Reach vector aligned boundaries
+        DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(work_buffer, vect_size_int8));
+        if (!ISALIGNED_N(work_buffer, vect_size_int8)) {
+            svuint8_t chars = svld1(svptrue_b8(), work_buffer - vect_size_int8);
+            const u8 *alligned_buffer = ROUNDDOWN_PTR(work_buffer, vect_size_int8);
+            ret_val = scanBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, work_buffer - vect_size_int8, false);
+            DEBUG_PRINTF("ret_val %p \n", ret_val);
+            if (ret_val >= alligned_buffer) return ret_val;
+            work_buffer = alligned_buffer;
+        }
+
+        while (work_buffer - vect_size_int8 >= buf) {
+            DEBUG_PRINTF("aligned %p \n", work_buffer);
+            // On large packet buffers, this prefetch appears to get us about 2%.
+            __builtin_prefetch(work_buffer - 16*64);
+
+            work_buffer -= vect_size_int8;
+            svuint8_t chars = svld1(svptrue_b8(), work_buffer);
+            ret_val = scanBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, work_buffer, false);
+            if (ret_val) return ret_val;
+        }
+    }
+
+    DEBUG_PRINTF("tail work_buffer %p e %p \n", buf, work_buffer);
+    // finish off head
+
+    if (work_buffer != buf) {
+        svuint8_t chars;
+        if (buf_end - buf < vect_size_int8) {
+            const svbool_t remaining_lanes = svwhilele_b8(0ll, buf_end - buf);
+            chars = svld1(remaining_lanes, buf);
+        } else {
+            chars = svld1(svptrue_b8(), buf);
+        }
+        ret_val = scanBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, buf, false);
+        DEBUG_PRINTF("ret_val %p \n", ret_val);
+        if (ret_val && ret_val < buf_end) return ret_val;
+    }
+
+    return buf - 1;
+}
+#else
 template <uint16_t S>
 static really_inline
 const u8 *fwdBlock(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars, const u8 *buf) {
@@ -77,13 +239,7 @@ const u8 *truffleExecReal(m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_highse
     const u8 *d = buf;
     const u8 *rv;
 
-    DEBUG_PRINTF("start %p end %p \n", d, buf_end);
-    assert(d < buf_end);
-
-    __builtin_prefetch(d +   64);
-    __builtin_prefetch(d + 2*64);
-    __builtin_prefetch(d + 3*64);
-    __builtin_prefetch(d + 4*64);
+    __builtin_prefetch(d + 16*64);
     DEBUG_PRINTF("start %p end %p \n", d, buf_end);
     assert(d < buf_end);
     if (d + S <= buf_end) {
@@ -98,7 +254,7 @@ const u8 *truffleExecReal(m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_highse
         }
 
         while(d + S <= buf_end) {
-            __builtin_prefetch(d + 64);
+            __builtin_prefetch(d + 16*64);
             DEBUG_PRINTF("d %p \n", d);
             SuperVector<S> chars = SuperVector<S>::load(d);
             rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d);
@@ -149,10 +305,7 @@ const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highse
     const u8 *d = buf_end;
     const u8 *rv;
 
-    __builtin_prefetch(d -   64);
-    __builtin_prefetch(d - 2*64);
-    __builtin_prefetch(d - 3*64);
-    __builtin_prefetch(d - 4*64);
+    __builtin_prefetch(d - 16*64);
     DEBUG_PRINTF("start %p end %p \n", buf, d);
     assert(d > buf);
     if (d - S >= buf) {
@@ -170,7 +323,7 @@ const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highse
         while (d - S >= buf) {
             DEBUG_PRINTF("aligned %p \n", d);
             // On large packet buffers, this prefetch appears to get us about 2%.
-            __builtin_prefetch(d - 64);
+            __builtin_prefetch(d - 16*64);
 
             d -= S;
             SuperVector<S> chars = SuperVector<S>::load(d);
@@ -196,3 +349,4 @@ const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highse
 
     return buf - 1;
 }
+#endif //HAVE_SVE

src/util/arch/arm/match.hpp

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2015-2017, Intel Corporation
  * Copyright (c) 2020-2021, VectorCamp PC
+ * Copyright (c) 2023, Arm Limited
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -99,3 +100,64 @@ const u8 *last_zero_match_inverted<16>(const u8 *buf, SuperVector<16> mask, u16
     }
 }
 
+#ifdef HAVE_SVE
+
+
+static really_inline
+uint64_t last_non_zero_real(svuint8_t mask) {
+	const svuint64_t leading_zeros = svclz_x(svptrue_b64(), svreinterpret_u64(mask));
+
+        uint64_t last_active_lane;
+
+        svbool_t remaining_mask = svptrue_b64();
+        uint64_t i = 0;
+        while(svptest_any(svptrue_b64(), remaining_mask)) {
+            svbool_t single_lane_mask = svpnext_b64(remaining_mask, svpfalse());
+            remaining_mask = sveor_z(svptrue_b64(), remaining_mask, single_lane_mask);
+            uint64_t active_element = svlastb(single_lane_mask, leading_zeros);
+            if(active_element<64) {
+                uint64_t lane_index = (i+1)*8 - (active_element/8) - 1;
+                last_active_lane = lane_index;
+            }
+            i++;
+        }
+        return last_active_lane;
+}
+
+/*
+ * It is assumed mask have the value 0 for all inactive lanes, if any.
+ */
+static really_inline
+uint64_t last_non_zero(const size_t vector_size_int_8, svuint8_t mask) {
+    const svbool_t result_pred = svcmpne(svptrue_b8(), mask, 0);
+
+    if (svptest_any(svptrue_b8(), result_pred)) {
+        return last_non_zero_real(mask);
+    } else {
+        return vector_size_int_8;
+    }
+}
+
+/*
+ * It is assumed mask have the value 0 for all inactive lanes, if any.
+ */
+static really_inline
+uint64_t first_non_zero(const size_t vector_size_int_8, svuint8_t mask) {
+    const svbool_t result_pred = svcmpne(svptrue_b8(), mask, 0);
+
+    if (svptest_any(svptrue_b8(), result_pred)) {
+
+        // We don't have a CTZ instruction but we can work around by reversing the lane order
+        const svuint64_t rev_large_res = svreinterpret_u64(svrev(mask));
+	// Now each pack of 8 leading 0 means one empty lane. So if we have 18 leading 0,
+        // that means the third lane have a matching character.
+	uint64_t first_active_lane = last_non_zero_real(svreinterpret_u8(rev_large_res));
+        // We reversed the lanes, so we reverse back the index
+	first_active_lane = (vector_size_int_8-1) - first_active_lane;
+        return first_active_lane;
+    } else {
+        return vector_size_int_8;
+    }
+}
+
+#endif //HAVE_SVE

src/util/supervector/supervector.hpp

@@ -138,7 +138,7 @@ struct BaseVector<64>
   static constexpr u16  previous_size = 32;
 };
 
-// 128 bit implementation
+// 256 bit implementation
 template <>
 struct BaseVector<32>
 {