diff --git a/src/hwlm/noodle_engine_simd.hpp b/src/hwlm/noodle_engine_simd.hpp new file mode 100644 index 00000000..98289d59 --- /dev/null +++ b/src/hwlm/noodle_engine_simd.hpp @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2017, Intel Corporation + * Copyright (c) 2020, 2021, VectorCamp PC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* SIMD engine agnostic noodle scan parts */ + +#include "util/simd/types.hpp" + +// using Z_TYPE = typename SuperVector::movemask_type; + +#if defined(HAVE_SIMD_512_BITS) +using Z_TYPE = u64a; +#define Z_BITS 64 +#define Z_SHIFT 63 +#define DOUBLE_LOAD_MASK(l, off) ((~0ULL) >> (Z_BITS -l)) +#define SINGLE_LOAD_MASK(l) (((1ULL) << l) - 1ULL) +#elif defined(HAVE_SIMD_256_BITS) +using Z_TYPE = u32; +#define Z_BITS 32 +#define Z_SHIFT 31 +#define DOUBLE_LOAD_MASK(l, off) ((((1ULL) << l) - 1ULL) << off) +#define SINGLE_LOAD_MASK(l) (((1ULL) << l) - 1ULL) +#elif defined(HAVE_SIMD_128_BITS) +using Z_TYPE = u32; +#define Z_BITS 32 +#define Z_SHIFT 0 +#define DOUBLE_LOAD_MASK(l, off) ((((1ULL) << l) - 1ULL) << off) +#define SINGLE_LOAD_MASK(l) (((1ULL) << l) - 1ULL) +#endif + +static u8 CASEMASK[] = { 0xff, 0xdf }; + +static really_inline +u8 caseClear8(u8 x, bool noCase) +{ + return static_cast(x & CASEMASK[(u8)noCase]); +} + +template +static really_inline SuperVector getMask(u8 c, bool noCase) { + u8 k = caseClear8(c, noCase); + return SuperVector(k); +} + +template +static really_inline SuperVector getCaseMask(void) { + return SuperVector(CASEMASK[1]); +} + +// The short scan routine. It is used both to scan data up to an +// alignment boundary if needed and to finish off data that the aligned scan +// function can't handle (due to small/unaligned chunk at end) +template +static really_inline +hwlm_error_t scanSingleUnaligned2(const struct noodTable *n, const u8 *buf, + SuperVector caseMask, SuperVector mask1, + const struct cb_info *cbi, size_t len, size_t start, + size_t end) { + const u8 *d = buf + start; + DEBUG_PRINTF("start %zu end %zu\n", start, end); + const size_t l = end - start; + //assert(l <= 64); + if (!l) { + return HWLM_SUCCESS; + } + + typename SuperVector::movemask_type mask = SINGLE_LOAD_MASK(l); + SuperVector v = SuperVector::loadu(d) & caseMask; + typename SuperVector::movemask_type z = mask & mask1.eqmask(v); + + return single_zscan(n, d, buf, &z, len, cbi); +} + +template +static really_inline +hwlm_error_t scanSingleFast2(const struct noodTable *n, const u8 *buf, + size_t len, SuperVector caseMask, SuperVector mask1, + const struct cb_info *cbi, size_t start, + size_t loops) { + const u8 *d = buf + start; + + for (size_t i = 0; i < loops; i++, d+= S) { + const u8 *base = ROUNDUP_PTR(d, 64); + // On large packet buffers, this prefetch appears to get us about 2%. + __builtin_prefetch(base + 4*S); + + SuperVector v = SuperVector::load(d) & caseMask; + typename SuperVector::movemask_type z = mask1.eqmask(v); + + hwlm_error_t result = single_zscan(n, d, buf, &z, len, cbi); + if (unlikely(result != HWLM_SUCCESS)) + return result; + } + return HWLM_SUCCESS; +} + +template +static really_inline +hwlm_error_t scanDoubleUnaligned2(const struct noodTable *n, const u8 *buf, + SuperVector caseMask, SuperVector mask1, SuperVector mask2, + const struct cb_info *cbi, size_t len, size_t offset, size_t start, size_t end) { + const u8 *d = buf + offset; + DEBUG_PRINTF("start %zu end %zu", start, end); + const size_t l = end - start; + assert(l <= S); + if (!l) { + return HWLM_SUCCESS; + } + u32 buf_off = start - offset; + + SuperVector v = SuperVector::loadu(d) & caseMask; + + typename SuperVector::movemask_type mask = DOUBLE_LOAD_MASK(l, buf_off); + typename SuperVector::movemask_type z1 = mask1.eqmask(v); + typename SuperVector::movemask_type z2 = mask2.eqmask(v); + typename SuperVector::movemask_type z = mask & (z1 << 1) & z2; +#if defined(HAVE_AVX512) && defined(BUILD_AVX512) + DEBUG_PRINTF("buf_off = %d\n", buf_off); + DEBUG_PRINTF("l = %ld, mask = 0x%016llx\n", l, mask); + DEBUG_PRINTF("\nz1 = 0x%016llx\n", z1); + DEBUG_PRINTF("z2 = 0x%016llx\n", z2); + DEBUG_PRINTF("z = 0x%016llx\n", z); + __mmask64 k = (~0ULL) >> (64 - l); + DEBUG_PRINTF("k = 0x%016llx\n", k); + + m512 v1 = loadu_maskz_m512(k, d); + v1 = and512(v1, caseMask.u.v512[0]); + + u64a z0_ = masked_eq512mask(k, mask1.u.v512[0], v1); + u64a z1_ = masked_eq512mask(k, mask2.u.v512[0], v1); + u64a z_ = (z0_ << 1) & z1_; + DEBUG_PRINTF("z0_ = 0x%016llx\n", z0_); + DEBUG_PRINTF("z1_ = 0x%016llx\n", z1_); + DEBUG_PRINTF("z_ = 0x%016llx\n", z_); + assert(z == z_); +#endif + + return double_zscan(n, d, buf, &z, len, cbi); +} + +template +static really_inline +hwlm_error_t scanDoubleFast2(const struct noodTable *n, const u8 *buf, + size_t len, SuperVector caseMask, SuperVector mask1, SuperVector mask2, + const struct cb_info *cbi, size_t start, size_t end/*loops*/) { + const u8 *d = buf + start, *e = buf + end; + //DEBUG_PRINTF("start %zu loops %zu \n", start, loops); + typename SuperVector::movemask_type lastz1{0}; + + //for (size_t i=0; i < loops; i++, d+= S) { + for (; d < e; d+= S) { + const u8 *base = ROUNDUP_PTR(d, 64); + // On large packet buffers, this prefetch appears to get us about 2%. + __builtin_prefetch(base + 4*S); + + SuperVector v = SuperVector::load(d) & caseMask; + typename SuperVector::movemask_type z1 = mask1.eqmask(v); + typename SuperVector::movemask_type z2 = mask2.eqmask(v); + typename SuperVector::movemask_type z = (z1 << 1 | lastz1) & z2; + lastz1 = z1 >> Z_SHIFT; + + hwlm_error_t result = double_zscan(n, d, buf, &z, len, cbi); + if (unlikely(result != HWLM_SUCCESS)) + return result; + } + return HWLM_SUCCESS; +}