From f77837130d9c188dade426ecb266ed7b3c0da829 Mon Sep 17 00:00:00 2001 From: Konstantinos Margaritis Date: Wed, 12 May 2021 20:18:05 +0300 Subject: [PATCH] delete separate implementations --- src/hwlm/noodle_engine_avx2.c | 138 ----------------------------- src/hwlm/noodle_engine_avx512.c | 149 -------------------------------- src/hwlm/noodle_engine_sse.c | 131 ---------------------------- 3 files changed, 418 deletions(-) delete mode 100644 src/hwlm/noodle_engine_avx2.c delete mode 100644 src/hwlm/noodle_engine_avx512.c delete mode 100644 src/hwlm/noodle_engine_sse.c diff --git a/src/hwlm/noodle_engine_avx2.c b/src/hwlm/noodle_engine_avx2.c deleted file mode 100644 index 0aebdc67..00000000 --- a/src/hwlm/noodle_engine_avx2.c +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (c) 2015-2017, Intel Corporation - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/* noodle scan parts for AVX */ - -static really_inline m256 getMask(u8 c, bool noCase) { - u8 k = caseClear8(c, noCase); - return set1_32x8(k); -} - -static really_inline m256 getCaseMask(void) { - return set1_32x8(0xdf); -} - -static really_inline -hwlm_error_t scanSingleUnaligned(const struct noodTable *n, const u8 *buf, - size_t len, size_t offset, m256 caseMask, m256 mask1, - const struct cb_info *cbi, size_t start, - size_t end) { - const u8 *d = buf + offset; - DEBUG_PRINTF("start %zu end %zu offset %zu\n", start, end, offset); - const size_t l = end - start; - m256 v = and256(loadu256(d), caseMask); - - u32 z = movemask256(eq256(mask1, v)); - - u32 buf_off = start - offset; - u32 mask = (u32)((u64a)(1ULL << l) - 1) << buf_off; - DEBUG_PRINTF("mask 0x%08x z 0x%08x\n", mask, z); - - z &= mask; - - return single_zscan(n, d, buf, z, len, cbi); -} - -static really_inline -hwlm_error_t scanDoubleUnaligned(const struct noodTable *n, const u8 *buf, - size_t len, size_t offset, m256 caseMask, m256 mask1, m256 mask2, - const struct cb_info *cbi, size_t start, - size_t end) { - const u8 *d = buf + offset; - DEBUG_PRINTF("start %zu end %zu offset %zu\n", start, end, offset); - size_t l = end - start; - m256 v = and256(loadu256(d), caseMask); - - u32 z0 = movemask256(eq256(mask1, v)); - u32 z1 = movemask256(eq256(mask2, v)); - u32 z = (z0 << 1) & z1; - - // mask out where we can't match - u32 buf_off = start - offset; - u32 mask = (u32)((u64a)(1ULL << l) - 1) << buf_off; - DEBUG_PRINTF("mask 0x%08x z 0x%08x\n", mask, z); - z &= mask; - - return double_zscan(n, d, buf, z, len, cbi); -} - -static really_inline -hwlm_error_t scanSingleFast(const struct noodTable *n, const u8 *buf, - size_t len, m256 caseMask, m256 mask1, - const struct cb_info *cbi, size_t start, - size_t end) { - const u8 *d = buf + start, *e = buf + end; - assert(d < e); - - for (; d < e; d += 32) { - m256 v = and256(load256(d), caseMask); - - u32 z = movemask256(eq256(mask1, v)); - - // On large packet buffers, this prefetch appears to get us about 2%. - __builtin_prefetch(ROUNDDOWN_PTR(d + 128, 64)); - - hwlm_error_t result = single_zscan(n, d, buf, z, len, cbi); - if (unlikely(result != HWLM_SUCCESS)) - return result; - - } - return HWLM_SUCCESS; -} - -static really_inline -hwlm_error_t scanDoubleFast(const struct noodTable *n, const u8 *buf, - size_t len, m256 caseMask, m256 mask1, - m256 mask2, const struct cb_info *cbi, size_t start, - size_t end) { - const u8 *d = buf + start, *e = buf + end; - DEBUG_PRINTF("start %zu end %zu \n", start, end); - assert(d < e); - u32 lastz0 = 0; - - for (; d < e; d += 32) { - m256 v = and256(load256(d), caseMask); - - // we have to pull the masks out of the AVX registers because we can't - // byte shift between the lanes - u32 z0 = movemask256(eq256(mask1, v)); - u32 z1 = movemask256(eq256(mask2, v)); - u32 z = (lastz0 | (z0 << 1)) & z1; - lastz0 = z0 >> 31; - - // On large packet buffers, this prefetch appears to get us about 2%. - __builtin_prefetch(ROUNDDOWN_PTR(d + 128, 64)); - - hwlm_error_t result = double_zscan(n, d, buf, z, len, cbi); - if (unlikely(result != HWLM_SUCCESS)) - return result; - - } - return HWLM_SUCCESS; -} - diff --git a/src/hwlm/noodle_engine_avx512.c b/src/hwlm/noodle_engine_avx512.c deleted file mode 100644 index f992e83f..00000000 --- a/src/hwlm/noodle_engine_avx512.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2017, Intel Corporation - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/* noodle scan parts for AVX512 */ - -static really_inline -m512 getMask(u8 c, bool noCase) { - u8 k = caseClear8(c, noCase); - return set1_64x8(k); -} - -static really_inline -m512 getCaseMask(void) { - return set1_64x8(CASE_CLEAR); -} - -// The short scan routine. It is used both to scan data up to an -// alignment boundary if needed and to finish off data that the aligned scan -// function can't handle (due to small/unaligned chunk at end) -static really_inline -hwlm_error_t scanSingleUnaligned(const struct noodTable *n, const u8 *buf, - size_t len, size_t offset, m512 caseMask, m512 mask1, - const struct cb_info *cbi, size_t start, - size_t end) { - const u8 *d = buf + offset; - DEBUG_PRINTF("start %zu end %zu offset %zu\n", start, end, offset); - const size_t l = end - start; - assert(l <= 64); - if (!l) { - return HWLM_SUCCESS; - } - - __mmask64 k = (~0ULL) >> (64 - l); - DEBUG_PRINTF("load mask 0x%016llx\n", k); - - m512 v = loadu_maskz_m512(k, d); - v = and512(v, caseMask); - - // reuse the load mask to indicate valid bytes - u64a z = masked_eq512mask(k, mask1, v); - - return single_zscan(n, d, buf, z, len, cbi); -} - -static really_inline -hwlm_error_t scanSingleFast(const struct noodTable *n, const u8 *buf, - size_t len, m512 caseMask, m512 mask1, - const struct cb_info *cbi, size_t start, - size_t end) { - const u8 *d = buf + start, *e = buf + end; - assert(d < e); - - for (; d < e; d += 64) { - m512 v = and512(load512(d), caseMask); - - u64a z = eq512mask(mask1, v); - - // On large packet buffers, this prefetch appears to get us about 2%. - __builtin_prefetch(d + 128); - - hwlm_error_t result = single_zscan(n, d, buf, z, len, cbi); - if (unlikely(result != HWLM_SUCCESS)) - return result; - } - return HWLM_SUCCESS; -} - -static really_inline -hwlm_error_t scanDoubleUnaligned(const struct noodTable *n, const u8 *buf, - size_t len, size_t offset, m512 caseMask, - m512 mask1, m512 mask2, - const struct cb_info *cbi, size_t start, - size_t end) { - const u8 *d = buf + offset; - DEBUG_PRINTF("start %zu end %zu offset %zu\n", start, end, offset); - const size_t l = end - start; - assert(l <= 64); - if (!l) { - return HWLM_SUCCESS; - } - - __mmask64 k = (~0ULL) >> (64 - l); - DEBUG_PRINTF("load mask 0x%016llx\n", k); - - m512 v = loadu_maskz_m512(k, d); - v = and512(v, caseMask); - - u64a z0 = masked_eq512mask(k, mask1, v); - u64a z1 = masked_eq512mask(k, mask2, v); - u64a z = (z0 << 1) & z1; - DEBUG_PRINTF("z 0x%016llx\n", z); - - return single_zscan(n, d, buf, z, len, cbi); -} - -static really_inline -hwlm_error_t scanDoubleFast(const struct noodTable *n, const u8 *buf, - size_t len, m512 caseMask, m512 mask1, - m512 mask2, const struct cb_info *cbi, size_t start, - size_t end) { - const u8 *d = buf + start, *e = buf + end; - DEBUG_PRINTF("start %zu end %zu \n", start, end); - assert(d < e); - u64a lastz0 = 0; - - for (; d < e; d += 64) { - m512 v = and512(load512(d), caseMask); - - // we have to pull the masks out of the AVX registers because we can't - // byte shift between the lanes - u64a z0 = eq512mask(mask1, v); - u64a z1 = eq512mask(mask2, v); - u64a z = (lastz0 | (z0 << 1)) & z1; - lastz0 = z0 >> 63; - - // On large packet buffers, this prefetch appears to get us about 2%. - __builtin_prefetch(d + 128); - - hwlm_error_t result = double_zscan(n, d, buf, z, len, cbi); - if (unlikely(result != HWLM_SUCCESS)) - return result; - } - return HWLM_SUCCESS; -} diff --git a/src/hwlm/noodle_engine_sse.c b/src/hwlm/noodle_engine_sse.c deleted file mode 100644 index e1da2083..00000000 --- a/src/hwlm/noodle_engine_sse.c +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2015-2017, Intel Corporation - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/* noodle scan parts for SSE */ - -static really_inline m128 getMask(u8 c, bool noCase) { - u8 k = caseClear8(c, noCase); - return set1_16x8(k); -} - -static really_inline m128 getCaseMask(void) { - return set1_16x8(0xdf); -} - -static really_inline -hwlm_error_t scanSingleUnaligned(const struct noodTable *n, const u8 *buf, - size_t len, size_t offset, m128 caseMask, m128 mask1, - const struct cb_info *cbi, size_t start, - size_t end) { - const u8 *d = buf + offset; - DEBUG_PRINTF("start %zu end %zu offset %zu\n", start, end, offset); - const size_t l = end - start; - - m128 v = and128(loadu128(d), caseMask); - - u32 buf_off = start - offset; - u32 mask = ((1 << l) - 1) << buf_off; - u32 z = mask & movemask128(eq128(mask1, v)); - DEBUG_PRINTF("mask 0x%08x z 0x%08x\n", mask, z); - - return single_zscan(n, d, buf, &z, len, cbi); -} - -static really_inline -hwlm_error_t scanDoubleUnaligned(const struct noodTable *n, const u8 *buf, - size_t len, size_t offset, - m128 caseMask, m128 mask1, m128 mask2, - const struct cb_info *cbi, size_t start, - size_t end) { - const u8 *d = buf + offset; - DEBUG_PRINTF("start %zu end %zu offset %zu\n", start, end, offset); - size_t l = end - start; - u32 buf_off = start - offset; - - m128 v = and128(loadu128(d), caseMask); - - // mask out where we can't match - u32 mask = ((1 << l) - 1) << buf_off; - u32 z = mask & movemask128(and128(lshiftbyte_m128(eq128(mask1, v), 1), eq128(mask2, v))); - DEBUG_PRINTF("mask 0x%08x z 0x%08x\n", mask, z); - - return double_zscan(n, d, buf, &z, len, cbi); -} - -static really_inline -hwlm_error_t scanSingleFast(const struct noodTable *n, const u8 *buf, - size_t len, m128 caseMask, m128 mask1, - const struct cb_info *cbi, size_t start, - size_t end) { - const u8 *d = buf + start, *e = buf + end; - assert(d < e); - - const u8 *base = ROUNDDOWN_PTR(d, 64); - for (; d < e; d += 16) { - m128 v = and128(load128(d), caseMask); - u32 z = movemask128(eq128(mask1, v)); - - // On large packet buffers, this prefetch appears to get us about 2%. - __builtin_prefetch(base + 128); - DEBUG_PRINTF("z 0x%08x\n", z); - - hwlm_error_t result = single_zscan(n, d, buf, &z, len, cbi); - if (unlikely(result != HWLM_SUCCESS)) - return result; - } - return HWLM_SUCCESS; -} - -static really_inline -hwlm_error_t scanDoubleFast(const struct noodTable *n, const u8 *buf, - size_t len, m128 caseMask, m128 mask1, - m128 mask2, const struct cb_info *cbi, size_t start, - size_t end) { - const u8 *d = buf + start, *e = buf + end; - assert(d < e); - m128 lastz1 = zeroes128(); - - const u8 *base = ROUNDDOWN_PTR(d, 64); - for (; d < e; d += 16) { - m128 v = and128(load128(d), caseMask); - m128 z1 = eq128(mask1, v); - m128 z2 = eq128(mask2, v); - u32 z = movemask128(and128(palignr(z1, lastz1, 15), z2)); - lastz1 = z1; - - // On large packet buffers, this prefetch appears to get us about 2%. - __builtin_prefetch(base + 128); - DEBUG_PRINTF("z 0x%08x\n", z); - - hwlm_error_t result = double_zscan(n, d, buf, &z, len, cbi); - if (unlikely(result != HWLM_SUCCESS)) - return result; - - } - return HWLM_SUCCESS; -}