github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-06-28 16:41:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Teddy macros unrolling - initial PR to test in CI (#294)

Browse Source 
Major refactoring of teddy and teddy_avx2, unrolling macros to C++ templated functions

---------

Co-authored-by: G.E <gregory.economou@vectorcamp.gr>
This commit is contained in:
g. economou 2024-06-26 22:35:33 +03:00 committed by GitHub
parent 0f4369bf22
commit aa832db892
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 1649 additions and 1880 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CMakeLists.txt
View File

@ -297,7 +297,7 @@ set (hs_exec_SRCS
src/fdr/fdr_confirm_runtime.h src/fdr/fdr_confirm_runtime.h
src/fdr/flood_runtime.h src/fdr/flood_runtime.h
src/fdr/fdr_loadval.h src/fdr/fdr_loadval.h
src/fdr/teddy.c src/fdr/teddy.cpp
src/fdr/teddy.h src/fdr/teddy.h
src/fdr/teddy_internal.h src/fdr/teddy_internal.h
src/fdr/teddy_runtime_common.h src/fdr/teddy_runtime_common.h
@ -441,9 +441,11 @@ set (hs_exec_SRCS
src/util/supervector/arch/ppc64el/impl.cpp) src/util/supervector/arch/ppc64el/impl.cpp)
endif() endif()
if (ARCH_IA32 OR ARCH_X86_64) if (ARCH_IA32 OR ARCH_X86_64)
set (hs_exec_avx2_SRCS set (hs_exec_avx2_SRCS
src/fdr/teddy_avx2.c src/fdr/teddy.cpp
src/fdr/teddy_fat.cpp
src/util/arch/x86/masked_move.c src/util/arch/x86/masked_move.c
src/util/arch/x86/masked_move.h src/util/arch/x86/masked_move.h
) )

28
src/fdr/fdr_confirm_runtime.h
View File

@ -1,5 +1,6 @@
/* /*
* Copyright (c) 2015-2019, Intel Corporation * Copyright (c) 2015-2019, Intel Corporation
* Copyright (c) 2024, VectorCamp PC
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met: * modification, are permitted provided that the following conditions are met:
@ -54,9 +55,14 @@ void confWithBit(const struct FDRConfirm *fdrc, const struct FDR_Runtime_Args *a
if (likely(!start)) { if (likely(!start)) {
return; return;
} }
// these cplusplus checks are needed because this is included in both fdr.c and teddy.cpp
#ifdef __cplusplus
const struct LitInfo *li
= reinterpret_cast<const struct LitInfo *>(reinterpret_cast<const u8 *>(fdrc) + start);
#else
const struct LitInfo *li const struct LitInfo *li
= (const struct LitInfo *)((const u8 *)fdrc + start); = (const struct LitInfo *)((const u8 *)fdrc + start);
#endif
struct hs_scratch *scratch = a->scratch; struct hs_scratch *scratch = a->scratch;
assert(!scratch->fdr_conf); assert(!scratch->fdr_conf);
@ -74,18 +80,20 @@ void confWithBit(const struct FDRConfirm *fdrc, const struct FDR_Runtime_Args *a
goto out; goto out;
} }
const u8 *loc = buf + i - li->size + 1; do{ // this do while is to block off the line below from the goto
const u8 *loc = buf + i - li->size + 1;
if (loc < buf) { if (loc < buf) {
u32 full_overhang = buf - loc; u32 full_overhang = buf - loc;
size_t len_history = a->len_history; size_t len_history = a->len_history;
// can't do a vectored confirm either if we don't have // can't do a vectored confirm either if we don't have
// the bytes // the bytes
if (full_overhang > len_history) { if (full_overhang > len_history) {
goto out; goto out;
}
} }
} }while(0);
assert(li->size <= sizeof(CONF_TYPE)); assert(li->size <= sizeof(CONF_TYPE));
if (unlikely(!(li->groups & *control))) { if (unlikely(!(li->groups & *control))) {

42
src/fdr/flood_runtime.h
View File

@ -1,5 +1,6 @@
/* /*
* Copyright (c) 2015-2017, Intel Corporation * Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2024, VectorCamp PC
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met: * modification, are permitted provided that the following conditions are met:
@ -37,6 +38,13 @@
#define FLOOD_MINIMUM_SIZE 256 #define FLOOD_MINIMUM_SIZE 256
#define FLOOD_BACKOFF_START 32 #define FLOOD_BACKOFF_START 32
// this is because this file is included in both fdr.c and teddy.cpp
#if defined __cplusplus
#define CU64A_P_CAST(X) reinterpret_cast<const u64a*>(X)
#else
#define CU64A_P_CAST(X) (const u64a *)(X)
#endif
static really_inline static really_inline
const u8 * nextFloodDetect(const u8 * buf, size_t len, u32 floodBackoff) { const u8 * nextFloodDetect(const u8 * buf, size_t len, u32 floodBackoff) {
// if we don't have a flood at either the start or end, // if we don't have a flood at either the start or end,
@ -47,18 +55,18 @@ const u8 * nextFloodDetect(const u8 * buf, size_t len, u32 floodBackoff) {
/* entry points in runtime.c prefetch relevant data */ /* entry points in runtime.c prefetch relevant data */
#ifndef FLOOD_32 #ifndef FLOOD_32
u64a x11 = *(const u64a *)ROUNDUP_PTR(buf, 8); u64a x11 = *CU64A_P_CAST(ROUNDUP_PTR(buf, 8));
u64a x12 = *(const u64a *)ROUNDUP_PTR(buf+8, 8); u64a x12 = *CU64A_P_CAST(ROUNDUP_PTR(buf+8, 8));
if (x11 == x12) { if (x11 == x12) {
return buf + floodBackoff; return buf + floodBackoff;
} }
u64a x21 = *(const u64a *)ROUNDUP_PTR(buf + len/2, 8); u64a x21 = *CU64A_P_CAST(ROUNDUP_PTR(buf + len/2, 8));
u64a x22 = *(const u64a *)ROUNDUP_PTR(buf + len/2 + 8, 8); u64a x22 = *CU64A_P_CAST(ROUNDUP_PTR(buf + len/2 + 8, 8));
if (x21 == x22) { if (x21 == x22) {
return buf + floodBackoff; return buf + floodBackoff;
} }
u64a x31 = *(const u64a *)ROUNDUP_PTR(buf + len - 24, 8); u64a x31 = *CU64A_P_CAST(ROUNDUP_PTR(buf + len - 24, 8));
u64a x32 = *(const u64a *)ROUNDUP_PTR(buf + len - 16, 8); u64a x32 = *CU64A_P_CAST(ROUNDUP_PTR(buf + len - 16, 8));
if (x31 == x32) { if (x31 == x32) {
return buf + floodBackoff; return buf + floodBackoff;
} }
@ -106,9 +114,15 @@ const u8 * floodDetect(const struct FDR * fdr,
// go from c to our FDRFlood structure // go from c to our FDRFlood structure
u8 c = buf[i]; u8 c = buf[i];
#ifdef __cplusplus
const u8 * fBase = (reinterpret_cast<const u8 *>(fdr)) + fdr->floodOffset;
u32 fIdx = (reinterpret_cast<const u32 *>(fBase))[c];
const struct FDRFlood * fsb = reinterpret_cast<const struct FDRFlood *>(fBase + sizeof(u32) * 256);
#else
const u8 * fBase = ((const u8 *)fdr) + fdr->floodOffset; const u8 * fBase = ((const u8 *)fdr) + fdr->floodOffset;
u32 fIdx = ((const u32 *)fBase)[c]; u32 fIdx = ((const u32 *)fBase)[c];
const struct FDRFlood * fsb = (const struct FDRFlood *)(fBase + sizeof(u32) * 256); const struct FDRFlood * fsb = (const struct FDRFlood *)(fBase + sizeof(u32) * 256);
#endif
const struct FDRFlood * fl = &fsb[fIdx]; const struct FDRFlood * fl = &fsb[fIdx];
#ifndef FLOOD_32 #ifndef FLOOD_32
@ -116,7 +130,7 @@ const u8 * floodDetect(const struct FDR * fdr,
cmpVal |= cmpVal << 8; cmpVal |= cmpVal << 8;
cmpVal |= cmpVal << 16; cmpVal |= cmpVal << 16;
cmpVal |= cmpVal << 32; cmpVal |= cmpVal << 32;
u64a probe = *(const u64a *)ROUNDUP_PTR(buf+i, 8); u64a probe = *CU64A_P_CAST(ROUNDUP_PTR(buf+i, 8));
#else #else
u32 cmpVal = c; u32 cmpVal = c;
cmpVal |= cmpVal << 8; cmpVal |= cmpVal << 8;
@ -139,16 +153,16 @@ const u8 * floodDetect(const struct FDR * fdr,
#ifndef FLOOD_32 #ifndef FLOOD_32
j -= (u32)((uintptr_t)buf + j) & 0x7; // push j back to yield 8-aligned addrs j -= (u32)((uintptr_t)buf + j) & 0x7; // push j back to yield 8-aligned addrs
for (; j + 32 < mainLoopLen; j += 32) { for (; j + 32 < mainLoopLen; j += 32) {
u64a v = *(const u64a *)(buf + j); u64a v = *CU64A_P_CAST(buf + j);
u64a v2 = *(const u64a *)(buf + j + 8); u64a v2 = *CU64A_P_CAST(buf + j + 8);
u64a v3 = *(const u64a *)(buf + j + 16); u64a v3 = *CU64A_P_CAST(buf + j + 16);
u64a v4 = *(const u64a *)(buf + j + 24); u64a v4 = *CU64A_P_CAST(buf + j + 24);
if ((v4 != cmpVal) || (v3 != cmpVal) || (v2 != cmpVal) || (v != cmpVal)) { if ((v4 != cmpVal) || (v3 != cmpVal) || (v2 != cmpVal) || (v != cmpVal)) {
break; break;
} }
} }
for (; j + 8 < mainLoopLen; j += 8) { for (; j + 8 < mainLoopLen; j += 8) {
u64a v = *(const u64a *)(buf + j); u64a v = *CU64A_P_CAST(buf + j);
if (v != cmpVal) { if (v != cmpVal) {
break; break;
} }
@ -172,7 +186,11 @@ const u8 * floodDetect(const struct FDR * fdr,
} }
#endif #endif
for (; j < mainLoopLen; j++) { for (; j < mainLoopLen; j++) {
#ifdef __cplusplus
u8 v = *(reinterpret_cast<const u8 *>(buf + j));
#else
u8 v = *(const u8 *)(buf + j); u8 v = *(const u8 *)(buf + j);
#endif
if (v != c) { if (v != c) {
break; break;
} }

1116
src/fdr/teddy.c
View File

File diff suppressed because it is too large Load Diff

862
src/fdr/teddy.cpp Normal file
View File

@ -0,0 +1,862 @@
/*
* Copyright (c) 2015-2020, Intel Corporation
* Copyright (c) 2024, VectorCamp PC
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief Teddy literal matcher: SSSE3 engine runtime.
*/
#include "fdr_internal.h"
#include "flood_runtime.h"
#include "teddy.h"
#include "teddy_internal.h"
#include "teddy_runtime_common.h"
#include "util/arch.h"
#include "util/simd_utils.h"
#ifdef ARCH_64_BIT
static really_inline
hwlm_error_t conf_chunk_64(u64a chunk, u8 bucket, u8 offset,
CautionReason reason, const u8 *pt,
const u32* confBase,
const struct FDR_Runtime_Args *a,
hwlm_group_t *control,
u32 *last_match) {
if (unlikely(chunk != ones_u64a)) {
chunk = ~chunk;
do_confWithBit_teddy(&chunk, bucket, offset, confBase, reason, a, pt,
control, last_match);
// adapted from CHECK_HWLM_TERMINATE_MATCHING
if (unlikely(*control == HWLM_TERMINATE_MATCHING)) {
return HWLM_TERMINATED;
}
}
return HWLM_SUCCESS;
}
#define CONF_CHUNK_64(chunk, bucket, off, reason, pt, confBase, a, control, last_match) \
if(conf_chunk_64(chunk, bucket, off, reason, pt, confBase, a, control, last_match) == HWLM_TERMINATED)return HWLM_TERMINATED;
#else // 32/64
static really_inline
hwlm_error_t conf_chunk_32(u32 chunk, u8 bucket, u8 offset,
CautionReason reason, const u8 *pt,
const u32* confBase,
const struct FDR_Runtime_Args *a,
hwlm_group_t *control,
u32 *last_match) {
if (unlikely(chunk != ones_u32)) {
chunk = ~chunk;
do_confWithBit_teddy(&chunk, bucket, offset, confBase, reason, a, pt,
control, last_match);
// adapted from CHECK_HWLM_TERMINATE_MATCHING
if (unlikely(*control == HWLM_TERMINATE_MATCHING)) {
return HWLM_TERMINATED;
}
}
return HWLM_SUCCESS;
}
#define CONF_CHUNK_32(chunk, bucket, off, reason, pt, confBase, a, control, last_match) \
if(conf_chunk_32(chunk, bucket, off, reason, pt, confBase, a, control, last_match) == HWLM_TERMINATED)return HWLM_TERMINATED;
#endif
#if defined(HAVE_AVX512VBMI) || defined(HAVE_AVX512) // common to both 512b's
static really_inline
const m512 *getDupMaskBase(const struct Teddy *teddy, u8 numMask) {
return (const m512 *)((const u8 *)teddy + ROUNDUP_CL(sizeof(struct Teddy))
+ ROUNDUP_CL(2 * numMask * sizeof(m256)));
}
#ifdef ARCH_64_BIT
static really_inline
hwlm_error_t confirm_teddy_64_512(m512 var, u8 bucket, u8 offset,
CautionReason reason, const u8 *ptr,
const struct FDR_Runtime_Args *a,
const u32* confBase, hwlm_group_t *control,
u32 *last_match) {
if (unlikely(diff512(var, ones512()))) {
m128 p128_0 = extract128from512(var, 0);
m128 p128_1 = extract128from512(var, 1);
m128 p128_2 = extract128from512(var, 2);
m128 p128_3 = extract128from512(var, 3);
u64a part1 = movq(p128_0);
u64a part2 = movq(rshiftbyte_m128(p128_0, 8));
u64a part3 = movq(p128_1);
u64a part4 = movq(rshiftbyte_m128(p128_1, 8));
u64a part5 = movq(p128_2);
u64a part6 = movq(rshiftbyte_m128(p128_2, 8));
u64a part7 = movq(p128_3);
u64a part8 = movq(rshiftbyte_m128(p128_3, 8));
CONF_CHUNK_64(part1, bucket, offset, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_64(part2, bucket, offset + 8, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_64(part3, bucket, offset + 16, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_64(part4, bucket, offset + 24, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_64(part5, bucket, offset + 32, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_64(part6, bucket, offset + 40, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_64(part7, bucket, offset + 48, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_64(part8, bucket, offset + 56, reason, ptr, confBase, a, control, last_match);
}
return HWLM_SUCCESS;
}
#define confirm_teddy_512_f confirm_teddy_64_512
#else // 32/64
static really_inline
hwlm_error_t confirm_teddy_32_512(m512 var, u8 bucket, u8 offset,
CautionReason reason, const u8 *ptr,
const struct FDR_Runtime_Args *a,
const u32* confBase, hwlm_group_t *control,
u32 *last_match) {
if (unlikely(diff512(var, ones512()))) {
m128 p128_0 = extract128from512(var, 0);
m128 p128_1 = extract128from512(var, 1);
m128 p128_2 = extract128from512(var, 2);
m128 p128_3 = extract128from512(var, 3);
u32 part1 = movd(p128_0);
u32 part2 = movd(rshiftbyte_m128(p128_0, 4));
u32 part3 = movd(rshiftbyte_m128(p128_0, 8));
u32 part4 = movd(rshiftbyte_m128(p128_0, 12));
u32 part5 = movd(p128_1);
u32 part6 = movd(rshiftbyte_m128(p128_1, 4));
u32 part7 = movd(rshiftbyte_m128(p128_1, 8));
u32 part8 = movd(rshiftbyte_m128(p128_1, 12));
u32 part9 = movd(p128_2);
u32 part10 = movd(rshiftbyte_m128(p128_2, 4));
u32 part11 = movd(rshiftbyte_m128(p128_2, 8));
u32 part12 = movd(rshiftbyte_m128(p128_2, 12));
u32 part13 = movd(p128_3);
u32 part14 = movd(rshiftbyte_m128(p128_3, 4));
u32 part15 = movd(rshiftbyte_m128(p128_3, 8));
u32 part16 = movd(rshiftbyte_m128(p128_3, 12));
CONF_CHUNK_32(part1, bucket, offset, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part2, bucket, offset + 4, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part3, bucket, offset + 8, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part4, bucket, offset + 12, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part5, bucket, offset + 16, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part6, bucket, offset + 20, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part7, bucket, offset + 24, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part8, bucket, offset + 28, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part9, bucket, offset + 32, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part10, bucket, offset + 36, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part11, bucket, offset + 40, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part12, bucket, offset + 44, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part13, bucket, offset + 48, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part14, bucket, offset + 52, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part15, bucket, offset + 56, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part16, bucket, offset + 60, reason, ptr, confBase, a, control, last_match);
}
return HWLM_SUCCESS;
}
#define confirm_teddy_512_f confirm_teddy_32_512
#endif // 32/64
#define CONFIRM_TEDDY_512(...) if(confirm_teddy_512_f(__VA_ARGS__, a, confBase, &control, &last_match) == HWLM_TERMINATED)return HWLM_TERMINATED;
#endif // AVX512VBMI or AVX512
#if defined(HAVE_AVX512VBMI) // VBMI strong teddy
#define TEDDY_VBMI_SL1_MASK 0xfffffffffffffffeULL
#define TEDDY_VBMI_SL2_MASK 0xfffffffffffffffcULL
#define TEDDY_VBMI_SL3_MASK 0xfffffffffffffff8ULL
template<int NMSK>
static really_inline
m512 prep_conf_teddy_512vbmi_templ(const m512 *lo_mask, const m512 *dup_mask,
const m512 *sl_msk, const m512 val) {
m512 lo = and512(val, *lo_mask);
m512 hi = and512(rshift64_m512(val, 4), *lo_mask);
m512 shuf_or_b0 = or512(pshufb_m512(dup_mask[0], lo),
pshufb_m512(dup_mask[1], hi));
if constexpr (NMSK == 1) return shuf_or_b0;
m512 shuf_or_b1 = or512(pshufb_m512(dup_mask[2], lo),
pshufb_m512(dup_mask[3], hi));
m512 sl1 = maskz_vpermb512(TEDDY_VBMI_SL1_MASK, sl_msk[0], shuf_or_b1);
if constexpr (NMSK == 2) return (or512(sl1, shuf_or_b0));
m512 shuf_or_b2 = or512(pshufb_m512(dup_mask[4], lo),
pshufb_m512(dup_mask[5], hi));
m512 sl2 = maskz_vpermb512(TEDDY_VBMI_SL2_MASK, sl_msk[1], shuf_or_b2);
if constexpr (NMSK == 3) return (or512(sl2, or512(sl1, shuf_or_b0)));
m512 shuf_or_b3 = or512(pshufb_m512(dup_mask[6], lo),
pshufb_m512(dup_mask[7], hi));
m512 sl3 = maskz_vpermb512(TEDDY_VBMI_SL3_MASK, sl_msk[2], shuf_or_b3);
return (or512(sl3, or512(sl2, or512(sl1, shuf_or_b0))));
}
#define TEDDY_VBMI_SL1_POS 15
#define TEDDY_VBMI_SL2_POS 14
#define TEDDY_VBMI_SL3_POS 13
#define TEDDY_VBMI_CONF_MASK_HEAD (0xffffffffffffffffULL >> n_sh)
#define TEDDY_VBMI_CONF_MASK_FULL (0xffffffffffffffffULL << n_sh)
#define TEDDY_VBMI_CONF_MASK_VAR(n) (0xffffffffffffffffULL >> (64 - n) << overlap)
#define TEDDY_VBMI_LOAD_MASK_PATCH (0xffffffffffffffffULL >> (64 - n_sh))
template<int NMSK>
hwlm_error_t fdr_exec_teddy_512vbmi_templ(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
const u8 *buf_end = a->buf + a->len;
const u8 *ptr = a->buf + a->start_offset;
u32 floodBackoff = FLOOD_BACKOFF_START;
const u8 *tryFloodDetect = a->firstFloodDetect;
u32 last_match = ones_u32;
const struct Teddy *teddy = (const struct Teddy *)fdr;
const size_t iterBytes = 64;
u32 n_sh = NMSK - 1;
const size_t loopBytes = 64 - n_sh;
DEBUG_PRINTF("params: buf %p len %zu start_offset %zu\n",
a->buf, a->len, a->start_offset);
const m128 *maskBase = getMaskBase(teddy);
m512 lo_mask = set1_64x8(0xf);
m512 dup_mask[NMSK * 2];
m512 sl_msk[NMSK - 1];
dup_mask[0] = set1_4x128(maskBase[0]);
dup_mask[1] = set1_4x128(maskBase[1]);
if constexpr (NMSK > 1){
dup_mask[2] = set1_4x128(maskBase[2]);
dup_mask[3] = set1_4x128(maskBase[3]);
sl_msk[0] = loadu512(p_sh_mask_arr + TEDDY_VBMI_SL1_POS);
}
if constexpr (NMSK > 2){
dup_mask[4] = set1_4x128(maskBase[4]);
dup_mask[5] = set1_4x128(maskBase[5]);
sl_msk[1] = loadu512(p_sh_mask_arr + TEDDY_VBMI_SL2_POS);
}
if constexpr (NMSK > 3){
dup_mask[6] = set1_4x128(maskBase[6]);
dup_mask[7] = set1_4x128(maskBase[7]);
sl_msk[2] = loadu512(p_sh_mask_arr + TEDDY_VBMI_SL3_POS);
}
const u32 *confBase = getConfBase(teddy);
u64a k = TEDDY_VBMI_CONF_MASK_FULL;
m512 p_mask = set_mask_m512(~k);
u32 overlap = 0;
u64a patch = 0;
if (likely(ptr + loopBytes <= buf_end)) {
m512 p_mask0 = set_mask_m512(~TEDDY_VBMI_CONF_MASK_HEAD);
m512 r_0 = prep_conf_teddy_512vbmi_templ<NMSK>(&lo_mask, dup_mask, sl_msk, loadu512(ptr));
r_0 = or512(r_0, p_mask0);
CONFIRM_TEDDY_512(r_0, 8, 0, VECTORING, ptr);
ptr += loopBytes;
overlap = n_sh;
patch = TEDDY_VBMI_LOAD_MASK_PATCH;
}
for (; ptr + loopBytes <= buf_end; ptr += loopBytes) {
__builtin_prefetch(ptr - n_sh + (64 * 2));
CHECK_FLOOD;
m512 r_0 = prep_conf_teddy_512vbmi_templ<NMSK>(&lo_mask, dup_mask, sl_msk, loadu512(ptr - n_sh));
r_0 = or512(r_0, p_mask);
CONFIRM_TEDDY_512(r_0, 8, 0, NOT_CAUTIOUS, ptr - n_sh);
}
assert(ptr + loopBytes > buf_end);
if (ptr < buf_end) {
u32 left = (u32)(buf_end - ptr);
u64a k1 = TEDDY_VBMI_CONF_MASK_VAR(left);
m512 p_mask1 = set_mask_m512(~k1);
m512 val_0 = loadu_maskz_m512(k1 | patch, ptr - overlap);
m512 r_0 = prep_conf_teddy_512vbmi_templ<NMSK>(&lo_mask, dup_mask, sl_msk, val_0);
r_0 = or512(r_0, p_mask1);
CONFIRM_TEDDY_512(r_0, 8, 0, VECTORING, ptr - overlap);
}
return HWLM_SUCCESS;
}
#define FDR_EXEC_TEDDY_FN fdr_exec_teddy_512vbmi_templ
#elif defined(HAVE_AVX512) // AVX512 reinforced teddy
/* both 512b versions use the same confirm teddy */
template <int NMSK>
static inline
m512 shift_or_512_templ(const m512 *dup_mask, m512 lo, m512 hi) {
return or512(lshift128_m512(or512(pshufb_m512(dup_mask[(NMSK - 1) * 2], lo),
pshufb_m512(dup_mask[(NMSK * 2) - 1], hi)),
NMSK - 1), shift_or_512_templ<NMSK - 1>(dup_mask, lo, hi));
}
template <>
m512 shift_or_512_templ<1>(const m512 *dup_mask, m512 lo, m512 hi){
return or512(pshufb_m512(dup_mask[0], lo), pshufb_m512(dup_mask[1], hi));
}
template <int NMSK>
static really_inline
m512 prep_conf_teddy_no_reinforcement_512_templ(const m512 *lo_mask,
const m512 *dup_mask,
const m512 val) {
m512 lo = and512(val, *lo_mask);
m512 hi = and512(rshift64_m512(val, 4), *lo_mask);
return shift_or_512_templ<NMSK>(dup_mask, lo, hi);
}
template <int NMSK>
static really_inline
m512 prep_conf_teddy_512_templ(const m512 *lo_mask, const m512 *dup_mask,
const u8 *ptr, const u64a *r_msk_base,
u32 *c_0, u32 *c_16, u32 *c_32, u32 *c_48) {
m512 lo = and512(load512(ptr), *lo_mask);
m512 hi = and512(rshift64_m512(load512(ptr), 4), *lo_mask);
*c_16 = *(ptr + 15);
*c_32 = *(ptr + 31);
*c_48 = *(ptr + 47);
m512 r_msk = set8x64(0ULL, r_msk_base[*c_48], 0ULL, r_msk_base[*c_32],
0ULL, r_msk_base[*c_16], 0ULL, r_msk_base[*c_0]);
*c_0 = *(ptr + 63);
return or512(shift_or_512_templ<NMSK>(dup_mask, lo, hi), r_msk);
}
#define PREP_CONF_FN_512(ptr, n) \
prep_conf_teddy_512_templ<n>(&lo_mask, dup_mask, ptr, r_msk_base, \
&c_0, &c_16, &c_32, &c_48)
template <int NMSK>
hwlm_error_t fdr_exec_teddy_512_templ(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
const u8 *buf_end = a->buf + a->len;
const u8 *ptr = a->buf + a->start_offset;
u32 floodBackoff = FLOOD_BACKOFF_START;
const u8 *tryFloodDetect = a->firstFloodDetect;
u32 last_match = ones_u32;
const struct Teddy *teddy = (const struct Teddy *)fdr;
const size_t iterBytes = 128;
DEBUG_PRINTF("params: buf %p len %zu start_offset %zu\n",
a->buf, a->len, a->start_offset);
const m128 *maskBase = getMaskBase(teddy);
m512 lo_mask = set1_64x8(0xf);
m512 dup_mask[NMSK * 2];
dup_mask[0] = set1_4x128(maskBase[0]);
dup_mask[1] = set1_4x128(maskBase[1]);
if constexpr (NMSK > 1){
dup_mask[2] = set1_4x128(maskBase[2]);
dup_mask[3] = set1_4x128(maskBase[3]);
}
if constexpr (NMSK > 2){
dup_mask[4] = set1_4x128(maskBase[4]);
dup_mask[5] = set1_4x128(maskBase[5]);
}
if constexpr (NMSK > 3){
dup_mask[6] = set1_4x128(maskBase[6]);
dup_mask[7] = set1_4x128(maskBase[7]);
}
const u32 *confBase = getConfBase(teddy);
const u64a *r_msk_base = getReinforcedMaskBase(teddy, NMSK);
u32 c_0 = 0x100;
u32 c_16 = 0x100;
u32 c_32 = 0x100;
u32 c_48 = 0x100;
const u8 *mainStart = ROUNDUP_PTR(ptr, 64);
DEBUG_PRINTF("derive: ptr: %p mainstart %p\n", ptr, mainStart);
if (ptr < mainStart) {
ptr = mainStart - 64;
m512 p_mask;
m512 val_0 = vectoredLoad512(&p_mask, ptr, a->start_offset,
a->buf, buf_end,
a->buf_history, a->len_history, NMSK);
m512 r_0 = prep_conf_teddy_no_reinforcement_512_templ<NMSK>(&lo_mask, dup_mask, val_0);
r_0 = or512(r_0, p_mask);
CONFIRM_TEDDY_512(r_0, 8, 0, VECTORING, ptr);
ptr += 64;
}
if (ptr + 64 <= buf_end) {
m512 r_0 = PREP_CONF_FN_512(ptr, NMSK);
CONFIRM_TEDDY_512(r_0, 8, 0, VECTORING, ptr);
ptr += 64;
}
for (; ptr + iterBytes <= buf_end; ptr += iterBytes) {
__builtin_prefetch(ptr + (iterBytes * 4));
CHECK_FLOOD;
m512 r_0 = PREP_CONF_FN_512(ptr, NMSK);
CONFIRM_TEDDY_512(r_0, 8, 0, NOT_CAUTIOUS, ptr);
m512 r_1 = PREP_CONF_FN_512(ptr + 64, NMSK);
CONFIRM_TEDDY_512(r_1, 8, 64, NOT_CAUTIOUS, ptr);
}
if (ptr + 64 <= buf_end) {
m512 r_0 = PREP_CONF_FN_512(ptr, NMSK);
CONFIRM_TEDDY_512(r_0, 8, 0, NOT_CAUTIOUS, ptr);
ptr += 64;
}
assert(ptr + 64 > buf_end);
if (ptr < buf_end) {
m512 p_mask;
m512 val_0 = vectoredLoad512(&p_mask, ptr, 0, ptr, buf_end,
a->buf_history, a->len_history, NMSK);
m512 r_0 = prep_conf_teddy_no_reinforcement_512_templ<NMSK>(&lo_mask, dup_mask,val_0);
r_0 = or512(r_0, p_mask);
CONFIRM_TEDDY_512(r_0, 8, 0, VECTORING, ptr);
}
return HWLM_SUCCESS;
}
#define FDR_EXEC_TEDDY_FN fdr_exec_teddy_512_templ
/* #endif // AVX512 vs AVX512VBMI * back to the original fully exclusive logic */
#elif defined(HAVE_AVX2) // not HAVE_AVX512 but HAVE_AVX2 reinforced teddy
#ifdef ARCH_64_BIT
hwlm_error_t confirm_teddy_64_256(m256 var, u8 bucket, u8 offset,
CautionReason reason, const u8 *ptr,
const struct FDR_Runtime_Args *a,
const u32* confBase, hwlm_group_t *control,
u32 *last_match) {
if (unlikely(diff256(var, ones256()))) {
m128 lo = movdq_lo(var);
m128 hi = movdq_hi(var);
u64a part1 = movq(lo);
u64a part2 = movq(rshiftbyte_m128(lo, 8));
u64a part3 = movq(hi);
u64a part4 = movq(rshiftbyte_m128(hi, 8));
CONF_CHUNK_64(part1, bucket, offset, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_64(part2, bucket, offset + 8, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_64(part3, bucket, offset + 16, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_64(part4, bucket, offset + 24, reason, ptr, confBase, a, control, last_match);
}
return HWLM_SUCCESS;
}
#define confirm_teddy_256_f confirm_teddy_64_256
#else
hwlm_error_t confirm_teddy_32_256(m256 var, u8 bucket, u8 offset,
CautionReason reason, const u8 *ptr,
const struct FDR_Runtime_Args *a,
const u32* confBase, hwlm_group_t *control,
u32 *last_match) {
if (unlikely(diff256(var, ones256()))) {
m128 lo = movdq_lo(var);
m128 hi = movdq_hi(var);
u32 part1 = movd(lo);
u32 part2 = movd(rshiftbyte_m128(lo, 4));
u32 part3 = movd(rshiftbyte_m128(lo, 8));
u32 part4 = movd(rshiftbyte_m128(lo, 12));
u32 part5 = movd(hi);
u32 part6 = movd(rshiftbyte_m128(hi, 4));
u32 part7 = movd(rshiftbyte_m128(hi, 8));
u32 part8 = movd(rshiftbyte_m128(hi, 12));
CONF_CHUNK_32(part1, bucket, offset, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part2, bucket, offset + 4, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part3, bucket, offset + 8, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part4, bucket, offset + 12, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part5, bucket, offset + 16, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part6, bucket, offset + 20, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part7, bucket, offset + 24, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part8, bucket, offset + 28, reason, ptr, confBase, a, control, last_match);
}
return HWLM_SUCCESS;
}
#define confirm_teddy_256_f confirm_teddy_32_256
#endif
#define CONFIRM_TEDDY_256(...) if(confirm_teddy_256_f(__VA_ARGS__, a, confBase, &control, &last_match) == HWLM_TERMINATED)return HWLM_TERMINATED;
/*
static really_inline
m256 vectoredLoad2x128(m256 *p_mask, const u8 *ptr, const size_t start_offset,
const u8 *lo, const u8 *hi,
const u8 *buf_history, size_t len_history,
const u32 nMasks) {
m128 p_mask128;
m256 ret = set1_2x128(vectoredLoad128(&p_mask128, ptr, start_offset, lo, hi,
buf_history, len_history, nMasks));
*p_mask = set1_2x128(p_mask128);
return ret;
}
*/
template <int NMSK>
static inline
m256 shift_or_256_templ(const m256 *dup_mask, m256 lo, m256 hi){
return or256(lshift128_m256(or256(pshufb_m256(dup_mask[(NMSK-1)*2], lo),
pshufb_m256(dup_mask[(NMSK*2)-1], hi)),
(NMSK-1)), shift_or_256_templ<NMSK-1>(dup_mask, lo, hi));
}
template<>
m256 shift_or_256_templ<1>(const m256 *dup_mask, m256 lo, m256 hi){
return or256(pshufb_m256(dup_mask[0], lo), pshufb_m256(dup_mask[1], hi));
}
template <int NMSK>
static really_inline
m256 prep_conf_teddy_no_reinforcement_256_templ(const m256 *lo_mask,
const m256 *dup_mask,
const m256 val) {
m256 lo = and256(val, *lo_mask);
m256 hi = and256(rshift64_m256(val, 4), *lo_mask);
return shift_or_256_templ<NMSK>(dup_mask, lo, hi);
}
template <int NMSK>
static really_inline
m256 prep_conf_teddy_256_templ(const m256 *lo_mask, const m256 *dup_mask,
const u8 *ptr, const u64a *r_msk_base,
u32 *c_0, u32 *c_128) {
m256 lo = and256(load256(ptr), *lo_mask);
m256 hi = and256(rshift64_m256(load256(ptr), 4), *lo_mask);
*c_128 = *(ptr + 15);
m256 r_msk = set4x64(0ULL, r_msk_base[*c_128], 0ULL, r_msk_base[*c_0]);
*c_0 = *(ptr + 31);
return or256(shift_or_256_templ<NMSK>(dup_mask, lo, hi), r_msk);
}
#define PREP_CONF_FN_256_NO_REINFORCEMENT(val, n) \
prep_conf_teddy_no_reinforcement_256_templ<n>(&lo_mask, dup_mask, val)
#define PREP_CONF_FN_256(ptr, n) \
prep_conf_teddy_256_templ<n>(&lo_mask, dup_mask, ptr, r_msk_base, &c_0, &c_128)
template <int NMSK>
hwlm_error_t fdr_exec_teddy_256_templ(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
const u8 *buf_end = a->buf + a->len;
const u8 *ptr = a->buf + a->start_offset;
u32 floodBackoff = FLOOD_BACKOFF_START;
const u8 *tryFloodDetect = a->firstFloodDetect;
u32 last_match = ones_u32;
const struct Teddy *teddy = (const struct Teddy *)fdr;
const size_t iterBytes = 64;
DEBUG_PRINTF("params: buf %p len %zu start_offset %zu\n",
a->buf, a->len, a->start_offset);
const m128 *maskBase = getMaskBase(teddy);
//PREPARE_MASKS_256;
m256 lo_mask = set1_32x8(0xf);
m256 dup_mask[NMSK * 2];
dup_mask[0] = set1_2x128(maskBase[0]);
dup_mask[1] = set1_2x128(maskBase[1]);
if constexpr (NMSK > 1){
dup_mask[2] = set1_2x128(maskBase[2]);
dup_mask[3] = set1_2x128(maskBase[3]);
}
if constexpr (NMSK > 2){
dup_mask[4] = set1_2x128(maskBase[4]);
dup_mask[5] = set1_2x128(maskBase[5]);
}
if constexpr (NMSK > 3){
dup_mask[6] = set1_2x128(maskBase[6]);
dup_mask[7] = set1_2x128(maskBase[7]);
}
const u32 *confBase = getConfBase(teddy);
const u64a *r_msk_base = getReinforcedMaskBase(teddy, NMSK);
u32 c_0 = 0x100;
u32 c_128 = 0x100;
const u8 *mainStart = ROUNDUP_PTR(ptr, 32);
DEBUG_PRINTF("derive: ptr: %p mainstart %p\n", ptr, mainStart);
if (ptr < mainStart) {
ptr = mainStart - 32;
m256 p_mask;
m256 val_0 = vectoredLoad256(&p_mask, ptr, a->start_offset,
a->buf, buf_end,
a->buf_history, a->len_history, NMSK);
m256 r_0 = PREP_CONF_FN_256_NO_REINFORCEMENT(val_0, NMSK);
r_0 = or256(r_0, p_mask);
CONFIRM_TEDDY_256(r_0, 8, 0, VECTORING, ptr);
ptr += 32;
}
if (ptr + 32 <= buf_end) {
m256 r_0 = PREP_CONF_FN_256(ptr, NMSK);
CONFIRM_TEDDY_256(r_0, 8, 0, VECTORING, ptr);
ptr += 32;
}
for (; ptr + iterBytes <= buf_end; ptr += iterBytes) {
__builtin_prefetch(ptr + (iterBytes * 4));
CHECK_FLOOD;
m256 r_0 = PREP_CONF_FN_256(ptr, NMSK);
CONFIRM_TEDDY_256(r_0, 8, 0, NOT_CAUTIOUS, ptr);
m256 r_1 = PREP_CONF_FN_256(ptr + 32, NMSK);
CONFIRM_TEDDY_256(r_1, 8, 32, NOT_CAUTIOUS, ptr);
}
if (ptr + 32 <= buf_end) {
m256 r_0 = PREP_CONF_FN_256(ptr, NMSK);
CONFIRM_TEDDY_256(r_0, 8, 0, NOT_CAUTIOUS, ptr);
ptr += 32;
}
assert(ptr + 32 > buf_end);
if (ptr < buf_end) {
m256 p_mask;
m256 val_0 = vectoredLoad256(&p_mask, ptr, 0, ptr, buf_end,
a->buf_history, a->len_history, NMSK);
m256 r_0 = PREP_CONF_FN_256_NO_REINFORCEMENT(val_0, NMSK);
r_0 = or256(r_0, p_mask);
CONFIRM_TEDDY_256(r_0, 8, 0, VECTORING, ptr);
}
return HWLM_SUCCESS;
}
#define FDR_EXEC_TEDDY_FN fdr_exec_teddy_256_templ
#else // not defined HAVE_AVX2
#ifdef ARCH_64_BIT
static really_inline
hwlm_error_t confirm_teddy_64_128(m128 var, u8 bucket, u8 offset,
CautionReason reason, const u8 *ptr,
const struct FDR_Runtime_Args *a,
const u32* confBase, hwlm_group_t *control,
u32 *last_match) {
if (unlikely(diff128(var, ones128()))) {
u64a lo = 0;
u64a hi = 0;
u64a __attribute__((aligned(16))) vec[2];
store128(vec, var);
lo = vec[0];
hi = vec[1];
CONF_CHUNK_64(lo, bucket, offset, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_64(hi, bucket, offset + 8, reason, ptr, confBase, a, control, last_match);
}
return HWLM_SUCCESS;
}
#define confirm_teddy_128_f confirm_teddy_64_128
#else // 32/64
static really_inline
hwlm_error_t confirm_teddy_32_128(m128 var, u8 bucket, u8 offset,
CautionReason reason, const u8 *ptr,
const struct FDR_Runtime_Args *a,
const u32* confBase, hwlm_group_t *control,
u32 *last_match) {
if (unlikely(diff128(var, ones128()))) {
u32 part1 = movd(var);
u32 part2 = movd(rshiftbyte_m128(var, 4));
u32 part3 = movd(rshiftbyte_m128(var, 8));
u32 part4 = movd(rshiftbyte_m128(var, 12));
CONF_CHUNK_32(part1, bucket, offset, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part2, bucket, offset + 4, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part3, bucket, offset + 8, reason, ptr, confBase, a, control, last_match);
CONF_CHUNK_32(part4, bucket, offset + 12, reason, ptr, confBase, a, control, last_match);
}
return HWLM_SUCCESS;
}
#define confirm_teddy_128_f confirm_teddy_32_128
#endif // 32/64
#define CONFIRM_TEDDY_128(...) if(confirm_teddy_128_f(__VA_ARGS__, a, confBase, &control, &last_match) == HWLM_TERMINATED)return HWLM_TERMINATED;
template <int NMSK>
static really_inline
m128 prep_conf_teddy_128_templ(const m128 *maskBase, m128 val) {
m128 mask = set1_16x8(0xf);
m128 lo = and128(val, mask);
m128 hi = and128(rshift64_m128(val, 4), mask);
m128 r1 = or128(pshufb_m128(maskBase[0 * 2], lo),
pshufb_m128(maskBase[0 * 2 + 1], hi));
if constexpr (NMSK == 1) return r1;
m128 res_1 = or128(pshufb_m128(maskBase[1 * 2], lo),
pshufb_m128(maskBase[1 * 2 + 1], hi));
m128 old_1 = zeroes128();
m128 res_shifted_1 = palignr(res_1, old_1, 16 - 1);
m128 r2 = or128(r1, res_shifted_1);
if constexpr (NMSK == 2) return r2;
m128 res_2 = or128(pshufb_m128(maskBase[2 * 2], lo),
pshufb_m128(maskBase[2 * 2 + 1], hi));
m128 res_shifted_2 = palignr(res_2, old_1, 16 - 2);
m128 r3 = or128(r2, res_shifted_2);
if constexpr (NMSK == 3) return r3;
m128 res_3 = or128(pshufb_m128(maskBase[3 * 2], lo),
pshufb_m128(maskBase[3 * 2 + 1], hi));
m128 res_shifted_3 = palignr(res_3, old_1, 16 - 3);
return or128(r3, res_shifted_3);
}
template <int NMSK>
hwlm_error_t fdr_exec_teddy_128_templ(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
const u8 *buf_end = a->buf + a->len;
const u8 *ptr = a->buf + a->start_offset;
u32 floodBackoff = FLOOD_BACKOFF_START;
const u8 *tryFloodDetect = a->firstFloodDetect;
u32 last_match = ones_u32;
const struct Teddy *teddy = reinterpret_cast<const struct Teddy *>(fdr);
const size_t iterBytes = 32;
DEBUG_PRINTF("params: buf %p len %zu start_offset %zu\n",
a->buf, a->len, a->start_offset);
const m128 *maskBase = getMaskBase(teddy);
const u32 *confBase = getConfBase(teddy);
const u8 *mainStart = ROUNDUP_PTR(ptr, 16);
DEBUG_PRINTF("derive: ptr: %p mainstart %p\n", ptr, mainStart);
if (ptr < mainStart) {
ptr = mainStart - 16;
m128 p_mask;
m128 val_0 = vectoredLoad128(&p_mask, ptr, a->start_offset,
a->buf, buf_end,
a->buf_history, a->len_history, NMSK);
m128 r_0 = prep_conf_teddy_128_templ<NMSK>(maskBase, val_0);
r_0 = or128(r_0, p_mask);
CONFIRM_TEDDY_128(r_0, 8, 0, VECTORING, ptr);
ptr += 16;
}
if (ptr + 16 <= buf_end) {
m128 r_0 = prep_conf_teddy_128_templ<NMSK>(maskBase, load128(ptr));
CONFIRM_TEDDY_128(r_0, 8, 0, VECTORING, ptr);
ptr += 16;
}
for (; ptr + iterBytes <= buf_end; ptr += iterBytes) {
__builtin_prefetch(ptr + (iterBytes * 4));
CHECK_FLOOD;
m128 r_0 = prep_conf_teddy_128_templ<NMSK>(maskBase, load128(ptr));
CONFIRM_TEDDY_128(r_0, 8, 0, NOT_CAUTIOUS, ptr);
m128 r_1 = prep_conf_teddy_128_templ<NMSK>(maskBase, load128(ptr + 16));
CONFIRM_TEDDY_128(r_1, 8, 16, NOT_CAUTIOUS, ptr);
}
if (ptr + 16 <= buf_end) {
m128 r_0 = prep_conf_teddy_128_templ<NMSK>(maskBase, load128(ptr));
CONFIRM_TEDDY_128(r_0, 8, 0, NOT_CAUTIOUS, ptr);
ptr += 16;
}
assert(ptr + 16 > buf_end);
if (ptr < buf_end) {
m128 p_mask;
m128 val_0 = vectoredLoad128(&p_mask, ptr, 0, ptr, buf_end,
a->buf_history, a->len_history, NMSK);
m128 r_0 = prep_conf_teddy_128_templ<NMSK>(maskBase, val_0);
r_0 = or128(r_0, p_mask);
CONFIRM_TEDDY_128(r_0, 8, 0, VECTORING, ptr);
}
return HWLM_SUCCESS;
}
#define FDR_EXEC_TEDDY_FN fdr_exec_teddy_128_templ
#endif // HAVE_AVX2 HAVE_AVX512
extern "C" {
hwlm_error_t fdr_exec_teddy_msks1(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_TEDDY_FN<1>(fdr, a, control);
}
hwlm_error_t fdr_exec_teddy_msks1_pck(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_TEDDY_FN<1>(fdr, a, control);
}
hwlm_error_t fdr_exec_teddy_msks2(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_TEDDY_FN<2>(fdr, a, control);
}
hwlm_error_t fdr_exec_teddy_msks2_pck(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_TEDDY_FN<2>(fdr, a, control);
}
hwlm_error_t fdr_exec_teddy_msks3(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_TEDDY_FN<3>(fdr, a, control);
}
hwlm_error_t fdr_exec_teddy_msks3_pck(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_TEDDY_FN<3>(fdr, a, control);
}
hwlm_error_t fdr_exec_teddy_msks4(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_TEDDY_FN<4>(fdr, a, control);
}
hwlm_error_t fdr_exec_teddy_msks4_pck(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_TEDDY_FN<4>(fdr, a, control);
}
} // extern

8
src/fdr/teddy.h
View File

@ -1,5 +1,6 @@
/* /*
* Copyright (c) 2016-2017, Intel Corporation * Copyright (c) 2016-2017, Intel Corporation
* Copyright (c) 2024, VectorCamp PC
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met: * modification, are permitted provided that the following conditions are met:
@ -39,6 +40,10 @@
struct FDR; // forward declaration from fdr_internal.h struct FDR; // forward declaration from fdr_internal.h
struct FDR_Runtime_Args; struct FDR_Runtime_Args;
#ifdef __cplusplus
extern "C" {
#endif
hwlm_error_t fdr_exec_teddy_msks1(const struct FDR *fdr, hwlm_error_t fdr_exec_teddy_msks1(const struct FDR *fdr,
const struct FDR_Runtime_Args *a, const struct FDR_Runtime_Args *a,
hwlm_group_t control); hwlm_group_t control);
@ -106,5 +111,8 @@ hwlm_error_t fdr_exec_fat_teddy_msks4_pck(const struct FDR *fdr,
hwlm_group_t control); hwlm_group_t control);
#endif /* HAVE_AVX2 */ #endif /* HAVE_AVX2 */
#ifdef __cplusplus
}
#endif
#endif /* TEDDY_H_ */ #endif /* TEDDY_H_ */

709
src/fdr/teddy_avx2.c
View File

@ -1,709 +0,0 @@
/*
* Copyright (c) 2016-2020, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief Teddy literal matcher: AVX2 engine runtime.
*/
#include "fdr_internal.h"
#include "flood_runtime.h"
#include "teddy.h"
#include "teddy_internal.h"
#include "teddy_runtime_common.h"
#include "util/arch.h"
#include "util/simd_utils.h"
#if defined(HAVE_AVX2)
const u8 ALIGN_AVX_DIRECTIVE p_mask_arr256[33][64] = {
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff},
{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
};
#if defined(HAVE_AVX512VBMI) // VBMI strong fat teddy
#define CONF_FAT_CHUNK_64(chunk, bucket, off, reason, pt, conf_fn) \
do { \
if (unlikely(chunk != ones_u64a)) { \
chunk = ~chunk; \
conf_fn(&chunk, bucket, off, confBase, reason, a, pt, \
&control, &last_match); \
CHECK_HWLM_TERMINATE_MATCHING; \
} \
} while(0)
#define CONF_FAT_CHUNK_32(chunk, bucket, off, reason, pt, conf_fn) \
do { \
if (unlikely(chunk != ones_u32)) { \
chunk = ~chunk; \
conf_fn(&chunk, bucket, off, confBase, reason, a, pt, \
&control, &last_match); \
CHECK_HWLM_TERMINATE_MATCHING; \
} \
} while(0)
static really_inline
const m512 *getDupMaskBase(const struct Teddy *teddy, u8 numMask) {
return (const m512 *)((const u8 *)teddy + ROUNDUP_CL(sizeof(struct Teddy))
+ ROUNDUP_CL(2 * numMask * sizeof(m256)));
}
#else
#define CONF_FAT_CHUNK_64(chunk, bucket, off, reason, conf_fn) \
do { \
if (unlikely(chunk != ones_u64a)) { \
chunk = ~chunk; \
conf_fn(&chunk, bucket, off, confBase, reason, a, ptr, \
&control, &last_match); \
CHECK_HWLM_TERMINATE_MATCHING; \
} \
} while(0)
#define CONF_FAT_CHUNK_32(chunk, bucket, off, reason, conf_fn) \
do { \
if (unlikely(chunk != ones_u32)) { \
chunk = ~chunk; \
conf_fn(&chunk, bucket, off, confBase, reason, a, ptr, \
&control, &last_match); \
CHECK_HWLM_TERMINATE_MATCHING; \
} \
} while(0)
static really_inline
const m256 *getMaskBase_fat(const struct Teddy *teddy) {
return (const m256 *)((const u8 *)teddy + ROUNDUP_CL(sizeof(struct Teddy)));
}
#endif
#if defined(HAVE_AVX512VBMI) // VBMI strong fat teddy
const u8 ALIGN_AVX_DIRECTIVE p_mask_interleave[64] = {
0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39,
8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47,
16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55,
24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63
};
#ifdef ARCH_64_BIT
#define CONFIRM_FAT_TEDDY(var, bucket, offset, reason, pt, conf_fn) \
do { \
if (unlikely(diff512(var, ones512()))) { \
m512 msk_interleave = load512(p_mask_interleave); \
m512 r = vpermb512(msk_interleave, var); \
m128 r0 = extract128from512(r, 0); \
m128 r1 = extract128from512(r, 1); \
m128 r2 = extract128from512(r, 2); \
m128 r3 = extract128from512(r, 3); \
u64a part1 = movq(r0); \
u64a part2 = extract64from128(r0, 1); \
u64a part3 = movq(r1); \
u64a part4 = extract64from128(r1, 1); \
u64a part5 = movq(r2); \
u64a part6 = extract64from128(r2, 1); \
u64a part7 = movq(r3); \
u64a part8 = extract64from128(r3, 1); \
CONF_FAT_CHUNK_64(part1, bucket, offset, reason, pt, conf_fn); \
CONF_FAT_CHUNK_64(part2, bucket, offset + 4, reason, pt, conf_fn); \
CONF_FAT_CHUNK_64(part3, bucket, offset + 8, reason, pt, conf_fn); \
CONF_FAT_CHUNK_64(part4, bucket, offset + 12, reason, pt, conf_fn); \
CONF_FAT_CHUNK_64(part5, bucket, offset + 16, reason, pt, conf_fn); \
CONF_FAT_CHUNK_64(part6, bucket, offset + 20, reason, pt, conf_fn); \
CONF_FAT_CHUNK_64(part7, bucket, offset + 24, reason, pt, conf_fn); \
CONF_FAT_CHUNK_64(part8, bucket, offset + 28, reason, pt, conf_fn); \
} \
} while(0)
#else
#define CONFIRM_FAT_TEDDY(var, bucket, offset, reason, pt, conf_fn) \
do { \
if (unlikely(diff512(var, ones512()))) { \
m512 msk_interleave = load512(p_mask_interleave); \
m512 r = vpermb512(msk_interleave, var); \
m128 r0 = extract128from512(r, 0); \
m128 r1 = extract128from512(r, 1); \
m128 r2 = extract128from512(r, 2); \
m128 r3 = extract128from512(r, 3); \
u32 part1 = movd(r0); \
u32 part2 = extract32from128(r0, 1); \
u32 part3 = extract32from128(r0, 2); \
u32 part4 = extract32from128(r0, 3); \
u32 part5 = movd(r1); \
u32 part6 = extract32from128(r1, 1); \
u32 part7 = extract32from128(r1, 2); \
u32 part8 = extract32from128(r1, 3); \
u32 part9 = movd(r2); \
u32 part10 = extract32from128(r2, 1); \
u32 part11 = extract32from128(r2, 2); \
u32 part12 = extract32from128(r2, 3); \
u32 part13 = movd(r3); \
u32 part14 = extract32from128(r3, 1); \
u32 part15 = extract32from128(r3, 2); \
u32 part16 = extract32from128(r3, 3); \
CONF_FAT_CHUNK_32(part1, bucket, offset, reason, pt, conf_fn); \
CONF_FAT_CHUNK_32(part2, bucket, offset + 2, reason, pt, conf_fn); \
CONF_FAT_CHUNK_32(part3, bucket, offset + 4, reason, pt, conf_fn); \
CONF_FAT_CHUNK_32(part4, bucket, offset + 6, reason, pt, conf_fn); \
CONF_FAT_CHUNK_32(part5, bucket, offset + 8, reason, pt, conf_fn); \
CONF_FAT_CHUNK_32(part6, bucket, offset + 10, reason, pt, conf_fn); \
CONF_FAT_CHUNK_32(part7, bucket, offset + 12, reason, pt, conf_fn); \
CONF_FAT_CHUNK_32(part8, bucket, offset + 14, reason, pt, conf_fn); \
CONF_FAT_CHUNK_32(part9, bucket, offset + 16, reason, pt, conf_fn); \
CONF_FAT_CHUNK_32(part10, bucket, offset + 18, reason, pt, conf_fn);\
CONF_FAT_CHUNK_32(part11, bucket, offset + 20, reason, pt, conf_fn);\
CONF_FAT_CHUNK_32(part12, bucket, offset + 22, reason, pt, conf_fn);\
CONF_FAT_CHUNK_32(part13, bucket, offset + 24, reason, pt, conf_fn);\
CONF_FAT_CHUNK_32(part14, bucket, offset + 26, reason, pt, conf_fn);\
CONF_FAT_CHUNK_32(part15, bucket, offset + 28, reason, pt, conf_fn);\
CONF_FAT_CHUNK_32(part16, bucket, offset + 30, reason, pt, conf_fn);\
} \
} while(0)
#endif
#define PREP_FAT_SHUF_MASK \
m512 lo = and512(val, *lo_mask); \
m512 hi = and512(rshift64_m512(val, 4), *lo_mask)
#define FAT_TEDDY_VBMI_PSHUFB_OR_M1 \
m512 shuf_or_b0 = or512(pshufb_m512(dup_mask[0], lo), \
pshufb_m512(dup_mask[1], hi));
#define FAT_TEDDY_VBMI_PSHUFB_OR_M2 \
FAT_TEDDY_VBMI_PSHUFB_OR_M1 \
m512 shuf_or_b1 = or512(pshufb_m512(dup_mask[2], lo), \
pshufb_m512(dup_mask[3], hi));
#define FAT_TEDDY_VBMI_PSHUFB_OR_M3 \
FAT_TEDDY_VBMI_PSHUFB_OR_M2 \
m512 shuf_or_b2 = or512(pshufb_m512(dup_mask[4], lo), \
pshufb_m512(dup_mask[5], hi));
#define FAT_TEDDY_VBMI_PSHUFB_OR_M4 \
FAT_TEDDY_VBMI_PSHUFB_OR_M3 \
m512 shuf_or_b3 = or512(pshufb_m512(dup_mask[6], lo), \
pshufb_m512(dup_mask[7], hi));
#define FAT_TEDDY_VBMI_SL1_MASK 0xfffffffefffffffeULL
#define FAT_TEDDY_VBMI_SL2_MASK 0xfffffffcfffffffcULL
#define FAT_TEDDY_VBMI_SL3_MASK 0xfffffff8fffffff8ULL
#define FAT_TEDDY_VBMI_SHIFT_M1
#define FAT_TEDDY_VBMI_SHIFT_M2 \
FAT_TEDDY_VBMI_SHIFT_M1 \
m512 sl1 = maskz_vpermb512(FAT_TEDDY_VBMI_SL1_MASK, sl_msk[0], shuf_or_b1);
#define FAT_TEDDY_VBMI_SHIFT_M3 \
FAT_TEDDY_VBMI_SHIFT_M2 \
m512 sl2 = maskz_vpermb512(FAT_TEDDY_VBMI_SL2_MASK, sl_msk[1], shuf_or_b2);
#define FAT_TEDDY_VBMI_SHIFT_M4 \
FAT_TEDDY_VBMI_SHIFT_M3 \
m512 sl3 = maskz_vpermb512(FAT_TEDDY_VBMI_SL3_MASK, sl_msk[2], shuf_or_b3);
#define FAT_SHIFT_OR_M1 \
shuf_or_b0
#define FAT_SHIFT_OR_M2 \
or512(sl1, FAT_SHIFT_OR_M1)
#define FAT_SHIFT_OR_M3 \
or512(sl2, FAT_SHIFT_OR_M2)
#define FAT_SHIFT_OR_M4 \
or512(sl3, FAT_SHIFT_OR_M3)
static really_inline
m512 prep_conf_fat_teddy_m1(const m512 *lo_mask, const m512 *dup_mask,
UNUSED const m512 *sl_msk, const m512 val) {
PREP_FAT_SHUF_MASK;
FAT_TEDDY_VBMI_PSHUFB_OR_M1;
FAT_TEDDY_VBMI_SHIFT_M1;
return FAT_SHIFT_OR_M1;
}
static really_inline
m512 prep_conf_fat_teddy_m2(const m512 *lo_mask, const m512 *dup_mask,
const m512 *sl_msk, const m512 val) {
PREP_FAT_SHUF_MASK;
FAT_TEDDY_VBMI_PSHUFB_OR_M2;
FAT_TEDDY_VBMI_SHIFT_M2;
return FAT_SHIFT_OR_M2;
}
static really_inline
m512 prep_conf_fat_teddy_m3(const m512 *lo_mask, const m512 *dup_mask,
const m512 *sl_msk, const m512 val) {
PREP_FAT_SHUF_MASK;
FAT_TEDDY_VBMI_PSHUFB_OR_M3;
FAT_TEDDY_VBMI_SHIFT_M3;
return FAT_SHIFT_OR_M3;
}
static really_inline
m512 prep_conf_fat_teddy_m4(const m512 *lo_mask, const m512 *dup_mask,
const m512 *sl_msk, const m512 val) {
PREP_FAT_SHUF_MASK;
FAT_TEDDY_VBMI_PSHUFB_OR_M4;
FAT_TEDDY_VBMI_SHIFT_M4;
return FAT_SHIFT_OR_M4;
}
#define PREP_CONF_FAT_FN(val, n) \
prep_conf_fat_teddy_m##n(&lo_mask, dup_mask, sl_msk, val)
#define FAT_TEDDY_VBMI_SL1_POS 15
#define FAT_TEDDY_VBMI_SL2_POS 14
#define FAT_TEDDY_VBMI_SL3_POS 13
#define FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M1
#define FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M2 \
FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M1 \
sl_msk[0] = loadu512(p_sh_mask_arr + FAT_TEDDY_VBMI_SL1_POS);
#define FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M3 \
FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M2 \
sl_msk[1] = loadu512(p_sh_mask_arr + FAT_TEDDY_VBMI_SL2_POS);
#define FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M4 \
FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M3 \
sl_msk[2] = loadu512(p_sh_mask_arr + FAT_TEDDY_VBMI_SL3_POS);
/*
* In FAT teddy, it needs 2 bytes to represent result of each position,
* so each nibble's(for example, lo nibble of last byte) FAT teddy mask
* has 16x2 bytes:
* |----------------------------------|----------------------------------|
* 16bytes (bucket 0..7 in each byte) 16bytes (bucket 8..15 in each byte)
* A B
* at runtime FAT teddy reads 16 bytes once and duplicate them to 32 bytes:
* |----------------------------------|----------------------------------|
* 16bytes input data (lo nibbles) 16bytes duplicated data (lo nibbles)
* X X
* then do pshufb_m256(AB, XX).
*
* In AVX512 reinforced FAT teddy, it reads 32 bytes once and duplicate them
* to 64 bytes:
* |----------------|----------------|----------------|----------------|
* X Y X Y
* in this case we need DUP_FAT_MASK to construct AABB:
* |----------------|----------------|----------------|----------------|
* A A B B
* then do pshufb_m512(AABB, XYXY).
*/
#define PREPARE_FAT_MASKS(n) \
m512 lo_mask = set1_64x8(0xf); \
m512 sl_msk[n - 1]; \
FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M##n
#define FAT_TEDDY_VBMI_CONF_MASK_HEAD (0xffffffffULL >> n_sh)
#define FAT_TEDDY_VBMI_CONF_MASK_FULL ((0xffffffffULL << n_sh) & 0xffffffffULL)
#define FAT_TEDDY_VBMI_CONF_MASK_VAR(n) (0xffffffffULL >> (32 - n) << overlap)
#define FAT_TEDDY_VBMI_LOAD_MASK_PATCH (0xffffffffULL >> (32 - n_sh))
#define FDR_EXEC_FAT_TEDDY(fdr, a, control, n_msk, conf_fn) \
do { \
const u8 *buf_end = a->buf + a->len; \
const u8 *ptr = a->buf + a->start_offset; \
u32 floodBackoff = FLOOD_BACKOFF_START; \
const u8 *tryFloodDetect = a->firstFloodDetect; \
u32 last_match = ones_u32; \
const struct Teddy *teddy = (const struct Teddy *)fdr; \
const size_t iterBytes = 32; \
u32 n_sh = n_msk - 1; \
const size_t loopBytes = 32 - n_sh; \
DEBUG_PRINTF("params: buf %p len %zu start_offset %zu\n", \
a->buf, a->len, a->start_offset); \
\
const m512 *dup_mask = getDupMaskBase(teddy, n_msk); \
PREPARE_FAT_MASKS(n_msk); \
const u32 *confBase = getConfBase(teddy); \
\
u64a k = FAT_TEDDY_VBMI_CONF_MASK_FULL; \
m512 p_mask = set_mask_m512(~((k << 32) | k)); \
u32 overlap = 0; \
u64a patch = 0; \
if (likely(ptr + loopBytes <= buf_end)) { \
u64a k0 = FAT_TEDDY_VBMI_CONF_MASK_HEAD; \
m512 p_mask0 = set_mask_m512(~((k0 << 32) | k0)); \
m512 r_0 = PREP_CONF_FAT_FN(set2x256(loadu256(ptr)), n_msk); \
r_0 = or512(r_0, p_mask0); \
CONFIRM_FAT_TEDDY(r_0, 16, 0, VECTORING, ptr, conf_fn); \
ptr += loopBytes; \
overlap = n_sh; \
patch = FAT_TEDDY_VBMI_LOAD_MASK_PATCH; \
} \
\
for (; ptr + loopBytes <= buf_end; ptr += loopBytes) { \
CHECK_FLOOD; \
m512 r_0 = PREP_CONF_FAT_FN(set2x256(loadu256(ptr - n_sh)), n_msk); \
r_0 = or512(r_0, p_mask); \
CONFIRM_FAT_TEDDY(r_0, 16, 0, NOT_CAUTIOUS, ptr - n_sh, conf_fn); \
} \
\
assert(ptr + loopBytes > buf_end); \
if (ptr < buf_end) { \
u32 left = (u32)(buf_end - ptr); \
u64a k1 = FAT_TEDDY_VBMI_CONF_MASK_VAR(left); \
m512 p_mask1 = set_mask_m512(~((k1 << 32) | k1)); \
m512 val_0 = set2x256(loadu_maskz_m256(k1 | patch, ptr - overlap)); \
m512 r_0 = PREP_CONF_FAT_FN(val_0, n_msk); \
r_0 = or512(r_0, p_mask1); \
CONFIRM_FAT_TEDDY(r_0, 16, 0, VECTORING, ptr - overlap, conf_fn); \
} \
\
return HWLM_SUCCESS; \
} while(0)
#else // !HAVE_AVX512VBMI, AVX2 normal fat teddy
#ifdef ARCH_64_BIT
#define CONFIRM_FAT_TEDDY(var, bucket, offset, reason, conf_fn) \
do { \
if (unlikely(diff256(var, ones256()))) { \
m256 swap = swap128in256(var); \
m256 r = interleave256lo(var, swap); \
u64a part1 = extractlow64from256(r); \
u64a part2 = extract64from256(r, 1); \
r = interleave256hi(var, swap); \
u64a part3 = extractlow64from256(r); \
u64a part4 = extract64from256(r, 1); \
CONF_FAT_CHUNK_64(part1, bucket, offset, reason, conf_fn); \
CONF_FAT_CHUNK_64(part2, bucket, offset + 4, reason, conf_fn); \
CONF_FAT_CHUNK_64(part3, bucket, offset + 8, reason, conf_fn); \
CONF_FAT_CHUNK_64(part4, bucket, offset + 12, reason, conf_fn); \
} \
} while(0)
#else
#define CONFIRM_FAT_TEDDY(var, bucket, offset, reason, conf_fn) \
do { \
if (unlikely(diff256(var, ones256()))) { \
m256 swap = swap128in256(var); \
m256 r = interleave256lo(var, swap); \
u32 part1 = extractlow32from256(r); \
u32 part2 = extract32from256(r, 1); \
u32 part3 = extract32from256(r, 2); \
u32 part4 = extract32from256(r, 3); \
r = interleave256hi(var, swap); \
u32 part5 = extractlow32from256(r); \
u32 part6 = extract32from256(r, 1); \
u32 part7 = extract32from256(r, 2); \
u32 part8 = extract32from256(r, 3); \
CONF_FAT_CHUNK_32(part1, bucket, offset, reason, conf_fn); \
CONF_FAT_CHUNK_32(part2, bucket, offset + 2, reason, conf_fn); \
CONF_FAT_CHUNK_32(part3, bucket, offset + 4, reason, conf_fn); \
CONF_FAT_CHUNK_32(part4, bucket, offset + 6, reason, conf_fn); \
CONF_FAT_CHUNK_32(part5, bucket, offset + 8, reason, conf_fn); \
CONF_FAT_CHUNK_32(part6, bucket, offset + 10, reason, conf_fn); \
CONF_FAT_CHUNK_32(part7, bucket, offset + 12, reason, conf_fn); \
CONF_FAT_CHUNK_32(part8, bucket, offset + 14, reason, conf_fn); \
} \
} while(0)
#endif
static really_inline
m256 vectoredLoad2x128(m256 *p_mask, const u8 *ptr, const size_t start_offset,
const u8 *lo, const u8 *hi,
const u8 *buf_history, size_t len_history,
const u32 nMasks) {
m128 p_mask128;
m256 ret = set1_2x128(vectoredLoad128(&p_mask128, ptr, start_offset, lo, hi,
buf_history, len_history, nMasks));
*p_mask = set1_2x128(p_mask128);
return ret;
}
static really_inline
m256 prep_conf_fat_teddy_m1(const m256 *maskBase, m256 val) {
m256 mask = set1_32x8(0xf);
m256 lo = and256(val, mask);
m256 hi = and256(rshift64_m256(val, 4), mask);
return or256(pshufb_m256(maskBase[0 * 2], lo),
pshufb_m256(maskBase[0 * 2 + 1], hi));
}
static really_inline
m256 prep_conf_fat_teddy_m2(const m256 *maskBase, m256 *old_1, m256 val) {
m256 mask = set1_32x8(0xf);
m256 lo = and256(val, mask);
m256 hi = and256(rshift64_m256(val, 4), mask);
m256 r = prep_conf_fat_teddy_m1(maskBase, val);
m256 res_1 = or256(pshufb_m256(maskBase[1 * 2], lo),
pshufb_m256(maskBase[1 * 2 + 1], hi));
m256 res_shifted_1 = vpalignr(res_1, *old_1, 16 - 1);
*old_1 = res_1;
return or256(r, res_shifted_1);
}
static really_inline
m256 prep_conf_fat_teddy_m3(const m256 *maskBase, m256 *old_1, m256 *old_2,
m256 val) {
m256 mask = set1_32x8(0xf);
m256 lo = and256(val, mask);
m256 hi = and256(rshift64_m256(val, 4), mask);
m256 r = prep_conf_fat_teddy_m2(maskBase, old_1, val);
m256 res_2 = or256(pshufb_m256(maskBase[2 * 2], lo),
pshufb_m256(maskBase[2 * 2 + 1], hi));
m256 res_shifted_2 = vpalignr(res_2, *old_2, 16 - 2);
*old_2 = res_2;
return or256(r, res_shifted_2);
}
static really_inline
m256 prep_conf_fat_teddy_m4(const m256 *maskBase, m256 *old_1, m256 *old_2,
m256 *old_3, m256 val) {
m256 mask = set1_32x8(0xf);
m256 lo = and256(val, mask);
m256 hi = and256(rshift64_m256(val, 4), mask);
m256 r = prep_conf_fat_teddy_m3(maskBase, old_1, old_2, val);
m256 res_3 = or256(pshufb_m256(maskBase[3 * 2], lo),
pshufb_m256(maskBase[3 * 2 + 1], hi));
m256 res_shifted_3 = vpalignr(res_3, *old_3, 16 - 3);
*old_3 = res_3;
return or256(r, res_shifted_3);
}
#define FDR_EXEC_FAT_TEDDY_RES_OLD_1 \
do { \
} while(0)
#define FDR_EXEC_FAT_TEDDY_RES_OLD_2 \
m256 res_old_1 = zeroes256();
#define FDR_EXEC_FAT_TEDDY_RES_OLD_3 \
m256 res_old_1 = zeroes256(); \
m256 res_old_2 = zeroes256();
#define FDR_EXEC_FAT_TEDDY_RES_OLD_4 \
m256 res_old_1 = zeroes256(); \
m256 res_old_2 = zeroes256(); \
m256 res_old_3 = zeroes256();
#define FDR_EXEC_FAT_TEDDY_RES_OLD(n) FDR_EXEC_FAT_TEDDY_RES_OLD_##n
#define PREP_CONF_FAT_FN_1(mask_base, val) \
prep_conf_fat_teddy_m1(mask_base, val)
#define PREP_CONF_FAT_FN_2(mask_base, val) \
prep_conf_fat_teddy_m2(mask_base, &res_old_1, val)
#define PREP_CONF_FAT_FN_3(mask_base, val) \
prep_conf_fat_teddy_m3(mask_base, &res_old_1, &res_old_2, val)
#define PREP_CONF_FAT_FN_4(mask_base, val) \
prep_conf_fat_teddy_m4(mask_base, &res_old_1, &res_old_2, &res_old_3, val)
#define PREP_CONF_FAT_FN(mask_base, val, n) \
PREP_CONF_FAT_FN_##n(mask_base, val)
#define FDR_EXEC_FAT_TEDDY(fdr, a, control, n_msk, conf_fn) \
do { \
const u8 *buf_end = a->buf + a->len; \
const u8 *ptr = a->buf + a->start_offset; \
u32 floodBackoff = FLOOD_BACKOFF_START; \
const u8 *tryFloodDetect = a->firstFloodDetect; \
u32 last_match = ones_u32; \
const struct Teddy *teddy = (const struct Teddy *)fdr; \
const size_t iterBytes = 32; \
DEBUG_PRINTF("params: buf %p len %zu start_offset %zu\n", \
a->buf, a->len, a->start_offset); \
\
const m256 *maskBase = getMaskBase_fat(teddy); \
const u32 *confBase = getConfBase(teddy); \
\
FDR_EXEC_FAT_TEDDY_RES_OLD(n_msk); \
const u8 *mainStart = ROUNDUP_PTR(ptr, 16); \
DEBUG_PRINTF("derive: ptr: %p mainstart %p\n", ptr, mainStart); \
if (ptr < mainStart) { \
ptr = mainStart - 16; \
m256 p_mask; \
m256 val_0 = vectoredLoad2x128(&p_mask, ptr, a->start_offset, \
a->buf, buf_end, \
a->buf_history, a->len_history, \
n_msk); \
m256 r_0 = PREP_CONF_FAT_FN(maskBase, val_0, n_msk); \
r_0 = or256(r_0, p_mask); \
CONFIRM_FAT_TEDDY(r_0, 16, 0, VECTORING, conf_fn); \
ptr += 16; \
} \
\
if (ptr + 16 <= buf_end) { \
m256 r_0 = PREP_CONF_FAT_FN(maskBase, load2x128(ptr), n_msk); \
CONFIRM_FAT_TEDDY(r_0, 16, 0, VECTORING, conf_fn); \
ptr += 16; \
} \
\
for ( ; ptr + iterBytes <= buf_end; ptr += iterBytes) { \
__builtin_prefetch(ptr + (iterBytes * 4)); \
CHECK_FLOOD; \
m256 r_0 = PREP_CONF_FAT_FN(maskBase, load2x128(ptr), n_msk); \
CONFIRM_FAT_TEDDY(r_0, 16, 0, NOT_CAUTIOUS, conf_fn); \
m256 r_1 = PREP_CONF_FAT_FN(maskBase, load2x128(ptr + 16), n_msk); \
CONFIRM_FAT_TEDDY(r_1, 16, 16, NOT_CAUTIOUS, conf_fn); \
} \
\
if (ptr + 16 <= buf_end) { \
m256 r_0 = PREP_CONF_FAT_FN(maskBase, load2x128(ptr), n_msk); \
CONFIRM_FAT_TEDDY(r_0, 16, 0, NOT_CAUTIOUS, conf_fn); \
ptr += 16; \
} \
\
assert(ptr + 16 > buf_end); \
if (ptr < buf_end) { \
m256 p_mask; \
m256 val_0 = vectoredLoad2x128(&p_mask, ptr, 0, ptr, buf_end, \
a->buf_history, a->len_history, \
n_msk); \
m256 r_0 = PREP_CONF_FAT_FN(maskBase, val_0, n_msk); \
r_0 = or256(r_0, p_mask); \
CONFIRM_FAT_TEDDY(r_0, 16, 0, VECTORING, conf_fn); \
} \
\
return HWLM_SUCCESS; \
} while(0)
#endif // HAVE_AVX512VBMI
hwlm_error_t fdr_exec_fat_teddy_msks1(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
FDR_EXEC_FAT_TEDDY(fdr, a, control, 1, do_confWithBit_teddy);
}
hwlm_error_t fdr_exec_fat_teddy_msks1_pck(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
FDR_EXEC_FAT_TEDDY(fdr, a, control, 1, do_confWithBit_teddy);
}
hwlm_error_t fdr_exec_fat_teddy_msks2(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
FDR_EXEC_FAT_TEDDY(fdr, a, control, 2, do_confWithBit_teddy);
}
hwlm_error_t fdr_exec_fat_teddy_msks2_pck(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
FDR_EXEC_FAT_TEDDY(fdr, a, control, 2, do_confWithBit_teddy);
}
hwlm_error_t fdr_exec_fat_teddy_msks3(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
FDR_EXEC_FAT_TEDDY(fdr, a, control, 3, do_confWithBit_teddy);
}
hwlm_error_t fdr_exec_fat_teddy_msks3_pck(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
FDR_EXEC_FAT_TEDDY(fdr, a, control, 3, do_confWithBit_teddy);
}
hwlm_error_t fdr_exec_fat_teddy_msks4(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
FDR_EXEC_FAT_TEDDY(fdr, a, control, 4, do_confWithBit_teddy);
}
hwlm_error_t fdr_exec_fat_teddy_msks4_pck(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
FDR_EXEC_FAT_TEDDY(fdr, a, control, 4, do_confWithBit_teddy);
}
#endif // HAVE_AVX2

570
src/fdr/teddy_fat.cpp Normal file
View File

@ -0,0 +1,570 @@
/*
* Copyright (c) 2015-2020, Intel Corporation
* Copyright (c) 2024, VectorCamp PC
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/* fat teddy for AVX2 and AVX512VBMI */
#include "fdr_internal.h"
#include "flood_runtime.h"
#include "teddy.h"
#include "teddy_internal.h"
#include "teddy_runtime_common.h"
#include "util/arch.h"
#include "util/simd_utils.h"
#if defined(HAVE_AVX2)
#ifdef ARCH_64_BIT
static really_inline
hwlm_error_t conf_chunk_64(u64a chunk, u8 bucket, u8 offset,
CautionReason reason, const u8 *pt,
const u32* confBase,
const struct FDR_Runtime_Args *a,
hwlm_group_t *control,
u32 *last_match) {
if (unlikely(chunk != ones_u64a)) {
chunk = ~chunk;
do_confWithBit_teddy(&chunk, bucket, offset, confBase, reason, a, pt,
control, last_match);
// adapted from CHECK_HWLM_TERMINATE_MATCHING
if (unlikely(*control == HWLM_TERMINATE_MATCHING)) {
return HWLM_TERMINATED;
}
}
return HWLM_SUCCESS;
}
#define CONF_FAT_CHUNK_64(chunk, bucket, off, reason, pt, confBase, a, control, last_match) \
if(conf_chunk_64(chunk, bucket, off, reason, pt, confBase, a, control, last_match) == HWLM_TERMINATED)return HWLM_TERMINATED;
#else
static really_inline
hwlm_error_t conf_chunk_32(u32 chunk, u8 bucket, u8 offset,
CautionReason reason, const u8 *pt,
const u32* confBase,
const struct FDR_Runtime_Args *a,
hwlm_group_t *control,
u32 *last_match) {
if (unlikely(chunk != ones_u32)) {
chunk = ~chunk;
do_confWithBit_teddy(&chunk, bucket, offset, confBase, reason, a, pt,
control, last_match);
// adapted from CHECK_HWLM_TERMINATE_MATCHING
if (unlikely(*control == HWLM_TERMINATE_MATCHING)) {
return HWLM_TERMINATED;
}
}
return HWLM_SUCCESS;
}
#define CONF_FAT_CHUNK_32(chunk, bucket, off, reason, pt, confBase, a, control, last_match) \
if(conf_chunk_32(chunk, bucket, off, reason, pt, confBase, a, control, last_match) == HWLM_TERMINATED)return HWLM_TERMINATED;
#endif
#if defined(HAVE_AVX512VBMI) // VBMI strong teddy
// fat 512 teddy is only with vbmi
static really_inline
const m512 *getDupMaskBase(const struct Teddy *teddy, u8 numMask) {
return (const m512 *)((const u8 *)teddy + ROUNDUP_CL(sizeof(struct Teddy))
+ ROUNDUP_CL(2 * numMask * sizeof(m256)));
}
const u8 ALIGN_AVX_DIRECTIVE p_mask_interleave[64] = {
0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39,
8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47,
16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55,
24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63
};
#ifdef ARCH_64_BIT
hwlm_error_t confirm_fat_teddy_64_512(m512 var, u8 bucket, u8 offset,
CautionReason reason, const u8 *ptr,
const struct FDR_Runtime_Args *a,
const u32* confBase, hwlm_group_t *control,
u32 *last_match) {
if (unlikely(diff512(var, ones512()))) {
m512 msk_interleave = load512(p_mask_interleave);
m512 r = vpermb512(msk_interleave, var);
m128 r0 = extract128from512(r, 0);
m128 r1 = extract128from512(r, 1);
m128 r2 = extract128from512(r, 2);
m128 r3 = extract128from512(r, 3);
u64a part1 = movq(r0);
u64a part2 = extract64from128(r0, 1);
u64a part3 = movq(r1);
u64a part4 = extract64from128(r1, 1);
u64a part5 = movq(r2);
u64a part6 = extract64from128(r2, 1);
u64a part7 = movq(r3);
u64a part8 = extract64from128(r3, 1);
CONF_FAT_CHUNK_64(part1, bucket, offset, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_64(part2, bucket, offset + 4, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_64(part3, bucket, offset + 8, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_64(part4, bucket, offset + 12, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_64(part5, bucket, offset + 16, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_64(part6, bucket, offset + 20, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_64(part7, bucket, offset + 24, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_64(part8, bucket, offset + 28, reason, ptr, confBase, a, control, last_match);
}
return HWLM_SUCCESS;
}
#define confirm_fat_teddy_512_f confirm_fat_teddy_64_512
#else // 32-64
hwlm_error_t confirm_fat_teddy_32_512(m512 var, u8 bucket, u8 offset,
CautionReason reason, const u8 *ptr,
const struct FDR_Runtime_Args *a,
const u32* confBase, hwlm_group_t *control,
u32 *last_match) {
if (unlikely(diff512(var, ones512()))) {
m512 msk_interleave = load512(p_mask_interleave);
m512 r = vpermb512(msk_interleave, var);
m128 r0 = extract128from512(r, 0);
m128 r1 = extract128from512(r, 1);
m128 r2 = extract128from512(r, 2);
m128 r3 = extract128from512(r, 3);
u32 part1 = movd(r0);
u32 part2 = extract32from128(r0, 1);
u32 part3 = extract32from128(r0, 2);
u32 part4 = extract32from128(r0, 3);
u32 part5 = movd(r1);
u32 part6 = extract32from128(r1, 1);
u32 part7 = extract32from128(r1, 2);
u32 part8 = extract32from128(r1, 3);
u32 part9 = movd(r2);
u32 part10 = extract32from128(r2, 1);
u32 part11 = extract32from128(r2, 2);
u32 part12 = extract32from128(r2, 3);
u32 part13 = movd(r3);
u32 part14 = extract32from128(r3, 1);
u32 part15 = extract32from128(r3, 2);
u32 part16 = extract32from128(r3, 3);
CONF_FAT_CHUNK_32(part1, bucket, offset, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part2, bucket, offset + 2, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part3, bucket, offset + 4, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part4, bucket, offset + 6, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part5, bucket, offset + 8, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part6, bucket, offset + 10, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part7, bucket, offset + 12, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part8, bucket, offset + 14, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part9, bucket, offset + 16, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part10, bucket, offset + 18, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part11, bucket, offset + 20, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part12, bucket, offset + 22, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part13, bucket, offset + 24, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part14, bucket, offset + 26, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part15, bucket, offset + 28, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part16, bucket, offset + 30, reason, ptr, confBase, a, control, last_match);
}
return HWLM_SUCCESS;
}
#define confirm_fat_teddy_512_f confirm_fat_teddy_32_512
#endif // 32/64
#define CONFIRM_FAT_TEDDY_512(...) if(confirm_fat_teddy_512_f(__VA_ARGS__, a, confBase, &control, &last_match) == HWLM_TERMINATED)return HWLM_TERMINATED;
#define TEDDY_VBMI_SL1_MASK 0xfffffffffffffffeULL
#define TEDDY_VBMI_SL2_MASK 0xfffffffffffffffcULL
#define TEDDY_VBMI_SL3_MASK 0xfffffffffffffff8ULL
#define FAT_TEDDY_VBMI_SL1_MASK 0xfffffffefffffffeULL
#define FAT_TEDDY_VBMI_SL2_MASK 0xfffffffcfffffffcULL
#define FAT_TEDDY_VBMI_SL3_MASK 0xfffffff8fffffff8ULL
#define FAT_TEDDY_VBMI_SL1_POS 15
#define FAT_TEDDY_VBMI_SL2_POS 14
#define FAT_TEDDY_VBMI_SL3_POS 13
#define FAT_TEDDY_VBMI_CONF_MASK_HEAD (0xffffffffULL >> n_sh)
#define FAT_TEDDY_VBMI_CONF_MASK_FULL ((0xffffffffULL << n_sh) & 0xffffffffULL)
#define FAT_TEDDY_VBMI_CONF_MASK_VAR(n) (0xffffffffULL >> (32 - n) << overlap)
#define FAT_TEDDY_VBMI_LOAD_MASK_PATCH (0xffffffffULL >> (32 - n_sh))
template<int NMSK>
static really_inline
m512 prep_conf_fat_teddy_512vbmi_templ(const m512 *lo_mask, const m512 *dup_mask,
const m512 *sl_msk, const m512 val) {
m512 lo = and512(val, *lo_mask);
m512 hi = and512(rshift64_m512(val, 4), *lo_mask);
m512 shuf_or_b0 = or512(pshufb_m512(dup_mask[0], lo),
pshufb_m512(dup_mask[1], hi));
if constexpr (NMSK == 1) return shuf_or_b0;
m512 shuf_or_b1 = or512(pshufb_m512(dup_mask[2], lo),
pshufb_m512(dup_mask[3], hi));
m512 sl1 = maskz_vpermb512(FAT_TEDDY_VBMI_SL1_MASK, sl_msk[0], shuf_or_b1);
if constexpr (NMSK == 2) return (or512(sl1, shuf_or_b0));
m512 shuf_or_b2 = or512(pshufb_m512(dup_mask[4], lo),
pshufb_m512(dup_mask[5], hi));
m512 sl2 = maskz_vpermb512(FAT_TEDDY_VBMI_SL2_MASK, sl_msk[1], shuf_or_b2);
if constexpr (NMSK == 3) return (or512(sl2, or512(sl1, shuf_or_b0)));
m512 shuf_or_b3 = or512(pshufb_m512(dup_mask[6], lo),
pshufb_m512(dup_mask[7], hi));
m512 sl3 = maskz_vpermb512(FAT_TEDDY_VBMI_SL3_MASK, sl_msk[2], shuf_or_b3);
return (or512(sl3, or512(sl2, or512(sl1, shuf_or_b0))));
}
#define TEDDY_VBMI_SL1_POS 15
#define TEDDY_VBMI_SL2_POS 14
#define TEDDY_VBMI_SL3_POS 13
#define TEDDY_VBMI_CONF_MASK_HEAD (0xffffffffffffffffULL >> n_sh)
#define TEDDY_VBMI_CONF_MASK_FULL (0xffffffffffffffffULL << n_sh)
#define TEDDY_VBMI_CONF_MASK_VAR(n) (0xffffffffffffffffULL >> (64 - n) << overlap)
#define TEDDY_VBMI_LOAD_MASK_PATCH (0xffffffffffffffffULL >> (64 - n_sh))
template<int NMSK>
hwlm_error_t fdr_exec_fat_teddy_512vbmi_templ(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
const u8 *buf_end = a->buf + a->len;
const u8 *ptr = a->buf + a->start_offset;
u32 floodBackoff = FLOOD_BACKOFF_START;
const u8 *tryFloodDetect = a->firstFloodDetect;
u32 last_match = ones_u32;
const struct Teddy *teddy = (const struct Teddy *)fdr;
const size_t iterBytes = 32;
u32 n_sh = NMSK - 1;
const size_t loopBytes = 32 - n_sh;
DEBUG_PRINTF("params: buf %p len %zu start_offset %zu\n",
a->buf, a->len, a->start_offset);
const m512 *dup_mask = getDupMaskBase(teddy, NMSK);
m512 lo_mask = set1_64x8(0xf);
m512 sl_msk[NMSK - 1];
if constexpr (NMSK > 1){
sl_msk[0] = loadu512(p_sh_mask_arr + FAT_TEDDY_VBMI_SL1_POS);
}
if constexpr (NMSK > 2){
sl_msk[1] = loadu512(p_sh_mask_arr + FAT_TEDDY_VBMI_SL2_POS);
}
if constexpr (NMSK > 3){
sl_msk[2] = loadu512(p_sh_mask_arr + FAT_TEDDY_VBMI_SL3_POS);
}
const u32 *confBase = getConfBase(teddy);
u64a k = FAT_TEDDY_VBMI_CONF_MASK_FULL;
m512 p_mask = set_mask_m512(~((k << 32) | k));
u32 overlap = 0;
u64a patch = 0;
if (likely(ptr + loopBytes <= buf_end)) {
u64a k0 = FAT_TEDDY_VBMI_CONF_MASK_HEAD;
m512 p_mask0 = set_mask_m512(~((k0 << 32) | k0));
m512 r_0 = prep_conf_fat_teddy_512vbmi_templ<NMSK>(&lo_mask, dup_mask, sl_msk, set2x256(loadu256(ptr)));
r_0 = or512(r_0, p_mask0);
CONFIRM_FAT_TEDDY_512(r_0, 16, 0, VECTORING, ptr);
ptr += loopBytes;
overlap = n_sh;
patch = FAT_TEDDY_VBMI_LOAD_MASK_PATCH;
}
for (; ptr + loopBytes <= buf_end; ptr += loopBytes) {
CHECK_FLOOD;
m512 r_0 = prep_conf_fat_teddy_512vbmi_templ<NMSK>(&lo_mask, dup_mask, sl_msk, set2x256(loadu256(ptr - n_sh)));
r_0 = or512(r_0, p_mask);
CONFIRM_FAT_TEDDY_512(r_0, 16, 0, NOT_CAUTIOUS, ptr - n_sh);
}
assert(ptr + loopBytes > buf_end);
if (ptr < buf_end) {
u32 left = (u32)(buf_end - ptr);
u64a k1 = FAT_TEDDY_VBMI_CONF_MASK_VAR(left);
m512 p_mask1 = set_mask_m512(~((k1 << 32) | k1));
m512 val_0 = set2x256(loadu_maskz_m256(k1 | patch, ptr - overlap));
m512 r_0 = prep_conf_fat_teddy_512vbmi_templ<NMSK>(&lo_mask, dup_mask, sl_msk, val_0);
r_0 = or512(r_0, p_mask1);
CONFIRM_FAT_TEDDY_512(r_0, 16, 0, VECTORING, ptr - overlap);
}
return HWLM_SUCCESS;
}
#define FDR_EXEC_FAT_TEDDY_FN fdr_exec_fat_teddy_512vbmi_templ
#elif defined(HAVE_AVX2) // not HAVE_AVX512 but HAVE_AVX2 reinforced teddy
#ifdef ARCH_64_BIT
extern "C" {
hwlm_error_t confirm_fat_teddy_64_256(m256 var, u8 bucket, u8 offset,
CautionReason reason, const u8 *ptr,
const struct FDR_Runtime_Args *a,
const u32* confBase, hwlm_group_t *control,
u32 *last_match) {
if (unlikely(diff256(var, ones256()))) {
m256 swap = swap128in256(var);
m256 r = interleave256lo(var, swap);
u64a part1 = extractlow64from256(r);
u64a part2 = extract64from256(r, 1);
r = interleave256hi(var, swap);
u64a part3 = extractlow64from256(r);
u64a part4 = extract64from256(r, 1);
CONF_FAT_CHUNK_64(part1, bucket, offset, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_64(part2, bucket, offset + 4, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_64(part3, bucket, offset + 8, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_64(part4, bucket, offset + 12, reason, ptr, confBase, a, control, last_match);
}
return HWLM_SUCCESS;
}
} // extern C
#define confirm_fat_teddy_256_f confirm_fat_teddy_64_256
#else
extern "C" {
hwlm_error_t confirm_fat_teddy_32_256(m256 var, u8 bucket, u8 offset,
CautionReason reason, const u8 *ptr,
const struct FDR_Runtime_Args *a,
const u32* confBase, hwlm_group_t *control,
u32 *last_match) {
if (unlikely(diff256(var, ones256()))) {
m256 swap = swap128in256(var);
m256 r = interleave256lo(var, swap);
u32 part1 = extractlow32from256(r);
u32 part2 = extract32from256(r, 1);
u32 part3 = extract32from256(r, 2);
u32 part4 = extract32from256(r, 3);
r = interleave256hi(var, swap);
u32 part5 = extractlow32from256(r);
u32 part6 = extract32from256(r, 1);
u32 part7 = extract32from256(r, 2);
u32 part8 = extract32from256(r, 3);
CONF_FAT_CHUNK_32(part1, bucket, offset, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part2, bucket, offset + 2, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part3, bucket, offset + 4, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part4, bucket, offset + 6, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part5, bucket, offset + 8, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part6, bucket, offset + 10, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part7, bucket, offset + 12, reason, ptr, confBase, a, control, last_match);
CONF_FAT_CHUNK_32(part8, bucket, offset + 14, reason, ptr, confBase, a, control, last_match);
}
return HWLM_SUCCESS;
}
} // extern C
#define confirm_fat_teddy_256_f confirm_fat_teddy_32_256
#endif
#define CONFIRM_FAT_TEDDY_256(...) if(confirm_fat_teddy_256_f(__VA_ARGS__, a, confBase, &control, &last_match) == HWLM_TERMINATED)return HWLM_TERMINATED;
static really_inline
const m256 *getMaskBase_fat(const struct Teddy *teddy) {
return (const m256 *)((const u8 *)teddy + ROUNDUP_CL(sizeof(struct Teddy)));
}
static really_inline
m256 vectoredLoad2x128(m256 *p_mask, const u8 *ptr, const size_t start_offset,
const u8 *lo, const u8 *hi,
const u8 *buf_history, size_t len_history,
const u32 nMasks) {
m128 p_mask128;
m256 ret = set1_2x128(vectoredLoad128(&p_mask128, ptr, start_offset, lo, hi,
buf_history, len_history, nMasks));
*p_mask = set1_2x128(p_mask128);
return ret;
}
template<int NMSK>
static really_inline
m256 prep_conf_fat_teddy_256_templ(const m256 *maskBase, m256 val,
m256* old_1, m256* old_2, m256* old_3){
m256 mask = set1_32x8(0xf);
m256 lo = and256(val, mask);
m256 hi = and256(rshift64_m256(val, 4), mask);
m256 r = or256(pshufb_m256(maskBase[0 * 2], lo),
pshufb_m256(maskBase[0 * 2 + 1], hi));
if constexpr (NMSK == 1) return r;
m256 res_1 = or256(pshufb_m256(maskBase[(NMSK-1) * 2], lo),
pshufb_m256(maskBase[(NMSK-1) * 2 + 1], hi));
m256 res_shifted_1 = vpalignr(res_1, *old_1, 16 - (NMSK-1));
*old_1 = res_1;
r = or256(r, res_shifted_1);
if constexpr (NMSK == 2) return r;
m256 res_2 = or256(pshufb_m256(maskBase[(NMSK-1) * 2], lo),
pshufb_m256(maskBase[(NMSK-1) * 2 + 1], hi));
m256 res_shifted_2 = vpalignr(res_2, *old_2, 16 - (NMSK-1));
*old_2 = res_2;
r = or256(r, res_shifted_2);
if constexpr (NMSK == 3) return r;
m256 res_3 = or256(pshufb_m256(maskBase[(NMSK-1) * 2], lo),
pshufb_m256(maskBase[(NMSK-1) * 2 + 1], hi));
m256 res_shifted_3 = vpalignr(res_3, *old_3, 16 - (NMSK-1));
*old_3 = res_3;
return or256(r, res_shifted_3);
}
template<int NMSK>
hwlm_error_t fdr_exec_fat_teddy_256_templ(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
const u8 *buf_end = a->buf + a->len;
const u8 *ptr = a->buf + a->start_offset;
u32 floodBackoff = FLOOD_BACKOFF_START;
const u8 *tryFloodDetect = a->firstFloodDetect;
u32 last_match = ones_u32;
const struct Teddy *teddy = (const struct Teddy *)fdr;
const size_t iterBytes = 32;
DEBUG_PRINTF("params: buf %p len %zu start_offset %zu\n",
a->buf, a->len, a->start_offset);
const m256 *maskBase = getMaskBase_fat(teddy);
const u32 *confBase = getConfBase(teddy);
m256 res_old_1 = zeroes256();
m256 res_old_2 = zeroes256();
m256 res_old_3 = zeroes256();
const u8 *mainStart = ROUNDUP_PTR(ptr, 16);
DEBUG_PRINTF("derive: ptr: %p mainstart %p\n", ptr, mainStart);
if (ptr < mainStart) {
ptr = mainStart - 16;
m256 p_mask;
m256 val_0 = vectoredLoad2x128(&p_mask, ptr, a->start_offset,
a->buf, buf_end,
a->buf_history, a->len_history,
NMSK);
m256 r_0 = prep_conf_fat_teddy_256_templ<NMSK>(maskBase, val_0, &res_old_1, &res_old_2, &res_old_3);
r_0 = or256(r_0, p_mask);
CONFIRM_FAT_TEDDY_256(r_0, 16, 0, VECTORING, ptr);
ptr += 16;
}
if (ptr + 16 <= buf_end) {
m256 r_0 = prep_conf_fat_teddy_256_templ<NMSK>(maskBase, load2x128(ptr), &res_old_1, &res_old_2, &res_old_3);
CONFIRM_FAT_TEDDY_256(r_0, 16, 0, VECTORING, ptr);
ptr += 16;
}
for ( ; ptr + iterBytes <= buf_end; ptr += iterBytes) {
__builtin_prefetch(ptr + (iterBytes * 4));
CHECK_FLOOD;
m256 r_0 = prep_conf_fat_teddy_256_templ<NMSK>(maskBase, load2x128(ptr), &res_old_1, &res_old_2, &res_old_3);
CONFIRM_FAT_TEDDY_256(r_0, 16, 0, NOT_CAUTIOUS, ptr);
m256 r_1 = prep_conf_fat_teddy_256_templ<NMSK>(maskBase, load2x128(ptr + 16), &res_old_1, &res_old_2, &res_old_3);
CONFIRM_FAT_TEDDY_256(r_1, 16, 16, NOT_CAUTIOUS, ptr);
}
if (ptr + 16 <= buf_end) {
m256 r_0 = prep_conf_fat_teddy_256_templ<NMSK>(maskBase, load2x128(ptr), &res_old_1, &res_old_2, &res_old_3);
CONFIRM_FAT_TEDDY_256(r_0, 16, 0, NOT_CAUTIOUS, ptr);
ptr += 16;
}
assert(ptr + 16 > buf_end);
if (ptr < buf_end) {
m256 p_mask;
m256 val_0 = vectoredLoad2x128(&p_mask, ptr, 0, ptr, buf_end,
a->buf_history, a->len_history,
NMSK);
m256 r_0 = prep_conf_fat_teddy_256_templ<NMSK>(maskBase, val_0, &res_old_1, &res_old_2, &res_old_3);
r_0 = or256(r_0, p_mask);
CONFIRM_FAT_TEDDY_256(r_0, 16, 0, VECTORING, ptr);
}
return HWLM_SUCCESS;
}
// this check is because it is possible to build with both AVX512VBMI and AVX2 defined,
// to replicate the behaviour of the original flow of control we give preference
// to the former. If we're building for both then this will be compiled multiple times
// with the desired variant defined by itself.
#ifndef FDR_EXEC_FAT_TEDDY_FN
#define FDR_EXEC_FAT_TEDDY_FN fdr_exec_fat_teddy_256_templ
#endif
#endif // HAVE_AVX2 for fat teddy
/* we only have fat teddy in these two modes */
// #if (defined(HAVE_AVX2) || defined(HAVE_AVX512VBMI)) && defined(FDR_EXEC_FAT_TEDDY_FN)
// #if defined(FDR_EXEC_FAT_TEDDY_FN)
extern "C" {
hwlm_error_t fdr_exec_fat_teddy_msks1(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_FAT_TEDDY_FN<1>(fdr, a, control);
}
hwlm_error_t fdr_exec_fat_teddy_msks1_pck(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_FAT_TEDDY_FN<1>(fdr, a, control);
}
hwlm_error_t fdr_exec_fat_teddy_msks2(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_FAT_TEDDY_FN<2>(fdr, a, control);
}
hwlm_error_t fdr_exec_fat_teddy_msks2_pck(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_FAT_TEDDY_FN<2>(fdr, a, control);
}
hwlm_error_t fdr_exec_fat_teddy_msks3(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_FAT_TEDDY_FN<3>(fdr, a, control);
}
hwlm_error_t fdr_exec_fat_teddy_msks3_pck(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_FAT_TEDDY_FN<3>(fdr, a, control);
}
hwlm_error_t fdr_exec_fat_teddy_msks4(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_FAT_TEDDY_FN<4>(fdr, a, control);
}
hwlm_error_t fdr_exec_fat_teddy_msks4_pck(const struct FDR *fdr,
const struct FDR_Runtime_Args *a,
hwlm_group_t control) {
return FDR_EXEC_FAT_TEDDY_FN<4>(fdr, a, control);
}
} // extern c
#endif // HAVE_AVX2 from the beginning

86
src/fdr/teddy_runtime_common.h
View File

@ -1,5 +1,6 @@
/* /*
* Copyright (c) 2016-2020, Intel Corporation * Copyright (c) 2016-2020, Intel Corporation
* Copyright (c) 2024, VectorCamp PC
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met: * modification, are permitted provided that the following conditions are met:
@ -40,10 +41,6 @@
#include "util/simd_utils.h" #include "util/simd_utils.h"
#include "util/uniform_ops.h" #include "util/uniform_ops.h"
extern const u8 ALIGN_DIRECTIVE p_mask_arr[17][32];
#if defined(HAVE_AVX2)
extern const u8 ALIGN_AVX_DIRECTIVE p_mask_arr256[33][64];
#endif
#if defined(HAVE_AVX512VBMI) #if defined(HAVE_AVX512VBMI)
static const u8 ALIGN_DIRECTIVE p_sh_mask_arr[80] = { static const u8 ALIGN_DIRECTIVE p_sh_mask_arr[80] = {
@ -142,6 +139,37 @@ void copyRuntBlock128(u8 *dst, const u8 *src, size_t len) {
// |----------|-------|----------------|............| // |----------|-------|----------------|............|
// 0 start start+offset end(<=16) // 0 start start+offset end(<=16)
// p_mask ffff.....ffffff..ff0000...........00ffff.......... // p_mask ffff.....ffffff..ff0000...........00ffff..........
// replace the p_mask_arr table.
// m is the length of the zone of bytes==0 , n is
// the offset where that zone begins. more specifically, there are
// 16-n bytes of 1's before the zone begins.
// m,n 4,7 - 4 bytes of 0s, and 16-7 bytes of 1's before that.
// 00 00 00 00 ff..ff
// ff ff ff ff ff ff ff ff 00 00 00 00 ff..ff
// m,n 15,15 - 15 bytes of 0s , f's high, but also with 16-15=1 byte of 1s
// in the beginning - which push the ff at the end off the high end , leaving
// ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
// m,n 15,16 - 15 bytes of 0s, ff high , with 16-16 = 0 ones on the low end
// before that, so,
// 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff
// so to get the one part, with the f's high, we start out with 1's and
// shift them up (right) by m+n.
// now to fill in any ones that belong on the low end we have to take
// some 1's and shift them down. the ones zone there needs to be 16-n long,
// meaning shifted down by 16-(16-n) , or of course just n.
// then we should be able to or these together.
static really_inline
m128 p_mask_gen(u8 m, u8 n){
m128 a = ones128();
m128 b = ones128();
m%=17; n%=17;
m+=(16-n); m%=17;
a = rshiftbyte_m128(a, n);
b = lshiftbyte_m128(b, m);
return or128(a, b);
}
static really_inline static really_inline
m128 vectoredLoad128(m128 *p_mask, const u8 *ptr, const size_t start_offset, m128 vectoredLoad128(m128 *p_mask, const u8 *ptr, const size_t start_offset,
const u8 *lo, const u8 *hi, const u8 *lo, const u8 *hi,
@ -161,13 +189,11 @@ m128 vectoredLoad128(m128 *p_mask, const u8 *ptr, const size_t start_offset,
uintptr_t avail = (uintptr_t)(hi - ptr); uintptr_t avail = (uintptr_t)(hi - ptr);
if (avail >= 16) { if (avail >= 16) {
assert(start_offset - start <= 16); assert(start_offset - start <= 16);
*p_mask = loadu128(p_mask_arr[16 - start_offset + start] *p_mask = p_mask_gen(16 - start_offset + start, 16 - start_offset + start);
+ 16 - start_offset + start);
return loadu128(ptr); return loadu128(ptr);
} }
assert(start_offset - start <= avail); assert(start_offset - start <= avail);
*p_mask = loadu128(p_mask_arr[avail - start_offset + start] *p_mask = p_mask_gen(avail - start_offset + start, 16 - start_offset + start);
+ 16 - start_offset + start);
copy_start = 0; copy_start = 0;
copy_len = avail; copy_len = avail;
} else { // start zone } else { // start zone
@ -180,8 +206,7 @@ m128 vectoredLoad128(m128 *p_mask, const u8 *ptr, const size_t start_offset,
} }
uintptr_t end = MIN(16, (uintptr_t)(hi - ptr)); uintptr_t end = MIN(16, (uintptr_t)(hi - ptr));
assert(start + start_offset <= end); assert(start + start_offset <= end);
*p_mask = loadu128(p_mask_arr[end - start - start_offset] *p_mask = p_mask_gen(end - start - start_offset, 16 - start - start_offset);
+ 16 - start - start_offset);
copy_start = start; copy_start = start;
copy_len = end - start; copy_len = end - start;
} }
@ -270,6 +295,20 @@ void copyRuntBlock256(u8 *dst, const u8 *src, size_t len) {
// |----------|-------|----------------|............| // |----------|-------|----------------|............|
// 0 start start+offset end(<=32) // 0 start start+offset end(<=32)
// p_mask ffff.....ffffff..ff0000...........00ffff.......... // p_mask ffff.....ffffff..ff0000...........00ffff..........
// like the pmask gen above this replaces the large array.
static really_inline
m256 fat_pmask_gen(u8 m, u8 n){
m256 a=ones256();
m256 b=ones256();
m%=33; n%=33;
m+=(32-n); m%=33;
a = rshift_byte_m256(a, m);
b = lshift_byte_m256(b, n);
return or256(a, b);
}
static really_inline static really_inline
m256 vectoredLoad256(m256 *p_mask, const u8 *ptr, const size_t start_offset, m256 vectoredLoad256(m256 *p_mask, const u8 *ptr, const size_t start_offset,
const u8 *lo, const u8 *hi, const u8 *lo, const u8 *hi,
@ -289,13 +328,11 @@ m256 vectoredLoad256(m256 *p_mask, const u8 *ptr, const size_t start_offset,
uintptr_t avail = (uintptr_t)(hi - ptr); uintptr_t avail = (uintptr_t)(hi - ptr);
if (avail >= 32) { if (avail >= 32) {
assert(start_offset - start <= 32); assert(start_offset - start <= 32);
*p_mask = loadu256(p_mask_arr256[32 - start_offset + start] *p_mask = fat_pmask_gen(32 - start_offset + start, 32 - start_offset + start);
+ 32 - start_offset + start);
return loadu256(ptr); return loadu256(ptr);
} }
assert(start_offset - start <= avail); assert(start_offset - start <= avail);
*p_mask = loadu256(p_mask_arr256[avail - start_offset + start] *p_mask = fat_pmask_gen(avail - start_offset + start, 32 - start_offset + start);
+ 32 - start_offset + start);
copy_start = 0; copy_start = 0;
copy_len = avail; copy_len = avail;
} else { //start zone } else { //start zone
@ -308,8 +345,7 @@ m256 vectoredLoad256(m256 *p_mask, const u8 *ptr, const size_t start_offset,
} }
uintptr_t end = MIN(32, (uintptr_t)(hi - ptr)); uintptr_t end = MIN(32, (uintptr_t)(hi - ptr));
assert(start + start_offset <= end); assert(start + start_offset <= end);
*p_mask = loadu256(p_mask_arr256[end - start - start_offset] *p_mask = fat_pmask_gen(end - start - start_offset, 32 - start - start_offset);
+ 32 - start - start_offset);
copy_start = start; copy_start = start;
copy_len = end - start; copy_len = end - start;
} }
@ -428,8 +464,13 @@ void do_confWithBit_teddy(TEDDY_CONF_TYPE *conf, u8 bucket, u8 offset,
if (!cf) { if (!cf) {
continue; continue;
} }
#ifdef __cplusplus
const struct FDRConfirm *fdrc = reinterpret_cast<const struct FDRConfirm *>
(reinterpret_cast<const u8 *>(confBase) + cf);
#else
const struct FDRConfirm *fdrc = (const struct FDRConfirm *) const struct FDRConfirm *fdrc = (const struct FDRConfirm *)
((const u8 *)confBase + cf); ((const u8 *)confBase + cf);
#endif
if (!(fdrc->groups & *control)) { if (!(fdrc->groups & *control)) {
continue; continue;
} }
@ -442,18 +483,31 @@ void do_confWithBit_teddy(TEDDY_CONF_TYPE *conf, u8 bucket, u8 offset,
static really_inline static really_inline
const m128 *getMaskBase(const struct Teddy *teddy) { const m128 *getMaskBase(const struct Teddy *teddy) {
#ifdef __cplusplus
return reinterpret_cast<const m128 *>(reinterpret_cast<const u8 *>(teddy) + ROUNDUP_CL(sizeof(struct Teddy)));
#else
return (const m128 *)((const u8 *)teddy + ROUNDUP_CL(sizeof(struct Teddy))); return (const m128 *)((const u8 *)teddy + ROUNDUP_CL(sizeof(struct Teddy)));
#endif
} }
static really_inline static really_inline
const u64a *getReinforcedMaskBase(const struct Teddy *teddy, u8 numMask) { const u64a *getReinforcedMaskBase(const struct Teddy *teddy, u8 numMask) {
#ifdef __cplusplus
return reinterpret_cast<const u64a *>(reinterpret_cast<const u8 *>(getMaskBase(teddy))
+ ROUNDUP_CL(2 * numMask * sizeof(m128)));
#else
return (const u64a *)((const u8 *)getMaskBase(teddy) return (const u64a *)((const u8 *)getMaskBase(teddy)
+ ROUNDUP_CL(2 * numMask * sizeof(m128))); + ROUNDUP_CL(2 * numMask * sizeof(m128)));
#endif
} }
static really_inline static really_inline
const u32 *getConfBase(const struct Teddy *teddy) { const u32 *getConfBase(const struct Teddy *teddy) {
#ifdef __cplusplus
return reinterpret_cast<const u32 *>(reinterpret_cast<const u8 *>(teddy) + teddy->confOffset);
#else
return (const u32 *)((const u8 *)teddy + teddy->confOffset); return (const u32 *)((const u8 *)teddy + teddy->confOffset);
#endif
} }
#endif /* TEDDY_RUNTIME_COMMON_H_ */ #endif /* TEDDY_RUNTIME_COMMON_H_ */

8
src/util/arch/arm/simd_utils.h
View File

@ -352,16 +352,20 @@ static really_really_inline
m128 rshiftbyte_m128(m128 a, unsigned b) { m128 rshiftbyte_m128(m128 a, unsigned b) {
if (b == 0) { if (b == 0) {
return a; return a;
} else if (b > 15) {
return zeroes128();
} }
return palignr(zeroes128(), a, b); else return palignr(zeroes128(), a, b);
} }
static really_really_inline static really_really_inline
m128 lshiftbyte_m128(m128 a, unsigned b) { m128 lshiftbyte_m128(m128 a, unsigned b) {
if (b == 0) { if (b == 0) {
return a; return a;
} else if (b > 15) {
return zeroes128();
} }
return palignr(a, zeroes128(), 16 - b); else return palignr(a, zeroes128(), 16 - b);
} }
static really_inline static really_inline

92
src/util/arch/x86/simd_utils.h
View File

@ -42,6 +42,7 @@
#include <string.h> // for memcpy #include <string.h> // for memcpy
#define ZEROES_8 0, 0, 0, 0, 0, 0, 0, 0 #define ZEROES_8 0, 0, 0, 0, 0, 0, 0, 0
#define ZEROES_31 ZEROES_8, ZEROES_8, ZEROES_8, 0, 0, 0, 0, 0, 0, 0 #define ZEROES_31 ZEROES_8, ZEROES_8, ZEROES_8, 0, 0, 0, 0, 0, 0, 0
#define ZEROES_32 ZEROES_8, ZEROES_8, ZEROES_8, ZEROES_8 #define ZEROES_32 ZEROES_8, ZEROES_8, ZEROES_8, ZEROES_8
@ -178,13 +179,11 @@ m128 load_m128_from_u64a(const u64a *p) {
#define CASE_RSHIFT_VECTOR(a, count) case count: return _mm_srli_si128((m128)(a), (count)); break; #define CASE_RSHIFT_VECTOR(a, count) case count: return _mm_srli_si128((m128)(a), (count)); break;
// we encounter cases where an argument slips past __builtin_constant_p but
// still fails to meet the (stricter) criteria demanded by the underlying
// intrinsic. in those cases we want to explicitly avoid the optimization.
static really_inline static really_inline
m128 rshiftbyte_m128(const m128 a, int count_immed) { m128 rshiftbyte_m128_nim(const m128 a, int count_immed) {
#if defined(HAVE__BUILTIN_CONSTANT_P) && !defined(VS_SIMDE_BACKEND)
if (__builtin_constant_p(count_immed)) {
return _mm_srli_si128(a, count_immed);
}
#endif
switch (count_immed) { switch (count_immed) {
case 0: return a; break; case 0: return a; break;
CASE_RSHIFT_VECTOR(a, 1); CASE_RSHIFT_VECTOR(a, 1);
@ -205,17 +204,26 @@ m128 rshiftbyte_m128(const m128 a, int count_immed) {
default: return zeroes128(); break; default: return zeroes128(); break;
} }
} }
static really_inline
m128 rshiftbyte_m128(const m128 a, int count_immed) {
#if defined(HAVE__BUILTIN_CONSTANT_P) && !defined(VS_SIMDE_BACKEND)
if (__builtin_constant_p(count_immed)) {
return _mm_srli_si128(a, count_immed);
}
#endif
return rshiftbyte_m128_nim(a, count_immed);
}
#undef CASE_RSHIFT_VECTOR #undef CASE_RSHIFT_VECTOR
#define CASE_LSHIFT_VECTOR(a, count) case count: return _mm_slli_si128((m128)(a), (count)); break; #define CASE_LSHIFT_VECTOR(a, count) case count: return _mm_slli_si128((m128)(a), (count)); break;
// we encounter cases where an argument slips past __builtin_constant_p but
// still fails to meet the (stricter) criteria demanded by the underlying
// intrinsic. in those cases we want to explicitly avoid the optimization.
static really_inline static really_inline
m128 lshiftbyte_m128(const m128 a, int count_immed) { m128 lshiftbyte_m128_nim(const m128 a, int count_immed) {
#if defined(HAVE__BUILTIN_CONSTANT_P) && !defined(VS_SIMDE_BACKEND)
if (__builtin_constant_p(count_immed)) {
return _mm_slli_si128(a, count_immed);
}
#endif
switch (count_immed) { switch (count_immed) {
case 0: return a; break; case 0: return a; break;
CASE_LSHIFT_VECTOR(a, 1); CASE_LSHIFT_VECTOR(a, 1);
@ -236,6 +244,16 @@ m128 lshiftbyte_m128(const m128 a, int count_immed) {
default: return zeroes128(); break; default: return zeroes128(); break;
} }
} }
static really_inline
m128 lshiftbyte_m128(const m128 a, int count_immed) {
#if defined(HAVE__BUILTIN_CONSTANT_P) && !defined(VS_SIMDE_BACKEND)
if (__builtin_constant_p(count_immed)) {
return _mm_slli_si128(a, count_immed);
}
#endif
return lshiftbyte_m128_nim(a, count_immed);
}
#undef CASE_LSHIFT_VECTOR #undef CASE_LSHIFT_VECTOR
#if defined(HAVE_SSE41) #if defined(HAVE_SSE41)
@ -500,6 +518,56 @@ static really_inline m256 ones256(void) {
return rv; return rv;
} }
// byte-granularity shifts of the whole 256 bits as a single chunk
static really_inline m256 lshift_byte_m256(m256 v, u8 n){
if(n==0)return v;
else {
union {
u8 c[32];
m128 val128[2];
m256 val256;
} u;
u.val256=v;
if(n < 16){
m128 c = lshiftbyte_m128_nim(u.val128[1], 16-n);
u.val128[1] = rshiftbyte_m128_nim(u.val128[1], n);
u.val128[0] = or128(c, rshiftbyte_m128_nim(u.val128[0], n));
return u.val256;
} else if(n==16){
u.val128[0] = u.val128[1]; u.val128[1]=zeroes128();
return u.val256;
} else if(n<32){
u.val128[0] = rshiftbyte_m128_nim(u.val128[0], n-16);
u.val128[1]=zeroes128();
return u.val256;
} else return zeroes256();
}
}
static really_inline m256 rshift_byte_m256(m256 v, u8 n){
if(n==0)return v;
else {
union {
m128 val128[2];
m256 val256;
} u;
u.val256=v;
if(n < 16){
m128 c = rshiftbyte_m128_nim(u.val128[0], 16-n);
u.val128[0] = lshiftbyte_m128_nim(u.val128[0], n);
u.val128[1] = or128(c, lshiftbyte_m128_nim(u.val128[1], n));
return u.val256;
} else if(n==16){
u.val128[1] = u.val128[0]; u.val128[0]=zeroes128();
return u.val256;
} else if(n<32){
u.val128[1] = lshiftbyte_m128_nim(u.val128[1], n-16);
u.val128[0]=zeroes128();
return u.val256;
} else return zeroes256();
}
}
static really_inline m256 add256(m256 a, m256 b) { static really_inline m256 add256(m256 a, m256 b) {
return _mm256_add_epi64(a, b); return _mm256_add_epi64(a, b);
} }