diff --git a/CMakeLists.txt b/CMakeLists.txt index 908b53fc..7ca7b994 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,7 +119,10 @@ endif() # Detect OS and if Fat Runtime is available include (${CMAKE_MODULE_PATH}/osdetection.cmake) -if (ARCH_IA32 OR ARCH_X86_64) +if(SIMDE_BACKEND) + include (${CMAKE_MODULE_PATH}/simde.cmake) + set(ARCH_FLAG march) +elseif (ARCH_IA32 OR ARCH_X86_64) include (${CMAKE_MODULE_PATH}/cflags-x86.cmake) set(ARCH_FLAG march) elseif (ARCH_ARM32 OR ARCH_AARCH64) @@ -128,10 +131,6 @@ elseif (ARCH_ARM32 OR ARCH_AARCH64) elseif (ARCH_PPC64EL) include (${CMAKE_MODULE_PATH}/cflags-ppc64le.cmake) set(ARCH_FLAG mcpu) -elseif(SIMDE_BACKEND) - include (${CMAKE_MODULE_PATH}/simde.cmake) - set(ARCH_FLAG march) -else() message(FATAL_ERROR "Unsupported platform") endif () @@ -243,8 +242,11 @@ set (hs_exec_common_SRCS src/util/arch/common/cpuid_flags.h src/util/multibit.c ) - -if (ARCH_IA32 OR ARCH_X86_64) +if (SIMDE_BACKEND) +set (hs_exec_common_SRCS + ${hs_exec_common_SRCS} + src/util/arch/simde/cpuid_flags.c) +elseif (ARCH_IA32 OR ARCH_X86_64) set (hs_exec_common_SRCS ${hs_exec_common_SRCS} src/util/arch/x86/cpuid_flags.c @@ -258,10 +260,6 @@ elseif (ARCH_PPC64EL) set (hs_exec_common_SRCS ${hs_exec_common_SRCS} src/util/arch/ppc64el/cpuid_flags.c) -elseif (SIMDE_BACKEND) -set (hs_exec_common_SRCS - ${hs_exec_common_SRCS} - src/util/arch/simde/cpuid_flags.c) endif () set (hs_exec_SRCS @@ -406,7 +404,12 @@ set (hs_exec_SRCS src/database.h ) -if (ARCH_IA32 OR ARCH_X86_64) +if (SIMDE_BACKEND) +set (hs_exec_SRCS + ${hs_exec_SRCS} + src/nfa/vermicelli_simd.cpp + src/util/supervector/arch/x86/impl.cpp) +elseif (ARCH_IA32 OR ARCH_X86_64) set (hs_exec_SRCS ${hs_exec_SRCS} src/nfa/vermicelli_simd.cpp @@ -420,11 +423,6 @@ set (hs_exec_SRCS ${hs_exec_SRCS} src/nfa/vermicelli_simd.cpp src/util/supervector/arch/ppc64el/impl.cpp) -elseif (SIMDE_BACKEND) -set (hs_exec_SRCS - ${hs_exec_SRCS} - src/nfa/vermicelli_simd.cpp - src/util/supervector/arch/simde/impl.cpp) endif() if (ARCH_IA32 OR ARCH_X86_64) diff --git a/cmake/archdetect.cmake b/cmake/archdetect.cmake index 387437eb..87c4c4e7 100644 --- a/cmake/archdetect.cmake +++ b/cmake/archdetect.cmake @@ -67,7 +67,10 @@ if (USE_CPU_NATIVE) message(STATUS "clang will tune for ${GNUCC_ARCH}, ${TUNE_FLAG}") endif() else() - if (ARCH_IA32 OR ARCH_X86_64) + if (SIMDE_BACKEND) + set(GNUCC_ARCH native) + set(TUNE_FLAG generic) + elseif (ARCH_IA32 OR ARCH_X86_64) set(GNUCC_ARCH native) set(TUNE_FLAG generic) elseif(ARCH_AARCH64) @@ -85,8 +88,8 @@ else() set(GNUCC_ARCH armv7a) set(TUNE_FLAG generic) elseif(ARCH_PPC64EL) - set(GNUCC_ARCH power9) - set(TUNE_FLAG power9) + set(GNUCC_ARCH power8) + set(TUNE_FLAG power8) else() set(GNUCC_ARCH native) set(TUNE_FLAG native) diff --git a/cmake/simde.cmake b/cmake/simde.cmake index b68c8e57..12c56c6d 100644 --- a/cmake/simde.cmake +++ b/cmake/simde.cmake @@ -1,5 +1,9 @@ -include_directories(${PROJECT_SOURCE_DIR}/simde/simde) +# include_directories(${PROJECT_SOURCE_DIR}/simde/simde) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSIMDE_BACKEND") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSIMDE_BACKEND") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DVS_SIMDE_BACKEND") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DVS_SIMDE_BACKEND") +if (SIMDE_NATIVE) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DVS_SIMDE_NATIVE -DSIMDE_ENABLE_OPENMP -fopenmp-simd") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DVS_SIMDE_NATIVE -DSIMDE_ENABLE_OPENMP -fopenmp-simd") +endif() diff --git a/src/hs_valid_platform.c b/src/hs_valid_platform.c index 067a05e6..74a8fc1e 100644 --- a/src/hs_valid_platform.c +++ b/src/hs_valid_platform.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2016-2017, Intel Corporation + * Copyright (c) 2020-2023, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -29,28 +30,30 @@ #include "config.h" #include "hs_common.h" #include "ue2common.h" +#if !defined(VS_SIMDE_BACKEND) #if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/arch/x86/cpuid_inline.h" #elif defined(ARCH_AARCH64) #include "util/arch/arm/cpuid_inline.h" #endif +#endif HS_PUBLIC_API hs_error_t HS_CDECL hs_valid_platform(void) { /* Hyperscan requires SSSE3, anything else is a bonus */ -#if defined(ARCH_IA32) || defined(ARCH_X86_64) +#if !defined(VS_SIMDE_BACKEND) && (defined(ARCH_IA32) || defined(ARCH_X86_64)) if (check_ssse3()) { return HS_SUCCESS; } else { return HS_ARCH_ERROR; } -#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) +#elif !defined(VS_SIMDE_BACKEND) && (defined(ARCH_ARM32) || defined(ARCH_AARCH64)) if (check_neon()) { return HS_SUCCESS; } else { return HS_ARCH_ERROR; } -#elif defined(ARCH_PPC64EL) || defined(SIMDE_BACKEND) +#elif defined(ARCH_PPC64EL) || defined(VS_SIMDE_BACKEND) return HS_SUCCESS; #endif } diff --git a/src/nfa/shufti_simd.hpp b/src/nfa/shufti_simd.hpp index 30df80bf..feeb54ab 100644 --- a/src/nfa/shufti_simd.hpp +++ b/src/nfa/shufti_simd.hpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation - * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2020-2023, VectorCamp PC * Copyright (c) 2021, Arm Limited * * Redistribution and use in source and binary forms, with or without @@ -52,13 +52,17 @@ template static really_inline SuperVector blockDoubleMask(SuperVector mask1_lo, SuperVector mask1_hi, SuperVector mask2_lo, SuperVector mask2_hi, SuperVector chars); -#if defined(ARCH_IA32) || defined(ARCH_X86_64) || defined(SIMDE_BACKEND) +#if defined(VS_SIMDE_BACKEND) #include "x86/shufti.hpp" -#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) +#else +#if defined(ARCH_IA32) || defined(ARCH_X86_64) +#include "x86/shufti.hpp" +#elif (defined(ARCH_ARM32) || defined(ARCH_AARCH64)) #include "arm/shufti.hpp" #elif defined(ARCH_PPC64EL) #include "ppc64el/shufti.hpp" #endif +#endif template static really_inline diff --git a/src/nfa/truffle_simd.hpp b/src/nfa/truffle_simd.hpp index 0214833c..c1028156 100644 --- a/src/nfa/truffle_simd.hpp +++ b/src/nfa/truffle_simd.hpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation - * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2020-2023, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -45,13 +45,17 @@ template static really_inline const SuperVector blockSingleMask(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector chars); -#if defined(ARCH_IA32) || defined(ARCH_X86_64) || defined(SIMDE_BACKEND) +#if defined(VS_SIMDE_BACKEND) +#include "x86/truffle.hpp" +#else +#if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "x86/truffle.hpp" #elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) #include "arm/truffle.hpp" #elif defined(ARCH_PPC64EL) #include "ppc64el/truffle.hpp" #endif +#endif template static really_inline diff --git a/src/nfa/vermicelli_simd.cpp b/src/nfa/vermicelli_simd.cpp index c5fbc39a..67ac1dac 100644 --- a/src/nfa/vermicelli_simd.cpp +++ b/src/nfa/vermicelli_simd.cpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2015-2020, Intel Corporation - * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2020-2023, VectorCamp PC * Copyright (c) 2021, Arm Limited * * Redistribution and use in source and binary forms, with or without @@ -71,13 +71,17 @@ const u8 *vermicelliDoubleMaskedBlock(SuperVector const data, SuperVector SuperVector const mask1, SuperVector const mask2, u8 const c1, u8 const c2, u8 const m1, u8 const m2, u8 const *buf, u16 const len); -#if defined(ARCH_IA32) || defined(ARCH_X86_64) || defined(SIMDE_BACKEND) +#if defined(VS_SIMDE_BACKEND) +#include "x86/vermicelli.hpp" +#else +#if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "x86/vermicelli.hpp" #elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) #include "arm/vermicelli.hpp" #elif defined(ARCH_PPC64EL) #include "ppc64el/vermicelli.hpp" #endif +#endif template static const u8 *vermicelliExecReal(SuperVector const chars, SuperVector const casemask, u8 const *buf, u8 const *buf_end) { diff --git a/src/util/arch/common/simd_utils.h b/src/util/arch/common/simd_utils.h index 2542f0f6..89190648 100644 --- a/src/util/arch/common/simd_utils.h +++ b/src/util/arch/common/simd_utils.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2015-2020, Intel Corporation - * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2020-2023, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,7 +41,7 @@ #include // for memcpy -#if !defined(HAVE_SIMD_128_BITS) && !defined(SIMDE_BACKEND) +#if !defined(HAVE_SIMD_128_BITS) && !defined(VS_SIMDE_BACKEND) #error "You need at least a 128-bit capable SIMD engine!" #endif // HAVE_SIMD_128_BITS @@ -88,7 +88,7 @@ static inline void print_m128_2x64(const char *label, m128 vec) { #define print_m128_2x64(label, vec) ; #endif -#if !defined(ARCH_IA32) && !defined(ARCH_X86_64) && !defined(SIMDE_BACKEND) +#if !defined(ARCH_IA32) && !defined(ARCH_X86_64) && !defined(VS_SIMDE_BACKEND) #define ZEROES_8 0, 0, 0, 0, 0, 0, 0, 0 #define ZEROES_31 ZEROES_8, ZEROES_8, ZEROES_8, 0, 0, 0, 0, 0, 0, 0 #define ZEROES_32 ZEROES_8, ZEROES_8, ZEROES_8, ZEROES_8 diff --git a/src/util/arch/simde/simd_utils.h b/src/util/arch/simde/simd_utils.h deleted file mode 100644 index b8e7d4a8..00000000 --- a/src/util/arch/simde/simd_utils.h +++ /dev/null @@ -1,388 +0,0 @@ -/* - * Copyright (c) 2015-2020, Intel Corporation - * Copyright (c) 2020-2021, VectorCamp PC - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/** \file - * \brief SIMD types and primitive operations. - */ - -#ifndef ARCH_SIMDE_SIMD_UTILS_H -#define ARCH_SIMDE_SIMD_UTILS_H - -#include "ue2common.h" -#include "util/simd_types.h" -#include "util/unaligned.h" -#include "util/intrinsics.h" - -#include // for memcpy - -#define ZEROES_8 0, 0, 0, 0, 0, 0, 0, 0 -#define ZEROES_31 ZEROES_8, ZEROES_8, ZEROES_8, 0, 0, 0, 0, 0, 0, 0 -#define ZEROES_32 ZEROES_8, ZEROES_8, ZEROES_8, ZEROES_8 - -/** \brief LUT for the mask1bit functions. */ -ALIGN_CL_DIRECTIVE static const u8 simd_onebit_masks[] = { - ZEROES_32, ZEROES_32, - ZEROES_31, 0x01, ZEROES_32, - ZEROES_31, 0x02, ZEROES_32, - ZEROES_31, 0x04, ZEROES_32, - ZEROES_31, 0x08, ZEROES_32, - ZEROES_31, 0x10, ZEROES_32, - ZEROES_31, 0x20, ZEROES_32, - ZEROES_31, 0x40, ZEROES_32, - ZEROES_31, 0x80, ZEROES_32, - ZEROES_32, ZEROES_32, -}; - -static really_inline m128 ones128(void) { - return (m128) _mm_set1_epi8(0xFF); -} - -static really_inline m128 zeroes128(void) { - return (m128) _mm_setzero_si128(); -} - -/** \brief Bitwise not for m128*/ -static really_inline m128 not128(m128 a) { - return (m128) _mm_xor_si128(a, ones128()); -} - -/** \brief Return 1 if a and b are different otherwise 0 */ -static really_inline int diff128(m128 a, m128 b) { - return (_mm_movemask_epi8(_mm_cmpeq_epi8(a, b)) ^ 0xffff); -} - -static really_inline int isnonzero128(m128 a) { - return !!diff128(a, zeroes128()); -} - -/** - * "Rich" version of diff128(). Takes two vectors a and b and returns a 4-bit - * mask indicating which 32-bit words contain differences. - */ -static really_inline u32 diffrich128(m128 a, m128 b) { - a = _mm_cmpeq_epi32(a, b); - return ~(_mm_movemask_ps(_mm_castsi128_ps(a))) & 0xf; -} - -/** - * "Rich" version of diff128(), 64-bit variant. Takes two vectors a and b and - * returns a 4-bit mask indicating which 64-bit words contain differences. - */ -static really_inline u32 diffrich64_128(m128 a, m128 b) { - a = _mm_cmpeq_epi64(a, b); - return ~(_mm_movemask_ps(_mm_castsi128_ps(a))) & 0x5; -} - -static really_really_inline -m128 add_2x64(m128 a, m128 b) { - return (m128) _mm_add_epi64(a, b); -} - -static really_really_inline -m128 sub_2x64(m128 a, m128 b) { - return (m128) _mm_sub_epi64(a, b); -} - -static really_really_inline -m128 lshift64_m128(m128 a, unsigned b) { - return _mm_slli_epi64(a, b); -} - -#define rshift64_m128(a, b) _mm_srli_epi64((a), (b)) -#define eq128(a, b) _mm_cmpeq_epi8((a), (b)) -#define eq64_m128(a, b) _mm_cmpeq_epi64((a), (b)) -#define movemask128(a) ((u32)_mm_movemask_epi8((a))) - -static really_inline m128 set1_16x8(u8 c) { - return _mm_set1_epi8(c); -} - -static really_inline m128 set1_4x32(u32 c) { - return _mm_set1_epi32(c); -} - -static really_inline m128 set1_2x64(u64a c) { - return _mm_set1_epi64x(c); -} - -static really_inline u32 movd(const m128 in) { - return _mm_cvtsi128_si32(in); -} - -static really_inline u64a movq(const m128 in) { - return _mm_cvtsi128_si64(in); -} - -/* another form of movq */ -static really_inline -m128 load_m128_from_u64a(const u64a *p) { - return _mm_set_epi64x(0LL, *p); -} - -#define CASE_RSHIFT_VECTOR(a, count) case count: return _mm_srli_si128((m128)(a), (count)); break; - -static really_inline -m128 rshiftbyte_m128(const m128 a, int count_immed) { - switch (count_immed) { - case 0: return a; break; - CASE_RSHIFT_VECTOR(a, 1); - CASE_RSHIFT_VECTOR(a, 2); - CASE_RSHIFT_VECTOR(a, 3); - CASE_RSHIFT_VECTOR(a, 4); - CASE_RSHIFT_VECTOR(a, 5); - CASE_RSHIFT_VECTOR(a, 6); - CASE_RSHIFT_VECTOR(a, 7); - CASE_RSHIFT_VECTOR(a, 8); - CASE_RSHIFT_VECTOR(a, 9); - CASE_RSHIFT_VECTOR(a, 10); - CASE_RSHIFT_VECTOR(a, 11); - CASE_RSHIFT_VECTOR(a, 12); - CASE_RSHIFT_VECTOR(a, 13); - CASE_RSHIFT_VECTOR(a, 14); - CASE_RSHIFT_VECTOR(a, 15); - default: return zeroes128(); break; - } -} -#undef CASE_RSHIFT_VECTOR - -#define CASE_LSHIFT_VECTOR(a, count) case count: return _mm_slli_si128((m128)(a), (count)); break; - -static really_inline -m128 lshiftbyte_m128(const m128 a, int count_immed) { - switch (count_immed) { - case 0: return a; break; - CASE_LSHIFT_VECTOR(a, 1); - CASE_LSHIFT_VECTOR(a, 2); - CASE_LSHIFT_VECTOR(a, 3); - CASE_LSHIFT_VECTOR(a, 4); - CASE_LSHIFT_VECTOR(a, 5); - CASE_LSHIFT_VECTOR(a, 6); - CASE_LSHIFT_VECTOR(a, 7); - CASE_LSHIFT_VECTOR(a, 8); - CASE_LSHIFT_VECTOR(a, 9); - CASE_LSHIFT_VECTOR(a, 10); - CASE_LSHIFT_VECTOR(a, 11); - CASE_LSHIFT_VECTOR(a, 12); - CASE_LSHIFT_VECTOR(a, 13); - CASE_LSHIFT_VECTOR(a, 14); - CASE_LSHIFT_VECTOR(a, 15); - default: return zeroes128(); break; - } -} -#undef CASE_LSHIFT_VECTOR - -#define extract32from128(a, imm) _mm_extract_epi32(a, imm) -#define extract64from128(a, imm) _mm_extract_epi64(a, imm) - -static really_inline m128 add128(m128 a, m128 b) { - return _mm_add_epi64(a, b); -} - -static really_inline m128 and128(m128 a, m128 b) { - return _mm_and_si128(a,b); -} - -static really_inline m128 xor128(m128 a, m128 b) { - return _mm_xor_si128(a,b); -} - -static really_inline m128 or128(m128 a, m128 b) { - return _mm_or_si128(a,b); -} - -static really_inline m128 andnot128(m128 a, m128 b) { - return _mm_andnot_si128(a, b); -} - -// aligned load -static really_inline m128 load128(const void *ptr) { - assert(ISALIGNED_N(ptr, alignof(m128))); - ptr = vectorscan_assume_aligned(ptr, 16); - return _mm_load_si128((const m128 *)ptr); -} - -// aligned store -static really_inline void store128(void *ptr, m128 a) { - assert(ISALIGNED_N(ptr, alignof(m128))); - ptr = vectorscan_assume_aligned(ptr, 16); - *(m128 *)ptr = a; -} - -// unaligned load -static really_inline m128 loadu128(const void *ptr) { - return _mm_loadu_si128((const m128 *)ptr); -} - -// unaligned store -static really_inline void storeu128(void *ptr, m128 a) { - _mm_storeu_si128 ((m128 *)ptr, a); -} - -// packed unaligned store of first N bytes -static really_inline -void storebytes128(void *ptr, m128 a, unsigned int n) { - assert(n <= sizeof(a)); - memcpy(ptr, &a, n); -} - -// packed unaligned load of first N bytes, pad with zero -static really_inline -m128 loadbytes128(const void *ptr, unsigned int n) { - m128 a = zeroes128(); - assert(n <= sizeof(a)); - memcpy(&a, ptr, n); - return a; -} - -static really_inline -m128 mask1bit128(unsigned int n) { - assert(n < sizeof(m128) * 8); - u32 mask_idx = ((n % 8) * 64) + 95; - mask_idx -= n / 8; - return loadu128(&simd_onebit_masks[mask_idx]); -} - -// switches on bit N in the given vector. -static really_inline -void setbit128(m128 *ptr, unsigned int n) { - *ptr = or128(mask1bit128(n), *ptr); -} - -// switches off bit N in the given vector. -static really_inline -void clearbit128(m128 *ptr, unsigned int n) { - *ptr = andnot128(mask1bit128(n), *ptr); -} - -// tests bit N in the given vector. -static really_inline -char testbit128(m128 val, unsigned int n) { - const m128 mask = mask1bit128(n); -#if defined(HAVE_SSE41) - return !_mm_testz_si128(mask, val); -#else - return isnonzero128(and128(mask, val)); -#endif -} - -// offset must be an immediate -#define palignr_imm(r, l, offset) _mm_alignr_epi8(r, l, offset) - -static really_inline -m128 pshufb_m128(m128 a, m128 b) { - return _mm_shuffle_epi8(a, b); -} - -#define CASE_ALIGN_VECTORS(a, b, offset) case offset: return palignr_imm((m128)(a), (m128)(b), (offset)); break; - -static really_really_inline -m128 palignr_sw(m128 r, m128 l, int offset) { - switch (offset) { - case 0: return l; break; - CASE_ALIGN_VECTORS(r, l, 1); - CASE_ALIGN_VECTORS(r, l, 2); - CASE_ALIGN_VECTORS(r, l, 3); - CASE_ALIGN_VECTORS(r, l, 4); - CASE_ALIGN_VECTORS(r, l, 5); - CASE_ALIGN_VECTORS(r, l, 6); - CASE_ALIGN_VECTORS(r, l, 7); - CASE_ALIGN_VECTORS(r, l, 8); - CASE_ALIGN_VECTORS(r, l, 9); - CASE_ALIGN_VECTORS(r, l, 10); - CASE_ALIGN_VECTORS(r, l, 11); - CASE_ALIGN_VECTORS(r, l, 12); - CASE_ALIGN_VECTORS(r, l, 13); - CASE_ALIGN_VECTORS(r, l, 14); - CASE_ALIGN_VECTORS(r, l, 15); - case 16: return r; break; - default: - return zeroes128(); - break; - } -} -#undef CASE_ALIGN_VECTORS - -static really_really_inline -m128 palignr(m128 r, m128 l, int offset) { -#if defined(HAVE__BUILTIN_CONSTANT_P) - if (__builtin_constant_p(offset)) { - return palignr_imm(r, l, offset); - } -#endif - return palignr_sw(r, l, offset); -} - -static really_inline -m128 variable_byte_shift_m128(m128 in, s32 amount) { - assert(amount >= -16 && amount <= 16); - if (amount < 0) { - return palignr(zeroes128(), in, -amount); - } else { - return palignr(in, zeroes128(), 16 - amount); - } -} -/* -static really_inline -m128 variable_byte_shift_m128(m128 in, s32 amount) { - assert(amount >= -16 && amount <= 16); - m128 shift_mask = loadu128(vbs_mask_data + 16 - amount); - return pshufb_m128(in, shift_mask); -}*/ - -static really_inline -m128 max_u8_m128(m128 a, m128 b) { - return _mm_max_epu8(a, b); -} - -static really_inline -m128 min_u8_m128(m128 a, m128 b) { - return _mm_min_epu8(a, b); -} - -static really_inline -m128 sadd_u8_m128(m128 a, m128 b) { - return _mm_adds_epu8(a, b); -} - -static really_inline -m128 sub_u8_m128(m128 a, m128 b) { - return _mm_sub_epi8(a, b); -} - -static really_inline -m128 set4x32(u32 x3, u32 x2, u32 x1, u32 x0) { - return _mm_set_epi32(x3, x2, x1, x0); -} - -static really_inline -m128 set2x64(u64a hi, u64a lo) { - return _mm_set_epi64x(hi, lo); -} - -#endif // ARCH_SIMDE_SIMD_UTILS_H diff --git a/src/util/arch/x86/simd_utils.h b/src/util/arch/x86/simd_utils.h index ba2bf26f..01429cf2 100644 --- a/src/util/arch/x86/simd_utils.h +++ b/src/util/arch/x86/simd_utils.h @@ -112,6 +112,16 @@ static really_inline u32 diffrich64_128(m128 a, m128 b) { #endif } +static really_really_inline +m128 add_2x64(m128 a, m128 b) { + return (m128) _mm_add_epi64(a, b); +} + +static really_really_inline +m128 sub_2x64(m128 a, m128 b) { + return (m128) _mm_sub_epi64(a, b); +} + static really_really_inline m128 lshift64_m128(m128 a, unsigned b) { #if defined(HAVE__BUILTIN_CONSTANT_P) @@ -124,8 +134,9 @@ m128 lshift64_m128(m128 a, unsigned b) { } #define rshift64_m128(a, b) _mm_srli_epi64((a), (b)) -#define eq128(a, b) _mm_cmpeq_epi8((a), (b)) -#define movemask128(a) ((u32)_mm_movemask_epi8((a))) +#define eq128(a, b) _mm_cmpeq_epi8((a), (b)) +#define eq64_m128(a, b) _mm_cmpeq_epi64((a), (b)) +#define movemask128(a) ((u32)_mm_movemask_epi8((a))) #if defined(HAVE_AVX512) static really_inline m128 cast512to128(const m512 in) { @@ -668,24 +679,6 @@ m256 combine2x128(m128 hi, m128 lo) { } #endif //AVX2 -#if defined(HAVE_SIMD_128_BITS) -/** - * "Rich" version of diff384(). Takes two vectors a and b and returns a 12-bit - * mask indicating which 32-bit words contain differences. - */ - -static really_inline u32 diffrich384(m384 a, m384 b) { - m128 z = zeroes128(); - a.lo = _mm_cmpeq_epi32(a.lo, b.lo); - a.mid = _mm_cmpeq_epi32(a.mid, b.mid); - a.hi = _mm_cmpeq_epi32(a.hi, b.hi); - m128 packed = _mm_packs_epi16(_mm_packs_epi32(a.lo, a.mid), - _mm_packs_epi32(a.hi, z)); - return ~(_mm_movemask_epi8(packed)) & 0xfff; -} - -#endif // HAVE_SIMD_128_BITS - /**** **** 512-bit Primitives ****/ diff --git a/src/util/bitutils.h b/src/util/bitutils.h index 7e006158..c67d5a85 100644 --- a/src/util/bitutils.h +++ b/src/util/bitutils.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2020-2023, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -44,13 +45,14 @@ #define DOUBLE_CASE_CLEAR 0xdfdf #define OCTO_CASE_CLEAR 0xdfdfdfdfdfdfdfdfULL - +#if !defined(VS_SIMDE_BACKEND) #if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/arch/x86/bitutils.h" #elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) #include "util/arch/arm/bitutils.h" #elif defined(ARCH_PPC64EL) #include "util/arch/ppc64el/bitutils.h" +#endif #else #include "util/arch/common/bitutils.h" #define clz32_impl clz32_impl_c diff --git a/src/util/match.hpp b/src/util/match.hpp index 68497349..6567b212 100644 --- a/src/util/match.hpp +++ b/src/util/match.hpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation - * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2020-2023, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -49,12 +49,16 @@ const u8 *first_zero_match_inverted(const u8 *buf, SuperVector v, u16 const l template const u8 *last_zero_match_inverted(const u8 *buf, SuperVector v, u16 len = S); -#if defined(ARCH_IA32) || defined(ARCH_X86_64) || defined(SIMDE_BACKEND) +#if defined(VS_SIMDE_BACKEND) +#include "util/arch/x86/match.hpp" +#else +#if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/arch/x86/match.hpp" #elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) #include "util/arch/arm/match.hpp" #elif defined(ARCH_PPC64EL) #include "util/arch/ppc64el/match.hpp" #endif +#endif #endif // MATCH_HPP diff --git a/src/util/simd_types.h b/src/util/simd_types.h index b9e2a492..e393d081 100644 --- a/src/util/simd_types.h +++ b/src/util/simd_types.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2023, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,7 +35,16 @@ #include "util/intrinsics.h" #include "ue2common.h" -#if defined(ARCH_IA32) || defined(ARCH_X86_64) +#if defined(VS_SIMDE_BACKEND) +#define VECTORSIZE 16 +#define SIMDE_ENABLE_NATIVE_ALIASES +#if !defined(VS_SIMDE_NATIVE) +#define SIMDE_NO_NATIVE +#endif +#include +typedef simde__m128i m128; +#define HAVE_SIMD_128_BITS +#elif defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/arch/x86/simd_types.h" #elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) #include "util/arch/arm/simd_types.h" @@ -42,14 +52,6 @@ #include "util/arch/ppc64el/simd_types.h" #endif -#if defined(SIMDE_BACKEND) -#define VECTORSIZE 16 -#define SIMDE_ENABLE_NATIVE_ALIASES -#define SIMDE_NO_NATIVE -#include "simde/simde/x86/sse4.2.h" -typedef simde__m128i m128; -#define HAVE_SIMD_128_BITS -#endif #if !defined(m256) && !defined(HAVE_SIMD_256_BITS) typedef struct ALIGN_AVX_DIRECTIVE {m128 lo; m128 hi;} m256; diff --git a/src/util/simd_utils.h b/src/util/simd_utils.h index 0ed66177..01c309b1 100644 --- a/src/util/simd_utils.h +++ b/src/util/simd_utils.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2020, Intel Corporation + * Copyright (c) 2023, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -61,14 +62,16 @@ extern const char vbs_mask_data[]; } #endif +#if defined(VS_SIMDE_BACKEND) +#include "util/arch/x86/simd_utils.h" +#else #if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/arch/x86/simd_utils.h" #elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) #include "util/arch/arm/simd_utils.h" #elif defined(ARCH_PPC64EL) #include "util/arch/ppc64el/simd_utils.h" -#elif defined(SIMDE_BACKEND) -#include "util/arch/simde/simd_utils.h" +#endif #endif #include "util/arch/common/simd_utils.h" diff --git a/src/util/supervector/arch/simde/impl.cpp b/src/util/supervector/arch/simde/impl.cpp deleted file mode 100644 index b1c9b631..00000000 --- a/src/util/supervector/arch/simde/impl.cpp +++ /dev/null @@ -1,530 +0,0 @@ -/* - * Copyright (c) 2015-2017, Intel Corporation - * Copyright (c) 2020-2021, VectorCamp PC - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef SIMD_IMPL_HPP -#define SIMD_IMPL_HPP - -#include -#include - -#include "ue2common.h" -#include "util/arch.h" -#include "util/unaligned.h" -#include "util/supervector/supervector.hpp" - -template<> -really_inline SuperVector<16>::SuperVector(SuperVector const &other) -{ - u.v128[0] = other.u.v128[0]; -} - -template<> -really_inline SuperVector<16>::SuperVector(typename base_type::type const v) -{ - u.v128[0] = v; -}; - -template<> -template<> -really_inline SuperVector<16>::SuperVector(int8_t const other) -{ - u.v128[0] = _mm_set1_epi8(other); -} - -template<> -template<> -really_inline SuperVector<16>::SuperVector(uint8_t const other) -{ - u.v128[0] = _mm_set1_epi8(static_cast(other)); -} - -template<> -template<> -really_inline SuperVector<16>::SuperVector(int16_t const other) -{ - u.v128[0] = _mm_set1_epi16(other); -} - -template<> -template<> -really_inline SuperVector<16>::SuperVector(uint16_t const other) -{ - u.v128[0] = _mm_set1_epi16(static_cast(other)); -} - -template<> -template<> -really_inline SuperVector<16>::SuperVector(int32_t const other) -{ - u.v128[0] = _mm_set1_epi32(other); -} - -template<> -template<> -really_inline SuperVector<16>::SuperVector(uint32_t const other) -{ - u.v128[0] = _mm_set1_epi32(static_cast(other)); -} - -template<> -template<> -really_inline SuperVector<16>::SuperVector(int64_t const other) -{ - u.v128[0] = _mm_set1_epi64x(other); -} - -template<> -template<> -really_inline SuperVector<16>::SuperVector(uint64_t const other) -{ - u.v128[0] = _mm_set1_epi64x(static_cast(other)); -} - -// Constants -template<> -really_inline SuperVector<16> SuperVector<16>::Ones() -{ - return {_mm_set1_epi8(0xFF)}; -} - -template<> -really_inline SuperVector<16> SuperVector<16>::Zeroes(void) -{ - return {_mm_set1_epi8(0)}; -} - -// Methods - -template <> -really_inline void SuperVector<16>::operator=(SuperVector<16> const &other) -{ - u.v128[0] = other.u.v128[0]; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::operator&(SuperVector<16> const &b) const -{ - return {_mm_and_si128(u.v128[0], b.u.v128[0])}; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::operator|(SuperVector<16> const &b) const -{ - return {_mm_or_si128(u.v128[0], b.u.v128[0])}; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::operator^(SuperVector<16> const &b) const -{ - return {_mm_xor_si128(u.v128[0], b.u.v128[0])}; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::operator!() const -{ - return {_mm_xor_si128(u.v128[0], u.v128[0])}; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::opandnot(SuperVector<16> const &b) const -{ - return {_mm_andnot_si128(u.v128[0], b.u.v128[0])}; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::operator==(SuperVector<16> const &b) const -{ - return {_mm_cmpeq_epi8(u.v128[0], b.u.v128[0])}; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::operator!=(SuperVector<16> const &b) const -{ - return !(*this == b); -} - -template <> -really_inline SuperVector<16> SuperVector<16>::operator>(SuperVector<16> const &b) const -{ - return {_mm_cmpgt_epi8(u.v128[0], b.u.v128[0])}; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::operator<(SuperVector<16> const &b) const -{ - return {_mm_cmplt_epi8(u.v128[0], b.u.v128[0])}; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::operator>=(SuperVector<16> const &b) const -{ - return !(*this < b); -} - -template <> -really_inline SuperVector<16> SuperVector<16>::operator<=(SuperVector<16> const &b) const -{ - return !(*this > b); -} - -template <> -really_inline SuperVector<16> SuperVector<16>::eq(SuperVector<16> const &b) const -{ - return (*this == b); -} - -template <> -really_inline typename SuperVector<16>::comparemask_type -SuperVector<16>::comparemask(void) const { - return (u32)_mm_movemask_epi8(u.v128[0]); -} - -template <> -really_inline typename SuperVector<16>::comparemask_type -SuperVector<16>::eqmask(SuperVector<16> const b) const { - return eq(b).comparemask(); -} - -template <> really_inline u32 SuperVector<16>::mask_width() { return 1; } - -template <> -really_inline typename SuperVector<16>::comparemask_type -SuperVector<16>::iteration_mask( - typename SuperVector<16>::comparemask_type mask) { - return mask; -} - -// template <> -// template -// really_inline SuperVector<16> SuperVector<16>::vshl_8_imm() const -// { -// const uint8_t i = N; -// return {_mm_slli_epi8(u.v128[0], i)}; -// } - -template <> -template -really_inline SuperVector<16> SuperVector<16>::vshl_16_imm() const -{ - return {_mm_slli_epi16(u.v128[0], N)}; -} - -template <> -template -really_inline SuperVector<16> SuperVector<16>::vshl_32_imm() const -{ - return {_mm_slli_epi32(u.v128[0], N)}; -} - -template <> -template -really_inline SuperVector<16> SuperVector<16>::vshl_64_imm() const -{ - return {_mm_slli_epi64(u.v128[0], N)}; -} - -template <> -template -really_inline SuperVector<16> SuperVector<16>::vshl_128_imm() const -{ - return {_mm_slli_si128(u.v128[0], N)}; -} - -template <> -template -really_inline SuperVector<16> SuperVector<16>::vshl_imm() const -{ - return vshl_128_imm(); -} - -// template <> -// template -// really_inline SuperVector<16> SuperVector<16>::vshr_8_imm() const -// { -// return {_mm_srli_epi8(u.v128[0], N)}; -// } - -template <> -template -really_inline SuperVector<16> SuperVector<16>::vshr_16_imm() const -{ - return {_mm_srli_epi16(u.v128[0], N)}; -} - -template <> -template -really_inline SuperVector<16> SuperVector<16>::vshr_32_imm() const -{ - return {_mm_srli_epi32(u.v128[0], N)}; -} - -template <> -template -really_inline SuperVector<16> SuperVector<16>::vshr_64_imm() const -{ - return {_mm_srli_epi64(u.v128[0], N)}; -} - -template <> -template -really_inline SuperVector<16> SuperVector<16>::vshr_128_imm() const -{ - return {_mm_srli_si128(u.v128[0], N)}; -} - -template <> -template -really_inline SuperVector<16> SuperVector<16>::vshr_imm() const -{ - return vshr_128_imm(); -} - -#if !defined(HS_OPTIMIZE) -template SuperVector<16> SuperVector<16>::vshl_16_imm<1>() const; -template SuperVector<16> SuperVector<16>::vshl_64_imm<1>() const; -template SuperVector<16> SuperVector<16>::vshl_64_imm<4>() const; -template SuperVector<16> SuperVector<16>::vshl_128_imm<1>() const; -template SuperVector<16> SuperVector<16>::vshl_128_imm<4>() const; -template SuperVector<16> SuperVector<16>::vshr_16_imm<1>() const; -template SuperVector<16> SuperVector<16>::vshr_64_imm<1>() const; -template SuperVector<16> SuperVector<16>::vshr_64_imm<4>() const; -template SuperVector<16> SuperVector<16>::vshr_128_imm<1>() const; -template SuperVector<16> SuperVector<16>::vshr_128_imm<4>() const; -#endif - -// template <> -// really_inline SuperVector<16> SuperVector<16>::vshl_8 (uint8_t const N) const -// { -// Unroller<0, 15>::iterator([&,v=this](int i) { if (N == i) return {_mm_slli_epi8(v->u.v128[0], i)}; }); -// if (N == 16) return Zeroes(); -// } - -template <> -really_inline SuperVector<16> SuperVector<16>::vshl_16 (uint8_t const N) const -{ -#if defined(HAVE__BUILTIN_CONSTANT_P) - if (__builtin_constant_p(N)) { - return {_mm_slli_epi16(u.v128[0], N)}; - } -#endif - if (N == 0) return *this; - if (N == 16) return Zeroes(); - SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_slli_epi16(v->u.v128[0], n)}; }); - return result; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::vshl_32 (uint8_t const N) const -{ - if (N == 0) return *this; - if (N == 16) return Zeroes(); - SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_slli_epi32(v->u.v128[0], n)}; }); - return result; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::vshl_64 (uint8_t const N) const -{ - if (N == 0) return *this; - if (N == 16) return Zeroes(); - SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_slli_epi64(v->u.v128[0], n)}; }); - return result; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::vshl_128(uint8_t const N) const -{ - if (N == 0) return *this; - if (N == 16) return Zeroes(); - SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_slli_si128(v->u.v128[0], n)}; }); - return result; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::vshl(uint8_t const N) const -{ - return vshl_128(N); -} - -// template <> -// really_inline SuperVector<16> SuperVector<16>::vshr_8 (uint8_t const N) const -// { -// SuperVector<16> result; -// Unroller<0, 15>::iterator([&,v=this](uint8_t const i) { if (N == i) result = {_mm_srli_epi8(v->u.v128[0], i)}; }); -// if (N == 16) result = Zeroes(); -// return result; -// } - -template <> -really_inline SuperVector<16> SuperVector<16>::vshr_16 (uint8_t const N) const -{ - if (N == 0) return *this; - if (N == 16) return Zeroes(); - SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_srli_epi16(v->u.v128[0], n)}; }); - return result; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::vshr_32 (uint8_t const N) const -{ - if (N == 0) return *this; - if (N == 16) return Zeroes(); - SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_srli_epi32(v->u.v128[0], n)}; }); - return result; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::vshr_64 (uint8_t const N) const -{ - if (N == 0) return *this; - if (N == 16) return Zeroes(); - SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_srli_epi64(v->u.v128[0], n)}; }); - return result; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::vshr_128(uint8_t const N) const -{ - if (N == 0) return *this; - if (N == 16) return Zeroes(); - SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_srli_si128(v->u.v128[0], n)}; }); - return result; -} - -template <> -really_inline SuperVector<16> SuperVector<16>::vshr(uint8_t const N) const -{ - return vshr_128(N); -} - -template <> -really_inline SuperVector<16> SuperVector<16>::operator>>(uint8_t const N) const -{ - return vshr_128(N); -} - -template <> -really_inline SuperVector<16> SuperVector<16>::operator<<(uint8_t const N) const -{ - return vshl_128(N); -} - -template<> -really_inline SuperVector<16> SuperVector<16>::Ones_vshr(uint8_t const N) -{ - if (N == 0) return Ones(); - else return Ones().vshr_128(N); -} - -template<> -really_inline SuperVector<16> SuperVector<16>::Ones_vshl(uint8_t const N) -{ - if (N == 0) return Ones(); - else return Ones().vshr_128(N); -} - -template <> -really_inline SuperVector<16> SuperVector<16>::loadu(void const *ptr) -{ - return _mm_loadu_si128((const m128 *)ptr); -} - -template <> -really_inline SuperVector<16> SuperVector<16>::load(void const *ptr) -{ - assert(ISALIGNED_N(ptr, alignof(SuperVector::size))); - ptr = vectorscan_assume_aligned(ptr, SuperVector::size); - return _mm_load_si128((const m128 *)ptr); -} - -template <> -really_inline SuperVector<16> SuperVector<16>::loadu_maskz(void const *ptr, uint8_t const len) -{ - SuperVector mask = Ones_vshr(16 -len); - SuperVector v = _mm_loadu_si128((const m128 *)ptr); - return mask & v; -} - -template<> -really_inline SuperVector<16> SuperVector<16>::alignr(SuperVector<16> &other, int8_t offset) -{ -#if defined(HAVE__BUILTIN_CONSTANT_P) - if (__builtin_constant_p(offset)) { - if (offset == 16) { - return *this; - } else { - return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], offset)}; - } - } -#endif - switch(offset) { - case 0: return other; break; - case 1: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 1)}; break; - case 2: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 2)}; break; - case 3: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 3)}; break; - case 4: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 4)}; break; - case 5: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 5)}; break; - case 6: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 6)}; break; - case 7: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 7)}; break; - case 8: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 8)}; break; - case 9: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 9)}; break; - case 10: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 10)}; break; - case 11: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 11)}; break; - case 12: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 12)}; break; - case 13: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 13)}; break; - case 14: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 14)}; break; - case 15: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 15)}; break; - default: break; - } - return *this; -} - -template<> -template<> -really_inline SuperVector<16> SuperVector<16>::pshufb(SuperVector<16> b) -{ - return {_mm_shuffle_epi8(u.v128[0], b.u.v128[0])}; -} - -template<> -really_inline SuperVector<16> SuperVector<16>::pshufb_maskz(SuperVector<16> b, uint8_t const len) -{ - SuperVector mask = Ones_vshr(16 -len); - return mask & pshufb(b); -} - -#endif // SIMD_IMPL_HPP diff --git a/src/util/supervector/supervector.hpp b/src/util/supervector/supervector.hpp index 730a6fd2..253907fa 100644 --- a/src/util/supervector/supervector.hpp +++ b/src/util/supervector/supervector.hpp @@ -34,6 +34,9 @@ #include #include +#if defined(VS_SIMDE_BACKEND) +#include "util/supervector/arch/x86/types.hpp" +#else #if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/supervector/arch/x86/types.hpp" #elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) @@ -41,6 +44,7 @@ #elif defined(ARCH_PPC64EL) #include "util/supervector/arch/ppc64el/types.hpp" #endif +#endif // VS_SIMDE_BACKEND #if defined(HAVE_SIMD_512_BITS) using Z_TYPE = u64a; @@ -57,7 +61,7 @@ using Z_TYPE = u32; #define DOUBLE_LOAD_MASK(l) (((1ULL) << (l)) - 1ULL) #define SINGLE_LOAD_MASK(l) (((1ULL) << (l)) - 1ULL) #elif defined(HAVE_SIMD_128_BITS) -#if defined(ARCH_ARM32) || defined(ARCH_AARCH64) +#if !defined(VS_SIMDE_BACKEND) && (defined(ARCH_ARM32) || defined(ARCH_AARCH64)) using Z_TYPE = u64a; #define Z_BITS 64 #define Z_POSSHIFT 2 @@ -175,7 +179,7 @@ public: typename BaseVector<32>::type ALIGN_ATTR(BaseVector<32>::size) v256[SIZE / BaseVector<32>::size]; typename BaseVector<64>::type ALIGN_ATTR(BaseVector<64>::size) v512[SIZE / BaseVector<64>::size]; -#if defined(ARCH_ARM32) || defined(ARCH_AARCH64) || defined(ARCH_PPC64EL) +#if !defined(VS_SIMDE_BACKEND) && (defined(ARCH_ARM32) || defined(ARCH_AARCH64) || defined(ARCH_PPC64EL)) uint64x2_t ALIGN_ATTR(BaseVector<16>::size) u64x2[SIZE / BaseVector<16>::size]; int64x2_t ALIGN_ATTR(BaseVector<16>::size) s64x2[SIZE / BaseVector<16>::size]; uint32x4_t ALIGN_ATTR(BaseVector<16>::size) u32x4[SIZE / BaseVector<16>::size]; @@ -382,14 +386,16 @@ struct Unroller }; #if defined(HS_OPTIMIZE) +#if defined(VS_SIMDE_BACKEND) +#include "util/supervector/arch/x86/impl.cpp" +#else #if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/supervector/arch/x86/impl.cpp" #elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) #include "util/supervector/arch/arm/impl.cpp" #elif defined(ARCH_PPC64EL) #include "util/supervector/arch/ppc64el/impl.cpp" -#elif defined(SIMDE_BACKEND) -#include "util/supervector/arch/simde/impl.cpp" +#endif #endif #endif diff --git a/unit/internal/simd_utils.cpp b/unit/internal/simd_utils.cpp index a9737bd2..272d5456 100644 --- a/unit/internal/simd_utils.cpp +++ b/unit/internal/simd_utils.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2023, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -667,7 +668,7 @@ TEST(SimdUtilsTest, movq) { ASSERT_EQ(0, memcmp(cmp, &r, sizeof(r))); #if defined(HAVE_SIMD_128_BITS) -#if defined(ARCH_IA32) || defined(ARCH_X86_64) || defined(SIMDE_BACKEND) +#if defined(ARCH_IA32) || defined(ARCH_X86_64) || defined(VS_SIMDE_BACKEND) simd = _mm_set_epi64x(~0LL, 0x123456789abcdef); #elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) int64x2_t a = { 0x123456789abcdefLL, ~0LL };