mirror of
https://github.com/VectorCamp/vectorscan.git
synced 2025-06-28 16:41:01 +03:00
fix SIMDe emulation builds on Arm, add native translation from x86 for comparison
This commit is contained in:
Konstantinos Margaritis
parent
dfacf75855
commit
f57928ea08
@ -119,7 +119,10 @@ endif()
|
||||
# Detect OS and if Fat Runtime is available
|
||||
include (${CMAKE_MODULE_PATH}/osdetection.cmake)
|
||||
|
||||
if (ARCH_IA32 OR ARCH_X86_64)
|
||||
if(SIMDE_BACKEND)
|
||||
include (${CMAKE_MODULE_PATH}/simde.cmake)
|
||||
set(ARCH_FLAG march)
|
||||
elseif (ARCH_IA32 OR ARCH_X86_64)
|
||||
include (${CMAKE_MODULE_PATH}/cflags-x86.cmake)
|
||||
set(ARCH_FLAG march)
|
||||
elseif (ARCH_ARM32 OR ARCH_AARCH64)
|
||||
@ -128,10 +131,6 @@ elseif (ARCH_ARM32 OR ARCH_AARCH64)
|
||||
elseif (ARCH_PPC64EL)
|
||||
include (${CMAKE_MODULE_PATH}/cflags-ppc64le.cmake)
|
||||
set(ARCH_FLAG mcpu)
|
||||
elseif(SIMDE_BACKEND)
|
||||
include (${CMAKE_MODULE_PATH}/simde.cmake)
|
||||
set(ARCH_FLAG march)
|
||||
else()
|
||||
message(FATAL_ERROR "Unsupported platform")
|
||||
endif ()
|
||||
|
||||
@ -243,8 +242,11 @@ set (hs_exec_common_SRCS
|
||||
src/util/arch/common/cpuid_flags.h
|
||||
src/util/multibit.c
|
||||
)
|
||||
|
||||
if (ARCH_IA32 OR ARCH_X86_64)
|
||||
if (SIMDE_BACKEND)
|
||||
set (hs_exec_common_SRCS
|
||||
${hs_exec_common_SRCS}
|
||||
src/util/arch/simde/cpuid_flags.c)
|
||||
elseif (ARCH_IA32 OR ARCH_X86_64)
|
||||
set (hs_exec_common_SRCS
|
||||
${hs_exec_common_SRCS}
|
||||
src/util/arch/x86/cpuid_flags.c
|
||||
@ -258,10 +260,6 @@ elseif (ARCH_PPC64EL)
|
||||
set (hs_exec_common_SRCS
|
||||
${hs_exec_common_SRCS}
|
||||
src/util/arch/ppc64el/cpuid_flags.c)
|
||||
elseif (SIMDE_BACKEND)
|
||||
set (hs_exec_common_SRCS
|
||||
${hs_exec_common_SRCS}
|
||||
src/util/arch/simde/cpuid_flags.c)
|
||||
endif ()
|
||||
|
||||
set (hs_exec_SRCS
|
||||
@ -406,7 +404,12 @@ set (hs_exec_SRCS
|
||||
src/database.h
|
||||
)
|
||||
|
||||
if (ARCH_IA32 OR ARCH_X86_64)
|
||||
if (SIMDE_BACKEND)
|
||||
set (hs_exec_SRCS
|
||||
${hs_exec_SRCS}
|
||||
src/nfa/vermicelli_simd.cpp
|
||||
src/util/supervector/arch/x86/impl.cpp)
|
||||
elseif (ARCH_IA32 OR ARCH_X86_64)
|
||||
set (hs_exec_SRCS
|
||||
${hs_exec_SRCS}
|
||||
src/nfa/vermicelli_simd.cpp
|
||||
@ -420,11 +423,6 @@ set (hs_exec_SRCS
|
||||
${hs_exec_SRCS}
|
||||
src/nfa/vermicelli_simd.cpp
|
||||
src/util/supervector/arch/ppc64el/impl.cpp)
|
||||
elseif (SIMDE_BACKEND)
|
||||
set (hs_exec_SRCS
|
||||
${hs_exec_SRCS}
|
||||
src/nfa/vermicelli_simd.cpp
|
||||
src/util/supervector/arch/simde/impl.cpp)
|
||||
endif()
|
||||
|
||||
if (ARCH_IA32 OR ARCH_X86_64)
|
||||
|
||||
@ -67,7 +67,10 @@ if (USE_CPU_NATIVE)
|
||||
message(STATUS "clang will tune for ${GNUCC_ARCH}, ${TUNE_FLAG}")
|
||||
endif()
|
||||
else()
|
||||
if (ARCH_IA32 OR ARCH_X86_64)
|
||||
if (SIMDE_BACKEND)
|
||||
set(GNUCC_ARCH native)
|
||||
set(TUNE_FLAG generic)
|
||||
elseif (ARCH_IA32 OR ARCH_X86_64)
|
||||
set(GNUCC_ARCH native)
|
||||
set(TUNE_FLAG generic)
|
||||
elseif(ARCH_AARCH64)
|
||||
@ -85,8 +88,8 @@ else()
|
||||
set(GNUCC_ARCH armv7a)
|
||||
set(TUNE_FLAG generic)
|
||||
elseif(ARCH_PPC64EL)
|
||||
set(GNUCC_ARCH power9)
|
||||
set(TUNE_FLAG power9)
|
||||
set(GNUCC_ARCH power8)
|
||||
set(TUNE_FLAG power8)
|
||||
else()
|
||||
set(GNUCC_ARCH native)
|
||||
set(TUNE_FLAG native)
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
include_directories(${PROJECT_SOURCE_DIR}/simde/simde)
|
||||
# include_directories(${PROJECT_SOURCE_DIR}/simde/simde)
|
||||
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSIMDE_BACKEND")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSIMDE_BACKEND")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DVS_SIMDE_BACKEND")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DVS_SIMDE_BACKEND")
|
||||
|
||||
if (SIMDE_NATIVE)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DVS_SIMDE_NATIVE -DSIMDE_ENABLE_OPENMP -fopenmp-simd")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DVS_SIMDE_NATIVE -DSIMDE_ENABLE_OPENMP -fopenmp-simd")
|
||||
endif()
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2016-2017, Intel Corporation
|
||||
* Copyright (c) 2020-2023, VectorCamp PC
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
@ -29,28 +30,30 @@
|
||||
#include "config.h"
|
||||
#include "hs_common.h"
|
||||
#include "ue2common.h"
|
||||
#if !defined(VS_SIMDE_BACKEND)
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64)
|
||||
#include "util/arch/x86/cpuid_inline.h"
|
||||
#elif defined(ARCH_AARCH64)
|
||||
#include "util/arch/arm/cpuid_inline.h"
|
||||
#endif
|
||||
#endif
|
||||
|
||||
HS_PUBLIC_API
|
||||
hs_error_t HS_CDECL hs_valid_platform(void) {
|
||||
/* Hyperscan requires SSSE3, anything else is a bonus */
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64)
|
||||
#if !defined(VS_SIMDE_BACKEND) && (defined(ARCH_IA32) || defined(ARCH_X86_64))
|
||||
if (check_ssse3()) {
|
||||
return HS_SUCCESS;
|
||||
} else {
|
||||
return HS_ARCH_ERROR;
|
||||
}
|
||||
#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64)
|
||||
#elif !defined(VS_SIMDE_BACKEND) && (defined(ARCH_ARM32) || defined(ARCH_AARCH64))
|
||||
if (check_neon()) {
|
||||
return HS_SUCCESS;
|
||||
} else {
|
||||
return HS_ARCH_ERROR;
|
||||
}
|
||||
#elif defined(ARCH_PPC64EL) || defined(SIMDE_BACKEND)
|
||||
#elif defined(ARCH_PPC64EL) || defined(VS_SIMDE_BACKEND)
|
||||
return HS_SUCCESS;
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2015-2017, Intel Corporation
|
||||
* Copyright (c) 2020-2021, VectorCamp PC
|
||||
* Copyright (c) 2020-2023, VectorCamp PC
|
||||
* Copyright (c) 2021, Arm Limited
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
@ -52,13 +52,17 @@ template <uint16_t S>
|
||||
static really_inline
|
||||
SuperVector<S> blockDoubleMask(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi, SuperVector<S> mask2_lo, SuperVector<S> mask2_hi, SuperVector<S> chars);
|
||||
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64) || defined(SIMDE_BACKEND)
|
||||
#if defined(VS_SIMDE_BACKEND)
|
||||
#include "x86/shufti.hpp"
|
||||
#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64)
|
||||
#else
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64)
|
||||
#include "x86/shufti.hpp"
|
||||
#elif (defined(ARCH_ARM32) || defined(ARCH_AARCH64))
|
||||
#include "arm/shufti.hpp"
|
||||
#elif defined(ARCH_PPC64EL)
|
||||
#include "ppc64el/shufti.hpp"
|
||||
#endif
|
||||
#endif
|
||||
|
||||
template <uint16_t S>
|
||||
static really_inline
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2015-2017, Intel Corporation
|
||||
* Copyright (c) 2020-2021, VectorCamp PC
|
||||
* Copyright (c) 2020-2023, VectorCamp PC
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
@ -45,13 +45,17 @@ template <uint16_t S>
|
||||
static really_inline
|
||||
const SuperVector<S> blockSingleMask(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars);
|
||||
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64) || defined(SIMDE_BACKEND)
|
||||
#if defined(VS_SIMDE_BACKEND)
|
||||
#include "x86/truffle.hpp"
|
||||
#else
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64)
|
||||
#include "x86/truffle.hpp"
|
||||
#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64)
|
||||
#include "arm/truffle.hpp"
|
||||
#elif defined(ARCH_PPC64EL)
|
||||
#include "ppc64el/truffle.hpp"
|
||||
#endif
|
||||
#endif
|
||||
|
||||
template <uint16_t S>
|
||||
static really_inline
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2015-2020, Intel Corporation
|
||||
* Copyright (c) 2020-2021, VectorCamp PC
|
||||
* Copyright (c) 2020-2023, VectorCamp PC
|
||||
* Copyright (c) 2021, Arm Limited
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
@ -71,13 +71,17 @@ const u8 *vermicelliDoubleMaskedBlock(SuperVector<S> const data, SuperVector<S>
|
||||
SuperVector<S> const mask1, SuperVector<S> const mask2,
|
||||
u8 const c1, u8 const c2, u8 const m1, u8 const m2, u8 const *buf, u16 const len);
|
||||
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64) || defined(SIMDE_BACKEND)
|
||||
#if defined(VS_SIMDE_BACKEND)
|
||||
#include "x86/vermicelli.hpp"
|
||||
#else
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64)
|
||||
#include "x86/vermicelli.hpp"
|
||||
#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64)
|
||||
#include "arm/vermicelli.hpp"
|
||||
#elif defined(ARCH_PPC64EL)
|
||||
#include "ppc64el/vermicelli.hpp"
|
||||
#endif
|
||||
#endif
|
||||
|
||||
template <uint16_t S>
|
||||
static const u8 *vermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const casemask, u8 const *buf, u8 const *buf_end) {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2015-2020, Intel Corporation
|
||||
* Copyright (c) 2020-2021, VectorCamp PC
|
||||
* Copyright (c) 2020-2023, VectorCamp PC
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
@ -41,7 +41,7 @@
|
||||
|
||||
#include <string.h> // for memcpy
|
||||
|
||||
#if !defined(HAVE_SIMD_128_BITS) && !defined(SIMDE_BACKEND)
|
||||
#if !defined(HAVE_SIMD_128_BITS) && !defined(VS_SIMDE_BACKEND)
|
||||
#error "You need at least a 128-bit capable SIMD engine!"
|
||||
#endif // HAVE_SIMD_128_BITS
|
||||
|
||||
@ -88,7 +88,7 @@ static inline void print_m128_2x64(const char *label, m128 vec) {
|
||||
#define print_m128_2x64(label, vec) ;
|
||||
#endif
|
||||
|
||||
#if !defined(ARCH_IA32) && !defined(ARCH_X86_64) && !defined(SIMDE_BACKEND)
|
||||
#if !defined(ARCH_IA32) && !defined(ARCH_X86_64) && !defined(VS_SIMDE_BACKEND)
|
||||
#define ZEROES_8 0, 0, 0, 0, 0, 0, 0, 0
|
||||
#define ZEROES_31 ZEROES_8, ZEROES_8, ZEROES_8, 0, 0, 0, 0, 0, 0, 0
|
||||
#define ZEROES_32 ZEROES_8, ZEROES_8, ZEROES_8, ZEROES_8
|
||||
|
||||
@ -1,388 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2015-2020, Intel Corporation
|
||||
* Copyright (c) 2020-2021, VectorCamp PC
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \brief SIMD types and primitive operations.
|
||||
*/
|
||||
|
||||
#ifndef ARCH_SIMDE_SIMD_UTILS_H
|
||||
#define ARCH_SIMDE_SIMD_UTILS_H
|
||||
|
||||
#include "ue2common.h"
|
||||
#include "util/simd_types.h"
|
||||
#include "util/unaligned.h"
|
||||
#include "util/intrinsics.h"
|
||||
|
||||
#include <string.h> // for memcpy
|
||||
|
||||
#define ZEROES_8 0, 0, 0, 0, 0, 0, 0, 0
|
||||
#define ZEROES_31 ZEROES_8, ZEROES_8, ZEROES_8, 0, 0, 0, 0, 0, 0, 0
|
||||
#define ZEROES_32 ZEROES_8, ZEROES_8, ZEROES_8, ZEROES_8
|
||||
|
||||
/** \brief LUT for the mask1bit functions. */
|
||||
ALIGN_CL_DIRECTIVE static const u8 simd_onebit_masks[] = {
|
||||
ZEROES_32, ZEROES_32,
|
||||
ZEROES_31, 0x01, ZEROES_32,
|
||||
ZEROES_31, 0x02, ZEROES_32,
|
||||
ZEROES_31, 0x04, ZEROES_32,
|
||||
ZEROES_31, 0x08, ZEROES_32,
|
||||
ZEROES_31, 0x10, ZEROES_32,
|
||||
ZEROES_31, 0x20, ZEROES_32,
|
||||
ZEROES_31, 0x40, ZEROES_32,
|
||||
ZEROES_31, 0x80, ZEROES_32,
|
||||
ZEROES_32, ZEROES_32,
|
||||
};
|
||||
|
||||
static really_inline m128 ones128(void) {
|
||||
return (m128) _mm_set1_epi8(0xFF);
|
||||
}
|
||||
|
||||
static really_inline m128 zeroes128(void) {
|
||||
return (m128) _mm_setzero_si128();
|
||||
}
|
||||
|
||||
/** \brief Bitwise not for m128*/
|
||||
static really_inline m128 not128(m128 a) {
|
||||
return (m128) _mm_xor_si128(a, ones128());
|
||||
}
|
||||
|
||||
/** \brief Return 1 if a and b are different otherwise 0 */
|
||||
static really_inline int diff128(m128 a, m128 b) {
|
||||
return (_mm_movemask_epi8(_mm_cmpeq_epi8(a, b)) ^ 0xffff);
|
||||
}
|
||||
|
||||
static really_inline int isnonzero128(m128 a) {
|
||||
return !!diff128(a, zeroes128());
|
||||
}
|
||||
|
||||
/**
|
||||
* "Rich" version of diff128(). Takes two vectors a and b and returns a 4-bit
|
||||
* mask indicating which 32-bit words contain differences.
|
||||
*/
|
||||
static really_inline u32 diffrich128(m128 a, m128 b) {
|
||||
a = _mm_cmpeq_epi32(a, b);
|
||||
return ~(_mm_movemask_ps(_mm_castsi128_ps(a))) & 0xf;
|
||||
}
|
||||
|
||||
/**
|
||||
* "Rich" version of diff128(), 64-bit variant. Takes two vectors a and b and
|
||||
* returns a 4-bit mask indicating which 64-bit words contain differences.
|
||||
*/
|
||||
static really_inline u32 diffrich64_128(m128 a, m128 b) {
|
||||
a = _mm_cmpeq_epi64(a, b);
|
||||
return ~(_mm_movemask_ps(_mm_castsi128_ps(a))) & 0x5;
|
||||
}
|
||||
|
||||
static really_really_inline
|
||||
m128 add_2x64(m128 a, m128 b) {
|
||||
return (m128) _mm_add_epi64(a, b);
|
||||
}
|
||||
|
||||
static really_really_inline
|
||||
m128 sub_2x64(m128 a, m128 b) {
|
||||
return (m128) _mm_sub_epi64(a, b);
|
||||
}
|
||||
|
||||
static really_really_inline
|
||||
m128 lshift64_m128(m128 a, unsigned b) {
|
||||
return _mm_slli_epi64(a, b);
|
||||
}
|
||||
|
||||
#define rshift64_m128(a, b) _mm_srli_epi64((a), (b))
|
||||
#define eq128(a, b) _mm_cmpeq_epi8((a), (b))
|
||||
#define eq64_m128(a, b) _mm_cmpeq_epi64((a), (b))
|
||||
#define movemask128(a) ((u32)_mm_movemask_epi8((a)))
|
||||
|
||||
static really_inline m128 set1_16x8(u8 c) {
|
||||
return _mm_set1_epi8(c);
|
||||
}
|
||||
|
||||
static really_inline m128 set1_4x32(u32 c) {
|
||||
return _mm_set1_epi32(c);
|
||||
}
|
||||
|
||||
static really_inline m128 set1_2x64(u64a c) {
|
||||
return _mm_set1_epi64x(c);
|
||||
}
|
||||
|
||||
static really_inline u32 movd(const m128 in) {
|
||||
return _mm_cvtsi128_si32(in);
|
||||
}
|
||||
|
||||
static really_inline u64a movq(const m128 in) {
|
||||
return _mm_cvtsi128_si64(in);
|
||||
}
|
||||
|
||||
/* another form of movq */
|
||||
static really_inline
|
||||
m128 load_m128_from_u64a(const u64a *p) {
|
||||
return _mm_set_epi64x(0LL, *p);
|
||||
}
|
||||
|
||||
#define CASE_RSHIFT_VECTOR(a, count) case count: return _mm_srli_si128((m128)(a), (count)); break;
|
||||
|
||||
static really_inline
|
||||
m128 rshiftbyte_m128(const m128 a, int count_immed) {
|
||||
switch (count_immed) {
|
||||
case 0: return a; break;
|
||||
CASE_RSHIFT_VECTOR(a, 1);
|
||||
CASE_RSHIFT_VECTOR(a, 2);
|
||||
CASE_RSHIFT_VECTOR(a, 3);
|
||||
CASE_RSHIFT_VECTOR(a, 4);
|
||||
CASE_RSHIFT_VECTOR(a, 5);
|
||||
CASE_RSHIFT_VECTOR(a, 6);
|
||||
CASE_RSHIFT_VECTOR(a, 7);
|
||||
CASE_RSHIFT_VECTOR(a, 8);
|
||||
CASE_RSHIFT_VECTOR(a, 9);
|
||||
CASE_RSHIFT_VECTOR(a, 10);
|
||||
CASE_RSHIFT_VECTOR(a, 11);
|
||||
CASE_RSHIFT_VECTOR(a, 12);
|
||||
CASE_RSHIFT_VECTOR(a, 13);
|
||||
CASE_RSHIFT_VECTOR(a, 14);
|
||||
CASE_RSHIFT_VECTOR(a, 15);
|
||||
default: return zeroes128(); break;
|
||||
}
|
||||
}
|
||||
#undef CASE_RSHIFT_VECTOR
|
||||
|
||||
#define CASE_LSHIFT_VECTOR(a, count) case count: return _mm_slli_si128((m128)(a), (count)); break;
|
||||
|
||||
static really_inline
|
||||
m128 lshiftbyte_m128(const m128 a, int count_immed) {
|
||||
switch (count_immed) {
|
||||
case 0: return a; break;
|
||||
CASE_LSHIFT_VECTOR(a, 1);
|
||||
CASE_LSHIFT_VECTOR(a, 2);
|
||||
CASE_LSHIFT_VECTOR(a, 3);
|
||||
CASE_LSHIFT_VECTOR(a, 4);
|
||||
CASE_LSHIFT_VECTOR(a, 5);
|
||||
CASE_LSHIFT_VECTOR(a, 6);
|
||||
CASE_LSHIFT_VECTOR(a, 7);
|
||||
CASE_LSHIFT_VECTOR(a, 8);
|
||||
CASE_LSHIFT_VECTOR(a, 9);
|
||||
CASE_LSHIFT_VECTOR(a, 10);
|
||||
CASE_LSHIFT_VECTOR(a, 11);
|
||||
CASE_LSHIFT_VECTOR(a, 12);
|
||||
CASE_LSHIFT_VECTOR(a, 13);
|
||||
CASE_LSHIFT_VECTOR(a, 14);
|
||||
CASE_LSHIFT_VECTOR(a, 15);
|
||||
default: return zeroes128(); break;
|
||||
}
|
||||
}
|
||||
#undef CASE_LSHIFT_VECTOR
|
||||
|
||||
#define extract32from128(a, imm) _mm_extract_epi32(a, imm)
|
||||
#define extract64from128(a, imm) _mm_extract_epi64(a, imm)
|
||||
|
||||
static really_inline m128 add128(m128 a, m128 b) {
|
||||
return _mm_add_epi64(a, b);
|
||||
}
|
||||
|
||||
static really_inline m128 and128(m128 a, m128 b) {
|
||||
return _mm_and_si128(a,b);
|
||||
}
|
||||
|
||||
static really_inline m128 xor128(m128 a, m128 b) {
|
||||
return _mm_xor_si128(a,b);
|
||||
}
|
||||
|
||||
static really_inline m128 or128(m128 a, m128 b) {
|
||||
return _mm_or_si128(a,b);
|
||||
}
|
||||
|
||||
static really_inline m128 andnot128(m128 a, m128 b) {
|
||||
return _mm_andnot_si128(a, b);
|
||||
}
|
||||
|
||||
// aligned load
|
||||
static really_inline m128 load128(const void *ptr) {
|
||||
assert(ISALIGNED_N(ptr, alignof(m128)));
|
||||
ptr = vectorscan_assume_aligned(ptr, 16);
|
||||
return _mm_load_si128((const m128 *)ptr);
|
||||
}
|
||||
|
||||
// aligned store
|
||||
static really_inline void store128(void *ptr, m128 a) {
|
||||
assert(ISALIGNED_N(ptr, alignof(m128)));
|
||||
ptr = vectorscan_assume_aligned(ptr, 16);
|
||||
*(m128 *)ptr = a;
|
||||
}
|
||||
|
||||
// unaligned load
|
||||
static really_inline m128 loadu128(const void *ptr) {
|
||||
return _mm_loadu_si128((const m128 *)ptr);
|
||||
}
|
||||
|
||||
// unaligned store
|
||||
static really_inline void storeu128(void *ptr, m128 a) {
|
||||
_mm_storeu_si128 ((m128 *)ptr, a);
|
||||
}
|
||||
|
||||
// packed unaligned store of first N bytes
|
||||
static really_inline
|
||||
void storebytes128(void *ptr, m128 a, unsigned int n) {
|
||||
assert(n <= sizeof(a));
|
||||
memcpy(ptr, &a, n);
|
||||
}
|
||||
|
||||
// packed unaligned load of first N bytes, pad with zero
|
||||
static really_inline
|
||||
m128 loadbytes128(const void *ptr, unsigned int n) {
|
||||
m128 a = zeroes128();
|
||||
assert(n <= sizeof(a));
|
||||
memcpy(&a, ptr, n);
|
||||
return a;
|
||||
}
|
||||
|
||||
static really_inline
|
||||
m128 mask1bit128(unsigned int n) {
|
||||
assert(n < sizeof(m128) * 8);
|
||||
u32 mask_idx = ((n % 8) * 64) + 95;
|
||||
mask_idx -= n / 8;
|
||||
return loadu128(&simd_onebit_masks[mask_idx]);
|
||||
}
|
||||
|
||||
// switches on bit N in the given vector.
|
||||
static really_inline
|
||||
void setbit128(m128 *ptr, unsigned int n) {
|
||||
*ptr = or128(mask1bit128(n), *ptr);
|
||||
}
|
||||
|
||||
// switches off bit N in the given vector.
|
||||
static really_inline
|
||||
void clearbit128(m128 *ptr, unsigned int n) {
|
||||
*ptr = andnot128(mask1bit128(n), *ptr);
|
||||
}
|
||||
|
||||
// tests bit N in the given vector.
|
||||
static really_inline
|
||||
char testbit128(m128 val, unsigned int n) {
|
||||
const m128 mask = mask1bit128(n);
|
||||
#if defined(HAVE_SSE41)
|
||||
return !_mm_testz_si128(mask, val);
|
||||
#else
|
||||
return isnonzero128(and128(mask, val));
|
||||
#endif
|
||||
}
|
||||
|
||||
// offset must be an immediate
|
||||
#define palignr_imm(r, l, offset) _mm_alignr_epi8(r, l, offset)
|
||||
|
||||
static really_inline
|
||||
m128 pshufb_m128(m128 a, m128 b) {
|
||||
return _mm_shuffle_epi8(a, b);
|
||||
}
|
||||
|
||||
#define CASE_ALIGN_VECTORS(a, b, offset) case offset: return palignr_imm((m128)(a), (m128)(b), (offset)); break;
|
||||
|
||||
static really_really_inline
|
||||
m128 palignr_sw(m128 r, m128 l, int offset) {
|
||||
switch (offset) {
|
||||
case 0: return l; break;
|
||||
CASE_ALIGN_VECTORS(r, l, 1);
|
||||
CASE_ALIGN_VECTORS(r, l, 2);
|
||||
CASE_ALIGN_VECTORS(r, l, 3);
|
||||
CASE_ALIGN_VECTORS(r, l, 4);
|
||||
CASE_ALIGN_VECTORS(r, l, 5);
|
||||
CASE_ALIGN_VECTORS(r, l, 6);
|
||||
CASE_ALIGN_VECTORS(r, l, 7);
|
||||
CASE_ALIGN_VECTORS(r, l, 8);
|
||||
CASE_ALIGN_VECTORS(r, l, 9);
|
||||
CASE_ALIGN_VECTORS(r, l, 10);
|
||||
CASE_ALIGN_VECTORS(r, l, 11);
|
||||
CASE_ALIGN_VECTORS(r, l, 12);
|
||||
CASE_ALIGN_VECTORS(r, l, 13);
|
||||
CASE_ALIGN_VECTORS(r, l, 14);
|
||||
CASE_ALIGN_VECTORS(r, l, 15);
|
||||
case 16: return r; break;
|
||||
default:
|
||||
return zeroes128();
|
||||
break;
|
||||
}
|
||||
}
|
||||
#undef CASE_ALIGN_VECTORS
|
||||
|
||||
static really_really_inline
|
||||
m128 palignr(m128 r, m128 l, int offset) {
|
||||
#if defined(HAVE__BUILTIN_CONSTANT_P)
|
||||
if (__builtin_constant_p(offset)) {
|
||||
return palignr_imm(r, l, offset);
|
||||
}
|
||||
#endif
|
||||
return palignr_sw(r, l, offset);
|
||||
}
|
||||
|
||||
static really_inline
|
||||
m128 variable_byte_shift_m128(m128 in, s32 amount) {
|
||||
assert(amount >= -16 && amount <= 16);
|
||||
if (amount < 0) {
|
||||
return palignr(zeroes128(), in, -amount);
|
||||
} else {
|
||||
return palignr(in, zeroes128(), 16 - amount);
|
||||
}
|
||||
}
|
||||
/*
|
||||
static really_inline
|
||||
m128 variable_byte_shift_m128(m128 in, s32 amount) {
|
||||
assert(amount >= -16 && amount <= 16);
|
||||
m128 shift_mask = loadu128(vbs_mask_data + 16 - amount);
|
||||
return pshufb_m128(in, shift_mask);
|
||||
}*/
|
||||
|
||||
static really_inline
|
||||
m128 max_u8_m128(m128 a, m128 b) {
|
||||
return _mm_max_epu8(a, b);
|
||||
}
|
||||
|
||||
static really_inline
|
||||
m128 min_u8_m128(m128 a, m128 b) {
|
||||
return _mm_min_epu8(a, b);
|
||||
}
|
||||
|
||||
static really_inline
|
||||
m128 sadd_u8_m128(m128 a, m128 b) {
|
||||
return _mm_adds_epu8(a, b);
|
||||
}
|
||||
|
||||
static really_inline
|
||||
m128 sub_u8_m128(m128 a, m128 b) {
|
||||
return _mm_sub_epi8(a, b);
|
||||
}
|
||||
|
||||
static really_inline
|
||||
m128 set4x32(u32 x3, u32 x2, u32 x1, u32 x0) {
|
||||
return _mm_set_epi32(x3, x2, x1, x0);
|
||||
}
|
||||
|
||||
static really_inline
|
||||
m128 set2x64(u64a hi, u64a lo) {
|
||||
return _mm_set_epi64x(hi, lo);
|
||||
}
|
||||
|
||||
#endif // ARCH_SIMDE_SIMD_UTILS_H
|
||||
@ -112,6 +112,16 @@ static really_inline u32 diffrich64_128(m128 a, m128 b) {
|
||||
#endif
|
||||
}
|
||||
|
||||
static really_really_inline
|
||||
m128 add_2x64(m128 a, m128 b) {
|
||||
return (m128) _mm_add_epi64(a, b);
|
||||
}
|
||||
|
||||
static really_really_inline
|
||||
m128 sub_2x64(m128 a, m128 b) {
|
||||
return (m128) _mm_sub_epi64(a, b);
|
||||
}
|
||||
|
||||
static really_really_inline
|
||||
m128 lshift64_m128(m128 a, unsigned b) {
|
||||
#if defined(HAVE__BUILTIN_CONSTANT_P)
|
||||
@ -124,8 +134,9 @@ m128 lshift64_m128(m128 a, unsigned b) {
|
||||
}
|
||||
|
||||
#define rshift64_m128(a, b) _mm_srli_epi64((a), (b))
|
||||
#define eq128(a, b) _mm_cmpeq_epi8((a), (b))
|
||||
#define movemask128(a) ((u32)_mm_movemask_epi8((a)))
|
||||
#define eq128(a, b) _mm_cmpeq_epi8((a), (b))
|
||||
#define eq64_m128(a, b) _mm_cmpeq_epi64((a), (b))
|
||||
#define movemask128(a) ((u32)_mm_movemask_epi8((a)))
|
||||
|
||||
#if defined(HAVE_AVX512)
|
||||
static really_inline m128 cast512to128(const m512 in) {
|
||||
@ -668,24 +679,6 @@ m256 combine2x128(m128 hi, m128 lo) {
|
||||
}
|
||||
#endif //AVX2
|
||||
|
||||
#if defined(HAVE_SIMD_128_BITS)
|
||||
/**
|
||||
* "Rich" version of diff384(). Takes two vectors a and b and returns a 12-bit
|
||||
* mask indicating which 32-bit words contain differences.
|
||||
*/
|
||||
|
||||
static really_inline u32 diffrich384(m384 a, m384 b) {
|
||||
m128 z = zeroes128();
|
||||
a.lo = _mm_cmpeq_epi32(a.lo, b.lo);
|
||||
a.mid = _mm_cmpeq_epi32(a.mid, b.mid);
|
||||
a.hi = _mm_cmpeq_epi32(a.hi, b.hi);
|
||||
m128 packed = _mm_packs_epi16(_mm_packs_epi32(a.lo, a.mid),
|
||||
_mm_packs_epi32(a.hi, z));
|
||||
return ~(_mm_movemask_epi8(packed)) & 0xfff;
|
||||
}
|
||||
|
||||
#endif // HAVE_SIMD_128_BITS
|
||||
|
||||
/****
|
||||
**** 512-bit Primitives
|
||||
****/
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2015-2017, Intel Corporation
|
||||
* Copyright (c) 2020-2023, VectorCamp PC
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
@ -44,13 +45,14 @@
|
||||
#define DOUBLE_CASE_CLEAR 0xdfdf
|
||||
#define OCTO_CASE_CLEAR 0xdfdfdfdfdfdfdfdfULL
|
||||
|
||||
|
||||
#if !defined(VS_SIMDE_BACKEND)
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64)
|
||||
#include "util/arch/x86/bitutils.h"
|
||||
#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64)
|
||||
#include "util/arch/arm/bitutils.h"
|
||||
#elif defined(ARCH_PPC64EL)
|
||||
#include "util/arch/ppc64el/bitutils.h"
|
||||
#endif
|
||||
#else
|
||||
#include "util/arch/common/bitutils.h"
|
||||
#define clz32_impl clz32_impl_c
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2015-2017, Intel Corporation
|
||||
* Copyright (c) 2020-2021, VectorCamp PC
|
||||
* Copyright (c) 2020-2023, VectorCamp PC
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
@ -49,12 +49,16 @@ const u8 *first_zero_match_inverted(const u8 *buf, SuperVector<S> v, u16 const l
|
||||
template <u16 S>
|
||||
const u8 *last_zero_match_inverted(const u8 *buf, SuperVector<S> v, u16 len = S);
|
||||
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64) || defined(SIMDE_BACKEND)
|
||||
#if defined(VS_SIMDE_BACKEND)
|
||||
#include "util/arch/x86/match.hpp"
|
||||
#else
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64)
|
||||
#include "util/arch/x86/match.hpp"
|
||||
#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64)
|
||||
#include "util/arch/arm/match.hpp"
|
||||
#elif defined(ARCH_PPC64EL)
|
||||
#include "util/arch/ppc64el/match.hpp"
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#endif // MATCH_HPP
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2015-2017, Intel Corporation
|
||||
* Copyright (c) 2023, VectorCamp PC
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
@ -34,7 +35,16 @@
|
||||
#include "util/intrinsics.h"
|
||||
#include "ue2common.h"
|
||||
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64)
|
||||
#if defined(VS_SIMDE_BACKEND)
|
||||
#define VECTORSIZE 16
|
||||
#define SIMDE_ENABLE_NATIVE_ALIASES
|
||||
#if !defined(VS_SIMDE_NATIVE)
|
||||
#define SIMDE_NO_NATIVE
|
||||
#endif
|
||||
#include <simde/x86/sse4.2.h>
|
||||
typedef simde__m128i m128;
|
||||
#define HAVE_SIMD_128_BITS
|
||||
#elif defined(ARCH_IA32) || defined(ARCH_X86_64)
|
||||
#include "util/arch/x86/simd_types.h"
|
||||
#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64)
|
||||
#include "util/arch/arm/simd_types.h"
|
||||
@ -42,14 +52,6 @@
|
||||
#include "util/arch/ppc64el/simd_types.h"
|
||||
#endif
|
||||
|
||||
#if defined(SIMDE_BACKEND)
|
||||
#define VECTORSIZE 16
|
||||
#define SIMDE_ENABLE_NATIVE_ALIASES
|
||||
#define SIMDE_NO_NATIVE
|
||||
#include "simde/simde/x86/sse4.2.h"
|
||||
typedef simde__m128i m128;
|
||||
#define HAVE_SIMD_128_BITS
|
||||
#endif
|
||||
|
||||
#if !defined(m256) && !defined(HAVE_SIMD_256_BITS)
|
||||
typedef struct ALIGN_AVX_DIRECTIVE {m128 lo; m128 hi;} m256;
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2015-2020, Intel Corporation
|
||||
* Copyright (c) 2023, VectorCamp PC
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
@ -61,14 +62,16 @@ extern const char vbs_mask_data[];
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(VS_SIMDE_BACKEND)
|
||||
#include "util/arch/x86/simd_utils.h"
|
||||
#else
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64)
|
||||
#include "util/arch/x86/simd_utils.h"
|
||||
#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64)
|
||||
#include "util/arch/arm/simd_utils.h"
|
||||
#elif defined(ARCH_PPC64EL)
|
||||
#include "util/arch/ppc64el/simd_utils.h"
|
||||
#elif defined(SIMDE_BACKEND)
|
||||
#include "util/arch/simde/simd_utils.h"
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#include "util/arch/common/simd_utils.h"
|
||||
|
||||
@ -1,530 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2015-2017, Intel Corporation
|
||||
* Copyright (c) 2020-2021, VectorCamp PC
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef SIMD_IMPL_HPP
|
||||
#define SIMD_IMPL_HPP
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
|
||||
#include "ue2common.h"
|
||||
#include "util/arch.h"
|
||||
#include "util/unaligned.h"
|
||||
#include "util/supervector/supervector.hpp"
|
||||
|
||||
template<>
|
||||
really_inline SuperVector<16>::SuperVector(SuperVector const &other)
|
||||
{
|
||||
u.v128[0] = other.u.v128[0];
|
||||
}
|
||||
|
||||
template<>
|
||||
really_inline SuperVector<16>::SuperVector(typename base_type::type const v)
|
||||
{
|
||||
u.v128[0] = v;
|
||||
};
|
||||
|
||||
template<>
|
||||
template<>
|
||||
really_inline SuperVector<16>::SuperVector(int8_t const other)
|
||||
{
|
||||
u.v128[0] = _mm_set1_epi8(other);
|
||||
}
|
||||
|
||||
template<>
|
||||
template<>
|
||||
really_inline SuperVector<16>::SuperVector(uint8_t const other)
|
||||
{
|
||||
u.v128[0] = _mm_set1_epi8(static_cast<int8_t>(other));
|
||||
}
|
||||
|
||||
template<>
|
||||
template<>
|
||||
really_inline SuperVector<16>::SuperVector(int16_t const other)
|
||||
{
|
||||
u.v128[0] = _mm_set1_epi16(other);
|
||||
}
|
||||
|
||||
template<>
|
||||
template<>
|
||||
really_inline SuperVector<16>::SuperVector(uint16_t const other)
|
||||
{
|
||||
u.v128[0] = _mm_set1_epi16(static_cast<int16_t>(other));
|
||||
}
|
||||
|
||||
template<>
|
||||
template<>
|
||||
really_inline SuperVector<16>::SuperVector(int32_t const other)
|
||||
{
|
||||
u.v128[0] = _mm_set1_epi32(other);
|
||||
}
|
||||
|
||||
template<>
|
||||
template<>
|
||||
really_inline SuperVector<16>::SuperVector(uint32_t const other)
|
||||
{
|
||||
u.v128[0] = _mm_set1_epi32(static_cast<int32_t>(other));
|
||||
}
|
||||
|
||||
template<>
|
||||
template<>
|
||||
really_inline SuperVector<16>::SuperVector(int64_t const other)
|
||||
{
|
||||
u.v128[0] = _mm_set1_epi64x(other);
|
||||
}
|
||||
|
||||
template<>
|
||||
template<>
|
||||
really_inline SuperVector<16>::SuperVector(uint64_t const other)
|
||||
{
|
||||
u.v128[0] = _mm_set1_epi64x(static_cast<int64_t>(other));
|
||||
}
|
||||
|
||||
// Constants
|
||||
template<>
|
||||
really_inline SuperVector<16> SuperVector<16>::Ones()
|
||||
{
|
||||
return {_mm_set1_epi8(0xFF)};
|
||||
}
|
||||
|
||||
template<>
|
||||
really_inline SuperVector<16> SuperVector<16>::Zeroes(void)
|
||||
{
|
||||
return {_mm_set1_epi8(0)};
|
||||
}
|
||||
|
||||
// Methods
|
||||
|
||||
template <>
|
||||
really_inline void SuperVector<16>::operator=(SuperVector<16> const &other)
|
||||
{
|
||||
u.v128[0] = other.u.v128[0];
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::operator&(SuperVector<16> const &b) const
|
||||
{
|
||||
return {_mm_and_si128(u.v128[0], b.u.v128[0])};
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::operator|(SuperVector<16> const &b) const
|
||||
{
|
||||
return {_mm_or_si128(u.v128[0], b.u.v128[0])};
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::operator^(SuperVector<16> const &b) const
|
||||
{
|
||||
return {_mm_xor_si128(u.v128[0], b.u.v128[0])};
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::operator!() const
|
||||
{
|
||||
return {_mm_xor_si128(u.v128[0], u.v128[0])};
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::opandnot(SuperVector<16> const &b) const
|
||||
{
|
||||
return {_mm_andnot_si128(u.v128[0], b.u.v128[0])};
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::operator==(SuperVector<16> const &b) const
|
||||
{
|
||||
return {_mm_cmpeq_epi8(u.v128[0], b.u.v128[0])};
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::operator!=(SuperVector<16> const &b) const
|
||||
{
|
||||
return !(*this == b);
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::operator>(SuperVector<16> const &b) const
|
||||
{
|
||||
return {_mm_cmpgt_epi8(u.v128[0], b.u.v128[0])};
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::operator<(SuperVector<16> const &b) const
|
||||
{
|
||||
return {_mm_cmplt_epi8(u.v128[0], b.u.v128[0])};
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::operator>=(SuperVector<16> const &b) const
|
||||
{
|
||||
return !(*this < b);
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::operator<=(SuperVector<16> const &b) const
|
||||
{
|
||||
return !(*this > b);
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::eq(SuperVector<16> const &b) const
|
||||
{
|
||||
return (*this == b);
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline typename SuperVector<16>::comparemask_type
|
||||
SuperVector<16>::comparemask(void) const {
|
||||
return (u32)_mm_movemask_epi8(u.v128[0]);
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline typename SuperVector<16>::comparemask_type
|
||||
SuperVector<16>::eqmask(SuperVector<16> const b) const {
|
||||
return eq(b).comparemask();
|
||||
}
|
||||
|
||||
template <> really_inline u32 SuperVector<16>::mask_width() { return 1; }
|
||||
|
||||
template <>
|
||||
really_inline typename SuperVector<16>::comparemask_type
|
||||
SuperVector<16>::iteration_mask(
|
||||
typename SuperVector<16>::comparemask_type mask) {
|
||||
return mask;
|
||||
}
|
||||
|
||||
// template <>
|
||||
// template<uint8_t N>
|
||||
// really_inline SuperVector<16> SuperVector<16>::vshl_8_imm() const
|
||||
// {
|
||||
// const uint8_t i = N;
|
||||
// return {_mm_slli_epi8(u.v128[0], i)};
|
||||
// }
|
||||
|
||||
template <>
|
||||
template<uint8_t N>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshl_16_imm() const
|
||||
{
|
||||
return {_mm_slli_epi16(u.v128[0], N)};
|
||||
}
|
||||
|
||||
template <>
|
||||
template<uint8_t N>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshl_32_imm() const
|
||||
{
|
||||
return {_mm_slli_epi32(u.v128[0], N)};
|
||||
}
|
||||
|
||||
template <>
|
||||
template<uint8_t N>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshl_64_imm() const
|
||||
{
|
||||
return {_mm_slli_epi64(u.v128[0], N)};
|
||||
}
|
||||
|
||||
template <>
|
||||
template<uint8_t N>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshl_128_imm() const
|
||||
{
|
||||
return {_mm_slli_si128(u.v128[0], N)};
|
||||
}
|
||||
|
||||
template <>
|
||||
template<uint8_t N>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshl_imm() const
|
||||
{
|
||||
return vshl_128_imm<N>();
|
||||
}
|
||||
|
||||
// template <>
|
||||
// template<uint8_t N>
|
||||
// really_inline SuperVector<16> SuperVector<16>::vshr_8_imm() const
|
||||
// {
|
||||
// return {_mm_srli_epi8(u.v128[0], N)};
|
||||
// }
|
||||
|
||||
template <>
|
||||
template<uint8_t N>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshr_16_imm() const
|
||||
{
|
||||
return {_mm_srli_epi16(u.v128[0], N)};
|
||||
}
|
||||
|
||||
template <>
|
||||
template<uint8_t N>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshr_32_imm() const
|
||||
{
|
||||
return {_mm_srli_epi32(u.v128[0], N)};
|
||||
}
|
||||
|
||||
template <>
|
||||
template<uint8_t N>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshr_64_imm() const
|
||||
{
|
||||
return {_mm_srli_epi64(u.v128[0], N)};
|
||||
}
|
||||
|
||||
template <>
|
||||
template<uint8_t N>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshr_128_imm() const
|
||||
{
|
||||
return {_mm_srli_si128(u.v128[0], N)};
|
||||
}
|
||||
|
||||
template <>
|
||||
template<uint8_t N>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshr_imm() const
|
||||
{
|
||||
return vshr_128_imm<N>();
|
||||
}
|
||||
|
||||
#if !defined(HS_OPTIMIZE)
|
||||
template SuperVector<16> SuperVector<16>::vshl_16_imm<1>() const;
|
||||
template SuperVector<16> SuperVector<16>::vshl_64_imm<1>() const;
|
||||
template SuperVector<16> SuperVector<16>::vshl_64_imm<4>() const;
|
||||
template SuperVector<16> SuperVector<16>::vshl_128_imm<1>() const;
|
||||
template SuperVector<16> SuperVector<16>::vshl_128_imm<4>() const;
|
||||
template SuperVector<16> SuperVector<16>::vshr_16_imm<1>() const;
|
||||
template SuperVector<16> SuperVector<16>::vshr_64_imm<1>() const;
|
||||
template SuperVector<16> SuperVector<16>::vshr_64_imm<4>() const;
|
||||
template SuperVector<16> SuperVector<16>::vshr_128_imm<1>() const;
|
||||
template SuperVector<16> SuperVector<16>::vshr_128_imm<4>() const;
|
||||
#endif
|
||||
|
||||
// template <>
|
||||
// really_inline SuperVector<16> SuperVector<16>::vshl_8 (uint8_t const N) const
|
||||
// {
|
||||
// Unroller<0, 15>::iterator([&,v=this](int i) { if (N == i) return {_mm_slli_epi8(v->u.v128[0], i)}; });
|
||||
// if (N == 16) return Zeroes();
|
||||
// }
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshl_16 (uint8_t const N) const
|
||||
{
|
||||
#if defined(HAVE__BUILTIN_CONSTANT_P)
|
||||
if (__builtin_constant_p(N)) {
|
||||
return {_mm_slli_epi16(u.v128[0], N)};
|
||||
}
|
||||
#endif
|
||||
if (N == 0) return *this;
|
||||
if (N == 16) return Zeroes();
|
||||
SuperVector result;
|
||||
Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_slli_epi16(v->u.v128[0], n)}; });
|
||||
return result;
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshl_32 (uint8_t const N) const
|
||||
{
|
||||
if (N == 0) return *this;
|
||||
if (N == 16) return Zeroes();
|
||||
SuperVector result;
|
||||
Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_slli_epi32(v->u.v128[0], n)}; });
|
||||
return result;
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshl_64 (uint8_t const N) const
|
||||
{
|
||||
if (N == 0) return *this;
|
||||
if (N == 16) return Zeroes();
|
||||
SuperVector result;
|
||||
Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_slli_epi64(v->u.v128[0], n)}; });
|
||||
return result;
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshl_128(uint8_t const N) const
|
||||
{
|
||||
if (N == 0) return *this;
|
||||
if (N == 16) return Zeroes();
|
||||
SuperVector result;
|
||||
Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_slli_si128(v->u.v128[0], n)}; });
|
||||
return result;
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshl(uint8_t const N) const
|
||||
{
|
||||
return vshl_128(N);
|
||||
}
|
||||
|
||||
// template <>
|
||||
// really_inline SuperVector<16> SuperVector<16>::vshr_8 (uint8_t const N) const
|
||||
// {
|
||||
// SuperVector<16> result;
|
||||
// Unroller<0, 15>::iterator([&,v=this](uint8_t const i) { if (N == i) result = {_mm_srli_epi8(v->u.v128[0], i)}; });
|
||||
// if (N == 16) result = Zeroes();
|
||||
// return result;
|
||||
// }
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshr_16 (uint8_t const N) const
|
||||
{
|
||||
if (N == 0) return *this;
|
||||
if (N == 16) return Zeroes();
|
||||
SuperVector result;
|
||||
Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_srli_epi16(v->u.v128[0], n)}; });
|
||||
return result;
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshr_32 (uint8_t const N) const
|
||||
{
|
||||
if (N == 0) return *this;
|
||||
if (N == 16) return Zeroes();
|
||||
SuperVector result;
|
||||
Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_srli_epi32(v->u.v128[0], n)}; });
|
||||
return result;
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshr_64 (uint8_t const N) const
|
||||
{
|
||||
if (N == 0) return *this;
|
||||
if (N == 16) return Zeroes();
|
||||
SuperVector result;
|
||||
Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_srli_epi64(v->u.v128[0], n)}; });
|
||||
return result;
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshr_128(uint8_t const N) const
|
||||
{
|
||||
if (N == 0) return *this;
|
||||
if (N == 16) return Zeroes();
|
||||
SuperVector result;
|
||||
Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {_mm_srli_si128(v->u.v128[0], n)}; });
|
||||
return result;
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::vshr(uint8_t const N) const
|
||||
{
|
||||
return vshr_128(N);
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::operator>>(uint8_t const N) const
|
||||
{
|
||||
return vshr_128(N);
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::operator<<(uint8_t const N) const
|
||||
{
|
||||
return vshl_128(N);
|
||||
}
|
||||
|
||||
template<>
|
||||
really_inline SuperVector<16> SuperVector<16>::Ones_vshr(uint8_t const N)
|
||||
{
|
||||
if (N == 0) return Ones();
|
||||
else return Ones().vshr_128(N);
|
||||
}
|
||||
|
||||
template<>
|
||||
really_inline SuperVector<16> SuperVector<16>::Ones_vshl(uint8_t const N)
|
||||
{
|
||||
if (N == 0) return Ones();
|
||||
else return Ones().vshr_128(N);
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::loadu(void const *ptr)
|
||||
{
|
||||
return _mm_loadu_si128((const m128 *)ptr);
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::load(void const *ptr)
|
||||
{
|
||||
assert(ISALIGNED_N(ptr, alignof(SuperVector::size)));
|
||||
ptr = vectorscan_assume_aligned(ptr, SuperVector::size);
|
||||
return _mm_load_si128((const m128 *)ptr);
|
||||
}
|
||||
|
||||
template <>
|
||||
really_inline SuperVector<16> SuperVector<16>::loadu_maskz(void const *ptr, uint8_t const len)
|
||||
{
|
||||
SuperVector mask = Ones_vshr(16 -len);
|
||||
SuperVector v = _mm_loadu_si128((const m128 *)ptr);
|
||||
return mask & v;
|
||||
}
|
||||
|
||||
template<>
|
||||
really_inline SuperVector<16> SuperVector<16>::alignr(SuperVector<16> &other, int8_t offset)
|
||||
{
|
||||
#if defined(HAVE__BUILTIN_CONSTANT_P)
|
||||
if (__builtin_constant_p(offset)) {
|
||||
if (offset == 16) {
|
||||
return *this;
|
||||
} else {
|
||||
return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], offset)};
|
||||
}
|
||||
}
|
||||
#endif
|
||||
switch(offset) {
|
||||
case 0: return other; break;
|
||||
case 1: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 1)}; break;
|
||||
case 2: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 2)}; break;
|
||||
case 3: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 3)}; break;
|
||||
case 4: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 4)}; break;
|
||||
case 5: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 5)}; break;
|
||||
case 6: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 6)}; break;
|
||||
case 7: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 7)}; break;
|
||||
case 8: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 8)}; break;
|
||||
case 9: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 9)}; break;
|
||||
case 10: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 10)}; break;
|
||||
case 11: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 11)}; break;
|
||||
case 12: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 12)}; break;
|
||||
case 13: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 13)}; break;
|
||||
case 14: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 14)}; break;
|
||||
case 15: return {_mm_alignr_epi8(u.v128[0], other.u.v128[0], 15)}; break;
|
||||
default: break;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
template<>
|
||||
template<>
|
||||
really_inline SuperVector<16> SuperVector<16>::pshufb<true>(SuperVector<16> b)
|
||||
{
|
||||
return {_mm_shuffle_epi8(u.v128[0], b.u.v128[0])};
|
||||
}
|
||||
|
||||
template<>
|
||||
really_inline SuperVector<16> SuperVector<16>::pshufb_maskz(SuperVector<16> b, uint8_t const len)
|
||||
{
|
||||
SuperVector mask = Ones_vshr(16 -len);
|
||||
return mask & pshufb(b);
|
||||
}
|
||||
|
||||
#endif // SIMD_IMPL_HPP
|
||||
@ -34,6 +34,9 @@
|
||||
#include <cstdio>
|
||||
#include <type_traits>
|
||||
|
||||
#if defined(VS_SIMDE_BACKEND)
|
||||
#include "util/supervector/arch/x86/types.hpp"
|
||||
#else
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64)
|
||||
#include "util/supervector/arch/x86/types.hpp"
|
||||
#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64)
|
||||
@ -41,6 +44,7 @@
|
||||
#elif defined(ARCH_PPC64EL)
|
||||
#include "util/supervector/arch/ppc64el/types.hpp"
|
||||
#endif
|
||||
#endif // VS_SIMDE_BACKEND
|
||||
|
||||
#if defined(HAVE_SIMD_512_BITS)
|
||||
using Z_TYPE = u64a;
|
||||
@ -57,7 +61,7 @@ using Z_TYPE = u32;
|
||||
#define DOUBLE_LOAD_MASK(l) (((1ULL) << (l)) - 1ULL)
|
||||
#define SINGLE_LOAD_MASK(l) (((1ULL) << (l)) - 1ULL)
|
||||
#elif defined(HAVE_SIMD_128_BITS)
|
||||
#if defined(ARCH_ARM32) || defined(ARCH_AARCH64)
|
||||
#if !defined(VS_SIMDE_BACKEND) && (defined(ARCH_ARM32) || defined(ARCH_AARCH64))
|
||||
using Z_TYPE = u64a;
|
||||
#define Z_BITS 64
|
||||
#define Z_POSSHIFT 2
|
||||
@ -175,7 +179,7 @@ public:
|
||||
typename BaseVector<32>::type ALIGN_ATTR(BaseVector<32>::size) v256[SIZE / BaseVector<32>::size];
|
||||
typename BaseVector<64>::type ALIGN_ATTR(BaseVector<64>::size) v512[SIZE / BaseVector<64>::size];
|
||||
|
||||
#if defined(ARCH_ARM32) || defined(ARCH_AARCH64) || defined(ARCH_PPC64EL)
|
||||
#if !defined(VS_SIMDE_BACKEND) && (defined(ARCH_ARM32) || defined(ARCH_AARCH64) || defined(ARCH_PPC64EL))
|
||||
uint64x2_t ALIGN_ATTR(BaseVector<16>::size) u64x2[SIZE / BaseVector<16>::size];
|
||||
int64x2_t ALIGN_ATTR(BaseVector<16>::size) s64x2[SIZE / BaseVector<16>::size];
|
||||
uint32x4_t ALIGN_ATTR(BaseVector<16>::size) u32x4[SIZE / BaseVector<16>::size];
|
||||
@ -382,14 +386,16 @@ struct Unroller<End, End>
|
||||
};
|
||||
|
||||
#if defined(HS_OPTIMIZE)
|
||||
#if defined(VS_SIMDE_BACKEND)
|
||||
#include "util/supervector/arch/x86/impl.cpp"
|
||||
#else
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64)
|
||||
#include "util/supervector/arch/x86/impl.cpp"
|
||||
#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64)
|
||||
#include "util/supervector/arch/arm/impl.cpp"
|
||||
#elif defined(ARCH_PPC64EL)
|
||||
#include "util/supervector/arch/ppc64el/impl.cpp"
|
||||
#elif defined(SIMDE_BACKEND)
|
||||
#include "util/supervector/arch/simde/impl.cpp"
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2015-2017, Intel Corporation
|
||||
* Copyright (c) 2023, VectorCamp PC
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
@ -667,7 +668,7 @@ TEST(SimdUtilsTest, movq) {
|
||||
ASSERT_EQ(0, memcmp(cmp, &r, sizeof(r)));
|
||||
|
||||
#if defined(HAVE_SIMD_128_BITS)
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64) || defined(SIMDE_BACKEND)
|
||||
#if defined(ARCH_IA32) || defined(ARCH_X86_64) || defined(VS_SIMDE_BACKEND)
|
||||
simd = _mm_set_epi64x(~0LL, 0x123456789abcdef);
|
||||
#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64)
|
||||
int64x2_t a = { 0x123456789abcdefLL, ~0LL };
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user