diff --git a/CMakeLists.txt b/CMakeLists.txt index 05e6a5c7..92abf6dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -277,8 +277,6 @@ elseif (ARCH_ARM32 OR ARCH_AARCH64) message(FATAL_ERROR "arm_sve.h is required to build for SVE.") endif() endif() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -flax-vector-conversions") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flax-vector-conversions") endif() CHECK_FUNCTION_EXISTS(posix_memalign HAVE_POSIX_MEMALIGN) diff --git a/src/util/arch/arm/match.hpp b/src/util/arch/arm/match.hpp index 46d84d06..e7f757bd 100644 --- a/src/util/arch/arm/match.hpp +++ b/src/util/arch/arm/match.hpp @@ -30,8 +30,8 @@ template <> really_really_inline const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> mask) { - uint32x4_t res_t = vreinterpretq_u32_u8(mask.u.v128[0]); - uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(res_t, res_t)), 0); + uint32x4_t m = mask.u.u32x4[0]; + uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0); if (vmax != 0) { typename SuperVector<16>::movemask_type z = mask.movemask(); DEBUG_PRINTF("z %08x\n", z); @@ -49,8 +49,8 @@ const u8 *firstMatch<16>(const u8 *buf, SuperVector<16> mask) { template <> really_really_inline const u8 *lastMatch<16>(const u8 *buf, SuperVector<16> mask) { - uint32x4_t res_t = vreinterpretq_u32_u8(mask.u.v128[0]); - uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(res_t, res_t)), 0); + uint32x4_t m = mask.u.u32x4[0]; + uint64_t vmax = vgetq_lane_u64 (vreinterpretq_u64_u32 (vpmaxq_u32(m, m)), 0); if (vmax != 0) { typename SuperVector<16>::movemask_type z = mask.movemask(); DEBUG_PRINTF("buf %p z %08x \n", buf, z); diff --git a/src/util/arch/arm/simd_utils.h b/src/util/arch/arm/simd_utils.h index 24851773..630cac93 100644 --- a/src/util/arch/arm/simd_utils.h +++ b/src/util/arch/arm/simd_utils.h @@ -100,7 +100,7 @@ static really_inline int isnonzero128(m128 a) { */ static really_inline u32 diffrich128(m128 a, m128 b) { static const uint32x4_t movemask = { 1, 2, 4, 8 }; - return vaddvq_u32(vandq_u32(vmvnq_s32(vceqq_s32((int32x4_t)a, (int32x4_t)b)), movemask)); + return vaddvq_u32(vandq_u32(vmvnq_u32(vceqq_u32((uint32x4_t)a, (uint32x4_t)b)), movemask)); } /** @@ -109,53 +109,53 @@ static really_inline u32 diffrich128(m128 a, m128 b) { */ static really_inline u32 diffrich64_128(m128 a, m128 b) { static const uint64x2_t movemask = { 1, 4 }; - return vaddvq_u64(vandq_u64(vmvnq_s32(vceqq_s64((int64x2_t)a, (int64x2_t)b)), movemask)); + return (u32) vaddvq_u64(vandq_u64((uint64x2_t)vmvnq_u32((uint32x4_t)vceqq_u64((uint64x2_t)a, (uint64x2_t)b)), movemask)); } static really_really_inline m128 add_2x64(m128 a, m128 b) { - return (m128) vaddq_u64((int64x2_t)a, (int64x2_t)b); + return (m128) vaddq_u64((uint64x2_t)a, (uint64x2_t)b); } static really_really_inline m128 sub_2x64(m128 a, m128 b) { - return (m128) vsubq_u64((int64x2_t)a, (int64x2_t)b); + return (m128) vsubq_u64((uint64x2_t)a, (uint64x2_t)b); } static really_really_inline m128 lshift_m128(m128 a, unsigned b) { - return (m128) vshlq_n_s32((int64x2_t)a, b); + return (m128) vshlq_n_u32((uint32x4_t)a, b); } static really_really_inline m128 rshift_m128(m128 a, unsigned b) { - return (m128) vshrq_n_s32((int64x2_t)a, b); + return (m128) vshrq_n_u32((uint32x4_t)a, b); } static really_really_inline m128 lshift64_m128(m128 a, unsigned b) { - return (m128) vshlq_n_s64((int64x2_t)a, b); + return (m128) vshlq_n_u64((uint64x2_t)a, b); } static really_really_inline m128 rshift64_m128(m128 a, unsigned b) { - return (m128) vshrq_n_s64((int64x2_t)a, b); + return (m128) vshrq_n_u64((uint64x2_t)a, b); } static really_inline m128 eq128(m128 a, m128 b) { - return (m128) vceqq_s8((int8x16_t)a, (int8x16_t)b); + return (m128) vceqq_u8((uint8x16_t)a, (uint8x16_t)b); } static really_inline m128 eq64_m128(m128 a, m128 b) { - return (m128) vceqq_u64((int64x2_t)a, (int64x2_t)b); + return (m128) vceqq_u64((uint64x2_t)a, (uint64x2_t)b); } static really_inline u32 movemask128(m128 a) { static const uint8x16_t powers = { 1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128 }; // Compute the mask from the input - uint64x2_t mask = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vandq_u8((uint8x16_t)a, powers)))); - uint64x2_t mask1 = (m128)vextq_s8(mask, zeroes128(), 7); + uint8x16_t mask = (uint8x16_t) vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vandq_u8((uint8x16_t)a, powers)))); + uint8x16_t mask1 = vextq_u8(mask, (uint8x16_t)zeroes128(), 7); mask = vorrq_u8(mask, mask1); // Get the resulting bytes @@ -187,7 +187,7 @@ static really_inline u64a movq(const m128 in) { /* another form of movq */ static really_inline m128 load_m128_from_u64a(const u64a *p) { - return (m128) vsetq_lane_u64(*p, zeroes128(), 0); + return (m128) vsetq_lane_u64(*p, (uint64x2_t) zeroes128(), 0); } static really_inline u32 extract32from128(const m128 in, unsigned imm) { @@ -220,10 +220,10 @@ static really_inline u64a extract64from128(const m128 in, unsigned imm) { #else switch (imm) { case 0: - return vgetq_lane_u64((uint32x4_t) in, 0); + return vgetq_lane_u64((uint64x2_t) in, 0); break; case 1: - return vgetq_lane_u64((uint32x4_t) in, 1); + return vgetq_lane_u64((uint64x2_t) in, 1); break; default: return 0; @@ -233,11 +233,11 @@ static really_inline u64a extract64from128(const m128 in, unsigned imm) { } static really_inline m128 low64from128(const m128 in) { - return vcombine_u64(vget_low_u64(in), vdup_n_u64(0)); + return (m128) vcombine_u64(vget_low_u64((uint64x2_t)in), vdup_n_u64(0)); } static really_inline m128 high64from128(const m128 in) { - return vcombine_u64(vget_high_u64(in), vdup_n_u64(0)); + return (m128) vcombine_u64(vget_high_u64((uint64x2_t)in), vdup_n_u64(0)); } static really_inline m128 add128(m128 a, m128 b) { @@ -257,7 +257,7 @@ static really_inline m128 or128(m128 a, m128 b) { } static really_inline m128 andnot128(m128 a, m128 b) { - return (m128) (m128) vandq_s8( vmvnq_s8(a), b); + return (m128) vandq_s8( vmvnq_s8((int8x16_t) a), (int8x16_t) b); } // aligned load @@ -401,12 +401,12 @@ m128 pshufb_m128(m128 a, m128 b) { static really_inline m128 max_u8_m128(m128 a, m128 b) { - return (m128) vmaxq_u8((int8x16_t)a, (int8x16_t)b); + return (m128) vmaxq_u8((uint8x16_t)a, (uint8x16_t)b); } static really_inline m128 min_u8_m128(m128 a, m128 b) { - return (m128) vminq_u8((int8x16_t)a, (int8x16_t)b); + return (m128) vminq_u8((uint8x16_t)a, (uint8x16_t)b); } static really_inline diff --git a/src/util/supervector/arch/arm/impl.cpp b/src/util/supervector/arch/arm/impl.cpp index 34e5486d..f804abeb 100644 --- a/src/util/supervector/arch/arm/impl.cpp +++ b/src/util/supervector/arch/arm/impl.cpp @@ -45,72 +45,114 @@ really_inline SuperVector<16>::SuperVector(typename base_type::type const v) template<> template<> -really_inline SuperVector<16>::SuperVector(int8x16_t const other) +really_inline SuperVector<16>::SuperVector(int8x16_t other) { - u.v128[0] = static_cast(other); + u.s8x16[0] = other; } template<> template<> -really_inline SuperVector<16>::SuperVector(uint8x16_t const other) +really_inline SuperVector<16>::SuperVector(uint8x16_t other) { - u.v128[0] = static_cast(other); + u.u8x16[0] = other; +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(int16x8_t other) +{ + u.s16x8[0] = other; +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(uint16x8_t other) +{ + u.u16x8[0] = other; +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(int32x4_t other) +{ + u.s32x4[0] = other; +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(uint32x4_t other) +{ + u.u32x4[0] = other; +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(int64x2_t other) +{ + u.s64x2[0] = other; +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(uint64x2_t other) +{ + u.u64x2[0] = other; } template<> template<> really_inline SuperVector<16>::SuperVector(int8_t const other) { - u.v128[0] = vdupq_n_s8(other); + u.s8x16[0] = vdupq_n_s8(other); } template<> template<> really_inline SuperVector<16>::SuperVector(uint8_t const other) { - u.v128[0] = vdupq_n_u8(other); + u.u8x16[0] = vdupq_n_u8(other); } template<> template<> really_inline SuperVector<16>::SuperVector(int16_t const other) { - u.v128[0] = vdupq_n_s16(other); + u.s16x8[0] = vdupq_n_s16(other); } template<> template<> really_inline SuperVector<16>::SuperVector(uint16_t const other) { - u.v128[0] = vdupq_n_u16(other); + u.u16x8[0] = vdupq_n_u16(other); } template<> template<> really_inline SuperVector<16>::SuperVector(int32_t const other) { - u.v128[0] = vdupq_n_s32(other); + u.s32x4[0] = vdupq_n_s32(other); } template<> template<> really_inline SuperVector<16>::SuperVector(uint32_t const other) { - u.v128[0] = vdupq_n_u32(other); + u.u32x4[0] = vdupq_n_u32(other); } template<> template<> really_inline SuperVector<16>::SuperVector(int64_t const other) { - u.v128[0] = vdupq_n_s64(other); + u.s64x2[0] = vdupq_n_s64(other); } template<> template<> really_inline SuperVector<16>::SuperVector(uint64_t const other) { - u.v128[0] = vdupq_n_u64(other); + u.u64x2[0] = vdupq_n_u64(other); } // Constants @@ -137,37 +179,37 @@ really_inline void SuperVector<16>::operator=(SuperVector<16> const &other) template <> really_inline SuperVector<16> SuperVector<16>::operator&(SuperVector<16> const &b) const { - return {vandq_s8(u.v128[0], b.u.v128[0])}; + return {vandq_u8(u.u8x16[0], b.u.u8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator|(SuperVector<16> const &b) const { - return {vorrq_s8(u.v128[0], b.u.v128[0])}; + return {vorrq_u8(u.u8x16[0], b.u.u8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator^(SuperVector<16> const &b) const { - return {veorq_s8(u.v128[0], b.u.v128[0])}; + return {veorq_u8(u.u8x16[0], b.u.u8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator!() const { - return {vmvnq_s8(u.v128[0])}; + return {vmvnq_u8(u.u8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::opandnot(SuperVector<16> const &b) const { - return {vandq_s8(vmvnq_s8(u.v128[0]), b.u.v128[0])}; + return {vandq_u8(vmvnq_u8(u.u8x16[0]), b.u.u8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator==(SuperVector<16> const &b) const { - return {vceqq_s8((int16x8_t)u.v128[0], (int16x8_t)b.u.v128[0])}; + return {vceqq_u8(u.u8x16[0], b.u.u8x16[0])}; } template <> @@ -179,25 +221,25 @@ really_inline SuperVector<16> SuperVector<16>::operator!=(SuperVector<16> const template <> really_inline SuperVector<16> SuperVector<16>::operator>(SuperVector<16> const &b) const { - return {vcgtq_s8((int16x8_t)u.v128[0], (int16x8_t)b.u.v128[0])}; + return {vcgtq_s8(u.s8x16[0], b.u.s8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator>=(SuperVector<16> const &b) const { - return {vcgeq_s8((int16x8_t)u.v128[0], (int16x8_t)b.u.v128[0])}; + return {vcgeq_u8(u.u8x16[0], b.u.u8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator<(SuperVector<16> const &b) const { - return {vcltq_s8((int16x8_t)u.v128[0], (int16x8_t)b.u.v128[0])}; + return {vcltq_s8(u.s8x16[0], b.u.s8x16[0])}; } template <> really_inline SuperVector<16> SuperVector<16>::operator<=(SuperVector<16> const &b) const { - return {vcgeq_s8((int16x8_t)u.v128[0], (int16x8_t)b.u.v128[0])}; + return {vcgeq_s8(u.s8x16[0], b.u.s8x16[0])}; } template <> @@ -212,9 +254,9 @@ really_inline typename SuperVector<16>::movemask_type SuperVector<16>::movemask( SuperVector powers{0x8040201008040201UL}; // Compute the mask from the input - uint64x2_t mask = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vandq_u8((uint16x8_t)u.v128[0], powers.u.v128[0])))); - uint64x2_t mask1 = (m128)vextq_s8(mask, vdupq_n_u8(0), 7); - mask = vorrq_u8(mask, mask1); + uint8x16_t mask = (uint8x16_t) vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vandq_u8(u.u8x16[0], powers.u.u8x16[0])))); + uint64x2_t mask1 = (uint64x2_t) vextq_u8(mask, vdupq_n_u8(0), 7); + mask = vorrq_u8(mask, (uint8x16_t) mask1); // Get the resulting bytes uint16_t output; @@ -232,35 +274,35 @@ template <> template really_inline SuperVector<16> SuperVector<16>::vshl_8_imm() const { - return {(m128)vshlq_n_s8(u.v128[0], N)}; + return {vshlq_n_u8(u.u8x16[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshl_16_imm() const { - return {(m128)vshlq_n_s16(u.v128[0], N)}; + return {vshlq_n_u16(u.u16x8[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshl_32_imm() const { - return {(m128)vshlq_n_s32(u.v128[0], N)}; + return {vshlq_n_u32(u.u32x4[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshl_64_imm() const { - return {(m128)vshlq_n_s64(u.v128[0], N)}; + return {vshlq_n_u64(u.u64x2[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshl_128_imm() const { - return {vextq_s8(vdupq_n_u8(0), (int16x8_t)u.v128[0], 16 - N)}; + return {vextq_u8(vdupq_n_u8(0), u.u8x16[0], 16 - N)}; } template <> @@ -274,35 +316,35 @@ template <> template really_inline SuperVector<16> SuperVector<16>::vshr_8_imm() const { - return {(m128)vshrq_n_s8(u.v128[0], N)}; + return {vshrq_n_u8(u.u8x16[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshr_16_imm() const { - return {(m128)vshrq_n_s16(u.v128[0], N)}; + return {vshrq_n_u16(u.u16x8[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshr_32_imm() const { - return {(m128)vshrq_n_s32(u.v128[0], N)}; + return {vshrq_n_u32(u.u32x4[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshr_64_imm() const { - return {(m128)vshrq_n_s64(u.v128[0], N)}; + return {vshrq_n_u64(u.u64x2[0], N)}; } template <> template really_inline SuperVector<16> SuperVector<16>::vshr_128_imm() const { - return {vextq_s8((int16x8_t)u.v128[0], vdupq_n_u8(0), N)}; + return {vextq_u8(u.u8x16[0], vdupq_n_u8(0), N)}; } template <> @@ -334,7 +376,7 @@ really_inline SuperVector<16> SuperVector<16>::vshl_8 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshlq_n_s8(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshlq_n_u8(u.u8x16[0], n)}; }); return result; } @@ -344,7 +386,7 @@ really_inline SuperVector<16> SuperVector<16>::vshl_16 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshlq_n_s16(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshlq_n_u16(u.u16x8[0], n)}; }); return result; } @@ -354,7 +396,7 @@ really_inline SuperVector<16> SuperVector<16>::vshl_32 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshlq_n_s32(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshlq_n_u32(u.u32x4[0], n)}; }); return result; } @@ -364,7 +406,7 @@ really_inline SuperVector<16> SuperVector<16>::vshl_64 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshlq_n_s64(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshlq_n_u64(u.u64x2[0], n)}; }); return result; } @@ -374,7 +416,7 @@ really_inline SuperVector<16> SuperVector<16>::vshl_128(uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vextq_s8(vdupq_n_u8(0), (int16x8_t)u.v128[0], 16 - n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vextq_u8(vdupq_n_u8(0), u.u8x16[0], 16 - n)}; }); return result; } @@ -390,7 +432,7 @@ really_inline SuperVector<16> SuperVector<16>::vshr_8 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshrq_n_s8(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshrq_n_u8(u.u8x16[0], n)}; }); return result; } @@ -400,7 +442,7 @@ really_inline SuperVector<16> SuperVector<16>::vshr_16 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshrq_n_s16(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshrq_n_u16(u.u16x8[0], n)}; }); return result; } @@ -410,7 +452,7 @@ really_inline SuperVector<16> SuperVector<16>::vshr_32 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshrq_n_s32(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshrq_n_u32(u.u32x4[0], n)}; }); return result; } @@ -420,7 +462,7 @@ really_inline SuperVector<16> SuperVector<16>::vshr_64 (uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {(m128)vshrq_n_s64(u.v128[0], n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vshrq_n_u64(u.u64x2[0], n)}; }); return result; } @@ -430,7 +472,7 @@ really_inline SuperVector<16> SuperVector<16>::vshr_128(uint8_t const N) const if (N == 0) return *this; if (N == 16) return Zeroes(); SuperVector result; - Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vextq_s8((int16x8_t)u.v128[0], vdupq_n_u8(0), n)}; }); + Unroller<1, 16>::iterator([&,v=this](auto const i) { constexpr uint8_t n = i.value; if (N == n) result = {vextq_u8(u.u8x16[0], vdupq_n_u8(0), n)}; }); return result; } @@ -444,7 +486,7 @@ really_inline SuperVector<16> SuperVector<16>::vshr(uint8_t const N) const template <> really_inline SuperVector<16> SuperVector<16>::operator>>(uint8_t const N) const { - return {vextq_s8((int16x8_t)u.v128[0], vdupq_n_u8(0), N)}; + return {vextq_u8(u.u8x16[0], vdupq_n_u8(0), N)}; } #else template <> @@ -458,7 +500,7 @@ really_inline SuperVector<16> SuperVector<16>::operator>>(uint8_t const N) const template <> really_inline SuperVector<16> SuperVector<16>::operator<<(uint8_t const N) const { - return {vextq_s8(vdupq_n_u8(0), (int16x8_t)u.v128[0], 16 - N)}; + return {vextq_u8(vdupq_n_u8(0), u.u8x16[0], 16 - N)}; } #else template <> @@ -512,7 +554,7 @@ really_inline SuperVector<16> SuperVector<16>::alignr(SuperVector<16> &other, in if (offset == 16) { return *this; } else { - return {vextq_s8((int16x8_t)other.u.v128[0], (int16x8_t)u.v128[0], offset)}; + return {vextq_u8(other.u.u8x16[0], u.u8x16[0], offset)}; } } #else @@ -521,21 +563,21 @@ really_inline SuperVector<16> SuperVector<16>::alignr(SuperVector<16> &other, in { switch(offset) { case 0: return other; break; - case 1: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 1)}; break; - case 2: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 2)}; break; - case 3: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 3)}; break; - case 4: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 4)}; break; - case 5: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 5)}; break; - case 6: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 6)}; break; - case 7: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 7)}; break; - case 8: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 8)}; break; - case 9: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 9)}; break; - case 10: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 10)}; break; - case 11: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 11)}; break; - case 12: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 12)}; break; - case 13: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 13)}; break; - case 14: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 14)}; break; - case 15: return {vextq_s8((int16x8_t) other.u.v128[0], (int16x8_t) u.v128[0], 15)}; break; + case 1: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 1)}; break; + case 2: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 2)}; break; + case 3: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 3)}; break; + case 4: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 4)}; break; + case 5: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 5)}; break; + case 6: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 6)}; break; + case 7: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 7)}; break; + case 8: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 8)}; break; + case 9: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 9)}; break; + case 10: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 10)}; break; + case 11: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 11)}; break; + case 12: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 12)}; break; + case 13: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 13)}; break; + case 14: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 14)}; break; + case 15: return {vextq_u8( other.u.u8x16[0], u.u8x16[0], 15)}; break; case 16: return *this; break; default: break; } @@ -547,7 +589,7 @@ template<> template<> really_inline SuperVector<16> SuperVector<16>::pshufb(SuperVector<16> b) { - return {vqtbl1q_s8((int8x16_t)u.v128[0], (uint8x16_t)b.u.v128[0])}; + return {vqtbl1q_u8(u.u8x16[0], b.u.u8x16[0])}; } template<> @@ -565,7 +607,7 @@ template<> really_inline SuperVector<16> SuperVector<16>::pshufb_maskz(SuperVector<16> b, uint8_t const len) { SuperVector mask = Ones_vshr(16 -len); - return mask & pshufb(b); + return mask & pshufb(b); } #endif // SIMD_IMPL_HPP diff --git a/src/util/supervector/supervector.hpp b/src/util/supervector/supervector.hpp index 76e167ce..e69e4b42 100644 --- a/src/util/supervector/supervector.hpp +++ b/src/util/supervector/supervector.hpp @@ -162,6 +162,18 @@ public: typename BaseVector<16>::type ALIGN_ATTR(BaseVector<16>::size) v128[SIZE / BaseVector<16>::size]; typename BaseVector<32>::type ALIGN_ATTR(BaseVector<32>::size) v256[SIZE / BaseVector<32>::size]; typename BaseVector<64>::type ALIGN_ATTR(BaseVector<64>::size) v512[SIZE / BaseVector<64>::size]; + +#if defined(ARCH_ARM32) || defined(ARCH_AARCH64) + uint64x2_t ALIGN_ATTR(BaseVector<16>::size) u64x2[SIZE / BaseVector<16>::size]; + int64x2_t ALIGN_ATTR(BaseVector<16>::size) s64x2[SIZE / BaseVector<16>::size]; + uint32x4_t ALIGN_ATTR(BaseVector<16>::size) u32x4[SIZE / BaseVector<16>::size]; + int32x4_t ALIGN_ATTR(BaseVector<16>::size) s32x4[SIZE / BaseVector<16>::size]; + uint16x8_t ALIGN_ATTR(BaseVector<16>::size) u16x8[SIZE / BaseVector<16>::size]; + int16x8_t ALIGN_ATTR(BaseVector<16>::size) s16x8[SIZE / BaseVector<16>::size]; + uint8x16_t ALIGN_ATTR(BaseVector<16>::size) u8x16[SIZE / BaseVector<16>::size]; + int8x16_t ALIGN_ATTR(BaseVector<16>::size) s8x16[SIZE / BaseVector<16>::size]; +#endif + uint64_t u64[SIZE / sizeof(uint64_t)]; int64_t s64[SIZE / sizeof(int64_t)]; uint32_t u32[SIZE / sizeof(uint32_t)]; @@ -180,7 +192,7 @@ public: SuperVector(typename base_type::type const v); template - SuperVector(T const other); + SuperVector(T other); SuperVector(SuperVector const lo, SuperVector const hi); SuperVector(previous_type const lo, previous_type const hi); diff --git a/unit/internal/simd_utils.cpp b/unit/internal/simd_utils.cpp index 2a9accae..9b206e1b 100644 --- a/unit/internal/simd_utils.cpp +++ b/unit/internal/simd_utils.cpp @@ -667,7 +667,7 @@ TEST(SimdUtilsTest, movq) { simd = _mm_set_epi64x(~0LL, 0x123456789abcdef); #elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) int64x2_t a = { 0x123456789abcdefLL, ~0LL }; - simd = vreinterpretq_s64_s8(a); + simd = vreinterpretq_s32_s64(a); #endif #endif r = movq(simd);