diff --git a/src/util/arch/common/simd_utils.hpp b/src/util/arch/common/simd_utils.hpp new file mode 100644 index 00000000..e69de29b diff --git a/src/util/simd/arch/x86/impl.hpp b/src/util/simd/arch/x86/impl.hpp new file mode 100644 index 00000000..90ad09e8 --- /dev/null +++ b/src/util/simd/arch/x86/impl.hpp @@ -0,0 +1,635 @@ +/* + * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SIMD_IMPL_HPP +#define SIMD_IMPL_HPP + +#include + +#if !defined(m128) && defined(HAVE_SSE2) +typedef __m128i m128; +#endif + +#if !defined(m128) && defined(HAVE_AVX2) +typedef __m256i m256; +#endif + +#if !defined(m512) && defined(HAVE_AVX512) +typedef __m512i m512; +#endif + +// 128-bit SSE implementation + +template<> +really_inline SuperVector<16>::SuperVector(SuperVector const &o) +{ + u.v128[0] = o.u.v128[0]; +} + +template<> +really_inline SuperVector<16>::SuperVector(typename base_type::type const v) +{ + u.v128[0] = v; +}; + +template<> +template<> +really_inline SuperVector<16>::SuperVector(int8_t const o) +{ + u.v128[0] = _mm_set1_epi8(o); +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(uint8_t const o) +{ + u.v128[0] = _mm_set1_epi8(static_cast(o)); +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(int16_t const o) +{ + u.v128[0] = _mm_set1_epi16(o); +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(uint16_t const o) +{ + u.v128[0] = _mm_set1_epi16(static_cast(o)); +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(int32_t const o) +{ + u.v128[0] = _mm_set1_epi32(o); +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(uint32_t const o) +{ + u.v128[0] = _mm_set1_epi32(static_cast(o)); +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(int64_t const o) +{ + u.v128[0] = _mm_set1_epi64x(o); +} + +template<> +template<> +really_inline SuperVector<16>::SuperVector(uint64_t const o) +{ + u.v128[0] = _mm_set1_epi64x(static_cast(o)); +} + +template <> +really_inline void SuperVector<16>::operator=(SuperVector<16> const &o) +{ + u.v128[0] = o.u.v128[0]; +} + +template <> +really_inline SuperVector<16> SuperVector<16>::operator&(SuperVector<16> const b) const +{ + return {_mm_and_si128(u.v128[0], b.u.v128[0])}; +} + +template <> +really_inline SuperVector<16> SuperVector<16>::eq(SuperVector<16> const b) const +{ + return {_mm_cmpeq_epi8(u.v128[0], b.u.v128[0])}; +} + +template <> +really_inline typename SuperVector<16>::movemask_type SuperVector<16>::movemask(void)const +{ + return _mm_movemask_epi8(u.v128[0]); +} + +template <> +really_inline typename SuperVector<16>::movemask_type SuperVector<16>::eqmask(SuperVector<16> const b) const +{ + return eq(b).movemask(); +} + +#ifndef DEBUG +template <> +really_inline SuperVector<16> SuperVector<16>::operator<<(uint8_t const N) const +{ + return {_mm_slli_si128(u.v128[0], N)}; +} +#else +template <> +really_inline SuperVector<16> SuperVector<16>::operator<<(uint8_t const N) const +{ + switch(N) { + case 0: return {_mm_slli_si128(u.v128[0], 0)}; break; + case 1: return {_mm_slli_si128(u.v128[0], 1)}; break; + case 2: return {_mm_slli_si128(u.v128[0], 2)}; break; + case 3: return {_mm_slli_si128(u.v128[0], 3)}; break; + case 4: return {_mm_slli_si128(u.v128[0], 4)}; break; + case 5: return {_mm_slli_si128(u.v128[0], 5)}; break; + case 6: return {_mm_slli_si128(u.v128[0], 6)}; break; + case 7: return {_mm_slli_si128(u.v128[0], 7)}; break; + case 8: return {_mm_slli_si128(u.v128[0], 8)}; break; + case 9: return {_mm_slli_si128(u.v128[0], 9)}; break; + case 10: return {_mm_slli_si128(u.v128[0], 10)}; break; + case 11: return {_mm_slli_si128(u.v128[0], 11)}; break; + case 12: return {_mm_slli_si128(u.v128[0], 12)}; break; + case 13: return {_mm_slli_si128(u.v128[0], 13)}; break; + case 14: return {_mm_slli_si128(u.v128[0], 14)}; break; + case 15: return {_mm_slli_si128(u.v128[0], 15)}; break; + default: break; + } + return *this; +} +#endif + + +template <> +really_inline SuperVector<16> SuperVector<16>::loadu(void const *ptr) +{ + return _mm_loadu_si128((const m128 *)ptr); +} + +template <> +really_inline SuperVector<16> SuperVector<16>::load(void const *ptr) +{ + assert(ISALIGNED_N(ptr, alignof(SuperVector::size))); + ptr = assume_aligned(ptr, SuperVector::size); + return _mm_load_si128((const m128 *)ptr); +} + +#ifndef DEBUG +template<> +really_inline SuperVector<16> SuperVector<16>::alignr(SuperVector<16> l, int8_t offset) +{ + return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], offset)}; +} +#else +template<> +really_inline SuperVector<16> SuperVector<16>::alignr(SuperVector<16> l, int8_t offset) +{ + switch(offset) { + case 0: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 0)};; break; + case 1: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 1)}; break; + case 2: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 2)}; break; + case 3: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 3)}; break; + case 4: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 4)}; break; + case 5: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 5)}; break; + case 6: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 6)}; break; + case 7: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 7)}; break; + case 8: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 8)}; break; + case 9: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 9)}; break; + case 10: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 10)}; break; + case 11: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 11)}; break; + case 12: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 12)}; break; + case 13: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 13)}; break; + case 14: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 14)}; break; + case 15: return {_mm_alignr_epi8(u.v128[0], l.u.v128[0], 15)}; break; + default: break; + } + return *this; +} +#endif + + +// Constants +template<> +really_inline SuperVector<16> SuperVector<16>::Ones(void) +{ + return {_mm_set1_epi8(0xFF)}; +} + +// Constants +template<> +really_inline SuperVector<16> SuperVector<16>::Zeroes(void) +{ + return {_mm_set1_epi8(0)}; +} + +// 256-bit AVX2 implementation +#if defined(HAVE_AVX2) +template<> +really_inline SuperVector<32>::SuperVector(SuperVector const &o) +{ + u.v256[0] = o.u.v256[0]; +} + +template<> +really_inline SuperVector<32>::SuperVector(typename base_type::type const v) +{ + u.v256[0] = v; +}; + +template<> +template<> +really_inline SuperVector<32>::SuperVector(int8_t const o) +{ + u.v256[0] = _mm256_set1_epi8(o); +} + +template<> +template<> +really_inline SuperVector<32>::SuperVector(uint8_t const o) +{ + u.v256[0] = _mm256_set1_epi8(static_cast(o)); +} + +template<> +template<> +really_inline SuperVector<32>::SuperVector(int16_t const o) +{ + u.v256[0] = _mm256_set1_epi16(o); +} + +template<> +template<> +really_inline SuperVector<32>::SuperVector(uint16_t const o) +{ + u.v256[0] = _mm256_set1_epi16(static_cast(o)); +} + +template<> +template<> +really_inline SuperVector<32>::SuperVector(int32_t const o) +{ + u.v256[0] = _mm256_set1_epi32(o); +} + +template<> +template<> +really_inline SuperVector<32>::SuperVector(uint32_t const o) +{ + u.v256[0] = _mm256_set1_epi32(static_cast(o)); +} + +template<> +template<> +really_inline SuperVector<32>::SuperVector(int64_t const o) +{ + u.v256[0] = _mm256_set1_epi64x(o); +} + +template<> +template<> +really_inline SuperVector<32>::SuperVector(uint64_t const o) +{ + u.v256[0] = _mm256_set1_epi64x(static_cast(o)); +} + +template <> +really_inline void SuperVector<32>::operator=(SuperVector<32> const &o) +{ + u.v256[0] = o.u.v256[0]; +} + +template <> +really_inline SuperVector<32> SuperVector<32>::operator&(SuperVector<32> const b) const +{ + return {_mm256_and_si256(u.v256[0], b.u.v256[0])}; +} + +template <> +really_inline SuperVector<32> SuperVector<32>::eq(SuperVector<32> const b) const +{ + return {_mm256_cmpeq_epi8(u.v256[0], b.u.v256[0])}; +} + +template <> +really_inline typename SuperVector<32>::movemask_type SuperVector<32>::movemask(void)const +{ + return _mm256_movemask_epi8(u.v256[0]); +} + +template <> +really_inline typename SuperVector<32>::movemask_type SuperVector<32>::eqmask(SuperVector<32> const b) const +{ + return eq(b).movemask(); +} + +#ifndef DEBUG +template <> +really_inline SuperVector<32> SuperVector<32>::operator<<(uint8_t const N) const +{ + return {_mm256_slli_si256(u.v256[0], N)}; +} +#else +template <> +really_inline SuperVector<32> SuperVector<32>::operator<<(uint8_t const N) const +{ + switch(N) { + case 0: return {_mm256_slli_si256(u.v256[0], 0)}; break; + case 1: return {_mm256_slli_si256(u.v256[0], 1)}; break; + case 2: return {_mm256_slli_si256(u.v256[0], 2)}; break; + case 3: return {_mm256_slli_si256(u.v256[0], 3)}; break; + case 4: return {_mm256_slli_si256(u.v256[0], 4)}; break; + case 5: return {_mm256_slli_si256(u.v256[0], 5)}; break; + case 6: return {_mm256_slli_si256(u.v256[0], 6)}; break; + case 7: return {_mm256_slli_si256(u.v256[0], 7)}; break; + case 8: return {_mm256_slli_si256(u.v256[0], 8)}; break; + case 9: return {_mm256_slli_si256(u.v256[0], 9)}; break; + case 10: return {_mm256_slli_si256(u.v256[0], 10)}; break; + case 11: return {_mm256_slli_si256(u.v256[0], 11)}; break; + case 12: return {_mm256_slli_si256(u.v256[0], 12)}; break; + case 13: return {_mm256_slli_si256(u.v256[0], 13)}; break; + case 14: return {_mm256_slli_si256(u.v256[0], 14)}; break; + case 15: return {_mm256_slli_si256(u.v256[0], 15)}; break; + default: break; + } + return *this; +} +#endif + +template <> +really_inline SuperVector<32> SuperVector<32>::loadu(void const *ptr) +{ + return {_mm256_loadu_si256((const m256 *)ptr)}; +} + +template <> +really_inline SuperVector<32> SuperVector<32>::load(void const *ptr) +{ + assert(ISALIGNED_N(ptr, alignof(SuperVector::size))); + ptr = assume_aligned(ptr, SuperVector::size); + return {_mm256_load_si256((const m256 *)ptr)}; +} +/* +static void print1_m128_16x8(const char *label, __m128i vector) { + uint8_t __attribute__((aligned((16)))) data[16]; + _mm_store_si128((__m128i*)data, vector); + printf("%s : ", label); + for(int i=0; i < 16; i++) + printf("%02x ", data[i]); + printf("\n"); +} + +static void print_m256_32x8(const char *label, __m256i vector) { + uint8_t __attribute__((aligned((32)))) data[32]; + _mm256_store_si256((__m256i*)data, vector); + printf("%s : ", label); + for(int i=0; i < 32; i++) + printf("%02x ", data[i]); + printf("\n"); +}*/ + +#ifndef DEBUG +template<> +really_inline SuperVector<32> SuperVector<32>::alignr(SuperVector<32> l, int8_t offset) +{ + return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], offset)}; +} +#else +template<> +really_inline SuperVector<32> SuperVector<32>::alignr(SuperVector<32> l, int8_t offset) +{ + switch(offset) { + case 0: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 0)};; break; + case 1: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 1)}; break; + case 2: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 2)}; break; + case 3: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 3)}; break; + case 4: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 4)}; break; + case 5: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 5)}; break; + case 6: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 6)}; break; + case 7: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 7)}; break; + case 8: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 8)}; break; + case 9: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 9)}; break; + case 10: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 10)}; break; + case 11: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 11)}; break; + case 12: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 12)}; break; + case 13: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 13)}; break; + case 14: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 14)}; break; + case 15: return {_mm256_alignr_epi8(u.v256[0], l.u.v256[0], 15)}; break; + default: break; + } + return *this; +} +#endif +/* +template<> +really_inline SuperVector<32> SuperVector<32>::alignr(SuperVector<32> l, int8_t offset) +{ + printf("offset = %d\n", offset); + //u.v256[0] = _mm256_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32); + //l.u.v256[0] = _mm256_set_epi8(101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132); + print_m256_32x8("this", u.v256[0]); + print_m256_32x8("l", l.u.v256[0]); + __m128i v1 = _mm256_extracti128_si256(u.v256[0], 0); + print1_m128_16x8("v1", v1); + __m128i v2 = _mm256_extracti128_si256(u.v256[0], 1); + print1_m128_16x8("v2", v2); + __m128i l1 = _mm256_extracti128_si256(l.u.v256[0], 0); + print1_m128_16x8("l1", l1); + __m128i y1 = _mm_alignr_epi8(v2, l1, 16 - offset); + print1_m128_16x8("y1", y1); + __m128i y2 = _mm_alignr_epi8(v2, v1, 16 - offset); + print1_m128_16x8("y2", y2); + print_m256_32x8("this", _mm256_set_m128i(y1, y2)); + return {_mm256_set_m128i(y1, y2)}; +}*/ + +// Constants +template<> +really_inline SuperVector<32> SuperVector<32>::Ones(void) +{ + return {_mm256_set1_epi8(0xFF)}; +} + +template<> +really_inline SuperVector<32> SuperVector<32>::Zeroes(void) +{ + return {_mm256_set1_epi8(0)}; +} + +#endif // HAVE_AVX2 + +// 512-bit AVX512 implementation +#if defined(HAVE_AVX512) +template<> +really_inline SuperVector<64>::SuperVector(SuperVector const &o) +{ + u.v512[0] = o.u.v512[0]; +} + +template<> +really_inline SuperVector<64>::SuperVector(typename base_type::type const v) +{ + u.v512[0] = v; +}; + +template<> +template<> +really_inline SuperVector<64>::SuperVector(int8_t const o) +{ + u.v512[0] = _mm512_set1_epi8(o); +} + +template<> +template<> +really_inline SuperVector<64>::SuperVector(uint8_t const o) +{ + u.v512[0] = _mm512_set1_epi8(static_cast(o)); +} + +template<> +template<> +really_inline SuperVector<64>::SuperVector(int16_t const o) +{ + u.v512[0] = _mm512_set1_epi16(o); +} + +template<> +template<> +really_inline SuperVector<64>::SuperVector(uint16_t const o) +{ + u.v512[0] = _mm512_set1_epi16(static_cast(o)); +} + +template<> +template<> +really_inline SuperVector<64>::SuperVector(int32_t const o) +{ + u.v512[0] = _mm512_set1_epi32(o); +} + +template<> +template<> +really_inline SuperVector<64>::SuperVector(uint32_t const o) +{ + u.v512[0] = _mm512_set1_epi32(static_cast(o)); +} + +template<> +template<> +really_inline SuperVector<64>::SuperVector(int64_t const o) +{ + u.v512[0] = _mm512_set1_epi64(o); +} + +template<> +template<> +really_inline SuperVector<64>::SuperVector(uint64_t const o) +{ + u.v512[0] = _mm512_set1_epi64(static_cast(o)); +} + +template <> +really_inline void SuperVector<64>::operator=(SuperVector<64> const &o) +{ + u.v512[0] = o.u.v512[0]; +} + +template <> +really_inline SuperVector<64> SuperVector<64>::operator&(SuperVector<64> const b) const +{ + return {_mm512_and_si512(u.v512[0], b.u.v512[0])}; +} + +template <> +really_inline typename SuperVector<64>::movemask_type SuperVector<64>::eqmask(SuperVector<64> const b) const +{ + return _mm512_cmpeq_epi8_mask(u.v512[0], b.u.v512[0]); +} + +// template <> +// really_inline SuperVector<64> SuperVector<64>::operator<<(uint8_t const N) const +// { +// return {_mm512_slli_si512(u.v512[0], N)}; +// } + +template <> +really_inline SuperVector<64> SuperVector<64>::loadu(void const *ptr) +{ + return {_mm512_loadu_si512((const m512 *)ptr)}; +} + +template <> +really_inline SuperVector<64> SuperVector<64>::load(void const *ptr) +{ + assert(ISALIGNED_N(ptr, alignof(SuperVector::size))); + ptr = assume_aligned(ptr, SuperVector::size); + return {_mm512_load_si512((const m512 *)ptr)}; +} + +#ifndef DEBUG +template<> +really_inline SuperVector<64> SuperVector<64>::alignr(SuperVector<64> l, int8_t offset) +{ + return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], offset)}; +} +#else +template<> +really_inline SuperVector<64> SuperVector<64>::alignr(SuperVector<64> l, int8_t offset) +{ + switch(offset) { + case 0: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 0)};; break; + case 1: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 1)}; break; + case 2: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 2)}; break; + case 3: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 3)}; break; + case 4: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 4)}; break; + case 5: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 5)}; break; + case 6: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 6)}; break; + case 7: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 7)}; break; + case 8: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 8)}; break; + case 9: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 9)}; break; + case 10: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 10)}; break; + case 11: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 11)}; break; + case 12: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 12)}; break; + case 13: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 13)}; break; + case 14: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 14)}; break; + case 15: return {_mm512_alignr_epi8(u.v512[0], l.u.v512[0], 15)}; break; + default: break; + } + return *this; +} +#endif + +// Constants +template<> +really_inline SuperVector<64> SuperVector<64>::Ones(void) +{ + return {_mm512_set1_epi8(0xFF)}; +} + +template<> +really_inline SuperVector<64> SuperVector<64>::Zeroes(void) +{ + return {_mm512_set1_epi8(0)}; +} + +#endif // HAVE_AVX512 + + +#endif // SIMD_IMPL_HPP diff --git a/src/util/simd/arch/x86/types.hpp b/src/util/simd/arch/x86/types.hpp new file mode 100644 index 00000000..1361d968 --- /dev/null +++ b/src/util/simd/arch/x86/types.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#if !defined(m128) && defined(HAVE_SSE2) +typedef __m128i m128; +#endif + +#if !defined(m128) && defined(HAVE_AVX2) +typedef __m256i m256; +#endif + +#if !defined(m512) && defined(HAVE_AVX512) +typedef __m512i m512; +#endif \ No newline at end of file diff --git a/src/util/simd/types.hpp b/src/util/simd/types.hpp new file mode 100644 index 00000000..16b7e69a --- /dev/null +++ b/src/util/simd/types.hpp @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SIMD_TYPES_HPP +#define SIMD_TYPES_HPP + +#include + +#if defined(ARCH_IA32) || defined(ARCH_X86_64) +#include "util/simd/arch/x86/types.hpp" +#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) +#include "util/simd/arch/arm/types.hpp" +#endif + +template +class SuperVector; + +using m128_t = SuperVector<16>; +using m256_t = SuperVector<32>; +using m512_t = SuperVector<64>; +using m1024_t = SuperVector<128>; + +// struct for inferring what underlying types to use +template +struct BaseVector +{ + static const bool is_valid = false; // for template matches specialisation + using type = void; + using movemask_type = uint32_t; +}; + +template <> +struct BaseVector<128> +{ + static constexpr bool is_valid = true; + static constexpr uint16_t size = 128; + using type = void; + using movemask_type = u64a; +}; + +template <> +struct BaseVector<64> +{ + static constexpr bool is_valid = true; + static constexpr uint16_t size = 64; + using type = m512; + using movemask_type = u64a; +}; + +// 128 bit implementation +template <> +struct BaseVector<32> +{ + static constexpr bool is_valid = true; + static constexpr uint16_t size = 32; + using type = m256; + using movemask_type = u32; +}; + +// 128 bit implementation +template <> +struct BaseVector<16> +{ + static constexpr bool is_valid = true; + static constexpr uint16_t size = 16; + using type = m128; + using movemask_type = u32; +}; + +template +class SuperVector : public BaseVector +{ + static_assert(BaseVector::is_valid, "invalid SuperVector size"); + +public: + + using base_type = BaseVector; + + union { + typename BaseVector<16>::type ALIGN_ATTR(BaseVector<16>::size) v128[SIZE / BaseVector<16>::size]; + typename BaseVector<32>::type ALIGN_ATTR(BaseVector<32>::size) v256[SIZE / BaseVector<32>::size]; + typename BaseVector<64>::type ALIGN_ATTR(BaseVector<64>::size) v512[SIZE / BaseVector<64>::size]; + uint64_t u64[SIZE / sizeof(uint64_t)]; + int64_t s64[SIZE / sizeof(int64_t)]; + uint32_t u32[SIZE / sizeof(uint32_t)]; + int32_t s32[SIZE / sizeof(int32_t)]; + uint16_t u16[SIZE / sizeof(uint16_t)]; + int16_t s16[SIZE / sizeof(int16_t)]; + uint8_t u8[SIZE / sizeof(uint8_t)]; + int8_t s8[SIZE / sizeof(int8_t)]; + float f32[SIZE / sizeof(float)]; + double f64[SIZE / sizeof(double)]; + } u; + + SuperVector(SuperVector const &o); + SuperVector(typename base_type::type const v); + + template + SuperVector(T const o); + + void operator=(SuperVector const &o); + SuperVector operator&(SuperVector const b) const; + SuperVector eq(SuperVector const b) const; + SuperVector operator<<(uint8_t const N) const; + typename base_type::movemask_type movemask(void) const; + typename base_type::movemask_type eqmask(SuperVector const b) const; + static SuperVector loadu(void const *ptr); + static SuperVector load(void const *ptr); + SuperVector alignr(SuperVector l, int8_t offset); + + // Constants + static SuperVector Ones(); + static SuperVector Zeroes(); +}; + +//class SuperVector<16>; +// class SuperVector<32>; +// class SuperVector<64>; +// class SuperVector<128>; + +#if defined(ARCH_IA32) || defined(ARCH_X86_64) +#include "util/simd/arch/x86/impl.hpp" +#elif defined(ARCH_ARM32) || defined(ARCH_AARCH64) +#include "util/simd/arch/arm/impl.hpp" +#endif + + +#endif /* SIMD_TYPES_H */ +