github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-09-29 11:16:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Move SVE functions into their own files.

Browse Source 
Change-Id: I995ba4b7d2b558ee403693ee45d747d414d3b177
This commit is contained in:
George Wort
2021-07-20 18:13:02 +01:00
committed by Konstantinos Margaritis
parent 6c6aee9682
commit a879715953
16 changed files with 676 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/util/arch/arm/bitutils.h
View File

@@ -119,24 +119,23 @@ m128 compress128_impl(m128 x, m128 m) {
return res;
}
#if defined(HAVE_SVE2_BITPERM)
#include "bitutils_sve.h"
#else
static really_inline
u32 expand32_impl(u32 x, u32 m) {
#if defined(HAVE_SVE2_BITPERM)
return svlasta(svpfalse(), svbdep(svdup_u32(x), m));
#else
return expand32_impl_c(x, m);
#endif
}
static really_inline
u64a expand64_impl(u64a x, u64a m) {
#if defined(HAVE_SVE2_BITPERM)
return svlasta(svpfalse(), svbdep(svdup_u64(x), m));
#else
return expand64_impl_c(x, m);
#endif
}
#endif // HAVE_SVE2_BITPERM
static really_inline
m128 expand128_impl(m128 x, m128 m) {
m128 one = set1_2x64(1);

49
src/util/arch/arm/bitutils_sve.h Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2021, Arm Limited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief Bit-twiddling primitives for SVE (ctz, compress etc)
*/
static really_inline
u32 expand32_impl(u32 x, u32 m) {
return svlasta(svpfalse(), svbdep(svdup_u32(x), m));
}
static really_inline
u64a expand64_impl(u64a x, u64a m) {
return svlasta(svpfalse(), svbdep(svdup_u64(x), m));
}
static really_inline
void bdep64x2(u64a *d, const u64a *x, const m128 *m) {
svbool_t pg = svptrue_pat_b64(SV_VL2);
svst1(pg, (uint64_t *)d, svbdep(svld1_u64(pg, (const uint64_t *)x),
svld1_u64(pg, (const uint64_t *)m)));
}

63
src/util/arch/arm/simd_utils.h
View File

@@ -43,70 +43,11 @@
#include "util/intrinsics.h"
#ifdef HAVE_SVE
really_really_inline
uint64_t accelSearchGetOffset(svbool_t matched) {
return svcntp_b8(svptrue_b8(), svbrkb_z(svptrue_b8(), matched));
}
really_really_inline
const u8 *accelSearchCheckMatched(const u8 *buf, svbool_t matched) {
if (unlikely(svptest_any(svptrue_b8(), matched))) {
const u8 *matchPos = buf + accelSearchGetOffset(matched);
DEBUG_PRINTF("match pos %p\n", matchPos);
return matchPos;
}
return NULL;
}
really_really_inline
const u8 *accelRevSearchCheckMatched(const u8 *buf, svbool_t matched) {
if (unlikely(svptest_any(svptrue_b8(), matched))) {
const u8 *matchPos = buf + (svcntb() -
svcntp_b8(svptrue_b8(), svbrka_z(svptrue_b8(), svrev_b8(matched))));
DEBUG_PRINTF("match pos %p\n", matchPos);
return matchPos;
}
return NULL;
}
static really_inline
svuint8_t getSVEMaskFrom128(m128 mask) {
return svld1_u8(svptrue_pat_b8(SV_VL16), (const uint8_t *)&mask);
}
#include "simd_utils_sve.h"
#endif
#ifdef HAVE_SVE2
static really_inline
svuint8_t getCharMaskSingle(const u8 c, bool noCase) {
if (noCase) {
uint16_t chars_u16 = (c & 0xdf) | ((c | 0x20) << 8);
return svreinterpret_u8(svdup_u16(chars_u16));
} else {
return svdup_u8(c);
}
}
static really_inline
svuint16_t getCharMaskDouble(const u8 c0, const u8 c1, bool noCase) {
if (noCase) {
const uint64_t lowerFirst = c0 & 0xdf;
const uint64_t upperFirst = c0 | 0x20;
const uint64_t lowerSecond = c1 & 0xdf;
const uint64_t upperSecond = c1 | 0x20;
const uint64_t chars = lowerFirst | (lowerSecond << 8)
| (lowerFirst << 16) | (upperSecond) << 24
| (upperFirst << 32) | (lowerSecond) << 40
| (upperFirst << 48) | (upperSecond) << 56;
return svreinterpret_u16(svdup_u64(chars));
} else {
uint16_t chars_u16 = c0 | (c1 << 8);
return svdup_u16(chars_u16);
}
}
#include "simd_utils_sve2.h"
#endif
#include <string.h> // for memcpy

62
src/util/arch/arm/simd_utils_sve.h Normal file
View File

@@ -0,0 +1,62 @@
/*
* Copyright (c) 2021, Arm Limited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief SVE primitive operations.
*/
really_really_inline
uint64_t accelSearchGetOffset(svbool_t matched) {
return svcntp_b8(svptrue_b8(), svbrkb_z(svptrue_b8(), matched));
}
really_really_inline
const u8 *accelSearchCheckMatched(const u8 *buf, svbool_t matched) {
if (unlikely(svptest_any(svptrue_b8(), matched))) {
const u8 *matchPos = buf + accelSearchGetOffset(matched);
DEBUG_PRINTF("match pos %p\n", matchPos);
return matchPos;
}
return NULL;
}
really_really_inline
const u8 *accelRevSearchCheckMatched(const u8 *buf, svbool_t matched) {
if (unlikely(svptest_any(svptrue_b8(), matched))) {
const u8 *matchPos = buf + (svcntb() -
svcntp_b8(svptrue_b8(), svbrka_z(svptrue_b8(), svrev_b8(matched))));
DEBUG_PRINTF("match pos %p\n", matchPos);
return matchPos;
}
return NULL;
}
static really_inline
svuint8_t getSVEMaskFrom128(m128 mask) {
return svld1_u8(svptrue_pat_b8(SV_VL16), (const uint8_t *)&mask);
}

59
src/util/arch/arm/simd_utils_sve2.h Normal file
View File

@@ -0,0 +1,59 @@
/*
* Copyright (c) 2021, Arm Limited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief SVE primitive operations.
*/
static really_inline
svuint8_t getCharMaskSingle(const u8 c, bool noCase) {
if (noCase) {
uint16_t chars_u16 = (c & 0xdf) | ((c | 0x20) << 8);
return svreinterpret_u8(svdup_u16(chars_u16));
} else {
return svdup_u8(c);
}
}
static really_inline
svuint16_t getCharMaskDouble(const u8 c0, const u8 c1, bool noCase) {
if (noCase) {
const uint64_t lowerFirst = c0 & 0xdf;
const uint64_t upperFirst = c0 | 0x20;
const uint64_t lowerSecond = c1 & 0xdf;
const uint64_t upperSecond = c1 | 0x20;
const uint64_t chars = lowerFirst | (lowerSecond << 8)
| (lowerFirst << 16) | (upperSecond) << 24
| (upperFirst << 32) | (lowerSecond) << 40
| (upperFirst << 48) | (upperSecond) << 56;
return svreinterpret_u16(svdup_u64(chars));
} else {
uint16_t chars_u16 = c0 | (c1 << 8);
return svdup_u16(chars_u16);
}
}

11
src/util/state_compress.c
View File

@@ -83,17 +83,6 @@ void loadcompressed64(u64a *x, const void *ptr, const u64a *m, u32 bytes) {
#endif
}
#if defined(HAVE_SVE2_BITPERM)
static really_inline
void bdep64x2(u64a *d, const u64a *x, const m128 *m) {
svbool_t pg = svptrue_pat_b64(SV_VL2);
svst1(pg, (uint64_t *)d, svbdep(svld1_u64(pg, (const uint64_t *)x),
svld1_u64(pg, (const uint64_t *)m)));
}
#endif // HAVE_SVE2_BITPERM
/*
* 128-bit store/load.
*/