github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-11-20 02:47:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add SVE2 support for vermicelli

Browse Source 
Change-Id: Ia025de53521fbaefe5fb1e4425aaf75c7d80a14e
This commit is contained in:
George Wort
2021-06-07 13:55:09 +01:00
committed by Konstantinos Margaritis
parent b2332218a4
commit acfa11a34f
8 changed files with 877 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files

281
src/nfa/vermicelli.h
View File

@@ -1,5 +1,6 @@
/*
* Copyright (c) 2015-2020, Intel Corporation
* Copyright (c) 2021, Arm Limited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -37,138 +38,16 @@
#include "util/simd_utils.h"
#include "util/unaligned.h"
#if !defined(HAVE_AVX512)
#include "vermicelli_common.h"
#endif
#ifdef HAVE_SVE2
#include "vermicelli_sve.h"
#else
#include "vermicelli_sse.h"
static really_inline
const u8 *vermicelliExec(char c, char nocase, const u8 *buf,
const u8 *buf_end) {
DEBUG_PRINTF("verm scan %s\\x%02hhx over %zu bytes\n",
nocase ? "nocase " : "", c, (size_t)(buf_end - buf));
assert(buf < buf_end);
VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */
// Handle small scans.
#ifdef HAVE_AVX512
if (buf_end - buf <= VERM_BOUNDARY) {
const u8 *ptr = nocase
? vermMiniNocase(chars, buf, buf_end, 0)
: vermMini(chars, buf, buf_end, 0);
if (ptr) {
return ptr;
}
return buf_end;
}
#else
if (buf_end - buf < VERM_BOUNDARY) {
for (; buf < buf_end; buf++) {
char cur = (char)*buf;
if (nocase) {
cur &= CASE_CLEAR;
}
if (cur == c) {
break;
}
}
return buf;
}
#endif
uintptr_t min = (uintptr_t)buf % VERM_BOUNDARY;
if (min) {
// Input isn't aligned, so we need to run one iteration with an
// unaligned load, then skip buf forward to the next aligned address.
// There's some small overlap here, but we don't mind scanning it twice
// if we can do it quickly, do we?
const u8 *ptr = nocase ? vermUnalignNocase(chars, buf, 0)
: vermUnalign(chars, buf, 0);
if (ptr) {
return ptr;
}
buf += VERM_BOUNDARY - min;
assert(buf < buf_end);
}
// Aligned loops from here on in
const u8 *ptr = nocase ? vermSearchAlignedNocase(chars, buf, buf_end - 1, 0)
: vermSearchAligned(chars, buf, buf_end - 1, 0);
if (ptr) {
return ptr;
}
// Tidy up the mess at the end
ptr = nocase ? vermUnalignNocase(chars, buf_end - VERM_BOUNDARY, 0)
: vermUnalign(chars, buf_end - VERM_BOUNDARY, 0);
return ptr ? ptr : buf_end;
}
/* like vermicelliExec except returns the address of the first character which
* is not c */
static really_inline
const u8 *nvermicelliExec(char c, char nocase, const u8 *buf,
const u8 *buf_end) {
DEBUG_PRINTF("nverm scan %s\\x%02hhx over %zu bytes\n",
nocase ? "nocase " : "", c, (size_t)(buf_end - buf));
assert(buf < buf_end);
VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */
// Handle small scans.
#ifdef HAVE_AVX512
if (buf_end - buf <= VERM_BOUNDARY) {
const u8 *ptr = nocase
? vermMiniNocase(chars, buf, buf_end, 1)
: vermMini(chars, buf, buf_end, 1);
if (ptr) {
return ptr;
}
return buf_end;
}
#else
if (buf_end - buf < VERM_BOUNDARY) {
for (; buf < buf_end; buf++) {
char cur = (char)*buf;
if (nocase) {
cur &= CASE_CLEAR;
}
if (cur != c) {
break;
}
}
return buf;
}
#endif
size_t min = (size_t)buf % VERM_BOUNDARY;
if (min) {
// Input isn't aligned, so we need to run one iteration with an
// unaligned load, then skip buf forward to the next aligned address.
// There's some small overlap here, but we don't mind scanning it twice
// if we can do it quickly, do we?
const u8 *ptr = nocase ? vermUnalignNocase(chars, buf, 1)
: vermUnalign(chars, buf, 1);
if (ptr) {
return ptr;
}
buf += VERM_BOUNDARY - min;
assert(buf < buf_end);
}
// Aligned loops from here on in
const u8 *ptr = nocase ? vermSearchAlignedNocase(chars, buf, buf_end - 1, 1)
: vermSearchAligned(chars, buf, buf_end - 1, 1);
if (ptr) {
return ptr;
}
// Tidy up the mess at the end
ptr = nocase ? vermUnalignNocase(chars, buf_end - VERM_BOUNDARY, 1)
: vermUnalign(chars, buf_end - VERM_BOUNDARY, 1);
return ptr ? ptr : buf_end;
}
static really_inline
const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf,
const u8 *buf_end) {
@@ -315,150 +194,6 @@ const u8 *vermicelliDoubleMaskedExec(char c1, char c2, char m1, char m2,
return buf_end;
}
// Reverse vermicelli scan. Provides exact semantics and returns (buf - 1) if
// character not found.
static really_inline
const u8 *rvermicelliExec(char c, char nocase, const u8 *buf,
const u8 *buf_end) {
DEBUG_PRINTF("rev verm scan %s\\x%02hhx over %zu bytes\n",
nocase ? "nocase " : "", c, (size_t)(buf_end - buf));
assert(buf < buf_end);
VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */
// Handle small scans.
#ifdef HAVE_AVX512
if (buf_end - buf <= VERM_BOUNDARY) {
const u8 *ptr = nocase
? rvermMiniNocase(chars, buf, buf_end, 0)
: rvermMini(chars, buf, buf_end, 0);
if (ptr) {
return ptr;
}
return buf - 1;
}
#else
if (buf_end - buf < VERM_BOUNDARY) {
for (buf_end--; buf_end >= buf; buf_end--) {
char cur = (char)*buf_end;
if (nocase) {
cur &= CASE_CLEAR;
}
if (cur == c) {
break;
}
}
return buf_end;
}
#endif
size_t min = (size_t)buf_end % VERM_BOUNDARY;
if (min) {
// Input isn't aligned, so we need to run one iteration with an
// unaligned load, then skip buf backward to the next aligned address.
// There's some small overlap here, but we don't mind scanning it twice
// if we can do it quickly, do we?
const u8 *ptr = nocase ? rvermUnalignNocase(chars,
buf_end - VERM_BOUNDARY,
0)
: rvermUnalign(chars, buf_end - VERM_BOUNDARY,
0);
if (ptr) {
return ptr;
}
buf_end -= min;
if (buf >= buf_end) {
return buf_end;
}
}
// Aligned loops from here on in.
const u8 *ptr = nocase ? rvermSearchAlignedNocase(chars, buf, buf_end, 0)
: rvermSearchAligned(chars, buf, buf_end, 0);
if (ptr) {
return ptr;
}
// Tidy up the mess at the end, return buf - 1 if not found.
ptr = nocase ? rvermUnalignNocase(chars, buf, 0)
: rvermUnalign(chars, buf, 0);
return ptr ? ptr : buf - 1;
}
/* like rvermicelliExec except returns the address of the last character which
* is not c */
static really_inline
const u8 *rnvermicelliExec(char c, char nocase, const u8 *buf,
const u8 *buf_end) {
DEBUG_PRINTF("rev verm scan %s\\x%02hhx over %zu bytes\n",
nocase ? "nocase " : "", c, (size_t)(buf_end - buf));
assert(buf < buf_end);
VERM_TYPE chars = VERM_SET_FN(c); /* nocase already uppercase */
// Handle small scans.
#ifdef HAVE_AVX512
if (buf_end - buf <= VERM_BOUNDARY) {
const u8 *ptr = nocase
? rvermMiniNocase(chars, buf, buf_end, 1)
: rvermMini(chars, buf, buf_end, 1);
if (ptr) {
return ptr;
}
return buf - 1;
}
#else
if (buf_end - buf < VERM_BOUNDARY) {
for (buf_end--; buf_end >= buf; buf_end--) {
char cur = (char)*buf_end;
if (nocase) {
cur &= CASE_CLEAR;
}
if (cur != c) {
break;
}
}
return buf_end;
}
#endif
size_t min = (size_t)buf_end % VERM_BOUNDARY;
if (min) {
// Input isn't aligned, so we need to run one iteration with an
// unaligned load, then skip buf backward to the next aligned address.
// There's some small overlap here, but we don't mind scanning it twice
// if we can do it quickly, do we?
const u8 *ptr = nocase ? rvermUnalignNocase(chars,
buf_end - VERM_BOUNDARY,
1)
: rvermUnalign(chars, buf_end - VERM_BOUNDARY,
1);
if (ptr) {
return ptr;
}
buf_end -= min;
if (buf >= buf_end) {
return buf_end;
}
}
// Aligned loops from here on in.
const u8 *ptr = nocase ? rvermSearchAlignedNocase(chars, buf, buf_end, 1)
: rvermSearchAligned(chars, buf, buf_end, 1);
if (ptr) {
return ptr;
}
// Tidy up the mess at the end, return buf - 1 if not found.
ptr = nocase ? rvermUnalignNocase(chars, buf, 1)
: rvermUnalign(chars, buf, 1);
return ptr ? ptr : buf - 1;
}
/* returns highest offset of c2 (NOTE: not c1) */
static really_inline
const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf,