github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-09-30 19:47:43 +03:00
Code Issues Packages Projects Releases Wiki Activity

noodle: behave like our other literal matchers

Browse Source 
Noodle now supports supplementary masks.
This commit is contained in:
Matthew Barr
2017-05-30 15:54:51 +10:00
parent 9c538a7522
commit 31a445a0e8
8 changed files with 307 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

246
src/hwlm/noodle_engine.c
View File

@@ -84,9 +84,8 @@ struct cb_info {
while (unlikely(z)) { \
Z_TYPE pos = JOIN(findAndClearLSB_, Z_BITS)(&z); \
size_t matchPos = d - buf + pos; \
DEBUG_PRINTF("match pos %zu\n", matchPos); \
hwlmcb_rv_t rv = final(buf, len, key, 1, 0, 0, noCase, cbi, \
matchPos); \
DEBUG_PRINTF("match pos %zu\n", matchPos); \
hwlmcb_rv_t rv = final(n, buf, len, 1, cbi, matchPos); \
RETURN_IF_TERMINATED(rv); \
} \
} while (0)
@@ -96,9 +95,8 @@ struct cb_info {
while (unlikely(z)) { \
Z_TYPE pos = JOIN(findAndClearLSB_, Z_BITS)(&z); \
size_t matchPos = d - buf + pos - 1; \
DEBUG_PRINTF("match pos %zu\n", matchPos); \
hwlmcb_rv_t rv = final(buf, len, key, keyLen, keyOffset, 1, \
noCase, cbi, matchPos); \
DEBUG_PRINTF("match pos %zu\n", matchPos); \
hwlmcb_rv_t rv = final(n, buf, len, 0, cbi, matchPos); \
RETURN_IF_TERMINATED(rv); \
} \
} while (0)
@@ -112,21 +110,28 @@ u8 caseClear8(u8 x, bool noCase) {
// is used only for single chars with case insensitivity used correctly,
// so it can go straight to the callback if we get this far.
static really_inline
hwlm_error_t final(const u8 *buf, size_t len, const u8 *key, size_t keyLen,
size_t keyOffset, bool is_double, bool noCase,
const struct cb_info *cbi, size_t pos) {
pos -= keyOffset;
if (is_double) {
if (pos + keyLen > len) {
return HWLM_SUCCESS;
}
if (cmpForward(buf + pos, key, keyLen, noCase)) { // ret 1 on mismatch
return HWLM_SUCCESS;
hwlm_error_t final(const struct noodTable *n, const u8 *buf, UNUSED size_t len,
char single, const struct cb_info *cbi, size_t pos) {
if (single) {
if (n->msk_len == 1) {
goto match;
}
}
pos += cbi->offsetAdj;
DEBUG_PRINTF("match @ %zu->%zu\n", pos, (pos + keyLen - 1));
hwlmcb_rv_t rv = cbi->cb(pos, (pos + keyLen - 1), cbi->id, cbi->ctx);
assert(len >= n->msk_len);
u64a v =
partial_load_u64a(buf + pos + n->key_offset - n->msk_len, n->msk_len);
DEBUG_PRINTF("v %016llx msk %016llx cmp %016llx\n", v, n->msk, n->cmp);
if ((v & n->msk) != n->cmp) {
/* mask didn't match */
return HWLM_SUCCESS;
}
match:
pos -= cbi->offsetAdj;
DEBUG_PRINTF("match @ %zu->%zu\n", pos + n->key_offset - n->lit_len,
pos + n->key_offset);
hwlmcb_rv_t rv = cbi->cb(pos + n->key_offset - n->lit_len,
pos + n->key_offset - 1, cbi->id, cbi->ctx);
if (rv == HWLM_TERMINATE_MATCHING) {
return HWLM_TERMINATED;
}
@@ -148,38 +153,43 @@ hwlm_error_t final(const u8 *buf, size_t len, const u8 *key, size_t keyLen,
#endif
static really_inline
hwlm_error_t scanSingleMain(const u8 *buf, size_t len, const u8 *key,
bool noCase, const struct cb_info *cbi) {
hwlm_error_t scanSingleMain(const struct noodTable *n, const u8 *buf,
size_t len, size_t start, bool noCase,
const struct cb_info *cbi) {
const MASK_TYPE mask1 = getMask(key[0], noCase);
const MASK_TYPE mask1 = getMask(n->key0, noCase);
const MASK_TYPE caseMask = getCaseMask();
size_t offset = start + n->msk_len - 1;
size_t end = len;
assert(offset < end);
#if !defined(HAVE_AVX512)
hwlm_error_t rv;
size_t end = len;
if (len < CHUNKSIZE) {
rv = scanSingleShort(buf, len, key, noCase, caseMask, mask1, cbi, 0, len);
if (end - offset < CHUNKSIZE) {
rv = scanSingleShort(n, buf, len, noCase, caseMask, mask1, cbi, offset,
end);
return rv;
}
if (len == CHUNKSIZE) {
rv = scanSingleUnaligned(buf, len, 0, key, noCase, caseMask, mask1, cbi,
0, len);
if (end - offset == CHUNKSIZE) {
rv = scanSingleUnaligned(n, buf, len, 0, noCase, caseMask, mask1, cbi,
offset, end);
return rv;
}
uintptr_t data = (uintptr_t)buf;
uintptr_t s2Start = ROUNDUP_N(data, CHUNKSIZE) - data;
uintptr_t s2Start = ROUNDUP_N(data + offset, CHUNKSIZE) - data;
uintptr_t last = data + end;
uintptr_t s2End = ROUNDDOWN_N(last, CHUNKSIZE) - data;
uintptr_t s3Start = len - CHUNKSIZE;
uintptr_t s3Start = end - CHUNKSIZE;
if (s2Start) {
if (offset != s2Start) {
// first scan out to the fast scan starting point
DEBUG_PRINTF("stage 1: -> %zu\n", s2Start);
rv = scanSingleUnaligned(buf, len, 0, key, noCase, caseMask, mask1, cbi,
0, s2Start);
rv = scanSingleUnaligned(n, buf, len, 0, noCase, caseMask, mask1, cbi,
offset, s2Start);
RETURN_IF_TERMINATED(rv);
}
@@ -187,68 +197,70 @@ hwlm_error_t scanSingleMain(const u8 *buf, size_t len, const u8 *key,
// scan as far as we can, bounded by the last point this key can
// possibly match
DEBUG_PRINTF("fast: ~ %zu -> %zu\n", s2Start, s2End);
rv = scanSingleFast(buf, len, key, noCase, caseMask, mask1, cbi,
s2Start, s2End);
rv = scanSingleFast(n, buf, len, noCase, caseMask, mask1, cbi, s2Start,
s2End);
RETURN_IF_TERMINATED(rv);
}
// if we are done bail out
if (s2End == end) {
if (s2End == len) {
return HWLM_SUCCESS;
}
DEBUG_PRINTF("stage 3: %zu -> %zu\n", s2End, end);
rv = scanSingleUnaligned(buf, len, s3Start, key, noCase, caseMask, mask1,
cbi, s2End, end);
DEBUG_PRINTF("stage 3: %zu -> %zu\n", s2End, len);
rv = scanSingleUnaligned(n, buf, len, s3Start, noCase, caseMask, mask1, cbi,
s2End, len);
return rv;
#else // HAVE_AVX512
return scanSingle512(buf, len, key, noCase, caseMask, mask1, cbi);
return scanSingle512(n, buf, len, noCase, caseMask, mask1, cbi, offset,
end);
#endif
}
static really_inline
hwlm_error_t scanDoubleMain(const u8 *buf, size_t len, const u8 *key,
size_t keyLen, size_t keyOffset, bool noCase,
hwlm_error_t scanDoubleMain(const struct noodTable *n, const u8 *buf,
size_t len, size_t start, bool noCase,
const struct cb_info *cbi) {
// we stop scanning for the key-fragment when the rest of the key can't
// possibly fit in the remaining buffer
size_t end = len - keyLen + keyOffset + 2;
size_t end = len - n->key_offset + 2;
// the first place the key can match
size_t offset = start + n->msk_len - n->key_offset;
const MASK_TYPE caseMask = getCaseMask();
const MASK_TYPE mask1 = getMask(key[keyOffset + 0], noCase);
const MASK_TYPE mask2 = getMask(key[keyOffset + 1], noCase);
const MASK_TYPE mask1 = getMask(n->key0, noCase);
const MASK_TYPE mask2 = getMask(n->key1, noCase);
#if !defined(HAVE_AVX512)
hwlm_error_t rv;
if (end - keyOffset < CHUNKSIZE) {
rv = scanDoubleShort(buf, len, key, keyLen, keyOffset, noCase, caseMask,
mask1, mask2, cbi, keyOffset, end);
if (end - offset < CHUNKSIZE) {
rv = scanDoubleShort(n, buf, len, noCase, caseMask, mask1, mask2, cbi,
offset, end);
return rv;
}
if (end - keyOffset == CHUNKSIZE) {
rv = scanDoubleUnaligned(buf, len, keyOffset, key, keyLen, keyOffset,
noCase, caseMask, mask1, mask2, cbi, keyOffset,
end);
if (end - offset == CHUNKSIZE) {
rv = scanDoubleUnaligned(n, buf, len, offset, noCase, caseMask, mask1,
mask2, cbi, offset, end);
return rv;
}
uintptr_t data = (uintptr_t)buf;
uintptr_t s2Start = ROUNDUP_N(data + keyOffset, CHUNKSIZE) - data;
uintptr_t s2Start = ROUNDUP_N(data + offset, CHUNKSIZE) - data;
uintptr_t s1End = s2Start + 1;
uintptr_t last = data + end;
uintptr_t s2End = ROUNDDOWN_N(last, CHUNKSIZE) - data;
uintptr_t s3Start = end - CHUNKSIZE;
uintptr_t off = keyOffset;
uintptr_t off = offset;
if (s2Start != keyOffset) {
if (s2Start != off) {
// first scan out to the fast scan starting point plus one char past to
// catch the key on the overlap
DEBUG_PRINTF("stage 1: -> %zu\n", s2Start);
rv = scanDoubleUnaligned(buf, len, keyOffset, key, keyLen, keyOffset,
noCase, caseMask, mask1, mask2, cbi, off,
s1End);
DEBUG_PRINTF("stage 1: %zu -> %zu\n", off, s2Start);
rv = scanDoubleUnaligned(n, buf, len, offset, noCase, caseMask, mask1,
mask2, cbi, off, s1End);
RETURN_IF_TERMINATED(rv);
}
off = s1End;
@@ -262,8 +274,8 @@ hwlm_error_t scanDoubleMain(const u8 *buf, size_t len, const u8 *key,
// scan as far as we can, bounded by the last point this key can
// possibly match
DEBUG_PRINTF("fast: ~ %zu -> %zu\n", s2Start, s3Start);
rv = scanDoubleFast(buf, len, key, keyLen, keyOffset, noCase, caseMask,
mask1, mask2, cbi, s2Start, s2End);
rv = scanDoubleFast(n, buf, len, noCase, caseMask, mask1, mask2, cbi,
s2Start, s2End);
RETURN_IF_TERMINATED(rv);
off = s2End;
}
@@ -274,98 +286,101 @@ hwlm_error_t scanDoubleMain(const u8 *buf, size_t len, const u8 *key,
}
DEBUG_PRINTF("stage 3: %zu -> %zu\n", s3Start, end);
rv = scanDoubleUnaligned(buf, len, s3Start, key, keyLen, keyOffset, noCase,
caseMask, mask1, mask2, cbi, off, end);
rv = scanDoubleUnaligned(n, buf, len, s3Start, noCase, caseMask, mask1,
mask2, cbi, off, end);
return rv;
#else // AVX512
return scanDouble512(buf, len, key, keyLen, keyOffset, noCase, caseMask,
mask1, mask2, cbi, keyOffset, end);
return scanDouble512(n, buf, len, noCase, caseMask, mask1, mask2, cbi,
offset, end);
#endif // AVX512
}
static really_inline
hwlm_error_t scanSingleNoCase(const u8 *buf, size_t len, const u8 *key,
hwlm_error_t scanSingleNoCase(const struct noodTable *n, const u8 *buf,
size_t len, size_t start,
const struct cb_info *cbi) {
return scanSingleMain(buf, len, key, 1, cbi);
return scanSingleMain(n, buf, len, start, 1, cbi);
}
static really_inline
hwlm_error_t scanSingleCase(const u8 *buf, size_t len, const u8 *key,
hwlm_error_t scanSingleCase(const struct noodTable *n, const u8 *buf,
size_t len, size_t start,
const struct cb_info *cbi) {
return scanSingleMain(buf, len, key, 0, cbi);
return scanSingleMain(n, buf, len, start, 0, cbi);
}
// Single-character specialisation, used when keyLen = 1
static really_inline
hwlm_error_t scanSingle(const u8 *buf, size_t len, const u8 *key, bool noCase,
const struct cb_info *cbi) {
if (!ourisalpha(key[0])) {
hwlm_error_t scanSingle(const struct noodTable *n, const u8 *buf, size_t len,
size_t start, bool noCase, const struct cb_info *cbi) {
if (!ourisalpha(n->key0)) {
noCase = 0; // force noCase off if we don't have an alphabetic char
}
// kinda ugly, but this forces constant propagation
if (noCase) {
return scanSingleNoCase(buf, len, key, cbi);
return scanSingleNoCase(n, buf, len, start, cbi);
} else {
return scanSingleCase(buf, len, key, cbi);
return scanSingleCase(n, buf, len, start, cbi);
}
}
static really_inline
hwlm_error_t scanDoubleNoCase(const u8 *buf, size_t len, const u8 *key,
size_t keyLen, size_t keyOffset,
hwlm_error_t scanDoubleNoCase(const struct noodTable *n, const u8 *buf,
size_t len, size_t start,
const struct cb_info *cbi) {
return scanDoubleMain(buf, len, key, keyLen, keyOffset, 1, cbi);
return scanDoubleMain(n, buf, len, start, 1, cbi);
}
static really_inline
hwlm_error_t scanDoubleCase(const u8 *buf, size_t len, const u8 *key,
size_t keyLen, size_t keyOffset,
hwlm_error_t scanDoubleCase(const struct noodTable *n, const u8 *buf,
size_t len, size_t start,
const struct cb_info *cbi) {
return scanDoubleMain(buf, len, key, keyLen, keyOffset, 0, cbi);
return scanDoubleMain(n, buf, len, start, 0, cbi);
}
static really_inline
hwlm_error_t scanDouble(const u8 *buf, size_t len, const u8 *key, size_t keyLen,
size_t keyOffset, bool noCase,
const struct cb_info *cbi) {
hwlm_error_t scanDouble(const struct noodTable *n, const u8 *buf, size_t len,
size_t start, bool noCase, const struct cb_info *cbi) {
// kinda ugly, but this forces constant propagation
if (noCase) {
return scanDoubleNoCase(buf, len, key, keyLen, keyOffset, cbi);
return scanDoubleNoCase(n, buf, len, start, cbi);
} else {
return scanDoubleCase(buf, len, key, keyLen, keyOffset, cbi);
return scanDoubleCase(n, buf, len, start, cbi);
}
}
// main entry point for the scan code
static really_inline
hwlm_error_t scan(const u8 *buf, size_t len, const u8 *key, size_t keyLen,
size_t keyOffset, bool noCase, const struct cb_info *cbi) {
if (len < keyLen) {
hwlm_error_t scan(const struct noodTable *n, const u8 *buf, size_t len,
size_t start, char single, bool noCase,
const struct cb_info *cbi) {
if (len - start < n->msk_len) {
// can't find string of length keyLen in a shorter buffer
return HWLM_SUCCESS;
}
if (keyLen == 1) {
assert(keyOffset == 0);
return scanSingle(buf, len, key, noCase, cbi);
if (single) {
return scanSingle(n, buf, len, start, noCase, cbi);
} else {
return scanDouble(buf, len, key, keyLen, keyOffset, noCase, cbi);
return scanDouble(n, buf, len, start, noCase, cbi);
}
}
/** \brief Block-mode scanner. */
hwlm_error_t noodExec(const struct noodTable *n, const u8 *buf, size_t len,
size_t offset_adj, HWLMCallback cb, void *ctxt) {
size_t start, HWLMCallback cb, void *ctxt) {
assert(n && buf);
struct cb_info cbi = { cb, n->id, ctxt, offset_adj };
DEBUG_PRINTF("nood scan of %zu bytes for %*s\n", len, n->len, n->str);
return scan(buf, len, n->str, n->len, n->key_offset, n->nocase, &cbi);
struct cb_info cbi = {cb, n->id, ctxt, 0};
DEBUG_PRINTF("nood scan of %zu bytes for %*s @ %p\n", len, n->lit_len,
(const char *)&n->cmp + n->msk_len - n->lit_len, buf);
return scan(n, buf, len, start, n->single, n->nocase, &cbi);
}
/** \brief Streaming-mode scanner. */
@@ -375,34 +390,49 @@ hwlm_error_t noodExecStreaming(const struct noodTable *n, const u8 *hbuf,
assert(n);
struct cb_info cbi = {cb, n->id, ctxt, 0};
hwlm_error_t rv;
DEBUG_PRINTF("nood scan of %zu bytes (%zu hlen) for %*s @ %p\n", len, hlen,
n->lit_len, (const char *)&n->cmp + n->msk_len - n->lit_len,
buf);
if (hlen) {
/*
* we have history, so build up a buffer from enough of the history
* buffer plus what we've been given to scan. Since this is relatively
* short, just check against msk+cmp per byte offset for matches.
*/
assert(hbuf);
u8 ALIGN_DIRECTIVE temp_buf[HWLM_LITERAL_MAX_LEN * 2];
memset(temp_buf, 0, sizeof(temp_buf));
size_t tl1 = MIN(n->len - 1, hlen);
size_t tl2 = MIN(n->len - 1, len);
size_t temp_len = tl1 + tl2;
assert(temp_len < sizeof(temp_buf));
assert(n->msk_len);
size_t tl1 = MIN((size_t)n->msk_len - 1, hlen);
size_t tl2 = MIN((size_t)n->msk_len - 1, len);
assert(tl1 + tl2 <= sizeof(temp_buf));
assert(tl1 <= sizeof(u64a));
assert(tl2 <= sizeof(u64a));
DEBUG_PRINTF("using %zu bytes of hist and %zu bytes of buf\n", tl1, tl2);
unaligned_store_u64a(temp_buf,
partial_load_u64a(hbuf + hlen - tl1, tl1));
unaligned_store_u64a(temp_buf + tl1, partial_load_u64a(buf, tl2));
cbi.offsetAdj = -tl1;
rv = scan(temp_buf, temp_len, n->str, n->len, n->key_offset, n->nocase,
&cbi);
if (rv == HWLM_TERMINATED) {
return HWLM_TERMINATED;
for (size_t i = 0; i < tl1; i++) {
u64a v = unaligned_load_u64a(temp_buf + i);
if ((v & n->msk) == n->cmp) {
size_t m_end = -tl1 + i + n->msk_len - 1;
size_t m_start = m_end - n->lit_len;
DEBUG_PRINTF("match @ %zu->%zu (i %zu)\n", m_start, m_end, i);
hwlmcb_rv_t rv = cb(m_start, m_end, n->id, ctxt);
if (rv == HWLM_TERMINATE_MATCHING) {
return HWLM_TERMINATED;
}
}
}
}
assert(buf);
cbi.offsetAdj = 0;
return scan(buf, len, n->str, n->len, n->key_offset, n->nocase, &cbi);
return scan(n, buf, len, 0, n->single, n->nocase, &cbi);
}