literal matchers: change context passed to callback to scratch

This commit is contained in:
Wang, Xiang W
2017-07-04 12:24:11 -04:00
committed by Matthew Barr
parent bc232d272f
commit 67a8f43355
17 changed files with 203 additions and 188 deletions

View File

@@ -32,6 +32,7 @@
#include "fdr_internal.h"
#include "fdr_loadval.h"
#include "flood_runtime.h"
#include "scratch.h"
#include "teddy.h"
#include "teddy_internal.h"
#include "util/arch.h"
@@ -824,8 +825,8 @@ static const FDRFUNCTYPE funcs[] = {
static const u8 fake_history[FAKE_HISTORY_SIZE];
hwlm_error_t fdrExec(const struct FDR *fdr, const u8 *buf, size_t len,
size_t start, HWLMCallback cb, void *ctxt,
hwlm_group_t groups) {
size_t start, HWLMCallback cb,
struct hs_scratch *scratch, hwlm_group_t groups) {
// We guarantee (for safezone construction) that it is safe to read 16
// bytes before the end of the history buffer.
const u8 *hbuf = fake_history + FAKE_HISTORY_SIZE;
@@ -837,7 +838,7 @@ hwlm_error_t fdrExec(const struct FDR *fdr, const u8 *buf, size_t len,
0,
start,
cb,
ctxt,
scratch,
nextFloodDetect(buf, len, FLOOD_BACKOFF_START),
0
};
@@ -851,7 +852,8 @@ hwlm_error_t fdrExec(const struct FDR *fdr, const u8 *buf, size_t len,
hwlm_error_t fdrExecStreaming(const struct FDR *fdr, const u8 *hbuf,
size_t hlen, const u8 *buf, size_t len,
size_t start, HWLMCallback cb, void *ctxt,
size_t start, HWLMCallback cb,
struct hs_scratch *scratch,
hwlm_group_t groups) {
struct FDR_Runtime_Args a = {
buf,
@@ -860,7 +862,7 @@ hwlm_error_t fdrExecStreaming(const struct FDR *fdr, const u8 *hbuf,
hlen,
start,
cb,
ctxt,
scratch,
nextFloodDetect(buf, len, FLOOD_BACKOFF_START),
/* we are guaranteed to always have 16 initialised bytes at the end of
* the history buffer (they may be garbage). */

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
* Copyright (c) 2015-2017, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -42,6 +42,7 @@ extern "C" {
#endif
struct FDR;
struct hs_scratch;
/**
* \brief Block-mode scan.
@@ -51,11 +52,11 @@ struct FDR;
* \param len Length of buffer to scan.
* \param start First offset in buf at which a match may end.
* \param cb Callback to call when a match is found.
* \param ctxt Caller-provided context pointer supplied to callback on match.
* \param scratch Scratch supplied to callback on match.
* \param groups Initial groups mask.
*/
hwlm_error_t fdrExec(const struct FDR *fdr, const u8 *buf, size_t len,
size_t start, HWLMCallback cb, void *ctxt,
size_t start, HWLMCallback cb, struct hs_scratch *scratch,
hwlm_group_t groups);
/**
@@ -68,12 +69,13 @@ hwlm_error_t fdrExec(const struct FDR *fdr, const u8 *buf, size_t len,
* \param len Length of buffer to scan (buf).
* \param start First offset in buf at which a match may end.
* \param cb Callback to call when a match is found.
* \param ctxt Caller-provided context pointer supplied to callback on match.
* \param scratch Scratch supplied to callback on match.
* \param groups Initial groups mask.
*/
hwlm_error_t fdrExecStreaming(const struct FDR *fdr, const u8 *hbuf,
size_t hlen, const u8 *buf, size_t len,
size_t start, HWLMCallback cb, void *ctxt,
size_t start, HWLMCallback cb,
struct hs_scratch *scratch,
hwlm_group_t groups);
#ifdef __cplusplus

View File

@@ -88,7 +88,7 @@ void confWithBit(const struct FDRConfirm *fdrc, const struct FDR_Runtime_Args *a
}
*last_match = li->id;
*control = a->cb(i, li->id, a->ctxt);
*control = a->cb(i, li->id, a->scratch);
out:
oldNext = li->next; // oldNext is either 0 or an 'adjust' value
li++;

View File

@@ -36,6 +36,8 @@
#include "ue2common.h"
#include "hwlm/hwlm.h" // for hwlm_group_t, HWLMCallback
struct hs_scratch;
typedef enum {
NOT_CAUTIOUS, //!< not near a boundary (quantify?)
VECTORING //!< potentially vectoring
@@ -95,7 +97,7 @@ struct FDR_Runtime_Args {
size_t len_history;
size_t start_offset;
HWLMCallback cb;
void *ctxt;
struct hs_scratch *scratch;
const u8 *firstFloodDetect;
const u64a histBytes;
};

View File

@@ -94,7 +94,7 @@ const u8 * floodDetect(const struct FDR * fdr,
const u8 * buf = a->buf;
const size_t len = a->len;
HWLMCallback cb = a->cb;
void * ctxt = a->ctxt;
struct hs_scratch *scratch = a->scratch;
const u8 * ptr = *ptrPtr;
// tryFloodDetect is never put in places where unconditional
@@ -197,67 +197,67 @@ const u8 * floodDetect(const struct FDR * fdr,
t += 4) {
DEBUG_PRINTF("aaa %u %llx\n", t, fl->groups[0]);
if (*control & fl->groups[0]) {
*control = cb(i + t + 0, fl->ids[0], ctxt);
*control = cb(i + t + 0, fl->ids[0], scratch);
}
if (*control & fl->groups[0]) {
*control = cb(i + t + 1, fl->ids[0], ctxt);
*control = cb(i + t + 1, fl->ids[0], scratch);
}
if (*control & fl->groups[0]) {
*control = cb(i + t + 2, fl->ids[0], ctxt);
*control = cb(i + t + 2, fl->ids[0], scratch);
}
if (*control & fl->groups[0]) {
*control = cb(i + t + 3, fl->ids[0], ctxt);
*control = cb(i + t + 3, fl->ids[0], scratch);
}
}
break;
case 2:
for (u32 t = 0; t < floodSize && (*control & fl->allGroups); t += 4) {
if (*control & fl->groups[0]) {
*control = cb(i + t, fl->ids[0], ctxt);
*control = cb(i + t, fl->ids[0], scratch);
}
if (*control & fl->groups[1]) {
*control = cb(i + t, fl->ids[1], ctxt);
*control = cb(i + t, fl->ids[1], scratch);
}
if (*control & fl->groups[0]) {
*control =
cb(i + t + 1, fl->ids[0], ctxt);
cb(i + t + 1, fl->ids[0], scratch);
}
if (*control & fl->groups[1]) {
*control = cb(i + t + 1, fl->ids[1], ctxt);
*control = cb(i + t + 1, fl->ids[1], scratch);
}
if (*control & fl->groups[0]) {
*control = cb(i + t + 2, fl->ids[0], ctxt);
*control = cb(i + t + 2, fl->ids[0], scratch);
}
if (*control & fl->groups[1]) {
*control = cb(i + t + 2, fl->ids[1], ctxt);
*control = cb(i + t + 2, fl->ids[1], scratch);
}
if (*control & fl->groups[0]) {
*control = cb(i + t + 3, fl->ids[0], ctxt);
*control = cb(i + t + 3, fl->ids[0], scratch);
}
if (*control & fl->groups[1]) {
*control = cb(i + t + 3, fl->ids[1], ctxt);
*control = cb(i + t + 3, fl->ids[1], scratch);
}
}
break;
case 3:
for (u32 t = 0; t < floodSize && (*control & fl->allGroups); t += 2) {
if (*control & fl->groups[0]) {
*control = cb(i + t, fl->ids[0], ctxt);
*control = cb(i + t, fl->ids[0], scratch);
}
if (*control & fl->groups[1]) {
*control = cb(i + t, fl->ids[1], ctxt);
*control = cb(i + t, fl->ids[1], scratch);
}
if (*control & fl->groups[2]) {
*control = cb(i + t, fl->ids[2], ctxt);
*control = cb(i + t, fl->ids[2], scratch);
}
if (*control & fl->groups[0]) {
*control = cb(i + t + 1, fl->ids[0], ctxt);
*control = cb(i + t + 1, fl->ids[0], scratch);
}
if (*control & fl->groups[1]) {
*control = cb(i + t + 1, fl->ids[1], ctxt);
*control = cb(i + t + 1, fl->ids[1], scratch);
}
if (*control & fl->groups[2]) {
*control = cb(i + t + 1, fl->ids[2], ctxt);
*control = cb(i + t + 1, fl->ids[2], scratch);
}
}
break;
@@ -266,40 +266,40 @@ const u8 * floodDetect(const struct FDR * fdr,
for (u32 t = 0; t < floodSize && (*control & fl->allGroups); t += 2) {
if (*control & fl->groups[0]) {
*control = cb(i + t, fl->ids[0], ctxt);
*control = cb(i + t, fl->ids[0], scratch);
}
if (*control & fl->groups[1]) {
*control = cb(i + t, fl->ids[1], ctxt);
*control = cb(i + t, fl->ids[1], scratch);
}
if (*control & fl->groups[2]) {
*control = cb(i + t, fl->ids[2], ctxt);
*control = cb(i + t, fl->ids[2], scratch);
}
if (*control & fl->groups[3]) {
*control = cb(i + t, fl->ids[3], ctxt);
*control = cb(i + t, fl->ids[3], scratch);
}
for (u32 t2 = 4; t2 < fl->idCount; t2++) {
if (*control & fl->groups[t2]) {
*control = cb(i + t, fl->ids[t2], ctxt);
*control = cb(i + t, fl->ids[t2], scratch);
}
}
if (*control & fl->groups[0]) {
*control = cb(i + t + 1, fl->ids[0], ctxt);
*control = cb(i + t + 1, fl->ids[0], scratch);
}
if (*control & fl->groups[1]) {
*control = cb(i + t + 1, fl->ids[1], ctxt);
*control = cb(i + t + 1, fl->ids[1], scratch);
}
if (*control & fl->groups[2]) {
*control = cb(i + t + 1, fl->ids[2], ctxt);
*control = cb(i + t + 1, fl->ids[2], scratch);
}
if (*control & fl->groups[3]) {
*control = cb(i + t + 1, fl->ids[3], ctxt);
*control = cb(i + t + 1, fl->ids[3], scratch);
}
for (u32 t2 = 4; t2 < fl->idCount; t2++) {
if (*control & fl->groups[t2]) {
*control = cb(i + t + 1, fl->ids[t2], ctxt);
*control = cb(i + t + 1, fl->ids[t2], scratch);
}
}
}
@@ -310,7 +310,7 @@ const u8 * floodDetect(const struct FDR * fdr,
for (u32 t = 0; t < floodSize && (*control & fl->allGroups); t++) {
for (u32 t2 = 0; t2 < fl->idCount; t2++) {
if (*control & fl->groups[t2]) {
*control = cb(i + t, fl->ids[t2], ctxt);
*control = cb(i + t, fl->ids[t2], scratch);
}
}
}

View File

@@ -170,7 +170,7 @@ void do_accel_streaming(const union AccelAux *aux, const u8 *hbuf, size_t hlen,
}
hwlm_error_t hwlmExec(const struct HWLM *t, const u8 *buf, size_t len,
size_t start, HWLMCallback cb, void *ctxt,
size_t start, HWLMCallback cb, struct hs_scratch *scratch,
hwlm_group_t groups) {
assert(t);
@@ -184,7 +184,7 @@ hwlm_error_t hwlmExec(const struct HWLM *t, const u8 *buf, size_t len,
if (t->type == HWLM_ENGINE_NOOD) {
DEBUG_PRINTF("calling noodExec\n");
return noodExec(HWLM_C_DATA(t), buf, len, start, cb, ctxt);
return noodExec(HWLM_C_DATA(t), buf, len, start, cb, scratch);
}
assert(t->type == HWLM_ENGINE_FDR);
@@ -195,12 +195,12 @@ hwlm_error_t hwlmExec(const struct HWLM *t, const u8 *buf, size_t len,
}
do_accel_block(aa, buf, len, &start);
DEBUG_PRINTF("calling frankie (groups=%08llx, start=%zu)\n", groups, start);
return fdrExec(HWLM_C_DATA(t), buf, len, start, cb, ctxt, groups);
return fdrExec(HWLM_C_DATA(t), buf, len, start, cb, scratch, groups);
}
hwlm_error_t hwlmExecStreaming(const struct HWLM *t, struct hs_scratch *scratch,
size_t len, size_t start, HWLMCallback cb,
void *ctxt, hwlm_group_t groups) {
hwlm_error_t hwlmExecStreaming(const struct HWLM *t, size_t len, size_t start,
HWLMCallback cb, struct hs_scratch *scratch,
hwlm_group_t groups) {
assert(t);
assert(scratch);
@@ -222,10 +222,10 @@ hwlm_error_t hwlmExecStreaming(const struct HWLM *t, struct hs_scratch *scratch,
// If we've been handed a start offset, we can use a block mode scan at
// that offset.
if (start) {
return noodExec(HWLM_C_DATA(t), buf, len, start, cb, ctxt);
return noodExec(HWLM_C_DATA(t), buf, len, start, cb, scratch);
} else {
return noodExecStreaming(HWLM_C_DATA(t), hbuf, hlen, buf, len, cb,
ctxt);
scratch);
}
}
@@ -238,5 +238,5 @@ hwlm_error_t hwlmExecStreaming(const struct HWLM *t, struct hs_scratch *scratch,
do_accel_streaming(aa, hbuf, hlen, buf, len, &start);
DEBUG_PRINTF("calling frankie (groups=%08llx, start=%zu)\n", groups, start);
return fdrExecStreaming(HWLM_C_DATA(t), hbuf, hlen, buf, len, start, cb,
ctxt, groups);
scratch, groups);
}

View File

@@ -95,7 +95,8 @@ struct HWLM;
* belonging to the literal which was active at the when the end match location
* was first reached.
*/
typedef hwlmcb_rv_t (*HWLMCallback)(size_t end, u32 id, void *context);
typedef hwlmcb_rv_t (*HWLMCallback)(size_t end, u32 id,
struct hs_scratch *scratch);
/** \brief Match strings in table.
*
@@ -112,29 +113,28 @@ typedef hwlmcb_rv_t (*HWLMCallback)(size_t end, u32 id, void *context);
* the first possible match of a literal which is in the initial group mask.
*/
hwlm_error_t hwlmExec(const struct HWLM *tab, const u8 *buf, size_t len,
size_t start, HWLMCallback callback, void *context,
hwlm_group_t groups);
size_t start, HWLMCallback callback,
struct hs_scratch *scratch, hwlm_group_t groups);
/** \brief As for \ref hwlmExec, but a streaming case across two buffers.
*
* \p scratch is used to access fdr_temp_buf and to access the history buffer,
* history length and the main buffer.
*
* \p len is the length of the main buffer to be scanned.
*
* \p start is an advisory hint representing the first offset at which a match
* may start. Some underlying literal matches may not respect it.
*
* \p scratch is used to access the history buffer, history length and
* the main buffer.
*
* Two buffers/lengths are provided. Matches that occur entirely within
* the history buffer will not be reported by this function. The offsets
* reported for the main buffer are relative to the start of that buffer (a
* match at byte 10 of the main buffer is reported as 10). Matches that start
* in the history buffer will have starts reported with 'negative' values.
*/
hwlm_error_t hwlmExecStreaming(const struct HWLM *tab,
struct hs_scratch *scratch, size_t len,
size_t start, HWLMCallback callback,
void *context, hwlm_group_t groups);
hwlm_error_t hwlmExecStreaming(const struct HWLM *tab, size_t len, size_t start,
HWLMCallback callback,
struct hs_scratch *scratch, hwlm_group_t groups);
#ifdef __cplusplus
} /* extern "C" */

View File

@@ -32,6 +32,7 @@
#include "hwlm.h"
#include "noodle_engine.h"
#include "noodle_internal.h"
#include "scratch.h"
#include "ue2common.h"
#include "util/arch.h"
#include "util/bitutils.h"
@@ -50,7 +51,7 @@
struct cb_info {
HWLMCallback cb; //!< callback function called on match
u32 id; //!< ID to pass to callback on match
void *ctx; //!< caller-supplied context to pass to callback
struct hs_scratch *scratch; //!< scratch to pass to callback
size_t offsetAdj; //!< used in streaming mode
};
@@ -129,7 +130,7 @@ hwlm_error_t final(const struct noodTable *n, const u8 *buf, UNUSED size_t len,
match:
pos -= cbi->offsetAdj;
DEBUG_PRINTF("match @ %zu\n", pos + n->key_offset);
hwlmcb_rv_t rv = cbi->cb(pos + n->key_offset - 1, cbi->id, cbi->ctx);
hwlmcb_rv_t rv = cbi->cb(pos + n->key_offset - 1, cbi->id, cbi->scratch);
if (rv == HWLM_TERMINATE_MATCHING) {
return HWLM_TERMINATED;
}
@@ -371,10 +372,11 @@ hwlm_error_t scan(const struct noodTable *n, const u8 *buf, size_t len,
/** \brief Block-mode scanner. */
hwlm_error_t noodExec(const struct noodTable *n, const u8 *buf, size_t len,
size_t start, HWLMCallback cb, void *ctxt) {
size_t start, HWLMCallback cb,
struct hs_scratch *scratch) {
assert(n && buf);
struct cb_info cbi = {cb, n->id, ctxt, 0};
struct cb_info cbi = {cb, n->id, scratch, 0};
DEBUG_PRINTF("nood scan of %zu bytes for %*s @ %p\n", len, n->msk_len,
(const char *)&n->cmp, buf);
@@ -384,7 +386,7 @@ hwlm_error_t noodExec(const struct noodTable *n, const u8 *buf, size_t len,
/** \brief Streaming-mode scanner. */
hwlm_error_t noodExecStreaming(const struct noodTable *n, const u8 *hbuf,
size_t hlen, const u8 *buf, size_t len,
HWLMCallback cb, void *ctxt) {
HWLMCallback cb, struct hs_scratch *scratch) {
assert(n);
if (len + hlen < n->msk_len) {
@@ -392,7 +394,7 @@ hwlm_error_t noodExecStreaming(const struct noodTable *n, const u8 *hbuf,
return HWLM_SUCCESS;
}
struct cb_info cbi = {cb, n->id, ctxt, 0};
struct cb_info cbi = {cb, n->id, scratch, 0};
DEBUG_PRINTF("nood scan of %zu bytes (%zu hlen) for %*s @ %p\n", len, hlen,
n->msk_len, (const char *)&n->cmp, buf);
@@ -425,7 +427,7 @@ hwlm_error_t noodExecStreaming(const struct noodTable *n, const u8 *hbuf,
if ((v & n->msk) == n->cmp) {
size_t m_end = -tl1 + i + n->msk_len - 1;
DEBUG_PRINTF("match @ %zu (i %zu)\n", m_end, i);
hwlmcb_rv_t rv = cb(m_end, n->id, ctxt);
hwlmcb_rv_t rv = cb(m_end, n->id, scratch);
if (rv == HWLM_TERMINATE_MATCHING) {
return HWLM_TERMINATED;
}

View File

@@ -41,15 +41,17 @@ extern "C"
#endif
struct noodTable;
struct hs_scratch;
/** \brief Block-mode scanner. */
hwlm_error_t noodExec(const struct noodTable *n, const u8 *buf, size_t len,
size_t start, HWLMCallback cb, void *ctxt);
size_t start, HWLMCallback cb,
struct hs_scratch *scratch);
/** \brief Streaming-mode scanner. */
hwlm_error_t noodExecStreaming(const struct noodTable *n, const u8 *hbuf,
size_t hlen, const u8 *buf, size_t len,
HWLMCallback cb, void *ctxt);
HWLMCallback cb, struct hs_scratch *scratch);
#ifdef __cplusplus
} /* extern "C" */

View File

@@ -66,8 +66,8 @@ void printMatch(const struct core_info *ci, u64a start, u64a end) {
}
#endif
hwlmcb_rv_t roseDelayRebuildCallback(size_t end, u32 id, void *ctx) {
struct hs_scratch *scratch = ctx;
hwlmcb_rv_t roseDelayRebuildCallback(size_t end, u32 id,
struct hs_scratch *scratch) {
struct RoseContext *tctx = &scratch->tctxt;
struct core_info *ci = &scratch->core_info;
const struct RoseEngine *t = ci->rose;
@@ -472,8 +472,7 @@ anchored_leftovers:;
}
static really_inline
hwlmcb_rv_t roseCallback_i(size_t end, u32 id, void *ctxt) {
struct hs_scratch *scratch = ctxt;
hwlmcb_rv_t roseCallback_i(size_t end, u32 id, struct hs_scratch *scratch) {
struct RoseContext *tctx = &scratch->tctxt;
const struct RoseEngine *t = scratch->core_info.rose;
@@ -519,15 +518,15 @@ hwlmcb_rv_t roseCallback_i(size_t end, u32 id, void *ctxt) {
return HWLM_TERMINATE_MATCHING;
}
hwlmcb_rv_t roseCallback(size_t end, u32 id, void *ctxt) {
return roseCallback_i(end, id, ctxt);
hwlmcb_rv_t roseCallback(size_t end, u32 id, struct hs_scratch *scratch) {
return roseCallback_i(end, id, scratch);
}
hwlmcb_rv_t roseFloatingCallback(size_t end, u32 id, void *ctxt) {
struct hs_scratch *scratch = ctxt;
hwlmcb_rv_t roseFloatingCallback(size_t end, u32 id,
struct hs_scratch *scratch) {
const struct RoseEngine *t = scratch->core_info.rose;
return roseCallback_i(end, id, ctxt) & t->floating_group_mask;
return roseCallback_i(end, id, scratch) & t->floating_group_mask;
}
/**

View File

@@ -52,9 +52,11 @@ int roseNfaAdaptor(u64a start, u64a end, ReportID id, void *context);
/* Callbacks, defined in match.c */
hwlmcb_rv_t roseCallback(size_t end, u32 id, void *ctx);
hwlmcb_rv_t roseFloatingCallback(size_t end, u32 id, void *ctx);
hwlmcb_rv_t roseDelayRebuildCallback(size_t end, u32 id, void *ctx);
hwlmcb_rv_t roseCallback(size_t end, u32 id, struct hs_scratch *scratch);
hwlmcb_rv_t roseFloatingCallback(size_t end, u32 id,
struct hs_scratch *scratch);
hwlmcb_rv_t roseDelayRebuildCallback(size_t end, u32 id,
struct hs_scratch *scratch);
int roseAnchoredCallback(u64a start, u64a end, u32 id, void *ctx);
/* Common code, used all over Rose runtime */

View File

@@ -46,7 +46,7 @@ void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch);
void roseStreamEodExec(const struct RoseEngine *t, u64a offset,
struct hs_scratch *scratch);
hwlmcb_rv_t roseCallback(size_t end, u32 id, void *context);
hwlmcb_rv_t roseCallback(size_t end, u32 id, struct hs_scratch *scratch);
int roseReportAdaptor(u64a start, u64a end, ReportID id, void *context);

View File

@@ -659,8 +659,8 @@ void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch) {
DEBUG_PRINTF("start=%zu\n", start);
DEBUG_PRINTF("BEGIN FLOATING (over %zu/%zu)\n", flen, length);
hwlmExecStreaming(ftable, scratch, flen, start, roseFloatingCallback,
scratch, tctxt->groups & t->floating_group_mask);
hwlmExecStreaming(ftable, flen, start, roseFloatingCallback, scratch,
tctxt->groups & t->floating_group_mask);
}
flush_delay_and_exit:

View File

@@ -764,7 +764,7 @@ void pureLiteralStreamExec(struct hs_stream *stream_state,
// start the match region at zero.
const size_t start = 0;
hwlmExecStreaming(ftable, scratch, len2, start, roseCallback, scratch,
hwlmExecStreaming(ftable, len2, start, roseCallback, scratch,
rose->initialGroups & rose->floating_group_mask);
if (!told_to_stop_matching(scratch) &&