github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-11-17 01:41:51 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rose: replace internal_report with program

Browse Source 
Replace the use of the internal_report structure (for reports from
engines, MPV etc) with the Rose program interpreter.

SOM processing was reworked to use a new som_operation structure that is
embedded in the appropriate instructions.
This commit is contained in:
Justin Viiret
2016-03-22 16:45:09 +11:00
committed by Matthew Barr
parent 68851742cc
commit 36150bbc19
31 changed files with 877 additions and 1356 deletions
Download Patch File Download Diff File Expand all files Collapse all files

335
src/report.h
View File

@@ -43,7 +43,6 @@
#include "som/som_runtime.h"
#include "util/exhaust.h"
#include "util/fatbit.h"
#include "util/internal_report.h"
static really_inline
int satisfiesMinLength(u64a min_len, u64a from_offset,
@@ -68,14 +67,12 @@ enum DedupeResult {
static really_inline
enum DedupeResult dedupeCatchup(const struct RoseEngine *rose,
const struct internal_report *ir,
struct hs_scratch *scratch, u64a offset,
u64a from_offset, u64a to_offset,
const char do_som) {
u64a from_offset, u64a to_offset, u32 dkey,
s32 offset_adjust, char is_external_report,
char quash_som, const char do_som) {
DEBUG_PRINTF("offset=%llu, match=[%llu,%llu], dkey=%u, do_som=%d\n", offset,
from_offset, to_offset, ir->dkey, do_som);
DEBUG_PRINTF("report type=%u, quashSom=%d\n", ir->type, ir->quashSom);
const u32 dkey = ir->dkey;
from_offset, to_offset, dkey, do_som);
// We should not have been called if there's no dedupe work to do.
assert(do_som || dkey != MO_INVALID_IDX);
@@ -99,10 +96,9 @@ enum DedupeResult dedupeCatchup(const struct RoseEngine *rose,
if (dkey != MO_INVALID_IDX) {
const u32 dkeyCount = rose->dkeyCount;
const s32 offset_adj = ir->offsetAdjust;
if (ir->type == EXTERNAL_CALLBACK || ir->quashSom) {
if (is_external_report || quash_som) {
DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset);
assert(offset_adj == 0 || offset_adj == -1);
assert(offset_adjust == 0 || offset_adjust == -1);
if (fatbit_set(deduper->log[to_offset % 2], dkeyCount, dkey)) {
/* we have already raised this report at this offset, squash
* dupe match. */
@@ -112,7 +108,7 @@ enum DedupeResult dedupeCatchup(const struct RoseEngine *rose,
} else if (do_som) {
/* SOM external event */
DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset);
assert(offset_adj == 0 || offset_adj == -1);
assert(offset_adjust == 0 || offset_adjust == -1);
u64a *starts = deduper->som_start_log[to_offset % 2];
if (fatbit_set(deduper->som_log[to_offset % 2], dkeyCount, dkey)) {
starts[dkey] = MIN(starts[dkey], from_offset);
@@ -121,7 +117,7 @@ enum DedupeResult dedupeCatchup(const struct RoseEngine *rose,
}
DEBUG_PRINTF("starts[%u]=%llu\n", dkey, starts[dkey]);
if (offset_adj) {
if (offset_adjust) {
deduper->som_log_dirty |= 1;
} else {
deduper->som_log_dirty |= 2;
@@ -134,176 +130,6 @@ enum DedupeResult dedupeCatchup(const struct RoseEngine *rose,
return DEDUPE_CONTINUE;
}
static really_inline
enum DedupeResult dedupeCatchupSom(const struct RoseEngine *rose,
const struct internal_report *ir,
struct hs_scratch *scratch, u64a offset,
u64a from_offset, u64a to_offset) {
DEBUG_PRINTF("offset=%llu, match=[%llu,%llu], dkey=%u\n", offset,
from_offset, to_offset, ir->dkey);
DEBUG_PRINTF("report type=%u, quashSom=%d\n", ir->type, ir->quashSom);
struct match_deduper *deduper = &scratch->deduper;
if (offset != deduper->current_report_offset) {
assert(deduper->current_report_offset == ~0ULL ||
deduper->current_report_offset < offset);
if (offset == deduper->current_report_offset + 1) {
fatbit_clear(deduper->log[offset % 2]);
} else {
fatbit_clear(deduper->log[0]);
fatbit_clear(deduper->log[1]);
}
if (flushStoredSomMatches(scratch, offset)) {
return DEDUPE_HALT;
}
deduper->current_report_offset = offset;
}
const u32 dkey = ir->dkey;
if (dkey != MO_INVALID_IDX) {
const u32 dkeyCount = rose->dkeyCount;
const s32 offset_adj = ir->offsetAdjust;
if (ir->quashSom) {
DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset);
assert(offset_adj == 0 || offset_adj == -1);
if (fatbit_set(deduper->log[to_offset % 2], dkeyCount, dkey)) {
/* we have already raised this report at this offset, squash
* dupe match. */
DEBUG_PRINTF("dedupe\n");
return DEDUPE_SKIP;
}
} else {
/* SOM external event */
DEBUG_PRINTF("checking dkey %u at offset %llu\n", dkey, to_offset);
assert(offset_adj == 0 || offset_adj == -1);
u64a *starts = deduper->som_start_log[to_offset % 2];
if (fatbit_set(deduper->som_log[to_offset % 2], dkeyCount, dkey)) {
starts[dkey] = MIN(starts[dkey], from_offset);
} else {
starts[dkey] = from_offset;
}
DEBUG_PRINTF("starts[%u]=%llu\n", dkey, starts[dkey]);
if (offset_adj) {
deduper->som_log_dirty |= 1;
} else {
deduper->som_log_dirty |= 2;
}
return DEDUPE_SKIP;
}
}
return DEDUPE_CONTINUE;
}
static really_inline
int roseAdaptor_i(u64a offset, ReportID id, struct hs_scratch *scratch,
char is_simple, char do_som) {
assert(id != MO_INVALID_IDX); // Should never get an invalid ID.
assert(scratch);
assert(scratch->magic == SCRATCH_MAGIC);
struct core_info *ci = &scratch->core_info;
const struct RoseEngine *rose = ci->rose;
DEBUG_PRINTF("internal report %u\n", id);
const struct internal_report *ir = getInternalReport(rose, id);
assert(isExternalReport(ir)); /* only external reports should reach here */
s32 offset_adj = ir->offsetAdjust;
u64a to_offset = offset;
u64a from_offset = 0;
u32 flags = 0;
#ifndef RELEASE_BUILD
if (offset_adj) {
// alert testing tools that we've got adjusted matches
flags |= HS_MATCH_FLAG_ADJUSTED;
}
#endif
DEBUG_PRINTF("internal match at %llu: IID=%u type=%hhu RID=%u "
"offsetAdj=%d\n", offset, id, ir->type, ir->onmatch,
offset_adj);
if (unlikely(can_stop_matching(scratch))) { /* ok - we are from rose */
DEBUG_PRINTF("pre broken - halting\n");
return MO_HALT_MATCHING;
}
if (!is_simple && ir->hasBounds) {
assert(ir->minOffset || ir->minLength || ir->maxOffset < MAX_OFFSET);
assert(ir->minOffset <= ir->maxOffset);
if (offset < ir->minOffset || offset > ir->maxOffset) {
DEBUG_PRINTF("match fell outside valid range %llu !: [%llu,%llu]\n",
offset, ir->minOffset, ir->maxOffset);
return ROSE_CONTINUE_MATCHING_NO_EXHAUST;
}
}
if (!is_simple && ir->ekey != INVALID_EKEY &&
unlikely(isExhausted(ci->rose, ci->exhaustionVector, ir->ekey))) {
DEBUG_PRINTF("ate exhausted match\n");
return MO_CONTINUE_MATCHING;
}
if (ir->type == EXTERNAL_CALLBACK) {
from_offset = 0;
} else if (do_som) {
from_offset = handleSomExternal(scratch, ir, to_offset);
}
to_offset += offset_adj;
assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset);
if (do_som && ir->minLength) {
if (!satisfiesMinLength(ir->minLength, from_offset, to_offset)) {
return ROSE_CONTINUE_MATCHING_NO_EXHAUST;
}
if (ir->quashSom) {
from_offset = 0;
}
}
DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n",
from_offset, to_offset, ir->onmatch, ci->userContext);
int halt = 0;
if (do_som || ir->dkey != MO_INVALID_IDX) {
enum DedupeResult dedupe_rv = dedupeCatchup(rose, ir, scratch, offset,
from_offset, to_offset, do_som);
switch (dedupe_rv) {
case DEDUPE_HALT:
halt = 1;
goto exit;
case DEDUPE_SKIP:
halt = 0;
goto exit;
case DEDUPE_CONTINUE:
break;
}
}
halt = ci->userCallback((unsigned int)ir->onmatch, from_offset, to_offset,
flags, ci->userContext);
exit:
if (halt) {
DEBUG_PRINTF("callback requested to terminate matches\n");
ci->status |= STATUS_TERMINATED;
return MO_HALT_MATCHING;
}
if (!is_simple && ir->ekey != INVALID_EKEY) {
markAsMatched(ci->rose, ci->exhaustionVector, ir->ekey);
return MO_CONTINUE_MATCHING;
} else {
return ROSE_CONTINUE_MATCHING_NO_EXHAUST;
}
}
/**
* \brief Deliver the given report to the user callback.
*
@@ -311,8 +137,8 @@ exit:
* that dedupe catchup has been done.
*/
static really_inline
int roseDeliverReport(u64a offset, UNUSED ReportID id, ReportID onmatch,
s32 offset_adjust, struct hs_scratch *scratch, u32 ekey) {
int roseDeliverReport(u64a offset, ReportID onmatch, s32 offset_adjust,
struct hs_scratch *scratch, u32 ekey) {
assert(scratch);
assert(scratch->magic == SCRATCH_MAGIC);
@@ -326,19 +152,7 @@ int roseDeliverReport(u64a offset, UNUSED ReportID id, ReportID onmatch,
}
#endif
#ifndef NDEBUG
// Assertions for development builds.
UNUSED const struct internal_report *ir = getInternalReport(ci->rose, id);
assert(isExternalReport(ir)); /* only external reports should reach here */
assert(!can_stop_matching(scratch));
assert(!ir->hasBounds ||
(offset >= ir->minOffset && offset <= ir->maxOffset));
assert(ir->type == EXTERNAL_CALLBACK);
assert(!ir->minLength);
assert(!ir->quashSom);
#endif
assert(ekey == INVALID_EKEY ||
!isExhausted(ci->rose, ci->exhaustionVector, ekey));
@@ -364,102 +178,6 @@ int roseDeliverReport(u64a offset, UNUSED ReportID id, ReportID onmatch,
}
}
static really_inline
int roseSomAdaptor_i(u64a from_offset, u64a to_offset, ReportID id,
struct hs_scratch *scratch, char is_simple) {
assert(id != MO_INVALID_IDX); // Should never get an invalid ID.
assert(scratch);
assert(scratch->magic == SCRATCH_MAGIC);
u32 flags = 0;
struct core_info *ci = &scratch->core_info;
const struct RoseEngine *rose = ci->rose;
const struct internal_report *ir = getInternalReport(rose, id);
/* internal events should be handled by rose directly */
assert(ir->type == EXTERNAL_CALLBACK);
DEBUG_PRINTF("internal match at %llu: IID=%u type=%hhu RID=%u "
"offsetAdj=%d\n", to_offset, id, ir->type, ir->onmatch,
ir->offsetAdjust);
if (unlikely(can_stop_matching(scratch))) {
DEBUG_PRINTF("pre broken - halting\n");
return MO_HALT_MATCHING;
}
if (!is_simple && ir->hasBounds) {
assert(ir->minOffset || ir->minLength || ir->maxOffset < MAX_OFFSET);
if (to_offset < ir->minOffset || to_offset > ir->maxOffset) {
DEBUG_PRINTF("match fell outside valid range %llu !: [%llu,%llu]\n",
to_offset, ir->minOffset, ir->maxOffset);
return MO_CONTINUE_MATCHING;
}
}
int halt = 0;
if (!is_simple && ir->ekey != INVALID_EKEY &&
unlikely(isExhausted(ci->rose, ci->exhaustionVector, ir->ekey))) {
DEBUG_PRINTF("ate exhausted match\n");
goto exit;
}
u64a offset = to_offset;
to_offset += ir->offsetAdjust;
assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset);
if (!is_simple && ir->minLength) {
if (!satisfiesMinLength(ir->minLength, from_offset, to_offset)) {
return MO_CONTINUE_MATCHING;
}
if (ir->quashSom) {
from_offset = 0;
}
}
DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n",
from_offset, to_offset, ir->onmatch, ci->userContext);
#ifndef RELEASE_BUILD
if (ir->offsetAdjust != 0) {
// alert testing tools that we've got adjusted matches
flags |= HS_MATCH_FLAG_ADJUSTED;
}
#endif
enum DedupeResult dedupe_rv =
dedupeCatchupSom(rose, ir, scratch, offset, from_offset, to_offset);
switch (dedupe_rv) {
case DEDUPE_HALT:
halt = 1;
goto exit;
case DEDUPE_SKIP:
halt = 0;
goto exit;
case DEDUPE_CONTINUE:
break;
}
halt = ci->userCallback((unsigned int)ir->onmatch, from_offset, to_offset,
flags, ci->userContext);
if (!is_simple && ir->ekey != INVALID_EKEY) {
markAsMatched(ci->rose, ci->exhaustionVector, ir->ekey);
}
exit:
if (halt) {
DEBUG_PRINTF("callback requested to terminate matches\n");
ci->status |= STATUS_TERMINATED;
return MO_HALT_MATCHING;
}
return MO_CONTINUE_MATCHING;
}
/**
* \brief Deliver the given SOM report to the user callback.
*
@@ -467,42 +185,34 @@ exit:
* that dedupe catchup has been done.
*/
static really_inline
int roseDeliverSomReport(u64a from_offset, u64a to_offset,
const struct internal_report *ir,
struct hs_scratch *scratch, char is_exhaustible) {
int roseDeliverSomReport(u64a from_offset, u64a to_offset, ReportID onmatch,
s32 offset_adjust, struct hs_scratch *scratch,
u32 ekey) {
assert(scratch);
assert(scratch->magic == SCRATCH_MAGIC);
assert(isExternalReport(ir)); /* only external reports should reach here */
struct core_info *ci = &scratch->core_info;
u32 flags = 0;
#ifndef RELEASE_BUILD
if (ir->offsetAdjust != 0) {
if (offset_adjust) {
// alert testing tools that we've got adjusted matches
flags |= HS_MATCH_FLAG_ADJUSTED;
}
#endif
assert(!can_stop_matching(scratch));
assert(!ir->hasBounds ||
(to_offset >= ir->minOffset && to_offset <= ir->maxOffset));
assert(ir->ekey == INVALID_EKEY ||
!isExhausted(ci->rose, ci->exhaustionVector, ir->ekey));
assert(ekey == INVALID_EKEY ||
!isExhausted(ci->rose, ci->exhaustionVector, ekey));
to_offset += ir->offsetAdjust;
to_offset += offset_adjust;
assert(from_offset == HS_OFFSET_PAST_HORIZON || from_offset <= to_offset);
assert(!ir->minLength ||
satisfiesMinLength(ir->minLength, from_offset, to_offset));
assert(!ir->quashSom || from_offset == 0);
DEBUG_PRINTF(">> reporting match @[%llu,%llu] for sig %u ctxt %p <<\n",
from_offset, to_offset, ir->onmatch, ci->userContext);
from_offset, to_offset, onmatch, ci->userContext);
int halt = ci->userCallback((unsigned int)ir->onmatch, from_offset,
to_offset, flags, ci->userContext);
int halt = ci->userCallback(onmatch, from_offset, to_offset, flags,
ci->userContext);
if (halt) {
DEBUG_PRINTF("callback requested to terminate matches\n");
@@ -510,9 +220,8 @@ int roseDeliverSomReport(u64a from_offset, u64a to_offset,
return MO_HALT_MATCHING;
}
if (is_exhaustible) {
assert(ir->ekey != INVALID_EKEY);
markAsMatched(ci->rose, ci->exhaustionVector, ir->ekey);
if (ekey != INVALID_EKEY) {
markAsMatched(ci->rose, ci->exhaustionVector, ekey);
return MO_CONTINUE_MATCHING;
} else {
return ROSE_CONTINUE_MATCHING_NO_EXHAUST;