github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-09-29 19:24:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rose: replace internal_report with program

Browse Source 
Replace the use of the internal_report structure (for reports from
engines, MPV etc) with the Rose program interpreter.

SOM processing was reworked to use a new som_operation structure that is
embedded in the appropriate instructions.
This commit is contained in:
Justin Viiret
2016-03-22 16:45:09 +11:00
committed by Matthew Barr
parent 68851742cc
commit 36150bbc19
31 changed files with 877 additions and 1356 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/util/exhaust.h
View File

@@ -34,10 +34,12 @@
#define EXHAUST_H
#include "rose/rose_internal.h"
#include "util/internal_report.h"
#include "util/multibit.h"
#include "ue2common.h"
/** Index meaning a given exhaustion key is invalid. */
#define INVALID_EKEY (~(u32)0)
/** \brief Test whether the given key (\a ekey) is set in the exhaustion vector
* \a evec. */
static really_inline

201
src/util/internal_report.h
View File

@@ -1,201 +0,0 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief Definition of an internal_report, along with convenience functions.
*/
#ifndef INTERNAL_REPORT_H
#define INTERNAL_REPORT_H
#include "ue2common.h"
/* internal_report::type values */
#define EXTERNAL_CALLBACK 0
#define EXTERNAL_CALLBACK_SOM_REL 1
#define INTERNAL_SOM_LOC_SET 2
#define INTERNAL_SOM_LOC_SET_IF_UNSET 3
#define INTERNAL_SOM_LOC_SET_IF_WRITABLE 4
#define INTERNAL_SOM_LOC_SET_SOM_REV_NFA 5
#define INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET 6
#define INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE 7
#define INTERNAL_SOM_LOC_COPY 8
#define INTERNAL_SOM_LOC_COPY_IF_WRITABLE 9
#define INTERNAL_SOM_LOC_MAKE_WRITABLE 10
#define EXTERNAL_CALLBACK_SOM_STORED 11
#define EXTERNAL_CALLBACK_SOM_ABS 12
#define EXTERNAL_CALLBACK_SOM_REV_NFA 13
/** set the som loc to the value in from_offset */
#define INTERNAL_SOM_LOC_SET_FROM 14
/** set the som loc to the value in from_offset */
#define INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE 15
#define INTERNAL_ROSE_CHAIN 16
/** Index meaning a given exhaustion key is invalid. */
#define INVALID_EKEY 0xffffffff
/** \brief All the data we use for handling a match, bytecode representation.
*
* Includes extparam constraints and bounds, exhaustion/dedupe keys, offset
* adjustment and SOM information.
*
* At compile time, this data is in the ue2::Report structure, which is
* converted to internal_report for layout in the bytecode.
*/
struct ALIGN_CL_DIRECTIVE internal_report {
/** \brief from EXTERNAL_ and INTERNAL_ defined above. */
u8 type;
/** \brief do we actually use minOffset, maxOffset */
u8 hasBounds;
/** \brief use SOM for minLength, but don't report it to user callback. */
u8 quashSom;
/** \brief min offset in the stream at which this report can match. */
u64a minOffset;
/** \brief max offset in the stream at which this report can match. */
u64a maxOffset;
/** \brief min match length (start of match to current offset) */
u64a minLength;
/** \brief Exhaustion key.
*
* If exhaustible, the ekey to check before reporting a match.
* Additionally after reporting a match the ekey will be set. If not
* exhaustible, this will be INVALID_EKEY. */
u32 ekey;
/** \brief Dedupe key. */
u32 dkey;
/** \brief Adjustment to add to the match offset when we report a match.
*
* This is usually used for reports attached to states that form part of a
* zero-width assertion, like '$'. */
s32 offsetAdjust;
/** \brief Match report ID, for external reports.
*
* - external callback -> external report id
* - internal_som_* -> som loc to modify,
* - INTERNAL_ROSE_CHAIN -> top event to push on
* - otherwise target subnfa. */
u32 onmatch;
union {
/** \brief SOM distance value, use varies according to type.
*
* - for EXTERNAL_CALLBACK_SOM_REL, from-offset is this many bytes
* before the to-offset.
* - for EXTERNAL_CALLBACK_SOM_ABS, set from-offset to this value.
* - for INTERNAL_SOM_LOC_COPY*, som location read_from.
*/
u64a somDistance;
/** \brief Index of the reverse nfa.
* Used by EXTERNAL_CALLBACK_SOM_REV_NFA and
* INTERNAL_SOM_LOC_SET_SOM_REV_NFA*
*/
u64a revNfaIndex;
/**
* Used by INTERNAL_ROSE_CHAIN, Number of bytes behind us that we are
* allowed to squash identical top events on the queue.
*/
u64a topSquashDistance;
} aux;
};
static really_inline
int isInternalSomReport(const struct internal_report *ri) {
switch (ri->type) {
case INTERNAL_SOM_LOC_SET:
case INTERNAL_SOM_LOC_SET_IF_UNSET:
case INTERNAL_SOM_LOC_SET_IF_WRITABLE:
case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
case INTERNAL_SOM_LOC_COPY:
case INTERNAL_SOM_LOC_COPY_IF_WRITABLE:
case INTERNAL_SOM_LOC_MAKE_WRITABLE:
case INTERNAL_SOM_LOC_SET_FROM:
case INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE:
return 1;
case EXTERNAL_CALLBACK:
case EXTERNAL_CALLBACK_SOM_REL:
case EXTERNAL_CALLBACK_SOM_STORED:
case EXTERNAL_CALLBACK_SOM_ABS:
case EXTERNAL_CALLBACK_SOM_REV_NFA:
case INTERNAL_ROSE_CHAIN:
return 0;
default:
break; // fall through
}
assert(0); // unknown?
return 0;
}
#ifndef NDEBUG
/* used in asserts */
static UNUSED
char isExternalReport(const struct internal_report *ir) {
switch (ir->type) {
case INTERNAL_SOM_LOC_SET:
case INTERNAL_SOM_LOC_SET_IF_UNSET:
case INTERNAL_SOM_LOC_SET_IF_WRITABLE:
case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
case INTERNAL_SOM_LOC_COPY:
case INTERNAL_SOM_LOC_COPY_IF_WRITABLE:
case INTERNAL_SOM_LOC_MAKE_WRITABLE:
case INTERNAL_SOM_LOC_SET_FROM:
case INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE:
case INTERNAL_ROSE_CHAIN:
return 0;
case EXTERNAL_CALLBACK:
case EXTERNAL_CALLBACK_SOM_REL:
case EXTERNAL_CALLBACK_SOM_STORED:
case EXTERNAL_CALLBACK_SOM_ABS:
case EXTERNAL_CALLBACK_SOM_REV_NFA:
return 1;
default:
break; // fall through
}
assert(0); // unknown?
return 1;
}
#endif
#endif // INTERNAL_REPORT_H

69
src/util/report.cpp
View File

@@ -1,69 +0,0 @@
/*
* Copyright (c) 2015, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "internal_report.h"
#include "report.h"
#include "report_manager.h"
namespace ue2 {
void writeInternalReport(const Report &report, const ReportManager &rm,
internal_report *ir) {
assert(ir);
assert(ISALIGNED(ir));
ir->type = report.type;
ir->hasBounds = report.hasBounds() ? 1 : 0;
ir->quashSom = report.quashSom ? 1 : 0;
ir->minOffset = report.minOffset;
ir->maxOffset = report.maxOffset;
ir->minLength = report.minLength;
ir->ekey = report.ekey;
ir->offsetAdjust = report.offsetAdjust;
ir->onmatch = report.onmatch;
switch (report.type) {
case INTERNAL_ROSE_CHAIN:
ir->aux.topSquashDistance = report.topSquashDistance;
break;
case EXTERNAL_CALLBACK_SOM_REV_NFA:
case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
ir->aux.revNfaIndex = report.revNfaIndex;
break;
default:
ir->aux.somDistance = report.somDistance;
break;
}
// Dedupe keys are managed by ReportManager.
ir->dkey = rm.getDkey(report);
}
} // namespace ue2

58
src/util/report.h
View File

@@ -34,31 +34,49 @@
#ifndef UTIL_REPORT_H
#define UTIL_REPORT_H
#include "internal_report.h"
#include "util/exhaust.h" // for INVALID_EKEY
#include "order_check.h"
#include "ue2common.h"
#include <cassert>
struct internal_report;
namespace ue2 {
class ReportManager;
enum ReportType {
EXTERNAL_CALLBACK,
EXTERNAL_CALLBACK_SOM_REL,
INTERNAL_SOM_LOC_SET,
INTERNAL_SOM_LOC_SET_IF_UNSET,
INTERNAL_SOM_LOC_SET_IF_WRITABLE,
INTERNAL_SOM_LOC_SET_SOM_REV_NFA,
INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET,
INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE,
INTERNAL_SOM_LOC_COPY,
INTERNAL_SOM_LOC_COPY_IF_WRITABLE,
INTERNAL_SOM_LOC_MAKE_WRITABLE,
EXTERNAL_CALLBACK_SOM_STORED,
EXTERNAL_CALLBACK_SOM_ABS,
EXTERNAL_CALLBACK_SOM_REV_NFA,
INTERNAL_SOM_LOC_SET_FROM,
INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE,
INTERNAL_ROSE_CHAIN,
EXTERNAL_CALLBACK_SOM_PASS
};
/**
* \brief All the data we use for handling a match.
*
* Includes extparam constraints and bounds, exhaustion/dedupe keys, offset
* adjustment and SOM information.
*
* The data in this structure is converted into an \ref internal_report in the
* bytecode.
* The data in this structure eventually becomes a list of Rose programs
* instructions.
*/
struct Report {
Report(u8 type_in, u32 onmatch_in) : type(type_in), onmatch(onmatch_in) {
assert(type <= INTERNAL_ROSE_CHAIN);
}
Report(ReportType type_in, u32 onmatch_in)
: type(type_in), onmatch(onmatch_in) {}
/** \brief True if this report has bounds from extended parameters, i.e.
* min offset, max offset, min length. */
@@ -66,8 +84,8 @@ struct Report {
return minOffset > 0 || maxOffset < MAX_OFFSET || minLength > 0;
}
/** \brief from EXTERNAL_ and INTERNAL_ defined in internal_report.h. */
u8 type;
/** \brief Type of this report. */
ReportType type;
/** \brief use SOM for minLength, but don't report it to user callback. */
bool quashSom = false;
@@ -147,6 +165,7 @@ bool isInternalSomReport(const Report &r) {
case EXTERNAL_CALLBACK_SOM_ABS:
case EXTERNAL_CALLBACK_SOM_REV_NFA:
case INTERNAL_ROSE_CHAIN:
case EXTERNAL_CALLBACK_SOM_PASS:
return false;
default:
break; // fall through
@@ -176,6 +195,7 @@ bool isExternalReport(const Report &r) {
case EXTERNAL_CALLBACK_SOM_STORED:
case EXTERNAL_CALLBACK_SOM_ABS:
case EXTERNAL_CALLBACK_SOM_REV_NFA:
case EXTERNAL_CALLBACK_SOM_PASS:
return true;
default:
break; // fall through
@@ -228,7 +248,7 @@ Report makeSomRelativeCallback(u32 report, s32 offsetAdjust, u64a distance) {
}
static inline
Report makeRoseTrigger(u32 event, u64a squashDistance) {
Report makeMpvTrigger(u32 event, u64a squashDistance) {
Report ir(INTERNAL_ROSE_CHAIN, event);
ir.ekey = INVALID_EKEY;
ir.topSquashDistance = squashDistance;
@@ -254,22 +274,6 @@ bool isSimpleExhaustible(const Report &ir) {
return true;
}
/** True if this report requires some of the more esoteric processing in the
* rose adaptor, rather than just firing a callback or doing SOM handling.
*/
static inline
bool isComplexReport(const Report &ir) {
if (ir.hasBounds() || ir.ekey != INVALID_EKEY) {
return true;
}
return false;
}
/** \brief Write the given Report into an internal_report structure. */
void writeInternalReport(const Report &report, const ReportManager &rm,
internal_report *ir);
} // namespace
#endif // UTIL_REPORT_H