github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-09-29 19:24:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

Logical Combination of patterns.

Browse Source
This commit is contained in:
Chang, Harry
2018-06-22 18:15:21 +08:00
parent 5895b8da25
commit 8a1c497f44
50 changed files with 2693 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
src/compiler/compiler.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -45,6 +45,7 @@
#include "parser/buildstate.h"
#include "parser/dump.h"
#include "parser/Component.h"
#include "parser/logical_combination.h"
#include "parser/parse_error.h"
#include "parser/Parser.h" // for flags
#include "parser/position.h"
@@ -111,7 +112,13 @@ ParsedExpression::ParsedExpression(unsigned index_in, const char *expression,
const hs_expr_ext *ext)
: expr(index_in, flags & HS_FLAG_ALLOWEMPTY, flags & HS_FLAG_SINGLEMATCH,
false, flags & HS_FLAG_PREFILTER, SOM_NONE, report, 0, MAX_OFFSET,
0, 0, 0) {
0, 0, 0, flags & HS_FLAG_QUIET) {
// We disallow SOM + Quiet.
if ((flags & HS_FLAG_QUIET) && (flags & HS_FLAG_SOM_LEFTMOST)) {
throw CompileError("HS_FLAG_QUIET is not supported in "
"combination with HS_FLAG_SOM_LEFTMOST.");
}
flags &= ~HS_FLAG_QUIET;
ParseMode mode(flags);
component = parse(expression, mode);
@@ -234,6 +241,45 @@ void addExpression(NG &ng, unsigned index, const char *expression,
DEBUG_PRINTF("index=%u, id=%u, flags=%u, expr='%s'\n", index, id, flags,
expression);
if (flags & HS_FLAG_COMBINATION) {
if (flags & ~(HS_FLAG_COMBINATION | HS_FLAG_QUIET |
HS_FLAG_SINGLEMATCH)) {
throw CompileError("only HS_FLAG_QUIET and HS_FLAG_SINGLEMATCH "
"are supported in combination "
"with HS_FLAG_COMBINATION.");
}
if (flags & HS_FLAG_QUIET) {
DEBUG_PRINTF("skip QUIET logical combination expression %u\n", id);
} else {
u32 ekey = INVALID_EKEY;
u64a min_offset = 0;
u64a max_offset = MAX_OFFSET;
if (flags & HS_FLAG_SINGLEMATCH) {
ekey = ng.rm.getExhaustibleKey(id);
}
if (ext) {
validateExt(*ext);
if (ext->flags & ~(HS_EXT_FLAG_MIN_OFFSET |
HS_EXT_FLAG_MAX_OFFSET)) {
throw CompileError("only HS_EXT_FLAG_MIN_OFFSET and "
"HS_EXT_FLAG_MAX_OFFSET extra flags "
"are supported in combination "
"with HS_FLAG_COMBINATION.");
}
if (ext->flags & HS_EXT_FLAG_MIN_OFFSET) {
min_offset = ext->min_offset;
}
if (ext->flags & HS_EXT_FLAG_MAX_OFFSET) {
max_offset = ext->max_offset;
}
}
ng.rm.pl.parseLogicalCombination(id, expression, ekey, min_offset,
max_offset);
DEBUG_PRINTF("parsed logical combination expression %u\n", id);
}
return;
}
// Ensure that our pattern isn't too long (in characters).
if (strlen(expression) > cc.grey.limitPatternLength) {
throw CompileError("Pattern length exceeds limit.");

9
src/compiler/expression_info.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, Intel Corporation
* Copyright (c) 2017-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -46,12 +46,12 @@ public:
bool highlander_in, bool utf8_in, bool prefilter_in,
som_type som_in, ReportID report_in, u64a min_offset_in,
u64a max_offset_in, u64a min_length_in, u32 edit_distance_in,
u32 hamm_distance_in)
u32 hamm_distance_in, bool quiet_in)
: index(index_in), report(report_in), allow_vacuous(allow_vacuous_in),
highlander(highlander_in), utf8(utf8_in), prefilter(prefilter_in),
som(som_in), min_offset(min_offset_in), max_offset(max_offset_in),
min_length(min_length_in), edit_distance(edit_distance_in),
hamm_distance(hamm_distance_in) {}
hamm_distance(hamm_distance_in), quiet(quiet_in) {}
/**
* \brief Index of the expression represented by this graph.
@@ -98,6 +98,9 @@ public:
*/
u32 edit_distance;
u32 hamm_distance;
/** \brief Quiet on match. */
bool quiet;
};
}

7
src/hs.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -245,6 +245,11 @@ hs_compile_multi_int(const char *const *expressions, const unsigned *flags,
}
}
// Check sub-expression ids
ng.rm.pl.validateSubIDs(ids, expressions, flags, elements);
// Renumber and assign lkey to reports
ng.rm.logicalKeyRenumber();
unsigned length = 0;
struct hs_database *out = build(ng, &length);

24
src/hs_compile.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -811,6 +811,28 @@ hs_error_t HS_CDECL hs_populate_platform(hs_platform_info_t *platform);
*/
#define HS_FLAG_SOM_LEFTMOST 256
/**
* Compile flag: Logical combination.
*
* This flag instructs Hyperscan to parse this expression as logical
* combination syntax.
* Logical constraints consist of operands, operators and parentheses.
* The operands are expression indices, and operators can be
* '!'(NOT), '&'(AND) or '|'(OR).
* For example:
* (101&102&103)|(104&!105)
* ((301|302)&303)&(304|305)
*/
#define HS_FLAG_COMBINATION 512
/**
* Compile flag: Don't do any match reporting.
*
* This flag instructs Hyperscan to ignore match reporting for this expression.
* It is designed to be used on the sub-expressions in logical combinations.
*/
#define HS_FLAG_QUIET 1024
/** @} */
/**

7
src/nfagraph/ng.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -577,7 +577,8 @@ bool NG::addHolder(NGHolder &g) {
}
bool NG::addLiteral(const ue2_literal &literal, u32 expr_index,
u32 external_report, bool highlander, som_type som) {
u32 external_report, bool highlander, som_type som,
bool quiet) {
assert(!literal.empty());
if (!cc.grey.shortcutLiterals) {
@@ -605,7 +606,7 @@ bool NG::addLiteral(const ue2_literal &literal, u32 expr_index,
} else {
u32 ekey = highlander ? rm.getExhaustibleKey(external_report)
: INVALID_EKEY;
Report r = makeECallback(external_report, 0, ekey);
Report r = makeECallback(external_report, 0, ekey, quiet);
id = rm.getInternalId(r);
}

4
src/nfagraph/ng.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -77,7 +77,7 @@ public:
/** \brief Adds a literal to Rose, used by literal shortcut passes (instead
* of using \ref addGraph) */
bool addLiteral(const ue2_literal &lit, u32 expr_index, u32 external_report,
bool highlander, som_type som);
bool highlander, som_type som, bool quiet);
/** \brief Maximum history in bytes available for use by SOM reverse NFAs,
* a hack for pattern support (see UE-1903). This is always set to the max

376
src/parser/logical_combination.cpp Normal file
View File

@@ -0,0 +1,376 @@
/*
* Copyright (c) 2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief Parse and build ParsedLogical::logicalTree and combInfoMap.
*/
#include "logical_combination.h"
#include "parser/parse_error.h"
#include "util/container.h"
#include "hs_compile.h"
#include <vector>
using namespace std;
namespace ue2 {
u32 ParsedLogical::getLogicalKey(u32 a) {
auto it = toLogicalKeyMap.find(a);
if (it == toLogicalKeyMap.end()) {
// get size before assigning to avoid wacky LHS shenanigans
u32 size = toLogicalKeyMap.size();
bool inserted;
tie(it, inserted) = toLogicalKeyMap.emplace(a, size);
assert(inserted);
}
DEBUG_PRINTF("%u -> lkey %u\n", it->first, it->second);
return it->second;
}
u32 ParsedLogical::getCombKey(u32 a) {
auto it = toCombKeyMap.find(a);
if (it == toCombKeyMap.end()) {
u32 size = toCombKeyMap.size();
bool inserted;
tie(it, inserted) = toCombKeyMap.emplace(a, size);
assert(inserted);
}
DEBUG_PRINTF("%u -> ckey %u\n", it->first, it->second);
return it->second;
}
void ParsedLogical::addRelateCKey(u32 lkey, u32 ckey) {
auto it = lkey2ckeys.find(lkey);
if (it == lkey2ckeys.end()) {
bool inserted;
tie(it, inserted) = lkey2ckeys.emplace(lkey, set<u32>());
assert(inserted);
}
it->second.insert(ckey);
DEBUG_PRINTF("lkey %u belongs to combination key %u\n",
it->first, ckey);
}
#define TRY_RENUM_OP(ckey) \
do { \
if (ckey & LOGICAL_OP_BIT) { \
ckey = (ckey & ~LOGICAL_OP_BIT) + toLogicalKeyMap.size(); \
} \
} while(0)
u32 ParsedLogical::logicalTreeAdd(u32 op, u32 left, u32 right) {
LogicalOp lop;
assert((LOGICAL_OP_BIT & (u32)logicalTree.size()) == 0);
lop.id = LOGICAL_OP_BIT | (u32)logicalTree.size();
lop.op = op;
lop.lo = left;
lop.ro = right;
logicalTree.push_back(lop);
return lop.id;
}
void ParsedLogical::combinationInfoAdd(UNUSED u32 ckey, u32 id, u32 ekey,
u32 lkey_start, u32 lkey_result,
u64a min_offset, u64a max_offset) {
assert(ckey == combInfoMap.size());
CombInfo ci;
ci.id = id;
ci.ekey = ekey;
ci.start = lkey_start;
ci.result = lkey_result;
ci.min_offset = min_offset;
ci.max_offset = max_offset;
combInfoMap.push_back(ci);
DEBUG_PRINTF("ckey %u (id %u) -> lkey %u..%u, ekey=0x%x\n", ckey, ci.id,
ci.start, ci.result, ci.ekey);
}
void ParsedLogical::validateSubIDs(const unsigned *ids,
const char *const *expressions,
const unsigned *flags,
unsigned elements) {
for (const auto &it : toLogicalKeyMap) {
bool unknown = true;
u32 i = 0;
for (i = 0; i < elements; i++) {
if ((ids ? ids[i] : 0) == it.first) {
unknown = false;
break;
}
}
if (unknown) {
throw CompileError("Unknown sub-expression id.");
}
if (contains(toCombKeyMap, it.first)) {
throw CompileError("Have combination of combination.");
}
if (flags && (flags[i] & HS_FLAG_SOM_LEFTMOST)) {
throw CompileError("Have SOM flag in sub-expression.");
}
if (flags && (flags[i] & HS_FLAG_PREFILTER)) {
throw CompileError("Have PREFILTER flag in sub-expression.");
}
hs_compile_error_t *compile_err = NULL;
hs_expr_info_t *info = NULL;
hs_error_t err = hs_expression_info(expressions[i], flags[i], &info,
&compile_err);
if (err != HS_SUCCESS) {
hs_free_compile_error(compile_err);
throw CompileError("Run hs_expression_info() failed.");
}
if (!info) {
throw CompileError("Get hs_expr_info_t failed.");
} else {
if (info->unordered_matches) {
throw CompileError("Have unordered match in sub-expressions.");
}
free(info);
}
}
}
void ParsedLogical::logicalKeyRenumber() {
// renumber operation lkey in op vector
for (auto &op : logicalTree) {
TRY_RENUM_OP(op.id);
TRY_RENUM_OP(op.lo);
TRY_RENUM_OP(op.ro);
}
// renumber operation lkey in info map
for (auto &ci : combInfoMap) {
TRY_RENUM_OP(ci.start);
TRY_RENUM_OP(ci.result);
}
}
struct LogicalOperator {
LogicalOperator(u32 op_in, u32 paren_in)
: op(op_in), paren(paren_in) {}
u32 op;
u32 paren;
};
static
u32 toOperator(char c) {
u32 op = UNKNOWN_OP;
switch (c) {
case '!' :
op = LOGICAL_OP_NOT;
break;
case '&' :
op = LOGICAL_OP_AND;
break;
case '|' :
op = LOGICAL_OP_OR;
break;
default:
break;
};
return op;
}
static
bool cmpOperator(const LogicalOperator &op1, const LogicalOperator &op2) {
if (op1.paren < op2.paren) {
return false;
}
if (op1.paren > op2.paren) {
return true;
}
assert(op1.paren == op2.paren);
if (op1.op > op2.op) {
return false;
}
if (op1.op < op2.op) {
return true;
}
return true;
}
static
u32 fetchSubID(const char *logical, u32 &digit, u32 end) {
if (digit == (u32)-1) { // no digit parsing in progress
return (u32)-1;
}
assert(end > digit);
if (end - digit > 9) {
throw LocatedParseError("Expression id too large");
}
u32 mult = 1;
u32 sum = 0;
for (u32 j = end - 1; (j >= digit) && (j != (u32)-1) ; j--) {
assert(isdigit(logical[j]));
sum += (logical[j] - '0') * mult;
mult *= 10;
}
digit = (u32)-1;
return sum;
}
static
void popOperator(vector<LogicalOperator> &op_stack, vector<u32> &subid_stack,
ParsedLogical &pl) {
if (subid_stack.empty()) {
throw LocatedParseError("Not enough operand");
}
u32 right = subid_stack.back();
subid_stack.pop_back();
u32 left = 0;
if (op_stack.back().op != LOGICAL_OP_NOT) {
if (subid_stack.empty()) {
throw LocatedParseError("Not enough operand");
}
left = subid_stack.back();
subid_stack.pop_back();
}
subid_stack.push_back(pl.logicalTreeAdd(op_stack.back().op, left, right));
op_stack.pop_back();
}
static
char getValue(const vector<char> &lv, u32 ckey) {
if (ckey & LOGICAL_OP_BIT) {
return lv[ckey & ~LOGICAL_OP_BIT];
} else {
return 0;
}
}
static
bool hasMatchFromPurelyNegative(const vector<LogicalOp> &tree,
u32 start, u32 result) {
vector<char> lv(tree.size());
assert(start <= result);
for (u32 i = start; i <= result; i++) {
assert(i & LOGICAL_OP_BIT);
const LogicalOp &op = tree[i & ~LOGICAL_OP_BIT];
assert(i == op.id);
switch (op.op) {
case LOGICAL_OP_NOT:
lv[op.id & ~LOGICAL_OP_BIT] = !getValue(lv, op.ro);
break;
case LOGICAL_OP_AND:
lv[op.id & ~LOGICAL_OP_BIT] = getValue(lv, op.lo) &
getValue(lv, op.ro);
break;
case LOGICAL_OP_OR:
lv[op.id & ~LOGICAL_OP_BIT] = getValue(lv, op.lo) |
getValue(lv, op.ro);
break;
default:
assert(0);
break;
}
}
return lv[result & ~LOGICAL_OP_BIT];
}
void ParsedLogical::parseLogicalCombination(unsigned id, const char *logical,
u32 ekey, u64a min_offset,
u64a max_offset) {
u32 ckey = getCombKey(id);
vector<LogicalOperator> op_stack;
vector<u32> subid_stack;
u32 lkey_start = INVALID_LKEY; // logical operation's lkey
u32 paren = 0; // parentheses
u32 digit = (u32)-1; // digit start offset, invalid offset is -1
u32 subid = (u32)-1;
u32 i;
try {
for (i = 0; logical[i]; i++) {
if (isdigit(logical[i])) {
if (digit == (u32)-1) { // new digit start
digit = i;
}
} else {
if ((subid = fetchSubID(logical, digit, i)) != (u32)-1) {
subid_stack.push_back(getLogicalKey(subid));
addRelateCKey(subid_stack.back(), ckey);
}
if (logical[i] == ' ') { // skip whitespace
continue;
}
if (logical[i] == '(') {
paren += 1;
} else if (logical[i] == ')') {
if (paren <= 0) {
throw LocatedParseError("Not enough left parentheses");
}
paren -= 1;
} else {
u32 prio = toOperator(logical[i]);
if (prio != UNKNOWN_OP) {
LogicalOperator op(prio, paren);
while (!op_stack.empty()
&& cmpOperator(op_stack.back(), op)) {
popOperator(op_stack, subid_stack, *this);
if (lkey_start == INVALID_LKEY) {
lkey_start = subid_stack.back();
}
}
op_stack.push_back(op);
} else {
throw LocatedParseError("Unknown character");
}
}
}
}
if (paren != 0) {
throw LocatedParseError("Not enough right parentheses");
}
if ((subid = fetchSubID(logical, digit, i)) != (u32)-1) {
subid_stack.push_back(getLogicalKey(subid));
addRelateCKey(subid_stack.back(), ckey);
}
while (!op_stack.empty()) {
popOperator(op_stack, subid_stack, *this);
if (lkey_start == INVALID_LKEY) {
lkey_start = subid_stack.back();
}
}
if (subid_stack.size() != 1) {
throw LocatedParseError("Not enough operator");
}
} catch (LocatedParseError &error) {
error.locate(i);
throw;
}
u32 lkey_result = subid_stack.back(); // logical operation's lkey
if (lkey_start == INVALID_LKEY) {
throw CompileError("No logical operation.");
}
if (hasMatchFromPurelyNegative(logicalTree, lkey_start, lkey_result)) {
throw CompileError("Has match from purely negative sub-expressions.");
}
combinationInfoAdd(ckey, id, ekey, lkey_start, lkey_result,
min_offset, max_offset);
}
} // namespace ue2

112
src/parser/logical_combination.h Normal file
View File

@@ -0,0 +1,112 @@
/*
* Copyright (c) 2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief Parse and build ParsedLogical::logicalTree and combInfoMap.
*/
#ifndef LOGICAL_COMBINATION_H
#define LOGICAL_COMBINATION_H
#include "util/logical.h"
#include <map>
#include <set>
#include <vector>
namespace ue2 {
class ParsedLogical {
friend class ReportManager;
public:
/** \brief Parse 1 logical expression \a logical, assign temporary ckey. */
void parseLogicalCombination(unsigned id, const char *logical, u32 ekey,
u64a min_offset, u64a max_offset);
/** \brief Check if all sub-expression id in combinations are valid. */
void validateSubIDs(const unsigned *ids, const char *const *expressions,
const unsigned *flags, unsigned elements);
/** \brief Renumber and assign final lkey for each logical operation
* after parsed all logical expressions. */
void logicalKeyRenumber();
/** \brief Fetch the lkey associated with the given expression id,
* assigning one if necessary. */
u32 getLogicalKey(u32 expressionId);
/** \brief Fetch the ckey associated with the given expression id,
* assigning one if necessary. */
u32 getCombKey(u32 expressionId);
/** \brief Add lkey's corresponding combination id. */
void addRelateCKey(u32 lkey, u32 ckey);
/** \brief Add one Logical Operation. */
u32 logicalTreeAdd(u32 op, u32 left, u32 right);
/** \brief Assign the combination info associated with the given ckey. */
void combinationInfoAdd(u32 ckey, u32 id, u32 ekey, u32 lkey_start,
u32 lkey_result, u64a min_offset, u64a max_offset);
const std::map<u32, u32> &getLkeyMap() const {
return toLogicalKeyMap;
}
const std::vector<LogicalOp> &getLogicalTree() const {
return logicalTree;
}
CombInfo getCombInfoById(u32 id) const {
u32 ckey = toCombKeyMap.at(id);
assert(ckey < combInfoMap.size());
return combInfoMap.at(ckey);
}
private:
/** \brief Mapping from ckey to combination info. */
std::vector<CombInfo> combInfoMap;
/** \brief Mapping from combination expression id to combination key,
* combination key is used in combination bit-vector cache. */
std::map<u32, u32> toCombKeyMap;
/** \brief Mapping from expression id to logical key, logical key is used
* as index in LogicalOp array. */
std::map<u32, u32> toLogicalKeyMap;
/** \brief Mapping from logical key to related combination keys. */
std::map<u32, std::set<u32>> lkey2ckeys;
/** \brief Logical constraints, each operation from postfix notation. */
std::vector<LogicalOp> logicalTree;
};
} // namespace ue2
#endif

4
src/parser/shortcut_literal.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -199,7 +199,7 @@ bool shortcutLiteral(NG &ng, const ParsedExpression &pe) {
DEBUG_PRINTF("constructed literal %s\n", dumpString(lit).c_str());
return ng.addLiteral(lit, expr.index, expr.report, expr.highlander,
expr.som);
expr.som, expr.quiet);
}
} // namespace ue2

90
src/report.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, Intel Corporation
* Copyright (c) 2016-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -42,6 +42,7 @@
#include "rose/runtime.h"
#include "som/som_runtime.h"
#include "util/exhaust.h"
#include "util/logical.h"
#include "util/fatbit.h"
enum DedupeResult {
@@ -151,6 +152,93 @@ void clearEvec(const struct RoseEngine *rose, char *evec) {
mmbit_clear((u8 *)evec, rose->ekeyCount);
}
/** \brief Test whether the given key (\a lkey) is set in the logical vector
* \a lvec. */
static really_inline
char getLogicalVal(const struct RoseEngine *rose, const char *lvec, u32 lkey) {
DEBUG_PRINTF("checking lkey matching %p %u\n", lvec, lkey);
assert(lkey != INVALID_LKEY);
assert(lkey < rose->lkeyCount + rose->lopCount);
return mmbit_isset((const u8 *)lvec, rose->lkeyCount + rose->lopCount,
lkey);
}
/** \brief Mark key \a lkey on in the logical vector. */
static really_inline
void setLogicalVal(const struct RoseEngine *rose, char *lvec, u32 lkey,
char val) {
DEBUG_PRINTF("marking as matched logical key %u\n", lkey);
assert(lkey != INVALID_LKEY);
assert(lkey < rose->lkeyCount + rose->lopCount);
switch (val) {
case 0:
mmbit_unset((u8 *)lvec, rose->lkeyCount + rose->lopCount, lkey);
break;
default:
mmbit_set((u8 *)lvec, rose->lkeyCount + rose->lopCount, lkey);
break;
}
}
/** \brief Mark key \a ckey on in the combination vector. */
static really_inline
void setCombinationActive(const struct RoseEngine *rose, char *cvec, u32 ckey) {
DEBUG_PRINTF("marking as active combination key %u\n", ckey);
assert(ckey != INVALID_CKEY);
assert(ckey < rose->ckeyCount);
mmbit_set((u8 *)cvec, rose->ckeyCount, ckey);
}
/** \brief Returns 1 if compliant to all logical combinations. */
static really_inline
char isLogicalCombination(const struct RoseEngine *rose, char *lvec,
u32 start, u32 result) {
const struct LogicalOp *logicalTree = (const struct LogicalOp *)
((const char *)rose + rose->logicalTreeOffset);
assert(start >= rose->lkeyCount);
assert(start <= result);
assert(result < rose->lkeyCount + rose->lopCount);
for (u32 i = start; i <= result; i++) {
const struct LogicalOp *op = logicalTree + (i - rose->lkeyCount);
assert(i == op->id);
assert(op->op <= LAST_LOGICAL_OP);
switch ((enum LogicalOpType)op->op) {
case LOGICAL_OP_NOT:
setLogicalVal(rose, lvec, op->id,
!getLogicalVal(rose, lvec, op->ro));
break;
case LOGICAL_OP_AND:
setLogicalVal(rose, lvec, op->id,
getLogicalVal(rose, lvec, op->lo) &
getLogicalVal(rose, lvec, op->ro)); // &&
break;
case LOGICAL_OP_OR:
setLogicalVal(rose, lvec, op->id,
getLogicalVal(rose, lvec, op->lo) |
getLogicalVal(rose, lvec, op->ro)); // ||
break;
}
}
return getLogicalVal(rose, lvec, result);
}
/** \brief Clear all keys in the logical vector. */
static really_inline
void clearLvec(const struct RoseEngine *rose, char *lvec, char *cvec) {
DEBUG_PRINTF("clearing lvec %p %u\n", lvec,
rose->lkeyCount + rose->lopCount);
DEBUG_PRINTF("clearing cvec %p %u\n", cvec, rose->ckeyCount);
mmbit_clear((u8 *)lvec, rose->lkeyCount + rose->lopCount);
mmbit_clear((u8 *)cvec, rose->ckeyCount);
}
/** \brief Clear all keys in the combination vector. */
static really_inline
void clearCvec(const struct RoseEngine *rose, char *cvec) {
DEBUG_PRINTF("clearing cvec %p %u\n", cvec, rose->ckeyCount);
mmbit_clear((u8 *)cvec, rose->ckeyCount);
}
/**
* \brief Deliver the given report to the user callback.
*

3
src/rose/block.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -145,6 +145,7 @@ void init_for_block(const struct RoseEngine *t, struct hs_scratch *scratch,
tctxt->lastEndOffset = 0;
tctxt->filledDelayedSlots = 0;
tctxt->lastMatchOffset = 0;
tctxt->lastCombMatchOffset = 0;
tctxt->minMatchOffset = 0;
tctxt->minNonMpvMatchOffset = 0;
tctxt->next_mpv_offset = 0;

8
src/rose/catchup.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -424,6 +424,12 @@ hwlmcb_rv_t roseCatchUpMPV_i(const struct RoseEngine *t, s64a loc,
}
done:
if (t->flushCombProgramOffset) {
if (roseRunFlushCombProgram(t, scratch, mpv_exec_end)
== HWLM_TERMINATE_MATCHING) {
return HWLM_TERMINATE_MATCHING;
}
}
updateMinMatchOffsetFromMpv(&scratch->tctxt, mpv_exec_end);
scratch->tctxt.next_mpv_offset
= MAX(next_pos_match_loc + scratch->core_info.buf_offset,

15
src/rose/catchup.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -51,6 +51,7 @@
#include "hwlm/hwlm.h"
#include "runtime.h"
#include "scratch.h"
#include "rose.h"
#include "rose_common.h"
#include "rose_internal.h"
#include "ue2common.h"
@@ -105,6 +106,12 @@ hwlmcb_rv_t roseCatchUpMPV(const struct RoseEngine *t, s64a loc,
assert(!can_stop_matching(scratch));
if (canSkipCatchUpMPV(t, scratch, cur_offset)) {
if (t->flushCombProgramOffset) {
if (roseRunFlushCombProgram(t, scratch, cur_offset)
== HWLM_TERMINATE_MATCHING) {
return HWLM_TERMINATE_MATCHING;
}
}
updateMinMatchOffsetFromMpv(&scratch->tctxt, cur_offset);
return HWLM_CONTINUE_MATCHING;
}
@@ -139,6 +146,12 @@ hwlmcb_rv_t roseCatchUpTo(const struct RoseEngine *t,
hwlmcb_rv_t rv;
if (!t->activeArrayCount
|| !mmbit_any(getActiveLeafArray(t, state), t->activeArrayCount)) {
if (t->flushCombProgramOffset) {
if (roseRunFlushCombProgram(t, scratch, end)
== HWLM_TERMINATE_MATCHING) {
return HWLM_TERMINATE_MATCHING;
}
}
updateMinMatchOffset(&scratch->tctxt, end);
rv = HWLM_CONTINUE_MATCHING;
} else {

18
src/rose/match.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -571,6 +571,22 @@ int roseRunBoundaryProgram(const struct RoseEngine *rose, u32 program,
return MO_CONTINUE_MATCHING;
}
/**
* \brief Execute a flush combination program.
*
* Returns MO_HALT_MATCHING if the stream is exhausted or the user has
* instructed us to halt, or MO_CONTINUE_MATCHING otherwise.
*/
int roseRunFlushCombProgram(const struct RoseEngine *rose,
struct hs_scratch *scratch, u64a end) {
hwlmcb_rv_t rv = roseRunProgram(rose, scratch, rose->flushCombProgramOffset,
0, end, 0);
if (rv == HWLM_TERMINATE_MATCHING) {
return MO_HALT_MATCHING;
}
return MO_CONTINUE_MATCHING;
}
int roseReportAdaptor(u64a start, u64a end, ReportID id, void *context) {
struct hs_scratch *scratch = context;
assert(scratch && scratch->magic == SCRATCH_MAGIC);

107
src/rose/program_runtime.h
View File

@@ -557,6 +557,22 @@ void roseHandleSomSom(struct hs_scratch *scratch,
setSomFromSomAware(scratch, sr, start, end);
}
static rose_inline
hwlmcb_rv_t roseSetExhaust(const struct RoseEngine *t,
struct hs_scratch *scratch, u32 ekey) {
assert(scratch);
assert(scratch->magic == SCRATCH_MAGIC);
struct core_info *ci = &scratch->core_info;
assert(!can_stop_matching(scratch));
assert(!isExhausted(ci->rose, ci->exhaustionVector, ekey));
markAsMatched(ci->rose, ci->exhaustionVector, ekey);
return roseHaltIfExhausted(t, scratch);
}
static really_inline
int reachHasBit(const u8 *reach, u8 c) {
return !!(reach[c / 8U] & (u8)1U << (c % 8U));
@@ -1822,6 +1838,56 @@ void updateSeqPoint(struct RoseContext *tctxt, u64a offset,
}
}
static rose_inline
hwlmcb_rv_t flushActiveCombinations(const struct RoseEngine *t,
struct hs_scratch *scratch) {
u8 *cvec = (u8 *)scratch->core_info.combVector;
if (!mmbit_any(cvec, t->ckeyCount)) {
return HWLM_CONTINUE_MATCHING;
}
u64a end = scratch->tctxt.lastCombMatchOffset;
for (u32 i = mmbit_iterate(cvec, t->ckeyCount, MMB_INVALID);
i != MMB_INVALID; i = mmbit_iterate(cvec, t->ckeyCount, i)) {
const struct CombInfo *combInfoMap = (const struct CombInfo *)
((const char *)t + t->combInfoMapOffset);
const struct CombInfo *ci = combInfoMap + i;
if ((ci->min_offset != 0) && (end < ci->min_offset)) {
DEBUG_PRINTF("halt: before min_offset=%llu\n", ci->min_offset);
continue;
}
if ((ci->max_offset != MAX_OFFSET) && (end > ci->max_offset)) {
DEBUG_PRINTF("halt: after max_offset=%llu\n", ci->max_offset);
continue;
}
DEBUG_PRINTF("check ekey %u\n", ci->ekey);
if (ci->ekey != INVALID_EKEY) {
assert(ci->ekey < t->ekeyCount);
const char *evec = scratch->core_info.exhaustionVector;
if (isExhausted(t, evec, ci->ekey)) {
DEBUG_PRINTF("ekey %u already set, match is exhausted\n",
ci->ekey);
continue;
}
}
DEBUG_PRINTF("check ckey %u\n", i);
char *lvec = scratch->core_info.logicalVector;
if (!isLogicalCombination(t, lvec, ci->start, ci->result)) {
DEBUG_PRINTF("Logical Combination Failed!\n");
continue;
}
DEBUG_PRINTF("Logical Combination Passed!\n");
if (roseReport(t, scratch, end, ci->id, 0,
ci->ekey) == HWLM_TERMINATE_MATCHING) {
return HWLM_TERMINATE_MATCHING;
}
}
clearCvec(t, (char *)cvec);
return HWLM_CONTINUE_MATCHING;
}
#define PROGRAM_CASE(name) \
case ROSE_INSTR_##name: { \
DEBUG_PRINTF("instruction: " #name " (pc=%u)\n", \
@@ -2587,6 +2653,47 @@ hwlmcb_rv_t roseRunProgram_i(const struct RoseEngine *t,
}
}
PROGRAM_NEXT_INSTRUCTION
PROGRAM_CASE(SET_LOGICAL) {
DEBUG_PRINTF("set logical value of lkey %u, offset_adjust=%d\n",
ri->lkey, ri->offset_adjust);
assert(ri->lkey != INVALID_LKEY);
assert(ri->lkey < t->lkeyCount);
char *lvec = scratch->core_info.logicalVector;
setLogicalVal(t, lvec, ri->lkey, 1);
updateLastCombMatchOffset(tctxt, end + ri->offset_adjust);
}
PROGRAM_NEXT_INSTRUCTION
PROGRAM_CASE(SET_COMBINATION) {
DEBUG_PRINTF("set ckey %u as active\n", ri->ckey);
assert(ri->ckey != INVALID_CKEY);
assert(ri->ckey < t->ckeyCount);
char *cvec = scratch->core_info.combVector;
setCombinationActive(t, cvec, ri->ckey);
}
PROGRAM_NEXT_INSTRUCTION
PROGRAM_CASE(FLUSH_COMBINATION) {
assert(end >= tctxt->lastCombMatchOffset);
if (end > tctxt->lastCombMatchOffset) {
if (flushActiveCombinations(t, scratch)
== HWLM_TERMINATE_MATCHING) {
return HWLM_TERMINATE_MATCHING;
}
}
}
PROGRAM_NEXT_INSTRUCTION
PROGRAM_CASE(SET_EXHAUST) {
updateSeqPoint(tctxt, end, from_mpv);
if (roseSetExhaust(t, scratch, ri->ekey)
== HWLM_TERMINATE_MATCHING) {
return HWLM_TERMINATE_MATCHING;
}
work_done = 1;
}
PROGRAM_NEXT_INSTRUCTION
}
}

5
src/rose/rose.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -53,4 +53,7 @@ int roseReportAdaptor(u64a start, u64a end, ReportID id, void *context);
int roseRunBoundaryProgram(const struct RoseEngine *rose, u32 program,
u64a stream_offset, struct hs_scratch *scratch);
int roseRunFlushCombProgram(const struct RoseEngine *rose,
struct hs_scratch *scratch, u64a end);
#endif // ROSE_H

38
src/rose/rose_build_bytecode.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -426,6 +426,17 @@ void fillStateOffsets(const RoseBuildImpl &build, u32 rolesWithStateCount,
curr_offset += mmbit_size(build.rm.numEkeys());
so->exhausted_size = mmbit_size(build.rm.numEkeys());
// Logical multibit.
so->logicalVec = curr_offset;
so->logicalVec_size = mmbit_size(build.rm.numLogicalKeys() +
build.rm.numLogicalOps());
curr_offset += so->logicalVec_size;
// Combination multibit.
so->combVec = curr_offset;
so->combVec_size = mmbit_size(build.rm.numCkeys());
curr_offset += so->combVec_size;
// SOM locations and valid/writeable multibit structures.
if (build.ssm.numSomSlots()) {
const u32 somWidth = build.ssm.somPrecision();
@@ -2469,6 +2480,18 @@ void writeLeftInfo(RoseEngineBlob &engine_blob, RoseEngine &proto,
proto.rosePrefixCount = countRosePrefixes(leftInfoTable);
}
static
void writeLogicalInfo(const ReportManager &rm, RoseEngineBlob &engine_blob,
RoseEngine &proto) {
const auto &tree = rm.getLogicalTree();
proto.logicalTreeOffset = engine_blob.add_range(tree);
const auto &combMap = rm.getCombInfoMap();
proto.combInfoMapOffset = engine_blob.add_range(combMap);
proto.lkeyCount = rm.numLogicalKeys();
proto.lopCount = rm.numLogicalOps();
proto.ckeyCount = rm.numCkeys();
}
static
void writeNfaInfo(const RoseBuildImpl &build, build_context &bc,
RoseEngine &proto, const set<u32> &no_retrigger_queues) {
@@ -3313,6 +3336,15 @@ RoseProgram makeEodProgram(const RoseBuildImpl &build, build_context &bc,
return program;
}
static
RoseProgram makeFlushCombProgram(const RoseEngine &t) {
RoseProgram program;
if (t.ckeyCount) {
addFlushCombinationProgram(program);
}
return program;
}
static
u32 history_required(const rose_literal_id &key) {
if (key.msk.size() < key.s.length()) {
@@ -3678,6 +3710,10 @@ bytecode_ptr<RoseEngine> RoseBuildImpl::buildFinalEngine(u32 minWidth) {
writeDkeyInfo(rm, bc.engine_blob, proto);
writeLeftInfo(bc.engine_blob, proto, leftInfoTable);
writeLogicalInfo(rm, bc.engine_blob, proto);
auto flushComb_prog = makeFlushCombProgram(proto);
proto.flushCombProgramOffset = writeProgram(bc, move(flushComb_prog));
// Build anchored matcher.
auto atable = buildAnchoredMatcher(*this, fragments, anchored_dfas);

53
src/rose/rose_build_dump.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -1469,6 +1469,25 @@ void dumpProgram(ofstream &os, const RoseEngine *t, const char *pc) {
}
PROGRAM_NEXT_INSTRUCTION
PROGRAM_CASE(SET_LOGICAL) {
os << " lkey " << ri->lkey << endl;
os << " offset_adjust " << ri->offset_adjust << endl;
}
PROGRAM_NEXT_INSTRUCTION
PROGRAM_CASE(SET_COMBINATION) {
os << " ckey " << ri->ckey << endl;
}
PROGRAM_NEXT_INSTRUCTION
PROGRAM_CASE(FLUSH_COMBINATION) {}
PROGRAM_NEXT_INSTRUCTION
PROGRAM_CASE(SET_EXHAUST) {
os << " ekey " << ri->ekey << endl;
}
PROGRAM_NEXT_INSTRUCTION
default:
os << " UNKNOWN (code " << int{code} << ")" << endl;
os << " <stopping>" << endl;
@@ -1523,6 +1542,23 @@ void dumpRoseEodPrograms(const RoseEngine *t, const string &filename) {
os.close();
}
static
void dumpRoseFlushCombPrograms(const RoseEngine *t, const string &filename) {
ofstream os(filename);
const char *base = (const char *)t;
if (t->flushCombProgramOffset) {
os << "Flush Combination Program @ " << t->flushCombProgramOffset
<< ":" << endl;
dumpProgram(os, t, base + t->flushCombProgramOffset);
os << endl;
} else {
os << "<No Flush Combination Program>" << endl;
}
os.close();
}
static
void dumpRoseReportPrograms(const RoseEngine *t, const string &filename) {
ofstream os(filename);
@@ -2028,6 +2064,10 @@ void roseDumpText(const RoseEngine *t, FILE *f) {
fprintf(f, " - history buffer : %u bytes\n", t->historyRequired);
fprintf(f, " - exhaustion vector : %u bytes\n",
t->stateOffsets.exhausted_size);
fprintf(f, " - logical vector : %u bytes\n",
t->stateOffsets.logicalVec_size);
fprintf(f, " - combination vector: %u bytes\n",
t->stateOffsets.combVec_size);
fprintf(f, " - role state mmbit : %u bytes\n", t->stateSize);
fprintf(f, " - long lit matcher : %u bytes\n", t->longLitStreamState);
fprintf(f, " - active array : %u bytes\n",
@@ -2092,6 +2132,11 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
DUMP_U32(t, mode);
DUMP_U32(t, historyRequired);
DUMP_U32(t, ekeyCount);
DUMP_U32(t, lkeyCount);
DUMP_U32(t, lopCount);
DUMP_U32(t, ckeyCount);
DUMP_U32(t, logicalTreeOffset);
DUMP_U32(t, combInfoMapOffset);
DUMP_U32(t, dkeyCount);
DUMP_U32(t, dkeyLogSize);
DUMP_U32(t, invDkeyOffset);
@@ -2127,6 +2172,7 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
DUMP_U32(t, leftOffset);
DUMP_U32(t, roseCount);
DUMP_U32(t, eodProgramOffset);
DUMP_U32(t, flushCombProgramOffset);
DUMP_U32(t, lastByteHistoryIterOffset);
DUMP_U32(t, minWidth);
DUMP_U32(t, minWidthExcludingBoundaries);
@@ -2150,6 +2196,10 @@ void roseDumpStructRaw(const RoseEngine *t, FILE *f) {
DUMP_U32(t, stateOffsets.history);
DUMP_U32(t, stateOffsets.exhausted);
DUMP_U32(t, stateOffsets.exhausted_size);
DUMP_U32(t, stateOffsets.logicalVec);
DUMP_U32(t, stateOffsets.logicalVec_size);
DUMP_U32(t, stateOffsets.combVec);
DUMP_U32(t, stateOffsets.combVec_size);
DUMP_U32(t, stateOffsets.activeLeafArray);
DUMP_U32(t, stateOffsets.activeLeafArray_size);
DUMP_U32(t, stateOffsets.activeLeftArray);
@@ -2200,6 +2250,7 @@ void roseDumpPrograms(const vector<LitFragment> &fragments, const RoseEngine *t,
const string &base) {
dumpRoseLitPrograms(fragments, t, base + "/rose_lit_programs.txt");
dumpRoseEodPrograms(t, base + "/rose_eod_programs.txt");
dumpRoseFlushCombPrograms(t, base + "/rose_flush_comb_programs.txt");
dumpRoseReportPrograms(t, base + "/rose_report_programs.txt");
dumpRoseAnchoredPrograms(t, base + "/rose_anchored_programs.txt");
dumpRoseDelayPrograms(t, base + "/rose_delay_programs.txt");

25
src/rose/rose_build_instructions.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, Intel Corporation
* Copyright (c) 2017-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -47,6 +47,7 @@ RoseInstrSuffixesEod::~RoseInstrSuffixesEod() = default;
RoseInstrMatcherEod::~RoseInstrMatcherEod() = default;
RoseInstrEnd::~RoseInstrEnd() = default;
RoseInstrClearWorkDone::~RoseInstrClearWorkDone() = default;
RoseInstrFlushCombination::~RoseInstrFlushCombination() = default;
using OffsetMap = RoseInstruction::OffsetMap;
@@ -644,4 +645,26 @@ void RoseInstrIncludedJump::write(void *dest, RoseEngineBlob &blob,
inst->squash = squash;
}
void RoseInstrSetLogical::write(void *dest, RoseEngineBlob &blob,
const OffsetMap &offset_map) const {
RoseInstrBase::write(dest, blob, offset_map);
auto *inst = static_cast<impl_type *>(dest);
inst->lkey = lkey;
inst->offset_adjust = offset_adjust;
}
void RoseInstrSetCombination::write(void *dest, RoseEngineBlob &blob,
const OffsetMap &offset_map) const {
RoseInstrBase::write(dest, blob, offset_map);
auto *inst = static_cast<impl_type *>(dest);
inst->ckey = ckey;
}
void RoseInstrSetExhaust::write(void *dest, RoseEngineBlob &blob,
const OffsetMap &offset_map) const {
RoseInstrBase::write(dest, blob, offset_map);
auto *inst = static_cast<impl_type *>(dest);
inst->ekey = ekey;
}
}

90
src/rose/rose_build_instructions.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, Intel Corporation
* Copyright (c) 2017-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -2144,6 +2144,94 @@ public:
}
};
class RoseInstrSetLogical
: public RoseInstrBaseNoTargets<ROSE_INSTR_SET_LOGICAL,
ROSE_STRUCT_SET_LOGICAL,
RoseInstrSetLogical> {
public:
u32 lkey;
s32 offset_adjust;
RoseInstrSetLogical(u32 lkey_in, s32 offset_adjust_in)
: lkey(lkey_in), offset_adjust(offset_adjust_in) {}
bool operator==(const RoseInstrSetLogical &ri) const {
return lkey == ri.lkey && offset_adjust == ri.offset_adjust;
}
size_t hash() const override {
return hash_all(opcode, lkey, offset_adjust);
}
void write(void *dest, RoseEngineBlob &blob,
const OffsetMap &offset_map) const override;
bool equiv_to(const RoseInstrSetLogical &ri, const OffsetMap &,
const OffsetMap &) const {
return lkey == ri.lkey && offset_adjust == ri.offset_adjust;
}
};
class RoseInstrSetCombination
: public RoseInstrBaseNoTargets<ROSE_INSTR_SET_COMBINATION,
ROSE_STRUCT_SET_COMBINATION,
RoseInstrSetCombination> {
public:
u32 ckey;
RoseInstrSetCombination(u32 ckey_in) : ckey(ckey_in) {}
bool operator==(const RoseInstrSetCombination &ri) const {
return ckey == ri.ckey;
}
size_t hash() const override {
return hash_all(opcode, ckey);
}
void write(void *dest, RoseEngineBlob &blob,
const OffsetMap &offset_map) const override;
bool equiv_to(const RoseInstrSetCombination &ri, const OffsetMap &,
const OffsetMap &) const {
return ckey == ri.ckey;
}
};
class RoseInstrFlushCombination
: public RoseInstrBaseTrivial<ROSE_INSTR_FLUSH_COMBINATION,
ROSE_STRUCT_FLUSH_COMBINATION,
RoseInstrFlushCombination> {
public:
~RoseInstrFlushCombination() override;
};
class RoseInstrSetExhaust
: public RoseInstrBaseNoTargets<ROSE_INSTR_SET_EXHAUST,
ROSE_STRUCT_SET_EXHAUST,
RoseInstrSetExhaust> {
public:
u32 ekey;
RoseInstrSetExhaust(u32 ekey_in) : ekey(ekey_in) {}
bool operator==(const RoseInstrSetExhaust &ri) const {
return ekey == ri.ekey;
}
size_t hash() const override {
return hash_all(opcode, ekey);
}
void write(void *dest, RoseEngineBlob &blob,
const OffsetMap &offset_map) const override;
bool equiv_to(const RoseInstrSetExhaust &ri, const OffsetMap &,
const OffsetMap &) const {
return ekey == ri.ekey;
}
};
class RoseInstrEnd
: public RoseInstrBaseTrivial<ROSE_INSTR_END, ROSE_STRUCT_END,
RoseInstrEnd> {

115
src/rose/rose_build_program.cpp
View File

@@ -313,6 +313,10 @@ void addMatcherEodProgram(RoseProgram &program) {
program.add_block(move(block));
}
void addFlushCombinationProgram(RoseProgram &program) {
program.add_before_end(make_unique<RoseInstrFlushCombination>());
}
static
void makeRoleCheckLeftfix(const RoseBuildImpl &build,
const map<RoseVertex, left_build_info> &leftfix_info,
@@ -496,6 +500,23 @@ void writeSomOperation(const Report &report, som_operation *op) {
}
}
static
void addLogicalSetRequired(const Report &report, ReportManager &rm,
RoseProgram &program) {
if (report.lkey == INVALID_LKEY) {
return;
}
// set matching status of current lkey
auto risl = make_unique<RoseInstrSetLogical>(report.lkey,
report.offsetAdjust);
program.add_before_end(move(risl));
// set current lkey's corresponding ckeys active, pending to check
for (auto ckey : rm.getRelateCKeys(report.lkey)) {
auto risc = make_unique<RoseInstrSetCombination>(ckey);
program.add_before_end(move(risc));
}
}
static
void makeReport(const RoseBuildImpl &build, const ReportID id,
const bool has_som, RoseProgram &program) {
@@ -542,38 +563,62 @@ void makeReport(const RoseBuildImpl &build, const ReportID id,
switch (report.type) {
case EXTERNAL_CALLBACK:
if (build.rm.numCkeys()) {
addFlushCombinationProgram(report_block);
}
if (!has_som) {
// Dedupe is only necessary if this report has a dkey, or if there
// are SOM reports to catch up.
bool needs_dedupe = build.rm.getDkey(report) != ~0U || build.hasSom;
if (report.ekey == INVALID_EKEY) {
if (needs_dedupe) {
report_block.add_before_end(
make_unique<RoseInstrDedupeAndReport>(
report.quashSom, build.rm.getDkey(report),
report.onmatch, report.offsetAdjust, end_inst));
if (!report.quiet) {
report_block.add_before_end(
make_unique<RoseInstrDedupeAndReport>(
report.quashSom, build.rm.getDkey(report),
report.onmatch, report.offsetAdjust, end_inst));
} else {
makeDedupe(build.rm, report, report_block);
}
} else {
report_block.add_before_end(make_unique<RoseInstrReport>(
report.onmatch, report.offsetAdjust));
if (!report.quiet) {
report_block.add_before_end(
make_unique<RoseInstrReport>(
report.onmatch, report.offsetAdjust));
}
}
} else {
if (needs_dedupe) {
makeDedupe(build.rm, report, report_block);
}
report_block.add_before_end(make_unique<RoseInstrReportExhaust>(
report.onmatch, report.offsetAdjust, report.ekey));
if (!report.quiet) {
report_block.add_before_end(
make_unique<RoseInstrReportExhaust>(
report.onmatch, report.offsetAdjust, report.ekey));
} else {
report_block.add_before_end(
make_unique<RoseInstrSetExhaust>(report.ekey));
}
}
} else { // has_som
makeDedupeSom(build.rm, report, report_block);
if (report.ekey == INVALID_EKEY) {
report_block.add_before_end(make_unique<RoseInstrReportSom>(
report.onmatch, report.offsetAdjust));
if (!report.quiet) {
report_block.add_before_end(make_unique<RoseInstrReportSom>(
report.onmatch, report.offsetAdjust));
}
} else {
report_block.add_before_end(
make_unique<RoseInstrReportSomExhaust>(
report.onmatch, report.offsetAdjust, report.ekey));
if (!report.quiet) {
report_block.add_before_end(
make_unique<RoseInstrReportSomExhaust>(
report.onmatch, report.offsetAdjust, report.ekey));
} else {
report_block.add_before_end(
make_unique<RoseInstrSetExhaust>(report.ekey));
}
}
}
addLogicalSetRequired(report, build.rm, report_block);
break;
case INTERNAL_SOM_LOC_SET:
case INTERNAL_SOM_LOC_SET_IF_UNSET:
@@ -586,6 +631,9 @@ void makeReport(const RoseBuildImpl &build, const ReportID id,
case INTERNAL_SOM_LOC_MAKE_WRITABLE:
case INTERNAL_SOM_LOC_SET_FROM:
case INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE:
if (build.rm.numCkeys()) {
addFlushCombinationProgram(report_block);
}
if (has_som) {
auto ri = make_unique<RoseInstrReportSomAware>();
writeSomOperation(report, &ri->som);
@@ -605,24 +653,48 @@ void makeReport(const RoseBuildImpl &build, const ReportID id,
case EXTERNAL_CALLBACK_SOM_STORED:
case EXTERNAL_CALLBACK_SOM_ABS:
case EXTERNAL_CALLBACK_SOM_REV_NFA:
if (build.rm.numCkeys()) {
addFlushCombinationProgram(report_block);
}
makeDedupeSom(build.rm, report, report_block);
if (report.ekey == INVALID_EKEY) {
report_block.add_before_end(make_unique<RoseInstrReportSom>(
report.onmatch, report.offsetAdjust));
if (!report.quiet) {
report_block.add_before_end(make_unique<RoseInstrReportSom>(
report.onmatch, report.offsetAdjust));
}
} else {
report_block.add_before_end(make_unique<RoseInstrReportSomExhaust>(
report.onmatch, report.offsetAdjust, report.ekey));
if (!report.quiet) {
report_block.add_before_end(
make_unique<RoseInstrReportSomExhaust>(
report.onmatch, report.offsetAdjust, report.ekey));
} else {
report_block.add_before_end(
make_unique<RoseInstrSetExhaust>(report.ekey));
}
}
addLogicalSetRequired(report, build.rm, report_block);
break;
case EXTERNAL_CALLBACK_SOM_PASS:
if (build.rm.numCkeys()) {
addFlushCombinationProgram(report_block);
}
makeDedupeSom(build.rm, report, report_block);
if (report.ekey == INVALID_EKEY) {
report_block.add_before_end(make_unique<RoseInstrReportSom>(
report.onmatch, report.offsetAdjust));
if (!report.quiet) {
report_block.add_before_end(make_unique<RoseInstrReportSom>(
report.onmatch, report.offsetAdjust));
}
} else {
report_block.add_before_end(make_unique<RoseInstrReportSomExhaust>(
report.onmatch, report.offsetAdjust, report.ekey));
if (!report.quiet) {
report_block.add_before_end(
make_unique<RoseInstrReportSomExhaust>(
report.onmatch, report.offsetAdjust, report.ekey));
} else {
report_block.add_before_end(
make_unique<RoseInstrSetExhaust>(report.ekey));
}
}
addLogicalSetRequired(report, build.rm, report_block);
break;
default:
@@ -630,7 +702,6 @@ void makeReport(const RoseBuildImpl &build, const ReportID id,
throw CompileError("Unable to generate bytecode.");
}
assert(!report_block.empty());
program.add_block(move(report_block));
}

3
src/rose/rose_build_program.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2017, Intel Corporation
* Copyright (c) 2016-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -187,6 +187,7 @@ struct ProgramBuild : noncopyable {
void addEnginesEodProgram(u32 eodNfaIterOffset, RoseProgram &program);
void addSuffixesEodProgram(RoseProgram &program);
void addMatcherEodProgram(RoseProgram &program);
void addFlushCombinationProgram(RoseProgram &program);
static constexpr u32 INVALID_QUEUE = ~0U;

26
src/rose/rose_internal.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -199,9 +199,25 @@ struct RoseStateOffsets {
* reports with that ekey should not be delivered to the user. */
u32 exhausted;
/** size of exhausted multibit */
/** size in bytes of exhausted multibit */
u32 exhausted_size;
/** Logical multibit.
*
* entry per logical key(operand/operator) (used by Logical Combination). */
u32 logicalVec;
/** size in bytes of logical multibit */
u32 logicalVec_size;
/** Combination multibit.
*
* entry per combination key (used by Logical Combination). */
u32 combVec;
/** size in bytes of combination multibit */
u32 combVec_size;
/** Multibit for active suffix/outfix engines. */
u32 activeLeafArray;
@@ -327,6 +343,11 @@ struct RoseEngine {
u32 mode; /**< scanning mode, one of HS_MODE_{BLOCK,STREAM,VECTORED} */
u32 historyRequired; /**< max amount of history required for streaming */
u32 ekeyCount; /**< number of exhaustion keys */
u32 lkeyCount; /**< number of logical keys */
u32 lopCount; /**< number of logical ops */
u32 ckeyCount; /**< number of combination keys */
u32 logicalTreeOffset; /**< offset to mapping from lkey to LogicalOp */
u32 combInfoMapOffset; /**< offset to mapping from ckey to combInfo */
u32 dkeyCount; /**< number of dedupe keys */
u32 dkeyLogSize; /**< size of fatbit for storing dkey log (bytes) */
u32 invDkeyOffset; /**< offset to table mapping from dkeys to the external
@@ -404,6 +425,7 @@ struct RoseEngine {
u32 roseCount;
u32 eodProgramOffset; //!< EOD program, otherwise 0.
u32 flushCombProgramOffset; /**< FlushCombination program, otherwise 0 */
u32 lastByteHistoryIterOffset; // if non-zero

42
src/rose/rose_program.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -183,7 +183,25 @@ enum RoseInstructionCode {
*/
ROSE_INSTR_INCLUDED_JUMP,
LAST_ROSE_INSTRUCTION = ROSE_INSTR_INCLUDED_JUMP //!< Sentinel.
/**
* \brief Set matching status of a sub-expression.
*/
ROSE_INSTR_SET_LOGICAL,
/**
* \brief Set combination status pending checking.
*/
ROSE_INSTR_SET_COMBINATION,
/**
* \brief Check if compliant with any logical constraints.
*/
ROSE_INSTR_FLUSH_COMBINATION,
/** \brief Mark as exhausted instead of report while quiet. */
ROSE_INSTR_SET_EXHAUST,
LAST_ROSE_INSTRUCTION = ROSE_INSTR_SET_EXHAUST //!< Sentinel.
};
struct ROSE_STRUCT_END {
@@ -636,4 +654,24 @@ struct ROSE_STRUCT_INCLUDED_JUMP {
u8 squash; //!< FDR confirm squash mask for included literal.
u32 child_offset; //!< Program offset of included literal.
};
struct ROSE_STRUCT_SET_LOGICAL {
u8 code; //!< From enum RoseInstructionCode.
u32 lkey; //!< Logical key to set.
s32 offset_adjust; //!< offsetAdjust from struct Report triggers the flush.
};
struct ROSE_STRUCT_SET_COMBINATION {
u8 code; //!< From enum RoseInstructionCode.
u32 ckey; //!< Combination key to set.
};
struct ROSE_STRUCT_FLUSH_COMBINATION {
u8 code; //!< From enum RoseInstructionCode.
};
struct ROSE_STRUCT_SET_EXHAUST {
u8 code; //!< From enum RoseInstructionCode.
u32 ekey; //!< Exhaustion key.
};
#endif // ROSE_ROSE_PROGRAM_H

11
src/rose/runtime.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -127,6 +127,15 @@ void updateLastMatchOffset(struct RoseContext *tctxt, u64a offset) {
tctxt->lastMatchOffset = offset;
}
static really_inline
void updateLastCombMatchOffset(struct RoseContext *tctxt, u64a offset) {
DEBUG_PRINTF("match @%llu, last match @%llu\n", offset,
tctxt->lastCombMatchOffset);
assert(offset >= tctxt->lastCombMatchOffset);
tctxt->lastCombMatchOffset = offset;
}
static really_inline
void updateMinMatchOffset(struct RoseContext *tctxt, u64a offset) {
DEBUG_PRINTF("min match now @%llu, was @%llu\n", offset,

4
src/rose/stream.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -578,6 +578,7 @@ void roseStreamExec(const struct RoseEngine *t, struct hs_scratch *scratch) {
tctxt->lastEndOffset = offset;
tctxt->filledDelayedSlots = 0;
tctxt->lastMatchOffset = 0;
tctxt->lastCombMatchOffset = offset;
tctxt->minMatchOffset = offset;
tctxt->minNonMpvMatchOffset = offset;
tctxt->next_mpv_offset = 0;
@@ -700,6 +701,7 @@ void roseStreamInitEod(const struct RoseEngine *t, u64a offset,
tctxt->lastEndOffset = offset;
tctxt->filledDelayedSlots = 0;
tctxt->lastMatchOffset = 0;
tctxt->lastCombMatchOffset = offset; /* DO NOT set 0 here! */
tctxt->minMatchOffset = offset;
tctxt->minNonMpvMatchOffset = offset;
tctxt->next_mpv_offset = offset;

48
src/runtime.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -356,6 +356,15 @@ hs_error_t HS_CDECL hs_scan(const hs_database_t *db, const char *data,
length, NULL, 0, 0, 0, flags);
clearEvec(rose, scratch->core_info.exhaustionVector);
if (rose->ckeyCount) {
scratch->core_info.logicalVector = scratch->bstate +
rose->stateOffsets.logicalVec;
scratch->core_info.combVector = scratch->bstate +
rose->stateOffsets.combVec;
scratch->tctxt.lastCombMatchOffset = 0;
clearLvec(rose, scratch->core_info.logicalVector,
scratch->core_info.combVector);
}
if (!length) {
if (rose->boundary.reportZeroEodOffset) {
@@ -436,6 +445,13 @@ done_scan:
scratch);
}
if (rose->flushCombProgramOffset) {
if (roseRunFlushCombProgram(rose, scratch, ~0ULL) == MO_HALT_MATCHING) {
unmarkScratchInUse(scratch);
return HS_SCAN_TERMINATED;
}
}
set_retval:
DEBUG_PRINTF("done. told_to_stop_matching=%d\n",
told_to_stop_matching(scratch));
@@ -500,6 +516,10 @@ void init_stream(struct hs_stream *s, const struct RoseEngine *rose,
roseInitState(rose, state);
clearEvec(rose, state + rose->stateOffsets.exhausted);
if (rose->ckeyCount) {
clearLvec(rose, state + rose->stateOffsets.logicalVec,
state + rose->stateOffsets.combVec);
}
// SOM state multibit structures.
initSomState(rose, state);
@@ -614,6 +634,13 @@ void report_eod_matches(hs_stream_t *id, hs_scratch_t *scratch,
getHistory(state, rose, id->offset),
getHistoryAmount(rose, id->offset), id->offset, status, 0);
if (rose->ckeyCount) {
scratch->core_info.logicalVector = state +
rose->stateOffsets.logicalVec;
scratch->core_info.combVector = state + rose->stateOffsets.combVec;
scratch->tctxt.lastCombMatchOffset = id->offset;
}
if (rose->somLocationCount) {
loadSomFromStream(scratch, id->offset);
}
@@ -657,6 +684,13 @@ void report_eod_matches(hs_stream_t *id, hs_scratch_t *scratch,
scratch->core_info.status |= STATUS_TERMINATED;
}
}
if (rose->flushCombProgramOffset && !told_to_stop_matching(scratch)) {
if (roseRunFlushCombProgram(rose, scratch, ~0ULL) == MO_HALT_MATCHING) {
DEBUG_PRINTF("told to stop matching\n");
scratch->core_info.status |= STATUS_TERMINATED;
}
}
}
HS_PUBLIC_API
@@ -849,6 +883,12 @@ hs_error_t hs_scan_stream_internal(hs_stream_t *id, const char *data,
populateCoreInfo(scratch, rose, state, onEvent, context, data, length,
getHistory(state, rose, id->offset), historyAmount,
id->offset, status, flags);
if (rose->ckeyCount) {
scratch->core_info.logicalVector = state +
rose->stateOffsets.logicalVec;
scratch->core_info.combVector = state + rose->stateOffsets.combVec;
scratch->tctxt.lastCombMatchOffset = id->offset;
}
assert(scratch->core_info.hlen <= id->offset
&& scratch->core_info.hlen <= rose->historyRequired);
@@ -894,6 +934,12 @@ hs_error_t hs_scan_stream_internal(hs_stream_t *id, const char *data,
}
}
if (rose->flushCombProgramOffset && !told_to_stop_matching(scratch)) {
if (roseRunFlushCombProgram(rose, scratch, ~0ULL) == MO_HALT_MATCHING) {
scratch->core_info.status |= STATUS_TERMINATED;
}
}
setStreamStatus(state, scratch->core_info.status);
if (likely(!can_stop_matching(scratch))) {

5
src/scratch.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -94,6 +94,8 @@ struct core_info {
const struct RoseEngine *rose;
char *state; /**< full stream state */
char *exhaustionVector; /**< pointer to evec for this stream */
char *logicalVector; /**< pointer to lvec for this stream */
char *combVector; /**< pointer to cvec for this stream */
const u8 *buf; /**< main scan buffer */
size_t len; /**< length of main scan buffer in bytes */
const u8 *hbuf; /**< history buffer */
@@ -115,6 +117,7 @@ struct RoseContext {
* stream */
u64a lastMatchOffset; /**< last match offset report up out of rose;
* used _only_ for debugging, asserts */
u64a lastCombMatchOffset; /**< last match offset of active combinations */
u64a minMatchOffset; /**< the earliest offset that we are still allowed to
* report */
u64a minNonMpvMatchOffset; /**< the earliest offset that non-mpv engines are

9
src/stream_compress_impl.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, Intel Corporation
* Copyright (c) 2017-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -148,6 +148,13 @@ size_t JOIN(sc_, FN_SUFFIX)(const struct RoseEngine *rose,
/* copy the exhaustion multibit */
COPY_MULTIBIT(stream_body + so->exhausted, rose->ekeyCount);
/* copy the logical multibit */
COPY_MULTIBIT(stream_body + so->logicalVec,
rose->lkeyCount + rose->lopCount);
/* copy the combination multibit */
COPY_MULTIBIT(stream_body + so->combVec, rose->ckeyCount);
/* copy nfa stream state for endfixes */
/* Note: in the expand case the active array has already been copied into
* the stream. */

77
src/util/logical.h Normal file
View File

@@ -0,0 +1,77 @@
/*
* Copyright (c) 2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* \brief Inline functions for manipulating logical combinations.
*/
#ifndef LOGICAL_H
#define LOGICAL_H
#include "ue2common.h"
/** Index meaning a given logical key is invalid. */
#define INVALID_LKEY (~(u32)0)
#define INVALID_CKEY INVALID_LKEY
/** Logical operation type, the priority is from high to low. */
enum LogicalOpType {
LOGICAL_OP_NOT,
LOGICAL_OP_AND,
LOGICAL_OP_OR,
LAST_LOGICAL_OP = LOGICAL_OP_OR //!< Sentinel.
};
#define UNKNOWN_OP (~(u32)0)
/** Logical Operation is consist of 4 parts. */
struct LogicalOp {
u32 id; //!< logical operator/operation id
u32 op; //!< LogicalOpType
u32 lo; //!< left operand
u32 ro; //!< right operand
};
/** Each logical combination has its info:
* It occupies a region in LogicalOp vector.
* It has an exhaustion key for single-match mode. */
struct CombInfo {
u32 id;
u32 ekey; //!< exhaustion key
u32 start; //!< ckey of logical operation to start calculating
u32 result; //!< ckey of logical operation to give final result
u64a min_offset;
u64a max_offset;
};
/** Temporarily use to seperate operations' id from reports' lkey
* when building logicalTree in shunting yard algorithm,
* operations' id will be finally renumbered following reports' lkey. */
#define LOGICAL_OP_BIT 0x80000000UL
#endif

18
src/util/report.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -36,6 +36,7 @@
#include "ue2common.h"
#include "util/exhaust.h" // for INVALID_EKEY
#include "util/logical.h" // for INVALID_LKEY
#include "util/hash.h"
#include "util/order_check.h"
@@ -107,6 +108,16 @@ struct Report {
* exhaustible, this will be INVALID_EKEY. */
u32 ekey = INVALID_EKEY;
/** \brief Logical Combination key in each combination.
*
* If in Logical Combination, the lkey to check before reporting a match.
* Additionally before checking the lkey will be set. If not
* in Logical Combination, this will be INVALID_LKEY. */
u32 lkey = INVALID_LKEY;
/** \brief Quiet flag for expressions in any logical combination. */
bool quiet = false;
/** \brief Adjustment to add to the match offset when we report a match.
*
* This is usually used for reports attached to states that form part of a
@@ -207,16 +218,17 @@ bool operator==(const Report &a, const Report &b) {
}
static inline
Report makeECallback(u32 report, s32 offsetAdjust, u32 ekey) {
Report makeECallback(u32 report, s32 offsetAdjust, u32 ekey, bool quiet) {
Report ir(EXTERNAL_CALLBACK, report);
ir.offsetAdjust = offsetAdjust;
ir.ekey = ekey;
ir.quiet = (u8)quiet;
return ir;
}
static inline
Report makeCallback(u32 report, s32 offsetAdjust) {
return makeECallback(report, offsetAdjust, INVALID_EKEY);
return makeECallback(report, offsetAdjust, INVALID_EKEY, false);
}
static inline

41
src/util/report_manager.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -95,6 +95,31 @@ u32 ReportManager::getExhaustibleKey(u32 a) {
return it->second;
}
const set<u32> &ReportManager::getRelateCKeys(u32 lkey) {
auto it = pl.lkey2ckeys.find(lkey);
assert(it != pl.lkey2ckeys.end());
return it->second;
}
void ReportManager::logicalKeyRenumber() {
pl.logicalKeyRenumber();
// assign to corresponding report
for (u32 i = 0; i < reportIds.size(); i++) {
Report &ir = reportIds[i];
if (contains(pl.toLogicalKeyMap, ir.onmatch)) {
ir.lkey = pl.toLogicalKeyMap.at(ir.onmatch);
}
}
}
const vector<LogicalOp> &ReportManager::getLogicalTree() const {
return pl.logicalTree;
}
const vector<CombInfo> &ReportManager::getCombInfoMap() const {
return pl.combInfoMap;
}
u32 ReportManager::getUnassociatedExhaustibleKey(void) {
u32 rv = toExhaustibleKeyMap.size();
bool inserted;
@@ -115,6 +140,18 @@ u32 ReportManager::numEkeys() const {
return (u32) toExhaustibleKeyMap.size();
}
u32 ReportManager::numLogicalKeys() const {
return (u32) pl.toLogicalKeyMap.size();
}
u32 ReportManager::numLogicalOps() const {
return (u32) pl.logicalTree.size();
}
u32 ReportManager::numCkeys() const {
return (u32) pl.toCombKeyMap.size();
}
bool ReportManager::patternSetCanExhaust() const {
return global_exhaust && !toExhaustibleKeyMap.empty();
}
@@ -219,7 +256,7 @@ Report ReportManager::getBasicInternalReport(const ExpressionInfo &expr,
ekey = getExhaustibleKey(expr.report);
}
return makeECallback(expr.report, adj, ekey);
return makeECallback(expr.report, adj, ekey, expr.quiet);
}
void ReportManager::setProgramOffset(ReportID id, u32 programOffset) {

28
src/util/report_manager.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2017, Intel Corporation
* Copyright (c) 2015-2018, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -38,6 +38,7 @@
#include "util/compile_error.h"
#include "util/noncopyable.h"
#include "util/report.h"
#include "parser/logical_combination.h"
#include <map>
#include <set>
@@ -80,6 +81,15 @@ public:
/** \brief Total number of exhaustion keys. */
u32 numEkeys() const;
/** \brief Total number of logical keys. */
u32 numLogicalKeys() const;
/** \brief Total number of logical operators. */
u32 numLogicalOps() const;
/** \brief Total number of combination keys. */
u32 numCkeys() const;
/** \brief True if the pattern set can exhaust (i.e. all patterns are
* highlander). */
bool patternSetCanExhaust() const;
@@ -110,6 +120,19 @@ public:
* assigning one if necessary. */
u32 getExhaustibleKey(u32 expressionIndex);
/** \brief Get lkey's corresponding ckeys. */
const std::set<u32> &getRelateCKeys(u32 lkey);
/** \brief Renumber lkey for logical operations, after parsed
* all logical expressions. */
void logicalKeyRenumber();
/** \brief Used in Rose for writing bytecode. */
const std::vector<LogicalOp> &getLogicalTree() const;
/** \brief Used in Rose for writing bytecode. */
const std::vector<CombInfo> &getCombInfoMap() const;
/** \brief Fetch the dedupe key associated with the given report. Returns
* ~0U if no dkey is needed. */
u32 getDkey(const Report &r) const;
@@ -122,6 +145,9 @@ public:
* set. */
u32 getProgramOffset(ReportID id) const;
/** \brief Parsed logical combination structure. */
ParsedLogical pl;
private:
/** \brief Grey box ref, for checking resource limits. */
const Grey &grey;