github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-09-29 19:24:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

fdr: move long literal handling into Rose

Browse Source 
Move the hash table used for long literal support in streaming mode from
FDR to Rose, and introduce new instructions CHECK_LONG_LIT and
CHECK_LONG_LIT_NOCASE for doing literal confirm for long literals.

This simplifies FDR confirm, and guarantees that HWLM matchers will only
be used for literals < 256 bytes long.
This commit is contained in:
Justin Viiret
2016-09-07 15:59:23 +10:00
committed by Matthew Barr
parent 6ed30194ce
commit 68bf473e2e
40 changed files with 1759 additions and 1310 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/fdr/fdr.c
View File

@@ -31,7 +31,6 @@
#include "fdr_confirm_runtime.h"
#include "fdr_internal.h"
#include "fdr_loadval.h"
#include "fdr_streaming_runtime.h"
#include "flood_runtime.h"
#include "teddy.h"
#include "teddy_internal.h"
@@ -809,8 +808,6 @@ hwlm_error_t fdrExec(const struct FDR *fdr, const u8 *buf, size_t len,
len,
hbuf,
0,
hbuf, // nocase
0,
start,
cb,
ctxt,
@@ -828,14 +825,12 @@ hwlm_error_t fdrExec(const struct FDR *fdr, const u8 *buf, size_t len,
hwlm_error_t fdrExecStreaming(const struct FDR *fdr, const u8 *hbuf,
size_t hlen, const u8 *buf, size_t len,
size_t start, HWLMCallback cb, void *ctxt,
hwlm_group_t groups, u8 *stream_state) {
hwlm_group_t groups) {
struct FDR_Runtime_Args a = {
buf,
len,
hbuf,
hlen,
hbuf, // nocase - start same as caseful, override later if needed
hlen, // nocase
start,
cb,
ctxt,
@@ -844,7 +839,6 @@ hwlm_error_t fdrExecStreaming(const struct FDR *fdr, const u8 *hbuf,
* the history buffer (they may be garbage). */
hbuf ? unaligned_load_u64a(hbuf + hlen - sizeof(u64a)) : (u64a)0
};
fdrUnpackState(fdr, &a, stream_state);
hwlm_error_t ret;
if (unlikely(a.start_offset >= a.len)) {
@@ -854,6 +848,5 @@ hwlm_error_t fdrExecStreaming(const struct FDR *fdr, const u8 *hbuf,
ret = funcs[fdr->engineID](fdr, &a, groups);
}
fdrPackState(fdr, &a, stream_state);
return ret;
}

7
src/fdr/fdr.h
View File

@@ -43,10 +43,6 @@ extern "C" {
struct FDR;
/** \brief Returns non-zero if the contents of the stream state indicate that
* there is active FDR history beyond the regularly used history. */
u32 fdrStreamStateActive(const struct FDR *fdr, const u8 *stream_state);
/**
* \brief Block-mode scan.
*
@@ -74,12 +70,11 @@ hwlm_error_t fdrExec(const struct FDR *fdr, const u8 *buf, size_t len,
* \param cb Callback to call when a match is found.
* \param ctxt Caller-provided context pointer supplied to callback on match.
* \param groups Initial groups mask.
* \param stream_state Persistent stream state for use by FDR.
*/
hwlm_error_t fdrExecStreaming(const struct FDR *fdr, const u8 *hbuf,
size_t hlen, const u8 *buf, size_t len,
size_t start, HWLMCallback cb, void *ctxt,
hwlm_group_t groups, u8 *stream_state);
hwlm_group_t groups);
#ifdef __cplusplus
}

23
src/fdr/fdr_compile.cpp
View File

@@ -39,6 +39,7 @@
#include "teddy_engine_description.h"
#include "grey.h"
#include "ue2common.h"
#include "hwlm/hwlm_build.h"
#include "util/alloc.h"
#include "util/compare.h"
#include "util/dump_mask.h"
@@ -495,14 +496,34 @@ FDRCompiler::build(pair<aligned_unique_ptr<u8>, size_t> &link) {
} // namespace
static
size_t maxMaskLen(const vector<hwlmLiteral> &lits) {
size_t rv = 0;
for (const auto &lit : lits) {
rv = max(rv, lit.msk.size());
}
return rv;
}
static
void setHistoryRequired(hwlmStreamingControl &stream_ctl,
const vector<hwlmLiteral> &lits) {
size_t max_mask_len = maxMaskLen(lits);
// we want enough history to manage the longest literal and the longest
// mask.
stream_ctl.literal_history_required = max(maxLen(lits), max_mask_len) - 1;
}
static
aligned_unique_ptr<FDR>
fdrBuildTableInternal(const vector<hwlmLiteral> &lits, bool make_small,
const target_t &target, const Grey &grey, u32 hint,
hwlmStreamingControl *stream_control) {
pair<aligned_unique_ptr<u8>, size_t> link(nullptr, 0);
if (stream_control) {
link = fdrBuildTableStreaming(lits, *stream_control);
setHistoryRequired(*stream_control, lits);
}
DEBUG_PRINTF("cpu has %s\n", target.has_avx2() ? "avx2" : "no-avx2");

2
src/fdr/fdr_confirm_compile.cpp
View File

@@ -339,7 +339,7 @@ getFDRConfirm(const vector<hwlmLiteral> &lits, bool applyOneCharOpt,
ptr = ROUNDUP_PTR(ptr, alignof(LitInfo));
if (next(i) == e) {
finalLI.next = 0x0;
finalLI.next = 0;
} else {
// our next field represents an adjustment on top of
// current address + the actual size of the literal

9
src/fdr/fdr_confirm_runtime.h
View File

@@ -74,10 +74,8 @@ void confWithBit(const struct FDRConfirm *fdrc, const struct FDR_Runtime_Args *a
if (loc < buf) {
u32 full_overhang = buf - loc;
const u8 *history = caseless ? a->buf_history_nocase
: a->buf_history;
size_t len_history = caseless ? a->len_history_nocase
: a->len_history;
const u8 *history = a->buf_history;
size_t len_history = a->len_history;
// can't do a vectored confirm either if we don't have
// the bytes
@@ -123,8 +121,7 @@ void confWithBit(const struct FDRConfirm *fdrc, const struct FDR_Runtime_Args *a
const u8 *loc2 = buf + i - li->extended_size + 1 - pullBackAmount;
if (loc2 < buf) {
u32 full_overhang = buf - loc2;
size_t len_history = caseless ? a->len_history_nocase
: a->len_history;
size_t len_history = a->len_history;
if (full_overhang > len_history) {
goto out;
}

2
src/fdr/fdr_internal.h
View File

@@ -100,8 +100,6 @@ struct FDR_Runtime_Args {
size_t len;
const u8 *buf_history;
size_t len_history;
const u8 *buf_history_nocase;
size_t len_history_nocase;
size_t start_offset;
HWLMCallback cb;
void *ctxt;

425
src/fdr/fdr_streaming_compile.cpp
View File

@@ -1,425 +0,0 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "fdr_internal.h"
#include "fdr_streaming_internal.h"
#include "fdr_compile_internal.h"
#include "hwlm/hwlm_build.h"
#include "util/alloc.h"
#include "util/bitutils.h"
#include "util/target_info.h"
#include "util/verify_types.h"
#include <algorithm>
#include <cassert>
#include <cstdio>
#include <cstring>
#include <deque>
#include <set>
#include <boost/dynamic_bitset.hpp>
using namespace std;
using boost::dynamic_bitset;
namespace ue2 {
namespace {
struct LongLitOrder {
bool operator()(const hwlmLiteral &i1, const hwlmLiteral &i2) const {
if (i1.nocase != i2.nocase) {
return i1.nocase < i2.nocase;
} else {
return i1.s < i2.s;
}
}
};
}
static
bool hwlmLitEqual(const hwlmLiteral &l1, const hwlmLiteral &l2) {
return l1.s == l2.s && l1.nocase == l2.nocase;
}
static
u32 roundUpToPowerOfTwo(u32 x) {
x -= 1;
x |= (x >> 1);
x |= (x >> 2);
x |= (x >> 4);
x |= (x >> 8);
x |= (x >> 16);
return x + 1;
}
/**
* \brief Creates a long literals vector containing all literals of length > max_len.
*
* The last char of each literal is trimmed as we're not interested in full
* matches, only partial matches.
*
* Literals are sorted (by caseful/caseless, then lexicographical order) and
* made unique.
*
* The ID of each literal is set to its position in the vector.
*
* \return False if there aren't any long literals.
*/
static
bool setupLongLits(const vector<hwlmLiteral> &lits,
vector<hwlmLiteral> &long_lits, size_t max_len) {
long_lits.reserve(lits.size());
for (const auto &lit : lits) {
if (lit.s.length() > max_len) {
hwlmLiteral tmp = lit; // copy
tmp.s.pop_back();
tmp.id = 0; // recalc later
tmp.groups = 0; // filled in later by hash bucket(s)
long_lits.push_back(move(tmp));
}
}
if (long_lits.empty()) {
return false;
}
// sort long_literals by caseful/caseless and in lexicographical order,
// remove duplicates
stable_sort(long_lits.begin(), long_lits.end(), LongLitOrder());
auto new_end = unique(long_lits.begin(), long_lits.end(), hwlmLitEqual);
long_lits.erase(new_end, long_lits.end());
// fill in ids; not currently used
for (auto i = long_lits.begin(), e = long_lits.end(); i != e; ++i) {
i->id = distance(long_lits.begin(), i);
}
return true;
}
// boundaries are the 'start' boundaries for each 'mode'
// so boundary[CASEFUL] is the index one above the largest caseful index
// positions[CASEFUL] is the # of positions in caseful strings (stream)
// hashedPositions[CASEFUL] is the # of positions in caseful strings
// (not returned - a temporary)
// hashEntries[CASEFUL] is the # of positions hashed for caseful strings
// (rounded up to the nearest power of two)
static
void analyzeLits(const vector<hwlmLiteral> &long_lits, size_t max_len,
u32 *boundaries, u32 *positions, u32 *hashEntries) {
u32 hashedPositions[MAX_MODES];
for (u32 m = CASEFUL; m < MAX_MODES; ++m) {
boundaries[m] = verify_u32(long_lits.size());
positions[m] = 0;
hashedPositions[m] = 0;
}
for (auto i = long_lits.begin(), e = long_lits.end(); i != e; ++i) {
if (i->nocase) {
boundaries[CASEFUL] = verify_u32(distance(long_lits.begin(), i));
break;
}
}
for (const auto &lit : long_lits) {
Modes m = lit.nocase ? CASELESS : CASEFUL;
for (u32 j = 1; j < lit.s.size() - max_len + 1; j++) {
hashedPositions[m]++;
}
positions[m] += lit.s.size();
}
for (u32 m = CASEFUL; m < MAX_MODES; m++) {
hashEntries[m] = hashedPositions[m]
? roundUpToPowerOfTwo(MAX(4096, hashedPositions[m]))
: 0;
}
#ifdef DEBUG_COMPILE
printf("analyzeLits:\n");
for (Modes m = CASEFUL; m < MAX_MODES; m++) {
printf("mode %s boundary %d positions %d hashedPositions %d "
"hashEntries %d\n",
(m == CASEFUL) ? "caseful" : "caseless", boundaries[m],
positions[m], hashedPositions[m], hashEntries[m]);
}
printf("\n");
#endif
}
static
u32 hashLit(const hwlmLiteral &l, u32 offset, size_t max_len, Modes m) {
return streaming_hash((const u8 *)l.s.c_str() + offset, max_len, m);
}
// sort by 'distance from start'
namespace {
struct OffsetIDFromEndOrder {
const vector<hwlmLiteral> &lits; // not currently used
explicit OffsetIDFromEndOrder(const vector<hwlmLiteral> &lits_in)
: lits(lits_in) {}
bool operator()(const pair<u32, u32> &i1, const pair<u32, u32> &i2) const {
if (i1.second != i2.second) {
// longest is 'first', so > not <
return i1.second > i2.second;
}
return i1.first < i2.first;
}
};
}
static
void fillHashes(const vector<hwlmLiteral> &long_lits, size_t max_len,
FDRSHashEntry *tab, size_t numEntries, Modes mode,
map<u32, u32> &litToOffsetVal) {
const u32 nbits = lg2(numEntries);
map<u32, deque<pair<u32, u32> > > bucketToLitOffPairs;
map<u32, u64a> bucketToBitfield;
for (const auto &lit : long_lits) {
if ((mode == CASELESS) != lit.nocase) {
continue;
}
for (u32 j = 1; j < lit.s.size() - max_len + 1; j++) {
u32 h = hashLit(lit, j, max_len, mode);
u32 h_ent = h & ((1U << nbits) - 1);
u32 h_low = (h >> nbits) & 63;
bucketToLitOffPairs[h_ent].emplace_back(lit.id, j);
bucketToBitfield[h_ent] |= (1ULL << h_low);
}
}
// this used to be a set<u32>, but a bitset is much much faster given that
// we're using it only for membership testing.
dynamic_bitset<> filledBuckets(numEntries); // all bits zero by default.
// sweep out bitfield entries and save the results swapped accordingly
// also, anything with bitfield entries is put in filledBuckets
for (const auto &m : bucketToBitfield) {
const u32 &bucket = m.first;
const u64a &contents = m.second;
tab[bucket].bitfield = contents;
filledBuckets.set(bucket);
}
// store out all our chains based on free values in our hash table.
// find nearest free locations that are empty (there will always be more
// entries than strings, at present)
for (auto &m : bucketToLitOffPairs) {
u32 bucket = m.first;
deque<pair<u32, u32>> &d = m.second;
// sort d by distance of the residual string (len minus our depth into
// the string). We need to put the 'furthest back' string first...
stable_sort(d.begin(), d.end(), OffsetIDFromEndOrder(long_lits));
while (1) {
// first time through is always at bucket, then we fill in links
filledBuckets.set(bucket);
FDRSHashEntry *ent = &tab[bucket];
u32 lit_id = d.front().first;
u32 offset = d.front().second;
ent->state = verify_u32(litToOffsetVal[lit_id] + offset + max_len);
ent->link = (u32)LINK_INVALID;
d.pop_front();
if (d.empty()) {
break;
}
// now, if there is another value
// find a bucket for it and put in 'bucket' and repeat
// all we really need to do is find something not in filledBuckets,
// ideally something close to bucket
// we search backward and forward from bucket, trying to stay as
// close as possible.
UNUSED bool found = false;
int bucket_candidate = 0;
for (u32 k = 1; k < numEntries * 2; k++) {
bucket_candidate = bucket + (((k & 1) == 0)
? (-(int)k / 2) : (k / 2));
if (bucket_candidate < 0 ||
(size_t)bucket_candidate >= numEntries) {
continue;
}
if (!filledBuckets.test(bucket_candidate)) {
found = true;
break;
}
}
assert(found);
bucket = bucket_candidate;
ent->link = bucket;
}
}
}
static
size_t maxMaskLen(const vector<hwlmLiteral> &lits) {
size_t rv = 0;
for (const auto &lit : lits) {
rv = max(rv, lit.msk.size());
}
return rv;
}
pair<aligned_unique_ptr<u8>, size_t>
fdrBuildTableStreaming(const vector<hwlmLiteral> &lits,
hwlmStreamingControl &stream_control) {
// refuse to compile if we are forced to have smaller than minimum
// history required for long-literal support, full stop
// otherwise, choose the maximum of the preferred history quantity
// (currently a fairly extravagant 32) or the already used history
// quantity - subject to the limitation of stream_control.history_max
const size_t MIN_HISTORY_REQUIRED = 32;
if (MIN_HISTORY_REQUIRED > stream_control.history_max) {
throw std::logic_error("Cannot set history to minimum history required");
}
size_t max_len =
MIN(stream_control.history_max,
MAX(MIN_HISTORY_REQUIRED, stream_control.history_min));
assert(max_len >= MIN_HISTORY_REQUIRED);
size_t max_mask_len = maxMaskLen(lits);
vector<hwlmLiteral> long_lits;
if (!setupLongLits(lits, long_lits, max_len) || false) {
// "Don't need to do anything" path, not really a fail
DEBUG_PRINTF("Streaming literal path produces no table\n");
// we want enough history to manage the longest literal and the longest
// mask.
stream_control.literal_history_required =
max(maxLen(lits), max_mask_len) - 1;
stream_control.literal_stream_state_required = 0;
return {nullptr, size_t{0}};
}
// Ensure that we have enough room for the longest mask.
if (max_mask_len) {
max_len = max(max_len, max_mask_len - 1);
}
u32 boundary[MAX_MODES];
u32 positions[MAX_MODES];
u32 hashEntries[MAX_MODES];
analyzeLits(long_lits, max_len, boundary, positions, hashEntries);
// first assess the size and find our caseless threshold
size_t headerSize = ROUNDUP_16(sizeof(FDRSTableHeader));
size_t litTabOffset = headerSize;
size_t litTabNumEntries = long_lits.size() + 1;
size_t litTabSize = ROUNDUP_16(litTabNumEntries * sizeof(FDRSLiteral));
size_t wholeLitTabOffset = litTabOffset + litTabSize;
size_t totalWholeLitTabSize = ROUNDUP_16(positions[CASEFUL] +
positions[CASELESS]);
size_t htOffset[MAX_MODES];
size_t htSize[MAX_MODES];
htOffset[CASEFUL] = wholeLitTabOffset + totalWholeLitTabSize;
htSize[CASEFUL] = hashEntries[CASEFUL] * sizeof(FDRSHashEntry);
htOffset[CASELESS] = htOffset[CASEFUL] + htSize[CASEFUL];
htSize[CASELESS] = hashEntries[CASELESS] * sizeof(FDRSHashEntry);
size_t tabSize = ROUNDUP_16(htOffset[CASELESS] + htSize[CASELESS]);
// need to add +2 to both of these to allow space for the actual largest
// value as well as handling the fact that we add one to the space when
// storing out a position to allow zero to mean "no stream state value"
u8 streamBits[MAX_MODES];
streamBits[CASEFUL] = lg2(roundUpToPowerOfTwo(positions[CASEFUL] + 2));
streamBits[CASELESS] = lg2(roundUpToPowerOfTwo(positions[CASELESS] + 2));
u32 tot_state_bytes = (streamBits[CASEFUL] + streamBits[CASELESS] + 7) / 8;
auto secondaryTable = aligned_zmalloc_unique<u8>(tabSize);
assert(secondaryTable); // otherwise would have thrown std::bad_alloc
// then fill it in
u8 * ptr = secondaryTable.get();
FDRSTableHeader * header = (FDRSTableHeader *)ptr;
// fill in header
header->pseudoEngineID = (u32)0xffffffff;
header->N = verify_u8(max_len); // u8 so doesn't matter; won't go > 255
for (u32 m = CASEFUL; m < MAX_MODES; ++m) {
header->boundary[m] = boundary[m];
header->hashOffset[m] = verify_u32(htOffset[m]);
header->hashNBits[m] = lg2(hashEntries[m]);
header->streamStateBits[m] = streamBits[m];
}
assert(tot_state_bytes < sizeof(u64a));
header->streamStateBytes = verify_u8(tot_state_bytes); // u8
ptr += headerSize;
// now fill in the rest
FDRSLiteral * litTabPtr = (FDRSLiteral *)ptr;
ptr += litTabSize;
map<u32, u32> litToOffsetVal;
for (auto i = long_lits.begin(), e = long_lits.end(); i != e; ++i) {
u32 entry = verify_u32(i - long_lits.begin());
u32 offset = verify_u32(ptr - secondaryTable.get());
// point the table entry to the string location
litTabPtr[entry].offset = offset;
litToOffsetVal[entry] = offset;
// copy the string into the string location
memcpy(ptr, i->s.c_str(), i->s.size());
ptr += i->s.size(); // and the string location
}
// fill in final lit table entry with current ptr (serves as end value)
litTabPtr[long_lits.size()].offset = verify_u32(ptr - secondaryTable.get());
// fill hash tables
ptr = secondaryTable.get() + htOffset[CASEFUL];
for (u32 m = CASEFUL; m < MAX_MODES; ++m) {
fillHashes(long_lits, max_len, (FDRSHashEntry *)ptr, hashEntries[m],
(Modes)m, litToOffsetVal);
ptr += htSize[m];
}
// tell the world what we did
stream_control.literal_history_required = max_len;
stream_control.literal_stream_state_required = tot_state_bytes;
return {move(secondaryTable), tabSize};
}
} // namespace ue2

152
src/fdr/fdr_streaming_internal.h
View File

@@ -1,152 +0,0 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef FDR_STREAMING_INTERNAL_H
#define FDR_STREAMING_INTERNAL_H
#include "ue2common.h"
#include "fdr_internal.h"
#include "util/unaligned.h"
// tertiary table:
// a header (FDRSTableHeader)
// long_lits.size()+1 entries holding an offset to the string in the
// 'whole literal table' (FDRSLiteral structure)
// the whole literal table - every string packed in (freeform)
// hash table (caseful) (FDRSHashEntry)
// hash table (caseless) (FDRSHashEntry)
enum Modes {
CASEFUL = 0,
CASELESS = 1,
MAX_MODES = 2
};
// We have one of these structures hanging off the 'link' of our secondary
// FDR table that handles streaming strings
struct FDRSTableHeader {
u32 pseudoEngineID; // set to 0xffffffff to indicate this isn't an FDR
// string id one beyond the maximum entry for this type of literal
// boundary[CASEFUL] is the end of the caseful literals
// boundary[CASELESS] is the end of the caseless literals and one beyond
// the largest literal id (the size of the littab)
u32 boundary[MAX_MODES];
// offsets are 0 if no such table exists
// offset from the base of the tertiary structure to the hash table
u32 hashOffset[MAX_MODES];
u32 hashNBits[MAX_MODES]; // lg2 of the size of the hash table
u8 streamStateBits[MAX_MODES];
u8 streamStateBytes; // total size of packed stream state in bytes
u8 N; // prefix lengths
u16 pad;
};
// One of these structures per literal entry in our secondary FDR table.
struct FDRSLiteral {
u32 offset;
// potentially - another u32 to point to the 'next lesser included literal'
// which would be a literal that overlaps this one in such a way that a
// failure to match _this_ literal can leave us in a state that we might
// still match that literal. Offset information might also be called for,
// in which case we might be wanting to use a FDRSLiteralOffset
};
typedef u32 FDRSLiteralOffset;
#define LINK_INVALID 0xffffffff
// One of these structures per hash table entry in our secondary FDR table
struct FDRSHashEntry {
u64a bitfield;
FDRSLiteralOffset state;
u32 link;
};
static really_inline
u32 get_start_lit_idx(const struct FDRSTableHeader * h, enum Modes m) {
return m == CASEFUL ? 0 : h->boundary[m-1];
}
static really_inline
u32 get_end_lit_idx(const struct FDRSTableHeader * h, enum Modes m) {
return h->boundary[m];
}
static really_inline
const struct FDRSLiteral * getLitTab(const struct FDRSTableHeader * h) {
return (const struct FDRSLiteral *) (((const u8 *)h) +
ROUNDUP_16(sizeof(struct FDRSTableHeader)));
}
static really_inline
u32 getBaseOffsetOfLits(const struct FDRSTableHeader * h, enum Modes m) {
return getLitTab(h)[get_start_lit_idx(h, m)].offset;
}
static really_inline
u32 packStateVal(const struct FDRSTableHeader * h, enum Modes m, u32 v) {
return v - getBaseOffsetOfLits(h, m) + 1;
}
static really_inline
u32 unpackStateVal(const struct FDRSTableHeader * h, enum Modes m, u32 v) {
return v + getBaseOffsetOfLits(h, m) - 1;
}
static really_inline
u32 has_bit(const struct FDRSHashEntry * ent, u32 bit) {
return (ent->bitfield >> bit) & 0x1;
}
static really_inline
u32 streaming_hash(const u8 *ptr, UNUSED size_t len, enum Modes mode) {
const u64a CASEMASK = 0xdfdfdfdfdfdfdfdfULL;
const u64a MULTIPLIER = 0x0b4e0ef37bc32127ULL;
assert(len >= 32);
u64a v1 = unaligned_load_u64a(ptr);
u64a v2 = unaligned_load_u64a(ptr + 8);
u64a v3 = unaligned_load_u64a(ptr + 16);
if (mode == CASELESS) {
v1 &= CASEMASK;
v2 &= CASEMASK;
v3 &= CASEMASK;
}
v1 *= MULTIPLIER;
v2 *= (MULTIPLIER*MULTIPLIER);
v3 *= (MULTIPLIER*MULTIPLIER*MULTIPLIER);
v1 >>= 32;
v2 >>= 32;
v3 >>= 32;
return v1 ^ v2 ^ v3;
}
#endif

368
src/fdr/fdr_streaming_runtime.h
View File

@@ -1,368 +0,0 @@
/*
* Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef FDR_STREAMING_RUNTIME_H
#define FDR_STREAMING_RUNTIME_H
#include "fdr_streaming_internal.h"
#include "util/partial_store.h"
#include <string.h>
static really_inline
const struct FDRSTableHeader * getSHDR(const struct FDR * fdr) {
const u8 * linkPtr = ((const u8 *)fdr) + fdr->link;
// test if it's not really a engineID, but a 'pseudo engine id'
assert(*(const u32 *)linkPtr == 0xffffffff);
assert(linkPtr);
return (const struct FDRSTableHeader *)linkPtr;
}
// Reads from stream state and unpacks values into stream state table.
static really_inline
void getStreamStates(const struct FDRSTableHeader * streamingTable,
const u8 * stream_state, u32 * table) {
assert(streamingTable);
assert(stream_state);
assert(table);
u8 ss_bytes = streamingTable->streamStateBytes;
u8 ssb = streamingTable->streamStateBits[CASEFUL];
UNUSED u8 ssb_nc = streamingTable->streamStateBits[CASELESS];
assert(ss_bytes == (ssb + ssb_nc + 7) / 8);
#if defined(ARCH_32_BIT)
// On 32-bit hosts, we may be able to avoid having to do any u64a
// manipulation at all.
if (ss_bytes <= 4) {
u32 ssb_mask = (1U << ssb) - 1;
u32 streamVal = partial_load_u32(stream_state, ss_bytes);
table[CASEFUL] = (u32)(streamVal & ssb_mask);
table[CASELESS] = (u32)(streamVal >> ssb);
return;
}
#endif
u64a ssb_mask = (1ULL << ssb) - 1;
u64a streamVal = partial_load_u64a(stream_state, ss_bytes);
table[CASEFUL] = (u32)(streamVal & ssb_mask);
table[CASELESS] = (u32)(streamVal >> (u64a)ssb);
}
#ifndef NDEBUG
// Defensive checking (used in assert) that these table values don't overflow
// outside the range available.
static really_inline UNUSED
u32 streamingTableOverflow(u32 * table, u8 ssb, u8 ssb_nc) {
u32 ssb_mask = (1ULL << (ssb)) - 1;
if (table[CASEFUL] & ~ssb_mask) {
return 1;
}
u32 ssb_nc_mask = (1ULL << (ssb_nc)) - 1;
if (table[CASELESS] & ~ssb_nc_mask) {
return 1;
}
return 0;
}
#endif
// Reads from stream state table and packs values into stream state.
static really_inline
void setStreamStates(const struct FDRSTableHeader * streamingTable,
u8 * stream_state, u32 * table) {
assert(streamingTable);
assert(stream_state);
assert(table);
u8 ss_bytes = streamingTable->streamStateBytes;
u8 ssb = streamingTable->streamStateBits[CASEFUL];
UNUSED u8 ssb_nc = streamingTable->streamStateBits[CASELESS];
assert(ss_bytes == (ssb + ssb_nc + 7) / 8);
assert(!streamingTableOverflow(table, ssb, ssb_nc));
#if defined(ARCH_32_BIT)
// On 32-bit hosts, we may be able to avoid having to do any u64a
// manipulation at all.
if (ss_bytes <= 4) {
u32 stagingStreamState = table[CASEFUL];
stagingStreamState |= (table[CASELESS] << ssb);
partial_store_u32(stream_state, stagingStreamState, ss_bytes);
return;
}
#endif
u64a stagingStreamState = (u64a)table[CASEFUL];
stagingStreamState |= (u64a)table[CASELESS] << ((u64a)ssb);
partial_store_u64a(stream_state, stagingStreamState, ss_bytes);
}
u32 fdrStreamStateActive(const struct FDR * fdr, const u8 * stream_state) {
if (!stream_state) {
return 0;
}
const struct FDRSTableHeader * streamingTable = getSHDR(fdr);
u8 ss_bytes = streamingTable->streamStateBytes;
// We just care if there are any bits set, and the test below is faster
// than a partial_load_u64a (especially on 32-bit hosts).
for (u32 i = 0; i < ss_bytes; i++) {
if (*stream_state) {
return 1;
}
++stream_state;
}
return 0;
}
// binary search for the literal index that contains the current state
static really_inline
u32 findLitTabEntry(const struct FDRSTableHeader * streamingTable,
u32 stateValue, enum Modes m) {
const struct FDRSLiteral * litTab = getLitTab(streamingTable);
u32 lo = get_start_lit_idx(streamingTable, m);
u32 hi = get_end_lit_idx(streamingTable, m);
// Now move stateValue back by one so that we're looking for the
// litTab entry that includes it the string, not the one 'one past' it
stateValue -= 1;
assert(lo != hi);
assert(litTab[lo].offset <= stateValue);
assert(litTab[hi].offset > stateValue);
// binary search to find the entry e such that:
// litTab[e].offsetToLiteral <= stateValue < litTab[e+1].offsetToLiteral
while (lo + 1 < hi) {
u32 mid = (lo + hi) / 2;
if (litTab[mid].offset <= stateValue) {
lo = mid;
} else { //(litTab[mid].offset > stateValue) {
hi = mid;
}
}
assert(litTab[lo].offset <= stateValue);
assert(litTab[hi].offset > stateValue);
return lo;
}
static really_inline
void fdrUnpackStateMode(struct FDR_Runtime_Args *a,
const struct FDRSTableHeader *streamingTable,
const struct FDRSLiteral * litTab,
const u32 *state_table,
const enum Modes m) {
if (!state_table[m]) {
return;
}
u32 stateValue = unpackStateVal(streamingTable, m, state_table[m]);
u32 idx = findLitTabEntry(streamingTable, stateValue, m);
size_t found_offset = litTab[idx].offset;
const u8 * found_buf = found_offset + (const u8 *)streamingTable;
size_t found_sz = stateValue - found_offset;
if (m == CASEFUL) {
a->buf_history = found_buf;
a->len_history = found_sz;
} else {
a->buf_history_nocase = found_buf;
a->len_history_nocase = found_sz;
}
}
static really_inline
void fdrUnpackState(const struct FDR * fdr, struct FDR_Runtime_Args * a,
const u8 * stream_state) {
// nothing to do if there's no stream state for the case
if (!stream_state) {
return;
}
const struct FDRSTableHeader * streamingTable = getSHDR(fdr);
const struct FDRSLiteral * litTab = getLitTab(streamingTable);
u32 state_table[MAX_MODES];
getStreamStates(streamingTable, stream_state, state_table);
fdrUnpackStateMode(a, streamingTable, litTab, state_table, CASEFUL);
fdrUnpackStateMode(a, streamingTable, litTab, state_table, CASELESS);
}
static really_inline
u32 do_single_confirm(const struct FDRSTableHeader *streamingTable,
const struct FDR_Runtime_Args *a, u32 hashState,
enum Modes m) {
const struct FDRSLiteral * litTab = getLitTab(streamingTable);
u32 idx = findLitTabEntry(streamingTable, hashState, m);
size_t found_offset = litTab[idx].offset;
const u8 * s1 = found_offset + (const u8 *)streamingTable;
assert(hashState > found_offset);
size_t l1 = hashState - found_offset;
const u8 * buf = a->buf;
size_t len = a->len;
const char nocase = m != CASEFUL;
if (l1 > len) {
const u8 * hist = nocase ? a->buf_history_nocase : a->buf_history;
size_t hist_len = nocase ? a->len_history_nocase : a->len_history;
if (l1 > len+hist_len) {
return 0; // Break out - not enough total history
}
size_t overhang = l1 - len;
assert(overhang <= hist_len);
if (cmpForward(hist + hist_len - overhang, s1, overhang, nocase)) {
return 0;
}
s1 += overhang;
l1 -= overhang;
}
// if we got here, we don't need history or we compared ok out of history
assert(l1 <= len);
if (cmpForward(buf + len - l1, s1, l1, nocase)) {
return 0;
}
return hashState; // our new state
}
static really_inline
void fdrFindStreamingHash(const struct FDR_Runtime_Args *a,
const struct FDRSTableHeader *streamingTable,
u8 hash_len, u32 *hashes) {
u8 tempbuf[128];
const u8 *base;
if (hash_len > a->len) {
assert(hash_len <= 128);
size_t overhang = hash_len - a->len;
assert(overhang <= a->len_history);
memcpy(tempbuf, a->buf_history + a->len_history - overhang, overhang);
memcpy(tempbuf + overhang, a->buf, a->len);
base = tempbuf;
} else {
assert(hash_len <= a->len);
base = a->buf + a->len - hash_len;
}
if (streamingTable->hashNBits[CASEFUL]) {
hashes[CASEFUL] = streaming_hash(base, hash_len, CASEFUL);
}
if (streamingTable->hashNBits[CASELESS]) {
hashes[CASELESS] = streaming_hash(base, hash_len, CASELESS);
}
}
static really_inline
const struct FDRSHashEntry *getEnt(const struct FDRSTableHeader *streamingTable,
u32 h, const enum Modes m) {
u32 nbits = streamingTable->hashNBits[m];
if (!nbits) {
return NULL;
}
u32 h_ent = h & ((1 << nbits) - 1);
u32 h_low = (h >> nbits) & 63;
const struct FDRSHashEntry *tab =
(const struct FDRSHashEntry *)((const u8 *)streamingTable
+ streamingTable->hashOffset[m]);
const struct FDRSHashEntry *ent = tab + h_ent;
if (!has_bit(ent, h_low)) {
return NULL;
}
return ent;
}
static really_inline
void fdrPackStateMode(u32 *state_table, const struct FDR_Runtime_Args *a,
const struct FDRSTableHeader *streamingTable,
const struct FDRSHashEntry *ent, const enum Modes m) {
assert(ent);
assert(streamingTable->hashNBits[m]);
const struct FDRSHashEntry *tab =
(const struct FDRSHashEntry *)((const u8 *)streamingTable
+ streamingTable->hashOffset[m]);
while (1) {
u32 tmp = 0;
if ((tmp = do_single_confirm(streamingTable, a, ent->state, m))) {
state_table[m] = packStateVal(streamingTable, m, tmp);
break;
}
if (ent->link == LINK_INVALID) {
break;
}
ent = tab + ent->link;
}
}
static really_inline
void fdrPackState(const struct FDR *fdr, const struct FDR_Runtime_Args *a,
u8 *stream_state) {
// nothing to do if there's no stream state for the case
if (!stream_state) {
return;
}
// get pointers to the streamer FDR and the tertiary structure
const struct FDRSTableHeader *streamingTable = getSHDR(fdr);
assert(streamingTable->N);
u32 state_table[MAX_MODES] = {0, 0};
// if we don't have enough history, we don't need to do anything
if (streamingTable->N <= a->len + a->len_history) {
u32 hashes[MAX_MODES] = {0, 0};
fdrFindStreamingHash(a, streamingTable, streamingTable->N, hashes);
const struct FDRSHashEntry *ent_ful = getEnt(streamingTable,
hashes[CASEFUL], CASEFUL);
const struct FDRSHashEntry *ent_less = getEnt(streamingTable,
hashes[CASELESS], CASELESS);
if (ent_ful) {
fdrPackStateMode(state_table, a, streamingTable, ent_ful,
CASEFUL);
}
if (ent_less) {
fdrPackStateMode(state_table, a, streamingTable, ent_less,
CASELESS);
}
}
setStreamStates(streamingTable, stream_state, state_table);
}
#endif