github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-09-29 11:16:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

multibit compression support

Browse Source
This commit is contained in:
Hong, Yang A
2017-05-08 15:56:28 -04:00
committed by Matthew Barr
parent 5f6291529f
commit 205a5bc98f
6 changed files with 1032 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/stream_compress.c
View File

@@ -32,6 +32,7 @@
#include "nfa/nfa_internal.h"
#include "rose/rose_internal.h"
#include "util/multibit.h"
#include "util/multibit_compress.h"
#include "util/uniform_ops.h"
#include <string.h>
@@ -57,7 +58,39 @@
DEBUG_PRINTF("co = %zu\n", currOffset); \
} while (0);
#define COPY_MULTIBIT_IN(p, total_bits) do { \
size_t sz; \
STREAM_QUAL u8 *bits = (STREAM_QUAL u8 *)p; \
BUF_QUAL u8 *comp = (BUF_QUAL u8 *)(buf + currOffset); \
if (!mmbit_compress(bits, total_bits, comp, &sz, \
buf_size - currOffset)) { \
return 0; /* error */ \
} \
currOffset += sz; \
DEBUG_PRINTF("co = %zu\n", currOffset); \
} while (0);
#define COPY_MULTIBIT_OUT(p, total_bits) do { \
size_t sz; \
STREAM_QUAL u8 *bits = (STREAM_QUAL u8 *)p; \
BUF_QUAL u8 *comp = (BUF_QUAL u8 *)(buf + currOffset); \
if (!mmbit_decompress(bits, total_bits, comp, &sz, \
buf_size - currOffset)) { \
return 0; /* error */ \
} \
currOffset += sz; \
DEBUG_PRINTF("co = %zu\n", currOffset); \
} while (0);
#define COPY_MULTIBIT_SIZE(p, total_bits) do { \
STREAM_QUAL u8 *bits = (STREAM_QUAL u8 *)p; \
size_t sz = mmbit_compsize(bits, total_bits); \
currOffset += sz; \
DEBUG_PRINTF("co = %zu\n", currOffset); \
} while (0);
#define COPY COPY_OUT
#define COPY_MULTIBIT COPY_MULTIBIT_OUT
#define ASSIGN(lhs, rhs) do { lhs = rhs; } while (0)
#define FN_SUFFIX expand
#define STREAM_QUAL
@@ -70,6 +103,7 @@ int expand_stream(struct hs_stream *stream, const struct RoseEngine *rose,
}
#define COPY COPY_IN
#define COPY_MULTIBIT COPY_MULTIBIT_IN
#define ASSIGN(lhs, rhs) do { } while (0)
#define FN_SUFFIX compress
#define STREAM_QUAL const
@@ -83,6 +117,7 @@ size_t compress_stream(char *buf, size_t buf_size,
}
#define COPY SIZE_COPY_IN
#define COPY_MULTIBIT COPY_MULTIBIT_SIZE
#define ASSIGN(lhs, rhs) do { } while (0)
#define FN_SUFFIX size
#define STREAM_QUAL const

16
src/stream_compress_impl.h
View File

@@ -29,10 +29,6 @@
#include "util/join.h"
#define COPY_FIELD(x) COPY(&x, sizeof(x))
/* TODO: replace with a multibit compress/expand call */
#define COPY_MULTIBIT(mm_p, mm_size_bytes) COPY(mm_p, mm_size_bytes)
#define COPY_LEFTFIXES JOIN(sc_left_, FN_SUFFIX)
#define COPY_SOM_INFO JOIN(sc_som_, FN_SUFFIX)
@@ -93,8 +89,8 @@ size_t COPY_SOM_INFO(const struct RoseEngine *rose, size_t currOffset,
assert(so->somValid);
assert(so->somWritable);
COPY_MULTIBIT(stream_body + so->somWritable, so->somMultibit_size);
COPY_MULTIBIT(stream_body + so->somValid, so->somMultibit_size);
COPY_MULTIBIT(stream_body + so->somWritable, rose->somLocationCount);
COPY_MULTIBIT(stream_body + so->somValid, rose->somLocationCount);
/* Copy only the som slots which contain valid values. */
/* Note: in the expand case the som valid array has been copied in. */
@@ -123,7 +119,7 @@ size_t JOIN(sc_, FN_SUFFIX)(const struct RoseEngine *rose,
ASSIGN(stream->rose, rose);
COPY(stream_body + ROSE_STATE_OFFSET_STATUS_FLAGS, 1);
COPY_MULTIBIT(stream_body + ROSE_STATE_OFFSET_ROLE_MMBIT, rose->stateSize);
COPY_MULTIBIT(stream_body + ROSE_STATE_OFFSET_ROLE_MMBIT, rose->rolesWithStateCount);
/* stream is valid in compress/size, and stream->offset has been set already
* on the expand side */
@@ -131,8 +127,8 @@ size_t JOIN(sc_, FN_SUFFIX)(const struct RoseEngine *rose,
u32 history = MIN((u32)offset, rose->historyRequired);
/* copy the active mmbits */
COPY_MULTIBIT(stream_body + so->activeLeafArray, so->activeLeafArray_size);
COPY_MULTIBIT(stream_body + so->activeLeftArray, so->activeLeftArray_size);
COPY_MULTIBIT(stream_body + so->activeLeafArray, rose->activeArrayCount);
COPY_MULTIBIT(stream_body + so->activeLeftArray, rose->activeLeftCount);
COPY(stream_body + so->longLitState, so->longLitState_size);
@@ -150,7 +146,7 @@ size_t JOIN(sc_, FN_SUFFIX)(const struct RoseEngine *rose,
COPY(stream_body + hend - history, history);
/* copy the exhaustion multibit */
COPY_MULTIBIT(stream_body + so->exhausted, so->exhausted_size);
COPY_MULTIBIT(stream_body + so->exhausted, rose->ekeyCount);
/* copy nfa stream state for endfixes */
/* Note: in the expand case the active array has already been copied into

204
src/util/multibit_compress.h Normal file
View File

@@ -0,0 +1,204 @@
/*
* Copyright (c) 2017, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** file
* \brief multibit compression API: compress / decompress / size
*/
#ifndef MULTIBIT_COMPRESS_H
#define MULTIBIT_COMPRESS_H
#include "multibit.h"
#ifdef __cplusplus
extern "C" {
#endif
/** \brief size API. */
static really_inline
size_t mmbit_compsize(const u8 *bits, u32 total_bits) {
// Deal with flat model.
if (total_bits <= MMB_FLAT_MAX_BITS) {
return (ROUNDUP_N(total_bits, 8) / 8);
}
// Deal with all cleared mmb.
if (mmb_load(bits) == 0) {
return sizeof(MMB_TYPE);
}
// Deal with normal pyramid mmb.
const u32 max_level = mmbit_maxlevel(total_bits);
u32 level = 0;
u32 key = 0;
u32 key_rem = 0;
u32 num_block = 0;
// Iteration-version of DFS
while (1) {
if (key_rem < MMB_KEY_BITS) {
const u8 *block_ptr = mmbit_get_level_root_const(bits, level) +
key * sizeof(MMB_TYPE);
MMB_TYPE block = mmb_load(block_ptr);
MMB_TYPE block_1 = block & ~mmb_mask_zero_to_nocheck(key_rem);
if (mmb_popcount(block) == mmb_popcount(block_1)) {
num_block++;
}
if (level < max_level && block_1) {
key = (key << MMB_KEY_SHIFT) + mmb_ctz(block_1);
key_rem = 0;
level++;
continue;
}
}
if (level-- == 0) {
return sizeof(MMB_TYPE) * num_block;
}
key_rem = (key & MMB_KEY_MASK) + 1;
key >>= MMB_KEY_SHIFT;
}
}
/** \brief compress API. */
static really_inline
char mmbit_compress(const u8 *bits, u32 total_bits, u8 *comp,
size_t *comp_space, size_t max_comp_space) {
UNUSED u8 *comp_init = comp;
// Compute comp_size first.
size_t comp_size = mmbit_compsize(bits, total_bits);
// Check whether out of writable range.
if (comp_size > max_comp_space) {
return 0;
}
*comp_space = comp_size; // Return comp_size outside.
// Deal with flat model.
if (total_bits <= MMB_FLAT_MAX_BITS) {
memcpy(comp, bits, comp_size);
return 1;
}
// Deal with all cleared mmb.
if (mmb_load(bits) == 0) {
memcpy(comp, bits, sizeof(MMB_TYPE));
return 1;
}
// Deal with normal pyramid mmb.
const u32 max_level = mmbit_maxlevel(total_bits);
u32 level = 0;
u32 key = 0;
u32 key_rem = 0;
// Iteration-version of DFS
while (1) {
if (key_rem < MMB_KEY_BITS) {
const u8 *block_ptr = mmbit_get_level_root_const(bits, level) +
key * sizeof(MMB_TYPE);
MMB_TYPE block = mmb_load(block_ptr);
MMB_TYPE block_1 = block & ~mmb_mask_zero_to_nocheck(key_rem);
if (mmb_popcount(block) == mmb_popcount(block_1)) {
memcpy(comp, &block, sizeof(MMB_TYPE));
comp += sizeof(MMB_TYPE);
}
if (level < max_level && block_1) {
key = (key << MMB_KEY_SHIFT) + mmb_ctz(block_1);
key_rem = 0;
level++;
continue;
}
}
if (level-- == 0) {
break;
}
key_rem = (key & MMB_KEY_MASK) + 1;
key >>= MMB_KEY_SHIFT;
}
assert((u32)(comp - comp_init) == comp_size);
return 1;
}
/** \brief decompress API. */
static really_inline
char mmbit_decompress(u8 *bits, u32 total_bits, const u8 *comp,
size_t *comp_space, size_t max_comp_space) {
UNUSED const u8 *comp_init = comp;
size_t comp_size;
// Deal with flat model.
if (total_bits <= MMB_FLAT_MAX_BITS) {
comp_size = ROUNDUP_N(total_bits, 8) / 8;
memcpy(bits, comp, comp_size);
*comp_space = comp_size;
return 1;
}
// Deal with all cleared mmb.
if (mmb_load(comp) == 0) {
comp_size = sizeof(MMB_TYPE);
memcpy(bits, comp, comp_size);
*comp_space = comp_size;
return 1;
}
// Deal with normal mmb.
u32 max_level = mmbit_maxlevel(total_bits);
u32 level = 0;
u32 key = 0;
u32 key_rem = 0;
UNUSED const u8 *comp_end = comp_init + max_comp_space;
// Iteration-version of DFS
memcpy(bits, comp, sizeof(MMB_TYPE)); // Copy root block first.
comp += sizeof(MMB_TYPE);
while (1) {
if (key_rem < MMB_KEY_BITS) {
u8 *block_ptr = mmbit_get_level_root(bits, level) +
key * sizeof(MMB_TYPE);
MMB_TYPE block = mmb_load(block_ptr);
MMB_TYPE block_1 = block & ~mmb_mask_zero_to_nocheck(key_rem);
if (level < max_level && block_1) {
key = (key << MMB_KEY_SHIFT) + mmb_ctz(block_1);
u8 *block_ptr_1 = mmbit_get_level_root(bits, level + 1) +
key * sizeof(MMB_TYPE);
memcpy(block_ptr_1, comp, sizeof(MMB_TYPE));
comp += sizeof(MMB_TYPE);
if (comp > comp_end) {
return 0; // Out of buffer.
}
key_rem = 0;
level++;
continue;
}
}
if (level-- == 0) {
break;
}
key_rem = (key & MMB_KEY_MASK) + 1;
key >>= MMB_KEY_SHIFT;
}
comp_size = (u32)(comp - comp_init);
*comp_space = comp_size;
return 1;
}
#ifdef __cplusplus
} // extern "C"
#endif
#endif // MULTBIT_COMPRESS_H