diff --git a/src/hs_common.h b/src/hs_common.h index b38d9505..4bf31146 100644 --- a/src/hs_common.h +++ b/src/hs_common.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, Intel Corporation + * Copyright (c) 2015-2016, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -500,6 +500,25 @@ const char *hs_version(void); */ #define HS_BAD_ALLOC (-9) +/** + * The scratch region was already in use. + * + * This error is returned when Hyperscan is able to detect that the scratch + * region given is already in use by another Hyperscan API call. + * + * A separate scratch region, allocated with @ref hs_alloc_scratch() or @ref + * hs_clone_scratch(), is required for every concurrent caller of the Hyperscan + * API. + * + * For example, this error might be returned when @ref hs_scan() has been + * called inside a callback delivered by a currently-executing @ref hs_scan() + * call using the same scratch region. + * + * Note: Not all concurrent uses of scratch regions may be detected. This error + * is intended as a best-effort debugging tool, not a guarantee. + */ +#define HS_SCRATCH_IN_USE (-10) + /** @} */ #ifdef __cplusplus diff --git a/src/runtime.c b/src/runtime.c index 852eaf92..95f21d84 100644 --- a/src/runtime.c +++ b/src/runtime.c @@ -319,8 +319,13 @@ hs_error_t hs_scan(const hs_database_t *db, const char *data, unsigned length, return HS_INVALID; } + if (unlikely(markScratchInUse(scratch))) { + return HS_SCRATCH_IN_USE; + } + if (rose->minWidth > length) { DEBUG_PRINTF("minwidth=%u > length=%u\n", rose->minWidth, length); + unmarkScratchInUse(scratch); return HS_SUCCESS; } @@ -394,12 +399,14 @@ hs_error_t hs_scan(const hs_database_t *db, const char *data, unsigned length, done_scan: if (told_to_stop_matching(scratch)) { + unmarkScratchInUse(scratch); return HS_SCAN_TERMINATED; } if (rose->hasSom) { int halt = flushStoredSomMatches(scratch, ~0ULL); if (halt) { + unmarkScratchInUse(scratch); return HS_SCAN_TERMINATED; } } @@ -412,7 +419,10 @@ done_scan: set_retval: DEBUG_PRINTF("done. told_to_stop_matching=%d\n", told_to_stop_matching(scratch)); - return told_to_stop_matching(scratch) ? HS_SCAN_TERMINATED : HS_SUCCESS; + hs_error_t rv = told_to_stop_matching(scratch) ? HS_SCAN_TERMINATED + : HS_SUCCESS; + unmarkScratchInUse(scratch); + return rv; } static really_inline @@ -674,7 +684,11 @@ hs_error_t hs_reset_and_copy_stream(hs_stream_t *to_id, if (!scratch || !validScratch(to_id->rose, scratch)) { return HS_INVALID; } + if (unlikely(markScratchInUse(scratch))) { + return HS_SCRATCH_IN_USE; + } report_eod_matches(to_id, scratch, onEvent, context); + unmarkScratchInUse(scratch); } size_t stateSize @@ -784,7 +798,10 @@ hs_error_t hs_scan_stream_internal(hs_stream_t *id, const char *data, unsigned length, UNUSED unsigned flags, hs_scratch_t *scratch, match_event_handler onEvent, void *context) { - if (unlikely(!id || !scratch || !data || !validScratch(id->rose, scratch))) { + assert(id); + assert(scratch); + + if (unlikely(!data)) { return HS_INVALID; } @@ -878,8 +895,18 @@ HS_PUBLIC_API hs_error_t hs_scan_stream(hs_stream_t *id, const char *data, unsigned length, unsigned flags, hs_scratch_t *scratch, match_event_handler onEvent, void *context) { - return hs_scan_stream_internal(id, data, length, flags, scratch, - onEvent, context); + if (unlikely(!id || !scratch || !data || + !validScratch(id->rose, scratch))) { + return HS_INVALID; + } + + if (unlikely(markScratchInUse(scratch))) { + return HS_SCRATCH_IN_USE; + } + hs_error_t rv = hs_scan_stream_internal(id, data, length, flags, scratch, + onEvent, context); + unmarkScratchInUse(scratch); + return rv; } HS_PUBLIC_API @@ -893,7 +920,11 @@ hs_error_t hs_close_stream(hs_stream_t *id, hs_scratch_t *scratch, if (!scratch || !validScratch(id->rose, scratch)) { return HS_INVALID; } + if (unlikely(markScratchInUse(scratch))) { + return HS_SCRATCH_IN_USE; + } report_eod_matches(id, scratch, onEvent, context); + unmarkScratchInUse(scratch); } hs_stream_free(id); @@ -913,7 +944,11 @@ hs_error_t hs_reset_stream(hs_stream_t *id, UNUSED unsigned int flags, if (!scratch || !validScratch(id->rose, scratch)) { return HS_INVALID; } + if (unlikely(markScratchInUse(scratch))) { + return HS_SCRATCH_IN_USE; + } report_eod_matches(id, scratch, onEvent, context); + unmarkScratchInUse(scratch); } init_stream(id, id->rose); @@ -995,6 +1030,10 @@ hs_error_t hs_scan_vector(const hs_database_t *db, const char * const * data, return HS_INVALID; } + if (unlikely(markScratchInUse(scratch))) { + return HS_SCRATCH_IN_USE; + } + hs_stream_t *id = (hs_stream_t *)(scratch->bstate); init_stream(id, rose); /* open stream */ @@ -1009,6 +1048,7 @@ hs_error_t hs_scan_vector(const hs_database_t *db, const char * const * data, = hs_scan_stream_internal(id, data[i], length[i], 0, scratch, onEvent, context); if (ret != HS_SUCCESS) { + unmarkScratchInUse(scratch); return ret; } } @@ -1018,9 +1058,12 @@ hs_error_t hs_scan_vector(const hs_database_t *db, const char * const * data, report_eod_matches(id, scratch, onEvent, context); if (told_to_stop_matching(scratch)) { + unmarkScratchInUse(scratch); return HS_SCAN_TERMINATED; } } + unmarkScratchInUse(scratch); + return HS_SUCCESS; } diff --git a/src/scratch.c b/src/scratch.c index 42db42ac..b496833a 100644 --- a/src/scratch.c +++ b/src/scratch.c @@ -129,6 +129,7 @@ hs_error_t alloc_scratch(const hs_scratch_t *proto, hs_scratch_t **scratch) { *s = *proto; s->magic = SCRATCH_MAGIC; + s->in_use = 1; s->scratchSize = alloc_size; s->scratch_alloc = (char *)s_tmp; @@ -254,6 +255,9 @@ hs_error_t hs_alloc_scratch(const hs_database_t *db, hs_scratch_t **scratch) { if ((*scratch)->magic != SCRATCH_MAGIC) { return HS_INVALID; } + if (markScratchInUse(*scratch)) { + return HS_SCRATCH_IN_USE; + } } const struct RoseEngine *rose = hs_get_bytecode(db); @@ -355,6 +359,7 @@ hs_error_t hs_alloc_scratch(const hs_database_t *db, hs_scratch_t **scratch) { hs_scratch_free(proto_tmp); /* kill off temp used for sizing */ } + unmarkScratchInUse(*scratch); return HS_SUCCESS; } @@ -384,6 +389,10 @@ hs_error_t hs_free_scratch(hs_scratch_t *scratch) { if (scratch->magic != SCRATCH_MAGIC) { return HS_INVALID; } + if (markScratchInUse(scratch)) { + return HS_SCRATCH_IN_USE; + } + scratch->magic = 0; assert(scratch->scratch_alloc); DEBUG_PRINTF("scratch %p is really at %p : freeing\n", scratch, diff --git a/src/scratch.h b/src/scratch.h index 21ec809c..f8e322f8 100644 --- a/src/scratch.h +++ b/src/scratch.h @@ -140,6 +140,7 @@ struct match_deduper { */ struct ALIGN_CL_DIRECTIVE hs_scratch { u32 magic; + u8 in_use; /**< non-zero when being used by an API call. */ char *scratch_alloc; /* user allocated scratch object */ u32 queueCount; u32 bStateSize; /**< sizeof block mode states */ @@ -198,6 +199,34 @@ char can_stop_matching(const struct hs_scratch *scratch) { return scratch->core_info.status & (STATUS_TERMINATED | STATUS_EXHAUSTED); } +/** + * \brief Mark scratch as in use. + * + * Returns non-zero if it was already in use, zero otherwise. + */ +static really_inline +char markScratchInUse(struct hs_scratch *scratch) { + DEBUG_PRINTF("marking scratch as in use\n"); + assert(scratch && scratch->magic == SCRATCH_MAGIC); + if (scratch->in_use) { + DEBUG_PRINTF("scratch already in use!\n"); + return 1; + } + scratch->in_use = 1; + return 0; +} + +/** + * \brief Mark scratch as no longer in use. + */ +static really_inline +void unmarkScratchInUse(struct hs_scratch *scratch) { + DEBUG_PRINTF("marking scratch as not in use\n"); + assert(scratch && scratch->magic == SCRATCH_MAGIC); + assert(scratch->in_use == 1); + scratch->in_use = 0; +} + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/unit/CMakeLists.txt b/unit/CMakeLists.txt index b3cc8cea..a893d3d5 100644 --- a/unit/CMakeLists.txt +++ b/unit/CMakeLists.txt @@ -97,6 +97,7 @@ set(unit_hyperscan_SOURCES hyperscan/multi.cpp hyperscan/order.cpp hyperscan/scratch_op.cpp + hyperscan/scratch_in_use.cpp hyperscan/serialize.cpp hyperscan/single.cpp hyperscan/som.cpp diff --git a/unit/hyperscan/scratch_in_use.cpp b/unit/hyperscan/scratch_in_use.cpp new file mode 100644 index 00000000..ddd4bf4e --- /dev/null +++ b/unit/hyperscan/scratch_in_use.cpp @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2016, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include "test_util.h" + +#include "hs.h" +#include "gtest/gtest.h" + +#include + +using namespace std; + +struct RescanContext { + RescanContext(const hs_database_t *db_in, hs_scratch_t *scratch_in) + : db(db_in), scratch(scratch_in) {} + const hs_database_t *db; + hs_scratch_t *scratch; + size_t matches = 0; +}; + +struct HyperscanDatabaseDeleter { + void operator()(hs_database_t *db) const { + hs_error_t err = hs_free_database(db); + EXPECT_EQ(HS_SUCCESS, err); + } +}; + +unique_ptr +makeDatabase(const char *expression, unsigned int flags, unsigned int mode) { + hs_database_t *db = nullptr; + hs_compile_error_t *compile_err = nullptr; + hs_error_t err = hs_compile(expression, flags, mode, nullptr, &db, + &compile_err); + EXPECT_EQ(HS_SUCCESS, err); + + return unique_ptr(db); +} + +// Generic block mode test that uses the given scan callback. +static +void runBlockTest(match_event_handler cb_func) { + auto db = makeDatabase("foo.*bar", 0, HS_MODE_BLOCK); + ASSERT_NE(nullptr, db.get()); + + hs_scratch_t *scratch = nullptr; + hs_error_t err = hs_alloc_scratch(db.get(), &scratch); + ASSERT_EQ(HS_SUCCESS, err); + ASSERT_TRUE(scratch != nullptr); + + RescanContext rc(db.get(), scratch); + const string data = "___foo___bar_"; + + err = hs_scan(db.get(), data.c_str(), data.length(), 0, scratch, + cb_func, &rc); + ASSERT_EQ(HS_SUCCESS, err); + ASSERT_EQ(1, rc.matches); + + // teardown + hs_free_scratch(scratch); +} + +// Generic streaming mode test that uses the given scan callback. +static +void runStreamingTest(match_event_handler cb_func) { + auto db = makeDatabase("foo.*bar", 0, HS_MODE_STREAM); + ASSERT_NE(nullptr, db.get()); + + hs_scratch_t *scratch = nullptr; + hs_error_t err = hs_alloc_scratch(db.get(), &scratch); + ASSERT_EQ(HS_SUCCESS, err); + ASSERT_TRUE(scratch != nullptr); + + hs_stream_t *stream = nullptr; + err = hs_open_stream(db.get(), 0, &stream); + ASSERT_EQ(HS_SUCCESS, err); + ASSERT_TRUE(stream != nullptr); + + RescanContext rc(db.get(), scratch); + const string data = "___foo___bar_"; + + err = hs_scan_stream(stream, data.c_str(), data.length(), 0, scratch, + cb_func, &rc); + ASSERT_EQ(HS_SUCCESS, err); + ASSERT_EQ(1, rc.matches); + + // teardown + hs_close_stream(stream, scratch, nullptr, nullptr); + hs_free_scratch(scratch); +} + +// Generic vectored mode test that uses the given scan callback. +static +void runVectoredTest(match_event_handler cb_func) { + auto db = makeDatabase("foo.*bar", 0, HS_MODE_VECTORED); + ASSERT_NE(nullptr, db.get()); + + hs_scratch_t *scratch = nullptr; + hs_error_t err = hs_alloc_scratch(db.get(), &scratch); + ASSERT_EQ(HS_SUCCESS, err); + ASSERT_TRUE(scratch != nullptr); + + RescanContext rc(db.get(), scratch); + const string data1 = "___foo_"; + const string data2 = "bar_"; + + const char *vec[] = {data1.c_str(), data2.c_str()}; + const unsigned int len[] = {unsigned(data1.length()), + unsigned(data2.length())}; + + err = hs_scan_vector(db.get(), vec, len, 2, 0, scratch, cb_func, &rc); + ASSERT_EQ(HS_SUCCESS, err); + ASSERT_EQ(1, rc.matches); + + // teardown + hs_free_scratch(scratch); +} + +static +int rescan_block_cb(unsigned, unsigned long long, unsigned long long, unsigned, + void *ctx) { + RescanContext *rctx = (RescanContext *)ctx; + rctx->matches++; + + const string data = "___foo___bar_"; + + hs_error_t err = hs_scan(rctx->db, data.c_str(), data.length(), 0, + rctx->scratch, dummy_cb, nullptr); + EXPECT_EQ(HS_SCRATCH_IN_USE, err); + return 0; +} + + +// Attempt to use in-use scratch inside block mode callback. +TEST(ScratchInUse, Block) { + runBlockTest(rescan_block_cb); +} + +static +int rescan_stream_cb(unsigned, unsigned long long, unsigned long long, unsigned, + void *ctx) { + RescanContext *rctx = (RescanContext *)ctx; + rctx->matches++; + + const string data = "___foo___bar_"; + + hs_stream_t *stream = nullptr; + hs_error_t err = hs_open_stream(rctx->db, 0, &stream); + EXPECT_EQ(HS_SUCCESS, err); + EXPECT_TRUE(stream != nullptr); + if (stream == nullptr) { + return 1; + } + + err = hs_scan_stream(stream, data.c_str(), data.length(), 0, + rctx->scratch, dummy_cb, nullptr); + EXPECT_EQ(HS_SCRATCH_IN_USE, err); + + hs_close_stream(stream, nullptr, nullptr, nullptr); + return 0; +} + +// Attempt to use in-use scratch inside streaming mode callback. +TEST(ScratchInUse, Streaming) { + runStreamingTest(rescan_stream_cb); +} + +static +int rescan_vector_cb(unsigned, unsigned long long, unsigned long long, unsigned, + void *ctx) { + RescanContext *rctx = (RescanContext *)ctx; + rctx->matches++; + + const string data1 = "___foo_"; + const string data2 = "bar_"; + + const char *vec[] = {data1.c_str(), data2.c_str()}; + const unsigned int len[] = {unsigned(data1.length()), + unsigned(data2.length())}; + + hs_error_t err = hs_scan_vector(rctx->db, vec, len, 2, 0, rctx->scratch, + dummy_cb, nullptr); + EXPECT_EQ(HS_SCRATCH_IN_USE, err); + return 0; +} + +// Attempt to use in-use scratch inside vectored mode callback. +TEST(ScratchInUse, Vectored) { + runVectoredTest(rescan_vector_cb); +} + +static +int rescan_realloc_cb(unsigned, unsigned long long, unsigned long long, + unsigned, void *ctx) { + RescanContext *rctx = (RescanContext *)ctx; + rctx->matches++; + + auto db = makeDatabase("another db", 0, HS_MODE_BLOCK); + hs_error_t err = hs_alloc_scratch(db.get(), &rctx->scratch); + EXPECT_EQ(HS_SCRATCH_IN_USE, err); + return 0; +} + +// Attempt to use hs_alloc_scratch on in-use scratch inside callback (block +// scan). +TEST(ScratchInUse, ReallocScratchBlock) { + runBlockTest(rescan_realloc_cb); +} + +// Attempt to use hs_alloc_scratch on in-use scratch inside callback (streaming +// scan). +TEST(ScratchInUse, ReallocScratchStreaming) { + runStreamingTest(rescan_realloc_cb); +} + +// Attempt to use hs_alloc_scratch on in-use scratch inside callback (vectored +// scan). +TEST(ScratchInUse, ReallocScratchVector) { + runVectoredTest(rescan_realloc_cb); +} + +static +int rescan_free_cb(unsigned, unsigned long long, unsigned long long, + unsigned, void *ctx) { + RescanContext *rctx = (RescanContext *)ctx; + rctx->matches++; + + hs_error_t err = hs_free_scratch(rctx->scratch); + EXPECT_EQ(HS_SCRATCH_IN_USE, err); + return 0; +} + +// Attempt to use hs_free_scratch on in-use scratch inside callback (block +// scan). +TEST(ScratchInUse, FreeScratchBlock) { + runBlockTest(rescan_free_cb); +} + +// Attempt to use hs_free_scratch on in-use scratch inside callback (streaming +// scan). +TEST(ScratchInUse, FreeScratchStreaming) { + runStreamingTest(rescan_free_cb); +} + +// Attempt to use hs_free_scratch on in-use scratch inside callback (vectored +// scan). +TEST(ScratchInUse, FreeScratchVector) { + runVectoredTest(rescan_free_cb); +} diff --git a/unit/hyperscan/test_util.h b/unit/hyperscan/test_util.h index 1ce0c182..fad6137c 100644 --- a/unit/hyperscan/test_util.h +++ b/unit/hyperscan/test_util.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, Intel Corporation + * Copyright (c) 2015-2016, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -29,7 +29,9 @@ #ifndef TEST_UTIL_H #define TEST_UTIL_H +#include #include +#include #include #include "hs.h"