diff --git a/src/nfa/shufti_simd.hpp b/src/nfa/shufti_simd.hpp index 6e9ff3e8..bc9916b5 100644 --- a/src/nfa/shufti_simd.hpp +++ b/src/nfa/shufti_simd.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -39,6 +40,7 @@ #include "util/unaligned.h" #include "util/supervector/supervector.hpp" +#include "util/match.hpp" template static really_inline @@ -62,41 +64,6 @@ typename SuperVector::movemask_type block(SuperVector mask_lo, SuperVector return t.eqmask(SuperVector::Zeroes()); } -template -const u8 *firstMatch(const u8 *buf, typename SuperVector::movemask_type z); - -template -const u8 *lastMatch(const u8 *buf, typename SuperVector::movemask_type z); - -template <> -really_inline -const u8 *firstMatch<16>(const u8 *buf, typename SuperVector<16>::movemask_type z) { - DEBUG_PRINTF("buf %p z %08x \n", buf, z); - DEBUG_PRINTF("z %08x\n", z); - if (unlikely(z != 0xffff)) { - u32 pos = ctz32(~z & 0xffff); - DEBUG_PRINTF("~z %08x\n", ~z); - DEBUG_PRINTF("match @ pos %u\n", pos); - assert(pos < 16); - return buf + pos; - } else { - return NULL; // no match - } -} - -template <> -really_inline -const u8 *firstMatch<64>(const u8 *buf, typename SuperVector<64>::movemask_type z) { - DEBUG_PRINTF("z 0x%016llx\n", z); - if (unlikely(z != ~0ULL)) { - u32 pos = ctz64(~z); - DEBUG_PRINTF("match @ pos %u\n", pos); - assert(pos < 64); - return buf + pos; - } else { - return NULL; // no match - } -} template static really_inline @@ -130,37 +97,6 @@ const u8 *shortShufti(SuperVector mask_lo, SuperVector mask_hi, const u8 * return firstMatch(buf, z); } - -template <> -really_inline -const u8 *lastMatch<16>(const u8 *buf, typename SuperVector<16>::movemask_type z) { - DEBUG_PRINTF("buf %p z %08x \n", buf, z); - DEBUG_PRINTF("z %08x\n", z); - if (unlikely(z != 0xffff)) { - u32 pos = clz32(~z & 0xffff); - DEBUG_PRINTF("~z %08x\n", ~z); - DEBUG_PRINTF("match @ pos %u\n", pos); - assert(pos >= 16 && pos < 32); - return buf + (31 - pos); - } else { - return NULL; // no match - } -} - -template <> -really_inline -const u8 *lastMatch<64>(const u8 *buf, typename SuperVector<64>::movemask_type z) { - DEBUG_PRINTF("z 0x%016llx\n", z); - if (unlikely(z != ~0ULL)) { - u32 pos = clz64(~z); - DEBUG_PRINTF("match @ pos %u\n", pos); - assert(pos < 64); - return buf + pos; - } else { - return NULL; // no match - } -} - template static really_inline const u8 *revBlock(SuperVector mask_lo, SuperVector mask_hi, SuperVector chars, @@ -171,7 +107,6 @@ const u8 *revBlock(SuperVector mask_lo, SuperVector mask_hi, SuperVector(buf, z); } - template const u8 *shuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *buf_end) { assert(buf && buf_end); diff --git a/src/nfa/truffle_simd.hpp b/src/nfa/truffle_simd.hpp index bf421300..943b1818 100644 --- a/src/nfa/truffle_simd.hpp +++ b/src/nfa/truffle_simd.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -39,76 +40,7 @@ #include "util/unaligned.h" #include "util/supervector/supervector.hpp" - - -template -const u8 *firstMatch(const u8 *buf, typename SuperVector::movemask_type z); - -template -const u8 *lastMatch(const u8 *buf, typename SuperVector::movemask_type z); - - -template <> -really_inline -const u8 *firstMatch<16>(const u8 *buf, typename SuperVector<16>::movemask_type z) { - DEBUG_PRINTF("buf %p z %08x \n", buf, z); - DEBUG_PRINTF("z %08x\n", z); - if (unlikely(z != 0xffff)) { - u32 pos = ctz32(~z & 0xffff); - DEBUG_PRINTF("~z %08x\n", ~z); - DEBUG_PRINTF("match @ pos %u\n", pos); - assert(pos < 16); - return buf + pos; - } else { - return NULL; // no match - } -} - -template <> -really_inline -const u8 *firstMatch<64>(const u8 *buf, typename SuperVector<64>::movemask_type z) { - DEBUG_PRINTF("z 0x%016llx\n", z); - if (unlikely(z != ~0ULL)) { - u32 pos = ctz64(~z); - DEBUG_PRINTF("match @ pos %u\n", pos); - assert(pos < 64); - return buf + pos; - } else { - return NULL; // no match - } -} - - -template <> -really_inline -const u8 *lastMatch<16>(const u8 *buf, typename SuperVector<16>::movemask_type z) { - DEBUG_PRINTF("buf %p z %08x \n", buf, z); - DEBUG_PRINTF("z %08x\n", z); - if (unlikely(z != 0xffff)) { - u32 pos = clz32(~z & 0xffff); - DEBUG_PRINTF("~z %08x\n", ~z); - DEBUG_PRINTF("match @ pos %u\n", pos); - assert(pos >= 16 && pos < 32); - return buf + (31 - pos); - } else { - return NULL; // no match - } -} - -template <> -really_inline -const u8 *lastMatch<64>(const u8 *buf, typename SuperVector<64>::movemask_type z) { - DEBUG_PRINTF("z 0x%016llx\n", z); - if (unlikely(z != ~0ULL)) { - u32 pos = clz64(~z); - DEBUG_PRINTF("match @ pos %u\n", pos); - assert(pos < 64); - return buf + pos; - } else { - return NULL; // no match - } -} - +#include "util/match.hpp" template static really_inline @@ -131,7 +63,6 @@ typename SuperVector::movemask_type block(SuperVector shuf_mask_lo_highcle return tmp.eqmask(SuperVector::Zeroes()); } - template static really_inline const u8 *truffleMini(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, const u8 *buf, const u8 *buf_end){ diff --git a/src/util/match.hpp b/src/util/match.hpp new file mode 100644 index 00000000..74da50d8 --- /dev/null +++ b/src/util/match.hpp @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2020-2021, VectorCamp PC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "ue2common.h" +#include "util/arch.h" +#include "util/bitutils.h" +#include "util/unaligned.h" + +#include "util/supervector/supervector.hpp" + +template +const u8 *firstMatch(const u8 *buf, typename SuperVector::movemask_type z); + +template +const u8 *lastMatch(const u8 *buf, typename SuperVector::movemask_type z); + +template <> +really_really_inline +const u8 *firstMatch<16>(const u8 *buf, typename SuperVector<16>::movemask_type z) { + DEBUG_PRINTF("buf %p z %08x \n", buf, z); + DEBUG_PRINTF("z %08x\n", z); + if (unlikely(z != 0xffff)) { + u32 pos = ctz32(~z & 0xffff); + DEBUG_PRINTF("~z %08x\n", ~z); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos < 16); + return buf + pos; + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *firstMatch<64>(const u8 *buf, typename SuperVector<64>::movemask_type z) { + DEBUG_PRINTF("z 0x%016llx\n", z); + if (unlikely(z != ~0ULL)) { + u32 pos = ctz64(~z); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos < 64); + return buf + pos; + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *lastMatch<16>(const u8 *buf, typename SuperVector<16>::movemask_type z) { + DEBUG_PRINTF("buf %p z %08x \n", buf, z); + DEBUG_PRINTF("z %08x\n", z); + if (unlikely(z != 0xffff)) { + u32 pos = clz32(~z & 0xffff); + DEBUG_PRINTF("~z %08x\n", ~z); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos >= 16 && pos < 32); + return buf + (31 - pos); + } else { + return NULL; // no match + } +} + +template <> +really_really_inline +const u8 *lastMatch<64>(const u8 *buf, typename SuperVector<64>::movemask_type z) { + DEBUG_PRINTF("z 0x%016llx\n", z); + if (unlikely(z != ~0ULL)) { + u32 pos = clz64(~z); + DEBUG_PRINTF("match @ pos %u\n", pos); + assert(pos < 64); + return buf + pos; + } else { + return NULL; // no match + } +} +