Initial commit of Hyperscan

src/util/alloc.cpp

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Aligned memory alloc/free.
+ */
+#include "ue2common.h"
+#include "alloc.h"
+
+#include <cstdlib>
+#include <cstring>
+
+namespace ue2 {
+
+// This is one of the simplest ways to catch failure where we aren't using an
+// aligned_(zmalloc|_free) pair - it will force death if the wrong free is used.
+// We use this whenever assertions are switched on.
+#if !defined(NDEBUG)
+#define HACK_OFFSET 64
+#else
+#define HACK_OFFSET 0
+#endif
+
+/* get us a posix_memalign from somewhere */
+#if !defined(HAVE_POSIX_MEMALIGN)
+# if defined(HAVE_MEMALIGN)
+    #define posix_memalign(A, B, C) ((*A = (void *)memalign(B, C)) == nullptr)
+# elif defined(HAVE__ALIGNED_MALLOC)
+    /* on Windows */
+    #include <malloc.h>
+    #define posix_memalign(A, B, C) ((*A = (void *)_aligned_malloc(C, B)) == nullptr)
+# else
+    #error no posix_memalign or memalign aligned malloc
+# endif
+#endif
+
+void *aligned_malloc_internal(size_t size, size_t align) {
+    void *mem;
+#if !defined(_WIN32)
+    int rv = posix_memalign(&mem, align, size);
+    if (rv != 0) {
+        DEBUG_PRINTF("posix_memalign returned %d when asked for %zu bytes\n",
+                     rv, size);
+        return nullptr;
+    }
+#else
+    if (nullptr == (mem = _aligned_malloc(size, align))) {
+        DEBUG_PRINTF("_aligned_malloc failed when asked for %zu bytes\n",
+                     size);
+        return nullptr;
+    }
+#endif
+
+    assert(mem);
+    return mem;
+}
+
+void aligned_free_internal(void *ptr) {
+    if (!ptr) {
+        return;
+    }
+
+#if defined(_WIN32)
+    _aligned_free(ptr);
+#else
+    free(ptr);
+#endif
+}
+
+/** \brief 64-byte aligned, zeroed malloc.
+ *
+ * Pointers should be freed with \ref aligned_free. If we are unable to
+ * allocate the requested number of bytes, this function will throw
+ * std::bad_alloc. */
+void *aligned_zmalloc(size_t size) {
+    // Really huge allocations are probably an indication that we've
+    // done something wrong.
+    assert(size < 1024 * 1024 * 1024); // 1GB
+
+    const size_t alloc_size = size + HACK_OFFSET;
+
+    void *mem = aligned_malloc_internal(alloc_size, 64);
+    if (!mem) {
+        DEBUG_PRINTF("unable to allocate %zu bytes\n", alloc_size);
+        throw std::bad_alloc();
+    }
+
+    DEBUG_PRINTF("alloced %p reporting %p\n", mem, (char *)mem + HACK_OFFSET);
+    assert(ISALIGNED_N(mem, 64));
+
+    memset(mem, 0, alloc_size);
+    return (void *)((char *)mem + HACK_OFFSET);
+}
+
+/** \brief Free a pointer allocated with \ref aligned_zmalloc. */
+void aligned_free(void *ptr) {
+    if (!ptr) {
+        return;
+    }
+
+    void *addr = (void *)((char *)ptr - HACK_OFFSET);
+    DEBUG_PRINTF("asked to free %p freeing %p\n", ptr, addr);
+
+    assert(ISALIGNED_N(addr, 64));
+    aligned_free_internal(addr);
+}
+
+} // namespace ue2

src/util/alloc.h

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Aligned memory alloc/free.
+ */
+
+#ifndef UTIL_ALLOC_H
+#define UTIL_ALLOC_H
+
+#include <cassert>
+#include <cstddef> // size_t
+#include <limits>
+#include <memory>
+#include <new>     // std::bad_alloc
+
+namespace ue2 {
+
+/** \brief 64-byte aligned, zeroed malloc.
+ *
+ * Pointers should be freed with \ref aligned_free. If we are unable to
+ * allocate the requested number of bytes, this function will throw
+ * std::bad_alloc. */
+void *aligned_zmalloc(size_t size);
+
+/** \brief Free a pointer allocated with \ref aligned_zmalloc. */
+void aligned_free(void *ptr);
+
+template <typename T> struct AlignedDeleter {
+    void operator()(T *ptr) const { aligned_free(ptr); }
+};
+template <typename T>
+using aligned_unique_ptr = std::unique_ptr<T, AlignedDeleter<T>>;
+
+/** \brief 64-byte aligned, zeroed malloc that returns an appropriately-typed
+ * aligned_unique_ptr.
+ *
+ * If the requested size cannot be allocated, throws std::bad_alloc.
+ */
+template <typename T>
+inline
+aligned_unique_ptr<T> aligned_zmalloc_unique(size_t size) {
+    T* ptr = static_cast<T *>(aligned_zmalloc(size));
+    assert(ptr); // Guaranteed by aligned_zmalloc.
+    return aligned_unique_ptr<T>(ptr);
+}
+
+/** \brief Internal use only, used by AlignedAllocator. */
+void *aligned_malloc_internal(size_t size, size_t align);
+
+/** \brief Internal use only, used by AlignedAllocator. */
+void aligned_free_internal(void *ptr);
+
+/** \brief Aligned allocator class for use with STL containers. Ensures that
+ * your objects are aligned to N bytes. */
+template <typename T, std::size_t N> class AlignedAllocator {
+public:
+    typedef T value_type;
+    typedef std::size_t size_type;
+    typedef std::ptrdiff_t difference_type;
+    typedef T *pointer;
+    typedef const T *const_pointer;
+    typedef T &reference;
+    typedef const T &const_reference;
+
+    template <typename U> struct rebind {
+        typedef AlignedAllocator<U, N> other;
+    };
+
+    pointer address(reference x) const { return &x; }
+    const_pointer address(const_reference x) const { return &x; }
+
+    size_type max_size() const {
+        return std::numeric_limits<size_type>::max() / sizeof(value_type);
+    }
+
+    pointer allocate(size_type size) const {
+        return static_cast<pointer>(
+            aligned_malloc_internal(size * sizeof(value_type), N));
+    }
+
+    void deallocate(pointer x, size_type) const { aligned_free_internal(x); }
+
+    void construct(pointer x, const value_type &val) const {
+        new (x) value_type(val);
+    }
+
+    void destroy(pointer p) const { p->~value_type(); }
+
+    bool operator==(const AlignedAllocator<T, N> &) const {
+        // All instances of AlignedAllocator can dealloc each others' memory.
+        return true;
+    }
+
+    bool operator!=(const AlignedAllocator<T, N> &) const {
+        // All instances of AlignedAllocator can dealloc each others' memory.
+        return false;
+    }
+};
+
+} // namespace ue2
+
+#endif

src/util/bitfield.h

@@ -0,0 +1,412 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Fast bitset class with find_first and find_next operations.
+ */
+
+#ifndef BITFIELD_H
+#define BITFIELD_H
+
+#include "ue2common.h"
+#include "popcount.h"
+#include "util/bitutils.h"
+
+#include <array>
+#include <cassert>
+
+#include <boost/dynamic_bitset.hpp>
+#include <boost/functional/hash/hash.hpp>
+
+namespace ue2 {
+
+/**
+ * \brief Templated bitset class with find_first and find_next operations.
+ *
+ * This is a simple (but hopefully fast) class to replace our use of
+ * std::bitset<>.
+ *
+ * Note: underlying storage is allocated as an array of 64-bit blocks. All
+ * mutating operations MUST ensure that the trailer (the bits between
+ * requested_size and the end of the array) is filled with zeroes; there's a
+ * clear_trailer member function for this.
+ */
+template<size_t requested_size>
+class bitfield {
+public:
+    /// Empty constructor, zero initializes all bits.
+    bitfield() : bits{{0}} {
+        assert(none());
+    }
+
+    bitfield(const boost::dynamic_bitset<> &a) : bits{{0}} {
+        assert(a.size() == requested_size);
+        assert(none());
+        for (auto i = a.find_first(); i != a.npos; i = a.find_next(i)) {
+            set(i);
+        }
+    }
+
+    /// Complete bitset equality.
+    bool operator==(const bitfield &a) const {
+        return bits == a.bits;
+    }
+
+    /// Inequality.
+    bool operator!=(const bitfield &a) const {
+        return bits != a.bits;
+    }
+
+    /// Ordering.
+    bool operator<(const bitfield &a) const {
+        return bits < a.bits;
+    }
+
+    /// Set all bits.
+    void setall() {
+        for (auto &e : bits) {
+            e = all_ones;
+        }
+        clear_trailer();
+    }
+
+    /// Set all bits (alias for bitset::setall, to match dynamic_bitset).
+    void set() {
+        setall();
+    }
+
+    /// Clear all bits.
+    void clear() {
+        for (auto &e : bits) {
+            e = 0;
+        }
+    }
+
+    /// Clear all bits (alias for bitset::clear).
+    void reset() {
+        clear();
+    }
+
+    /// Clear bit N.
+    void clear(size_t n) {
+        assert(n < size());
+        bits[getword(n)] &= ~maskbit(n);
+    }
+
+    /// Set bit N.
+    void set(size_t n) {
+        assert(n < size());
+        bits[getword(n)] |= maskbit(n);
+    }
+
+    /// Test bit N.
+    bool test(size_t n) const {
+        assert(n < size());
+        return bits[getword(n)] & maskbit(n);
+    }
+
+    /// Flip bit N.
+    void flip(size_t n) {
+        assert(n < size());
+        bits[getword(n)] ^= maskbit(n);
+    }
+
+    /// Flip all bits.
+    void flip() {
+        for (auto &e : bits) {
+            e = ~e;
+        }
+        clear_trailer();
+    }
+
+    /// Switch on the bit in the range [from, to], inclusive.
+    void set_range(size_t from, size_t to) {
+        assert(from <= to);
+        assert(to < requested_size);
+
+        if (from / block_size == to / block_size) {
+            // Small case, our indices are in the same block.
+            block_type block = all_ones << (from % block_size);
+            if (to % block_size != block_size - 1) {
+                block &= maskbit(to + 1) - 1;
+            }
+            bits[from / block_size] |= block;
+            return;
+        }
+
+        // Large case, work in block units. Write a partial mask, then a
+        // run of all-ones blocks, then a partial mask at the end.
+        size_t i = from;
+        if (i % block_size) {
+            block_type block = all_ones << (i % block_size);
+            bits[i / block_size] |= block;
+            i = ROUNDUP_N(i, block_size);
+        }
+
+        for (; i + block_size <= to + 1; i += block_size) {
+            bits[i / block_size] = all_ones;
+        }
+
+        if (i <= to) {
+            assert(to - i + 1 < block_size);
+            bits[i / block_size] |= (maskbit(to + 1) - 1);
+        }
+    }
+
+    /// Returns total number of bits.
+    static constexpr size_t size() {
+        return requested_size;
+    }
+
+    /// Returns number of bits set on.
+    size_t count() const {
+        static_assert(block_size == 64, "adjust popcount for block_type");
+        size_t sum = 0;
+        for (const auto &e : bits) {
+            sum += popcount64(e);
+        }
+        assert(sum <= size());
+        return sum;
+    }
+
+    /// Are no bits set?
+    bool none() const {
+        for (const auto &e : bits) {
+            if (e != 0) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /// Is any bit set?
+    bool any() const {
+        return !none();
+    }
+
+    /// Are all bits set?
+    bool all() const {
+        for (size_t i = 0; i < bits.size() - 1; i++) {
+            if (bits[i] != all_ones) {
+                return false;
+            }
+        }
+        size_t rem = requested_size % block_size;
+        block_type exp = rem ? ((block_type{1} << rem) - 1) : all_ones;
+        return *bits.rbegin() == exp;
+    }
+
+    /// Returns first bit set, or bitfield::npos if none set.
+    size_t find_first() const {
+        for (size_t i = 0; i < bits.size(); i++) {
+            if (bits[i] != 0) {
+                return (i * block_size) + word_ctz(i);
+            }
+        }
+        return npos;
+    }
+
+    // Returns last bit set, or bitfield::npos if none set.
+    size_t find_last() const {
+        for (int i = bits.size() - 1; i >= 0; i--) {
+            if (bits[i]) {
+                static_assert(block_size == 64, "adjust clz for block_type");
+                return (i * block_size) + block_size - 1 - clz64(bits[i]);
+            }
+        }
+        return npos;
+    }
+
+    /// Returns next bit set, or bitfield::npos if none set after 'last'.
+    size_t find_next(size_t last) const {
+        if (last >= size()) {
+            return npos;
+        }
+
+        // check current word.
+        size_t i = getword(last);
+        block_type lastword = bits[i];
+
+        if ((last % block_size) != (block_size - 1)) {
+            lastword &= (all_ones << ((last % block_size) + 1));
+
+            if (lastword) {
+                static_assert(block_size == 64, "adjust ctz for block_type");
+                return (i * block_size) + ctz64(lastword);
+            }
+        }
+
+        // check the rest.
+        for (i++; i < bits.size(); i++) {
+            if (bits[i]) {
+                return (i * block_size) + word_ctz(i);
+            }
+        }
+
+        return npos;
+    }
+
+    size_t find_nth(size_t n) const {
+        assert(n < npos);
+
+        static_assert(block_size == 64, "adjust for block_type");
+
+        size_t sum = 0;
+        for (size_t i = 0; i < bits.size(); i++) {
+            block_type block = bits[i];
+            size_t aftersum = sum + popcount64(block);
+            if (aftersum > n) { // Block contains the nth bit.
+                for (; sum < n; sum++) {
+                    assert(block);
+                    block &= (block - 1);
+                }
+                assert(block);
+                size_t bit = (i * block_size) + ctz64(block);
+                assert(test(bit));
+                return bit;
+            }
+            sum = aftersum;
+        }
+
+        assert(count() < n + 1);
+        return npos;
+    }
+
+    /// Bitwise OR.
+    bitfield operator|(const bitfield &a) const {
+        bitfield cr;
+        for (size_t i = 0; i < bits.size(); i++) {
+            cr.bits[i] = bits[i] | a.bits[i];
+        }
+        return cr;
+    }
+
+    /// Bitwise OR-equals.
+    void operator|=(const bitfield &a) {
+        for (size_t i = 0; i < bits.size(); i++) {
+            bits[i] |= a.bits[i];
+        }
+    }
+
+    /// Bitwise AND.
+    bitfield operator&(const bitfield &a) const {
+        bitfield cr;
+        for (size_t i = 0; i < bits.size(); i++) {
+            cr.bits[i] = bits[i] & a.bits[i];
+        }
+        return cr;
+    }
+
+    /// Bitwise AND-equals.
+    void operator&=(const bitfield &a) {
+        for (size_t i = 0; i < bits.size(); i++) {
+            bits[i] &= a.bits[i];
+        }
+    }
+
+    /// Bitwise XOR.
+    bitfield operator^(const bitfield &a) const {
+        bitfield cr;
+        for (size_t i = 0; i < bits.size(); i++) {
+            cr.bits[i] = bits[i] ^ a.bits[i];
+        }
+        return cr;
+    }
+
+    /// Bitwise XOR-equals.
+    void operator^=(const bitfield &a) {
+        for (size_t i = 0; i < bits.size(); i++) {
+            bits[i] ^= a.bits[i];
+        }
+    }
+
+    /// Bitwise complement.
+    bitfield operator~(void) const {
+        bitfield cr(*this);
+        cr.flip();
+        return cr;
+    }
+
+    /// Simple hash.
+    size_t hash() const {
+        return boost::hash_range(std::begin(bits), std::end(bits));
+    }
+
+    /// Sentinel value meaning "no more bits", used by find_first and
+    /// find_next.
+    static constexpr size_t npos = requested_size;
+
+private:
+    /// Underlying block type.
+    using block_type = u64a;
+
+    /// A block filled with on bits.
+    static constexpr block_type all_ones = ~block_type{0};
+
+    /// Size of a block.
+    static constexpr size_t block_size = sizeof(block_type) * 8;
+
+    static size_t getword(size_t n) {
+        return n / block_size;
+    }
+
+    static block_type maskbit(size_t n) {
+        return (block_type{1} << (n % block_size));
+    }
+
+    size_t word_ctz(size_t n) const {
+        static_assert(block_size == 64, "adjust ctz call for block type");
+        return ctz64(bits[n]);
+    }
+
+    /// Ensures that bits between our requested size and the end of storage are
+    /// zero.
+    void clear_trailer() {
+        size_t final_bits = requested_size % block_size;
+        if (final_bits) {
+            bits.back() &= ((block_type{1} << final_bits) - 1);
+        }
+    }
+
+    /// Size of storage array of blocks.
+    static constexpr size_t num_blocks =
+        (requested_size + block_size - 1) / block_size;
+
+    /// Underlying storage.
+    std::array<block_type, num_blocks> bits;
+};
+
+/** \brief Boost-style hash free function. */
+template<size_t requested_size>
+size_t hash_value(const bitfield<requested_size> &b) {
+    return b.hash();
+}
+
+} // namespace ue2
+
+#endif // BITFIELD_H

src/util/bitutils.h

@@ -0,0 +1,436 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Bit-twiddling primitives (ctz, compress etc)
+ */
+
+#ifndef BITUTILS_H
+#define BITUTILS_H
+
+#include "ue2common.h"
+#include "popcount.h"
+
+#ifdef __cplusplus
+# if defined(HAVE_CXX_X86INTRIN_H)
+#  define USE_X86INTRIN_H
+# endif
+#else // C, baby
+# if defined(HAVE_C_X86INTRIN_H)
+#  define USE_X86INTRIN_H
+# endif
+#endif
+
+#ifdef __cplusplus
+# if defined(HAVE_CXX_INTRIN_H)
+#  define USE_INTRIN_H
+# endif
+#else // C, baby
+# if defined(HAVE_C_INTRIN_H)
+#  define USE_INTRIN_H
+# endif
+#endif
+
+#if defined(USE_X86INTRIN_H)
+#include <x86intrin.h>
+#elif defined(USE_INTRIN_H)
+#include <intrin.h>
+#endif
+
+// MSVC has a different form of inline asm
+#ifdef _WIN32
+#define NO_ASM
+#endif
+
+#define CASE_BIT          0x20
+#define CASE_CLEAR        0xdf
+#define DOUBLE_CASE_CLEAR 0xdfdf
+
+static really_inline
+u32 clz32(u32 x) {
+    assert(x); // behaviour not defined for x == 0
+#if defined(_WIN32)
+    unsigned long r;
+    _BitScanReverse(&r, x);
+    return 31 - r;
+#else
+    return (u32)__builtin_clz(x);
+#endif
+}
+
+static really_inline
+u32 clz64(u64a x) {
+    assert(x); // behaviour not defined for x == 0
+#if defined(_WIN32)
+    unsigned long r;
+    _BitScanReverse64(&r, x);
+    return 63 - r;
+#else
+    return (u32)__builtin_clzll(x);
+#endif
+}
+
+// CTZ (count trailing zero) implementations.
+static really_inline
+u32 ctz32(u32 x) {
+    assert(x); // behaviour not defined for x == 0
+#if defined(_WIN32)
+    unsigned long r;
+    _BitScanForward(&r, x);
+    return r;
+#else
+    return (u32)__builtin_ctz(x);
+#endif
+}
+
+static really_inline
+u32 ctz64(u64a x) {
+    assert(x); // behaviour not defined for x == 0
+#if defined(_WIN32)
+    unsigned long r;
+    _BitScanForward64(&r, x);
+    return r;
+#else
+    return (u32)__builtin_ctzll(x);
+#endif
+}
+
+static really_inline
+u32 lg2(u32 x) {
+    if (!x) {
+        return 0;
+    }
+    return 31 - clz32(x);
+}
+
+static really_inline
+u64a lg2_64(u64a x) {
+    if (!x) {
+        return 0;
+    }
+    return 63 - clz64(x);
+}
+
+static really_inline
+u32 findAndClearLSB_32(u32 *v) {
+    assert(*v != 0); // behaviour not defined in this case
+#ifndef NO_ASM
+    u32 val = *v, offset;
+    __asm__ ("bsf %1, %0\n"
+             "btr %0, %1\n"
+             : "=r" (offset), "=r" (val)
+             : "1" (val));
+    *v = val;
+#else
+    u32 val = *v;
+    u32 offset = ctz32(val);
+    *v = val & (val - 1);
+#endif
+
+    assert(offset < 32);
+    return offset;
+}
+
+static really_inline
+u32 findAndClearLSB_64(u64a *v) {
+    assert(*v != 0); // behaviour not defined in this case
+
+#ifdef ARCH_64_BIT
+#if defined(ARCH_X86_64) && !defined(NO_ASM)
+    u64a val = *v, offset;
+    __asm__ ("bsfq %1, %0\n"
+             "btrq %0, %1\n"
+             : "=r" (offset), "=r" (val)
+             : "1" (val));
+    *v = val;
+#else
+    // generic variant using gcc's builtin on 64-bit
+    u64a val = *v, offset;
+    offset = ctz64(val);
+    *v = val & (val - 1);
+#endif // ARCH_X86_64
+#else
+    // fall back to doing things with two 32-bit cases, since gcc-4.1 doesn't
+    // inline calls to __builtin_ctzll
+    u32 v1 = *v;
+    u32 v2 = (*v >> 32);
+    u32 offset;
+    if (v1) {
+        offset = findAndClearLSB_32(&v1);
+        *v = (u64a)v1 | ((u64a)v2 << 32);
+    } else {
+        offset = findAndClearLSB_32(&v2) + 32;
+        *v = (u64a)v2 << 32;
+    }
+#endif
+
+    assert(offset < 64);
+    return (u32)offset;
+}
+
+static really_inline
+u32 findAndClearMSB_32(u32 *v) {
+    assert(*v != 0); // behaviour not defined in this case
+#ifndef NO_ASM
+    u32 val = *v, offset;
+    __asm__ ("bsr %1, %0\n"
+             "btr %0, %1\n"
+             : "=r" (offset), "=r" (val)
+             : "1" (val));
+    *v = val;
+#else
+    u32 val = *v;
+    u32 offset = 31 - clz32(val);
+    *v = val & ~(1 << offset);
+#endif
+    assert(offset < 32);
+    return offset;
+}
+
+static really_inline
+u32 findAndClearMSB_64(u64a *v) {
+    assert(*v != 0); // behaviour not defined in this case
+
+#ifdef ARCH_64_BIT
+#if defined(ARCH_X86_64) && !defined(NO_ASM)
+    u64a val = *v, offset;
+    __asm__ ("bsrq %1, %0\n"
+             "btrq %0, %1\n"
+             : "=r" (offset), "=r" (val)
+             : "1" (val));
+    *v = val;
+#else
+    // generic variant using gcc's builtin on 64-bit
+    u64a val = *v, offset;
+    offset = 63 - clz64(val);
+    *v = val & ~(1ULL << offset);
+#endif // ARCH_X86_64
+#else
+    // fall back to doing things with two 32-bit cases, since gcc-4.1 doesn't
+    // inline calls to __builtin_ctzll
+    u32 v1 = *v;
+    u32 v2 = (*v >> 32);
+    u32 offset;
+    if (v2) {
+        offset = findAndClearMSB_32(&v2) + 32;
+        *v = ((u64a)v2 << 32) | (u64a)v1;
+    } else {
+        offset = findAndClearMSB_32(&v1);
+        *v = (u64a)v1;
+    }
+#endif
+
+    assert(offset < 64);
+    return (u32)offset;
+}
+
+static really_inline
+u32 compress32(u32 x, u32 m) {
+#if defined(__BMI2__)
+    // BMI2 has a single instruction for this operation.
+    return _pext_u32(x, m);
+#endif
+
+    // Return zero quickly on trivial cases
+    if ((x & m) == 0) {
+        return 0;
+    }
+
+    u32 mk, mp, mv, t;
+
+    x &= m; // clear irrelevant bits
+
+    mk = ~m << 1; // we will count 0's to right
+    for (u32 i = 0; i < 5; i++) {
+        mp = mk ^ (mk << 1);
+        mp ^= mp << 2;
+        mp ^= mp << 4;
+        mp ^= mp << 8;
+        mp ^= mp << 16;
+
+        mv = mp & m; // bits to move
+        m = (m ^ mv) | (mv >> (1 << i)); // compress m
+        t = x & mv;
+        x = (x ^ t) | (t >> (1 << i)); // compress x
+        mk = mk & ~mp;
+    }
+
+    return x;
+}
+
+static really_inline
+u64a compress64(u64a x, u64a m) {
+#if defined(ARCH_X86_64) && defined(__BMI2__)
+    // BMI2 has a single instruction for this operation.
+    return _pext_u64(x, m);
+#endif
+
+    // Return zero quickly on trivial cases
+    if ((x & m) == 0) {
+        return 0;
+    }
+
+    u64a mk, mp, mv, t;
+
+    x &= m; // clear irrelevant bits
+
+    mk = ~m << 1; // we will count 0's to right
+    for (u32 i = 0; i < 6; i++) {
+        mp = mk ^ (mk << 1);
+        mp ^= mp << 2;
+        mp ^= mp << 4;
+        mp ^= mp << 8;
+        mp ^= mp << 16;
+        mp ^= mp << 32;
+
+        mv = mp & m; // bits to move
+        m = (m ^ mv) | (mv >> (1 << i)); // compress m
+        t = x & mv;
+        x = (x ^ t) | (t >> (1 << i)); // compress x
+        mk = mk & ~mp;
+    }
+
+    return x;
+}
+
+static really_inline
+u32 expand32(u32 x, u32 m) {
+#if defined(__BMI2__)
+    // BMI2 has a single instruction for this operation.
+    return _pdep_u32(x, m);
+#endif
+
+    // Return zero quickly on trivial cases
+    if (!x || !m) {
+        return 0;
+    }
+
+    u32 m0, mk, mp, mv, t;
+    u32 array[5];
+
+    m0 = m; // save original mask
+    mk = ~m << 1; // we will count 0's to right
+
+    for (int i = 0; i < 5; i++) {
+        mp = mk ^ (mk << 1); // parallel suffix
+        mp = mp ^ (mp << 2);
+        mp = mp ^ (mp << 4);
+        mp = mp ^ (mp << 8);
+        mp = mp ^ (mp << 16);
+        mv = mp & m; // bits to move
+        array[i] = mv;
+        m = (m ^ mv) | (mv >> (1 << i)); // compress m
+        mk = mk & ~mp;
+    }
+
+    for (int i = 4; i >= 0; i--) {
+        mv = array[i];
+        t = x << (1 << i);
+        x = (x & ~mv) | (t & mv);
+    }
+
+    return x & m0; // clear out extraneous bits
+}
+
+static really_inline
+u64a expand64(u64a x, u64a m) {
+#if defined(ARCH_X86_64) && defined(__BMI2__)
+    // BMI2 has a single instruction for this operation.
+    return _pdep_u64(x, m);
+#endif
+
+    // Return zero quickly on trivial cases
+    if (!x || !m) {
+        return 0;
+    }
+
+    u64a m0, mk, mp, mv, t;
+    u64a array[6];
+
+    m0 = m; // save original mask
+    mk = ~m << 1; // we will count 0's to right
+
+    for (int i = 0; i < 6; i++) {
+        mp = mk ^ (mk << 1); // parallel suffix
+        mp = mp ^ (mp << 2);
+        mp = mp ^ (mp << 4);
+        mp = mp ^ (mp << 8);
+        mp = mp ^ (mp << 16);
+        mp = mp ^ (mp << 32);
+        mv = mp & m; // bits to move
+        array[i] = mv;
+        m = (m ^ mv) | (mv >> (1 << i)); // compress m
+        mk = mk & ~mp;
+    }
+
+    for (int i = 5; i >= 0; i--) {
+        mv = array[i];
+        t = x << (1 << i);
+        x = (x & ~mv) | (t & mv);
+    }
+
+    return x & m0; // clear out extraneous bits
+}
+
+
+/* returns the first set bit after begin (if not ~0U). If no bit is set after
+ * begin returns ~0U
+ */
+static really_inline
+u32 bf64_iterate(u64a bitfield, u32 begin) {
+    if (begin != ~0U) {
+        /* switch off all bits at or below begin. Note: not legal to shift by
+         * by size of the datatype or larger. */
+        assert(begin <= 63);
+        bitfield &= ~((2ULL << begin) - 1);
+    }
+
+    if (!bitfield) {
+        return ~0U;
+    }
+
+    return ctz64(bitfield);
+}
+
+static really_inline
+char bf64_set(u64a *bitfield, u32 i) {
+    assert(i < 64);
+    u64a mask = 1ULL << i;
+    char was_set = !!(*bitfield & mask);
+    *bitfield |= mask;
+
+    return was_set;
+}
+
+static really_inline
+void bf64_unset(u64a *bitfield, u32 i) {
+    assert(i < 64);
+    *bitfield &= ~(1ULL << i);
+}
+
+#endif // BITUTILS_H

src/util/boundary_reports.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef BOUNDARY_REPORTS_H
+#define BOUNDARY_REPORTS_H
+
+#include "ue2common.h"
+
+#include <set>
+#include <boost/core/noncopyable.hpp>
+
+namespace ue2 {
+
+struct BoundaryReports : boost::noncopyable {
+    std::set<ReportID> report_at_0; /* set of internal reports to fire
+                                     * unconditionally at offset 0 */
+    std::set<ReportID> report_at_0_eod; /* set of internal reports to fire
+                                         * unconditionally at offset 0 if it is
+                                         * eod */
+    std::set<ReportID> report_at_eod; /* set of internal reports to fire
+                                       * unconditionally at eod */
+};
+
+} // namespace ue2
+
+#endif

src/util/charreach.cpp

@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Class for representing character reachability.
+ *
+ * This is a simple (but hopefully fast) class for representing 8-bit character
+ * reachability, along with a bunch of useful operations.
+ */
+#include "ue2common.h"
+#include "charreach.h"
+#include "charreach_util.h"
+#include "compare.h"
+#include "unicode_def.h"
+
+#include <cassert>
+#include <string>
+
+namespace ue2 {
+
+/// Switch on the bits corresponding to the characters in \a s.
+void CharReach::set(const std::string &s) {
+    for (const auto &c : s) {
+        set(c);
+    }
+}
+
+/// Bitwise OR.
+CharReach CharReach::operator|(const CharReach &a) const {
+    CharReach cr(*this);
+    cr.bits |= a.bits;
+    return cr;
+}
+
+/// Bitwise OR-equals.
+void CharReach::operator|=(const CharReach &a) {
+    bits |= a.bits;
+}
+
+/// Bitwise AND.
+CharReach CharReach::operator&(const CharReach &a) const {
+    CharReach cr(*this);
+    cr.bits &= a.bits;
+    return cr;
+}
+
+/// Bitwise AND-equals.
+void CharReach::operator&=(const CharReach &a) {
+    bits &= a.bits;
+}
+
+/// Bitwise complement.
+CharReach CharReach::operator~(void) const {
+    CharReach cr(*this);
+    cr.flip();
+    return cr;
+}
+
+/// Bitwise XOR.
+CharReach CharReach::operator^(const CharReach &a) const {
+    CharReach cr(*this);
+    cr.bits ^= a.bits;
+    return cr;
+}
+
+/// Do we only contain bits representing alpha characters?
+bool CharReach::isAlpha() const {
+    if (none()) {
+        return false;
+    }
+    for (size_t i = find_first(); i != npos; i = find_next(i)) {
+        if (!ourisalpha((char)i)) {
+            return false;
+        }
+    }
+    return true;
+}
+
+/// Do we represent an uppercase/lowercase pair?
+bool CharReach::isCaselessChar() const {
+    if (count() != 2) {
+        return false;
+    }
+    size_t first = find_first();
+    size_t second = find_next(first);
+    assert(first != npos && second != npos);
+    return (char)first == mytoupper((char)second);
+}
+
+/// Do we represent a cheapskate caseless set?
+bool CharReach::isBit5Insensitive() const {
+    for (size_t i = find_first(); i != npos; i = find_next(i)) {
+        if (!test((char)i ^ 0x20)) {
+            return false;
+        }
+    }
+    return true;
+}
+
+/// Return a string containing the characters that are switched on.
+std::string CharReach::to_string() const {
+    std::string s;
+    for (size_t i = find_first(); i != npos; i = find_next(i)) {
+        s += (char)i;
+    }
+    return s;
+}
+
+/** \brief True iff there is a non-empty intersection between \a and \a b */
+bool overlaps(const CharReach &a, const CharReach &b) {
+    return (a & b).any();
+}
+
+/** \brief True iff \a small is a subset of \a big. */
+bool isSubsetOf(const CharReach &small, const CharReach &big) {
+    return small.isSubsetOf(big);
+}
+
+/// True if this character class is a subset of \a other.
+bool CharReach::isSubsetOf(const CharReach &other) const {
+    return (bits & other.bits) == bits;
+}
+
+void make_caseless(CharReach *cr) {
+    for (char c = 'A'; c <= 'Z'; c++) {
+        if (cr->test(c) || cr->test(mytolower(c))) {
+            cr->set(c);
+            cr->set(mytolower(c));
+        }
+    }
+}
+
+bool isutf8ascii(const CharReach &cr) {
+    return (cr & ~CharReach(0x0, 0x7f)).none();
+}
+
+bool isutf8start(const CharReach &cr) {
+    return (cr & CharReach(0x0, UTF_CONT_MAX)).none();
+}
+
+void fill_bitvector(const CharReach &cr, u8 *bits) {
+    assert(bits);
+    std::fill_n(bits, 32, 0);
+    for (size_t i = cr.find_first(); i != cr.npos; i = cr.find_next(i)) {
+        bits[i / 8U] |= (u8)1U << (i % 8U);
+    }
+}
+
+void make_and_cmp_mask(const CharReach &cr, u8 *and_mask, u8 *cmp_mask) {
+    u8 lo = 0xff;
+    u8 hi = 0;
+
+    for (size_t c = cr.find_first(); c != cr.npos; c = cr.find_next(c)) {
+        hi |= (u8)c;
+        lo &= (u8)c;
+    }
+
+    *and_mask = ~(lo ^ hi);
+    *cmp_mask = lo;
+}
+
+} // namespace ue2

src/util/charreach.h

@@ -0,0 +1,191 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Class for representing character reachability.
+ *
+ * This is a simple (but hopefully fast) class for representing 8-bit character
+ * reachability, along with a bunch of useful operations.
+ */
+
+#ifndef NG_CHARREACH_H
+#define NG_CHARREACH_H
+
+#include "ue2common.h"
+#include "util/bitfield.h"
+
+#include <string>
+
+namespace ue2 {
+
+class CharReach {
+private:
+    /// Underlying storage.
+    ue2::bitfield<256> bits;
+
+public:
+    static constexpr size_t npos = decltype(bits)::npos; //!< One past the max value.
+
+    /// Empty constructor.
+    CharReach() {}
+
+    /// Constructor for a character class containing a single char.
+    explicit CharReach(unsigned char c) { set(c); }
+
+    /// Constructor for a character class representing a contiguous range of
+    /// chars, inclusive.
+    CharReach(unsigned char from, unsigned char to) { setRange(from, to); }
+
+    /// Constructor for a character class based on the set of chars in a
+    /// string.
+    explicit CharReach(const std::string &str) { set(str); }
+
+    /// Returns total capacity.
+    static constexpr size_t size() { return npos; }
+
+    /// Returns a CharReach with complete reachability (a "dot").
+    static CharReach dot() { return CharReach(0, 255); }
+
+    /// Complete bitset equality.
+    bool operator==(const CharReach &a) const { return bits == a.bits; }
+
+    /// Inequality.
+    bool operator!=(const CharReach &a) const { return bits != a.bits; }
+
+    /// Ordering.
+    bool operator<(const CharReach &a) const { return bits < a.bits; }
+
+    /// Set all bits.
+    void setall() { bits.setall(); }
+
+    /// Clear all bits.
+    void clear() { bits.clear(); }
+
+    /// Clear bit N.
+    void clear(unsigned char n) { bits.clear(n); }
+
+    /// Set bit N.
+    void set(unsigned char n) { bits.set(n); }
+
+    /// Test bit N.
+    bool test(unsigned char n) const { return bits.test(n); }
+
+    /// Flip bit N.
+    void flip(unsigned char n) { bits.flip(n); }
+
+    /// Flip all bits.
+    void flip() { bits.flip(); }
+
+    // Switch on the bit in the range (from, to), inclusive.
+    void setRange(unsigned char from, unsigned char to) {
+        bits.set_range(from, to);
+    }
+
+    // Switch on the bits corresponding to the characters in \a s.
+    void set(const std::string &s);
+
+    /// Returns number of bits set on.
+    size_t count() const { return bits.count(); }
+
+    /// Are no bits set?
+    bool none() const { return bits.none(); }
+
+    /// Is any bit set?
+    bool any() const { return bits.any(); }
+
+    /// Are all bits set?
+    bool all() const { return bits.all(); }
+
+    /// Returns first bit set, or CharReach::npos if none set.
+    size_t find_first() const { return bits.find_first(); }
+
+    /// Returns last bit set, or CharReach::npos if none set.
+    size_t find_last() const { return bits.find_last(); }
+
+    /// Returns next bit set, or CharReach::npos if none set after n.
+    size_t find_next(size_t last) const { return bits.find_next(last); }
+
+    /// Returns (zero-based) N'th bit set, or CharReach::npos if fewer than
+    /// N + 1 bits are on.
+    size_t find_nth(size_t n) const { return bits.find_nth(n); }
+
+    /// Bitwise OR.
+    CharReach operator|(const CharReach &a) const;
+
+    /// Bitwise OR-equals.
+    void operator|=(const CharReach &a);
+
+    /// Bitwise AND.
+    CharReach operator&(const CharReach &a) const;
+
+    /// Bitwise AND-equals.
+    void operator&=(const CharReach &a);
+
+    /// Bitwise XOR.
+    CharReach operator^(const CharReach &a) const;
+
+    /// Bitwise complement.
+    CharReach operator~(void) const;
+
+    /// Do we only contain bits representing alpha characters?
+    bool isAlpha() const;
+
+    /// Do we represent an uppercase/lowercase pair?
+    bool isCaselessChar() const;
+
+    /// Do we represent a cheapskate caseless set?
+    bool isBit5Insensitive() const;
+
+    /// Return a string containing the characters that are switched on.
+    std::string to_string() const;
+
+    /// Hash of enabled bits.
+    size_t hash() const { return bits.hash(); }
+
+    /// True if this character class is a subset of \a other.
+    bool isSubsetOf(const CharReach &other) const;
+};
+
+/** \brief True iff there is a non-empty intersection between \a and \a b */
+bool overlaps(const CharReach &a, const CharReach &b);
+
+/** \brief True iff \a small is a subset of \a big. */
+bool isSubsetOf(const CharReach &small, const CharReach &big);
+
+bool isutf8ascii(const CharReach &cr);
+bool isutf8start(const CharReach &cr);
+
+/** \brief Boost-style hash free function. */
+static really_inline
+size_t hash_value(const CharReach &cr) {
+    return cr.hash();
+}
+
+} // namespace ue2
+
+#endif // NG_CHARREACH_H

src/util/charreach_util.h

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef CHARREACH_UTIL_H
+#define CHARREACH_UTIL_H
+
+#include <map>
+#include <set>
+
+#include "charreach.h"
+
+namespace ue2 {
+
+template<typename T>
+std::map<CharReach, std::set<T> >
+make_disjoint(const std::map<CharReach, std::set<T> > &in) {
+    using namespace std;
+
+    map<u8, set<T> > by_char;
+    for (typename map<CharReach, set<T> >::const_iterator it = in.begin();
+         it != in.end(); ++it) {
+        const CharReach &cr = it->first;
+        for (size_t j = cr.find_first(); j != CharReach::npos;
+             j = cr.find_next(j)) {
+            by_char[j].insert(it->second.begin(), it->second.end());
+        }
+    }
+
+    map<set<T>, CharReach> rev;
+    for (typename map<u8, set<T> >::const_iterator it = by_char.begin();
+         it != by_char.end(); ++it) {
+        rev[it->second].set(it->first);
+    }
+
+    map<CharReach, set<T> > out;
+    for (typename map<set<T>, CharReach>::const_iterator it = rev.begin();
+         it != rev.end(); ++it) {
+        assert(out.find(it->second) == out.end());
+        out[it->second] = it->first;
+    }
+
+    return out;
+}
+
+void make_caseless(CharReach *cr);
+
+/**
+ * \brief Fill a bitvector with the contents of the given CharReach.
+ *
+ * \a bits should point at an array of 32 bytes.
+ */
+void fill_bitvector(const CharReach &cr, u8 *bits);
+
+/**
+ * \brief Generate and and compare masks for checking the char reach.
+ *
+ * Any character c in cr will be result in (c & and_mask) == cmp_mask being true.
+ * Note: characters not in cr may also pass the and/cmp checks.
+ */
+void make_and_cmp_mask(const CharReach &cr, u8 *and_mask, u8 *cmp_mask);
+
+} // namespace ue2
+
+#endif

src/util/compare.h

@@ -0,0 +1,171 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef COMPARE_H
+#define COMPARE_H
+
+#include "unaligned.h"
+#include "ue2common.h"
+
+/* Our own definitions of tolower, toupper and isalpha are provided to prevent
+ * us from going out to libc for these tests. */
+
+static really_inline
+char myisupper(const char c) {
+    return ((c >= 'A') && (c <= 'Z'));
+}
+
+static really_inline
+char myislower(const char c) {
+    return ((c >= 'a') && (c <= 'z'));
+}
+
+static really_inline
+char mytolower(const char c) {
+    if (myisupper(c)) {
+        return c + 0x20;
+    }
+    return c;
+}
+
+static really_inline
+char mytoupper(const char c) {
+    if (myislower(c)) {
+        return c - 0x20;
+    }
+    return c;
+}
+
+/* this is a slightly warped definition of `alpha'. What we really
+ * mean is: does this character have different uppercase and lowercase forms?
+ */
+static really_inline char ourisalpha(const char c) {
+    return mytolower(c) != mytoupper(c);
+}
+
+static really_inline char ourisprint(const char c) {
+    return c >= 0x20 && c <= 0x7e;
+}
+
+// Paul Hsieh's SWAR toupper; used because it doesn't
+// matter whether we go toupper or tolower. We should
+// probably change the other one
+static really_inline
+u32 theirtoupper32(const u32 x) {
+    u32 b = 0x80808080ul | x;
+    u32 c = b - 0x61616161ul;
+    u32 d = ~(b - 0x7b7b7b7bul);
+    u32 e = (c & d) & (~x & 0x80808080ul);
+    return x - (e >> 2);
+}
+
+// 64-bit variant.
+static really_inline
+u64a theirtoupper64(const u64a x) {
+    u64a b = 0x8080808080808080ull | x;
+    u64a c = b - 0x6161616161616161ull;
+    u64a d = ~(b - 0x7b7b7b7b7b7b7b7bull);
+    u64a e = (c & d) & (~x & 0x8080808080808080ull);
+    u64a v = x - (e >> 2);
+    return v;
+}
+
+static really_inline
+int cmpNocaseNaive(const u8 *p1, const u8 *p2, size_t len) {
+    const u8 *pEnd = (const u8 *)p1 + len;
+    for (; p1 < pEnd; p1++, p2++)
+        if (mytolower(*p1) != mytolower(*p2))
+            return 1;
+    return 0;
+}
+
+static really_inline
+int cmpCaseNaive(const u8 *p1, const u8 *p2, size_t len) {
+    const u8 *pEnd = (const u8 *)p1 + len;
+    for (; p1 < pEnd; p1++, p2++)
+        if (*p1 != *p2)
+            return 1;
+    return 0;
+}
+
+#ifdef ARCH_64_BIT
+#  define CMP_T        u64a
+#  define ULOAD(x)     unaligned_load_u64a(x)
+#  define TOUPPER(x)   theirtoupper64(x)
+#else
+#  define CMP_T        u32
+#  define ULOAD(x)     unaligned_load_u32(x)
+#  define TOUPPER(x)   theirtoupper32(x)
+#endif
+
+#define CMP_SIZE sizeof(CMP_T)
+
+#if defined(ARCH_IA32)
+static UNUSED never_inline
+#else
+static really_inline
+#endif
+int cmpForward(const u8 *p1, const u8 *p2, size_t len, char nocase) {
+    if (len < CMP_SIZE) {
+        return nocase ? cmpNocaseNaive(p1, p2, len)
+                      : cmpCaseNaive(p1, p2, len);
+    }
+
+    const u8 *p1_end = p1 + len - CMP_SIZE;
+    const u8 *p2_end = p2 + len - CMP_SIZE;
+
+    if (nocase) { // Case-insensitive version.
+        for (; p1 < p1_end; p1 += CMP_SIZE, p2 += CMP_SIZE) {
+            if (TOUPPER(ULOAD(p1)) != TOUPPER(ULOAD(p2))) {
+                return 1;
+            }
+        }
+        if (TOUPPER(ULOAD(p1_end)) != TOUPPER(ULOAD(p2_end))) {
+            return 1;
+        }
+    } else { // Case-sensitive version.
+        for (; p1 < p1_end; p1 += CMP_SIZE, p2 += CMP_SIZE) {
+            if (ULOAD(p1) != ULOAD(p2)) {
+                return 1;
+            }
+        }
+        if (ULOAD(p1_end) != ULOAD(p2_end)) {
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
+#undef CMP_T
+#undef ULOAD
+#undef TOUPPER
+#undef CMP_SIZE
+
+#endif
+

src/util/compile_context.cpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Global compile context, describes compile environment.
+ */
+#include "compile_context.h"
+#include "grey.h"
+
+namespace ue2 {
+
+CompileContext::CompileContext(bool in_isStreaming, bool in_isVectored,
+                               const target_t &in_target_info,
+                               const Grey &in_grey)
+    : streaming(in_isStreaming || in_isVectored),
+      vectored(in_isVectored),
+      target_info(in_target_info),
+      grey(in_grey) {
+}
+
+} // namespace ue2

src/util/compile_context.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Global compile context, describes compile environment.
+ */
+
+#ifndef COMPILE_CONTEXT_H
+#define COMPILE_CONTEXT_H
+
+#include "target_info.h"
+#include "grey.h"
+
+namespace ue2 {
+
+/** \brief Structure for describing the compile environment: grey box settings,
+ * target arch, mode flags, etc. */
+struct CompileContext {
+    CompileContext(bool isStreaming, bool isVectored,
+                   const target_t &target_info, const Grey &grey);
+
+    const bool streaming; /* streaming or vectored mode */
+    const bool vectored;
+
+    /** \brief Target platform info. */
+    const target_t target_info;
+
+    /** \brief Greybox structure, allows tuning of all sorts of behaviour. */
+    const Grey grey;
+};
+
+} // namespace ue2
+
+#endif

src/util/compile_error.cpp

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "compile_error.h"
+
+using namespace std;
+
+namespace ue2 {
+
+CompileError::CompileError(const string &why)
+    : reason(why), hasIndex(false), index(0) {
+    assert(!why.empty());
+    assert(*why.rbegin() == '.');
+}
+
+CompileError::CompileError(unsigned int idx, const string &why)
+    : reason(why), hasIndex(true), index(idx) {
+    assert(!why.empty());
+    assert(*why.rbegin() == '.');
+}
+
+void CompileError::setExpressionIndex(u32 expr_index) {
+    hasIndex = true;
+    index = expr_index;
+}
+
+CompileError::~CompileError() {}
+
+ResourceLimitError::ResourceLimitError()
+    : CompileError("Resource limit exceeded.") {}
+
+ResourceLimitError::~ResourceLimitError() {}
+
+} // namespace ue2

src/util/compile_error.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UTIL_COMPILE_ERROR_H
+#define UTIL_COMPILE_ERROR_H
+
+#include <cassert>
+#include <stdexcept>
+#include <string>
+
+#include "ue2common.h"
+
+namespace ue2 {
+
+/** \brief Error thrown by the compiler, can reference a specific expression
+ * index. */
+class CompileError {
+public:
+    // Note: 'why' should describe why the error occurred and end with a
+    // full stop, but no line break.
+    explicit CompileError(const std::string &why);
+    CompileError(u32 index, const std::string &why);
+
+    virtual ~CompileError();
+
+    void setExpressionIndex(u32 index);
+
+    std::string reason; //!< Reason for the error
+    bool hasIndex; //!< Does it reference a specific expression?
+    u32 index; //!< The index of the expression referred to
+};
+
+/** \brief Error thrown by the compiler when an arbitrary resource limit (as
+ * specified in the grey box) is exceeded. */
+class ResourceLimitError : public CompileError {
+public:
+    ResourceLimitError();
+    ~ResourceLimitError() override;
+};
+
+} // namespace ue2
+
+#endif

src/util/container.h

@@ -0,0 +1,193 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Convenience template functions for containers.
+ */
+
+#ifndef UTIL_CONTAINER_H
+#define UTIL_CONTAINER_H
+
+#include <algorithm>
+#include <set>
+#include <utility>
+
+namespace ue2 {
+
+// Existence check for associative containers.
+template<typename C>
+bool contains(const C &container, const typename C::key_type &key) {
+    return container.find(key) != container.end();
+}
+
+template<typename C, typename It>
+bool contains_any_of(const C &container, const std::pair<It, It> &range) {
+    return std::find_first_of(range.first, range.second, container.begin(),
+                              container.end()) != range.second;
+}
+
+template<typename C, typename It>
+void insert(C *container, const std::pair<It, It> &range) {
+    container->insert(range.first, range.second);
+}
+
+template<typename C, typename It>
+void insert(C *container, typename C::iterator pos,
+            const std::pair<It, It> &range) {
+    container->insert(pos, range.first, range.second);
+}
+
+template<typename C, typename D>
+void insert(C *container, const D &donor) {
+    container->insert(donor.begin(), donor.end());
+}
+
+template<typename C, typename D>
+void insert(C *container, typename C::iterator pos, const D &donor) {
+    container->insert(pos, donor.begin(), donor.end());
+}
+
+/**
+ * \brief Constructs a vector from a range bounded by the given pair of iterators. */
+template <typename It>
+auto make_vector_from(const std::pair<It, It> &range)
+    -> std::vector<decltype(*range.first)> {
+    using T = decltype(*range.first);
+    return std::vector<T>(range.first, range.second);
+}
+
+/** \brief Returns a set containing the keys in the given associative
+ * container. */
+template <typename C>
+std::set<typename C::key_type> assoc_keys(const C &container) {
+    std::set<typename C::key_type> keys;
+    for (const auto &elem : container) {
+        keys.insert(elem.first);
+    }
+    return keys;
+}
+
+template<typename T>
+typename std::vector<T>::size_type byte_length(const std::vector<T> &vec) {
+    return vec.size() * sizeof(T);
+}
+
+template<typename OrderedContainer1, typename OrderedContainer2>
+bool is_subset_of(const OrderedContainer1 &small, const OrderedContainer2 &big) {
+    static_assert(std::is_same<typename OrderedContainer1::value_type,
+                               typename OrderedContainer2::value_type>::value,
+                  "Both containers should have the same value_type");
+    auto sit = small.begin();
+    auto bit = big.begin();
+    if (small.size() > big.size()) {
+        return false;
+    }
+
+    while (sit != small.end()) {
+        if (bit == big.end()) {
+            return false;
+        }
+
+        if (*sit == *bit) {
+            ++sit;
+            ++bit;
+            continue;
+        }
+        if (*bit < *sit) {
+            ++bit;
+            continue;
+        }
+
+        return false;
+    }
+    return true;
+}
+
+template<typename OrderedContainer1, typename OrderedContainer2>
+bool has_intersection(const OrderedContainer1 &a, const OrderedContainer2 &b) {
+    static_assert(std::is_same<typename OrderedContainer1::value_type,
+                               typename OrderedContainer2::value_type>::value,
+                  "Both containers should have the same value_type");
+    auto ait = a.begin();
+    auto bit = b.begin();
+    while (ait != a.end() && bit != b.end()) {
+        if (*ait == *bit) {
+            return true;
+        }
+
+        if (*ait < *bit) {
+            ++ait;
+        } else {
+            ++bit;
+        }
+    }
+
+    return false;
+}
+
+/**
+ * \brief Erase the elements (by value) in the donor container from the given
+ * container.
+ */
+template<typename C, typename D>
+void erase_all(C *container, const D &donor) {
+    for (const auto &elem : donor) {
+        container->erase(elem);
+    }
+}
+
+} // namespace ue2
+
+#ifdef DUMP_SUPPORT
+
+#include <sstream>
+#include <string>
+
+namespace ue2 {
+
+/**
+ * \brief Dump a container of stream-printable objects into a comma-separated
+ * list in a string.
+ */
+template<class C>
+std::string as_string_list(const C &c) {
+    std::ostringstream oss;
+    for (auto it = c.begin(); it != c.end(); ++it) {
+        if (it != c.begin()) {
+            oss << ", ";
+        }
+        oss << *it;
+    }
+    return oss.str();
+}
+
+} // namespace ue2
+
+#endif // DUMP_SUPPORT
+
+#endif // UTIL_CONTAINER_H

src/util/cpuid_flags.c

@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "cpuid_flags.h"
+#include "ue2common.h"
+#include "hs_compile.h" // for HS_MODE_ flags
+#include "hs_internal.h"
+
+#ifndef _WIN32
+#include <cpuid.h>
+#endif
+
+// ECX
+#define SSE3 (1 << 0)
+#define SSSE3 (1 << 9)
+#define SSE4_1 (1 << 19)
+#define SSE4_2 (1 << 20)
+
+// EDX
+#define SSE (1 << 25)
+#define SSE2 (1 << 25)
+#define HTT (1 << 28)
+
+// Structured Extended Feature Flags Enumeration Leaf ECX values
+#define BMI (1 << 3)
+#define AVX2 (1 << 5)
+#define BMI2 (1 << 8)
+
+static __inline
+void cpuid(unsigned int op, unsigned int leaf, unsigned int *eax,
+           unsigned int *ebx, unsigned int *ecx, unsigned int *edx) {
+#ifndef _WIN32
+    __cpuid_count(op, leaf, *eax, *ebx, *ecx, *edx);
+#else
+    unsigned int a[4];
+    __cpuidex(a, op, leaf);
+    *eax = a[0];
+    *ebx = a[1];
+    *ecx = a[2];
+    *edx = a[3];
+#endif
+}
+
+u64a cpuid_flags(void) {
+    unsigned int eax, ebx, ecx, edx;
+    u64a cap = 0;
+
+    // version info
+    cpuid(1, 0, &eax, &ebx, &ecx, &edx);
+
+    /* ECX and EDX contain capability flags */
+
+    ecx = 0;
+    cpuid(7, 0, &eax, &ebx, &ecx, &edx);
+
+    if (ebx & AVX2) {
+        cap |= HS_CPU_FEATURES_AVX2;
+    }
+
+#if !defined(__AVX2__)
+    cap &= ~HS_CPU_FEATURES_AVX2;
+#endif
+
+    return cap;
+}
+
+struct family_id {
+    u32 full_family;
+    u32 full_model;
+    u32 tune;
+};
+
+/* from table 35-1 of the Intel 64 and IA32 Arch. Software Developer's Manual
+ * and "Intel Architecture and Processor Identification With CPUID Model and
+ * Family Numbers" */
+static const struct family_id known_microarch[] = {
+    { 0x6, 0x37, HS_TUNE_FAMILY_SLM }, /* baytrail */
+    { 0x6, 0x4D, HS_TUNE_FAMILY_SLM }, /* avoton, rangley */
+
+    { 0x6, 0x3C, HS_TUNE_FAMILY_HSW }, /* haswell */
+    { 0x6, 0x45, HS_TUNE_FAMILY_HSW }, /* haswell */
+    { 0x6, 0x46, HS_TUNE_FAMILY_HSW }, /* haswell */
+    { 0x6, 0x3F, HS_TUNE_FAMILY_HSW }, /* haswell */
+
+    { 0x6, 0x3E, HS_TUNE_FAMILY_IVB }, /* ivybridge */
+    { 0x6, 0x3A, HS_TUNE_FAMILY_IVB }, /* ivybridge */
+
+    { 0x6, 0x2A, HS_TUNE_FAMILY_SNB }, /* sandybridge */
+    { 0x6, 0x2D, HS_TUNE_FAMILY_SNB }, /* sandybridge */
+
+    { 0x6, 0x3D, HS_TUNE_FAMILY_BDW }, /* broadwell Core-M */
+    { 0x6, 0x4F, HS_TUNE_FAMILY_BDW }, /* broadwell xeon */
+    { 0x6, 0x56, HS_TUNE_FAMILY_BDW }, /* broadwell xeon-d */
+
+//    { 0x6, 0x25, HS_TUNE_FAMILY_GENERIC }, /* westmere */
+//    { 0x6, 0x2C, HS_TUNE_FAMILY_GENERIC }, /* westmere */
+//    { 0x6, 0x2F, HS_TUNE_FAMILY_GENERIC }, /* westmere */
+
+//    { 0x6, 0x1E, HS_TUNE_FAMILY_GENERIC }, /* nehalem */
+//    { 0x6, 0x1A, HS_TUNE_FAMILY_GENERIC }, /* nehalem */
+//    { 0x6, 0x2E, HS_TUNE_FAMILY_GENERIC }, /* nehalem */
+
+//    { 0x6, 0x17, HS_TUNE_FAMILY_GENERIC }, /* penryn */
+//    { 0x6, 0x1D, HS_TUNE_FAMILY_GENERIC }, /* penryn */
+
+};
+
+#ifdef DUMP_SUPPORT
+static UNUSED
+const char *dumpTune(u32 tune) {
+#define T_CASE(x) case x: return #x;
+    switch (tune) {
+        T_CASE(HS_TUNE_FAMILY_SLM);
+        T_CASE(HS_TUNE_FAMILY_HSW);
+        T_CASE(HS_TUNE_FAMILY_SNB);
+        T_CASE(HS_TUNE_FAMILY_IVB);
+        T_CASE(HS_TUNE_FAMILY_BDW);
+    }
+#undef T_CASE
+    return "unknown";
+}
+#endif
+
+u32 cpuid_tune(void) {
+    unsigned int eax, ebx, ecx, edx;
+
+    cpuid(1, 0, &eax, &ebx, &ecx, &edx);
+
+    u32 family = (eax >> 8) & 0xf;
+    u32 model = 0;
+
+    if (family == 0x6 || family == 0xf) {
+        model = ((eax >> 4) & 0xf) | ((eax >> 12) & 0xf0);
+    } else {
+        model = (eax >> 4) & 0xf;
+    }
+
+    DEBUG_PRINTF("family = %xh model = %xh\n", family, model);
+    for (u32 i = 0; i < ARRAY_LENGTH(known_microarch); i++) {
+        if (family != known_microarch[i].full_family) {
+            continue;
+        }
+
+        if (model != known_microarch[i].full_model) {
+            continue;
+        }
+
+        u32 tune = known_microarch[i].tune;
+        DEBUG_PRINTF("found tune flag %s\n", dumpTune(tune) );
+        return tune;
+    }
+
+    return HS_TUNE_FAMILY_GENERIC;
+}

src/util/cpuid_flags.h

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef CPUID_H_53FFCB14B257C2
+#define CPUID_H_53FFCB14B257C2
+
+#include "ue2common.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/* returns HS_CPU_FEATURES_* mask.  */
+u64a cpuid_flags(void);
+
+u32 cpuid_tune(void);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* CPUID_H_53FFCB14B257C2 */
+

src/util/depth.cpp

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Data types used to represent depth quantities.
+ */
+#include "depth.h"
+#include "ue2common.h"
+
+#include <algorithm> // std::min, std::max
+
+namespace ue2 {
+
+DepthMinMax unionDepthMinMax(const DepthMinMax &a, const DepthMinMax &b) {
+    DepthMinMax rv;
+
+    if (a.min.is_unreachable()) {
+        rv.min = b.min;
+    } else if (b.min.is_unreachable()) {
+        rv.min = a.min;
+    } else {
+        rv.min = std::min(a.min, b.min);
+    }
+
+    if (a.max.is_infinite() || b.max.is_infinite()) {
+        rv.max = depth::infinity();
+    } else if (a.max.is_unreachable()) {
+        rv.max = b.max;
+    } else if (b.max.is_unreachable()) {
+        rv.max = a.max;
+    } else {
+        rv.max = std::max(a.max, b.max);
+    }
+
+    return rv;
+}
+
+} // namespace ue2
+
+#ifdef DUMP_SUPPORT
+
+#include <sstream>
+#include <string>
+
+namespace ue2 {
+
+std::string depth::str() const {
+    if (is_unreachable()) {
+        return "unr";
+    } else if (is_infinite()) {
+        return "inf";
+    }
+    std::ostringstream oss;
+    oss << val;
+    return oss.str();
+}
+
+std::string DepthMinMax::str() const {
+    std::ostringstream oss;
+    oss << "[" << min.str() << "," << max.str() << "]";
+    return oss.str();
+}
+
+} // namespace ue2
+
+#endif // DUMP_SUPPORT

src/util/depth.h

@@ -0,0 +1,260 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Data types used to represent depth quantities.
+ */
+
+#ifndef DEPTH_H
+#define DEPTH_H
+
+#include "ue2common.h"
+
+#ifdef DUMP_SUPPORT
+#include <string>
+#endif
+
+#include <boost/functional/hash/hash_fwd.hpp>
+
+namespace ue2 {
+
+/**
+ * \brief Exception thrown if a depth operation overflows.
+ */
+struct DepthOverflowError {};
+
+/**
+ * \brief Type used to represent depth information; value is either a count,
+ * or the special values "infinity" and "unreachable".
+ */
+class depth {
+public:
+    depth() : val(val_unreachable) {}
+
+    depth(u32 v) : val(v) {
+        if (v > max_value()) {
+            DEBUG_PRINTF("depth %u too large to represent!\n", v);
+            throw DepthOverflowError();
+        }
+    }
+
+    static depth unreachable() {
+        depth d;
+        d.val = val_unreachable;
+        return d;
+    }
+
+    static depth infinity() {
+        depth d;
+        d.val = val_infinity;
+        return d;
+    }
+
+    /** \brief Returns the max finite value representable as a depth. */
+    static constexpr u32 max_value() { return val_infinity - 1; }
+
+    bool is_finite() const { return val < val_infinity; }
+    bool is_infinite() const { return val == val_infinity; }
+    bool is_unreachable() const { return val == val_unreachable; }
+    bool is_reachable() const { return !is_unreachable(); }
+
+    /** \brief Convert a finite depth to an integer. */
+    operator u32() const {
+        if (!is_finite()) {
+            throw DepthOverflowError();
+        }
+        return val;
+    }
+
+    bool operator<(const depth &d) const { return val < d.val; }
+    bool operator>(const depth &d) const { return val > d.val; }
+    bool operator<=(const depth &d) const { return val <= d.val; }
+    bool operator>=(const depth &d) const { return val >= d.val; }
+    bool operator==(const depth &d) const { return val == d.val; }
+    bool operator!=(const depth &d) const { return val != d.val; }
+
+    // The following comparison operators exist for use against integer types
+    // that are bigger than what we can safely convert to depth (such as those
+    // in extparam).
+
+    bool operator<(u64a d) const {
+        if (!is_finite()) {
+            return false;
+        }
+        return val < d;
+    }
+    bool operator<=(u64a d) const {
+        if (!is_finite()) {
+            return false;
+        }
+        return val <= d;
+    }
+    bool operator==(u64a d) const {
+        if (!is_finite()) {
+            return false;
+        }
+        return val == d;
+    }
+    bool operator>(u64a d) const { return !(*this <= d); }
+    bool operator>=(u64a d) const { return !(*this < d); }
+    bool operator!=(u64a d) const { return !(*this == d); }
+
+    depth operator+(const depth &d) const {
+        if (is_unreachable() || d.is_unreachable()) {
+            return unreachable();
+        }
+        if (is_infinite() || d.is_infinite()) {
+            return infinity();
+        }
+
+        u64a rv = val + d.val;
+        if (rv >= val_infinity) {
+            DEBUG_PRINTF("depth %llu too large to represent!\n", rv);
+            throw DepthOverflowError();
+        }
+
+        return depth((u32)rv);
+    }
+
+    depth &operator+=(const depth &d) {
+        depth rv = *this + d;
+        *this = rv;
+        return *this;
+    }
+
+    depth operator-(const depth &d) const {
+        if (!d.is_finite()) {
+            throw DepthOverflowError();
+        }
+
+        if (is_unreachable()) {
+            return unreachable();
+        }
+        if (is_infinite()) {
+            return infinity();
+        }
+
+        if (val < d.val) {
+            throw DepthOverflowError();
+        }
+
+        u32 rv = val - d.val;
+        return depth(rv);
+    }
+
+    depth &operator-=(const depth &d) {
+        depth rv = *this - d;
+        *this = rv;
+        return *this;
+    }
+
+    depth operator+(s32 d) const {
+        if (is_unreachable()) {
+            return unreachable();
+        }
+        if (is_infinite()) {
+            return infinity();
+        }
+
+        s64a rv = val + d;
+        if (rv < 0 || (u64a)rv >= val_infinity) {
+            DEBUG_PRINTF("depth %llu too large to represent!\n", rv);
+            throw DepthOverflowError();
+        }
+
+        return depth((u32)rv);
+    }
+
+    depth operator+=(s32 d) {
+        depth rv = *this + d;
+        *this = rv;
+        return *this;
+    }
+
+#ifdef DUMP_SUPPORT
+    /** \brief Render as a string, useful for debugging. */
+    std::string str() const;
+#endif
+
+    friend size_t hash_value(const depth &d) {
+        return d.val;
+    }
+
+private:
+    static constexpr u32 val_infinity = (1u << 31) - 1;
+    static constexpr u32 val_unreachable = 1u << 31;
+
+    u32 val;
+};
+
+/**
+ * \brief Encapsulates a min/max pair.
+ */
+struct DepthMinMax {
+    depth min;
+    depth max;
+
+    DepthMinMax() : min(depth::infinity()), max(depth(0)) {}
+    DepthMinMax(const depth &mn, const depth &mx) : min(mn), max(mx) {}
+
+    bool operator<(const DepthMinMax &b) const {
+        if (min != b.min) {
+            return min < b.min;
+        }
+        return max < b.max;
+    }
+
+    bool operator==(const DepthMinMax &b) const {
+        return min == b.min && max == b.max;
+    }
+
+    bool operator!=(const DepthMinMax &b) const {
+        return !(*this == b);
+    }
+
+#ifdef DUMP_SUPPORT
+    /** \brief Render as a string, useful for debugging. */
+    std::string str() const;
+#endif
+};
+
+inline size_t hash_value(const DepthMinMax &d) {
+    size_t val = 0;
+    boost::hash_combine(val, d.min);
+    boost::hash_combine(val, d.max);
+    return val;
+}
+
+/**
+ * \brief Merge two DepthMinMax values together to produce their union.
+ */
+DepthMinMax unionDepthMinMax(const DepthMinMax &a, const DepthMinMax &b);
+
+} // namespace ue2
+
+#endif // DEPTH_H

src/util/determinise.h

@@ -0,0 +1,200 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief DFA determinisation code.
+ */
+
+#ifndef DETERMINISE_H
+#define DETERMINISE_H
+
+#include "nfagraph/ng_holder.h"
+#include "charreach.h"
+#include "container.h"
+#include "ue2common.h"
+
+#include <array>
+#include <algorithm>
+#include <vector>
+
+namespace ue2 {
+
+#define DETERMINISE_RESERVE_SIZE 10
+
+/* Automaton details:
+ *
+ * const vector<StateSet> initial()
+ *     returns initial states to start determinising from. StateSets in the
+ *     initial() vector will given consecutive ids starting from 1, in the order
+ *     that they appear.
+ *
+ *  void reports(StateSet s, flat_set<ReportID> *out)
+ *     fills out with any reports that need to be raised for stateset.
+ *
+ *  void reportsEod(StateSet s, flat_set<ReportID> *out)
+ *     fills out with any reports that need to be raised for stateset at EOD.
+ *
+ *  void transition(const StateSet &in, StateSet *next)
+ *     fills the next array such next[i] is the stateset that in transitions to
+ *     on seeing symbol i (i is in the compressed alphabet of the automaton).
+ *
+ *  u16 alphasize
+ *     size of the compressed alphabet
+ */
+
+/** \brief determinises some sort of nfa
+ *  \param n the automaton to determinise
+ *  \param dstates_out output dfa states
+ *  \param state_limit limit on the number of dfa states to construct
+ *  \param statesets_out a mapping from DFA state to the set of NFA states in
+ *         the automaton
+ *  \return zero on success
+ */
+template<class Auto, class ds>
+never_inline
+int determinise(Auto &n, std::vector<ds> &dstates_out, dstate_id_t state_limit,
+                std::vector<typename Auto::StateSet> *statesets_out = nullptr) {
+    DEBUG_PRINTF("the determinator\n");
+    typedef typename Auto::StateSet StateSet;
+    typedef typename Auto::StateMap DstateIdMap;
+    DstateIdMap dstate_ids;
+    std::vector<StateSet> statesets;
+
+    const size_t alphabet_size = n.alphasize;
+
+    std::vector<ds> dstates;
+    dstates.reserve(DETERMINISE_RESERVE_SIZE);
+    statesets.reserve(DETERMINISE_RESERVE_SIZE);
+
+    dstate_ids[n.dead] = DEAD_STATE;
+    dstates.push_back(ds(alphabet_size));
+    std::fill_n(dstates[0].next.begin(), alphabet_size, DEAD_STATE);
+
+    statesets.push_back(n.dead);
+
+    const std::vector<StateSet> &init = n.initial();
+    for (u32 i = 0; i < init.size(); i++) {
+        statesets.push_back(init[i]);
+        assert(!contains(dstate_ids, init[i]));
+        dstate_ids[init[i]] = dstates.size();
+        dstates.push_back(ds(alphabet_size));
+    }
+
+    std::vector<StateSet> succs(alphabet_size, n.dead);
+    for (dstate_id_t curr_id = DEAD_STATE; curr_id < dstates.size();
+         curr_id++) {
+        StateSet &curr = statesets[curr_id];
+
+        DEBUG_PRINTF("curr: %hu\n", curr_id);
+
+        /* fill in accepts */
+        n.reports(curr, dstates[curr_id].reports);
+        n.reportsEod(curr, dstates[curr_id].reports_eod);
+
+        if (!dstates[curr_id].reports.empty()) {
+            DEBUG_PRINTF("curr: %hu: is accept\n", curr_id);
+        }
+
+        if (!dstates[curr_id].reports.empty()) {
+            /* only external reports set ekeys */
+            if (n.canPrune(dstates[curr_id].reports)) {
+                /* we only transition to dead on characters, TOPs leave us
+                 * alone */
+                std::fill_n(dstates[curr_id].next.begin(), alphabet_size,
+                            DEAD_STATE);
+                dstates[curr_id].next[n.alpha[TOP]] = curr_id;
+                continue;
+            }
+        }
+
+        /* fill in successor states */
+        n.transition(curr, &succs[0]);
+        for (symbol_t s = 0; s < n.alphasize; s++) {
+            dstate_id_t succ_id;
+            if (s && succs[s] == succs[s - 1]) {
+                succ_id = dstates[curr_id].next[s - 1];
+            } else {
+                typename DstateIdMap::const_iterator dstate_id_iter;
+                dstate_id_iter = dstate_ids.find(succs[s]);
+
+                if (dstate_id_iter != dstate_ids.end()) {
+                    succ_id = dstate_id_iter->second;
+
+                    if (succ_id > curr_id && !dstates[succ_id].daddy
+                        && n.unalpha[s] < N_CHARS) {
+                        dstates[succ_id].daddy = curr_id;
+                    }
+                } else {
+                    statesets.push_back(succs[s]);
+                    succ_id = dstates.size();
+                    dstate_ids[succs[s]] = succ_id;
+                    dstates.push_back(ds(alphabet_size));
+                    dstates.back().daddy = n.unalpha[s] < N_CHARS ? curr_id : 0;
+                }
+
+                DEBUG_PRINTF("-->%hu on %02hx\n", succ_id, n.unalpha[s]);
+            }
+
+            if (succ_id >= state_limit) {
+                DEBUG_PRINTF("succ_id %hu >= state_limit %hu\n",
+                             succ_id, state_limit);
+                return -2;
+            }
+
+            dstates[curr_id].next[s] = succ_id;
+        }
+    }
+
+    dstates_out = dstates;
+    if (statesets_out) {
+        statesets_out->swap(statesets);
+    }
+    DEBUG_PRINTF("ok\n");
+    return 0;
+}
+
+static inline
+std::vector<CharReach> populateCR(const NGHolder &g,
+                                  const std::vector<NFAVertex> &v_by_index,
+                                  const std::array<u16, ALPHABET_SIZE> &alpha) {
+    std::vector<CharReach> cr_by_index(v_by_index.size());
+
+    for (size_t i = 0; i < v_by_index.size(); i++) {
+        const CharReach &cr = g[v_by_index[i]].char_reach;
+        CharReach &cr_out = cr_by_index[i];
+        for (size_t s = cr.find_first(); s != cr.npos; s = cr.find_next(s)) {
+            cr_out.set(alpha[s]);
+        }
+    }
+
+    return cr_by_index;
+}
+
+} // namespace ue2
+
+#endif

src/util/dump_charclass.cpp

@@ -0,0 +1,258 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Dump code for character classes (expressed as CharReach objects).
+ */
+
+#include "config.h"
+
+// Everything in this file is dump code
+#if defined(DUMP_SUPPORT)
+
+#include "charreach.h"
+#include "dump_charclass.h"
+
+#include <cctype>
+#include <iomanip>
+#include <ostream>
+#include <sstream>
+
+using std::string;
+using std::ostream;
+
+namespace ue2 {
+
+static
+void describeChar(ostream &os, char c, enum cc_output_t out_type) {
+    // these characters must always be escaped
+    static const string escaped("^-[].");
+    static const string dot_single_escaped("\"\'");
+
+    const string backslash((out_type == CC_OUT_DOT ? 2 : 1), '\\');
+
+    if (isgraph(c) && c != '\\') {
+        if (escaped.find(c) != string::npos) {
+            os << backslash << c;
+        } else if (out_type == CC_OUT_DOT
+                   && dot_single_escaped.find(c) != string::npos) {
+            os << '\\' << c;
+        } else {
+            os << c;
+        }
+    } else if (c == 0x09) {
+        os << backslash << 't';
+    } else if (c == 0x0a) {
+        os << backslash << 'n';
+    } else if (c == 0x0d) {
+        os << backslash << 'r';
+    } else {
+        os << backslash << 'x' << std::hex << std::setw(2)
+           << std::setfill('0') << (unsigned)(c & 0xff) << std::dec;
+    }
+}
+
+static
+void describeRange(ostream &os, unsigned char c1, unsigned char c2, enum cc_output_t out_type) {
+    assert(c1 <= c2);
+    if (c1 == c2) {
+        describeChar(os, (char)c1, out_type);
+    }
+    else if (c2 - c1 < 4) {
+        // render as individual chars
+        do {
+            describeChar(os, (char)c1, out_type);
+        } while (c1++ != c2);
+    } else {
+        // range
+        describeChar(os, (char)c1, out_type);
+        os << '-';
+        describeChar(os, (char)c2, out_type);
+    }
+}
+
+static
+bool extractMnemonic(ostream &os, CharReach &cr, enum cc_output_t out_type) {
+    const string backslash((out_type == CC_OUT_DOT ? 2 : 1), '\\');
+
+    // \w (word characters: any letter, digit, or underscore)
+    static const CharReach words(string("_0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWYXZ"));
+    if (words == (cr & words)) {
+        cr &= ~words;
+        os << backslash << 'w';
+        return true;
+    }
+
+    // \d (digits)
+    static const CharReach digits(string("0123456789"));
+    if (digits == (cr & digits)) {
+        cr &= ~digits;
+        os << backslash << 'd';
+        return true;
+    }
+
+    // \s (whitespace)
+    static const CharReach whitespace(string("\x09\x0a\x0b\x0c\x0d\x20", 6));
+    if (whitespace == (cr & whitespace)) {
+        cr &= ~whitespace;
+        os << backslash << 's';
+        return true;
+    }
+
+    return false;
+}
+
+static
+bool isContiguous(const CharReach &cr, size_t &first, size_t &last) {
+    first = cr.find_first();
+    size_t c = first;
+    while (c != CharReach::npos) {
+        last = c;
+        c = cr.find_next(c);
+        if (c == CharReach::npos) {
+            break;
+        }
+        if (c != last + 1) {
+            return false;
+        }
+    }
+    return true;
+}
+
+static
+size_t describeClassInt(ostream &os, const CharReach &incr, size_t maxLength,
+        enum cc_output_t out_type) {
+
+    // Approx size of output
+    size_t i = 0;
+    // one we can break
+    CharReach cr(incr);
+
+    // If we can be rendered as a single range, do it
+    size_t first = 0, last = 0;
+    if (isContiguous(cr, first, last)) {
+        describeRange(os, first, last, out_type);
+        i = 2;
+        return i;
+    }
+
+    // Extract any mnemonics
+    while (extractMnemonic(os, cr, out_type)) {
+        if (++i == maxLength) {
+            os << "...]";
+            return maxLength;
+        }
+    }
+
+    if (cr.none()) {
+        // all mnemonics, all the time
+        return i;
+    }
+
+    // Render charclass as a series of ranges
+    size_t c_start = cr.find_first();
+    size_t c = c_start, c_last = 0;
+    while (c != CharReach::npos) {
+        c_last = c;
+        c = cr.find_next(c);
+        if (c != c_last + 1 || c_last == 0xff) {
+            describeRange(os, c_start, c_last, out_type);
+            c_start = c;
+            if (++i == maxLength && c != CharReach::npos) {
+                os << "...]";
+                return maxLength;
+            }
+        }
+    }
+    return i;
+}
+
+////
+//// Functions exported outside this unit.
+////
+
+// C++ iostreams interface
+void describeClass(ostream &os, const CharReach &incr, size_t maxLength,
+                   enum cc_output_t out_type) {
+    if (incr.all()) {
+        os << "<any>";
+        return;
+    }
+
+    if (incr.none()) {
+        os << "<empty>";
+        return;
+    }
+
+    if (incr.count() == 1) {
+        describeChar(os, (char)incr.find_first(), out_type);
+        return;
+    }
+    if ((~incr).count() == 1) {
+        os << "[^";
+        describeChar(os, (char)(~incr).find_first(), out_type);
+        os << ']';
+        return;
+    }
+
+    // build up a normal string and a negated one, and see which is shorter
+    std::ostringstream out;
+    int out_count = describeClassInt(out, incr, maxLength, out_type);
+
+    std::ostringstream neg;
+    UNUSED int neg_count = describeClassInt(neg, ~incr, maxLength, out_type);
+
+    if (out.tellp() <= neg.tellp()) {
+        if (out_count > 1) {
+            os << '[' << out.str() << ']';
+        } else {
+            os << out.str();
+        }
+    } else {
+        // TODO: negated single mnemonics
+        os << "[^" << neg.str() << ']';
+    }
+}
+
+// Version that returns a string, for convenience.
+string describeClass(const CharReach &cr, size_t maxLength,
+                     enum cc_output_t out_type) {
+    std::ostringstream oss;
+    describeClass(oss, cr, maxLength, out_type);
+    return oss.str();
+}
+
+// C stdio wrapper
+void describeClass(FILE *f, const CharReach &cr, size_t maxLength,
+                   enum cc_output_t out_type) {
+    fprintf(f, "%s", describeClass(cr, maxLength, out_type).c_str());
+}
+
+} // namespace ue2
+
+#endif // DUMP_SUPPORT

src/util/dump_charclass.h

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Dump code for character classes (expressed as CharReach objects).
+ */
+
+#ifndef DUMP_CHARCLASS_H
+#define DUMP_CHARCLASS_H
+
+#include "ue2common.h"
+
+#include <cstdio>
+#include <ostream>
+#include <string>
+
+namespace ue2 {
+
+enum cc_output_t {
+    CC_OUT_TEXT, //!< unescaped text output
+    CC_OUT_DOT   //!< escaped DOT label output
+};
+
+class CharReach;
+
+void describeClass(std::ostream &os, const CharReach &cr, size_t maxLength,
+                   enum cc_output_t out_type);
+
+std::string describeClass(const CharReach &cr, size_t maxLength = 16,
+                          enum cc_output_t out_type = CC_OUT_TEXT);
+
+void describeClass(FILE *f, const CharReach &cr, size_t maxLength,
+                   enum cc_output_t out_type);
+
+} // namespace ue2
+
+#endif // DUMP_CHARCLASS_H

src/util/dump_mask.cpp

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Dump code for bitmasks.
+ *
+ * Note that these functions are only emitted in builds with DUMP_SUPPORT.
+ */
+
+#include "config.h"
+
+#ifdef DUMP_SUPPORT
+
+#include "ue2common.h"
+#include "util/dump_mask.h"
+
+#include <string>
+
+namespace ue2 {
+
+std::string dumpMask(const u8 *mask, size_t len) {
+    std::string s;
+    s.reserve(len + len / 8);
+
+    for (size_t i = 0; i < len; i++) {
+        if ((i % 8) == 0 && i != 0) {
+            s.push_back(' ');
+        }
+
+        s.push_back((mask[i / 8] >> (i % 8)) & 0x1 ? '1' : '0');
+    }
+
+    return s;
+}
+
+} // namespace ue2
+
+#endif // DUMP_SUPPORT

src/util/dump_mask.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Dump code for bitmasks.
+ *
+ * Note that these functions are only emitted in builds with DUMP_SUPPORT.
+ */
+
+#ifndef DUMP_MASK_H
+#define DUMP_MASK_H
+
+#ifdef DUMP_SUPPORT
+
+#include "ue2common.h"
+#include <string>
+
+namespace ue2 {
+
+/**
+ * Returns a representation of the given mask in binary, as a string of 1s and
+ * 0s.
+ */
+std::string dumpMask(const u8 *mask, size_t len);
+
+} // namespace ue2
+
+#endif // DUMP_SUPPORT
+
+#endif // DUMP_MASK_H

src/util/exhaust.h

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Inline functions for manipulating exhaustion vector.
+ */
+
+#ifndef EXHAUST_H
+#define EXHAUST_H
+
+#include "rose/rose_internal.h"
+#include "ue2common.h"
+#include <string.h>
+
+/** \brief Sentinel value meaning no further exhaustion keys. */
+#define END_EXHAUST (~(u32)0)
+
+/** \brief Test whether the given key (\a eoff) is set in the exhaustion vector
+ * \a evec. */
+static really_inline
+int isExhausted(const char *evec, u32 eoff) {
+    DEBUG_PRINTF("checking exhaustion %p %u\n", evec, eoff);
+    return eoff != END_EXHAUST && (evec[eoff >> 3] & (1 << (eoff % 8)));
+}
+
+/** \brief Returns 1 if all exhaustion keys in the bitvector are on. */
+static really_inline
+int isAllExhausted(const struct RoseEngine *t, const char *evec_in) {
+    if (!t->canExhaust) {
+        return 0; /* pattern set is inexhaustible */
+    }
+
+    const u8 *evec = (const u8 *)evec_in;
+
+    u32 whole_bytes = t->ekeyCount / 8;
+    for (u32 i = 0; i < whole_bytes; i++) {
+        if (evec[i] != 0xff) {
+            DEBUG_PRINTF("unexhausted pattern in byte %u\n", i);
+            return 0;
+        }
+    }
+
+    u32 rem = t->ekeyCount % 8;
+    if (t->ekeyCount % 8) {
+        u8 mask = (1 << rem) - 1;
+        if (evec[whole_bytes] != (char)mask) {
+            DEBUG_PRINTF("unexhausted pattern (%hhu) in final byte\n", mask);
+            return 0;
+        }
+    }
+
+    DEBUG_PRINTF("pattern set is exhausted\n");
+    return 1;
+}
+
+/** \brief Mark key \a eoff on in the exhaustion vector. */
+static really_inline
+void markAsMatched(char *evec, u32 eoff) {
+    if (eoff != END_EXHAUST) {
+        DEBUG_PRINTF("marking as exhausted key %u\n", eoff);
+        evec[eoff >> 3] |= 1 << (eoff % 8);
+    }
+}
+
+/** \brief Clear all keys in the exhaustion vector. */
+static really_inline
+void clearEvec(char *ev, const struct RoseEngine *t) {
+    size_t size = (t->ekeyCount + 7) / 8;
+    DEBUG_PRINTF("clearing evec %p %zu\n", ev, size);
+    memset(ev, 0, size);
+}
+
+#endif

src/util/fatbit.c

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fatbit.h"
+#include "multibit.h"
+
+u32 fatbit_size(u32 total_bits) {
+    return MAX(sizeof(struct fatbit), mmbit_size(total_bits));
+}

src/util/fatbit.h

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef FATBIT_H
+#define FATBIT_H
+
+/** \file
+ * \brief Multibit: fast bitset structure for use in scratch.
+ * Uses more space than mmbit, to avoid partial words for hopefully a taddy more
+ * performance.
+ *
+ * API is also trimmed down.
+ */
+
+#include "multibit.h"
+#include "ue2common.h"
+
+#define MIN_FAT_SIZE 32
+
+struct fatbit {
+    union {
+        u64a flat[MIN_FAT_SIZE / sizeof(u64a)];
+        u8 raw[MIN_FAT_SIZE];
+    } fb_int;
+    u64a tail[];
+};
+
+static really_inline
+void fatbit_clear(struct fatbit *bits) {
+    assert(ISALIGNED(bits));
+    memset(bits, 0, sizeof(struct fatbit));
+}
+
+static really_inline
+char fatbit_set(struct fatbit *bits, u32 total_bits, u32 key) {
+    return mmbit_set(bits->fb_int.raw, total_bits, key);
+}
+
+static really_inline
+void fatbit_unset(struct fatbit *bits, u32 total_bits, u32 key) {
+     mmbit_unset(bits->fb_int.raw, total_bits, key);
+}
+
+static really_inline
+char fatbit_isset(const struct fatbit *bits, u32 total_bits, u32 key) {
+    return mmbit_isset(bits->fb_int.raw, total_bits, key);
+}
+
+static really_inline
+u32 fatbit_iterate(const struct fatbit *bits, u32 total_bits, u32 it_in) {
+    /* TODO: iterate_flat could be specialised as we don't have to worry about
+     * partial blocks. */
+    return mmbit_iterate(bits->fb_int.raw, total_bits, it_in);
+}
+
+/** \brief Return the size in bytes of a fatbit that can store the given
+ * number of bits.
+ *
+ * Not for use in performance-critical code, implementation is in fatbit.c.
+ */
+u32 fatbit_size(u32 total_bits);
+
+#endif

src/util/graph.h

@@ -0,0 +1,318 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Functions for graph manipulation that aren't in the base BGL toolkit.
+ */
+
+#ifndef UTIL_GRAPH_H
+#define UTIL_GRAPH_H
+
+#include "container.h"
+#include "ue2common.h"
+#include "util/graph_range.h"
+#include "util/ue2_containers.h"
+
+#include <boost/graph/adjacency_iterator.hpp>
+#include <boost/graph/adjacency_list.hpp>
+#include <boost/graph/depth_first_search.hpp>
+#include <boost/graph/graph_traits.hpp>
+
+namespace ue2 {
+
+/** \brief True if the given vertex has no out-edges. */
+template<class Graph>
+bool isLeafNode(const typename Graph::vertex_descriptor& v, const Graph& g) {
+    typename Graph::adjacency_iterator ai, ae;
+    std::tie(ai, ae) = adjacent_vertices(v, g);
+    return ai == ae; // no out edges
+}
+
+/** \brief True if the out-degree of vertex \a v is greater than the given
+ * limit. */
+template<class Graph>
+bool hasGreaterOutDegree(size_t limit,
+                         const typename Graph::vertex_descriptor& v,
+                         const Graph& g) {
+    typename Graph::out_edge_iterator ei, ee;
+    for (std::tie(ei, ee) = out_edges(v, g); ei != ee; ++ei) {
+        if (limit-- == 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/** \brief Returns true if the in-degree of vertex \a v is greater than the
+ * given limit. */
+template<class Graph>
+bool hasGreaterInDegree(size_t limit,
+                        const typename Graph::vertex_descriptor& v,
+                        const Graph& g) {
+    typename Graph::in_edge_iterator ei, ee;
+    for (std::tie(ei, ee) = in_edges(v, g); ei != ee; ++ei) {
+        if (limit-- == 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/**
+ * \brief True if the degree of vertex \a v is greater than the given limit.
+ */
+template <class Graph>
+bool has_greater_degree(size_t limit,
+                        const typename Graph::vertex_descriptor &v,
+                        const Graph &g) {
+    typename Graph::in_edge_iterator ei, ee;
+    for (std::tie(ei, ee) = in_edges(v, g); ei != ee; ++ei) {
+        if (limit-- == 0) {
+            return true;
+        }
+    }
+    typename Graph::out_edge_iterator oi, oe;
+    for (std::tie(oi, oe) = out_edges(v, g); oi != oe; ++oi) {
+        if (limit-- == 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/** \brief True if vertex \a v has an edge to itself. */
+template<class Graph>
+bool hasSelfLoop(const typename Graph::vertex_descriptor &v, const Graph &g) {
+    return edge(v, v, g).second;
+}
+
+/** \brief True if any vertex in [it, end) has an edge to itself. */
+template<class Graph, class Iterator>
+bool anySelfLoop(const Graph &g, Iterator it, const Iterator &end) {
+    for (; it != end; ++it) {
+        if (hasSelfLoop(*it, g)) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+/** \brief Returns the out-degree of vertex \a v, ignoring self-loops. */
+template<class Graph>
+size_t proper_out_degree(const typename Graph::vertex_descriptor &v,
+                         const Graph &g) {
+    return out_degree(v, g) - (edge(v, v, g).second ? 1 : 0);
+}
+
+/** \brief Returns the in-degree of vertex \a v, ignoring self-loops. */
+template<class Graph>
+size_t proper_in_degree(const typename Graph::vertex_descriptor &v,
+                        const Graph &g) {
+    return in_degree(v, g) - (edge(v, v, g).second ? 1 : 0);
+}
+
+/** \brief Returns true iff the in-degree of vertex \a v is \a expected */
+template<class Graph>
+bool in_degree_equal_to(const typename Graph::vertex_descriptor &v,
+                        const Graph &g, size_t expected) {
+    size_t seen = 0;
+    typename Graph::in_edge_iterator ei, ee;
+    for (std::tie(ei, ee) = in_edges(v, g);; ++ei, seen++) {
+        if (seen == expected) {
+            return ei == ee;
+        }
+        if (ei == ee) {
+            return false;
+        }
+    }
+}
+
+/** \brief same as edge(s, t, g) by finds edge by inspecting in-edges of target.
+ * Should be used when it is known that t has a small in-degree and when s
+ * may have a large out-degree.
+ */
+template<class Graph>
+std::pair<typename Graph::edge_descriptor, bool>
+edge_by_target(const typename Graph::vertex_descriptor &s,
+               const typename Graph::vertex_descriptor &t, const Graph &g) {
+    typename Graph::in_edge_iterator ei, ee;
+    for (std::tie(ei, ee) = in_edges(t, g); ei != ee; ++ei) {
+        if (source(*ei, g) == s) {
+            return std::make_pair(*ei, true);
+        }
+    }
+
+    return std::make_pair(typename Graph::edge_descriptor(), false);
+}
+
+
+/** \brief True if vertex \a v has at least one successor. */
+template<class Graph>
+bool has_successor(const typename Graph::vertex_descriptor &v, const Graph &g) {
+    typename Graph::adjacency_iterator ai, ae;
+    std::tie(ai, ae) = adjacent_vertices(v, g);
+
+    return ai != ae;
+}
+
+/** \brief True if vertex \a v has at least one successor other than itself. */
+template<class Graph>
+bool has_proper_successor(const typename Graph::vertex_descriptor &v,
+                          const Graph &g) {
+    typename Graph::adjacency_iterator ai, ae;
+    std::tie(ai, ae) = adjacent_vertices(v, g);
+    if (ai == ae) {
+        return false;
+    }
+    if (*ai == v) {
+        ++ai; // skip self-loop
+    }
+
+    return ai != ae;
+}
+
+/** \brief A version of clear_vertex that explicitly removes in- and out-edges
+ * for vertex \a v. For many graphs, this is faster than the BGL clear_vertex
+ * function, which walks the graph's full edge list. */
+template <class Graph>
+void clear_vertex_faster(typename Graph::vertex_descriptor v, Graph &g) {
+    typename Graph::in_edge_iterator ei, ee;
+    tie(ei, ee) = in_edges(v, g);
+    while (ei != ee) {
+        remove_edge(*ei++, g);
+    }
+
+    typename Graph::out_edge_iterator oi, oe;
+    tie(oi, oe) = out_edges(v, g);
+    while (oi != oe) {
+        // NOTE: version that takes out_edge_iterator is faster according to
+        // the BGL docs.
+        remove_edge(oi++, g);
+    }
+}
+
+/** \brief Find the set of vertices that are reachable from the vertices in \a
+ * sources. */
+template<class Graph, class SourceCont, class OutCont>
+void find_reachable(const Graph &g, const SourceCont &sources, OutCont *out) {
+    using vertex_descriptor = typename Graph::vertex_descriptor;
+    ue2::unordered_map<vertex_descriptor, boost::default_color_type> colours;
+
+    for (auto v : sources) {
+        boost::depth_first_visit(g, v,
+                                 boost::make_dfs_visitor(boost::null_visitor()),
+                                 boost::make_assoc_property_map(colours));
+    }
+
+    for (const auto &e : colours) {
+        out->insert(e.first);
+    }
+}
+
+/** \brief Find the set of vertices that are NOT reachable from the vertices in
+ * \a sources. */
+template<class Graph, class SourceCont, class OutCont>
+void find_unreachable(const Graph &g, const SourceCont &sources, OutCont *out) {
+    using vertex_descriptor = typename Graph::vertex_descriptor;
+    ue2::unordered_set<vertex_descriptor> reachable;
+
+    find_reachable(g, sources, &reachable);
+
+    for (const auto &v : vertices_range(g)) {
+        if (!contains(reachable, v)) {
+            out->insert(v);
+        }
+    }
+}
+
+template <class Graph>
+bool has_parallel_edge(const Graph &g) {
+    using vertex_descriptor = typename Graph::vertex_descriptor;
+    ue2::unordered_set<std::pair<vertex_descriptor, vertex_descriptor>> seen;
+    for (const auto &e : edges_range(g)) {
+        auto u = source(e, g);
+        auto v = target(e, g);
+        if (!seen.emplace(u, v).second) {
+            return true;
+        }
+    }
+    return false;
+}
+
+struct found_back_edge {};
+struct detect_back_edges : public boost::default_dfs_visitor {
+    explicit detect_back_edges(bool ignore_self_in)
+        : ignore_self(ignore_self_in) {}
+    template <class Graph>
+    void back_edge(const typename Graph::edge_descriptor &e,
+                   const Graph &g) const {
+        if (ignore_self && source(e, g) == target(e, g)) {
+            return;
+        }
+        throw found_back_edge();
+    }
+    bool ignore_self;
+};
+
+template <class Graph>
+bool is_dag(const Graph &g, bool ignore_self_loops = false) {
+    try {
+        depth_first_search(g, visitor(detect_back_edges(ignore_self_loops)));
+    } catch (const found_back_edge &) {
+        return false;
+    }
+
+    return true;
+}
+
+template <class Graph>
+std::pair<typename Graph::edge_descriptor, bool>
+add_edge_if_not_present(typename Graph::vertex_descriptor u,
+                        typename Graph::vertex_descriptor v, Graph &g) {
+    std::pair<typename Graph::edge_descriptor, bool> e = edge(u, v, g);
+    if (!e.second) {
+        e = add_edge(u, v, g);
+    }
+    return e;
+}
+
+template <class Graph>
+std::pair<typename Graph::edge_descriptor, bool> add_edge_if_not_present(
+    typename Graph::vertex_descriptor u, typename Graph::vertex_descriptor v,
+    const typename Graph::edge_property_type &prop, Graph &g) {
+    std::pair<typename Graph::edge_descriptor, bool> e = edge(u, v, g);
+    if (!e.second) {
+        e = add_edge(u, v, prop, g);
+    }
+    return e;
+}
+
+} // namespace ue2
+
+#endif // UTIL_GRAPH_H

src/util/graph_range.h

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Convenience functions allowing range-based-for over BGL graphs.
+ *
+ * Normally with the BGL in C++98 you need to do this to iterate over graph
+ * elements:
+ *
+ *        Graph:out_edge_iterator ei, ee;
+ *        for (tie(ei, ee) = out_edges(v, g); ei != ee; ++ei) {
+ *            do_thing_with_edge(*ei, g);
+ *        }
+ *
+ * But now, with C++11 range-based-for and these functions, you can do this
+ * instead:
+ *
+ *        for (auto e : out_edges_range(v, g)) {
+ *            do_thing_with_edge(e, g);
+ *        }
+ *
+ * This syntax is much more compact and keeps the iterator vars from cluttering
+ * the outer scope.
+ */
+
+#ifndef UTIL_GRAPH_RANGE_H
+#define UTIL_GRAPH_RANGE_H
+
+#include <boost/graph/adjacency_list.hpp>
+#include <boost/range/iterator_range.hpp>
+
+namespace ue2 {
+
+/** Adapts a pair of iterators into a range. */
+template <class Iter>
+inline boost::iterator_range<Iter> pair_range(const std::pair<Iter, Iter> &p) {
+    return boost::make_iterator_range(p.first, p.second);
+}
+
+/** vertices(g) */
+template <class Graph>
+inline auto vertices_range(const Graph &g)
+    -> decltype(pair_range(vertices(g))) {
+    return pair_range(vertices(g));
+}
+
+/** edges(g) */
+template <class Graph>
+inline auto edges_range(const Graph &g) -> decltype(pair_range(edges(g))) {
+    return pair_range(edges(g));
+}
+
+/** out_edges(v, g) */
+template <class Graph>
+inline auto out_edges_range(const typename Graph::vertex_descriptor &v,
+                            const Graph &g)
+    -> decltype(pair_range(out_edges(v, g))) {
+    return pair_range(out_edges(v, g));
+}
+
+/** in_edges(v, g) */
+template <class Graph>
+inline auto in_edges_range(const typename Graph::vertex_descriptor &v,
+                           const Graph &g)
+    -> decltype(pair_range(in_edges(v, g))) {
+    return pair_range(in_edges(v, g));
+}
+
+/** adjacent_vertices(v, g) */
+template <class Graph>
+inline auto adjacent_vertices_range(const typename Graph::vertex_descriptor &v,
+                                    const Graph &g)
+    -> decltype(pair_range(adjacent_vertices(v, g))) {
+    return pair_range(adjacent_vertices(v, g));
+}
+
+/** inv_adjacent_vertices(v, g) */
+template <class Graph>
+inline auto inv_adjacent_vertices_range(
+    const typename Graph::vertex_descriptor &v, const Graph &g)
+    -> decltype(pair_range(inv_adjacent_vertices(v, g))) {
+    return pair_range(inv_adjacent_vertices(v, g));
+}
+
+} // namespace ue2
+
+#endif // UTIL_GRAPH_RANGE_H

src/util/internal_report.h

@@ -0,0 +1,201 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Definition of an internal_report, along with convenience functions.
+ */
+
+#ifndef INTERNAL_REPORT_H
+#define INTERNAL_REPORT_H
+
+#include "ue2common.h"
+
+/* internal_report::type values */
+
+#define EXTERNAL_CALLBACK                             0
+#define EXTERNAL_CALLBACK_SOM_REL                     1
+#define INTERNAL_SOM_LOC_SET                          2
+#define INTERNAL_SOM_LOC_SET_IF_UNSET                 3
+#define INTERNAL_SOM_LOC_SET_IF_WRITABLE              4
+#define INTERNAL_SOM_LOC_SET_SOM_REV_NFA              5
+#define INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET     6
+#define INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE  7
+#define INTERNAL_SOM_LOC_COPY                         8
+#define INTERNAL_SOM_LOC_COPY_IF_WRITABLE             9
+#define INTERNAL_SOM_LOC_MAKE_WRITABLE               10
+#define EXTERNAL_CALLBACK_SOM_STORED                 11
+#define EXTERNAL_CALLBACK_SOM_ABS                    12
+#define EXTERNAL_CALLBACK_SOM_REV_NFA                13
+
+/** set the som loc to the value in from_offset */
+#define INTERNAL_SOM_LOC_SET_FROM                    14
+
+/** set the som loc to the value in from_offset */
+#define INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE        15
+#define INTERNAL_ROSE_CHAIN                          16
+
+/** Index meaning a given exhaustion key is invalid. */
+#define INVALID_EKEY      0xffffffff
+
+/** \brief All the data we use for handling a match, bytecode representation.
+ *
+ * Includes extparam constraints and bounds, exhaustion/dedupe keys, offset
+ * adjustment and SOM information.
+ *
+ * At compile time, this data is in the ue2::Report structure, which is
+ * converted to internal_report for layout in the bytecode.
+ */
+struct ALIGN_CL_DIRECTIVE internal_report {
+    /** \brief from EXTERNAL_ and INTERNAL_ defined above. */
+    u8 type;
+
+    /** \brief do we actually use minOffset, maxOffset */
+    u8 hasBounds;
+
+    /** \brief use SOM for minLength, but don't report it to user callback. */
+    u8 quashSom;
+
+    /** \brief min offset in the stream at which this report can match. */
+    u64a minOffset;
+
+    /** \brief max offset in the stream at which this report can match. */
+    u64a maxOffset;
+
+    /** \brief min match length (start of match to current offset) */
+    u64a minLength;
+
+    /** \brief Exhaustion key.
+     *
+     * If exhaustible, the ekey to check before reporting a match.
+     * Additionally after reporting a match the ekey will be set. If not
+     * exhaustible, this will be INVALID_EKEY. */
+    u32 ekey;
+
+    /** \brief Dedupe key. */
+    u32 dkey;
+
+    /** \brief Adjustment to add to the match offset when we report a match.
+     *
+     * This is usually used for reports attached to states that form part of a
+     * zero-width assertion, like '$'. */
+    s32 offsetAdjust;
+
+    /** \brief Match report ID, for external reports.
+     *
+     * - external callback -> external report id
+     * - internal_som_* -> som loc to modify,
+     * - INTERNAL_ROSE_CHAIN -> top event to push on
+     * - otherwise target subnfa. */
+    u32 onmatch;
+
+    union {
+        /** \brief SOM distance value, use varies according to type.
+         *
+         *  - for EXTERNAL_CALLBACK_SOM_REL, from-offset is this many bytes
+         *    before the to-offset.
+         *  - for EXTERNAL_CALLBACK_SOM_ABS, set from-offset to this value.
+         *  - for INTERNAL_SOM_LOC_COPY*, som location read_from.
+         */
+        u64a somDistance;
+
+        /** \brief Index of the reverse nfa.
+         * Used by EXTERNAL_CALLBACK_SOM_REV_NFA and
+         *  INTERNAL_SOM_LOC_SET_SOM_REV_NFA*
+         */
+        u64a revNfaIndex;
+
+        /**
+         * Used by INTERNAL_ROSE_CHAIN, Number of bytes behind us that we are
+         * allowed to squash identical top events on the queue.
+         */
+        u64a topSquashDistance;
+    } aux;
+};
+
+static really_inline
+int isInternalSomReport(const struct internal_report *ri) {
+    switch (ri->type) {
+    case INTERNAL_SOM_LOC_SET:
+    case INTERNAL_SOM_LOC_SET_IF_UNSET:
+    case INTERNAL_SOM_LOC_SET_IF_WRITABLE:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
+    case INTERNAL_SOM_LOC_COPY:
+    case INTERNAL_SOM_LOC_COPY_IF_WRITABLE:
+    case INTERNAL_SOM_LOC_MAKE_WRITABLE:
+    case INTERNAL_SOM_LOC_SET_FROM:
+    case INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE:
+        return 1;
+    case EXTERNAL_CALLBACK:
+    case EXTERNAL_CALLBACK_SOM_REL:
+    case EXTERNAL_CALLBACK_SOM_STORED:
+    case EXTERNAL_CALLBACK_SOM_ABS:
+    case EXTERNAL_CALLBACK_SOM_REV_NFA:
+    case INTERNAL_ROSE_CHAIN:
+        return 0;
+    default:
+        break; // fall through
+    }
+    assert(0); // unknown?
+    return 0;
+}
+
+#ifndef NDEBUG
+/* used in asserts */
+static UNUSED
+char isExternalReport(const struct internal_report *ir) {
+    switch (ir->type) {
+    case INTERNAL_SOM_LOC_SET:
+    case INTERNAL_SOM_LOC_SET_IF_UNSET:
+    case INTERNAL_SOM_LOC_SET_IF_WRITABLE:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
+    case INTERNAL_SOM_LOC_COPY:
+    case INTERNAL_SOM_LOC_COPY_IF_WRITABLE:
+    case INTERNAL_SOM_LOC_MAKE_WRITABLE:
+    case INTERNAL_SOM_LOC_SET_FROM:
+    case INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE:
+    case INTERNAL_ROSE_CHAIN:
+        return 0;
+    case EXTERNAL_CALLBACK:
+    case EXTERNAL_CALLBACK_SOM_REL:
+    case EXTERNAL_CALLBACK_SOM_STORED:
+    case EXTERNAL_CALLBACK_SOM_ABS:
+    case EXTERNAL_CALLBACK_SOM_REV_NFA:
+        return 1;
+    default:
+        break; // fall through
+    }
+    assert(0); // unknown?
+    return 1;
+}
+#endif
+
+#endif // INTERNAL_REPORT_H

src/util/join.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef JOIN_H
+
+#define JOIN(x, y) JOIN_AGAIN(x, y)
+#define JOIN_AGAIN(x, y) x ## y
+
+#endif

src/util/make_unique.h

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UTIL_MAKE_UNIQUE_H
+#define UTIL_MAKE_UNIQUE_H
+
+#if (defined(_WIN32) || defined(_WIN64)) && (_MSC_VER > 1700)
+// VC++ 2013 onwards has make_unique in the STL
+#define USE_STD
+#include <memory>
+#else
+#include <boost/smart_ptr/make_unique.hpp>
+#endif
+
+namespace ue2 {
+#if defined(USE_STD)
+	using std::make_unique;
+#else
+	using boost::make_unique;
+#endif
+}
+
+#undef USE_STD
+#endif // UTIL_MAKE_UNIQUE_H

src/util/masked_move.c

@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "ue2common.h"
+#include "masked_move.h"
+
+#if defined(__AVX2__)
+/* masks for masked moves */
+
+/* magic mask for maskload (vmmaskmovq) - described in UE-2424 */
+const u32 mm_mask_mask[16] ALIGN_CL_DIRECTIVE = {
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0xff000000U,
+    0xfe000000U,
+    0xfc000000U,
+    0xf8000000U,
+    0xf0000000U,
+    0xe0000000U,
+    0xc0000000U,
+    0x80000000U,
+};
+
+const u32 mm_shuffle_end[32][8] = {
+    { 0x03020100U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x02010080U, 0x80808003U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x01008080U, 0x80800302U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x00808080U, 0x80030201U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x03020100U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x02010080U, 0x80808003U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x01008080U, 0x80800302U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x00808080U, 0x80030201U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x03020100U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x02010080U, 0x80808003U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x01008080U, 0x80800302U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x00808080U, 0x80030201U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x03020100U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x02010080U, 0x80808003U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x01008080U, 0x80800302U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x00808080U, 0x80030201U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x03020100U, 0x80808080U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x02010080U, 0x80808003U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x01008080U, 0x80800302U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x00808080U, 0x80030201U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x03020100U, 0x80808080U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x02010080U, 0x80808003U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x01008080U, 0x80800302U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x00808080U, 0x80030201U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x03020100U, 0x80808080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x02010080U, 0x80808003U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x01008080U, 0x80800302U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x00808080U, 0x80030201U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x03020100U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x02010080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x01008080U, },
+    { 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x80808080U, 0x00808080U, },
+};
+#endif // AVX2

src/util/masked_move.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef MASKED_MOVE_H
+#define MASKED_MOVE_H
+
+#if defined(__AVX2__)
+
+#include "unaligned.h"
+#include "simd_utils.h"
+#include "simd_utils_ssse3.h"
+
+extern const u32 mm_mask_mask[16];
+extern const u32 mm_shuffle_end[32][8];
+
+/* load mask for len bytes from start of buffer */
+static really_inline m256
+_get_mm_mask_end(u32 len) {
+    assert(len <= 32);
+    const u8 *masky = (const u8 *)mm_mask_mask;
+    m256 mask = load256(masky + 32);
+    mask = _mm256_sll_epi32(mask, _mm_cvtsi32_si128(8 - (len >> 2)));
+    return mask;
+}
+
+/*
+ * masked_move256_len: Will load len bytes from *buf into m256
+ * _______________________________
+ * |0<----len---->|            32|
+ * -------------------------------
+ */
+static really_inline m256
+masked_move256_len(const u8 *buf, const u32 len) {
+    assert(len >= 4);
+
+    m256 lmask = _get_mm_mask_end(len);
+
+    u32 end = unaligned_load_u32(buf + len - 4);
+    m256 preshufend = _mm256_broadcastq_epi64(_mm_cvtsi32_si128(end));
+    m256 v = _mm256_maskload_epi32((const int *)buf, lmask);
+    m256 shufend = vpshufb(preshufend, loadu256(&mm_shuffle_end[len - 4]));
+    m256 target = or256(v, shufend);
+
+    return target;
+}
+
+#endif /* AVX2 */
+#endif /* MASKED_MOVE_H */
+

src/util/multibit.c

@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Multibit: lookup tables and support code.
+ *
+ * This C file contains the constant tables used by multibit, so we don't end
+ * up creating copies of them for every unit that uses it.
+ */
+
+#include "multibit.h"
+#include "ue2common.h"
+
+const u8 mmbit_keyshift_lut[32] = {
+    30, 30, 24, 24, 24, 24, 24, 24, 18, 18, 18,
+    18, 18, 18, 12, 12, 12, 12, 12, 12, 6, 6,
+    6,  6,  6,  6,  0,  0,  0,  0,  0,  0
+};
+
+// The only actually valid values of ks are as shown in the LUT above, but a
+// division is just too expensive.
+const u8 mmbit_maxlevel_from_keyshift_lut[32] = {
+    0, 0, 0, 0, 0, 0,
+    1, 1, 1, 1, 1, 1,
+    2, 2, 2, 2, 2, 2,
+    3, 3, 3, 3, 3, 3,
+    4, 4, 4, 4, 4, 4,
+    5, 5
+};
+
+const u8 mmbit_maxlevel_direct_lut[32] = {
+    5, 5, 4, 4, 4, 4, 4, 4, 3, 3, 3,
+    3, 3, 3, 2, 2, 2, 2, 2, 2, 1, 1,
+    1, 1, 1, 1, 0, 0, 0, 0, 0, 0
+};
+
+#define ZERO_TO_LUT(x) ((1ULL << x) - 1)
+
+const u64a mmbit_zero_to_lut[65] = {
+    ZERO_TO_LUT(0),
+    ZERO_TO_LUT(1),
+    ZERO_TO_LUT(2),
+    ZERO_TO_LUT(3),
+    ZERO_TO_LUT(4),
+    ZERO_TO_LUT(5),
+    ZERO_TO_LUT(6),
+    ZERO_TO_LUT(7),
+    ZERO_TO_LUT(8),
+    ZERO_TO_LUT(9),
+    ZERO_TO_LUT(10),
+    ZERO_TO_LUT(11),
+    ZERO_TO_LUT(12),
+    ZERO_TO_LUT(13),
+    ZERO_TO_LUT(14),
+    ZERO_TO_LUT(15),
+    ZERO_TO_LUT(16),
+    ZERO_TO_LUT(17),
+    ZERO_TO_LUT(18),
+    ZERO_TO_LUT(19),
+    ZERO_TO_LUT(20),
+    ZERO_TO_LUT(21),
+    ZERO_TO_LUT(22),
+    ZERO_TO_LUT(23),
+    ZERO_TO_LUT(24),
+    ZERO_TO_LUT(25),
+    ZERO_TO_LUT(26),
+    ZERO_TO_LUT(27),
+    ZERO_TO_LUT(28),
+    ZERO_TO_LUT(29),
+    ZERO_TO_LUT(30),
+    ZERO_TO_LUT(31),
+    ZERO_TO_LUT(32),
+    ZERO_TO_LUT(33),
+    ZERO_TO_LUT(34),
+    ZERO_TO_LUT(35),
+    ZERO_TO_LUT(36),
+    ZERO_TO_LUT(37),
+    ZERO_TO_LUT(38),
+    ZERO_TO_LUT(39),
+    ZERO_TO_LUT(40),
+    ZERO_TO_LUT(41),
+    ZERO_TO_LUT(42),
+    ZERO_TO_LUT(43),
+    ZERO_TO_LUT(44),
+    ZERO_TO_LUT(45),
+    ZERO_TO_LUT(46),
+    ZERO_TO_LUT(47),
+    ZERO_TO_LUT(48),
+    ZERO_TO_LUT(49),
+    ZERO_TO_LUT(50),
+    ZERO_TO_LUT(51),
+    ZERO_TO_LUT(52),
+    ZERO_TO_LUT(53),
+    ZERO_TO_LUT(54),
+    ZERO_TO_LUT(55),
+    ZERO_TO_LUT(56),
+    ZERO_TO_LUT(57),
+    ZERO_TO_LUT(58),
+    ZERO_TO_LUT(59),
+    ZERO_TO_LUT(60),
+    ZERO_TO_LUT(61),
+    ZERO_TO_LUT(62),
+    ZERO_TO_LUT(63),
+    ~0ULL
+};
+
+const u32 mmbit_root_offset_from_level[7] = {
+    0,
+    1,
+    1 + (1 << MMB_KEY_SHIFT),
+    1 + (1 << MMB_KEY_SHIFT) + (1 << MMB_KEY_SHIFT * 2),
+    1 + (1 << MMB_KEY_SHIFT) + (1 << MMB_KEY_SHIFT * 2) + (1 << MMB_KEY_SHIFT * 3),
+    1 + (1 << MMB_KEY_SHIFT) + (1 << MMB_KEY_SHIFT * 2) + (1 << MMB_KEY_SHIFT * 3) + (1 << MMB_KEY_SHIFT * 4),
+    1 + (1 << MMB_KEY_SHIFT) + (1 << MMB_KEY_SHIFT * 2) + (1 << MMB_KEY_SHIFT * 3) + (1 << MMB_KEY_SHIFT * 4) + (1 << MMB_KEY_SHIFT * 5),
+};
+
+u32 mmbit_size(u32 total_bits) {
+    MDEBUG_PRINTF("%u\n", total_bits);
+
+    // UE-2228: multibit has bugs in very, very large cases that we should be
+    // protected against at compile time by resource limits.
+    assert(total_bits <= 1U << 30);
+
+    // Flat model multibit structures are just stored as a bit vector.
+    if (total_bits <= MMB_FLAT_MAX_BITS) {
+        return ROUNDUP_N(total_bits, 8) / 8;
+    }
+
+    u32 current_level = 1;
+    u32 total = 0;
+    while (current_level * MMB_KEY_BITS < total_bits) {
+        total += current_level;
+        current_level <<= MMB_KEY_SHIFT;
+    }
+    total += (total_bits + MMB_KEY_BITS - 1)/MMB_KEY_BITS;
+    return sizeof(MMB_TYPE) * total;
+}
+
+#ifdef DUMP_SUPPORT
+
+#include <stdio.h>
+#include <stdlib.h>
+
+/** \brief Dump a sparse iterator's keys to stdout. */
+void mmbit_sparse_iter_dump(const struct mmbit_sparse_iter *it,
+                            u32 total_bits) {
+    // Expediency and future-proofing: create a temporary multibit of the right
+    // size with all the bits on, then walk it with this sparse iterator.
+    size_t bytes = mmbit_size(total_bits);
+    u8 *bits = malloc(bytes);
+    if (!bits) {
+        printf("Failed to alloc %zu bytes for temp multibit", bytes);
+        return;
+    }
+    for (u32 i = 0; i < total_bits; i++) {
+        mmbit_set_i(bits, total_bits, i);
+    }
+
+    struct mmbit_sparse_state s[MAX_SPARSE_ITER_STATES];
+    u32 idx = 0;
+    for (u32 i = mmbit_sparse_iter_begin(bits, total_bits, &idx, it, s);
+             i != MMB_INVALID;
+             i = mmbit_sparse_iter_next(bits, total_bits, i, &idx, it, s)) {
+        printf("%u ", i);
+    }
+
+    printf("(%u keys)", idx + 1);
+
+    free(bits);
+}
+
+#endif // DUMP_SUPPORT

src/util/multibit_build.cpp

@@ -0,0 +1,299 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Multibit: build code (for sparse iterators)
+ */
+#include "multibit.h"
+#include "multibit_build.h"
+#include "scatter.h"
+#include "ue2common.h"
+#include "rose/rose_build_scatter.h"
+
+#include <cassert>
+#include <cstring> // for memset
+#include <map>
+#include <queue>
+#include <vector>
+
+using namespace std;
+
+namespace ue2 {
+
+namespace {
+struct TreeNode {
+    MMB_TYPE mask = 0;
+    u32 depth = 0;
+    map<u32, TreeNode> children; // keyed by rkey
+};
+} // namespace
+
+static
+void addNode(TreeNode &tree, u32 depth, u32 key, s32 ks, u32 rkey) {
+    u32 bit = (key >> ks) & MMB_KEY_MASK;
+    DEBUG_PRINTF("depth=%u, key=%u, ks=%d, rkey=%u, bit=%u\n", depth, key, ks,
+                 rkey, bit);
+    mmb_set(&tree.mask, bit); // add bit to this level
+    tree.depth = depth; // record depth
+    // next level
+    rkey = (rkey << MMB_KEY_SHIFT) + bit;
+    ks -= MMB_KEY_SHIFT;
+    depth++;
+    if (ks >= 0) {
+        addNode(tree.children[rkey], depth, key, ks, rkey);
+    }
+}
+
+static
+void bfs(vector<mmbit_sparse_iter> &out, const TreeNode &tree) {
+    queue<const TreeNode *> q;
+    q.push(&tree);
+
+    vector<u32> levels;
+    u32 depth = 0;
+
+    DEBUG_PRINTF("walking q\n");
+
+    while (!q.empty()) {
+        const TreeNode *t = q.front();
+        q.pop();
+
+        if (depth != t->depth) {
+            depth = t->depth;
+            levels.push_back(out.size());
+        }
+
+        DEBUG_PRINTF("pop: mask=0x%08llx, depth=%u, children.size()=%zu\n",
+                     t->mask, t->depth, t->children.size());
+
+        out.push_back(mmbit_sparse_iter());
+        memset(&out.back(), 0, sizeof(mmbit_sparse_iter));
+        mmbit_sparse_iter &record = out.back();
+        record.mask = t->mask;
+        record.val = 0;
+
+        for (auto &e : t->children) {
+            q.push(&e.second);
+        }
+    }
+
+    // val for records in non-last levels is the iterator array start offset
+    // for that iterator record's children
+    u32 start = 0;
+    for (size_t i = 0; i < levels.size(); i++) {
+        u32 start_next = levels[i];
+        u32 population = 0;
+        DEBUG_PRINTF("next level starts at %u\n", start_next);
+        for (u32 j = start; j < start_next; j++) {
+            out[j].val = start_next + population;
+            DEBUG_PRINTF("  children of %u start at %u\n", j, out[j].val);
+            population += mmb_popcount(out[j].mask);
+        }
+        start = start_next;
+    }
+
+    // val for records in the last level is the cumulative popcount
+    u32 population = 0;
+    for (size_t i = start; i < out.size(); i++) {
+        DEBUG_PRINTF("last level: i=%zu, population=%u\n", i, population);
+        out[i].val = population;
+        population += mmb_popcount(out[i].mask);
+    }
+}
+
+/** \brief Construct a sparse iterator over the values in \a bits for a
+ * multibit of size \a total_bits. */
+void mmbBuildSparseIterator(vector<mmbit_sparse_iter> &out,
+                            const vector<u32> &bits, u32 total_bits) {
+    assert(out.empty());
+    assert(!bits.empty());
+    assert(total_bits > 0);
+
+    DEBUG_PRINTF("building sparse iter for %zu of %u bits\n",
+                 bits.size(), total_bits);
+
+    s32 ks = (total_bits > 1 ? mmbit_keyshift(total_bits) : 0);
+
+    // Construct an intermediate tree
+    TreeNode tree;
+    for (const auto &bit : bits) {
+        assert(bit < total_bits);
+        addNode(tree, 0, bit, ks, 0);
+    }
+
+    // From our intermediate tree, lay the data out with a breadth-first walk
+    bfs(out, tree);
+    assert(!out.empty());
+
+#ifdef DEBUG
+    DEBUG_PRINTF("dump of iterator tree:\n");
+    for (size_t i = 0; i < out.size(); ++i) {
+        printf("    %zu:\tmask=0x%08llx, val=%u\n", i, out[i].mask, out[i].val);
+    }
+#endif
+
+    DEBUG_PRINTF("iter has %zu records\n", out.size());
+}
+
+template<typename T>
+static
+void add_scatter(vector<T> *out, u32 offset, u64a mask) {
+    T su;
+    memset(&su, 0, sizeof(su));
+    su.offset = offset;
+    su.val = mask;
+    out->push_back(su);
+    DEBUG_PRINTF("add %llu at offset %u\n", mask, offset);
+}
+
+static
+u32 mmbit_get_level_root_offset(u32 level) {
+    return mmbit_root_offset_from_level[level] * sizeof(MMB_TYPE);
+}
+
+void mmbBuildInitRangePlan(u32 total_bits, u32 begin, u32 end,
+                           scatter_plan_raw *out) {
+    DEBUG_PRINTF("building scatter plan for [%u, %u]/%u\n", begin, end,
+                 total_bits);
+    if (!total_bits) {
+        return;
+    }
+
+    if (total_bits <= MMB_FLAT_MAX_BITS) {
+        // Handle flat model cases: first a bunch of 64-bit full-sized blocks,
+        // then a single runt block at the end.
+        u32 dest = 0; // dest offset
+        u32 bits = total_bits;
+        u32 base = 0;
+        for (; bits > 64; bits -= 64, base += 64, dest += 8) {
+            MMB_TYPE mask = get_flat_masks(base, begin, end);
+            add_scatter(&out->p_u64a, dest, mask);
+        }
+
+        // Last chunk.
+        assert(bits > 0 && bits <= 64);
+
+        MMB_TYPE mask = get_flat_masks(base, begin, end);
+        if (bits <= 8) {
+            add_scatter(&out->p_u8, dest + 0, mask);
+        } else if (bits <= 16) {
+            add_scatter(&out->p_u16, dest + 0, mask);
+        } else if (bits <= 24) {
+            add_scatter(&out->p_u16, dest + 0, mask);
+            add_scatter(&out->p_u8,  dest + 2, mask >> 16);
+        } else if (bits <= 32) {
+            add_scatter(&out->p_u32, dest + 0, mask);
+        } else if (bits <= 40) {
+            add_scatter(&out->p_u32, dest + 0, mask);
+            add_scatter(&out->p_u8,  dest + 4, mask >> 32);
+        } else if (bits <= 48) {
+            add_scatter(&out->p_u32, dest + 0, mask);
+            add_scatter(&out->p_u16, dest + 4, mask >> 32);
+        } else if (bits <= 56) {
+            add_scatter(&out->p_u32, dest + 0, mask);
+            add_scatter(&out->p_u16, dest + 4, mask >> 32);
+            add_scatter(&out->p_u8,  dest + 6, mask >> 48);
+        } else {
+            add_scatter(&out->p_u64a, dest + 0, mask);
+        }
+        return;
+    }
+
+    /* handle the multilevel case */
+    s32 ks = mmbit_keyshift(total_bits);
+    u32 level = 0;
+    assert(sizeof(MMB_TYPE) == sizeof(u64a));
+
+    if (begin == end) {
+        add_scatter(&out->p_u64a, 0, 0);
+        return;
+    }
+
+    for (;;) {
+        u32 block_offset = mmbit_get_level_root_offset(level);
+        u32 k1 = begin >> ks, k2 = end >> ks;
+
+        // Summary blocks need to account for the runt block on the end.
+        if ((k2 << ks) != end) {
+            k2++;
+        }
+
+        // Partial block to deal with beginning.
+        block_offset += k1 / MMB_KEY_BITS;
+        if (k1 % MMB_KEY_BITS) {
+            u32 idx = k1 / MMB_KEY_BITS;
+            u32 block_end = (idx + 1) * MMB_KEY_BITS;
+
+            // Because k1 % MMB_KEY_BITS != 0, we can avoid checking edge cases
+            // here (see the branch in mmb_mask_zero_to).
+            MMB_TYPE mask = (-MMB_ONE) << (k1 % MMB_KEY_BITS);
+
+            if (k2 < block_end) {
+                assert(k2 % MMB_KEY_BITS);
+                mask &= mmb_mask_zero_to_nocheck(k2 % MMB_KEY_BITS);
+                add_scatter(&out->p_u64a, block_offset, mask);
+                goto next_level;
+            } else {
+                add_scatter(&out->p_u64a, block_offset, mask);
+                k1 = block_end;
+                block_offset += sizeof(MMB_TYPE);
+            }
+        }
+
+        // Write blocks filled with ones until we get to the last block.
+        for (; k1 < (k2 & ~MMB_KEY_MASK); k1 += MMB_KEY_BITS) {
+            add_scatter(&out->p_u64a, block_offset, -MMB_ONE);
+            block_offset += sizeof(MMB_TYPE);
+        }
+
+        // Final block.
+        if (likely(k1 < k2)) {
+            // Again, if k2 was at a block boundary, it would have been handled
+            // by the previous loop, so we know k2 % MMB_KEY_BITS != 0 and can
+            // avoid the branch in mmb_mask_zero_to here.
+            assert(k2 % MMB_KEY_BITS);
+            MMB_TYPE mask = mmb_mask_zero_to_nocheck(k2 % MMB_KEY_BITS);
+
+            add_scatter(&out->p_u64a, block_offset, mask);
+        }
+
+    next_level:
+        if (ks == 0) {
+            break; // Last level is done, finished.
+        }
+
+        ks -= MMB_KEY_SHIFT;
+        level++;
+    }
+}
+
+void mmbBuildClearPlan(u32 total_bits, scatter_plan_raw *out) {
+    return mmbBuildInitRangePlan(total_bits, 0, 0, out);
+}
+
+} // namespace ue2

src/util/multibit_build.h

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Multibit: build code (for sparse iterators)
+ */
+
+#ifndef MULTIBIT_BUILD_H
+#define MULTIBIT_BUILD_H
+
+#include "multibit_internal.h"
+
+#include <vector>
+
+/** \brief Comparator for \ref mmbit_sparse_iter structures. */
+static inline
+bool operator<(const mmbit_sparse_iter &a, const mmbit_sparse_iter &b) {
+    if (a.mask != b.mask) {
+        return a.mask < b.mask;
+    }
+    return a.val < b.val;
+}
+
+namespace ue2 {
+
+/** \brief Construct a sparse iterator over the values in \a bits for a
+ * multibit of size \a total_bits. */
+void mmbBuildSparseIterator(std::vector<mmbit_sparse_iter> &out,
+                            const std::vector<u32> &bits, u32 total_bits);
+
+struct scatter_plan_raw;
+
+void mmbBuildInitRangePlan(u32 total_bits, u32 begin, u32 end,
+                           scatter_plan_raw *out);
+void mmbBuildClearPlan(u32 total_bits, scatter_plan_raw *out);
+
+} // namespace ue2
+
+#endif // MULTIBIT_BUILD_H

src/util/multibit_internal.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Multibit: data structures.
+ *
+ * If all you need is the sizes of multibit's few structures, then including
+ * this file is a much better idea than including all of multibit.h.
+ */
+#ifndef MULTIBIT_INTERNAL_H
+#define MULTIBIT_INTERNAL_H
+
+#include "ue2common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** \brief Sentinel value meaning "no key found". */
+#define MMB_INVALID 0xffffffffu
+
+typedef u64a MMB_TYPE; /**< Basic block type for mmbit operations. */
+#define MMB_MAX_LEVEL 6 /**< Maximum level in the mmbit pyramid. */
+
+/** \brief Sparse iterator record type.
+ *
+ * A sparse iterator is a tree of these records, where val identifies the
+ * offset of the result for leaf nodes and points to the next record for
+ * intermediate nodes. Built by the code in multibit_build.cpp.
+ */
+struct mmbit_sparse_iter {
+    MMB_TYPE mask;
+    u32 val;
+};
+
+/** \brief Sparse iterator runtime state type.
+ *
+ * An array of these records (one per "level" in the multibit pyramid) is used
+ * to store the current iteration state.
+ */
+struct mmbit_sparse_state {
+    MMB_TYPE mask; //!< \brief masked last block read at this level.
+    u32 itkey;     //!< \brief iterator offset for this level.
+};
+
+/** \brief Maximum number of \ref mmbit_sparse_state that could be needed. */
+#define MAX_SPARSE_ITER_STATES (6 + 1)
+
+/** \brief Return the size in bytes of a multibit that can store the given
+ * number of bits.
+ *
+ * Not for use in performance-critical code, implementation is in multibit.c.
+ */
+u32 mmbit_size(u32 total_bits);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // MULTIBIT_INTERNAL_H

src/util/order_check.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* for implementing operator<, assumes objects are a and b */
+#define ORDER_CHECK(field) do {    \
+        if (a.field < b.field) {   \
+            return 1;              \
+        }                          \
+        if (b.field < a.field) {   \
+            return 0;              \
+        }                          \
+    } while (0)

src/util/pack_bits.h

@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Functions for packing/unpacking arrays.
+ */
+
+#ifndef UTIL_PACK_BITS_H
+#define UTIL_PACK_BITS_H
+
+#include "ue2common.h"
+#include "unaligned.h"
+#include "partial_store.h"
+
+/**
+ * \brief Pack bits from an array of 32-bit words into \a out.
+ *
+ * \param out Output array. Must be large enough to store sum(bits).
+ * \param v Input array.
+ * \param bits Number of low bits in the corresponding element of \a v to pack.
+ * \param elements Size of the \a v and \a bits arrays.
+ */
+static really_inline
+void pack_bits_32(char *out, const u32 *v, const u32 *bits,
+                  const unsigned int elements);
+
+/**
+ * \brief Pack bits from an array of 64-bit words into \a out.
+ *
+ * \param out Output array. Must be large enough to store sum(bits).
+ * \param v Input array.
+ * \param bits Number of low bits in the corresponding element of \a v to pack.
+ * \param elements Size of the \a v and \a bits arrays.
+ */
+static really_inline
+void pack_bits_64(char *out, const u64a *v, const u32 *bits,
+                  const unsigned int elements);
+
+/**
+ * \brief Unpack bits into an array of 32-bit words according to the counts
+ * given.
+ *
+ * \param v Output array.
+ * \param in Packed input array.
+ * \param bits Number of bits to unpack into the corresponding element of \a v.
+ * \param elements Size of the \a v and \a bits arrays.
+ */
+static really_inline
+void unpack_bits_32(u32 *v, const u8 *in, const u32 *bits,
+                    const unsigned int elements);
+
+/**
+ * \brief Unpack bits into an array of 64-bit words according to the counts
+ * given.
+ *
+ * \param v Output array.
+ * \param in Packed input array.
+ * \param bits Number of bits to unpack into the corresponding element of \a v.
+ * \param elements Size of the \a v and \a bits arrays.
+ */
+static really_inline
+void unpack_bits_64(u64a *v, const u8 *in, const u32 *bits,
+                    const unsigned int elements);
+
+/*
+ * Inline implementations follow.
+ */
+
+static really_inline
+void pack_bits_32(char *out, const u32 *v, const u32 *bits,
+                     const unsigned int elements) {
+    u32 write = 0; // accumulator
+    u32 idx = 0;   // acc holds this many bits
+
+    for (unsigned int i = 0; i < elements; i++) {
+        assert(bits[i] <= 32);
+        write |= (v[i] << idx);
+        idx += bits[i];
+        if (idx >= 32) {
+            unaligned_store_u32(out, write);
+            out += 4;
+            idx -= 32;
+            u32 leftover = bits[i] - idx;
+            if (leftover == 32) {
+                write = 0;
+            } else {
+                assert(leftover < 32);
+                write = v[i] >> leftover;
+            }
+        }
+    }
+
+    // There might be a write left over.
+    partial_store_u32(out, write, (idx + 7) / 8);
+}
+
+static really_inline
+void pack_bits_64(char *out, const u64a *v, const u32 *bits,
+                     const unsigned int elements) {
+    u64a write = 0; // accumulator
+    u32 idx = 0;    // acc holds this many bits
+
+    for (unsigned int i = 0; i < elements; i++) {
+        assert(bits[i] <= 64);
+        write |= (v[i] << idx);
+        idx += bits[i];
+        if (idx >= 64) {
+            unaligned_store_u64a(out, write);
+            out += 8;
+            idx -= 64;
+            u32 leftover = bits[i] - idx;
+            if (leftover == 64) {
+                write = 0;
+            } else {
+                assert(leftover < 64);
+                write = v[i] >> leftover;
+            }
+        }
+    }
+
+    // There might be a write left over.
+    DEBUG_PRINTF("partial store of idx=%u\n", idx);
+    partial_store_u64a(out, write, (idx + 7) / 8);
+}
+
+static really_inline
+void unpack_bits_32(u32 *v, const u8 *in, const u32 *bits,
+                   const unsigned int elements) {
+    u32 used = 0; // bits used from *in
+
+    for (unsigned int i = 0; i < elements; i++) {
+        assert(bits[i] <= 32);
+        u32 v_out = 0;   // accumulator for v[i]
+        u32 b = bits[i]; // bits left to read for v[i]
+        u32 vidx = 0;    // bits written to v[i]
+
+        while (b) {
+            u32 read = *in >> used;
+            u32 bits_read = 8 - used;
+
+            if (b <= bits_read) {
+                u32 mask = read & ((1U << b) - 1);
+                v_out |= mask << vidx;
+                vidx += b;
+                used += b;
+                b = 0;
+                if (used < 8) {
+                    continue; // more from this *in
+                }
+            } else {
+                v_out |= read << vidx;
+                vidx += bits_read;
+                b -= bits_read;
+            }
+
+            used = 0;
+            in++;
+        }
+
+        v[i] = v_out;
+    }
+}
+
+static really_inline
+void unpack_bits_64(u64a *v, const u8 *in, const u32 *bits,
+                    const unsigned int elements) {
+    u32 used = 0; // bits used from *in
+
+    for (unsigned int i = 0; i < elements; i++) {
+        assert(bits[i] <= 64);
+        u64a v_out = 0;  // accumulator for v[i]
+        u32 b = bits[i]; // bits left to read for v[i]
+        u32 vidx = 0;    // bits written to v[i]
+
+        while (b) {
+            u64a read = *in >> used;
+            u32 bits_read = 8 - used;
+
+            if (b <= bits_read) {
+                u64a mask = read & ((1U << b) - 1);
+                v_out |= mask << vidx;
+                vidx += b;
+                used += b;
+                b = 0;
+                if (used < 8) {
+                    continue; // more from this *in
+                }
+            } else {
+                v_out |= read << vidx;
+                vidx += bits_read;
+                b -= bits_read;
+            }
+
+            used = 0;
+            in++;
+        }
+
+        v[i] = v_out;
+    }
+}
+
+#endif // UTIL_PACK_BITS_H

src/util/partial_store.h

@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PARTIAL_STORE_H
+#define PARTIAL_STORE_H
+
+#include "ue2common.h"
+#include "unaligned.h"
+
+/* loads/stores the least significant bytes of the values. */
+
+static really_inline
+void partial_store_u32(void *ptr, u32 value, u32 numBytes) {
+    assert(numBytes <= 4);
+    switch (numBytes) {
+    case 4:
+        unaligned_store_u32(ptr, value);
+        break;
+    case 3:
+        unaligned_store_u16(ptr, (u16)value);
+        *((u8 *)ptr + 2) = (u8)(value >> 16);
+        break;
+    case 2:
+        unaligned_store_u16(ptr, (u16)value);
+        break;
+    case 1:
+        *(u8 *)ptr = (u8)value;
+        break;
+    case 0:
+        break;
+    }
+}
+
+static really_inline
+u32 partial_load_u32(const void *ptr, u32 numBytes) {
+    u32 value;
+    assert(numBytes <= 4);
+    switch (numBytes) {
+    case 4:
+        value = unaligned_load_u32(ptr);
+        return value;
+    case 3:
+        value = unaligned_load_u16(ptr);
+        value |= ((u32)(*((const u8 *)ptr + 2)) << 16);
+        return value;
+    case 2:
+        value = unaligned_load_u16(ptr);
+        return value;
+    case 1:
+        value = *(const u8 *)ptr;
+        return value;
+    case 0:
+        break;
+    }
+
+    return 0;
+}
+
+static really_inline
+void partial_store_u64a(void *ptr, u64a value, u32 numBytes) {
+    assert(numBytes <= 8);
+    switch (numBytes) {
+    case 8:
+        unaligned_store_u64a(ptr, value);
+        break;
+    case 7:
+        unaligned_store_u32(ptr, (u32)value);
+        unaligned_store_u16((u8 *)ptr + 4, (u16)(value >> 32));
+        *((u8 *)ptr + 6) = (u8)(value >> 48);
+        break;
+    case 6:
+        unaligned_store_u32(ptr, (u32)value);
+        unaligned_store_u16((u8 *)ptr + 4, (u16)(value >> 32));
+        break;
+    case 5:
+        unaligned_store_u32(ptr, (u32)value);
+        *((u8 *)ptr + 4) = (u8)(value >> 32);
+        break;
+    case 4:
+        unaligned_store_u32(ptr, (u32)value);
+        break;
+    case 3:
+        unaligned_store_u16(ptr, (u16)value);
+        *((u8 *)ptr + 2) = (u8)(value >> 16);
+        break;
+    case 2:
+        unaligned_store_u16(ptr, (u16)value);
+        break;
+    case 1:
+        *(u8 *)ptr = (u8)value;
+        break;
+    case 0:
+        break;
+    }
+}
+
+static really_inline
+u64a partial_load_u64a(const void *ptr, u32 numBytes) {
+    u64a value;
+    assert(numBytes <= 8);
+    switch (numBytes) {
+    case 8:
+        value = unaligned_load_u64a(ptr);
+        return value;
+    case 7:
+        value = unaligned_load_u32(ptr);
+        value |= (u64a)unaligned_load_u16((const u8 *)ptr + 4) << 32;
+        value |= (u64a)(*((const u8 *)ptr + 6)) << 48;
+        return value;
+    case 6:
+        value = unaligned_load_u32(ptr);
+        value |= (u64a)unaligned_load_u16((const u8 *)ptr + 4) << 32;
+        return value;
+    case 5:
+        value = unaligned_load_u32(ptr);
+        value |= (u64a)(*((const u8 *)ptr + 4)) << 32;
+        return value;
+    case 4:
+        value = unaligned_load_u32(ptr);
+        return value;
+    case 3:
+        value = unaligned_load_u16(ptr);
+        value |= (u64a)(*((const u8 *)ptr + 2)) << 16;
+        return value;
+    case 2:
+        value = unaligned_load_u16(ptr);
+        return value;
+    case 1:
+        value = *(const u8 *)ptr;
+        return value;
+    case 0:
+        break;
+    }
+
+    return 0;
+}
+
+#endif

src/util/partitioned_set.h

@@ -0,0 +1,266 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PARTITIONED_SET_H
+#define PARTITIONED_SET_H
+
+#include "container.h"
+#include "ue2_containers.h"
+#include "ue2common.h"
+
+#include <algorithm>
+#include <vector>
+
+#include <boost/core/noncopyable.hpp>
+#include <boost/dynamic_bitset.hpp>
+
+namespace ue2 {
+
+static constexpr size_t INVALID_SUBSET = ~(size_t)0;
+
+/**
+ * partition_set represents a partitioning of a set of integers [0, n) into
+ * disjoint non-empty subsets.
+ *
+ * The subsets themselves are also indexed by integers.
+ *
+ * The underlying integer type for the set members is parameterized.
+ */
+
+template<typename T>
+class partitioned_set : boost::noncopyable {
+public:
+    class subset {
+    public:
+        typedef typename std::vector<T>::const_iterator const_iterator;
+
+        size_t size() const {
+            assert(members.size());
+            return members.size();
+        }
+
+        const_iterator begin() const {
+            return members.begin();
+        }
+
+        const_iterator end() const {
+            return members.end();
+        }
+
+    private:
+        std::vector<T> members; /**< sorted members of the subset */
+
+        friend class partitioned_set;
+    };
+
+    /** returns the number of subsets in the partition */
+    size_t size() const { return subsets.size(); }
+
+    /** returns the subset with the given index */
+    const subset &operator[](size_t subset_index) const {
+        assert(subset_index < size());
+        return subsets[subset_index];
+    }
+
+    /**
+     * Splits the subset with the given subset_index based on whether its
+     * members are also members of the splitter set.
+     *
+     * The smaller of the intersection and difference is placed into a new
+     * subset, the index of which is returned. The larger part remains with the
+     * subset index.
+     *
+     * If the set was not split (due to there being no overlap with splitter or
+     * being a complete subset), INVALID_SUBSET is returned.
+     */
+    size_t split(size_t subset_index,
+                 const typename ue2::flat_set<T> &splitter) {
+        assert(!splitter.empty());
+        if (splitter.empty()) {
+            return INVALID_SUBSET;
+        }
+
+        subset &orig = subsets[subset_index];
+
+        assert(orig.size());
+
+        split_temp_diff.clear();
+        split_temp_inter.clear();
+
+        auto sp_it = splitter.begin();
+        auto sp_e = splitter.end();
+
+        /* subset members are always in sorted order. */
+        assert(std::is_sorted(orig.members.begin(), orig.members.end()));
+
+        if (orig.members.back() < *sp_it) {
+            /* first splitter is greater than all our members */
+            return INVALID_SUBSET;
+        }
+
+        if (orig.members.front() > *splitter.rbegin()) {
+            /* last splitter is less than all our members */
+            return INVALID_SUBSET;
+        }
+
+        for (auto it = orig.members.begin(); it != orig.members.end(); ++it) {
+            T member = *it;
+            assert(member < member_to_subset.size());
+
+            while (sp_it != sp_e && *sp_it < member) {
+                ++sp_it;
+            }
+            if (sp_it == sp_e) {
+                split_temp_diff.insert(split_temp_diff.end(), it,
+                                       orig.members.end());
+                break;
+            }
+
+            if (*sp_it > member) {
+                split_temp_diff.push_back(member);
+            } else {
+                split_temp_inter.push_back(member);
+            }
+        }
+
+        assert(split_temp_diff.size() + split_temp_inter.size() == orig.size());
+
+        if (split_temp_inter.empty()) {
+            assert(split_temp_diff == orig.members);
+            return INVALID_SUBSET;
+        }
+
+        if (split_temp_diff.empty()) {
+            assert(split_temp_inter == orig.members);
+            return INVALID_SUBSET;
+        }
+
+        assert(MIN(split_temp_inter[0], split_temp_diff[0]) == orig.members[0]);
+
+        /* work out which is the bigger half */
+        std::vector<T> *big;
+        std::vector<T> *small;
+        if (split_temp_diff.size() > split_temp_inter.size()) {
+            big = &split_temp_diff;
+            small = &split_temp_inter;
+        } else {
+            big = &split_temp_inter;
+            small = &split_temp_diff;
+        }
+
+        /* larger subset replaces the input subset */
+        std::vector<T> temp_i;
+        insert(&temp_i, temp_i.end(), *big);
+        orig.members.swap(temp_i);
+
+        /* smaller subset is placed in the new subset  */
+        size_t new_index = subsets.size();
+        subsets.push_back(subset());
+        insert(&subsets.back().members, subsets.back().members.end(), *small);
+
+        for (const auto &e : *small) {
+            member_to_subset[e] = new_index;
+        }
+
+        return new_index;
+    }
+
+    /**
+     * Returns all subsets which have a member in keys.
+     */
+    void find_overlapping(const typename ue2::flat_set<T> &keys,
+                          std::vector<size_t> *containing) const {
+        boost::dynamic_bitset<> seen(subsets.size()); // all zero by default.
+
+        for (const auto &key : keys) {
+            assert(key < member_to_subset.size());
+            size_t sub = member_to_subset[key];
+            assert(sub < subsets.size());
+            seen.set(sub);
+        }
+
+        for (size_t i = seen.find_first(); i != seen.npos;
+             i = seen.find_next(i)) {
+            containing->push_back(i);
+        }
+    }
+
+    /**
+     * Creates a partitioned set containing elements [0, state_to_subset.size() )
+     *
+     * The initial subset that an element belongs to is given by the
+     * corresponding entry in state_to_subset. The subsets should be identified
+     * by a dense range of indices starting from 0.
+     */
+    explicit partitioned_set(const std::vector<size_t> &state_to_subset) {
+        assert(!state_to_subset.empty());
+
+        subsets.reserve(state_to_subset.size());
+        member_to_subset.resize(state_to_subset.size());
+
+        split_temp_inter.reserve(state_to_subset.size());
+        split_temp_diff.reserve(state_to_subset.size());
+
+        size_t subset_count = 0;
+        for (const auto &sub : state_to_subset) {
+            assert(sub != INVALID_SUBSET);
+            ENSURE_AT_LEAST(&subset_count, sub + 1);
+        }
+        assert(subset_count <= state_to_subset.size());
+
+        subsets.resize(subset_count);
+        for (size_t i = 0; i < state_to_subset.size(); i++) {
+             /* ensure that our underlying type is big enough to hold all our
+              * set members */
+            assert(i == (size_t)(T)i);
+
+            size_t sub = state_to_subset[i];
+            assert(sub < subsets.size());
+
+            member_to_subset[i] = sub;
+            subsets[sub].members.push_back(i);
+        }
+
+        /* none of the subsets should be empty */
+        assert(std::all_of(subsets.begin(), subsets.end(),
+                           [](const subset &sub){ return sub.size() > 0; }));
+    }
+
+private:
+    std::vector<size_t> member_to_subset;
+    std::vector<subset> subsets;
+
+    std::vector<T> split_temp_inter; /**< used internally by split to hold the
+                                      * intersection. */
+    std::vector<T> split_temp_diff; /**< used internally by split to hold the
+                                     * set difference. */
+};
+
+} // namespace
+
+#endif

src/util/popcount.h

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Platform specific popcount functions
+ */
+
+#ifndef POPCOUNT_H_075D843B4545B6
+#define POPCOUNT_H_075D843B4545B6
+
+#include "ue2common.h"
+
+// We have a native popcount where the compiler has defined __POPCNT__.
+#if defined(__POPCNT__)
+#define HAVE_POPCOUNT_INSTR
+#endif
+
+#if defined(_WIN32) && defined(__AVX__) // TODO: fix win preproc
+#define HAVE_POPCOUNT_INSTR
+#define __builtin_popcount __popcnt
+#define __builtin_popcountll __popcnt64
+#endif
+
+static really_inline
+u32 popcount32(u32 x) {
+#if defined(HAVE_POPCOUNT_INSTR)
+    // Single-instruction builtin.
+    return (u32)__builtin_popcount(x);
+#else
+    // Fast branch-free version from bit-twiddling hacks as most Intel
+    // processors do not have a POPCNT instruction.
+    x -= (x >> 1) & 0x55555555;
+    x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
+    return (((x + (x >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
+#endif
+}
+
+static really_inline
+u32 popcount64(u64a x) {
+#if defined(HAVE_POPCOUNT_INSTR)
+    // Single-instruction builtin.
+    return (u32)__builtin_popcountll(x);
+#elif defined(ARCH_X86_64)
+    // Fast branch-free version from bit-twiddling hacks as most Intel
+    // processors do not have a POPCNT instruction.
+    x -= (x >> 1) & 0x5555555555555555;
+    x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333);
+    x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f;
+    return (x * 0x0101010101010101) >> 56;
+#else
+    // Synthesise from two 32-bit cases.
+    return popcount32(x >> 32) + popcount32(x);
+#endif
+}
+
+#endif /* POPCOUNT_H_075D843B4545B6 */
+

src/util/pqueue.h

@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PQUEUE_H
+#define PQUEUE_H
+
+#include "ue2common.h"
+
+static really_inline u32
+pq_left(u32 i) {
+    return (i << 1) + 1;
+}
+
+static really_inline u32
+pq_right(u32 i) {
+    return (i << 1) + 2;
+}
+
+static really_inline
+u32 pq_parent(u32 i) {
+    return (i - 1) >> 1;
+}
+
+static really_inline
+void pq_sift(PQ_T *items, u32 start, u32 end) {
+    u32 j = start;
+    PQ_T j_temp = items[j];
+
+    while (pq_left(j) < end) {
+        u32 max_child;
+
+        if (pq_right(j) < end && PQ_COMP(items, pq_right(j), pq_left(j))) {
+            max_child = pq_right(j);
+        } else {
+            max_child = pq_left(j);
+        }
+
+        if (PQ_COMP_B(items, max_child, j_temp)) {
+            items[j] = items[max_child];
+            j = max_child;
+        } else {
+            /* j is already less than its children. We know heap property
+             * is already maintained for children we are done */
+            break;
+        }
+    }
+    items[j] = j_temp;
+}
+
+static really_inline
+PQ_T *pq_top(PQ_T *items) {
+    return items;
+}
+
+static really_inline
+void pq_pop(PQ_T *items, u32 item_count) {
+    item_count--;
+    items[0] = items[item_count];
+    pq_sift(items, 0, item_count);
+}
+
+static really_inline
+void pq_insert(PQ_T *items, u32 item_count, PQ_T new_item) {
+    u32 pos = item_count;
+    while (pos) {
+        u32 parent = pq_parent(pos);
+        if (!PQ_COMP_B(items, parent, new_item)) {
+            items[pos] = items[parent];
+            pos = parent;
+        } else {
+            break;
+        }
+    }
+    items[pos] = new_item;
+}
+
+static really_inline
+void pq_replace_top(PQ_T *items, u32 item_count, PQ_T new_item) {
+    items[0] = new_item;
+    pq_sift(items, 0, item_count);
+}
+
+#endif
+

src/util/queue_index_factory.h

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief QueueIndexFactory used to hand out NFA queues at compile time.
+ */
+#ifndef UTIL_QUEUE_INDEX_FACTORY_H
+#define UTIL_QUEUE_INDEX_FACTORY_H
+
+#include "ue2common.h"
+
+#include <boost/core/noncopyable.hpp>
+
+namespace ue2 {
+
+class QueueIndexFactory : boost::noncopyable {
+public:
+    QueueIndexFactory() : val(0) {}
+    u32 get_queue() { return val++; }
+    u32 allocated_count() const { return val; }
+
+private:
+    u32 val;
+};
+
+} // namespace ue2
+
+#endif // UTIL_QUEUE_INDEX_FACTORY_H

src/util/report.cpp

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "internal_report.h"
+#include "report.h"
+#include "report_manager.h"
+
+namespace ue2 {
+
+void writeInternalReport(const Report &report, const ReportManager &rm,
+                         internal_report *ir) {
+    assert(ir);
+    assert(ISALIGNED(ir));
+
+    ir->type = report.type;
+    ir->hasBounds = report.hasBounds() ? 1 : 0;
+    ir->quashSom = report.quashSom ? 1 : 0;
+    ir->minOffset = report.minOffset;
+    ir->maxOffset = report.maxOffset;
+    ir->minLength = report.minLength;
+    ir->ekey = report.ekey;
+    ir->offsetAdjust = report.offsetAdjust;
+    ir->onmatch = report.onmatch;
+
+    switch (report.type) {
+    case INTERNAL_ROSE_CHAIN:
+        ir->aux.topSquashDistance = report.topSquashDistance;
+        break;
+    case EXTERNAL_CALLBACK_SOM_REV_NFA:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
+        ir->aux.revNfaIndex = report.revNfaIndex;
+        break;
+    default:
+        ir->aux.somDistance = report.somDistance;
+        break;
+    }
+
+    // Dedupe keys are managed by ReportManager.
+    ir->dkey = rm.getDkey(report);
+}
+
+} // namespace ue2

src/util/report.h

@@ -0,0 +1,270 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Report structure used to manage data associated with a report at
+ * compile time.
+ */
+
+#ifndef UTIL_REPORT_H
+#define UTIL_REPORT_H
+
+#include "internal_report.h"
+#include "order_check.h"
+#include "ue2common.h"
+
+#include <cassert>
+
+struct internal_report;
+
+namespace ue2 {
+
+class ReportManager;
+
+/**
+ * \brief All the data we use for handling a match.
+ *
+ * Includes extparam constraints and bounds, exhaustion/dedupe keys, offset
+ * adjustment and SOM information.
+ *
+ * The data in this structure is converted into an \ref internal_report in the
+ * bytecode.
+ */
+struct Report {
+    Report(u8 type_in, u32 onmatch_in) : type(type_in), onmatch(onmatch_in) {
+        assert(type <= INTERNAL_ROSE_CHAIN);
+    }
+
+    /** \brief True if this report has bounds from extended parameters, i.e.
+     * min offset, max offset, min length. */
+    bool hasBounds() const {
+        return minOffset > 0 || maxOffset < MAX_OFFSET || minLength > 0;
+    }
+
+    /** \brief from EXTERNAL_ and INTERNAL_ defined in internal_report.h. */
+    u8 type;
+
+    /** \brief use SOM for minLength, but don't report it to user callback. */
+    bool quashSom = false;
+
+    /** \brief min offset in the stream at which this report can match. */
+    u64a minOffset = 0;
+
+    /** \brief max offset in the stream at which this report can match. */
+    u64a maxOffset = MAX_OFFSET;
+
+    /** \brief min match length (start of match to current offset) */
+    u64a minLength = 0;
+
+    /** \brief Exhaustion key.
+     *
+     * If exhaustible, the ekey to check before reporting a match.
+     * Additionally after reporting a match the ekey will be set. If not
+     * exhaustible, this will be INVALID_EKEY. */
+    u32 ekey = INVALID_EKEY;
+
+    /** \brief Adjustment to add to the match offset when we report a match.
+     *
+     * This is usually used for reports attached to states that form part of a
+     * zero-width assertion, like '$'. */
+    s32 offsetAdjust = 0;
+
+    /** \brief Match report ID, for external reports.
+     *
+     * - external callback -> external report id
+     * - internal_som_* -> som loc to modify
+     * - INTERNAL_ROSE_CHAIN -> top event to push on
+     * - otherwise -> target subnfa */
+    u32 onmatch;
+
+    /** \brief Index of the reverse nfa.
+     *
+     * Used by EXTERNAL_CALLBACK_SOM_REV_NFA and
+     * INTERNAL_SOM_LOC_SET_SOM_REV_NFA*.
+     */
+    u32 revNfaIndex = 0;
+
+    /** \brief SOM distance value, use varies according to type.
+     *
+     *  - for EXTERNAL_CALLBACK_SOM_REL, from-offset is this many bytes
+     *    before the to-offset.
+     *  - for EXTERNAL_CALLBACK_SOM_ABS, set from-offset to this value.
+     *  - for INTERNAL_SOM_LOC_COPY*, som location read_from.
+     */
+    u64a somDistance = 0;
+
+    /** \brief Number of bytes behind us that we are allowed to squash
+     * identical top events on the queue.
+     *
+     * Used by INTERNAL_ROSE_CHAIN.
+     */
+    u64a topSquashDistance = 0;
+};
+
+static inline
+bool isInternalSomReport(const Report &r) {
+    switch (r.type) {
+    case INTERNAL_SOM_LOC_SET:
+    case INTERNAL_SOM_LOC_SET_IF_UNSET:
+    case INTERNAL_SOM_LOC_SET_IF_WRITABLE:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
+    case INTERNAL_SOM_LOC_COPY:
+    case INTERNAL_SOM_LOC_COPY_IF_WRITABLE:
+    case INTERNAL_SOM_LOC_MAKE_WRITABLE:
+    case INTERNAL_SOM_LOC_SET_FROM:
+    case INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE:
+        return true;
+    case EXTERNAL_CALLBACK:
+    case EXTERNAL_CALLBACK_SOM_REL:
+    case EXTERNAL_CALLBACK_SOM_STORED:
+    case EXTERNAL_CALLBACK_SOM_ABS:
+    case EXTERNAL_CALLBACK_SOM_REV_NFA:
+    case INTERNAL_ROSE_CHAIN:
+        return false;
+    default:
+        break; // fall through
+    }
+    assert(0); // unknown?
+    return false;
+}
+
+static inline
+bool isExternalReport(const Report &r) {
+    switch (r.type) {
+    case INTERNAL_SOM_LOC_SET:
+    case INTERNAL_SOM_LOC_SET_IF_UNSET:
+    case INTERNAL_SOM_LOC_SET_IF_WRITABLE:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_UNSET:
+    case INTERNAL_SOM_LOC_SET_SOM_REV_NFA_IF_WRITABLE:
+    case INTERNAL_SOM_LOC_COPY:
+    case INTERNAL_SOM_LOC_COPY_IF_WRITABLE:
+    case INTERNAL_SOM_LOC_MAKE_WRITABLE:
+    case INTERNAL_SOM_LOC_SET_FROM:
+    case INTERNAL_SOM_LOC_SET_FROM_IF_WRITABLE:
+    case INTERNAL_ROSE_CHAIN:
+        return false;
+    case EXTERNAL_CALLBACK:
+    case EXTERNAL_CALLBACK_SOM_REL:
+    case EXTERNAL_CALLBACK_SOM_STORED:
+    case EXTERNAL_CALLBACK_SOM_ABS:
+    case EXTERNAL_CALLBACK_SOM_REV_NFA:
+        return true;
+    default:
+        break; // fall through
+    }
+    assert(0); // unknown?
+    return true;
+}
+
+static inline
+bool operator<(const Report &a, const Report &b) {
+    ORDER_CHECK(type);
+    ORDER_CHECK(quashSom);
+    ORDER_CHECK(ekey);
+    ORDER_CHECK(offsetAdjust);
+    ORDER_CHECK(onmatch);
+    ORDER_CHECK(minOffset);
+    ORDER_CHECK(maxOffset);
+    ORDER_CHECK(minLength);
+    ORDER_CHECK(somDistance);
+    ORDER_CHECK(revNfaIndex);
+    ORDER_CHECK(topSquashDistance);
+    return false;
+}
+
+static inline
+Report makeECallback(u32 report, s32 offsetAdjust, u32 ekey) {
+    Report ir(EXTERNAL_CALLBACK, report);
+    ir.offsetAdjust = offsetAdjust;
+    ir.ekey = ekey;
+    return ir;
+}
+
+static inline
+Report makeCallback(u32 report, s32 offsetAdjust) {
+    return makeECallback(report, offsetAdjust, INVALID_EKEY);
+}
+
+static inline
+Report makeSomRelativeCallback(u32 report, s32 offsetAdjust, u64a distance) {
+    Report ir(EXTERNAL_CALLBACK_SOM_REL, report);
+    ir.offsetAdjust = offsetAdjust;
+    ir.ekey = INVALID_EKEY;
+    ir.somDistance = distance;
+    return ir;
+}
+
+static inline
+Report makeRoseTrigger(u32 event, u64a squashDistance) {
+    Report ir(INTERNAL_ROSE_CHAIN, event);
+    ir.ekey = INVALID_EKEY;
+    ir.topSquashDistance = squashDistance;
+    return ir;
+}
+
+/** simple exhaustible: exhaustible and if the first attempted match does not
+ * succeed, no later matches will succeed either  */
+static inline
+bool isSimpleExhaustible(const Report &ir) {
+    if (ir.ekey == INVALID_EKEY) {
+        return false;
+    }
+
+    if (ir.hasBounds() && (ir.minOffset || ir.minLength)) {
+        return false;
+    }
+
+    if (!isExternalReport(ir)) {
+        return false;
+    }
+
+    return true;
+}
+
+/** True if this report requires some of the more esoteric processing in the
+ * rose adaptor, rather than just firing a callback or doing SOM handling.
+ */
+static inline
+bool isComplexReport(const Report &ir) {
+    if (ir.hasBounds() || ir.ekey != INVALID_EKEY) {
+        return true;
+    }
+
+    return false;
+}
+
+/** \brief Write the given Report into an internal_report structure. */
+void writeInternalReport(const Report &report, const ReportManager &rm,
+                         internal_report *ir);
+
+} // namespace
+
+#endif // UTIL_REPORT_H

src/util/report_manager.cpp

@@ -0,0 +1,250 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief ReportManager: tracks Report structures, exhaustion and dedupe keys.
+ */
+#include "grey.h"
+#include "report_manager.h"
+#include "ue2common.h"
+#include "nfagraph/ng.h"
+#include "rose/rose_build.h"
+#include "util/compile_error.h"
+#include "util/container.h"
+
+#include <deque>
+#include <map>
+#include <sstream>
+#include <vector>
+
+using namespace std;
+
+namespace ue2 {
+
+ReportManager::ReportManager(const Grey &g)
+    : grey(g), freeEIndex(0), global_exhaust(true) {}
+
+u32 ReportManager::getInternalId(const Report &ir) {
+    auto it = reportIdToInternalMap.find(ir);
+    if (it != reportIdToInternalMap.end()) {
+        DEBUG_PRINTF("existing report %zu\n", it->second);
+        return it->second;
+    }
+
+    // Construct a new internal report and assign it a ReportID.
+
+    if (numReports() >= grey.limitReportCount) {
+        throw ResourceLimitError();
+    }
+
+    u32 size = reportIds.size();
+    reportIds.push_back(ir);
+    reportIdToInternalMap[ir] = size;
+    DEBUG_PRINTF("new report %u\n", size);
+    return size;
+}
+
+const Report &ReportManager::getReport(u32 id) const {
+    assert(id < reportIds.size());
+    return reportIds.at(id);
+}
+
+size_t ReportManager::numReports() const {
+    return reportIds.size();
+}
+
+u32 ReportManager::getExhaustibleKey(u32 a) {
+    auto it = toExhaustibleKeyMap.find(a);
+    if (it == toExhaustibleKeyMap.end()) {
+        // get size before assigning to avoid wacky LHS shenanigans
+        u32 size = toExhaustibleKeyMap.size();
+        bool inserted;
+        tie(it, inserted) = toExhaustibleKeyMap.emplace(s64a{a}, size);
+        assert(inserted);
+    }
+
+    DEBUG_PRINTF("%lld -> ekey %u\n", it->first, it->second);
+    return it->second;
+}
+
+u32 ReportManager::getUnassociatedExhaustibleKey(void) {
+    u32 rv = toExhaustibleKeyMap.size();
+    bool inserted;
+    map<s64a, u32>::const_iterator it;
+    tie(it, inserted) = toExhaustibleKeyMap.emplace(--freeEIndex, rv);
+    assert(inserted);
+    assert(it->second == rv);
+
+    return rv;
+}
+
+u32 ReportManager::numDkeys() const {
+    DEBUG_PRINTF("%zu dkeys\n", reportIdToDedupeKey.size());
+    return reportIdToDedupeKey.size();
+}
+
+u32 ReportManager::numEkeys() const {
+    return (u32) toExhaustibleKeyMap.size();
+}
+
+bool ReportManager::patternSetCanExhaust() const {
+    return global_exhaust && !toExhaustibleKeyMap.empty();
+}
+
+vector<ReportID> ReportManager::getDkeyToReportTable() const {
+    vector<ReportID> rv(reportIdToDedupeKey.size());
+
+    for (const auto &m : reportIdToDedupeKey) {
+        assert(m.second < rv.size());
+        rv[m.second] = m.first;
+    }
+
+    return rv;
+}
+
+void ReportManager::assignDkeys(const RoseBuild *rose) {
+    unique_ptr<RoseDedupeAux> dedupe = rose->generateDedupeAux();
+
+    DEBUG_PRINTF("assigning...\n");
+
+    map<u32, set<ReportID>> ext_to_int;
+
+    for (u32 i = 0; i < reportIds.size(); i++) {
+        const Report &ir = reportIds[i];
+
+        /* need to populate dkey */
+        if (isExternalReport(ir)) {
+            ext_to_int[ir.onmatch].insert(i);
+        }
+    }
+
+    for (const auto &m : ext_to_int) {
+        u32 ext = m.first;
+
+        if (!dedupe->requiresDedupeSupport(m.second)) {
+            DEBUG_PRINTF("%u does not require dedupe\n", ext);
+            continue; /* no dedupe required for this set */
+        }
+
+        u32 dkey = reportIdToDedupeKey.size();
+        reportIdToDedupeKey[ext] = dkey;
+        DEBUG_PRINTF("ext=%u -> dkey=%u\n", ext, dkey);
+    }
+}
+
+u32 ReportManager::getDkey(const Report &r) const {
+    if (!isExternalReport(r)) {
+        return ~u32{0};
+    }
+
+    auto it = reportIdToDedupeKey.find(r.onmatch);
+    if (it == reportIdToDedupeKey.end()) {
+        return ~u32{0};
+    }
+    return it->second;
+}
+
+void ReportManager::registerExtReport(ReportID id,
+                                      const external_report_info &ext) {
+    if (contains(externalIdMap, id)) {
+        const external_report_info &eri = externalIdMap.at(id);
+        if (eri.highlander != ext.highlander) {
+            /* we have a problem */
+            ostringstream out;
+            out << "Expression (index " << ext.first_pattern_index
+                << ") with match ID " << id << " ";
+            if (!ext.highlander) {
+                out << "did not specify ";
+            } else {
+                out << "specified ";
+            }
+            out << "HS_FLAG_SINGLEMATCH whereas previous expression (index "
+                << eri.first_pattern_index << ") with the same match ID did";
+            if (ext.highlander) {
+                out << " not";
+            }
+            out << ".";
+            throw CompileError(ext.first_pattern_index, out.str());
+        }
+    } else {
+        externalIdMap.emplace(id, ext);
+    }
+
+    // Any non-highlander pattern will render us not globally exhaustible.
+    if (!ext.highlander) {
+        global_exhaust = false;
+    }
+}
+
+Report ReportManager::getBasicInternalReport(const NGWrapper &g, s32 adj) {
+    /* validate that we are not violating highlander constraints, this will
+     * throw a CompileError if so. */
+    registerExtReport(g.reportId,
+                      external_report_info(g.highlander, g.expressionIndex));
+
+    /* create the internal report */
+    u32 ekey = INVALID_EKEY;
+    if (g.highlander) {
+        /* all patterns with the same report id share an ekey */
+        ekey = getExhaustibleKey(g.reportId);
+    }
+
+    return makeECallback(g.reportId, adj, ekey);
+}
+
+static
+void ekeysUnion(std::set<u32> *ekeys, u32 more) {
+    if (!ekeys->empty()) {
+        if (more == INVALID_EKEY) {
+            ekeys->clear();
+        } else {
+            ekeys->insert(more);
+        }
+    }
+}
+
+set<u32> reportsToEkeys(const set<ReportID> &reports, const ReportManager &rm) {
+    assert(!reports.empty());
+
+    set<u32> ekeys;
+
+    for (auto it = reports.begin(), ite = reports.end(); it != ite; ++it) {
+        u32 e = rm.getReport(*it).ekey;
+        if (it == reports.begin()) {
+            if (e != INVALID_EKEY) {
+                ekeys.insert(e);
+            }
+        } else {
+            ekeysUnion(&ekeys, e);
+        }
+    }
+
+    return ekeys;
+}
+
+} // namespace ue2

src/util/report_manager.h

@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief ReportManager: tracks Report structures, exhaustion and
+ * dedupe keys.
+ */
+
+#ifndef REPORT_MANAGER_H
+#define REPORT_MANAGER_H
+
+#include "ue2common.h"
+#include "util/compile_error.h"
+#include "util/report.h"
+
+#include <map>
+#include <set>
+#include <vector>
+#include <boost/core/noncopyable.hpp>
+
+namespace ue2 {
+
+struct Grey;
+class RoseBuild;
+class NGWrapper;
+
+struct external_report_info {
+    external_report_info(bool h, u32 fpi)
+    : highlander(h), first_pattern_index(fpi) { }
+    const bool highlander;
+    const u32 first_pattern_index;
+};
+
+/** \brief Tracks Report structures, exhaustion and dedupe keys. */
+class ReportManager : boost::noncopyable {
+public:
+    explicit ReportManager(const Grey &g);
+
+    /** \brief Fetch the ID associated with the given Report. */
+    u32 getInternalId(const Report &r);
+
+    /** \brief Fetch the Report associated with \a id. */
+    const Report &getReport(u32 id) const;
+
+    /** \brief Total number of reports. */
+    size_t numReports() const;
+
+    /** \brief Return an unused exhaustion key (the next available one). */
+    u32 getUnassociatedExhaustibleKey(void);
+
+    /** \brief Total number of dedupe keys. */
+    u32 numDkeys() const;
+
+    /** \brief Total number of exhaustion keys. */
+    u32 numEkeys() const;
+
+    /** \brief True if the pattern set can exhaust (i.e. all patterns are
+     * highlander). */
+    bool patternSetCanExhaust() const;
+
+    void assignDkeys(const RoseBuild *rose);
+
+    std::vector<ReportID> getDkeyToReportTable() const;
+
+    /** \brief Return a const reference to the table of Report
+     * structures. */
+    const std::vector<Report> &reports() const { return reportIds; }
+
+    /**
+     * Get a simple internal report corresponding to the wrapper. An ekey will
+     * be setup as required.
+     *
+     * Note: this function may throw a CompileError if constraints on external
+     * match id are violated (mixed highlander status for example).
+     */
+    Report getBasicInternalReport(const NGWrapper &g, s32 adj = 0);
+
+    /** \brief Register an external report and validate that we are not
+     * violating highlander constraints (which will cause an exception to be
+     * thrown). */
+    void registerExtReport(ReportID id, const external_report_info &ext);
+
+    /** \brief Fetch the ekey associated with the given expression index,
+     * assigning one if necessary. */
+    u32 getExhaustibleKey(u32 expressionIndex);
+
+    /** \brief Fetch the dedupe key associated with the given report. Returns
+     * ~0U if no dkey is needed. */
+    u32 getDkey(const Report &r) const;
+
+private:
+    /** \brief Grey box ref, for checking resource limits. */
+    const Grey &grey;
+
+    /** \brief Report structures, indexed by ID. */
+    std::vector<Report> reportIds;
+
+    /** \brief Mapping from Report to ID (inverse of \ref reportIds
+     * vector).  */
+    std::map<Report, size_t> reportIdToInternalMap;
+
+    /** \brief Mapping from ReportID to dedupe key. */
+    std::map<ReportID, u32> reportIdToDedupeKey;
+
+    /** \brief Mapping from external match ids to information about that
+     * id. */
+    std::map<ReportID, external_report_info> externalIdMap;
+
+    /** \brief Mapping from expression index to exhaustion key. */
+    std::map<s64a, u32> toExhaustibleKeyMap;
+
+    /** \brief Unallocated expression index, used for \ref
+     * getUnassociatedExhaustibleKey.
+     *
+     * TODO: work out why this is signed.
+     */
+    s64a freeEIndex;
+
+    /** \brief True if database is globally exhaustible (all patterns must be
+     * highlander for this to be the case). */
+    bool global_exhaust;
+};
+
+std::set<u32> reportsToEkeys(const std::set<ReportID> &reports,
+                             const ReportManager &rm);
+
+} // namespace ue2
+
+#endif

src/util/scatter.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UTIL_SCATTER_H
+#define UTIL_SCATTER_H
+
+#include "ue2common.h"
+
+#define SCATTER_STRUCT(t) \
+    struct scatter_unit_##t { u32 offset; t val; };
+
+SCATTER_STRUCT(u64a)
+SCATTER_STRUCT(u32)
+SCATTER_STRUCT(u16)
+SCATTER_STRUCT(u8)
+
+struct scatter_full_plan {
+    u32 s_u64a_offset;
+    u32 s_u64a_count;
+    u32 s_u32_offset;
+    u32 s_u32_count;
+    u32 s_u16_offset;
+    u32 s_u16_count;
+    u32 s_u8_count;
+    u32 s_u8_offset;
+};
+
+#undef SCATTER_STRUCT
+
+#endif

src/util/scatter_runtime.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UTIL_SCATTER_RUNTIME_H
+#define UTIL_SCATTER_RUNTIME_H
+
+#include "scatter.h"
+
+#include "uniform_ops.h"
+
+#define SCATTER_DEF(t)                                                        \
+static really_inline                                                          \
+void scatter_##t(void *out, const struct scatter_unit_##t *plan, u32 count) { \
+    for (u32 i = 0; i < count; i++) {                                         \
+        const struct scatter_unit_##t *item = plan + i;                       \
+        DEBUG_PRINTF("storing %llu into offset %u\n", (u64a)item->val, \
+                     item->offset);                                     \
+        storeu_##t((char *)out + item->offset, item->val);              \
+    }                                                                         \
+}
+
+SCATTER_DEF(u64a)
+SCATTER_DEF(u32)
+SCATTER_DEF(u16)
+SCATTER_DEF(u8)
+
+#undef SCATTER_DEF
+
+static really_inline
+void scatter(void *out, const void *base, const struct scatter_full_plan *p) {
+#define RUN_SUB(t)                                      \
+    if (p->s_##t##_offset) {                            \
+        assert(p->s_##t##_count);                       \
+        const struct scatter_unit_##t *pp               \
+            = (const void *)(b + p->s_##t##_offset);    \
+        scatter_##t(out, pp, p->s_##t##_count);         \
+    }
+
+    const char *b = base;
+
+    RUN_SUB(u64a);
+    RUN_SUB(u32);
+    RUN_SUB(u16);
+    RUN_SUB(u8);
+
+#undef RUN_SUB
+}
+
+#endif

src/util/shuffle.h

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Naive dynamic shuffles.
+ *
+ * These are written with the assumption that the provided masks are sparsely
+ * populated and never contain more than 32 on bits. Other implementations will
+ * be faster and actually correct if these assumptions don't hold true.
+ */
+
+#ifndef SHUFFLE_H
+#define SHUFFLE_H
+
+#include "config.h"
+#include "bitutils.h"
+#include "simd_utils.h"
+#include "ue2common.h"
+
+#if defined(__BMI2__) || (defined(_WIN32) && defined(__AVX2__))
+#define HAVE_PEXT
+#endif
+
+static really_inline
+u32 shuffleDynamic32(u32 x, u32 mask) {
+#if defined(HAVE_PEXT)
+    // Intel BMI2 can do this operation in one instruction.
+    return _pext_u32(x, mask);
+#endif
+
+    u32 result = 0, num = 1;
+    while (mask != 0) {
+        u32 bit = findAndClearLSB_32(&mask);
+        if (x & (1U << bit)) {
+            assert(num != 0); // more than 32 bits!
+            result |= num;
+        }
+        num <<= 1;
+    }
+    return result;
+}
+
+static really_inline
+u32 shuffleDynamic64(u64a x, u64a mask) {
+#if defined(HAVE_PEXT) && defined(ARCH_64_BIT)
+    // Intel BMI2 can do this operation in one instruction.
+    return _pext_u64(x, mask);
+#endif
+
+    u32 result = 0, num = 1;
+    while (mask != 0) {
+        u32 bit = findAndClearLSB_64(&mask);
+        if (x & (1ULL << bit)) {
+            assert(num != 0); // more than 32 bits!
+            result |= num;
+        }
+        num <<= 1;
+    }
+    return result;
+}
+
+#undef HAVE_PEXT
+
+#endif // SHUFFLE_H

src/util/shuffle_ssse3.h

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SHUFFLE_SSSE3_H
+#define SHUFFLE_SSSE3_H
+
+#include "simd_utils_ssse3.h"
+
+#ifdef DEBUG
+#include "compare.h"
+static really_inline void shufDumpMsk(m128 msk) {
+    u8 * mskAsU8 = (u8 *)&msk;
+    for (int i = 0; i < 16; i++) {
+        u8 c = mskAsU8[i];
+        for (int j = 0; j < 8; j++) {
+            if ((c >> (7-j)) & 0x1)
+                printf("1");
+            else
+                printf("0");
+        }
+        printf(" ");
+    }
+}
+
+static really_inline void shufDumpMskAsChars(m128 msk) {
+    u8 * mskAsU8 = (u8 *)&msk;
+    for (int i = 0; i < 16; i++) {
+        u8 c = mskAsU8[i];
+        if (ourisprint(c))
+            printf("%c",c);
+        else
+            printf(".");
+    }
+}
+#endif
+
+#if !defined(NO_SSSE3)
+static really_inline
+u32 shufflePshufb128(m128 s, const m128 permute, const m128 compare) {
+    m128 shuffled = pshufb(s, permute);
+    m128 compared = and128(shuffled, compare);
+#ifdef DEBUG
+    printf("State:   ");  shufDumpMsk(s);       printf("\n");
+    printf("Permute: ");  shufDumpMsk(permute); printf("\n");
+    printf("Compare: ");  shufDumpMsk(compare); printf("\n");
+    printf("Shuffled: "); shufDumpMsk(shuffled); printf("\n");
+    printf("Compared: "); shufDumpMsk(compared); printf("\n");
+#endif
+    u16 rv = ~cmpmsk8(compared, shuffled);
+    return (u32)rv;
+}
+#endif // NO_SSSE3
+
+#endif // SHUFFLE_SSSE3_H

src/util/simd_types.h

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SIMD_TYPES_H
+#define SIMD_TYPES_H
+
+#include "config.h"
+#include "ue2common.h"
+
+// more recent headers are bestest, but only if we can use them
+#ifdef __cplusplus
+# if defined(HAVE_CXX_X86INTRIN_H)
+#  define USE_X86INTRIN_H
+# endif
+#else // C
+# if defined(HAVE_C_X86INTRIN_H)
+#  define USE_X86INTRIN_H
+# endif
+#endif
+
+#ifdef __cplusplus
+# if defined(HAVE_CXX_INTRIN_H)
+#  define USE_INTRIN_H
+# endif
+#else // C
+# if defined(HAVE_C_INTRIN_H)
+#  define USE_INTRIN_H
+# endif
+#endif
+
+#if defined(USE_X86INTRIN_H)
+#include <x86intrin.h>
+#elif defined(USE_INTRIN_H)
+#include <intrin.h>
+#else
+#error no intrinsics!
+#endif
+
+typedef __m128i m128;
+#if defined(__AVX2__)
+typedef __m256i m256;
+#else
+typedef struct ALIGN_AVX_DIRECTIVE {m128 lo; m128 hi;} m256;
+#endif
+
+// these should align to 16 and 32 respectively
+typedef struct {m128 lo; m128 mid; m128 hi;} m384;
+typedef struct {m256 lo; m256 hi;} m512;
+
+#endif /* SIMD_TYPES_H */
+

src/util/simd_utils_ssse3.h

@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief SIMD primitives specifically for Intel SSSE3 platforms.
+ */
+
+#ifndef SIMD_UTILS_SSSE3_H_E27DF795C9AA02
+#define SIMD_UTILS_SSSE3_H_E27DF795C9AA02
+
+#if !defined(_WIN32) && !defined(__SSSE3__)
+#error SSSE3 instructions must be enabled
+#endif
+
+#include "simd_utils.h"
+#include "ue2common.h"
+
+// we may already have x86intrin.h
+#if !defined(USE_X86INTRIN_H)
+#if defined(HAVE_C_INTRIN_H)
+#include <intrin.h>
+#elif defined(HAVE_TMMINTRIN_H)
+#include <tmmintrin.h> // SSSE3 intrinsics
+#else
+#define I_HAVE_BROKEN_INTRINSICS
+#endif
+#endif
+
+
+#if !defined(I_HAVE_BROKEN_INTRINSICS)
+// newish compilers get this right
+#define palignr(r, l, offset) _mm_alignr_epi8(r, l, offset)
+#else
+// must be inline, even in weak-sauce debug builds.
+// oldish compilers either don't have the intrinsic, or force one arg through memory
+static really_really_inline
+m128 palignr(m128 r, m128 l, const int offset) {
+    __asm__ ("palignr   %2,%1,%0" : "+x"(r) : "x"(l), "i"(offset));
+    return r;
+}
+#endif
+
+
+static really_inline
+m128 pshufb(m128 a, m128 b) {
+    m128 result;
+#if !defined(I_HAVE_BROKEN_INTRINSICS)
+    result = _mm_shuffle_epi8(a, b);
+#else
+    __asm__("pshufb\t%1,%0" : "=x"(result) : "xm"(b), "0"(a));
+#endif
+    return result;
+}
+
+#if defined(__AVX2__)
+
+static really_inline
+m256 vpshufb(m256 a, m256 b) {
+    return _mm256_shuffle_epi8(a, b);
+}
+#if defined(USE_GCC_COMPOUND_STATEMENTS)
+#define vpalignr(r, l, offset) ({                   \
+    m256 res = _mm256_alignr_epi8(r, l, offset);    \
+    res;                                            \
+})
+#else
+static really_inline
+m256 vpalignr(m256 r, m256 l, const int offset) {
+    return _mm256_alignr_epi8(r, l, offset);
+}
+#endif
+
+#else // not __AVX2__
+
+static really_inline
+m256 vpshufb(m256 a, m256 b) {
+    m256 rv;
+    rv.lo = pshufb(a.lo, b.lo);
+    rv.hi = pshufb(a.hi, b.hi);
+    return rv;
+}
+
+/* palignr requires the offset to be an immediate, which we can do with a
+ * compound macro, otherwise we have to enumerate the offsets and hope the
+ * compiler can throw the rest away. */
+#if defined(USE_GCC_COMPOUND_STATEMENTS)
+#define vpalignr(r, l, offset) ({           \
+    m256 res;                               \
+    res.lo = palignr(r.lo, l.lo, offset);   \
+    res.hi = palignr(r.hi, l.hi, offset);   \
+    res;                                    \
+})
+#else
+#define VPALIGN_CASE(N) case N: \
+		res.lo = palignr(r.lo, l.lo, N); \
+		res.hi = palignr(r.hi, l.hi, N); \
+		return res;
+static really_inline
+m256 vpalignr(m256 r, m256 l, const int offset) {
+	m256 res;
+	switch (offset) {
+	VPALIGN_CASE(0)
+	VPALIGN_CASE(1)
+	VPALIGN_CASE(2)
+	VPALIGN_CASE(3)
+	VPALIGN_CASE(4)
+	VPALIGN_CASE(5)
+	VPALIGN_CASE(6)
+	VPALIGN_CASE(7)
+	VPALIGN_CASE(8)
+	VPALIGN_CASE(9)
+	VPALIGN_CASE(10)
+	VPALIGN_CASE(11)
+	VPALIGN_CASE(12)
+	VPALIGN_CASE(13)
+	VPALIGN_CASE(14)
+	VPALIGN_CASE(15)
+	default:
+		assert(0);
+		return zeroes256();
+	}
+}
+#undef VPALIGN_CASE
+#endif
+#endif // __AVX2__
+
+#endif /* SIMD_UTILS_SSSE3_H_E27DF795C9AA02 */
+

src/util/state_compress.c

@@ -0,0 +1,604 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Mask-based state compression, used by the NFA.
+ */
+#include "config.h"
+#include "ue2common.h"
+#include "bitutils.h"
+#include "unaligned.h"
+#include "pack_bits.h"
+#include "partial_store.h"
+#include "popcount.h"
+#include "state_compress.h"
+
+#include <string.h>
+
+/*
+ * 32-bit store/load.
+ */
+
+void storecompressed32(void *ptr, const u32 *x, const u32 *m, u32 bytes) {
+    assert(popcount32(*m) <= bytes * 8);
+
+    u32 v = compress32(*x, *m);
+    partial_store_u32(ptr, v, bytes);
+}
+
+void loadcompressed32(u32 *x, const void *ptr, const u32 *m, u32 bytes) {
+    assert(popcount32(*m) <= bytes * 8);
+
+    u32 v = partial_load_u32(ptr, bytes);
+    *x = expand32(v, *m);
+}
+
+/*
+ * 64-bit store/load.
+ */
+
+void storecompressed64(void *ptr, const u64a *x, const u64a *m, u32 bytes) {
+    assert(popcount64(*m) <= bytes * 8);
+
+    u64a v = compress64(*x, *m);
+    partial_store_u64a(ptr, v, bytes);
+}
+
+void loadcompressed64(u64a *x, const void *ptr, const u64a *m, u32 bytes) {
+    assert(popcount64(*m) <= bytes * 8);
+
+    u64a v = partial_load_u64a(ptr, bytes);
+    *x = expand64(v, *m);
+}
+
+/*
+ * 128-bit store/load.
+ */
+
+#if defined(ARCH_32_BIT)
+static really_inline
+void storecompressed128_32bit(void *ptr, m128 xvec, m128 mvec) {
+    // First, decompose our vectors into 32-bit chunks.
+    u32 x[4];
+    memcpy(x, &xvec, sizeof(xvec));
+    u32 m[4];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    // Count the number of bits of compressed state we're writing out per
+    // chunk.
+    u32 bits[4] = { popcount32(m[0]), popcount32(m[1]),
+                    popcount32(m[2]), popcount32(m[3]) };
+
+    // Compress each 32-bit chunk individually.
+    u32 v[4] = { compress32(x[0], m[0]), compress32(x[1], m[1]),
+                 compress32(x[2], m[2]), compress32(x[3], m[3]) };
+
+    // Write packed data out.
+    pack_bits_32(ptr, v, bits, 4);
+}
+#endif
+
+#if defined(ARCH_64_BIT)
+static really_inline
+void storecompressed128_64bit(void *ptr, m128 xvec, m128 mvec) {
+    // First, decompose our vectors into 64-bit chunks.
+    u64a x[2];
+    memcpy(x, &xvec, sizeof(xvec));
+    u64a m[2];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    // Count the number of bits of compressed state we're writing out per
+    // chunk.
+    u32 bits[2] = { popcount64(m[0]), popcount64(m[1]) };
+
+    // Compress each 64-bit chunk individually.
+    u64a v[2] = { compress64(x[0], m[0]), compress64(x[1], m[1]) };
+
+    // Write packed data out.
+    pack_bits_64(ptr, v, bits, 2);
+}
+#endif
+
+void storecompressed128(void *ptr, const m128 *x, const m128 *m,
+                        UNUSED u32 bytes) {
+#if defined(ARCH_64_BIT)
+    storecompressed128_64bit(ptr, *x, *m);
+#else
+    storecompressed128_32bit(ptr, *x, *m);
+#endif
+}
+
+#if defined(ARCH_32_BIT)
+static really_inline
+m128 loadcompressed128_32bit(const void *ptr, m128 mvec) {
+    // First, decompose our vectors into 32-bit chunks.
+    u32 m[8];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    u32 bits[4] = { popcount32(m[0]), popcount32(m[1]),
+                    popcount32(m[2]), popcount32(m[3]) };
+    u32 v[4];
+
+    unpack_bits_32(v, (const u8 *)ptr, bits, 4);
+
+    u32 x[4] = { expand32(v[0], m[0]), expand32(v[1], m[1]),
+                 expand32(v[2], m[2]), expand32(v[3], m[3]) };
+
+    return _mm_set_epi32(x[3], x[2], x[1], x[0]);
+}
+#endif
+
+#if defined(ARCH_64_BIT)
+static really_inline
+m128 loadcompressed128_64bit(const void *ptr, m128 mvec) {
+    // First, decompose our vectors into 64-bit chunks.
+    u64a m[2] = { movq(mvec), movq(_mm_srli_si128(mvec, 8)) };
+
+    u32 bits[2] = { popcount64(m[0]), popcount64(m[1]) };
+    u64a v[2];
+
+    unpack_bits_64(v, (const u8 *)ptr, bits, 2);
+
+    u64a x[2] = { expand64(v[0], m[0]), expand64(v[1], m[1]) };
+
+    return _mm_set_epi64x(x[1], x[0]);
+}
+#endif
+
+void loadcompressed128(m128 *x, const void *ptr, const m128 *m,
+                       UNUSED u32 bytes) {
+#if defined(ARCH_64_BIT)
+    *x = loadcompressed128_64bit(ptr, *m);
+#else
+    *x = loadcompressed128_32bit(ptr, *m);
+#endif
+}
+
+/*
+ * 256-bit store/load.
+ */
+
+#if defined(ARCH_32_BIT)
+static really_inline
+void storecompressed256_32bit(void *ptr, m256 xvec, m256 mvec) {
+    // First, decompose our vectors into 32-bit chunks.
+    u32 x[8];
+    memcpy(x, &xvec, sizeof(xvec));
+    u32 m[8];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    // Count the number of bits of compressed state we're writing out per
+    // chunk.
+    u32 bits[8] = { popcount32(m[0]), popcount32(m[1]),
+                    popcount32(m[2]), popcount32(m[3]),
+                    popcount32(m[4]), popcount32(m[5]),
+                    popcount32(m[6]), popcount32(m[7])};
+
+    // Compress each 32-bit chunk individually.
+    u32 v[8] = { compress32(x[0], m[0]), compress32(x[1], m[1]),
+                 compress32(x[2], m[2]), compress32(x[3], m[3]),
+                 compress32(x[4], m[4]), compress32(x[5], m[5]),
+                 compress32(x[6], m[6]), compress32(x[7], m[7]) };
+
+    // Write packed data out.
+    pack_bits_32(ptr, v, bits, 8);
+}
+#endif
+
+#if defined(ARCH_64_BIT)
+static really_really_inline
+void storecompressed256_64bit(void *ptr, m256 xvec, m256 mvec) {
+    // First, decompose our vectors into 64-bit chunks.
+    u64a x[4];
+    memcpy(x, &xvec, sizeof(xvec));
+    u64a m[4];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    // Count the number of bits of compressed state we're writing out per
+    // chunk.
+    u32 bits[4] = { popcount64(m[0]), popcount64(m[1]),
+                    popcount64(m[2]), popcount64(m[3]) };
+
+    // Compress each 64-bit chunk individually.
+    u64a v[4] = { compress64(x[0], m[0]), compress64(x[1], m[1]),
+                  compress64(x[2], m[2]), compress64(x[3], m[3]) };
+
+    // Write packed data out.
+    pack_bits_64(ptr, v, bits, 4);
+}
+#endif
+
+void storecompressed256(void *ptr, const m256 *x, const m256 *m,
+                        UNUSED u32 bytes) {
+#if defined(ARCH_64_BIT)
+    storecompressed256_64bit(ptr, *x, *m);
+#else
+    storecompressed256_32bit(ptr, *x, *m);
+#endif
+}
+
+#if defined(ARCH_32_BIT)
+static really_inline
+m256 loadcompressed256_32bit(const void *ptr, m256 mvec) {
+    // First, decompose our vectors into 32-bit chunks.
+    u32 m[8];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    u32 bits[8] = { popcount32(m[0]), popcount32(m[1]),
+                    popcount32(m[2]), popcount32(m[3]),
+                    popcount32(m[4]), popcount32(m[5]),
+                    popcount32(m[6]), popcount32(m[7])};
+    u32 v[8];
+
+    unpack_bits_32(v, (const u8 *)ptr, bits, 8);
+
+    u32 x[8] = { expand32(v[0], m[0]), expand32(v[1], m[1]),
+                 expand32(v[2], m[2]), expand32(v[3], m[3]),
+                 expand32(v[4], m[4]), expand32(v[5], m[5]),
+                 expand32(v[6], m[6]), expand32(v[7], m[7]) };
+
+#if !defined(__AVX2__)
+    m256 xvec = { .lo = _mm_set_epi32(x[3], x[2], x[1], x[0]),
+                  .hi = _mm_set_epi32(x[7], x[6], x[5], x[4]) };
+#else
+    m256 xvec = _mm256_set_epi32(x[7], x[6], x[5], x[4],
+                                 x[3], x[2], x[1], x[0]);
+#endif
+    return xvec;
+}
+#endif
+
+#if defined(ARCH_64_BIT)
+static really_inline
+m256 loadcompressed256_64bit(const void *ptr, m256 mvec) {
+    // First, decompose our vectors into 64-bit chunks.
+    u64a m[4];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    u32 bits[4] = { popcount64(m[0]), popcount64(m[1]),
+                    popcount64(m[2]), popcount64(m[3]) };
+    u64a v[4];
+
+    unpack_bits_64(v, (const u8 *)ptr, bits, 4);
+
+    u64a x[4] = { expand64(v[0], m[0]), expand64(v[1], m[1]),
+                  expand64(v[2], m[2]), expand64(v[3], m[3]) };
+
+#if !defined(__AVX2__)
+    m256 xvec = { .lo = _mm_set_epi64x(x[1], x[0]),
+                  .hi = _mm_set_epi64x(x[3], x[2]) };
+#else
+    m256 xvec = _mm256_set_epi64x(x[3], x[2], x[1], x[0]);
+#endif
+    return xvec;
+}
+#endif
+
+void loadcompressed256(m256 *x, const void *ptr, const m256 *m,
+                       UNUSED u32 bytes) {
+#if defined(ARCH_64_BIT)
+    *x = loadcompressed256_64bit(ptr, *m);
+#else
+    *x = loadcompressed256_32bit(ptr, *m);
+#endif
+}
+
+/*
+ * 384-bit store/load.
+ */
+
+#if defined(ARCH_32_BIT)
+static really_inline
+void storecompressed384_32bit(void *ptr, m384 xvec, m384 mvec) {
+    // First, decompose our vectors into 32-bit chunks.
+    u32 x[12];
+    memcpy(x, &xvec, sizeof(xvec));
+    u32 m[12];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    // Count the number of bits of compressed state we're writing out per
+    // chunk.
+    u32 bits[12] = { popcount32(m[0]), popcount32(m[1]),
+                     popcount32(m[2]), popcount32(m[3]),
+                     popcount32(m[4]), popcount32(m[5]),
+                     popcount32(m[6]), popcount32(m[7]),
+                     popcount32(m[8]), popcount32(m[9]),
+                     popcount32(m[10]), popcount32(m[11]) };
+
+    // Compress each 32-bit chunk individually.
+    u32 v[12] = { compress32(x[0], m[0]), compress32(x[1], m[1]),
+                  compress32(x[2], m[2]), compress32(x[3], m[3]),
+                  compress32(x[4], m[4]), compress32(x[5], m[5]),
+                  compress32(x[6], m[6]), compress32(x[7], m[7]),
+                  compress32(x[8], m[8]), compress32(x[9], m[9]),
+                  compress32(x[10], m[10]), compress32(x[11], m[11])};
+
+    // Write packed data out.
+    pack_bits_32(ptr, v, bits, 12);
+}
+#endif
+
+#if defined(ARCH_64_BIT)
+static really_inline
+void storecompressed384_64bit(void *ptr, m384 xvec, m384 mvec) {
+    // First, decompose our vectors into 64-bit chunks.
+    u64a x[6];
+    memcpy(x, &xvec, sizeof(xvec));
+    u64a m[6];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    // Count the number of bits of compressed state we're writing out per
+    // chunk.
+    u32 bits[6] = { popcount64(m[0]), popcount64(m[1]),
+                    popcount64(m[2]), popcount64(m[3]),
+                    popcount64(m[4]), popcount64(m[5]) };
+
+    // Compress each 64-bit chunk individually.
+    u64a v[6] = { compress64(x[0], m[0]), compress64(x[1], m[1]),
+                  compress64(x[2], m[2]), compress64(x[3], m[3]),
+                  compress64(x[4], m[4]), compress64(x[5], m[5]) };
+
+    // Write packed data out.
+    pack_bits_64(ptr, v, bits, 6);
+}
+#endif
+
+void storecompressed384(void *ptr, const m384 *x, const m384 *m,
+                        UNUSED u32 bytes) {
+#if defined(ARCH_64_BIT)
+    storecompressed384_64bit(ptr, *x, *m);
+#else
+    storecompressed384_32bit(ptr, *x, *m);
+#endif
+}
+
+#if defined(ARCH_32_BIT)
+static really_inline
+m384 loadcompressed384_32bit(const void *ptr, m384 mvec) {
+    // First, decompose our vectors into 32-bit chunks.
+    u32 m[12];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    u32 bits[12] = { popcount32(m[0]), popcount32(m[1]),
+                     popcount32(m[2]), popcount32(m[3]),
+                     popcount32(m[4]), popcount32(m[5]),
+                     popcount32(m[6]), popcount32(m[7]),
+                     popcount32(m[8]), popcount32(m[9]),
+                     popcount32(m[10]), popcount32(m[11]) };
+    u32 v[12];
+
+    unpack_bits_32(v, (const u8 *)ptr, bits, 12);
+
+    u32 x[12] = { expand32(v[0], m[0]), expand32(v[1], m[1]),
+                  expand32(v[2], m[2]), expand32(v[3], m[3]),
+                  expand32(v[4], m[4]), expand32(v[5], m[5]),
+                  expand32(v[6], m[6]), expand32(v[7], m[7]),
+                  expand32(v[8], m[8]), expand32(v[9], m[9]),
+                  expand32(v[10], m[10]), expand32(v[11], m[11]) };
+
+    m384 xvec = { .lo = _mm_set_epi32(x[3], x[2], x[1], x[0]),
+                  .mid = _mm_set_epi32(x[7], x[6], x[5], x[4]),
+                  .hi = _mm_set_epi32(x[11], x[10], x[9], x[8]) };
+    return xvec;
+}
+#endif
+
+#if defined(ARCH_64_BIT)
+static really_inline
+m384 loadcompressed384_64bit(const void *ptr, m384 mvec) {
+    // First, decompose our vectors into 64-bit chunks.
+    u64a m[6];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    u32 bits[6] = { popcount64(m[0]), popcount64(m[1]),
+                    popcount64(m[2]), popcount64(m[3]),
+                    popcount64(m[4]), popcount64(m[5]) };
+    u64a v[6];
+
+    unpack_bits_64(v, (const u8 *)ptr, bits, 6);
+
+    u64a x[6] = { expand64(v[0], m[0]), expand64(v[1], m[1]),
+                  expand64(v[2], m[2]), expand64(v[3], m[3]),
+                  expand64(v[4], m[4]), expand64(v[5], m[5]) };
+
+    m384 xvec = { .lo = _mm_set_epi64x(x[1], x[0]),
+                  .mid = _mm_set_epi64x(x[3], x[2]),
+                  .hi = _mm_set_epi64x(x[5], x[4]) };
+    return xvec;
+}
+#endif
+
+void loadcompressed384(m384 *x, const void *ptr, const m384 *m,
+                       UNUSED u32 bytes) {
+#if defined(ARCH_64_BIT)
+    *x = loadcompressed384_64bit(ptr, *m);
+#else
+    *x = loadcompressed384_32bit(ptr, *m);
+#endif
+}
+
+/*
+ * 512-bit store/load.
+ */
+
+#if defined(ARCH_32_BIT)
+static really_inline
+void storecompressed512_32bit(void *ptr, m512 xvec, m512 mvec) {
+    // First, decompose our vectors into 32-bit chunks.
+    u32 x[16];
+    memcpy(x, &xvec, sizeof(xvec));
+    u32 m[16];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    // Count the number of bits of compressed state we're writing out per
+    // chunk.
+    u32 bits[16] = { popcount32(m[0]), popcount32(m[1]),
+                     popcount32(m[2]), popcount32(m[3]),
+                     popcount32(m[4]), popcount32(m[5]),
+                     popcount32(m[6]), popcount32(m[7]),
+                     popcount32(m[8]), popcount32(m[9]),
+                     popcount32(m[10]), popcount32(m[11]),
+                     popcount32(m[12]), popcount32(m[13]),
+                     popcount32(m[14]), popcount32(m[15])};
+
+    // Compress each 32-bit chunk individually.
+    u32 v[16] = { compress32(x[0], m[0]), compress32(x[1], m[1]),
+                  compress32(x[2], m[2]), compress32(x[3], m[3]),
+                  compress32(x[4], m[4]), compress32(x[5], m[5]),
+                  compress32(x[6], m[6]), compress32(x[7], m[7]),
+                  compress32(x[8], m[8]), compress32(x[9], m[9]),
+                  compress32(x[10], m[10]), compress32(x[11], m[11]),
+                  compress32(x[12], m[12]), compress32(x[13], m[13]),
+                  compress32(x[14], m[14]), compress32(x[15], m[15]) };
+
+    // Write packed data out.
+    pack_bits_32(ptr, v, bits, 16);
+}
+#endif
+
+#if defined(ARCH_64_BIT)
+static really_inline
+void storecompressed512_64bit(void *ptr, m512 xvec, m512 mvec) {
+    // First, decompose our vectors into 64-bit chunks.
+    u64a m[8];
+    memcpy(m, &mvec, sizeof(mvec));
+    u64a x[8];
+    memcpy(x, &xvec, sizeof(xvec));
+
+    // Count the number of bits of compressed state we're writing out per
+    // chunk.
+    u32 bits[8] = { popcount64(m[0]), popcount64(m[1]),
+                    popcount64(m[2]), popcount64(m[3]),
+                    popcount64(m[4]), popcount64(m[5]),
+                    popcount64(m[6]), popcount64(m[7]) };
+
+    // Compress each 64-bit chunk individually.
+    u64a v[8] = { compress64(x[0], m[0]), compress64(x[1], m[1]),
+                  compress64(x[2], m[2]), compress64(x[3], m[3]),
+                  compress64(x[4], m[4]), compress64(x[5], m[5]),
+                  compress64(x[6], m[6]), compress64(x[7], m[7]) };
+
+    // Write packed data out.
+    pack_bits_64(ptr, v, bits, 8);
+}
+#endif
+
+void storecompressed512(void *ptr, const m512 *x, const m512 *m,
+                        UNUSED u32 bytes) {
+#if defined(ARCH_64_BIT)
+    storecompressed512_64bit(ptr, *x, *m);
+#else
+    storecompressed512_32bit(ptr, *x, *m);
+#endif
+}
+
+#if defined(ARCH_32_BIT)
+static really_inline
+m512 loadcompressed512_32bit(const void *ptr, m512 mvec) {
+    // First, decompose our vectors into 32-bit chunks.
+    u32 m[16];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    u32 bits[16] = { popcount32(m[0]), popcount32(m[1]),
+                     popcount32(m[2]), popcount32(m[3]),
+                     popcount32(m[4]), popcount32(m[5]),
+                     popcount32(m[6]), popcount32(m[7]),
+                     popcount32(m[8]), popcount32(m[9]),
+                     popcount32(m[10]), popcount32(m[11]),
+                     popcount32(m[12]), popcount32(m[13]),
+                     popcount32(m[14]), popcount32(m[15]) };
+    u32 v[16];
+
+    unpack_bits_32(v, (const u8 *)ptr, bits, 16);
+
+    u32 x[16] = { expand32(v[0], m[0]), expand32(v[1], m[1]),
+                  expand32(v[2], m[2]), expand32(v[3], m[3]),
+                  expand32(v[4], m[4]), expand32(v[5], m[5]),
+                  expand32(v[6], m[6]), expand32(v[7], m[7]),
+                  expand32(v[8], m[8]), expand32(v[9], m[9]),
+                  expand32(v[10], m[10]), expand32(v[11], m[11]),
+                  expand32(v[12], m[12]), expand32(v[13], m[13]),
+                  expand32(v[14], m[14]), expand32(v[15], m[15]) };
+
+    m512 xvec;
+#if !defined(__AVX2__)
+    xvec.lo.lo = _mm_set_epi32(x[3], x[2], x[1], x[0]);
+    xvec.lo.hi = _mm_set_epi32(x[7], x[6], x[5], x[4]);
+    xvec.hi.lo = _mm_set_epi32(x[11], x[10], x[9], x[8]);
+    xvec.hi.hi = _mm_set_epi32(x[15], x[14], x[13], x[12]);
+#else
+    xvec.lo = _mm256_set_epi32(x[7], x[6], x[5], x[4],
+                               x[3], x[2], x[1], x[0]);
+    xvec.hi = _mm256_set_epi32(x[15], x[14], x[13], x[12],
+                               x[11], x[10], x[9], x[8]);
+#endif
+    return xvec;
+}
+#endif
+
+#if defined(ARCH_64_BIT)
+static really_inline
+m512 loadcompressed512_64bit(const void *ptr, m512 mvec) {
+    // First, decompose our vectors into 64-bit chunks.
+    u64a m[8];
+    memcpy(m, &mvec, sizeof(mvec));
+
+    u32 bits[8] = { popcount64(m[0]), popcount64(m[1]),
+                    popcount64(m[2]), popcount64(m[3]),
+                    popcount64(m[4]), popcount64(m[5]),
+                    popcount64(m[6]), popcount64(m[7]) };
+    u64a v[8];
+
+    unpack_bits_64(v, (const u8 *)ptr, bits, 8);
+
+    u64a x[8] = { expand64(v[0], m[0]), expand64(v[1], m[1]),
+                  expand64(v[2], m[2]), expand64(v[3], m[3]),
+                  expand64(v[4], m[4]), expand64(v[5], m[5]),
+                  expand64(v[6], m[6]), expand64(v[7], m[7]) };
+
+#if !defined(__AVX2__)
+    m512 xvec = { .lo = { _mm_set_epi64x(x[1], x[0]),
+                          _mm_set_epi64x(x[3], x[2]) },
+                  .hi = { _mm_set_epi64x(x[5], x[4]),
+                          _mm_set_epi64x(x[7], x[6]) } };
+#else
+    m512 xvec = { .lo = _mm256_set_epi64x(x[3], x[2], x[1], x[0]),
+                  .hi = _mm256_set_epi64x(x[7], x[6], x[5], x[4])};
+#endif
+    return xvec;
+}
+#endif
+
+void loadcompressed512(m512 *x, const void *ptr, const m512 *m,
+                       UNUSED u32 bytes) {
+#if defined(ARCH_64_BIT)
+    *x = loadcompressed512_64bit(ptr, *m);
+#else
+    *x = loadcompressed512_32bit(ptr, *m);
+#endif
+}

src/util/state_compress.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Mask-based state compression, used by the NFA.
+ */
+
+#ifndef STATE_COMPRESS_H
+#define STATE_COMPRESS_H
+
+#include "simd_utils.h"
+#include "ue2common.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/* Note: bytes is not used by implementations >= 128 */
+
+void storecompressed32(void *ptr, const u32 *x, const u32 *m, u32 bytes);
+void loadcompressed32(u32 *x, const void *ptr, const u32 *m, u32 bytes);
+
+void storecompressed64(void *ptr, const u64a *x, const u64a *m, u32 bytes);
+void loadcompressed64(u64a *x, const void *ptr, const u64a *m, u32 bytes);
+
+void storecompressed128(void *ptr, const m128 *x, const m128 *m, u32 bytes);
+void loadcompressed128(m128 *x, const void *ptr, const m128 *m, u32 bytes);
+
+void storecompressed256(void *ptr, const m256 *x, const m256 *m, u32 bytes);
+void loadcompressed256(m256 *x, const void *ptr, const m256 *m, u32 bytes);
+
+void storecompressed384(void *ptr, const m384 *x, const m384 *m, u32 bytes);
+void loadcompressed384(m384 *x, const void *ptr, const m384 *m, u32 bytes);
+
+void storecompressed512(void *ptr, const m512 *x, const m512 *m, u32 bytes);
+void loadcompressed512(m512 *x, const void *ptr, const m512 *m, u32 bytes);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif

src/util/target_info.cpp

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "hs_compile.h" // for various hs_platform_info flags
+#include "target_info.h"
+#include "util/cpuid_flags.h"
+
+namespace ue2 {
+
+target_t get_current_target(void) {
+    hs_platform_info p;
+    p.cpu_features = cpuid_flags();
+    p.tune = cpuid_tune();
+
+    return target_t(p);
+}
+
+bool target_t::can_run_on_code_built_for(const target_t &code_target) const {
+    if (!has_avx2() && code_target.has_avx2()) {
+        return false;
+    }
+
+    return true;
+}
+
+target_t::target_t(const hs_platform_info &p)
+    : tune(p.tune), cpu_features(p.cpu_features) {}
+
+bool target_t::has_avx2(void) const {
+    return (cpu_features & HS_CPU_FEATURES_AVX2);
+}
+
+bool target_t::is_atom_class(void) const {
+    return tune == HS_TUNE_FAMILY_SLM;
+}
+
+} // namespace ue2

src/util/target_info.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef TARGET_INFO_H
+#define TARGET_INFO_H
+
+#include "ue2common.h"
+
+struct hs_platform_info;
+
+namespace ue2 {
+
+struct target_t {
+    explicit target_t(const hs_platform_info &pi);
+
+    bool has_avx2(void) const;
+
+    bool is_atom_class(void) const;
+
+    // This asks: can this target (the object) run on code that was built for
+    // "code_target". Very wordy but less likely to be misinterpreted than
+    // is_compatible() or some such.
+    bool can_run_on_code_built_for(const target_t &code_target) const;
+
+private:
+    u32 tune;
+    u64a cpu_features;
+};
+
+target_t get_current_target(void);
+
+} // namespace ue2
+
+#endif

src/util/ue2_containers.h

@@ -0,0 +1,603 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UTIL_UE2_CONTAINERS_H_
+#define UTIL_UE2_CONTAINERS_H_
+
+#include "ue2common.h"
+
+#include <algorithm>
+#include <iterator>
+#include <type_traits>
+#include <utility>
+
+#include <boost/iterator/iterator_facade.hpp>
+#include <boost/unordered/unordered_map.hpp>
+#include <boost/unordered/unordered_set.hpp>
+
+namespace ue2 {
+
+/** \brief Unordered set container implemented internally as a hash table. */
+using boost::unordered_set;
+
+/** \brief Unordered map container implemented internally as a hash table. */
+using boost::unordered_map;
+
+namespace flat_detail {
+
+// Iterator facade that wraps an underlying iterator, so that we get our
+// own iterator types.
+template <class WrappedIter, class Value>
+class iter_wrapper
+    : public boost::iterator_facade<iter_wrapper<WrappedIter, Value>, Value,
+                                    boost::random_access_traversal_tag> {
+public:
+    iter_wrapper() {}
+    explicit iter_wrapper(const WrappedIter &it_in) : it(it_in) {}
+
+    // Templated copy-constructor to allow for interoperable iterator and
+    // const_iterator.
+private:
+    template <class, class> friend class iter_wrapper;
+
+public:
+    template <class OtherIter, class OtherValue>
+    iter_wrapper(const iter_wrapper<OtherIter, OtherValue> &other,
+                 typename std::enable_if<std::is_convertible<
+                     OtherIter, WrappedIter>::value>::type * = nullptr)
+        : it(other.it) {}
+
+    WrappedIter get() const { return it; }
+
+private:
+    friend class boost::iterator_core_access;
+
+    WrappedIter it;
+
+    void increment() { ++it; }
+    void decrement() { --it; }
+    void advance(size_t n) { it += n; }
+    typename WrappedIter::difference_type
+    distance_to(const iter_wrapper &other) const {
+        return other.it - it;
+    }
+    bool equal(const iter_wrapper &other) const { return it == other.it; }
+    Value &dereference() const { return *it; }
+};
+
+} // namespace flat_detail
+
+/**
+ * \brief Set container implemented internally as a sorted vector. Use this
+ * rather than std::set for small sets as it's faster, uses less memory and
+ * incurs less malloc time.
+ *
+ * Note: we used to use boost::flat_set, but have run into problems with all
+ * the extra machinery it instantiates.
+ */
+template <class T, class Compare = std::less<T>,
+          class Allocator = std::allocator<T>>
+class flat_set {
+    // Underlying storage is a sorted std::vector.
+    using StorageT = std::vector<T, Allocator>;
+
+    Compare comp;
+    StorageT data;
+
+public:
+    // Member types.
+    using key_type = T;
+    using value_type = T;
+    using size_type = typename StorageT::size_type;
+    using difference_type = typename StorageT::difference_type;
+    using key_compare = Compare;
+    using value_compare = Compare;
+    using allocator_type = Allocator;
+    using reference = value_type &;
+    using const_reference = const value_type &;
+    using pointer = typename std::allocator_traits<Allocator>::pointer;
+    using const_pointer = typename std::allocator_traits<Allocator>::const_pointer;
+
+    // Iterator types.
+
+    using iterator = flat_detail::iter_wrapper<typename StorageT::iterator,
+                                               const value_type>;
+    using const_iterator =
+        flat_detail::iter_wrapper<typename StorageT::const_iterator,
+                                  const value_type>;
+
+    using reverse_iterator = std::reverse_iterator<iterator>;
+    using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+
+    // Constructors.
+
+    flat_set(const Compare &compare = Compare(),
+             const Allocator &alloc = Allocator())
+        : comp(compare), data(alloc) {}
+
+    template <class InputIt>
+    flat_set(InputIt first, InputIt last, const Compare &compare = Compare(),
+             const Allocator &alloc = Allocator())
+        : comp(compare), data(alloc) {
+        insert(first, last);
+    }
+
+    flat_set(std::initializer_list<value_type> init,
+             const Compare &compare = Compare(),
+             const Allocator &alloc = Allocator())
+        : comp(compare), data(alloc) {
+        insert(init.begin(), init.end());
+    }
+
+    flat_set(const flat_set &) = default;
+    flat_set(flat_set &&) = default;
+    flat_set &operator=(const flat_set &) = default;
+    flat_set &operator=(flat_set &&) = default;
+
+    // Other members.
+
+    allocator_type get_allocator() const {
+        return data.get_allocator();
+    }
+
+    // Iterators.
+
+    iterator begin() { return iterator(data.begin()); }
+    const_iterator cbegin() const { return const_iterator(data.cbegin()); }
+    const_iterator begin() const { return cbegin(); }
+
+    iterator end() { return iterator(data.end()); }
+    const_iterator cend() const { return const_iterator(data.cend()); }
+    const_iterator end() const { return cend(); }
+
+    reverse_iterator rbegin() { return reverse_iterator(end()); }
+    const_reverse_iterator crbegin() const {
+        return const_reverse_iterator(cend());
+    }
+    const_reverse_iterator rbegin() const { return crbegin(); }
+
+    reverse_iterator rend() { return reverse_iterator(begin()); }
+    const_reverse_iterator crend() const {
+        return const_reverse_iterator(cbegin());
+    }
+    const_reverse_iterator rend() const { return crend(); }
+
+    // Capacity.
+
+    bool empty() const { return data.empty(); }
+    size_t size() const { return data.size(); }
+    size_t max_size() const { return data.max_size(); }
+
+    // Modifiers.
+
+    void clear() {
+        data.clear();
+    }
+
+    std::pair<iterator, bool> insert(const value_type &value) {
+        auto it = std::lower_bound(data.begin(), data.end(), value, comp);
+        if (it == data.end() || comp(value, *it)) {
+            return std::make_pair(iterator(data.insert(it, value)), true);
+        }
+        return std::make_pair(iterator(it), false);
+    }
+
+    std::pair<iterator, bool> insert(value_type &&value) {
+        auto it = std::lower_bound(data.begin(), data.end(), value, comp);
+        if (it == data.end() || comp(value, *it)) {
+            return std::make_pair(iterator(data.insert(it, std::move(value))),
+                                  true);
+        }
+        return std::make_pair(iterator(it), false);
+    }
+
+    template <class InputIt>
+    void insert(InputIt first, InputIt second) {
+        for (; first != second; ++first) {
+            insert(*first);
+        }
+    }
+
+    void insert(std::initializer_list<value_type> ilist) {
+        insert(ilist.begin(), ilist.end());
+    }
+
+    template<class...Args>
+    std::pair<iterator, bool> emplace(Args&&... args) {
+        return insert(value_type(std::forward<Args>(args)...));
+    }
+
+    void erase(iterator pos) {
+        data.erase(pos.get());
+    }
+
+    void erase(iterator first, iterator last) {
+        data.erase(first.get(), last.get());
+    }
+
+    void erase(const key_type &key) {
+        auto it = find(key);
+        if (it != end()) {
+            erase(it);
+        }
+    }
+
+    void swap(flat_set &a) {
+        using std::swap;
+        swap(comp, a.comp);
+        swap(data, a.data);
+    }
+
+    // Lookup.
+
+    size_type count(const value_type &value) const {
+        return find(value) != end() ? 1 : 0;
+    }
+
+    iterator find(const value_type &value) {
+        auto it = std::lower_bound(data.begin(), data.end(), value, comp);
+        if (it != data.end() && comp(value, *it)) {
+            it = data.end();
+        }
+        return iterator(it);
+    }
+
+    const_iterator find(const value_type &value) const {
+        auto it = std::lower_bound(data.begin(), data.end(), value, comp);
+        if (it != data.end() && comp(value, *it)) {
+            it = data.end();
+        }
+        return const_iterator(it);
+    }
+
+    // Observers.
+
+    key_compare key_comp() const {
+        return comp;
+    }
+
+    value_compare value_comp() const {
+        return comp;
+    }
+
+    // Operators.
+
+    bool operator==(const flat_set &a) const {
+        return data == a.data;
+    }
+    bool operator!=(const flat_set &a) const {
+        return data != a.data;
+    }
+    bool operator<(const flat_set &a) const {
+        return data < a.data;
+    }
+    bool operator<=(const flat_set &a) const {
+        return data <= a.data;
+    }
+    bool operator>(const flat_set &a) const {
+        return data > a.data;
+    }
+    bool operator>=(const flat_set &a) const {
+        return data >= a.data;
+    }
+
+    // Free swap function for ADL.
+    friend void swap(flat_set &a, flat_set &b) {
+        a.swap(b);
+    }
+};
+
+/**
+ * \brief Map container implemented internally as a sorted vector. Use this
+ * rather than std::map for small sets as it's faster, uses less memory and
+ * incurs less malloc time.
+ *
+ * Note: we used to use boost::flat_map, but have run into problems with all
+ * the extra machinery it instantiates.
+ *
+ * Note: ue2::flat_map does NOT provide mutable iterators, as (given the way
+ * the data is stored) it is difficult to provide a real mutable iterator that
+ * wraps std::pair<const Key, T>. Instead, all iterators are const, and you
+ * should use flat_map::at() or flat_map::operator[] to mutate the contents of
+ * the container.
+ */
+template <class Key, class T, class Compare = std::less<Key>,
+          class Allocator = std::allocator<std::pair<Key, T>>>
+class flat_map {
+public:
+    // Member types.
+    using key_type = Key;
+    using mapped_type = T;
+    using value_type = std::pair<const Key, T>;
+
+private:
+    // Underlying storage is a sorted std::vector.
+    using storage_type = std::pair<key_type, mapped_type>;
+    using StorageT = std::vector<storage_type, Allocator>;
+
+    Compare comp;
+    StorageT data;
+
+public:
+    // More Member types.
+    using size_type = typename StorageT::size_type;
+    using difference_type = typename StorageT::difference_type;
+    using key_compare = Compare;
+    using allocator_type = Allocator;
+    using reference = value_type &;
+    using const_reference = const value_type &;
+    using pointer = typename std::allocator_traits<Allocator>::pointer;
+    using const_pointer = typename std::allocator_traits<Allocator>::const_pointer;
+
+public:
+    using const_iterator =
+        flat_detail::iter_wrapper<typename StorageT::const_iterator,
+                                  const storage_type>;
+
+    using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+
+    // All iterators are const for flat_map.
+    using iterator = const_iterator;
+    using reverse_iterator = const_reverse_iterator;
+
+    // Constructors.
+
+    flat_map(const Compare &compare = Compare(),
+             const Allocator &alloc = Allocator())
+        : comp(compare), data(alloc) {}
+
+    template <class InputIt>
+    flat_map(InputIt first, InputIt last, const Compare &compare = Compare(),
+             const Allocator &alloc = Allocator())
+        : comp(compare), data(alloc) {
+        insert(first, last);
+    }
+
+    flat_map(std::initializer_list<value_type> init,
+             const Compare &compare = Compare(),
+             const Allocator &alloc = Allocator())
+        : comp(compare), data(alloc) {
+        insert(init.begin(), init.end());
+    }
+
+    flat_map(const flat_map &) = default;
+    flat_map(flat_map &&) = default;
+    flat_map &operator=(const flat_map &) = default;
+    flat_map &operator=(flat_map &&) = default;
+
+    // Other members.
+
+    allocator_type get_allocator() const {
+        return data.get_allocator();
+    }
+
+    // Iterators.
+
+    const_iterator cbegin() const { return const_iterator(data.cbegin()); }
+    const_iterator begin() const { return cbegin(); }
+
+    const_iterator cend() const { return const_iterator(data.cend()); }
+    const_iterator end() const { return cend(); }
+
+    const_reverse_iterator crbegin() const {
+        return const_reverse_iterator(cend());
+    }
+    const_reverse_iterator rbegin() const { return crbegin(); }
+
+    const_reverse_iterator crend() const {
+        return const_reverse_iterator(cbegin());
+    }
+    const_reverse_iterator rend() const { return crend(); }
+
+    // Capacity.
+
+    bool empty() const { return data.empty(); }
+    size_t size() const { return data.size(); }
+    size_t max_size() const { return data.max_size(); }
+
+private:
+    using storage_iterator = typename StorageT::iterator;
+    using storage_const_iterator = typename StorageT::const_iterator;
+
+    storage_iterator data_lower_bound(const key_type &key) {
+        return std::lower_bound(
+            data.begin(), data.end(), key,
+            [&](const storage_type &elem, const key_type &k) {
+                return comp(elem.first, k);
+            });
+    }
+
+    storage_const_iterator
+    data_lower_bound(const key_type &key) const {
+        return std::lower_bound(
+            data.begin(), data.end(), key,
+            [&](const storage_type &elem, const key_type &k) {
+                return comp(elem.first, k);
+            });
+    }
+
+    std::pair<storage_iterator, bool> data_insert(const value_type &value) {
+        auto it = data_lower_bound(value.first);
+        if (it == data.end() || comp(value.first, it->first)) {
+            return std::make_pair(data.insert(it, value), true);
+        }
+        return std::make_pair(it, false);
+    }
+
+    std::pair<storage_iterator, bool> data_insert(value_type &&value) {
+        auto it = data_lower_bound(value.first);
+        if (it == data.end() || comp(value.first, it->first)) {
+            return std::make_pair(data.insert(it, std::move(value)), true);
+        }
+        return std::make_pair(it, false);
+    }
+
+    storage_iterator data_find(const key_type &key) {
+        auto it = data_lower_bound(key);
+        if (it != data.end() && comp(key, it->first)) {
+            it = data.end();
+        }
+        return it;
+    }
+
+    storage_const_iterator data_find(const key_type &key) const {
+        auto it = data_lower_bound(key);
+        if (it != data.end() && comp(key, it->first)) {
+            it = data.end();
+        }
+        return it;
+    }
+
+public:
+    // Modifiers.
+
+    void clear() {
+        data.clear();
+    }
+
+    std::pair<iterator, bool> insert(const value_type &value) {
+        auto rv = data_insert(value);
+        return std::make_pair(iterator(rv.first), rv.second);
+    }
+
+    std::pair<iterator, bool> insert(value_type &&value) {
+        auto rv = data_insert(std::move(value));
+        return std::make_pair(iterator(rv.first), rv.second);
+    }
+
+    template <class InputIt>
+    void insert(InputIt first, InputIt second) {
+        for (; first != second; ++first) {
+            insert(*first);
+        }
+    }
+
+    void insert(std::initializer_list<value_type> ilist) {
+        insert(ilist.begin(), ilist.end());
+    }
+
+    template<class...Args>
+    std::pair<iterator, bool> emplace(Args&&... args) {
+        return insert(value_type(std::forward<Args>(args)...));
+    }
+
+    void erase(iterator pos) {
+        // Convert to a non-const storage iterator via pointer arithmetic.
+        storage_iterator it = data.begin() + distance(begin(), pos);
+        data.erase(it);
+    }
+
+    void erase(iterator first, iterator last) {
+        // Convert to a non-const storage iterator via pointer arithmetic.
+        storage_iterator data_first = data.begin() + distance(begin(), first);
+        storage_iterator data_last = data.begin() + distance(begin(), last);
+        data.erase(data_first, data_last);
+    }
+
+    void erase(const key_type &key) {
+        auto it = find(key);
+        if (it != end()) {
+            erase(it);
+        }
+    }
+
+    void swap(flat_map &a) {
+        using std::swap;
+        swap(comp, a.comp);
+        swap(data, a.data);
+    }
+
+    // Lookup.
+
+    size_type count(const key_type &key) const {
+        return find(key) != end() ? 1 : 0;
+    }
+
+    const_iterator find(const key_type &key) const {
+        return const_iterator(data_find(key));
+    }
+
+    // Element access.
+
+    mapped_type &at(const key_type &key) {
+        auto it = data_find(key);
+        if (it == data.end()) {
+            throw std::out_of_range("element not found");
+        }
+        return it->second;
+    }
+
+    const mapped_type &at(const key_type &key) const {
+        auto it = data_find(key);
+        if (it == data.end()) {
+            throw std::out_of_range("element not found");
+        }
+        return it->second;
+    }
+
+    mapped_type &operator[](const key_type &key) {
+        auto p = data_insert(value_type(key, mapped_type()));
+        return p.first->second;
+    }
+
+    // Observers.
+
+    key_compare key_comp() const {
+        return comp;
+    }
+
+    // Operators.
+
+    bool operator==(const flat_map &a) const {
+        return data == a.data;
+    }
+    bool operator!=(const flat_map &a) const {
+        return data != a.data;
+    }
+    bool operator<(const flat_map &a) const {
+        return data < a.data;
+    }
+    bool operator<=(const flat_map &a) const {
+        return data <= a.data;
+    }
+    bool operator>(const flat_map &a) const {
+        return data > a.data;
+    }
+    bool operator>=(const flat_map &a) const {
+        return data >= a.data;
+    }
+
+    // Free swap function for ADL.
+    friend void swap(flat_map &a, flat_map &b) {
+        a.swap(b);
+    }
+};
+
+} // namespace
+
+#endif // UTIL_UE2_CONTAINERS_H_

src/util/ue2string.cpp

@@ -0,0 +1,393 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Tools for string manipulation, ue2_literal definition.
+ */
+#include "charreach.h"
+#include "compare.h"
+#include "ue2string.h"
+
+#include <algorithm>
+#include <iomanip>
+#include <sstream>
+#include <string>
+
+using namespace std;
+
+namespace ue2 {
+
+#if defined(DUMP_SUPPORT) || defined(DEBUG)
+
+// Escape a string so that it's screen-printable
+string escapeString(const string &s) {
+    ostringstream os;
+    for (unsigned int i = 0; i < s.size(); ++i) {
+        char c = s[i];
+        if (0x20 <= c && c <= 0x7e && c != '\\') {
+            os << c;
+        } else if (c == '\n') {
+            os << "\\n";
+        } else if (c == '\r') {
+            os << "\\r";
+        } else if (c == '\t') {
+            os << "\\t";
+        } else {
+            os << "\\x" << hex << setw(2) << setfill('0')
+                << (unsigned)(c & 0xff) << dec;
+        }
+    }
+    return os.str();
+}
+
+string escapeString(const ue2_literal &lit) {
+    ostringstream os;
+    for (ue2_literal::const_iterator it = lit.begin(); it != lit.end(); ++it) {
+        char c = it->c;
+        if (0x20 <= c && c <= 0x7e && c != '\\') {
+            os << c;
+        } else if (c == '\n') {
+            os << "\\n";
+        } else {
+            os << "\\x" << hex << setw(2) << setfill('0')
+                << (unsigned)(c & 0xff) << dec;
+        }
+    }
+    return os.str();
+}
+
+// escape any metacharacters in a literal string
+string escapeStringMeta(const string &s) {
+    ostringstream os;
+    for (unsigned int i = 0; i < s.size(); ++i) {
+        char c = s[i];
+        switch (c) {
+            case '#': case '$': case '(': case ')':
+            case '*': case '+': case '.': case '/':
+            case '?': case '[': case ']': case '^':
+            case '|':
+                os << "\\" << c; break;
+            default:
+                os << c; break;
+        }
+    }
+    return os.str();
+}
+
+string dotEscapeString(const string &s) {
+    string ss = escapeString(s);
+    string out;
+    out.reserve(ss.size());
+    for (size_t i = 0; i != ss.size(); i++) {
+        char c = ss[i];
+        switch (c) {
+        case '\"':
+        case '\\':
+            out.push_back('\\');
+        // fall through
+        default:
+            out.push_back(c);
+            break;
+        }
+    }
+    return out;
+}
+
+string dumpString(const ue2_literal &lit) {
+    string s = escapeString(lit.get_string());
+    if (lit.any_nocase()) {
+        s += " (nocase)";
+    }
+
+    return s;
+}
+#endif
+
+void upperString(string &s) {
+    transform(s.begin(), s.end(), s.begin(), (int(*)(int)) mytoupper);
+}
+
+size_t maxStringOverlap(const string &a, const string &b, bool nocase) {
+    size_t lena = a.length(), lenb = b.length();
+    const char *astart = a.c_str();
+    const char *bstart = b.c_str();
+    const char *aend = astart + lena;
+    size_t i = lenb;
+
+    for (; i > lena; i--) {
+        if (!cmp(astart, bstart + i - lena, lena, nocase)) {
+            return i;
+        }
+    }
+
+    for (; i && cmp(aend - i, bstart, i, nocase); i--) {
+        ;
+    }
+
+    return i;
+}
+
+size_t maxStringOverlap(const ue2_literal &a, const ue2_literal &b) {
+    /* todo: handle nocase better */
+    return maxStringOverlap(a.get_string(), b.get_string(),
+                            a.any_nocase() || b.any_nocase());
+}
+
+size_t maxStringSelfOverlap(const string &a, bool nocase) {
+    size_t lena = a.length();
+    const char *astart = a.c_str();
+    const char *bstart = a.c_str();
+    const char *aend = astart + lena;
+    size_t i = lena - 1;
+
+    for (; i && cmp(aend - i, bstart, i, nocase); i--) {
+        ;
+    }
+
+    return i;
+}
+
+u32 cmp(const char *a, const char *b, size_t len, bool nocase) {
+    return cmpForward((const u8 *)a, (const u8 *)b, len, nocase);
+}
+
+case_iter::case_iter(const ue2_literal &ss) : s(ss.get_string()),
+                                              s_orig(ss.get_string()) {
+    for (ue2_literal::const_iterator it = ss.begin(); it != ss.end(); ++it) {
+        nocase.push_back(it->nocase);
+    }
+}
+
+case_iter caseIterateBegin(const ue2_literal &s) {
+    return case_iter(s);
+}
+
+case_iter caseIterateEnd() {
+    return case_iter(ue2_literal());
+}
+
+case_iter &case_iter::operator++ () {
+    for (size_t i = s.length(); i != 0; i--) {
+        char lower = mytolower(s[i - 1]);
+        if (nocase[i - 1] && lower != s[i - 1]) {
+            s[i - 1] = lower;
+            copy(s_orig.begin() + i, s_orig.end(), s.begin() + i);
+            return *this;
+        }
+    }
+
+    s.clear();
+    return *this;
+}
+
+static
+string toUpperString(string s) {
+    upperString(s);
+    return s;
+}
+
+ue2_literal::elem::operator CharReach () const {
+    if (!nocase) {
+        return CharReach(c);
+    } else {
+        CharReach rv;
+        rv.set(mytoupper(c));
+        rv.set(mytolower(c));
+        return rv;
+    }
+}
+
+ue2_literal::ue2_literal(const std::string &s_in, bool nc_in)
+    : s(nc_in ? toUpperString(s_in) : s_in), nocase(s_in.size(), nc_in) {
+    if (nc_in) {
+        // Quash nocase bit for non-alpha chars
+        for (size_t i = 0; i < s.length(); i++) {
+            if (!ourisalpha(s[i])) {
+                nocase[i] = false;
+            }
+        }
+    }
+}
+
+ue2_literal::ue2_literal(char c, bool nc)
+    : s(1, nc ? mytoupper(c) : c), nocase(1, ourisalpha(c) ? nc : false) {}
+
+ue2_literal ue2_literal::substr(size_type pos, size_type n) const {
+    ue2_literal rv;
+    rv.s = s.substr(pos, n);
+    size_type upper = nocase.size();
+    if (n != string::npos && n + pos < nocase.size()) {
+        upper = n + pos;
+    }
+    rv.nocase.insert(rv.nocase.end(), nocase.begin() + pos,
+                     nocase.begin() + upper);
+    return rv;
+}
+
+ue2_literal &ue2_literal::erase(size_type pos, size_type n) {
+    s.erase(pos, n);
+    size_type upper = nocase.size();
+    if (n != string::npos && n + pos < nocase.size()) {
+        upper = n + pos;
+    }
+    nocase.erase(nocase.begin() + pos, nocase.begin() + upper);
+    return *this;
+}
+
+void ue2_literal::push_back(char c, bool nc) {
+    assert(!nc || ourisalpha(c));
+    if (nc) {
+        c = mytoupper(c);
+    }
+    nocase.push_back(nc);
+    s.push_back(c);
+}
+
+// Return a copy of this literal in reverse order.
+ue2_literal reverse_literal(const ue2_literal &in) {
+    ue2_literal rv;
+    if (in.empty()) {
+        return rv;
+    }
+
+    for (ue2_literal::const_iterator it = in.end(); it != in.begin();) {
+        --it;
+        rv.push_back(it->c, it->nocase);
+    }
+    return rv;
+}
+
+bool ue2_literal::operator<(const ue2_literal &b) const {
+    if (s < b.s) {
+        return true;
+    }
+    if (s > b.s) {
+        return false;
+    }
+    return nocase < b.nocase;
+}
+
+ue2_literal operator+(const ue2_literal &a, const ue2_literal &b) {
+    ue2_literal rv;
+    rv.s = a.s + b.s;
+    rv.nocase = a.nocase;
+    rv.nocase.insert(rv.nocase.end(), b.nocase.begin(), b.nocase.end());
+    return rv;
+}
+
+void ue2_literal::operator+=(const ue2_literal &b) {
+    s += b.s;
+    nocase.insert(nocase.end(), b.nocase.begin(), b.nocase.end());
+}
+
+bool ue2_literal::any_nocase() const {
+    return find(nocase.begin(), nocase.end(), true) != nocase.end();
+}
+
+bool mixed_sensitivity(const ue2_literal &s) {
+    bool cs = false;
+    bool nc = false;
+    for (ue2_literal::const_iterator it = s.begin(); it != s.end(); ++it) {
+        if (!ourisalpha(it->c)) {
+            continue;
+        }
+        if (it->nocase) {
+            nc = true;
+        } else {
+            cs = true;
+        }
+    }
+
+    return cs && nc;
+}
+
+void make_nocase(ue2_literal *lit) {
+    ue2_literal rv;
+
+    for (ue2_literal::const_iterator it = lit->begin(); it != lit->end();
+         ++it) {
+        rv.push_back(it->c, ourisalpha(it->c));
+    }
+
+    lit->swap(rv);
+}
+
+static
+bool testchar(char c, const CharReach &cr, bool nocase) {
+    if (nocase) {
+        return cr.test((unsigned char)mytolower(c))
+            || cr.test((unsigned char)mytoupper(c));
+    } else {
+        return cr.test((unsigned char)c);
+    }
+}
+
+// Returns true if the given literal contains a char in the given CharReach
+bool contains(const ue2_literal &s, const CharReach &cr) {
+    for (ue2_literal::const_iterator it = s.begin(), ite = s.end();
+         it != ite; ++it) {
+        if (testchar(it->c, cr, it->nocase)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+size_t maxStringSelfOverlap(const ue2_literal &a) {
+    /* overly conservative if only part of the string is nocase, TODO: fix */
+    return maxStringSelfOverlap(a.get_string(), a.any_nocase());
+}
+
+size_t minStringPeriod(const ue2_literal &a) {
+    return a.length() - maxStringSelfOverlap(a);
+}
+
+// Returns true if `a' is a suffix of (or equal to) `b'.
+bool isSuffix(const ue2_literal &a, const ue2_literal &b) {
+    size_t alen = a.length(), blen = b.length();
+    if (alen > blen) {
+        return false;
+    }
+    return equal(a.begin(), a.end(), b.begin() + (blen - alen));
+}
+
+bool is_flood(const ue2_literal &s) {
+    assert(!s.empty());
+
+    ue2_literal::const_iterator it = s.begin(), ite = s.end();
+    ue2_literal::elem f = *it;
+    for (++it; it != ite; ++it) {
+        if (*it != f) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+} // namespace ue2

src/util/ue2string.h

@@ -0,0 +1,253 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Tools for string manipulation, ue2_literal definition.
+ */
+
+#ifndef UE2STRING_H
+#define UE2STRING_H
+
+#include "ue2common.h"
+#include "util/charreach.h"
+
+#include <iterator>
+#include <string>
+#include <vector>
+
+#include <boost/iterator/iterator_facade.hpp>
+
+namespace ue2 {
+
+/// Force the given string to upper-case.
+void upperString(std::string &s);
+
+size_t maxStringOverlap(const std::string &a, const std::string &b,
+                        bool nocase);
+
+size_t maxStringSelfOverlap(const std::string &a, bool nocase);
+
+/// Compares two strings, returns non-zero if they're different.
+u32 cmp(const char *a, const char *b, size_t len, bool nocase);
+
+class CharReach;
+
+struct ue2_literal {
+public:
+    /// Single element proxy, pointed to by our const_iterator.
+    struct elem {
+        elem() : c(0), nocase(false) {}
+        elem(char c_in, bool nc_in) : c(c_in), nocase(nc_in) {}
+        bool operator==(const elem &o) const {
+            return c == o.c && nocase == o.nocase;
+        }
+        bool operator!=(const elem &o) const {
+            return c != o.c || nocase != o.nocase;
+        }
+        operator CharReach() const;
+        char c;
+        bool nocase;
+    };
+
+    /// Boost iterator_facade lets us synthesize an iterator simply.
+    class const_iterator : public boost::iterator_facade<
+            const_iterator,
+            elem const,
+            boost::random_access_traversal_tag,
+            elem const> {
+    public:
+        const_iterator() {}
+    private:
+        friend class boost::iterator_core_access;
+        void increment() {
+            ++it; ++it_nc;
+        }
+        void decrement() {
+            --it; --it_nc;
+        }
+        void advance(size_t n) {
+            it += n; it_nc += n;
+        }
+        difference_type distance_to(const const_iterator &other) const {
+            return other.it - it;
+        }
+        bool equal(const const_iterator &other) const {
+            return it == other.it;
+        }
+        const elem dereference() const {
+            return elem(*it, *it_nc);
+        }
+
+        friend struct ue2_literal;
+        const_iterator(const std::string::const_iterator &it_in,
+                       const std::vector<bool>::const_iterator &it_nc_in)
+            : it(it_in), it_nc(it_nc_in) {}
+
+        std::string::const_iterator it;
+        std::vector<bool>::const_iterator it_nc;
+    };
+
+    using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+
+    typedef std::string::size_type size_type;
+
+    ue2_literal() {}
+    ue2_literal(const std::string &s_in, bool nc_in);
+    ue2_literal(char c, bool nc_in);
+    ue2_literal(const ue2_literal &) = default;
+    ue2_literal(ue2_literal &&) = default;
+    ue2_literal &operator=(const ue2_literal &) = default;
+    ue2_literal &operator=(ue2_literal &&) = default;
+
+    size_type length() const { return s.length(); }
+    bool empty() const { return s.empty(); }
+    ue2_literal substr(size_type pos, size_type n = std::string::npos) const;
+    const char *c_str() const { return s.c_str(); }
+    bool any_nocase() const;
+
+    const_iterator begin() const {
+        return const_iterator(s.begin(), nocase.begin());
+    }
+
+    const_iterator end() const {
+        return const_iterator(s.end(), nocase.end());
+    }
+
+    const_reverse_iterator rbegin() const {
+        return const_reverse_iterator(end());
+    }
+
+    const_reverse_iterator rend() const {
+        return const_reverse_iterator(begin());
+    }
+
+    ue2_literal &erase(size_type pos = 0, size_type n = std::string::npos);
+    void push_back(const elem &e) {
+        push_back(e.c, e.nocase);
+    }
+
+    void push_back(char c, bool nc);
+    const elem back() const { return elem(*s.rbegin(), nocase.back()); }
+    friend ue2_literal operator+(const ue2_literal &a, const ue2_literal &b);
+
+    void operator+=(const ue2_literal &b);
+    bool operator==(const ue2_literal &b) const {
+        return s == b.s && nocase == b.nocase;
+    }
+    bool operator!=(const ue2_literal &b) const {
+        return !(*this == b);
+    }
+    bool operator<(const ue2_literal &b) const;
+
+    void clear(void) { s.clear(); nocase.clear(); }
+
+    const std::string &get_string() const { return s; }
+
+    void swap(ue2_literal &other) {
+        s.swap(other.s);
+        nocase.swap(other.nocase);
+    }
+
+private:
+    std::string s;
+    std::vector<bool> nocase; /* for trolling value */
+};
+
+
+/// Return a reversed copy of this literal.
+ue2_literal reverse_literal(const ue2_literal &in);
+
+// Escape any meta characters in a string
+std::string escapeStringMeta(const std::string &s);
+
+/** Note: may be overly conservative if only partially nocase */
+size_t maxStringSelfOverlap(const ue2_literal &a);
+size_t minStringPeriod(const ue2_literal &a);
+size_t maxStringOverlap(const ue2_literal &a, const ue2_literal &b);
+
+/** \brief True iff the literal cannot be considered entirely case-sensitive
+ * nor entirely case-insensitive */
+bool mixed_sensitivity(const ue2_literal &lit);
+
+void make_nocase(ue2_literal *lit);
+
+struct case_iter {
+    explicit case_iter(const ue2_literal &ss);
+    const std::string &operator*() const { return s; } /* limited lifetime */
+    case_iter &operator++ ();
+    bool operator!=(const case_iter &b) const { return s != b.s; }
+private:
+    std::string s;
+    std::string s_orig;
+    std::vector<bool> nocase;
+};
+
+case_iter caseIterateBegin(const ue2_literal &lit);
+case_iter caseIterateEnd();
+
+/** \brief True if there is any overlap between the characters in \a s and the
+ * set characters in \a cr.
+ *
+ * Note: this means that if \a s is nocase, then \a cr only needs to have
+ * either the lower-case or upper-case version of a letter set.  */
+bool contains(const ue2_literal &s, const CharReach &cr);
+
+/// Returns true if \a a is a suffix of (or equal to) \a b.
+bool isSuffix(const ue2_literal &a, const ue2_literal &b);
+
+static inline
+std::vector<CharReach> as_cr_seq(const ue2_literal &s) {
+    std::vector<CharReach> rv;
+    rv.reserve(s.length());
+    rv.insert(rv.end(), s.begin(), s.end());
+    return rv;
+}
+
+/** \brief True if the given literal consists entirely of a flood of the same
+ * character. */
+bool is_flood(const ue2_literal &s);
+
+#if defined(DUMP_SUPPORT) || defined(DEBUG)
+/* Utility functions for debugging/dumping */
+
+/// Escape a string so it's dot-printable.
+std::string dotEscapeString(const std::string &s);
+
+std::string dumpString(const ue2_literal &lit);
+
+/// Escape a string so that it's screen-printable.
+std::string escapeString(const std::string &s);
+
+/// Escape a ue2_literal so that it's screen-printable.
+std::string escapeString(const ue2_literal &lit);
+
+#endif
+
+} // namespace ue2
+
+#endif

src/util/unaligned.h

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Helper functions for unaligned loads and stores.
+ */
+
+#ifndef UNALIGNED_H
+#define UNALIGNED_H
+
+#include "ue2common.h"
+
+#if !defined(_WIN32)
+#define PACKED__MAY_ALIAS __attribute__((packed, may_alias))
+#else
+#define PACKED__MAY_ALIAS
+#pragma pack(push, 1) // pack everything until told otherwise
+#endif
+
+/// Perform an unaligned 16-bit load
+static really_inline
+u16 unaligned_load_u16(const void *ptr) {
+    struct unaligned { u16 u; } PACKED__MAY_ALIAS;
+    const struct unaligned *uptr = (const struct unaligned *)ptr;
+    return uptr->u;
+}
+
+/// Perform an unaligned 32-bit load
+static really_inline
+u32 unaligned_load_u32(const void *ptr) {
+    struct unaligned { u32 u; } PACKED__MAY_ALIAS;
+    const struct unaligned *uptr = (const struct unaligned *)ptr;
+    return uptr->u;
+}
+
+/// Perform an unaligned 64-bit load
+static really_inline
+u64a unaligned_load_u64a(const void *ptr) {
+    struct unaligned { u64a u; } PACKED__MAY_ALIAS;
+    const struct unaligned *uptr = (const struct unaligned *)ptr;
+    return uptr->u;
+}
+
+/// Perform an unaligned 16-bit store
+static really_inline
+void unaligned_store_u16(void *ptr, u16 val) {
+    struct unaligned { u16 u; } PACKED__MAY_ALIAS;
+    struct unaligned *uptr = (struct unaligned *)ptr;
+    uptr->u = val;
+}
+
+/// Perform an unaligned 32-bit store
+static really_inline
+void unaligned_store_u32(void *ptr, u32 val) {
+    struct unaligned { u32 u; } PACKED__MAY_ALIAS;
+    struct unaligned *uptr = (struct unaligned *)ptr;
+    uptr->u = val;
+}
+
+/// Perform an unaligned 64-bit store
+static really_inline
+void unaligned_store_u64a(void *ptr, u64a val) {
+    struct unaligned { u64a u; } PACKED__MAY_ALIAS;
+    struct unaligned *uptr = (struct unaligned *)ptr;
+    uptr->u = val;
+}
+#if defined(_WIN32)
+#pragma pack(pop)
+#endif // win32
+
+#undef PACKED__MAY_ALIAS
+
+#endif // UNALIGNED_H

src/util/unicode_def.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UNICODE_DEF_H
+#define UNICODE_DEF_H
+
+#include "ue2common.h"
+
+#define MAX_UNICODE 0x10FFFF
+#define INVALID_UNICODE 0xffffffff /* unicode could never go above 2^31 */
+
+#define UTF_2CHAR_MIN (1U << 7)
+#define UTF_3CHAR_MIN (1U << 11)
+#define UTF_4CHAR_MIN (1U << 16)
+#define UTF_CONT_SHIFT 6
+#define UTF_CONT_BYTE_RANGE (1U << UTF_CONT_SHIFT)
+#define UTF_CONT_BYTE_HEADER  ((u8)0x80) /* 10xx xxxx */
+#define UTF_TWO_BYTE_HEADER   ((u8)0xc0) /* 110x xxxx */
+#define UTF_THREE_BYTE_HEADER ((u8)0xe0) /* 1110 xxxx */
+#define UTF_FOUR_BYTE_HEADER  ((u8)0xf0) /* 1111 0xxx */
+
+#define UTF_CONT_BYTE_VALUE_MASK 0x3f
+
+#define UTF_CONT_MIN UTF_CONT_BYTE_HEADER
+#define UTF_CONT_MAX (UTF_TWO_BYTE_HEADER - 1)
+
+#define UTF_TWO_BYTE_MIN UTF_TWO_BYTE_HEADER
+#define UTF_TWO_BYTE_MAX (UTF_THREE_BYTE_HEADER - 1)
+
+#define UTF_THREE_BYTE_MIN UTF_THREE_BYTE_HEADER
+#define UTF_THREE_BYTE_MAX (UTF_FOUR_BYTE_HEADER - 1)
+
+#define UTF_FOUR_BYTE_MIN UTF_FOUR_BYTE_HEADER
+#define UTF_FOUR_BYTE_MAX ((u8)0xf4)
+
+#define UTF_CONT_CR CharReach(UTF_CONT_MIN, UTF_CONT_MAX)
+#define UTF_ASCII_CR CharReach(0, 127)
+#define UTF_START_CR CharReach(UTF_TWO_BYTE_MIN, UTF_FOUR_BYTE_MAX)
+#define UTF_TWO_START_CR CharReach(UTF_TWO_BYTE_MIN, UTF_TWO_BYTE_MAX)
+#define UTF_THREE_START_CR CharReach(UTF_THREE_BYTE_MIN, UTF_THREE_BYTE_MAX)
+#define UTF_FOUR_START_CR CharReach(UTF_FOUR_BYTE_MIN, UTF_FOUR_BYTE_MAX)
+
+#define UNICODE_SURROGATE_MIN 0xd800
+#define UNICODE_SURROGATE_MAX 0xdfff
+
+#ifdef __cplusplus
+
+namespace ue2 {
+typedef u32 unichar; /* represents a unicode code point */
+
+static UNUSED
+u8 makeContByte(u8 val) {
+    return UTF_CONT_BYTE_HEADER | (val & UTF_CONT_BYTE_VALUE_MASK);
+}
+
+} // namespace
+
+#endif // __cplusplus
+
+#endif

src/util/unicode_set.h

@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UNICODE_SET
+#define UNICODE_SET
+
+#include "unicode_def.h"
+
+#include <boost/icl/interval_set.hpp>
+
+namespace ue2 {
+
+class CodePointSet {
+public:
+    typedef boost::icl::closed_interval<unichar> interval;
+    typedef boost::icl::interval_set<unichar, std::less, interval> implT;
+    typedef implT::const_iterator const_iterator;
+
+    CodePointSet(void) {}
+
+    explicit CodePointSet(const interval &st) : impl(st) {}
+
+    bool none(void) const {
+        return impl.empty();
+    }
+
+    void set(unichar c) {
+        assert(c <= MAX_UNICODE);
+        impl.insert(c);
+    }
+
+    void unset(unichar c) {
+        assert(c <= MAX_UNICODE);
+        impl.subtract(c);
+    }
+
+    void setRange(unichar from, unichar to) { /* inclusive */
+        assert(from <= to);
+        assert(to <= MAX_UNICODE);
+        impl.insert(interval(from, to));
+    }
+
+    void unsetRange(unichar from, unichar to) { /* inclusive */
+        assert(from <= to);
+        assert(to <= MAX_UNICODE);
+        impl.subtract(interval(from, to));
+    }
+
+    void flip(void) {
+        impl = implT(interval(0, MAX_UNICODE)) - impl;
+    }
+
+    void operator|=(const CodePointSet &a) {
+        impl += a.impl;
+    }
+
+    const_iterator begin(void) const {
+        return impl.begin();
+    }
+
+    const_iterator end(void) const {
+        return impl.end();
+    }
+
+    size_t count(void) const {
+        return cardinality(impl);
+    }
+
+    CodePointSet operator~(void) const {
+        CodePointSet rv = *this;
+        rv.flip();
+        return rv;
+    }
+
+    bool operator==(const CodePointSet &a) const {
+        return is_element_equal(impl, a.impl);
+    }
+
+    bool operator!=(const CodePointSet &a) const {
+        return !is_element_equal(impl, a.impl);
+    }
+
+    bool isSubset(const CodePointSet &a) const {
+        // Check that adding an interval set has no effect
+        return ((impl + a.impl) == impl);
+    }
+
+    void operator-=(const CodePointSet &a) {
+        impl -= a.impl;
+    }
+
+    /* finds the nth set codepoint, returns INVALID_UNICODE on failure */
+    unichar at(size_t pos) const {
+        for (const_iterator i = begin(), e = end(); i != e; ++i) {
+            size_t int_count = cardinality(*i);
+            if (int_count <= pos) {
+                /* not in this interval, check next */
+                pos -= int_count;
+                continue;
+            } else {
+                return lower(*i) + pos;
+            }
+        }
+
+        return INVALID_UNICODE;
+    }
+
+    void swap(CodePointSet &other) { impl.swap(other.impl); }
+
+private:
+    implT impl;
+};
+
+} // namespace ue2
+
+#endif

src/util/uniform_ops.h

@@ -0,0 +1,226 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Uniformly-named primitives named by target type.
+ *
+ * The following are a set of primitives named by target type, so that we can
+ * macro the hell out of all our NFA implementations. Hurrah!
+ */
+
+#ifndef UNIFORM_OPS_H
+#define UNIFORM_OPS_H
+
+#include "ue2common.h"
+#include "simd_utils.h"
+#include "unaligned.h"
+
+// Aligned loads
+#define load_u8(a)          (*(const u8 *)(a))
+#define load_u16(a)         (*(const u16 *)(a))
+#define load_u32(a)         (*(const u32 *)(a))
+#define load_u64a(a)        (*(const u64a *)(a))
+#define load_m128(a)        load128(a)
+#define load_m256(a)        load256(a)
+#define load_m384(a)        load384(a)
+#define load_m512(a)        load512(a)
+
+// Unaligned loads
+#define loadu_u8(a)          (*(const u8 *)(a))
+#define loadu_u16(a)         unaligned_load_u16((const u8 *)(a))
+#define loadu_u32(a)         unaligned_load_u32((const u8 *)(a))
+#define loadu_u64a(a)        unaligned_load_u64a((const u8 *)(a))
+#define loadu_m128(a)        loadu128(a)
+#define loadu_m256(a)        loadu256(a)
+#define loadu_m384(a)        loadu384(a)
+#define loadu_m512(a)        loadu512(a)
+
+// Aligned stores
+#define store_u8(ptr, a)    do { *(u8 *)(ptr) = (a); } while(0)
+#define store_u16(ptr, a)   do { *(u16 *)(ptr) = (a); } while(0)
+#define store_u32(ptr, a)   do { *(u32 *)(ptr) = (a); } while(0)
+#define store_u64a(ptr, a)  do { *(u64a *)(ptr) = (a); } while(0)
+#define store_m128(ptr, a)  store128(ptr, a)
+#define store_m256(ptr, a)  store256(ptr, a)
+#define store_m384(ptr, a)  store384(ptr, a)
+#define store_m512(ptr, a)  store512(ptr, a)
+
+// Unaligned stores
+#define storeu_u8(ptr, a)    do { *(u8 *)(ptr) = (a); } while(0)
+#define storeu_u16(ptr, a)   unaligned_store_u16(ptr, a)
+#define storeu_u32(ptr, a)   unaligned_store_u32(ptr, a)
+#define storeu_u64a(ptr, a)  unaligned_store_u64a(ptr, a)
+#define storeu_m128(ptr, a)  storeu128(ptr, a)
+#define storeu_m256(ptr, a)  storeu256(ptr, a)
+#define storeu_m384(ptr, a)  storeu384(ptr, a)
+#define storeu_m512(ptr, a)  storeu512(ptr, a)
+
+#define zero_u8             0
+#define zero_u32            0
+#define zero_u64a           0
+#define zero_m128           zeroes128()
+#define zero_m256           zeroes256()
+#define zero_m384           zeroes384()
+#define zero_m512           zeroes512()
+
+#define ones_u8             0xff
+#define ones_u32            0xfffffffful
+#define ones_u64a           0xffffffffffffffffull
+#define ones_m128           ones128()
+#define ones_m256           ones256()
+#define ones_m384           ones384()
+#define ones_m512           ones512()
+
+#define or_u8(a, b)         ((a) | (b))
+#define or_u32(a, b)        ((a) | (b))
+#define or_u64a(a, b)       ((a) | (b))
+#define or_m128(a, b)       (or128(a, b))
+#define or_m256(a, b)       (or256(a, b))
+#define or_m384(a, b)       (or384(a, b))
+#define or_m512(a, b)       (or512(a, b))
+
+#define and_u8(a, b)        ((a) & (b))
+#define and_u32(a, b)       ((a) & (b))
+#define and_u64a(a, b)      ((a) & (b))
+#define and_m128(a, b)      (and128(a, b))
+#define and_m256(a, b)      (and256(a, b))
+#define and_m384(a, b)      (and384(a, b))
+#define and_m512(a, b)      (and512(a, b))
+
+#define not_u8(a)           (~(a))
+#define not_u32(a)          (~(a))
+#define not_u64a(a)         (~(a))
+#define not_m128(a)         (not128(a))
+#define not_m256(a)         (not256(a))
+#define not_m384(a)         (not384(a))
+#define not_m512(a)         (not512(a))
+
+#define andnot_u8(a, b)     ((~(a)) & (b))
+#define andnot_u32(a, b)    ((~(a)) & (b))
+#define andnot_u64a(a, b)   ((~(a)) & (b))
+#define andnot_m128(a, b)   (andnot128(a, b))
+#define andnot_m256(a, b)   (andnot256(a, b))
+#define andnot_m384(a, b)   (andnot384(a, b))
+#define andnot_m512(a, b)   (andnot512(a, b))
+
+#define shift_u32(a, b)     ((a) << (b))
+#define shift_u64a(a, b)    ((a) << (b))
+#define shift_m128(a, b)    (shift128(a, b))
+#define shift_m256(a, b)    (shift256(a, b))
+#define shift_m384(a, b)    (shift384(a, b))
+#define shift_m512(a, b)    (shift512(a, b))
+
+#define isZero_u8(a)        ((a) == 0)
+#define isZero_u32(a)       ((a) == 0)
+#define isZero_u64a(a)      ((a) == 0)
+#define isZero_m128(a)      (!isnonzero128(a))
+#define isZero_m256(a)      (!isnonzero256(a))
+#define isZero_m384(a)      (!isnonzero384(a))
+#define isZero_m512(a)      (!isnonzero512(a))
+
+#define isNonZero_u8(a)     ((a) != 0)
+#define isNonZero_u32(a)    ((a) != 0)
+#define isNonZero_u64a(a)   ((a) != 0)
+#define isNonZero_m128(a)   (isnonzero128(a))
+#define isNonZero_m256(a)   (isnonzero256(a))
+#define isNonZero_m384(a)   (isnonzero384(a))
+#define isNonZero_m512(a)   (isnonzero512(a))
+
+#define diffrich_u32(a, b)  ((a) != (b))
+#define diffrich_u64a(a, b) ((a) != (b) ? 3 : 0) //TODO: impl 32bit granularity
+#define diffrich_m128(a, b) (diffrich128(a, b))
+#define diffrich_m256(a, b) (diffrich256(a, b))
+#define diffrich_m384(a, b) (diffrich384(a, b))
+#define diffrich_m512(a, b) (diffrich512(a, b))
+
+#define diffrich64_u32(a, b)  ((a) != (b))
+#define diffrich64_u64a(a, b) ((a) != (b) ? 1 : 0)
+#define diffrich64_m128(a, b) (diffrich64_128(a, b))
+#define diffrich64_m256(a, b) (diffrich64_256(a, b))
+#define diffrich64_m384(a, b) (diffrich64_384(a, b))
+#define diffrich64_m512(a, b) (diffrich64_512(a, b))
+
+#define noteq_u8(a, b)      ((a) != (b))
+#define noteq_u32(a, b)     ((a) != (b))
+#define noteq_u64a(a, b)    ((a) != (b))
+#define noteq_m128(a, b)    (diff128(a, b))
+#define noteq_m256(a, b)    (diff256(a, b))
+#define noteq_m384(a, b)    (diff384(a, b))
+#define noteq_m512(a, b)    (diff512(a, b))
+
+#define partial_store_m128(ptr, v, sz) storebytes128(ptr, v, sz)
+#define partial_store_m256(ptr, v, sz) storebytes256(ptr, v, sz)
+#define partial_store_m384(ptr, v, sz) storebytes384(ptr, v, sz)
+#define partial_store_m512(ptr, v, sz) storebytes512(ptr, v, sz)
+
+#define partial_load_m128(ptr, sz) loadbytes128(ptr, sz)
+#define partial_load_m256(ptr, sz) loadbytes256(ptr, sz)
+#define partial_load_m384(ptr, sz) loadbytes384(ptr, sz)
+#define partial_load_m512(ptr, sz) loadbytes512(ptr, sz)
+
+#define store_compressed_u32(ptr, x, m)     storecompressed32(ptr, x, m)
+#define store_compressed_u64a(ptr, x, m)    storecompressed64(ptr, x, m)
+#define store_compressed_m128(ptr, x, m)    storecompressed128(ptr, x, m)
+#define store_compressed_m256(ptr, x, m)    storecompressed256(ptr, x, m)
+#define store_compressed_m384(ptr, x, m)    storecompressed384(ptr, x, m)
+#define store_compressed_m512(ptr, x, m)    storecompressed512(ptr, x, m)
+
+#define load_compressed_u32(x, ptr, m)      loadcompressed32(x, ptr, m)
+#define load_compressed_u64a(x, ptr, m)     loadcompressed64(x, ptr, m)
+#define load_compressed_m128(x, ptr, m)     loadcompressed128(x, ptr, m)
+#define load_compressed_m256(x, ptr, m)     loadcompressed256(x, ptr, m)
+#define load_compressed_m384(x, ptr, m)     loadcompressed384(x, ptr, m)
+#define load_compressed_m512(x, ptr, m)     loadcompressed512(x, ptr, m)
+
+static really_inline void clearbit_u32(u32 *p, u32 n) {
+    assert(n < sizeof(*p) * 8);
+    *p &= ~(1U << n);
+}
+static really_inline void clearbit_u64a(u64a *p, u32 n) {
+    assert(n < sizeof(*p) * 8);
+    *p &= ~(1ULL << n);
+}
+#define clearbit_m128(ptr, n)   (clearbit128(ptr, n))
+#define clearbit_m256(ptr, n)   (clearbit256(ptr, n))
+#define clearbit_m384(ptr, n)   (clearbit384(ptr, n))
+#define clearbit_m512(ptr, n)   (clearbit512(ptr, n))
+
+static really_inline char testbit_u32(const u32 *p, u32 n) {
+    assert(n < sizeof(*p) * 8);
+    return !!(*p & (1U << n));
+}
+static really_inline char testbit_u64a(const u64a *p, u32 n) {
+    assert(n < sizeof(*p) * 8);
+    return !!(*p & (1ULL << n));
+}
+#define testbit_m128(ptr, n)    (testbit128(ptr, n))
+#define testbit_m256(ptr, n)    (testbit256(ptr, n))
+#define testbit_m384(ptr, n)    (testbit384(ptr, n))
+#define testbit_m512(ptr, n)    (testbit512(ptr, n))
+
+#endif

src/util/verify_types.h

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UTIL_VERIFY_TYPES
+#define UTIL_VERIFY_TYPES
+
+#include "ue2common.h"
+
+#include <cassert>
+
+namespace ue2 {
+
+template<typename Int_T>
+static UNUSED u8 verify_u8(Int_T val) {
+    assert(val == (Int_T)((u8)val)); // there and back again
+    return (u8)(val);
+}
+
+template<typename Int_T>
+static UNUSED s8 verify_s8(Int_T val) {
+    assert(val == (Int_T)((s8)val)); // there and back again
+    return (s8)(val);
+}
+
+template<typename Int_T>
+static UNUSED s16 verify_s16(Int_T val) {
+    assert(val == (Int_T)((s16)val)); // there and back again
+    return (s16)(val);
+}
+
+template<typename Int_T>
+static UNUSED u16 verify_u16(Int_T val) {
+    assert(val == (Int_T)((u16)val)); // there and back again
+    return (u16)(val);
+}
+
+template<typename Int_T>
+static UNUSED s32 verify_s32(Int_T val) {
+    assert(val == (Int_T)((s32)val)); // there and back again
+    return (s32)(val);
+}
+
+template<typename Int_T>
+static UNUSED u32 verify_u32(Int_T val) {
+    assert(val == (Int_T)((u32)val)); // there and back again
+    return (u32)(val);
+}
+
+} // namespace ue2
+
+#endif // UTIL_VERIFY_TYPES