multibit, fatbit: make _size build-time only

This commit makes mmbit_size() and fatbit_size compile-time only, and
adds a resource limit for very large multibits.

src/util/fatbit.h

@@ -40,6 +40,10 @@
 #include "multibit.h"
 #include "ue2common.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define MIN_FAT_SIZE 32
 
 struct fatbit {
@@ -82,11 +86,8 @@ u32 fatbit_iterate(const struct fatbit *bits, u32 total_bits, u32 it_in) {
     return mmbit_iterate(bits->fb_int.raw, total_bits, it_in);
 }
 
-/** \brief Return the size in bytes of a fatbit that can store the given
- * number of bits.
- *
- * Not for use in performance-critical code, implementation is in fatbit.c.
- */
-u32 fatbit_size(u32 total_bits);
+#ifdef __cplusplus
+} // extern "C"
+#endif
 
 #endif

src/util/fatbit_build.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -26,9 +26,19 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "fatbit_build.h"
+
 #include "fatbit.h"
-#include "multibit.h"
+#include "multibit_build.h"
+
+#include <algorithm>
+
+using namespace std;
+
+namespace ue2 {
 
 u32 fatbit_size(u32 total_bits) {
-    return MAX(sizeof(struct fatbit), mmbit_size(total_bits));
+    return max(u32{sizeof(struct fatbit)}, mmbit_size(total_bits));
 }
+
+} // namespace ue2

src/util/fatbit_build.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2016, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Fatbit: build code
+ */
+
+#ifndef FATBIT_BUILD_H
+#define FATBIT_BUILD_H
+
+#include "ue2common.h"
+
+namespace ue2 {
+
+/**
+ * \brief Return the size in bytes of a fatbit that can store the given
+ * number of bits.
+ */
+u32 fatbit_size(u32 total_bits);
+
+} // namespace ue2
+
+#endif // FATBIT_BUILD_H

src/util/multibit.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -138,62 +138,3 @@ const u32 mmbit_root_offset_from_level[7] = {
     1 + (1 << MMB_KEY_SHIFT) + (1 << MMB_KEY_SHIFT * 2) + (1 << MMB_KEY_SHIFT * 3) + (1 << MMB_KEY_SHIFT * 4),
     1 + (1 << MMB_KEY_SHIFT) + (1 << MMB_KEY_SHIFT * 2) + (1 << MMB_KEY_SHIFT * 3) + (1 << MMB_KEY_SHIFT * 4) + (1 << MMB_KEY_SHIFT * 5),
 };
-
-u32 mmbit_size(u32 total_bits) {
-    MDEBUG_PRINTF("%u\n", total_bits);
-
-    // Flat model multibit structures are just stored as a bit vector.
-    if (total_bits <= MMB_FLAT_MAX_BITS) {
-        return ROUNDUP_N(total_bits, 8) / 8;
-    }
-
-    u64a current_level = 1; // Number of blocks on current level.
-    u64a total = 0;         // Total number of blocks.
-    while (current_level * MMB_KEY_BITS < total_bits) {
-        total += current_level;
-        current_level <<= MMB_KEY_SHIFT;
-    }
-
-    // Last level is a one-for-one bit vector. It needs room for total_bits
-    // elements, rounded up to the nearest block.
-    u64a last_level = ((u64a)total_bits + MMB_KEY_BITS - 1) / MMB_KEY_BITS;
-    total += last_level;
-
-    assert(total * sizeof(MMB_TYPE) <= UINT32_MAX);
-    return (u32)(total * sizeof(MMB_TYPE));
-}
-
-#ifdef DUMP_SUPPORT
-
-#include <stdio.h>
-#include <stdlib.h>
-
-/** \brief Dump a sparse iterator's keys to stdout. */
-void mmbit_sparse_iter_dump(const struct mmbit_sparse_iter *it,
-                            u32 total_bits) {
-    // Expediency and future-proofing: create a temporary multibit of the right
-    // size with all the bits on, then walk it with this sparse iterator.
-    size_t bytes = mmbit_size(total_bits);
-    u8 *bits = malloc(bytes);
-    if (!bits) {
-        printf("Failed to alloc %zu bytes for temp multibit", bytes);
-        return;
-    }
-    for (u32 i = 0; i < total_bits; i++) {
-        mmbit_set_i(bits, total_bits, i);
-    }
-
-    struct mmbit_sparse_state s[MAX_SPARSE_ITER_STATES];
-    u32 idx = 0;
-    for (u32 i = mmbit_sparse_iter_begin(bits, total_bits, &idx, it, s);
-             i != MMB_INVALID;
-             i = mmbit_sparse_iter_next(bits, total_bits, i, &idx, it, s)) {
-        printf("%u ", i);
-    }
-
-    printf("(%u keys)", idx + 1);
-
-    free(bits);
-}
-
-#endif // DUMP_SUPPORT

src/util/multibit_build.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -34,6 +34,7 @@
 #include "scatter.h"
 #include "ue2common.h"
 #include "rose/rose_build_scatter.h"
+#include "util/compile_error.h"
 
 #include <cassert>
 #include <cstring> // for memset
@@ -45,6 +46,32 @@ using namespace std;
 
 namespace ue2 {
 
+u32 mmbit_size(u32 total_bits) {
+    if (total_bits > MMB_MAX_BITS) {
+        throw ResourceLimitError();
+    }
+
+    // Flat model multibit structures are just stored as a bit vector.
+    if (total_bits <= MMB_FLAT_MAX_BITS) {
+        return ROUNDUP_N(total_bits, 8) / 8;
+    }
+
+    u64a current_level = 1; // Number of blocks on current level.
+    u64a total = 0;         // Total number of blocks.
+    while (current_level * MMB_KEY_BITS < total_bits) {
+        total += current_level;
+        current_level <<= MMB_KEY_SHIFT;
+    }
+
+    // Last level is a one-for-one bit vector. It needs room for total_bits
+    // elements, rounded up to the nearest block.
+    u64a last_level = ((u64a)total_bits + MMB_KEY_BITS - 1) / MMB_KEY_BITS;
+    total += last_level;
+
+    assert(total * sizeof(MMB_TYPE) <= UINT32_MAX);
+    return (u32)(total * sizeof(MMB_TYPE));
+}
+
 namespace {
 struct TreeNode {
     MMB_TYPE mask = 0;
@@ -133,6 +160,7 @@ void mmbBuildSparseIterator(vector<mmbit_sparse_iter> &out,
     assert(out.empty());
     assert(!bits.empty());
     assert(total_bits > 0);
+    assert(total_bits <= MMB_MAX_BITS);
 
     DEBUG_PRINTF("building sparse iter for %zu of %u bits\n",
                  bits.size(), total_bits);

src/util/multibit_build.h

@@ -50,6 +50,15 @@ size_t hash_value(const mmbit_sparse_iter &iter) {
 
 namespace ue2 {
 
+/**
+ * \brief Return the size in bytes of a multibit that can store the given
+ * number of bits.
+ *
+ * This will throw a resource limit assertion if the requested mmbit is too
+ * large.
+ */
+u32 mmbit_size(u32 total_bits);
+
 /** \brief Construct a sparse iterator over the values in \a bits for a
  * multibit of size \a total_bits. */
 void mmbBuildSparseIterator(std::vector<mmbit_sparse_iter> &out,

src/util/multibit_internal.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -47,6 +47,9 @@ extern "C" {
 typedef u64a MMB_TYPE; /**< Basic block type for mmbit operations. */
 #define MMB_MAX_LEVEL 6 /**< Maximum level in the mmbit pyramid. */
 
+/** \brief Maximum number of keys (bits) in a multibit. */
+#define MMB_MAX_BITS (1U << 31)
+
 /** \brief Sparse iterator record type.
  *
  * A sparse iterator is a tree of these records, where val identifies the
@@ -71,13 +74,6 @@ struct mmbit_sparse_state {
 /** \brief Maximum number of \ref mmbit_sparse_state that could be needed. */
 #define MAX_SPARSE_ITER_STATES (6 + 1)
 
-/** \brief Return the size in bytes of a multibit that can store the given
- * number of bits.
- *
- * Not for use in performance-critical code, implementation is in multibit.c.
- */
-u32 mmbit_size(u32 total_bits);
-
 #ifdef __cplusplus
 } // extern "C"
 #endif