vectorscan/chimera/ch_scratch.c

/*
 * Copyright (c) 2018, Intel Corporation
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *  * Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *  * Neither the name of Intel Corporation nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/** \file
 * \brief Chimera: scratch space alloc.
 */

#include <string.h>

#include "allocator.h"
#include "ch.h"
#include "hs.h"
#include "hs_internal.h"
#include "ue2common.h"
#include "ch_alloc.h"
#include "ch_internal.h"
#include "ch_scratch.h"
#include "ch_database.h"

static
size_t getPatternDataSize(const ch_scratch_t *s) {
    size_t numCapturingStructs =
        s->patternCount * (s->maxCaptureGroups + 1);
    return (sizeof(struct ch_patterndata) * s->patternCount) +
           alignof(struct ch_capture) + // padding
           (sizeof(struct ch_capture) * numCapturingStructs);
}

static
void initPatternData(const ch_scratch_t *s) {
    // ch_capture array is aligned, directly after the patterndata array.
    char *ptr = (char *)s->patternData +
            (sizeof(struct ch_patterndata) * s->patternCount);
    struct ch_capture *cap = (struct ch_capture *)
            (ROUNDUP_PTR(ptr, alignof(struct ch_capture)));

    for (u32 i = 0; i < s->patternCount; i++) {
        struct ch_patterndata *pd = &s->patternData[i];
        pd->match = cap;
        DEBUG_PRINTF("pattern %u: pd=%p, match=%p\n", i, pd, pd->match);
        cap += (s->maxCaptureGroups + 1);
    }
}

static
ch_error_t alloc_scratch(const ch_scratch_t *proto, ch_scratch_t **scratch) {
    size_t ovectorSize = (proto->maxCaptureGroups + 1) * sizeof(int) * 3;
    size_t capturedSize =
        sizeof(struct ch_capture) * (proto->maxCaptureGroups + 1);
    size_t patternDataSize = getPatternDataSize(proto);
    size_t activeSize = proto->activeSize;
    size_t queueSize = proto->patternCount * sizeof(struct queue_item);

    // max padding for alignment below.
    size_t padding = alignof(int) + alignof(struct ch_capture) +
                     alignof(struct ch_patterndata) +
                     alignof(struct queue_item);

    size_t allocSize = sizeof(ch_scratch_t) + ovectorSize + capturedSize +
                       patternDataSize + activeSize + queueSize + padding
                       + 256; /* padding for cacheline alignment */
    ch_scratch_t *s;
    ch_scratch_t *s_tmp = ch_scratch_alloc(allocSize);
    ch_error_t err = ch_check_alloc(s_tmp);
    if (err != CH_SUCCESS) {
        ch_scratch_free(s_tmp);
        *scratch = NULL;
        return err;
    }

    memset(s_tmp, 0, allocSize);
    s = ROUNDUP_PTR(s_tmp, 64);
    // Set ordinary members.
    *s = *proto;

    s->magic = CH_SCRATCH_MAGIC;
    s->in_use = 0;
    s->scratch_alloc = (char *)s_tmp;

    // Set pointers internal to allocation.

    char *ptr = (char *)s + sizeof(*s);
    ptr = ROUNDUP_PTR(ptr, alignof(int));
    s->ovector = (int *)ptr;
    ptr += ovectorSize;

    ptr = ROUNDUP_PTR(ptr, alignof(struct ch_capture));
    s->captured = (struct ch_capture *)ptr;
    ptr += capturedSize;

    ptr = ROUNDUP_PTR(ptr, alignof(struct ch_patterndata));
    s->patternData = (struct ch_patterndata *)ptr;
    ptr += patternDataSize;

    // Pre-fill pattern data, setting captureOffsets
    initPatternData(s);

    ptr = ROUNDUP_PTR(ptr, alignof(struct queue_item));
    s->pq.item = (struct queue_item *)ptr;
    ptr += queueSize;

    s->active = (u8 *)ptr;

    // Store size.
    s->scratchSize = allocSize;

    // We should never overrun our allocation.
    assert((ptr + activeSize) - (char *)s <= (ptrdiff_t)allocSize);

    *scratch = s;
    return CH_SUCCESS;
}

HS_PUBLIC_API
ch_error_t HS_CDECL ch_alloc_scratch(const ch_database_t *hydb,
                                     ch_scratch_t **scratch) {
    if (!hydb || !scratch) {
        DEBUG_PRINTF("invalid args\n");
        return CH_INVALID;
    }

    DEBUG_PRINTF("hydb=%p, &scratch=%p\n", hydb, scratch);
    ch_error_t rv = hydbIsValid(hydb);
    if (rv != CH_SUCCESS) {
        DEBUG_PRINTF("invalid database\n");
        return rv;
    }

    if (*scratch != NULL) {
        /* has to be aligned before we can do anything with it */
        if (!ISALIGNED_CL(*scratch)) {
            return CH_INVALID;
        }
        if ((*scratch)->magic != CH_SCRATCH_MAGIC) {
            return CH_INVALID;
        }
        if (markScratchInUse(*scratch)) {
            return CH_SCRATCH_IN_USE;
        }
    }

    // We allocate a prototype of the scratch header to do our sizing with.
    ch_scratch_t *proto;
    ch_scratch_t *proto_tmp = ch_scratch_alloc(sizeof(ch_scratch_t) + 256);
    ch_error_t proto_ret = ch_check_alloc(proto_tmp);
    if (proto_ret != CH_SUCCESS) {
        ch_scratch_free(proto_tmp);
        ch_scratch_free(*scratch);
        *scratch = NULL;
        return proto_ret;
    }

    proto = ROUNDUP_PTR(proto_tmp, 64);

    int resize = 0;
    if (*scratch) {
        *proto = **scratch;
    } else {
        memset(proto, 0, sizeof(*proto));
        resize = 1;
    }
    proto->scratch_alloc = (char *)proto_tmp;

    const struct ch_bytecode *db = ch_get_bytecode(hydb);

    if (db->maxCaptureGroups > proto->maxCaptureGroups) {
        proto->maxCaptureGroups = db->maxCaptureGroups;
        resize = 1;
    }

    if (db->patternCount > proto->patternCount) {
        proto->patternCount = db->patternCount;
        proto->activeSize = db->activeSize;
        resize = 1;
    }

    if (resize) {
        if (*scratch) {
            ch_scratch_free((*scratch)->scratch_alloc);
        }

        ch_error_t alloc_ret = alloc_scratch(proto, scratch);
        ch_scratch_free(proto_tmp);
        if (alloc_ret != CH_SUCCESS) {
            *scratch = NULL;
            return alloc_ret;
        }
    } else {
        ch_scratch_free(proto_tmp);
        unmarkScratchInUse(*scratch);
    }

    if (db->flags & CHIMERA_FLAG_NO_MULTIMATCH) {
        return CH_SUCCESS;
    }

    // We may still have to realloc the underlying Hyperscan scratch.
    rv = hs_alloc_scratch(getHyperscanDatabase(db),
                           &(*scratch)->multi_scratch);
    if (rv != HS_SUCCESS) {
        DEBUG_PRINTF("hs_alloc_scratch for multi_scratch failed\n");
        hs_free_scratch((*scratch)->multi_scratch);
        ch_scratch_free((*scratch)->scratch_alloc);
        *scratch = NULL;
        return rv;
    }

    return CH_SUCCESS;
}

HS_PUBLIC_API
ch_error_t HS_CDECL ch_clone_scratch(const ch_scratch_t *src,
                                     ch_scratch_t **dest) {
    if (!dest || !src || !ISALIGNED_CL(src) ||
        src->magic != CH_SCRATCH_MAGIC) {
        DEBUG_PRINTF("scratch invalid\n");
        return CH_INVALID;
    }

    ch_error_t ret = alloc_scratch(src, dest);
    if (ret != CH_SUCCESS) {
        DEBUG_PRINTF("alloc_scratch failed\n");
        *dest = NULL;
        return ret;
    }

    if (src->multi_scratch) {
        (*dest)->multi_scratch = NULL;
        ret = hs_clone_scratch(src->multi_scratch, &(*dest)->multi_scratch);
        if (ret != HS_SUCCESS) {
            DEBUG_PRINTF("hs_clone_scratch(multi_scratch,...) failed\n");
            ch_scratch_free(*dest);
            return ret;
        }
    }

    return CH_SUCCESS;
}

HS_PUBLIC_API
ch_error_t HS_CDECL ch_free_scratch(ch_scratch_t *scratch) {
    ch_error_t ret = CH_SUCCESS;
    if (scratch) {
        /* has to be aligned before we can do anything with it */
        if (!ISALIGNED_CL(scratch)) {
            return CH_INVALID;
        }
        if (scratch->magic != CH_SCRATCH_MAGIC) {
            return CH_INVALID;
        }
        if (markScratchInUse(scratch)) {
            return CH_SCRATCH_IN_USE;
        }

        if (scratch->multi_scratch) {
            ret = hs_free_scratch(scratch->multi_scratch);
        }

        scratch->magic = 0;
        assert(scratch->scratch_alloc);
        DEBUG_PRINTF("scratch %p is really at %p : freeing\n", scratch,
                     scratch->scratch_alloc);
        ch_scratch_free(scratch->scratch_alloc);
    }

    return ret;
}

/** Not public, but used for info from our internal tools. Note that in the
 * hybrid matcher the scratch is definitely not a contiguous memory region. */
HS_PUBLIC_API
ch_error_t HS_CDECL ch_scratch_size(const ch_scratch_t *scratch, size_t *size) {
    ch_error_t ret = CH_SUCCESS;
    if (!size || !scratch || !ISALIGNED_CL(scratch) ||
        scratch->magic != CH_SCRATCH_MAGIC) {
        return CH_INVALID;
    } else {
        size_t multi_size = 0;

        if (scratch->multi_scratch) {
            ret = hs_scratch_size(scratch->multi_scratch, &multi_size);
        }
        if (ret) {
            multi_size = 0;
        }

        *size = scratch->scratchSize + multi_size;
    }

    return ret;
}