mirror of
https://github.com/VectorCamp/vectorscan.git
synced 2025-06-28 16:41:01 +03:00
hsbench: sqlite output support
This commit is contained in:
parent
193bdb628e
commit
2f17ef032c
@ -38,6 +38,9 @@ SET(hsbench_SOURCES
|
||||
huge.cpp
|
||||
huge.h
|
||||
main.cpp
|
||||
sqldb.cpp
|
||||
sqldb.h
|
||||
sqldb_bind.h
|
||||
thread_barrier.h
|
||||
timer.h
|
||||
)
|
||||
|
@ -42,4 +42,9 @@ extern bool forceEditDistance;
|
||||
extern unsigned editDistance;
|
||||
extern bool printCompressSize;
|
||||
|
||||
struct SqlFailure {
|
||||
explicit SqlFailure(const std::string &s) : message(s) {}
|
||||
std::string message;
|
||||
};
|
||||
|
||||
#endif // COMMON_H
|
||||
|
@ -34,6 +34,7 @@
|
||||
#include "expressions.h"
|
||||
#include "heapstats.h"
|
||||
#include "huge.h"
|
||||
#include "sqldb.h"
|
||||
#include "timer.h"
|
||||
|
||||
#include "database.h"
|
||||
@ -113,7 +114,8 @@ int onMatchEcho(unsigned int id, unsigned long long, unsigned long long to,
|
||||
return 0;
|
||||
}
|
||||
|
||||
EngineHyperscan::EngineHyperscan(hs_database_t *db_in) : db(db_in) {
|
||||
EngineHyperscan::EngineHyperscan(hs_database_t *db_in, CompileStats cs)
|
||||
: db(db_in), compile_stats(std::move(cs)) {
|
||||
assert(db);
|
||||
}
|
||||
|
||||
@ -234,6 +236,43 @@ void EngineHyperscan::streamCompressExpand(EngineStream &stream,
|
||||
}
|
||||
}
|
||||
|
||||
void EngineHyperscan::printStats() const {
|
||||
// Output summary information.
|
||||
if (!compile_stats.sigs_name.empty()) {
|
||||
printf("Signature set: %s\n", compile_stats.sigs_name.c_str());
|
||||
}
|
||||
printf("Signatures: %s\n", compile_stats.signatures.c_str());
|
||||
printf("Hyperscan info: %s\n", compile_stats.db_info.c_str());
|
||||
printf("Expression count: %'zu\n", compile_stats.expressionCount);
|
||||
printf("Bytecode size: %'zu bytes\n", compile_stats.compiledSize);
|
||||
printf("Database CRC: 0x%x\n", compile_stats.crc32);
|
||||
if (compile_stats.streaming) {
|
||||
printf("Stream state size: %'zu bytes\n", compile_stats.streamSize);
|
||||
}
|
||||
printf("Scratch size: %'zu bytes\n", compile_stats.scratchSize);
|
||||
printf("Compile time: %'0.3Lf seconds\n", compile_stats.compileSecs);
|
||||
printf("Peak heap usage: %'u bytes\n", compile_stats.peakMemorySize);
|
||||
}
|
||||
|
||||
void EngineHyperscan::sqlStats(SqlDB &sqldb) const {
|
||||
ostringstream crc;
|
||||
crc << "0x" << hex << compile_stats.crc32;
|
||||
|
||||
static const std::string Q =
|
||||
"INSERT INTO Compile ("
|
||||
"sigsName, signatures, dbInfo, exprCount, dbSize, crc, streaming,"
|
||||
"streamSize, scratchSize, compileSecs, peakMemory) "
|
||||
"VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)";
|
||||
|
||||
sqldb.insert_all(Q, compile_stats.sigs_name, compile_stats.signatures,
|
||||
compile_stats.db_info, compile_stats.expressionCount,
|
||||
compile_stats.compiledSize, crc.str(),
|
||||
compile_stats.streaming ? "TRUE" : "FALSE",
|
||||
compile_stats.streamSize, compile_stats.scratchSize,
|
||||
compile_stats.compileSecs, compile_stats.peakMemorySize);
|
||||
}
|
||||
|
||||
|
||||
static
|
||||
unsigned makeModeFlags(ScanMode scan_mode) {
|
||||
switch (scan_mode) {
|
||||
@ -281,7 +320,8 @@ string dbFilename(const std::string &name, unsigned mode) {
|
||||
|
||||
std::unique_ptr<EngineHyperscan>
|
||||
buildEngineHyperscan(const ExpressionMap &expressions, ScanMode scan_mode,
|
||||
const std::string &name, UNUSED const ue2::Grey &grey) {
|
||||
const std::string &name, const std::string &sigs_name,
|
||||
UNUSED const ue2::Grey &grey) {
|
||||
if (expressions.empty()) {
|
||||
assert(0);
|
||||
return nullptr;
|
||||
@ -292,7 +332,6 @@ buildEngineHyperscan(const ExpressionMap &expressions, ScanMode scan_mode,
|
||||
size_t streamSize = 0;
|
||||
size_t scratchSize = 0;
|
||||
unsigned int peakMemorySize = 0;
|
||||
unsigned int crc = 0;
|
||||
std::string db_info;
|
||||
|
||||
unsigned int mode = makeModeFlags(scan_mode);
|
||||
@ -393,8 +432,6 @@ buildEngineHyperscan(const ExpressionMap &expressions, ScanMode scan_mode,
|
||||
}
|
||||
assert(compiledSize > 0);
|
||||
|
||||
crc = db->crc32;
|
||||
|
||||
if (saveDatabases) {
|
||||
saveDatabase(db, dbFilename(name, mode).c_str());
|
||||
}
|
||||
@ -431,18 +468,24 @@ buildEngineHyperscan(const ExpressionMap &expressions, ScanMode scan_mode,
|
||||
}
|
||||
hs_free_scratch(scratch);
|
||||
|
||||
// Output summary information.
|
||||
printf("Signatures: %s\n", name.c_str());
|
||||
printf("Hyperscan info: %s\n", db_info.c_str());
|
||||
printf("Expression count: %'zu\n", expressions.size());
|
||||
printf("Bytecode size: %'zu bytes\n", compiledSize);
|
||||
printf("Database CRC: 0x%x\n", crc);
|
||||
if (mode & HS_MODE_STREAM) {
|
||||
printf("Stream state size: %'zu bytes\n", streamSize);
|
||||
// Collect summary information.
|
||||
CompileStats cs;
|
||||
cs.sigs_name = sigs_name;
|
||||
if (!sigs_name.empty()) {
|
||||
const auto pos = name.find_last_of('/');
|
||||
cs.signatures = name.substr(pos + 1);
|
||||
} else {
|
||||
cs.signatures = name;
|
||||
}
|
||||
printf("Scratch size: %'zu bytes\n", scratchSize);
|
||||
printf("Compile time: %'0.3Lf seconds\n", compileSecs);
|
||||
printf("Peak heap usage: %'u bytes\n", peakMemorySize);
|
||||
cs.db_info = db_info;
|
||||
cs.expressionCount = expressions.size();
|
||||
cs.compiledSize = compiledSize;
|
||||
cs.crc32 = db->crc32;
|
||||
cs.streaming = mode & HS_MODE_STREAM;
|
||||
cs.streamSize = streamSize;
|
||||
cs.scratchSize = scratchSize;
|
||||
cs.compileSecs = compileSecs;
|
||||
cs.peakMemorySize = peakMemorySize;
|
||||
|
||||
return ue2::make_unique<EngineHyperscan>(db);
|
||||
return ue2::make_unique<EngineHyperscan>(db, std::move(cs));
|
||||
}
|
||||
|
@ -31,9 +31,11 @@
|
||||
|
||||
#include "expressions.h"
|
||||
#include "common.h"
|
||||
#include "sqldb.h"
|
||||
#include "hs_runtime.h"
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
/** Structure for the result of a single complete scan. */
|
||||
@ -42,6 +44,21 @@ struct ResultEntry {
|
||||
unsigned int matches = 0; //!< Count of matches found.
|
||||
};
|
||||
|
||||
/** Infomation about the database compile */
|
||||
struct CompileStats {
|
||||
std::string sigs_name;
|
||||
std::string signatures;
|
||||
std::string db_info;
|
||||
size_t expressionCount = 0;
|
||||
size_t compiledSize = 0;
|
||||
uint32_t crc32 = 0;
|
||||
bool streaming;
|
||||
size_t streamSize = 0;
|
||||
size_t scratchSize = 0;
|
||||
long double compileSecs = 0;
|
||||
unsigned int peakMemorySize = 0;
|
||||
};
|
||||
|
||||
/** Engine context which is allocated on a per-thread basis. */
|
||||
class EngineContext {
|
||||
public:
|
||||
@ -62,7 +79,7 @@ public:
|
||||
/** Hyperscan Engine for scanning data. */
|
||||
class EngineHyperscan {
|
||||
public:
|
||||
explicit EngineHyperscan(hs_database_t *db);
|
||||
explicit EngineHyperscan(hs_database_t *db, CompileStats cs);
|
||||
~EngineHyperscan();
|
||||
|
||||
std::unique_ptr<EngineContext> makeContext() const;
|
||||
@ -86,8 +103,13 @@ public:
|
||||
void streamScan(EngineStream &stream, const char *data, unsigned int len,
|
||||
unsigned int id, ResultEntry &result) const;
|
||||
|
||||
void printStats() const;
|
||||
|
||||
void sqlStats(SqlDB &db) const;
|
||||
|
||||
private:
|
||||
hs_database_t *db;
|
||||
CompileStats compile_stats;
|
||||
};
|
||||
|
||||
namespace ue2 {
|
||||
@ -96,6 +118,7 @@ struct Grey;
|
||||
|
||||
std::unique_ptr<EngineHyperscan>
|
||||
buildEngineHyperscan(const ExpressionMap &expressions, ScanMode scan_mode,
|
||||
const std::string &name, const ue2::Grey &grey);
|
||||
const std::string &name, const std::string &sigs_name,
|
||||
const ue2::Grey &grey);
|
||||
|
||||
#endif // ENGINEHYPERSCAN_H
|
||||
|
@ -32,6 +32,7 @@
|
||||
#include "data_corpus.h"
|
||||
#include "engine_hyperscan.h"
|
||||
#include "expressions.h"
|
||||
#include "sqldb.h"
|
||||
#include "thread_barrier.h"
|
||||
#include "timer.h"
|
||||
#include "util/expression_path.h"
|
||||
@ -89,10 +90,14 @@ ScanMode scan_mode = ScanMode::STREAMING;
|
||||
unsigned repeats = 20;
|
||||
string exprPath("");
|
||||
string corpusFile("");
|
||||
string sqloutFile("");
|
||||
string sigName(""); // info only
|
||||
vector<unsigned int> threadCores;
|
||||
Timer totalTimer;
|
||||
double totalSecs = 0;
|
||||
|
||||
SqlDB out_db;
|
||||
|
||||
typedef void (*thread_func_t)(void *context);
|
||||
|
||||
class ThreadContext : boost::noncopyable {
|
||||
@ -188,6 +193,8 @@ void usage(const char *error) {
|
||||
printf("\n");
|
||||
printf(" --per-scan Display per-scan Mbit/sec results.\n");
|
||||
printf(" --echo-matches Display all matches that occur during scan.\n");
|
||||
printf(" --sql-out FILE Output sqlite db.\n");
|
||||
printf(" -S NAME Signature set name (for sqlite db).\n");
|
||||
printf("\n\n");
|
||||
|
||||
if (error) {
|
||||
@ -207,28 +214,30 @@ struct BenchmarkSigs {
|
||||
static
|
||||
void processArgs(int argc, char *argv[], vector<BenchmarkSigs> &sigSets,
|
||||
UNUSED unique_ptr<Grey> &grey) {
|
||||
const char options[] = "-b:c:Cd:e:E:G:hi:n:No:p:sVw:z:"
|
||||
const char options[] = "-b:c:Cd:e:E:G:hi:n:No:p:sS:Vw:z:"
|
||||
#ifdef HAVE_DECL_PTHREAD_SETAFFINITY_NP
|
||||
"T:" // add the thread flag
|
||||
#endif
|
||||
;
|
||||
int in_sigfile = 0;
|
||||
int do_per_scan = 0;
|
||||
int do_echo_matches = 0;
|
||||
int do_compress = 0;
|
||||
int do_compress_size = 0;
|
||||
int do_echo_matches = 0;
|
||||
int do_sql_output = 0;
|
||||
int option_index = 0;
|
||||
vector<string> sigFiles;
|
||||
|
||||
static struct option longopts[] = {
|
||||
{"per-scan", 0, &do_per_scan, 1},
|
||||
{"echo-matches", 0, &do_echo_matches, 1},
|
||||
{"compress-stream", 0, &do_compress, 1},
|
||||
{"print-compress-size", 0, &do_compress_size, 1},
|
||||
{"per-scan", no_argument, &do_per_scan, 1},
|
||||
{"echo-matches", no_argument, &do_echo_matches, 1},
|
||||
{"compress-stream", no_argument, &do_compress, 1},
|
||||
{"sql-out", required_argument, &do_sql_output, 1},
|
||||
{nullptr, 0, nullptr, 0}
|
||||
};
|
||||
|
||||
for (;;) {
|
||||
int c = getopt_long(argc, argv, options, longopts, nullptr);
|
||||
int c = getopt_long(argc, argv, options, longopts, &option_index);
|
||||
if (c < 0) {
|
||||
break;
|
||||
}
|
||||
@ -294,6 +303,9 @@ void processArgs(int argc, char *argv[], vector<BenchmarkSigs> &sigSets,
|
||||
case 'V':
|
||||
scan_mode = ScanMode::VECTORED;
|
||||
break;
|
||||
case 'S':
|
||||
sigName.assign(optarg);
|
||||
break;
|
||||
#ifdef HAVE_DECL_PTHREAD_SETAFFINITY_NP
|
||||
case 'T':
|
||||
if (!strToList(optarg, threadCores)) {
|
||||
@ -321,14 +333,19 @@ void processArgs(int argc, char *argv[], vector<BenchmarkSigs> &sigSets,
|
||||
saveDatabases = true;
|
||||
serializePath = optarg;
|
||||
break;
|
||||
case 0:
|
||||
if (do_sql_output) {
|
||||
sqloutFile.assign(optarg);
|
||||
do_sql_output = 0;
|
||||
}
|
||||
break;
|
||||
case 1:
|
||||
if (in_sigfile) {
|
||||
sigFiles.push_back(optarg);
|
||||
in_sigfile = 2;
|
||||
break;
|
||||
}
|
||||
case 0:
|
||||
break;
|
||||
/* fallthrough */
|
||||
default:
|
||||
usage("Unrecognised command line argument.");
|
||||
exit(1);
|
||||
@ -726,6 +743,67 @@ void displayResults(const vector<unique_ptr<ThreadContext>> &threads,
|
||||
}
|
||||
}
|
||||
|
||||
/** Dump per-scan throughput data to sql. */
|
||||
static
|
||||
void sqlPerScanResults(const vector<unique_ptr<ThreadContext>> &threads,
|
||||
u64a bytesPerRun, u64a scan_id) {
|
||||
static const std::string Q =
|
||||
"INSERT INTO ScanResults (scan_id, thread, scan, throughput) "
|
||||
"VALUES (?1, ?2, ?3, ?4)";
|
||||
|
||||
for (const auto &t : threads) {
|
||||
const auto &results = t->results;
|
||||
for (size_t j = 0; j != results.size(); j++) {
|
||||
const auto &r = results[j];
|
||||
double mbps = calc_mbps(r.seconds, bytesPerRun);
|
||||
out_db.insert_all(Q, scan_id, t->num, j, mbps);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Dump benchmark results to sql. */
|
||||
static
|
||||
void sqlResults(const vector<unique_ptr<ThreadContext>> &threads,
|
||||
const vector<DataBlock> &corpus_blocks) {
|
||||
u64a bytesPerRun = byte_size(corpus_blocks);
|
||||
u64a matchesPerRun = threads[0]->results[0].matches;
|
||||
|
||||
u64a scan_id = out_db.lastRowId();
|
||||
|
||||
// Sanity check: all of our results should have the same match count.
|
||||
for (const auto &t : threads) {
|
||||
if (!all_of(begin(t->results), end(t->results),
|
||||
[&matchesPerRun](const ResultEntry &e) {
|
||||
return e.matches == matchesPerRun;
|
||||
})) {
|
||||
printf("\nWARNING: PER-SCAN MATCH COUNTS ARE INCONSISTENT!\n\n");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
u64a totalBytes = bytesPerRun * repeats * threads.size();
|
||||
double matchRate = ((double)matchesPerRun * 1024) / bytesPerRun;
|
||||
|
||||
const auto pos = corpusFile.find_last_of('/');
|
||||
const auto corpus = corpusFile.substr(pos + 1);
|
||||
|
||||
static const std::string Q =
|
||||
"INSERT INTO Scan (scan_id, corpusFile, totalSecs, "
|
||||
"bytesPerRun, blockSize, blockCount, totalBytes, "
|
||||
"totalBlocks, matchesPerRun, matchRate, overallTput) "
|
||||
"VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)";
|
||||
|
||||
out_db.insert_all(
|
||||
Q, scan_id, corpus, totalSecs, bytesPerRun, corpus_blocks.size(),
|
||||
scan_mode == ScanMode::BLOCK ? 1 : count_streams(corpus_blocks),
|
||||
totalBytes, corpus_blocks.size() * repeats * threads.size(),
|
||||
matchesPerRun, matchRate, calc_mbps(totalSecs, totalBytes));
|
||||
|
||||
if (display_per_scan) {
|
||||
sqlPerScanResults(threads, bytesPerRun, scan_id);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a thread context for this scanning mode.
|
||||
*
|
||||
@ -798,10 +876,15 @@ void runBenchmark(const EngineHyperscan &db,
|
||||
t->join();
|
||||
}
|
||||
|
||||
// Display global results.
|
||||
displayResults(threads, corpus_blocks);
|
||||
if (sqloutFile.empty()) {
|
||||
// Display global results.
|
||||
displayResults(threads, corpus_blocks);
|
||||
} else {
|
||||
// write to sqlite file
|
||||
sqlResults(threads, corpus_blocks);
|
||||
out_db.exec("END");
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
/** Main driver. */
|
||||
@ -842,22 +925,39 @@ int main(int argc, char *argv[]) {
|
||||
printf("Corpus data error: %s\n", e.msg.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
for (const auto &s : sigSets) {
|
||||
auto exprMap = limitToSignatures(exprMapTemplate, s.sigs);
|
||||
if (exprMap.empty()) {
|
||||
continue;
|
||||
try {
|
||||
if (!sqloutFile.empty()) {
|
||||
out_db.open(sqloutFile);
|
||||
}
|
||||
|
||||
auto engine = buildEngineHyperscan(exprMap, scan_mode, s.name, *grey);
|
||||
if (!engine) {
|
||||
printf("Error: expressions failed to compile.\n");
|
||||
exit(1);
|
||||
for (const auto &s : sigSets) {
|
||||
auto exprMap = limitToSignatures(exprMapTemplate, s.sigs);
|
||||
if (exprMap.empty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto engine = buildEngineHyperscan(exprMap, scan_mode, s.name,
|
||||
sigName, *grey);
|
||||
if (!engine) {
|
||||
printf("Error: expressions failed to compile.\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (sqloutFile.empty()) {
|
||||
// Display global results.
|
||||
engine->printStats();
|
||||
printf("\n");
|
||||
|
||||
} else {
|
||||
out_db.exec("BEGIN");
|
||||
engine->sqlStats(out_db);
|
||||
}
|
||||
|
||||
runBenchmark(*engine, corpus_blocks);
|
||||
}
|
||||
|
||||
printf("\n");
|
||||
|
||||
runBenchmark(*engine, corpus_blocks);
|
||||
} catch (const SqlFailure &f) {
|
||||
cerr << f.message << '\n';
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
167
tools/hsbench/sqldb.cpp
Normal file
167
tools/hsbench/sqldb.cpp
Normal file
@ -0,0 +1,167 @@
|
||||
/*
|
||||
* Copyright (c) 2017, Intel Corporation
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "config.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "sqldb.h"
|
||||
#include "ue2common.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace {
|
||||
|
||||
static
|
||||
sqlite3 *initDB(const string &filename) {
|
||||
sqlite3 *db;
|
||||
int status;
|
||||
status = sqlite3_open_v2(filename.c_str(), &db,
|
||||
SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nullptr);
|
||||
|
||||
if (status != SQLITE_OK) {
|
||||
ostringstream oss;
|
||||
oss << "Unable to open database '" << filename
|
||||
<< "': " << sqlite3_errmsg(db);
|
||||
status = sqlite3_close(db);
|
||||
assert(status == SQLITE_OK);
|
||||
throw SqlFailure(oss.str());
|
||||
}
|
||||
|
||||
// create tables
|
||||
static const string c("CREATE TABLE Compile ("
|
||||
"id INTEGER PRIMARY KEY,"
|
||||
"sigsName TEXT, "
|
||||
"signatures TEXT, "
|
||||
"dbInfo TEXT, "
|
||||
"exprCount INTEGER, "
|
||||
"dbSize INTEGER,"
|
||||
"crc TEXT, "
|
||||
"streaming TEXT, "
|
||||
"streamSize INTEGER, "
|
||||
"scratchSize INTEGER, "
|
||||
"compileSecs DOUBLE, "
|
||||
"peakMemory INTEGER"
|
||||
");");
|
||||
|
||||
static const string s("CREATE TABLE Scan (id INTEGER PRIMARY KEY,"
|
||||
"corpusFile TEXT, scan_id INTEGER, "
|
||||
"totalSecs DOUBLE, bytesPerRun INTEGER, "
|
||||
"blockSize INTEGER, blockCount INTEGER, "
|
||||
"totalBytes INTEGER, totalBlocks INTEGER, "
|
||||
"matchesPerRun INTEGER, "
|
||||
"matchRate DOUBLE, overallTput DOUBLE);");
|
||||
|
||||
static const string sr(
|
||||
"CREATE TABLE ScanResults ( id INTEGER PRIMARY KEY, "
|
||||
"scan_id INTEGER, thread INTEGER, scan INTEGER, throughput DOUBLE );");
|
||||
|
||||
static const string create_query = c + s + sr;
|
||||
|
||||
sqlite3_stmt *statement;
|
||||
const char *pzTail = create_query.c_str();
|
||||
|
||||
while (strlen(pzTail)) {
|
||||
status =
|
||||
sqlite3_prepare(db, pzTail, strlen(pzTail), &statement, &pzTail);
|
||||
if (status != SQLITE_OK) {
|
||||
goto fail;
|
||||
}
|
||||
status = sqlite3_step(statement);
|
||||
if (status != SQLITE_DONE && status != SQLITE_ROW) {
|
||||
goto fail;
|
||||
}
|
||||
status = sqlite3_finalize(statement);
|
||||
if (status != SQLITE_OK) {
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
|
||||
return db;
|
||||
|
||||
fail:
|
||||
ostringstream oss;
|
||||
oss << "Unable to create tables: " << sqlite3_errmsg(db);
|
||||
status = sqlite3_close(db);
|
||||
assert(status == SQLITE_OK);
|
||||
throw SqlFailure(oss.str());
|
||||
}
|
||||
} // namespace
|
||||
|
||||
SqlDB::~SqlDB() {
|
||||
if (db) {
|
||||
sqlite3_close(db);
|
||||
}
|
||||
db = nullptr;
|
||||
}
|
||||
|
||||
void SqlDB::open(const string &filename) {
|
||||
if (!ifstream(filename)) {
|
||||
// file doesn't exist, go set up some tables
|
||||
db = initDB(filename);
|
||||
} else {
|
||||
int status;
|
||||
status = sqlite3_open_v2(filename.c_str(), &db, SQLITE_OPEN_READWRITE,
|
||||
nullptr);
|
||||
|
||||
if (status != SQLITE_OK) {
|
||||
ostringstream oss;
|
||||
oss << "Unable to open database '" << filename
|
||||
<< "': " << sqlite3_errmsg(db);
|
||||
throw SqlFailure(oss.str());
|
||||
}
|
||||
}
|
||||
|
||||
exec("PRAGMA synchronous = off;");
|
||||
exec("PRAGMA encoding = 'UTF-8';");
|
||||
}
|
||||
|
||||
void SqlDB::exec(const string &query) {
|
||||
assert(db);
|
||||
int status;
|
||||
status = sqlite3_exec(db, query.c_str(), nullptr, nullptr, nullptr);
|
||||
if (status != SQLITE_OK) {
|
||||
ostringstream oss;
|
||||
oss << "Unable to run sqlite query: " << sqlite3_errmsg(db);
|
||||
sqlite3_close(db);
|
||||
throw SqlFailure(oss.str());
|
||||
}
|
||||
}
|
||||
|
||||
u64a SqlDB::lastRowId() {
|
||||
assert(db);
|
||||
return sqlite3_last_insert_rowid(db);
|
||||
}
|
86
tools/hsbench/sqldb.h
Normal file
86
tools/hsbench/sqldb.h
Normal file
@ -0,0 +1,86 @@
|
||||
/*
|
||||
* Copyright (c) 2017, Intel Corporation
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
#ifndef SQLDB_H_
|
||||
#define SQLDB_H_
|
||||
|
||||
#include "ue2common.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "sqldb_bind.h"
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
class SqlDB {
|
||||
public:
|
||||
SqlDB() : db(nullptr) {};
|
||||
~SqlDB();
|
||||
void open(const std::string &filename);
|
||||
void exec(const std::string &query);
|
||||
u64a lastRowId();
|
||||
|
||||
template <typename... Args>
|
||||
void insert_all(const std::string &query, Args&&... args) {
|
||||
sqlite3_stmt *stmt;
|
||||
const char *tail;
|
||||
|
||||
int rc = sqlite3_prepare(db, query.c_str(), query.size(), &stmt, &tail);
|
||||
if (rc != SQLITE_OK) {
|
||||
std::ostringstream oss;
|
||||
oss << "Unable to prepare query: " << sqlite3_errmsg(db);
|
||||
throw SqlFailure(oss.str());
|
||||
}
|
||||
|
||||
// only one statement per function call
|
||||
assert(strlen(tail) == 0);
|
||||
|
||||
// perform templated binds to this statement
|
||||
ue2_sqlite::bind_args(stmt, 1, args...);
|
||||
|
||||
rc = sqlite3_step(stmt);
|
||||
if (rc != SQLITE_DONE) {
|
||||
std::ostringstream oss;
|
||||
oss << "Unable to run insert: " << sqlite3_errmsg(db);
|
||||
throw SqlFailure(oss.str());
|
||||
}
|
||||
|
||||
rc = sqlite3_finalize(stmt);
|
||||
if (rc != SQLITE_OK) {
|
||||
std::ostringstream oss;
|
||||
oss << "Unable to finalize statement: " << sqlite3_errmsg(db);
|
||||
throw SqlFailure(oss.str());
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
sqlite3 *db;
|
||||
};
|
||||
|
||||
#endif /* SQLDB_H_ */
|
90
tools/hsbench/sqldb_bind.h
Normal file
90
tools/hsbench/sqldb_bind.h
Normal file
@ -0,0 +1,90 @@
|
||||
/*
|
||||
* Copyright (c) 2017, Intel Corporation
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
#ifndef SQLDB_BIND_H_
|
||||
#define SQLDB_BIND_H_
|
||||
|
||||
#include "ue2common.h"
|
||||
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
namespace ue2_sqlite {
|
||||
|
||||
inline
|
||||
int bind_impl(sqlite3_stmt *stmt, int param, const unsigned long &val) {
|
||||
return sqlite3_bind_int64(stmt, param, val);
|
||||
}
|
||||
|
||||
inline
|
||||
int bind_impl(sqlite3_stmt *stmt, int param, const unsigned int &val) {
|
||||
return sqlite3_bind_int(stmt, param, val);
|
||||
}
|
||||
|
||||
inline
|
||||
int bind_impl(sqlite3_stmt *stmt, int param, const u64a &val) {
|
||||
return sqlite3_bind_int64(stmt, param, val);
|
||||
}
|
||||
|
||||
inline
|
||||
int bind_impl(sqlite3_stmt *stmt, int param, const double &val) {
|
||||
return sqlite3_bind_double(stmt, param, val);
|
||||
}
|
||||
|
||||
inline
|
||||
int bind_impl(sqlite3_stmt *stmt, int param, const long double &val) {
|
||||
return sqlite3_bind_double(stmt, param, val);
|
||||
}
|
||||
|
||||
inline
|
||||
int bind_impl(sqlite3_stmt *stmt, int param, const std::string &val) {
|
||||
return sqlite3_bind_text(stmt, param, val.c_str(), val.size(),
|
||||
SQLITE_TRANSIENT);
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void bind_args(sqlite3_stmt *stmt, int param, T obj) {
|
||||
int rc = bind_impl(stmt, param, obj);
|
||||
if (rc != SQLITE_OK) {
|
||||
std::ostringstream oss;
|
||||
oss << "SQL value bind failed for param #: " << param;
|
||||
throw SqlFailure(oss.str());
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T, typename... Args>
|
||||
void bind_args(sqlite3_stmt *stmt, int param, T obj, Args&&... args) {
|
||||
bind_args(stmt, param, obj);
|
||||
bind_args(stmt, param + 1, args...);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
#endif /* SQLDB_BIND_H_ */
|
Loading…
x
Reference in New Issue
Block a user