github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-06-28 16:41:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #237 from gtsoul-tech/feature/microbenchmarkingCSV

Browse Source 
Microbenchmarking tool changed color output to csv output
This commit is contained in:
Konstantinos Margaritis 2024-04-02 17:33:28 +03:00 committed by GitHub
commit c2700cafd9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 101 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
benchmarks/benchmarks.cpp
View File

@ -26,32 +26,30 @@
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
*/ */
#include <iostream>
#include <chrono> #include <chrono>
#include <cstdlib>
#include <cstring> #include <cstring>
#include <ctime> #include <ctime>
#include <cstdlib>
#include <memory>
#include <functional> #include <functional>
#include <iostream>
#include <memory>
#include "benchmarks.hpp" #include "benchmarks.hpp"
#define MAX_LOOPS 1000000000 #define MAX_LOOPS 1000000000
#define MAX_MATCHES 5 #define MAX_MATCHES 5
#define N 8 #define N 8
struct hlmMatchEntry { struct hlmMatchEntry {
size_t to; size_t to;
u32 id; u32 id;
hlmMatchEntry(size_t end, u32 identifier) : hlmMatchEntry(size_t end, u32 identifier) : to(end), id(identifier) {}
to(end), id(identifier) {}
}; };
std::vector<hlmMatchEntry> ctxt; std::vector<hlmMatchEntry> ctxt;
static static hwlmcb_rv_t hlmSimpleCallback(size_t to, u32 id,
hwlmcb_rv_t hlmSimpleCallback(size_t to, u32 id, UNUSED struct hs_scratch *scratch) {
UNUSED struct hs_scratch *scratch) {
DEBUG_PRINTF("match @%zu = %u\n", to, id); DEBUG_PRINTF("match @%zu = %u\n", to, id);
ctxt.push_back(hlmMatchEntry(to, id)); ctxt.push_back(hlmMatchEntry(to, id));
@ -59,10 +57,12 @@ hwlmcb_rv_t hlmSimpleCallback(size_t to, u32 id,
return HWLM_CONTINUE_MATCHING; return HWLM_CONTINUE_MATCHING;
} }
template<typename InitFunc, typename BenchFunc> template <typename InitFunc, typename BenchFunc>
static void run_benchmarks(int size, int loops, int max_matches, bool is_reverse, MicroBenchmark &bench, InitFunc &&init, BenchFunc &&func) { static void run_benchmarks(int size, int loops, int max_matches,
bool is_reverse, MicroBenchmark &bench,
InitFunc &&init, BenchFunc &&func) {
init(bench); init(bench);
double total_sec = 0.0; double total_sec = 0.0;
u64a total_size = 0; u64a total_size = 0;
double bw = 0.0; double bw = 0.0;
double avg_bw = 0.0; double avg_bw = 0.0;
@ -70,29 +70,31 @@ static void run_benchmarks(int size, int loops, int max_matches, bool is_reverse
double avg_time = 0.0; double avg_time = 0.0;
if (max_matches) { if (max_matches) {
int pos = 0; int pos = 0;
for(int j = 0; j < max_matches - 1; j++) { for (int j = 0; j < max_matches - 1; j++) {
bench.buf[pos] = 'b'; bench.buf[pos] = 'b';
pos = (j+1) *size / max_matches ; pos = (j + 1) * size / max_matches;
bench.buf[pos] = 'a'; bench.buf[pos] = 'a';
u64a actual_size = 0; u64a actual_size = 0;
auto start = std::chrono::steady_clock::now(); auto start = std::chrono::steady_clock::now();
for(int i = 0; i < loops; i++) { for (int i = 0; i < loops; i++) {
const u8 *res = func(bench); const u8 *res = func(bench);
if (is_reverse) if (is_reverse)
actual_size += bench.buf.data() + size - res; actual_size += bench.buf.data() + size - res;
else else
actual_size += res - bench.buf.data(); actual_size += res - bench.buf.data();
} }
auto end = std::chrono::steady_clock::now(); auto end = std::chrono::steady_clock::now();
double dt = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count(); double dt = std::chrono::duration_cast<std::chrono::microseconds>(
end - start)
.count();
total_sec += dt; total_sec += dt;
/*convert microseconds to seconds*/ /*convert microseconds to seconds*/
/*calculate bandwidth*/ /*calculate bandwidth*/
bw = (actual_size / dt) * 1000000.0 / 1048576.0; bw = (actual_size / dt) * 1000000.0 / 1048576.0;
/*std::cout << "act_size = " << act_size << std::endl; /*std::cout << "act_size = " << act_size << std::endl;
std::cout << "dt = " << dt << std::endl; std::cout << "dt = " << dt << std::endl;
std::cout << "bw = " << bw << std::endl;*/ std::cout << "bw = " << bw << std::endl;*/
avg_bw += bw; avg_bw += bw;
/*convert to MB/s*/ /*convert to MB/s*/
max_bw = std::max(bw, max_bw); max_bw = std::max(bw, max_bw);
/*calculate average time*/ /*calculate average time*/
@ -100,18 +102,20 @@ static void run_benchmarks(int size, int loops, int max_matches, bool is_reverse
} }
avg_time /= max_matches; avg_time /= max_matches;
avg_bw /= max_matches; avg_bw /= max_matches;
total_sec /= 1000000.0; total_sec /= 1000000.0;
/*convert average time to us*/ /*convert average time to us*/
printf(KMAG "%s: %u matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " printf("%-18s, %-12u, %-10u, %-6u, %-10.3f, %-9.3f, %-8.3f, %-7.3f\n",
KBLU "average time per call =" RST " %.3f μs," KBLU " max bandwidth = " RST " %.3f MB/s," KBLU " average bandwidth =" RST " %.3f MB/s \n", bench.label, max_matches, size, loops, total_sec, avg_time,
bench.label, max_matches, size ,loops, total_sec, avg_time, max_bw, avg_bw); max_bw, avg_bw);
} else { } else {
auto start = std::chrono::steady_clock::now(); auto start = std::chrono::steady_clock::now();
for (int i = 0; i < loops; i++) { for (int i = 0; i < loops; i++) {
const u8 *res = func(bench); const u8 *res = func(bench);
} }
auto end = std::chrono::steady_clock::now(); auto end = std::chrono::steady_clock::now();
total_sec += std::chrono::duration_cast<std::chrono::microseconds>(end - start).count(); total_sec +=
std::chrono::duration_cast<std::chrono::microseconds>(end - start)
.count();
/*calculate transferred size*/ /*calculate transferred size*/
total_size = size * loops; total_size = size * loops;
/*calculate average time*/ /*calculate average time*/
@ -122,117 +126,126 @@ static void run_benchmarks(int size, int loops, int max_matches, bool is_reverse
max_bw = total_size / total_sec; max_bw = total_size / total_sec;
/*convert to MB/s*/ /*convert to MB/s*/
max_bw /= 1048576.0; max_bw /= 1048576.0;
printf(KMAG "%s: no matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " printf("%-18s, %-12s, %-10u, %-6u, %-10.3f, %-9.3f, %-8.3f, %-7s\n",
KBLU "average time per call =" RST " %.3f μs ," KBLU " bandwidth = " RST " %.3f MB/s \n", bench.label, "0", size, loops, total_sec, avg_time, max_bw, "0");
bench.label, size ,loops, total_sec, avg_time, max_bw );
} }
} }
int main(){ int main() {
int matches[] = {0, MAX_MATCHES}; int matches[] = {0, MAX_MATCHES};
std::vector<size_t> sizes; std::vector<size_t> sizes;
for (size_t i = 0; i < N; i++) sizes.push_back(16000 << i*2); for (size_t i = 0; i < N; i++)
const char charset[] = "aAaAaAaAAAaaaaAAAAaaaaAAAAAAaaaAAaaa"; sizes.push_back(16000 << i * 2);
const char charset[] = "aAaAaAaAAAaaaaAAAAaaaaAAAAAAaaaAAaaa";
printf("%-18s, %-12s, %-10s, %-6s, %-10s, %-9s, %-8s, %-7s\n", "Matcher",
"max_matches", "size", "loops", "total_sec", "avg_time", "max_bw",
"avg_bw");
for (int m = 0; m < 2; m++) { for (int m = 0; m < 2; m++) {
for (size_t i = 0; i < std::size(sizes); i++) { for (size_t i = 0; i < std::size(sizes); i++) {
MicroBenchmark bench("Shufti", sizes[i]); MicroBenchmark bench("Shufti", sizes[i]);
run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, run_benchmarks(
sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench,
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
b.chars.set('a'); b.chars.set('a');
ue2::shuftiBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); ue2::shuftiBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi);
memset(b.buf.data(), 'b', b.size); memset(b.buf.data(), 'b', b.size);
}, },
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
return shuftiExec(b.lo, b.hi, b.buf.data(), b.buf.data() + b.size); return shuftiExec(b.lo, b.hi, b.buf.data(),
} b.buf.data() + b.size);
); });
} }
for (size_t i = 0; i < std::size(sizes); i++) { for (size_t i = 0; i < std::size(sizes); i++) {
MicroBenchmark bench("Reverse Shufti", sizes[i]); MicroBenchmark bench("Reverse Shufti", sizes[i]);
run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench, run_benchmarks(
sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench,
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
b.chars.set('a'); b.chars.set('a');
ue2::shuftiBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); ue2::shuftiBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi);
memset(b.buf.data(), 'b', b.size); memset(b.buf.data(), 'b', b.size);
}, },
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
return rshuftiExec(b.lo, b.hi, b.buf.data(), b.buf.data() + b.size); return rshuftiExec(b.lo, b.hi, b.buf.data(),
} b.buf.data() + b.size);
); });
} }
for (size_t i = 0; i < std::size(sizes); i++) { for (size_t i = 0; i < std::size(sizes); i++) {
MicroBenchmark bench("Truffle", sizes[i]); MicroBenchmark bench("Truffle", sizes[i]);
run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, run_benchmarks(
sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench,
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
b.chars.set('a'); b.chars.set('a');
ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi);
memset(b.buf.data(), 'b', b.size); memset(b.buf.data(), 'b', b.size);
}, },
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
return truffleExec(b.lo, b.hi, b.buf.data(), b.buf.data() + b.size); return truffleExec(b.lo, b.hi, b.buf.data(),
} b.buf.data() + b.size);
); });
} }
for (size_t i = 0; i < std::size(sizes); i++) { for (size_t i = 0; i < std::size(sizes); i++) {
MicroBenchmark bench("Reverse Truffle", sizes[i]); MicroBenchmark bench("Reverse Truffle", sizes[i]);
run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench, run_benchmarks(
sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench,
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
b.chars.set('a'); b.chars.set('a');
ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi);
memset(b.buf.data(), 'b', b.size); memset(b.buf.data(), 'b', b.size);
}, },
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
return rtruffleExec(b.lo, b.hi, b.buf.data(), b.buf.data() + b.size); return rtruffleExec(b.lo, b.hi, b.buf.data(),
} b.buf.data() + b.size);
); });
} }
for (size_t i = 0; i < std::size(sizes); i++) { for (size_t i = 0; i < std::size(sizes); i++) {
MicroBenchmark bench("Vermicelli", sizes[i]); MicroBenchmark bench("Vermicelli", sizes[i]);
run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, run_benchmarks(
sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench,
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
b.chars.set('a'); b.chars.set('a');
ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi);
memset(b.buf.data(), 'b', b.size); memset(b.buf.data(), 'b', b.size);
}, },
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
return vermicelliExec('a', 'b', b.buf.data(), b.buf.data() + b.size); return vermicelliExec('a', 'b', b.buf.data(),
} b.buf.data() + b.size);
); });
} }
for (size_t i = 0; i < std::size(sizes); i++) { for (size_t i = 0; i < std::size(sizes); i++) {
MicroBenchmark bench("Reverse Vermicelli", sizes[i]); MicroBenchmark bench("Reverse Vermicelli", sizes[i]);
run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench, run_benchmarks(
sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench,
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
b.chars.set('a'); b.chars.set('a');
ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi);
memset(b.buf.data(), 'b', b.size); memset(b.buf.data(), 'b', b.size);
}, },
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
return rvermicelliExec('a', 'b', b.buf.data(), b.buf.data() + b.size); return rvermicelliExec('a', 'b', b.buf.data(),
} b.buf.data() + b.size);
); });
} }
for (size_t i = 0; i < std::size(sizes); i++) { for (size_t i = 0; i < std::size(sizes); i++) {
//we imitate the noodle unit tests // we imitate the noodle unit tests
std::string str; std::string str;
const size_t char_len = 5; const size_t char_len = 5;
str.resize(char_len + 1); str.resize(char_len + 1);
for (size_t j=0; j < char_len; j++) { for (size_t j = 0; j < char_len; j++) {
srand (time(NULL)); srand(time(NULL));
int key = rand() % + 36 ; int key = rand() % +36;
str[char_len] = charset[key]; str[char_len] = charset[key];
str[char_len + 1] = '\0'; str[char_len + 1] = '\0';
} }
MicroBenchmark bench("Noodle", sizes[i]); MicroBenchmark bench("Noodle", sizes[i]);
run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, run_benchmarks(
sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench,
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
ctxt.clear(); ctxt.clear();
memset(b.buf.data(), 'a', b.size); memset(b.buf.data(), 'a', b.size);
@ -242,10 +255,10 @@ int main(){
assert(b.nt != nullptr); assert(b.nt != nullptr);
}, },
[&](MicroBenchmark &b) { [&](MicroBenchmark &b) {
noodExec(b.nt.get(), b.buf.data(), b.size, 0, hlmSimpleCallback, &b.scratch); noodExec(b.nt.get(), b.buf.data(), b.size, 0,
hlmSimpleCallback, &b.scratch);
return b.buf.data() + b.size; return b.buf.data() + b.size;
} });
);
} }
} }

46
benchmarks/benchmarks.hpp
View File

@ -26,44 +26,32 @@
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
*/ */
#include "hwlm/hwlm_literal.h"
#include "hwlm/noodle_build.h"
#include "hwlm/noodle_engine.h"
#include "hwlm/noodle_internal.h"
#include "nfa/shufti.h" #include "nfa/shufti.h"
#include "nfa/shufticompile.h" #include "nfa/shufticompile.h"
#include "nfa/truffle.h" #include "nfa/truffle.h"
#include "nfa/trufflecompile.h" #include "nfa/trufflecompile.h"
#include "nfa/vermicelli.hpp" #include "nfa/vermicelli.hpp"
#include "hwlm/noodle_build.h"
#include "hwlm/noodle_engine.h"
#include "hwlm/noodle_internal.h"
#include "hwlm/hwlm_literal.h"
#include "util/bytecode_ptr.h"
#include "scratch.h" #include "scratch.h"
#include "util/bytecode_ptr.h"
/*define colour control characters*/ class MicroBenchmark {
#define RST "\x1B[0m"
#define KRED "\x1B[31m"
#define KGRN "\x1B[32m"
#define KYEL "\x1B[33m"
#define KBLU "\x1B[34m"
#define KMAG "\x1B[35m"
#define KCYN "\x1B[36m"
#define KWHT "\x1B[37m"
class MicroBenchmark
{
public: public:
char const *label; char const *label;
size_t size; size_t size;
// Shufti/Truffle // Shufti/Truffle
m128 lo, hi; m128 lo, hi;
ue2::CharReach chars; ue2::CharReach chars;
std::vector<u8> buf; std::vector<u8> buf;
// Noodle // Noodle
struct hs_scratch scratch; struct hs_scratch scratch;
ue2::bytecode_ptr<noodTable> nt; ue2::bytecode_ptr<noodTable> nt;
MicroBenchmark(char const *label_, size_t size_) MicroBenchmark(char const *label_, size_t size_)
:label(label_), size(size_), buf(size_) { : label(label_), size(size_), buf(size_){};
};
}; };