From 50a62a17ffbfdbf4a9a30010bec61ebc271d95f8 Mon Sep 17 00:00:00 2001 From: gtsoul-tech Date: Mon, 1 Apr 2024 16:05:13 +0300 Subject: [PATCH 1/4] changed color output to csv output --- benchmarks/benchmarks.cpp | 168 ++++++++++++++++++++++---------------- 1 file changed, 97 insertions(+), 71 deletions(-) diff --git a/benchmarks/benchmarks.cpp b/benchmarks/benchmarks.cpp index 91cab3f8..c6e453ef 100644 --- a/benchmarks/benchmarks.cpp +++ b/benchmarks/benchmarks.cpp @@ -26,32 +26,30 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#include #include +#include #include #include -#include -#include #include +#include +#include #include "benchmarks.hpp" -#define MAX_LOOPS 1000000000 -#define MAX_MATCHES 5 -#define N 8 +#define MAX_LOOPS 1000000000 +#define MAX_MATCHES 5 +#define N 8 struct hlmMatchEntry { size_t to; u32 id; - hlmMatchEntry(size_t end, u32 identifier) : - to(end), id(identifier) {} + hlmMatchEntry(size_t end, u32 identifier) : to(end), id(identifier) {} }; std::vector ctxt; -static -hwlmcb_rv_t hlmSimpleCallback(size_t to, u32 id, - UNUSED struct hs_scratch *scratch) { +static hwlmcb_rv_t hlmSimpleCallback(size_t to, u32 id, + UNUSED struct hs_scratch *scratch) { DEBUG_PRINTF("match @%zu = %u\n", to, id); ctxt.push_back(hlmMatchEntry(to, id)); @@ -59,10 +57,12 @@ hwlmcb_rv_t hlmSimpleCallback(size_t to, u32 id, return HWLM_CONTINUE_MATCHING; } -template -static void run_benchmarks(int size, int loops, int max_matches, bool is_reverse, MicroBenchmark &bench, InitFunc &&init, BenchFunc &&func) { +template +static void run_benchmarks(int size, int loops, int max_matches, + bool is_reverse, MicroBenchmark &bench, + InitFunc &&init, BenchFunc &&func) { init(bench); - double total_sec = 0.0; + double total_sec = 0.0; u64a total_size = 0; double bw = 0.0; double avg_bw = 0.0; @@ -70,29 +70,31 @@ static void run_benchmarks(int size, int loops, int max_matches, bool is_reverse double avg_time = 0.0; if (max_matches) { int pos = 0; - for(int j = 0; j < max_matches - 1; j++) { + for (int j = 0; j < max_matches - 1; j++) { bench.buf[pos] = 'b'; - pos = (j+1) *size / max_matches ; + pos = (j + 1) * size / max_matches; bench.buf[pos] = 'a'; u64a actual_size = 0; auto start = std::chrono::steady_clock::now(); - for(int i = 0; i < loops; i++) { + for (int i = 0; i < loops; i++) { const u8 *res = func(bench); - if (is_reverse) - actual_size += bench.buf.data() + size - res; - else - actual_size += res - bench.buf.data(); + if (is_reverse) + actual_size += bench.buf.data() + size - res; + else + actual_size += res - bench.buf.data(); } auto end = std::chrono::steady_clock::now(); - double dt = std::chrono::duration_cast(end - start).count(); + double dt = std::chrono::duration_cast( + end - start) + .count(); total_sec += dt; /*convert microseconds to seconds*/ /*calculate bandwidth*/ - bw = (actual_size / dt) * 1000000.0 / 1048576.0; - /*std::cout << "act_size = " << act_size << std::endl; - std::cout << "dt = " << dt << std::endl; - std::cout << "bw = " << bw << std::endl;*/ - avg_bw += bw; + bw = (actual_size / dt) * 1000000.0 / 1048576.0; + /*std::cout << "act_size = " << act_size << std::endl; + std::cout << "dt = " << dt << std::endl; + std::cout << "bw = " << bw << std::endl;*/ + avg_bw += bw; /*convert to MB/s*/ max_bw = std::max(bw, max_bw); /*calculate average time*/ @@ -100,18 +102,28 @@ static void run_benchmarks(int size, int loops, int max_matches, bool is_reverse } avg_time /= max_matches; avg_bw /= max_matches; - total_sec /= 1000000.0; + total_sec /= 1000000.0; /*convert average time to us*/ - printf(KMAG "%s: %u matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " - KBLU "average time per call =" RST " %.3f μs," KBLU " max bandwidth = " RST " %.3f MB/s," KBLU " average bandwidth =" RST " %.3f MB/s \n", - bench.label, max_matches, size ,loops, total_sec, avg_time, max_bw, avg_bw); + /* Keeping the color output + printf(KMAG "%s: %u matches, %u * %u iterations," KBLU + " total elapsed time =" RST " %.3f s, " KBLU + "average time per call =" RST " %.3f μs," KBLU + " max bandwidth = " RST " %.3f MB/s," KBLU + " average bandwidth =" RST " %.3f MB/s \n", + bench.label, max_matches, size, loops, total_sec, avg_time, + max_bw, avg_bw); + */ + printf("%s,%u,%u,%u,%.3f,%.3f,%.3f,%.3f\n", bench.label, max_matches, + size, loops, total_sec, avg_time, max_bw, avg_bw); } else { auto start = std::chrono::steady_clock::now(); for (int i = 0; i < loops; i++) { const u8 *res = func(bench); } auto end = std::chrono::steady_clock::now(); - total_sec += std::chrono::duration_cast(end - start).count(); + total_sec += + std::chrono::duration_cast(end - start) + .count(); /*calculate transferred size*/ total_size = size * loops; /*calculate average time*/ @@ -122,117 +134,131 @@ static void run_benchmarks(int size, int loops, int max_matches, bool is_reverse max_bw = total_size / total_sec; /*convert to MB/s*/ max_bw /= 1048576.0; - printf(KMAG "%s: no matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " - KBLU "average time per call =" RST " %.3f μs ," KBLU " bandwidth = " RST " %.3f MB/s \n", - bench.label, size ,loops, total_sec, avg_time, max_bw ); + /*Keeping the color output + printf(KMAG "%s: no matches, %u * %u iterations," KBLU " total elapsed + time =" RST " %.3f s, " KBLU "average time per call =" RST " %.3f μs ," + KBLU " bandwidth = " RST " %.3f MB/s \n", bench.label, size ,loops, + total_sec, avg_time, max_bw ); + */ + printf("%s,0,%u,%u,%.3f,%.3f,%.3f,0\n", bench.label, size, loops, + total_sec, avg_time, max_bw); } } -int main(){ +int main() { int matches[] = {0, MAX_MATCHES}; std::vector sizes; - for (size_t i = 0; i < N; i++) sizes.push_back(16000 << i*2); - const char charset[] = "aAaAaAaAAAaaaaAAAAaaaaAAAAAAaaaAAaaa"; - + for (size_t i = 0; i < N; i++) + sizes.push_back(16000 << i * 2); + const char charset[] = "aAaAaAaAAAaaaaAAAAaaaaAAAAAAaaaAAaaa"; + printf("Bench Label, max_matches, size,loops, total_sec, avg_time, " + "max_bw, avg_bw\n"); for (int m = 0; m < 2; m++) { for (size_t i = 0; i < std::size(sizes); i++) { MicroBenchmark bench("Shufti", sizes[i]); - run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, + run_benchmarks( + sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, [&](MicroBenchmark &b) { b.chars.set('a'); ue2::shuftiBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); memset(b.buf.data(), 'b', b.size); }, [&](MicroBenchmark &b) { - return shuftiExec(b.lo, b.hi, b.buf.data(), b.buf.data() + b.size); - } - ); + return shuftiExec(b.lo, b.hi, b.buf.data(), + b.buf.data() + b.size); + }); } for (size_t i = 0; i < std::size(sizes); i++) { MicroBenchmark bench("Reverse Shufti", sizes[i]); - run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench, + run_benchmarks( + sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench, [&](MicroBenchmark &b) { b.chars.set('a'); ue2::shuftiBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); memset(b.buf.data(), 'b', b.size); }, [&](MicroBenchmark &b) { - return rshuftiExec(b.lo, b.hi, b.buf.data(), b.buf.data() + b.size); - } - ); + return rshuftiExec(b.lo, b.hi, b.buf.data(), + b.buf.data() + b.size); + }); } for (size_t i = 0; i < std::size(sizes); i++) { MicroBenchmark bench("Truffle", sizes[i]); - run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, + run_benchmarks( + sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, [&](MicroBenchmark &b) { b.chars.set('a'); ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); memset(b.buf.data(), 'b', b.size); }, [&](MicroBenchmark &b) { - return truffleExec(b.lo, b.hi, b.buf.data(), b.buf.data() + b.size); - } - ); + return truffleExec(b.lo, b.hi, b.buf.data(), + b.buf.data() + b.size); + }); } for (size_t i = 0; i < std::size(sizes); i++) { MicroBenchmark bench("Reverse Truffle", sizes[i]); - run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench, + run_benchmarks( + sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench, [&](MicroBenchmark &b) { b.chars.set('a'); ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); memset(b.buf.data(), 'b', b.size); }, [&](MicroBenchmark &b) { - return rtruffleExec(b.lo, b.hi, b.buf.data(), b.buf.data() + b.size); - } - ); + return rtruffleExec(b.lo, b.hi, b.buf.data(), + b.buf.data() + b.size); + }); } for (size_t i = 0; i < std::size(sizes); i++) { MicroBenchmark bench("Vermicelli", sizes[i]); - run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, + run_benchmarks( + sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, [&](MicroBenchmark &b) { b.chars.set('a'); ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); memset(b.buf.data(), 'b', b.size); }, [&](MicroBenchmark &b) { - return vermicelliExec('a', 'b', b.buf.data(), b.buf.data() + b.size); - } - ); + return vermicelliExec('a', 'b', b.buf.data(), + b.buf.data() + b.size); + }); } for (size_t i = 0; i < std::size(sizes); i++) { MicroBenchmark bench("Reverse Vermicelli", sizes[i]); - run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench, + run_benchmarks( + sizes[i], MAX_LOOPS / sizes[i], matches[m], true, bench, [&](MicroBenchmark &b) { b.chars.set('a'); ue2::truffleBuildMasks(b.chars, (u8 *)&b.lo, (u8 *)&b.hi); memset(b.buf.data(), 'b', b.size); }, [&](MicroBenchmark &b) { - return rvermicelliExec('a', 'b', b.buf.data(), b.buf.data() + b.size); - } - ); + return rvermicelliExec('a', 'b', b.buf.data(), + b.buf.data() + b.size); + }); } for (size_t i = 0; i < std::size(sizes); i++) { - //we imitate the noodle unit tests + // we imitate the noodle unit tests std::string str; const size_t char_len = 5; str.resize(char_len + 1); - for (size_t j=0; j < char_len; j++) { - srand (time(NULL)); - int key = rand() % + 36 ; + for (size_t j = 0; j < char_len; j++) { + srand(time(NULL)); + int key = rand() % +36; str[char_len] = charset[key]; str[char_len + 1] = '\0'; } MicroBenchmark bench("Noodle", sizes[i]); - run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, + run_benchmarks( + sizes[i], MAX_LOOPS / sizes[i], matches[m], false, bench, [&](MicroBenchmark &b) { ctxt.clear(); memset(b.buf.data(), 'a', b.size); @@ -242,10 +268,10 @@ int main(){ assert(b.nt != nullptr); }, [&](MicroBenchmark &b) { - noodExec(b.nt.get(), b.buf.data(), b.size, 0, hlmSimpleCallback, &b.scratch); + noodExec(b.nt.get(), b.buf.data(), b.size, 0, + hlmSimpleCallback, &b.scratch); return b.buf.data() + b.size; - } - ); + }); } } From b5a29155e4d4dee44f94ec44622bf8431e676ce4 Mon Sep 17 00:00:00 2001 From: gtsoul-tech Date: Tue, 2 Apr 2024 11:28:00 +0300 Subject: [PATCH 2/4] removed color output code --- benchmarks/benchmarks.cpp | 15 ------------- benchmarks/benchmarks.hpp | 46 +++++++++++++++------------------------ 2 files changed, 17 insertions(+), 44 deletions(-) diff --git a/benchmarks/benchmarks.cpp b/benchmarks/benchmarks.cpp index c6e453ef..14cccc10 100644 --- a/benchmarks/benchmarks.cpp +++ b/benchmarks/benchmarks.cpp @@ -104,15 +104,6 @@ static void run_benchmarks(int size, int loops, int max_matches, avg_bw /= max_matches; total_sec /= 1000000.0; /*convert average time to us*/ - /* Keeping the color output - printf(KMAG "%s: %u matches, %u * %u iterations," KBLU - " total elapsed time =" RST " %.3f s, " KBLU - "average time per call =" RST " %.3f μs," KBLU - " max bandwidth = " RST " %.3f MB/s," KBLU - " average bandwidth =" RST " %.3f MB/s \n", - bench.label, max_matches, size, loops, total_sec, avg_time, - max_bw, avg_bw); - */ printf("%s,%u,%u,%u,%.3f,%.3f,%.3f,%.3f\n", bench.label, max_matches, size, loops, total_sec, avg_time, max_bw, avg_bw); } else { @@ -134,12 +125,6 @@ static void run_benchmarks(int size, int loops, int max_matches, max_bw = total_size / total_sec; /*convert to MB/s*/ max_bw /= 1048576.0; - /*Keeping the color output - printf(KMAG "%s: no matches, %u * %u iterations," KBLU " total elapsed - time =" RST " %.3f s, " KBLU "average time per call =" RST " %.3f μs ," - KBLU " bandwidth = " RST " %.3f MB/s \n", bench.label, size ,loops, - total_sec, avg_time, max_bw ); - */ printf("%s,0,%u,%u,%.3f,%.3f,%.3f,0\n", bench.label, size, loops, total_sec, avg_time, max_bw); } diff --git a/benchmarks/benchmarks.hpp b/benchmarks/benchmarks.hpp index 974d2234..13f66fa5 100644 --- a/benchmarks/benchmarks.hpp +++ b/benchmarks/benchmarks.hpp @@ -26,44 +26,32 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include "hwlm/hwlm_literal.h" +#include "hwlm/noodle_build.h" +#include "hwlm/noodle_engine.h" +#include "hwlm/noodle_internal.h" #include "nfa/shufti.h" #include "nfa/shufticompile.h" #include "nfa/truffle.h" #include "nfa/trufflecompile.h" #include "nfa/vermicelli.hpp" -#include "hwlm/noodle_build.h" -#include "hwlm/noodle_engine.h" -#include "hwlm/noodle_internal.h" -#include "hwlm/hwlm_literal.h" -#include "util/bytecode_ptr.h" #include "scratch.h" +#include "util/bytecode_ptr.h" -/*define colour control characters*/ -#define RST "\x1B[0m" -#define KRED "\x1B[31m" -#define KGRN "\x1B[32m" -#define KYEL "\x1B[33m" -#define KBLU "\x1B[34m" -#define KMAG "\x1B[35m" -#define KCYN "\x1B[36m" -#define KWHT "\x1B[37m" - -class MicroBenchmark -{ +class MicroBenchmark { public: - char const *label; - size_t size; + char const *label; + size_t size; - // Shufti/Truffle - m128 lo, hi; - ue2::CharReach chars; - std::vector buf; + // Shufti/Truffle + m128 lo, hi; + ue2::CharReach chars; + std::vector buf; - // Noodle - struct hs_scratch scratch; - ue2::bytecode_ptr nt; + // Noodle + struct hs_scratch scratch; + ue2::bytecode_ptr nt; - MicroBenchmark(char const *label_, size_t size_) - :label(label_), size(size_), buf(size_) { - }; + MicroBenchmark(char const *label_, size_t size_) + : label(label_), size(size_), buf(size_){}; }; From 62a275e5764efeb8202af18e80174f210b9a3993 Mon Sep 17 00:00:00 2001 From: gtsoul-tech Date: Tue, 2 Apr 2024 13:32:51 +0300 Subject: [PATCH 3/4] change first column name csv --- benchmarks/benchmarks.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/benchmarks.cpp b/benchmarks/benchmarks.cpp index 14cccc10..fd7aed47 100644 --- a/benchmarks/benchmarks.cpp +++ b/benchmarks/benchmarks.cpp @@ -136,7 +136,7 @@ int main() { for (size_t i = 0; i < N; i++) sizes.push_back(16000 << i * 2); const char charset[] = "aAaAaAaAAAaaaaAAAAaaaaAAAAAAaaaAAaaa"; - printf("Bench Label, max_matches, size,loops, total_sec, avg_time, " + printf("Matcher, max_matches, size,loops, total_sec, avg_time, " "max_bw, avg_bw\n"); for (int m = 0; m < 2; m++) { for (size_t i = 0; i < std::size(sizes); i++) { From 3670e52c873e5631871030e3559111ea0d3529a3 Mon Sep 17 00:00:00 2001 From: gtsoul-tech Date: Tue, 2 Apr 2024 14:56:27 +0300 Subject: [PATCH 4/4] output tabulated and csv --- benchmarks/benchmarks.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/benchmarks/benchmarks.cpp b/benchmarks/benchmarks.cpp index fd7aed47..b6106ed4 100644 --- a/benchmarks/benchmarks.cpp +++ b/benchmarks/benchmarks.cpp @@ -104,8 +104,9 @@ static void run_benchmarks(int size, int loops, int max_matches, avg_bw /= max_matches; total_sec /= 1000000.0; /*convert average time to us*/ - printf("%s,%u,%u,%u,%.3f,%.3f,%.3f,%.3f\n", bench.label, max_matches, - size, loops, total_sec, avg_time, max_bw, avg_bw); + printf("%-18s, %-12u, %-10u, %-6u, %-10.3f, %-9.3f, %-8.3f, %-7.3f\n", + bench.label, max_matches, size, loops, total_sec, avg_time, + max_bw, avg_bw); } else { auto start = std::chrono::steady_clock::now(); for (int i = 0; i < loops; i++) { @@ -125,8 +126,8 @@ static void run_benchmarks(int size, int loops, int max_matches, max_bw = total_size / total_sec; /*convert to MB/s*/ max_bw /= 1048576.0; - printf("%s,0,%u,%u,%.3f,%.3f,%.3f,0\n", bench.label, size, loops, - total_sec, avg_time, max_bw); + printf("%-18s, %-12s, %-10u, %-6u, %-10.3f, %-9.3f, %-8.3f, %-7s\n", + bench.label, "0", size, loops, total_sec, avg_time, max_bw, "0"); } } @@ -136,8 +137,9 @@ int main() { for (size_t i = 0; i < N; i++) sizes.push_back(16000 << i * 2); const char charset[] = "aAaAaAaAAAaaaaAAAAaaaaAAAAAAaaaAAaaa"; - printf("Matcher, max_matches, size,loops, total_sec, avg_time, " - "max_bw, avg_bw\n"); + printf("%-18s, %-12s, %-10s, %-6s, %-10s, %-9s, %-8s, %-7s\n", "Matcher", + "max_matches", "size", "loops", "total_sec", "avg_time", "max_bw", + "avg_bw"); for (int m = 0; m < 2; m++) { for (size_t i = 0; i < std::size(sizes); i++) { MicroBenchmark bench("Shufti", sizes[i]);