diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index debfc0ca..f56a5f5b 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -1,4 +1,4 @@ -add_executable(benchmarks benchmarks.cpp shufti.cpp truffle.cpp noodle.cpp) -set_source_files_properties(shufti.cpp PROPERTIES COMPILE_FLAGS +add_executable(benchmarks benchmarks.cpp noodle.cpp) +set_source_files_properties(benchmarks.cpp PROPERTIES COMPILE_FLAGS "-Wall -Wno-unused-variable") target_link_libraries(benchmarks hs) \ No newline at end of file diff --git a/benchmarks/benchmarks.cpp b/benchmarks/benchmarks.cpp index be224ba9..ce680334 100644 --- a/benchmarks/benchmarks.cpp +++ b/benchmarks/benchmarks.cpp @@ -1,25 +1,161 @@ -#include "benchmarks.hpp" #include -#include -#include -#include -#include -#include +#include +#include +#include +#include #include +#include "nfa/shufti.h" +#include "nfa/shufticompile.h" +#include "nfa/truffle.h" +#include "nfa/trufflecompile.h" +#include "benchmarks.hpp" + #define MAX_LOOPS 500000000 #define MAX_MATCHES 10 +/* +void shuffle_init(){ + m128 lo, hi; + ue2::CharReach chars; + chars.set('a'); + shuftiBuildMasks(chars, (u8 *)&lo, (u8 *)&hi); + std::unique_ptr kt1 ( new u8[size] ); + memset(kt1.get(),'b',size); +} +*/ + +/* +void truffle_init(){ + m128 lo, hi; + ue2::CharReach chars; + chars.set('a'); + truffleBuildMasks(chars, (u8 *)&lo, (u8 *)&hi); + std::unique_ptr kt1 ( new u8[size] ); + memset(kt1.get(),'b',size); +} +*/ + +/* +struct hlmMatchEntry { + size_t to; + u32 id; + hlmMatchEntry(size_t end, u32 identifier) : + to(end), id(identifier) {} +}; + +std::vector ctxt; + +static +hwlmcb_rv_t hlmSimpleCallback(size_t to, u32 id, + UNUSED struct hs_scratch *scratch) { + DEBUG_PRINTF("match @%zu = %u\n", to, id); + + ctxt.push_back(hlmMatchEntry(to, id)); + + return HWLM_CONTINUE_MATCHING; +} + +void noodle_init(){ + ctxt.clear(); + std::unique_ptr data ( new u8[size] ); + memset(data.get(), 'a', size); + double total_sec = 0.0; + u64a transferred_size = 0; + double avg_time = 0.0; + double max_bw = 0.0; + double bandwitdh = 0.0; + u32 id = 1000; + ue2::hwlmLiteral lit(std::string(lit_str, lit_len), nocase, id); + auto n = ue2::noodBuildTable(lit); + assert(n != nullptr); + struct hs_scratch scratch; +} +*/ + +void run_benchmarks(int size, int loops, int M, bool has_match, std::function function) { + m128 lo, hi; + ue2::CharReach chars; + chars.set('a'); + shuftiBuildMasks(chars, (u8 *)&lo, (u8 *)&hi); + std::unique_ptr kt1 ( new u8[size] ); + memset(kt1.get(),'b',size); + double total_sec = 0.0; + u64a transferred_size = 0; + double bandwidth = 0.0; + double max_bw = 0.0; + double avg_time = 0.0; + if (has_match) { + int pos = 0; + for(int j = 0; j < M; j++) { + kt1[pos] = 'b'; + pos = (j*size) / M ; + kt1[pos] = 'a'; + unsigned long act_size = 0; + auto start = std::chrono::steady_clock::now(); + for(int i = 0; i < loops; i++) { + const u8 *res = function(lo, hi, kt1.get(), kt1.get() + size); + act_size += res - kt1.get(); + } + auto end = std::chrono::steady_clock::now(); + double dt = std::chrono::duration_cast(end - start).count(); + total_sec += dt; + /*convert microseconds to seconds*/ + total_sec /= 1000000.0; + /*calculate bandwidth*/ + bandwidth += (act_size / dt) * 1000000.0; + /*convert to MB/s*/ + bandwidth = bandwidth / 1048576.0; + max_bw = std::max(bandwidth ,max_bw); + /*calculate average time*/ + avg_time += total_sec / loops; + } + avg_time /= M; + bandwidth /= M; + /*convert average time to us*/ + avg_time *= 1000000.0; + printf(KMAG "case with %u matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " + KBLU "average time per call =" RST " %.3f μs," KBLU " max bandwidth = " RST " %.3f MB/s," KBLU " average bandwidth =" RST " %.3f MB/s \n", + M, size ,loops, total_sec, avg_time, max_bw, bandwidth); + } else { + auto start = std::chrono::steady_clock::now(); + for (int i = 0; i < loops; i++) { + function(lo, hi, kt1.get(), kt1.get() + size); + } + auto end = std::chrono::steady_clock::now(); + total_sec += std::chrono::duration_cast(end - start).count(); + /*calculate transferred size*/ + transferred_size = size * loops; + /*calculate average time*/ + avg_time = total_sec / loops; + /*convert microseconds to seconds*/ + total_sec /= 1000000.0; + /*calculate maximum bandwidth*/ + max_bw = transferred_size / total_sec; + /*convert to MB/s*/ + max_bw /= 1048576.0; + printf(KMAG "case without matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " + KBLU "average time per call =" RST " %.3f μs ," KBLU " bandwidth = " RST " %.3f MB/s \n", + size ,loops, total_sec, avg_time, max_bw); + } +} + + int main(){ - std::function functions[] = { shufti_benchmarks, rshufti_benchmarks, truffle_benchmarks, rtruffle_benchmarks }; + std::function functions[] = {shuftiExec, rshuftiExec, truffleExec, rtruffleExec}; int sizes[] = { 16000, 32000, 64000, 120000, 1600000, 2000000, 2500000, 3500000, 150000000, 250000000, 350000000, 500000000 }; + std::string labels[] = {"\x1B[33m shuftiExec Benchmarks \x1B[0m\n", "\x1B[33m rshuftiExec Benchmarks \x1B[0m\n", + "\x1B[33m triffleExec Benchmarks \x1B[0m\n", "\x1B[33m triffleExec Benchmarks \x1B[0m\n"}; const char charset[] = "aAaAaAaAAAaaaaAAAAaaaaAAAAAAaaaAAaaa"; + for (size_t i = 0; i < std::size(sizes); i++) { for(size_t j = 0; j < std::size(functions); j++) { - functions[j](sizes[i], MAX_LOOPS / sizes[i], MAX_MATCHES, false); - functions[j](sizes[i], MAX_LOOPS / sizes[i], MAX_MATCHES, true); + std::cout << labels[j]; + run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], MAX_MATCHES, false, functions[j]); + run_benchmarks(sizes[i], MAX_LOOPS / sizes[i], MAX_MATCHES, true, functions[j]); } } + for(size_t i=0; i < std::size(sizes); i++){ //we imitate the noodle unit tests for (int char_len = 1; char_len < 9; char_len++) { diff --git a/benchmarks/benchmarks.hpp b/benchmarks/benchmarks.hpp index 894ed9ce..621dfb25 100644 --- a/benchmarks/benchmarks.hpp +++ b/benchmarks/benchmarks.hpp @@ -1,3 +1,5 @@ +#include + /*define colour control characters*/ #define RST "\x1B[0m" #define KRED "\x1B[31m" @@ -8,8 +10,6 @@ #define KCYN "\x1B[36m" #define KWHT "\x1B[37m" -void shufti_benchmarks(int size, int loops, int M, bool has_match); -void rshufti_benchmarks(int size, int loops, int M, bool has_match); -void truffle_benchmarks(int size, int loops, int M, bool has_match); -void rtruffle_benchmarks(int size, int loops, int M, bool has_match); -void noodle_benchmarks(int size, int M, const char *lit_str, int lit_len, char nocase); \ No newline at end of file + +void noodle_benchmarks(int size, int M, const char *lit_str, int lit_len, char nocase); +void run_benchmarks(int size, int loops, int M, bool has_match, std::function function); \ No newline at end of file diff --git a/benchmarks/shufti.cpp b/benchmarks/shufti.cpp deleted file mode 100644 index f6c2be5c..00000000 --- a/benchmarks/shufti.cpp +++ /dev/null @@ -1,149 +0,0 @@ -#include "nfa/shufti.h" -#include "nfa/shufticompile.h" -#include "benchmarks.hpp" -#include -#include -#include -#include -#include -#include - -void shufti_benchmarks(int size, int loops, int M, bool has_match) { - m128 lo, hi; - ue2::CharReach chars; - chars.set('a'); - int ret = shuftiBuildMasks(chars, (u8 *)&lo, (u8 *)&hi); - std::unique_ptr kt1 ( new u8[size] ); - memset(kt1.get(),'b',size); - double total_sec = 0.0; - u64a transferred_size = 0; - double bandwitdh = 0.0; - double max_bw = 0.0; - double avg_time = 0.0; - if (has_match) { - int pos = 0; - for(int j = 0; j < M; j++) { - kt1[pos] = 'b'; - srand (time(NULL)); - pos = rand() % size + 0; - kt1[pos] = 'a'; - unsigned long act_size = 0; - auto start = std::chrono::steady_clock::now(); - for(int i = 0; i < loops; i++) { - const u8 *res = shuftiExec(lo, hi, kt1.get(), kt1.get() + size); - act_size += res - kt1.get(); - } - auto end = std::chrono::steady_clock::now(); - double dt = std::chrono::duration_cast(end - start).count(); - total_sec += dt; - /*convert microseconds to seconds*/ - total_sec /= 1000000.0; - /*calculate bandwidth*/ - bandwitdh += act_size / total_sec; - /*convert to MB/s*/ - bandwitdh = bandwitdh / 1048576.0; - max_bw = std::max(bandwitdh ,max_bw); - /*calculate average time*/ - avg_time += total_sec / loops; - } - avg_time /= M; - bandwitdh /= M; - /*convert average time to us*/ - avg_time *= 1000000.0; - printf(KMAG "ShuftiExec: case with %u matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " - KBLU "average time per call =" RST " %.3f μs," KBLU " bandwidth = " RST " %.3f MB/s," KBLU " average bandwidth =" RST " %.3f MB/s \n", - M, size ,loops, total_sec, avg_time, max_bw, bandwitdh); - } else { - auto start = std::chrono::steady_clock::now(); - for (int i = 0; i < loops; i++) { - shuftiExec(lo, hi, kt1.get(), kt1.get() + size); - } - auto end = std::chrono::steady_clock::now(); - total_sec += std::chrono::duration_cast(end - start).count(); - /*calculate transferred size*/ - transferred_size = size * loops; - /*calculate average time*/ - avg_time = total_sec / loops; - /*convert microseconds to seconds*/ - total_sec /= 1000000.0; - /*calculate maximum bandwidth*/ - max_bw = transferred_size / total_sec; - /*convert to MB/s*/ - max_bw /= 1048576.0; - /*calculate average bandwidth*/ - bandwitdh = max_bw / loops; - printf(KMAG "ShuftiExec: case without matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " - KBLU "average time per call =" RST " %.3f μs ," KBLU " bandwidth = " RST " %.3f MB/s," KBLU " average bandwidth =" RST " %.3f MB/s \n", - size ,loops, total_sec, avg_time, max_bw, bandwitdh); - } -} - -void rshufti_benchmarks(int size, int loops, int M, bool has_match) { - m128 lo, hi; - ue2::CharReach chars; - chars.set('a'); - int ret = shuftiBuildMasks(chars, (u8 *)&lo, (u8 *)&hi); - std::unique_ptr kt1 ( new u8[size] ); - memset(kt1.get(),'b',size); - double total_sec = 0.0; - u64a transferred_size = 0; - double bandwitdh = 0.0; - double max_bw = 0.0; - double avg_time = 0.0; - if (has_match) { - int pos = 0; - for(int j = 0; j < M; j++) { - kt1[pos] = 'b'; - srand (time(NULL)); - pos = rand() % size + 0; - kt1[pos] = 'a'; - unsigned long act_size = 0; - auto start = std::chrono::steady_clock::now(); - for(int i = 0; i < loops; i++) { - const u8 *res = rshuftiExec(lo, hi, kt1.get(), kt1.get() + size); - act_size += res - kt1.get(); - } - auto end = std::chrono::steady_clock::now(); - double dt = std::chrono::duration_cast(end - start).count(); - total_sec += dt; - /*convert microseconds to seconds*/ - total_sec /= 1000000.0; - /*calculate bandwidth*/ - bandwitdh += act_size / total_sec; - /*convert to MB/s*/ - bandwitdh = bandwitdh / 1048576.0; - max_bw = std::max(bandwitdh ,max_bw); - /*calculate average time*/ - avg_time += total_sec / loops; - } - avg_time /= M; - bandwitdh /= M; - /*convert average time to μs*/ - avg_time *= 1000000.0; - printf(KMAG "rShuftiExec: case with %u matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " - KBLU "average time per call =" RST " %.3f μs," KBLU " bandwidth = " RST " %.3f MB/s," KBLU " average bandwidth =" RST " %.3f MB/s \n", - M, size ,loops, total_sec, avg_time, max_bw, bandwitdh); - } else { - auto start = std::chrono::steady_clock::now(); - for (int i = 0; i < loops; i++) { - rshuftiExec(lo, hi, kt1.get(), kt1.get() + size); - } - auto end = std::chrono::steady_clock::now(); - total_sec += std::chrono::duration_cast(end - start).count(); - /*calculate transferred size*/ - transferred_size = size * loops; - /*calculate average time*/ - avg_time = total_sec / loops; - /*convert microseconds to seconds*/ - total_sec /= 1000000.0; - /*calculate maximum bandwidth*/ - max_bw = transferred_size / total_sec; - /*convert to MB/s*/ - max_bw /= 1048576.0; - /*calculate average bandwidth*/ - bandwitdh = max_bw / loops; - printf(KMAG "rShuftiExec: case without matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " - KBLU "average time per call =" RST " %.3f μs ," KBLU " bandwidth = " RST " %.3f MB/s," KBLU " average bandwidth =" RST " %.3f MB/s \n", - size ,loops, total_sec, avg_time, max_bw, bandwitdh); - } -} diff --git a/benchmarks/truffle.cpp b/benchmarks/truffle.cpp deleted file mode 100644 index d521c2b1..00000000 --- a/benchmarks/truffle.cpp +++ /dev/null @@ -1,149 +0,0 @@ -#include "nfa/truffle.h" -#include "nfa/trufflecompile.h" -#include "benchmarks.hpp" -#include -#include -#include -#include -#include - -void truffle_benchmarks(int size, int loops, int M, bool has_match) { - m128 lo, hi; - ue2::CharReach chars; - chars.set('a'); - truffleBuildMasks(chars, (u8 *)&lo, (u8 *)&hi); - std::unique_ptr kt1 ( new u8[size] ); - memset(kt1.get(),'b',size); - double total_sec = 0.0; - u64a transferred_size = 0; - double bandwitdh = 0.0; - double max_bw = 0.0; - double avg_time = 0.0; - if (has_match) { - int pos = 0; - for(int j = 0; j < M; j++) { - kt1[pos] = 'b'; - srand (time(NULL)); - pos = rand() % size + 0; - kt1[pos] = 'a'; - unsigned long act_size = 0; - auto start = std::chrono::steady_clock::now(); - for(int i = 0; i < loops; i++) { - const u8 *res = truffleExec(lo, hi, kt1.get(), kt1.get() + size); - act_size += res - kt1.get(); - } - auto end = std::chrono::steady_clock::now(); - double dt = std::chrono::duration_cast(end - start).count(); - total_sec += dt; - /*convert microseconds to seconds*/ - total_sec /= 1000000.0; - /*calculate bandwidth*/ - bandwitdh += act_size / total_sec; - /*convert to MB/s*/ - bandwitdh = bandwitdh / 1048576.0; - max_bw = std::max(bandwitdh ,max_bw); - /*calculate average time*/ - avg_time += total_sec / loops; - } - avg_time /= M; - bandwitdh /= M; - /*convert average time to us*/ - avg_time *= 1000000.0; - printf(KMAG "TruffleExec: case with %u matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " - KBLU "average time per call =" RST " %.3f μs," KBLU " bandwidth = " RST " %.3f MB/s," KBLU " average bandwidth =" RST " %.3f MB/s \n", - M, size ,loops, total_sec, avg_time, max_bw, bandwitdh); - } else { - auto start = std::chrono::steady_clock::now(); - for (int i = 0; i < loops; i++) { - truffleExec(lo, hi, kt1.get(), kt1.get() + size); - } - auto end = std::chrono::steady_clock::now(); - total_sec += std::chrono::duration_cast(end - start).count(); - /*calculate transferred size*/ - transferred_size = size * loops; - /*calculate average time*/ - avg_time = total_sec / loops; - /*convert microseconds to seconds*/ - total_sec /= 1000000.0; - /*calculate maximum bandwidth*/ - max_bw = transferred_size / total_sec; - /*convert to MB/s*/ - max_bw /= 1048576.0; - /*calculate average bandwidth*/ - bandwitdh = max_bw / loops; - printf(KMAG "TruffleExec case without matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " - KBLU "average time per call =" RST " %.3f μs ," KBLU " bandwidth = " RST " %.3f MB/s," KBLU " average bandwidth =" RST " %.3f MB/s \n", - size ,loops, total_sec, avg_time, max_bw, bandwitdh); - } -} - - -void rtruffle_benchmarks(int size, int loops, int M, bool has_match) { - m128 lo, hi; - ue2::CharReach chars; - chars.set('a'); - truffleBuildMasks(chars, (u8 *)&lo, (u8 *)&hi); - std::unique_ptr kt1 ( new u8[size] ); - memset(kt1.get(),'b',size); - double total_sec = 0.0; - u64a transferred_size = 0; - double bandwitdh = 0.0; - double max_bw = 0.0; - double avg_time = 0.0; - if (has_match) { - int pos = 0; - for(int j = 0; j < M; j++) { - kt1[pos] = 'b'; - srand (time(NULL)); - pos = rand() % size + 0; - kt1[pos] = 'a'; - unsigned long act_size = 0; - auto start = std::chrono::steady_clock::now(); - for(int i = 0; i < loops; i++) { - const u8 *res = rtruffleExec(lo, hi, kt1.get(), kt1.get() + size); - act_size += res - kt1.get(); - } - auto end = std::chrono::steady_clock::now(); - double dt = std::chrono::duration_cast(end - start).count(); - total_sec += dt; - /*convert microseconds to seconds*/ - total_sec /= 1000000.0; - /*calculate bandwidth*/ - bandwitdh += act_size / total_sec; - /*convert to MB/s*/ - bandwitdh = bandwitdh / 1048576.0; - max_bw = std::max(bandwitdh ,max_bw); - /*calculate average time*/ - avg_time += total_sec / loops; - } - avg_time /= M; - bandwitdh /= M; - /*convert average time to us*/ - avg_time *= 1000000.0; - printf(KMAG "rTruffleExec: case with %u matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " - KBLU "average time per call =" RST " %.3f μs," KBLU " bandwidth = " RST " %.3f MB/s," KBLU " average bandwidth =" RST " %.3f MB/s \n", - M, size ,loops, total_sec, avg_time, max_bw, bandwitdh); - } else { - auto start = std::chrono::steady_clock::now(); - for (int i = 0; i < loops; i++) { - rtruffleExec(lo, hi, kt1.get(), kt1.get() + size); - } - auto end = std::chrono::steady_clock::now(); - total_sec += std::chrono::duration_cast(end - start).count(); - /*calculate transferred size*/ - transferred_size = size * loops; - /*calculate average time*/ - avg_time = total_sec / loops; - /*convert microseconds to seconds*/ - total_sec /= 1000000.0; - /*calculate maximum bandwidth*/ - max_bw = transferred_size / total_sec; - /*convert to MB/s*/ - max_bw /= 1048576.0; - /*calculate average bandwidth*/ - bandwitdh = max_bw / loops; - printf(KMAG "rTruffleExec case without matches, %u * %u iterations," KBLU " total elapsed time =" RST " %.3f s, " - KBLU "average time per call =" RST " %.3f μs ," KBLU " bandwidth = " RST " %.3f MB/s," KBLU " average bandwidth =" RST " %.3f MB/s \n", - size ,loops, total_sec, avg_time, max_bw, bandwitdh); - } -}