From 2b9636ccc0945b97b2d056f1ddcdbb319fe61ed6 Mon Sep 17 00:00:00 2001
From: apostolos <apostolos.tapsas@vectorcamp.gr>
Date: Tue, 7 Sep 2021 11:01:10 +0300
Subject: [PATCH] benchmarks output fixes

---
 benchmarks/benchmarks.cpp | 12 +++---
 benchmarks/noodle.cpp     | 18 ++++++---
 benchmarks/shufti.cpp     | 84 +++++++++++++++++++++++++++++----------
 benchmarks/truffle.cpp    | 83 ++++++++++++++++++++++++++++----------
 4 files changed, 144 insertions(+), 53 deletions(-)

diff --git a/benchmarks/benchmarks.cpp b/benchmarks/benchmarks.cpp
index 34f1ccfc..7d669cc0 100644
--- a/benchmarks/benchmarks.cpp
+++ b/benchmarks/benchmarks.cpp
@@ -6,8 +6,8 @@
 #include <functional>
 int main(){
     int sizes[]=   { 16000, 32000, 64000, 120000, 1600000, 2000000, 2500000, 3500000, 150000000, 250000000, 350000000, 500000000};
-    int iters[]=   { 16000, 32000, 64000, 120000,    3000,    3000,    3000,    2000,        25,         3,         3,         2};
-    int exp_len[]= { 16000, 32000, 64000, 120000, 1000000, 1000000, 1500000, 3500000,  10000000,  20000000,  30000000,  40000000};
+    int loops[]=   {  6000,  6000,  6000,   6000,    1000,    1000,    1000,    1000,        50,        50,        25,        25};
+    int exp_len[]= {  2000,  2000,  2000,   2000,     250,     250,     250,     250,        10,        10,         5,         5};
     const char charset[] = "aAaAaAaAAAaaaaAAAAaaaaAAAAAAaaaAAaaa";
     std::string labels[] = {"\x1B[33m shuftiExec Benchmarks(kbytes)  \x1B[0m\n", "\x1B[33m rshuftiExec Benchmarks(kbytes)  \x1B[0m\n",
                             "\x1B[33m truffleExec Benchmarks(kbytes) \x1B[0m\n", "\x1B[33m rtruffleExec Benchmarks(kbytes) \x1B[0m\n", 
@@ -17,15 +17,13 @@ int main(){
                             "\x1B[33m truffleExec Benchmarks(Gbytes) \x1B[0m\n", "\x1B[33m rtruffleExec Benchmarks(Gbytes) \x1B[0m\n"
                         };
     std::function<void(int,int,int,bool)>  functions[] = { shufti_benchmarks, rshufti_benchmarks, truffle_benchmarks, rtruffle_benchmarks };
-    
     for (int i=0; i<12; i++) {
         std::cout << labels[i];
         for(int j=0; j<4; j++){
-            functions[j](sizes[i],iters[i],exp_len[i],false);
-            functions[j](sizes[i],iters[i],exp_len[i],true);  
+            functions[j](sizes[i],loops[i],exp_len[i],false);
+            functions[j](sizes[i],loops[i],exp_len[i],true);  
         } 
     }
-    
     for(int i=0; i<12; i++){
         if(i==0){
             std::cout<<std::endl <<"\x1B[33m noodle Benchmarks(kbytes) \x1B[0m"<<std::endl;
@@ -44,7 +42,7 @@ int main(){
                 str[char_len] = charset[key];
                 str[char_len + 1] = '\0';
             }
-            noodle_benchmarks(sizes[i], iters[i], str,char_len,0);
+            noodle_benchmarks(sizes[i], loops[i], str,char_len,0);
             delete [] str;    
         }
     }
diff --git a/benchmarks/noodle.cpp b/benchmarks/noodle.cpp
index 86222f80..607f1f05 100644
--- a/benchmarks/noodle.cpp
+++ b/benchmarks/noodle.cpp
@@ -34,19 +34,25 @@ void noodle_benchmarks(int size, int M, const char *lit_str, int lit_len, char n
     u8 *data = new u8[size];
     memset(data, 'a', size);
     double total_sec = 0;
+    long double bw = 0;
     u32 id = 1000;
     ue2::hwlmLiteral lit(std::string(lit_str, lit_len), nocase, id);
     auto n = ue2::noodBuildTable(lit);
     assert(n != nullptr);
     struct hs_scratch scratch;
-    for (int i = 0; i < M; i++){
-        auto start = std::chrono::steady_clock::now(); 
+    auto start = std::chrono::steady_clock::now(); 
+    for (int i = 0; i < M; i++){ 
         noodExec(n.get(), data, size, 0, hlmSimpleCallback, &scratch); 
-        auto end = std::chrono::steady_clock::now();
-        std::chrono::duration<double> noodExec_elapsed_seconds = end-start;
-        total_sec += noodExec_elapsed_seconds.count();
     }
+    auto end = std::chrono::steady_clock::now();
+    total_sec += std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+    /*average time*/
     total_sec /= M;
-    std::cout<<"\x1B[35m Case with match in random pos and size: "<<size<<" lit_len: "<<lit_len<<" nocase: "<<(int)nocase<<"\x1B[36m noodExec elapsetime: \x1B[0m"<<total_sec<<"\x1B[36m bandwidth: \x1B[0m"<<(size/total_sec)<<std::endl;    
+    /*average size*/
+    size /=M;
+    double mb_size = (double) size / 1048576;
+    bw = mb_size / total_sec;
+    std::cout << "\x1B[35m Case with match in random pos and size: "<< size <<" lit_len: "<< lit_len <<" nocase: "<< (int)nocase
+              << "\x1B[36m noodExec elapsetime: \x1B[0m" << total_sec << " (μs) \x1B[36m bandwidth: \x1B[0m" << bw <<" (MB/μs)" << std::endl;    
     delete [] data;
 }
\ No newline at end of file
diff --git a/benchmarks/shufti.cpp b/benchmarks/shufti.cpp
index 45390b0d..78afc90d 100644
--- a/benchmarks/shufti.cpp
+++ b/benchmarks/shufti.cpp
@@ -1,9 +1,11 @@
 #include "nfa/shufti.h"
+#include "nfa/shufticompile.h"
 #include "benchmarks.hpp"
 #include <iostream>
 #include <chrono>
 #include <cstring>
 #include <ctime>
+#include <cstdlib>
 
 /*
 #define RST  "\x1B[0m"
@@ -19,9 +21,13 @@
 
 void shufti_benchmarks(int size, int loops, int M, bool has_match) { 
     m128 lo, hi;
-    char *kt1 = new char[size];
+    ue2::CharReach chars;
+    chars.set('a');
+    int ret = shuftiBuildMasks(chars, (u8 *)&lo, (u8 *)&hi);
+    u8 *kt1 = new u8[size];
     memset(kt1,'b',size);
     double total_sec = 0;
+    double bw = 0;
     if (has_match){
         int pos = 0;
         for(int j=0; j<M; j++){
@@ -29,34 +35,55 @@ void shufti_benchmarks(int size, int loops, int M, bool has_match) {
             srand (time(NULL));
             pos = rand() % size + 0;
             kt1[pos] = 'a';
+            unsigned long act_size = 0;
             auto start = std::chrono::steady_clock::now();
-            for (int i = 0; i < loops/M; i++) {
-                shuftiExec(lo, hi, (u8 *)kt1 + i, (u8 *)kt1 + strlen(kt1));
+            for (int i = 0; i < loops; i++) {
+                const u8 *res = shuftiExec(lo, hi, kt1, kt1 + size);
+                act_size += res - kt1;
             }
             auto end = std::chrono::steady_clock::now();
-            std::chrono::duration<double> shuftiExec_elapsed_seconds = end-start;
-            total_sec += shuftiExec_elapsed_seconds.count();
+            double dt = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+            /*average time*/
+            dt /= loops;
+            total_sec += dt;
+            /*average size*/
+            act_size /= loops;
+            double mb_size = (double) act_size / 1048576;
+            bw += mb_size / dt;
         }
         total_sec /= M;
-        std::cout<<"\x1B[35m Case with match in random pos and size: "<<size<<" for "<<loops<<" loops ("<< M <<" random possisions checked):"<<"\x1B[36m shuftiExec elapsetime: \x1B[0m"<<total_sec<<"\x1B[36m bandwidth: \x1B[0m"<<(size/total_sec)<<std::endl;
+        bw /= M;
+        std::cout << "\x1B[35m Case with match in random pos and size: " << size << " for " << loops <<" loops ("
+                  << M << " random possisions checked): \x1B[36m shuftiExec elapsetime: \x1B[0m" << (total_sec)
+                  << " (μs) \x1B[36m bandwidth: \x1B[0m" << bw <<" (MB/μs)"<<std::endl;
     } else {
         auto start = std::chrono::steady_clock::now();
         for (int i = 0; i < loops; i++) {
-            shuftiExec(lo, hi, (u8 *)kt1 + i, (u8 *)kt1 + strlen(kt1));
+            shuftiExec(lo, hi, kt1, kt1 + size);
         }
         auto end = std::chrono::steady_clock::now();
-        std::chrono::duration<double> shuftiExec_elapsed_seconds = end-start;
-        total_sec += shuftiExec_elapsed_seconds.count();
-        std::cout<<"\x1B[35m Case with no match in random pos and size: "<<size<<" for "<<loops<<" loops:"<<"\x1B[36m shuftiExec elapsetime: \x1B[0m"<<total_sec<<"\x1B[36m bandwidth: \x1B[0m"<<(size/total_sec)<<std::endl;
+        total_sec += std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+        /*average time*/
+        total_sec /= loops;
+        double mb_size = (double) size / 1048576;
+        /*average size*/
+        mb_size /= loops;
+        bw = mb_size / total_sec;
+        std::cout<<"\x1B[35m Case with no match in random pos and size: "<<size<<" for "<<loops<<" loops:"
+                 <<"\x1B[36m shuftiExec elapsetime: \x1B[0m"<<(total_sec)<<" (μs) \x1B[36m bandwidth: \x1B[0m"<< bw <<" (MB/μs)"<<std::endl;
     }
     delete [] kt1;
 }
 
 void rshufti_benchmarks(int size, int loops, int M, bool has_match) {
     m128 lo, hi;
-    char *kt1 = new char[size];
+    ue2::CharReach chars;
+    chars.set('a');
+    int ret = shuftiBuildMasks(chars, (u8 *)&lo, (u8 *)&hi);
+    u8 *kt1 = new u8[size];
     memset(kt1,'b',size);
     double total_sec = 0;
+    long double bw = 0;
     if (has_match){
         int pos = 0;
         for(int j=0; j<M; j++){
@@ -64,25 +91,42 @@ void rshufti_benchmarks(int size, int loops, int M, bool has_match) {
             srand (time(NULL));
             pos = rand() % size + 0;
             kt1[pos] = 'a';
+            unsigned long act_size = 0;
             auto start = std::chrono::steady_clock::now();
-            for (int i = 0; i < loops/M; i++) {
-                rshuftiExec(lo, hi, (u8 *)kt1 + i, (u8 *)kt1 + strlen(kt1));
+            for (int i = 0; i < loops; i++) {
+                const u8 *res = rshuftiExec(lo, hi, kt1, kt1 + size);
+                act_size += res - kt1; 
             }
             auto end = std::chrono::steady_clock::now();
-            std::chrono::duration<double> shuftiExec_elapsed_seconds = end-start;
-            total_sec += shuftiExec_elapsed_seconds.count();
+            double dt = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+            /*average time*/
+            dt /= loops;
+            total_sec += dt;
+            /*average size*/
+            act_size /= loops;
+            double mb_size = (double) act_size / 1048576;
+            bw += mb_size / dt;
         }
         total_sec /= M;
-        std::cout<<"\x1B[35m Case with match in random pos and size: "<<size<<" for "<<loops<<" loops ("<< M <<" random possisions checked):"<<"\x1B[36m rshuftiExec elapsetime: \x1B[0m"<<total_sec<<"\x1B[36m bandwidth: \x1B[0m"<<(size/total_sec)<<std::endl;
+        bw /= M;
+        std::cout << "\x1B[35m Case with match in random pos and size: " << size << " for " << loops<<" loops ("
+                  << M << " random possisions checked): \x1B[36m rshuftiExec elapsetime: \x1B[0m" << total_sec
+                  <<" (μs) \x1B[36m bandwidth: \x1B[0m" << bw <<" (MB/μs)"<<std::endl;
     } else {
         auto start = std::chrono::steady_clock::now();
         for (int i = 0; i < loops; i++) {
-            rshuftiExec(lo, hi, (u8 *)kt1 + i, (u8 *)kt1 + strlen(kt1));
+            rshuftiExec(lo, hi, kt1, kt1 + size);
         }
         auto end = std::chrono::steady_clock::now();
-        std::chrono::duration<double> shuftiExec_elapsed_seconds = end-start;
-        total_sec += shuftiExec_elapsed_seconds.count();
-        std::cout<<"\x1B[35m Case with no match in random pos and size: "<<size<<" for "<<loops<<" loops:"<<"\x1B[36m rshuftiExec elapsetime: \x1B[0m"<<total_sec<<"\x1B[36m bandwidth: \x1B[0m"<<(size/total_sec)<<std::endl;
+        total_sec += std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+        /*average time*/
+        total_sec /=loops;
+        double mb_size = (double) size / 1048576;
+        /*average size*/
+        mb_size /=loops;
+        bw = mb_size / total_sec;
+        std::cout<<"\x1B[35m Case with no match in random pos and size: "<< size <<" for "<< loops <<" loops:"
+                 <<"\x1B[36m rshuftiExec elapsetime: \x1B[0m"<< total_sec <<" (μs) \x1B[36m bandwidth: \x1B[0m"<< bw <<" (MB/μs)"<<std::endl;
     }
     delete [] kt1;
 }
diff --git a/benchmarks/truffle.cpp b/benchmarks/truffle.cpp
index abcecdb1..a0d1b0ff 100644
--- a/benchmarks/truffle.cpp
+++ b/benchmarks/truffle.cpp
@@ -1,4 +1,5 @@
 #include "nfa/truffle.h"
+#include "nfa/trufflecompile.h"
 #include "benchmarks.hpp"
 #include <iostream>
 #include <chrono>
@@ -19,9 +20,13 @@
 
 void truffle_benchmarks(int size, int loops, int M, bool has_match) {
     m128 lo, hi;
-    char *kt1 = new char[size];
+    ue2::CharReach chars;
+    chars.set('a');
+    truffleBuildMasks(chars, (u8 *)&lo, (u8 *)&hi);
+    u8*kt1 = new u8[size];
     memset(kt1,'b',size);
     double total_sec = 0;
+    long double bw = 0;
     if (has_match){
         int pos = 0;
         for(int j=0; j<M; j++){
@@ -29,25 +34,42 @@ void truffle_benchmarks(int size, int loops, int M, bool has_match) {
             srand (time(NULL));
             pos = rand() % size + 0;
             kt1[pos] = 'a';
+            unsigned long act_size = 0;
             auto start = std::chrono::steady_clock::now();
-            for (int i = 0; i < loops/M; i++) {
-                truffleExec(lo, hi, (u8 *)kt1 + i, (u8 *)kt1 + strlen(kt1));
+            for (int i = 0; i < loops; i++) {
+                const u8 *res = truffleExec(lo, hi, kt1, kt1 + size);
+                act_size += res - kt1;
             }
             auto end = std::chrono::steady_clock::now();
-            std::chrono::duration<double> shuftiExec_elapsed_seconds = end-start;
-            total_sec += shuftiExec_elapsed_seconds.count();
+            double dt = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+            /*average time*/
+            dt /= loops;
+            total_sec += dt;
+            /*average size*/
+            act_size /= loops;
+            double mb_size = (double) act_size / 1048576;
+            bw += mb_size / dt;
         }
         total_sec /= M;
-        std::cout<<"\x1B[35m Case with match in random pos and size: "<<size<<" for "<<loops<<" loops ("<< M <<" random possisions checked):"<<"\x1B[36m truffleExec elapsetime: \x1B[0m"<<total_sec<<"\x1B[36m bandwidth: \x1B[0m"<<(size/total_sec)<<std::endl;
+        bw /= M;
+        std::cout << "\x1B[35m Case with match in random pos and size: " << size << " for "<< loops <<" loops ("
+                  << M <<" random possisions checked): \x1B[36m truffleExec elapsetime: \x1B[0m" << total_sec 
+                  << "(μs) \x1B[36m bandwidth: \x1B[0m"<< bw << "(MB/μs)" <<std::endl;
     } else {
         auto start = std::chrono::steady_clock::now();
         for (int i = 0; i < loops; i++) {
-            truffleExec(lo, hi, (u8 *)kt1 + i, (u8 *)kt1 + strlen(kt1));
+            truffleExec(lo, hi, kt1, kt1 + size);
         }
         auto end = std::chrono::steady_clock::now();
-        std::chrono::duration<double> shuftiExec_elapsed_seconds = end-start;
-        total_sec += shuftiExec_elapsed_seconds.count();
-        std::cout<<"\x1B[35m Case with no match in random pos and size: "<<size<<" for "<<loops<<" loops:"<<"\x1B[36m truffleExec elapsetime: \x1B[0m"<<total_sec<<"\x1B[36m bandwidth: \x1B[0m"<<(size/total_sec)<<std::endl;
+        total_sec += std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+        /*average time*/
+        total_sec /= loops;
+        /*average size*/
+        size /= loops;
+        double mb_size = (double) size / 1048576;
+        bw = mb_size / total_sec;
+        std::cout<<"\x1B[35m Case with no match in random pos and size: "<< size <<" for "<< loops <<" loops:"
+                 <<"\x1B[36m truffleExec elapsetime: \x1B[0m" << total_sec << " μs \x1B[36m bandwidth: \x1B[0m"<< bw <<" (MB/μs)"<<std::endl;
     }
     delete [] kt1;
 }
@@ -55,9 +77,13 @@ void truffle_benchmarks(int size, int loops, int M, bool has_match) {
 
 void rtruffle_benchmarks(int size, int loops, int M, bool has_match) {  
     m128 lo, hi;
-    char *kt1 = new char[size];
+    ue2::CharReach chars;
+    chars.set('a');
+    truffleBuildMasks(chars, (u8 *)&lo, (u8 *)&hi);
+    u8 *kt1 = new u8[size];
     memset(kt1,'b',size);
     double total_sec = 0;
+    long double bw = 0;
     if (has_match){
         int pos = 0;
         for(int j=0; j<M; j++){
@@ -65,25 +91,42 @@ void rtruffle_benchmarks(int size, int loops, int M, bool has_match) {
             srand (time(NULL));
             pos = rand() % size + 0;
             kt1[pos] = 'a';
+            unsigned long act_size = 0;
             auto start = std::chrono::steady_clock::now();
-            for (int i = 0; i < loops/M; i++) {
-                rtruffleExec(lo, hi, (u8 *)kt1 + i, (u8 *)kt1 + strlen(kt1));
+            for (int i = 0; i < loops; i++) {
+                const u8 *res = rtruffleExec(lo, hi, kt1, kt1 + size);
+                act_size += res - kt1;
             }
             auto end = std::chrono::steady_clock::now();
-            std::chrono::duration<double> shuftiExec_elapsed_seconds = end-start;
-            total_sec += shuftiExec_elapsed_seconds.count();
+            double dt = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+            /*average time*/
+            dt /= loops;
+            total_sec += dt;
+            /*average size*/
+            act_size /= loops;
+            double mb_size = (double) act_size / 1048576;
+            bw += mb_size / dt;
         }
         total_sec /= M;
-        std::cout<<"\x1B[35m Case with match in random pos and size: "<<size<<" for "<<loops<<" loops ("<< M <<" random possisions checked):"<<"\x1B[36m rtruffleExec elapsetime: \x1B[0m"<<total_sec<<"\x1B[36m bandwidth: \x1B[0m"<<(size/total_sec)<<std::endl;
+        bw /= M;
+        std::cout<<"\x1B[35m Case with match in random pos and size: "<<size<<" for "<<loops<<" loops ("
+                 << M <<" random possisions checked):"<<"\x1B[36m rtruffleExec elapsetime: \x1B[0m" 
+                 << total_sec <<" (μs) \x1B[36m bandwidth: \x1B[0m"<< bw <<"(ΜΒ/μs)"<<std::endl;
     } else {
         auto start = std::chrono::steady_clock::now();
         for (int i = 0; i < loops; i++) {
-            rtruffleExec(lo, hi, (u8 *)kt1 + i, (u8 *)kt1 + strlen(kt1));
+            rtruffleExec(lo, hi, kt1, kt1 + size);
         }
         auto end = std::chrono::steady_clock::now();
-        std::chrono::duration<double> shuftiExec_elapsed_seconds = end-start;
-        total_sec += shuftiExec_elapsed_seconds.count();
-        std::cout<<"\x1B[35m Case with no match in random pos and size: "<<size<<" for "<<loops<<" loops:"<<"\x1B[36m rtruffleExec elapsetime: \x1B[0m"<<total_sec<<"\x1B[36m bandwidth: \x1B[0m"<<(size/total_sec)<<std::endl;
+        total_sec += std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+        /*average time*/
+        total_sec /= loops;
+        /*average size*/
+        size /=loops;
+        double mb_size = (double) size / 1048576;
+        bw = mb_size / total_sec;
+        std::cout<<"\x1B[35m Case with no match in random pos and size: "<< size <<" for "<< loops <<" loops:"
+                 <<"\x1B[36m rtruffleExec elapsetime: \x1B[0m" << total_sec <<" (μs) \x1B[36m bandwidth: \x1B[0m"<< bw <<" (MB/μs)"<<std::endl;
     }
     delete [] kt1;
 }