From 205bcbb0603d86a412c8c31307735288f78d6d6c Mon Sep 17 00:00:00 2001 From: "G.E." Date: Wed, 6 Mar 2024 10:48:56 +0100 Subject: [PATCH 01/33] support building on NetBSD --- README.md | 11 +++++++++++ cmake/osdetection.cmake | 6 +++++- examples/patbench.cc | 5 +++++ examples/pcapscan.cc | 5 +++++ src/util/popcount.h | 8 ++++++++ 5 files changed, 34 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7f7c2f53..cf2e5f40 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,17 @@ Assuming an existing HomeBrew installation: % brew install boost cmake gcc libpcap pkg-config ragel sqlite ``` +### NetBSD +In NetBSD you will almost certainly need to have a newer compiler installed. +Using the example of gcc12 from pkgsrc, one will need to set three +environment variables before starting: +export CC="/usr/pkg/gcc12/bin/cc" +export CXX="/usr/pkg/gcc12/bin/g++" +export CXXFLAGS="-gdwarf-4" + +Then continue with the build as below. + + ## Configure & build In order to configure with `cmake` first create and cd into a build directory: diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 59f2b342..0e730fe3 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -6,7 +6,11 @@ if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") -if (ARCH_IA32 OR ARCH_X86_64) +if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + set(NETBSD true) +endif(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + +if (LINUX AND (ARCH_IA32 OR ARCH_X86_64)) option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" ON) else() option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" OFF) diff --git a/examples/patbench.cc b/examples/patbench.cc index 8180d2a9..7bb5020a 100644 --- a/examples/patbench.cc +++ b/examples/patbench.cc @@ -134,7 +134,12 @@ #include #include #include +#ifdef __NetBSD__ +#include +#include +#else #include +#endif /* __NetBSD__ */ #include #include diff --git a/examples/pcapscan.cc b/examples/pcapscan.cc index 12b94438..349ef6a9 100644 --- a/examples/pcapscan.cc +++ b/examples/pcapscan.cc @@ -68,7 +68,12 @@ #include #include #include +#ifdef __NetBSD__ +#include +#include +#else #include +#endif /* __NetBSD__ */ #include #include diff --git a/src/util/popcount.h b/src/util/popcount.h index d90a0d50..e9f689e9 100644 --- a/src/util/popcount.h +++ b/src/util/popcount.h @@ -36,7 +36,12 @@ #include "ue2common.h" #include "util/arch.h" +#ifdef __NetBSD__ +#include +#endif + +#ifndef __NetBSD__ static really_inline u32 popcount32(u32 x) { return __builtin_popcount(x); @@ -51,6 +56,7 @@ u32 popcount32(u32 x) { // return (((x + (x >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24; // #endif } +#endif /* __NetBSD__ */ static really_inline u32 popcount32x4(u32 const *x) { @@ -61,6 +67,7 @@ u32 popcount32x4(u32 const *x) { return sum; } +#ifndef __NetBSD__ static really_inline u32 popcount64(u64a x) { return __builtin_popcountll(x); @@ -81,6 +88,7 @@ u32 popcount64(u64a x) { // return popcount32(x >> 32) + popcount32(x); // #endif } +#endif /* __NetBSD__ */ static really_inline u32 popcount64x4(u64a const *x) { From 8835b342215611ae3ba6ee409de6c4ca9a674da3 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Wed, 6 Mar 2024 12:31:35 +0100 Subject: [PATCH 02/33] updated readme to reflect FreeBSD build --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cf2e5f40..67124678 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Assuming an existing HomeBrew installation: % brew install boost cmake gcc libpcap pkg-config ragel sqlite ``` -### NetBSD +### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. Using the example of gcc12 from pkgsrc, one will need to set three environment variables before starting: @@ -121,6 +121,12 @@ export CC="/usr/pkg/gcc12/bin/cc" export CXX="/usr/pkg/gcc12/bin/g++" export CXXFLAGS="-gdwarf-4" +In FreeBSD similarly, using gcc12 from pkg +export CC=/usr/local/bin/gcc +export CXX=/usr/local/bin/g++ +export CXXFLAGS="-gdwarf-4" + + Then continue with the build as below. From c0784c4aa56e00b71c85132dfd505a62582528d7 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Wed, 6 Mar 2024 14:57:28 +0100 Subject: [PATCH 03/33] lets rather not add lines of code not yet used anywhere --- cmake/osdetection.cmake | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 0e730fe3..e4d0bf57 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -6,10 +6,6 @@ if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") -if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") - set(NETBSD true) -endif(CMAKE_SYSTEM_NAME MATCHES "NetBSD") - if (LINUX AND (ARCH_IA32 OR ARCH_X86_64)) option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" ON) else() From 67452eebe3aeafb2bdeef3a41168d708aa2f105b Mon Sep 17 00:00:00 2001 From: "G.E." Date: Wed, 6 Mar 2024 15:28:20 +0100 Subject: [PATCH 04/33] minor change for more certain success in netbsd --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 67124678..fb9b1bba 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,8 @@ Using the example of gcc12 from pkgsrc, one will need to set three environment variables before starting: export CC="/usr/pkg/gcc12/bin/cc" export CXX="/usr/pkg/gcc12/bin/g++" -export CXXFLAGS="-gdwarf-4" +export CXXFLAGS="-gdwarf-4 -Wno-redundant-move" + In FreeBSD similarly, using gcc12 from pkg export CC=/usr/local/bin/gcc From d40367fad8c6ae528fe0df9fc23e2c3e91d93274 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 6 Mar 2024 15:50:26 +0100 Subject: [PATCH 05/33] formatting in readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index fb9b1bba..7bf2d046 100644 --- a/README.md +++ b/README.md @@ -117,15 +117,19 @@ Assuming an existing HomeBrew installation: In NetBSD you will almost certainly need to have a newer compiler installed. Using the example of gcc12 from pkgsrc, one will need to set three environment variables before starting: +``` export CC="/usr/pkg/gcc12/bin/cc" export CXX="/usr/pkg/gcc12/bin/g++" export CXXFLAGS="-gdwarf-4 -Wno-redundant-move" +``` In FreeBSD similarly, using gcc12 from pkg +``` export CC=/usr/local/bin/gcc export CXX=/usr/local/bin/g++ export CXXFLAGS="-gdwarf-4" +``` Then continue with the build as below. From 739dd200ac81535d71e0707a2716e141b8688c5d Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 6 Mar 2024 15:53:03 +0100 Subject: [PATCH 06/33] more readme format tinkering --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7bf2d046..2009d220 100644 --- a/README.md +++ b/README.md @@ -126,8 +126,8 @@ export CXXFLAGS="-gdwarf-4 -Wno-redundant-move" In FreeBSD similarly, using gcc12 from pkg ``` -export CC=/usr/local/bin/gcc -export CXX=/usr/local/bin/g++ +export CC="/usr/local/bin/gcc" +export CXX="/usr/local/bin/g++" export CXXFLAGS="-gdwarf-4" ``` From 2e897ad4a8f702ccdbfd74331a83c67abe4659d3 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 6 Mar 2024 16:39:34 +0100 Subject: [PATCH 07/33] lets not disable warnings --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2009d220..0fb15607 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ environment variables before starting: ``` export CC="/usr/pkg/gcc12/bin/cc" export CXX="/usr/pkg/gcc12/bin/g++" -export CXXFLAGS="-gdwarf-4 -Wno-redundant-move" +export CXXFLAGS="-gdwarf-4" ``` From 642b0e92fa43caa23b1fe6646e280d3187118100 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 6 Mar 2024 17:05:55 +0100 Subject: [PATCH 08/33] maybe netbsd is more pedantic about this? --- tools/hscollider/UltimateTruth.cpp | 2 +- tools/hscollider/main.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/hscollider/UltimateTruth.cpp b/tools/hscollider/UltimateTruth.cpp index 93d432c3..e33ce2fb 100644 --- a/tools/hscollider/UltimateTruth.cpp +++ b/tools/hscollider/UltimateTruth.cpp @@ -1079,7 +1079,7 @@ shared_ptr UltimateTruth::compile(const set &ids, } } - return std::move(db); + return db; } bool UltimateTruth::allocScratch(shared_ptr db) { diff --git a/tools/hscollider/main.cpp b/tools/hscollider/main.cpp index dcc5c1b6..0decc810 100644 --- a/tools/hscollider/main.cpp +++ b/tools/hscollider/main.cpp @@ -1830,11 +1830,11 @@ unique_ptr buildCorpora(const vector &corporaFiles, exit_with_fail(); } } - return std::move(c); /* move allows unique_ptr conversion */ + return c; } else { auto c = std::make_unique( exprMap, corpus_gen_prop, force_utf8, force_prefilter); - return std::move(c); + return c; } } From 33444e7ce060d5a9b3b60d9b10e0ba61c22ac1e9 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Tue, 12 Mar 2024 12:37:08 +0200 Subject: [PATCH 09/33] incremental improvement in cleanliness --- cmake/cflags-generic.cmake | 4 ++++ cmake/osdetection.cmake | 4 ++++ src/util/arch.h | 4 ++++ src/util/popcount.h | 12 ++++-------- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/cmake/cflags-generic.cmake b/cmake/cflags-generic.cmake index 29b746a1..93fa789d 100644 --- a/cmake/cflags-generic.cmake +++ b/cmake/cflags-generic.cmake @@ -27,6 +27,10 @@ CHECK_INCLUDE_FILES(unistd.h HAVE_UNISTD_H) CHECK_FUNCTION_EXISTS(posix_memalign HAVE_POSIX_MEMALIGN) CHECK_FUNCTION_EXISTS(_aligned_malloc HAVE__ALIGNED_MALLOC) +if(FREEBSD OR NETBSD) + set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -gdwarf-4") +endif() + # these end up in the config file CHECK_C_COMPILER_FLAG(-fvisibility=hidden HAS_C_HIDDEN) CHECK_CXX_COMPILER_FLAG(-fvisibility=hidden HAS_CXX_HIDDEN) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index e4d0bf57..0e730fe3 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -6,6 +6,10 @@ if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") +if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + set(NETBSD true) +endif(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + if (LINUX AND (ARCH_IA32 OR ARCH_X86_64)) option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" ON) else() diff --git a/src/util/arch.h b/src/util/arch.h index 1e8d2fbd..9a9dde1b 100644 --- a/src/util/arch.h +++ b/src/util/arch.h @@ -42,6 +42,10 @@ #elif defined(ARCH_PPC64EL) #include "util/arch/ppc64el/ppc64el.h" #endif +#ifdef __NetBSD__ +#include +#define HAVE_LOCAL_POPCOUNT +#endif #endif // UTIL_ARCH_X86_H_ diff --git a/src/util/popcount.h b/src/util/popcount.h index e9f689e9..a02e3bc5 100644 --- a/src/util/popcount.h +++ b/src/util/popcount.h @@ -36,12 +36,8 @@ #include "ue2common.h" #include "util/arch.h" -#ifdef __NetBSD__ -#include -#endif - -#ifndef __NetBSD__ +#ifndef HAVE_LOCAL_POPCOUNT static really_inline u32 popcount32(u32 x) { return __builtin_popcount(x); @@ -56,7 +52,7 @@ u32 popcount32(u32 x) { // return (((x + (x >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24; // #endif } -#endif /* __NetBSD__ */ +#endif /* HAVE_LOCAL_POPCOUNT */ static really_inline u32 popcount32x4(u32 const *x) { @@ -67,7 +63,7 @@ u32 popcount32x4(u32 const *x) { return sum; } -#ifndef __NetBSD__ +#ifndef HAVE_LOCAL_POPCOUNT static really_inline u32 popcount64(u64a x) { return __builtin_popcountll(x); @@ -88,7 +84,7 @@ u32 popcount64(u64a x) { // return popcount32(x >> 32) + popcount32(x); // #endif } -#endif /* __NetBSD__ */ +#endif /* HAVE_LOCAL_POPCOUNT */ static really_inline u32 popcount64x4(u64a const *x) { From 9591ae34e50c0ddd2801dd4316984b8ebae3d270 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Tue, 12 Mar 2024 12:59:16 +0200 Subject: [PATCH 10/33] another small cleanup in readme --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 0fb15607..5dafd0ea 100644 --- a/README.md +++ b/README.md @@ -115,12 +115,11 @@ Assuming an existing HomeBrew installation: ### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. -Using the example of gcc12 from pkgsrc, one will need to set three +Using the example of gcc12 from pkgsrc, one will need to set two environment variables before starting: ``` export CC="/usr/pkg/gcc12/bin/cc" export CXX="/usr/pkg/gcc12/bin/g++" -export CXXFLAGS="-gdwarf-4" ``` @@ -128,7 +127,6 @@ In FreeBSD similarly, using gcc12 from pkg ``` export CC="/usr/local/bin/gcc" export CXX="/usr/local/bin/g++" -export CXXFLAGS="-gdwarf-4" ``` From feeb6a57ac0d94ae093fc4cdc8c41bd7072672d7 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Tue, 12 Mar 2024 14:22:39 +0200 Subject: [PATCH 11/33] moved HAVE_BUILTIN_POPCOUNT def to cmake --- README.md | 8 +++++++- cmake/cflags-generic.cmake | 5 +++++ src/util/arch.h | 1 - src/util/popcount.h | 8 ++++---- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5dafd0ea..2ecc0961 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,8 @@ Assuming an existing HomeBrew installation: ### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. +This will not replace the one in the standard base distribution, and +cmake will probably find the wrong compiler when it checks automatically. Using the example of gcc12 from pkgsrc, one will need to set two environment variables before starting: ``` @@ -123,7 +125,11 @@ export CXX="/usr/pkg/gcc12/bin/g++" ``` -In FreeBSD similarly, using gcc12 from pkg +In FreeBSD similarly, if you install another compiler, cmake might not +find the right one. Worse is if it finds, say, g++ and configures it for +C++ but leaves cc for C - FreeBSD's default cc is clang, and the binaries +generated between g++ and clang will not play nicely together. +Using the example of gcc12 from pkg: ``` export CC="/usr/local/bin/gcc" export CXX="/usr/local/bin/g++" diff --git a/cmake/cflags-generic.cmake b/cmake/cflags-generic.cmake index 93fa789d..35bbe9f4 100644 --- a/cmake/cflags-generic.cmake +++ b/cmake/cflags-generic.cmake @@ -31,6 +31,11 @@ if(FREEBSD OR NETBSD) set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -gdwarf-4") endif() +if(NETBSD) + set(EXTRA_C_FLAGS "${EXTRA_CXX_FLAGS} -DHAVE_BUILTIN_POPCOUNT") + set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -DHAVE_BUILTIN_POPCOUNT") +endif() + # these end up in the config file CHECK_C_COMPILER_FLAG(-fvisibility=hidden HAS_C_HIDDEN) CHECK_CXX_COMPILER_FLAG(-fvisibility=hidden HAS_CXX_HIDDEN) diff --git a/src/util/arch.h b/src/util/arch.h index 9a9dde1b..1963b405 100644 --- a/src/util/arch.h +++ b/src/util/arch.h @@ -44,7 +44,6 @@ #endif #ifdef __NetBSD__ #include -#define HAVE_LOCAL_POPCOUNT #endif #endif // UTIL_ARCH_X86_H_ diff --git a/src/util/popcount.h b/src/util/popcount.h index a02e3bc5..34f386b7 100644 --- a/src/util/popcount.h +++ b/src/util/popcount.h @@ -37,7 +37,7 @@ #include "ue2common.h" #include "util/arch.h" -#ifndef HAVE_LOCAL_POPCOUNT +#ifndef HAVE_BUILTIN_POPCOUNT static really_inline u32 popcount32(u32 x) { return __builtin_popcount(x); @@ -52,7 +52,7 @@ u32 popcount32(u32 x) { // return (((x + (x >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24; // #endif } -#endif /* HAVE_LOCAL_POPCOUNT */ +#endif /* HAVE_BUILTIN_POPCOUNT */ static really_inline u32 popcount32x4(u32 const *x) { @@ -63,7 +63,7 @@ u32 popcount32x4(u32 const *x) { return sum; } -#ifndef HAVE_LOCAL_POPCOUNT +#ifndef HAVE_BUILTIN_POPCOUNT static really_inline u32 popcount64(u64a x) { return __builtin_popcountll(x); @@ -84,7 +84,7 @@ u32 popcount64(u64a x) { // return popcount32(x >> 32) + popcount32(x); // #endif } -#endif /* HAVE_LOCAL_POPCOUNT */ +#endif /* HAVE_BUILTIN_POPCOUNT */ static really_inline u32 popcount64x4(u64a const *x) { From 10e96c3f53b6ac009e58e454a8a91a084b4dddf9 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Tue, 12 Mar 2024 13:58:50 +0100 Subject: [PATCH 12/33] typo fix --- cmake/cflags-generic.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cflags-generic.cmake b/cmake/cflags-generic.cmake index 35bbe9f4..6bc60502 100644 --- a/cmake/cflags-generic.cmake +++ b/cmake/cflags-generic.cmake @@ -32,7 +32,7 @@ if(FREEBSD OR NETBSD) endif() if(NETBSD) - set(EXTRA_C_FLAGS "${EXTRA_CXX_FLAGS} -DHAVE_BUILTIN_POPCOUNT") + set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} -DHAVE_BUILTIN_POPCOUNT") set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -DHAVE_BUILTIN_POPCOUNT") endif() From f64239261092641936084d107ffac512d4e3d056 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Tue, 12 Mar 2024 16:50:57 +0200 Subject: [PATCH 13/33] shortened freebsd text --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 2ecc0961..e82517f0 100644 --- a/README.md +++ b/README.md @@ -125,10 +125,7 @@ export CXX="/usr/pkg/gcc12/bin/g++" ``` -In FreeBSD similarly, if you install another compiler, cmake might not -find the right one. Worse is if it finds, say, g++ and configures it for -C++ but leaves cc for C - FreeBSD's default cc is clang, and the binaries -generated between g++ and clang will not play nicely together. +In FreeBSD similarly, you might want to install a different compiler. Using the example of gcc12 from pkg: ``` export CC="/usr/local/bin/gcc" From 24db99586c375fc9b83cf47c8b0a6fa8f44fb94d Mon Sep 17 00:00:00 2001 From: "G.E." Date: Wed, 13 Mar 2024 17:33:23 +0200 Subject: [PATCH 14/33] whitespace editing in readme --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index e82517f0..764f5e8c 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,6 @@ export CC="/usr/pkg/gcc12/bin/cc" export CXX="/usr/pkg/gcc12/bin/g++" ``` - In FreeBSD similarly, you might want to install a different compiler. Using the example of gcc12 from pkg: ``` @@ -132,7 +131,6 @@ export CC="/usr/local/bin/gcc" export CXX="/usr/local/bin/g++" ``` - Then continue with the build as below. From 5468d0c492440ca9334e5d16ca192cff6c8b9f5f Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 27 Mar 2024 16:31:02 +0200 Subject: [PATCH 15/33] more verbose instructions for preparing BSD systems. --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 764f5e8c..717888e0 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,15 @@ Assuming an existing HomeBrew installation: ### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. +When using pkgsrc, you would typically do this using something +similar to +``` +pkg_add gcc12-12.3.0.tgz +``` +Version numbers etc will of course vary. One would either download the +binary package or build it using pkgsrc. +See https://www.netbsd.org/docs/software/packages.html for more information. + This will not replace the one in the standard base distribution, and cmake will probably find the wrong compiler when it checks automatically. Using the example of gcc12 from pkgsrc, one will need to set two @@ -126,6 +135,12 @@ export CXX="/usr/pkg/gcc12/bin/g++" In FreeBSD similarly, you might want to install a different compiler. Using the example of gcc12 from pkg: +installing the desired compiler: +``` +pkg install gcc12 +``` +and then before beginning the cmake and build process, set +the environment variables to point to this compiler: ``` export CC="/usr/local/bin/gcc" export CXX="/usr/local/bin/g++" From 6e8cef574f5a07ca40daf302622f24dec7f6612b Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 27 Mar 2024 16:35:21 +0200 Subject: [PATCH 16/33] also added note for CC/CXX vars in fbsd/ppc which are different. --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 717888e0..f1622030 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,15 @@ export CC="/usr/local/bin/gcc" export CXX="/usr/local/bin/g++" ``` +A further note in FreeBSD, on the PowerPC platform, for some reason, +the gcc12 package installs to a slightly different name, on FreeBSD/ppc, +gcc12 will be found using: +``` +export CC="/usr/local/bin/gcc12" +export CXX="/usr/local/bin/g++12" +``` + + Then continue with the build as below. From c6b5f1cff64324f8d2118aed89008adb154e4e72 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 27 Mar 2024 17:19:56 +0200 Subject: [PATCH 17/33] more system prep info for bsd --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f1622030..5edc6376 100644 --- a/README.md +++ b/README.md @@ -115,13 +115,17 @@ Assuming an existing HomeBrew installation: ### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. +Also you will need to install cmake, boost and ragel. When using pkgsrc, you would typically do this using something similar to ``` pkg_add gcc12-12.3.0.tgz +pkg_add boost-headers-1.83.0.tgz boost-jam-1.83.0.tgz boost-libs-1.83.0nb1.tgz +pkg_add ragel-6.10.tgz +pkg_add cmake-3.28.1.tgz ``` Version numbers etc will of course vary. One would either download the -binary package or build it using pkgsrc. +binary packages or build them using pkgsrc. See https://www.netbsd.org/docs/software/packages.html for more information. This will not replace the one in the standard base distribution, and @@ -134,10 +138,14 @@ export CXX="/usr/pkg/gcc12/bin/g++" ``` In FreeBSD similarly, you might want to install a different compiler. +You will also, as in NetBSD, need to install cmake, boost and ragel packages. Using the example of gcc12 from pkg: installing the desired compiler: ``` pkg install gcc12 +pkg install boost-all +pkg install ragel +pkg install cmake ``` and then before beginning the cmake and build process, set the environment variables to point to this compiler: From 18fd2f44593050454960d899e76f0fd2655d6e3b Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 27 Mar 2024 17:24:08 +0200 Subject: [PATCH 18/33] some more bsd detail --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5edc6376..400bc025 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,11 @@ pkg_add ragel-6.10.tgz pkg_add cmake-3.28.1.tgz ``` Version numbers etc will of course vary. One would either download the -binary packages or build them using pkgsrc. +binary packages or build them using pkgsrc. There exist some NetBSD pkg +tools like ```pkgin``` which help download e.g. dependencies as binary packages, +but overall NetBSD leaves a lot of detail exposed to the user. +The main package system used in NetBSD is pkgsrc and one will probably +want to read up more about it than is in the scope of this document. See https://www.netbsd.org/docs/software/packages.html for more information. This will not replace the one in the standard base distribution, and From d8f50478fca4c788013cee56b4adfafc598d10d0 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 27 Mar 2024 17:28:24 +0200 Subject: [PATCH 19/33] also sqlite info for bsd --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 400bc025..fca411ff 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Assuming an existing HomeBrew installation: ### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. -Also you will need to install cmake, boost and ragel. +Also you will need to install cmake, sqlite, boost and ragel. When using pkgsrc, you would typically do this using something similar to ``` @@ -123,6 +123,7 @@ pkg_add gcc12-12.3.0.tgz pkg_add boost-headers-1.83.0.tgz boost-jam-1.83.0.tgz boost-libs-1.83.0nb1.tgz pkg_add ragel-6.10.tgz pkg_add cmake-3.28.1.tgz +pkg_add sqlite3-3.44.2.tgz ``` Version numbers etc will of course vary. One would either download the binary packages or build them using pkgsrc. There exist some NetBSD pkg @@ -142,7 +143,7 @@ export CXX="/usr/pkg/gcc12/bin/g++" ``` In FreeBSD similarly, you might want to install a different compiler. -You will also, as in NetBSD, need to install cmake, boost and ragel packages. +You will also, as in NetBSD, need to install cmake, sqlite, boost and ragel packages. Using the example of gcc12 from pkg: installing the desired compiler: ``` @@ -150,6 +151,7 @@ pkg install gcc12 pkg install boost-all pkg install ragel pkg install cmake +pkg install sqlite ``` and then before beginning the cmake and build process, set the environment variables to point to this compiler: From e167f4a8b2f8a9de976082fa5d234df278e7d2c4 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 27 Mar 2024 17:33:25 +0200 Subject: [PATCH 20/33] slightly clearer comments in netbsd section --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fca411ff..e6965710 100644 --- a/README.md +++ b/README.md @@ -133,8 +133,8 @@ The main package system used in NetBSD is pkgsrc and one will probably want to read up more about it than is in the scope of this document. See https://www.netbsd.org/docs/software/packages.html for more information. -This will not replace the one in the standard base distribution, and -cmake will probably find the wrong compiler when it checks automatically. +This will not replace the compiler in the standard base distribution, and +cmake will probably find the base dist's compiler when it checks automatically. Using the example of gcc12 from pkgsrc, one will need to set two environment variables before starting: ``` From b054e6c4881cfec1778d7cc2b345a744be8cc190 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Thu, 11 Apr 2024 09:49:43 +0300 Subject: [PATCH 21/33] added copyright info for modified src files --- examples/patbench.cc | 1 + examples/pcapscan.cc | 1 + src/util/arch.h | 1 + src/util/popcount.h | 2 +- tools/hscollider/UltimateTruth.cpp | 1 + tools/hscollider/main.cpp | 1 + 6 files changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/patbench.cc b/examples/patbench.cc index 7bb5020a..1f965f13 100644 --- a/examples/patbench.cc +++ b/examples/patbench.cc @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/examples/pcapscan.cc b/examples/pcapscan.cc index 349ef6a9..bd5493a4 100644 --- a/examples/pcapscan.cc +++ b/examples/pcapscan.cc @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/util/arch.h b/src/util/arch.h index 1963b405..ed762797 100644 --- a/src/util/arch.h +++ b/src/util/arch.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2017-2020, Intel Corporation + * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/util/popcount.h b/src/util/popcount.h index 34f386b7..bca787db 100644 --- a/src/util/popcount.h +++ b/src/util/popcount.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation - * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2020-2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/tools/hscollider/UltimateTruth.cpp b/tools/hscollider/UltimateTruth.cpp index e33ce2fb..6b933384 100644 --- a/tools/hscollider/UltimateTruth.cpp +++ b/tools/hscollider/UltimateTruth.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2019, Intel Corporation + * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/tools/hscollider/main.cpp b/tools/hscollider/main.cpp index 0decc810..3d4312cd 100644 --- a/tools/hscollider/main.cpp +++ b/tools/hscollider/main.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2019, Intel Corporation + * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: From abe954acdd2f46f799c5ae1eda948e7d60352379 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Thu, 11 Apr 2024 15:06:18 +0300 Subject: [PATCH 22/33] adding libpcap to the bsd packages to install for building/running --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index e6965710..2f5da86e 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,8 @@ Assuming an existing HomeBrew installation: ### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. Also you will need to install cmake, sqlite, boost and ragel. +Also, libpcap is necessary for some of the benchmarks, so let's install that +as well. When using pkgsrc, you would typically do this using something similar to ``` @@ -124,6 +126,7 @@ pkg_add boost-headers-1.83.0.tgz boost-jam-1.83.0.tgz boost-libs-1.83.0nb1 pkg_add ragel-6.10.tgz pkg_add cmake-3.28.1.tgz pkg_add sqlite3-3.44.2.tgz +pkg_add libpcap-1.10.4.tgz ``` Version numbers etc will of course vary. One would either download the binary packages or build them using pkgsrc. There exist some NetBSD pkg @@ -152,6 +155,7 @@ pkg install boost-all pkg install ragel pkg install cmake pkg install sqlite +pkg install libpcap ``` and then before beginning the cmake and build process, set the environment variables to point to this compiler: From d381c12fcfe20ce439d021d866423245df6ec5f7 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Thu, 11 Apr 2024 17:47:16 +0300 Subject: [PATCH 23/33] added ccache for freebsd install recommendations --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2f5da86e..1baeaac8 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,7 @@ pkg install ragel pkg install cmake pkg install sqlite pkg install libpcap +pkg install ccache ``` and then before beginning the cmake and build process, set the environment variables to point to this compiler: From aea40433fce4349bc83f6f012d9ce28701bda4a0 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Mon, 15 Apr 2024 11:07:45 +0300 Subject: [PATCH 24/33] tiny edit to readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1baeaac8..f6a10fca 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ export CC="/usr/local/bin/gcc" export CXX="/usr/local/bin/g++" ``` -A further note in FreeBSD, on the PowerPC platform, for some reason, +A further note in FreeBSD, on the PowerPC and ARM platforms, the gcc12 package installs to a slightly different name, on FreeBSD/ppc, gcc12 will be found using: ``` From f872c289bd6f98c067444fb93ad0c03154b4a1b3 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Mon, 15 Apr 2024 11:44:22 +0300 Subject: [PATCH 25/33] one more place to fix where clang in bsd is more picky than gcc in linux --- src/rose/rose_build_program.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rose/rose_build_program.cpp b/src/rose/rose_build_program.cpp index 1e0fe24b..861855b5 100644 --- a/src/rose/rose_build_program.cpp +++ b/src/rose/rose_build_program.cpp @@ -1161,7 +1161,7 @@ static really_inline void nibUpdate(map &nib, u32 hi_lo) { u16 hi = hi_lo >> 16; u16 lo = hi_lo & 0xffff; - for (const auto pairs : nib) { + for (const auto &pairs : nib) { u32 old = pairs.first; if ((old >> 16) == hi || (old & 0xffff) == lo) { if (!nib[old | hi_lo]) { From 1ef8e5dddcfdf7e72024add842c323945f6d4d63 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Mon, 15 Apr 2024 11:52:33 +0300 Subject: [PATCH 26/33] first try at getting cmake to leave custom shared lib paths with the binary, with package added compilers we need to find the right std libs from the compiler we added, not the base install libs. --- cmake/osdetection.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 0e730fe3..1f35f1d3 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -4,6 +4,7 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") From b4115b821be1970c88dbfcf3df28c5050d3c0e23 Mon Sep 17 00:00:00 2001 From: Gregory Economou Date: Mon, 15 Apr 2024 14:59:08 +0300 Subject: [PATCH 27/33] united the static fat runtime dispatcher with the BSD support. --- README.md | 7 +- cmake/build_wrapper.sh | 12 +- cmake/osdetection.cmake | 35 +++--- src/dispatcher.c | 249 +++++++++++++++++++++++++++++++++------- 4 files changed, 240 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index f6a10fca..2e68d2e6 100644 --- a/README.md +++ b/README.md @@ -173,9 +173,14 @@ export CC="/usr/local/bin/gcc12" export CXX="/usr/local/bin/g++12" ``` - Then continue with the build as below. +A note about running in FreeBSD: if you built a dynamically linked binary +with an alternative compiler, the libraries specific to the compiler that +built the binary will probably not be found and the base distro libraries +in /lib will be found instead. Adjust LD_LIBRARY_PATH appropriately. For +example, with gcc12 installed from pkg, one would want to use +```export LD_LIBRARY_PATH=/usr/local/lib/gcc12/``` ## Configure & build diff --git a/cmake/build_wrapper.sh b/cmake/build_wrapper.sh index 895610c0..d0a7e923 100755 --- a/cmake/build_wrapper.sh +++ b/cmake/build_wrapper.sh @@ -15,13 +15,21 @@ SYMSFILE=$(mktemp -p /tmp ${PREFIX}_rename.syms.XXXXX) KEEPSYMS=$(mktemp -p /tmp keep.syms.XXXXX) # find the libc used by gcc LIBC_SO=$("$@" --print-file-name=libc.so.6) +NM_FLAG="-f" +if [ `uname` = "FreeBSD" ]; then + # for freebsd, we will specify the name, + # we will leave it work as is in linux + LIBC_SO=/lib/libc.so.7 + # also, in BSD, the nm flag -F corresponds to the -f flag in linux. + NM_FLAG="-F" +fi cp ${KEEPSYMS_IN} ${KEEPSYMS} # get all symbols from libc and turn them into patterns -nm -f p -g -D ${LIBC_SO} | sed -s 's/\([^ @]*\).*/^\1$/' >> ${KEEPSYMS} +nm ${NM_FLAG} p -g -D ${LIBC_SO} | sed 's/\([^ @]*\).*/^\1$/' >> ${KEEPSYMS} # build the object "$@" # rename the symbols in the object -nm -f p -g ${OUT} | cut -f1 -d' ' | grep -v -f ${KEEPSYMS} | sed -e "s/\(.*\)/\1\ ${PREFIX}_\1/" >> ${SYMSFILE} +nm ${NM_FLAG} p -g ${OUT} | cut -f1 -d' ' | grep -v -f ${KEEPSYMS} | sed -e "s/\(.*\)/\1\ ${PREFIX}_\1/" >> ${SYMSFILE} if test -s ${SYMSFILE} then objcopy --redefine-syms=${SYMSFILE} ${OUT} diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 1f35f1d3..6e7178cb 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -4,14 +4,13 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) - set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") set(NETBSD true) endif(CMAKE_SYSTEM_NAME MATCHES "NetBSD") -if (LINUX AND (ARCH_IA32 OR ARCH_X86_64)) +if (ARCH_IA32 OR ARCH_X86_64) option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" ON) else() option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" OFF) @@ -19,26 +18,22 @@ endif() if (FAT_RUNTIME) message("Checking Fat Runtime Requirements...") - if (NOT LINUX) - message(FATAL_ERROR "Fat runtime is only supported on Linux OS") - else() - if (USE_CPU_NATIVE AND FAT_RUNTIME) - message(FATAL_ERROR "Fat runtime is not compatible with Native CPU detection") - endif() + if (USE_CPU_NATIVE AND FAT_RUNTIME) + message(FATAL_ERROR "Fat runtime is not compatible with Native CPU detection") + endif() - if (NOT (ARCH_IA32 OR ARCH_X86_64 OR ARCH_AARCH64)) - message(FATAL_ERROR "Fat runtime is only supported on Intel and Aarch64 architectures") + if (NOT (ARCH_IA32 OR ARCH_X86_64 OR ARCH_AARCH64)) + message(FATAL_ERROR "Fat runtime is only supported on Intel and Aarch64 architectures") + else() + message(STATUS "Building Fat runtime for multiple microarchitectures") + message(STATUS "generator is ${CMAKE_GENERATOR}") + if (NOT (CMAKE_GENERATOR MATCHES "Unix Makefiles" OR + (CMAKE_VERSION VERSION_GREATER "3.0" AND CMAKE_GENERATOR MATCHES "Ninja"))) + message (FATAL_ERROR "Building the fat runtime requires the Unix Makefiles generator, or Ninja with CMake v3.0 or higher") else() - message(STATUS "Building Fat runtime for multiple microarchitectures") - message(STATUS "generator is ${CMAKE_GENERATOR}") - if (NOT (CMAKE_GENERATOR MATCHES "Unix Makefiles" OR - (CMAKE_VERSION VERSION_GREATER "3.0" AND CMAKE_GENERATOR MATCHES "Ninja"))) - message (FATAL_ERROR "Building the fat runtime requires the Unix Makefiles generator, or Ninja with CMake v3.0 or higher") - else() - include (${CMAKE_MODULE_PATH}/attrib.cmake) - if (NOT HAS_C_ATTR_IFUNC) - message(FATAL_ERROR "Compiler does not support ifunc attribute, cannot build fat runtime") - endif() + include (${CMAKE_MODULE_PATH}/attrib.cmake) + if (NOT HAS_C_ATTR_IFUNC) + message(FATAL_ERROR "Compiler does not support ifunc attribute, cannot build fat runtime") endif() endif() endif() diff --git a/src/dispatcher.c b/src/dispatcher.c index a817e744..b704b0a5 100644 --- a/src/dispatcher.c +++ b/src/dispatcher.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2016-2020, Intel Corporation + * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -30,6 +31,39 @@ #include "hs_common.h" #include "hs_runtime.h" #include "ue2common.h" + +/* streamlining the dispatch to eliminate runtime checking/branching: + * what we want to do is, first call to the function will run the resolve + * code and set the static resolved/dispatch pointer to point to the + * correct function. Subsequent calls to the function will go directly to + * the resolved ptr. The simplest way to accomplish this is, to + * initially set the pointer to the resolve function. + * To accomplish this in a manner invisible to the user, + * we do involve some rather ugly/confusing macros in here. + * There are four macros that assemble the code for each function + * we want to dispatch in this manner: + * CREATE_DISPATCH + * this generates the declarations for the candidate target functions, + * for the fat_dispatch function pointer, for the resolve_ function, + * points the function pointer to the resolve function, and contains + * most of the definition of the resolve function. The very end of the + * resolve function is completed by the next macro, because in the + * CREATE_DISPATCH macro we have the argument list with the arg declarations, + * which is needed to generate correct function signatures, but we + * can't generate from this, in a macro, a _call_ to one of those functions. + * CONNECT_ARGS_1 + * this macro fills in the actual call at the end of the resolve function, + * with the correct arg list. hence the name connect args. + * CONNECT_DISPATCH_2 + * this macro likewise gives up the beginning of the definition of the + * actual entry point function (the 'real name' that's called by the user) + * but again in the pass-through call, cannot invoke the target without + * getting the arg list , which is supplied by the final macro, + * CONNECT_ARGS_3 + * + */ + + #if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/arch/x86/cpuid_inline.h" #include "util/join.h" @@ -57,30 +91,38 @@ return (RTYPE)HS_ARCH_ERROR; \ } \ \ - /* resolver */ \ - static RTYPE (*JOIN(resolve_, NAME)(void))(__VA_ARGS__) { \ - if (check_avx512vbmi()) { \ - return JOIN(avx512vbmi_, NAME); \ - } \ - if (check_avx512()) { \ - return JOIN(avx512_, NAME); \ - } \ - if (check_avx2()) { \ - return JOIN(avx2_, NAME); \ - } \ - if (check_sse42() && check_popcnt()) { \ - return JOIN(corei7_, NAME); \ - } \ - if (check_ssse3()) { \ - return JOIN(core2_, NAME); \ - } \ - /* anything else is fail */ \ - return JOIN(error_, NAME); \ - } \ + /* dispatch routing pointer for this function */ \ + /* initially point it at the resolve function */ \ + static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__); \ + static RTYPE (* JOIN(fat_dispatch_, NAME))(__VA_ARGS__) = \ + &JOIN(resolve_, NAME); \ \ - /* function */ \ - HS_PUBLIC_API \ - RTYPE NAME(__VA_ARGS__) __attribute__((ifunc("resolve_" #NAME))) + /* resolver */ \ + static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__) { \ + if (check_avx512vbmi()) { \ + fat_dispatch_ ## NAME = &JOIN(avx512vbmi_, NAME); \ + } \ + else if (check_avx512()) { \ + fat_dispatch_ ## NAME = &JOIN(avx512_, NAME); \ + } \ + else if (check_avx2()) { \ + fat_dispatch_ ## NAME = &JOIN(avx2_, NAME); \ + } \ + else if (check_sse42() && check_popcnt()) { \ + fat_dispatch_ ## NAME = &JOIN(corei7_, NAME); \ + } \ + else if (check_ssse3()) { \ + fat_dispatch_ ## NAME = &JOIN(core2_, NAME); \ + } else { \ + /* anything else is fail */ \ + fat_dispatch_ ## NAME = &JOIN(error_, NAME); \ + } \ + + + +/* the rest of the function is completed in the CONNECT_ARGS_1 macro. */ + + #elif defined(ARCH_AARCH64) #include "util/arch/arm/cpuid_inline.h" @@ -97,99 +139,226 @@ return (RTYPE)HS_ARCH_ERROR; \ } \ \ - /* resolver */ \ - static RTYPE (*JOIN(resolve_, NAME)(void))(__VA_ARGS__) { \ - if (check_sve2()) { \ - return JOIN(sve2_, NAME); \ - } \ - if (check_sve()) { \ - return JOIN(sve_, NAME); \ - } \ - if (check_neon()) { \ - return JOIN(neon_, NAME); \ - } \ - /* anything else is fail */ \ - return JOIN(error_, NAME); \ - } \ + /* dispatch routing pointer for this function */ \ + /* initially point it at the resolve function */ \ + static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__); \ + static RTYPE (* JOIN(fat_dispatch_, NAME))(__VA_ARGS__) = \ + &JOIN(resolve_, NAME); \ \ - /* function */ \ - HS_PUBLIC_API \ - RTYPE NAME(__VA_ARGS__) __attribute__((ifunc("resolve_" #NAME))) + /* resolver */ \ + static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__) { \ + if (check_sve2()) { \ + fat_dispatch_ ## NAME = &JOIN(sve2_, NAME); \ + } \ + else if (check_sve()) { \ + fat_dispatch_ ## NAME = &JOIN(sve_, NAME); \ + } \ + else if (check_neon()) { \ + fat_dispatch_ ## NAME = &JOIN(neon_, NAME); \ + } else { \ + /* anything else is fail */ \ + fat_dispatch_ ## NAME = &JOIN(error_, NAME); \ + } \ + + +/* the rest of the function is completed in the CONNECT_ARGS_1 macro. */ + #endif + +#define CONNECT_ARGS_1(RTYPE, NAME, ...) \ + return (*fat_dispatch_ ## NAME)(__VA_ARGS__); \ + } \ + + +#define CONNECT_DISPATCH_2(RTYPE, NAME, ...) \ + /* new function */ \ + HS_PUBLIC_API \ + RTYPE NAME(__VA_ARGS__) { \ + + +#define CONNECT_ARGS_3(RTYPE, NAME, ...) \ + return (*fat_dispatch_ ## NAME)(__VA_ARGS__); \ + } \ + + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" + +/* this gets a bit ugly to compose the static redirect functions, + * as we necessarily need first the typed arg list and then just the arg + * names, twice in a row, to define the redirect function and the + * dispatch function call */ + CREATE_DISPATCH(hs_error_t, hs_scan, const hs_database_t *db, const char *data, unsigned length, unsigned flags, hs_scratch_t *scratch, match_event_handler onEvent, void *userCtx); +CONNECT_ARGS_1(hs_error_t, hs_scan, db, data, length, flags, scratch, onEvent, userCtx); +CONNECT_DISPATCH_2(hs_error_t, hs_scan, const hs_database_t *db, const char *data, + unsigned length, unsigned flags, hs_scratch_t *scratch, + match_event_handler onEvent, void *userCtx); +CONNECT_ARGS_3(hs_error_t, hs_scan, db, data, length, flags, scratch, onEvent, userCtx); CREATE_DISPATCH(hs_error_t, hs_stream_size, const hs_database_t *database, size_t *stream_size); +CONNECT_ARGS_1(hs_error_t, hs_stream_size, database, stream_size); +CONNECT_DISPATCH_2(hs_error_t, hs_stream_size, const hs_database_t *database, + size_t *stream_size); +CONNECT_ARGS_3(hs_error_t, hs_stream_size, database, stream_size); CREATE_DISPATCH(hs_error_t, hs_database_size, const hs_database_t *db, size_t *size); +CONNECT_ARGS_1(hs_error_t, hs_database_size, db, size); +CONNECT_DISPATCH_2(hs_error_t, hs_database_size, const hs_database_t *db, + size_t *size); +CONNECT_ARGS_3(hs_error_t, hs_database_size, db, size); + CREATE_DISPATCH(hs_error_t, dbIsValid, const hs_database_t *db); +CONNECT_ARGS_1(hs_error_t, dbIsValid, db); +CONNECT_DISPATCH_2(hs_error_t, dbIsValid, const hs_database_t *db); +CONNECT_ARGS_3(hs_error_t, dbIsValid, db); + CREATE_DISPATCH(hs_error_t, hs_free_database, hs_database_t *db); +CONNECT_ARGS_1(hs_error_t, hs_free_database, db); +CONNECT_DISPATCH_2(hs_error_t, hs_free_database, hs_database_t *db); +CONNECT_ARGS_3(hs_error_t, hs_free_database, db); CREATE_DISPATCH(hs_error_t, hs_open_stream, const hs_database_t *db, unsigned int flags, hs_stream_t **stream); +CONNECT_ARGS_1(hs_error_t, hs_open_stream, db, flags, stream); +CONNECT_DISPATCH_2(hs_error_t, hs_open_stream, const hs_database_t *db, + unsigned int flags, hs_stream_t **stream); +CONNECT_ARGS_3(hs_error_t, hs_open_stream, db, flags, stream); CREATE_DISPATCH(hs_error_t, hs_scan_stream, hs_stream_t *id, const char *data, unsigned int length, unsigned int flags, hs_scratch_t *scratch, match_event_handler onEvent, void *ctxt); +CONNECT_ARGS_1(hs_error_t, hs_scan_stream, id, data, length, flags, scratch, onEvent, ctxt); +CONNECT_DISPATCH_2(hs_error_t, hs_scan_stream, hs_stream_t *id, const char *data, + unsigned int length, unsigned int flags, hs_scratch_t *scratch, + match_event_handler onEvent, void *ctxt); +CONNECT_ARGS_3(hs_error_t, hs_scan_stream, id, data, length, flags, scratch, onEvent, ctxt); CREATE_DISPATCH(hs_error_t, hs_close_stream, hs_stream_t *id, hs_scratch_t *scratch, match_event_handler onEvent, void *ctxt); +CONNECT_ARGS_1(hs_error_t, hs_close_stream, id, scratch, onEvent, ctxt); +CONNECT_DISPATCH_2(hs_error_t, hs_close_stream, hs_stream_t *id, + hs_scratch_t *scratch, match_event_handler onEvent, void *ctxt); +CONNECT_ARGS_3(hs_error_t, hs_close_stream, id, scratch, onEvent, ctxt); CREATE_DISPATCH(hs_error_t, hs_scan_vector, const hs_database_t *db, const char *const *data, const unsigned int *length, unsigned int count, unsigned int flags, hs_scratch_t *scratch, match_event_handler onevent, void *context); +CONNECT_ARGS_1(hs_error_t, hs_scan_vector, db, data, length, count, flags, scratch, onevent, context); +CONNECT_DISPATCH_2(hs_error_t, hs_scan_vector, const hs_database_t *db, + const char *const *data, const unsigned int *length, + unsigned int count, unsigned int flags, hs_scratch_t *scratch, + match_event_handler onevent, void *context); +CONNECT_ARGS_3(hs_error_t, hs_scan_vector, db, data, length, count, flags, scratch, onevent, context); CREATE_DISPATCH(hs_error_t, hs_database_info, const hs_database_t *db, char **info); +CONNECT_ARGS_1(hs_error_t, hs_database_info, db, info); +CONNECT_DISPATCH_2(hs_error_t, hs_database_info, const hs_database_t *db, char **info); +CONNECT_ARGS_3(hs_error_t, hs_database_info, db, info); CREATE_DISPATCH(hs_error_t, hs_copy_stream, hs_stream_t **to_id, const hs_stream_t *from_id); +CONNECT_ARGS_1(hs_error_t, hs_copy_stream, to_id, from_id); +CONNECT_DISPATCH_2(hs_error_t, hs_copy_stream, hs_stream_t **to_id, + const hs_stream_t *from_id); +CONNECT_ARGS_3(hs_error_t, hs_copy_stream, to_id, from_id); CREATE_DISPATCH(hs_error_t, hs_reset_stream, hs_stream_t *id, unsigned int flags, hs_scratch_t *scratch, match_event_handler onEvent, void *context); +CONNECT_ARGS_1(hs_error_t, hs_reset_stream, id, flags, scratch, onEvent, context); +CONNECT_DISPATCH_2(hs_error_t, hs_reset_stream, hs_stream_t *id, + unsigned int flags, hs_scratch_t *scratch, + match_event_handler onEvent, void *context); +CONNECT_ARGS_3(hs_error_t, hs_reset_stream, id, flags, scratch, onEvent, context); CREATE_DISPATCH(hs_error_t, hs_reset_and_copy_stream, hs_stream_t *to_id, const hs_stream_t *from_id, hs_scratch_t *scratch, match_event_handler onEvent, void *context); +CONNECT_ARGS_1(hs_error_t, hs_reset_and_copy_stream, to_id, from_id, scratch, onEvent, context); +CONNECT_DISPATCH_2(hs_error_t, hs_reset_and_copy_stream, hs_stream_t *to_id, + const hs_stream_t *from_id, hs_scratch_t *scratch, + match_event_handler onEvent, void *context); +CONNECT_ARGS_3(hs_error_t, hs_reset_and_copy_stream, to_id, from_id, scratch, onEvent, context); CREATE_DISPATCH(hs_error_t, hs_serialize_database, const hs_database_t *db, char **bytes, size_t *length); +CONNECT_ARGS_1(hs_error_t, hs_serialize_database, db, bytes, length); +CONNECT_DISPATCH_2(hs_error_t, hs_serialize_database, const hs_database_t *db, + char **bytes, size_t *length); +CONNECT_ARGS_3(hs_error_t, hs_serialize_database, db, bytes, length); CREATE_DISPATCH(hs_error_t, hs_deserialize_database, const char *bytes, const size_t length, hs_database_t **db); +CONNECT_ARGS_1(hs_error_t, hs_deserialize_database, bytes, length, db); +CONNECT_DISPATCH_2(hs_error_t, hs_deserialize_database, const char *bytes, + const size_t length, hs_database_t **db); +CONNECT_ARGS_3(hs_error_t, hs_deserialize_database, bytes, length, db); CREATE_DISPATCH(hs_error_t, hs_deserialize_database_at, const char *bytes, const size_t length, hs_database_t *db); +CONNECT_ARGS_1(hs_error_t, hs_deserialize_database_at, bytes, length, db); +CONNECT_DISPATCH_2(hs_error_t, hs_deserialize_database_at, const char *bytes, + const size_t length, hs_database_t *db); +CONNECT_ARGS_3(hs_error_t, hs_deserialize_database_at, bytes, length, db); CREATE_DISPATCH(hs_error_t, hs_serialized_database_info, const char *bytes, size_t length, char **info); +CONNECT_ARGS_1(hs_error_t, hs_serialized_database_info, bytes, length, info); +CONNECT_DISPATCH_2(hs_error_t, hs_serialized_database_info, const char *bytes, + size_t length, char **info); +CONNECT_ARGS_3(hs_error_t, hs_serialized_database_info, bytes, length, info); CREATE_DISPATCH(hs_error_t, hs_serialized_database_size, const char *bytes, const size_t length, size_t *deserialized_size); +CONNECT_ARGS_1(hs_error_t, hs_serialized_database_size, bytes, length, deserialized_size); +CONNECT_DISPATCH_2(hs_error_t, hs_serialized_database_size, const char *bytes, + const size_t length, size_t *deserialized_size); +CONNECT_ARGS_3(hs_error_t, hs_serialized_database_size, bytes, length, deserialized_size); CREATE_DISPATCH(hs_error_t, hs_compress_stream, const hs_stream_t *stream, char *buf, size_t buf_space, size_t *used_space); +CONNECT_ARGS_1(hs_error_t, hs_compress_stream, stream, + buf, buf_space, used_space); +CONNECT_DISPATCH_2(hs_error_t, hs_compress_stream, const hs_stream_t *stream, + char *buf, size_t buf_space, size_t *used_space); +CONNECT_ARGS_3(hs_error_t, hs_compress_stream, stream, + buf, buf_space, used_space); CREATE_DISPATCH(hs_error_t, hs_expand_stream, const hs_database_t *db, hs_stream_t **stream, const char *buf,size_t buf_size); +CONNECT_ARGS_1(hs_error_t, hs_expand_stream, db, stream, buf,buf_size); +CONNECT_DISPATCH_2(hs_error_t, hs_expand_stream, const hs_database_t *db, + hs_stream_t **stream, const char *buf,size_t buf_size); +CONNECT_ARGS_3(hs_error_t, hs_expand_stream, db, stream, buf,buf_size); CREATE_DISPATCH(hs_error_t, hs_reset_and_expand_stream, hs_stream_t *to_stream, const char *buf, size_t buf_size, hs_scratch_t *scratch, match_event_handler onEvent, void *context); +CONNECT_ARGS_1(hs_error_t, hs_reset_and_expand_stream, to_stream, + buf, buf_size, scratch, onEvent, context); +CONNECT_DISPATCH_2(hs_error_t, hs_reset_and_expand_stream, hs_stream_t *to_stream, + const char *buf, size_t buf_size, hs_scratch_t *scratch, + match_event_handler onEvent, void *context); +CONNECT_ARGS_3(hs_error_t, hs_reset_and_expand_stream, to_stream, + buf, buf_size, scratch, onEvent, context); /** INTERNALS **/ CREATE_DISPATCH(u32, Crc32c_ComputeBuf, u32 inCrc32, const void *buf, size_t bufLen); +CONNECT_ARGS_1(u32, Crc32c_ComputeBuf, inCrc32, buf, bufLen); +CONNECT_DISPATCH_2(u32, Crc32c_ComputeBuf, u32 inCrc32, const void *buf, size_t bufLen); +CONNECT_ARGS_3(u32, Crc32c_ComputeBuf, inCrc32, buf, bufLen); #pragma GCC diagnostic pop #pragma GCC diagnostic pop + From 1055fd7ed8c8b288bb911475ba062cd12cebd221 Mon Sep 17 00:00:00 2001 From: Gregory Economou Date: Mon, 15 Apr 2024 16:50:58 +0300 Subject: [PATCH 28/33] probably fixed the bit about not finding the right libs in some bsd installs --- cmake/osdetection.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 6e7178cb..874031b9 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -4,6 +4,9 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true) + set(CMAKE_SKIP_INSTALL_RPATH false) + set(CMAKE_SKIP_RPATH false) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") From 7faa32275eb93bab5afaab941ce204974dac3ca4 Mon Sep 17 00:00:00 2001 From: Gregory Economou Date: Tue, 16 Apr 2024 10:17:53 +0300 Subject: [PATCH 29/33] bit hacky but it works for setting rpath in freebsd --- cmake/osdetection.cmake | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 874031b9..23f6a5d5 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -4,9 +4,12 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) - set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true) - set(CMAKE_SKIP_INSTALL_RPATH false) - set(CMAKE_SKIP_RPATH false) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + if(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc12") + set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc12") + elseif(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc13") + set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc13") + endif() endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") From 87b4787455a45b211805b2fbba2db92bfb4f6821 Mon Sep 17 00:00:00 2001 From: Gregory Economou Date: Tue, 16 Apr 2024 13:09:05 +0300 Subject: [PATCH 30/33] made the rpath finding a bit more flexible than just hardcoded gcc12 and gcc13 --- cmake/osdetection.cmake | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 23f6a5d5..c163a7ff 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -5,11 +5,7 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - if(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc12") - set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc12") - elseif(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc13") - set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc13") - endif() + execute_process(COMMAND "sh" "-c" "realpath `dirname \\`${CMAKE_C_COMPILER} --print-file-name=libgcc_s.so\\``" OUTPUT_VARIABLE CMAKE_BUILD_RPATH) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") From 645156333df9d65ac7a00eacb029b9cecff2f029 Mon Sep 17 00:00:00 2001 From: Gregory Economou Date: Tue, 16 Apr 2024 15:04:20 +0300 Subject: [PATCH 31/33] cmake adds newlines to variables set from command output, trashing the usefulness of the flexible method of setting the rpath var. back to the clunky manual setting. --- cmake/osdetection.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index c163a7ff..23f6a5d5 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -5,7 +5,11 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - execute_process(COMMAND "sh" "-c" "realpath `dirname \\`${CMAKE_C_COMPILER} --print-file-name=libgcc_s.so\\``" OUTPUT_VARIABLE CMAKE_BUILD_RPATH) + if(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc12") + set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc12") + elseif(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc13") + set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc13") + endif() endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") From 7ee5f7913eae4b18a08d9268d139580eb863d9e9 Mon Sep 17 00:00:00 2001 From: Gregory Economou Date: Tue, 16 Apr 2024 15:18:58 +0300 Subject: [PATCH 32/33] added a fixme for the clunky rpath setting logic --- cmake/osdetection.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 23f6a5d5..3369447a 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -5,6 +5,7 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + #FIXME: find a nicer and more general way of doing this if(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc12") set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc12") elseif(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc13") From 5ebc19674cbc16bbbd93f32b89fc095c3209d121 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Tue, 16 Apr 2024 17:43:11 +0300 Subject: [PATCH 33/33] removing the dispatcher.c changes from this branch/PR --- src/dispatcher.c | 229 +++++++---------------------------------------- 1 file changed, 30 insertions(+), 199 deletions(-) diff --git a/src/dispatcher.c b/src/dispatcher.c index b704b0a5..a817e744 100644 --- a/src/dispatcher.c +++ b/src/dispatcher.c @@ -1,6 +1,5 @@ /* * Copyright (c) 2016-2020, Intel Corporation - * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,39 +30,6 @@ #include "hs_common.h" #include "hs_runtime.h" #include "ue2common.h" - -/* streamlining the dispatch to eliminate runtime checking/branching: - * what we want to do is, first call to the function will run the resolve - * code and set the static resolved/dispatch pointer to point to the - * correct function. Subsequent calls to the function will go directly to - * the resolved ptr. The simplest way to accomplish this is, to - * initially set the pointer to the resolve function. - * To accomplish this in a manner invisible to the user, - * we do involve some rather ugly/confusing macros in here. - * There are four macros that assemble the code for each function - * we want to dispatch in this manner: - * CREATE_DISPATCH - * this generates the declarations for the candidate target functions, - * for the fat_dispatch function pointer, for the resolve_ function, - * points the function pointer to the resolve function, and contains - * most of the definition of the resolve function. The very end of the - * resolve function is completed by the next macro, because in the - * CREATE_DISPATCH macro we have the argument list with the arg declarations, - * which is needed to generate correct function signatures, but we - * can't generate from this, in a macro, a _call_ to one of those functions. - * CONNECT_ARGS_1 - * this macro fills in the actual call at the end of the resolve function, - * with the correct arg list. hence the name connect args. - * CONNECT_DISPATCH_2 - * this macro likewise gives up the beginning of the definition of the - * actual entry point function (the 'real name' that's called by the user) - * but again in the pass-through call, cannot invoke the target without - * getting the arg list , which is supplied by the final macro, - * CONNECT_ARGS_3 - * - */ - - #if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/arch/x86/cpuid_inline.h" #include "util/join.h" @@ -91,38 +57,30 @@ return (RTYPE)HS_ARCH_ERROR; \ } \ \ - /* dispatch routing pointer for this function */ \ - /* initially point it at the resolve function */ \ - static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__); \ - static RTYPE (* JOIN(fat_dispatch_, NAME))(__VA_ARGS__) = \ - &JOIN(resolve_, NAME); \ - \ /* resolver */ \ - static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__) { \ + static RTYPE (*JOIN(resolve_, NAME)(void))(__VA_ARGS__) { \ if (check_avx512vbmi()) { \ - fat_dispatch_ ## NAME = &JOIN(avx512vbmi_, NAME); \ + return JOIN(avx512vbmi_, NAME); \ } \ - else if (check_avx512()) { \ - fat_dispatch_ ## NAME = &JOIN(avx512_, NAME); \ + if (check_avx512()) { \ + return JOIN(avx512_, NAME); \ } \ - else if (check_avx2()) { \ - fat_dispatch_ ## NAME = &JOIN(avx2_, NAME); \ + if (check_avx2()) { \ + return JOIN(avx2_, NAME); \ } \ - else if (check_sse42() && check_popcnt()) { \ - fat_dispatch_ ## NAME = &JOIN(corei7_, NAME); \ + if (check_sse42() && check_popcnt()) { \ + return JOIN(corei7_, NAME); \ } \ - else if (check_ssse3()) { \ - fat_dispatch_ ## NAME = &JOIN(core2_, NAME); \ - } else { \ - /* anything else is fail */ \ - fat_dispatch_ ## NAME = &JOIN(error_, NAME); \ + if (check_ssse3()) { \ + return JOIN(core2_, NAME); \ } \ - - - -/* the rest of the function is completed in the CONNECT_ARGS_1 macro. */ - - + /* anything else is fail */ \ + return JOIN(error_, NAME); \ + } \ + \ + /* function */ \ + HS_PUBLIC_API \ + RTYPE NAME(__VA_ARGS__) __attribute__((ifunc("resolve_" #NAME))) #elif defined(ARCH_AARCH64) #include "util/arch/arm/cpuid_inline.h" @@ -139,226 +97,99 @@ return (RTYPE)HS_ARCH_ERROR; \ } \ \ - /* dispatch routing pointer for this function */ \ - /* initially point it at the resolve function */ \ - static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__); \ - static RTYPE (* JOIN(fat_dispatch_, NAME))(__VA_ARGS__) = \ - &JOIN(resolve_, NAME); \ - \ /* resolver */ \ - static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__) { \ + static RTYPE (*JOIN(resolve_, NAME)(void))(__VA_ARGS__) { \ if (check_sve2()) { \ - fat_dispatch_ ## NAME = &JOIN(sve2_, NAME); \ + return JOIN(sve2_, NAME); \ } \ - else if (check_sve()) { \ - fat_dispatch_ ## NAME = &JOIN(sve_, NAME); \ + if (check_sve()) { \ + return JOIN(sve_, NAME); \ } \ - else if (check_neon()) { \ - fat_dispatch_ ## NAME = &JOIN(neon_, NAME); \ - } else { \ - /* anything else is fail */ \ - fat_dispatch_ ## NAME = &JOIN(error_, NAME); \ + if (check_neon()) { \ + return JOIN(neon_, NAME); \ } \ - - -/* the rest of the function is completed in the CONNECT_ARGS_1 macro. */ - + /* anything else is fail */ \ + return JOIN(error_, NAME); \ + } \ + \ + /* function */ \ + HS_PUBLIC_API \ + RTYPE NAME(__VA_ARGS__) __attribute__((ifunc("resolve_" #NAME))) #endif - -#define CONNECT_ARGS_1(RTYPE, NAME, ...) \ - return (*fat_dispatch_ ## NAME)(__VA_ARGS__); \ - } \ - - -#define CONNECT_DISPATCH_2(RTYPE, NAME, ...) \ - /* new function */ \ - HS_PUBLIC_API \ - RTYPE NAME(__VA_ARGS__) { \ - - -#define CONNECT_ARGS_3(RTYPE, NAME, ...) \ - return (*fat_dispatch_ ## NAME)(__VA_ARGS__); \ - } \ - - #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" - -/* this gets a bit ugly to compose the static redirect functions, - * as we necessarily need first the typed arg list and then just the arg - * names, twice in a row, to define the redirect function and the - * dispatch function call */ - CREATE_DISPATCH(hs_error_t, hs_scan, const hs_database_t *db, const char *data, unsigned length, unsigned flags, hs_scratch_t *scratch, match_event_handler onEvent, void *userCtx); -CONNECT_ARGS_1(hs_error_t, hs_scan, db, data, length, flags, scratch, onEvent, userCtx); -CONNECT_DISPATCH_2(hs_error_t, hs_scan, const hs_database_t *db, const char *data, - unsigned length, unsigned flags, hs_scratch_t *scratch, - match_event_handler onEvent, void *userCtx); -CONNECT_ARGS_3(hs_error_t, hs_scan, db, data, length, flags, scratch, onEvent, userCtx); CREATE_DISPATCH(hs_error_t, hs_stream_size, const hs_database_t *database, size_t *stream_size); -CONNECT_ARGS_1(hs_error_t, hs_stream_size, database, stream_size); -CONNECT_DISPATCH_2(hs_error_t, hs_stream_size, const hs_database_t *database, - size_t *stream_size); -CONNECT_ARGS_3(hs_error_t, hs_stream_size, database, stream_size); CREATE_DISPATCH(hs_error_t, hs_database_size, const hs_database_t *db, size_t *size); -CONNECT_ARGS_1(hs_error_t, hs_database_size, db, size); -CONNECT_DISPATCH_2(hs_error_t, hs_database_size, const hs_database_t *db, - size_t *size); -CONNECT_ARGS_3(hs_error_t, hs_database_size, db, size); - CREATE_DISPATCH(hs_error_t, dbIsValid, const hs_database_t *db); -CONNECT_ARGS_1(hs_error_t, dbIsValid, db); -CONNECT_DISPATCH_2(hs_error_t, dbIsValid, const hs_database_t *db); -CONNECT_ARGS_3(hs_error_t, dbIsValid, db); - CREATE_DISPATCH(hs_error_t, hs_free_database, hs_database_t *db); -CONNECT_ARGS_1(hs_error_t, hs_free_database, db); -CONNECT_DISPATCH_2(hs_error_t, hs_free_database, hs_database_t *db); -CONNECT_ARGS_3(hs_error_t, hs_free_database, db); CREATE_DISPATCH(hs_error_t, hs_open_stream, const hs_database_t *db, unsigned int flags, hs_stream_t **stream); -CONNECT_ARGS_1(hs_error_t, hs_open_stream, db, flags, stream); -CONNECT_DISPATCH_2(hs_error_t, hs_open_stream, const hs_database_t *db, - unsigned int flags, hs_stream_t **stream); -CONNECT_ARGS_3(hs_error_t, hs_open_stream, db, flags, stream); CREATE_DISPATCH(hs_error_t, hs_scan_stream, hs_stream_t *id, const char *data, unsigned int length, unsigned int flags, hs_scratch_t *scratch, match_event_handler onEvent, void *ctxt); -CONNECT_ARGS_1(hs_error_t, hs_scan_stream, id, data, length, flags, scratch, onEvent, ctxt); -CONNECT_DISPATCH_2(hs_error_t, hs_scan_stream, hs_stream_t *id, const char *data, - unsigned int length, unsigned int flags, hs_scratch_t *scratch, - match_event_handler onEvent, void *ctxt); -CONNECT_ARGS_3(hs_error_t, hs_scan_stream, id, data, length, flags, scratch, onEvent, ctxt); CREATE_DISPATCH(hs_error_t, hs_close_stream, hs_stream_t *id, hs_scratch_t *scratch, match_event_handler onEvent, void *ctxt); -CONNECT_ARGS_1(hs_error_t, hs_close_stream, id, scratch, onEvent, ctxt); -CONNECT_DISPATCH_2(hs_error_t, hs_close_stream, hs_stream_t *id, - hs_scratch_t *scratch, match_event_handler onEvent, void *ctxt); -CONNECT_ARGS_3(hs_error_t, hs_close_stream, id, scratch, onEvent, ctxt); CREATE_DISPATCH(hs_error_t, hs_scan_vector, const hs_database_t *db, const char *const *data, const unsigned int *length, unsigned int count, unsigned int flags, hs_scratch_t *scratch, match_event_handler onevent, void *context); -CONNECT_ARGS_1(hs_error_t, hs_scan_vector, db, data, length, count, flags, scratch, onevent, context); -CONNECT_DISPATCH_2(hs_error_t, hs_scan_vector, const hs_database_t *db, - const char *const *data, const unsigned int *length, - unsigned int count, unsigned int flags, hs_scratch_t *scratch, - match_event_handler onevent, void *context); -CONNECT_ARGS_3(hs_error_t, hs_scan_vector, db, data, length, count, flags, scratch, onevent, context); CREATE_DISPATCH(hs_error_t, hs_database_info, const hs_database_t *db, char **info); -CONNECT_ARGS_1(hs_error_t, hs_database_info, db, info); -CONNECT_DISPATCH_2(hs_error_t, hs_database_info, const hs_database_t *db, char **info); -CONNECT_ARGS_3(hs_error_t, hs_database_info, db, info); CREATE_DISPATCH(hs_error_t, hs_copy_stream, hs_stream_t **to_id, const hs_stream_t *from_id); -CONNECT_ARGS_1(hs_error_t, hs_copy_stream, to_id, from_id); -CONNECT_DISPATCH_2(hs_error_t, hs_copy_stream, hs_stream_t **to_id, - const hs_stream_t *from_id); -CONNECT_ARGS_3(hs_error_t, hs_copy_stream, to_id, from_id); CREATE_DISPATCH(hs_error_t, hs_reset_stream, hs_stream_t *id, unsigned int flags, hs_scratch_t *scratch, match_event_handler onEvent, void *context); -CONNECT_ARGS_1(hs_error_t, hs_reset_stream, id, flags, scratch, onEvent, context); -CONNECT_DISPATCH_2(hs_error_t, hs_reset_stream, hs_stream_t *id, - unsigned int flags, hs_scratch_t *scratch, - match_event_handler onEvent, void *context); -CONNECT_ARGS_3(hs_error_t, hs_reset_stream, id, flags, scratch, onEvent, context); CREATE_DISPATCH(hs_error_t, hs_reset_and_copy_stream, hs_stream_t *to_id, const hs_stream_t *from_id, hs_scratch_t *scratch, match_event_handler onEvent, void *context); -CONNECT_ARGS_1(hs_error_t, hs_reset_and_copy_stream, to_id, from_id, scratch, onEvent, context); -CONNECT_DISPATCH_2(hs_error_t, hs_reset_and_copy_stream, hs_stream_t *to_id, - const hs_stream_t *from_id, hs_scratch_t *scratch, - match_event_handler onEvent, void *context); -CONNECT_ARGS_3(hs_error_t, hs_reset_and_copy_stream, to_id, from_id, scratch, onEvent, context); CREATE_DISPATCH(hs_error_t, hs_serialize_database, const hs_database_t *db, char **bytes, size_t *length); -CONNECT_ARGS_1(hs_error_t, hs_serialize_database, db, bytes, length); -CONNECT_DISPATCH_2(hs_error_t, hs_serialize_database, const hs_database_t *db, - char **bytes, size_t *length); -CONNECT_ARGS_3(hs_error_t, hs_serialize_database, db, bytes, length); CREATE_DISPATCH(hs_error_t, hs_deserialize_database, const char *bytes, const size_t length, hs_database_t **db); -CONNECT_ARGS_1(hs_error_t, hs_deserialize_database, bytes, length, db); -CONNECT_DISPATCH_2(hs_error_t, hs_deserialize_database, const char *bytes, - const size_t length, hs_database_t **db); -CONNECT_ARGS_3(hs_error_t, hs_deserialize_database, bytes, length, db); CREATE_DISPATCH(hs_error_t, hs_deserialize_database_at, const char *bytes, const size_t length, hs_database_t *db); -CONNECT_ARGS_1(hs_error_t, hs_deserialize_database_at, bytes, length, db); -CONNECT_DISPATCH_2(hs_error_t, hs_deserialize_database_at, const char *bytes, - const size_t length, hs_database_t *db); -CONNECT_ARGS_3(hs_error_t, hs_deserialize_database_at, bytes, length, db); CREATE_DISPATCH(hs_error_t, hs_serialized_database_info, const char *bytes, size_t length, char **info); -CONNECT_ARGS_1(hs_error_t, hs_serialized_database_info, bytes, length, info); -CONNECT_DISPATCH_2(hs_error_t, hs_serialized_database_info, const char *bytes, - size_t length, char **info); -CONNECT_ARGS_3(hs_error_t, hs_serialized_database_info, bytes, length, info); CREATE_DISPATCH(hs_error_t, hs_serialized_database_size, const char *bytes, const size_t length, size_t *deserialized_size); -CONNECT_ARGS_1(hs_error_t, hs_serialized_database_size, bytes, length, deserialized_size); -CONNECT_DISPATCH_2(hs_error_t, hs_serialized_database_size, const char *bytes, - const size_t length, size_t *deserialized_size); -CONNECT_ARGS_3(hs_error_t, hs_serialized_database_size, bytes, length, deserialized_size); CREATE_DISPATCH(hs_error_t, hs_compress_stream, const hs_stream_t *stream, char *buf, size_t buf_space, size_t *used_space); -CONNECT_ARGS_1(hs_error_t, hs_compress_stream, stream, - buf, buf_space, used_space); -CONNECT_DISPATCH_2(hs_error_t, hs_compress_stream, const hs_stream_t *stream, - char *buf, size_t buf_space, size_t *used_space); -CONNECT_ARGS_3(hs_error_t, hs_compress_stream, stream, - buf, buf_space, used_space); CREATE_DISPATCH(hs_error_t, hs_expand_stream, const hs_database_t *db, hs_stream_t **stream, const char *buf,size_t buf_size); -CONNECT_ARGS_1(hs_error_t, hs_expand_stream, db, stream, buf,buf_size); -CONNECT_DISPATCH_2(hs_error_t, hs_expand_stream, const hs_database_t *db, - hs_stream_t **stream, const char *buf,size_t buf_size); -CONNECT_ARGS_3(hs_error_t, hs_expand_stream, db, stream, buf,buf_size); CREATE_DISPATCH(hs_error_t, hs_reset_and_expand_stream, hs_stream_t *to_stream, const char *buf, size_t buf_size, hs_scratch_t *scratch, match_event_handler onEvent, void *context); -CONNECT_ARGS_1(hs_error_t, hs_reset_and_expand_stream, to_stream, - buf, buf_size, scratch, onEvent, context); -CONNECT_DISPATCH_2(hs_error_t, hs_reset_and_expand_stream, hs_stream_t *to_stream, - const char *buf, size_t buf_size, hs_scratch_t *scratch, - match_event_handler onEvent, void *context); -CONNECT_ARGS_3(hs_error_t, hs_reset_and_expand_stream, to_stream, - buf, buf_size, scratch, onEvent, context); /** INTERNALS **/ CREATE_DISPATCH(u32, Crc32c_ComputeBuf, u32 inCrc32, const void *buf, size_t bufLen); -CONNECT_ARGS_1(u32, Crc32c_ComputeBuf, inCrc32, buf, bufLen); -CONNECT_DISPATCH_2(u32, Crc32c_ComputeBuf, u32 inCrc32, const void *buf, size_t bufLen); -CONNECT_ARGS_3(u32, Crc32c_ComputeBuf, inCrc32, buf, bufLen); #pragma GCC diagnostic pop #pragma GCC diagnostic pop -