From 12f61d15ed935c02a45060c54a2846623fe5374f Mon Sep 17 00:00:00 2001 From: "G.E." Date: Wed, 6 Mar 2024 10:48:56 +0100 Subject: [PATCH 01/33] support building on NetBSD --- README.md | 11 +++++++++++ cmake/osdetection.cmake | 6 +++++- examples/patbench.cc | 5 +++++ examples/pcapscan.cc | 5 +++++ src/util/popcount.h | 8 ++++++++ 5 files changed, 34 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7f7c2f53..cf2e5f40 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,17 @@ Assuming an existing HomeBrew installation: % brew install boost cmake gcc libpcap pkg-config ragel sqlite ``` +### NetBSD +In NetBSD you will almost certainly need to have a newer compiler installed. +Using the example of gcc12 from pkgsrc, one will need to set three +environment variables before starting: +export CC="/usr/pkg/gcc12/bin/cc" +export CXX="/usr/pkg/gcc12/bin/g++" +export CXXFLAGS="-gdwarf-4" + +Then continue with the build as below. + + ## Configure & build In order to configure with `cmake` first create and cd into a build directory: diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 59f2b342..0e730fe3 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -6,7 +6,11 @@ if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") -if (ARCH_IA32 OR ARCH_X86_64) +if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + set(NETBSD true) +endif(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + +if (LINUX AND (ARCH_IA32 OR ARCH_X86_64)) option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" ON) else() option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" OFF) diff --git a/examples/patbench.cc b/examples/patbench.cc index 8180d2a9..7bb5020a 100644 --- a/examples/patbench.cc +++ b/examples/patbench.cc @@ -134,7 +134,12 @@ #include #include #include +#ifdef __NetBSD__ +#include +#include +#else #include +#endif /* __NetBSD__ */ #include #include diff --git a/examples/pcapscan.cc b/examples/pcapscan.cc index 12b94438..349ef6a9 100644 --- a/examples/pcapscan.cc +++ b/examples/pcapscan.cc @@ -68,7 +68,12 @@ #include #include #include +#ifdef __NetBSD__ +#include +#include +#else #include +#endif /* __NetBSD__ */ #include #include diff --git a/src/util/popcount.h b/src/util/popcount.h index d90a0d50..e9f689e9 100644 --- a/src/util/popcount.h +++ b/src/util/popcount.h @@ -36,7 +36,12 @@ #include "ue2common.h" #include "util/arch.h" +#ifdef __NetBSD__ +#include +#endif + +#ifndef __NetBSD__ static really_inline u32 popcount32(u32 x) { return __builtin_popcount(x); @@ -51,6 +56,7 @@ u32 popcount32(u32 x) { // return (((x + (x >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24; // #endif } +#endif /* __NetBSD__ */ static really_inline u32 popcount32x4(u32 const *x) { @@ -61,6 +67,7 @@ u32 popcount32x4(u32 const *x) { return sum; } +#ifndef __NetBSD__ static really_inline u32 popcount64(u64a x) { return __builtin_popcountll(x); @@ -81,6 +88,7 @@ u32 popcount64(u64a x) { // return popcount32(x >> 32) + popcount32(x); // #endif } +#endif /* __NetBSD__ */ static really_inline u32 popcount64x4(u64a const *x) { From a05c891146bc68df35c45b374af9e0fab797fe81 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Wed, 6 Mar 2024 12:31:35 +0100 Subject: [PATCH 02/33] updated readme to reflect FreeBSD build --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cf2e5f40..67124678 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Assuming an existing HomeBrew installation: % brew install boost cmake gcc libpcap pkg-config ragel sqlite ``` -### NetBSD +### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. Using the example of gcc12 from pkgsrc, one will need to set three environment variables before starting: @@ -121,6 +121,12 @@ export CC="/usr/pkg/gcc12/bin/cc" export CXX="/usr/pkg/gcc12/bin/g++" export CXXFLAGS="-gdwarf-4" +In FreeBSD similarly, using gcc12 from pkg +export CC=/usr/local/bin/gcc +export CXX=/usr/local/bin/g++ +export CXXFLAGS="-gdwarf-4" + + Then continue with the build as below. From 30ae8505c308d68ef39d3890f2cce606d5483cc8 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Wed, 6 Mar 2024 14:57:28 +0100 Subject: [PATCH 03/33] lets rather not add lines of code not yet used anywhere --- cmake/osdetection.cmake | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 0e730fe3..e4d0bf57 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -6,10 +6,6 @@ if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") -if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") - set(NETBSD true) -endif(CMAKE_SYSTEM_NAME MATCHES "NetBSD") - if (LINUX AND (ARCH_IA32 OR ARCH_X86_64)) option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" ON) else() From a6a35e044cb4ac83832d9ee372a74cb6bcc40828 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Wed, 6 Mar 2024 15:28:20 +0100 Subject: [PATCH 04/33] minor change for more certain success in netbsd --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 67124678..fb9b1bba 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,8 @@ Using the example of gcc12 from pkgsrc, one will need to set three environment variables before starting: export CC="/usr/pkg/gcc12/bin/cc" export CXX="/usr/pkg/gcc12/bin/g++" -export CXXFLAGS="-gdwarf-4" +export CXXFLAGS="-gdwarf-4 -Wno-redundant-move" + In FreeBSD similarly, using gcc12 from pkg export CC=/usr/local/bin/gcc From 4739c76a1152bacc6c4fa61b6f2636ca1bb2f055 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 6 Mar 2024 15:50:26 +0100 Subject: [PATCH 05/33] formatting in readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index fb9b1bba..7bf2d046 100644 --- a/README.md +++ b/README.md @@ -117,15 +117,19 @@ Assuming an existing HomeBrew installation: In NetBSD you will almost certainly need to have a newer compiler installed. Using the example of gcc12 from pkgsrc, one will need to set three environment variables before starting: +``` export CC="/usr/pkg/gcc12/bin/cc" export CXX="/usr/pkg/gcc12/bin/g++" export CXXFLAGS="-gdwarf-4 -Wno-redundant-move" +``` In FreeBSD similarly, using gcc12 from pkg +``` export CC=/usr/local/bin/gcc export CXX=/usr/local/bin/g++ export CXXFLAGS="-gdwarf-4" +``` Then continue with the build as below. From f7a4d41c63c2a391bca915799896f5ecc77dcd06 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 6 Mar 2024 15:53:03 +0100 Subject: [PATCH 06/33] more readme format tinkering --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7bf2d046..2009d220 100644 --- a/README.md +++ b/README.md @@ -126,8 +126,8 @@ export CXXFLAGS="-gdwarf-4 -Wno-redundant-move" In FreeBSD similarly, using gcc12 from pkg ``` -export CC=/usr/local/bin/gcc -export CXX=/usr/local/bin/g++ +export CC="/usr/local/bin/gcc" +export CXX="/usr/local/bin/g++" export CXXFLAGS="-gdwarf-4" ``` From afcbd28d3be1c8634bab544bf08433a55ca2b0a5 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 6 Mar 2024 16:39:34 +0100 Subject: [PATCH 07/33] lets not disable warnings --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2009d220..0fb15607 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ environment variables before starting: ``` export CC="/usr/pkg/gcc12/bin/cc" export CXX="/usr/pkg/gcc12/bin/g++" -export CXXFLAGS="-gdwarf-4 -Wno-redundant-move" +export CXXFLAGS="-gdwarf-4" ``` From 523db6051d70de55a38111c7b0f153b312e74ef4 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 6 Mar 2024 17:05:55 +0100 Subject: [PATCH 08/33] maybe netbsd is more pedantic about this? --- tools/hscollider/UltimateTruth.cpp | 2 +- tools/hscollider/main.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/hscollider/UltimateTruth.cpp b/tools/hscollider/UltimateTruth.cpp index 93d432c3..e33ce2fb 100644 --- a/tools/hscollider/UltimateTruth.cpp +++ b/tools/hscollider/UltimateTruth.cpp @@ -1079,7 +1079,7 @@ shared_ptr UltimateTruth::compile(const set &ids, } } - return std::move(db); + return db; } bool UltimateTruth::allocScratch(shared_ptr db) { diff --git a/tools/hscollider/main.cpp b/tools/hscollider/main.cpp index dcc5c1b6..0decc810 100644 --- a/tools/hscollider/main.cpp +++ b/tools/hscollider/main.cpp @@ -1830,11 +1830,11 @@ unique_ptr buildCorpora(const vector &corporaFiles, exit_with_fail(); } } - return std::move(c); /* move allows unique_ptr conversion */ + return c; } else { auto c = std::make_unique( exprMap, corpus_gen_prop, force_utf8, force_prefilter); - return std::move(c); + return c; } } From d6d7a96c44a1a4dfb09f2d7f595fe7907a5727f6 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Tue, 12 Mar 2024 12:37:08 +0200 Subject: [PATCH 09/33] incremental improvement in cleanliness --- cmake/cflags-generic.cmake | 4 ++++ cmake/osdetection.cmake | 4 ++++ src/util/arch.h | 4 ++++ src/util/popcount.h | 12 ++++-------- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/cmake/cflags-generic.cmake b/cmake/cflags-generic.cmake index 29b746a1..93fa789d 100644 --- a/cmake/cflags-generic.cmake +++ b/cmake/cflags-generic.cmake @@ -27,6 +27,10 @@ CHECK_INCLUDE_FILES(unistd.h HAVE_UNISTD_H) CHECK_FUNCTION_EXISTS(posix_memalign HAVE_POSIX_MEMALIGN) CHECK_FUNCTION_EXISTS(_aligned_malloc HAVE__ALIGNED_MALLOC) +if(FREEBSD OR NETBSD) + set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -gdwarf-4") +endif() + # these end up in the config file CHECK_C_COMPILER_FLAG(-fvisibility=hidden HAS_C_HIDDEN) CHECK_CXX_COMPILER_FLAG(-fvisibility=hidden HAS_CXX_HIDDEN) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index e4d0bf57..0e730fe3 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -6,6 +6,10 @@ if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") +if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + set(NETBSD true) +endif(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + if (LINUX AND (ARCH_IA32 OR ARCH_X86_64)) option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" ON) else() diff --git a/src/util/arch.h b/src/util/arch.h index 1e8d2fbd..9a9dde1b 100644 --- a/src/util/arch.h +++ b/src/util/arch.h @@ -42,6 +42,10 @@ #elif defined(ARCH_PPC64EL) #include "util/arch/ppc64el/ppc64el.h" #endif +#ifdef __NetBSD__ +#include +#define HAVE_LOCAL_POPCOUNT +#endif #endif // UTIL_ARCH_X86_H_ diff --git a/src/util/popcount.h b/src/util/popcount.h index e9f689e9..a02e3bc5 100644 --- a/src/util/popcount.h +++ b/src/util/popcount.h @@ -36,12 +36,8 @@ #include "ue2common.h" #include "util/arch.h" -#ifdef __NetBSD__ -#include -#endif - -#ifndef __NetBSD__ +#ifndef HAVE_LOCAL_POPCOUNT static really_inline u32 popcount32(u32 x) { return __builtin_popcount(x); @@ -56,7 +52,7 @@ u32 popcount32(u32 x) { // return (((x + (x >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24; // #endif } -#endif /* __NetBSD__ */ +#endif /* HAVE_LOCAL_POPCOUNT */ static really_inline u32 popcount32x4(u32 const *x) { @@ -67,7 +63,7 @@ u32 popcount32x4(u32 const *x) { return sum; } -#ifndef __NetBSD__ +#ifndef HAVE_LOCAL_POPCOUNT static really_inline u32 popcount64(u64a x) { return __builtin_popcountll(x); @@ -88,7 +84,7 @@ u32 popcount64(u64a x) { // return popcount32(x >> 32) + popcount32(x); // #endif } -#endif /* __NetBSD__ */ +#endif /* HAVE_LOCAL_POPCOUNT */ static really_inline u32 popcount64x4(u64a const *x) { From 226645eaf1178ffa7af766e934241728a5a6d3f7 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Tue, 12 Mar 2024 12:59:16 +0200 Subject: [PATCH 10/33] another small cleanup in readme --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 0fb15607..5dafd0ea 100644 --- a/README.md +++ b/README.md @@ -115,12 +115,11 @@ Assuming an existing HomeBrew installation: ### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. -Using the example of gcc12 from pkgsrc, one will need to set three +Using the example of gcc12 from pkgsrc, one will need to set two environment variables before starting: ``` export CC="/usr/pkg/gcc12/bin/cc" export CXX="/usr/pkg/gcc12/bin/g++" -export CXXFLAGS="-gdwarf-4" ``` @@ -128,7 +127,6 @@ In FreeBSD similarly, using gcc12 from pkg ``` export CC="/usr/local/bin/gcc" export CXX="/usr/local/bin/g++" -export CXXFLAGS="-gdwarf-4" ``` From 0045a2bdc75c643002c1806cf73a53ec1c8eea06 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Tue, 12 Mar 2024 14:22:39 +0200 Subject: [PATCH 11/33] moved HAVE_BUILTIN_POPCOUNT def to cmake --- README.md | 8 +++++++- cmake/cflags-generic.cmake | 5 +++++ src/util/arch.h | 1 - src/util/popcount.h | 8 ++++---- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5dafd0ea..2ecc0961 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,8 @@ Assuming an existing HomeBrew installation: ### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. +This will not replace the one in the standard base distribution, and +cmake will probably find the wrong compiler when it checks automatically. Using the example of gcc12 from pkgsrc, one will need to set two environment variables before starting: ``` @@ -123,7 +125,11 @@ export CXX="/usr/pkg/gcc12/bin/g++" ``` -In FreeBSD similarly, using gcc12 from pkg +In FreeBSD similarly, if you install another compiler, cmake might not +find the right one. Worse is if it finds, say, g++ and configures it for +C++ but leaves cc for C - FreeBSD's default cc is clang, and the binaries +generated between g++ and clang will not play nicely together. +Using the example of gcc12 from pkg: ``` export CC="/usr/local/bin/gcc" export CXX="/usr/local/bin/g++" diff --git a/cmake/cflags-generic.cmake b/cmake/cflags-generic.cmake index 93fa789d..35bbe9f4 100644 --- a/cmake/cflags-generic.cmake +++ b/cmake/cflags-generic.cmake @@ -31,6 +31,11 @@ if(FREEBSD OR NETBSD) set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -gdwarf-4") endif() +if(NETBSD) + set(EXTRA_C_FLAGS "${EXTRA_CXX_FLAGS} -DHAVE_BUILTIN_POPCOUNT") + set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -DHAVE_BUILTIN_POPCOUNT") +endif() + # these end up in the config file CHECK_C_COMPILER_FLAG(-fvisibility=hidden HAS_C_HIDDEN) CHECK_CXX_COMPILER_FLAG(-fvisibility=hidden HAS_CXX_HIDDEN) diff --git a/src/util/arch.h b/src/util/arch.h index 9a9dde1b..1963b405 100644 --- a/src/util/arch.h +++ b/src/util/arch.h @@ -44,7 +44,6 @@ #endif #ifdef __NetBSD__ #include -#define HAVE_LOCAL_POPCOUNT #endif #endif // UTIL_ARCH_X86_H_ diff --git a/src/util/popcount.h b/src/util/popcount.h index a02e3bc5..34f386b7 100644 --- a/src/util/popcount.h +++ b/src/util/popcount.h @@ -37,7 +37,7 @@ #include "ue2common.h" #include "util/arch.h" -#ifndef HAVE_LOCAL_POPCOUNT +#ifndef HAVE_BUILTIN_POPCOUNT static really_inline u32 popcount32(u32 x) { return __builtin_popcount(x); @@ -52,7 +52,7 @@ u32 popcount32(u32 x) { // return (((x + (x >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24; // #endif } -#endif /* HAVE_LOCAL_POPCOUNT */ +#endif /* HAVE_BUILTIN_POPCOUNT */ static really_inline u32 popcount32x4(u32 const *x) { @@ -63,7 +63,7 @@ u32 popcount32x4(u32 const *x) { return sum; } -#ifndef HAVE_LOCAL_POPCOUNT +#ifndef HAVE_BUILTIN_POPCOUNT static really_inline u32 popcount64(u64a x) { return __builtin_popcountll(x); @@ -84,7 +84,7 @@ u32 popcount64(u64a x) { // return popcount32(x >> 32) + popcount32(x); // #endif } -#endif /* HAVE_LOCAL_POPCOUNT */ +#endif /* HAVE_BUILTIN_POPCOUNT */ static really_inline u32 popcount64x4(u64a const *x) { From d0498f942d1d94957d9e7165b983278a614227b8 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Tue, 12 Mar 2024 13:58:50 +0100 Subject: [PATCH 12/33] typo fix --- cmake/cflags-generic.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cflags-generic.cmake b/cmake/cflags-generic.cmake index 35bbe9f4..6bc60502 100644 --- a/cmake/cflags-generic.cmake +++ b/cmake/cflags-generic.cmake @@ -32,7 +32,7 @@ if(FREEBSD OR NETBSD) endif() if(NETBSD) - set(EXTRA_C_FLAGS "${EXTRA_CXX_FLAGS} -DHAVE_BUILTIN_POPCOUNT") + set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} -DHAVE_BUILTIN_POPCOUNT") set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -DHAVE_BUILTIN_POPCOUNT") endif() From b006d7f620a9f09925a8d7d7e5a0569d43db9387 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Tue, 12 Mar 2024 16:50:57 +0200 Subject: [PATCH 13/33] shortened freebsd text --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 2ecc0961..e82517f0 100644 --- a/README.md +++ b/README.md @@ -125,10 +125,7 @@ export CXX="/usr/pkg/gcc12/bin/g++" ``` -In FreeBSD similarly, if you install another compiler, cmake might not -find the right one. Worse is if it finds, say, g++ and configures it for -C++ but leaves cc for C - FreeBSD's default cc is clang, and the binaries -generated between g++ and clang will not play nicely together. +In FreeBSD similarly, you might want to install a different compiler. Using the example of gcc12 from pkg: ``` export CC="/usr/local/bin/gcc" From 1ea53768a6f34f83b8deb93c6f5215b5a7c39d2e Mon Sep 17 00:00:00 2001 From: "G.E." Date: Wed, 13 Mar 2024 17:33:23 +0200 Subject: [PATCH 14/33] whitespace editing in readme --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index e82517f0..764f5e8c 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,6 @@ export CC="/usr/pkg/gcc12/bin/cc" export CXX="/usr/pkg/gcc12/bin/g++" ``` - In FreeBSD similarly, you might want to install a different compiler. Using the example of gcc12 from pkg: ``` @@ -132,7 +131,6 @@ export CC="/usr/local/bin/gcc" export CXX="/usr/local/bin/g++" ``` - Then continue with the build as below. From 17c78ff23c4537f8c007e028e964bf3170d168e3 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 27 Mar 2024 16:31:02 +0200 Subject: [PATCH 15/33] more verbose instructions for preparing BSD systems. --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 764f5e8c..717888e0 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,15 @@ Assuming an existing HomeBrew installation: ### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. +When using pkgsrc, you would typically do this using something +similar to +``` +pkg_add gcc12-12.3.0.tgz +``` +Version numbers etc will of course vary. One would either download the +binary package or build it using pkgsrc. +See https://www.netbsd.org/docs/software/packages.html for more information. + This will not replace the one in the standard base distribution, and cmake will probably find the wrong compiler when it checks automatically. Using the example of gcc12 from pkgsrc, one will need to set two @@ -126,6 +135,12 @@ export CXX="/usr/pkg/gcc12/bin/g++" In FreeBSD similarly, you might want to install a different compiler. Using the example of gcc12 from pkg: +installing the desired compiler: +``` +pkg install gcc12 +``` +and then before beginning the cmake and build process, set +the environment variables to point to this compiler: ``` export CC="/usr/local/bin/gcc" export CXX="/usr/local/bin/g++" From dc371fb68295647923ce4f88a6ae202c2aa9822d Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 27 Mar 2024 16:35:21 +0200 Subject: [PATCH 16/33] also added note for CC/CXX vars in fbsd/ppc which are different. --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 717888e0..f1622030 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,15 @@ export CC="/usr/local/bin/gcc" export CXX="/usr/local/bin/g++" ``` +A further note in FreeBSD, on the PowerPC platform, for some reason, +the gcc12 package installs to a slightly different name, on FreeBSD/ppc, +gcc12 will be found using: +``` +export CC="/usr/local/bin/gcc12" +export CXX="/usr/local/bin/g++12" +``` + + Then continue with the build as below. From e239f482fd0f3fd7bd20407cd5fbe76ef173beca Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 27 Mar 2024 17:19:56 +0200 Subject: [PATCH 17/33] more system prep info for bsd --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f1622030..5edc6376 100644 --- a/README.md +++ b/README.md @@ -115,13 +115,17 @@ Assuming an existing HomeBrew installation: ### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. +Also you will need to install cmake, boost and ragel. When using pkgsrc, you would typically do this using something similar to ``` pkg_add gcc12-12.3.0.tgz +pkg_add boost-headers-1.83.0.tgz boost-jam-1.83.0.tgz boost-libs-1.83.0nb1.tgz +pkg_add ragel-6.10.tgz +pkg_add cmake-3.28.1.tgz ``` Version numbers etc will of course vary. One would either download the -binary package or build it using pkgsrc. +binary packages or build them using pkgsrc. See https://www.netbsd.org/docs/software/packages.html for more information. This will not replace the one in the standard base distribution, and @@ -134,10 +138,14 @@ export CXX="/usr/pkg/gcc12/bin/g++" ``` In FreeBSD similarly, you might want to install a different compiler. +You will also, as in NetBSD, need to install cmake, boost and ragel packages. Using the example of gcc12 from pkg: installing the desired compiler: ``` pkg install gcc12 +pkg install boost-all +pkg install ragel +pkg install cmake ``` and then before beginning the cmake and build process, set the environment variables to point to this compiler: From e2ce866462a8aa1e65d4401b9ca50fab6eed5cc7 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 27 Mar 2024 17:24:08 +0200 Subject: [PATCH 18/33] some more bsd detail --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5edc6376..400bc025 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,11 @@ pkg_add ragel-6.10.tgz pkg_add cmake-3.28.1.tgz ``` Version numbers etc will of course vary. One would either download the -binary packages or build them using pkgsrc. +binary packages or build them using pkgsrc. There exist some NetBSD pkg +tools like ```pkgin``` which help download e.g. dependencies as binary packages, +but overall NetBSD leaves a lot of detail exposed to the user. +The main package system used in NetBSD is pkgsrc and one will probably +want to read up more about it than is in the scope of this document. See https://www.netbsd.org/docs/software/packages.html for more information. This will not replace the one in the standard base distribution, and From 42653b8a31fe332a6efd642d8b0670a4a3076a63 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 27 Mar 2024 17:28:24 +0200 Subject: [PATCH 19/33] also sqlite info for bsd --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 400bc025..fca411ff 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Assuming an existing HomeBrew installation: ### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. -Also you will need to install cmake, boost and ragel. +Also you will need to install cmake, sqlite, boost and ragel. When using pkgsrc, you would typically do this using something similar to ``` @@ -123,6 +123,7 @@ pkg_add gcc12-12.3.0.tgz pkg_add boost-headers-1.83.0.tgz boost-jam-1.83.0.tgz boost-libs-1.83.0nb1.tgz pkg_add ragel-6.10.tgz pkg_add cmake-3.28.1.tgz +pkg_add sqlite3-3.44.2.tgz ``` Version numbers etc will of course vary. One would either download the binary packages or build them using pkgsrc. There exist some NetBSD pkg @@ -142,7 +143,7 @@ export CXX="/usr/pkg/gcc12/bin/g++" ``` In FreeBSD similarly, you might want to install a different compiler. -You will also, as in NetBSD, need to install cmake, boost and ragel packages. +You will also, as in NetBSD, need to install cmake, sqlite, boost and ragel packages. Using the example of gcc12 from pkg: installing the desired compiler: ``` @@ -150,6 +151,7 @@ pkg install gcc12 pkg install boost-all pkg install ragel pkg install cmake +pkg install sqlite ``` and then before beginning the cmake and build process, set the environment variables to point to this compiler: From e20ba37208b0917f1025538e83e245038b934241 Mon Sep 17 00:00:00 2001 From: isildur-g Date: Wed, 27 Mar 2024 17:33:25 +0200 Subject: [PATCH 20/33] slightly clearer comments in netbsd section --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fca411ff..e6965710 100644 --- a/README.md +++ b/README.md @@ -133,8 +133,8 @@ The main package system used in NetBSD is pkgsrc and one will probably want to read up more about it than is in the scope of this document. See https://www.netbsd.org/docs/software/packages.html for more information. -This will not replace the one in the standard base distribution, and -cmake will probably find the wrong compiler when it checks automatically. +This will not replace the compiler in the standard base distribution, and +cmake will probably find the base dist's compiler when it checks automatically. Using the example of gcc12 from pkgsrc, one will need to set two environment variables before starting: ``` From 773d57d890d9683fdb02f060603c67227c949f92 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Thu, 11 Apr 2024 09:49:43 +0300 Subject: [PATCH 21/33] added copyright info for modified src files --- examples/patbench.cc | 1 + examples/pcapscan.cc | 1 + src/util/arch.h | 1 + src/util/popcount.h | 2 +- tools/hscollider/UltimateTruth.cpp | 1 + tools/hscollider/main.cpp | 1 + 6 files changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/patbench.cc b/examples/patbench.cc index 7bb5020a..1f965f13 100644 --- a/examples/patbench.cc +++ b/examples/patbench.cc @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/examples/pcapscan.cc b/examples/pcapscan.cc index 349ef6a9..bd5493a4 100644 --- a/examples/pcapscan.cc +++ b/examples/pcapscan.cc @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/util/arch.h b/src/util/arch.h index 1963b405..ed762797 100644 --- a/src/util/arch.h +++ b/src/util/arch.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2017-2020, Intel Corporation + * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/util/popcount.h b/src/util/popcount.h index 34f386b7..bca787db 100644 --- a/src/util/popcount.h +++ b/src/util/popcount.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2015-2017, Intel Corporation - * Copyright (c) 2020-2021, VectorCamp PC + * Copyright (c) 2020-2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/tools/hscollider/UltimateTruth.cpp b/tools/hscollider/UltimateTruth.cpp index e33ce2fb..6b933384 100644 --- a/tools/hscollider/UltimateTruth.cpp +++ b/tools/hscollider/UltimateTruth.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2019, Intel Corporation + * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/tools/hscollider/main.cpp b/tools/hscollider/main.cpp index 0decc810..3d4312cd 100644 --- a/tools/hscollider/main.cpp +++ b/tools/hscollider/main.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015-2019, Intel Corporation + * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: From 372053ba4f3b58b6b38a8002c3533cec8bea163f Mon Sep 17 00:00:00 2001 From: "G.E." Date: Thu, 11 Apr 2024 15:06:18 +0300 Subject: [PATCH 22/33] adding libpcap to the bsd packages to install for building/running --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index e6965710..2f5da86e 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,8 @@ Assuming an existing HomeBrew installation: ### *BSD In NetBSD you will almost certainly need to have a newer compiler installed. Also you will need to install cmake, sqlite, boost and ragel. +Also, libpcap is necessary for some of the benchmarks, so let's install that +as well. When using pkgsrc, you would typically do this using something similar to ``` @@ -124,6 +126,7 @@ pkg_add boost-headers-1.83.0.tgz boost-jam-1.83.0.tgz boost-libs-1.83.0nb1 pkg_add ragel-6.10.tgz pkg_add cmake-3.28.1.tgz pkg_add sqlite3-3.44.2.tgz +pkg_add libpcap-1.10.4.tgz ``` Version numbers etc will of course vary. One would either download the binary packages or build them using pkgsrc. There exist some NetBSD pkg @@ -152,6 +155,7 @@ pkg install boost-all pkg install ragel pkg install cmake pkg install sqlite +pkg install libpcap ``` and then before beginning the cmake and build process, set the environment variables to point to this compiler: From d0b39914dffe12951acd36995bb4732e9149e18c Mon Sep 17 00:00:00 2001 From: "G.E." Date: Thu, 11 Apr 2024 17:47:16 +0300 Subject: [PATCH 23/33] added ccache for freebsd install recommendations --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2f5da86e..1baeaac8 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,7 @@ pkg install ragel pkg install cmake pkg install sqlite pkg install libpcap +pkg install ccache ``` and then before beginning the cmake and build process, set the environment variables to point to this compiler: From 0376319a937a6d2aed9e9cc111aec48774fbe592 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Mon, 15 Apr 2024 11:07:45 +0300 Subject: [PATCH 24/33] tiny edit to readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1baeaac8..f6a10fca 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ export CC="/usr/local/bin/gcc" export CXX="/usr/local/bin/g++" ``` -A further note in FreeBSD, on the PowerPC platform, for some reason, +A further note in FreeBSD, on the PowerPC and ARM platforms, the gcc12 package installs to a slightly different name, on FreeBSD/ppc, gcc12 will be found using: ``` From b0916df82560d35a0f301efc4db44eaa3a132be0 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Mon, 15 Apr 2024 11:44:22 +0300 Subject: [PATCH 25/33] one more place to fix where clang in bsd is more picky than gcc in linux --- src/rose/rose_build_program.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rose/rose_build_program.cpp b/src/rose/rose_build_program.cpp index 1e0fe24b..861855b5 100644 --- a/src/rose/rose_build_program.cpp +++ b/src/rose/rose_build_program.cpp @@ -1161,7 +1161,7 @@ static really_inline void nibUpdate(map &nib, u32 hi_lo) { u16 hi = hi_lo >> 16; u16 lo = hi_lo & 0xffff; - for (const auto pairs : nib) { + for (const auto &pairs : nib) { u32 old = pairs.first; if ((old >> 16) == hi || (old & 0xffff) == lo) { if (!nib[old | hi_lo]) { From 2e86b8524d6c5ee994eb52f4a619a17f58d2feb1 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Mon, 15 Apr 2024 11:52:33 +0300 Subject: [PATCH 26/33] first try at getting cmake to leave custom shared lib paths with the binary, with package added compilers we need to find the right std libs from the compiler we added, not the base install libs. --- cmake/osdetection.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 0e730fe3..1f35f1d3 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -4,6 +4,7 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") From d96206a12f3642f1e5009c598c5115da560993f7 Mon Sep 17 00:00:00 2001 From: Gregory Economou Date: Mon, 15 Apr 2024 14:59:08 +0300 Subject: [PATCH 27/33] united the static fat runtime dispatcher with the BSD support. --- README.md | 7 +- cmake/build_wrapper.sh | 12 +- cmake/osdetection.cmake | 35 +++--- src/dispatcher.c | 249 +++++++++++++++++++++++++++++++++------- 4 files changed, 240 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index f6a10fca..2e68d2e6 100644 --- a/README.md +++ b/README.md @@ -173,9 +173,14 @@ export CC="/usr/local/bin/gcc12" export CXX="/usr/local/bin/g++12" ``` - Then continue with the build as below. +A note about running in FreeBSD: if you built a dynamically linked binary +with an alternative compiler, the libraries specific to the compiler that +built the binary will probably not be found and the base distro libraries +in /lib will be found instead. Adjust LD_LIBRARY_PATH appropriately. For +example, with gcc12 installed from pkg, one would want to use +```export LD_LIBRARY_PATH=/usr/local/lib/gcc12/``` ## Configure & build diff --git a/cmake/build_wrapper.sh b/cmake/build_wrapper.sh index 895610c0..d0a7e923 100755 --- a/cmake/build_wrapper.sh +++ b/cmake/build_wrapper.sh @@ -15,13 +15,21 @@ SYMSFILE=$(mktemp -p /tmp ${PREFIX}_rename.syms.XXXXX) KEEPSYMS=$(mktemp -p /tmp keep.syms.XXXXX) # find the libc used by gcc LIBC_SO=$("$@" --print-file-name=libc.so.6) +NM_FLAG="-f" +if [ `uname` = "FreeBSD" ]; then + # for freebsd, we will specify the name, + # we will leave it work as is in linux + LIBC_SO=/lib/libc.so.7 + # also, in BSD, the nm flag -F corresponds to the -f flag in linux. + NM_FLAG="-F" +fi cp ${KEEPSYMS_IN} ${KEEPSYMS} # get all symbols from libc and turn them into patterns -nm -f p -g -D ${LIBC_SO} | sed -s 's/\([^ @]*\).*/^\1$/' >> ${KEEPSYMS} +nm ${NM_FLAG} p -g -D ${LIBC_SO} | sed 's/\([^ @]*\).*/^\1$/' >> ${KEEPSYMS} # build the object "$@" # rename the symbols in the object -nm -f p -g ${OUT} | cut -f1 -d' ' | grep -v -f ${KEEPSYMS} | sed -e "s/\(.*\)/\1\ ${PREFIX}_\1/" >> ${SYMSFILE} +nm ${NM_FLAG} p -g ${OUT} | cut -f1 -d' ' | grep -v -f ${KEEPSYMS} | sed -e "s/\(.*\)/\1\ ${PREFIX}_\1/" >> ${SYMSFILE} if test -s ${SYMSFILE} then objcopy --redefine-syms=${SYMSFILE} ${OUT} diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 1f35f1d3..6e7178cb 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -4,14 +4,13 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) - set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") set(NETBSD true) endif(CMAKE_SYSTEM_NAME MATCHES "NetBSD") -if (LINUX AND (ARCH_IA32 OR ARCH_X86_64)) +if (ARCH_IA32 OR ARCH_X86_64) option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" ON) else() option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" OFF) @@ -19,26 +18,22 @@ endif() if (FAT_RUNTIME) message("Checking Fat Runtime Requirements...") - if (NOT LINUX) - message(FATAL_ERROR "Fat runtime is only supported on Linux OS") - else() - if (USE_CPU_NATIVE AND FAT_RUNTIME) - message(FATAL_ERROR "Fat runtime is not compatible with Native CPU detection") - endif() + if (USE_CPU_NATIVE AND FAT_RUNTIME) + message(FATAL_ERROR "Fat runtime is not compatible with Native CPU detection") + endif() - if (NOT (ARCH_IA32 OR ARCH_X86_64 OR ARCH_AARCH64)) - message(FATAL_ERROR "Fat runtime is only supported on Intel and Aarch64 architectures") + if (NOT (ARCH_IA32 OR ARCH_X86_64 OR ARCH_AARCH64)) + message(FATAL_ERROR "Fat runtime is only supported on Intel and Aarch64 architectures") + else() + message(STATUS "Building Fat runtime for multiple microarchitectures") + message(STATUS "generator is ${CMAKE_GENERATOR}") + if (NOT (CMAKE_GENERATOR MATCHES "Unix Makefiles" OR + (CMAKE_VERSION VERSION_GREATER "3.0" AND CMAKE_GENERATOR MATCHES "Ninja"))) + message (FATAL_ERROR "Building the fat runtime requires the Unix Makefiles generator, or Ninja with CMake v3.0 or higher") else() - message(STATUS "Building Fat runtime for multiple microarchitectures") - message(STATUS "generator is ${CMAKE_GENERATOR}") - if (NOT (CMAKE_GENERATOR MATCHES "Unix Makefiles" OR - (CMAKE_VERSION VERSION_GREATER "3.0" AND CMAKE_GENERATOR MATCHES "Ninja"))) - message (FATAL_ERROR "Building the fat runtime requires the Unix Makefiles generator, or Ninja with CMake v3.0 or higher") - else() - include (${CMAKE_MODULE_PATH}/attrib.cmake) - if (NOT HAS_C_ATTR_IFUNC) - message(FATAL_ERROR "Compiler does not support ifunc attribute, cannot build fat runtime") - endif() + include (${CMAKE_MODULE_PATH}/attrib.cmake) + if (NOT HAS_C_ATTR_IFUNC) + message(FATAL_ERROR "Compiler does not support ifunc attribute, cannot build fat runtime") endif() endif() endif() diff --git a/src/dispatcher.c b/src/dispatcher.c index a817e744..b704b0a5 100644 --- a/src/dispatcher.c +++ b/src/dispatcher.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2016-2020, Intel Corporation + * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -30,6 +31,39 @@ #include "hs_common.h" #include "hs_runtime.h" #include "ue2common.h" + +/* streamlining the dispatch to eliminate runtime checking/branching: + * what we want to do is, first call to the function will run the resolve + * code and set the static resolved/dispatch pointer to point to the + * correct function. Subsequent calls to the function will go directly to + * the resolved ptr. The simplest way to accomplish this is, to + * initially set the pointer to the resolve function. + * To accomplish this in a manner invisible to the user, + * we do involve some rather ugly/confusing macros in here. + * There are four macros that assemble the code for each function + * we want to dispatch in this manner: + * CREATE_DISPATCH + * this generates the declarations for the candidate target functions, + * for the fat_dispatch function pointer, for the resolve_ function, + * points the function pointer to the resolve function, and contains + * most of the definition of the resolve function. The very end of the + * resolve function is completed by the next macro, because in the + * CREATE_DISPATCH macro we have the argument list with the arg declarations, + * which is needed to generate correct function signatures, but we + * can't generate from this, in a macro, a _call_ to one of those functions. + * CONNECT_ARGS_1 + * this macro fills in the actual call at the end of the resolve function, + * with the correct arg list. hence the name connect args. + * CONNECT_DISPATCH_2 + * this macro likewise gives up the beginning of the definition of the + * actual entry point function (the 'real name' that's called by the user) + * but again in the pass-through call, cannot invoke the target without + * getting the arg list , which is supplied by the final macro, + * CONNECT_ARGS_3 + * + */ + + #if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/arch/x86/cpuid_inline.h" #include "util/join.h" @@ -57,30 +91,38 @@ return (RTYPE)HS_ARCH_ERROR; \ } \ \ - /* resolver */ \ - static RTYPE (*JOIN(resolve_, NAME)(void))(__VA_ARGS__) { \ - if (check_avx512vbmi()) { \ - return JOIN(avx512vbmi_, NAME); \ - } \ - if (check_avx512()) { \ - return JOIN(avx512_, NAME); \ - } \ - if (check_avx2()) { \ - return JOIN(avx2_, NAME); \ - } \ - if (check_sse42() && check_popcnt()) { \ - return JOIN(corei7_, NAME); \ - } \ - if (check_ssse3()) { \ - return JOIN(core2_, NAME); \ - } \ - /* anything else is fail */ \ - return JOIN(error_, NAME); \ - } \ + /* dispatch routing pointer for this function */ \ + /* initially point it at the resolve function */ \ + static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__); \ + static RTYPE (* JOIN(fat_dispatch_, NAME))(__VA_ARGS__) = \ + &JOIN(resolve_, NAME); \ \ - /* function */ \ - HS_PUBLIC_API \ - RTYPE NAME(__VA_ARGS__) __attribute__((ifunc("resolve_" #NAME))) + /* resolver */ \ + static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__) { \ + if (check_avx512vbmi()) { \ + fat_dispatch_ ## NAME = &JOIN(avx512vbmi_, NAME); \ + } \ + else if (check_avx512()) { \ + fat_dispatch_ ## NAME = &JOIN(avx512_, NAME); \ + } \ + else if (check_avx2()) { \ + fat_dispatch_ ## NAME = &JOIN(avx2_, NAME); \ + } \ + else if (check_sse42() && check_popcnt()) { \ + fat_dispatch_ ## NAME = &JOIN(corei7_, NAME); \ + } \ + else if (check_ssse3()) { \ + fat_dispatch_ ## NAME = &JOIN(core2_, NAME); \ + } else { \ + /* anything else is fail */ \ + fat_dispatch_ ## NAME = &JOIN(error_, NAME); \ + } \ + + + +/* the rest of the function is completed in the CONNECT_ARGS_1 macro. */ + + #elif defined(ARCH_AARCH64) #include "util/arch/arm/cpuid_inline.h" @@ -97,99 +139,226 @@ return (RTYPE)HS_ARCH_ERROR; \ } \ \ - /* resolver */ \ - static RTYPE (*JOIN(resolve_, NAME)(void))(__VA_ARGS__) { \ - if (check_sve2()) { \ - return JOIN(sve2_, NAME); \ - } \ - if (check_sve()) { \ - return JOIN(sve_, NAME); \ - } \ - if (check_neon()) { \ - return JOIN(neon_, NAME); \ - } \ - /* anything else is fail */ \ - return JOIN(error_, NAME); \ - } \ + /* dispatch routing pointer for this function */ \ + /* initially point it at the resolve function */ \ + static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__); \ + static RTYPE (* JOIN(fat_dispatch_, NAME))(__VA_ARGS__) = \ + &JOIN(resolve_, NAME); \ \ - /* function */ \ - HS_PUBLIC_API \ - RTYPE NAME(__VA_ARGS__) __attribute__((ifunc("resolve_" #NAME))) + /* resolver */ \ + static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__) { \ + if (check_sve2()) { \ + fat_dispatch_ ## NAME = &JOIN(sve2_, NAME); \ + } \ + else if (check_sve()) { \ + fat_dispatch_ ## NAME = &JOIN(sve_, NAME); \ + } \ + else if (check_neon()) { \ + fat_dispatch_ ## NAME = &JOIN(neon_, NAME); \ + } else { \ + /* anything else is fail */ \ + fat_dispatch_ ## NAME = &JOIN(error_, NAME); \ + } \ + + +/* the rest of the function is completed in the CONNECT_ARGS_1 macro. */ + #endif + +#define CONNECT_ARGS_1(RTYPE, NAME, ...) \ + return (*fat_dispatch_ ## NAME)(__VA_ARGS__); \ + } \ + + +#define CONNECT_DISPATCH_2(RTYPE, NAME, ...) \ + /* new function */ \ + HS_PUBLIC_API \ + RTYPE NAME(__VA_ARGS__) { \ + + +#define CONNECT_ARGS_3(RTYPE, NAME, ...) \ + return (*fat_dispatch_ ## NAME)(__VA_ARGS__); \ + } \ + + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" + +/* this gets a bit ugly to compose the static redirect functions, + * as we necessarily need first the typed arg list and then just the arg + * names, twice in a row, to define the redirect function and the + * dispatch function call */ + CREATE_DISPATCH(hs_error_t, hs_scan, const hs_database_t *db, const char *data, unsigned length, unsigned flags, hs_scratch_t *scratch, match_event_handler onEvent, void *userCtx); +CONNECT_ARGS_1(hs_error_t, hs_scan, db, data, length, flags, scratch, onEvent, userCtx); +CONNECT_DISPATCH_2(hs_error_t, hs_scan, const hs_database_t *db, const char *data, + unsigned length, unsigned flags, hs_scratch_t *scratch, + match_event_handler onEvent, void *userCtx); +CONNECT_ARGS_3(hs_error_t, hs_scan, db, data, length, flags, scratch, onEvent, userCtx); CREATE_DISPATCH(hs_error_t, hs_stream_size, const hs_database_t *database, size_t *stream_size); +CONNECT_ARGS_1(hs_error_t, hs_stream_size, database, stream_size); +CONNECT_DISPATCH_2(hs_error_t, hs_stream_size, const hs_database_t *database, + size_t *stream_size); +CONNECT_ARGS_3(hs_error_t, hs_stream_size, database, stream_size); CREATE_DISPATCH(hs_error_t, hs_database_size, const hs_database_t *db, size_t *size); +CONNECT_ARGS_1(hs_error_t, hs_database_size, db, size); +CONNECT_DISPATCH_2(hs_error_t, hs_database_size, const hs_database_t *db, + size_t *size); +CONNECT_ARGS_3(hs_error_t, hs_database_size, db, size); + CREATE_DISPATCH(hs_error_t, dbIsValid, const hs_database_t *db); +CONNECT_ARGS_1(hs_error_t, dbIsValid, db); +CONNECT_DISPATCH_2(hs_error_t, dbIsValid, const hs_database_t *db); +CONNECT_ARGS_3(hs_error_t, dbIsValid, db); + CREATE_DISPATCH(hs_error_t, hs_free_database, hs_database_t *db); +CONNECT_ARGS_1(hs_error_t, hs_free_database, db); +CONNECT_DISPATCH_2(hs_error_t, hs_free_database, hs_database_t *db); +CONNECT_ARGS_3(hs_error_t, hs_free_database, db); CREATE_DISPATCH(hs_error_t, hs_open_stream, const hs_database_t *db, unsigned int flags, hs_stream_t **stream); +CONNECT_ARGS_1(hs_error_t, hs_open_stream, db, flags, stream); +CONNECT_DISPATCH_2(hs_error_t, hs_open_stream, const hs_database_t *db, + unsigned int flags, hs_stream_t **stream); +CONNECT_ARGS_3(hs_error_t, hs_open_stream, db, flags, stream); CREATE_DISPATCH(hs_error_t, hs_scan_stream, hs_stream_t *id, const char *data, unsigned int length, unsigned int flags, hs_scratch_t *scratch, match_event_handler onEvent, void *ctxt); +CONNECT_ARGS_1(hs_error_t, hs_scan_stream, id, data, length, flags, scratch, onEvent, ctxt); +CONNECT_DISPATCH_2(hs_error_t, hs_scan_stream, hs_stream_t *id, const char *data, + unsigned int length, unsigned int flags, hs_scratch_t *scratch, + match_event_handler onEvent, void *ctxt); +CONNECT_ARGS_3(hs_error_t, hs_scan_stream, id, data, length, flags, scratch, onEvent, ctxt); CREATE_DISPATCH(hs_error_t, hs_close_stream, hs_stream_t *id, hs_scratch_t *scratch, match_event_handler onEvent, void *ctxt); +CONNECT_ARGS_1(hs_error_t, hs_close_stream, id, scratch, onEvent, ctxt); +CONNECT_DISPATCH_2(hs_error_t, hs_close_stream, hs_stream_t *id, + hs_scratch_t *scratch, match_event_handler onEvent, void *ctxt); +CONNECT_ARGS_3(hs_error_t, hs_close_stream, id, scratch, onEvent, ctxt); CREATE_DISPATCH(hs_error_t, hs_scan_vector, const hs_database_t *db, const char *const *data, const unsigned int *length, unsigned int count, unsigned int flags, hs_scratch_t *scratch, match_event_handler onevent, void *context); +CONNECT_ARGS_1(hs_error_t, hs_scan_vector, db, data, length, count, flags, scratch, onevent, context); +CONNECT_DISPATCH_2(hs_error_t, hs_scan_vector, const hs_database_t *db, + const char *const *data, const unsigned int *length, + unsigned int count, unsigned int flags, hs_scratch_t *scratch, + match_event_handler onevent, void *context); +CONNECT_ARGS_3(hs_error_t, hs_scan_vector, db, data, length, count, flags, scratch, onevent, context); CREATE_DISPATCH(hs_error_t, hs_database_info, const hs_database_t *db, char **info); +CONNECT_ARGS_1(hs_error_t, hs_database_info, db, info); +CONNECT_DISPATCH_2(hs_error_t, hs_database_info, const hs_database_t *db, char **info); +CONNECT_ARGS_3(hs_error_t, hs_database_info, db, info); CREATE_DISPATCH(hs_error_t, hs_copy_stream, hs_stream_t **to_id, const hs_stream_t *from_id); +CONNECT_ARGS_1(hs_error_t, hs_copy_stream, to_id, from_id); +CONNECT_DISPATCH_2(hs_error_t, hs_copy_stream, hs_stream_t **to_id, + const hs_stream_t *from_id); +CONNECT_ARGS_3(hs_error_t, hs_copy_stream, to_id, from_id); CREATE_DISPATCH(hs_error_t, hs_reset_stream, hs_stream_t *id, unsigned int flags, hs_scratch_t *scratch, match_event_handler onEvent, void *context); +CONNECT_ARGS_1(hs_error_t, hs_reset_stream, id, flags, scratch, onEvent, context); +CONNECT_DISPATCH_2(hs_error_t, hs_reset_stream, hs_stream_t *id, + unsigned int flags, hs_scratch_t *scratch, + match_event_handler onEvent, void *context); +CONNECT_ARGS_3(hs_error_t, hs_reset_stream, id, flags, scratch, onEvent, context); CREATE_DISPATCH(hs_error_t, hs_reset_and_copy_stream, hs_stream_t *to_id, const hs_stream_t *from_id, hs_scratch_t *scratch, match_event_handler onEvent, void *context); +CONNECT_ARGS_1(hs_error_t, hs_reset_and_copy_stream, to_id, from_id, scratch, onEvent, context); +CONNECT_DISPATCH_2(hs_error_t, hs_reset_and_copy_stream, hs_stream_t *to_id, + const hs_stream_t *from_id, hs_scratch_t *scratch, + match_event_handler onEvent, void *context); +CONNECT_ARGS_3(hs_error_t, hs_reset_and_copy_stream, to_id, from_id, scratch, onEvent, context); CREATE_DISPATCH(hs_error_t, hs_serialize_database, const hs_database_t *db, char **bytes, size_t *length); +CONNECT_ARGS_1(hs_error_t, hs_serialize_database, db, bytes, length); +CONNECT_DISPATCH_2(hs_error_t, hs_serialize_database, const hs_database_t *db, + char **bytes, size_t *length); +CONNECT_ARGS_3(hs_error_t, hs_serialize_database, db, bytes, length); CREATE_DISPATCH(hs_error_t, hs_deserialize_database, const char *bytes, const size_t length, hs_database_t **db); +CONNECT_ARGS_1(hs_error_t, hs_deserialize_database, bytes, length, db); +CONNECT_DISPATCH_2(hs_error_t, hs_deserialize_database, const char *bytes, + const size_t length, hs_database_t **db); +CONNECT_ARGS_3(hs_error_t, hs_deserialize_database, bytes, length, db); CREATE_DISPATCH(hs_error_t, hs_deserialize_database_at, const char *bytes, const size_t length, hs_database_t *db); +CONNECT_ARGS_1(hs_error_t, hs_deserialize_database_at, bytes, length, db); +CONNECT_DISPATCH_2(hs_error_t, hs_deserialize_database_at, const char *bytes, + const size_t length, hs_database_t *db); +CONNECT_ARGS_3(hs_error_t, hs_deserialize_database_at, bytes, length, db); CREATE_DISPATCH(hs_error_t, hs_serialized_database_info, const char *bytes, size_t length, char **info); +CONNECT_ARGS_1(hs_error_t, hs_serialized_database_info, bytes, length, info); +CONNECT_DISPATCH_2(hs_error_t, hs_serialized_database_info, const char *bytes, + size_t length, char **info); +CONNECT_ARGS_3(hs_error_t, hs_serialized_database_info, bytes, length, info); CREATE_DISPATCH(hs_error_t, hs_serialized_database_size, const char *bytes, const size_t length, size_t *deserialized_size); +CONNECT_ARGS_1(hs_error_t, hs_serialized_database_size, bytes, length, deserialized_size); +CONNECT_DISPATCH_2(hs_error_t, hs_serialized_database_size, const char *bytes, + const size_t length, size_t *deserialized_size); +CONNECT_ARGS_3(hs_error_t, hs_serialized_database_size, bytes, length, deserialized_size); CREATE_DISPATCH(hs_error_t, hs_compress_stream, const hs_stream_t *stream, char *buf, size_t buf_space, size_t *used_space); +CONNECT_ARGS_1(hs_error_t, hs_compress_stream, stream, + buf, buf_space, used_space); +CONNECT_DISPATCH_2(hs_error_t, hs_compress_stream, const hs_stream_t *stream, + char *buf, size_t buf_space, size_t *used_space); +CONNECT_ARGS_3(hs_error_t, hs_compress_stream, stream, + buf, buf_space, used_space); CREATE_DISPATCH(hs_error_t, hs_expand_stream, const hs_database_t *db, hs_stream_t **stream, const char *buf,size_t buf_size); +CONNECT_ARGS_1(hs_error_t, hs_expand_stream, db, stream, buf,buf_size); +CONNECT_DISPATCH_2(hs_error_t, hs_expand_stream, const hs_database_t *db, + hs_stream_t **stream, const char *buf,size_t buf_size); +CONNECT_ARGS_3(hs_error_t, hs_expand_stream, db, stream, buf,buf_size); CREATE_DISPATCH(hs_error_t, hs_reset_and_expand_stream, hs_stream_t *to_stream, const char *buf, size_t buf_size, hs_scratch_t *scratch, match_event_handler onEvent, void *context); +CONNECT_ARGS_1(hs_error_t, hs_reset_and_expand_stream, to_stream, + buf, buf_size, scratch, onEvent, context); +CONNECT_DISPATCH_2(hs_error_t, hs_reset_and_expand_stream, hs_stream_t *to_stream, + const char *buf, size_t buf_size, hs_scratch_t *scratch, + match_event_handler onEvent, void *context); +CONNECT_ARGS_3(hs_error_t, hs_reset_and_expand_stream, to_stream, + buf, buf_size, scratch, onEvent, context); /** INTERNALS **/ CREATE_DISPATCH(u32, Crc32c_ComputeBuf, u32 inCrc32, const void *buf, size_t bufLen); +CONNECT_ARGS_1(u32, Crc32c_ComputeBuf, inCrc32, buf, bufLen); +CONNECT_DISPATCH_2(u32, Crc32c_ComputeBuf, u32 inCrc32, const void *buf, size_t bufLen); +CONNECT_ARGS_3(u32, Crc32c_ComputeBuf, inCrc32, buf, bufLen); #pragma GCC diagnostic pop #pragma GCC diagnostic pop + From b31243892953c9495a49ed1e38f28383e7a74448 Mon Sep 17 00:00:00 2001 From: Gregory Economou Date: Mon, 15 Apr 2024 16:50:58 +0300 Subject: [PATCH 28/33] probably fixed the bit about not finding the right libs in some bsd installs --- cmake/osdetection.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 6e7178cb..874031b9 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -4,6 +4,9 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true) + set(CMAKE_SKIP_INSTALL_RPATH false) + set(CMAKE_SKIP_RPATH false) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") From 12e95b2c5c520f59b6330634d92bc36081d66230 Mon Sep 17 00:00:00 2001 From: Gregory Economou Date: Tue, 16 Apr 2024 10:17:53 +0300 Subject: [PATCH 29/33] bit hacky but it works for setting rpath in freebsd --- cmake/osdetection.cmake | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 874031b9..23f6a5d5 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -4,9 +4,12 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) - set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true) - set(CMAKE_SKIP_INSTALL_RPATH false) - set(CMAKE_SKIP_RPATH false) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + if(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc12") + set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc12") + elseif(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc13") + set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc13") + endif() endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") From 00b1a50977747f3ac8836b160d9450956a332477 Mon Sep 17 00:00:00 2001 From: Gregory Economou Date: Tue, 16 Apr 2024 13:09:05 +0300 Subject: [PATCH 30/33] made the rpath finding a bit more flexible than just hardcoded gcc12 and gcc13 --- cmake/osdetection.cmake | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 23f6a5d5..c163a7ff 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -5,11 +5,7 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - if(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc12") - set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc12") - elseif(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc13") - set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc13") - endif() + execute_process(COMMAND "sh" "-c" "realpath `dirname \\`${CMAKE_C_COMPILER} --print-file-name=libgcc_s.so\\``" OUTPUT_VARIABLE CMAKE_BUILD_RPATH) endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") From 81722040f8ecc576cf8e6af922e6f033b9d45657 Mon Sep 17 00:00:00 2001 From: Gregory Economou Date: Tue, 16 Apr 2024 15:04:20 +0300 Subject: [PATCH 31/33] cmake adds newlines to variables set from command output, trashing the usefulness of the flexible method of setting the rpath var. back to the clunky manual setting. --- cmake/osdetection.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index c163a7ff..23f6a5d5 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -5,7 +5,11 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - execute_process(COMMAND "sh" "-c" "realpath `dirname \\`${CMAKE_C_COMPILER} --print-file-name=libgcc_s.so\\``" OUTPUT_VARIABLE CMAKE_BUILD_RPATH) + if(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc12") + set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc12") + elseif(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc13") + set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc13") + endif() endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") From b6b0ab1a9bcb4c32a6851e9435372385f7e6f1ee Mon Sep 17 00:00:00 2001 From: Gregory Economou Date: Tue, 16 Apr 2024 15:18:58 +0300 Subject: [PATCH 32/33] added a fixme for the clunky rpath setting logic --- cmake/osdetection.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/osdetection.cmake b/cmake/osdetection.cmake index 23f6a5d5..3369447a 100644 --- a/cmake/osdetection.cmake +++ b/cmake/osdetection.cmake @@ -5,6 +5,7 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(FREEBSD true) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + #FIXME: find a nicer and more general way of doing this if(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc12") set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc12") elseif(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc13") From 6e306a508edb097378f1b9f2c88a5ac90fd3dff4 Mon Sep 17 00:00:00 2001 From: "G.E." Date: Tue, 16 Apr 2024 17:43:11 +0300 Subject: [PATCH 33/33] removing the dispatcher.c changes from this branch/PR --- src/dispatcher.c | 229 +++++++---------------------------------------- 1 file changed, 30 insertions(+), 199 deletions(-) diff --git a/src/dispatcher.c b/src/dispatcher.c index b704b0a5..a817e744 100644 --- a/src/dispatcher.c +++ b/src/dispatcher.c @@ -1,6 +1,5 @@ /* * Copyright (c) 2016-2020, Intel Corporation - * Copyright (c) 2024, VectorCamp PC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,39 +30,6 @@ #include "hs_common.h" #include "hs_runtime.h" #include "ue2common.h" - -/* streamlining the dispatch to eliminate runtime checking/branching: - * what we want to do is, first call to the function will run the resolve - * code and set the static resolved/dispatch pointer to point to the - * correct function. Subsequent calls to the function will go directly to - * the resolved ptr. The simplest way to accomplish this is, to - * initially set the pointer to the resolve function. - * To accomplish this in a manner invisible to the user, - * we do involve some rather ugly/confusing macros in here. - * There are four macros that assemble the code for each function - * we want to dispatch in this manner: - * CREATE_DISPATCH - * this generates the declarations for the candidate target functions, - * for the fat_dispatch function pointer, for the resolve_ function, - * points the function pointer to the resolve function, and contains - * most of the definition of the resolve function. The very end of the - * resolve function is completed by the next macro, because in the - * CREATE_DISPATCH macro we have the argument list with the arg declarations, - * which is needed to generate correct function signatures, but we - * can't generate from this, in a macro, a _call_ to one of those functions. - * CONNECT_ARGS_1 - * this macro fills in the actual call at the end of the resolve function, - * with the correct arg list. hence the name connect args. - * CONNECT_DISPATCH_2 - * this macro likewise gives up the beginning of the definition of the - * actual entry point function (the 'real name' that's called by the user) - * but again in the pass-through call, cannot invoke the target without - * getting the arg list , which is supplied by the final macro, - * CONNECT_ARGS_3 - * - */ - - #if defined(ARCH_IA32) || defined(ARCH_X86_64) #include "util/arch/x86/cpuid_inline.h" #include "util/join.h" @@ -91,38 +57,30 @@ return (RTYPE)HS_ARCH_ERROR; \ } \ \ - /* dispatch routing pointer for this function */ \ - /* initially point it at the resolve function */ \ - static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__); \ - static RTYPE (* JOIN(fat_dispatch_, NAME))(__VA_ARGS__) = \ - &JOIN(resolve_, NAME); \ - \ /* resolver */ \ - static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__) { \ + static RTYPE (*JOIN(resolve_, NAME)(void))(__VA_ARGS__) { \ if (check_avx512vbmi()) { \ - fat_dispatch_ ## NAME = &JOIN(avx512vbmi_, NAME); \ + return JOIN(avx512vbmi_, NAME); \ } \ - else if (check_avx512()) { \ - fat_dispatch_ ## NAME = &JOIN(avx512_, NAME); \ + if (check_avx512()) { \ + return JOIN(avx512_, NAME); \ } \ - else if (check_avx2()) { \ - fat_dispatch_ ## NAME = &JOIN(avx2_, NAME); \ + if (check_avx2()) { \ + return JOIN(avx2_, NAME); \ } \ - else if (check_sse42() && check_popcnt()) { \ - fat_dispatch_ ## NAME = &JOIN(corei7_, NAME); \ + if (check_sse42() && check_popcnt()) { \ + return JOIN(corei7_, NAME); \ } \ - else if (check_ssse3()) { \ - fat_dispatch_ ## NAME = &JOIN(core2_, NAME); \ - } else { \ - /* anything else is fail */ \ - fat_dispatch_ ## NAME = &JOIN(error_, NAME); \ + if (check_ssse3()) { \ + return JOIN(core2_, NAME); \ } \ - - - -/* the rest of the function is completed in the CONNECT_ARGS_1 macro. */ - - + /* anything else is fail */ \ + return JOIN(error_, NAME); \ + } \ + \ + /* function */ \ + HS_PUBLIC_API \ + RTYPE NAME(__VA_ARGS__) __attribute__((ifunc("resolve_" #NAME))) #elif defined(ARCH_AARCH64) #include "util/arch/arm/cpuid_inline.h" @@ -139,226 +97,99 @@ return (RTYPE)HS_ARCH_ERROR; \ } \ \ - /* dispatch routing pointer for this function */ \ - /* initially point it at the resolve function */ \ - static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__); \ - static RTYPE (* JOIN(fat_dispatch_, NAME))(__VA_ARGS__) = \ - &JOIN(resolve_, NAME); \ - \ /* resolver */ \ - static RTYPE JOIN(resolve_, NAME)(__VA_ARGS__) { \ + static RTYPE (*JOIN(resolve_, NAME)(void))(__VA_ARGS__) { \ if (check_sve2()) { \ - fat_dispatch_ ## NAME = &JOIN(sve2_, NAME); \ + return JOIN(sve2_, NAME); \ } \ - else if (check_sve()) { \ - fat_dispatch_ ## NAME = &JOIN(sve_, NAME); \ + if (check_sve()) { \ + return JOIN(sve_, NAME); \ } \ - else if (check_neon()) { \ - fat_dispatch_ ## NAME = &JOIN(neon_, NAME); \ - } else { \ - /* anything else is fail */ \ - fat_dispatch_ ## NAME = &JOIN(error_, NAME); \ + if (check_neon()) { \ + return JOIN(neon_, NAME); \ } \ - - -/* the rest of the function is completed in the CONNECT_ARGS_1 macro. */ - + /* anything else is fail */ \ + return JOIN(error_, NAME); \ + } \ + \ + /* function */ \ + HS_PUBLIC_API \ + RTYPE NAME(__VA_ARGS__) __attribute__((ifunc("resolve_" #NAME))) #endif - -#define CONNECT_ARGS_1(RTYPE, NAME, ...) \ - return (*fat_dispatch_ ## NAME)(__VA_ARGS__); \ - } \ - - -#define CONNECT_DISPATCH_2(RTYPE, NAME, ...) \ - /* new function */ \ - HS_PUBLIC_API \ - RTYPE NAME(__VA_ARGS__) { \ - - -#define CONNECT_ARGS_3(RTYPE, NAME, ...) \ - return (*fat_dispatch_ ## NAME)(__VA_ARGS__); \ - } \ - - #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" - -/* this gets a bit ugly to compose the static redirect functions, - * as we necessarily need first the typed arg list and then just the arg - * names, twice in a row, to define the redirect function and the - * dispatch function call */ - CREATE_DISPATCH(hs_error_t, hs_scan, const hs_database_t *db, const char *data, unsigned length, unsigned flags, hs_scratch_t *scratch, match_event_handler onEvent, void *userCtx); -CONNECT_ARGS_1(hs_error_t, hs_scan, db, data, length, flags, scratch, onEvent, userCtx); -CONNECT_DISPATCH_2(hs_error_t, hs_scan, const hs_database_t *db, const char *data, - unsigned length, unsigned flags, hs_scratch_t *scratch, - match_event_handler onEvent, void *userCtx); -CONNECT_ARGS_3(hs_error_t, hs_scan, db, data, length, flags, scratch, onEvent, userCtx); CREATE_DISPATCH(hs_error_t, hs_stream_size, const hs_database_t *database, size_t *stream_size); -CONNECT_ARGS_1(hs_error_t, hs_stream_size, database, stream_size); -CONNECT_DISPATCH_2(hs_error_t, hs_stream_size, const hs_database_t *database, - size_t *stream_size); -CONNECT_ARGS_3(hs_error_t, hs_stream_size, database, stream_size); CREATE_DISPATCH(hs_error_t, hs_database_size, const hs_database_t *db, size_t *size); -CONNECT_ARGS_1(hs_error_t, hs_database_size, db, size); -CONNECT_DISPATCH_2(hs_error_t, hs_database_size, const hs_database_t *db, - size_t *size); -CONNECT_ARGS_3(hs_error_t, hs_database_size, db, size); - CREATE_DISPATCH(hs_error_t, dbIsValid, const hs_database_t *db); -CONNECT_ARGS_1(hs_error_t, dbIsValid, db); -CONNECT_DISPATCH_2(hs_error_t, dbIsValid, const hs_database_t *db); -CONNECT_ARGS_3(hs_error_t, dbIsValid, db); - CREATE_DISPATCH(hs_error_t, hs_free_database, hs_database_t *db); -CONNECT_ARGS_1(hs_error_t, hs_free_database, db); -CONNECT_DISPATCH_2(hs_error_t, hs_free_database, hs_database_t *db); -CONNECT_ARGS_3(hs_error_t, hs_free_database, db); CREATE_DISPATCH(hs_error_t, hs_open_stream, const hs_database_t *db, unsigned int flags, hs_stream_t **stream); -CONNECT_ARGS_1(hs_error_t, hs_open_stream, db, flags, stream); -CONNECT_DISPATCH_2(hs_error_t, hs_open_stream, const hs_database_t *db, - unsigned int flags, hs_stream_t **stream); -CONNECT_ARGS_3(hs_error_t, hs_open_stream, db, flags, stream); CREATE_DISPATCH(hs_error_t, hs_scan_stream, hs_stream_t *id, const char *data, unsigned int length, unsigned int flags, hs_scratch_t *scratch, match_event_handler onEvent, void *ctxt); -CONNECT_ARGS_1(hs_error_t, hs_scan_stream, id, data, length, flags, scratch, onEvent, ctxt); -CONNECT_DISPATCH_2(hs_error_t, hs_scan_stream, hs_stream_t *id, const char *data, - unsigned int length, unsigned int flags, hs_scratch_t *scratch, - match_event_handler onEvent, void *ctxt); -CONNECT_ARGS_3(hs_error_t, hs_scan_stream, id, data, length, flags, scratch, onEvent, ctxt); CREATE_DISPATCH(hs_error_t, hs_close_stream, hs_stream_t *id, hs_scratch_t *scratch, match_event_handler onEvent, void *ctxt); -CONNECT_ARGS_1(hs_error_t, hs_close_stream, id, scratch, onEvent, ctxt); -CONNECT_DISPATCH_2(hs_error_t, hs_close_stream, hs_stream_t *id, - hs_scratch_t *scratch, match_event_handler onEvent, void *ctxt); -CONNECT_ARGS_3(hs_error_t, hs_close_stream, id, scratch, onEvent, ctxt); CREATE_DISPATCH(hs_error_t, hs_scan_vector, const hs_database_t *db, const char *const *data, const unsigned int *length, unsigned int count, unsigned int flags, hs_scratch_t *scratch, match_event_handler onevent, void *context); -CONNECT_ARGS_1(hs_error_t, hs_scan_vector, db, data, length, count, flags, scratch, onevent, context); -CONNECT_DISPATCH_2(hs_error_t, hs_scan_vector, const hs_database_t *db, - const char *const *data, const unsigned int *length, - unsigned int count, unsigned int flags, hs_scratch_t *scratch, - match_event_handler onevent, void *context); -CONNECT_ARGS_3(hs_error_t, hs_scan_vector, db, data, length, count, flags, scratch, onevent, context); CREATE_DISPATCH(hs_error_t, hs_database_info, const hs_database_t *db, char **info); -CONNECT_ARGS_1(hs_error_t, hs_database_info, db, info); -CONNECT_DISPATCH_2(hs_error_t, hs_database_info, const hs_database_t *db, char **info); -CONNECT_ARGS_3(hs_error_t, hs_database_info, db, info); CREATE_DISPATCH(hs_error_t, hs_copy_stream, hs_stream_t **to_id, const hs_stream_t *from_id); -CONNECT_ARGS_1(hs_error_t, hs_copy_stream, to_id, from_id); -CONNECT_DISPATCH_2(hs_error_t, hs_copy_stream, hs_stream_t **to_id, - const hs_stream_t *from_id); -CONNECT_ARGS_3(hs_error_t, hs_copy_stream, to_id, from_id); CREATE_DISPATCH(hs_error_t, hs_reset_stream, hs_stream_t *id, unsigned int flags, hs_scratch_t *scratch, match_event_handler onEvent, void *context); -CONNECT_ARGS_1(hs_error_t, hs_reset_stream, id, flags, scratch, onEvent, context); -CONNECT_DISPATCH_2(hs_error_t, hs_reset_stream, hs_stream_t *id, - unsigned int flags, hs_scratch_t *scratch, - match_event_handler onEvent, void *context); -CONNECT_ARGS_3(hs_error_t, hs_reset_stream, id, flags, scratch, onEvent, context); CREATE_DISPATCH(hs_error_t, hs_reset_and_copy_stream, hs_stream_t *to_id, const hs_stream_t *from_id, hs_scratch_t *scratch, match_event_handler onEvent, void *context); -CONNECT_ARGS_1(hs_error_t, hs_reset_and_copy_stream, to_id, from_id, scratch, onEvent, context); -CONNECT_DISPATCH_2(hs_error_t, hs_reset_and_copy_stream, hs_stream_t *to_id, - const hs_stream_t *from_id, hs_scratch_t *scratch, - match_event_handler onEvent, void *context); -CONNECT_ARGS_3(hs_error_t, hs_reset_and_copy_stream, to_id, from_id, scratch, onEvent, context); CREATE_DISPATCH(hs_error_t, hs_serialize_database, const hs_database_t *db, char **bytes, size_t *length); -CONNECT_ARGS_1(hs_error_t, hs_serialize_database, db, bytes, length); -CONNECT_DISPATCH_2(hs_error_t, hs_serialize_database, const hs_database_t *db, - char **bytes, size_t *length); -CONNECT_ARGS_3(hs_error_t, hs_serialize_database, db, bytes, length); CREATE_DISPATCH(hs_error_t, hs_deserialize_database, const char *bytes, const size_t length, hs_database_t **db); -CONNECT_ARGS_1(hs_error_t, hs_deserialize_database, bytes, length, db); -CONNECT_DISPATCH_2(hs_error_t, hs_deserialize_database, const char *bytes, - const size_t length, hs_database_t **db); -CONNECT_ARGS_3(hs_error_t, hs_deserialize_database, bytes, length, db); CREATE_DISPATCH(hs_error_t, hs_deserialize_database_at, const char *bytes, const size_t length, hs_database_t *db); -CONNECT_ARGS_1(hs_error_t, hs_deserialize_database_at, bytes, length, db); -CONNECT_DISPATCH_2(hs_error_t, hs_deserialize_database_at, const char *bytes, - const size_t length, hs_database_t *db); -CONNECT_ARGS_3(hs_error_t, hs_deserialize_database_at, bytes, length, db); CREATE_DISPATCH(hs_error_t, hs_serialized_database_info, const char *bytes, size_t length, char **info); -CONNECT_ARGS_1(hs_error_t, hs_serialized_database_info, bytes, length, info); -CONNECT_DISPATCH_2(hs_error_t, hs_serialized_database_info, const char *bytes, - size_t length, char **info); -CONNECT_ARGS_3(hs_error_t, hs_serialized_database_info, bytes, length, info); CREATE_DISPATCH(hs_error_t, hs_serialized_database_size, const char *bytes, const size_t length, size_t *deserialized_size); -CONNECT_ARGS_1(hs_error_t, hs_serialized_database_size, bytes, length, deserialized_size); -CONNECT_DISPATCH_2(hs_error_t, hs_serialized_database_size, const char *bytes, - const size_t length, size_t *deserialized_size); -CONNECT_ARGS_3(hs_error_t, hs_serialized_database_size, bytes, length, deserialized_size); CREATE_DISPATCH(hs_error_t, hs_compress_stream, const hs_stream_t *stream, char *buf, size_t buf_space, size_t *used_space); -CONNECT_ARGS_1(hs_error_t, hs_compress_stream, stream, - buf, buf_space, used_space); -CONNECT_DISPATCH_2(hs_error_t, hs_compress_stream, const hs_stream_t *stream, - char *buf, size_t buf_space, size_t *used_space); -CONNECT_ARGS_3(hs_error_t, hs_compress_stream, stream, - buf, buf_space, used_space); CREATE_DISPATCH(hs_error_t, hs_expand_stream, const hs_database_t *db, hs_stream_t **stream, const char *buf,size_t buf_size); -CONNECT_ARGS_1(hs_error_t, hs_expand_stream, db, stream, buf,buf_size); -CONNECT_DISPATCH_2(hs_error_t, hs_expand_stream, const hs_database_t *db, - hs_stream_t **stream, const char *buf,size_t buf_size); -CONNECT_ARGS_3(hs_error_t, hs_expand_stream, db, stream, buf,buf_size); CREATE_DISPATCH(hs_error_t, hs_reset_and_expand_stream, hs_stream_t *to_stream, const char *buf, size_t buf_size, hs_scratch_t *scratch, match_event_handler onEvent, void *context); -CONNECT_ARGS_1(hs_error_t, hs_reset_and_expand_stream, to_stream, - buf, buf_size, scratch, onEvent, context); -CONNECT_DISPATCH_2(hs_error_t, hs_reset_and_expand_stream, hs_stream_t *to_stream, - const char *buf, size_t buf_size, hs_scratch_t *scratch, - match_event_handler onEvent, void *context); -CONNECT_ARGS_3(hs_error_t, hs_reset_and_expand_stream, to_stream, - buf, buf_size, scratch, onEvent, context); /** INTERNALS **/ CREATE_DISPATCH(u32, Crc32c_ComputeBuf, u32 inCrc32, const void *buf, size_t bufLen); -CONNECT_ARGS_1(u32, Crc32c_ComputeBuf, inCrc32, buf, bufLen); -CONNECT_DISPATCH_2(u32, Crc32c_ComputeBuf, u32 inCrc32, const void *buf, size_t bufLen); -CONNECT_ARGS_3(u32, Crc32c_ComputeBuf, inCrc32, buf, bufLen); #pragma GCC diagnostic pop #pragma GCC diagnostic pop -