Merge pull request #232 from isildur-g/develop

minor changes to build in BSD (Net and Free)

README.md

@@ -113,6 +113,75 @@ Assuming an existing HomeBrew installation:
 % brew install boost cmake gcc libpcap pkg-config ragel sqlite
 ```
 
+### *BSD
+In NetBSD you will almost certainly need to have a newer compiler installed. 
+Also you will need to install cmake, sqlite, boost and ragel. 
+Also, libpcap is necessary for some of the benchmarks, so let's install that 
+as well.
+When using pkgsrc, you would typically do this using something
+similar to
+```
+pkg_add gcc12-12.3.0.tgz
+pkg_add boost-headers-1.83.0.tgz  boost-jam-1.83.0.tgz      boost-libs-1.83.0nb1.tgz
+pkg_add ragel-6.10.tgz
+pkg_add cmake-3.28.1.tgz
+pkg_add sqlite3-3.44.2.tgz
+pkg_add libpcap-1.10.4.tgz
+```
+Version numbers etc will of course vary. One would either download the
+binary packages or build them using pkgsrc. There exist some NetBSD pkg 
+tools like ```pkgin``` which help download e.g. dependencies as binary packages,
+but overall NetBSD leaves a lot of detail exposed to the user.
+The main package system used in NetBSD is pkgsrc and one will probably
+want to read up more about it than is in the scope of this document.
+See https://www.netbsd.org/docs/software/packages.html for more information.
+
+This will not replace the compiler in the standard base distribution, and
+cmake will probably find the base dist's compiler when it checks automatically.
+Using the example of gcc12 from pkgsrc, one will need to set two
+environment variables before starting: 
+```
+export CC="/usr/pkg/gcc12/bin/cc"
+export CXX="/usr/pkg/gcc12/bin/g++"
+```
+
+In FreeBSD similarly, you might want to install a different compiler.
+You will also, as in NetBSD, need to install cmake, sqlite, boost and ragel packages.
+Using the example of gcc12 from pkg:
+installing the desired compiler: 
+```
+pkg install gcc12
+pkg install boost-all
+pkg install ragel
+pkg install cmake
+pkg install sqlite
+pkg install libpcap
+pkg install ccache
+```
+and then before beginning the cmake and build process, set
+the environment variables to point to this compiler: 
+```
+export CC="/usr/local/bin/gcc"
+export CXX="/usr/local/bin/g++"
+```
+
+A further note in FreeBSD, on the PowerPC and ARM platforms, 
+the gcc12 package installs to a slightly different name, on FreeBSD/ppc, 
+gcc12 will be found using: 
+```
+export CC="/usr/local/bin/gcc12"
+export CXX="/usr/local/bin/g++12"
+```
+
+Then continue with the build as below. 
+
+A note about running in FreeBSD: if you built a dynamically linked binary
+with an alternative compiler, the libraries specific to the compiler that
+built the binary will probably not be found and the base distro libraries
+in /lib will be found instead. Adjust LD_LIBRARY_PATH appropriately. For
+example, with gcc12 installed from pkg, one would want to use
+```export LD_LIBRARY_PATH=/usr/local/lib/gcc12/``` 
+
 ## Configure & build
 
 In order to configure with `cmake` first create and cd into a build directory:

cmake/build_wrapper.sh

@@ -15,13 +15,21 @@ SYMSFILE=$(mktemp -p /tmp ${PREFIX}_rename.syms.XXXXX)
 KEEPSYMS=$(mktemp -p /tmp keep.syms.XXXXX)
 # find the libc used by gcc
 LIBC_SO=$("$@" --print-file-name=libc.so.6)
+NM_FLAG="-f"
+if [ `uname` = "FreeBSD" ]; then
+    # for freebsd, we will specify the name, 
+    # we will leave it work as is in linux
+    LIBC_SO=/lib/libc.so.7
+    # also, in BSD, the nm flag -F corresponds to the -f flag in linux.
+    NM_FLAG="-F"
+fi
 cp ${KEEPSYMS_IN} ${KEEPSYMS}
 # get all symbols from libc and turn them into patterns
-nm -f p -g -D ${LIBC_SO} | sed -s 's/\([^ @]*\).*/^\1$/' >> ${KEEPSYMS}
+nm ${NM_FLAG} p -g -D ${LIBC_SO} | sed 's/\([^ @]*\).*/^\1$/' >> ${KEEPSYMS}
 # build the object
 "$@"
 # rename the symbols in the object
-nm -f p -g ${OUT} | cut -f1 -d' ' | grep -v -f ${KEEPSYMS} | sed -e "s/\(.*\)/\1\ ${PREFIX}_\1/" >> ${SYMSFILE}
+nm ${NM_FLAG} p -g ${OUT} | cut -f1 -d' ' | grep -v -f ${KEEPSYMS} | sed -e "s/\(.*\)/\1\ ${PREFIX}_\1/" >> ${SYMSFILE}
 if test -s ${SYMSFILE}
 then
     objcopy --redefine-syms=${SYMSFILE} ${OUT}

cmake/cflags-generic.cmake

@@ -27,6 +27,15 @@ CHECK_INCLUDE_FILES(unistd.h HAVE_UNISTD_H)
 CHECK_FUNCTION_EXISTS(posix_memalign HAVE_POSIX_MEMALIGN)
 CHECK_FUNCTION_EXISTS(_aligned_malloc HAVE__ALIGNED_MALLOC)
 
+if(FREEBSD OR NETBSD)
+    set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -gdwarf-4")
+endif()
+
+if(NETBSD)
+    set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} -DHAVE_BUILTIN_POPCOUNT")
+    set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -DHAVE_BUILTIN_POPCOUNT")
+endif()
+
 # these end up in the config file
 CHECK_C_COMPILER_FLAG(-fvisibility=hidden HAS_C_HIDDEN)
 CHECK_CXX_COMPILER_FLAG(-fvisibility=hidden HAS_CXX_HIDDEN)

cmake/osdetection.cmake

@@ -4,8 +4,19 @@ endif(CMAKE_SYSTEM_NAME MATCHES "Linux")
 
 if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
     set(FREEBSD true)
+    set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+    #FIXME: find a nicer and more general way of doing this
+    if(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc12")
+        set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc12")
+    elseif(CMAKE_C_COMPILER MATCHES "/usr/local/bin/gcc13")
+        set(CMAKE_BUILD_RPATH "/usr/local/lib/gcc13")
+    endif()
 endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
 
+if(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
+    set(NETBSD true)
+endif(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
+
 if (ARCH_IA32 OR ARCH_X86_64)
   option(FAT_RUNTIME "Build a library that supports multiple microarchitectures" ON)
 else()
@@ -14,9 +25,6 @@ endif()
 
 if (FAT_RUNTIME)
     message("Checking Fat Runtime Requirements...")
-    if (NOT LINUX)
-        message(FATAL_ERROR "Fat runtime is only supported on Linux OS")
-    else()
     if (USE_CPU_NATIVE AND FAT_RUNTIME)
         message(FATAL_ERROR "Fat runtime is not compatible with Native CPU detection")
     endif()
@@ -36,7 +44,6 @@ if (FAT_RUNTIME)
             endif()
         endif()
     endif()
-    endif()
     if (NOT RELEASE_BUILD)
         message(FATAL_ERROR "Fat runtime is only built on Release builds")
     endif()

examples/patbench.cc

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2015-2017, Intel Corporation
+ * Copyright (c) 2024, VectorCamp PC
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -134,7 +135,12 @@
 #include <netinet/tcp.h>
 #include <netinet/udp.h>
 #include <netinet/ip_icmp.h>
+#ifdef __NetBSD__
+#include <net/ethertypes.h>
+#include <net/if_ether.h>
+#else
 #include <net/ethernet.h>
+#endif /* __NetBSD__ */
 #include <arpa/inet.h>
 
 #include <pcap.h>

examples/pcapscan.cc

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2015-2016, Intel Corporation
+ * Copyright (c) 2024, VectorCamp PC
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -68,7 +69,12 @@
 #include <netinet/tcp.h>
 #include <netinet/udp.h>
 #include <netinet/ip_icmp.h>
+#ifdef __NetBSD__
+#include <net/ethertypes.h>
+#include <net/if_ether.h>
+#else
 #include <net/ethernet.h>
+#endif /* __NetBSD__ */
 #include <arpa/inet.h>
 
 #include <pcap.h>

src/rose/rose_build_program.cpp

@@ -1161,7 +1161,7 @@ static really_inline
 void nibUpdate(map<u32, u16> &nib, u32 hi_lo) {
     u16 hi = hi_lo >> 16;
     u16 lo = hi_lo & 0xffff;
-    for (const auto pairs : nib) {
+    for (const auto &pairs : nib) {
         u32 old = pairs.first;
         if ((old >> 16) == hi || (old & 0xffff) == lo) {
             if (!nib[old | hi_lo]) {

src/util/arch.h

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2017-2020, Intel Corporation
+ * Copyright (c) 2024, VectorCamp PC
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -42,6 +43,9 @@
 #elif defined(ARCH_PPC64EL)
 #include "util/arch/ppc64el/ppc64el.h"
 #endif
+#ifdef __NetBSD__
+#include <strings.h>
+#endif
 
 #endif // UTIL_ARCH_X86_H_
 

src/util/popcount.h

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2015-2017, Intel Corporation
- * Copyright (c) 2020-2021, VectorCamp PC
+ * Copyright (c) 2020-2024, VectorCamp PC
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -37,6 +37,7 @@
 #include "ue2common.h"
 #include "util/arch.h"
 
+#ifndef HAVE_BUILTIN_POPCOUNT
 static really_inline
 u32 popcount32(u32 x) {
     return __builtin_popcount(x);
@@ -51,6 +52,7 @@ u32 popcount32(u32 x) {
 //     return (((x + (x >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
 // #endif
 }
+#endif /* HAVE_BUILTIN_POPCOUNT */
 
 static really_inline
 u32 popcount32x4(u32 const *x) {
@@ -61,6 +63,7 @@ u32 popcount32x4(u32 const *x) {
     return sum;
 }
 
+#ifndef HAVE_BUILTIN_POPCOUNT
 static really_inline
 u32 popcount64(u64a x) {
     return __builtin_popcountll(x);
@@ -81,6 +84,7 @@ u32 popcount64(u64a x) {
 //     return popcount32(x >> 32) + popcount32(x);
 // #endif
 }
+#endif /* HAVE_BUILTIN_POPCOUNT */
 
 static really_inline
 u32 popcount64x4(u64a const *x) {

tools/hscollider/UltimateTruth.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2015-2019, Intel Corporation
+ * Copyright (c) 2024, VectorCamp PC
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -1079,7 +1080,7 @@ shared_ptr<BaseDB> UltimateTruth::compile(const set<unsigned> &ids,
         }
     }
 
-    return std::move(db);
+    return db;
 }
 
 bool UltimateTruth::allocScratch(shared_ptr<const BaseDB> db) {

tools/hscollider/main.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2015-2019, Intel Corporation
+ * Copyright (c) 2024, VectorCamp PC
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -1830,11 +1831,11 @@ unique_ptr<CorporaSource> buildCorpora(const vector<string> &corporaFiles,
                 exit_with_fail();
             }
         }
-        return std::move(c); /* move allows unique_ptr<CorporaSource> conversion */
+        return c;
     } else {
         auto c = std::make_unique<NfaGeneratedCorpora>(
             exprMap, corpus_gen_prop, force_utf8, force_prefilter);
-        return std::move(c);
+        return c;
     }
 }