Move limex specific shuffle utils and ssse3 funcs

src/nfa/limex_accel.c

@@ -35,6 +35,7 @@
 #include "accel.h"
 #include "limex_internal.h"
 #include "limex_limits.h"
+#include "limex_shuffle.h"
 #include "nfa_internal.h"
 #include "shufti.h"
 #include "truffle.h"
@@ -44,10 +45,7 @@
 #include "ue2common.h"
 #include "vermicelli.h"
 #include "util/bitutils.h"
-#include "util/shuffle.h"
 #include "util/simd_utils.h"
-#include "util/simd_utils_ssse3.h"
-#include "util/shuffle_ssse3.h"
 
 static really_inline
 size_t accelScanWrapper(const u8 *accelTable, const union AccelAux *aux,
@@ -80,7 +78,7 @@ size_t accelScanWrapper(const u8 *accelTable, const union AccelAux *aux,
 size_t doAccel32(u32 s, u32 accel, const u8 *accelTable,
                  const union AccelAux *aux, const u8 *input, size_t i,
                  size_t end) {
-    u32 idx = shuffleDynamic32(s, accel);
+    u32 idx = packedExtract32(s, accel);
     return accelScanWrapper(accelTable, aux, input, idx, i, end);
 }
 
@@ -92,7 +90,7 @@ size_t doAccel128(const m128 *state, const struct LimExNFA128 *limex,
     DEBUG_PRINTF("using PSHUFB for 128-bit shuffle\n");
     m128 accelPerm = limex->accelPermute;
     m128 accelComp = limex->accelCompare;
-    idx = shufflePshufb128(s, accelPerm, accelComp);
+    idx = packedExtract128(s, accelPerm, accelComp);
     return accelScanWrapper(accelTable, aux, input, idx, i, end);
 }
 
@@ -105,17 +103,13 @@ size_t doAccel256(const m256 *state, const struct LimExNFA256 *limex,
     m256 accelPerm = limex->accelPermute;
     m256 accelComp = limex->accelCompare;
 #if !defined(__AVX2__)
-    u32 idx1 = shufflePshufb128(s.lo, accelPerm.lo, accelComp.lo);
-    u32 idx2 = shufflePshufb128(s.hi, accelPerm.hi, accelComp.hi);
-#else
-    // TODO: learn you some avx2 shuffles for great good
-    u32 idx1 = shufflePshufb128(movdq_lo(s), movdq_lo(accelPerm),
-                                movdq_lo(accelComp));
-    u32 idx2 = shufflePshufb128(movdq_hi(s), movdq_hi(accelPerm),
-                                movdq_hi(accelComp));
-#endif
+    u32 idx1 = packedExtract128(s.lo, accelPerm.lo, accelComp.lo);
+    u32 idx2 = packedExtract128(s.hi, accelPerm.hi, accelComp.hi);
     assert((idx1 & idx2) == 0); // should be no shared bits
     idx = idx1 | idx2;
+#else
+    idx = packedExtract256(s, accelPerm, accelComp);
+#endif
     return accelScanWrapper(accelTable, aux, input, idx, i, end);
 }
 
@@ -127,9 +121,9 @@ size_t doAccel384(const m384 *state, const struct LimExNFA384 *limex,
     DEBUG_PRINTF("using PSHUFB for 384-bit shuffle\n");
     m384 accelPerm = limex->accelPermute;
     m384 accelComp = limex->accelCompare;
-    u32 idx1 = shufflePshufb128(s.lo, accelPerm.lo, accelComp.lo);
-    u32 idx2 = shufflePshufb128(s.mid, accelPerm.mid, accelComp.mid);
-    u32 idx3 = shufflePshufb128(s.hi, accelPerm.hi, accelComp.hi);
+    u32 idx1 = packedExtract128(s.lo, accelPerm.lo, accelComp.lo);
+    u32 idx2 = packedExtract128(s.mid, accelPerm.mid, accelComp.mid);
+    u32 idx3 = packedExtract128(s.hi, accelPerm.hi, accelComp.hi);
     assert((idx1 & idx2 & idx3) == 0); // should be no shared bits
     idx = idx1 | idx2 | idx3;
     return accelScanWrapper(accelTable, aux, input, idx, i, end);
@@ -144,21 +138,17 @@ size_t doAccel512(const m512 *state, const struct LimExNFA512 *limex,
     m512 accelPerm = limex->accelPermute;
     m512 accelComp = limex->accelCompare;
 #if !defined(__AVX2__)
-    u32 idx1 = shufflePshufb128(s.lo.lo, accelPerm.lo.lo, accelComp.lo.lo);
-    u32 idx2 = shufflePshufb128(s.lo.hi, accelPerm.lo.hi, accelComp.lo.hi);
-    u32 idx3 = shufflePshufb128(s.hi.lo, accelPerm.hi.lo, accelComp.hi.lo);
-    u32 idx4 = shufflePshufb128(s.hi.hi, accelPerm.hi.hi, accelComp.hi.hi);
-#else
-    u32 idx1 = shufflePshufb128(movdq_lo(s.lo), movdq_lo(accelPerm.lo),
-                                movdq_lo(accelComp.lo));
-    u32 idx2 = shufflePshufb128(movdq_hi(s.lo), movdq_hi(accelPerm.lo),
-                                movdq_hi(accelComp.lo));
-    u32 idx3 = shufflePshufb128(movdq_lo(s.hi), movdq_lo(accelPerm.hi),
-                                movdq_lo(accelComp.hi));
-    u32 idx4 = shufflePshufb128(movdq_hi(s.hi), movdq_hi(accelPerm.hi),
-                                movdq_hi(accelComp.hi));
-#endif
+    u32 idx1 = packedExtract128(s.lo.lo, accelPerm.lo.lo, accelComp.lo.lo);
+    u32 idx2 = packedExtract128(s.lo.hi, accelPerm.lo.hi, accelComp.lo.hi);
+    u32 idx3 = packedExtract128(s.hi.lo, accelPerm.hi.lo, accelComp.hi.lo);
+    u32 idx4 = packedExtract128(s.hi.hi, accelPerm.hi.hi, accelComp.hi.hi);
     assert((idx1 & idx2 & idx3 & idx4) == 0); // should be no shared bits
     idx = idx1 | idx2 | idx3 | idx4;
+#else
+    u32 idx1 = packedExtract256(s.lo, accelPerm.lo, accelComp.lo);
+    u32 idx2 = packedExtract256(s.hi, accelPerm.hi, accelComp.hi);
+    assert((idx1 & idx2) == 0); // should be no shared bits
+    idx = idx1 | idx2;
+#endif
     return accelScanWrapper(accelTable, aux, input, idx, i, end);
 }

src/nfa/limex_shuffle.h

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2015-2016, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Naive dynamic shuffles.
+ *
+ * These are written with the assumption that the provided masks are sparsely
+ * populated and never contain more than 32 on bits. Other implementations will
+ * be faster and actually correct if these assumptions don't hold true.
+ */
+
+#ifndef LIMEX_SHUFFLE_H
+#define LIMEX_SHUFFLE_H
+
+#include "ue2common.h"
+#include "util/bitutils.h"
+#include "util/simd_utils.h"
+
+#if defined(__BMI2__) || (defined(_WIN32) && defined(__AVX2__))
+#define HAVE_PEXT
+#endif
+
+static really_inline
+u32 packedExtract32(u32 x, u32 mask) {
+#if defined(HAVE_PEXT)
+    // Intel BMI2 can do this operation in one instruction.
+    return _pext_u32(x, mask);
+#else
+
+    u32 result = 0, num = 1;
+    while (mask != 0) {
+        u32 bit = findAndClearLSB_32(&mask);
+        if (x & (1U << bit)) {
+            assert(num != 0); // more than 32 bits!
+            result |= num;
+        }
+        num <<= 1;
+    }
+    return result;
+#endif
+}
+
+static really_inline
+u32 packedExtract64(u64a x, u64a mask) {
+#if defined(HAVE_PEXT) && defined(ARCH_64_BIT)
+    // Intel BMI2 can do this operation in one instruction.
+    return _pext_u64(x, mask);
+#else
+
+    u32 result = 0, num = 1;
+    while (mask != 0) {
+        u32 bit = findAndClearLSB_64(&mask);
+        if (x & (1ULL << bit)) {
+            assert(num != 0); // more than 32 bits!
+            result |= num;
+        }
+        num <<= 1;
+    }
+    return result;
+#endif
+}
+
+#undef HAVE_PEXT
+
+static really_inline
+u32 packedExtract128(m128 s, const m128 permute, const m128 compare) {
+    m128 shuffled = pshufb(s, permute);
+    m128 compared = and128(shuffled, compare);
+    u16 rv = ~cmpmsk8(compared, shuffled);
+    return (u32)rv;
+}
+
+#if defined(__AVX2__)
+static really_inline
+u32 packedExtract256(m256 s, const m256 permute, const m256 compare) {
+    // vpshufb doesn't cross lanes, so this is a bit of a cheat
+    m256 shuffled = vpshufb(s, permute);
+    m256 compared = and256(shuffled, compare);
+    u32 rv = ~movemask256(eq256(compared, shuffled));
+    // stitch the lane-wise results back together
+    return (u32)((rv >> 16) | (rv & 0xffffU));
+}
+#endif // AVX2
+
+#endif // LIMEX_SHUFFLE_H

src/nfa/multishufti_avx2.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -31,7 +31,6 @@
 #include "ue2common.h"
 #include "util/bitutils.h"
 #include "util/simd_utils.h"
-#include "util/simd_utils_ssse3.h"
 
 static really_inline
 const u8 *JOIN(MATCH_ALGO, fwdBlock)(m256 mask_lo, m256 mask_hi, m256 chars,

src/nfa/multishufti_sse.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -31,7 +31,6 @@
 #include "ue2common.h"
 #include "util/bitutils.h"
 #include "util/simd_utils.h"
-#include "util/simd_utils_ssse3.h"
 
 /* Normal SSSE3 shufti */
 

src/nfa/multitruffle.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -32,7 +32,6 @@
 #include "multitruffle.h"
 #include "util/bitutils.h"
 #include "util/simd_utils.h"
-#include "util/simd_utils_ssse3.h"
 
 #include "multiaccel_common.h"
 

src/nfa/shufti.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -40,7 +40,6 @@
 
 #include "shufti_common.h"
 
-#include "util/simd_utils_ssse3.h"
 
 /** \brief Naive byte-by-byte implementation. */
 static really_inline

src/nfa/shufti_common.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -34,7 +34,6 @@
 #include "util/bitutils.h"
 #include "util/simd_utils.h"
 #include "util/unaligned.h"
-#include "util/simd_utils_ssse3.h"
 
 /*
  * Common stuff for all versions of shufti (single, multi and multidouble)

src/nfa/truffle.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -35,7 +35,6 @@
 #include "truffle.h"
 #include "util/bitutils.h"
 #include "util/simd_utils.h"
-#include "util/simd_utils_ssse3.h"
 
 #include "truffle_common.h"
 

src/nfa/truffle_common.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -31,7 +31,6 @@
 
 #include "util/bitutils.h"
 #include "util/simd_utils.h"
-#include "util/simd_utils_ssse3.h"
 
 /*
  * Common stuff for all versions of truffle (single, multi and multidouble)