github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-06-28 16:41:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

complete refactoring and unification of Vermicelli functions

Browse Source
This commit is contained in:
Konstantinos Margaritis 2021-11-01 16:51:18 +02:00
parent f4a490ac00
commit 44dc75a3ea
2 changed files with 127 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/nfa/vermicelli.hpp
View File

@ -75,4 +75,12 @@ const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, con
} }
#endif #endif
#ifdef __cplusplus
extern "C" {
#endif
const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end);
#ifdef __cplusplus
}
#endif
#endif /* VERMICELLI_HPP */ #endif /* VERMICELLI_HPP */

240
src/nfa/vermicelli_simd.cpp
View File

@ -41,48 +41,16 @@
template <uint16_t S> template <uint16_t S>
static really_inline static really_inline
const u8 *vermicelliSingleBlock(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) { const u8 *vermicelliBlock(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
SuperVector<S> mask = chars.eq(casemask & data); SuperVector<S> mask = chars.eq(casemask & data);
return first_non_zero_match<S>(buf, mask); return first_non_zero_match<S>(buf, mask);
} }
template <uint16_t S>
static really_inline
const u8 *rvermicelliSingleBlock(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
SuperVector<S> mask = chars.eq(casemask & data);
return last_non_zero_match<S>(buf, mask);
}
template <uint16_t S> template <uint16_t S>
static really_inline static really_inline
const u8 *vermicelliDoubleBlock(SuperVector<S> data, SuperVector<S> chars1, SuperVector<S> chars2, SuperVector<S> casemask, const u8 *vermicelliBlockNeg(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
const u8 *buf/*, SuperVector<S> *lastmask1, size_t len = S*/) {
// lastmask1->print8("lastmask1");
data.print8("data");
chars1.print8("chars1");
chars2.print8("chars2");
casemask.print8("casemask");
SuperVector<S> v = casemask & data;
v.print8("v");
SuperVector<S> mask1 = chars1.eq(v);
mask1.print8("mask1");
SuperVector<S> mask2 = chars2.eq(v);
mask2.print8("mask2");
SuperVector<S> mask = (mask1 & (mask2 >> 1));
mask.print8("mask");
DEBUG_PRINTF("len = %ld\n", len);
// *lastmask1 = mask1 >> (len -1);
// lastmask1->print8("lastmask1");
return first_non_zero_match<S>(buf, mask);
}
template <uint16_t S>
static really_inline
const u8 *vermicelliSingleBlockNeg(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
SuperVector<S> mask = chars.eq(casemask & data); SuperVector<S> mask = chars.eq(casemask & data);
return first_zero_match_inverted<S>(buf, mask); return first_zero_match_inverted<S>(buf, mask);
@ -90,36 +58,58 @@ const u8 *vermicelliSingleBlockNeg(SuperVector<S> data, SuperVector<S> chars, Su
template <uint16_t S> template <uint16_t S>
static really_inline static really_inline
const u8 *rvermicelliSingleBlockNeg(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) { const u8 *rvermicelliBlock(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
SuperVector<S> mask = chars.eq(casemask & data);
return last_non_zero_match<S>(buf, mask);
}
template <uint16_t S>
static really_inline
const u8 *rvermicelliBlockNeg(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
SuperVector<S> mask = chars.eq(casemask & data); SuperVector<S> mask = chars.eq(casemask & data);
return last_zero_match_inverted<S>(buf, mask); return last_zero_match_inverted<S>(buf, mask);
} }
/*
template <uint16_t S> template <uint16_t S>
static really_inline static really_inline
const u8 *vermicelliDoubleBlockNeg(SuperVector<S> data, SuperVector<S> chars1, SuperVector<S> chars2, SuperVector<S> casemask, const u8 *vermicelliDoubleBlock(SuperVector<S> data, SuperVector<S> chars1, SuperVector<S> chars2, SuperVector<S> casemask,
const u8 *buf, size_t len = S) { u8 const c1, u8 const c2, u8 const casechar, const u8 *buf) {
// lastmask1.print8("lastmask1");
data.print8("data");
chars1.print8("chars1");
chars2.print8("chars2");
casemask.print8("casemask");
SuperVector<S> v = casemask & data; SuperVector<S> v = casemask & data;
v.print8("v");
SuperVector<S> mask1 = chars1.eq(v); SuperVector<S> mask1 = chars1.eq(v);
mask1.print8("mask1");
SuperVector<S> mask2 = chars2.eq(v); SuperVector<S> mask2 = chars2.eq(v);
mask2.print8("mask2"); SuperVector<S> mask = mask1 & (mask2 >> 1);
SuperVector<S> mask = (mask1 & (mask2 >> 1));// | lastmask1;
mask.print8("mask");
DEBUG_PRINTF("len = %ld\n", len);
// lastmask1 = mask << (len -1);
// lastmask1.print8("lastmask1");
return last_zero_match_inverted<S>(buf, mask); DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]);
}*/ bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1));
DEBUG_PRINTF("partial = %d\n", partial_match);
if (partial_match) return buf - 1;
return first_non_zero_match<S>(buf, mask);
}
template <uint16_t S>
static really_inline
const u8 *rvermicelliDoubleBlock(SuperVector<S> data, SuperVector<S> chars1, SuperVector<S> chars2, SuperVector<S> casemask,
u8 const c1, u8 const c2, u8 const casechar, const u8 *buf) {
SuperVector<S> v = casemask & data;
SuperVector<S> mask1 = chars1.eq(v);
SuperVector<S> mask2 = chars2.eq(v);
SuperVector<S> mask = (mask1 << 1)& mask2;
DEBUG_PRINTF("buf[0] = %02hhx, buf[-1] = %02hhx\n", buf[0], buf[-1]);
bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1));
DEBUG_PRINTF("partial = %d\n", partial_match);
if (partial_match) {
mask = mask | (SuperVector<S>::Ones() >> (S-1));
}
return last_non_zero_match<S>(buf, mask);
}
template <uint16_t S> template <uint16_t S>
static const u8 *vermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const casemask, const u8 *buf, const u8 *buf_end) { static const u8 *vermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const casemask, const u8 *buf, const u8 *buf_end) {
@ -142,7 +132,7 @@ static const u8 *vermicelliExecReal(SuperVector<S> const chars, SuperVector<S> c
DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S));
if (!ISALIGNED_N(d, S)) { if (!ISALIGNED_N(d, S)) {
SuperVector<S> data = SuperVector<S>::loadu(d); SuperVector<S> data = SuperVector<S>::loadu(d);
rv = vermicelliSingleBlock(data, chars, casemask, d); rv = vermicelliBlock(data, chars, casemask, d);
if (rv) return rv; if (rv) return rv;
d = ROUNDUP_PTR(d, S); d = ROUNDUP_PTR(d, S);
} }
@ -151,7 +141,7 @@ static const u8 *vermicelliExecReal(SuperVector<S> const chars, SuperVector<S> c
__builtin_prefetch(d + 64); __builtin_prefetch(d + 64);
DEBUG_PRINTF("d %p \n", d); DEBUG_PRINTF("d %p \n", d);
SuperVector<S> data = SuperVector<S>::load(d); SuperVector<S> data = SuperVector<S>::load(d);
rv = vermicelliSingleBlock(data, chars, casemask, d); rv = vermicelliBlock(data, chars, casemask, d);
if (rv) return rv; if (rv) return rv;
d += S; d += S;
} }
@ -162,7 +152,7 @@ static const u8 *vermicelliExecReal(SuperVector<S> const chars, SuperVector<S> c
if (d != buf_end) { if (d != buf_end) {
SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d); SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d);
rv = vermicelliSingleBlock(data, chars, casemask, d); rv = vermicelliBlock(data, chars, casemask, d);
DEBUG_PRINTF("rv %p \n", rv); DEBUG_PRINTF("rv %p \n", rv);
if (rv && rv < buf_end) return rv; if (rv && rv < buf_end) return rv;
} }
@ -180,8 +170,6 @@ static const u8 *nvermicelliExecReal(SuperVector<S> const chars, SuperVector<S>
const u8 *d = buf; const u8 *d = buf;
const u8 *rv; const u8 *rv;
__builtin_prefetch(d + 64); __builtin_prefetch(d + 64);
__builtin_prefetch(d + 2*64); __builtin_prefetch(d + 2*64);
__builtin_prefetch(d + 3*64); __builtin_prefetch(d + 3*64);
@ -193,7 +181,7 @@ static const u8 *nvermicelliExecReal(SuperVector<S> const chars, SuperVector<S>
DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S));
if (!ISALIGNED_N(d, S)) { if (!ISALIGNED_N(d, S)) {
SuperVector<S> data = SuperVector<S>::loadu(d); SuperVector<S> data = SuperVector<S>::loadu(d);
rv = vermicelliSingleBlockNeg(data, chars, casemask, d); rv = vermicelliBlockNeg(data, chars, casemask, d);
if (rv) return rv; if (rv) return rv;
d = ROUNDUP_PTR(d, S); d = ROUNDUP_PTR(d, S);
} }
@ -202,7 +190,7 @@ static const u8 *nvermicelliExecReal(SuperVector<S> const chars, SuperVector<S>
__builtin_prefetch(d + 64); __builtin_prefetch(d + 64);
DEBUG_PRINTF("d %p \n", d); DEBUG_PRINTF("d %p \n", d);
SuperVector<S> data = SuperVector<S>::load(d); SuperVector<S> data = SuperVector<S>::load(d);
rv = vermicelliSingleBlockNeg(data, chars, casemask, d); rv = vermicelliBlockNeg(data, chars, casemask, d);
if (rv) return rv; if (rv) return rv;
d += S; d += S;
} }
@ -213,7 +201,7 @@ static const u8 *nvermicelliExecReal(SuperVector<S> const chars, SuperVector<S>
if (d != buf_end) { if (d != buf_end) {
SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d); SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d);
rv = vermicelliSingleBlockNeg(data, chars, casemask, d); rv = vermicelliBlockNeg(data, chars, casemask, d);
DEBUG_PRINTF("rv %p \n", rv); DEBUG_PRINTF("rv %p \n", rv);
if (rv && rv < buf_end) return rv; if (rv && rv < buf_end) return rv;
} }
@ -244,7 +232,7 @@ const u8 *rvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const c
DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S));
if (!ISALIGNED_N(d, S)) { if (!ISALIGNED_N(d, S)) {
SuperVector<S> data = SuperVector<S>::loadu(d - S); SuperVector<S> data = SuperVector<S>::loadu(d - S);
rv = rvermicelliSingleBlock(data, chars, casemask, d - S); rv = rvermicelliBlock(data, chars, casemask, d - S);
DEBUG_PRINTF("rv %p \n", rv); DEBUG_PRINTF("rv %p \n", rv);
if (rv) return rv; if (rv) return rv;
d = ROUNDDOWN_PTR(d, S); d = ROUNDDOWN_PTR(d, S);
@ -257,7 +245,7 @@ const u8 *rvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const c
d -= S; d -= S;
SuperVector<S> data = SuperVector<S>::load(d); SuperVector<S> data = SuperVector<S>::load(d);
rv = rvermicelliSingleBlock(data, chars, casemask, d); rv = rvermicelliBlock(data, chars, casemask, d);
if (rv) return rv; if (rv) return rv;
} }
} }
@ -267,7 +255,7 @@ const u8 *rvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const c
if (d != buf) { if (d != buf) {
SuperVector<S> data = SuperVector<S>::loadu(buf); SuperVector<S> data = SuperVector<S>::loadu(buf);
rv = rvermicelliSingleBlock(data, chars, casemask, buf); rv = rvermicelliBlock(data, chars, casemask, buf);
DEBUG_PRINTF("rv %p \n", rv); DEBUG_PRINTF("rv %p \n", rv);
if (rv && rv < buf_end) return rv; if (rv && rv < buf_end) return rv;
} }
@ -298,7 +286,7 @@ const u8 *rnvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const
DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S)); DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S));
if (!ISALIGNED_N(d, S)) { if (!ISALIGNED_N(d, S)) {
SuperVector<S> data = SuperVector<S>::loadu(d - S); SuperVector<S> data = SuperVector<S>::loadu(d - S);
rv = rvermicelliSingleBlockNeg(data, chars, casemask, d - S); rv = rvermicelliBlockNeg(data, chars, casemask, d - S);
DEBUG_PRINTF("rv %p \n", rv); DEBUG_PRINTF("rv %p \n", rv);
if (rv) return rv; if (rv) return rv;
d = ROUNDDOWN_PTR(d, S); d = ROUNDDOWN_PTR(d, S);
@ -311,7 +299,7 @@ const u8 *rnvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const
d -= S; d -= S;
SuperVector<S> data = SuperVector<S>::load(d); SuperVector<S> data = SuperVector<S>::load(d);
rv = rvermicelliSingleBlockNeg(data, chars, casemask, d); rv = rvermicelliBlockNeg(data, chars, casemask, d);
if (rv) return rv; if (rv) return rv;
} }
} }
@ -321,7 +309,7 @@ const u8 *rnvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const
if (d != buf) { if (d != buf) {
SuperVector<S> data = SuperVector<S>::loadu(buf); SuperVector<S> data = SuperVector<S>::loadu(buf);
rv = rvermicelliSingleBlockNeg(data, chars, casemask, buf); rv = rvermicelliBlockNeg(data, chars, casemask, buf);
DEBUG_PRINTF("rv %p \n", rv); DEBUG_PRINTF("rv %p \n", rv);
if (rv && rv < buf_end) return rv; if (rv && rv < buf_end) return rv;
} }
@ -355,7 +343,7 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector<
DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S)); DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S));
if (!ISALIGNED_N(d, S)) { if (!ISALIGNED_N(d, S)) {
SuperVector<S> data = SuperVector<S>::loadu(d); SuperVector<S> data = SuperVector<S>::loadu(d);
rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, &lastmask1); rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, &lastmask1);
if (rv) return rv; if (rv) return rv;
d = ROUNDUP_PTR(d, S); d = ROUNDUP_PTR(d, S);
} }
@ -364,11 +352,8 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector<
__builtin_prefetch(d + 64); __builtin_prefetch(d + 64);
DEBUG_PRINTF("d %p \n", d); DEBUG_PRINTF("d %p \n", d);
SuperVector<S> data = SuperVector<S>::load(d); SuperVector<S> data = SuperVector<S>::load(d);
rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, &lastmask1); rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, &lastmask1);
if (rv) { if (rv) return rv;
bool partial_match = (((rv[0] & casechar) == c2) && ((rv[-1] & casechar) == c1));
return rv - partial_match;
}
d += S; d += S;
} }
} }
@ -378,7 +363,7 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector<
if (d != buf_end) { if (d != buf_end) {
SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d); SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d);
rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, buf_end - d); rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, buf_end - d);
DEBUG_PRINTF("rv %p \n", rv); DEBUG_PRINTF("rv %p \n", rv);
if (rv && rv < buf_end) return rv; if (rv && rv < buf_end) return rv;
} }
@ -396,60 +381,63 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector<
} }
// /* returns highest offset of c2 (NOTE: not c1) */ // /* returns highest offset of c2 (NOTE: not c1) */
// static really_inline template <uint16_t S>
// const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *rvermicelliDoubleExecReal(char c1, char c2, SuperVector<S> const casemask, const u8 *buf, const u8 *buf_end) {
// const u8 *buf_end) { assert(buf && buf_end);
// DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n", assert(buf < buf_end);
// nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf)); DEBUG_PRINTF("rverm %p len %zu\n", buf, buf_end - buf);
// assert(buf < buf_end); DEBUG_PRINTF("b %s\n", buf);
char s[255];
snprintf(s, buf_end - buf + 1, "%s", buf);
DEBUG_PRINTF("b %s\n", s);
// VERM_TYPE chars1 = VERM_SET_FN(c1); /* nocase already uppercase */ const u8 *d = buf_end;
// VERM_TYPE chars2 = VERM_SET_FN(c2); /* nocase already uppercase */ const u8 *rv;
const SuperVector<VECTORSIZE> chars1 = SuperVector<VECTORSIZE>::dup_u8(c1);
const SuperVector<VECTORSIZE> chars2 = SuperVector<VECTORSIZE>::dup_u8(c2);
const u8 casechar = casemask.u.u8[0];
// #ifdef HAVE_AVX512 __builtin_prefetch(d - 64);
// if (buf_end - buf <= VERM_BOUNDARY) { __builtin_prefetch(d - 2*64);
// const u8 *ptr = nocase __builtin_prefetch(d - 3*64);
// ? rdvermMiniNocase(chars1, chars2, buf, buf_end) __builtin_prefetch(d - 4*64);
// : rdvermMini(chars1, chars2, buf, buf_end); DEBUG_PRINTF("start %p end %p \n", buf, d);
assert(d > buf);
if (d - S >= buf) {
// Reach vector aligned boundaries
DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S));
if (!ISALIGNED_N(d, S)) {
SuperVector<S> data = SuperVector<S>::loadu(d - S);
rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d - S);
DEBUG_PRINTF("rv %p \n", rv);
if (rv && rv < buf_end) return rv;
d = ROUNDDOWN_PTR(d, S);
}
// if (ptr) { while (d - S >= buf) {
// return ptr; DEBUG_PRINTF("aligned %p \n", d);
// } // On large packet buffers, this prefetch appears to get us about 2%.
__builtin_prefetch(d - 64);
// // check for partial match at end ??? d -= S;
// return buf - 1; SuperVector<S> data = SuperVector<S>::load(d);
// } rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);
// #endif if (rv) return rv;
}
}
// assert((buf_end - buf) >= VERM_BOUNDARY); DEBUG_PRINTF("tail d %p e %p \n", buf, d);
// size_t min = (size_t)buf_end % VERM_BOUNDARY; // finish off head
// if (min) {
// // input not aligned, so we need to run one iteration with an unaligned
// // load, then skip buf forward to the next aligned address. There's
// // some small overlap here, but we don't mind scanning it twice if we
// // can do it quickly, do we?
// const u8 *ptr = nocase ? rdvermPreconditionNocase(chars1, chars2,
// buf_end - VERM_BOUNDARY)
// : rdvermPrecondition(chars1, chars2,
// buf_end - VERM_BOUNDARY);
// if (ptr) { if (d != buf) {
// return ptr; SuperVector<S> data = SuperVector<S>::loadu(buf);
// } rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, buf);
DEBUG_PRINTF("rv %p \n", rv);
if (rv && rv < buf_end) return rv;
}
// buf_end -= min; return buf - 1;
// if (buf >= buf_end) { }
// return buf_end;
// }
// }
// // Aligned loops from here on in
// if (nocase) {
// return rdvermSearchAlignedNocase(chars1, chars2, c1, c2, buf, buf_end);
// } else {
// return rdvermSearchAligned(chars1, chars2, c1, c2, buf, buf_end);
// }
// }
extern "C" const u8 *vermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) { extern "C" const u8 *vermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) {
DEBUG_PRINTF("verm scan %s\\x%02hhx over %zu bytes\n", DEBUG_PRINTF("verm scan %s\\x%02hhx over %zu bytes\n",
@ -506,3 +494,13 @@ extern "C" const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u
return vermicelliDoubleExecReal<VECTORSIZE>(c1, c2, casemask, buf, buf_end); return vermicelliDoubleExecReal<VECTORSIZE>(c1, c2, casemask, buf, buf_end);
} }
extern "C" const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end) {
DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n",
nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf));
assert(buf < buf_end);
const SuperVector<VECTORSIZE> casemask{nocase ? getCaseMask<VECTORSIZE>() : SuperVector<VECTORSIZE>::Ones()};
return rvermicelliDoubleExecReal<VECTORSIZE>(c1, c2, casemask, buf, buf_end);
}