fdr: move long literal handling into Rose

Move the hash table used for long literal support in streaming mode from
FDR to Rose, and introduce new instructions CHECK_LONG_LIT and
CHECK_LONG_LIT_NOCASE for doing literal confirm for long literals.

This simplifies FDR confirm, and guarantees that HWLM matchers will only
be used for literals < 256 bytes long.

src/hwlm/hwlm.c

@@ -200,8 +200,7 @@ hwlm_error_t hwlmExec(const struct HWLM *t, const u8 *buf, size_t len,
 
 hwlm_error_t hwlmExecStreaming(const struct HWLM *t, struct hs_scratch *scratch,
                                size_t len, size_t start, HWLMCallback cb,
-                               void *ctxt, hwlm_group_t groups,
-                               u8 *stream_state) {
+                               void *ctxt, hwlm_group_t groups) {
     const u8 *hbuf = scratch->core_info.hbuf;
     const size_t hlen = scratch->core_info.hlen;
     const u8 *buf = scratch->core_info.buf;
@@ -234,13 +233,10 @@ hwlm_error_t hwlmExecStreaming(const struct HWLM *t, struct hs_scratch *scratch,
             DEBUG_PRINTF("using hq accel %hhu\n", t->accel1.accel_type);
             aa = &t->accel1;
         }
-        // if no active stream state, use acceleration
-        if (!fdrStreamStateActive(HWLM_C_DATA(t), stream_state)) {
-            do_accel_streaming(aa, hbuf, hlen, buf, len, &start);
-        }
+        do_accel_streaming(aa, hbuf, hlen, buf, len, &start);
         DEBUG_PRINTF("calling frankie (groups=%08llx, start=%zu)\n", groups,
                      start);
         return fdrExecStreaming(HWLM_C_DATA(t), hbuf, hlen, buf, len,
-                                start, cb, ctxt, groups, stream_state);
+                                start, cb, ctxt, groups);
     }
 }

src/hwlm/hwlm.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -132,8 +132,7 @@ hwlm_error_t hwlmExec(const struct HWLM *tab, const u8 *buf, size_t len,
 hwlm_error_t hwlmExecStreaming(const struct HWLM *tab,
                                struct hs_scratch *scratch, size_t len,
                                size_t start, HWLMCallback callback,
-                               void *context, hwlm_group_t groups,
-                               u8 *stream_state);
+                               void *context, hwlm_group_t groups);
 
 #ifdef __cplusplus
 }       /* extern "C" */

src/hwlm/hwlm_build.cpp

@@ -552,6 +552,12 @@ aligned_unique_ptr<HWLM> hwlmBuild(const vector<hwlmLiteral> &lits,
 
     if (stream_control) {
         assert(stream_control->history_min <= stream_control->history_max);
+
+        // We should not have been passed any literals that are too long to
+        // match with a maximally-sized history buffer.
+        assert(all_of(begin(lits), end(lits), [&](const hwlmLiteral &lit) {
+            return lit.s.length() <= stream_control->history_max + 1;
+        }));
     }
 
     // Check that we haven't exceeded the maximum number of literals.
@@ -602,7 +608,6 @@ aligned_unique_ptr<HWLM> hwlmBuild(const vector<hwlmLiteral> &lits,
             stream_control->literal_history_required = lit.s.length() - 1;
             assert(stream_control->literal_history_required
                    <= stream_control->history_max);
-            stream_control->literal_stream_state_required = 0;
         }
         eng = move(noodle);
     } else {

src/hwlm/hwlm_build.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -63,10 +63,6 @@ struct hwlmStreamingControl {
     /** \brief OUT parameter: History required by the literal matcher to
      * correctly match all literals. */
     size_t literal_history_required;
-
-    /** OUT parameter: Stream state required by literal matcher in bytes. Can
-     * be zero, and generally will be small (0-8 bytes). */
-    size_t literal_stream_state_required;
 };
 
 /** \brief Build an \ref HWLM literal matcher runtime structure for a group of

src/hwlm/hwlm_literal.cpp

@@ -86,6 +86,7 @@ hwlmLiteral::hwlmLiteral(const std::string &s_in, bool nocase_in,
                          const vector<u8> &msk_in, const vector<u8> &cmp_in)
     : s(s_in), id(id_in), nocase(nocase_in), noruns(noruns_in),
       groups(groups_in), msk(msk_in), cmp(cmp_in) {
+    assert(s.size() <= HWLM_LITERAL_MAX_LEN);
     assert(msk.size() <= HWLM_MASKLEN);
     assert(msk.size() == cmp.size());
 

src/hwlm/hwlm_literal.h

@@ -41,6 +41,9 @@
 
 namespace ue2 {
 
+/** \brief Max length of the literal passed to HWLM. */
+#define HWLM_LITERAL_MAX_LEN 255
+
 /** \brief Max length of the hwlmLiteral::msk and hwlmLiteral::cmp vectors. */
 #define HWLM_MASKLEN 8