aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/libs/hyperscan/src/rose/match.c
diff options
context:
space:
mode:
authorbnagaev <bnagaev@yandex-team.ru>2022-02-10 16:47:04 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:47:04 +0300
commitc74559fb88da8adac0d9186cfa55a6b13c47695f (patch)
treeb83306b6e37edeea782e9eed673d89286c4fef35 /contrib/libs/hyperscan/src/rose/match.c
parentd6449ba66291ff0c0d352c82e6eb3efb4c8a7e8d (diff)
downloadydb-c74559fb88da8adac0d9186cfa55a6b13c47695f.tar.gz
Restoring authorship annotation for <bnagaev@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'contrib/libs/hyperscan/src/rose/match.c')
-rw-r--r--contrib/libs/hyperscan/src/rose/match.c782
1 files changed, 391 insertions, 391 deletions
diff --git a/contrib/libs/hyperscan/src/rose/match.c b/contrib/libs/hyperscan/src/rose/match.c
index c7f8189cd2..84d3b1fdc2 100644
--- a/contrib/libs/hyperscan/src/rose/match.c
+++ b/contrib/libs/hyperscan/src/rose/match.c
@@ -1,240 +1,240 @@
-/*
+/*
* Copyright (c) 2015-2019, Intel Corporation
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "catchup.h"
-#include "match.h"
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Intel Corporation nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "catchup.h"
+#include "match.h"
#include "program_runtime.h"
-#include "rose.h"
-#include "util/bitutils.h"
-#include "util/fatbit.h"
-
-#if defined(DEBUG) || defined(DUMP_SUPPORT)
-#include "util/compare.h"
-/** A debugging crutch: print a hex-escaped version of the match for our
- * perusal. The start and end offsets are stream offsets. */
-static UNUSED
-void printMatch(const struct core_info *ci, u64a start, u64a end) {
- assert(start <= end);
- assert(end <= ci->buf_offset + ci->len);
-
- printf("'");
- u64a i = start;
- for (; i <= MIN(ci->buf_offset, end); i++) {
- u64a h_idx = ci->buf_offset - i;
- u8 c = h_idx >= ci->hlen ? '?' : ci->hbuf[ci->hlen - h_idx - 1];
- if (ourisprint(c) && c != '\'') {
- printf("%c", c);
- } else {
- printf("\\x%02x", c);
- }
- }
- for (; i <= end; i++) {
- u64a b_idx = i - ci->buf_offset - 1;
- u8 c = b_idx >= ci->len ? '?' : ci->buf[b_idx];
- if (ourisprint(c) && c != '\'') {
- printf("%c", c);
- } else {
- printf("\\x%02x", c);
- }
- }
- printf("'");
-}
-#endif
-
+#include "rose.h"
+#include "util/bitutils.h"
+#include "util/fatbit.h"
+
+#if defined(DEBUG) || defined(DUMP_SUPPORT)
+#include "util/compare.h"
+/** A debugging crutch: print a hex-escaped version of the match for our
+ * perusal. The start and end offsets are stream offsets. */
+static UNUSED
+void printMatch(const struct core_info *ci, u64a start, u64a end) {
+ assert(start <= end);
+ assert(end <= ci->buf_offset + ci->len);
+
+ printf("'");
+ u64a i = start;
+ for (; i <= MIN(ci->buf_offset, end); i++) {
+ u64a h_idx = ci->buf_offset - i;
+ u8 c = h_idx >= ci->hlen ? '?' : ci->hbuf[ci->hlen - h_idx - 1];
+ if (ourisprint(c) && c != '\'') {
+ printf("%c", c);
+ } else {
+ printf("\\x%02x", c);
+ }
+ }
+ for (; i <= end; i++) {
+ u64a b_idx = i - ci->buf_offset - 1;
+ u8 c = b_idx >= ci->len ? '?' : ci->buf[b_idx];
+ if (ourisprint(c) && c != '\'') {
+ printf("%c", c);
+ } else {
+ printf("\\x%02x", c);
+ }
+ }
+ printf("'");
+}
+#endif
+
hwlmcb_rv_t roseDelayRebuildCallback(size_t end, u32 id,
struct hs_scratch *scratch) {
- struct RoseContext *tctx = &scratch->tctxt;
- struct core_info *ci = &scratch->core_info;
+ struct RoseContext *tctx = &scratch->tctxt;
+ struct core_info *ci = &scratch->core_info;
const struct RoseEngine *t = ci->rose;
- size_t rb_len = MIN(ci->hlen, t->delayRebuildLength);
-
- u64a real_end = ci->buf_offset - rb_len + end + 1; // index after last byte
-
-#ifdef DEBUG
+ size_t rb_len = MIN(ci->hlen, t->delayRebuildLength);
+
+ u64a real_end = ci->buf_offset - rb_len + end + 1; // index after last byte
+
+#ifdef DEBUG
DEBUG_PRINTF("REBUILD MATCH id=%u end offset@%llu]: ", id, real_end);
u64a start = real_end < 8 ? 1 : real_end - 7;
printMatch(ci, start, real_end);
- printf("\n");
-#endif
-
+ printf("\n");
+#endif
+
DEBUG_PRINTF("STATE groups=0x%016llx\n", tctx->groups);
-
+
assert(id && id < t->size); // id is a program offset
const u64a som = 0;
const u8 flags = 0;
UNUSED hwlmcb_rv_t rv =
roseRunProgram(t, scratch, id, som, real_end, flags);
assert(rv != HWLM_TERMINATE_MATCHING);
-
+
/* we are just repopulating the delay queue, groups should be
- * already set from the original scan. */
-
- return tctx->groups;
-}
-
-static really_inline
-hwlmcb_rv_t ensureMpvQueueFlushed(const struct RoseEngine *t,
- struct hs_scratch *scratch, u32 qi, s64a loc,
+ * already set from the original scan. */
+
+ return tctx->groups;
+}
+
+static really_inline
+hwlmcb_rv_t ensureMpvQueueFlushed(const struct RoseEngine *t,
+ struct hs_scratch *scratch, u32 qi, s64a loc,
char in_chained) {
return ensureQueueFlushed_i(t, scratch, qi, loc, 1, in_chained);
-}
-
+}
+
hwlmcb_rv_t roseHandleChainMatch(const struct RoseEngine *t,
struct hs_scratch *scratch, u32 event,
u64a top_squash_distance, u64a end,
char in_catchup) {
assert(event == MQE_TOP || event >= MQE_TOP_FIRST);
- struct core_info *ci = &scratch->core_info;
-
+ struct core_info *ci = &scratch->core_info;
+
u8 *aa = getActiveLeafArray(t, scratch->core_info.state);
- u32 aaCount = t->activeArrayCount;
- struct fatbit *activeQueues = scratch->aqa;
- u32 qCount = t->queueCount;
-
+ u32 aaCount = t->activeArrayCount;
+ struct fatbit *activeQueues = scratch->aqa;
+ u32 qCount = t->queueCount;
+
const u32 qi = 0; /* MPV is always queue 0 if it exists */
- struct mq *q = &scratch->queues[qi];
- const struct NfaInfo *info = getNfaInfoByQueue(t, qi);
-
- s64a loc = (s64a)end - ci->buf_offset;
- assert(loc <= (s64a)ci->len && loc >= -(s64a)ci->hlen);
-
- if (!mmbit_set(aa, aaCount, qi)) {
+ struct mq *q = &scratch->queues[qi];
+ const struct NfaInfo *info = getNfaInfoByQueue(t, qi);
+
+ s64a loc = (s64a)end - ci->buf_offset;
+ assert(loc <= (s64a)ci->len && loc >= -(s64a)ci->hlen);
+
+ if (!mmbit_set(aa, aaCount, qi)) {
initQueue(q, qi, t, scratch);
- nfaQueueInitState(q->nfa, q);
- pushQueueAt(q, 0, MQE_START, loc);
- fatbit_set(activeQueues, qCount, qi);
- } else if (info->no_retrigger) {
- DEBUG_PRINTF("yawn\n");
- /* nfa only needs one top; we can go home now */
- return HWLM_CONTINUE_MATCHING;
- } else if (!fatbit_set(activeQueues, qCount, qi)) {
+ nfaQueueInitState(q->nfa, q);
+ pushQueueAt(q, 0, MQE_START, loc);
+ fatbit_set(activeQueues, qCount, qi);
+ } else if (info->no_retrigger) {
+ DEBUG_PRINTF("yawn\n");
+ /* nfa only needs one top; we can go home now */
+ return HWLM_CONTINUE_MATCHING;
+ } else if (!fatbit_set(activeQueues, qCount, qi)) {
initQueue(q, qi, t, scratch);
- loadStreamState(q->nfa, q, 0);
- pushQueueAt(q, 0, MQE_START, 0);
- } else if (isQueueFull(q)) {
- DEBUG_PRINTF("queue %u full -> catching up nfas\n", qi);
- /* we know it is a chained nfa and the suffixes/outfixes must already
- * be known to be consistent */
+ loadStreamState(q->nfa, q, 0);
+ pushQueueAt(q, 0, MQE_START, 0);
+ } else if (isQueueFull(q)) {
+ DEBUG_PRINTF("queue %u full -> catching up nfas\n", qi);
+ /* we know it is a chained nfa and the suffixes/outfixes must already
+ * be known to be consistent */
if (ensureMpvQueueFlushed(t, scratch, qi, loc, in_catchup)
- == HWLM_TERMINATE_MATCHING) {
+ == HWLM_TERMINATE_MATCHING) {
DEBUG_PRINTF("terminating...\n");
- return HWLM_TERMINATE_MATCHING;
- }
- }
-
+ return HWLM_TERMINATE_MATCHING;
+ }
+ }
+
if (top_squash_distance) {
assert(q->cur < q->end);
- struct mq_item *last = &q->items[q->end - 1];
- if (last->type == event
+ struct mq_item *last = &q->items[q->end - 1];
+ if (last->type == event
&& last->location >= loc - (s64a)top_squash_distance) {
- last->location = loc;
- goto event_enqueued;
- }
- }
-
- pushQueue(q, event, loc);
-
-event_enqueued:
- if (q_cur_loc(q) == (s64a)ci->len) {
- /* we may not run the nfa; need to ensure state is fine */
- DEBUG_PRINTF("empty run\n");
- pushQueueNoMerge(q, MQE_END, loc);
- char alive = nfaQueueExec(q->nfa, q, loc);
- if (alive) {
+ last->location = loc;
+ goto event_enqueued;
+ }
+ }
+
+ pushQueue(q, event, loc);
+
+event_enqueued:
+ if (q_cur_loc(q) == (s64a)ci->len) {
+ /* we may not run the nfa; need to ensure state is fine */
+ DEBUG_PRINTF("empty run\n");
+ pushQueueNoMerge(q, MQE_END, loc);
+ char alive = nfaQueueExec(q->nfa, q, loc);
+ if (alive) {
scratch->tctxt.mpv_inactive = 0;
- q->cur = q->end = 0;
- pushQueueAt(q, 0, MQE_START, loc);
- } else {
- mmbit_unset(aa, aaCount, qi);
- fatbit_unset(scratch->aqa, qCount, qi);
- }
- }
-
- DEBUG_PRINTF("added mpv event at %lld\n", loc);
+ q->cur = q->end = 0;
+ pushQueueAt(q, 0, MQE_START, loc);
+ } else {
+ mmbit_unset(aa, aaCount, qi);
+ fatbit_unset(scratch->aqa, qCount, qi);
+ }
+ }
+
+ DEBUG_PRINTF("added mpv event at %lld\n", loc);
scratch->tctxt.next_mpv_offset = 0; /* the top event may result in matches
* earlier than expected */
- return HWLM_CONTINUE_MATCHING;
-}
-
+ return HWLM_CONTINUE_MATCHING;
+}
+
int roseAnchoredCallback(u64a start, u64a end, u32 id, void *ctx) {
struct hs_scratch *scratch = ctx;
assert(scratch && scratch->magic == SCRATCH_MAGIC);
struct RoseContext *tctxt = &scratch->tctxt;
- struct core_info *ci = &scratch->core_info;
+ struct core_info *ci = &scratch->core_info;
const struct RoseEngine *t = ci->rose;
-
- u64a real_end = ci->buf_offset + end; // index after last byte
-
- DEBUG_PRINTF("MATCH id=%u offsets=[???,%llu]\n", id, real_end);
+
+ u64a real_end = ci->buf_offset + end; // index after last byte
+
+ DEBUG_PRINTF("MATCH id=%u offsets=[???,%llu]\n", id, real_end);
DEBUG_PRINTF("STATE groups=0x%016llx\n", tctxt->groups);
-
+
if (can_stop_matching(scratch)) {
- DEBUG_PRINTF("received a match when we're already dead!\n");
- return MO_HALT_MATCHING;
- }
-
- /* delayed literals need to be delivered before real literals; however
- * delayed literals only come from the floating table so if we are going
- * to deliver a literal here it must be too early for a delayed literal */
-
- /* no history checks from anchored region and we are before the flush
- * boundary */
-
- if (real_end <= t->floatingMinLiteralMatchOffset) {
+ DEBUG_PRINTF("received a match when we're already dead!\n");
+ return MO_HALT_MATCHING;
+ }
+
+ /* delayed literals need to be delivered before real literals; however
+ * delayed literals only come from the floating table so if we are going
+ * to deliver a literal here it must be too early for a delayed literal */
+
+ /* no history checks from anchored region and we are before the flush
+ * boundary */
+
+ if (real_end <= t->floatingMinLiteralMatchOffset) {
roseFlushLastByteHistory(t, scratch, real_end);
- tctxt->lastEndOffset = real_end;
- }
-
+ tctxt->lastEndOffset = real_end;
+ }
+
// Note that the "id" we have been handed is the program offset.
const u8 flags = ROSE_PROG_FLAG_IN_ANCHORED;
if (roseRunProgram(t, scratch, id, start, real_end, flags)
== HWLM_TERMINATE_MATCHING) {
assert(can_stop_matching(scratch));
- DEBUG_PRINTF("caller requested termination\n");
- return MO_HALT_MATCHING;
- }
-
+ DEBUG_PRINTF("caller requested termination\n");
+ return MO_HALT_MATCHING;
+ }
+
DEBUG_PRINTF("DONE groups=0x%016llx\n", tctxt->groups);
-
- return MO_CONTINUE_MATCHING;
-}
-
+
+ return MO_CONTINUE_MATCHING;
+}
+
/**
* \brief Run the program for the given literal ID, with the interpreter
* inlined into this call.
*
* Assumes not in_anchored.
*/
-static really_inline
+static really_inline
hwlmcb_rv_t roseProcessMatchInline(const struct RoseEngine *t,
struct hs_scratch *scratch, u64a end,
u32 id) {
- DEBUG_PRINTF("id=%u\n", id);
+ DEBUG_PRINTF("id=%u\n", id);
assert(id && id < t->size); // id is an offset into bytecode
const u64a som = 0;
const u8 flags = 0;
@@ -243,296 +243,296 @@ hwlmcb_rv_t roseProcessMatchInline(const struct RoseEngine *t,
} else {
return roseRunProgram(t, scratch, id, som, end, flags);
}
-}
-
-static rose_inline
+}
+
+static rose_inline
hwlmcb_rv_t playDelaySlot(const struct RoseEngine *t,
struct hs_scratch *scratch,
struct fatbit **delaySlots, u32 vicIndex,
u64a offset) {
- /* assert(!tctxt->in_anchored); */
- assert(vicIndex < DELAY_SLOT_COUNT);
+ /* assert(!tctxt->in_anchored); */
+ assert(vicIndex < DELAY_SLOT_COUNT);
const struct fatbit *vicSlot = delaySlots[vicIndex];
u32 delay_count = t->delay_count;
-
+
if (offset < t->floatingMinLiteralMatchOffset) {
- DEBUG_PRINTF("too soon\n");
- return HWLM_CONTINUE_MATCHING;
- }
-
+ DEBUG_PRINTF("too soon\n");
+ return HWLM_CONTINUE_MATCHING;
+ }
+
struct RoseContext *tctxt = &scratch->tctxt;
roseFlushLastByteHistory(t, scratch, offset);
- tctxt->lastEndOffset = offset;
-
+ tctxt->lastEndOffset = offset;
+
const u32 *programs = getByOffset(t, t->delayProgramOffset);
-
+
for (u32 it = fatbit_iterate(vicSlot, delay_count, MMB_INVALID);
it != MMB_INVALID; it = fatbit_iterate(vicSlot, delay_count, it)) {
- UNUSED rose_group old_groups = tctxt->groups;
-
+ UNUSED rose_group old_groups = tctxt->groups;
+
DEBUG_PRINTF("DELAYED MATCH id=%u offset=%llu\n", it, offset);
const u64a som = 0;
const u8 flags = 0;
hwlmcb_rv_t rv = roseRunProgram(t, scratch, programs[it], som, offset,
flags);
DEBUG_PRINTF("DONE groups=0x%016llx\n", tctxt->groups);
-
+
/* delayed literals can't safely set groups.
- * However we may be setting groups that successors already have
- * worked out that we don't need to match the group */
- DEBUG_PRINTF("groups in %016llx out %016llx\n", old_groups,
- tctxt->groups);
-
- if (rv == HWLM_TERMINATE_MATCHING) {
- return HWLM_TERMINATE_MATCHING;
- }
- }
-
- return HWLM_CONTINUE_MATCHING;
-}
-
-static really_inline
+ * However we may be setting groups that successors already have
+ * worked out that we don't need to match the group */
+ DEBUG_PRINTF("groups in %016llx out %016llx\n", old_groups,
+ tctxt->groups);
+
+ if (rv == HWLM_TERMINATE_MATCHING) {
+ return HWLM_TERMINATE_MATCHING;
+ }
+ }
+
+ return HWLM_CONTINUE_MATCHING;
+}
+
+static really_inline
hwlmcb_rv_t flushAnchoredLiteralAtLoc(const struct RoseEngine *t,
struct hs_scratch *scratch,
u32 curr_loc) {
struct RoseContext *tctxt = &scratch->tctxt;
struct fatbit *curr_row = getAnchoredLiteralLog(scratch)[curr_loc - 1];
u32 region_width = t->anchored_count;
-
+
const u32 *programs = getByOffset(t, t->anchoredProgramOffset);
- DEBUG_PRINTF("report matches at curr loc\n");
+ DEBUG_PRINTF("report matches at curr loc\n");
for (u32 it = fatbit_iterate(curr_row, region_width, MMB_INVALID);
it != MMB_INVALID; it = fatbit_iterate(curr_row, region_width, it)) {
- DEBUG_PRINTF("it = %u/%u\n", it, region_width);
-
- rose_group old_groups = tctxt->groups;
+ DEBUG_PRINTF("it = %u/%u\n", it, region_width);
+
+ rose_group old_groups = tctxt->groups;
DEBUG_PRINTF("ANCH REPLAY MATCH id=%u offset=%u\n", it, curr_loc);
const u64a som = 0;
const u8 flags = 0;
hwlmcb_rv_t rv = roseRunProgram(t, scratch, programs[it], som, curr_loc,
flags);
DEBUG_PRINTF("DONE groups=0x%016llx\n", tctxt->groups);
-
+
/* anchored literals can't safely set groups.
* However we may be setting groups that successors already
- * have worked out that we don't need to match the group */
- DEBUG_PRINTF("groups in %016llx out %016llx\n", old_groups,
- tctxt->groups);
- tctxt->groups &= old_groups;
-
- if (rv == HWLM_TERMINATE_MATCHING) {
- return HWLM_TERMINATE_MATCHING;
- }
- }
-
- /* clear row; does not invalidate iteration */
- bf64_unset(&scratch->al_log_sum, curr_loc - 1);
-
- return HWLM_CONTINUE_MATCHING;
-}
-
-static really_inline
+ * have worked out that we don't need to match the group */
+ DEBUG_PRINTF("groups in %016llx out %016llx\n", old_groups,
+ tctxt->groups);
+ tctxt->groups &= old_groups;
+
+ if (rv == HWLM_TERMINATE_MATCHING) {
+ return HWLM_TERMINATE_MATCHING;
+ }
+ }
+
+ /* clear row; does not invalidate iteration */
+ bf64_unset(&scratch->al_log_sum, curr_loc - 1);
+
+ return HWLM_CONTINUE_MATCHING;
+}
+
+static really_inline
u32 anchored_it_begin(struct hs_scratch *scratch) {
struct RoseContext *tctxt = &scratch->tctxt;
- if (tctxt->lastEndOffset >= scratch->anchored_literal_region_len) {
- return MMB_INVALID;
- }
- u32 begin = tctxt->lastEndOffset;
- begin--;
-
+ if (tctxt->lastEndOffset >= scratch->anchored_literal_region_len) {
+ return MMB_INVALID;
+ }
+ u32 begin = tctxt->lastEndOffset;
+ begin--;
+
return bf64_iterate(scratch->al_log_sum, begin);
-}
-
-static really_inline
+}
+
+static really_inline
hwlmcb_rv_t flushAnchoredLiterals(const struct RoseEngine *t,
struct hs_scratch *scratch,
- u32 *anchored_it_param, u64a to_off) {
+ u32 *anchored_it_param, u64a to_off) {
struct RoseContext *tctxt = &scratch->tctxt;
- u32 anchored_it = *anchored_it_param;
- /* catch up any remaining anchored matches */
- for (; anchored_it != MMB_INVALID && anchored_it < to_off;
- anchored_it = bf64_iterate(scratch->al_log_sum, anchored_it)) {
- assert(anchored_it < scratch->anchored_literal_region_len);
- DEBUG_PRINTF("loc_it = %u\n", anchored_it);
- u32 curr_off = anchored_it + 1;
+ u32 anchored_it = *anchored_it_param;
+ /* catch up any remaining anchored matches */
+ for (; anchored_it != MMB_INVALID && anchored_it < to_off;
+ anchored_it = bf64_iterate(scratch->al_log_sum, anchored_it)) {
+ assert(anchored_it < scratch->anchored_literal_region_len);
+ DEBUG_PRINTF("loc_it = %u\n", anchored_it);
+ u32 curr_off = anchored_it + 1;
roseFlushLastByteHistory(t, scratch, curr_off);
- tctxt->lastEndOffset = curr_off;
-
+ tctxt->lastEndOffset = curr_off;
+
if (flushAnchoredLiteralAtLoc(t, scratch, curr_off)
- == HWLM_TERMINATE_MATCHING) {
- return HWLM_TERMINATE_MATCHING;
- }
- }
-
- *anchored_it_param = anchored_it;
- return HWLM_CONTINUE_MATCHING;
-}
-
-static really_inline
+ == HWLM_TERMINATE_MATCHING) {
+ return HWLM_TERMINATE_MATCHING;
+ }
+ }
+
+ *anchored_it_param = anchored_it;
+ return HWLM_CONTINUE_MATCHING;
+}
+
+static really_inline
hwlmcb_rv_t playVictims(const struct RoseEngine *t, struct hs_scratch *scratch,
u32 *anchored_it, u64a lastEnd, u64a victimDelaySlots,
struct fatbit **delaySlots) {
- while (victimDelaySlots) {
- u32 vic = findAndClearLSB_64(&victimDelaySlots);
- DEBUG_PRINTF("vic = %u\n", vic);
- u64a vicOffset = vic + (lastEnd & ~(u64a)DELAY_MASK);
-
+ while (victimDelaySlots) {
+ u32 vic = findAndClearLSB_64(&victimDelaySlots);
+ DEBUG_PRINTF("vic = %u\n", vic);
+ u64a vicOffset = vic + (lastEnd & ~(u64a)DELAY_MASK);
+
if (flushAnchoredLiterals(t, scratch, anchored_it, vicOffset)
- == HWLM_TERMINATE_MATCHING) {
- return HWLM_TERMINATE_MATCHING;
- }
-
+ == HWLM_TERMINATE_MATCHING) {
+ return HWLM_TERMINATE_MATCHING;
+ }
+
if (playDelaySlot(t, scratch, delaySlots, vic % DELAY_SLOT_COUNT,
vicOffset) == HWLM_TERMINATE_MATCHING) {
- return HWLM_TERMINATE_MATCHING;
- }
- }
-
- return HWLM_CONTINUE_MATCHING;
-}
-
-/* call flushQueuedLiterals instead */
+ return HWLM_TERMINATE_MATCHING;
+ }
+ }
+
+ return HWLM_CONTINUE_MATCHING;
+}
+
+/* call flushQueuedLiterals instead */
hwlmcb_rv_t flushQueuedLiterals_i(const struct RoseEngine *t,
struct hs_scratch *scratch, u64a currEnd) {
struct RoseContext *tctxt = &scratch->tctxt;
- u64a lastEnd = tctxt->delayLastEndOffset;
- DEBUG_PRINTF("flushing backed up matches @%llu up from %llu\n", currEnd,
- lastEnd);
-
- assert(currEnd != lastEnd); /* checked in main entry point */
-
+ u64a lastEnd = tctxt->delayLastEndOffset;
+ DEBUG_PRINTF("flushing backed up matches @%llu up from %llu\n", currEnd,
+ lastEnd);
+
+ assert(currEnd != lastEnd); /* checked in main entry point */
+
u32 anchored_it = anchored_it_begin(scratch);
-
- if (!tctxt->filledDelayedSlots) {
- DEBUG_PRINTF("no delayed, no flush\n");
- goto anchored_leftovers;
- }
-
- {
+
+ if (!tctxt->filledDelayedSlots) {
+ DEBUG_PRINTF("no delayed, no flush\n");
+ goto anchored_leftovers;
+ }
+
+ {
struct fatbit **delaySlots = getDelaySlots(scratch);
-
- u32 lastIndex = lastEnd & DELAY_MASK;
- u32 currIndex = currEnd & DELAY_MASK;
-
- int wrapped = (lastEnd | DELAY_MASK) < currEnd;
-
- u64a victimDelaySlots; /* needs to be twice as wide as the number of
- * slots. */
-
- DEBUG_PRINTF("hello %08x\n", tctxt->filledDelayedSlots);
- if (!wrapped) {
- victimDelaySlots = tctxt->filledDelayedSlots;
-
- DEBUG_PRINTF("unwrapped %016llx %08x\n", victimDelaySlots,
- tctxt->filledDelayedSlots);
- /* index vars < 32 so 64bit shifts are safe */
-
- /* clear all slots at last index and below, */
- victimDelaySlots &= ~((1LLU << (lastIndex + 1)) - 1);
-
- /* clear all slots above curr index */
- victimDelaySlots &= (1LLU << (currIndex + 1)) - 1;
-
- tctxt->filledDelayedSlots &= ~victimDelaySlots;
-
- DEBUG_PRINTF("unwrapped %016llx %08x\n", victimDelaySlots,
- tctxt->filledDelayedSlots);
- } else {
- DEBUG_PRINTF("wrapped %08x\n", tctxt->filledDelayedSlots);
-
- /* 1st half: clear all slots at last index and below, */
- u64a first_half = tctxt->filledDelayedSlots;
- first_half &= ~((1ULL << (lastIndex + 1)) - 1);
- tctxt->filledDelayedSlots &= (1ULL << (lastIndex + 1)) - 1;
-
- u64a second_half = tctxt->filledDelayedSlots;
-
- if (currEnd > lastEnd + DELAY_SLOT_COUNT) {
- /* 2nd half: clear all slots above last index */
- second_half &= (1ULL << (lastIndex + 1)) - 1;
- } else {
- /* 2nd half: clear all slots above curr index */
- second_half &= (1ULL << (currIndex + 1)) - 1;
- }
- tctxt->filledDelayedSlots &= ~second_half;
-
- victimDelaySlots = first_half | (second_half << DELAY_SLOT_COUNT);
-
- DEBUG_PRINTF("-- %016llx %016llx = %016llx (li %u)\n", first_half,
- second_half, victimDelaySlots, lastIndex);
- }
-
+
+ u32 lastIndex = lastEnd & DELAY_MASK;
+ u32 currIndex = currEnd & DELAY_MASK;
+
+ int wrapped = (lastEnd | DELAY_MASK) < currEnd;
+
+ u64a victimDelaySlots; /* needs to be twice as wide as the number of
+ * slots. */
+
+ DEBUG_PRINTF("hello %08x\n", tctxt->filledDelayedSlots);
+ if (!wrapped) {
+ victimDelaySlots = tctxt->filledDelayedSlots;
+
+ DEBUG_PRINTF("unwrapped %016llx %08x\n", victimDelaySlots,
+ tctxt->filledDelayedSlots);
+ /* index vars < 32 so 64bit shifts are safe */
+
+ /* clear all slots at last index and below, */
+ victimDelaySlots &= ~((1LLU << (lastIndex + 1)) - 1);
+
+ /* clear all slots above curr index */
+ victimDelaySlots &= (1LLU << (currIndex + 1)) - 1;
+
+ tctxt->filledDelayedSlots &= ~victimDelaySlots;
+
+ DEBUG_PRINTF("unwrapped %016llx %08x\n", victimDelaySlots,
+ tctxt->filledDelayedSlots);
+ } else {
+ DEBUG_PRINTF("wrapped %08x\n", tctxt->filledDelayedSlots);
+
+ /* 1st half: clear all slots at last index and below, */
+ u64a first_half = tctxt->filledDelayedSlots;
+ first_half &= ~((1ULL << (lastIndex + 1)) - 1);
+ tctxt->filledDelayedSlots &= (1ULL << (lastIndex + 1)) - 1;
+
+ u64a second_half = tctxt->filledDelayedSlots;
+
+ if (currEnd > lastEnd + DELAY_SLOT_COUNT) {
+ /* 2nd half: clear all slots above last index */
+ second_half &= (1ULL << (lastIndex + 1)) - 1;
+ } else {
+ /* 2nd half: clear all slots above curr index */
+ second_half &= (1ULL << (currIndex + 1)) - 1;
+ }
+ tctxt->filledDelayedSlots &= ~second_half;
+
+ victimDelaySlots = first_half | (second_half << DELAY_SLOT_COUNT);
+
+ DEBUG_PRINTF("-- %016llx %016llx = %016llx (li %u)\n", first_half,
+ second_half, victimDelaySlots, lastIndex);
+ }
+
if (playVictims(t, scratch, &anchored_it, lastEnd, victimDelaySlots,
delaySlots) == HWLM_TERMINATE_MATCHING) {
- return HWLM_TERMINATE_MATCHING;
- }
- }
-
-anchored_leftovers:;
+ return HWLM_TERMINATE_MATCHING;
+ }
+ }
+
+anchored_leftovers:;
hwlmcb_rv_t rv = flushAnchoredLiterals(t, scratch, &anchored_it, currEnd);
- tctxt->delayLastEndOffset = currEnd;
- return rv;
-}
-
+ tctxt->delayLastEndOffset = currEnd;
+ return rv;
+}
+
static really_inline
hwlmcb_rv_t roseCallback_i(size_t end, u32 id, struct hs_scratch *scratch) {
struct RoseContext *tctx = &scratch->tctxt;
const struct RoseEngine *t = scratch->core_info.rose;
- u64a real_end = end + tctx->lit_offset_adjust;
-
-#if defined(DEBUG)
+ u64a real_end = end + tctx->lit_offset_adjust;
+
+#if defined(DEBUG)
DEBUG_PRINTF("MATCH id=%u end offset@%llu: ", id, real_end);
u64a start = real_end < 8 ? 1 : real_end - 7;
printMatch(&scratch->core_info, start, real_end);
- printf("\n");
-#endif
- DEBUG_PRINTF("last end %llu\n", tctx->lastEndOffset);
-
+ printf("\n");
+#endif
+ DEBUG_PRINTF("last end %llu\n", tctx->lastEndOffset);
+
DEBUG_PRINTF("STATE groups=0x%016llx\n", tctx->groups);
-
+
if (can_stop_matching(scratch)) {
- DEBUG_PRINTF("received a match when we're already dead!\n");
- return HWLM_TERMINATE_MATCHING;
- }
-
+ DEBUG_PRINTF("received a match when we're already dead!\n");
+ return HWLM_TERMINATE_MATCHING;
+ }
+
hwlmcb_rv_t rv = flushQueuedLiterals(t, scratch, real_end);
- /* flushDelayed may have advanced tctx->lastEndOffset */
-
+ /* flushDelayed may have advanced tctx->lastEndOffset */
+
if (real_end >= t->floatingMinLiteralMatchOffset) {
roseFlushLastByteHistory(t, scratch, real_end);
- tctx->lastEndOffset = real_end;
- }
-
- if (rv == HWLM_TERMINATE_MATCHING) {
- return HWLM_TERMINATE_MATCHING;
- }
-
+ tctx->lastEndOffset = real_end;
+ }
+
+ if (rv == HWLM_TERMINATE_MATCHING) {
+ return HWLM_TERMINATE_MATCHING;
+ }
+
rv = roseProcessMatchInline(t, scratch, real_end, id);
-
+
DEBUG_PRINTF("DONE groups=0x%016llx\n", tctx->groups);
-
- if (rv != HWLM_TERMINATE_MATCHING) {
- return tctx->groups;
- }
-
+
+ if (rv != HWLM_TERMINATE_MATCHING) {
+ return tctx->groups;
+ }
+
assert(can_stop_matching(scratch));
- DEBUG_PRINTF("user requested halt\n");
- return HWLM_TERMINATE_MATCHING;
-}
-
+ DEBUG_PRINTF("user requested halt\n");
+ return HWLM_TERMINATE_MATCHING;
+}
+
hwlmcb_rv_t roseCallback(size_t end, u32 id, struct hs_scratch *scratch) {
return roseCallback_i(end, id, scratch);
}
-
+
hwlmcb_rv_t roseFloatingCallback(size_t end, u32 id,
struct hs_scratch *scratch) {
const struct RoseEngine *t = scratch->core_info.rose;
-
+
return roseCallback_i(end, id, scratch) & t->floating_group_mask;
}
-
+
/**
* \brief Execute a boundary report program.
*
@@ -542,12 +542,12 @@ hwlmcb_rv_t roseFloatingCallback(size_t end, u32 id,
int roseRunBoundaryProgram(const struct RoseEngine *rose, u32 program,
u64a stream_offset, struct hs_scratch *scratch) {
DEBUG_PRINTF("running boundary program at offset %u\n", program);
-
+
if (can_stop_matching(scratch)) {
DEBUG_PRINTF("can stop matching\n");
return MO_HALT_MATCHING;
}
-
+
if (rose->hasSom && scratch->deduper.current_report_offset == ~0ULL) {
/* we cannot delay the initialization of the som deduper logs any longer
* as we are reporting matches. This is done explicitly as we are
@@ -557,13 +557,13 @@ int roseRunBoundaryProgram(const struct RoseEngine *rose, u32 program,
fatbit_clear(scratch->deduper.som_log[1]);
scratch->deduper.som_log_dirty = 0;
}
-
+
// Keep assertions in program report path happy. At offset zero, there can
// have been no earlier reports. At EOD, all earlier reports should have
// been handled and we will have been caught up to the stream offset by the
// time we are running boundary report programs.
scratch->tctxt.minMatchOffset = stream_offset;
-
+
const u64a som = 0;
const u8 flags = 0;
hwlmcb_rv_t rv = roseRunProgram(rose, scratch, program, som, stream_offset,
@@ -573,7 +573,7 @@ int roseRunBoundaryProgram(const struct RoseEngine *rose, u32 program,
}
return MO_CONTINUE_MATCHING;
-}
+}
/**
* \brief Execute a flush combination program.