diff --git a/CHANGELOG.md b/CHANGELOG.md index 19a92b909..8de3a8d6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,30 @@ This is a list of notable changes to Hyperscan, in reverse chronological order. +## [5.4.0] 2020-12-31 +- Improvement on literal matcher "Fat Teddy" performance, including + support for Intel(R) AVX-512 Vector Byte Manipulation Instructions (Intel(R) + AVX-512 VBMI). +- Introduce a new 32-state shuffle-based DFA engine ("Sheng32"). This improves + scanning performance by leveraging AVX-512 VBMI. +- Introduce a new 64-state shuffle-based DFA engine ("Sheng64"). This improves + scanning performance by leveraging AVX-512 VBMI. +- Introduce a new shuffle-based hybrid DFA engine ("McSheng64"). This improves + scanning performance by leveraging AVX-512 VBMI. +- Improvement on exceptional state handling performance for LimEx NFA, including + support for AVX-512 VBMI. +- Improvement on lookaround performance with new models, including support for + AVX-512. +- Improvement on DFA state space efficiency. +- Optimization on decision of NFA/DFA generation. +- hsbench: add CSV dump support for hsbench. +- Bugfix for cmake error on Icelake under release mode. +- Bugfix in find_vertices_in_cycles() to avoid self-loop checking in SCC. +- Bugfix for issue #270: fix return value handling in chimera. +- Bugfix for issue #284: use correct free function in logical combination. +- Add BUILD_EXAMPLES cmake option to enable example code compilation. (#260) +- Some typo fixing. (#242, #259) + ## [5.3.0] 2020-05-15 - Improvement on literal matcher "Teddy" performance, including support for Intel(R) AVX-512 Vector Byte Manipulation Instructions (Intel(R) AVX-512 diff --git a/CMakeLists.txt b/CMakeLists.txt index 59c6e6e2f..8bc6077fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required (VERSION 2.8.11) project (hyperscan C CXX) set (HS_MAJOR_VERSION 5) -set (HS_MINOR_VERSION 3) +set (HS_MINOR_VERSION 4) set (HS_PATCH_VERSION 0) set (HS_VERSION ${HS_MAJOR_VERSION}.${HS_MINOR_VERSION}.${HS_PATCH_VERSION}) @@ -133,6 +133,13 @@ CMAKE_DEPENDENT_OPTION(DISABLE_ASSERTS "Disable assert(); Asserts are enabled in option(BUILD_AVX512 "Experimental: support avx512 in the fat runtime" OFF) +option(BUILD_AVX512VBMI "Experimental: support avx512vbmi in the fat runtime" + OFF) + +if (BUILD_AVX512VBMI) + set(BUILD_AVX512 ON) +endif () + option(WINDOWS_ICC "Use Intel C++ Compiler on Windows, default off, requires ICC to be set in project" OFF) # TODO: per platform config files? @@ -277,6 +284,7 @@ else() set(SKYLAKE_FLAG "-xCORE-AVX512") else () set(SKYLAKE_FLAG "-march=skylake-avx512") + set(ICELAKE_FLAG "-march=icelake-server") endif () endif() @@ -1197,6 +1205,9 @@ else (FAT_RUNTIME) if (NOT BUILD_AVX512) set (DISPATCHER_DEFINE "-DDISABLE_AVX512_DISPATCH") endif (NOT BUILD_AVX512) + if (NOT BUILD_AVX512VBMI) + set (DISPATCHER_DEFINE "${DISPATCHER_DEFINE} -DDISABLE_AVX512VBMI_DISPATCH") + endif (NOT BUILD_AVX512VBMI) set_source_files_properties(src/dispatcher.c PROPERTIES COMPILE_FLAGS "-Wno-unused-parameter -Wno-unused-function ${DISPATCHER_DEFINE}") @@ -1229,6 +1240,14 @@ else (FAT_RUNTIME) RULE_LAUNCH_COMPILE "${BUILD_WRAPPER} avx512 ${CMAKE_MODULE_PATH}/keep.syms.in" ) endif (BUILD_AVX512) + if (BUILD_AVX512VBMI) + add_library(hs_exec_avx512vbmi OBJECT ${hs_exec_SRCS} ${hs_exec_avx2_SRCS}) + list(APPEND RUNTIME_LIBS $) + set_target_properties(hs_exec_avx512vbmi PROPERTIES + COMPILE_FLAGS "${ICELAKE_FLAG}" + RULE_LAUNCH_COMPILE "${BUILD_WRAPPER} avx512vbmi ${CMAKE_MODULE_PATH}/keep.syms.in" + ) + endif (BUILD_AVX512VBMI) add_library(hs_exec_common OBJECT ${hs_exec_common_SRCS} @@ -1287,6 +1306,15 @@ else (FAT_RUNTIME) RULE_LAUNCH_COMPILE "${BUILD_WRAPPER} avx512 ${CMAKE_MODULE_PATH}/keep.syms.in" ) endif (BUILD_AVX512) + if (BUILD_AVX512VBMI) + add_library(hs_exec_shared_avx512vbmi OBJECT ${hs_exec_SRCS} ${hs_exec_avx2_SRCS}) + list(APPEND RUNTIME_SHLIBS $) + set_target_properties(hs_exec_shared_avx512vbmi PROPERTIES + COMPILE_FLAGS "${ICELAKE_FLAG}" + POSITION_INDEPENDENT_CODE TRUE + RULE_LAUNCH_COMPILE "${BUILD_WRAPPER} avx512vbmi ${CMAKE_MODULE_PATH}/keep.syms.in" + ) + endif (BUILD_AVX512VBMI) add_library(hs_exec_common_shared OBJECT ${hs_exec_common_SRCS} src/dispatcher.c @@ -1380,7 +1408,7 @@ if (NOT BUILD_STATIC_LIBS) add_library(hs ALIAS hs_shared) endif () - -if(NOT WIN32) +option(BUILD_EXAMPLES "Build Hyperscan example code (default TRUE)" TRUE) +if(NOT WIN32 AND BUILD_EXAMPLES) add_subdirectory(examples) endif() diff --git a/chimera/ch_common.h b/chimera/ch_common.h index 8caa44407..bdb0bafa9 100644 --- a/chimera/ch_common.h +++ b/chimera/ch_common.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Intel Corporation + * Copyright (c) 2018-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -345,6 +345,16 @@ ch_error_t HS_CDECL ch_set_scratch_allocator(ch_alloc_t alloc_func, */ #define CH_SCRATCH_IN_USE (-10) +/** + * Unexpected internal error from Hyperscan. + * + * This error indicates that there was unexpected matching behaviors from + * Hyperscan. This could be related to invalid usage of scratch space or + * invalid memory operations by users. + * + */ +#define CH_UNKNOWN_HS_ERROR (-13) + /** * Returned when pcre_exec (called for some expressions internally from @ref * ch_scan) failed due to a fatal error. diff --git a/chimera/ch_runtime.c b/chimera/ch_runtime.c index 212bbc7be..fdb5b992b 100644 --- a/chimera/ch_runtime.c +++ b/chimera/ch_runtime.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Intel Corporation + * Copyright (c) 2018-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -419,6 +419,7 @@ int HS_CDECL multiCallback(unsigned int id, unsigned long long from, DEBUG_PRINTF("user callback told us to skip this pattern\n"); pd->scanStart = hyctx->length; ret = HS_SUCCESS; + hyctx->scratch->ret = ret; } else if (ret == CH_FAIL_INTERNAL) { return ret; } @@ -590,11 +591,24 @@ ch_error_t ch_scan_i(const ch_database_t *hydb, if (!(db->flags & CHIMERA_FLAG_NO_MULTIMATCH)) { ret = scanHyperscan(&hyctx, data, length); - if (ret != HS_SUCCESS && scratch->ret != CH_SUCCESS) { - DEBUG_PRINTF("Hyperscan returned error %d\n", scratch->ret); + // Errors from pcre scan. + if (scratch->ret == CH_CALLBACK_TERMINATE) { + DEBUG_PRINTF("Pcre terminates scan\n"); + unmarkScratchInUse(scratch); + return CH_SCAN_TERMINATED; + } else if (scratch->ret != CH_SUCCESS) { + DEBUG_PRINTF("Pcre internal error\n"); unmarkScratchInUse(scratch); return scratch->ret; } + // Errors from Hyperscan scan. Note Chimera could terminate + // Hyperscan callback on purpose so this is not counted as an error. + if (ret != HS_SUCCESS && ret != HS_SCAN_TERMINATED) { + assert(scratch->ret == CH_SUCCESS); + DEBUG_PRINTF("Hyperscan returned error %d\n", ret); + unmarkScratchInUse(scratch); + return ret; + } } DEBUG_PRINTF("Flush priority queue\n"); diff --git a/cmake/arch.cmake b/cmake/arch.cmake index cced49c69..eb4791e6b 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -17,10 +17,21 @@ if (BUILD_AVX512) endif () endif () +if (BUILD_AVX512VBMI) + CHECK_C_COMPILER_FLAG(${ICELAKE_FLAG} HAS_ARCH_ICELAKE) + if (NOT HAS_ARCH_ICELAKE) + message (FATAL_ERROR "AVX512VBMI not supported by compiler") + endif () +endif () + if (FAT_RUNTIME) # test the highest level microarch to make sure everything works if (BUILD_AVX512) - set (CMAKE_REQUIRED_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_C_FLAGS} ${SKYLAKE_FLAG}") + if (BUILD_AVX512VBMI) + set (CMAKE_REQUIRED_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_C_FLAGS} ${ICELAKE_FLAG}") + else () + set (CMAKE_REQUIRED_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_C_FLAGS} ${SKYLAKE_FLAG}") + endif (BUILD_AVX512VBMI) else () set (CMAKE_REQUIRED_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_C_FLAGS} -march=core-avx2") endif () @@ -80,6 +91,9 @@ if (FAT_RUNTIME) if (BUILD_AVX512 AND NOT HAVE_AVX512) message(FATAL_ERROR "AVX512 support requested but not supported") endif () + if (BUILD_AVX512VBMI AND NOT HAVE_AVX512VBMI) + message(FATAL_ERROR "AVX512VBMI support requested but not supported") + endif () else (NOT FAT_RUNTIME) if (NOT HAVE_AVX2) message(STATUS "Building without AVX2 support") @@ -87,6 +101,9 @@ else (NOT FAT_RUNTIME) if (NOT HAVE_AVX512) message(STATUS "Building without AVX512 support") endif () + if (NOT HAVE_AVX512VBMI) + message(STATUS "Building without AVX512VBMI support") + endif () if (NOT HAVE_SSSE3) message(FATAL_ERROR "A minimum of SSSE3 compiler support is required") endif () diff --git a/cmake/config.h.in b/cmake/config.h.in index 203f0afde..5454643e7 100644 --- a/cmake/config.h.in +++ b/cmake/config.h.in @@ -24,6 +24,9 @@ /* Define if building AVX-512 in the fat runtime. */ #cmakedefine BUILD_AVX512 +/* Define if building AVX512VBMI in the fat runtime. */ +#cmakedefine BUILD_AVX512VBMI + /* Define to 1 if `backtrace' works. */ #cmakedefine HAVE_BACKTRACE diff --git a/doc/dev-reference/chimera.rst b/doc/dev-reference/chimera.rst index 883cb5a0a..d35b116f5 100644 --- a/doc/dev-reference/chimera.rst +++ b/doc/dev-reference/chimera.rst @@ -212,7 +212,7 @@ space is required for that context. In the absence of recursive scanning, only one such space is required per thread and can (and indeed should) be allocated before data scanning is to commence. -In a scenario where a set of expressions are compiled by a single "master" +In a scenario where a set of expressions are compiled by a single "main" thread and data will be scanned by multiple "worker" threads, the convenience function :c:func:`ch_clone_scratch` allows multiple copies of an existing scratch space to be made for each thread (rather than forcing the caller to pass diff --git a/doc/dev-reference/compilation.rst b/doc/dev-reference/compilation.rst index 205b7348b..6f5541ecf 100644 --- a/doc/dev-reference/compilation.rst +++ b/doc/dev-reference/compilation.rst @@ -64,21 +64,21 @@ interpreted independently. No syntax association happens between any adjacent characters. For example, given an expression written as :regexp:`/bc?/`. We could say it is -a regluar expression, with the meaning that character ``b`` followed by nothing +a regular expression, with the meaning that character ``b`` followed by nothing or by one character ``c``. On the other view, we could also say it is a pure literal expression, with the meaning that this is a character sequence of 3-byte length, containing characters ``b``, ``c`` and ``?``. In regular case, the question mark character ``?`` has a particular syntax role called 0-1 quantifier, -which has an syntax association with the character ahead of it. Similar -characters exist in regular grammer like ``[``, ``]``, ``(``, ``)``, ``{``, +which has a syntax association with the character ahead of it. Similar +characters exist in regular grammar like ``[``, ``]``, ``(``, ``)``, ``{``, ``}``, ``-``, ``*``, ``+``, ``\``, ``|``, ``/``, ``:``, ``^``, ``.``, ``$``. While in pure literal case, all these meta characters lost extra meanings expect for that they are just common ASCII codes. Hyperscan is initially designed to process common regular expressions. It is -hence embedded with a complex parser to do comprehensive regular grammer -interpretion. Particularly, the identification of above meta characters is the -basic step for the interpretion of far more complex regular grammers. +hence embedded with a complex parser to do comprehensive regular grammar +interpretation. Particularly, the identification of above meta characters is the +basic step for the interpretation of far more complex regular grammars. However in real cases, patterns may not always be regular expressions. They could just be pure literals. Problem will come if the pure literals contain @@ -165,7 +165,7 @@ The following regex constructs are supported by Hyperscan: :regexp:`{n,}` are supported with limitations. * For arbitrary repeated sub-patterns: *n* and *m* should be either small - or infinite, e.g. :regexp:`(a|b}{4}`, :regexp:`(ab?c?d){4,10}` or + or infinite, e.g. :regexp:`(a|b){4}`, :regexp:`(ab?c?d){4,10}` or :regexp:`(ab(cd)*){6,}`. * For single-character width sub-patterns such as :regexp:`[^\\a]` or diff --git a/doc/dev-reference/getting_started.rst b/doc/dev-reference/getting_started.rst index b38128733..aaff15ba2 100644 --- a/doc/dev-reference/getting_started.rst +++ b/doc/dev-reference/getting_started.rst @@ -263,17 +263,19 @@ the current platform is supported by Hyperscan. As of this release, the variants of the runtime that are built, and the CPU capability that is required, are the following: -+----------+-------------------------------+---------------------------+ -| Variant | CPU Feature Flag(s) Required | gcc arch flag | -+==========+===============================+===========================+ -| Core 2 | ``SSSE3`` | ``-march=core2`` | -+----------+-------------------------------+---------------------------+ -| Core i7 | ``SSE4_2`` and ``POPCNT`` | ``-march=corei7`` | -+----------+-------------------------------+---------------------------+ -| AVX 2 | ``AVX2`` | ``-march=core-avx2`` | -+----------+-------------------------------+---------------------------+ -| AVX 512 | ``AVX512BW`` (see note below) | ``-march=skylake-avx512`` | -+----------+-------------------------------+---------------------------+ ++--------------+---------------------------------+---------------------------+ +| Variant | CPU Feature Flag(s) Required | gcc arch flag | ++==============+=================================+===========================+ +| Core 2 | ``SSSE3`` | ``-march=core2`` | ++--------------+---------------------------------+---------------------------+ +| Core i7 | ``SSE4_2`` and ``POPCNT`` | ``-march=corei7`` | ++--------------+---------------------------------+---------------------------+ +| AVX 2 | ``AVX2`` | ``-march=core-avx2`` | ++--------------+---------------------------------+---------------------------+ +| AVX 512 | ``AVX512BW`` (see note below) | ``-march=skylake-avx512`` | ++--------------+---------------------------------+---------------------------+ +| AVX 512 VBMI | ``AVX512VBMI`` (see note below) | ``-march=icelake-server`` | ++--------------+---------------------------------+---------------------------+ .. note:: @@ -287,6 +289,16 @@ capability that is required, are the following: cmake -DBUILD_AVX512=on <...> + Hyperscan v5.3 adds support for AVX512VBMI instructions - in particular the + ``AVX512VBMI`` instruction set that was introduced on Intel "Icelake" Xeon + processors - however the AVX512VBMI runtime variant is **not** enabled by + default in fat runtime builds as not all toolchains support AVX512VBMI + instruction sets. To build an AVX512VBMI runtime, the CMake variable + ``BUILD_AVX512VBMI`` must be enabled manually during configuration. For + example: :: + + cmake -DBUILD_AVX512VBMI=on <...> + As the fat runtime requires compiler, libc, and binutils support, at this time it will only be enabled for Linux builds where the compiler supports the `indirect function "ifunc" function attribute diff --git a/doc/dev-reference/runtime.rst b/doc/dev-reference/runtime.rst index d64ec540d..396521c94 100644 --- a/doc/dev-reference/runtime.rst +++ b/doc/dev-reference/runtime.rst @@ -178,7 +178,7 @@ space is required for that context. In the absence of recursive scanning, only one such space is required per thread and can (and indeed should) be allocated before data scanning is to commence. -In a scenario where a set of expressions are compiled by a single "master" +In a scenario where a set of expressions are compiled by a single "main" thread and data will be scanned by multiple "worker" threads, the convenience function :c:func:`hs_clone_scratch` allows multiple copies of an existing scratch space to be made for each thread (rather than forcing the caller to pass diff --git a/src/compiler/compiler.cpp b/src/compiler/compiler.cpp index 666eefc9c..5751bd64f 100644 --- a/src/compiler/compiler.cpp +++ b/src/compiler/compiler.cpp @@ -458,6 +458,9 @@ platform_t target_to_platform(const target_t &target_info) { if (!target_info.has_avx512()) { p |= HS_PLATFORM_NOAVX512; } + if (!target_info.has_avx512vbmi()) { + p |= HS_PLATFORM_NOAVX512VBMI; + } return p; } diff --git a/src/database.c b/src/database.c index 1a79800e2..6adf1419d 100644 --- a/src/database.c +++ b/src/database.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -115,7 +115,8 @@ static hs_error_t db_check_platform(const u64a p) { if (p != hs_current_platform && p != (hs_current_platform | hs_current_platform_no_avx2) - && p != (hs_current_platform | hs_current_platform_no_avx512)) { + && p != (hs_current_platform | hs_current_platform_no_avx512) + && p != (hs_current_platform | hs_current_platform_no_avx512vbmi)) { return HS_DB_PLATFORM_ERROR; } // passed all checks @@ -370,9 +371,11 @@ hs_error_t print_database_string(char **s, u32 version, const platform_t plat, u8 minor = (version >> 16) & 0xff; u8 major = (version >> 24) & 0xff; - const char *features = (plat & HS_PLATFORM_NOAVX512) - ? (plat & HS_PLATFORM_NOAVX2) ? "" : "AVX2" - : "AVX512"; + const char *features = (plat & HS_PLATFORM_NOAVX512VBMI) + ? (plat & HS_PLATFORM_NOAVX512) + ? (plat & HS_PLATFORM_NOAVX2) ? "" : "AVX2" + : "AVX512" + : "AVX512VBMI"; const char *mode = NULL; diff --git a/src/database.h b/src/database.h index 5715ed677..f122f97be 100644 --- a/src/database.h +++ b/src/database.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -55,6 +55,7 @@ extern "C" #define HS_PLATFORM_NOAVX2 (4<<13) #define HS_PLATFORM_NOAVX512 (8<<13) +#define HS_PLATFORM_NOAVX512VBMI (0x10<<13) /** \brief Platform features bitmask. */ typedef u64a platform_t; @@ -66,6 +67,9 @@ const platform_t hs_current_platform = { #endif #if !defined(HAVE_AVX512) HS_PLATFORM_NOAVX512 | +#endif +#if !defined(HAVE_AVX512VBMI) + HS_PLATFORM_NOAVX512VBMI | #endif 0, }; @@ -74,12 +78,20 @@ static UNUSED const platform_t hs_current_platform_no_avx2 = { HS_PLATFORM_NOAVX2 | HS_PLATFORM_NOAVX512 | + HS_PLATFORM_NOAVX512VBMI | 0, }; static UNUSED const platform_t hs_current_platform_no_avx512 = { HS_PLATFORM_NOAVX512 | + HS_PLATFORM_NOAVX512VBMI | + 0, +}; + +static UNUSED +const platform_t hs_current_platform_no_avx512vbmi = { + HS_PLATFORM_NOAVX512VBMI | 0, }; diff --git a/src/dispatcher.c b/src/dispatcher.c index a786b806d..9a8afa623 100644 --- a/src/dispatcher.c +++ b/src/dispatcher.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,8 +38,14 @@ #define check_avx512() (0) #endif +#if defined(DISABLE_AVX512VBMI_DISPATCH) +#define avx512vbmi_ disabled_ +#define check_avx512vbmi() (0) +#endif + #define CREATE_DISPATCH(RTYPE, NAME, ...) \ /* create defns */ \ + RTYPE JOIN(avx512vbmi_, NAME)(__VA_ARGS__); \ RTYPE JOIN(avx512_, NAME)(__VA_ARGS__); \ RTYPE JOIN(avx2_, NAME)(__VA_ARGS__); \ RTYPE JOIN(corei7_, NAME)(__VA_ARGS__); \ @@ -52,6 +58,9 @@ \ /* resolver */ \ static RTYPE (*JOIN(resolve_, NAME)(void))(__VA_ARGS__) { \ + if (check_avx512vbmi()) { \ + return JOIN(avx512vbmi_, NAME); \ + } \ if (check_avx512()) { \ return JOIN(avx512_, NAME); \ } \ diff --git a/src/fdr/fdr_dump.cpp b/src/fdr/fdr_dump.cpp index f4cd1f44e..1dda751ac 100644 --- a/src/fdr/fdr_dump.cpp +++ b/src/fdr/fdr_dump.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -107,6 +107,25 @@ void dumpTeddyReinforced(const u8 *rmsk, const u32 num_tables, FILE *f) { } } +static +void dumpTeddyDupMasks(const u8 *dmsk, u32 numMasks, FILE *f) { + // dump nibble masks + u32 maskWidth = 2; + fprintf(f, " dup nibble masks:\n"); + for (u32 i = 0; i < numMasks * 2; i++) { + fprintf(f, " -%d%s: ", 1 + i / 2, (i % 2) ? "hi" : "lo"); + for (u32 j = 0; j < 16 * maskWidth * 2; j++) { + u8 val = dmsk[i * 16 * maskWidth * 2 + j]; + for (u32 k = 0; k < 8; k++) { + fprintf(f, "%s", ((val >> k) & 0x1) ? "1" : "0"); + } + fprintf(f, " "); + } + fprintf(f, "\n"); + } + fprintf(f, "\n"); +} + static void dumpTeddyMasks(const u8 *baseMsk, u32 numMasks, u32 maskWidth, FILE *f) { // dump nibble masks @@ -146,12 +165,17 @@ void dumpTeddy(const Teddy *teddy, FILE *f) { u32 maskWidth = des->getNumBuckets() / 8; size_t headerSize = sizeof(Teddy); - size_t maskLen = des->numMasks * 16 * 2 * maskWidth; const u8 *teddy_base = (const u8 *)teddy; const u8 *baseMsk = teddy_base + ROUNDUP_CL(headerSize); - const u8 *rmsk = baseMsk + ROUNDUP_CL(maskLen); dumpTeddyMasks(baseMsk, des->numMasks, maskWidth, f); - dumpTeddyReinforced(rmsk, maskWidth, f); + size_t maskLen = des->numMasks * 16 * 2 * maskWidth; + const u8 *rdmsk = baseMsk + ROUNDUP_CL(maskLen); + if (maskWidth == 1) { // reinforcement table in Teddy + dumpTeddyReinforced(rdmsk, maskWidth, f); + } else { // dup nibble mask table in Fat Teddy + assert(maskWidth == 2); + dumpTeddyDupMasks(rdmsk, des->numMasks, f); + } dumpConfirms(teddy, teddy->confOffset, des->getNumBuckets(), f); } diff --git a/src/fdr/teddy.c b/src/fdr/teddy.c index 960e2a415..e6f547619 100644 --- a/src/fdr/teddy.c +++ b/src/fdr/teddy.c @@ -284,14 +284,6 @@ m512 prep_conf_teddy_m4(const m512 *lo_mask, const m512 *dup_mask, #define PREP_CONF_FN(val, n) \ prep_conf_teddy_m##n(&lo_mask, dup_mask, sl_msk, val) -const u8 ALIGN_DIRECTIVE p_sh_mask_arr[80] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f -}; - #define TEDDY_VBMI_SL1_POS 15 #define TEDDY_VBMI_SL2_POS 14 #define TEDDY_VBMI_SL3_POS 13 diff --git a/src/fdr/teddy_avx2.c b/src/fdr/teddy_avx2.c index 20ea938cf..6a6b27a5f 100644 --- a/src/fdr/teddy_avx2.c +++ b/src/fdr/teddy_avx2.c @@ -109,6 +109,36 @@ const u8 ALIGN_AVX_DIRECTIVE p_mask_arr256[33][64] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} }; +#if defined(HAVE_AVX512VBMI) // VBMI strong fat teddy + +#define CONF_FAT_CHUNK_64(chunk, bucket, off, reason, pt, conf_fn) \ +do { \ + if (unlikely(chunk != ones_u64a)) { \ + chunk = ~chunk; \ + conf_fn(&chunk, bucket, off, confBase, reason, a, pt, \ + &control, &last_match); \ + CHECK_HWLM_TERMINATE_MATCHING; \ + } \ +} while(0) + +#define CONF_FAT_CHUNK_32(chunk, bucket, off, reason, pt, conf_fn) \ +do { \ + if (unlikely(chunk != ones_u32)) { \ + chunk = ~chunk; \ + conf_fn(&chunk, bucket, off, confBase, reason, a, pt, \ + &control, &last_match); \ + CHECK_HWLM_TERMINATE_MATCHING; \ + } \ +} while(0) + +static really_inline +const m512 *getDupMaskBase(const struct Teddy *teddy, u8 numMask) { + return (const m512 *)((const u8 *)teddy + ROUNDUP_CL(sizeof(struct Teddy)) + + ROUNDUP_CL(2 * numMask * sizeof(m256))); +} + +#else + #define CONF_FAT_CHUNK_64(chunk, bucket, off, reason, conf_fn) \ do { \ if (unlikely(chunk != ones_u64a)) { \ @@ -134,203 +164,200 @@ const m256 *getMaskBase_fat(const struct Teddy *teddy) { return (const m256 *)((const u8 *)teddy + ROUNDUP_CL(sizeof(struct Teddy))); } -#if defined(HAVE_AVX512_REVERT) // revert to AVX2 Fat Teddy +#endif -static really_inline -const u64a *getReinforcedMaskBase_fat(const struct Teddy *teddy, u8 numMask) { - return (const u64a *)((const u8 *)getMaskBase_fat(teddy) - + ROUNDUP_CL(2 * numMask * sizeof(m256))); -} +#if defined(HAVE_AVX512VBMI) // VBMI strong fat teddy + +const u8 ALIGN_AVX_DIRECTIVE p_mask_interleave[64] = { + 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, + 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47, + 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, + 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63 +}; #ifdef ARCH_64_BIT -#define CONFIRM_FAT_TEDDY(var, bucket, offset, reason, conf_fn) \ +#define CONFIRM_FAT_TEDDY(var, bucket, offset, reason, pt, conf_fn) \ do { \ if (unlikely(diff512(var, ones512()))) { \ - m512 swap = swap256in512(var); \ - m512 r = interleave512lo(var, swap); \ + m512 msk_interleave = load512(p_mask_interleave); \ + m512 r = vpermb512(msk_interleave, var); \ m128 r0 = extract128from512(r, 0); \ m128 r1 = extract128from512(r, 1); \ + m128 r2 = extract128from512(r, 2); \ + m128 r3 = extract128from512(r, 3); \ u64a part1 = movq(r0); \ u64a part2 = extract64from128(r0, 1); \ - u64a part5 = movq(r1); \ - u64a part6 = extract64from128(r1, 1); \ - r = interleave512hi(var, swap); \ - r0 = extract128from512(r, 0); \ - r1 = extract128from512(r, 1); \ - u64a part3 = movq(r0); \ - u64a part4 = extract64from128(r0, 1); \ - u64a part7 = movq(r1); \ - u64a part8 = extract64from128(r1, 1); \ - CONF_FAT_CHUNK_64(part1, bucket, offset, reason, conf_fn); \ - CONF_FAT_CHUNK_64(part2, bucket, offset + 4, reason, conf_fn); \ - CONF_FAT_CHUNK_64(part3, bucket, offset + 8, reason, conf_fn); \ - CONF_FAT_CHUNK_64(part4, bucket, offset + 12, reason, conf_fn); \ - CONF_FAT_CHUNK_64(part5, bucket, offset + 16, reason, conf_fn); \ - CONF_FAT_CHUNK_64(part6, bucket, offset + 20, reason, conf_fn); \ - CONF_FAT_CHUNK_64(part7, bucket, offset + 24, reason, conf_fn); \ - CONF_FAT_CHUNK_64(part8, bucket, offset + 28, reason, conf_fn); \ + u64a part3 = movq(r1); \ + u64a part4 = extract64from128(r1, 1); \ + u64a part5 = movq(r2); \ + u64a part6 = extract64from128(r2, 1); \ + u64a part7 = movq(r3); \ + u64a part8 = extract64from128(r3, 1); \ + CONF_FAT_CHUNK_64(part1, bucket, offset, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_64(part2, bucket, offset + 4, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_64(part3, bucket, offset + 8, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_64(part4, bucket, offset + 12, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_64(part5, bucket, offset + 16, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_64(part6, bucket, offset + 20, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_64(part7, bucket, offset + 24, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_64(part8, bucket, offset + 28, reason, pt, conf_fn); \ } \ } while(0) #else -#define CONFIRM_FAT_TEDDY(var, bucket, offset, reason, conf_fn) \ +#define CONFIRM_FAT_TEDDY(var, bucket, offset, reason, pt, conf_fn) \ do { \ if (unlikely(diff512(var, ones512()))) { \ - m512 swap = swap256in512(var); \ - m512 r = interleave512lo(var, swap); \ + m512 msk_interleave = load512(p_mask_interleave); \ + m512 r = vpermb512(msk_interleave, var); \ m128 r0 = extract128from512(r, 0); \ m128 r1 = extract128from512(r, 1); \ + m128 r2 = extract128from512(r, 2); \ + m128 r3 = extract128from512(r, 3); \ u32 part1 = movd(r0); \ u32 part2 = extract32from128(r0, 1); \ u32 part3 = extract32from128(r0, 2); \ u32 part4 = extract32from128(r0, 3); \ - u32 part9 = movd(r1); \ - u32 part10 = extract32from128(r1, 1); \ - u32 part11 = extract32from128(r1, 2); \ - u32 part12 = extract32from128(r1, 3); \ - r = interleave512hi(var, swap); \ - r0 = extract128from512(r, 0); \ - r1 = extract128from512(r, 1); \ - u32 part5 = movd(r0); \ - u32 part6 = extract32from128(r0, 1); \ - u32 part7 = extract32from128(r0, 2); \ - u32 part8 = extract32from128(r0, 3); \ - u32 part13 = movd(r1); \ - u32 part14 = extract32from128(r1, 1); \ - u32 part15 = extract32from128(r1, 2); \ - u32 part16 = extract32from128(r1, 3); \ - CONF_FAT_CHUNK_32(part1, bucket, offset, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part2, bucket, offset + 2, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part3, bucket, offset + 4, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part4, bucket, offset + 6, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part5, bucket, offset + 8, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part6, bucket, offset + 10, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part7, bucket, offset + 12, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part8, bucket, offset + 14, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part9, bucket, offset + 16, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part10, bucket, offset + 18, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part11, bucket, offset + 20, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part12, bucket, offset + 22, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part13, bucket, offset + 24, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part14, bucket, offset + 26, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part15, bucket, offset + 28, reason, conf_fn); \ - CONF_FAT_CHUNK_32(part16, bucket, offset + 30, reason, conf_fn); \ + u32 part5 = movd(r1); \ + u32 part6 = extract32from128(r1, 1); \ + u32 part7 = extract32from128(r1, 2); \ + u32 part8 = extract32from128(r1, 3); \ + u32 part9 = movd(r2); \ + u32 part10 = extract32from128(r2, 1); \ + u32 part11 = extract32from128(r2, 2); \ + u32 part12 = extract32from128(r2, 3); \ + u32 part13 = movd(r3); \ + u32 part14 = extract32from128(r3, 1); \ + u32 part15 = extract32from128(r3, 2); \ + u32 part16 = extract32from128(r3, 3); \ + CONF_FAT_CHUNK_32(part1, bucket, offset, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_32(part2, bucket, offset + 2, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_32(part3, bucket, offset + 4, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_32(part4, bucket, offset + 6, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_32(part5, bucket, offset + 8, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_32(part6, bucket, offset + 10, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_32(part7, bucket, offset + 12, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_32(part8, bucket, offset + 14, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_32(part9, bucket, offset + 16, reason, pt, conf_fn); \ + CONF_FAT_CHUNK_32(part10, bucket, offset + 18, reason, pt, conf_fn);\ + CONF_FAT_CHUNK_32(part11, bucket, offset + 20, reason, pt, conf_fn);\ + CONF_FAT_CHUNK_32(part12, bucket, offset + 22, reason, pt, conf_fn);\ + CONF_FAT_CHUNK_32(part13, bucket, offset + 24, reason, pt, conf_fn);\ + CONF_FAT_CHUNK_32(part14, bucket, offset + 26, reason, pt, conf_fn);\ + CONF_FAT_CHUNK_32(part15, bucket, offset + 28, reason, pt, conf_fn);\ + CONF_FAT_CHUNK_32(part16, bucket, offset + 30, reason, pt, conf_fn);\ } \ } while(0) #endif -static really_inline -m512 vectoredLoad2x256(m512 *p_mask, const u8 *ptr, const size_t start_offset, - const u8 *lo, const u8 *hi, - const u8 *buf_history, size_t len_history, - const u32 nMasks) { - m256 p_mask256; - m512 ret = set2x256(vectoredLoad256(&p_mask256, ptr, start_offset, lo, hi, - buf_history, len_history, nMasks)); - *p_mask = set2x256(p_mask256); - return ret; -} - -#define PREP_FAT_SHUF_MASK_NO_REINFORCEMENT(val) \ +#define PREP_FAT_SHUF_MASK \ m512 lo = and512(val, *lo_mask); \ m512 hi = and512(rshift64_m512(val, 4), *lo_mask) -#define PREP_FAT_SHUF_MASK \ - PREP_FAT_SHUF_MASK_NO_REINFORCEMENT(set2x256(load256(ptr))); \ - *c_16 = *(ptr + 15); \ - m512 r_msk = set512_64(0ULL, r_msk_base_hi[*c_16], \ - 0ULL, r_msk_base_hi[*c_0], \ - 0ULL, r_msk_base_lo[*c_16], \ - 0ULL, r_msk_base_lo[*c_0]); \ - *c_0 = *(ptr + 31) - -#define FAT_SHIFT_OR_M1 \ - or512(pshufb_m512(dup_mask[0], lo), pshufb_m512(dup_mask[1], hi)) - -#define FAT_SHIFT_OR_M2 \ - or512(lshift128_m512(or512(pshufb_m512(dup_mask[2], lo), \ - pshufb_m512(dup_mask[3], hi)), \ - 1), FAT_SHIFT_OR_M1) - -#define FAT_SHIFT_OR_M3 \ - or512(lshift128_m512(or512(pshufb_m512(dup_mask[4], lo), \ - pshufb_m512(dup_mask[5], hi)), \ - 2), FAT_SHIFT_OR_M2) - -#define FAT_SHIFT_OR_M4 \ - or512(lshift128_m512(or512(pshufb_m512(dup_mask[6], lo), \ - pshufb_m512(dup_mask[7], hi)), \ - 3), FAT_SHIFT_OR_M3) +#define FAT_TEDDY_VBMI_PSHUFB_OR_M1 \ + m512 shuf_or_b0 = or512(pshufb_m512(dup_mask[0], lo), \ + pshufb_m512(dup_mask[1], hi)); -static really_inline -m512 prep_conf_fat_teddy_no_reinforcement_m1(const m512 *lo_mask, - const m512 *dup_mask, - const m512 val) { - PREP_FAT_SHUF_MASK_NO_REINFORCEMENT(val); - return FAT_SHIFT_OR_M1; -} +#define FAT_TEDDY_VBMI_PSHUFB_OR_M2 \ + FAT_TEDDY_VBMI_PSHUFB_OR_M1 \ + m512 shuf_or_b1 = or512(pshufb_m512(dup_mask[2], lo), \ + pshufb_m512(dup_mask[3], hi)); -static really_inline -m512 prep_conf_fat_teddy_no_reinforcement_m2(const m512 *lo_mask, - const m512 *dup_mask, - const m512 val) { - PREP_FAT_SHUF_MASK_NO_REINFORCEMENT(val); - return FAT_SHIFT_OR_M2; -} +#define FAT_TEDDY_VBMI_PSHUFB_OR_M3 \ + FAT_TEDDY_VBMI_PSHUFB_OR_M2 \ + m512 shuf_or_b2 = or512(pshufb_m512(dup_mask[4], lo), \ + pshufb_m512(dup_mask[5], hi)); -static really_inline -m512 prep_conf_fat_teddy_no_reinforcement_m3(const m512 *lo_mask, - const m512 *dup_mask, - const m512 val) { - PREP_FAT_SHUF_MASK_NO_REINFORCEMENT(val); - return FAT_SHIFT_OR_M3; -} +#define FAT_TEDDY_VBMI_PSHUFB_OR_M4 \ + FAT_TEDDY_VBMI_PSHUFB_OR_M3 \ + m512 shuf_or_b3 = or512(pshufb_m512(dup_mask[6], lo), \ + pshufb_m512(dup_mask[7], hi)); -static really_inline -m512 prep_conf_fat_teddy_no_reinforcement_m4(const m512 *lo_mask, - const m512 *dup_mask, - const m512 val) { - PREP_FAT_SHUF_MASK_NO_REINFORCEMENT(val); - return FAT_SHIFT_OR_M4; -} +#define FAT_TEDDY_VBMI_SL1_MASK 0xfffffffefffffffeULL +#define FAT_TEDDY_VBMI_SL2_MASK 0xfffffffcfffffffcULL +#define FAT_TEDDY_VBMI_SL3_MASK 0xfffffff8fffffff8ULL + +#define FAT_TEDDY_VBMI_SHIFT_M1 + +#define FAT_TEDDY_VBMI_SHIFT_M2 \ + FAT_TEDDY_VBMI_SHIFT_M1 \ + m512 sl1 = maskz_vpermb512(FAT_TEDDY_VBMI_SL1_MASK, sl_msk[0], shuf_or_b1); + +#define FAT_TEDDY_VBMI_SHIFT_M3 \ + FAT_TEDDY_VBMI_SHIFT_M2 \ + m512 sl2 = maskz_vpermb512(FAT_TEDDY_VBMI_SL2_MASK, sl_msk[1], shuf_or_b2); + +#define FAT_TEDDY_VBMI_SHIFT_M4 \ + FAT_TEDDY_VBMI_SHIFT_M3 \ + m512 sl3 = maskz_vpermb512(FAT_TEDDY_VBMI_SL3_MASK, sl_msk[2], shuf_or_b3); + +#define FAT_SHIFT_OR_M1 \ + shuf_or_b0 + +#define FAT_SHIFT_OR_M2 \ + or512(sl1, FAT_SHIFT_OR_M1) + +#define FAT_SHIFT_OR_M3 \ + or512(sl2, FAT_SHIFT_OR_M2) + +#define FAT_SHIFT_OR_M4 \ + or512(sl3, FAT_SHIFT_OR_M3) static really_inline m512 prep_conf_fat_teddy_m1(const m512 *lo_mask, const m512 *dup_mask, - const u8 *ptr, const u64a *r_msk_base_lo, - const u64a *r_msk_base_hi, u32 *c_0, u32 *c_16) { + UNUSED const m512 *sl_msk, const m512 val) { PREP_FAT_SHUF_MASK; - return or512(FAT_SHIFT_OR_M1, r_msk); + FAT_TEDDY_VBMI_PSHUFB_OR_M1; + FAT_TEDDY_VBMI_SHIFT_M1; + return FAT_SHIFT_OR_M1; } static really_inline m512 prep_conf_fat_teddy_m2(const m512 *lo_mask, const m512 *dup_mask, - const u8 *ptr, const u64a *r_msk_base_lo, - const u64a *r_msk_base_hi, u32 *c_0, u32 *c_16) { + const m512 *sl_msk, const m512 val) { PREP_FAT_SHUF_MASK; - return or512(FAT_SHIFT_OR_M2, r_msk); + FAT_TEDDY_VBMI_PSHUFB_OR_M2; + FAT_TEDDY_VBMI_SHIFT_M2; + return FAT_SHIFT_OR_M2; } static really_inline m512 prep_conf_fat_teddy_m3(const m512 *lo_mask, const m512 *dup_mask, - const u8 *ptr, const u64a *r_msk_base_lo, - const u64a *r_msk_base_hi, u32 *c_0, u32 *c_16) { + const m512 *sl_msk, const m512 val) { PREP_FAT_SHUF_MASK; - return or512(FAT_SHIFT_OR_M3, r_msk); + FAT_TEDDY_VBMI_PSHUFB_OR_M3; + FAT_TEDDY_VBMI_SHIFT_M3; + return FAT_SHIFT_OR_M3; } static really_inline m512 prep_conf_fat_teddy_m4(const m512 *lo_mask, const m512 *dup_mask, - const u8 *ptr, const u64a *r_msk_base_lo, - const u64a *r_msk_base_hi, u32 *c_0, u32 *c_16) { + const m512 *sl_msk, const m512 val) { PREP_FAT_SHUF_MASK; - return or512(FAT_SHIFT_OR_M4, r_msk); + FAT_TEDDY_VBMI_PSHUFB_OR_M4; + FAT_TEDDY_VBMI_SHIFT_M4; + return FAT_SHIFT_OR_M4; } -#define PREP_CONF_FAT_FN_NO_REINFORCEMENT(val, n) \ - prep_conf_fat_teddy_no_reinforcement_m##n(&lo_mask, dup_mask, val) +#define PREP_CONF_FAT_FN(val, n) \ + prep_conf_fat_teddy_m##n(&lo_mask, dup_mask, sl_msk, val) -#define PREP_CONF_FAT_FN(ptr, n) \ - prep_conf_fat_teddy_m##n(&lo_mask, dup_mask, ptr, \ - r_msk_base_lo, r_msk_base_hi, &c_0, &c_16) +#define FAT_TEDDY_VBMI_SL1_POS 15 +#define FAT_TEDDY_VBMI_SL2_POS 14 +#define FAT_TEDDY_VBMI_SL3_POS 13 + +#define FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M1 + +#define FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M2 \ + FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M1 \ + sl_msk[0] = loadu512(p_sh_mask_arr + FAT_TEDDY_VBMI_SL1_POS); + +#define FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M3 \ + FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M2 \ + sl_msk[1] = loadu512(p_sh_mask_arr + FAT_TEDDY_VBMI_SL2_POS); + +#define FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M4 \ + FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M3 \ + sl_msk[2] = loadu512(p_sh_mask_arr + FAT_TEDDY_VBMI_SL3_POS); /* * In FAT teddy, it needs 2 bytes to represent result of each position, @@ -355,31 +382,15 @@ m512 prep_conf_fat_teddy_m4(const m512 *lo_mask, const m512 *dup_mask, * then do pshufb_m512(AABB, XYXY). */ -#define DUP_FAT_MASK(a) mask_set2x256(set2x256(swap128in256(a)), 0xC3, a) - -#define PREPARE_FAT_MASKS_1 \ - dup_mask[0] = DUP_FAT_MASK(maskBase[0]); \ - dup_mask[1] = DUP_FAT_MASK(maskBase[1]); - -#define PREPARE_FAT_MASKS_2 \ - PREPARE_FAT_MASKS_1 \ - dup_mask[2] = DUP_FAT_MASK(maskBase[2]); \ - dup_mask[3] = DUP_FAT_MASK(maskBase[3]); - -#define PREPARE_FAT_MASKS_3 \ - PREPARE_FAT_MASKS_2 \ - dup_mask[4] = DUP_FAT_MASK(maskBase[4]); \ - dup_mask[5] = DUP_FAT_MASK(maskBase[5]); - -#define PREPARE_FAT_MASKS_4 \ - PREPARE_FAT_MASKS_3 \ - dup_mask[6] = DUP_FAT_MASK(maskBase[6]); \ - dup_mask[7] = DUP_FAT_MASK(maskBase[7]); - #define PREPARE_FAT_MASKS(n) \ m512 lo_mask = set64x8(0xf); \ - m512 dup_mask[n * 2]; \ - PREPARE_FAT_MASKS_##n + m512 sl_msk[n - 1]; \ + FAT_TEDDY_VBMI_LOAD_SHIFT_MASK_M##n + +#define FAT_TEDDY_VBMI_CONF_MASK_HEAD (0xffffffffULL >> n_sh) +#define FAT_TEDDY_VBMI_CONF_MASK_FULL ((0xffffffffULL << n_sh) & 0xffffffffULL) +#define FAT_TEDDY_VBMI_CONF_MASK_VAR(n) (0xffffffffULL >> (32 - n) << overlap) +#define FAT_TEDDY_VBMI_LOAD_MASK_PATCH (0xffffffffULL >> (32 - n_sh)) #define FDR_EXEC_FAT_TEDDY(fdr, a, control, n_msk, conf_fn) \ do { \ @@ -389,67 +400,53 @@ do { \ const u8 *tryFloodDetect = a->firstFloodDetect; \ u32 last_match = ones_u32; \ const struct Teddy *teddy = (const struct Teddy *)fdr; \ - const size_t iterBytes = 64; \ + const size_t iterBytes = 32; \ + u32 n_sh = n_msk - 1; \ + const size_t loopBytes = 32 - n_sh; \ DEBUG_PRINTF("params: buf %p len %zu start_offset %zu\n", \ a->buf, a->len, a->start_offset); \ \ - const m256 *maskBase = getMaskBase_fat(teddy); \ + const m512 *dup_mask = getDupMaskBase(teddy, n_msk); \ PREPARE_FAT_MASKS(n_msk); \ const u32 *confBase = getConfBase(teddy); \ \ - const u64a *r_msk_base_lo = getReinforcedMaskBase_fat(teddy, n_msk); \ - const u64a *r_msk_base_hi = r_msk_base_lo + (N_CHARS + 1); \ - u32 c_0 = 0x100; \ - u32 c_16 = 0x100; \ - const u8 *mainStart = ROUNDUP_PTR(ptr, 32); \ - DEBUG_PRINTF("derive: ptr: %p mainstart %p\n", ptr, mainStart); \ - if (ptr < mainStart) { \ - ptr = mainStart - 32; \ - m512 p_mask; \ - m512 val_0 = vectoredLoad2x256(&p_mask, ptr, a->start_offset, \ - a->buf, buf_end, \ - a->buf_history, a->len_history, n_msk); \ - m512 r_0 = PREP_CONF_FAT_FN_NO_REINFORCEMENT(val_0, n_msk); \ - r_0 = or512(r_0, p_mask); \ - CONFIRM_FAT_TEDDY(r_0, 16, 0, VECTORING, conf_fn); \ - ptr += 32; \ + u64a k = FAT_TEDDY_VBMI_CONF_MASK_FULL; \ + m512 p_mask = set_mask_m512(~((k << 32) | k)); \ + u32 overlap = 0; \ + u64a patch = 0; \ + if (likely(ptr + loopBytes <= buf_end)) { \ + u64a k0 = FAT_TEDDY_VBMI_CONF_MASK_HEAD; \ + m512 p_mask0 = set_mask_m512(~((k0 << 32) | k0)); \ + m512 r_0 = PREP_CONF_FAT_FN(set2x256(loadu256(ptr)), n_msk); \ + r_0 = or512(r_0, p_mask0); \ + CONFIRM_FAT_TEDDY(r_0, 16, 0, VECTORING, ptr, conf_fn); \ + ptr += loopBytes; \ + overlap = n_sh; \ + patch = FAT_TEDDY_VBMI_LOAD_MASK_PATCH; \ } \ \ - if (ptr + 32 <= buf_end) { \ - m512 r_0 = PREP_CONF_FAT_FN(ptr, n_msk); \ - CONFIRM_FAT_TEDDY(r_0, 16, 0, VECTORING, conf_fn); \ - ptr += 32; \ - } \ - \ - for (; ptr + iterBytes <= buf_end; ptr += iterBytes) { \ - __builtin_prefetch(ptr + (iterBytes * 4)); \ + for (; ptr + loopBytes <= buf_end; ptr += loopBytes) { \ CHECK_FLOOD; \ - m512 r_0 = PREP_CONF_FAT_FN(ptr, n_msk); \ - CONFIRM_FAT_TEDDY(r_0, 16, 0, NOT_CAUTIOUS, conf_fn); \ - m512 r_1 = PREP_CONF_FAT_FN(ptr + 32, n_msk); \ - CONFIRM_FAT_TEDDY(r_1, 16, 32, NOT_CAUTIOUS, conf_fn); \ - } \ - \ - if (ptr + 32 <= buf_end) { \ - m512 r_0 = PREP_CONF_FAT_FN(ptr, n_msk); \ - CONFIRM_FAT_TEDDY(r_0, 16, 0, NOT_CAUTIOUS, conf_fn); \ - ptr += 32; \ + m512 r_0 = PREP_CONF_FAT_FN(set2x256(loadu256(ptr - n_sh)), n_msk); \ + r_0 = or512(r_0, p_mask); \ + CONFIRM_FAT_TEDDY(r_0, 16, 0, NOT_CAUTIOUS, ptr - n_sh, conf_fn); \ } \ \ - assert(ptr + 32 > buf_end); \ + assert(ptr + loopBytes > buf_end); \ if (ptr < buf_end) { \ - m512 p_mask; \ - m512 val_0 = vectoredLoad2x256(&p_mask, ptr, 0, ptr, buf_end, \ - a->buf_history, a->len_history, n_msk); \ - m512 r_0 = PREP_CONF_FAT_FN_NO_REINFORCEMENT(val_0, n_msk); \ - r_0 = or512(r_0, p_mask); \ - CONFIRM_FAT_TEDDY(r_0, 16, 0, VECTORING, conf_fn); \ + u32 left = (u32)(buf_end - ptr); \ + u64a k1 = FAT_TEDDY_VBMI_CONF_MASK_VAR(left); \ + m512 p_mask1 = set_mask_m512(~((k1 << 32) | k1)); \ + m512 val_0 = set2x256(loadu_maskz_m256(k1 | patch, ptr - overlap)); \ + m512 r_0 = PREP_CONF_FAT_FN(val_0, n_msk); \ + r_0 = or512(r_0, p_mask1); \ + CONFIRM_FAT_TEDDY(r_0, 16, 0, VECTORING, ptr - overlap, conf_fn); \ } \ \ return HWLM_SUCCESS; \ } while(0) -#else // HAVE_AVX512 +#else // !HAVE_AVX512VBMI, AVX2 normal fat teddy #ifdef ARCH_64_BIT #define CONFIRM_FAT_TEDDY(var, bucket, offset, reason, conf_fn) \ @@ -659,7 +656,7 @@ do { \ return HWLM_SUCCESS; \ } while(0) -#endif // HAVE_AVX512 +#endif // HAVE_AVX512VBMI hwlm_error_t fdr_exec_fat_teddy_msks1(const struct FDR *fdr, const struct FDR_Runtime_Args *a, diff --git a/src/fdr/teddy_compile.cpp b/src/fdr/teddy_compile.cpp index 9a1e54a15..eae9c2c13 100644 --- a/src/fdr/teddy_compile.cpp +++ b/src/fdr/teddy_compile.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -353,6 +353,89 @@ void fillReinforcedMsk(u8 *rmsk, u16 c, u32 j, u8 bmsk) { } } +static +void fillDupNibbleMasks(const map> &bucketToLits, + const vector &lits, + u32 numMasks, size_t maskLen, + u8 *baseMsk) { + u32 maskWidth = 2; + memset(baseMsk, 0xff, maskLen); + + for (const auto &b2l : bucketToLits) { + const u32 &bucket_id = b2l.first; + const vector &ids = b2l.second; + const u8 bmsk = 1U << (bucket_id % 8); + + for (const LiteralIndex &lit_id : ids) { + const hwlmLiteral &l = lits[lit_id]; + DEBUG_PRINTF("putting lit %u into bucket %u\n", lit_id, bucket_id); + const u32 sz = verify_u32(l.s.size()); + + // fill in masks + for (u32 j = 0; j < numMasks; j++) { + const u32 msk_id_lo = j * 2 * maskWidth + (bucket_id / 8); + const u32 msk_id_hi = (j * 2 + 1) * maskWidth + (bucket_id / 8); + const u32 lo_base0 = msk_id_lo * 32; + const u32 lo_base1 = msk_id_lo * 32 + 16; + const u32 hi_base0 = msk_id_hi * 32; + const u32 hi_base1 = msk_id_hi * 32 + 16; + + // if we don't have a char at this position, fill in i + // locations in these masks with '1' + if (j >= sz) { + for (u32 n = 0; n < 16; n++) { + baseMsk[lo_base0 + n] &= ~bmsk; + baseMsk[lo_base1 + n] &= ~bmsk; + baseMsk[hi_base0 + n] &= ~bmsk; + baseMsk[hi_base1 + n] &= ~bmsk; + } + } else { + u8 c = l.s[sz - 1 - j]; + // if we do have a char at this position + const u32 hiShift = 4; + u32 n_hi = (c >> hiShift) & 0xf; + u32 n_lo = c & 0xf; + + if (j < l.msk.size() && l.msk[l.msk.size() - 1 - j]) { + u8 m = l.msk[l.msk.size() - 1 - j]; + u8 m_hi = (m >> hiShift) & 0xf; + u8 m_lo = m & 0xf; + u8 cmp = l.cmp[l.msk.size() - 1 - j]; + u8 cmp_lo = cmp & 0xf; + u8 cmp_hi = (cmp >> hiShift) & 0xf; + + for (u8 cm = 0; cm < 0x10; cm++) { + if ((cm & m_lo) == (cmp_lo & m_lo)) { + baseMsk[lo_base0 + cm] &= ~bmsk; + baseMsk[lo_base1 + cm] &= ~bmsk; + } + if ((cm & m_hi) == (cmp_hi & m_hi)) { + baseMsk[hi_base0 + cm] &= ~bmsk; + baseMsk[hi_base1 + cm] &= ~bmsk; + } + } + } else { + if (l.nocase && ourisalpha(c)) { + u32 cmHalfClear = (0xdf >> hiShift) & 0xf; + u32 cmHalfSet = (0x20 >> hiShift) & 0xf; + baseMsk[hi_base0 + (n_hi & cmHalfClear)] &= ~bmsk; + baseMsk[hi_base1 + (n_hi & cmHalfClear)] &= ~bmsk; + baseMsk[hi_base0 + (n_hi | cmHalfSet)] &= ~bmsk; + baseMsk[hi_base1 + (n_hi | cmHalfSet)] &= ~bmsk; + } else { + baseMsk[hi_base0 + n_hi] &= ~bmsk; + baseMsk[hi_base1 + n_hi] &= ~bmsk; + } + baseMsk[lo_base0 + n_lo] &= ~bmsk; + baseMsk[lo_base1 + n_lo] &= ~bmsk; + } + } + } + } + } +} + static void fillNibbleMasks(const map> &bucketToLits, @@ -479,14 +562,17 @@ bytecode_ptr TeddyCompiler::build() { size_t headerSize = sizeof(Teddy); size_t maskLen = eng.numMasks * 16 * 2 * maskWidth; - size_t reinforcedMaskLen = RTABLE_SIZE * maskWidth; + size_t reinforcedDupMaskLen = RTABLE_SIZE * maskWidth; + if (maskWidth == 2) { // dup nibble mask table in Fat Teddy + reinforcedDupMaskLen = maskLen * 2; + } auto floodTable = setupFDRFloodControl(lits, eng, grey); auto confirmTable = setupFullConfs(lits, eng, bucketToLits, make_small); // Note: we place each major structure here on a cacheline boundary. size_t size = ROUNDUP_CL(headerSize) + ROUNDUP_CL(maskLen) + - ROUNDUP_CL(reinforcedMaskLen) + + ROUNDUP_CL(reinforcedDupMaskLen) + ROUNDUP_CL(confirmTable.size()) + floodTable.size(); auto fdr = make_zeroed_bytecode_ptr(size, 64); @@ -502,7 +588,7 @@ bytecode_ptr TeddyCompiler::build() { // Write confirm structures. u8 *ptr = teddy_base + ROUNDUP_CL(headerSize) + ROUNDUP_CL(maskLen) + - ROUNDUP_CL(reinforcedMaskLen); + ROUNDUP_CL(reinforcedDupMaskLen); assert(ISALIGNED_CL(ptr)); teddy->confOffset = verify_u32(ptr - teddy_base); memcpy(ptr, confirmTable.get(), confirmTable.size()); @@ -519,9 +605,16 @@ bytecode_ptr TeddyCompiler::build() { fillNibbleMasks(bucketToLits, lits, eng.numMasks, maskWidth, maskLen, baseMsk); - // Write reinforcement masks. - u8 *reinforcedMsk = baseMsk + ROUNDUP_CL(maskLen); - fillReinforcedTable(bucketToLits, lits, reinforcedMsk, maskWidth); + if (maskWidth == 1) { // reinforcement table in Teddy + // Write reinforcement masks. + u8 *reinforcedMsk = baseMsk + ROUNDUP_CL(maskLen); + fillReinforcedTable(bucketToLits, lits, reinforcedMsk, maskWidth); + } else { // dup nibble mask table in Fat Teddy + assert(maskWidth == 2); + u8 *dupMsk = baseMsk + ROUNDUP_CL(maskLen); + fillDupNibbleMasks(bucketToLits, lits, eng.numMasks, + reinforcedDupMaskLen, dupMsk); + } return fdr; } diff --git a/src/fdr/teddy_runtime_common.h b/src/fdr/teddy_runtime_common.h index 730850cb7..b76800eb0 100644 --- a/src/fdr/teddy_runtime_common.h +++ b/src/fdr/teddy_runtime_common.h @@ -45,6 +45,16 @@ extern const u8 ALIGN_DIRECTIVE p_mask_arr[17][32]; extern const u8 ALIGN_AVX_DIRECTIVE p_mask_arr256[33][64]; #endif +#if defined(HAVE_AVX512VBMI) +static const u8 ALIGN_DIRECTIVE p_sh_mask_arr[80] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f +}; +#endif + #ifdef ARCH_64_BIT #define TEDDY_CONF_TYPE u64a #define TEDDY_FIND_AND_CLEAR_LSB(conf) findAndClearLSB_64(conf) diff --git a/src/hs.cpp b/src/hs.cpp index ab54105c5..eac588891 100644 --- a/src/hs.cpp +++ b/src/hs.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2019, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -120,9 +120,10 @@ bool checkMode(unsigned int mode, hs_compile_error **comp_error) { static bool checkPlatform(const hs_platform_info *p, hs_compile_error **comp_error) { - static constexpr u32 HS_TUNE_LAST = HS_TUNE_FAMILY_GLM; + static constexpr u32 HS_TUNE_LAST = HS_TUNE_FAMILY_ICX; static constexpr u32 HS_CPU_FEATURES_ALL = - HS_CPU_FEATURES_AVX2 | HS_CPU_FEATURES_AVX512; + HS_CPU_FEATURES_AVX2 | HS_CPU_FEATURES_AVX512 | + HS_CPU_FEATURES_AVX512VBMI; if (!p) { return true; diff --git a/src/hs.h b/src/hs.h index 105919fb8..2fe5d248b 100644 --- a/src/hs.h +++ b/src/hs.h @@ -42,7 +42,7 @@ /* The current Hyperscan version information. */ #define HS_MAJOR 5 -#define HS_MINOR 3 +#define HS_MINOR 4 #define HS_PATCH 0 #include "hs_compile.h" diff --git a/src/hs_compile.h b/src/hs_compile.h index 081d46387..b318c29db 100644 --- a/src/hs_compile.h +++ b/src/hs_compile.h @@ -1034,6 +1034,15 @@ hs_error_t HS_CDECL hs_populate_platform(hs_platform_info_t *platform); */ #define HS_CPU_FEATURES_AVX512 (1ULL << 3) +/** + * CPU features flag - Intel(R) Advanced Vector Extensions 512 + * Vector Byte Manipulation Instructions (Intel(R) AVX512VBMI) + * + * Setting this flag indicates that the target platform supports AVX512VBMI + * instructions. Using AVX512VBMI implies the use of AVX512. + */ +#define HS_CPU_FEATURES_AVX512VBMI (1ULL << 4) + /** @} */ /** @@ -1114,6 +1123,22 @@ hs_error_t HS_CDECL hs_populate_platform(hs_platform_info_t *platform); */ #define HS_TUNE_FAMILY_GLM 8 +/** + * Tuning Parameter - Intel(R) microarchitecture code name Icelake + * + * This indicates that the compiled database should be tuned for the + * Icelake microarchitecture. + */ +#define HS_TUNE_FAMILY_ICL 9 + +/** + * Tuning Parameter - Intel(R) microarchitecture code name Icelake Server + * + * This indicates that the compiled database should be tuned for the + * Icelake Server microarchitecture. + */ +#define HS_TUNE_FAMILY_ICX 10 + /** @} */ /** diff --git a/src/nfa/limex_compile.cpp b/src/nfa/limex_compile.cpp index bbb266051..9233ae515 100644 --- a/src/nfa/limex_compile.cpp +++ b/src/nfa/limex_compile.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -85,6 +85,18 @@ namespace ue2 { */ static constexpr u32 NO_STATE = ~0; +/* Maximum number of states taken as a small NFA */ +static constexpr u32 MAX_SMALL_NFA_STATES = 64; + +/* Maximum bounded repeat upper bound to consider as a fast NFA */ +static constexpr u64a MAX_REPEAT_SIZE = 200; + +/* Maximum bounded repeat char reach size to consider as a fast NFA */ +static constexpr u32 MAX_REPEAT_CHAR_REACH = 26; + +/* Minimum bounded repeat trigger distance to consider as a fast NFA */ +static constexpr u8 MIN_REPEAT_TRIGGER_DISTANCE = 6; + namespace { struct precalcAccel { @@ -1910,7 +1922,8 @@ struct Factory { } static - void writeExceptions(const map> &exceptionMap, + void writeExceptions(const build_info &args, + const map> &exceptionMap, const vector &repeatOffsets, implNFA_t *limex, const u32 exceptionsOffset, const u32 reportListOffset) { @@ -1962,6 +1975,59 @@ struct Factory { limex->exceptionOffset = exceptionsOffset; limex->exceptionCount = ecount; + + if (args.num_states > 64 && args.cc.target_info.has_avx512vbmi()) { + const u8 *exceptionMask = (const u8 *)(&limex->exceptionMask); + u8 *shufMask = (u8 *)&limex->exceptionShufMask; + u8 *bitMask = (u8 *)&limex->exceptionBitMask; + u8 *andMask = (u8 *)&limex->exceptionAndMask; + + u32 tot_cnt = 0; + u32 pos = 0; + bool valid = true; + size_t tot = sizeof(limex->exceptionMask); + size_t base = 0; + + // We normally have up to 64 exceptions to handle, + // but treat 384 state Limex differently to simplify operations + size_t limit = 64; + if (args.num_states > 256 && args.num_states <= 384) { + limit = 48; + } + + for (size_t i = 0; i < tot; i++) { + if (!exceptionMask[i]) { + continue; + } + u32 bit_cnt = popcount32(exceptionMask[i]); + + tot_cnt += bit_cnt; + if (tot_cnt > limit) { + valid = false; + break; + } + + u32 emsk = exceptionMask[i]; + while (emsk) { + u32 t = findAndClearLSB_32(&emsk); + bitMask[pos] = 1U << t; + andMask[pos] = 1U << t; + shufMask[pos++] = i + base; + + if (pos == 32 && + (args.num_states > 128 && args.num_states <= 256)) { + base += 32; + } + } + } + // Avoid matching unused bytes + for (u32 i = pos; i < 64; i++) { + bitMask[i] = 0xff; + } + if (valid) { + setLimexFlag(limex, LIMEX_FLAG_EXTRACT_EXP); + } + } } static @@ -2287,7 +2353,7 @@ struct Factory { writeRepeats(repeats, repeatOffsets, limex, repeatOffsetsOffset, repeatsOffset); - writeExceptions(exceptionMap, repeatOffsets, limex, exceptionsOffset, + writeExceptions(args, exceptionMap, repeatOffsets, limex, exceptionsOffset, reportListOffset); writeLimexMasks(args, limex); @@ -2422,6 +2488,68 @@ bool isSane(const NGHolder &h, const map> &tops, } #endif // NDEBUG +static +bool isFast(const build_info &args) { + const NGHolder &h = args.h; + const u32 num_states = args.num_states; + + if (num_states > MAX_SMALL_NFA_STATES) { + return false; + } + + unordered_map pos_trigger; + for (u32 i = 0; i < args.repeats.size(); i++) { + const BoundedRepeatData &br = args.repeats[i]; + assert(!contains(pos_trigger, br.pos_trigger)); + pos_trigger[br.pos_trigger] = br.repeatMax <= MAX_REPEAT_SIZE; + } + + // Small NFA without bounded repeat should be fast. + if (pos_trigger.empty()) { + return true; + } + + vector cur; + unordered_set visited; + for (const auto &m : args.tops) { + for (NFAVertex v : m.second) { + cur.push_back(v); + visited.insert(v); + } + } + + u8 pos_dist = 0; + while (!cur.empty()) { + vector next; + for (const auto &v : cur) { + if (contains(pos_trigger, v)) { + const CharReach &cr = h[v].char_reach; + if (!pos_trigger[v] && cr.count() > MAX_REPEAT_CHAR_REACH) { + return false; + } + } + for (const auto &w : adjacent_vertices_range(v, h)) { + if (w == v) { + continue; + } + u32 j = args.state_ids.at(w); + if (j == NO_STATE) { + continue; + } + if (!contains(visited, w)) { + next.push_back(w); + visited.insert(w); + } + } + } + if (++pos_dist >= MIN_REPEAT_TRIGGER_DISTANCE) { + break; + } + swap(cur, next); + } + return true; +} + static u32 max_state(const unordered_map &state_ids) { u32 rv = 0; @@ -2442,7 +2570,7 @@ bytecode_ptr generate(NGHolder &h, const unordered_map &squashMap, const map> &tops, const set &zombies, bool do_accel, - bool stateCompression, u32 hint, + bool stateCompression, bool &fast, u32 hint, const CompileContext &cc) { const u32 num_states = max_state(states) + 1; DEBUG_PRINTF("total states: %u\n", num_states); @@ -2497,6 +2625,7 @@ bytecode_ptr generate(NGHolder &h, if (nfa) { DEBUG_PRINTF("successful build with NFA engine: %s\n", nfa_type_name(limex_model)); + fast = isFast(arg); return nfa; } } diff --git a/src/nfa/limex_compile.h b/src/nfa/limex_compile.h index a08e0ae56..4afdcdb3e 100644 --- a/src/nfa/limex_compile.h +++ b/src/nfa/limex_compile.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -78,6 +78,7 @@ bytecode_ptr generate(NGHolder &g, const std::set &zombies, bool do_accel, bool stateCompression, + bool &fast, u32 hint, const CompileContext &cc); diff --git a/src/nfa/limex_exceptional.h b/src/nfa/limex_exceptional.h index e770c3278..6c7335f1b 100644 --- a/src/nfa/limex_exceptional.h +++ b/src/nfa/limex_exceptional.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -47,6 +47,8 @@ #define AND_STATE JOIN(and_, STATE_T) #define EQ_STATE(a, b) (!JOIN(noteq_, STATE_T)((a), (b))) #define OR_STATE JOIN(or_, STATE_T) +#define EXPAND_STATE JOIN(expand_, STATE_T) +#define SHUFFLE_BYTE_STATE JOIN(shuffle_byte_, STATE_T) #define TESTBIT_STATE JOIN(testbit_, STATE_T) #define EXCEPTION_T JOIN(struct NFAException, SIZE) #define CONTEXT_T JOIN(NFAContext, SIZE) @@ -208,7 +210,7 @@ int RUN_EXCEPTION_FN(const EXCEPTION_T *e, STATE_ARG, /** \brief Process all of the exceptions associated with the states in the \a * estate. */ static really_inline -int PE_FN(STATE_ARG, ESTATE_ARG, u32 diffmask, STATE_T *succ, +int PE_FN(STATE_ARG, ESTATE_ARG, UNUSED u32 diffmask, STATE_T *succ, const struct IMPL_NFA_T *limex, const EXCEPTION_T *exceptions, u64a offset, struct CONTEXT_T *ctx, char in_rev, char flags) { assert(diffmask > 0); // guaranteed by caller macro @@ -233,6 +235,72 @@ int PE_FN(STATE_ARG, ESTATE_ARG, u32 diffmask, STATE_T *succ, ctx->local_succ = ZERO_STATE; #endif + struct proto_cache new_cache = {0, NULL}; + enum CacheResult cacheable = CACHE_RESULT; + +#if defined(HAVE_AVX512VBMI) && SIZE > 64 + if (likely(limex->flags & LIMEX_FLAG_EXTRACT_EXP)) { + m512 emask = EXPAND_STATE(*STATE_ARG_P); + emask = SHUFFLE_BYTE_STATE(load_m512(&limex->exceptionShufMask), emask); + emask = and512(emask, load_m512(&limex->exceptionAndMask)); + u64a word = eq512mask(emask, load_m512(&limex->exceptionBitMask)); + + do { + u32 bit = FIND_AND_CLEAR_FN(&word); + const EXCEPTION_T *e = &exceptions[bit]; + + if (!RUN_EXCEPTION_FN(e, STATE_ARG_NAME, succ, +#ifndef BIG_MODEL + &local_succ, +#endif + limex, offset, ctx, &new_cache, &cacheable, + in_rev, flags)) { + return PE_RV_HALT; + } + } while (word); + } else { + // A copy of the estate as an array of GPR-sized chunks. + CHUNK_T chunks[sizeof(STATE_T) / sizeof(CHUNK_T)]; + CHUNK_T emask_chunks[sizeof(STATE_T) / sizeof(CHUNK_T)]; +#ifdef ESTATE_ON_STACK + memcpy(chunks, &estate, sizeof(STATE_T)); +#else + memcpy(chunks, estatep, sizeof(STATE_T)); +#endif + memcpy(emask_chunks, &limex->exceptionMask, sizeof(STATE_T)); + + u32 base_index[sizeof(STATE_T) / sizeof(CHUNK_T)]; + base_index[0] = 0; + for (s32 i = 0; i < (s32)ARRAY_LENGTH(base_index) - 1; i++) { + base_index[i + 1] = base_index[i] + POPCOUNT_FN(emask_chunks[i]); + } + + do { + u32 t = findAndClearLSB_32(&diffmask); +#ifdef ARCH_64_BIT + t >>= 1; // Due to diffmask64, which leaves holes in the bitmask. +#endif + assert(t < ARRAY_LENGTH(chunks)); + CHUNK_T word = chunks[t]; + assert(word != 0); + do { + u32 bit = FIND_AND_CLEAR_FN(&word); + u32 local_index = RANK_IN_MASK_FN(emask_chunks[t], bit); + u32 idx = local_index + base_index[t]; + const EXCEPTION_T *e = &exceptions[idx]; + + if (!RUN_EXCEPTION_FN(e, STATE_ARG_NAME, succ, +#ifndef BIG_MODEL + &local_succ, +#endif + limex, offset, ctx, &new_cache, &cacheable, + in_rev, flags)) { + return PE_RV_HALT; + } + } while (word); + } while (diffmask); + } +#else // A copy of the estate as an array of GPR-sized chunks. CHUNK_T chunks[sizeof(STATE_T) / sizeof(CHUNK_T)]; CHUNK_T emask_chunks[sizeof(STATE_T) / sizeof(CHUNK_T)]; @@ -243,9 +311,6 @@ int PE_FN(STATE_ARG, ESTATE_ARG, u32 diffmask, STATE_T *succ, #endif memcpy(emask_chunks, &limex->exceptionMask, sizeof(STATE_T)); - struct proto_cache new_cache = {0, NULL}; - enum CacheResult cacheable = CACHE_RESULT; - u32 base_index[sizeof(STATE_T) / sizeof(CHUNK_T)]; base_index[0] = 0; for (s32 i = 0; i < (s32)ARRAY_LENGTH(base_index) - 1; i++) { @@ -276,6 +341,7 @@ int PE_FN(STATE_ARG, ESTATE_ARG, u32 diffmask, STATE_T *succ, } } while (word); } while (diffmask); +#endif #ifndef BIG_MODEL *succ = OR_STATE(*succ, local_succ); @@ -307,6 +373,8 @@ int PE_FN(STATE_ARG, ESTATE_ARG, u32 diffmask, STATE_T *succ, #undef AND_STATE #undef EQ_STATE #undef OR_STATE +#undef EXPAND_STATE +#undef SHUFFLE_BYTE_STATE #undef TESTBIT_STATE #undef PE_FN #undef RUN_EXCEPTION_FN diff --git a/src/nfa/limex_internal.h b/src/nfa/limex_internal.h index db703f039..23b1bd970 100644 --- a/src/nfa/limex_internal.h +++ b/src/nfa/limex_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -86,6 +86,7 @@ #define LIMEX_FLAG_COMPRESS_STATE 1 /**< pack state into stream state */ #define LIMEX_FLAG_COMPRESS_MASKED 2 /**< use reach mask-based compression */ #define LIMEX_FLAG_CANNOT_DIE 4 /**< limex cannot have no states on */ +#define LIMEX_FLAG_EXTRACT_EXP 8 /**< use limex exception bit extraction */ enum LimExTrigger { LIMEX_TRIGGER_NONE = 0, @@ -157,6 +158,9 @@ struct LimExNFA##size { \ u_##size shift[MAX_SHIFT_COUNT]; \ u32 shiftCount; /**< number of shift masks used */ \ u8 shiftAmount[MAX_SHIFT_COUNT]; /**< shift amount for each mask */ \ + m512 exceptionShufMask; /**< exception byte shuffle mask */ \ + m512 exceptionBitMask; /**< exception bit mask */ \ + m512 exceptionAndMask; /**< exception and mask */ \ }; CREATE_NFA_LIMEX(32) diff --git a/src/nfa/mcclellancompile.cpp b/src/nfa/mcclellancompile.cpp index c1a4f87fc..27ec1716e 100644 --- a/src/nfa/mcclellancompile.cpp +++ b/src/nfa/mcclellancompile.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2018, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -1477,6 +1477,7 @@ bytecode_ptr mcclellanCompile_i(raw_dfa &raw, accel_dfa_build_strat &strat, bytecode_ptr nfa; if (!using8bit) { + // Wide state optimization if (cc.grey.allowWideStates && strat.getType() == McClellan && !is_triggered(raw.kind)) { find_wide_state(info); @@ -1486,19 +1487,22 @@ bytecode_ptr mcclellanCompile_i(raw_dfa &raw, accel_dfa_build_strat &strat, bool any_cyclic_near_anchored_state = is_cyclic_near(raw, raw.start_anchored); - for (u32 i = 0; i < info.size(); i++) { - if (info.is_widestate(i)) { - continue; + // Sherman optimization + if (info.impl_alpha_size > 16) { + for (u32 i = 0; i < info.size(); i++) { + if (info.is_widestate(i)) { + continue; + } + find_better_daddy(info, i, using8bit, + any_cyclic_near_anchored_state, + trust_daddy_states, cc.grey); + total_daddy += info.extra[i].daddytaken; } - find_better_daddy(info, i, using8bit, - any_cyclic_near_anchored_state, - trust_daddy_states, cc.grey); - total_daddy += info.extra[i].daddytaken; - } - DEBUG_PRINTF("daddy %hu/%zu states=%zu alpha=%hu\n", total_daddy, - info.size() * info.impl_alpha_size, info.size(), - info.impl_alpha_size); + DEBUG_PRINTF("daddy %hu/%zu states=%zu alpha=%hu\n", total_daddy, + info.size() * info.impl_alpha_size, info.size(), + info.impl_alpha_size); + } nfa = mcclellanCompile16(info, cc, accel_states); } else { diff --git a/src/nfa/mcsheng.c b/src/nfa/mcsheng.c index 4619ff6fd..22cac119f 100644 --- a/src/nfa/mcsheng.c +++ b/src/nfa/mcsheng.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2018, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -1184,7 +1184,7 @@ char nfaExecMcSheng16_reportCurrent(const struct NFA *n, struct mq *q) { static char mcshengHasAccept(const struct mcsheng *m, const struct mstate_aux *aux, - ReportID report) { + ReportID report) { assert(m && aux); if (!aux->accept) { @@ -1405,3 +1405,1332 @@ char nfaExecMcSheng16_expandState(UNUSED const struct NFA *nfa, void *dest, *(u16 *)dest = unaligned_load_u16(src); return 0; } + +#if defined(HAVE_AVX512VBMI) +static really_inline +const struct mstate_aux *get_aux64(const struct mcsheng64 *m, u32 s) { + const char *nfa = (const char *)m - sizeof(struct NFA); + const struct mstate_aux *aux + = s + (const struct mstate_aux *)(nfa + m->aux_offset); + + assert(ISALIGNED(aux)); + return aux; +} + +static really_inline +u32 mcshengEnableStarts64(const struct mcsheng64 *m, u32 s) { + const struct mstate_aux *aux = get_aux64(m, s); + + DEBUG_PRINTF("enabling starts %u->%hu\n", s, aux->top); + return aux->top; +} + +static really_inline +char doComplexReport64(NfaCallback cb, void *ctxt, const struct mcsheng64 *m, + u32 s, u64a loc, char eod, u32 *cached_accept_state, + u32 *cached_accept_id) { + DEBUG_PRINTF("reporting state = %u, loc=%llu, eod %hhu\n", + s & STATE_MASK, loc, eod); + + if (!eod && s == *cached_accept_state) { + if (cb(0, loc, *cached_accept_id, ctxt) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; /* termination requested */ + } + + return MO_CONTINUE_MATCHING; /* continue execution */ + } + + const struct mstate_aux *aux = get_aux64(m, s); + size_t offset = eod ? aux->accept_eod : aux->accept; + + assert(offset); + const struct report_list *rl + = (const void *)((const char *)m + offset - sizeof(struct NFA)); + assert(ISALIGNED(rl)); + + DEBUG_PRINTF("report list size %u\n", rl->count); + u32 count = rl->count; + + if (!eod && count == 1) { + *cached_accept_state = s; + *cached_accept_id = rl->report[0]; + + DEBUG_PRINTF("reporting %u\n", rl->report[0]); + if (cb(0, loc, rl->report[0], ctxt) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; /* termination requested */ + } + + return MO_CONTINUE_MATCHING; /* continue execution */ + } + + for (u32 i = 0; i < count; i++) { + DEBUG_PRINTF("reporting %u\n", rl->report[i]); + if (cb(0, loc, rl->report[i], ctxt) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; /* termination requested */ + } + } + + return MO_CONTINUE_MATCHING; /* continue execution */ +} + +static really_inline +u32 doSheng64(const struct mcsheng64 *m, const u8 **c_inout, const u8 *soft_c_end, + const u8 *hard_c_end, u32 s_in, char do_accel) { + assert(s_in < m->sheng_end); + assert(s_in); /* should not already be dead */ + assert(soft_c_end <= hard_c_end); + DEBUG_PRINTF("s_in = %u (adjusted %u)\n", s_in, s_in - 1); + m512 s = set64x8(s_in - 1); + const u8 *c = *c_inout; + const u8 *c_end = hard_c_end - SHENG_CHUNK + 1; + if (!do_accel) { + c_end = MIN(soft_c_end, hard_c_end - SHENG_CHUNK + 1); + } + + const m512 *masks = m->sheng_succ_masks; + u8 sheng_limit = m->sheng_end - 1; /* - 1: no dead state */ + u8 sheng_stop_limit = do_accel ? m->sheng_accel_limit : sheng_limit; + + /* When we use movd to get a u32 containing our state, it will have 4 lanes + * all duplicating the state. We can create versions of our limits with 4 + * copies to directly compare against, this prevents us generating code to + * extract a single copy of the state from the u32 for checking. */ + u32 sheng_stop_limit_x4 = sheng_stop_limit * 0x01010101; + +#if defined(HAVE_BMI2) && defined(ARCH_64_BIT) + u32 sheng_limit_x4 = sheng_limit * 0x01010101; + m512 simd_stop_limit = set16x32(sheng_stop_limit_x4); + m512 accel_delta = set64x8(sheng_limit - sheng_stop_limit); + DEBUG_PRINTF("end %hhu, accel %hu --> limit %hhu\n", sheng_limit, + m->sheng_accel_limit, sheng_stop_limit); +#endif + +#define SHENG64_SINGLE_ITER do { \ + m512 succ_mask = masks[*(c++)]; \ + s = vpermb512(s, succ_mask); \ + u32 s_gpr_x4 = movd512(s); /* convert to u8 */ \ + DEBUG_PRINTF("c %hhu (%c) --> s %u\n", c[-1], c[-1], s_gpr_x4); \ + if (s_gpr_x4 >= sheng_stop_limit_x4) { \ + s_gpr = s_gpr_x4; \ + goto exit; \ + } \ + } while (0) + + u8 s_gpr; + while (c < c_end) { +#if defined(HAVE_BMI2) && defined(ARCH_64_BIT) + /* This version uses pext for efficiently bitbashing out scaled + * versions of the bytes to process from a u64a */ + + u64a data_bytes = unaligned_load_u64a(c); + u64a cc0 = pdep64(data_bytes, 0x3fc0); /* extract scaled low byte */ + data_bytes &= ~0xffULL; /* clear low bits for scale space */ + + m512 succ_mask0 = load512((const char *)masks + cc0); + s = vpermb512(s, succ_mask0); + m512 s_max = s; + m512 s_max0 = s_max; + DEBUG_PRINTF("c %02llx --> s %u\n", cc0 >> 6, movd512(s)); + +#define SHENG64_SINGLE_UNROLL_ITER(iter) \ + assert(iter); \ + u64a cc##iter = pext64(data_bytes, mcsheng64_pext_mask[iter]); \ + assert(cc##iter == (u64a)c[iter] << 6); \ + m512 succ_mask##iter = load512((const char *)masks + cc##iter); \ + s = vpermb512(s, succ_mask##iter); \ + if (do_accel && iter == 7) { \ + /* in the final iteration we also have to check against accel */ \ + m512 s_temp = sadd_u8_m512(s, accel_delta); \ + s_max = max_u8_m512(s_max, s_temp); \ + } else { \ + s_max = max_u8_m512(s_max, s); \ + } \ + m512 s_max##iter = s_max; \ + DEBUG_PRINTF("c %02llx --> s %u max %u\n", cc##iter >> 6, \ + movd512(s), movd512(s_max)); + + SHENG64_SINGLE_UNROLL_ITER(1); + SHENG64_SINGLE_UNROLL_ITER(2); + SHENG64_SINGLE_UNROLL_ITER(3); + SHENG64_SINGLE_UNROLL_ITER(4); + SHENG64_SINGLE_UNROLL_ITER(5); + SHENG64_SINGLE_UNROLL_ITER(6); + SHENG64_SINGLE_UNROLL_ITER(7); + + if (movd512(s_max7) >= sheng_limit_x4) { + DEBUG_PRINTF("exit found\n"); + + /* Explicitly check the last byte as it is more likely as it also + * checks for acceleration. */ + if (movd512(s_max6) < sheng_limit_x4) { + c += SHENG_CHUNK; + s_gpr = movq512(s); + assert(s_gpr >= sheng_stop_limit); + goto exit; + } + + /* use shift-xor to create a register containing all of the max + * values */ + m512 blended = rshift64_m512(s_max0, 56); + blended = xor512(blended, rshift64_m512(s_max1, 48)); + blended = xor512(blended, rshift64_m512(s_max2, 40)); + blended = xor512(blended, rshift64_m512(s_max3, 32)); + blended = xor512(blended, rshift64_m512(s_max4, 24)); + blended = xor512(blended, rshift64_m512(s_max5, 16)); + blended = xor512(blended, rshift64_m512(s_max6, 8)); + blended = xor512(blended, s); + blended = xor512(blended, rshift64_m512(blended, 8)); + DEBUG_PRINTF("blended %016llx\n", movq512(blended)); + + m512 final = min_u8_m512(blended, simd_stop_limit); + m512 cmp = sub_u8_m512(final, simd_stop_limit); + m128 tmp = cast512to128(cmp); + u64a stops = ~movemask128(tmp); + assert(stops); + u32 earliest = ctz32(stops); + DEBUG_PRINTF("stops %02llx, earliest %u\n", stops, earliest); + assert(earliest < 8); + c += earliest + 1; + s_gpr = movq512(blended) >> (earliest * 8); + assert(s_gpr >= sheng_stop_limit); + goto exit; + } else { + c += SHENG_CHUNK; + } +#else + SHENG64_SINGLE_ITER; + SHENG64_SINGLE_ITER; + SHENG64_SINGLE_ITER; + SHENG64_SINGLE_ITER; + + SHENG64_SINGLE_ITER; + SHENG64_SINGLE_ITER; + SHENG64_SINGLE_ITER; + SHENG64_SINGLE_ITER; +#endif + } + + assert(c_end - c < SHENG_CHUNK); + if (c < soft_c_end) { + assert(soft_c_end - c < SHENG_CHUNK); + switch (soft_c_end - c) { + case 7: + SHENG64_SINGLE_ITER; // fallthrough + case 6: + SHENG64_SINGLE_ITER; // fallthrough + case 5: + SHENG64_SINGLE_ITER; // fallthrough + case 4: + SHENG64_SINGLE_ITER; // fallthrough + case 3: + SHENG64_SINGLE_ITER; // fallthrough + case 2: + SHENG64_SINGLE_ITER; // fallthrough + case 1: + SHENG64_SINGLE_ITER; // fallthrough + } + } + + assert(c >= soft_c_end); + + s_gpr = movq512(s); +exit: + assert(c <= hard_c_end); + DEBUG_PRINTF("%zu from end; s %hhu\n", c_end - c, s_gpr); + assert(c >= soft_c_end || s_gpr >= sheng_stop_limit); + /* undo state adjustment to match mcclellan view */ + if (s_gpr == sheng_limit) { + s_gpr = 0; + } else if (s_gpr < sheng_limit) { + s_gpr++; + } + + *c_inout = c; + return s_gpr; +} + +static really_inline +const char *findShermanState64(UNUSED const struct mcsheng64 *m, + const char *sherman_base_offset, + u32 sherman_base, u32 s) { + const char *rv + = sherman_base_offset + SHERMAN_FIXED_SIZE * (s - sherman_base); + assert(rv < (const char *)m + m->length - sizeof(struct NFA)); + UNUSED u8 type = *(const u8 *)(rv + SHERMAN_TYPE_OFFSET); + assert(type == SHERMAN_STATE); + return rv; +} + +static really_inline +const u8 *run_mcsheng_accel64(const struct mcsheng64 *m, + const struct mstate_aux *aux, u32 s, + const u8 **min_accel_offset, + const u8 *c, const u8 *c_end) { + DEBUG_PRINTF("skipping\n"); + u32 accel_offset = aux[s].accel_offset; + + assert(aux[s].accel_offset); + assert(accel_offset >= m->aux_offset); + assert(!m->sherman_offset || accel_offset < m->sherman_offset); + + const union AccelAux *aaux = (const void *)((const char *)m + accel_offset); + const u8 *c2 = run_accel(aaux, c, c_end); + + if (c2 < *min_accel_offset + BAD_ACCEL_DIST) { + *min_accel_offset = c2 + BIG_ACCEL_PENALTY; + } else { + *min_accel_offset = c2 + SMALL_ACCEL_PENALTY; + } + + if (*min_accel_offset >= c_end - ACCEL_MIN_LEN) { + *min_accel_offset = c_end; + } + + DEBUG_PRINTF("advanced %zd, next accel chance in %zd/%zd\n", + c2 - c, *min_accel_offset - c2, c_end - c2); + + return c2; +} + +static really_inline +u32 doNormal64_16(const struct mcsheng64 *m, const u8 **c_inout, const u8 *end, + u32 s, char do_accel, enum MatchMode mode) { + const u8 *c = *c_inout; + const u16 *succ_table + = (const u16 *)((const char *)m + sizeof(struct mcsheng64)); + assert(ISALIGNED_N(succ_table, 2)); + u32 sheng_end = m->sheng_end; + u32 sherman_base = m->sherman_limit; + const char *sherman_base_offset + = (const char *)m - sizeof(struct NFA) + m->sherman_offset; + u32 as = m->alphaShift; + + /* Adjust start of succ table so we can index into using state id (rather + * than adjust to normal id). As we will not be processing states with low + * state ids, we will not be accessing data before the succ table. Note: due + * to the size of the sheng tables, the succ_table pointer will still be + * inside the engine.*/ + succ_table -= sheng_end << as; + s &= STATE_MASK; + while (c < end && s >= sheng_end) { + u8 cprime = m->remap[*c]; + DEBUG_PRINTF("c: %02hhx '%c' cp:%02hhx (s=%u)\n", *c, + ourisprint(*c) ? *c : '?', cprime, s); + if (s < sherman_base) { + DEBUG_PRINTF("doing normal\n"); + assert(s < m->state_count); + s = succ_table[(s << as) + cprime]; + } else { + const char *sherman_state + = findShermanState64(m, sherman_base_offset, sherman_base, s); + DEBUG_PRINTF("doing sherman (%u)\n", s); + s = doSherman16(sherman_state, cprime, succ_table, as); + } + + DEBUG_PRINTF("s: %u (%u)\n", s, s & STATE_MASK); + c++; + + if (do_accel && (s & ACCEL_FLAG)) { + break; + } + if (mode != NO_MATCHES && (s & ACCEPT_FLAG)) { + break; + } + + s &= STATE_MASK; + } + + *c_inout = c; + return s; +} + +static really_inline +char mcsheng64Exec16_i(const struct mcsheng64 *m, u32 *state, const u8 *buf, + size_t len, u64a offAdj, NfaCallback cb, void *ctxt, + char single, const u8 **c_final, enum MatchMode mode) { + assert(ISALIGNED_N(state, 2)); + if (!len) { + if (mode == STOP_AT_MATCH) { + *c_final = buf; + } + return MO_ALIVE; + } + + u32 s = *state; + const u8 *c = buf; + const u8 *c_end = buf + len; + const u8 sheng_end = m->sheng_end; + const struct mstate_aux *aux + = (const struct mstate_aux *)((const char *)m + m->aux_offset + - sizeof(struct NFA)); + + s &= STATE_MASK; + + u32 cached_accept_id = 0; + u32 cached_accept_state = 0; + + DEBUG_PRINTF("s: %u, len %zu\n", s, len); + + const u8 *min_accel_offset = c; + if (!m->has_accel || len < ACCEL_MIN_LEN) { + min_accel_offset = c_end; + goto without_accel; + } + + goto with_accel; + +without_accel: + do { + assert(c < min_accel_offset); + int do_accept; + if (!s) { + goto exit; + } else if (s < sheng_end) { + s = doSheng64(m, &c, min_accel_offset, c_end, s, 0); + do_accept = mode != NO_MATCHES && get_aux64(m, s)->accept; + } else { + s = doNormal64_16(m, &c, min_accel_offset, s, 0, mode); + + do_accept = mode != NO_MATCHES && (s & ACCEPT_FLAG); + } + + if (do_accept) { + if (mode == STOP_AT_MATCH) { + *state = s & STATE_MASK; + *c_final = c - 1; + return MO_MATCHES_PENDING; + } + + u64a loc = (c - 1) - buf + offAdj + 1; + + if (single) { + DEBUG_PRINTF("reporting %u\n", m->arb_report); + if (cb(0, loc, m->arb_report, ctxt) == MO_HALT_MATCHING) { + return MO_DEAD; /* termination requested */ + } + } else if (doComplexReport64(cb, ctxt, m, s & STATE_MASK, loc, 0, + &cached_accept_state, + &cached_accept_id) + == MO_HALT_MATCHING) { + return MO_DEAD; + } + } + + assert(c <= c_end); /* sheng is fuzzy for min_accel_offset */ + } while (c < min_accel_offset); + + if (c == c_end) { + goto exit; + } + +with_accel: + do { + assert(c < c_end); + int do_accept; + + if (!s) { + goto exit; + } else if (s < sheng_end) { + if (s > m->sheng_accel_limit) { + c = run_mcsheng_accel64(m, aux, s, &min_accel_offset, c, c_end); + if (c == c_end) { + goto exit; + } else { + goto without_accel; + } + } + s = doSheng64(m, &c, c_end, c_end, s, 1); + do_accept = mode != NO_MATCHES && get_aux64(m, s)->accept; + } else { + if (s & ACCEL_FLAG) { + DEBUG_PRINTF("skipping\n"); + s &= STATE_MASK; + c = run_mcsheng_accel64(m, aux, s, &min_accel_offset, c, c_end); + if (c == c_end) { + goto exit; + } else { + goto without_accel; + } + } + + s = doNormal64_16(m, &c, c_end, s, 1, mode); + do_accept = mode != NO_MATCHES && (s & ACCEPT_FLAG); + } + + if (do_accept) { + if (mode == STOP_AT_MATCH) { + *state = s & STATE_MASK; + *c_final = c - 1; + return MO_MATCHES_PENDING; + } + + u64a loc = (c - 1) - buf + offAdj + 1; + + if (single) { + DEBUG_PRINTF("reporting %u\n", m->arb_report); + if (cb(0, loc, m->arb_report, ctxt) == MO_HALT_MATCHING) { + return MO_DEAD; /* termination requested */ + } + } else if (doComplexReport64(cb, ctxt, m, s & STATE_MASK, loc, 0, + &cached_accept_state, + &cached_accept_id) + == MO_HALT_MATCHING) { + return MO_DEAD; + } + } + + assert(c <= c_end); + } while (c < c_end); + +exit: + s &= STATE_MASK; + + if (mode == STOP_AT_MATCH) { + *c_final = c_end; + } + *state = s; + + return MO_ALIVE; +} + +static never_inline +char mcsheng64Exec16_i_cb(const struct mcsheng64 *m, u32 *state, const u8 *buf, + size_t len, u64a offAdj, NfaCallback cb, void *ctxt, + char single, const u8 **final_point) { + return mcsheng64Exec16_i(m, state, buf, len, offAdj, cb, ctxt, single, + final_point, CALLBACK_OUTPUT); +} + +static never_inline +char mcsheng64Exec16_i_sam(const struct mcsheng64 *m, u32 *state, const u8 *buf, + size_t len, u64a offAdj, NfaCallback cb, void *ctxt, + char single, const u8 **final_point) { + return mcsheng64Exec16_i(m, state, buf, len, offAdj, cb, ctxt, single, + final_point, STOP_AT_MATCH); +} + +static never_inline +char mcsheng64Exec16_i_nm(const struct mcsheng64 *m, u32 *state, const u8 *buf, + size_t len, u64a offAdj, NfaCallback cb, void *ctxt, + char single, const u8 **final_point) { + return mcsheng64Exec16_i(m, state, buf, len, offAdj, cb, ctxt, single, + final_point, NO_MATCHES); +} + +static really_inline +char mcsheng64Exec16_i_ni(const struct mcsheng64 *m, u32 *state, const u8 *buf, + size_t len, u64a offAdj, NfaCallback cb, void *ctxt, + char single, const u8 **final_point, + enum MatchMode mode) { + if (mode == CALLBACK_OUTPUT) { + return mcsheng64Exec16_i_cb(m, state, buf, len, offAdj, cb, ctxt, + single, final_point); + } else if (mode == STOP_AT_MATCH) { + return mcsheng64Exec16_i_sam(m, state, buf, len, offAdj, cb, ctxt, + single, final_point); + } else { + assert (mode == NO_MATCHES); + return mcsheng64Exec16_i_nm(m, state, buf, len, offAdj, cb, ctxt, + single, final_point); + } +} + +static really_inline +u32 doNormal64_8(const struct mcsheng64 *m, const u8 **c_inout, const u8 *end, u32 s, + char do_accel, enum MatchMode mode) { + const u8 *c = *c_inout; + u32 sheng_end = m->sheng_end; + u32 accel_limit = m->accel_limit_8; + u32 accept_limit = m->accept_limit_8; + + const u32 as = m->alphaShift; + const u8 *succ_table = (const u8 *)((const char *)m + + sizeof(struct mcsheng64)); + /* Adjust start of succ table so we can index into using state id (rather + * than adjust to normal id). As we will not be processing states with low + * state ids, we will not be accessing data before the succ table. Note: due + * to the size of the sheng tables, the succ_table pointer will still be + * inside the engine.*/ + succ_table -= sheng_end << as; + + assert(s >= sheng_end); + while (c < end && s >= sheng_end) { + u8 cprime = m->remap[*c]; + DEBUG_PRINTF("c: %02hhx '%c' cp:%02hhx\n", *c, + ourisprint(*c) ? *c : '?', cprime); + s = succ_table[(s << as) + cprime]; + + DEBUG_PRINTF("s: %u\n", s); + c++; + if (do_accel) { + if (s >= accel_limit) { + break; + } + } else { + if (mode != NO_MATCHES && s >= accept_limit) { + break; + } + } + } + *c_inout = c; + return s; +} + +static really_inline +char mcsheng64Exec8_i(const struct mcsheng64 *m, u32 *state, const u8 *buf, + size_t len, u64a offAdj, NfaCallback cb, void *ctxt, + char single, const u8 **c_final, enum MatchMode mode) { + if (!len) { + *c_final = buf; + return MO_ALIVE; + } + u32 s = *state; + const u8 *c = buf; + const u8 *c_end = buf + len; + const u8 sheng_end = m->sheng_end; + + const struct mstate_aux *aux + = (const struct mstate_aux *)((const char *)m + m->aux_offset + - sizeof(struct NFA)); + u32 accept_limit = m->accept_limit_8; + + u32 cached_accept_id = 0; + u32 cached_accept_state = 0; + + DEBUG_PRINTF("accel %hu, accept %u\n", m->accel_limit_8, accept_limit); + + DEBUG_PRINTF("s: %u, len %zu\n", s, len); + + const u8 *min_accel_offset = c; + if (!m->has_accel || len < ACCEL_MIN_LEN) { + min_accel_offset = c_end; + goto without_accel; + } + + goto with_accel; + +without_accel: + do { + assert(c < min_accel_offset); + if (!s) { + goto exit; + } else if (s < sheng_end) { + s = doSheng64(m, &c, min_accel_offset, c_end, s, 0); + } else { + s = doNormal64_8(m, &c, min_accel_offset, s, 0, mode); + assert(c <= min_accel_offset); + } + + if (mode != NO_MATCHES && s >= accept_limit) { + if (mode == STOP_AT_MATCH) { + DEBUG_PRINTF("match - pausing\n"); + *state = s; + *c_final = c - 1; + return MO_MATCHES_PENDING; + } + + u64a loc = (c - 1) - buf + offAdj + 1; + if (single) { + DEBUG_PRINTF("reporting %u\n", m->arb_report); + if (cb(0, loc, m->arb_report, ctxt) == MO_HALT_MATCHING) { + return MO_DEAD; + } + } else if (doComplexReport64(cb, ctxt, m, s, loc, 0, + &cached_accept_state, + &cached_accept_id) + == MO_HALT_MATCHING) { + return MO_DEAD; + } + } + + assert(c <= c_end); /* sheng is fuzzy for min_accel_offset */ + } while (c < min_accel_offset); + + if (c == c_end) { + goto exit; + } + +with_accel: + do { + u32 accel_limit = m->accel_limit_8; + + assert(c < c_end); + if (!s) { + goto exit; + } else if (s < sheng_end) { + if (s > m->sheng_accel_limit) { + c = run_mcsheng_accel64(m, aux, s, &min_accel_offset, c, c_end); + if (c == c_end) { + goto exit; + } else { + goto without_accel; + } + } + s = doSheng64(m, &c, c_end, c_end, s, 1); + } else { + if (s >= accel_limit && aux[s].accel_offset) { + c = run_mcsheng_accel64(m, aux, s, &min_accel_offset, c, c_end); + if (c == c_end) { + goto exit; + } else { + goto without_accel; + } + } + s = doNormal64_8(m, &c, c_end, s, 1, mode); + } + + if (mode != NO_MATCHES && s >= accept_limit) { + if (mode == STOP_AT_MATCH) { + DEBUG_PRINTF("match - pausing\n"); + *state = s; + *c_final = c - 1; + return MO_MATCHES_PENDING; + } + + u64a loc = (c - 1) - buf + offAdj + 1; + if (single) { + DEBUG_PRINTF("reporting %u\n", m->arb_report); + if (cb(0, loc, m->arb_report, ctxt) == MO_HALT_MATCHING) { + return MO_DEAD; + } + } else if (doComplexReport64(cb, ctxt, m, s, loc, 0, + &cached_accept_state, + &cached_accept_id) + == MO_HALT_MATCHING) { + return MO_DEAD; + } + } + + assert(c <= c_end); + } while (c < c_end); + +exit: + *state = s; + if (mode == STOP_AT_MATCH) { + *c_final = c_end; + } + return MO_ALIVE; +} + +static never_inline +char mcsheng64Exec8_i_cb(const struct mcsheng64 *m, u32 *state, const u8 *buf, + size_t len, u64a offAdj, NfaCallback cb, void *ctxt, + char single, const u8 **final_point) { + return mcsheng64Exec8_i(m, state, buf, len, offAdj, cb, ctxt, single, + final_point, CALLBACK_OUTPUT); +} + +static never_inline +char mcsheng64Exec8_i_sam(const struct mcsheng64 *m, u32 *state, const u8 *buf, + size_t len, u64a offAdj, NfaCallback cb, void *ctxt, + char single, const u8 **final_point) { + return mcsheng64Exec8_i(m, state, buf, len, offAdj, cb, ctxt, single, + final_point, STOP_AT_MATCH); +} + +static never_inline +char mcsheng64Exec8_i_nm(const struct mcsheng64 *m, u32 *state, const u8 *buf, + size_t len, u64a offAdj, NfaCallback cb, void *ctxt, + char single, const u8 **final_point) { + return mcsheng64Exec8_i(m, state, buf, len, offAdj, cb, ctxt, single, + final_point, NO_MATCHES); +} + +static really_inline +char mcsheng64Exec8_i_ni(const struct mcsheng64 *m, u32 *state, const u8 *buf, + size_t len, u64a offAdj, NfaCallback cb, void *ctxt, + char single, const u8 **final_point, + enum MatchMode mode) { + if (mode == CALLBACK_OUTPUT) { + return mcsheng64Exec8_i_cb(m, state, buf, len, offAdj, cb, ctxt, single, + final_point); + } else if (mode == STOP_AT_MATCH) { + return mcsheng64Exec8_i_sam(m, state, buf, len, offAdj, cb, ctxt, + single, final_point); + } else { + assert(mode == NO_MATCHES); + return mcsheng64Exec8_i_nm(m, state, buf, len, offAdj, cb, ctxt, single, + final_point); + } +} + +static really_inline +char mcshengCheckEOD64(const struct NFA *nfa, u32 s, u64a offset, + NfaCallback cb, void *ctxt) { + const struct mcsheng64 *m = getImplNfa(nfa); + const struct mstate_aux *aux = get_aux64(m, s); + + if (!aux->accept_eod) { + return MO_CONTINUE_MATCHING; + } + return doComplexReport64(cb, ctxt, m, s, offset, 1, NULL, NULL); +} + +static really_inline +char nfaExecMcSheng64_16_Q2i(const struct NFA *n, u64a offset, const u8 *buffer, + const u8 *hend, NfaCallback cb, void *context, + struct mq *q, char single, s64a end, + enum MatchMode mode) { + assert(n->type == MCSHENG_64_NFA_16); + const struct mcsheng64 *m = getImplNfa(n); + s64a sp; + + assert(ISALIGNED_N(q->state, 2)); + u32 s = *(u16 *)q->state; + + if (q->report_current) { + assert(s); + assert(get_aux64(m, s)->accept); + + int rv; + if (single) { + DEBUG_PRINTF("reporting %u\n", m->arb_report); + rv = cb(0, q_cur_offset(q), m->arb_report, context); + } else { + u32 cached_accept_id = 0; + u32 cached_accept_state = 0; + + rv = doComplexReport64(cb, context, m, s, q_cur_offset(q), 0, + &cached_accept_state, &cached_accept_id); + } + + q->report_current = 0; + + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + } + + sp = q_cur_loc(q); + q->cur++; + + const u8 *cur_buf = sp < 0 ? hend : buffer; + + assert(q->cur); + if (mode != NO_MATCHES && q->items[q->cur - 1].location > end) { + DEBUG_PRINTF("this is as far as we go\n"); + q->cur--; + q->items[q->cur].type = MQE_START; + q->items[q->cur].location = end; + *(u16 *)q->state = s; + return MO_ALIVE; + } + + while (1) { + assert(q->cur < q->end); + s64a ep = q->items[q->cur].location; + if (mode != NO_MATCHES) { + ep = MIN(ep, end); + } + + assert(ep >= sp); + + s64a local_ep = ep; + if (sp < 0) { + local_ep = MIN(0, ep); + } + + /* do main buffer region */ + const u8 *final_look; + char rv = mcsheng64Exec16_i_ni(m, &s, cur_buf + sp, local_ep - sp, + offset + sp, cb, context, single, + &final_look, mode); + if (rv == MO_DEAD) { + *(u16 *)q->state = 0; + return MO_DEAD; + } + if (mode == STOP_AT_MATCH && rv == MO_MATCHES_PENDING) { + DEBUG_PRINTF("this is as far as we go\n"); + DEBUG_PRINTF("state %u final_look %zd\n", s, final_look - cur_buf); + + assert(q->cur); + assert(final_look != cur_buf + local_ep); + + q->cur--; + q->items[q->cur].type = MQE_START; + q->items[q->cur].location = final_look - cur_buf + 1; /* due to + * early -1 */ + *(u16 *)q->state = s; + return MO_MATCHES_PENDING; + } + + assert(rv == MO_ALIVE); + assert(q->cur); + if (mode != NO_MATCHES && q->items[q->cur].location > end) { + DEBUG_PRINTF("this is as far as we go\n"); + q->cur--; + q->items[q->cur].type = MQE_START; + q->items[q->cur].location = end; + *(u16 *)q->state = s; + return MO_ALIVE; + } + + sp = local_ep; + + if (sp == 0) { + cur_buf = buffer; + } + + if (sp != ep) { + continue; + } + + switch (q->items[q->cur].type) { + case MQE_TOP: + assert(sp + offset || !s); + if (sp + offset == 0) { + s = m->start_anchored; + break; + } + s = mcshengEnableStarts64(m, s); + break; + case MQE_END: + *(u16 *)q->state = s; + q->cur++; + return s ? MO_ALIVE : MO_DEAD; + default: + assert(!"invalid queue event"); + } + + q->cur++; + } +} + +static really_inline +char nfaExecMcSheng64_8_Q2i(const struct NFA *n, u64a offset, const u8 *buffer, + const u8 *hend, NfaCallback cb, void *context, + struct mq *q, char single, s64a end, + enum MatchMode mode) { + assert(n->type == MCSHENG_64_NFA_8); + const struct mcsheng64 *m = getImplNfa(n); + s64a sp; + + u32 s = *(u8 *)q->state; + + if (q->report_current) { + assert(s); + assert(s >= m->accept_limit_8); + + int rv; + if (single) { + DEBUG_PRINTF("reporting %u\n", m->arb_report); + + rv = cb(0, q_cur_offset(q), m->arb_report, context); + } else { + u32 cached_accept_id = 0; + u32 cached_accept_state = 0; + + rv = doComplexReport64(cb, context, m, s, q_cur_offset(q), 0, + &cached_accept_state, &cached_accept_id); + } + + q->report_current = 0; + + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + } + + sp = q_cur_loc(q); + q->cur++; + + const u8 *cur_buf = sp < 0 ? hend : buffer; + + if (mode != NO_MATCHES && q->items[q->cur - 1].location > end) { + DEBUG_PRINTF("this is as far as we go\n"); + q->cur--; + q->items[q->cur].type = MQE_START; + q->items[q->cur].location = end; + *(u8 *)q->state = s; + return MO_ALIVE; + } + + while (1) { + DEBUG_PRINTF("%s @ %llu\n", q->items[q->cur].type == MQE_TOP ? "TOP" : + q->items[q->cur].type == MQE_END ? "END" : "???", + q->items[q->cur].location + offset); + assert(q->cur < q->end); + s64a ep = q->items[q->cur].location; + if (mode != NO_MATCHES) { + ep = MIN(ep, end); + } + + assert(ep >= sp); + + s64a local_ep = ep; + if (sp < 0) { + local_ep = MIN(0, ep); + } + + const u8 *final_look; + char rv = mcsheng64Exec8_i_ni(m, &s, cur_buf + sp, local_ep - sp, + offset + sp, cb, context, single, + &final_look, mode); + if (rv == MO_HALT_MATCHING) { + *(u8 *)q->state = 0; + return MO_DEAD; + } + if (mode == STOP_AT_MATCH && rv == MO_MATCHES_PENDING) { + DEBUG_PRINTF("this is as far as we go\n"); + DEBUG_PRINTF("state %u final_look %zd\n", s, final_look - cur_buf); + + assert(q->cur); + assert(final_look != cur_buf + local_ep); + + q->cur--; + q->items[q->cur].type = MQE_START; + q->items[q->cur].location = final_look - cur_buf + 1; /* due to + * early -1 */ + *(u8 *)q->state = s; + return MO_MATCHES_PENDING; + } + + assert(rv == MO_ALIVE); + assert(q->cur); + if (mode != NO_MATCHES && q->items[q->cur].location > end) { + DEBUG_PRINTF("this is as far as we go\n"); + assert(q->cur); + q->cur--; + q->items[q->cur].type = MQE_START; + q->items[q->cur].location = end; + *(u8 *)q->state = s; + return MO_ALIVE; + } + + sp = local_ep; + + if (sp == 0) { + cur_buf = buffer; + } + + if (sp != ep) { + continue; + } + + switch (q->items[q->cur].type) { + case MQE_TOP: + assert(sp + offset || !s); + if (sp + offset == 0) { + s = (u8)m->start_anchored; + break; + } + s = mcshengEnableStarts64(m, s); + break; + case MQE_END: + *(u8 *)q->state = s; + q->cur++; + return s ? MO_ALIVE : MO_DEAD; + default: + assert(!"invalid queue event"); + } + + q->cur++; + } +} + +char nfaExecMcSheng64_8_Q(const struct NFA *n, struct mq *q, s64a end) { + u64a offset = q->offset; + const u8 *buffer = q->buffer; + NfaCallback cb = q->cb; + void *context = q->context; + assert(n->type == MCSHENG_64_NFA_8); + const struct mcsheng64 *m = getImplNfa(n); + const u8 *hend = q->history + q->hlength; + + return nfaExecMcSheng64_8_Q2i(n, offset, buffer, hend, cb, context, q, + m->flags & MCSHENG_FLAG_SINGLE, end, + CALLBACK_OUTPUT); +} + +char nfaExecMcSheng64_16_Q(const struct NFA *n, struct mq *q, s64a end) { + u64a offset = q->offset; + const u8 *buffer = q->buffer; + NfaCallback cb = q->cb; + void *context = q->context; + assert(n->type == MCSHENG_64_NFA_16); + const struct mcsheng64 *m = getImplNfa(n); + const u8 *hend = q->history + q->hlength; + + return nfaExecMcSheng64_16_Q2i(n, offset, buffer, hend, cb, context, q, + m->flags & MCSHENG_FLAG_SINGLE, end, + CALLBACK_OUTPUT); +} + +char nfaExecMcSheng64_8_reportCurrent(const struct NFA *n, struct mq *q) { + const struct mcsheng64 *m = getImplNfa(n); + NfaCallback cb = q->cb; + void *ctxt = q->context; + u32 s = *(u8 *)q->state; + u8 single = m->flags & MCSHENG_FLAG_SINGLE; + u64a offset = q_cur_offset(q); + assert(q_cur_type(q) == MQE_START); + assert(s); + + if (s >= m->accept_limit_8) { + if (single) { + DEBUG_PRINTF("reporting %u\n", m->arb_report); + cb(0, offset, m->arb_report, ctxt); + } else { + u32 cached_accept_id = 0; + u32 cached_accept_state = 0; + + doComplexReport64(cb, ctxt, m, s, offset, 0, &cached_accept_state, + &cached_accept_id); + } + } + + return 0; +} + +char nfaExecMcSheng64_16_reportCurrent(const struct NFA *n, struct mq *q) { + const struct mcsheng64 *m = getImplNfa(n); + NfaCallback cb = q->cb; + void *ctxt = q->context; + u32 s = *(u16 *)q->state; + const struct mstate_aux *aux = get_aux64(m, s); + u8 single = m->flags & MCSHENG_FLAG_SINGLE; + u64a offset = q_cur_offset(q); + assert(q_cur_type(q) == MQE_START); + DEBUG_PRINTF("state %u\n", s); + assert(s); + + if (aux->accept) { + if (single) { + DEBUG_PRINTF("reporting %u\n", m->arb_report); + cb(0, offset, m->arb_report, ctxt); + } else { + u32 cached_accept_id = 0; + u32 cached_accept_state = 0; + + doComplexReport64(cb, ctxt, m, s, offset, 0, &cached_accept_state, + &cached_accept_id); + } + } + + return 0; +} + +static +char mcshengHasAccept64(const struct mcsheng64 *m, const struct mstate_aux *aux, + ReportID report) { + assert(m && aux); + + if (!aux->accept) { + return 0; + } + + const struct report_list *rl = (const struct report_list *) + ((const char *)m + aux->accept - sizeof(struct NFA)); + assert(ISALIGNED_N(rl, 4)); + + DEBUG_PRINTF("report list has %u entries\n", rl->count); + + for (u32 i = 0; i < rl->count; i++) { + if (rl->report[i] == report) { + return 1; + } + } + + return 0; +} + +char nfaExecMcSheng64_8_inAccept(const struct NFA *n, ReportID report, + struct mq *q) { + assert(n && q); + + const struct mcsheng64 *m = getImplNfa(n); + u8 s = *(u8 *)q->state; + DEBUG_PRINTF("checking accepts for %hhu\n", s); + + return mcshengHasAccept64(m, get_aux64(m, s), report); +} + +char nfaExecMcSheng64_8_inAnyAccept(const struct NFA *n, struct mq *q) { + assert(n && q); + + const struct mcsheng64 *m = getImplNfa(n); + u8 s = *(u8 *)q->state; + DEBUG_PRINTF("checking accepts for %hhu\n", s); + + return !!get_aux64(m, s)->accept; +} + +char nfaExecMcSheng64_16_inAccept(const struct NFA *n, ReportID report, + struct mq *q) { + assert(n && q); + + const struct mcsheng64 *m = getImplNfa(n); + u16 s = *(u16 *)q->state; + DEBUG_PRINTF("checking accepts for %hu\n", s); + + return mcshengHasAccept64(m, get_aux64(m, s), report); +} + +char nfaExecMcSheng64_16_inAnyAccept(const struct NFA *n, struct mq *q) { + assert(n && q); + + const struct mcsheng64 *m = getImplNfa(n); + u16 s = *(u16 *)q->state; + DEBUG_PRINTF("checking accepts for %hu\n", s); + + return !!get_aux64(m, s)->accept; +} + +char nfaExecMcSheng64_8_Q2(const struct NFA *n, struct mq *q, s64a end) { + u64a offset = q->offset; + const u8 *buffer = q->buffer; + NfaCallback cb = q->cb; + void *context = q->context; + assert(n->type == MCSHENG_64_NFA_8); + const struct mcsheng64 *m = getImplNfa(n); + const u8 *hend = q->history + q->hlength; + + return nfaExecMcSheng64_8_Q2i(n, offset, buffer, hend, cb, context, q, + m->flags & MCSHENG_FLAG_SINGLE, end, + STOP_AT_MATCH); +} + +char nfaExecMcSheng64_16_Q2(const struct NFA *n, struct mq *q, s64a end) { + u64a offset = q->offset; + const u8 *buffer = q->buffer; + NfaCallback cb = q->cb; + void *context = q->context; + assert(n->type == MCSHENG_64_NFA_16); + const struct mcsheng64 *m = getImplNfa(n); + const u8 *hend = q->history + q->hlength; + + return nfaExecMcSheng64_16_Q2i(n, offset, buffer, hend, cb, context, q, + m->flags & MCSHENG_FLAG_SINGLE, end, + STOP_AT_MATCH); +} + +char nfaExecMcSheng64_8_QR(const struct NFA *n, struct mq *q, ReportID report) { + u64a offset = q->offset; + const u8 *buffer = q->buffer; + NfaCallback cb = q->cb; + void *context = q->context; + assert(n->type == MCSHENG_64_NFA_8); + const struct mcsheng64 *m = getImplNfa(n); + const u8 *hend = q->history + q->hlength; + + char rv = nfaExecMcSheng64_8_Q2i(n, offset, buffer, hend, cb, context, q, + m->flags & MCSHENG_FLAG_SINGLE, + 0 /* end */, NO_MATCHES); + if (rv && nfaExecMcSheng64_8_inAccept(n, report, q)) { + return MO_MATCHES_PENDING; + } else { + return rv; + } +} + +char nfaExecMcSheng64_16_QR(const struct NFA *n, struct mq *q, ReportID report) { + u64a offset = q->offset; + const u8 *buffer = q->buffer; + NfaCallback cb = q->cb; + void *context = q->context; + assert(n->type == MCSHENG_64_NFA_16); + const struct mcsheng64 *m = getImplNfa(n); + const u8 *hend = q->history + q->hlength; + + char rv = nfaExecMcSheng64_16_Q2i(n, offset, buffer, hend, cb, context, q, + m->flags & MCSHENG_FLAG_SINGLE, + 0 /* end */, NO_MATCHES); + + if (rv && nfaExecMcSheng64_16_inAccept(n, report, q)) { + return MO_MATCHES_PENDING; + } else { + return rv; + } +} + +char nfaExecMcSheng64_8_initCompressedState(const struct NFA *nfa, u64a offset, + void *state, UNUSED u8 key) { + const struct mcsheng64 *m = getImplNfa(nfa); + u8 s = offset ? m->start_floating : m->start_anchored; + if (s) { + *(u8 *)state = s; + return 1; + } + return 0; +} + +char nfaExecMcSheng64_16_initCompressedState(const struct NFA *nfa, u64a offset, + void *state, UNUSED u8 key) { + const struct mcsheng64 *m = getImplNfa(nfa); + u16 s = offset ? m->start_floating : m->start_anchored; + if (s) { + unaligned_store_u16(state, s); + return 1; + } + return 0; +} + +char nfaExecMcSheng64_8_testEOD(const struct NFA *nfa, const char *state, + UNUSED const char *streamState, u64a offset, + NfaCallback callback, void *context) { + return mcshengCheckEOD64(nfa, *(const u8 *)state, offset, callback, + context); +} + +char nfaExecMcSheng64_16_testEOD(const struct NFA *nfa, const char *state, + UNUSED const char *streamState, u64a offset, + NfaCallback callback, void *context) { + assert(ISALIGNED_N(state, 2)); + return mcshengCheckEOD64(nfa, *(const u16 *)state, offset, callback, + context); +} + +char nfaExecMcSheng64_8_queueInitState(UNUSED const struct NFA *nfa, struct mq *q) { + assert(nfa->scratchStateSize == 1); + *(u8 *)q->state = 0; + return 0; +} + +char nfaExecMcSheng64_16_queueInitState(UNUSED const struct NFA *nfa, struct mq *q) { + assert(nfa->scratchStateSize == 2); + assert(ISALIGNED_N(q->state, 2)); + *(u16 *)q->state = 0; + return 0; +} + +char nfaExecMcSheng64_8_queueCompressState(UNUSED const struct NFA *nfa, + const struct mq *q, UNUSED s64a loc) { + void *dest = q->streamState; + const void *src = q->state; + assert(nfa->scratchStateSize == 1); + assert(nfa->streamStateSize == 1); + *(u8 *)dest = *(const u8 *)src; + return 0; +} + +char nfaExecMcSheng64_8_expandState(UNUSED const struct NFA *nfa, void *dest, + const void *src, UNUSED u64a offset, + UNUSED u8 key) { + assert(nfa->scratchStateSize == 1); + assert(nfa->streamStateSize == 1); + *(u8 *)dest = *(const u8 *)src; + return 0; +} + +char nfaExecMcSheng64_16_queueCompressState(UNUSED const struct NFA *nfa, + const struct mq *q, + UNUSED s64a loc) { + void *dest = q->streamState; + const void *src = q->state; + assert(nfa->scratchStateSize == 2); + assert(nfa->streamStateSize == 2); + assert(ISALIGNED_N(src, 2)); + unaligned_store_u16(dest, *(const u16 *)(src)); + return 0; +} + +char nfaExecMcSheng64_16_expandState(UNUSED const struct NFA *nfa, void *dest, + const void *src, UNUSED u64a offset, + UNUSED u8 key) { + assert(nfa->scratchStateSize == 2); + assert(nfa->streamStateSize == 2); + assert(ISALIGNED_N(dest, 2)); + *(u16 *)dest = unaligned_load_u16(src); + return 0; +} +#endif diff --git a/src/nfa/mcsheng.h b/src/nfa/mcsheng.h index 19fd69614..0329e1212 100644 --- a/src/nfa/mcsheng.h +++ b/src/nfa/mcsheng.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -80,5 +80,78 @@ char nfaExecMcSheng16_expandState(const struct NFA *nfa, void *dest, #define nfaExecMcSheng16_B_Reverse NFA_API_NO_IMPL #define nfaExecMcSheng16_zombie_status NFA_API_ZOMBIE_NO_IMPL +#if defined(HAVE_AVX512VBMI) +/* 64-8 bit Sheng-McClellan hybrid */ +char nfaExecMcSheng64_8_testEOD(const struct NFA *nfa, const char *state, + const char *streamState, u64a offset, + NfaCallback callback, void *context); +char nfaExecMcSheng64_8_Q(const struct NFA *n, struct mq *q, s64a end); +char nfaExecMcSheng64_8_Q2(const struct NFA *n, struct mq *q, s64a end); +char nfaExecMcSheng64_8_QR(const struct NFA *n, struct mq *q, ReportID report); +char nfaExecMcSheng64_8_reportCurrent(const struct NFA *n, struct mq *q); +char nfaExecMcSheng64_8_inAccept(const struct NFA *n, ReportID report, + struct mq *q); +char nfaExecMcSheng64_8_inAnyAccept(const struct NFA *n, struct mq *q); +char nfaExecMcSheng64_8_queueInitState(const struct NFA *n, struct mq *q); +char nfaExecMcSheng64_8_initCompressedState(const struct NFA *n, u64a offset, + void *state, u8 key); +char nfaExecMcSheng64_8_queueCompressState(const struct NFA *nfa, + const struct mq *q, s64a loc); +char nfaExecMcSheng64_8_expandState(const struct NFA *nfa, void *dest, + const void *src, u64a offset, u8 key); + +#define nfaExecMcSheng64_8_B_Reverse NFA_API_NO_IMPL +#define nfaExecMcSheng64_8_zombie_status NFA_API_ZOMBIE_NO_IMPL + +/* 64-16 bit Sheng-McClellan hybrid */ +char nfaExecMcSheng64_16_testEOD(const struct NFA *nfa, const char *state, + const char *streamState, u64a offset, + NfaCallback callback, void *context); +char nfaExecMcSheng64_16_Q(const struct NFA *n, struct mq *q, s64a end); +char nfaExecMcSheng64_16_Q2(const struct NFA *n, struct mq *q, s64a end); +char nfaExecMcSheng64_16_QR(const struct NFA *n, struct mq *q, ReportID report); +char nfaExecMcSheng64_16_reportCurrent(const struct NFA *n, struct mq *q); +char nfaExecMcSheng64_16_inAccept(const struct NFA *n, ReportID report, + struct mq *q); +char nfaExecMcSheng64_16_inAnyAccept(const struct NFA *n, struct mq *q); +char nfaExecMcSheng64_16_queueInitState(const struct NFA *n, struct mq *q); +char nfaExecMcSheng64_16_initCompressedState(const struct NFA *n, u64a offset, + void *state, u8 key); +char nfaExecMcSheng64_16_queueCompressState(const struct NFA *nfa, + const struct mq *q, s64a loc); +char nfaExecMcSheng64_16_expandState(const struct NFA *nfa, void *dest, + const void *src, u64a offset, u8 key); +#define nfaExecMcSheng64_16_B_Reverse NFA_API_NO_IMPL +#define nfaExecMcSheng64_16_zombie_status NFA_API_ZOMBIE_NO_IMPL +#else // !HAVE_AVX512VBMI +#define nfaExecMcSheng64_8_B_Reverse NFA_API_NO_IMPL +#define nfaExecMcSheng64_8_zombie_status NFA_API_ZOMBIE_NO_IMPL +#define nfaExecMcSheng64_8_Q NFA_API_NO_IMPL +#define nfaExecMcSheng64_8_Q2 NFA_API_NO_IMPL +#define nfaExecMcSheng64_8_QR NFA_API_NO_IMPL +#define nfaExecMcSheng64_8_inAccept NFA_API_NO_IMPL +#define nfaExecMcSheng64_8_inAnyAccept NFA_API_NO_IMPL +#define nfaExecMcSheng64_8_queueInitState NFA_API_NO_IMPL +#define nfaExecMcSheng64_8_queueCompressState NFA_API_NO_IMPL +#define nfaExecMcSheng64_8_expandState NFA_API_NO_IMPL +#define nfaExecMcSheng64_8_initCompressedState NFA_API_NO_IMPL +#define nfaExecMcSheng64_8_testEOD NFA_API_NO_IMPL +#define nfaExecMcSheng64_8_reportCurrent NFA_API_NO_IMPL + +#define nfaExecMcSheng64_16_B_Reverse NFA_API_NO_IMPL +#define nfaExecMcSheng64_16_zombie_status NFA_API_ZOMBIE_NO_IMPL +#define nfaExecMcSheng64_16_Q NFA_API_NO_IMPL +#define nfaExecMcSheng64_16_Q2 NFA_API_NO_IMPL +#define nfaExecMcSheng64_16_QR NFA_API_NO_IMPL +#define nfaExecMcSheng64_16_inAccept NFA_API_NO_IMPL +#define nfaExecMcSheng64_16_inAnyAccept NFA_API_NO_IMPL +#define nfaExecMcSheng64_16_queueInitState NFA_API_NO_IMPL +#define nfaExecMcSheng64_16_queueCompressState NFA_API_NO_IMPL +#define nfaExecMcSheng64_16_expandState NFA_API_NO_IMPL +#define nfaExecMcSheng64_16_initCompressedState NFA_API_NO_IMPL +#define nfaExecMcSheng64_16_testEOD NFA_API_NO_IMPL +#define nfaExecMcSheng64_16_reportCurrent NFA_API_NO_IMPL + +#endif //end of HAVE_AVX512VBMI #endif diff --git a/src/nfa/mcsheng_compile.cpp b/src/nfa/mcsheng_compile.cpp index 871ca4fb1..fb75e49a3 100644 --- a/src/nfa/mcsheng_compile.cpp +++ b/src/nfa/mcsheng_compile.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -64,7 +64,6 @@ #include #include #include - #include using namespace std; @@ -244,6 +243,106 @@ void populateBasicInfo(size_t state_size, const dfa_info &info, } } +static +mstate_aux *getAux64(NFA *n, dstate_id_t i) { + mcsheng64 *m = (mcsheng64 *)getMutableImplNfa(n); + mstate_aux *aux_base = (mstate_aux *)((char *)n + m->aux_offset); + + mstate_aux *aux = aux_base + i; + assert((const char *)aux < (const char *)n + m->length); + return aux; +} + +static +void createShuffleMasks64(mcsheng64 *m, const dfa_info &info, + dstate_id_t sheng_end, + const map &accel_escape_info) { + DEBUG_PRINTF("using first %hu states for a sheng\n", sheng_end); + assert(sheng_end > DEAD_STATE + 1); + assert(sheng_end <= sizeof(m512) + 1); + vector> masks; + masks.resize(info.alpha_size); + /* -1 to avoid wasting a slot as we do not include dead state */ + vector raw_ids; + raw_ids.resize(sheng_end - 1); + for (dstate_id_t s = DEAD_STATE + 1; s < info.states.size(); s++) { + assert(info.implId(s)); /* should not map to DEAD_STATE */ + if (info.is_sheng(s)) { + raw_ids[info.extra[s].sheng_id] = s; + } + } + for (u32 i = 0; i < info.alpha_size; i++) { + if (i == info.alpha_remap[TOP]) { + continue; + } + auto &mask = masks[i]; + assert(sizeof(mask) == sizeof(m512)); + mask.fill(0); + + for (dstate_id_t sheng_id = 0; sheng_id < sheng_end - 1; sheng_id++) { + dstate_id_t raw_id = raw_ids[sheng_id]; + dstate_id_t next_id = info.implId(info.states[raw_id].next[i]); + if (next_id == DEAD_STATE) { + next_id = sheng_end - 1; + } else if (next_id < sheng_end) { + next_id--; + } + DEBUG_PRINTF("%hu: %u->next %hu\n", sheng_id, i, next_id); + mask[sheng_id] = verify_u8(next_id); + } + } + for (u32 i = 0; i < N_CHARS; i++) { + assert(info.alpha_remap[i] != info.alpha_remap[TOP]); + memcpy((u8 *)&m->sheng_succ_masks[i], + (u8 *)masks[info.alpha_remap[i]].data(), sizeof(m512)); + } + m->sheng_end = sheng_end; + m->sheng_accel_limit = sheng_end - 1; + + for (dstate_id_t s : raw_ids) { + if (contains(accel_escape_info, s)) { + LIMIT_TO_AT_MOST(&m->sheng_accel_limit, info.extra[s].sheng_id); + } + } +} + +static +void populateBasicInfo64(size_t state_size, const dfa_info &info, + u32 total_size, u32 aux_offset, u32 accel_offset, + u32 accel_count, ReportID arb, bool single, NFA *nfa) { + assert(state_size == sizeof(u16) || state_size == sizeof(u8)); + + nfa->length = total_size; + nfa->nPositions = info.states.size(); + + nfa->scratchStateSize = verify_u32(state_size); + nfa->streamStateSize = verify_u32(state_size); + + if (state_size == sizeof(u8)) { + nfa->type = MCSHENG_64_NFA_8; + } else { + nfa->type = MCSHENG_64_NFA_16; + } + + mcsheng64 *m = (mcsheng64 *)getMutableImplNfa(nfa); + for (u32 i = 0; i < 256; i++) { + m->remap[i] = verify_u8(info.alpha_remap[i]); + } + m->alphaShift = info.getAlphaShift(); + m->length = total_size; + m->aux_offset = aux_offset; + m->accel_offset = accel_offset; + m->arb_report = arb; + m->state_count = verify_u16(info.size()); + m->start_anchored = info.implId(info.raw.start_anchored); + m->start_floating = info.implId(info.raw.start_floating); + m->has_accel = accel_count ? 1 : 0; + + if (single) { + m->flags |= MCSHENG_FLAG_SINGLE; + } +} + static size_t calcShermanRegionSize(const dfa_info &info) { size_t rv = 0; @@ -272,7 +371,7 @@ void fillInAux(mstate_aux *aux, dstate_id_t i, const dfa_info &info, /* returns false on error */ static bool allocateImplId16(dfa_info &info, dstate_id_t sheng_end, - dstate_id_t *sherman_base) { + dstate_id_t *sherman_base) { info.states[0].impl_id = 0; /* dead is always 0 */ vector norm; @@ -382,6 +481,7 @@ CharReach get_edge_reach(dstate_id_t u, dstate_id_t v, const dfa_info &info) { } #define MAX_SHENG_STATES 16 +#define MAX_SHENG64_STATES 64 #define MAX_SHENG_LEAKINESS 0.05 using LeakinessCache = ue2_unordered_map, double>; @@ -435,7 +535,8 @@ double leakiness(const RdfaGraph &g, dfa_info &info, static dstate_id_t find_sheng_states(dfa_info &info, - map &accel_escape_info) { + map &accel_escape_info, + size_t max_sheng_states) { RdfaGraph g(info.raw); auto cyclics = find_vertices_in_cycles(g); @@ -470,7 +571,7 @@ dstate_id_t find_sheng_states(dfa_info &info, flat_set considered = { DEAD_STATE }; bool seen_back_edge = false; while (!to_consider.empty() - && sheng_states.size() < MAX_SHENG_STATES) { + && sheng_states.size() < max_sheng_states) { auto v = to_consider.front(); to_consider.pop_front(); if (!considered.insert(g[v].index).second) { @@ -616,6 +717,80 @@ void fill_in_succ_table_16(NFA *nfa, const dfa_info &info, } } +static +void fill_in_aux_info64(NFA *nfa, const dfa_info &info, + const map &accel_escape_info, + u32 accel_offset, UNUSED u32 accel_end_offset, + const vector &reports, + const vector &reports_eod, + u32 report_base_offset, + const raw_report_info &ri) { + mcsheng64 *m = (mcsheng64 *)getMutableImplNfa(nfa); + + vector reportOffsets; + + ri.fillReportLists(nfa, report_base_offset, reportOffsets); + + for (u32 i = 0; i < info.size(); i++) { + u16 impl_id = info.implId(i); + mstate_aux *this_aux = getAux64(nfa, impl_id); + + fillInAux(this_aux, i, info, reports, reports_eod, reportOffsets); + if (contains(accel_escape_info, i)) { + this_aux->accel_offset = accel_offset; + accel_offset += info.strat.accelSize(); + assert(accel_offset <= accel_end_offset); + assert(ISALIGNED_N(accel_offset, alignof(union AccelAux))); + info.strat.buildAccel(i, accel_escape_info.at(i), + (void *)((char *)m + this_aux->accel_offset)); + } + } +} + +static +u16 get_edge_flags64(NFA *nfa, dstate_id_t target_impl_id) { + mstate_aux *aux = getAux64(nfa, target_impl_id); + u16 flags = 0; + + if (aux->accept) { + flags |= ACCEPT_FLAG; + } + + if (aux->accel_offset) { + flags |= ACCEL_FLAG; + } + + return flags; +} + +static +void fill_in_succ_table_64_16(NFA *nfa, const dfa_info &info, + dstate_id_t sheng_end, + UNUSED dstate_id_t sherman_base) { + u16 *succ_table = (u16 *)((char *)nfa + sizeof(NFA) + sizeof(mcsheng64)); + + u8 alphaShift = info.getAlphaShift(); + assert(alphaShift <= 8); + + for (size_t i = 0; i < info.size(); i++) { + if (!info.is_normal(i)) { + assert(info.implId(i) < sheng_end || info.is_sherman(i)); + continue; + } + + assert(info.implId(i) < sherman_base); + u16 normal_id = verify_u16(info.implId(i) - sheng_end); + + for (size_t s = 0; s < info.impl_alpha_size; s++) { + dstate_id_t raw_succ = info.states[i].next[s]; + u16 &entry = succ_table[((size_t)normal_id << alphaShift) + s]; + + entry = info.implId(raw_succ); + entry |= get_edge_flags64(nfa, entry); + } + } +} + #define MAX_SHERMAN_LIST_LEN 8 static @@ -842,17 +1017,20 @@ bytecode_ptr mcshengCompile16(dfa_info &info, dstate_id_t sheng_end, assert(info.getAlphaShift() <= 8); - u16 total_daddy = 0; - for (u32 i = 0; i < info.size(); i++) { - find_better_daddy(info, i, - is_cyclic_near(info.raw, info.raw.start_anchored), - grey); - total_daddy += info.extra[i].daddytaken; - } + // Sherman optimization + if (info.impl_alpha_size > 16) { + u16 total_daddy = 0; + for (u32 i = 0; i < info.size(); i++) { + find_better_daddy(info, i, + is_cyclic_near(info.raw, info.raw.start_anchored), + grey); + total_daddy += info.extra[i].daddytaken; + } - DEBUG_PRINTF("daddy %hu/%zu states=%zu alpha=%hu\n", total_daddy, - info.size() * info.impl_alpha_size, info.size(), - info.impl_alpha_size); + DEBUG_PRINTF("daddy %hu/%zu states=%zu alpha=%hu\n", total_daddy, + info.size() * info.impl_alpha_size, info.size(), + info.impl_alpha_size); + } u16 sherman_limit; if (!allocateImplId16(info, sheng_end, &sherman_limit)) { @@ -931,6 +1109,160 @@ void fill_in_succ_table_8(NFA *nfa, const dfa_info &info, } } +static +void fill_in_sherman64(NFA *nfa, dfa_info &info, UNUSED u16 sherman_limit) { + char *nfa_base = (char *)nfa; + mcsheng64 *m = (mcsheng64 *)getMutableImplNfa(nfa); + char *sherman_table = nfa_base + m->sherman_offset; + + assert(ISALIGNED_16(sherman_table)); + for (size_t i = 0; i < info.size(); i++) { + if (!info.is_sherman(i)) { + continue; + } + u16 fs = verify_u16(info.implId(i)); + DEBUG_PRINTF("building sherman %zu impl %hu\n", i, fs); + + assert(fs >= sherman_limit); + + char *curr_sherman_entry + = sherman_table + (fs - m->sherman_limit) * SHERMAN_FIXED_SIZE; + assert(curr_sherman_entry <= nfa_base + m->length); + + u8 len = verify_u8(info.impl_alpha_size - info.extra[i].daddytaken); + assert(len <= 9); + dstate_id_t d = info.states[i].daddy; + + *(u8 *)(curr_sherman_entry + SHERMAN_TYPE_OFFSET) = SHERMAN_STATE; + *(u8 *)(curr_sherman_entry + SHERMAN_LEN_OFFSET) = len; + *(u16 *)(curr_sherman_entry + SHERMAN_DADDY_OFFSET) = info.implId(d); + u8 *chars = (u8 *)(curr_sherman_entry + SHERMAN_CHARS_OFFSET); + + for (u16 s = 0; s < info.impl_alpha_size; s++) { + if (info.states[i].next[s] != info.states[d].next[s]) { + *(chars++) = (u8)s; + } + } + + u16 *states = (u16 *)(curr_sherman_entry + SHERMAN_STATES_OFFSET(len)); + for (u16 s = 0; s < info.impl_alpha_size; s++) { + if (info.states[i].next[s] != info.states[d].next[s]) { + DEBUG_PRINTF("s overrider %hu dad %hu char next %hu\n", fs, + info.implId(d), + info.implId(info.states[i].next[s])); + u16 entry_val = info.implId(info.states[i].next[s]); + entry_val |= get_edge_flags64(nfa, entry_val); + unaligned_store_u16((u8 *)states++, entry_val); + } + } + } +} + +static +bytecode_ptr mcsheng64Compile16(dfa_info&info, dstate_id_t sheng_end, + const map&accel_escape_info, + const Grey &grey) { + DEBUG_PRINTF("building mcsheng 64-16\n"); + + vector reports; /* index in ri for the appropriate report list */ + vector reports_eod; /* as above */ + ReportID arb; + u8 single; + + assert(info.getAlphaShift() <= 8); + + // Sherman optimization + if (info.impl_alpha_size > 16) { + u16 total_daddy = 0; + for (u32 i = 0; i < info.size(); i++) { + find_better_daddy(info, i, + is_cyclic_near(info.raw, info.raw.start_anchored), + grey); + total_daddy += info.extra[i].daddytaken; + } + + DEBUG_PRINTF("daddy %hu/%zu states=%zu alpha=%hu\n", total_daddy, + info.size() * info.impl_alpha_size, info.size(), + info.impl_alpha_size); + } + + u16 sherman_limit; + if (!allocateImplId16(info, sheng_end, &sherman_limit)) { + DEBUG_PRINTF("failed to allocate state numbers, %zu states total\n", + info.size()); + return nullptr; + } + u16 count_real_states = sherman_limit - sheng_end; + + auto ri = info.strat.gatherReports(reports, reports_eod, &single, &arb); + + size_t tran_size = (1 << info.getAlphaShift()) * sizeof(u16) + * count_real_states; + + size_t aux_size = sizeof(mstate_aux) * info.size(); + + size_t aux_offset = ROUNDUP_16(sizeof(NFA) + sizeof(mcsheng64) + tran_size); + size_t accel_size = info.strat.accelSize() * accel_escape_info.size(); + size_t accel_offset = ROUNDUP_N(aux_offset + aux_size + + ri->getReportListSize(), 32); + size_t sherman_offset = ROUNDUP_16(accel_offset + accel_size); + size_t sherman_size = calcShermanRegionSize(info); + + size_t total_size = sherman_offset + sherman_size; + + accel_offset -= sizeof(NFA); /* adj accel offset to be relative to m */ + assert(ISALIGNED_N(accel_offset, alignof(union AccelAux))); + + auto nfa = make_zeroed_bytecode_ptr(total_size); + mcsheng64 *m = (mcsheng64 *)getMutableImplNfa(nfa.get()); + + populateBasicInfo64(sizeof(u16), info, total_size, aux_offset, accel_offset, + accel_escape_info.size(), arb, single, nfa.get()); + createShuffleMasks64(m, info, sheng_end, accel_escape_info); + + /* copy in the mc header information */ + m->sherman_offset = sherman_offset; + m->sherman_end = total_size; + m->sherman_limit = sherman_limit; + + DEBUG_PRINTF("%hu sheng, %hu norm, %zu total\n", sheng_end, + count_real_states, info.size()); + + fill_in_aux_info64(nfa.get(), info, accel_escape_info, accel_offset, + sherman_offset - sizeof(NFA), reports, reports_eod, + aux_offset + aux_size, *ri); + + fill_in_succ_table_64_16(nfa.get(), info, sheng_end, sherman_limit); + + fill_in_sherman64(nfa.get(), info, sherman_limit); + + return nfa; +} + +static +void fill_in_succ_table_64_8(NFA *nfa, const dfa_info &info, + dstate_id_t sheng_end) { + u8 *succ_table = (u8 *)nfa + sizeof(NFA) + sizeof(mcsheng64); + + u8 alphaShift = info.getAlphaShift(); + assert(alphaShift <= 8); + + for (size_t i = 0; i < info.size(); i++) { + assert(!info.is_sherman(i)); + if (!info.is_normal(i)) { + assert(info.implId(i) < sheng_end); + continue; + } + u8 normal_id = verify_u8(info.implId(i) - sheng_end); + + for (size_t s = 0; s < info.impl_alpha_size; s++) { + dstate_id_t raw_succ = info.states[i].next[s]; + succ_table[((size_t)normal_id << alphaShift) + s] + = info.implId(raw_succ); + } + } +} + static void allocateImplId8(dfa_info &info, dstate_id_t sheng_end, const map &accel_escape_info, @@ -1028,6 +1360,58 @@ bytecode_ptr mcshengCompile8(dfa_info &info, dstate_id_t sheng_end, return nfa; } +static +bytecode_ptr mcsheng64Compile8(dfa_info &info, dstate_id_t sheng_end, + const map &accel_escape_info) { + DEBUG_PRINTF("building mcsheng 64-8\n"); + + vector reports; + vector reports_eod; + ReportID arb; + u8 single; + + auto ri = info.strat.gatherReports(reports, reports_eod, &single, &arb); + + size_t normal_count = info.size() - sheng_end; + + size_t tran_size = sizeof(u8) * (1 << info.getAlphaShift()) * normal_count; + size_t aux_size = sizeof(mstate_aux) * info.size(); + size_t aux_offset = ROUNDUP_16(sizeof(NFA) + sizeof(mcsheng64) + tran_size); + size_t accel_size = info.strat.accelSize() * accel_escape_info.size(); + size_t accel_offset = ROUNDUP_N(aux_offset + aux_size + + ri->getReportListSize(), 32); + size_t total_size = accel_offset + accel_size; + + DEBUG_PRINTF("aux_size %zu\n", aux_size); + DEBUG_PRINTF("aux_offset %zu\n", aux_offset); + DEBUG_PRINTF("rl size %u\n", ri->getReportListSize()); + DEBUG_PRINTF("accel_size %zu\n", accel_size); + DEBUG_PRINTF("accel_offset %zu\n", accel_offset); + DEBUG_PRINTF("total_size %zu\n", total_size); + + accel_offset -= sizeof(NFA); /* adj accel offset to be relative to m */ + assert(ISALIGNED_N(accel_offset, alignof(union AccelAux))); + + auto nfa = make_zeroed_bytecode_ptr(total_size); + mcsheng64 *m = (mcsheng64 *)getMutableImplNfa(nfa.get()); + + allocateImplId8(info, sheng_end, accel_escape_info, &m->accel_limit_8, + &m->accept_limit_8); + + populateBasicInfo64(sizeof(u8), info, total_size, aux_offset, accel_offset, + accel_escape_info.size(), arb, single, nfa.get()); + createShuffleMasks64(m, info, sheng_end, accel_escape_info); + + fill_in_aux_info64(nfa.get(), info, accel_escape_info, accel_offset, + total_size - sizeof(NFA), reports, reports_eod, + aux_offset + aux_size, *ri); + + fill_in_succ_table_64_8(nfa.get(), info, sheng_end); + DEBUG_PRINTF("rl size %zu\n", ri->size()); + + return nfa; +} + bytecode_ptr mcshengCompile(raw_dfa &raw, const CompileContext &cc, const ReportManager &rm) { if (!cc.grey.allowMcSheng) { @@ -1047,19 +1431,83 @@ bytecode_ptr mcshengCompile(raw_dfa &raw, const CompileContext &cc, map accel_escape_info = info.strat.getAccelInfo(cc.grey); + auto old_states = info.states; + dstate_id_t sheng_end = find_sheng_states(info, accel_escape_info, MAX_SHENG_STATES); - dstate_id_t sheng_end = find_sheng_states(info, accel_escape_info); if (sheng_end <= DEAD_STATE + 1) { + info.states = old_states; return nullptr; } bytecode_ptr nfa; + if (!using8bit) { nfa = mcshengCompile16(info, sheng_end, accel_escape_info, cc.grey); } else { nfa = mcshengCompile8(info, sheng_end, accel_escape_info); } + if (!nfa) { + info.states = old_states; + return nfa; + } + + if (has_eod_reports) { + nfa->flags |= NFA_ACCEPTS_EOD; + } + + DEBUG_PRINTF("compile done\n"); + return nfa; +} + +bytecode_ptr mcshengCompile64(raw_dfa &raw, const CompileContext &cc, + const ReportManager &rm) { + if (!cc.grey.allowMcSheng) { + return nullptr; + } + + if (!cc.target_info.has_avx512vbmi()) { + DEBUG_PRINTF("McSheng64 failed, no HS_CPU_FEATURES_AVX512VBMI!\n"); + return nullptr; + } + + mcclellan_build_strat mbs(raw, rm, false); + dfa_info info(mbs); + bool using8bit = cc.grey.allowMcClellan8 && info.size() <= 256; + + if (!cc.streaming) { /* TODO: work out if we can do the strip in streaming + * mode with our semantics */ + raw.stripExtraEodReports(); + } + + bool has_eod_reports = raw.hasEodReports(); + + map accel_escape_info + = info.strat.getAccelInfo(cc.grey); + bool using64state = false; /*default flag*/ + dstate_id_t sheng_end64; + sheng_end64 = find_sheng_states(info, accel_escape_info, MAX_SHENG64_STATES); + + if (sheng_end64 <= DEAD_STATE + 1) { + return nullptr; + } else { + using64state = true; + } + + bytecode_ptr nfa; + + if (using64state) { + assert((sheng_end64 > 17) && (sheng_end64 <= 65)); + if (!using8bit) { + nfa = mcsheng64Compile16(info, sheng_end64, accel_escape_info, cc.grey); + } else { + assert(using8bit); + nfa = mcsheng64Compile8(info, sheng_end64, accel_escape_info); + assert(nfa); + assert(nfa->type == MCSHENG_64_NFA_8); + } + } + if (!nfa) { return nfa; } diff --git a/src/nfa/mcsheng_compile.h b/src/nfa/mcsheng_compile.h index 487ab45f4..3a79b46a2 100644 --- a/src/nfa/mcsheng_compile.h +++ b/src/nfa/mcsheng_compile.h @@ -42,7 +42,8 @@ struct raw_dfa; bytecode_ptr mcshengCompile(raw_dfa &raw, const CompileContext &cc, const ReportManager &rm); - +bytecode_ptr mcshengCompile64(raw_dfa &raw, const CompileContext &cc, + const ReportManager &rm); bool has_accel_mcsheng(const NFA *nfa); } // namespace ue2 diff --git a/src/nfa/mcsheng_data.c b/src/nfa/mcsheng_data.c index eaf3cbbb3..0701b4b31 100644 --- a/src/nfa/mcsheng_data.c +++ b/src/nfa/mcsheng_data.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,3 +41,15 @@ const u64a mcsheng_pext_mask[8] = { 0x00ff00000000000f, 0xff0000000000000f, }; +#if defined(HAVE_AVX512VBMI) +const u64a mcsheng64_pext_mask[8] = { + 0, /* dummy */ + 0x000000000000ff3f, + 0x0000000000ff003f, + 0x00000000ff00003f, + 0x000000ff0000003f, + 0x0000ff000000003f, + 0x00ff00000000003f, + 0xff0000000000003f, +}; +#endif diff --git a/src/nfa/mcsheng_dump.cpp b/src/nfa/mcsheng_dump.cpp index 2b5630799..7cef82f4d 100644 --- a/src/nfa/mcsheng_dump.cpp +++ b/src/nfa/mcsheng_dump.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -174,6 +174,124 @@ void describeEdge(FILE *f, const mcsheng *m, const u16 *t, u16 i) { } } +static +const mstate_aux *getAux64(const NFA *n, dstate_id_t i) { + auto *m = (const mcsheng64 *)getImplNfa(n); + auto *aux_base = (const mstate_aux *)((const char *)n + m->aux_offset); + + const mstate_aux *aux = aux_base + i; + + assert((const char *)aux < (const char *)n + m->length); + return aux; +} + +static +void next_states64(const NFA *n, u16 s, u16 *t) { + const mcsheng64 *m = (const mcsheng64 *)getImplNfa(n); + const mstate_aux *aux = getAux64(n, s); + const u32 as = m->alphaShift; + assert(s != DEAD_STATE); + + if (s < m->sheng_end) { + for (u16 c = 0; c < N_CHARS; c++) { + u8 sheng_s = s - 1; + auto trans_for_c = (const char *)&m->sheng_succ_masks[c]; + assert(sheng_s < sizeof(m512)); + u8 raw_succ = trans_for_c[sheng_s]; + if (raw_succ == m->sheng_end - 1) { + t[c] = DEAD_STATE; + } else if (raw_succ < m->sheng_end) { + t[c] = raw_succ + 1; + } else { + t[c] = raw_succ; + } + } + } else if (n->type == MCSHENG_64_NFA_8) { + const u8 *succ_table = (const u8 *)((const char *)m + sizeof(mcsheng64)); + for (u16 c = 0; c < N_CHARS; c++) { + u32 normal_id = s - m->sheng_end; + t[c] = succ_table[(normal_id << as) + m->remap[c]]; + } + } else { + u16 base_s = s; + const char *winfo_base = (const char *)n + m->sherman_offset; + const char *state_base + = winfo_base + SHERMAN_FIXED_SIZE * (s - m->sherman_limit); + + if (s >= m->sherman_limit) { + base_s = unaligned_load_u16(state_base + SHERMAN_DADDY_OFFSET); + assert(base_s >= m->sheng_end); + } + + const u16 *succ_table = (const u16 *)((const char *)m + + sizeof(mcsheng64)); + for (u16 c = 0; c < N_CHARS; c++) { + u32 normal_id = base_s - m->sheng_end; + t[c] = succ_table[(normal_id << as) + m->remap[c]]; + } + + if (s >= m->sherman_limit) { + UNUSED char type = *(state_base + SHERMAN_TYPE_OFFSET); + assert(type == SHERMAN_STATE); + u8 len = *(const u8 *)(SHERMAN_LEN_OFFSET + state_base); + const char *chars = state_base + SHERMAN_CHARS_OFFSET; + const u16 *states = (const u16 *)(state_base + + SHERMAN_STATES_OFFSET(len)); + + for (u8 i = 0; i < len; i++) { + for (u16 c = 0; c < N_CHARS; c++) { + if (m->remap[c] == chars[i]) { + t[c] = unaligned_load_u16((const u8*)&states[i]); + } + } + } + } + + for (u16 c = 0; c < N_CHARS; c++) { + t[c] &= STATE_MASK; + } + + } + + t[TOP] = aux->top & STATE_MASK; +} + +static +void describeEdge64(FILE *f, const mcsheng64 *m, const u16 *t, u16 i) { + for (u16 s = 0; s < N_CHARS; s++) { + if (!t[s]) { + continue; + } + + u16 ss; + for (ss = 0; ss < s; ss++) { + if (t[s] == t[ss]) { + break; + } + } + + if (ss != s) { + continue; + } + + CharReach reach; + for (ss = s; ss < 256; ss++) { + if (t[s] == t[ss]) { + reach.set(ss); + } + } + + fprintf(f, "%u -> %u [ ", i, t[s]); + if (i < m->sheng_end && t[s] < m->sheng_end) { + fprintf(f, "color = red, fontcolor = red "); + } + fprintf(f, "label = \""); + describeClass(f, reach, 5, CC_OUT_DOT); + + fprintf(f, "\" ];\n"); + } +} + static void dumpAccelDot(FILE *f, u16 i, const union AccelAux *accel) { switch(accel->accel_type) { @@ -256,6 +374,66 @@ void describeNode(const NFA *n, const mcsheng *m, u16 i, FILE *f) { } +static +void describeNode64(const NFA *n, const mcsheng64 *m, u16 i, FILE *f) { + const mstate_aux *aux = getAux64(n, i); + + bool isSherman = m->sherman_limit && i >= m->sherman_limit; + + fprintf(f, "%u [ width = 1, fixedsize = true, fontsize = 12, " + "label = \"%u%s\" ]; \n", i, i, isSherman ? "w":""); + + if (aux->accel_offset) { + dumpAccelDot(f, i, (const union AccelAux *) + ((const char *)m + aux->accel_offset)); + } + + if (i && i < m->sheng_end) { + fprintf(f, "%u [color = red, fontcolor = red]; \n", i); + } + + if (aux->accept_eod) { + fprintf(f, "%u [ color = darkorchid ];\n", i); + } + + if (aux->accept) { + fprintf(f, "%u [ shape = doublecircle ];\n", i); + } + + if (aux->top && aux->top != i) { + fprintf(f, "%u -> %u [color = darkgoldenrod weight=0.1 ]\n", i, + aux->top); + } + + if (i == m->start_anchored) { + fprintf(f, "STARTA -> %u [color = blue ]\n", i); + } + + if (i == m->start_floating) { + fprintf(f, "STARTF -> %u [color = red ]\n", i); + } + + if (isSherman) { + const char *winfo_base = (const char *)n + m->sherman_offset; + const char *state_base + = winfo_base + SHERMAN_FIXED_SIZE * (i - m->sherman_limit); + assert(state_base < (const char *)m + m->length - sizeof(NFA)); + UNUSED u8 type = *(const u8 *)(state_base + SHERMAN_TYPE_OFFSET); + assert(type == SHERMAN_STATE); + fprintf(f, "%u [ fillcolor = lightblue style=filled ];\n", i); + u16 daddy = *(const u16 *)(state_base + SHERMAN_DADDY_OFFSET); + if (daddy) { + fprintf(f, "%u -> %u [ color=royalblue style=dashed weight=0.1]\n", + i, daddy); + } + } + + if (i && i < m->sheng_end) { + fprintf(f, "subgraph cluster_sheng { %u } \n", i); + } + +} + static void dumpDotPreambleDfa(FILE *f) { dumpDotPreamble(f); @@ -392,6 +570,131 @@ void dump_text_8(const NFA *nfa, FILE *f) { dumpTextReverse(nfa, f); } +static +void dump64_dot_16(const NFA *nfa, FILE *f) { + auto *m = (const mcsheng64 *)getImplNfa(nfa); + + dumpDotPreambleDfa(f); + + for (u16 i = 1; i < m->state_count; i++) { + describeNode64(nfa, m, i, f); + + u16 t[ALPHABET_SIZE]; + + next_states64(nfa, i, t); + + describeEdge64(f, m, t, i); + } + + fprintf(f, "}\n"); +} + +static +void dump64_dot_8(const NFA *nfa, FILE *f) { + auto m = (const mcsheng64 *)getImplNfa(nfa); + + dumpDotPreambleDfa(f); + + for (u16 i = 1; i < m->state_count; i++) { + describeNode64(nfa, m, i, f); + + u16 t[ALPHABET_SIZE]; + + next_states64(nfa, i, t); + + describeEdge64(f, m, t, i); + } + + fprintf(f, "}\n"); +} + +static +void dumpAccelMasks64(FILE *f, const mcsheng64 *m, const mstate_aux *aux) { + fprintf(f, "\n"); + fprintf(f, "Acceleration\n"); + fprintf(f, "------------\n"); + + for (u16 i = 0; i < m->state_count; i++) { + if (!aux[i].accel_offset) { + continue; + } + + auto accel = (const AccelAux *)((const char *)m + aux[i].accel_offset); + fprintf(f, "%05hu ", i); + dumpAccelInfo(f, *accel); + } +} + +static +void describeAlphabet64(FILE *f, const mcsheng64 *m) { + map rev; + + for (u16 i = 0; i < N_CHARS; i++) { + rev[m->remap[i]].clear(); + } + + for (u16 i = 0; i < N_CHARS; i++) { + rev[m->remap[i]].set(i); + } + + map::const_iterator it; + fprintf(f, "\nAlphabet\n"); + for (it = rev.begin(); it != rev.end(); ++it) { + fprintf(f, "%3hhu: ", it->first); + describeClass(f, it->second, 10240, CC_OUT_TEXT); + fprintf(f, "\n"); + } + fprintf(f, "\n"); +} + +static +void dumpCommonHeader64(FILE *f, const mcsheng64 *m) { + fprintf(f, "report: %u, states: %u, length: %u\n", m->arb_report, + m->state_count, m->length); + fprintf(f, "astart: %hu, fstart: %hu\n", m->start_anchored, + m->start_floating); + fprintf(f, "single accept: %d, has_accel: %d\n", + !!(int)m->flags & MCSHENG_FLAG_SINGLE, m->has_accel); + fprintf(f, "sheng_end: %hu\n", m->sheng_end); + fprintf(f, "sheng_accel_limit: %hu\n", m->sheng_accel_limit); +} + +static +void dump64_text_8(const NFA *nfa, FILE *f) { + auto m = (const mcsheng64 *)getImplNfa(nfa); + auto aux = (const mstate_aux *)((const char *)nfa + m->aux_offset); + + fprintf(f, "mcsheng 64-8\n"); + dumpCommonHeader64(f, m); + fprintf(f, "accel_limit: %hu, accept_limit %hu\n", m->accel_limit_8, + m->accept_limit_8); + fprintf(f, "\n"); + + describeAlphabet64(f, m); + dumpAccelMasks64(f, m, aux); + + fprintf(f, "\n"); + dumpTextReverse(nfa, f); +} + +static +void dump64_text_16(const NFA *nfa, FILE *f) { + auto *m = (const mcsheng64 *)getImplNfa(nfa); + auto *aux = (const mstate_aux *)((const char *)nfa + m->aux_offset); + + fprintf(f, "mcsheng 64-16\n"); + dumpCommonHeader64(f, m); + fprintf(f, "sherman_limit: %d, sherman_end: %d\n", (int)m->sherman_limit, + (int)m->sherman_end); + fprintf(f, "\n"); + + describeAlphabet64(f, m); + dumpAccelMasks64(f, m, aux); + + fprintf(f, "\n"); + dumpTextReverse(nfa, f); +} + void nfaExecMcSheng16_dump(const NFA *nfa, const string &base) { assert(nfa->type == MCSHENG_NFA_16); dump_text_16(nfa, StdioFile(base + ".txt", "w")); @@ -404,4 +707,16 @@ void nfaExecMcSheng8_dump(const NFA *nfa, const string &base) { dump_dot_8(nfa, StdioFile(base + ".dot", "w")); } +void nfaExecMcSheng64_16_dump(UNUSED const NFA *nfa, UNUSED const string &base) { + assert(nfa->type == MCSHENG_64_NFA_16); + dump64_text_16(nfa, StdioFile(base + ".txt", "w")); + dump64_dot_16(nfa, StdioFile(base + ".dot", "w")); +} + +void nfaExecMcSheng64_8_dump(UNUSED const NFA *nfa, UNUSED const string &base) { + assert(nfa->type == MCSHENG_64_NFA_8); + dump64_text_8(nfa, StdioFile(base + ".txt", "w")); + dump64_dot_8(nfa, StdioFile(base + ".dot", "w")); +} + } // namespace ue2 diff --git a/src/nfa/mcsheng_dump.h b/src/nfa/mcsheng_dump.h index 1b6993674..26e6cfda7 100644 --- a/src/nfa/mcsheng_dump.h +++ b/src/nfa/mcsheng_dump.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -42,7 +42,8 @@ namespace ue2 { void nfaExecMcSheng8_dump(const struct NFA *nfa, const std::string &base); void nfaExecMcSheng16_dump(const struct NFA *nfa, const std::string &base); - +void nfaExecMcSheng64_8_dump(const struct NFA *nfa, const std::string &base); +void nfaExecMcSheng64_16_dump(const struct NFA *nfa, const std::string &base); } // namespace ue2 #endif // DUMP_SUPPORT diff --git a/src/nfa/mcsheng_internal.h b/src/nfa/mcsheng_internal.h index bb45ae23f..d98557462 100644 --- a/src/nfa/mcsheng_internal.h +++ b/src/nfa/mcsheng_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2018, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -92,4 +92,33 @@ struct mcsheng { * representing the data from a u64a. */ extern const u64a mcsheng_pext_mask[8]; +struct mcsheng64 { + u16 state_count; /**< total number of states */ + u32 length; /**< length of dfa in bytes */ + u16 start_anchored; /**< anchored start state */ + u16 start_floating; /**< floating start state */ + u32 aux_offset; /**< offset of the aux structures relative to the start of + * the nfa structure */ + u32 sherman_offset; /**< offset of array of sherman state offsets the + * state_info structures relative to the start of the + * nfa structure */ + u32 sherman_end; /**< offset of the end of the state_info structures + * relative to the start of the nfa structure */ + u16 sheng_end; /**< first non-sheng state */ + u16 sheng_accel_limit; /**< first sheng accel state. state given in terms of + * internal sheng ids */ + u16 accel_limit_8; /**< 8 bit, lowest accelerable state */ + u16 accept_limit_8; /**< 8 bit, lowest accept state */ + u16 sherman_limit; /**< lowest sherman state */ + u8 alphaShift; + u8 flags; + u8 has_accel; /**< 1 iff there are any accel plans */ + u8 remap[256]; /**< remaps characters to a smaller alphabet */ + ReportID arb_report; /**< one of the accepts that this dfa may raise */ + u32 accel_offset; /**< offset of accel structures from start of McClellan */ + m512 sheng_succ_masks[N_CHARS]; +}; + +extern const u64a mcsheng64_pext_mask[8]; + #endif diff --git a/src/nfa/nfa_api_dispatch.c b/src/nfa/nfa_api_dispatch.c index f4b7552ef..75cac4b48 100644 --- a/src/nfa/nfa_api_dispatch.c +++ b/src/nfa/nfa_api_dispatch.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -76,6 +76,10 @@ DISPATCH_CASE(TAMARAMA_NFA, Tamarama, dbnt_func); \ DISPATCH_CASE(MCSHENG_NFA_8, McSheng8, dbnt_func); \ DISPATCH_CASE(MCSHENG_NFA_16, McSheng16, dbnt_func); \ + DISPATCH_CASE(SHENG_NFA_32, Sheng32, dbnt_func); \ + DISPATCH_CASE(SHENG_NFA_64, Sheng64, dbnt_func); \ + DISPATCH_CASE(MCSHENG_64_NFA_8, McSheng64_8, dbnt_func); \ + DISPATCH_CASE(MCSHENG_64_NFA_16, McSheng64_16, dbnt_func); \ default: \ assert(0); \ } diff --git a/src/nfa/nfa_build_util.cpp b/src/nfa/nfa_build_util.cpp index 9185ccdd7..47153163e 100644 --- a/src/nfa/nfa_build_util.cpp +++ b/src/nfa/nfa_build_util.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -181,7 +181,6 @@ enum NFACategory {NFA_LIMEX, NFA_OTHER}; static const nfa_dispatch_fn has_repeats_other_than_firsts; \ static const u32 stateAlign = \ MAX(mlt_align, alignof(RepeatControl)); \ - static const bool fast = mlt_size <= 64; \ }; \ const nfa_dispatch_fn NFATraits::has_accel \ = has_accel_limex; \ @@ -210,7 +209,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 1; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -226,7 +224,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 2; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -242,7 +239,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 8; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -258,7 +254,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 8; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -274,7 +269,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 8; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -290,7 +284,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 8; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -306,7 +299,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 8; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -322,7 +314,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 8; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -338,7 +329,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 8; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -354,7 +344,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 8; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -370,7 +359,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 8; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -386,7 +374,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 1; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -402,7 +389,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 64; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -418,7 +404,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 1; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -434,7 +419,6 @@ template<> struct NFATraits { UNUSED static const char *name; static const NFACategory category = NFA_OTHER; static const u32 stateAlign = 2; - static const bool fast = true; static const nfa_dispatch_fn has_accel; static const nfa_dispatch_fn has_repeats; static const nfa_dispatch_fn has_repeats_other_than_firsts; @@ -446,6 +430,65 @@ const nfa_dispatch_fn NFATraits::has_repeats_other_than_firsts = const char *NFATraits::name = "Shengy McShengFace 16"; #endif +template<> struct NFATraits { + UNUSED static const char *name; + static const NFACategory category = NFA_OTHER; + static const u32 stateAlign = 1; + static const nfa_dispatch_fn has_accel; + static const nfa_dispatch_fn has_repeats; + static const nfa_dispatch_fn has_repeats_other_than_firsts; +}; +const nfa_dispatch_fn NFATraits::has_accel = has_accel_sheng; +const nfa_dispatch_fn NFATraits::has_repeats = dispatch_false; +const nfa_dispatch_fn NFATraits::has_repeats_other_than_firsts = dispatch_false; +#if defined(DUMP_SUPPORT) +const char *NFATraits::name = "Sheng 32"; +#endif + +template<> struct NFATraits { + UNUSED static const char *name; + static const NFACategory category = NFA_OTHER; + static const u32 stateAlign = 1; + static const nfa_dispatch_fn has_accel; + static const nfa_dispatch_fn has_repeats; + static const nfa_dispatch_fn has_repeats_other_than_firsts; +}; +const nfa_dispatch_fn NFATraits::has_accel = has_accel_sheng; +const nfa_dispatch_fn NFATraits::has_repeats = dispatch_false; +const nfa_dispatch_fn NFATraits::has_repeats_other_than_firsts = dispatch_false; +#if defined(DUMP_SUPPORT) +const char *NFATraits::name = "Sheng 64"; +#endif + +template<> struct NFATraits { + UNUSED static const char *name; + static const NFACategory category = NFA_OTHER; + static const u32 stateAlign = 1; + static const nfa_dispatch_fn has_accel; + static const nfa_dispatch_fn has_repeats; + static const nfa_dispatch_fn has_repeats_other_than_firsts; +}; +const nfa_dispatch_fn NFATraits::has_accel = has_accel_mcsheng; +const nfa_dispatch_fn NFATraits::has_repeats = dispatch_false; +const nfa_dispatch_fn NFATraits::has_repeats_other_than_firsts = dispatch_false; +#if defined(DUMP_SUPPORT) +const char *NFATraits::name = "Shengy64 McShengFace 8"; +#endif + +template<> struct NFATraits { + UNUSED static const char *name; + static const NFACategory category = NFA_OTHER; + static const u32 stateAlign = 2; + static const nfa_dispatch_fn has_accel; + static const nfa_dispatch_fn has_repeats; + static const nfa_dispatch_fn has_repeats_other_than_firsts; +}; +const nfa_dispatch_fn NFATraits::has_accel = has_accel_mcsheng; +const nfa_dispatch_fn NFATraits::has_repeats = dispatch_false; +const nfa_dispatch_fn NFATraits::has_repeats_other_than_firsts = dispatch_false; +#if defined(DUMP_SUPPORT) +const char *NFATraits::name = "Shengy64 McShengFace 16"; +#endif } // namespace #if defined(DUMP_SUPPORT) @@ -473,20 +516,6 @@ u32 state_alignment(const NFA &nfa) { return DISPATCH_BY_NFA_TYPE((NFAEngineType)nfa.type, getStateAlign, nullptr); } -namespace { -template -struct getFastness { - static u32 call(void *) { - return NFATraits::fast; - } -}; -} - -bool is_fast(const NFA &nfa) { - NFAEngineType t = (NFAEngineType)nfa.type; - return DISPATCH_BY_NFA_TYPE(t, getFastness, nullptr); -} - namespace { template struct is_limex { diff --git a/src/nfa/nfa_build_util.h b/src/nfa/nfa_build_util.h index 92a1091ec..ee7a30949 100644 --- a/src/nfa/nfa_build_util.h +++ b/src/nfa/nfa_build_util.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -47,10 +47,6 @@ std::string describe(const NFA &nfa); // For a given NFA, retrieve the alignment required by its uncompressed state. u32 state_alignment(const NFA &nfa); -/* returns true if the nfa is considered 'fast'. TODO: work out what we mean by - * fast. */ -bool is_fast(const NFA &n); - bool has_bounded_repeats_other_than_firsts(const NFA &n); bool has_bounded_repeats(const NFA &n); diff --git a/src/nfa/nfa_dump_dispatch.cpp b/src/nfa/nfa_dump_dispatch.cpp index 5607ed27a..bc8c175d3 100644 --- a/src/nfa/nfa_dump_dispatch.cpp +++ b/src/nfa/nfa_dump_dispatch.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -81,6 +81,10 @@ namespace ue2 { DISPATCH_CASE(TAMARAMA_NFA, Tamarama, dbnt_func); \ DISPATCH_CASE(MCSHENG_NFA_8, McSheng8, dbnt_func); \ DISPATCH_CASE(MCSHENG_NFA_16, McSheng16, dbnt_func); \ + DISPATCH_CASE(SHENG_NFA_32, Sheng32, dbnt_func); \ + DISPATCH_CASE(SHENG_NFA_64, Sheng64, dbnt_func); \ + DISPATCH_CASE(MCSHENG_64_NFA_8, McSheng64_8, dbnt_func); \ + DISPATCH_CASE(MCSHENG_64_NFA_16, McSheng64_16, dbnt_func); \ default: \ assert(0); \ } diff --git a/src/nfa/nfa_internal.h b/src/nfa/nfa_internal.h index 9d2808225..ad27e28b1 100644 --- a/src/nfa/nfa_internal.h +++ b/src/nfa/nfa_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -72,6 +72,10 @@ enum NFAEngineType { TAMARAMA_NFA, /**< magic nfa container */ MCSHENG_NFA_8, /**< magic pseudo nfa */ MCSHENG_NFA_16, /**< magic pseudo nfa */ + SHENG_NFA_32, /**< magic pseudo nfa */ + SHENG_NFA_64, /**< magic pseudo nfa */ + MCSHENG_64_NFA_8, /**< magic pseudo nfa */ + MCSHENG_64_NFA_16, /**< magic pseudo nfa */ /** \brief bogus NFA - not used */ INVALID_NFA }; @@ -148,7 +152,8 @@ static really_inline int isMcClellanType(u8 t) { /** \brief True if the given type (from NFA::type) is a Sheng-McClellan hybrid * DFA. */ static really_inline int isShengMcClellanType(u8 t) { - return t == MCSHENG_NFA_8 || t == MCSHENG_NFA_16; + return t == MCSHENG_NFA_8 || t == MCSHENG_NFA_16 || + t == MCSHENG_64_NFA_8 || t == MCSHENG_64_NFA_16; } /** \brief True if the given type (from NFA::type) is a Gough DFA. */ @@ -157,10 +162,25 @@ static really_inline int isGoughType(u8 t) { } /** \brief True if the given type (from NFA::type) is a Sheng DFA. */ -static really_inline int isShengType(u8 t) { +static really_inline int isSheng16Type(u8 t) { return t == SHENG_NFA; } +/** \brief True if the given type (from NFA::type) is a Sheng32 DFA. */ +static really_inline int isSheng32Type(u8 t) { + return t == SHENG_NFA_32; +} + +/** \brief True if the given type (from NFA::type) is a Sheng64 DFA. */ +static really_inline int isSheng64Type(u8 t) { + return t == SHENG_NFA_64; +} + +/** \brief True if the given type (from NFA::type) is a Sheng16/32/64 DFA. */ +static really_inline int isShengType(u8 t) { + return t == SHENG_NFA || t == SHENG_NFA_32 || t == SHENG_NFA_64; +} + /** * \brief True if the given type (from NFA::type) is a McClellan, Gough or * Sheng DFA. diff --git a/src/nfa/sheng.c b/src/nfa/sheng.c index 4f30910b5..3f36e2189 100644 --- a/src/nfa/sheng.c +++ b/src/nfa/sheng.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2018, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -154,6 +154,205 @@ char fireReports(const struct sheng *sh, NfaCallback cb, void *ctxt, return MO_CONTINUE_MATCHING; /* continue execution */ } +#if defined(HAVE_AVX512VBMI) +// Sheng32 +static really_inline +const struct sheng32 *get_sheng32(const struct NFA *n) { + return (const struct sheng32 *)getImplNfa(n); +} + +static really_inline +const struct sstate_aux *get_aux32(const struct sheng32 *sh, u8 id) { + u32 offset = sh->aux_offset - sizeof(struct NFA) + + (id & SHENG32_STATE_MASK) * sizeof(struct sstate_aux); + DEBUG_PRINTF("Getting aux for state %u at offset %llu\n", + id & SHENG32_STATE_MASK, (u64a)offset + sizeof(struct NFA)); + return (const struct sstate_aux *)((const char *) sh + offset); +} + +static really_inline +const union AccelAux *get_accel32(const struct sheng32 *sh, u8 id) { + const struct sstate_aux *saux = get_aux32(sh, id); + DEBUG_PRINTF("Getting accel aux at offset %u\n", saux->accel); + const union AccelAux *aux = (const union AccelAux *) + ((const char *)sh + saux->accel - sizeof(struct NFA)); + return aux; +} + +static really_inline +const struct report_list *get_rl32(const struct sheng32 *sh, + const struct sstate_aux *aux) { + DEBUG_PRINTF("Getting report list at offset %u\n", aux->accept); + return (const struct report_list *) + ((const char *)sh + aux->accept - sizeof(struct NFA)); +} + +static really_inline +const struct report_list *get_eod_rl32(const struct sheng32 *sh, + const struct sstate_aux *aux) { + DEBUG_PRINTF("Getting EOD report list at offset %u\n", aux->accept); + return (const struct report_list *) + ((const char *)sh + aux->accept_eod - sizeof(struct NFA)); +} + +static really_inline +char sheng32HasAccept(const struct sheng32 *sh, const struct sstate_aux *aux, + ReportID report) { + assert(sh && aux); + + const struct report_list *rl = get_rl32(sh, aux); + assert(ISALIGNED_N(rl, 4)); + + DEBUG_PRINTF("report list has %u entries\n", rl->count); + + for (u32 i = 0; i < rl->count; i++) { + if (rl->report[i] == report) { + DEBUG_PRINTF("reporting %u\n", rl->report[i]); + return 1; + } + } + + return 0; +} + +static really_inline +char fireReports32(const struct sheng32 *sh, NfaCallback cb, void *ctxt, + const u8 state, u64a loc, u8 *const cached_accept_state, + ReportID *const cached_accept_id, char eod) { + DEBUG_PRINTF("reporting matches @ %llu\n", loc); + + if (!eod && state == *cached_accept_state) { + DEBUG_PRINTF("reporting %u\n", *cached_accept_id); + if (cb(0, loc, *cached_accept_id, ctxt) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; /* termination requested */ + } + + return MO_CONTINUE_MATCHING; /* continue execution */ + } + const struct sstate_aux *aux = get_aux32(sh, state); + const struct report_list *rl = eod ? get_eod_rl32(sh, aux) : + get_rl32(sh, aux); + assert(ISALIGNED(rl)); + + DEBUG_PRINTF("report list has %u entries\n", rl->count); + u32 count = rl->count; + + if (!eod && count == 1) { + *cached_accept_state = state; + *cached_accept_id = rl->report[0]; + + DEBUG_PRINTF("reporting %u\n", rl->report[0]); + if (cb(0, loc, rl->report[0], ctxt) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; /* termination requested */ + } + + return MO_CONTINUE_MATCHING; /* continue execution */ + } + + for (u32 i = 0; i < count; i++) { + DEBUG_PRINTF("reporting %u\n", rl->report[i]); + if (cb(0, loc, rl->report[i], ctxt) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; /* termination requested */ + } + } + return MO_CONTINUE_MATCHING; /* continue execution */ +} + +// Sheng64 +static really_inline +const struct sheng64 *get_sheng64(const struct NFA *n) { + return (const struct sheng64 *)getImplNfa(n); +} + +static really_inline +const struct sstate_aux *get_aux64(const struct sheng64 *sh, u8 id) { + u32 offset = sh->aux_offset - sizeof(struct NFA) + + (id & SHENG64_STATE_MASK) * sizeof(struct sstate_aux); + DEBUG_PRINTF("Getting aux for state %u at offset %llu\n", + id & SHENG64_STATE_MASK, (u64a)offset + sizeof(struct NFA)); + return (const struct sstate_aux *)((const char *) sh + offset); +} + +static really_inline +const struct report_list *get_rl64(const struct sheng64 *sh, + const struct sstate_aux *aux) { + DEBUG_PRINTF("Getting report list at offset %u\n", aux->accept); + return (const struct report_list *) + ((const char *)sh + aux->accept - sizeof(struct NFA)); +} + +static really_inline +const struct report_list *get_eod_rl64(const struct sheng64 *sh, + const struct sstate_aux *aux) { + DEBUG_PRINTF("Getting EOD report list at offset %u\n", aux->accept); + return (const struct report_list *) + ((const char *)sh + aux->accept_eod - sizeof(struct NFA)); +} + +static really_inline +char sheng64HasAccept(const struct sheng64 *sh, const struct sstate_aux *aux, + ReportID report) { + assert(sh && aux); + + const struct report_list *rl = get_rl64(sh, aux); + assert(ISALIGNED_N(rl, 4)); + + DEBUG_PRINTF("report list has %u entries\n", rl->count); + + for (u32 i = 0; i < rl->count; i++) { + if (rl->report[i] == report) { + DEBUG_PRINTF("reporting %u\n", rl->report[i]); + return 1; + } + } + + return 0; +} + +static really_inline +char fireReports64(const struct sheng64 *sh, NfaCallback cb, void *ctxt, + const u8 state, u64a loc, u8 *const cached_accept_state, + ReportID *const cached_accept_id, char eod) { + DEBUG_PRINTF("reporting matches @ %llu\n", loc); + + if (!eod && state == *cached_accept_state) { + DEBUG_PRINTF("reporting %u\n", *cached_accept_id); + if (cb(0, loc, *cached_accept_id, ctxt) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; /* termination requested */ + } + + return MO_CONTINUE_MATCHING; /* continue execution */ + } + const struct sstate_aux *aux = get_aux64(sh, state); + const struct report_list *rl = eod ? get_eod_rl64(sh, aux) : + get_rl64(sh, aux); + assert(ISALIGNED(rl)); + + DEBUG_PRINTF("report list has %u entries\n", rl->count); + u32 count = rl->count; + + if (!eod && count == 1) { + *cached_accept_state = state; + *cached_accept_id = rl->report[0]; + + DEBUG_PRINTF("reporting %u\n", rl->report[0]); + if (cb(0, loc, rl->report[0], ctxt) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; /* termination requested */ + } + + return MO_CONTINUE_MATCHING; /* continue execution */ + } + + for (u32 i = 0; i < count; i++) { + DEBUG_PRINTF("reporting %u\n", rl->report[i]); + if (cb(0, loc, rl->report[i], ctxt) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; /* termination requested */ + } + } + return MO_CONTINUE_MATCHING; /* continue execution */ +} +#endif // end of HAVE_AVX512VBMI + /* include Sheng function definitions */ #include "sheng_defs.h" @@ -671,3 +870,1008 @@ char nfaExecSheng_expandState(UNUSED const struct NFA *nfa, void *dest, *(u8 *)dest = *(const u8 *)src; return 0; } + +#if defined(HAVE_AVX512VBMI) +// Sheng32 +static really_inline +char runSheng32Cb(const struct sheng32 *sh, NfaCallback cb, void *ctxt, + u64a offset, u8 *const cached_accept_state, + ReportID *const cached_accept_id, const u8 *cur_buf, + const u8 *start, const u8 *end, u8 can_die, + u8 has_accel, u8 single, const u8 **scanned, u8 *state) { + DEBUG_PRINTF("Scanning %llu bytes (offset %llu) in callback mode\n", + (u64a)(end - start), offset); + DEBUG_PRINTF("start: %lli end: %lli\n", (s64a)(start - cur_buf), + (s64a)(end - cur_buf)); + DEBUG_PRINTF("can die: %u has accel: %u single: %u\n", !!can_die, + !!has_accel, !!single); + int rv; + /* scan and report all matches */ + if (can_die) { + if (has_accel) { + rv = sheng32_4_coda(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + } else { + rv = sheng32_4_cod(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + } + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + rv = sheng32_cod(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + *scanned, end, scanned); + } else { + if (has_accel) { + rv = sheng32_4_coa(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + } else { + rv = sheng32_4_co(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + } + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + rv = sheng32_co(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + *scanned, end, scanned); + } + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + return MO_ALIVE; +} + +static really_inline +void runSheng32Nm(const struct sheng32 *sh, NfaCallback cb, void *ctxt, + u64a offset, u8 *const cached_accept_state, + ReportID *const cached_accept_id, const u8 *cur_buf, + const u8 *start, const u8 *end, u8 can_die, u8 has_accel, + u8 single, const u8 **scanned, u8 *state) { + DEBUG_PRINTF("Scanning %llu bytes (offset %llu) in nomatch mode\n", + (u64a)(end - start), offset); + DEBUG_PRINTF("start: %lli end: %lli\n", (s64a)(start - cur_buf), + (s64a)(end - cur_buf)); + DEBUG_PRINTF("can die: %u has accel: %u single: %u\n", !!can_die, + !!has_accel, !!single); + /* just scan the buffer */ + if (can_die) { + if (has_accel) { + sheng32_4_nmda(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + } else { + sheng32_4_nmd(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + } + sheng32_nmd(state, cb, ctxt, sh, cached_accept_state, cached_accept_id, + single, offset, cur_buf, *scanned, end, scanned); + } else { + sheng32_4_nm(state, cb, ctxt, sh, cached_accept_state, cached_accept_id, + single, offset, cur_buf, start, end, scanned); + sheng32_nm(state, cb, ctxt, sh, cached_accept_state, cached_accept_id, + single, offset, cur_buf, *scanned, end, scanned); + } +} + +static really_inline +char runSheng32Sam(const struct sheng32 *sh, NfaCallback cb, void *ctxt, + u64a offset, u8 *const cached_accept_state, + ReportID *const cached_accept_id, const u8 *cur_buf, + const u8 *start, const u8 *end, u8 can_die, u8 has_accel, + u8 single, const u8 **scanned, u8 *state) { + DEBUG_PRINTF("Scanning %llu bytes (offset %llu) in stop at match mode\n", + (u64a)(end - start), offset); + DEBUG_PRINTF("start: %lli end: %lli\n", (s64a)(start - cur_buf), + (s64a)(end - cur_buf)); + DEBUG_PRINTF("can die: %u has accel: %u single: %u\n", !!can_die, + !!has_accel, !!single); + int rv; + /* scan until first match */ + if (can_die) { + if (has_accel) { + rv = sheng32_4_samda(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + } else { + rv = sheng32_4_samd(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + } + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + /* if we stopped before we expected, we found a match */ + if (rv == MO_MATCHES_PENDING) { + return MO_MATCHES_PENDING; + } + + rv = sheng32_samd(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + *scanned, end, scanned); + } else { + if (has_accel) { + rv = sheng32_4_sama(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + } else { + rv = sheng32_4_sam(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + } + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + /* if we stopped before we expected, we found a match */ + if (rv == MO_MATCHES_PENDING) { + return MO_MATCHES_PENDING; + } + + rv = sheng32_sam(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + *scanned, end, scanned); + } + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + /* if we stopped before we expected, we found a match */ + if (rv == MO_MATCHES_PENDING) { + return MO_MATCHES_PENDING; + } + return MO_ALIVE; +} + +static never_inline +char runSheng32(const struct sheng32 *sh, struct mq *q, s64a b_end, + enum MatchMode mode) { + u8 state = *(u8 *)q->state; + u8 can_die = sh->flags & SHENG_FLAG_CAN_DIE; + u8 has_accel = sh->flags & SHENG_FLAG_HAS_ACCEL; + u8 single = sh->flags & SHENG_FLAG_SINGLE_REPORT; + + u8 cached_accept_state = 0; + ReportID cached_accept_id = 0; + + DEBUG_PRINTF("starting Sheng32 execution in state %u\n", + state & SHENG32_STATE_MASK); + + if (q->report_current) { + DEBUG_PRINTF("reporting current pending matches\n"); + assert(sh); + + q->report_current = 0; + + int rv; + if (single) { + rv = fireSingleReport(q->cb, q->context, sh->report, + q_cur_offset(q)); + } else { + rv = fireReports32(sh, q->cb, q->context, state, q_cur_offset(q), + &cached_accept_state, &cached_accept_id, 0); + } + if (rv == MO_HALT_MATCHING) { + DEBUG_PRINTF("exiting in state %u\n", state & SHENG32_STATE_MASK); + return MO_DEAD; + } + + DEBUG_PRINTF("proceeding with matching\n"); + } + + assert(q_cur_type(q) == MQE_START); + s64a start = q_cur_loc(q); + + DEBUG_PRINTF("offset: %lli, location: %lli, mode: %s\n", q->offset, start, + mode == CALLBACK_OUTPUT ? "CALLBACK OUTPUT" : + mode == NO_MATCHES ? "NO MATCHES" : + mode == STOP_AT_MATCH ? "STOP AT MATCH" : "???"); + + DEBUG_PRINTF("processing event @ %lli: %s\n", q->offset + q_cur_loc(q), + q_cur_type(q) == MQE_START ? "START" : + q_cur_type(q) == MQE_TOP ? "TOP" : + q_cur_type(q) == MQE_END ? "END" : "???"); + + const u8* cur_buf; + if (start < 0) { + DEBUG_PRINTF("negative location, scanning history\n"); + DEBUG_PRINTF("min location: %zd\n", -q->hlength); + cur_buf = q->history + q->hlength; + } else { + DEBUG_PRINTF("positive location, scanning buffer\n"); + DEBUG_PRINTF("max location: %lli\n", b_end); + cur_buf = q->buffer; + } + + /* if we our queue event is past our end */ + if (mode != NO_MATCHES && q_cur_loc(q) > b_end) { + DEBUG_PRINTF("current location past buffer end\n"); + DEBUG_PRINTF("setting q location to %llu\n", b_end); + DEBUG_PRINTF("exiting in state %u\n", state & SHENG32_STATE_MASK); + q->items[q->cur].location = b_end; + return MO_ALIVE; + } + + q->cur++; + + s64a cur_start = start; + + while (1) { + DEBUG_PRINTF("processing event @ %lli: %s\n", q->offset + q_cur_loc(q), + q_cur_type(q) == MQE_START ? "START" : + q_cur_type(q) == MQE_TOP ? "TOP" : + q_cur_type(q) == MQE_END ? "END" : "???"); + s64a end = q_cur_loc(q); + if (mode != NO_MATCHES) { + end = MIN(end, b_end); + } + assert(end <= (s64a) q->length); + s64a cur_end = end; + + /* we may cross the border between history and current buffer */ + if (cur_start < 0) { + cur_end = MIN(0, cur_end); + } + + DEBUG_PRINTF("start: %lli end: %lli\n", start, end); + + /* don't scan zero length buffer */ + if (cur_start != cur_end) { + const u8 * scanned = cur_buf; + char rv; + + if (mode == NO_MATCHES) { + runSheng32Nm(sh, q->cb, q->context, q->offset, + &cached_accept_state, &cached_accept_id, cur_buf, + cur_buf + cur_start, cur_buf + cur_end, can_die, + has_accel, single, &scanned, &state); + } else if (mode == CALLBACK_OUTPUT) { + rv = runSheng32Cb(sh, q->cb, q->context, q->offset, + &cached_accept_state, &cached_accept_id, + cur_buf, cur_buf + cur_start, cur_buf + cur_end, + can_die, has_accel, single, &scanned, &state); + if (rv == MO_DEAD) { + DEBUG_PRINTF("exiting in state %u\n", + state & SHENG32_STATE_MASK); + return MO_DEAD; + } + } else if (mode == STOP_AT_MATCH) { + rv = runSheng32Sam(sh, q->cb, q->context, q->offset, + &cached_accept_state, &cached_accept_id, + cur_buf, cur_buf + cur_start, + cur_buf + cur_end, can_die, has_accel, single, + &scanned, &state); + if (rv == MO_DEAD) { + DEBUG_PRINTF("exiting in state %u\n", + state & SHENG32_STATE_MASK); + return rv; + } else if (rv == MO_MATCHES_PENDING) { + assert(q->cur); + DEBUG_PRINTF("found a match, setting q location to %zd\n", + scanned - cur_buf + 1); + q->cur--; + q->items[q->cur].type = MQE_START; + q->items[q->cur].location = + scanned - cur_buf + 1; /* due to exiting early */ + *(u8 *)q->state = state; + DEBUG_PRINTF("exiting in state %u\n", + state & SHENG32_STATE_MASK); + return rv; + } + } else { + assert(!"invalid scanning mode!"); + } + assert(scanned == cur_buf + cur_end); + + cur_start = cur_end; + } + + /* if we our queue event is past our end */ + if (mode != NO_MATCHES && q_cur_loc(q) > b_end) { + DEBUG_PRINTF("current location past buffer end\n"); + DEBUG_PRINTF("setting q location to %llu\n", b_end); + DEBUG_PRINTF("exiting in state %u\n", state & SHENG32_STATE_MASK); + q->cur--; + q->items[q->cur].type = MQE_START; + q->items[q->cur].location = b_end; + *(u8 *)q->state = state; + return MO_ALIVE; + } + + /* crossing over into actual buffer */ + if (cur_start == 0) { + DEBUG_PRINTF("positive location, scanning buffer\n"); + DEBUG_PRINTF("max offset: %lli\n", b_end); + cur_buf = q->buffer; + } + + /* continue scanning the same buffer */ + if (end != cur_end) { + continue; + } + + switch (q_cur_type(q)) { + case MQE_END: + *(u8 *)q->state = state; + q->cur++; + DEBUG_PRINTF("exiting in state %u\n", state & SHENG32_STATE_MASK); + if (can_die) { + return (state & SHENG32_STATE_DEAD) ? MO_DEAD : MO_ALIVE; + } + return MO_ALIVE; + case MQE_TOP: + if (q->offset + cur_start == 0) { + DEBUG_PRINTF("Anchored start, going to state %u\n", + sh->anchored); + state = sh->anchored; + } else { + u8 new_state = get_aux32(sh, state)->top; + DEBUG_PRINTF("Top event %u->%u\n", state & SHENG32_STATE_MASK, + new_state & SHENG32_STATE_MASK); + state = new_state; + } + break; + default: + assert(!"invalid queue event"); + break; + } + q->cur++; + } +} + +char nfaExecSheng32_B(const struct NFA *n, u64a offset, const u8 *buffer, + size_t length, NfaCallback cb, void *context) { + DEBUG_PRINTF("smallwrite Sheng32\n"); + assert(n->type == SHENG_NFA_32); + const struct sheng32 *sh = getImplNfa(n); + u8 state = sh->anchored; + u8 can_die = sh->flags & SHENG_FLAG_CAN_DIE; + u8 has_accel = sh->flags & SHENG_FLAG_HAS_ACCEL; + u8 single = sh->flags & SHENG_FLAG_SINGLE_REPORT; + u8 cached_accept_state = 0; + ReportID cached_accept_id = 0; + + /* scan and report all matches */ + int rv; + s64a end = length; + const u8 *scanned; + + rv = runSheng32Cb(sh, cb, context, offset, &cached_accept_state, + &cached_accept_id, buffer, buffer, buffer + end, can_die, + has_accel, single, &scanned, &state); + if (rv == MO_DEAD) { + DEBUG_PRINTF("exiting in state %u\n", + state & SHENG32_STATE_MASK); + return MO_DEAD; + } + + DEBUG_PRINTF("%u\n", state & SHENG32_STATE_MASK); + + const struct sstate_aux *aux = get_aux32(sh, state); + + if (aux->accept_eod) { + DEBUG_PRINTF("Reporting EOD matches\n"); + fireReports32(sh, cb, context, state, end + offset, + &cached_accept_state, &cached_accept_id, 1); + } + + return state & SHENG32_STATE_DEAD ? MO_DEAD : MO_ALIVE; +} + +char nfaExecSheng32_Q(const struct NFA *n, struct mq *q, s64a end) { + const struct sheng32 *sh = get_sheng32(n); + char rv = runSheng32(sh, q, end, CALLBACK_OUTPUT); + return rv; +} + +char nfaExecSheng32_Q2(const struct NFA *n, struct mq *q, s64a end) { + const struct sheng32 *sh = get_sheng32(n); + char rv = runSheng32(sh, q, end, STOP_AT_MATCH); + return rv; +} + +char nfaExecSheng32_QR(const struct NFA *n, struct mq *q, ReportID report) { + assert(q_cur_type(q) == MQE_START); + + const struct sheng32 *sh = get_sheng32(n); + char rv = runSheng32(sh, q, 0 /* end */, NO_MATCHES); + + if (rv && nfaExecSheng32_inAccept(n, report, q)) { + return MO_MATCHES_PENDING; + } + return rv; +} + +char nfaExecSheng32_inAccept(const struct NFA *n, ReportID report, + struct mq *q) { + assert(n && q); + + const struct sheng32 *sh = get_sheng32(n); + u8 s = *(const u8 *)q->state; + DEBUG_PRINTF("checking accepts for %u\n", (u8)(s & SHENG32_STATE_MASK)); + + const struct sstate_aux *aux = get_aux32(sh, s); + + if (!aux->accept) { + return 0; + } + + return sheng32HasAccept(sh, aux, report); +} + +char nfaExecSheng32_inAnyAccept(const struct NFA *n, struct mq *q) { + assert(n && q); + + const struct sheng32 *sh = get_sheng32(n); + u8 s = *(const u8 *)q->state; + DEBUG_PRINTF("checking accepts for %u\n", (u8)(s & SHENG32_STATE_MASK)); + + const struct sstate_aux *aux = get_aux32(sh, s); + return !!aux->accept; +} + +char nfaExecSheng32_testEOD(const struct NFA *nfa, const char *state, + UNUSED const char *streamState, u64a offset, + NfaCallback cb, void *ctxt) { + assert(nfa); + + const struct sheng32 *sh = get_sheng32(nfa); + u8 s = *(const u8 *)state; + DEBUG_PRINTF("checking EOD accepts for %u\n", (u8)(s & SHENG32_STATE_MASK)); + + const struct sstate_aux *aux = get_aux32(sh, s); + + if (!aux->accept_eod) { + return MO_CONTINUE_MATCHING; + } + + return fireReports32(sh, cb, ctxt, s, offset, NULL, NULL, 1); +} + +char nfaExecSheng32_reportCurrent(const struct NFA *n, struct mq *q) { + const struct sheng32 *sh = (const struct sheng32 *)getImplNfa(n); + NfaCallback cb = q->cb; + void *ctxt = q->context; + u8 s = *(u8 *)q->state; + const struct sstate_aux *aux = get_aux32(sh, s); + u64a offset = q_cur_offset(q); + u8 cached_state_id = 0; + ReportID cached_report_id = 0; + assert(q_cur_type(q) == MQE_START); + + if (aux->accept) { + if (sh->flags & SHENG_FLAG_SINGLE_REPORT) { + fireSingleReport(cb, ctxt, sh->report, offset); + } else { + fireReports32(sh, cb, ctxt, s, offset, &cached_state_id, + &cached_report_id, 0); + } + } + + return 0; +} + +char nfaExecSheng32_initCompressedState(const struct NFA *nfa, u64a offset, + void *state, UNUSED u8 key) { + const struct sheng32 *sh = get_sheng32(nfa); + u8 *s = (u8 *)state; + *s = offset ? sh->floating: sh->anchored; + return !(*s & SHENG32_STATE_DEAD); +} + +char nfaExecSheng32_queueInitState(const struct NFA *nfa, struct mq *q) { + assert(nfa->scratchStateSize == 1); + + /* starting in floating state */ + const struct sheng32 *sh = get_sheng32(nfa); + *(u8 *)q->state = sh->floating; + DEBUG_PRINTF("starting in floating state\n"); + return 0; +} + +char nfaExecSheng32_queueCompressState(UNUSED const struct NFA *nfa, + const struct mq *q, UNUSED s64a loc) { + void *dest = q->streamState; + const void *src = q->state; + assert(nfa->scratchStateSize == 1); + assert(nfa->streamStateSize == 1); + *(u8 *)dest = *(const u8 *)src; + return 0; +} + +char nfaExecSheng32_expandState(UNUSED const struct NFA *nfa, void *dest, + const void *src, UNUSED u64a offset, + UNUSED u8 key) { + assert(nfa->scratchStateSize == 1); + assert(nfa->streamStateSize == 1); + *(u8 *)dest = *(const u8 *)src; + return 0; +} + +// Sheng64 +static really_inline +char runSheng64Cb(const struct sheng64 *sh, NfaCallback cb, void *ctxt, + u64a offset, u8 *const cached_accept_state, + ReportID *const cached_accept_id, const u8 *cur_buf, + const u8 *start, const u8 *end, u8 can_die, + u8 single, const u8 **scanned, u8 *state) { + DEBUG_PRINTF("Scanning %llu bytes (offset %llu) in callback mode\n", + (u64a)(end - start), offset); + DEBUG_PRINTF("start: %lli end: %lli\n", (s64a)(start - cur_buf), + (s64a)(end - cur_buf)); + DEBUG_PRINTF("can die: %u single: %u\n", !!can_die, !!single); + int rv; + /* scan and report all matches */ + if (can_die) { + rv = sheng64_4_cod(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + rv = sheng64_cod(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + *scanned, end, scanned); + } else { + rv = sheng64_4_co(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + rv = sheng64_co(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + *scanned, end, scanned); + } + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + return MO_ALIVE; +} + +static really_inline +void runSheng64Nm(const struct sheng64 *sh, NfaCallback cb, void *ctxt, + u64a offset, u8 *const cached_accept_state, + ReportID *const cached_accept_id, const u8 *cur_buf, + const u8 *start, const u8 *end, u8 can_die, + u8 single, const u8 **scanned, u8 *state) { + DEBUG_PRINTF("Scanning %llu bytes (offset %llu) in nomatch mode\n", + (u64a)(end - start), offset); + DEBUG_PRINTF("start: %lli end: %lli\n", (s64a)(start - cur_buf), + (s64a)(end - cur_buf)); + DEBUG_PRINTF("can die: %u single: %u\n", !!can_die, !!single); + /* just scan the buffer */ + if (can_die) { + sheng64_4_nmd(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + sheng64_nmd(state, cb, ctxt, sh, cached_accept_state, cached_accept_id, + single, offset, cur_buf, *scanned, end, scanned); + } else { + sheng64_4_nm(state, cb, ctxt, sh, cached_accept_state, cached_accept_id, + single, offset, cur_buf, start, end, scanned); + sheng64_nm(state, cb, ctxt, sh, cached_accept_state, cached_accept_id, + single, offset, cur_buf, *scanned, end, scanned); + } +} + +static really_inline +char runSheng64Sam(const struct sheng64 *sh, NfaCallback cb, void *ctxt, + u64a offset, u8 *const cached_accept_state, + ReportID *const cached_accept_id, const u8 *cur_buf, + const u8 *start, const u8 *end, u8 can_die, + u8 single, const u8 **scanned, u8 *state) { + DEBUG_PRINTF("Scanning %llu bytes (offset %llu) in stop at match mode\n", + (u64a)(end - start), offset); + DEBUG_PRINTF("start: %lli end: %lli\n", (s64a)(start - cur_buf), + (s64a)(end - cur_buf)); + DEBUG_PRINTF("can die: %u single: %u\n", !!can_die, !!single); + int rv; + /* scan until first match */ + if (can_die) { + rv = sheng64_4_samd(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + /* if we stopped before we expected, we found a match */ + if (rv == MO_MATCHES_PENDING) { + return MO_MATCHES_PENDING; + } + + rv = sheng64_samd(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + *scanned, end, scanned); + } else { + rv = sheng64_4_sam(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + start, end, scanned); + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + /* if we stopped before we expected, we found a match */ + if (rv == MO_MATCHES_PENDING) { + return MO_MATCHES_PENDING; + } + + rv = sheng64_sam(state, cb, ctxt, sh, cached_accept_state, + cached_accept_id, single, offset, cur_buf, + *scanned, end, scanned); + } + if (rv == MO_HALT_MATCHING) { + return MO_DEAD; + } + /* if we stopped before we expected, we found a match */ + if (rv == MO_MATCHES_PENDING) { + return MO_MATCHES_PENDING; + } + return MO_ALIVE; +} + +static never_inline +char runSheng64(const struct sheng64 *sh, struct mq *q, s64a b_end, + enum MatchMode mode) { + u8 state = *(u8 *)q->state; + u8 can_die = sh->flags & SHENG_FLAG_CAN_DIE; + u8 single = sh->flags & SHENG_FLAG_SINGLE_REPORT; + + u8 cached_accept_state = 0; + ReportID cached_accept_id = 0; + + DEBUG_PRINTF("starting Sheng64 execution in state %u\n", + state & SHENG64_STATE_MASK); + + if (q->report_current) { + DEBUG_PRINTF("reporting current pending matches\n"); + assert(sh); + + q->report_current = 0; + + int rv; + if (single) { + rv = fireSingleReport(q->cb, q->context, sh->report, + q_cur_offset(q)); + } else { + rv = fireReports64(sh, q->cb, q->context, state, q_cur_offset(q), + &cached_accept_state, &cached_accept_id, 0); + } + if (rv == MO_HALT_MATCHING) { + DEBUG_PRINTF("exiting in state %u\n", state & SHENG64_STATE_MASK); + return MO_DEAD; + } + + DEBUG_PRINTF("proceeding with matching\n"); + } + + assert(q_cur_type(q) == MQE_START); + s64a start = q_cur_loc(q); + + DEBUG_PRINTF("offset: %lli, location: %lli, mode: %s\n", q->offset, start, + mode == CALLBACK_OUTPUT ? "CALLBACK OUTPUT" : + mode == NO_MATCHES ? "NO MATCHES" : + mode == STOP_AT_MATCH ? "STOP AT MATCH" : "???"); + + DEBUG_PRINTF("processing event @ %lli: %s\n", q->offset + q_cur_loc(q), + q_cur_type(q) == MQE_START ? "START" : + q_cur_type(q) == MQE_TOP ? "TOP" : + q_cur_type(q) == MQE_END ? "END" : "???"); + + const u8* cur_buf; + if (start < 0) { + DEBUG_PRINTF("negative location, scanning history\n"); + DEBUG_PRINTF("min location: %zd\n", -q->hlength); + cur_buf = q->history + q->hlength; + } else { + DEBUG_PRINTF("positive location, scanning buffer\n"); + DEBUG_PRINTF("max location: %lli\n", b_end); + cur_buf = q->buffer; + } + + /* if we our queue event is past our end */ + if (mode != NO_MATCHES && q_cur_loc(q) > b_end) { + DEBUG_PRINTF("current location past buffer end\n"); + DEBUG_PRINTF("setting q location to %llu\n", b_end); + DEBUG_PRINTF("exiting in state %u\n", state & SHENG64_STATE_MASK); + q->items[q->cur].location = b_end; + return MO_ALIVE; + } + + q->cur++; + + s64a cur_start = start; + + while (1) { + DEBUG_PRINTF("processing event @ %lli: %s\n", q->offset + q_cur_loc(q), + q_cur_type(q) == MQE_START ? "START" : + q_cur_type(q) == MQE_TOP ? "TOP" : + q_cur_type(q) == MQE_END ? "END" : "???"); + s64a end = q_cur_loc(q); + if (mode != NO_MATCHES) { + end = MIN(end, b_end); + } + assert(end <= (s64a) q->length); + s64a cur_end = end; + + /* we may cross the border between history and current buffer */ + if (cur_start < 0) { + cur_end = MIN(0, cur_end); + } + + DEBUG_PRINTF("start: %lli end: %lli\n", start, end); + + /* don't scan zero length buffer */ + if (cur_start != cur_end) { + const u8 * scanned = cur_buf; + char rv; + + if (mode == NO_MATCHES) { + runSheng64Nm(sh, q->cb, q->context, q->offset, + &cached_accept_state, &cached_accept_id, cur_buf, + cur_buf + cur_start, cur_buf + cur_end, can_die, + single, &scanned, &state); + } else if (mode == CALLBACK_OUTPUT) { + rv = runSheng64Cb(sh, q->cb, q->context, q->offset, + &cached_accept_state, &cached_accept_id, + cur_buf, cur_buf + cur_start, cur_buf + cur_end, + can_die, single, &scanned, &state); + if (rv == MO_DEAD) { + DEBUG_PRINTF("exiting in state %u\n", + state & SHENG64_STATE_MASK); + return MO_DEAD; + } + } else if (mode == STOP_AT_MATCH) { + rv = runSheng64Sam(sh, q->cb, q->context, q->offset, + &cached_accept_state, &cached_accept_id, + cur_buf, cur_buf + cur_start, + cur_buf + cur_end, can_die, single, + &scanned, &state); + if (rv == MO_DEAD) { + DEBUG_PRINTF("exiting in state %u\n", + state & SHENG64_STATE_MASK); + return rv; + } else if (rv == MO_MATCHES_PENDING) { + assert(q->cur); + DEBUG_PRINTF("found a match, setting q location to %zd\n", + scanned - cur_buf + 1); + q->cur--; + q->items[q->cur].type = MQE_START; + q->items[q->cur].location = + scanned - cur_buf + 1; /* due to exiting early */ + *(u8 *)q->state = state; + DEBUG_PRINTF("exiting in state %u\n", + state & SHENG64_STATE_MASK); + return rv; + } + } else { + assert(!"invalid scanning mode!"); + } + assert(scanned == cur_buf + cur_end); + + cur_start = cur_end; + } + + /* if we our queue event is past our end */ + if (mode != NO_MATCHES && q_cur_loc(q) > b_end) { + DEBUG_PRINTF("current location past buffer end\n"); + DEBUG_PRINTF("setting q location to %llu\n", b_end); + DEBUG_PRINTF("exiting in state %u\n", state & SHENG64_STATE_MASK); + q->cur--; + q->items[q->cur].type = MQE_START; + q->items[q->cur].location = b_end; + *(u8 *)q->state = state; + return MO_ALIVE; + } + + /* crossing over into actual buffer */ + if (cur_start == 0) { + DEBUG_PRINTF("positive location, scanning buffer\n"); + DEBUG_PRINTF("max offset: %lli\n", b_end); + cur_buf = q->buffer; + } + + /* continue scanning the same buffer */ + if (end != cur_end) { + continue; + } + + switch (q_cur_type(q)) { + case MQE_END: + *(u8 *)q->state = state; + q->cur++; + DEBUG_PRINTF("exiting in state %u\n", state & SHENG64_STATE_MASK); + if (can_die) { + return (state & SHENG64_STATE_DEAD) ? MO_DEAD : MO_ALIVE; + } + return MO_ALIVE; + case MQE_TOP: + if (q->offset + cur_start == 0) { + DEBUG_PRINTF("Anchored start, going to state %u\n", + sh->anchored); + state = sh->anchored; + } else { + u8 new_state = get_aux64(sh, state)->top; + DEBUG_PRINTF("Top event %u->%u\n", state & SHENG64_STATE_MASK, + new_state & SHENG64_STATE_MASK); + state = new_state; + } + break; + default: + assert(!"invalid queue event"); + break; + } + q->cur++; + } +} + +char nfaExecSheng64_B(const struct NFA *n, u64a offset, const u8 *buffer, + size_t length, NfaCallback cb, void *context) { + DEBUG_PRINTF("smallwrite Sheng64\n"); + assert(n->type == SHENG_NFA_64); + const struct sheng64 *sh = getImplNfa(n); + u8 state = sh->anchored; + u8 can_die = sh->flags & SHENG_FLAG_CAN_DIE; + u8 single = sh->flags & SHENG_FLAG_SINGLE_REPORT; + u8 cached_accept_state = 0; + ReportID cached_accept_id = 0; + + /* scan and report all matches */ + int rv; + s64a end = length; + const u8 *scanned; + + rv = runSheng64Cb(sh, cb, context, offset, &cached_accept_state, + &cached_accept_id, buffer, buffer, buffer + end, can_die, + single, &scanned, &state); + if (rv == MO_DEAD) { + DEBUG_PRINTF("exiting in state %u\n", + state & SHENG64_STATE_MASK); + return MO_DEAD; + } + + DEBUG_PRINTF("%u\n", state & SHENG64_STATE_MASK); + + const struct sstate_aux *aux = get_aux64(sh, state); + + if (aux->accept_eod) { + DEBUG_PRINTF("Reporting EOD matches\n"); + fireReports64(sh, cb, context, state, end + offset, + &cached_accept_state, &cached_accept_id, 1); + } + + return state & SHENG64_STATE_DEAD ? MO_DEAD : MO_ALIVE; +} + +char nfaExecSheng64_Q(const struct NFA *n, struct mq *q, s64a end) { + const struct sheng64 *sh = get_sheng64(n); + char rv = runSheng64(sh, q, end, CALLBACK_OUTPUT); + return rv; +} + +char nfaExecSheng64_Q2(const struct NFA *n, struct mq *q, s64a end) { + const struct sheng64 *sh = get_sheng64(n); + char rv = runSheng64(sh, q, end, STOP_AT_MATCH); + return rv; +} + +char nfaExecSheng64_QR(const struct NFA *n, struct mq *q, ReportID report) { + assert(q_cur_type(q) == MQE_START); + + const struct sheng64 *sh = get_sheng64(n); + char rv = runSheng64(sh, q, 0 /* end */, NO_MATCHES); + + if (rv && nfaExecSheng64_inAccept(n, report, q)) { + return MO_MATCHES_PENDING; + } + return rv; +} + +char nfaExecSheng64_inAccept(const struct NFA *n, ReportID report, + struct mq *q) { + assert(n && q); + + const struct sheng64 *sh = get_sheng64(n); + u8 s = *(const u8 *)q->state; + DEBUG_PRINTF("checking accepts for %u\n", (u8)(s & SHENG64_STATE_MASK)); + + const struct sstate_aux *aux = get_aux64(sh, s); + + if (!aux->accept) { + return 0; + } + + return sheng64HasAccept(sh, aux, report); +} + +char nfaExecSheng64_inAnyAccept(const struct NFA *n, struct mq *q) { + assert(n && q); + + const struct sheng64 *sh = get_sheng64(n); + u8 s = *(const u8 *)q->state; + DEBUG_PRINTF("checking accepts for %u\n", (u8)(s & SHENG64_STATE_MASK)); + + const struct sstate_aux *aux = get_aux64(sh, s); + return !!aux->accept; +} + +char nfaExecSheng64_testEOD(const struct NFA *nfa, const char *state, + UNUSED const char *streamState, u64a offset, + NfaCallback cb, void *ctxt) { + assert(nfa); + + const struct sheng64 *sh = get_sheng64(nfa); + u8 s = *(const u8 *)state; + DEBUG_PRINTF("checking EOD accepts for %u\n", (u8)(s & SHENG64_STATE_MASK)); + + const struct sstate_aux *aux = get_aux64(sh, s); + + if (!aux->accept_eod) { + return MO_CONTINUE_MATCHING; + } + + return fireReports64(sh, cb, ctxt, s, offset, NULL, NULL, 1); +} + +char nfaExecSheng64_reportCurrent(const struct NFA *n, struct mq *q) { + const struct sheng64 *sh = (const struct sheng64 *)getImplNfa(n); + NfaCallback cb = q->cb; + void *ctxt = q->context; + u8 s = *(u8 *)q->state; + const struct sstate_aux *aux = get_aux64(sh, s); + u64a offset = q_cur_offset(q); + u8 cached_state_id = 0; + ReportID cached_report_id = 0; + assert(q_cur_type(q) == MQE_START); + + if (aux->accept) { + if (sh->flags & SHENG_FLAG_SINGLE_REPORT) { + fireSingleReport(cb, ctxt, sh->report, offset); + } else { + fireReports64(sh, cb, ctxt, s, offset, &cached_state_id, + &cached_report_id, 0); + } + } + + return 0; +} + +char nfaExecSheng64_initCompressedState(const struct NFA *nfa, u64a offset, + void *state, UNUSED u8 key) { + const struct sheng64 *sh = get_sheng64(nfa); + u8 *s = (u8 *)state; + *s = offset ? sh->floating: sh->anchored; + return !(*s & SHENG64_STATE_DEAD); +} + +char nfaExecSheng64_queueInitState(const struct NFA *nfa, struct mq *q) { + assert(nfa->scratchStateSize == 1); + + /* starting in floating state */ + const struct sheng64 *sh = get_sheng64(nfa); + *(u8 *)q->state = sh->floating; + DEBUG_PRINTF("starting in floating state\n"); + return 0; +} + +char nfaExecSheng64_queueCompressState(UNUSED const struct NFA *nfa, + const struct mq *q, UNUSED s64a loc) { + void *dest = q->streamState; + const void *src = q->state; + assert(nfa->scratchStateSize == 1); + assert(nfa->streamStateSize == 1); + *(u8 *)dest = *(const u8 *)src; + return 0; +} + +char nfaExecSheng64_expandState(UNUSED const struct NFA *nfa, void *dest, + const void *src, UNUSED u64a offset, + UNUSED u8 key) { + assert(nfa->scratchStateSize == 1); + assert(nfa->streamStateSize == 1); + *(u8 *)dest = *(const u8 *)src; + return 0; +} +#endif // end of HAVE_AVX512VBMI diff --git a/src/nfa/sheng.h b/src/nfa/sheng.h index 84a2b6b51..7b90e3034 100644 --- a/src/nfa/sheng.h +++ b/src/nfa/sheng.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -58,4 +58,86 @@ char nfaExecSheng_reportCurrent(const struct NFA *n, struct mq *q); char nfaExecSheng_B(const struct NFA *n, u64a offset, const u8 *buffer, size_t length, NfaCallback cb, void *context); +#if defined(HAVE_AVX512VBMI) +#define nfaExecSheng32_B_Reverse NFA_API_NO_IMPL +#define nfaExecSheng32_zombie_status NFA_API_ZOMBIE_NO_IMPL + +char nfaExecSheng32_Q(const struct NFA *n, struct mq *q, s64a end); +char nfaExecSheng32_Q2(const struct NFA *n, struct mq *q, s64a end); +char nfaExecSheng32_QR(const struct NFA *n, struct mq *q, ReportID report); +char nfaExecSheng32_inAccept(const struct NFA *n, ReportID report, + struct mq *q); +char nfaExecSheng32_inAnyAccept(const struct NFA *n, struct mq *q); +char nfaExecSheng32_queueInitState(const struct NFA *nfa, struct mq *q); +char nfaExecSheng32_queueCompressState(const struct NFA *nfa, + const struct mq *q, s64a loc); +char nfaExecSheng32_expandState(const struct NFA *nfa, void *dest, + const void *src, u64a offset, u8 key); +char nfaExecSheng32_initCompressedState(const struct NFA *nfa, u64a offset, + void *state, u8 key); +char nfaExecSheng32_testEOD(const struct NFA *nfa, const char *state, + const char *streamState, u64a offset, + NfaCallback callback, void *context); +char nfaExecSheng32_reportCurrent(const struct NFA *n, struct mq *q); + +char nfaExecSheng32_B(const struct NFA *n, u64a offset, const u8 *buffer, + size_t length, NfaCallback cb, void *context); + +#define nfaExecSheng64_B_Reverse NFA_API_NO_IMPL +#define nfaExecSheng64_zombie_status NFA_API_ZOMBIE_NO_IMPL + +char nfaExecSheng64_Q(const struct NFA *n, struct mq *q, s64a end); +char nfaExecSheng64_Q2(const struct NFA *n, struct mq *q, s64a end); +char nfaExecSheng64_QR(const struct NFA *n, struct mq *q, ReportID report); +char nfaExecSheng64_inAccept(const struct NFA *n, ReportID report, + struct mq *q); +char nfaExecSheng64_inAnyAccept(const struct NFA *n, struct mq *q); +char nfaExecSheng64_queueInitState(const struct NFA *nfa, struct mq *q); +char nfaExecSheng64_queueCompressState(const struct NFA *nfa, + const struct mq *q, s64a loc); +char nfaExecSheng64_expandState(const struct NFA *nfa, void *dest, + const void *src, u64a offset, u8 key); +char nfaExecSheng64_initCompressedState(const struct NFA *nfa, u64a offset, + void *state, u8 key); +char nfaExecSheng64_testEOD(const struct NFA *nfa, const char *state, + const char *streamState, u64a offset, + NfaCallback callback, void *context); +char nfaExecSheng64_reportCurrent(const struct NFA *n, struct mq *q); + +char nfaExecSheng64_B(const struct NFA *n, u64a offset, const u8 *buffer, + size_t length, NfaCallback cb, void *context); + +#else // !HAVE_AVX512VBMI + +#define nfaExecSheng32_B_Reverse NFA_API_NO_IMPL +#define nfaExecSheng32_zombie_status NFA_API_ZOMBIE_NO_IMPL +#define nfaExecSheng32_Q NFA_API_NO_IMPL +#define nfaExecSheng32_Q2 NFA_API_NO_IMPL +#define nfaExecSheng32_QR NFA_API_NO_IMPL +#define nfaExecSheng32_inAccept NFA_API_NO_IMPL +#define nfaExecSheng32_inAnyAccept NFA_API_NO_IMPL +#define nfaExecSheng32_queueInitState NFA_API_NO_IMPL +#define nfaExecSheng32_queueCompressState NFA_API_NO_IMPL +#define nfaExecSheng32_expandState NFA_API_NO_IMPL +#define nfaExecSheng32_initCompressedState NFA_API_NO_IMPL +#define nfaExecSheng32_testEOD NFA_API_NO_IMPL +#define nfaExecSheng32_reportCurrent NFA_API_NO_IMPL +#define nfaExecSheng32_B NFA_API_NO_IMPL + +#define nfaExecSheng64_B_Reverse NFA_API_NO_IMPL +#define nfaExecSheng64_zombie_status NFA_API_ZOMBIE_NO_IMPL +#define nfaExecSheng64_Q NFA_API_NO_IMPL +#define nfaExecSheng64_Q2 NFA_API_NO_IMPL +#define nfaExecSheng64_QR NFA_API_NO_IMPL +#define nfaExecSheng64_inAccept NFA_API_NO_IMPL +#define nfaExecSheng64_inAnyAccept NFA_API_NO_IMPL +#define nfaExecSheng64_queueInitState NFA_API_NO_IMPL +#define nfaExecSheng64_queueCompressState NFA_API_NO_IMPL +#define nfaExecSheng64_expandState NFA_API_NO_IMPL +#define nfaExecSheng64_initCompressedState NFA_API_NO_IMPL +#define nfaExecSheng64_testEOD NFA_API_NO_IMPL +#define nfaExecSheng64_reportCurrent NFA_API_NO_IMPL +#define nfaExecSheng64_B NFA_API_NO_IMPL +#endif // end of HAVE_AVX512VBMI + #endif /* SHENG_H_ */ diff --git a/src/nfa/sheng_defs.h b/src/nfa/sheng_defs.h index 26bdbcee2..390af7522 100644 --- a/src/nfa/sheng_defs.h +++ b/src/nfa/sheng_defs.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -52,6 +52,43 @@ u8 hasInterestingStates(const u8 a, const u8 b, const u8 c, const u8 d) { return (a | b | c | d) & (SHENG_STATE_FLAG_MASK); } +#if defined(HAVE_AVX512VBMI) +static really_inline +u8 isDeadState32(const u8 a) { + return a & SHENG32_STATE_DEAD; +} + +static really_inline +u8 isAcceptState32(const u8 a) { + return a & SHENG32_STATE_ACCEPT; +} + +static really_inline +u8 isAccelState32(const u8 a) { + return a & SHENG32_STATE_ACCEL; +} + +static really_inline +u8 hasInterestingStates32(const u8 a, const u8 b, const u8 c, const u8 d) { + return (a | b | c | d) & (SHENG32_STATE_FLAG_MASK); +} + +static really_inline +u8 isDeadState64(const u8 a) { + return a & SHENG64_STATE_DEAD; +} + +static really_inline +u8 isAcceptState64(const u8 a) { + return a & SHENG64_STATE_ACCEPT; +} + +static really_inline +u8 hasInterestingStates64(const u8 a, const u8 b, const u8 c, const u8 d) { + return (a | b | c | d) & (SHENG64_STATE_FLAG_MASK); +} +#endif + /* these functions should be optimized out, used by NO_MATCHES mode */ static really_inline u8 dummyFunc4(UNUSED const u8 a, UNUSED const u8 b, UNUSED const u8 c, @@ -71,66 +108,162 @@ u8 dummyFunc(UNUSED const u8 a) { #define SHENG_IMPL sheng_cod #define DEAD_FUNC isDeadState #define ACCEPT_FUNC isAcceptState +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_cod +#define DEAD_FUNC32 isDeadState32 +#define ACCEPT_FUNC32 isAcceptState32 +#define SHENG64_IMPL sheng64_cod +#define DEAD_FUNC64 isDeadState64 +#define ACCEPT_FUNC64 isAcceptState64 +#endif #define STOP_AT_MATCH 0 #include "sheng_impl.h" #undef SHENG_IMPL #undef DEAD_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef DEAD_FUNC32 +#undef ACCEPT_FUNC32 +#undef SHENG64_IMPL +#undef DEAD_FUNC64 +#undef ACCEPT_FUNC64 +#endif #undef STOP_AT_MATCH /* callback output, can't die */ #define SHENG_IMPL sheng_co #define DEAD_FUNC dummyFunc #define ACCEPT_FUNC isAcceptState +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_co +#define DEAD_FUNC32 dummyFunc +#define ACCEPT_FUNC32 isAcceptState32 +#define SHENG64_IMPL sheng64_co +#define DEAD_FUNC64 dummyFunc +#define ACCEPT_FUNC64 isAcceptState64 +#endif #define STOP_AT_MATCH 0 #include "sheng_impl.h" #undef SHENG_IMPL #undef DEAD_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef DEAD_FUNC32 +#undef ACCEPT_FUNC32 +#undef SHENG64_IMPL +#undef DEAD_FUNC64 +#undef ACCEPT_FUNC64 +#endif #undef STOP_AT_MATCH /* stop at match, can die */ #define SHENG_IMPL sheng_samd #define DEAD_FUNC isDeadState #define ACCEPT_FUNC isAcceptState +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_samd +#define DEAD_FUNC32 isDeadState32 +#define ACCEPT_FUNC32 isAcceptState32 +#define SHENG64_IMPL sheng64_samd +#define DEAD_FUNC64 isDeadState64 +#define ACCEPT_FUNC64 isAcceptState64 +#endif #define STOP_AT_MATCH 1 #include "sheng_impl.h" #undef SHENG_IMPL #undef DEAD_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef DEAD_FUNC32 +#undef ACCEPT_FUNC32 +#undef SHENG64_IMPL +#undef DEAD_FUNC64 +#undef ACCEPT_FUNC64 +#endif #undef STOP_AT_MATCH /* stop at match, can't die */ #define SHENG_IMPL sheng_sam #define DEAD_FUNC dummyFunc #define ACCEPT_FUNC isAcceptState +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_sam +#define DEAD_FUNC32 dummyFunc +#define ACCEPT_FUNC32 isAcceptState32 +#define SHENG64_IMPL sheng64_sam +#define DEAD_FUNC64 dummyFunc +#define ACCEPT_FUNC64 isAcceptState64 +#endif #define STOP_AT_MATCH 1 #include "sheng_impl.h" #undef SHENG_IMPL #undef DEAD_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef DEAD_FUNC32 +#undef ACCEPT_FUNC32 +#undef SHENG64_IMPL +#undef DEAD_FUNC64 +#undef ACCEPT_FUNC64 +#endif #undef STOP_AT_MATCH /* no match, can die */ #define SHENG_IMPL sheng_nmd #define DEAD_FUNC isDeadState #define ACCEPT_FUNC dummyFunc +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_nmd +#define DEAD_FUNC32 isDeadState32 +#define ACCEPT_FUNC32 dummyFunc +#define SHENG64_IMPL sheng64_nmd +#define DEAD_FUNC64 isDeadState64 +#define ACCEPT_FUNC64 dummyFunc +#endif #define STOP_AT_MATCH 0 #include "sheng_impl.h" #undef SHENG_IMPL #undef DEAD_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef DEAD_FUNC32 +#undef ACCEPT_FUNC32 +#undef SHENG64_IMPL +#undef DEAD_FUNC64 +#undef ACCEPT_FUNC64 +#endif #undef STOP_AT_MATCH /* no match, can't die */ #define SHENG_IMPL sheng_nm #define DEAD_FUNC dummyFunc #define ACCEPT_FUNC dummyFunc +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_nm +#define DEAD_FUNC32 dummyFunc +#define ACCEPT_FUNC32 dummyFunc +#define SHENG64_IMPL sheng64_nm +#define DEAD_FUNC64 dummyFunc +#define ACCEPT_FUNC64 dummyFunc +#endif #define STOP_AT_MATCH 0 #include "sheng_impl.h" #undef SHENG_IMPL #undef DEAD_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef DEAD_FUNC32 +#undef ACCEPT_FUNC32 +#undef SHENG64_IMPL +#undef DEAD_FUNC64 +#undef ACCEPT_FUNC64 +#endif #undef STOP_AT_MATCH /* @@ -144,6 +277,16 @@ u8 dummyFunc(UNUSED const u8 a) { #define INNER_ACCEL_FUNC isAccelState #define OUTER_ACCEL_FUNC dummyFunc #define ACCEPT_FUNC isAcceptState +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_4_coda +#define INTERESTING_FUNC32 hasInterestingStates32 +#define INNER_DEAD_FUNC32 isDeadState32 +#define OUTER_DEAD_FUNC32 dummyFunc +#define INNER_ACCEL_FUNC32 isAccelState32 +#define OUTER_ACCEL_FUNC32 dummyFunc +#define ACCEPT_FUNC32 isAcceptState32 +#define NO_SHENG64_IMPL +#endif #define STOP_AT_MATCH 0 #include "sheng_impl4.h" #undef SHENG_IMPL @@ -153,6 +296,16 @@ u8 dummyFunc(UNUSED const u8 a) { #undef INNER_ACCEL_FUNC #undef OUTER_ACCEL_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef INTERESTING_FUNC32 +#undef INNER_DEAD_FUNC32 +#undef OUTER_DEAD_FUNC32 +#undef INNER_ACCEL_FUNC32 +#undef OUTER_ACCEL_FUNC32 +#undef ACCEPT_FUNC32 +#undef NO_SHENG64_IMPL +#endif #undef STOP_AT_MATCH /* callback output, can die, not accelerated */ @@ -163,6 +316,20 @@ u8 dummyFunc(UNUSED const u8 a) { #define INNER_ACCEL_FUNC dummyFunc #define OUTER_ACCEL_FUNC dummyFunc #define ACCEPT_FUNC isAcceptState +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_4_cod +#define INTERESTING_FUNC32 hasInterestingStates32 +#define INNER_DEAD_FUNC32 isDeadState32 +#define OUTER_DEAD_FUNC32 dummyFunc +#define INNER_ACCEL_FUNC32 dummyFunc +#define OUTER_ACCEL_FUNC32 dummyFunc +#define ACCEPT_FUNC32 isAcceptState32 +#define SHENG64_IMPL sheng64_4_cod +#define INTERESTING_FUNC64 hasInterestingStates64 +#define INNER_DEAD_FUNC64 isDeadState64 +#define OUTER_DEAD_FUNC64 dummyFunc +#define ACCEPT_FUNC64 isAcceptState64 +#endif #define STOP_AT_MATCH 0 #include "sheng_impl4.h" #undef SHENG_IMPL @@ -172,6 +339,20 @@ u8 dummyFunc(UNUSED const u8 a) { #undef INNER_ACCEL_FUNC #undef OUTER_ACCEL_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef INTERESTING_FUNC32 +#undef INNER_DEAD_FUNC32 +#undef OUTER_DEAD_FUNC32 +#undef INNER_ACCEL_FUNC32 +#undef OUTER_ACCEL_FUNC32 +#undef ACCEPT_FUNC32 +#undef SHENG64_IMPL +#undef INTERESTING_FUNC64 +#undef INNER_DEAD_FUNC64 +#undef OUTER_DEAD_FUNC64 +#undef ACCEPT_FUNC64 +#endif #undef STOP_AT_MATCH /* callback output, can't die, accelerated */ @@ -182,6 +363,16 @@ u8 dummyFunc(UNUSED const u8 a) { #define INNER_ACCEL_FUNC isAccelState #define OUTER_ACCEL_FUNC dummyFunc #define ACCEPT_FUNC isAcceptState +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_4_coa +#define INTERESTING_FUNC32 hasInterestingStates32 +#define INNER_DEAD_FUNC32 dummyFunc +#define OUTER_DEAD_FUNC32 dummyFunc +#define INNER_ACCEL_FUNC32 isAccelState32 +#define OUTER_ACCEL_FUNC32 dummyFunc +#define ACCEPT_FUNC32 isAcceptState32 +#define NO_SHENG64_IMPL +#endif #define STOP_AT_MATCH 0 #include "sheng_impl4.h" #undef SHENG_IMPL @@ -191,6 +382,16 @@ u8 dummyFunc(UNUSED const u8 a) { #undef INNER_ACCEL_FUNC #undef OUTER_ACCEL_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef INTERESTING_FUNC32 +#undef INNER_DEAD_FUNC32 +#undef OUTER_DEAD_FUNC32 +#undef INNER_ACCEL_FUNC32 +#undef OUTER_ACCEL_FUNC32 +#undef ACCEPT_FUNC32 +#undef NO_SHENG64_IMPL +#endif #undef STOP_AT_MATCH /* callback output, can't die, not accelerated */ @@ -201,6 +402,20 @@ u8 dummyFunc(UNUSED const u8 a) { #define INNER_ACCEL_FUNC dummyFunc #define OUTER_ACCEL_FUNC dummyFunc #define ACCEPT_FUNC isAcceptState +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_4_co +#define INTERESTING_FUNC32 hasInterestingStates32 +#define INNER_DEAD_FUNC32 dummyFunc +#define OUTER_DEAD_FUNC32 dummyFunc +#define INNER_ACCEL_FUNC32 dummyFunc +#define OUTER_ACCEL_FUNC32 dummyFunc +#define ACCEPT_FUNC32 isAcceptState32 +#define SHENG64_IMPL sheng64_4_co +#define INTERESTING_FUNC64 hasInterestingStates64 +#define INNER_DEAD_FUNC64 dummyFunc +#define OUTER_DEAD_FUNC64 dummyFunc +#define ACCEPT_FUNC64 isAcceptState64 +#endif #define STOP_AT_MATCH 0 #include "sheng_impl4.h" #undef SHENG_IMPL @@ -210,6 +425,20 @@ u8 dummyFunc(UNUSED const u8 a) { #undef INNER_ACCEL_FUNC #undef OUTER_ACCEL_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef INTERESTING_FUNC32 +#undef INNER_DEAD_FUNC32 +#undef OUTER_DEAD_FUNC32 +#undef INNER_ACCEL_FUNC32 +#undef OUTER_ACCEL_FUNC32 +#undef ACCEPT_FUNC32 +#undef SHENG64_IMPL +#undef INTERESTING_FUNC64 +#undef INNER_DEAD_FUNC64 +#undef OUTER_DEAD_FUNC64 +#undef ACCEPT_FUNC64 +#endif #undef STOP_AT_MATCH /* stop at match, can die, accelerated */ @@ -220,6 +449,16 @@ u8 dummyFunc(UNUSED const u8 a) { #define INNER_ACCEL_FUNC isAccelState #define OUTER_ACCEL_FUNC dummyFunc #define ACCEPT_FUNC isAcceptState +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_4_samda +#define INTERESTING_FUNC32 hasInterestingStates32 +#define INNER_DEAD_FUNC32 isDeadState32 +#define OUTER_DEAD_FUNC32 dummyFunc +#define INNER_ACCEL_FUNC32 isAccelState32 +#define OUTER_ACCEL_FUNC32 dummyFunc +#define ACCEPT_FUNC32 isAcceptState32 +#define NO_SHENG64_IMPL +#endif #define STOP_AT_MATCH 1 #include "sheng_impl4.h" #undef SHENG_IMPL @@ -229,6 +468,16 @@ u8 dummyFunc(UNUSED const u8 a) { #undef INNER_ACCEL_FUNC #undef OUTER_ACCEL_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef INTERESTING_FUNC32 +#undef INNER_DEAD_FUNC32 +#undef OUTER_DEAD_FUNC32 +#undef INNER_ACCEL_FUNC32 +#undef OUTER_ACCEL_FUNC32 +#undef ACCEPT_FUNC32 +#undef NO_SHENG64_IMPL +#endif #undef STOP_AT_MATCH /* stop at match, can die, not accelerated */ @@ -239,6 +488,20 @@ u8 dummyFunc(UNUSED const u8 a) { #define INNER_ACCEL_FUNC dummyFunc #define OUTER_ACCEL_FUNC dummyFunc #define ACCEPT_FUNC isAcceptState +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_4_samd +#define INTERESTING_FUNC32 hasInterestingStates32 +#define INNER_DEAD_FUNC32 isDeadState32 +#define OUTER_DEAD_FUNC32 dummyFunc +#define INNER_ACCEL_FUNC32 dummyFunc +#define OUTER_ACCEL_FUNC32 dummyFunc +#define ACCEPT_FUNC32 isAcceptState32 +#define SHENG64_IMPL sheng64_4_samd +#define INTERESTING_FUNC64 hasInterestingStates64 +#define INNER_DEAD_FUNC64 isDeadState64 +#define OUTER_DEAD_FUNC64 dummyFunc +#define ACCEPT_FUNC64 isAcceptState64 +#endif #define STOP_AT_MATCH 1 #include "sheng_impl4.h" #undef SHENG_IMPL @@ -248,6 +511,20 @@ u8 dummyFunc(UNUSED const u8 a) { #undef INNER_ACCEL_FUNC #undef OUTER_ACCEL_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef INTERESTING_FUNC32 +#undef INNER_DEAD_FUNC32 +#undef OUTER_DEAD_FUNC32 +#undef INNER_ACCEL_FUNC32 +#undef OUTER_ACCEL_FUNC32 +#undef ACCEPT_FUNC32 +#undef SHENG64_IMPL +#undef INTERESTING_FUNC64 +#undef INNER_DEAD_FUNC64 +#undef OUTER_DEAD_FUNC64 +#undef ACCEPT_FUNC64 +#endif #undef STOP_AT_MATCH /* stop at match, can't die, accelerated */ @@ -258,6 +535,16 @@ u8 dummyFunc(UNUSED const u8 a) { #define INNER_ACCEL_FUNC isAccelState #define OUTER_ACCEL_FUNC dummyFunc #define ACCEPT_FUNC isAcceptState +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_4_sama +#define INTERESTING_FUNC32 hasInterestingStates32 +#define INNER_DEAD_FUNC32 dummyFunc +#define OUTER_DEAD_FUNC32 dummyFunc +#define INNER_ACCEL_FUNC32 isAccelState32 +#define OUTER_ACCEL_FUNC32 dummyFunc +#define ACCEPT_FUNC32 isAcceptState32 +#define NO_SHENG64_IMPL +#endif #define STOP_AT_MATCH 1 #include "sheng_impl4.h" #undef SHENG_IMPL @@ -267,6 +554,16 @@ u8 dummyFunc(UNUSED const u8 a) { #undef INNER_ACCEL_FUNC #undef OUTER_ACCEL_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef INTERESTING_FUNC32 +#undef INNER_DEAD_FUNC32 +#undef OUTER_DEAD_FUNC32 +#undef INNER_ACCEL_FUNC32 +#undef OUTER_ACCEL_FUNC32 +#undef ACCEPT_FUNC32 +#undef NO_SHENG64_IMPL +#endif #undef STOP_AT_MATCH /* stop at match, can't die, not accelerated */ @@ -277,6 +574,20 @@ u8 dummyFunc(UNUSED const u8 a) { #define INNER_ACCEL_FUNC dummyFunc #define OUTER_ACCEL_FUNC dummyFunc #define ACCEPT_FUNC isAcceptState +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_4_sam +#define INTERESTING_FUNC32 hasInterestingStates32 +#define INNER_DEAD_FUNC32 dummyFunc +#define OUTER_DEAD_FUNC32 dummyFunc +#define INNER_ACCEL_FUNC32 dummyFunc +#define OUTER_ACCEL_FUNC32 dummyFunc +#define ACCEPT_FUNC32 isAcceptState32 +#define SHENG64_IMPL sheng64_4_sam +#define INTERESTING_FUNC64 hasInterestingStates64 +#define INNER_DEAD_FUNC64 dummyFunc +#define OUTER_DEAD_FUNC64 dummyFunc +#define ACCEPT_FUNC64 isAcceptState64 +#endif #define STOP_AT_MATCH 1 #include "sheng_impl4.h" #undef SHENG_IMPL @@ -286,6 +597,20 @@ u8 dummyFunc(UNUSED const u8 a) { #undef INNER_ACCEL_FUNC #undef OUTER_ACCEL_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef INTERESTING_FUNC32 +#undef INNER_DEAD_FUNC32 +#undef OUTER_DEAD_FUNC32 +#undef INNER_ACCEL_FUNC32 +#undef OUTER_ACCEL_FUNC32 +#undef ACCEPT_FUNC32 +#undef SHENG64_IMPL +#undef INTERESTING_FUNC64 +#undef INNER_DEAD_FUNC64 +#undef OUTER_DEAD_FUNC64 +#undef ACCEPT_FUNC64 +#endif #undef STOP_AT_MATCH /* no-match have interesting func as dummy, and die/accel checks are outer */ @@ -298,6 +623,16 @@ u8 dummyFunc(UNUSED const u8 a) { #define INNER_ACCEL_FUNC dummyFunc #define OUTER_ACCEL_FUNC isAccelState #define ACCEPT_FUNC dummyFunc +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_4_nmda +#define INTERESTING_FUNC32 dummyFunc4 +#define INNER_DEAD_FUNC32 dummyFunc +#define OUTER_DEAD_FUNC32 isDeadState32 +#define INNER_ACCEL_FUNC32 dummyFunc +#define OUTER_ACCEL_FUNC32 isAccelState32 +#define ACCEPT_FUNC32 dummyFunc +#define NO_SHENG64_IMPL +#endif #define STOP_AT_MATCH 0 #include "sheng_impl4.h" #undef SHENG_IMPL @@ -307,6 +642,16 @@ u8 dummyFunc(UNUSED const u8 a) { #undef INNER_ACCEL_FUNC #undef OUTER_ACCEL_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef INTERESTING_FUNC32 +#undef INNER_DEAD_FUNC32 +#undef OUTER_DEAD_FUNC32 +#undef INNER_ACCEL_FUNC32 +#undef OUTER_ACCEL_FUNC32 +#undef ACCEPT_FUNC32 +#undef NO_SHENG64_IMPL +#endif #undef STOP_AT_MATCH /* no match, can die, not accelerated */ @@ -317,6 +662,20 @@ u8 dummyFunc(UNUSED const u8 a) { #define INNER_ACCEL_FUNC dummyFunc #define OUTER_ACCEL_FUNC dummyFunc #define ACCEPT_FUNC dummyFunc +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_4_nmd +#define INTERESTING_FUNC32 dummyFunc4 +#define INNER_DEAD_FUNC32 dummyFunc +#define OUTER_DEAD_FUNC32 isDeadState32 +#define INNER_ACCEL_FUNC32 dummyFunc +#define OUTER_ACCEL_FUNC32 dummyFunc +#define ACCEPT_FUNC32 dummyFunc +#define SHENG64_IMPL sheng64_4_nmd +#define INTERESTING_FUNC64 dummyFunc4 +#define INNER_DEAD_FUNC64 dummyFunc +#define OUTER_DEAD_FUNC64 isDeadState64 +#define ACCEPT_FUNC64 dummyFunc +#endif #define STOP_AT_MATCH 0 #include "sheng_impl4.h" #undef SHENG_IMPL @@ -326,6 +685,20 @@ u8 dummyFunc(UNUSED const u8 a) { #undef INNER_ACCEL_FUNC #undef OUTER_ACCEL_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef INTERESTING_FUNC32 +#undef INNER_DEAD_FUNC32 +#undef OUTER_DEAD_FUNC32 +#undef INNER_ACCEL_FUNC32 +#undef OUTER_ACCEL_FUNC32 +#undef ACCEPT_FUNC32 +#undef SHENG64_IMPL +#undef INTERESTING_FUNC64 +#undef INNER_DEAD_FUNC64 +#undef OUTER_DEAD_FUNC64 +#undef ACCEPT_FUNC64 +#endif #undef STOP_AT_MATCH /* there is no performance benefit in accelerating a no-match case that can't @@ -339,6 +712,20 @@ u8 dummyFunc(UNUSED const u8 a) { #define INNER_ACCEL_FUNC dummyFunc #define OUTER_ACCEL_FUNC dummyFunc #define ACCEPT_FUNC dummyFunc +#if defined(HAVE_AVX512VBMI) +#define SHENG32_IMPL sheng32_4_nm +#define INTERESTING_FUNC32 dummyFunc4 +#define INNER_DEAD_FUNC32 dummyFunc +#define OUTER_DEAD_FUNC32 dummyFunc +#define INNER_ACCEL_FUNC32 dummyFunc +#define OUTER_ACCEL_FUNC32 dummyFunc +#define ACCEPT_FUNC32 dummyFunc +#define SHENG64_IMPL sheng64_4_nm +#define INTERESTING_FUNC64 dummyFunc4 +#define INNER_DEAD_FUNC64 dummyFunc +#define OUTER_DEAD_FUNC64 dummyFunc +#define ACCEPT_FUNC64 dummyFunc +#endif #define STOP_AT_MATCH 0 #include "sheng_impl4.h" #undef SHENG_IMPL @@ -348,6 +735,20 @@ u8 dummyFunc(UNUSED const u8 a) { #undef INNER_ACCEL_FUNC #undef OUTER_ACCEL_FUNC #undef ACCEPT_FUNC +#if defined(HAVE_AVX512VBMI) +#undef SHENG32_IMPL +#undef INTERESTING_FUNC32 +#undef INNER_DEAD_FUNC32 +#undef OUTER_DEAD_FUNC32 +#undef INNER_ACCEL_FUNC32 +#undef OUTER_ACCEL_FUNC32 +#undef ACCEPT_FUNC32 +#undef SHENG64_IMPL +#undef INTERESTING_FUNC64 +#undef INNER_DEAD_FUNC64 +#undef OUTER_DEAD_FUNC64 +#undef ACCEPT_FUNC64 +#endif #undef STOP_AT_MATCH #endif // SHENG_DEFS_H diff --git a/src/nfa/sheng_impl.h b/src/nfa/sheng_impl.h index 9552fe15d..fb8ee1683 100644 --- a/src/nfa/sheng_impl.h +++ b/src/nfa/sheng_impl.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -95,3 +95,127 @@ char SHENG_IMPL(u8 *state, NfaCallback cb, void *ctxt, const struct sheng *s, *scan_end = cur_buf; return MO_CONTINUE_MATCHING; } + +#if defined(HAVE_AVX512VBMI) +static really_inline +char SHENG32_IMPL(u8 *state, NfaCallback cb, void *ctxt, + const struct sheng32 *s, + u8 *const cached_accept_state, + ReportID *const cached_accept_id, + u8 single, u64a base_offset, const u8 *buf, const u8 *start, + const u8 *end, const u8 **scan_end) { + DEBUG_PRINTF("Starting DFA execution in state %u\n", + *state & SHENG32_STATE_MASK); + const u8 *cur_buf = start; + if (DEAD_FUNC32(*state)) { + DEBUG_PRINTF("Dead on arrival\n"); + *scan_end = end; + return MO_CONTINUE_MATCHING; + } + DEBUG_PRINTF("Scanning %lli bytes\n", (s64a)(end - start)); + + m512 cur_state = set64x8(*state); + const m512 *masks = s->succ_masks; + + while (likely(cur_buf != end)) { + const u8 c = *cur_buf; + const m512 succ_mask = masks[c]; + cur_state = vpermb512(cur_state, succ_mask); + const u8 tmp = movd512(cur_state); + + DEBUG_PRINTF("c: %02hhx '%c'\n", c, ourisprint(c) ? c : '?'); + DEBUG_PRINTF("s: %u (flag: %u)\n", tmp & SHENG32_STATE_MASK, + tmp & SHENG32_STATE_FLAG_MASK); + + if (unlikely(ACCEPT_FUNC32(tmp))) { + DEBUG_PRINTF("Accept state %u reached\n", tmp & SHENG32_STATE_MASK); + u64a match_offset = base_offset + (cur_buf - buf) + 1; + DEBUG_PRINTF("Match @ %llu\n", match_offset); + if (STOP_AT_MATCH) { + DEBUG_PRINTF("Stopping at match @ %lli\n", + (u64a)(cur_buf - start)); + *state = tmp; + *scan_end = cur_buf; + return MO_MATCHES_PENDING; + } + if (single) { + if (fireSingleReport(cb, ctxt, s->report, match_offset) == + MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } else { + if (fireReports32(s, cb, ctxt, tmp, match_offset, + cached_accept_state, cached_accept_id, + 0) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } + } + cur_buf++; + } + *state = movd512(cur_state); + *scan_end = cur_buf; + return MO_CONTINUE_MATCHING; +} + +static really_inline +char SHENG64_IMPL(u8 *state, NfaCallback cb, void *ctxt, + const struct sheng64 *s, + u8 *const cached_accept_state, + ReportID *const cached_accept_id, + u8 single, u64a base_offset, const u8 *buf, const u8 *start, + const u8 *end, const u8 **scan_end) { + DEBUG_PRINTF("Starting DFA execution in state %u\n", + *state & SHENG64_STATE_MASK); + const u8 *cur_buf = start; + if (DEAD_FUNC64(*state)) { + DEBUG_PRINTF("Dead on arrival\n"); + *scan_end = end; + return MO_CONTINUE_MATCHING; + } + DEBUG_PRINTF("Scanning %lli bytes\n", (s64a)(end - start)); + + m512 cur_state = set64x8(*state); + const m512 *masks = s->succ_masks; + + while (likely(cur_buf != end)) { + const u8 c = *cur_buf; + const m512 succ_mask = masks[c]; + cur_state = vpermb512(cur_state, succ_mask); + const u8 tmp = movd512(cur_state); + + DEBUG_PRINTF("c: %02hhx '%c'\n", c, ourisprint(c) ? c : '?'); + DEBUG_PRINTF("s: %u (flag: %u)\n", tmp & SHENG64_STATE_MASK, + tmp & SHENG64_STATE_FLAG_MASK); + + if (unlikely(ACCEPT_FUNC64(tmp))) { + DEBUG_PRINTF("Accept state %u reached\n", tmp & SHENG64_STATE_MASK); + u64a match_offset = base_offset + (cur_buf - buf) + 1; + DEBUG_PRINTF("Match @ %llu\n", match_offset); + if (STOP_AT_MATCH) { + DEBUG_PRINTF("Stopping at match @ %lli\n", + (u64a)(cur_buf - start)); + *state = tmp; + *scan_end = cur_buf; + return MO_MATCHES_PENDING; + } + if (single) { + if (fireSingleReport(cb, ctxt, s->report, match_offset) == + MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } else { + if (fireReports64(s, cb, ctxt, tmp, match_offset, + cached_accept_state, cached_accept_id, + 0) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } + } + cur_buf++; + } + *state = movd512(cur_state); + *scan_end = cur_buf; + return MO_CONTINUE_MATCHING; +} +#endif diff --git a/src/nfa/sheng_impl4.h b/src/nfa/sheng_impl4.h index 740322010..440e7396e 100644 --- a/src/nfa/sheng_impl4.h +++ b/src/nfa/sheng_impl4.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -282,3 +282,430 @@ char SHENG_IMPL(u8 *state, NfaCallback cb, void *ctxt, const struct sheng *s, *scan_end = cur_buf; return MO_CONTINUE_MATCHING; } + +#if defined(HAVE_AVX512VBMI) +static really_inline +char SHENG32_IMPL(u8 *state, NfaCallback cb, void *ctxt, + const struct sheng32 *s, + u8 *const cached_accept_state, + ReportID *const cached_accept_id, + u8 single, u64a base_offset, const u8 *buf, const u8 *start, + const u8 *end, const u8 **scan_end) { + DEBUG_PRINTF("Starting DFAx4 execution in state %u\n", + *state & SHENG32_STATE_MASK); + const u8 *cur_buf = start; + const u8 *min_accel_dist = start; + base_offset++; + DEBUG_PRINTF("Scanning %llu bytes\n", (u64a)(end - start)); + + if (INNER_ACCEL_FUNC32(*state) || OUTER_ACCEL_FUNC32(*state)) { + DEBUG_PRINTF("Accel state reached @ 0\n"); + const union AccelAux *aaux = + get_accel32(s, *state & SHENG32_STATE_MASK); + const u8 *new_offset = run_accel(aaux, cur_buf, end); + if (new_offset < cur_buf + BAD_ACCEL_DIST) { + min_accel_dist = new_offset + BIG_ACCEL_PENALTY; + } else { + min_accel_dist = new_offset + SMALL_ACCEL_PENALTY; + } + DEBUG_PRINTF("Next accel chance: %llu\n", + (u64a)(min_accel_dist - start)); + DEBUG_PRINTF("Accel scanned %zu bytes\n", new_offset - cur_buf); + cur_buf = new_offset; + DEBUG_PRINTF("New offset: %lli\n", (s64a)(cur_buf - start)); + } + if (INNER_DEAD_FUNC32(*state) || OUTER_DEAD_FUNC32(*state)) { + DEBUG_PRINTF("Dead on arrival\n"); + *scan_end = end; + return MO_CONTINUE_MATCHING; + } + + m512 cur_state = set64x8(*state); + const m512 *masks = s->succ_masks; + + while (likely(end - cur_buf >= 4)) { + const u8 *b1 = cur_buf; + const u8 *b2 = cur_buf + 1; + const u8 *b3 = cur_buf + 2; + const u8 *b4 = cur_buf + 3; + const u8 c1 = *b1; + const u8 c2 = *b2; + const u8 c3 = *b3; + const u8 c4 = *b4; + + const m512 succ_mask1 = masks[c1]; + cur_state = vpermb512(cur_state, succ_mask1); + const u8 a1 = movd512(cur_state); + + const m512 succ_mask2 = masks[c2]; + cur_state = vpermb512(cur_state, succ_mask2); + const u8 a2 = movd512(cur_state); + + const m512 succ_mask3 = masks[c3]; + cur_state = vpermb512(cur_state, succ_mask3); + const u8 a3 = movd512(cur_state); + + const m512 succ_mask4 = masks[c4]; + cur_state = vpermb512(cur_state, succ_mask4); + const u8 a4 = movd512(cur_state); + + DEBUG_PRINTF("c: %02hhx '%c'\n", c1, ourisprint(c1) ? c1 : '?'); + DEBUG_PRINTF("s: %u (flag: %u)\n", a1 & SHENG32_STATE_MASK, + a1 & SHENG32_STATE_FLAG_MASK); + + DEBUG_PRINTF("c: %02hhx '%c'\n", c2, ourisprint(c2) ? c2 : '?'); + DEBUG_PRINTF("s: %u (flag: %u)\n", a2 & SHENG32_STATE_MASK, + a2 & SHENG32_STATE_FLAG_MASK); + + DEBUG_PRINTF("c: %02hhx '%c'\n", c3, ourisprint(c3) ? c3 : '?'); + DEBUG_PRINTF("s: %u (flag: %u)\n", a3 & SHENG32_STATE_MASK, + a3 & SHENG32_STATE_FLAG_MASK); + + DEBUG_PRINTF("c: %02hhx '%c'\n", c4, ourisprint(c4) ? c4 : '?'); + DEBUG_PRINTF("s: %u (flag: %u)\n", a4 & SHENG32_STATE_MASK, + a4 & SHENG32_STATE_FLAG_MASK); + + if (unlikely(INTERESTING_FUNC32(a1, a2, a3, a4))) { + if (ACCEPT_FUNC32(a1)) { + u64a match_offset = base_offset + b1 - buf; + DEBUG_PRINTF("Accept state %u reached\n", + a1 & SHENG32_STATE_MASK); + DEBUG_PRINTF("Match @ %llu\n", match_offset); + if (STOP_AT_MATCH) { + DEBUG_PRINTF("Stopping at match @ %lli\n", + (s64a)(b1 - start)); + *scan_end = b1; + *state = a1; + return MO_MATCHES_PENDING; + } + if (single) { + if (fireSingleReport(cb, ctxt, s->report, match_offset) == + MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } else { + if (fireReports32(s, cb, ctxt, a1, match_offset, + cached_accept_state, cached_accept_id, + 0) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } + } + if (ACCEPT_FUNC32(a2)) { + u64a match_offset = base_offset + b2 - buf; + DEBUG_PRINTF("Accept state %u reached\n", + a2 & SHENG32_STATE_MASK); + DEBUG_PRINTF("Match @ %llu\n", match_offset); + if (STOP_AT_MATCH) { + DEBUG_PRINTF("Stopping at match @ %lli\n", + (s64a)(b2 - start)); + *scan_end = b2; + *state = a2; + return MO_MATCHES_PENDING; + } + if (single) { + if (fireSingleReport(cb, ctxt, s->report, match_offset) == + MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } else { + if (fireReports32(s, cb, ctxt, a2, match_offset, + cached_accept_state, cached_accept_id, + 0) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } + } + if (ACCEPT_FUNC32(a3)) { + u64a match_offset = base_offset + b3 - buf; + DEBUG_PRINTF("Accept state %u reached\n", + a3 & SHENG32_STATE_MASK); + DEBUG_PRINTF("Match @ %llu\n", match_offset); + if (STOP_AT_MATCH) { + DEBUG_PRINTF("Stopping at match @ %lli\n", + (s64a)(b3 - start)); + *scan_end = b3; + *state = a3; + return MO_MATCHES_PENDING; + } + if (single) { + if (fireSingleReport(cb, ctxt, s->report, match_offset) == + MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } else { + if (fireReports32(s, cb, ctxt, a3, match_offset, + cached_accept_state, cached_accept_id, + 0) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } + } + if (ACCEPT_FUNC32(a4)) { + u64a match_offset = base_offset + b4 - buf; + DEBUG_PRINTF("Accept state %u reached\n", + a4 & SHENG32_STATE_MASK); + DEBUG_PRINTF("Match @ %llu\n", match_offset); + if (STOP_AT_MATCH) { + DEBUG_PRINTF("Stopping at match @ %lli\n", + (s64a)(b4 - start)); + *scan_end = b4; + *state = a4; + return MO_MATCHES_PENDING; + } + if (single) { + if (fireSingleReport(cb, ctxt, s->report, match_offset) == + MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } else { + if (fireReports32(s, cb, ctxt, a4, match_offset, + cached_accept_state, cached_accept_id, + 0) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } + } + if (INNER_DEAD_FUNC32(a4)) { + DEBUG_PRINTF("Dead state reached @ %lli\n", (s64a)(b4 - buf)); + *scan_end = end; + *state = a4; + return MO_CONTINUE_MATCHING; + } + if (cur_buf > min_accel_dist && INNER_ACCEL_FUNC32(a4)) { + DEBUG_PRINTF("Accel state reached @ %lli\n", (s64a)(b4 - buf)); + const union AccelAux *aaux = + get_accel32(s, a4 & SHENG32_STATE_MASK); + const u8 *new_offset = run_accel(aaux, cur_buf + 4, end); + if (new_offset < cur_buf + 4 + BAD_ACCEL_DIST) { + min_accel_dist = new_offset + BIG_ACCEL_PENALTY; + } else { + min_accel_dist = new_offset + SMALL_ACCEL_PENALTY; + } + DEBUG_PRINTF("Next accel chance: %llu\n", + (u64a)(min_accel_dist - start)); + DEBUG_PRINTF("Accel scanned %llu bytes\n", + (u64a)(new_offset - cur_buf - 4)); + cur_buf = new_offset; + DEBUG_PRINTF("New offset: %llu\n", (u64a)(cur_buf - buf)); + continue; + } + } + if (OUTER_DEAD_FUNC32(a4)) { + DEBUG_PRINTF("Dead state reached @ %lli\n", (s64a)(cur_buf - buf)); + *scan_end = end; + *state = a4; + return MO_CONTINUE_MATCHING; + }; + if (cur_buf > min_accel_dist && OUTER_ACCEL_FUNC32(a4)) { + DEBUG_PRINTF("Accel state reached @ %lli\n", (s64a)(b4 - buf)); + const union AccelAux *aaux = + get_accel32(s, a4 & SHENG32_STATE_MASK); + const u8 *new_offset = run_accel(aaux, cur_buf + 4, end); + if (new_offset < cur_buf + 4 + BAD_ACCEL_DIST) { + min_accel_dist = new_offset + BIG_ACCEL_PENALTY; + } else { + min_accel_dist = new_offset + SMALL_ACCEL_PENALTY; + } + DEBUG_PRINTF("Next accel chance: %llu\n", + (u64a)(min_accel_dist - start)); + DEBUG_PRINTF("Accel scanned %llu bytes\n", + (u64a)(new_offset - cur_buf - 4)); + cur_buf = new_offset; + DEBUG_PRINTF("New offset: %llu\n", (u64a)(cur_buf - buf)); + continue; + }; + cur_buf += 4; + } + *state = movd512(cur_state); + *scan_end = cur_buf; + return MO_CONTINUE_MATCHING; +} + +#ifndef NO_SHENG64_IMPL +static really_inline +char SHENG64_IMPL(u8 *state, NfaCallback cb, void *ctxt, + const struct sheng64 *s, + u8 *const cached_accept_state, + ReportID *const cached_accept_id, + u8 single, u64a base_offset, const u8 *buf, const u8 *start, + const u8 *end, const u8 **scan_end) { + DEBUG_PRINTF("Starting DFAx4 execution in state %u\n", + *state & SHENG64_STATE_MASK); + const u8 *cur_buf = start; + base_offset++; + DEBUG_PRINTF("Scanning %llu bytes\n", (u64a)(end - start)); + + if (INNER_DEAD_FUNC64(*state) || OUTER_DEAD_FUNC64(*state)) { + DEBUG_PRINTF("Dead on arrival\n"); + *scan_end = end; + return MO_CONTINUE_MATCHING; + } + + m512 cur_state = set64x8(*state); + const m512 *masks = s->succ_masks; + + while (likely(end - cur_buf >= 4)) { + const u8 *b1 = cur_buf; + const u8 *b2 = cur_buf + 1; + const u8 *b3 = cur_buf + 2; + const u8 *b4 = cur_buf + 3; + const u8 c1 = *b1; + const u8 c2 = *b2; + const u8 c3 = *b3; + const u8 c4 = *b4; + + const m512 succ_mask1 = masks[c1]; + cur_state = vpermb512(cur_state, succ_mask1); + const u8 a1 = movd512(cur_state); + + const m512 succ_mask2 = masks[c2]; + cur_state = vpermb512(cur_state, succ_mask2); + const u8 a2 = movd512(cur_state); + + const m512 succ_mask3 = masks[c3]; + cur_state = vpermb512(cur_state, succ_mask3); + const u8 a3 = movd512(cur_state); + + const m512 succ_mask4 = masks[c4]; + cur_state = vpermb512(cur_state, succ_mask4); + const u8 a4 = movd512(cur_state); + + DEBUG_PRINTF("c: %02hhx '%c'\n", c1, ourisprint(c1) ? c1 : '?'); + DEBUG_PRINTF("s: %u (flag: %u)\n", a1 & SHENG64_STATE_MASK, + a1 & SHENG64_STATE_FLAG_MASK); + + DEBUG_PRINTF("c: %02hhx '%c'\n", c2, ourisprint(c2) ? c2 : '?'); + DEBUG_PRINTF("s: %u (flag: %u)\n", a2 & SHENG64_STATE_MASK, + a2 & SHENG64_STATE_FLAG_MASK); + + DEBUG_PRINTF("c: %02hhx '%c'\n", c3, ourisprint(c3) ? c3 : '?'); + DEBUG_PRINTF("s: %u (flag: %u)\n", a3 & SHENG64_STATE_MASK, + a3 & SHENG64_STATE_FLAG_MASK); + + DEBUG_PRINTF("c: %02hhx '%c'\n", c4, ourisprint(c4) ? c4 : '?'); + DEBUG_PRINTF("s: %u (flag: %u)\n", a4 & SHENG64_STATE_MASK, + a4 & SHENG64_STATE_FLAG_MASK); + + if (unlikely(INTERESTING_FUNC64(a1, a2, a3, a4))) { + if (ACCEPT_FUNC64(a1)) { + u64a match_offset = base_offset + b1 - buf; + DEBUG_PRINTF("Accept state %u reached\n", + a1 & SHENG64_STATE_MASK); + DEBUG_PRINTF("Match @ %llu\n", match_offset); + if (STOP_AT_MATCH) { + DEBUG_PRINTF("Stopping at match @ %lli\n", + (s64a)(b1 - start)); + *scan_end = b1; + *state = a1; + return MO_MATCHES_PENDING; + } + if (single) { + if (fireSingleReport(cb, ctxt, s->report, match_offset) == + MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } else { + if (fireReports64(s, cb, ctxt, a1, match_offset, + cached_accept_state, cached_accept_id, + 0) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } + } + if (ACCEPT_FUNC64(a2)) { + u64a match_offset = base_offset + b2 - buf; + DEBUG_PRINTF("Accept state %u reached\n", + a2 & SHENG64_STATE_MASK); + DEBUG_PRINTF("Match @ %llu\n", match_offset); + if (STOP_AT_MATCH) { + DEBUG_PRINTF("Stopping at match @ %lli\n", + (s64a)(b2 - start)); + *scan_end = b2; + *state = a2; + return MO_MATCHES_PENDING; + } + if (single) { + if (fireSingleReport(cb, ctxt, s->report, match_offset) == + MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } else { + if (fireReports64(s, cb, ctxt, a2, match_offset, + cached_accept_state, cached_accept_id, + 0) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } + } + if (ACCEPT_FUNC64(a3)) { + u64a match_offset = base_offset + b3 - buf; + DEBUG_PRINTF("Accept state %u reached\n", + a3 & SHENG64_STATE_MASK); + DEBUG_PRINTF("Match @ %llu\n", match_offset); + if (STOP_AT_MATCH) { + DEBUG_PRINTF("Stopping at match @ %lli\n", + (s64a)(b3 - start)); + *scan_end = b3; + *state = a3; + return MO_MATCHES_PENDING; + } + if (single) { + if (fireSingleReport(cb, ctxt, s->report, match_offset) == + MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } else { + if (fireReports64(s, cb, ctxt, a3, match_offset, + cached_accept_state, cached_accept_id, + 0) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } + } + if (ACCEPT_FUNC64(a4)) { + u64a match_offset = base_offset + b4 - buf; + DEBUG_PRINTF("Accept state %u reached\n", + a4 & SHENG64_STATE_MASK); + DEBUG_PRINTF("Match @ %llu\n", match_offset); + if (STOP_AT_MATCH) { + DEBUG_PRINTF("Stopping at match @ %lli\n", + (s64a)(b4 - start)); + *scan_end = b4; + *state = a4; + return MO_MATCHES_PENDING; + } + if (single) { + if (fireSingleReport(cb, ctxt, s->report, match_offset) == + MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } else { + if (fireReports64(s, cb, ctxt, a4, match_offset, + cached_accept_state, cached_accept_id, + 0) == MO_HALT_MATCHING) { + return MO_HALT_MATCHING; + } + } + } + if (INNER_DEAD_FUNC64(a4)) { + DEBUG_PRINTF("Dead state reached @ %lli\n", (s64a)(b4 - buf)); + *scan_end = end; + *state = a4; + return MO_CONTINUE_MATCHING; + } + } + if (OUTER_DEAD_FUNC64(a4)) { + DEBUG_PRINTF("Dead state reached @ %lli\n", (s64a)(cur_buf - buf)); + *scan_end = end; + *state = a4; + return MO_CONTINUE_MATCHING; + } + cur_buf += 4; + } + *state = movd512(cur_state); + *scan_end = cur_buf; + return MO_CONTINUE_MATCHING; +} +#endif // !NO_SHENG64_IMPL +#endif diff --git a/src/nfa/sheng_internal.h b/src/nfa/sheng_internal.h index ff843ebee..98536886c 100644 --- a/src/nfa/sheng_internal.h +++ b/src/nfa/sheng_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,6 +38,17 @@ #define SHENG_STATE_MASK 0xF #define SHENG_STATE_FLAG_MASK 0x70 +#define SHENG32_STATE_ACCEPT 0x20 +#define SHENG32_STATE_DEAD 0x40 +#define SHENG32_STATE_ACCEL 0x80 +#define SHENG32_STATE_MASK 0x1F +#define SHENG32_STATE_FLAG_MASK 0xE0 + +#define SHENG64_STATE_ACCEPT 0x40 +#define SHENG64_STATE_DEAD 0x80 +#define SHENG64_STATE_MASK 0x3F +#define SHENG64_STATE_FLAG_MASK 0xC0 + #define SHENG_FLAG_SINGLE_REPORT 0x1 #define SHENG_FLAG_CAN_DIE 0x2 #define SHENG_FLAG_HAS_ACCEL 0x4 @@ -67,4 +78,30 @@ struct sheng { ReportID report; }; +struct sheng32 { + m512 succ_masks[256]; + u32 length; + u32 aux_offset; + u32 report_offset; + u32 accel_offset; + u8 n_states; + u8 anchored; + u8 floating; + u8 flags; + ReportID report; +}; + +struct sheng64 { + m512 succ_masks[256]; + u32 length; + u32 aux_offset; + u32 report_offset; + u32 accel_offset; + u8 n_states; + u8 anchored; + u8 floating; + u8 flags; + ReportID report; +}; + #endif /* SHENG_INTERNAL_H_ */ diff --git a/src/nfa/shengcompile.cpp b/src/nfa/shengcompile.cpp index c4094cedc..aa3faeb09 100644 --- a/src/nfa/shengcompile.cpp +++ b/src/nfa/shengcompile.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -301,6 +301,28 @@ void dumpShuffleMask(const u8 chr, const u8 *buf, unsigned sz) { } DEBUG_PRINTF("chr %3u: %s\n", chr, o.str().c_str()); } + +static really_inline +void dumpShuffleMask32(const u8 chr, const u8 *buf, unsigned sz) { + stringstream o; + + for (unsigned i = 0; i < sz; i++) { + o.width(2); + o << (buf[i] & SHENG32_STATE_MASK) << " "; + } + DEBUG_PRINTF("chr %3u: %s\n", chr, o.str().c_str()); +} + +static really_inline +void dumpShuffleMask64(const u8 chr, const u8 *buf, unsigned sz) { + stringstream o; + + for (unsigned i = 0; i < sz; i++) { + o.width(2); + o << (buf[i] & SHENG64_STATE_MASK) << " "; + } + DEBUG_PRINTF("chr %3u: %s\n", chr, o.str().c_str()); +} #endif static @@ -311,9 +333,16 @@ void fillAccelOut(const map &accel_escape_info, } } +template static -u8 getShengState(dstate &state, dfa_info &info, - map &accelInfo) { +u8 getShengState(UNUSED dstate &state, UNUSED dfa_info &info, + UNUSED map &accelInfo) { + return 0; +} + +template <> +u8 getShengState(dstate &state, dfa_info &info, + map &accelInfo) { u8 s = state.impl_id; if (!state.reports.empty()) { s |= SHENG_STATE_ACCEPT; @@ -327,11 +356,41 @@ u8 getShengState(dstate &state, dfa_info &info, return s; } +template <> +u8 getShengState(dstate &state, dfa_info &info, + map &accelInfo) { + u8 s = state.impl_id; + if (!state.reports.empty()) { + s |= SHENG32_STATE_ACCEPT; + } + if (info.isDead(state)) { + s |= SHENG32_STATE_DEAD; + } + if (accelInfo.find(info.raw_id(state.impl_id)) != accelInfo.end()) { + s |= SHENG32_STATE_ACCEL; + } + return s; +} + +template <> +u8 getShengState(dstate &state, dfa_info &info, + UNUSED map &accelInfo) { + u8 s = state.impl_id; + if (!state.reports.empty()) { + s |= SHENG64_STATE_ACCEPT; + } + if (info.isDead(state)) { + s |= SHENG64_STATE_DEAD; + } + return s; +} + +template static void fillAccelAux(struct NFA *n, dfa_info &info, map &accelInfo) { DEBUG_PRINTF("Filling accel aux structures\n"); - sheng *s = (sheng *)getMutableImplNfa(n); + T *s = (T *)getMutableImplNfa(n); u32 offset = s->accel_offset; for (dstate_id_t i = 0; i < info.size(); i++) { @@ -349,11 +408,21 @@ void fillAccelAux(struct NFA *n, dfa_info &info, } } +template static -void populateBasicInfo(struct NFA *n, dfa_info &info, - map &accelInfo, u32 aux_offset, - u32 report_offset, u32 accel_offset, u32 total_size, - u32 dfa_size) { +void populateBasicInfo(UNUSED struct NFA *n, UNUSED dfa_info &info, + UNUSED map &accelInfo, + UNUSED u32 aux_offset, UNUSED u32 report_offset, + UNUSED u32 accel_offset, UNUSED u32 total_size, + UNUSED u32 dfa_size) { +} + +template <> +void populateBasicInfo(struct NFA *n, dfa_info &info, + map &accelInfo, + u32 aux_offset, u32 report_offset, + u32 accel_offset, u32 total_size, + u32 dfa_size) { n->length = total_size; n->scratchStateSize = 1; n->streamStateSize = 1; @@ -369,14 +438,65 @@ void populateBasicInfo(struct NFA *n, dfa_info &info, s->length = dfa_size; s->flags |= info.can_die ? SHENG_FLAG_CAN_DIE : 0; - s->anchored = getShengState(info.anchored, info, accelInfo); - s->floating = getShengState(info.floating, info, accelInfo); + s->anchored = getShengState(info.anchored, info, accelInfo); + s->floating = getShengState(info.floating, info, accelInfo); +} + +template <> +void populateBasicInfo(struct NFA *n, dfa_info &info, + map &accelInfo, + u32 aux_offset, u32 report_offset, + u32 accel_offset, u32 total_size, + u32 dfa_size) { + n->length = total_size; + n->scratchStateSize = 1; + n->streamStateSize = 1; + n->nPositions = info.size(); + n->type = SHENG_NFA_32; + n->flags |= info.raw.hasEodReports() ? NFA_ACCEPTS_EOD : 0; + + sheng32 *s = (sheng32 *)getMutableImplNfa(n); + s->aux_offset = aux_offset; + s->report_offset = report_offset; + s->accel_offset = accel_offset; + s->n_states = info.size(); + s->length = dfa_size; + s->flags |= info.can_die ? SHENG_FLAG_CAN_DIE : 0; + + s->anchored = getShengState(info.anchored, info, accelInfo); + s->floating = getShengState(info.floating, info, accelInfo); } +template <> +void populateBasicInfo(struct NFA *n, dfa_info &info, + map &accelInfo, + u32 aux_offset, u32 report_offset, + u32 accel_offset, u32 total_size, + u32 dfa_size) { + n->length = total_size; + n->scratchStateSize = 1; + n->streamStateSize = 1; + n->nPositions = info.size(); + n->type = SHENG_NFA_64; + n->flags |= info.raw.hasEodReports() ? NFA_ACCEPTS_EOD : 0; + + sheng64 *s = (sheng64 *)getMutableImplNfa(n); + s->aux_offset = aux_offset; + s->report_offset = report_offset; + s->accel_offset = accel_offset; + s->n_states = info.size(); + s->length = dfa_size; + s->flags |= info.can_die ? SHENG_FLAG_CAN_DIE : 0; + + s->anchored = getShengState(info.anchored, info, accelInfo); + s->floating = getShengState(info.floating, info, accelInfo); +} + +template static void fillTops(NFA *n, dfa_info &info, dstate_id_t id, map &accelInfo) { - sheng *s = (sheng *)getMutableImplNfa(n); + T *s = (T *)getMutableImplNfa(n); u32 aux_base = s->aux_offset; DEBUG_PRINTF("Filling tops for state %u\n", id); @@ -393,13 +513,14 @@ void fillTops(NFA *n, dfa_info &info, dstate_id_t id, DEBUG_PRINTF("Top transition for state %u: %u\n", id, top_state.impl_id); - aux->top = getShengState(top_state, info, accelInfo); + aux->top = getShengState(top_state, info, accelInfo); } +template static void fillAux(NFA *n, dfa_info &info, dstate_id_t id, vector &reports, vector &reports_eod, vector &report_offsets) { - sheng *s = (sheng *)getMutableImplNfa(n); + T *s = (T *)getMutableImplNfa(n); u32 aux_base = s->aux_offset; auto raw_id = info.raw_id(id); @@ -419,60 +540,97 @@ void fillAux(NFA *n, dfa_info &info, dstate_id_t id, vector &reports, DEBUG_PRINTF("EOD report list offset: %u\n", aux->accept_eod); } +template static void fillSingleReport(NFA *n, ReportID r_id) { - sheng *s = (sheng *)getMutableImplNfa(n); + T *s = (T *)getMutableImplNfa(n); DEBUG_PRINTF("Single report ID: %u\n", r_id); s->report = r_id; s->flags |= SHENG_FLAG_SINGLE_REPORT; } +template static -void createShuffleMasks(sheng *s, dfa_info &info, - map &accelInfo) { +bool createShuffleMasks(UNUSED T *s, UNUSED dfa_info &info, + UNUSED map &accelInfo) { + return true; +} + +template <> +bool createShuffleMasks(sheng *s, dfa_info &info, + map &accelInfo) { for (u16 chr = 0; chr < 256; chr++) { u8 buf[16] = {0}; for (dstate_id_t idx = 0; idx < info.size(); idx++) { auto &succ_state = info.next(idx, chr); - buf[idx] = getShengState(succ_state, info, accelInfo); + buf[idx] = getShengState(succ_state, info, accelInfo); } #ifdef DEBUG dumpShuffleMask(chr, buf, sizeof(buf)); #endif memcpy(&s->shuffle_masks[chr], buf, sizeof(m128)); } + return true; } -bool has_accel_sheng(const NFA *) { - return true; /* consider the sheng region as accelerated */ -} +template <> +bool createShuffleMasks(sheng32 *s, dfa_info &info, + map &accelInfo) { + for (u16 chr = 0; chr < 256; chr++) { + u8 buf[64] = {0}; -bytecode_ptr shengCompile(raw_dfa &raw, const CompileContext &cc, - const ReportManager &rm, bool only_accel_init, - set *accel_states) { - if (!cc.grey.allowSheng) { - DEBUG_PRINTF("Sheng is not allowed!\n"); - return nullptr; - } + assert(info.size() <= 32); + for (dstate_id_t idx = 0; idx < info.size(); idx++) { + auto &succ_state = info.next(idx, chr); - sheng_build_strat strat(raw, rm, only_accel_init); - dfa_info info(strat); + buf[idx] = getShengState(succ_state, info, accelInfo); + buf[32 + idx] = buf[idx]; + } +#ifdef DEBUG + dumpShuffleMask32(chr, buf, sizeof(buf)); +#endif + memcpy(&s->succ_masks[chr], buf, sizeof(m512)); + } + return true; +} - DEBUG_PRINTF("Trying to compile a %zu state Sheng\n", raw.states.size()); +template <> +bool createShuffleMasks(sheng64 *s, dfa_info &info, + map &accelInfo) { + for (u16 chr = 0; chr < 256; chr++) { + u8 buf[64] = {0}; - DEBUG_PRINTF("Anchored start state id: %u, floating start state id: %u\n", - raw.start_anchored, raw.start_floating); + assert(info.size() <= 64); + for (dstate_id_t idx = 0; idx < info.size(); idx++) { + auto &succ_state = info.next(idx, chr); - DEBUG_PRINTF("This DFA %s die so effective number of states is %zu\n", - info.can_die ? "can" : "cannot", info.size()); - if (info.size() > 16) { - DEBUG_PRINTF("Too many states\n"); - return nullptr; + if (accelInfo.find(info.raw_id(succ_state.impl_id)) + != accelInfo.end()) { + return false; + } + buf[idx] = getShengState(succ_state, info, accelInfo); + } +#ifdef DEBUG + dumpShuffleMask64(chr, buf, sizeof(buf)); +#endif + memcpy(&s->succ_masks[chr], buf, sizeof(m512)); } + return true; +} + +bool has_accel_sheng(const NFA *) { + return true; /* consider the sheng region as accelerated */ +} +template +static +bytecode_ptr shengCompile_int(raw_dfa &raw, const CompileContext &cc, + set *accel_states, + sheng_build_strat &strat, + dfa_info &info) { if (!cc.streaming) { /* TODO: work out if we can do the strip in streaming * mode with our semantics */ raw.stripExtraEodReports(); @@ -487,7 +645,7 @@ bytecode_ptr shengCompile(raw_dfa &raw, const CompileContext &cc, DEBUG_PRINTF("Anchored start state: %u, floating start state: %u\n", info.anchored.impl_id, info.floating.impl_id); - u32 nfa_size = ROUNDUP_16(sizeof(NFA) + sizeof(sheng)); + u32 nfa_size = ROUNDUP_16(sizeof(NFA) + sizeof(T)); vector reports, eod_reports, report_offsets; u8 isSingle = 0; ReportID single_report = 0; @@ -509,29 +667,128 @@ bytecode_ptr shengCompile(raw_dfa &raw, const CompileContext &cc, auto nfa = make_zeroed_bytecode_ptr(total_size); - populateBasicInfo(nfa.get(), info, accelInfo, nfa_size, reports_offset, - accel_offset, total_size, total_size - sizeof(NFA)); + populateBasicInfo(nfa.get(), info, accelInfo, nfa_size, + reports_offset, accel_offset, total_size, + total_size - sizeof(NFA)); DEBUG_PRINTF("Setting up aux and report structures\n"); ri->fillReportLists(nfa.get(), reports_offset, report_offsets); for (dstate_id_t idx = 0; idx < info.size(); idx++) { - fillTops(nfa.get(), info, idx, accelInfo); - fillAux(nfa.get(), info, idx, reports, eod_reports, report_offsets); + fillTops(nfa.get(), info, idx, accelInfo); + fillAux(nfa.get(), info, idx, reports, eod_reports, + report_offsets); } if (isSingle) { - fillSingleReport(nfa.get(), single_report); + fillSingleReport(nfa.get(), single_report); } - fillAccelAux(nfa.get(), info, accelInfo); + fillAccelAux(nfa.get(), info, accelInfo); if (accel_states) { fillAccelOut(accelInfo, accel_states); } - createShuffleMasks((sheng *)getMutableImplNfa(nfa.get()), info, accelInfo); + if (!createShuffleMasks((T *)getMutableImplNfa(nfa.get()), info, accelInfo)) { + return nullptr; + } + + return nfa; +} + +bytecode_ptr shengCompile(raw_dfa &raw, const CompileContext &cc, + const ReportManager &rm, bool only_accel_init, + set *accel_states) { + if (!cc.grey.allowSheng) { + DEBUG_PRINTF("Sheng is not allowed!\n"); + return nullptr; + } + + sheng_build_strat strat(raw, rm, only_accel_init); + dfa_info info(strat); + + DEBUG_PRINTF("Trying to compile a %zu state Sheng\n", raw.states.size()); + + DEBUG_PRINTF("Anchored start state id: %u, floating start state id: %u\n", + raw.start_anchored, raw.start_floating); + + DEBUG_PRINTF("This DFA %s die so effective number of states is %zu\n", + info.can_die ? "can" : "cannot", info.size()); + if (info.size() > 16) { + DEBUG_PRINTF("Too many states\n"); + return nullptr; + } + return shengCompile_int(raw, cc, accel_states, strat, info); +} + +bytecode_ptr sheng32Compile(raw_dfa &raw, const CompileContext &cc, + const ReportManager &rm, bool only_accel_init, + set *accel_states) { + if (!cc.grey.allowSheng) { + DEBUG_PRINTF("Sheng is not allowed!\n"); + return nullptr; + } + + if (!cc.target_info.has_avx512vbmi()) { + DEBUG_PRINTF("Sheng32 failed, no HS_CPU_FEATURES_AVX512VBMI!\n"); + return nullptr; + } + + sheng_build_strat strat(raw, rm, only_accel_init); + dfa_info info(strat); + + DEBUG_PRINTF("Trying to compile a %zu state Sheng\n", raw.states.size()); + + DEBUG_PRINTF("Anchored start state id: %u, floating start state id: %u\n", + raw.start_anchored, raw.start_floating); + + DEBUG_PRINTF("This DFA %s die so effective number of states is %zu\n", + info.can_die ? "can" : "cannot", info.size()); + assert(info.size() > 16); + if (info.size() > 32) { + DEBUG_PRINTF("Too many states\n"); + return nullptr; + } + + return shengCompile_int(raw, cc, accel_states, strat, info); +} + +bytecode_ptr sheng64Compile(raw_dfa &raw, const CompileContext &cc, + const ReportManager &rm, bool only_accel_init, + set *accel_states) { + if (!cc.grey.allowSheng) { + DEBUG_PRINTF("Sheng is not allowed!\n"); + return nullptr; + } + + if (!cc.target_info.has_avx512vbmi()) { + DEBUG_PRINTF("Sheng64 failed, no HS_CPU_FEATURES_AVX512VBMI!\n"); + return nullptr; + } + + sheng_build_strat strat(raw, rm, only_accel_init); + dfa_info info(strat); + + DEBUG_PRINTF("Trying to compile a %zu state Sheng\n", raw.states.size()); + + DEBUG_PRINTF("Anchored start state id: %u, floating start state id: %u\n", + raw.start_anchored, raw.start_floating); + + DEBUG_PRINTF("This DFA %s die so effective number of states is %zu\n", + info.can_die ? "can" : "cannot", info.size()); + assert(info.size() > 32); + if (info.size() > 64) { + DEBUG_PRINTF("Too many states\n"); + return nullptr; + } + vector old_states; + old_states = info.states; + auto nfa = shengCompile_int(raw, cc, accel_states, strat, info); + if (!nfa) { + info.states = old_states; + } return nfa; } diff --git a/src/nfa/shengcompile.h b/src/nfa/shengcompile.h index d795b3623..256f4a4e5 100644 --- a/src/nfa/shengcompile.h +++ b/src/nfa/shengcompile.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2018, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -71,6 +71,14 @@ bytecode_ptr shengCompile(raw_dfa &raw, const CompileContext &cc, const ReportManager &rm, bool only_accel_init, std::set *accel_states = nullptr); +bytecode_ptr sheng32Compile(raw_dfa &raw, const CompileContext &cc, + const ReportManager &rm, bool only_accel_init, + std::set *accel_states = nullptr); + +bytecode_ptr sheng64Compile(raw_dfa &raw, const CompileContext &cc, + const ReportManager &rm, bool only_accel_init, + std::set *accel_states = nullptr); + struct sheng_escape_info { CharReach outs; CharReach outs2_single; diff --git a/src/nfa/shengdump.cpp b/src/nfa/shengdump.cpp index 99fda76fd..6eb784077 100644 --- a/src/nfa/shengdump.cpp +++ b/src/nfa/shengdump.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -51,7 +51,7 @@ namespace ue2 { static const sstate_aux *get_aux(const NFA *n, dstate_id_t i) { - assert(n && isShengType(n->type)); + assert(n && isSheng16Type(n->type)); const sheng *s = (const sheng *)getImplNfa(n); const sstate_aux *aux_base = @@ -64,6 +64,36 @@ const sstate_aux *get_aux(const NFA *n, dstate_id_t i) { return aux; } +static +const sstate_aux *get_aux32(const NFA *n, dstate_id_t i) { + assert(n && isSheng32Type(n->type)); + + const sheng32 *s = (const sheng32 *)getImplNfa(n); + const sstate_aux *aux_base = + (const sstate_aux *)((const char *)n + s->aux_offset); + + const sstate_aux *aux = aux_base + i; + + assert((const char *)aux < (const char *)s + s->length); + + return aux; +} + +static +const sstate_aux *get_aux64(const NFA *n, dstate_id_t i) { + assert(n && isSheng64Type(n->type)); + + const sheng64 *s = (const sheng64 *)getImplNfa(n); + const sstate_aux *aux_base = + (const sstate_aux *)((const char *)n + s->aux_offset); + + const sstate_aux *aux = aux_base + i; + + assert((const char *)aux < (const char *)s + s->length); + + return aux; +} + static void dumpHeader(FILE *f, const sheng *s) { fprintf(f, "number of states: %u, DFA engine size: %u\n", s->n_states, @@ -79,6 +109,36 @@ void dumpHeader(FILE *f, const sheng *s) { !!(s->flags & SHENG_FLAG_SINGLE_REPORT)); } +static +void dumpHeader32(FILE *f, const sheng32 *s) { + fprintf(f, "number of states: %u, DFA engine size: %u\n", s->n_states, + s->length); + fprintf(f, "aux base offset: %u, reports base offset: %u, " + "accel offset: %u\n", + s->aux_offset, s->report_offset, s->accel_offset); + fprintf(f, "anchored start state: %u, floating start state: %u\n", + s->anchored & SHENG32_STATE_MASK, s->floating & SHENG32_STATE_MASK); + fprintf(f, "has accel: %u can die: %u single report: %u\n", + !!(s->flags & SHENG_FLAG_HAS_ACCEL), + !!(s->flags & SHENG_FLAG_CAN_DIE), + !!(s->flags & SHENG_FLAG_SINGLE_REPORT)); +} + +static +void dumpHeader64(FILE *f, const sheng64 *s) { + fprintf(f, "number of states: %u, DFA engine size: %u\n", s->n_states, + s->length); + fprintf(f, "aux base offset: %u, reports base offset: %u, " + "accel offset: %u\n", + s->aux_offset, s->report_offset, s->accel_offset); + fprintf(f, "anchored start state: %u, floating start state: %u\n", + s->anchored & SHENG64_STATE_MASK, s->floating & SHENG64_STATE_MASK); + fprintf(f, "has accel: %u can die: %u single report: %u\n", + !!(s->flags & SHENG_FLAG_HAS_ACCEL), + !!(s->flags & SHENG_FLAG_CAN_DIE), + !!(s->flags & SHENG_FLAG_SINGLE_REPORT)); +} + static void dumpAux(FILE *f, u32 state, const sstate_aux *aux) { fprintf(f, "state id: %u, reports offset: %u, EOD reports offset: %u, " @@ -87,6 +147,22 @@ void dumpAux(FILE *f, u32 state, const sstate_aux *aux) { aux->top & SHENG_STATE_MASK); } +static +void dumpAux32(FILE *f, u32 state, const sstate_aux *aux) { + fprintf(f, "state id: %u, reports offset: %u, EOD reports offset: %u, " + "accel offset: %u, top: %u\n", + state, aux->accept, aux->accept_eod, aux->accel, + aux->top & SHENG32_STATE_MASK); +} + +static +void dumpAux64(FILE *f, u32 state, const sstate_aux *aux) { + fprintf(f, "state id: %u, reports offset: %u, EOD reports offset: %u, " + "accel offset: %u, top: %u\n", + state, aux->accept, aux->accept_eod, aux->accel, + aux->top & SHENG64_STATE_MASK); +} + static void dumpReports(FILE *f, const report_list *rl) { fprintf(f, "reports count: %u\n", rl->count); @@ -115,6 +191,46 @@ void dumpMasks(FILE *f, const sheng *s) { } } +static +void dumpMasks32(FILE *f, const sheng32 *s) { + for (u32 chr = 0; chr < 256; chr++) { + u8 buf[64]; + m512 succ_mask = s->succ_masks[chr]; + memcpy(buf, &succ_mask, sizeof(m512)); + + fprintf(f, "%3u: ", chr); + for (u32 pos = 0; pos < 64; pos++) { + u8 c = buf[pos]; + if (c & SHENG32_STATE_FLAG_MASK) { + fprintf(f, "%2u* ", c & SHENG32_STATE_MASK); + } else { + fprintf(f, "%2u ", c & SHENG32_STATE_MASK); + } + } + fprintf(f, "\n"); + } +} + +static +void dumpMasks64(FILE *f, const sheng64 *s) { + for (u32 chr = 0; chr < 256; chr++) { + u8 buf[64]; + m512 succ_mask = s->succ_masks[chr]; + memcpy(buf, &succ_mask, sizeof(m512)); + + fprintf(f, "%3u: ", chr); + for (u32 pos = 0; pos < 64; pos++) { + u8 c = buf[pos]; + if (c & SHENG64_STATE_FLAG_MASK) { + fprintf(f, "%2u* ", c & SHENG64_STATE_MASK); + } else { + fprintf(f, "%2u ", c & SHENG64_STATE_MASK); + } + } + fprintf(f, "\n"); + } +} + static void nfaExecSheng_dumpText(const NFA *nfa, FILE *f) { assert(nfa->type == SHENG_NFA); @@ -153,6 +269,82 @@ void nfaExecSheng_dumpText(const NFA *nfa, FILE *f) { fprintf(f, "\n"); } +static +void nfaExecSheng32_dumpText(const NFA *nfa, FILE *f) { + assert(nfa->type == SHENG_NFA_32); + const sheng32 *s = (const sheng32 *)getImplNfa(nfa); + + fprintf(f, "sheng32 DFA\n"); + dumpHeader32(f, s); + + for (u32 state = 0; state < s->n_states; state++) { + const sstate_aux *aux = get_aux32(nfa, state); + dumpAux32(f, state, aux); + if (aux->accept) { + fprintf(f, "report list:\n"); + const report_list *rl = + (const report_list *)((const char *)nfa + aux->accept); + dumpReports(f, rl); + } + if (aux->accept_eod) { + fprintf(f, "EOD report list:\n"); + const report_list *rl = + (const report_list *)((const char *)nfa + aux->accept_eod); + dumpReports(f, rl); + } + if (aux->accel) { + fprintf(f, "accel:\n"); + const AccelAux *accel = + (const AccelAux *)((const char *)nfa + aux->accel); + dumpAccelInfo(f, *accel); + } + } + + fprintf(f, "\n"); + + dumpMasks32(f, s); + + fprintf(f, "\n"); +} + +static +void nfaExecSheng64_dumpText(const NFA *nfa, FILE *f) { + assert(nfa->type == SHENG_NFA_64); + const sheng64 *s = (const sheng64 *)getImplNfa(nfa); + + fprintf(f, "sheng64 DFA\n"); + dumpHeader64(f, s); + + for (u32 state = 0; state < s->n_states; state++) { + const sstate_aux *aux = get_aux64(nfa, state); + dumpAux64(f, state, aux); + if (aux->accept) { + fprintf(f, "report list:\n"); + const report_list *rl = + (const report_list *)((const char *)nfa + aux->accept); + dumpReports(f, rl); + } + if (aux->accept_eod) { + fprintf(f, "EOD report list:\n"); + const report_list *rl = + (const report_list *)((const char *)nfa + aux->accept_eod); + dumpReports(f, rl); + } + if (aux->accel) { + fprintf(f, "accel:\n"); + const AccelAux *accel = + (const AccelAux *)((const char *)nfa + aux->accel); + dumpAccelInfo(f, *accel); + } + } + + fprintf(f, "\n"); + + dumpMasks64(f, s); + + fprintf(f, "\n"); +} + static void dumpDotPreambleDfa(FILE *f) { dumpDotPreamble(f); @@ -163,8 +355,14 @@ void dumpDotPreambleDfa(FILE *f) { fprintf(f, "0 [style=invis];\n"); } +template static -void describeNode(const NFA *n, const sheng *s, u16 i, FILE *f) { +void describeNode(UNUSED const NFA *n, UNUSED const T *s, UNUSED u16 i, + UNUSED FILE *f) { +} + +template <> +void describeNode(const NFA *n, const sheng *s, u16 i, FILE *f) { const sstate_aux *aux = get_aux(n, i); fprintf(f, "%u [ width = 1, fixedsize = true, fontsize = 12, " @@ -193,6 +391,66 @@ void describeNode(const NFA *n, const sheng *s, u16 i, FILE *f) { } } +template <> +void describeNode(const NFA *n, const sheng32 *s, u16 i, FILE *f) { + const sstate_aux *aux = get_aux32(n, i); + + fprintf(f, "%u [ width = 1, fixedsize = true, fontsize = 12, " + "label = \"%u\" ]; \n", + i, i); + + if (aux->accept_eod) { + fprintf(f, "%u [ color = darkorchid ];\n", i); + } + + if (aux->accept) { + fprintf(f, "%u [ shape = doublecircle ];\n", i); + } + + if (aux->top && (aux->top & SHENG32_STATE_MASK) != i) { + fprintf(f, "%u -> %u [color = darkgoldenrod weight=0.1 ]\n", i, + aux->top & SHENG32_STATE_MASK); + } + + if (i == (s->anchored & SHENG32_STATE_MASK)) { + fprintf(f, "STARTA -> %u [color = blue ]\n", i); + } + + if (i == (s->floating & SHENG32_STATE_MASK)) { + fprintf(f, "STARTF -> %u [color = red ]\n", i); + } +} + +template <> +void describeNode(const NFA *n, const sheng64 *s, u16 i, FILE *f) { + const sstate_aux *aux = get_aux64(n, i); + + fprintf(f, "%u [ width = 1, fixedsize = true, fontsize = 12, " + "label = \"%u\" ]; \n", + i, i); + + if (aux->accept_eod) { + fprintf(f, "%u [ color = darkorchid ];\n", i); + } + + if (aux->accept) { + fprintf(f, "%u [ shape = doublecircle ];\n", i); + } + + if (aux->top && (aux->top & SHENG64_STATE_MASK) != i) { + fprintf(f, "%u -> %u [color = darkgoldenrod weight=0.1 ]\n", i, + aux->top & SHENG64_STATE_MASK); + } + + if (i == (s->anchored & SHENG64_STATE_MASK)) { + fprintf(f, "STARTA -> %u [color = blue ]\n", i); + } + + if (i == (s->floating & SHENG64_STATE_MASK)) { + fprintf(f, "STARTF -> %u [color = red ]\n", i); + } +} + static void describeEdge(FILE *f, const u16 *t, u16 i) { for (u16 s = 0; s < N_CHARS; s++) { @@ -228,7 +486,7 @@ void describeEdge(FILE *f, const u16 *t, u16 i) { static void shengGetTransitions(const NFA *n, u16 state, u16 *t) { - assert(isShengType(n->type)); + assert(isSheng16Type(n->type)); const sheng *s = (const sheng *)getImplNfa(n); const sstate_aux *aux = get_aux(n, state); @@ -244,6 +502,42 @@ void shengGetTransitions(const NFA *n, u16 state, u16 *t) { t[TOP] = aux->top & SHENG_STATE_MASK; } +static +void sheng32GetTransitions(const NFA *n, u16 state, u16 *t) { + assert(isSheng32Type(n->type)); + const sheng32 *s = (const sheng32 *)getImplNfa(n); + const sstate_aux *aux = get_aux32(n, state); + + for (unsigned i = 0; i < N_CHARS; i++) { + u8 buf[64]; + m512 succ_mask = s->succ_masks[i]; + + memcpy(buf, &succ_mask, sizeof(m512)); + + t[i] = buf[state] & SHENG32_STATE_MASK; + } + + t[TOP] = aux->top & SHENG32_STATE_MASK; +} + +static +void sheng64GetTransitions(const NFA *n, u16 state, u16 *t) { + assert(isSheng64Type(n->type)); + const sheng64 *s = (const sheng64 *)getImplNfa(n); + const sstate_aux *aux = get_aux64(n, state); + + for (unsigned i = 0; i < N_CHARS; i++) { + u8 buf[64]; + m512 succ_mask = s->succ_masks[i]; + + memcpy(buf, &succ_mask, sizeof(m512)); + + t[i] = buf[state] & SHENG64_STATE_MASK; + } + + t[TOP] = aux->top & SHENG64_STATE_MASK; +} + static void nfaExecSheng_dumpDot(const NFA *nfa, FILE *f) { assert(nfa->type == SHENG_NFA); @@ -252,7 +546,7 @@ void nfaExecSheng_dumpDot(const NFA *nfa, FILE *f) { dumpDotPreambleDfa(f); for (u16 i = 1; i < s->n_states; i++) { - describeNode(nfa, s, i, f); + describeNode(nfa, s, i, f); u16 t[ALPHABET_SIZE]; @@ -264,10 +558,62 @@ void nfaExecSheng_dumpDot(const NFA *nfa, FILE *f) { fprintf(f, "}\n"); } +static +void nfaExecSheng32_dumpDot(const NFA *nfa, FILE *f) { + assert(nfa->type == SHENG_NFA_32); + const sheng32 *s = (const sheng32 *)getImplNfa(nfa); + + dumpDotPreambleDfa(f); + + for (u16 i = 1; i < s->n_states; i++) { + describeNode(nfa, s, i, f); + + u16 t[ALPHABET_SIZE]; + + sheng32GetTransitions(nfa, i, t); + + describeEdge(f, t, i); + } + + fprintf(f, "}\n"); +} + +static +void nfaExecSheng64_dumpDot(const NFA *nfa, FILE *f) { + assert(nfa->type == SHENG_NFA_64); + const sheng64 *s = (const sheng64 *)getImplNfa(nfa); + + dumpDotPreambleDfa(f); + + for (u16 i = 1; i < s->n_states; i++) { + describeNode(nfa, s, i, f); + + u16 t[ALPHABET_SIZE]; + + sheng64GetTransitions(nfa, i, t); + + describeEdge(f, t, i); + } + + fprintf(f, "}\n"); +} + void nfaExecSheng_dump(const NFA *nfa, const string &base) { assert(nfa->type == SHENG_NFA); nfaExecSheng_dumpText(nfa, StdioFile(base + ".txt", "w")); nfaExecSheng_dumpDot(nfa, StdioFile(base + ".dot", "w")); } +void nfaExecSheng32_dump(UNUSED const NFA *nfa, UNUSED const string &base) { + assert(nfa->type == SHENG_NFA_32); + nfaExecSheng32_dumpText(nfa, StdioFile(base + ".txt", "w")); + nfaExecSheng32_dumpDot(nfa, StdioFile(base + ".dot", "w")); +} + +void nfaExecSheng64_dump(UNUSED const NFA *nfa, UNUSED const string &base) { + assert(nfa->type == SHENG_NFA_64); + nfaExecSheng64_dumpText(nfa, StdioFile(base + ".txt", "w")); + nfaExecSheng64_dumpDot(nfa, StdioFile(base + ".dot", "w")); +} + } // namespace ue2 diff --git a/src/nfa/shengdump.h b/src/nfa/shengdump.h index 2bdffeb9a..321536742 100644 --- a/src/nfa/shengdump.h +++ b/src/nfa/shengdump.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,6 +38,8 @@ struct NFA; namespace ue2 { void nfaExecSheng_dump(const struct NFA *nfa, const std::string &base); +void nfaExecSheng32_dump(const struct NFA *nfa, const std::string &base); +void nfaExecSheng64_dump(const struct NFA *nfa, const std::string &base); } // namespace ue2 diff --git a/src/nfagraph/ng_limex.cpp b/src/nfagraph/ng_limex.cpp index 922100e7a..2f0a55eab 100644 --- a/src/nfagraph/ng_limex.cpp +++ b/src/nfagraph/ng_limex.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -632,8 +632,8 @@ bytecode_ptr constructNFA(const NGHolder &h_in, const ReportManager *rm, const map &fixed_depth_tops, const map>> &triggers, - bool compress_state, bool do_accel, bool impl_test_only, u32 hint, - const CompileContext &cc) { + bool compress_state, bool do_accel, bool impl_test_only, + bool &fast, u32 hint, const CompileContext &cc) { if (!has_managed_reports(h_in)) { rm = nullptr; } else { @@ -684,19 +684,19 @@ constructNFA(const NGHolder &h_in, const ReportManager *rm, } return generate(*h, state_ids, repeats, reportSquashMap, squashMap, tops, - zombies, do_accel, compress_state, hint, cc); + zombies, do_accel, compress_state, fast, hint, cc); } bytecode_ptr constructNFA(const NGHolder &h_in, const ReportManager *rm, const map &fixed_depth_tops, const map>> &triggers, - bool compress_state, const CompileContext &cc) { + bool compress_state, bool &fast, const CompileContext &cc) { const u32 hint = INVALID_NFA; const bool do_accel = cc.grey.accelerateNFA; const bool impl_test_only = false; return constructNFA(h_in, rm, fixed_depth_tops, triggers, compress_state, - do_accel, impl_test_only, hint, cc); + do_accel, impl_test_only, fast, hint, cc); } #ifndef RELEASE_BUILD @@ -705,11 +705,11 @@ bytecode_ptr constructNFA(const NGHolder &h_in, const ReportManager *rm, const map &fixed_depth_tops, const map>> &triggers, - bool compress_state, u32 hint, const CompileContext &cc) { + bool compress_state, bool &fast, u32 hint, const CompileContext &cc) { const bool do_accel = cc.grey.accelerateNFA; const bool impl_test_only = false; - return constructNFA(h_in, rm, fixed_depth_tops, triggers, - compress_state, do_accel, impl_test_only, hint, cc); + return constructNFA(h_in, rm, fixed_depth_tops, triggers, compress_state, + do_accel, impl_test_only, fast, hint, cc); } #endif // RELEASE_BUILD @@ -739,9 +739,10 @@ bytecode_ptr constructReversedNFA_i(const NGHolder &h_in, u32 hint, vector repeats; unordered_map reportSquashMap; unordered_map squashMap; + UNUSED bool fast = false; return generate(h, state_ids, repeats, reportSquashMap, squashMap, tops, - zombies, false, false, hint, cc); + zombies, false, false, fast, hint, cc); } bytecode_ptr constructReversedNFA(const NGHolder &h_in, diff --git a/src/nfagraph/ng_limex.h b/src/nfagraph/ng_limex.h index 9bf46d693..7eba2eff0 100644 --- a/src/nfagraph/ng_limex.h +++ b/src/nfagraph/ng_limex.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -100,7 +100,7 @@ bytecode_ptr constructNFA(const NGHolder &g, const ReportManager *rm, const std::map &fixed_depth_tops, const std::map>> &triggers, - bool compress_state, const CompileContext &cc); + bool compress_state, bool &fast, const CompileContext &cc); /** * \brief Build a reverse NFA from the graph given, which should have already @@ -129,7 +129,7 @@ bytecode_ptr constructNFA(const NGHolder &g, const ReportManager *rm, const std::map &fixed_depth_tops, const std::map>> &triggers, - bool compress_state, u32 hint, const CompileContext &cc); + bool compress_state, bool &fast, u32 hint, const CompileContext &cc); /** * \brief Build a reverse NFA (with model type hint) from the graph given, diff --git a/src/nfagraph/ng_literal_analysis.cpp b/src/nfagraph/ng_literal_analysis.cpp index ea0def021..d25ac43e8 100644 --- a/src/nfagraph/ng_literal_analysis.cpp +++ b/src/nfagraph/ng_literal_analysis.cpp @@ -69,14 +69,14 @@ struct LitGraphVertexProps { LitGraphVertexProps() = default; explicit LitGraphVertexProps(ue2_literal::elem c_in) : c(move(c_in)) {} ue2_literal::elem c; // string element (char + bool) - size_t index; // managed by ue2_graph + size_t index = 0; // managed by ue2_graph }; struct LitGraphEdgeProps { LitGraphEdgeProps() = default; explicit LitGraphEdgeProps(u64a score_in) : score(score_in) {} u64a score = NO_LITERAL_AT_EDGE_SCORE; - size_t index; // managed by ue2_graph + size_t index = 0; // managed by ue2_graph }; struct LitGraph diff --git a/src/parser/logical_combination.cpp b/src/parser/logical_combination.cpp index 49e060c98..de017a110 100644 --- a/src/parser/logical_combination.cpp +++ b/src/parser/logical_combination.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019, Intel Corporation + * Copyright (c) 2018-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,6 +33,7 @@ #include "parser/parse_error.h" #include "util/container.h" #include "hs_compile.h" +#include "allocator.h" #include @@ -151,7 +152,7 @@ void ParsedLogical::validateSubIDs(const unsigned *ids, if (info->unordered_matches) { throw CompileError("Have unordered match in sub-expressions."); } - free(info); + hs_misc_free(info); } } } diff --git a/src/rose/program_runtime.c b/src/rose/program_runtime.c index 0f2d1083b..ff5a5099c 100644 --- a/src/rose/program_runtime.c +++ b/src/rose/program_runtime.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2019, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -767,10 +767,10 @@ int roseCheckMask32(const struct core_info *ci, const u8 *and_mask, c_shift = c_len - ci->len; c_len = ci->len; } - copy_upto_32_bytes((u8 *)&data - offset, ci->buf, c_len); + copy_upto_64_bytes((u8 *)&data - offset, ci->buf, c_len); } assert(h_shift + h_len + c_len + c_shift == 32); - copy_upto_32_bytes((u8 *)&data + h_shift, ci->hbuf + h_offset, h_len); + copy_upto_64_bytes((u8 *)&data + h_shift, ci->hbuf + h_offset, h_len); } else { if (offset + 32 > (s64a)ci->len) { if (offset >= (s64a)ci->len) { @@ -779,7 +779,7 @@ int roseCheckMask32(const struct core_info *ci, const u8 *and_mask, } c_len = ci->len - offset; c_shift = 32 - c_len; - copy_upto_32_bytes((u8 *)&data, ci->buf + offset, c_len); + copy_upto_64_bytes((u8 *)&data, ci->buf + offset, c_len); } else { data = loadu256(ci->buf + offset); } @@ -800,12 +800,90 @@ int roseCheckMask32(const struct core_info *ci, const u8 *and_mask, return 0; } -// get 128/256 bits data from history and current buffer. +#ifdef HAVE_AVX512 +static rose_inline +int roseCheckMask64(const struct core_info *ci, const u8 *and_mask, + const u8 *cmp_mask, const u64a neg_mask, + s32 checkOffset, u64a end) { + const s64a base_offset = (s64a)end - ci->buf_offset; + s64a offset = base_offset + checkOffset; + DEBUG_PRINTF("end %lld base_offset %lld\n", end, base_offset); + DEBUG_PRINTF("checkOffset %d offset %lld\n", checkOffset, offset); + + if (unlikely(checkOffset < 0 && (u64a)(0 - checkOffset) > end)) { + DEBUG_PRINTF("too early, fail\n"); + return 0; + } + + m512 data = zeroes512(); // consists of the following four parts. + s32 c_shift = 0; // blank bytes after current. + s32 h_shift = 0; // blank bytes before history. + s32 h_len = 64; // number of bytes from history buffer. + s32 c_len = 0; // number of bytes from current buffer. + /* h_shift + h_len + c_len + c_shift = 64 need to be hold.*/ + + if (offset < 0) { + s32 h_offset = 0; // the start offset in history buffer. + if (offset < -(s64a)ci->hlen) { + if (offset + 64 <= -(s64a)ci->hlen) { + DEBUG_PRINTF("all before history\n"); + return 1; + } + h_shift = -(offset + (s64a)ci->hlen); + h_len = 64 - h_shift; + } else { + h_offset = ci->hlen + offset; + } + if (offset + 64 > 0) { + // part in current buffer. + c_len = offset + 64; + h_len = -(offset + h_shift); + if (c_len > (s64a)ci->len) { + // out of current buffer. + c_shift = c_len - ci->len; + c_len = ci->len; + } + copy_upto_64_bytes((u8 *)&data - offset, ci->buf, c_len); + } + assert(h_shift + h_len + c_len + c_shift == 64); + copy_upto_64_bytes((u8 *)&data + h_shift, ci->hbuf + h_offset, h_len); + } else { + if (offset + 64 > (s64a)ci->len) { + if (offset >= (s64a)ci->len) { + DEBUG_PRINTF("all in the future.\n"); + return 1; + } + c_len = ci->len - offset; + c_shift = 64 - c_len; + copy_upto_64_bytes((u8 *)&data, ci->buf + offset, c_len); + } else { + data = loadu512(ci->buf + offset); + } + } + DEBUG_PRINTF("h_shift %d c_shift %d\n", h_shift, c_shift); + DEBUG_PRINTF("h_len %d c_len %d\n", h_len, c_len); + // we use valid_data_mask to blind bytes before history/in the future. + u64a valid_data_mask; + valid_data_mask = (~0ULL) << (h_shift + c_shift) >> (c_shift); + + m512 and_mask_m512 = loadu512(and_mask); + m512 cmp_mask_m512 = loadu512(cmp_mask); + + if (validateMask64(data, valid_data_mask, and_mask_m512, + cmp_mask_m512, neg_mask)) { + DEBUG_PRINTF("Mask64 passed\n"); + return 1; + } + return 0; +} +#endif + +// get 128/256/512 bits data from history and current buffer. // return data and valid_data_mask. static rose_inline -u32 getBufferDataComplex(const struct core_info *ci, const s64a loc, +u64a getBufferDataComplex(const struct core_info *ci, const s64a loc, u8 *data, const u32 data_len) { - assert(data_len == 16 || data_len == 32); + assert(data_len == 16 || data_len == 32 || data_len == 64); s32 c_shift = 0; // blank bytes after current. s32 h_shift = 0; // blank bytes before history. s32 h_len = data_len; // number of bytes from history buffer. @@ -831,10 +909,10 @@ u32 getBufferDataComplex(const struct core_info *ci, const s64a loc, c_shift = c_len - ci->len; c_len = ci->len; } - copy_upto_32_bytes(data - loc, ci->buf, c_len); + copy_upto_64_bytes(data - loc, ci->buf, c_len); } assert(h_shift + h_len + c_len + c_shift == (s32)data_len); - copy_upto_32_bytes(data + h_shift, ci->hbuf + h_offset, h_len); + copy_upto_64_bytes(data + h_shift, ci->hbuf + h_offset, h_len); } else { if (loc + data_len > (s64a)ci->len) { if (loc >= (s64a)ci->len) { @@ -843,8 +921,14 @@ u32 getBufferDataComplex(const struct core_info *ci, const s64a loc, } c_len = ci->len - loc; c_shift = data_len - c_len; - copy_upto_32_bytes(data, ci->buf + loc, c_len); + copy_upto_64_bytes(data, ci->buf + loc, c_len); } else { +#ifdef HAVE_AVX512 + if (data_len == 64) { + storeu512(data, loadu512(ci->buf + loc)); + return ~0ULL; + } +#endif if (data_len == 16) { storeu128(data, loadu128(ci->buf + loc)); return 0xffff; @@ -857,6 +941,11 @@ u32 getBufferDataComplex(const struct core_info *ci, const s64a loc, DEBUG_PRINTF("h_shift %d c_shift %d\n", h_shift, c_shift); DEBUG_PRINTF("h_len %d c_len %d\n", h_len, c_len); +#ifdef HAVE_AVX512 + if (data_len == 64) { + return (~0ULL) << (h_shift + c_shift) >> c_shift; + } +#endif if (data_len == 16) { return (u16)(0xffff << (h_shift + c_shift)) >> c_shift; } else { @@ -886,6 +975,19 @@ m256 getData256(const struct core_info *ci, s64a offset, u32 *valid_data_mask) { return *(m256 *)data; } +#ifdef HAVE_AVX512 +static rose_inline +m512 getData512(const struct core_info *ci, s64a offset, u64a *valid_data_mask) { + if (offset > 0 && offset + sizeof(m512) <= ci->len) { + *valid_data_mask = ~0ULL; + return loadu512(ci->buf + offset); + } + ALIGN_CL_DIRECTIVE u8 data[sizeof(m512)]; + *valid_data_mask = getBufferDataComplex(ci, offset, data, 64); + return *(m512 *)data; +} +#endif + static rose_inline int roseCheckShufti16x8(const struct core_info *ci, const u8 *nib_mask, const u8 *bucket_select_mask, u32 neg_mask, @@ -1025,6 +1127,83 @@ int roseCheckShufti32x16(const struct core_info *ci, const u8 *hi_mask, } } +#ifdef HAVE_AVX512 +static rose_inline +int roseCheckShufti64x8(const struct core_info *ci, const u8 *hi_mask, + const u8 *lo_mask, const u8 *bucket_select_mask, + u64a neg_mask, s32 checkOffset, u64a end) { + const s64a base_offset = (s64a)end - ci->buf_offset; + s64a offset = base_offset + checkOffset; + DEBUG_PRINTF("end %lld base_offset %lld\n", end, base_offset); + DEBUG_PRINTF("checkOffset %d offset %lld\n", checkOffset, offset); + + if (unlikely(checkOffset < 0 && (u64a)(0 - checkOffset) > end)) { + DEBUG_PRINTF("too early, fail\n"); + return 0; + } + + u64a valid_data_mask = 0; + m512 data = getData512(ci, offset, &valid_data_mask); + + if (unlikely(!valid_data_mask)) { + return 1; + } + + m512 hi_mask_m512 = loadu512(hi_mask); + m512 lo_mask_m512 = loadu512(lo_mask); + m512 bucket_select_mask_m512 = loadu512(bucket_select_mask); + if (validateShuftiMask64x8(data, hi_mask_m512, lo_mask_m512, + bucket_select_mask_m512, + neg_mask, valid_data_mask)) { + DEBUG_PRINTF("check shufti 64x8 successfully\n"); + return 1; + } else { + return 0; + } +} + +static rose_inline +int roseCheckShufti64x16(const struct core_info *ci, const u8 *hi_mask_1, + const u8 *hi_mask_2, const u8 *lo_mask_1, + const u8 *lo_mask_2, const u8 *bucket_select_mask_hi, + const u8 *bucket_select_mask_lo, u64a neg_mask, + s32 checkOffset, u64a end) { + const s64a base_offset = (s64a)end - ci->buf_offset; + s64a offset = base_offset + checkOffset; + DEBUG_PRINTF("end %lld base_offset %lld\n", end, base_offset); + DEBUG_PRINTF("checkOffset %d offset %lld\n", checkOffset, offset); + + if (unlikely(checkOffset < 0 && (u64a)(0 - checkOffset) > end)) { + DEBUG_PRINTF("too early, fail\n"); + return 0; + } + + u64a valid_data_mask = 0; + m512 data = getData512(ci, offset, &valid_data_mask); + if (unlikely(!valid_data_mask)) { + return 1; + } + + m512 hi_mask_1_m512 = loadu512(hi_mask_1); + m512 hi_mask_2_m512 = loadu512(hi_mask_2); + m512 lo_mask_1_m512 = loadu512(lo_mask_1); + m512 lo_mask_2_m512 = loadu512(lo_mask_2); + + m512 bucket_select_mask_hi_m512 = loadu512(bucket_select_mask_hi); + m512 bucket_select_mask_lo_m512 = loadu512(bucket_select_mask_lo); + if (validateShuftiMask64x16(data, hi_mask_1_m512, hi_mask_2_m512, + lo_mask_1_m512, lo_mask_2_m512, + bucket_select_mask_hi_m512, + bucket_select_mask_lo_m512, + neg_mask, valid_data_mask)) { + DEBUG_PRINTF("check shufti 64x16 successfully\n"); + return 1; + } else { + return 0; + } +} +#endif + static rose_inline int roseCheckSingleLookaround(const struct RoseEngine *t, const struct hs_scratch *scratch, @@ -2068,6 +2247,12 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t, &&LABEL_ROSE_INSTR_FLUSH_COMBINATION, &&LABEL_ROSE_INSTR_SET_EXHAUST, &&LABEL_ROSE_INSTR_LAST_FLUSH_COMBINATION +#ifdef HAVE_AVX512 + , + &&LABEL_ROSE_INSTR_CHECK_SHUFTI_64x8, //!< Check 64-byte data by 8-bucket shufti. + &&LABEL_ROSE_INSTR_CHECK_SHUFTI_64x16, //!< Check 64-byte data by 16-bucket shufti. + &&LABEL_ROSE_INSTR_CHECK_MASK_64 //!< 64-bytes and/cmp/neg mask check. +#endif }; #endif @@ -2258,6 +2443,45 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t, } PROGRAM_NEXT_INSTRUCTION +#ifdef HAVE_AVX512 + PROGRAM_CASE(CHECK_MASK_64) { + struct core_info *ci = &scratch->core_info; + if (!roseCheckMask64(ci, ri->and_mask, ri->cmp_mask, + ri->neg_mask, ri->offset, end)) { + assert(ri->fail_jump); + pc += ri->fail_jump; + PROGRAM_NEXT_INSTRUCTION_JUMP + } + } + PROGRAM_NEXT_INSTRUCTION + + PROGRAM_CASE(CHECK_SHUFTI_64x8) { + const struct core_info *ci = &scratch->core_info; + if (!roseCheckShufti64x8(ci, ri->hi_mask, ri->lo_mask, + ri->bucket_select_mask, + ri->neg_mask, ri->offset, end)) { + assert(ri->fail_jump); + pc += ri->fail_jump; + PROGRAM_NEXT_INSTRUCTION_JUMP; + } + } + PROGRAM_NEXT_INSTRUCTION + + PROGRAM_CASE(CHECK_SHUFTI_64x16) { + const struct core_info *ci = &scratch->core_info; + if (!roseCheckShufti64x16(ci, ri->hi_mask_1, ri->hi_mask_2, + ri->lo_mask_1, ri->lo_mask_2, + ri->bucket_select_mask_hi, + ri->bucket_select_mask_lo, + ri->neg_mask, ri->offset, end)) { + assert(ri->fail_jump); + pc += ri->fail_jump; + PROGRAM_NEXT_INSTRUCTION_JUMP; + } + } + PROGRAM_NEXT_INSTRUCTION +#endif + PROGRAM_CASE(CHECK_INFIX) { if (!roseTestInfix(t, scratch, ri->queue, ri->lag, ri->report, end)) { @@ -2945,6 +3169,19 @@ hwlmcb_rv_t roseRunProgram_l(const struct RoseEngine *t, } L_PROGRAM_NEXT_INSTRUCTION +#ifdef HAVE_AVX512 + L_PROGRAM_CASE(CHECK_MASK_64) { + struct core_info *ci = &scratch->core_info; + if (!roseCheckMask64(ci, ri->and_mask, ri->cmp_mask, + ri->neg_mask, ri->offset, end)) { + assert(ri->fail_jump); + pc += ri->fail_jump; + L_PROGRAM_NEXT_INSTRUCTION_JUMP + } + } + L_PROGRAM_NEXT_INSTRUCTION +#endif + L_PROGRAM_CASE(CHECK_BYTE) { const struct core_info *ci = &scratch->core_info; if (!roseCheckByte(ci, ri->and_mask, ri->cmp_mask, diff --git a/src/rose/rose_build_bytecode.cpp b/src/rose/rose_build_bytecode.cpp index 5cbb5c848..df464c280 100644 --- a/src/rose/rose_build_bytecode.cpp +++ b/src/rose/rose_build_bytecode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2019, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -554,7 +554,8 @@ void findFixedDepthTops(const RoseGraph &g, const set &triggers, */ static bytecode_ptr pickImpl(bytecode_ptr dfa_impl, - bytecode_ptr nfa_impl) { + bytecode_ptr nfa_impl, + bool fast_nfa) { assert(nfa_impl); assert(dfa_impl); assert(isDfaType(dfa_impl->type)); @@ -584,7 +585,7 @@ bytecode_ptr pickImpl(bytecode_ptr dfa_impl, return nfa_impl; } } else { - if (n_accel) { + if (n_accel && fast_nfa) { return nfa_impl; } else { return dfa_impl; @@ -632,6 +633,15 @@ bytecode_ptr getDfa(raw_dfa &rdfa, bool is_transient, * bytecode and that they are usually run on small blocks */ dfa = mcshengCompile(rdfa, cc, rm); } + if (!dfa) { + dfa = sheng32Compile(rdfa, cc, rm, false); + } + if (!dfa) { + dfa = sheng64Compile(rdfa, cc, rm, false); + } + if (!dfa && !is_transient) { + dfa = mcshengCompile64(rdfa, cc, rm); + } if (!dfa) { // Sheng wasn't successful, so unleash McClellan! dfa = mcclellanCompile(rdfa, cc, rm, false); @@ -678,20 +688,21 @@ buildSuffix(const ReportManager &rm, const SomSlotManager &ssm, } } + bool fast_nfa = false; auto n = constructNFA(holder, &rm, fixed_depth_tops, triggers, - compress_state, cc); + compress_state, fast_nfa, cc); assert(n); if (oneTop && cc.grey.roseMcClellanSuffix) { if (cc.grey.roseMcClellanSuffix == 2 || n->nPositions > 128 || - !has_bounded_repeats_other_than_firsts(*n)) { + !has_bounded_repeats_other_than_firsts(*n) || !fast_nfa) { auto rdfa = buildMcClellan(holder, &rm, false, triggers.at(0), cc.grey); if (rdfa) { auto d = getDfa(*rdfa, false, cc, rm); assert(d); if (cc.grey.roseMcClellanSuffix != 2) { - n = pickImpl(move(d), move(n)); + n = pickImpl(move(d), move(n), fast_nfa); } else { n = move(d); } @@ -826,23 +837,24 @@ bytecode_ptr makeLeftNfa(const RoseBuildImpl &tbi, left_id &left, n = constructLBR(*left.graph(), triggers.begin()->second, cc, rm); } + bool fast_nfa = false; if (!n && left.graph()) { map>> triggers; if (left.graph()->kind == NFA_INFIX) { findTriggerSequences(tbi, infixTriggers.at(left), &triggers); } n = constructNFA(*left.graph(), nullptr, fixed_depth_tops, triggers, - compress_state, cc); + compress_state, fast_nfa, cc); } if (cc.grey.roseMcClellanPrefix == 1 && is_prefix && !left.dfa() && left.graph() - && (!n || !has_bounded_repeats_other_than_firsts(*n) || !is_fast(*n))) { + && (!n || !has_bounded_repeats_other_than_firsts(*n) || !fast_nfa)) { auto rdfa = buildMcClellan(*left.graph(), nullptr, cc.grey); if (rdfa) { auto d = getDfa(*rdfa, is_transient, cc, rm); assert(d); - n = pickImpl(move(d), move(n)); + n = pickImpl(move(d), move(n), fast_nfa); } } @@ -1627,17 +1639,18 @@ class OutfixBuilder : public boost::static_visitor> { const map fixed_depth_tops; /* no tops */ const map>> triggers; /* no tops */ bool compress_state = cc.streaming; + bool fast_nfa = false; auto n = constructNFA(h, &rm, fixed_depth_tops, triggers, - compress_state, cc); + compress_state, fast_nfa, cc); // Try for a DFA upgrade. if (n && cc.grey.roseMcClellanOutfix && - !has_bounded_repeats_other_than_firsts(*n)) { + (!has_bounded_repeats_other_than_firsts(*n) || !fast_nfa)) { auto rdfa = buildMcClellan(h, &rm, cc.grey); if (rdfa) { auto d = getDfa(*rdfa, false, cc, rm); if (d) { - n = pickImpl(move(d), move(n)); + n = pickImpl(move(d), move(n), fast_nfa); } } } diff --git a/src/rose/rose_build_dump.cpp b/src/rose/rose_build_dump.cpp index 8999daef2..dbc938a5c 100644 --- a/src/rose/rose_build_dump.cpp +++ b/src/rose/rose_build_dump.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2019, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -757,13 +757,12 @@ CharReach shufti2cr(const u8 *lo, const u8 *hi, u8 bucket_mask) { static void dumpLookaroundShufti(ofstream &os, u32 len, const u8 *lo, const u8 *hi, - const u8 *bucket_mask, u32 neg_mask, s32 offset) { - assert(len == 16 || len == 32); + const u8 *bucket_mask, u64a neg_mask, s32 offset) { + assert(len == 16 || len == 32 || len == 64); os << " contents:" << endl; for (u32 idx = 0; idx < len; idx++) { CharReach cr = shufti2cr(lo, hi, bucket_mask[idx]); - - if (neg_mask & (1U << idx)) { + if (neg_mask & (1ULL << idx)) { cr.flip(); } @@ -779,14 +778,13 @@ void dumpLookaroundShufti(ofstream &os, u32 len, const u8 *lo, const u8 *hi, static void dumpLookaroundShufti(ofstream &os, u32 len, const u8 *lo, const u8 *hi, const u8 *lo_2, const u8 *hi_2, const u8 *bucket_mask, - const u8 *bucket_mask_2, u32 neg_mask, s32 offset) { - assert(len == 16 || len == 32); + const u8 *bucket_mask_2, u64a neg_mask, s32 offset) { + assert(len == 16 || len == 32 || len == 64); os << " contents:" << endl; for (u32 idx = 0; idx < len; idx++) { CharReach cr = shufti2cr(lo, hi, bucket_mask[idx]); cr |= shufti2cr(lo_2, hi_2, bucket_mask_2[idx]); - - if (neg_mask & (1U << idx)) { + if (neg_mask & (1ULL << idx)) { cr.flip(); } @@ -970,6 +968,20 @@ void dumpProgram(ofstream &os, const RoseEngine *t, const char *pc) { } PROGRAM_NEXT_INSTRUCTION + PROGRAM_CASE(CHECK_MASK_64) { + os << " and_mask " + << dumpStrMask(ri->and_mask, sizeof(ri->and_mask)) + << endl; + os << " cmp_mask " + << dumpStrMask(ri->cmp_mask, sizeof(ri->cmp_mask)) + << endl; + os << " neg_mask 0x" << std::hex << std::setw(8) + << std::setfill('0') << ri->neg_mask << std::dec << endl; + os << " offset " << ri->offset << endl; + os << " fail_jump " << offset + ri->fail_jump << endl; + } + PROGRAM_NEXT_INSTRUCTION + PROGRAM_CASE(CHECK_BYTE) { os << " and_mask 0x" << std::hex << std::setw(2) << std::setfill('0') << u32{ri->and_mask} << std::dec @@ -1072,6 +1084,60 @@ void dumpProgram(ofstream &os, const RoseEngine *t, const char *pc) { } PROGRAM_NEXT_INSTRUCTION + PROGRAM_CASE(CHECK_SHUFTI_64x8) { + os << " hi_mask " + << dumpStrMask(ri->hi_mask, sizeof(ri->hi_mask)) + << endl; + os << " lo_mask " + << dumpStrMask(ri->hi_mask, sizeof(ri->hi_mask)) + << endl; + os << " bucket_select_mask " + << dumpStrMask(ri->bucket_select_mask, + sizeof(ri->bucket_select_mask)) + << endl; + os << " neg_mask 0x" << std::hex << std::setw(8) + << std::setfill('0') << ri->neg_mask << std::dec << endl; + os << " offset " << ri->offset << endl; + os << " fail_jump " << offset + ri->fail_jump << endl; + dumpLookaroundShufti(os, 64, ri->lo_mask, ri->hi_mask, + ri->bucket_select_mask, ri->neg_mask, + ri->offset); + } + PROGRAM_NEXT_INSTRUCTION + + PROGRAM_CASE(CHECK_SHUFTI_64x16) { + os << " hi_mask_1 " + << dumpStrMask(ri->hi_mask_1, sizeof(ri->hi_mask_1)) + << endl; + os << " hi_mask_2 " + << dumpStrMask(ri->hi_mask_2, sizeof(ri->hi_mask_2)) + << endl; + os << " lo_mask_1 " + << dumpStrMask(ri->lo_mask_1, sizeof(ri->lo_mask_1)) + << endl; + os << " lo_mask_2 " + << dumpStrMask(ri->lo_mask_2, sizeof(ri->lo_mask_2)) + << endl; + os << " bucket_select_mask_hi " + << dumpStrMask(ri->bucket_select_mask_hi, + sizeof(ri->bucket_select_mask_hi)) + << endl; + os << " bucket_select_mask_lo " + << dumpStrMask(ri->bucket_select_mask_lo, + sizeof(ri->bucket_select_mask_lo)) + << endl; + os << " neg_mask 0x" << std::hex << std::setw(8) + << std::setfill('0') << ri->neg_mask << std::dec << endl; + os << " offset " << ri->offset << endl; + os << " fail_jump " << offset + ri->fail_jump << endl; + dumpLookaroundShufti(os, 64, ri->lo_mask_1, ri->hi_mask_1, + ri->lo_mask_2, ri->hi_mask_2, + ri->bucket_select_mask_lo, + ri->bucket_select_mask_hi, + ri->neg_mask, ri->offset); + } + PROGRAM_NEXT_INSTRUCTION + PROGRAM_CASE(CHECK_INFIX) { os << " queue " << ri->queue << endl; os << " lag " << ri->lag << endl; diff --git a/src/rose/rose_build_groups.cpp b/src/rose/rose_build_groups.cpp index c670e6033..209889e55 100644 --- a/src/rose/rose_build_groups.cpp +++ b/src/rose/rose_build_groups.cpp @@ -96,7 +96,7 @@ bool eligibleForAlwaysOnGroup(const RoseBuildImpl &build, u32 id) { static bool requires_group_assignment(const rose_literal_id &lit, const rose_literal_info &info) { - if (lit.delay) { /* we will check the shadow's master */ + if (lit.delay) { /* we will check the shadow's leader */ return false; } diff --git a/src/rose/rose_build_instructions.cpp b/src/rose/rose_build_instructions.cpp index c503f7311..f96221b24 100644 --- a/src/rose/rose_build_instructions.cpp +++ b/src/rose/rose_build_instructions.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019, Intel Corporation + * Copyright (c) 2017-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -162,6 +162,17 @@ void RoseInstrCheckMask32::write(void *dest, RoseEngineBlob &blob, inst->fail_jump = calc_jump(offset_map, this, target); } +void RoseInstrCheckMask64::write(void *dest, RoseEngineBlob &blob, + const OffsetMap &offset_map) const { + RoseInstrBase::write(dest, blob, offset_map); + auto *inst = static_cast(dest); + copy(begin(and_mask), end(and_mask), inst->and_mask); + copy(begin(cmp_mask), end(cmp_mask), inst->cmp_mask); + inst->neg_mask = neg_mask; + inst->offset = offset; + inst->fail_jump = calc_jump(offset_map, this, target); +} + void RoseInstrCheckByte::write(void *dest, RoseEngineBlob &blob, const OffsetMap &offset_map) const { RoseInstrBase::write(dest, blob, offset_map); @@ -227,6 +238,36 @@ void RoseInstrCheckShufti32x16::write(void *dest, RoseEngineBlob &blob, inst->fail_jump = calc_jump(offset_map, this, target); } +void RoseInstrCheckShufti64x8::write(void *dest, RoseEngineBlob &blob, + const OffsetMap &offset_map) const { + RoseInstrBase::write(dest, blob, offset_map); + auto *inst = static_cast(dest); + copy(begin(hi_mask), end(hi_mask), inst->hi_mask); + copy(begin(lo_mask), end(lo_mask), inst->lo_mask); + copy(begin(bucket_select_mask), end(bucket_select_mask), + inst->bucket_select_mask); + inst->neg_mask = neg_mask; + inst->offset = offset; + inst->fail_jump = calc_jump(offset_map, this, target); +} + +void RoseInstrCheckShufti64x16::write(void *dest, RoseEngineBlob &blob, + const OffsetMap &offset_map) const { + RoseInstrBase::write(dest, blob, offset_map); + auto *inst = static_cast(dest); + copy(begin(hi_mask_1), end(hi_mask_1), inst->hi_mask_1); + copy(begin(hi_mask_2), end(hi_mask_2), inst->hi_mask_2); + copy(begin(lo_mask_1), end(lo_mask_1), inst->lo_mask_1); + copy(begin(lo_mask_2), end(lo_mask_2), inst->lo_mask_2); + copy(begin(bucket_select_mask_hi), end(bucket_select_mask_hi), + inst->bucket_select_mask_hi); + copy(begin(bucket_select_mask_lo), end(bucket_select_mask_lo), + inst->bucket_select_mask_lo); + inst->neg_mask = neg_mask; + inst->offset = offset; + inst->fail_jump = calc_jump(offset_map, this, target); +} + void RoseInstrCheckInfix::write(void *dest, RoseEngineBlob &blob, const OffsetMap &offset_map) const { RoseInstrBase::write(dest, blob, offset_map); diff --git a/src/rose/rose_build_instructions.h b/src/rose/rose_build_instructions.h index 306a4166c..f18f4a471 100644 --- a/src/rose/rose_build_instructions.h +++ b/src/rose/rose_build_instructions.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019, Intel Corporation + * Copyright (c) 2017-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -519,6 +519,43 @@ class RoseInstrCheckMask32 } }; +class RoseInstrCheckMask64 + : public RoseInstrBaseOneTarget { +public: + std::array and_mask; + std::array cmp_mask; + u64a neg_mask; + s32 offset; + const RoseInstruction *target; + + RoseInstrCheckMask64(std::array and_mask_in, + std::array cmp_mask_in, u64a neg_mask_in, + s32 offset_in, const RoseInstruction *target_in) + : and_mask(std::move(and_mask_in)), cmp_mask(std::move(cmp_mask_in)), + neg_mask(neg_mask_in), offset(offset_in), target(target_in) {} + bool operator==(const RoseInstrCheckMask64 &ri) const { + return and_mask == ri.and_mask && cmp_mask == ri.cmp_mask && + neg_mask == ri.neg_mask && offset == ri.offset && + target == ri.target; + } + + size_t hash() const override { + return hash_all(opcode, and_mask, cmp_mask, neg_mask, offset); + } + + void write(void *dest, RoseEngineBlob &blob, + const OffsetMap &offset_map) const override; + + bool equiv_to(const RoseInstrCheckMask64 &ri, const OffsetMap &offsets, + const OffsetMap &other_offsets) const { + return and_mask == ri.and_mask && cmp_mask == ri.cmp_mask && + neg_mask == ri.neg_mask && offset == ri.offset && + offsets.at(target) == other_offsets.at(ri.target); + } +}; + class RoseInstrCheckByte : public RoseInstrBaseOneTarget { +public: + std::array hi_mask; + std::array lo_mask; + std::array bucket_select_mask; + u64a neg_mask; + s32 offset; + const RoseInstruction *target; + + RoseInstrCheckShufti64x8(std::array hi_mask_in, + std::array lo_mask_in, + std::array bucket_select_mask_in, + u64a neg_mask_in, s32 offset_in, + const RoseInstruction *target_in) + : hi_mask(std::move(hi_mask_in)), lo_mask(std::move(lo_mask_in)), + bucket_select_mask(std::move(bucket_select_mask_in)), + neg_mask(neg_mask_in), offset(offset_in), target(target_in) {} + + bool operator==(const RoseInstrCheckShufti64x8 &ri) const { + return hi_mask == ri.hi_mask && lo_mask == ri.lo_mask && + bucket_select_mask == ri.bucket_select_mask && + neg_mask == ri.neg_mask && offset == ri.offset && + target == ri.target; + } + + size_t hash() const override { + return hash_all(opcode, hi_mask, lo_mask, bucket_select_mask, neg_mask, + offset); + } + + void write(void *dest, RoseEngineBlob &blob, + const OffsetMap &offset_map) const override; + + bool equiv_to(const RoseInstrCheckShufti64x8 &ri, const OffsetMap &offsets, + const OffsetMap &other_offsets) const { + return hi_mask == ri.hi_mask && lo_mask == ri.lo_mask && + bucket_select_mask == ri.bucket_select_mask && + neg_mask == ri.neg_mask && offset == ri.offset && + offsets.at(target) == other_offsets.at(ri.target); + } +}; + +class RoseInstrCheckShufti64x16 + : public RoseInstrBaseOneTarget { +public: + std::array hi_mask_1; + std::array hi_mask_2; + std::array lo_mask_1; + std::array lo_mask_2; + std::array bucket_select_mask_hi; + std::array bucket_select_mask_lo; + u64a neg_mask; + s32 offset; + const RoseInstruction *target; + + RoseInstrCheckShufti64x16(std::array hi_mask_1_in, + std::array hi_mask_2_in, + std::array lo_mask_1_in, + std::array lo_mask_2_in, + std::array bucket_select_mask_hi_in, + std::array bucket_select_mask_lo_in, + u64a neg_mask_in, s32 offset_in, + const RoseInstruction *target_in) + : hi_mask_1(std::move(hi_mask_1_in)), hi_mask_2(std::move(hi_mask_2_in)), + lo_mask_1(std::move(lo_mask_1_in)), lo_mask_2(std::move(lo_mask_2_in)), + bucket_select_mask_hi(std::move(bucket_select_mask_hi_in)), + bucket_select_mask_lo(std::move(bucket_select_mask_lo_in)), + neg_mask(neg_mask_in), offset(offset_in), target(target_in) {} + + bool operator==(const RoseInstrCheckShufti64x16 &ri) const { + return hi_mask_1 == ri.hi_mask_1 && hi_mask_2 == ri.hi_mask_2 && + lo_mask_1 == ri.lo_mask_1 && lo_mask_2 == ri.lo_mask_2 && + bucket_select_mask_hi == ri.bucket_select_mask_hi && + bucket_select_mask_lo == ri.bucket_select_mask_lo && + neg_mask == ri.neg_mask && offset == ri.offset && + target == ri.target; + } + + size_t hash() const override { + return hash_all(opcode, hi_mask_1, hi_mask_2, lo_mask_1, lo_mask_2, + bucket_select_mask_hi, bucket_select_mask_lo, neg_mask, + offset); + } + + void write(void *dest, RoseEngineBlob &blob, + const OffsetMap &offset_map) const override; + + bool equiv_to(const RoseInstrCheckShufti64x16 &ri, const OffsetMap &offsets, + const OffsetMap &other_offsets) const { + return hi_mask_1 == ri.hi_mask_1 && hi_mask_2 == ri.hi_mask_2 && + lo_mask_1 == ri.lo_mask_1 && lo_mask_2 == ri.lo_mask_2 && + bucket_select_mask_hi == ri.bucket_select_mask_hi && + bucket_select_mask_lo == ri.bucket_select_mask_lo && + neg_mask == ri.neg_mask && offset == ri.offset && + offsets.at(target) == other_offsets.at(ri.target); + } +}; + class RoseInstrCheckInfix : public RoseInstrBaseOneTarget &look, return true; } +static +bool makeRoleMask64(const vector &look, + RoseProgram &program, const target_t &target) { + if (!target.has_avx512()) { + return false; + } + + if (look.back().offset >= look.front().offset + 64) { + return false; + } + s32 base_offset = verify_s32(look.front().offset); + array and_mask, cmp_mask; + and_mask.fill(0); + cmp_mask.fill(0); + u64a neg_mask = 0; + for (const auto &entry : look) { + u8 andmask_u8, cmpmask_u8, flip; + if (!checkReachWithFlip(entry.reach, andmask_u8, cmpmask_u8, flip)) { + return false; + } + u32 shift = entry.offset - base_offset; + assert(shift < 64); + and_mask[shift] = andmask_u8; + cmp_mask[shift] = cmpmask_u8; + if (flip) { + neg_mask |= 1ULL << shift; + } + } + + DEBUG_PRINTF("and_mask %s\n", + convertMaskstoString(and_mask.data(), 64).c_str()); + DEBUG_PRINTF("cmp_mask %s\n", + convertMaskstoString(cmp_mask.data(), 64).c_str()); + DEBUG_PRINTF("neg_mask %llx\n", neg_mask); + DEBUG_PRINTF("base_offset %d\n", base_offset); + + const auto *end_inst = program.end_instruction(); + auto ri = make_unique(and_mask, cmp_mask, neg_mask, + base_offset, end_inst); + program.add_before_end(move(ri)); + return true; +} + // Sorting by the size of every bucket. // Used in map, cmpNibble>. struct cmpNibble { @@ -1084,6 +1127,7 @@ void getAllBuckets(const vector &look, } else { neg_mask ^= 1ULL << (entry.offset - base_offset); } + map lo2hi; // We treat Ascii Table as a 16x16 grid. // Push every row in cr into lo2hi and mark the row number. @@ -1237,6 +1281,7 @@ makeCheckShufti16x16(u32 offset_range, u8 bucket_idx, (hi_mask, lo_mask, bucket_select_mask_32, neg_mask & 0xffff, base_offset, end_inst); } + static unique_ptr makeCheckShufti32x16(u32 offset_range, u8 bucket_idx, @@ -1255,10 +1300,83 @@ makeCheckShufti32x16(u32 offset_range, u8 bucket_idx, } static -bool makeRoleShufti(const vector &look, RoseProgram &program) { +unique_ptr +makeCheckShufti64x8(u32 offset_range, u8 bucket_idx, + const array &hi_mask, const array &lo_mask, + const array &bucket_select_mask, + u64a neg_mask, s32 base_offset, + const RoseInstruction *end_inst) { + if (offset_range > 64 || bucket_idx > 8) { + return nullptr; + } + + array hi_mask_64; + array lo_mask_64; + copy(hi_mask.begin(), hi_mask.begin() + 16, hi_mask_64.begin()); + copy(hi_mask.begin(), hi_mask.begin() + 16, hi_mask_64.begin() + 16); + copy(hi_mask.begin(), hi_mask.begin() + 16, hi_mask_64.begin() + 32); + copy(hi_mask.begin(), hi_mask.begin() + 16, hi_mask_64.begin() + 48); + copy(lo_mask.begin(), lo_mask.begin() + 16, lo_mask_64.begin()); + copy(lo_mask.begin(), lo_mask.begin() + 16, lo_mask_64.begin() + 16); + copy(lo_mask.begin(), lo_mask.begin() + 16, lo_mask_64.begin() + 32); + copy(lo_mask.begin(), lo_mask.begin() + 16, lo_mask_64.begin() + 48); + + return make_unique + (hi_mask_64, lo_mask_64, bucket_select_mask, + neg_mask, base_offset, end_inst); +} + +static +unique_ptr +makeCheckShufti64x16(u32 offset_range, u8 bucket_idx, + const array &hi_mask, const array &lo_mask, + const array &bucket_select_mask_lo, + const array &bucket_select_mask_hi, + u64a neg_mask, s32 base_offset, + const RoseInstruction *end_inst) { + if (offset_range > 64 || bucket_idx > 16) { + return nullptr; + } + + array hi_mask_1; + array hi_mask_2; + array lo_mask_1; + array lo_mask_2; + + copy(hi_mask.begin(), hi_mask.begin() + 16, hi_mask_1.begin()); + copy(hi_mask.begin(), hi_mask.begin() + 16, hi_mask_1.begin() + 16); + copy(hi_mask.begin(), hi_mask.begin() + 16, hi_mask_1.begin() + 32); + copy(hi_mask.begin(), hi_mask.begin() + 16, hi_mask_1.begin() + 48); + copy(hi_mask.begin() + 16, hi_mask.begin() + 32, hi_mask_2.begin()); + copy(hi_mask.begin() + 16, hi_mask.begin() + 32, hi_mask_2.begin() + 16); + copy(hi_mask.begin() + 16, hi_mask.begin() + 32, hi_mask_2.begin() + 32); + copy(hi_mask.begin() + 16, hi_mask.begin() + 32, hi_mask_2.begin() + 48); + + copy(lo_mask.begin(), lo_mask.begin() + 16, lo_mask_1.begin()); + copy(lo_mask.begin(), lo_mask.begin() + 16, lo_mask_1.begin() + 16); + copy(lo_mask.begin(), lo_mask.begin() + 16, lo_mask_1.begin() + 32); + copy(lo_mask.begin(), lo_mask.begin() + 16, lo_mask_1.begin() + 48); + copy(lo_mask.begin() + 16, lo_mask.begin() + 32, lo_mask_2.begin()); + copy(lo_mask.begin() + 16, lo_mask.begin() + 32, lo_mask_2.begin() + 16); + copy(lo_mask.begin() + 16, lo_mask.begin() + 32, lo_mask_2.begin() + 32); + copy(lo_mask.begin() + 16, lo_mask.begin() + 32, lo_mask_2.begin() + 48); + + return make_unique + (hi_mask_1, hi_mask_2, lo_mask_1, lo_mask_2, bucket_select_mask_hi, + bucket_select_mask_lo, neg_mask, base_offset, end_inst); +} +static +bool makeRoleShufti(const vector &look, RoseProgram &program, + const target_t &target) { + s32 offset_limit; + if (target.has_avx512()) { + offset_limit = 64; + } else { + offset_limit = 32; + } s32 base_offset = verify_s32(look.front().offset); - if (look.back().offset >= base_offset + 32) { + if (look.back().offset >= base_offset + offset_limit) { return false; } @@ -1266,17 +1384,40 @@ bool makeRoleShufti(const vector &look, RoseProgram &program) { u64a neg_mask_64; array hi_mask; array lo_mask; + array bucket_select_hi_64; // for AVX512 + array bucket_select_lo_64; // for AVX512 array bucket_select_hi; array bucket_select_lo; hi_mask.fill(0); lo_mask.fill(0); + bucket_select_hi_64.fill(0); + bucket_select_lo_64.fill(0); bucket_select_hi.fill(0); // will not be used in 16x8 and 32x8. bucket_select_lo.fill(0); - if (!getShuftiMasks(look, hi_mask, lo_mask, bucket_select_hi.data(), - bucket_select_lo.data(), neg_mask_64, bucket_idx, 32)) { - return false; + if (target.has_avx512()) { + if (!getShuftiMasks(look, hi_mask, lo_mask, bucket_select_hi_64.data(), + bucket_select_lo_64.data(), neg_mask_64, bucket_idx, + 32)) { + return false; + } + copy(bucket_select_hi_64.begin(), bucket_select_hi_64.begin() + 32, + bucket_select_hi.begin()); + copy(bucket_select_lo_64.begin(), bucket_select_lo_64.begin() + 32, + bucket_select_lo.begin()); + + DEBUG_PRINTF("bucket_select_hi_64 %s\n", + convertMaskstoString(bucket_select_hi_64.data(), 64).c_str()); + DEBUG_PRINTF("bucket_select_lo_64 %s\n", + convertMaskstoString(bucket_select_lo_64.data(), 64).c_str()); + } else { + if (!getShuftiMasks(look, hi_mask, lo_mask, bucket_select_hi.data(), + bucket_select_lo.data(), neg_mask_64, bucket_idx, + 32)) { + return false; + } } + u32 neg_mask = (u32)neg_mask_64; DEBUG_PRINTF("hi_mask %s\n", @@ -1299,6 +1440,13 @@ bool makeRoleShufti(const vector &look, RoseProgram &program) { bucket_select_lo, neg_mask, base_offset, end_inst); } + if (target.has_avx512()) { + if (!ri) { + ri = makeCheckShufti64x8(offset_range, bucket_idx, hi_mask, lo_mask, + bucket_select_lo_64, neg_mask_64, + base_offset, end_inst); + } + } if (!ri) { ri = makeCheckShufti16x16(offset_range, bucket_idx, hi_mask, lo_mask, bucket_select_lo, bucket_select_hi, @@ -1309,6 +1457,13 @@ bool makeRoleShufti(const vector &look, RoseProgram &program) { bucket_select_lo, bucket_select_hi, neg_mask, base_offset, end_inst); } + if (target.has_avx512()) { + if (!ri) { + ri = makeCheckShufti64x16(offset_range, bucket_idx, hi_mask, lo_mask, + bucket_select_lo_64, bucket_select_hi_64, + neg_mask_64, base_offset, end_inst); + } + } assert(ri); program.add_before_end(move(ri)); @@ -1321,7 +1476,7 @@ bool makeRoleShufti(const vector &look, RoseProgram &program) { */ static void makeLookaroundInstruction(const vector &look, - RoseProgram &program) { + RoseProgram &program, const target_t &target) { assert(!look.empty()); if (makeRoleByte(look, program)) { @@ -1345,7 +1500,11 @@ void makeLookaroundInstruction(const vector &look, return; } - if (makeRoleShufti(look, program)) { + if (makeRoleMask64(look, program, target)) { + return; + } + + if (makeRoleShufti(look, program, target)) { return; } @@ -1386,7 +1545,7 @@ void makeCheckLitMaskInstruction(const RoseBuildImpl &build, u32 lit_id, return; // all caseful chars handled by HWLM mask. } - makeLookaroundInstruction(look, program); + makeLookaroundInstruction(look, program, build.cc.target_info); } static @@ -1730,7 +1889,7 @@ void makeRoleLookaround(const RoseBuildImpl &build, findLookaroundMasks(build, v, look_more); mergeLookaround(look, look_more); if (!look.empty()) { - makeLookaroundInstruction(look, program); + makeLookaroundInstruction(look, program, build.cc.target_info); } return; } diff --git a/src/rose/rose_program.h b/src/rose/rose_program.h index e5485476b..7e21303cb 100644 --- a/src/rose/rose_program.h +++ b/src/rose/rose_program.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2019, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -208,7 +208,11 @@ enum RoseInstructionCode { */ ROSE_INSTR_LAST_FLUSH_COMBINATION, - LAST_ROSE_INSTRUCTION = ROSE_INSTR_LAST_FLUSH_COMBINATION //!< Sentinel. + ROSE_INSTR_CHECK_SHUFTI_64x8, //!< Check 64-byte data by 8-bucket shufti. + ROSE_INSTR_CHECK_SHUFTI_64x16, //!< Check 64-byte data by 16-bucket shufti. + ROSE_INSTR_CHECK_MASK_64, //!< 64-bytes and/cmp/neg mask check. + + LAST_ROSE_INSTRUCTION = ROSE_INSTR_CHECK_MASK_64 //!< Sentinel. }; struct ROSE_STRUCT_END { @@ -285,6 +289,15 @@ struct ROSE_STRUCT_CHECK_MASK_32 { u32 fail_jump; //!< Jump forward this many bytes on failure. }; +struct ROSE_STRUCT_CHECK_MASK_64 { + u8 code; //!< From enum RoseInstructionCode. + u8 and_mask[64]; //!< 64-byte and mask. + u8 cmp_mask[64]; //!< 64-byte cmp mask. + u64a neg_mask; //!< negation mask with 32 bits. + s32 offset; //!< Relative offset of the first byte. + u32 fail_jump; //!< Jump forward this many bytes on failure. +}; + struct ROSE_STRUCT_CHECK_BYTE { u8 code; //!< From enum RoseInstructionCode. u8 and_mask; //!< 8-bits and mask. @@ -336,6 +349,29 @@ struct ROSE_STRUCT_CHECK_SHUFTI_32x16 { u32 fail_jump; //!< Jump forward this many bytes on failure. }; +struct ROSE_STRUCT_CHECK_SHUFTI_64x8 { + u8 code; //!< From enum RoseInstructionCode. + u8 hi_mask[64]; //!< High nibble mask in shufti. + u8 lo_mask[64]; //!< Low nibble mask in shufti. + u8 bucket_select_mask[64]; //!< Mask for bucket assigning. + u64a neg_mask; //!< 64 bits negation mask. + s32 offset; //!< Relative offset of the first byte. + u32 fail_jump; //!< Jump forward this many bytes on failure. +}; + +struct ROSE_STRUCT_CHECK_SHUFTI_64x16 { + u8 code; //!< From enum RoseInstructionCode. + u8 hi_mask_1[64]; //!< 4 copies of 0-15 High nibble mask. + u8 hi_mask_2[64]; //!< 4 copies of 16-32 High nibble mask. + u8 lo_mask_1[64]; //!< 4 copies of 0-15 Low nibble mask. + u8 lo_mask_2[64]; //!< 4 copies of 16-32 Low nibble mask. + u8 bucket_select_mask_hi[64]; //!< Bucket mask for high 8 buckets. + u8 bucket_select_mask_lo[64]; //!< Bucket mask for low 8 buckets. + u64a neg_mask; //!< 64 bits negation mask. + s32 offset; //!< Relative offset of the first byte. + u32 fail_jump; //!< Jump forward this many bytes on failure. +}; + struct ROSE_STRUCT_CHECK_INFIX { u8 code; //!< From enum RoseInstructionCode. u32 queue; //!< Queue of leftfix to check. diff --git a/src/rose/stream_long_lit.h b/src/rose/stream_long_lit.h index 348676087..df9b57f4e 100644 --- a/src/rose/stream_long_lit.h +++ b/src/rose/stream_long_lit.h @@ -201,12 +201,12 @@ const u8 *prepScanBuffer(const struct core_info *ci, } else { // Copy: first chunk from history buffer. assert(overhang <= ci->hlen); - copy_upto_32_bytes(tempbuf, ci->hbuf + ci->hlen - overhang, + copy_upto_64_bytes(tempbuf, ci->hbuf + ci->hlen - overhang, overhang); // Copy: second chunk from current buffer. size_t copy_buf_len = LONG_LIT_HASH_LEN - overhang; assert(copy_buf_len <= ci->len); - copy_upto_32_bytes(tempbuf + overhang, ci->buf, copy_buf_len); + copy_upto_64_bytes(tempbuf + overhang, ci->buf, copy_buf_len); // Read from our temporary buffer for the hash. base = tempbuf; } diff --git a/src/rose/validate_mask.h b/src/rose/validate_mask.h index ac8cc312e..8191db52f 100644 --- a/src/rose/validate_mask.h +++ b/src/rose/validate_mask.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,6 +41,17 @@ void validateMask32Print(const u8 *mask) { } printf("\n"); } + +#ifdef HAVE_AVX512 +static +void validateMask64Print(const u8 *mask) { + int i; + for (i = 0; i < 64; i++) { + printf("%02x ", mask[i]); + } + printf("\n"); +} +#endif #endif // check positive bytes in cmp_result. @@ -115,4 +126,29 @@ int validateMask32(const m256 data, const u32 valid_data_mask, } } +#ifdef HAVE_AVX512 +static really_inline +int validateMask64(const m512 data, const u64a valid_data_mask, + const m512 and_mask, const m512 cmp_mask, + const u64a neg_mask) { + u64a cmp_result = ~eq512mask(and512(data, and_mask), cmp_mask); +#ifdef DEBUG + DEBUG_PRINTF("data\n"); + validateMask64Print((const u8 *)&data); + DEBUG_PRINTF("cmp_result\n"); + validateMask64Print((const u8 *)&cmp_result); +#endif + DEBUG_PRINTF("cmp_result %016llx neg_mask %016llx\n", cmp_result, neg_mask); + DEBUG_PRINTF("valid_data_mask %016llx\n", valid_data_mask); + + if ((cmp_result & valid_data_mask) == (neg_mask & valid_data_mask)) { + DEBUG_PRINTF("checkCompareResult64 passed\n"); + return 1; + } else { + DEBUG_PRINTF("checkCompareResult64 failed\n"); + return 0; + } +} +#endif + #endif diff --git a/src/rose/validate_shufti.h b/src/rose/validate_shufti.h index 1dc855d99..351df36a7 100644 --- a/src/rose/validate_shufti.h +++ b/src/rose/validate_shufti.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -175,6 +175,84 @@ int validateShuftiMask32x16(const m256 data, return !cmp_result; } +#ifdef HAVE_AVX512 +static really_inline +int validateShuftiMask64x8(const m512 data, const m512 hi_mask, + const m512 lo_mask, const m512 and_mask, + const u64a neg_mask, const u64a valid_data_mask) { + m512 low4bits = set64x8(0xf); + m512 c_lo = pshufb_m512(lo_mask, and512(data, low4bits)); + m512 c_hi = pshufb_m512(hi_mask, + rshift64_m512(andnot512(low4bits, data), 4)); + m512 t = and512(c_lo, c_hi); + u64a nresult = eq512mask(and512(t, and_mask), zeroes512()); +#ifdef DEBUG + DEBUG_PRINTF("data\n"); + dumpMask(&data, 64); + DEBUG_PRINTF("hi_mask\n"); + dumpMask(&hi_mask, 64); + DEBUG_PRINTF("lo_mask\n"); + dumpMask(&lo_mask, 64); + DEBUG_PRINTF("c_lo\n"); + dumpMask(&c_lo, 64); + DEBUG_PRINTF("c_hi\n"); + dumpMask(&c_hi, 64); + DEBUG_PRINTF("nresult %llx\n", nresult); + DEBUG_PRINTF("valid_data_mask %llx\n", valid_data_mask); +#endif + u64a cmp_result = (nresult ^ neg_mask) & valid_data_mask; + return !cmp_result; +} + +static really_inline +int validateShuftiMask64x16(const m512 data, + const m512 hi_mask_1, const m512 hi_mask_2, + const m512 lo_mask_1, const m512 lo_mask_2, + const m512 and_mask_hi, const m512 and_mask_lo, + const u64a neg_mask, const u64a valid_data_mask) { + m512 low4bits = set64x8(0xf); + m512 data_lo = and512(data, low4bits); + m512 data_hi = and512(rshift64_m512(data, 4), low4bits); + m512 c_lo_1 = pshufb_m512(lo_mask_1, data_lo); + m512 c_lo_2 = pshufb_m512(lo_mask_2, data_lo); + m512 c_hi_1 = pshufb_m512(hi_mask_1, data_hi); + m512 c_hi_2 = pshufb_m512(hi_mask_2, data_hi); + m512 t1 = and512(c_lo_1, c_hi_1); + m512 t2 = and512(c_lo_2, c_hi_2); + m512 result = or512(and512(t1, and_mask_lo), and512(t2, and_mask_hi)); + u64a nresult = eq512mask(result, zeroes512()); +#ifdef DEBUG + DEBUG_PRINTF("data\n"); + dumpMask(&data, 64); + DEBUG_PRINTF("data_lo\n"); + dumpMask(&data_lo, 64); + DEBUG_PRINTF("data_hi\n"); + dumpMask(&data_hi, 64); + DEBUG_PRINTF("hi_mask_1\n"); + dumpMask(&hi_mask_1, 64); + DEBUG_PRINTF("hi_mask_2\n"); + dumpMask(&hi_mask_2, 64); + DEBUG_PRINTF("lo_mask_1\n"); + dumpMask(&lo_mask_1, 64); + DEBUG_PRINTF("lo_mask_2\n"); + dumpMask(&lo_mask_2, 64); + DEBUG_PRINTF("c_lo_1\n"); + dumpMask(&c_lo_1, 64); + DEBUG_PRINTF("c_lo_2\n"); + dumpMask(&c_lo_2, 64); + DEBUG_PRINTF("c_hi_1\n"); + dumpMask(&c_hi_1, 64); + DEBUG_PRINTF("c_hi_2\n"); + dumpMask(&c_hi_2, 64); + DEBUG_PRINTF("result\n"); + dumpMask(&result, 64); + DEBUG_PRINTF("valid_data_mask %llx\n", valid_data_mask); +#endif + u64a cmp_result = (nresult ^ neg_mask) & valid_data_mask; + return !cmp_result; +} +#endif + static really_inline int checkMultipath32(u32 data, u32 hi_bits, u32 lo_bits) { u32 t = ~(data | hi_bits); diff --git a/src/smallwrite/smallwrite_build.cpp b/src/smallwrite/smallwrite_build.cpp index 345edfe95..4eb4801db 100644 --- a/src/smallwrite/smallwrite_build.cpp +++ b/src/smallwrite/smallwrite_build.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -793,6 +793,12 @@ bytecode_ptr getDfa(raw_dfa &rdfa, const CompileContext &cc, bytecode_ptr dfa = nullptr; if (cc.grey.allowSmallWriteSheng) { dfa = shengCompile(rdfa, cc, rm, only_accel_init, &accel_states); + if (!dfa) { + dfa = sheng32Compile(rdfa, cc, rm, only_accel_init, &accel_states); + } + if (!dfa) { + dfa = sheng64Compile(rdfa, cc, rm, only_accel_init, &accel_states); + } } if (!dfa) { dfa = mcclellanCompile(rdfa, cc, rm, only_accel_init, diff --git a/src/util/copybytes.h b/src/util/copybytes.h index 872b8d289..7f37d96bc 100644 --- a/src/util/copybytes.h +++ b/src/util/copybytes.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Intel Corporation + * Copyright (c) 2016-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,7 +33,7 @@ #include "simd_utils.h" static really_inline -void copy_upto_32_bytes(u8 *dst, const u8 *src, unsigned int len) { +void copy_upto_64_bytes(u8 *dst, const u8 *src, unsigned int len) { switch (len) { case 0: break; @@ -72,14 +72,41 @@ void copy_upto_32_bytes(u8 *dst, const u8 *src, unsigned int len) { case 16: storeu128(dst, loadu128(src)); break; + case 17: + case 18: + case 19: + case 20: + case 21: + case 22: + case 23: + case 24: + case 25: + case 26: + case 27: + case 28: + case 29: + case 30: + case 31: + storeu128(dst + len - 16, loadu128(src + len - 16)); + storeu128(dst, loadu128(src)); + break; case 32: storeu256(dst, loadu256(src)); break; +#ifdef HAVE_AVX512 + case 64: + storebytes512(dst, loadu512(src), 64); + break; default: - assert(len < 32); - storeu128(dst + len - 16, loadu128(src + len - 16)); - storeu128(dst, loadu128(src)); + assert(len < 64); + u64a k = (1ULL << len) - 1; + storeu_mask_m512(dst, k, loadu_maskz_m512(k, src)); + break; +#else + default: + assert(0); break; +#endif } } diff --git a/src/util/cpuid_flags.c b/src/util/cpuid_flags.c index 0b529c0bf..c00ce58e2 100644 --- a/src/util/cpuid_flags.c +++ b/src/util/cpuid_flags.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -50,6 +50,11 @@ u64a cpuid_flags(void) { cap |= HS_CPU_FEATURES_AVX512; } + if (check_avx512vbmi()) { + DEBUG_PRINTF("AVX512VBMI enabled\n"); + cap |= HS_CPU_FEATURES_AVX512VBMI; + } + #if !defined(FAT_RUNTIME) && !defined(HAVE_AVX2) cap &= ~HS_CPU_FEATURES_AVX2; #endif @@ -59,6 +64,11 @@ u64a cpuid_flags(void) { cap &= ~HS_CPU_FEATURES_AVX512; #endif +#if (!defined(FAT_RUNTIME) && !defined(HAVE_AVX512VBMI)) || \ + (defined(FAT_RUNTIME) && !defined(BUILD_AVX512VBMI)) + cap &= ~HS_CPU_FEATURES_AVX512VBMI; +#endif + return cap; } @@ -105,6 +115,11 @@ static const struct family_id known_microarch[] = { { 0x6, 0x8E, HS_TUNE_FAMILY_SKL }, /* Kabylake Mobile */ { 0x6, 0x9E, HS_TUNE_FAMILY_SKL }, /* Kabylake desktop */ + { 0x6, 0x7D, HS_TUNE_FAMILY_ICL }, /* Icelake */ + { 0x6, 0x7E, HS_TUNE_FAMILY_ICL }, /* Icelake */ + { 0x6, 0x6A, HS_TUNE_FAMILY_ICX }, /* Icelake Xeon-D */ + { 0x6, 0x6C, HS_TUNE_FAMILY_ICX }, /* Icelake Xeon */ + }; #ifdef DUMP_SUPPORT @@ -120,6 +135,8 @@ const char *dumpTune(u32 tune) { T_CASE(HS_TUNE_FAMILY_BDW); T_CASE(HS_TUNE_FAMILY_SKL); T_CASE(HS_TUNE_FAMILY_SKX); + T_CASE(HS_TUNE_FAMILY_ICL); + T_CASE(HS_TUNE_FAMILY_ICX); } #undef T_CASE return "unknown"; diff --git a/src/util/cpuid_inline.h b/src/util/cpuid_inline.h index b6768cc26..b7b424528 100644 --- a/src/util/cpuid_inline.h +++ b/src/util/cpuid_inline.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Intel Corporation + * Copyright (c) 2017-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -74,11 +74,12 @@ void cpuid(unsigned int op, unsigned int leaf, unsigned int *eax, #define CPUID_HTT (1 << 28) // Structured Extended Feature Flags Enumeration Leaf ECX values +#define CPUID_AVX512VBMI (1 << 1) + +// Structured Extended Feature Flags Enumeration Leaf EBX values #define CPUID_BMI (1 << 3) #define CPUID_AVX2 (1 << 5) #define CPUID_BMI2 (1 << 8) - -// Structured Extended Feature Flags Enumeration Leaf EBX values #define CPUID_AVX512F (1 << 16) #define CPUID_AVX512BW (1 << 30) @@ -186,6 +187,51 @@ int check_avx512(void) { #endif } +static inline +int check_avx512vbmi(void) { +#if defined(__INTEL_COMPILER) + return _may_i_use_cpu_feature(_FEATURE_AVX512VBMI); +#else + unsigned int eax, ebx, ecx, edx; + + cpuid(1, 0, &eax, &ebx, &ecx, &edx); + + /* check XSAVE is enabled by OS */ + if (!(ecx & CPUID_XSAVE)) { + DEBUG_PRINTF("AVX and XSAVE not supported\n"); + return 0; + } + + /* check that AVX 512 registers are enabled by OS */ + u64a xcr0 = xgetbv(0); + if ((xcr0 & CPUID_XCR0_AVX512) != CPUID_XCR0_AVX512) { + DEBUG_PRINTF("AVX512 registers not enabled\n"); + return 0; + } + + /* ECX and EDX contain capability flags */ + ecx = 0; + cpuid(7, 0, &eax, &ebx, &ecx, &edx); + + if (!(ebx & CPUID_AVX512F)) { + DEBUG_PRINTF("AVX512F (AVX512 Foundation) instructions not enabled\n"); + return 0; + } + + if (!(ebx & CPUID_AVX512BW)) { + DEBUG_PRINTF("AVX512BW instructions not enabled\n"); + return 0; + } + + if (ecx & CPUID_AVX512VBMI) { + DEBUG_PRINTF("AVX512VBMI instructions enabled\n"); + return 1; + } + + return 0; +#endif +} + static inline int check_ssse3(void) { unsigned int eax, ebx, ecx, edx; diff --git a/src/util/graph.h b/src/util/graph.h index 660afd029..3e18dae55 100644 --- a/src/util/graph.h +++ b/src/util/graph.h @@ -170,6 +170,7 @@ find_vertices_in_cycles(const Graph &g) { assert(!comp.empty()); if (comp.size() > 1) { insert(&rv, comp); + continue; } vertex_descriptor v = *comp.begin(); if (hasSelfLoop(v, g)) { diff --git a/src/util/simd_utils.h b/src/util/simd_utils.h index 42223133d..d1f060b07 100644 --- a/src/util/simd_utils.h +++ b/src/util/simd_utils.h @@ -138,6 +138,12 @@ m128 lshift64_m128(m128 a, unsigned b) { #define eq128(a, b) _mm_cmpeq_epi8((a), (b)) #define movemask128(a) ((u32)_mm_movemask_epi8((a))) +#if defined(HAVE_AVX512) +static really_inline m128 cast512to128(const m512 in) { + return _mm512_castsi512_si128(in); +} +#endif + static really_inline m128 set16x8(u8 c) { return _mm_set1_epi8(c); } @@ -156,6 +162,12 @@ static really_inline u32 movd512(const m512 in) { // so we use 2-step convertions to work around. return _mm_cvtsi128_si32(_mm512_castsi512_si128(in)); } + +static really_inline u64a movq512(const m512 in) { + // NOTE: seems AVX512 doesn't support _mm512_cvtsi512_si64(in), + // so we use 2-step convertions to work around. + return _mm_cvtsi128_si64(_mm512_castsi512_si128(in)); +} #endif static really_inline u64a movq(const m128 in) { @@ -211,6 +223,24 @@ static really_inline m128 or128(m128 a, m128 b) { return _mm_or_si128(a,b); } +#if defined(HAVE_AVX512VBMI) +static really_inline m512 expand128(m128 a) { + return _mm512_broadcast_i32x4(a); +} + +static really_inline m512 expand256(m256 a) { + return _mm512_broadcast_i64x4(a); +} + +static really_inline m512 expand384(m384 a) { + u64a *lo = (u64a*)&a.lo; + u64a *mid = (u64a*)&a.mid; + u64a *hi = (u64a*)&a.hi; + return _mm512_set_epi64(0ULL, 0ULL, hi[1], hi[0], mid[1], mid[0], + lo[1], lo[0]); +} +#endif + static really_inline m128 andnot128(m128 a, m128 b) { return _mm_andnot_si128(a, b); } @@ -1000,6 +1030,11 @@ m512 set8x64(u64a a) { return _mm512_set1_epi64(a); } +static really_inline +m512 set16x32(u32 a) { + return _mm512_set1_epi32(a); +} + static really_inline m512 set512_64(u64a hi_3, u64a hi_2, u64a hi_1, u64a hi_0, u64a lo_3, u64a lo_2, u64a lo_1, u64a lo_0) { @@ -1017,6 +1052,26 @@ static really_inline m512 set4x128(m128 a) { return _mm512_broadcast_i32x4(a); } + +static really_inline +m512 sadd_u8_m512(m512 a, m512 b) { + return _mm512_adds_epu8(a, b); +} + +static really_inline +m512 max_u8_m512(m512 a, m512 b) { + return _mm512_max_epu8(a, b); +} + +static really_inline +m512 min_u8_m512(m512 a, m512 b) { + return _mm512_min_epu8(a, b); +} + +static really_inline +m512 sub_u8_m512(m512 a, m512 b) { + return _mm512_sub_epi8(a, b); +} #endif static really_inline @@ -1204,6 +1259,22 @@ m512 loadu512(const void *ptr) { #endif } +// unaligned store +static really_inline +void storeu512(void *ptr, m512 a) { +#if defined(HAVE_AVX512) + _mm512_storeu_si512((m512 *)ptr, a); +#elif defined(HAVE_AVX2) + storeu256(ptr, a.lo); + storeu256((char *)ptr + 32, a.hi); +#else + storeu128(ptr, a.lo.lo); + storeu128((char *)ptr + 16, a.lo.hi); + storeu128((char *)ptr + 32, a.hi.lo); + storeu128((char *)ptr + 48, a.hi.hi); +#endif +} + #if defined(HAVE_AVX512) static really_inline m512 loadu_maskz_m512(__mmask64 k, const void *ptr) { @@ -1215,10 +1286,20 @@ m512 loadu_mask_m512(m512 src, __mmask64 k, const void *ptr) { return _mm512_mask_loadu_epi8(src, k, ptr); } +static really_inline +void storeu_mask_m512(void *ptr, __mmask64 k, m512 a) { + _mm512_mask_storeu_epi8(ptr, k, a); +} + static really_inline m512 set_mask_m512(__mmask64 k) { return _mm512_movm_epi8(k); } + +static really_inline +m256 loadu_maskz_m256(__mmask32 k, const void *ptr) { + return _mm256_maskz_loadu_epi8(k, ptr); +} #endif // packed unaligned store of first N bytes diff --git a/src/util/target_info.cpp b/src/util/target_info.cpp index 3a41e0207..66ba5f5ac 100644 --- a/src/util/target_info.cpp +++ b/src/util/target_info.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -50,6 +50,10 @@ bool target_t::can_run_on_code_built_for(const target_t &code_target) const { return false; } + if (!has_avx512vbmi() && code_target.has_avx512vbmi()) { + return false; + } + return true; } @@ -64,6 +68,10 @@ bool target_t::has_avx512(void) const { return cpu_features & HS_CPU_FEATURES_AVX512; } +bool target_t::has_avx512vbmi(void) const { + return cpu_features & HS_CPU_FEATURES_AVX512VBMI; +} + bool target_t::is_atom_class(void) const { return tune == HS_TUNE_FAMILY_SLM || tune == HS_TUNE_FAMILY_GLM; } diff --git a/src/util/target_info.h b/src/util/target_info.h index 794b29855..f64573aed 100644 --- a/src/util/target_info.h +++ b/src/util/target_info.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -42,6 +42,8 @@ struct target_t { bool has_avx512(void) const; + bool has_avx512vbmi(void) const; + bool is_atom_class(void) const; // This asks: can this target (the object) run on code that was built for diff --git a/src/util/uniform_ops.h b/src/util/uniform_ops.h index 3385e4418..262104aca 100644 --- a/src/util/uniform_ops.h +++ b/src/util/uniform_ops.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -101,6 +101,18 @@ #define or_m384(a, b) (or384(a, b)) #define or_m512(a, b) (or512(a, b)) +#if defined(HAVE_AVX512VBMI) +#define expand_m128(a) (expand128(a)) +#define expand_m256(a) (expand256(a)) +#define expand_m384(a) (expand384(a)) +#define expand_m512(a) (a) + +#define shuffle_byte_m128(a, b) (pshufb_m512(b, a)) +#define shuffle_byte_m256(a, b) (vpermb512(a, b)) +#define shuffle_byte_m384(a, b) (vpermb512(a, b)) +#define shuffle_byte_m512(a, b) (vpermb512(a, b)) +#endif + #define and_u8(a, b) ((a) & (b)) #define and_u32(a, b) ((a) & (b)) #define and_u64a(a, b) ((a) & (b)) diff --git a/tools/hsbench/engine.h b/tools/hsbench/engine.h index e41f9948c..aea1c8162 100644 --- a/tools/hsbench/engine.h +++ b/tools/hsbench/engine.h @@ -88,6 +88,8 @@ class Engine : boost::noncopyable { virtual void printStats() const = 0; + virtual void printCsvStats() const = 0; + virtual void sqlStats(SqlDB &db) const = 0; }; diff --git a/tools/hsbench/engine_chimera.cpp b/tools/hsbench/engine_chimera.cpp index 8a15c5bee..24a99d61f 100644 --- a/tools/hsbench/engine_chimera.cpp +++ b/tools/hsbench/engine_chimera.cpp @@ -187,6 +187,16 @@ void EngineChimera::printStats() const { #endif } +void EngineChimera::printCsvStats() const { + printf(",\"%s\"", compile_stats.signatures.c_str()); + printf(",\"%zu\"", compile_stats.expressionCount); + printf(",\"0x%x\"", compile_stats.crc32); + printf(",\"%zu\"", compile_stats.compiledSize); + printf(",\"%zu\"", compile_stats.scratchSize); + printf(",\"%0.3Lf\"", compile_stats.compileSecs); + printf(",\"%u\"", compile_stats.peakMemorySize); +} + void EngineChimera::sqlStats(SqlDB &sqldb) const { ostringstream crc; crc << "0x" << hex << compile_stats.crc32; diff --git a/tools/hsbench/engine_chimera.h b/tools/hsbench/engine_chimera.h index 8e2cd0f6c..187dec8cb 100644 --- a/tools/hsbench/engine_chimera.h +++ b/tools/hsbench/engine_chimera.h @@ -89,6 +89,8 @@ class EngineChimera : public Engine { void printStats() const; + void printCsvStats() const; + void sqlStats(SqlDB &db) const; private: diff --git a/tools/hsbench/engine_hyperscan.cpp b/tools/hsbench/engine_hyperscan.cpp index 79c58f77d..c94b42af7 100644 --- a/tools/hsbench/engine_hyperscan.cpp +++ b/tools/hsbench/engine_hyperscan.cpp @@ -276,6 +276,17 @@ void EngineHyperscan::printStats() const { #endif } +void EngineHyperscan::printCsvStats() const { + printf(",\"%s\"", compile_stats.signatures.c_str()); + printf(",\"%zu\"", compile_stats.expressionCount); + printf(",\"0x%x\"", compile_stats.crc32); + printf(",\"%zu\"", compile_stats.compiledSize); + printf(",\"%zu\"", compile_stats.streamSize); + printf(",\"%zu\"", compile_stats.scratchSize); + printf(",\"%0.3Lf\"", compile_stats.compileSecs); + printf(",\"%u\"", compile_stats.peakMemorySize); +} + void EngineHyperscan::sqlStats(SqlDB &sqldb) const { ostringstream crc; crc << "0x" << hex << compile_stats.crc32; diff --git a/tools/hsbench/engine_hyperscan.h b/tools/hsbench/engine_hyperscan.h index a8105d753..afbdf098d 100644 --- a/tools/hsbench/engine_hyperscan.h +++ b/tools/hsbench/engine_hyperscan.h @@ -65,8 +65,8 @@ class EngineHSContext : public EngineContext { class EngineHSStream : public EngineStream { public: ~EngineHSStream(); - hs_stream_t *id; - EngineHSContext *ctx; + hs_stream_t *id = nullptr; + EngineHSContext *ctx = nullptr; }; /** Hyperscan Engine for scanning data. */ @@ -98,6 +98,8 @@ class EngineHyperscan : public Engine { void printStats() const; + void printCsvStats() const; + void sqlStats(SqlDB &db) const; private: diff --git a/tools/hsbench/engine_pcre.cpp b/tools/hsbench/engine_pcre.cpp index 85616e987..23ab9d176 100644 --- a/tools/hsbench/engine_pcre.cpp +++ b/tools/hsbench/engine_pcre.cpp @@ -227,6 +227,15 @@ void EnginePCRE::printStats() const { #endif } +void EnginePCRE::printCsvStats() const { + printf(",\"%s\"", compile_stats.signatures.c_str()); + printf(",\"%zu\"", compile_stats.expressionCount); + printf(",\"%zu\"", compile_stats.compiledSize); + printf(",\"%zu\"", compile_stats.scratchSize); + printf(",\"%0.3Lf\"", compile_stats.compileSecs); + printf(",\"%u\"", compile_stats.peakMemorySize); +} + void EnginePCRE::sqlStats(SqlDB &sqldb) const { ostringstream crc; diff --git a/tools/hsbench/engine_pcre.h b/tools/hsbench/engine_pcre.h index 2e7dad9c5..9569bef48 100644 --- a/tools/hsbench/engine_pcre.h +++ b/tools/hsbench/engine_pcre.h @@ -62,7 +62,7 @@ class EnginePCREContext : public EngineContext{ struct PcreDB { bool highlander = false; bool utf8 = false; - u32 id; + u32 id = 0; pcre *db = nullptr; pcre_extra *extra = nullptr; }; @@ -97,6 +97,8 @@ class EnginePCRE : public Engine { void printStats() const; + void printCsvStats() const; + void sqlStats(SqlDB &db) const; private: diff --git a/tools/hsbench/main.cpp b/tools/hsbench/main.cpp index 4e65c8e0b..1c91813b2 100644 --- a/tools/hsbench/main.cpp +++ b/tools/hsbench/main.cpp @@ -98,6 +98,7 @@ bool display_per_scan = false; ScanMode scan_mode = ScanMode::STREAMING; bool useHybrid = false; bool usePcre = false; +bool dumpCsvOut = false; unsigned repeats = 20; string exprPath(""); string corpusFile(""); @@ -211,6 +212,7 @@ void usage(const char *error) { printf(" Benchmark with threads on specified CPUs or CPU" " range.\n"); #endif + printf(" -C Dump CSV output for tput matrix.\n"); printf(" -i DIR Don't compile, load from files in DIR" " instead.\n"); printf(" -w DIR After compiling, save to files in DIR.\n"); @@ -275,6 +277,9 @@ void processArgs(int argc, char *argv[], vector &sigSets, case 'c': corpusFile.assign(optarg); break; + case 'C': + dumpCsvOut = true; + break; case 'd': { unsigned dist; if (!fromString(optarg, dist)) { @@ -849,6 +854,40 @@ void displayResults(const vector> &threads, } } +/** Dump benchmark results to csv. */ +static +void displayCsvResults(const vector> &threads, + const vector &corpus_blocks) { + u64a bytesPerRun = byte_size(corpus_blocks); + u64a matchesPerRun = threads[0]->results[0].matches; + + // Sanity check: all of our results should have the same match count. + for (const auto &t : threads) { + if (!all_of(begin(t->results), end(t->results), + [&matchesPerRun](const ResultEntry &e) { + return e.matches == matchesPerRun; + })) { + printf("\nWARNING: PER-SCAN MATCH COUNTS ARE INCONSISTENT!\n\n"); + break; + } + } + + u64a totalBytes = bytesPerRun * repeats * threads.size(); + u64a totalBlocks = corpus_blocks.size() * repeats * threads.size(); + printf(",\"%0.3f\"", totalSecs); + printf(",\"%0.2Lf\"", calc_mbps(totalSecs, totalBytes)); + + assert(bytesPerRun); + double matchRate = ((double)matchesPerRun * 1024) / bytesPerRun; + printf(",\"%llu\"", matchesPerRun); + printf(",\"%0.3f\"", matchRate); + + double blockRate = (double)totalBlocks / (double)totalSecs; + printf(",\"%0.2f\"", blockRate); + printf("\n"); +} + + /** Dump per-scan throughput data to sql. */ static void sqlPerScanResults(const vector> &threads, @@ -982,7 +1021,9 @@ void runBenchmark(const Engine &db, t->join(); } - if (sqloutFile.empty()) { + if (dumpCsvOut) { + displayCsvResults(threads, corpus_blocks); + } else if (sqloutFile.empty()) { // Display global results. displayResults(threads, corpus_blocks); } else { @@ -1059,7 +1100,9 @@ int HS_CDECL main(int argc, char *argv[]) { exit(1); } - if (sqloutFile.empty()) { + if (dumpCsvOut) { + engine->printCsvStats(); + } else if (sqloutFile.empty()) { // Display global results. engine->printStats(); printf("\n"); diff --git a/unit/internal/limex_nfa.cpp b/unit/internal/limex_nfa.cpp index c70ceeae1..28433c968 100644 --- a/unit/internal/limex_nfa.cpp +++ b/unit/internal/limex_nfa.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, Intel Corporation + * Copyright (c) 2015-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -83,9 +83,10 @@ class LimExModelTest : public TestWithParam { const map fixed_depth_tops; const map>> triggers; bool compress_state = false; + bool fast_nfa = false; nfa = constructNFA(*g, &rm, fixed_depth_tops, triggers, compress_state, - type, cc); + fast_nfa, type, cc); ASSERT_TRUE(nfa != nullptr); full_state = make_bytecode_ptr(nfa->scratchStateSize, 64); @@ -376,9 +377,10 @@ class LimExZombieTest : public TestWithParam { const map fixed_depth_tops; const map>> triggers; bool compress_state = false; + bool fast_nfa = false; nfa = constructNFA(*g, &rm, fixed_depth_tops, triggers, compress_state, - type, cc); + fast_nfa, type, cc); ASSERT_TRUE(nfa != nullptr); full_state = make_bytecode_ptr(nfa->scratchStateSize, 64);