Merge branch 'master' into anilm3/abseil-containers

.clang-tidy

@@ -1,7 +1,7 @@
 ---
 # readability-function-cognitive-complexity temporarily disabled until clang-tidy is fixed
 # right now emalloc causes it to misbehave
-Checks: '*,-bugprone-reserved-identifier,-hicpp-signed-bitwise,-llvmlibc-restrict-system-libc-headers,-altera-unroll-loops,-hicpp-named-parameter,-cert-dcl37-c,-cert-dcl51-cpp,-read,-cppcoreguidelines-init-variables,-cppcoreguidelines-avoid-non-const-global-variables,-altera-id-dependent-backward-branch,-performance-no-int-to-ptr,-altera-struct-pack-align,-google-readability-casting,-modernize-use-trailing-return-type,-llvmlibc-implementation-in-namespace,-llvmlibc-callee-namespace,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-fuchsia-default-arguments-declarations,-fuchsia-overloaded-operator,-cppcoreguidelines-pro-type-union-access,-fuchsia-default-arguments-calls,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-google-readability-todo,-llvm-header-guard,-readability-function-cognitive-complexity,-readability-identifier-length,-cppcoreguidelines-owning-memory,-cert-err58-cpp,-fuchsia-statically-constructed-objects,-google-build-using-namespace,-hicpp-avoid-goto,-cppcoreguidelines-avoid-goto,-hicpp-no-array-decay,-cppcoreguidelines-pro-bounds-array-to-pointer-decay'
+Checks: '*,-bugprone-reserved-identifier,-hicpp-signed-bitwise,-llvmlibc-restrict-system-libc-headers,-altera-unroll-loops,-hicpp-named-parameter,-cert-dcl37-c,-cert-dcl51-cpp,-read,-cppcoreguidelines-init-variables,-cppcoreguidelines-avoid-non-const-global-variables,-altera-id-dependent-backward-branch,-performance-no-int-to-ptr,-altera-struct-pack-align,-google-readability-casting,-modernize-use-trailing-return-type,-llvmlibc-implementation-in-namespace,-llvmlibc-callee-namespace,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-fuchsia-default-arguments-declarations,-fuchsia-overloaded-operator,-cppcoreguidelines-pro-type-union-access,-fuchsia-default-arguments-calls,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-google-readability-todo,-llvm-header-guard,-readability-function-cognitive-complexity,-readability-identifier-length,-cppcoreguidelines-owning-memory,-cert-err58-cpp,-fuchsia-statically-constructed-objects,-google-build-using-namespace,-hicpp-avoid-goto,-cppcoreguidelines-avoid-goto,-hicpp-no-array-decay,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-cppcoreguidelines-pro-bounds-constant-array-index,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers'
 WarningsAsErrors: '*'
 HeaderFilterRegex: ''
 AnalyzeTemporaryDtors: false

.github/workflows/fuzz.yml

@@ -26,8 +26,9 @@ jobs:
 
  - name: Install deps
  run: |
+ DEBIAN_FRONTEND="noninteractive" sudo apt-get -y remove python3-lldb-14
  sudo .github/workflows/scripts/llvm.sh 15
- DEBIAN_FRONTEND="noninteractive" sudo apt-get -y install python3 apt-transport-https build-essential wget cmake git clang-15 libfuzzer-15-dev
+ DEBIAN_FRONTEND="noninteractive" sudo apt-get -y install libfuzzer-15-dev
 
  - name: Build
  run: ./fuzzing/build.sh

.github/workflows/scripts/llvm.sh

@@ -20,7 +20,7 @@ usage() {
  exit 1;
 }
 
-CURRENT_LLVM_STABLE=15
+CURRENT_LLVM_STABLE=16
 BASE_URL="http://apt.llvm.org"
 
 # Check for required tools
@@ -50,9 +50,9 @@ source /etc/os-release
 DISTRO=${DISTRO,,}
 case ${DISTRO} in
  debian)
- if [[ "${VERSION}" == "unstable" ]] || [[ "${VERSION}" == "testing" ]] || [[ "${VERSION_CODENAME}" == "bookworm" ]]; then
-  # For now, bookworm == sid.
-  # TODO change when bookworm is released
+ # Debian Trixie has a workaround because of
+ # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1038383
+ if [[ "${VERSION}" == "unstable" ]] || [[ "${VERSION}" == "testing" ]] || [[ "${VERSION_CODENAME}" == "trixie" ]]; then
  CODENAME=unstable
  LINKNAME=
  else
@@ -124,7 +124,8 @@ LLVM_VERSION_PATTERNS[12]="-12"
 LLVM_VERSION_PATTERNS[13]="-13"
 LLVM_VERSION_PATTERNS[14]="-14"
 LLVM_VERSION_PATTERNS[15]="-15"
-LLVM_VERSION_PATTERNS[16]=""
+LLVM_VERSION_PATTERNS[16]="-16"
+LLVM_VERSION_PATTERNS[17]=""
 
 if [ ! ${LLVM_VERSION_PATTERNS[$LLVM_VERSION]+_} ]; then
  echo "This script does not support LLVM version $LLVM_VERSION"
@@ -150,9 +151,15 @@ fi
 
 
 # install everything
-if [[ -z "`apt-key list | grep -i llvm`" ]]; then
+
+if [[ ! -f /etc/apt/trusted.gpg.d/apt.llvm.org.asc ]]; then
  # download GPG key once
- wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+ wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
+fi
+
+if [[ -z "`apt-key list 2> /dev/null | grep -i llvm`" ]]; then
+ # Delete the key in the old format
+ apt-key del AF4F7421
 fi
 add-apt-repository "${REPO_NAME}"
 apt-get update
@@ -161,5 +168,8 @@ if [[ $ALL -eq 1 ]]; then
  # same as in test-install.sh
  # No worries if we have dups
  PKG="$PKG clang-tidy-$LLVM_VERSION clang-format-$LLVM_VERSION clang-tools-$LLVM_VERSION llvm-$LLVM_VERSION-dev lld-$LLVM_VERSION lldb-$LLVM_VERSION llvm-$LLVM_VERSION-tools libomp-$LLVM_VERSION-dev libc++-$LLVM_VERSION-dev libc++abi-$LLVM_VERSION-dev libclang-common-$LLVM_VERSION-dev libclang-$LLVM_VERSION-dev libclang-cpp$LLVM_VERSION-dev libunwind-$LLVM_VERSION-dev"
+ if test $LLVM_VERSION -gt 14; then
+ PKG="$PKG libclang-rt-$LLVM_VERSION-dev libpolly-$LLVM_VERSION-dev"
+ fi
 fi
 apt-get install -y $PKG

.github/workflows/test.yml

@@ -163,6 +163,7 @@ jobs:
 
  - name: Install clang-{tidy,format}
  run: |
+ DEBIAN_FRONTEND="noninteractive" sudo apt-get -y remove python3-lldb-14
  sudo .github/workflows/scripts/llvm.sh 15
  sudo apt-get install -y clang-tidy-15 clang-format-15
 

CMakeLists.txt

@@ -14,7 +14,7 @@ if(CMAKE_OSX_ARCHITECTURES MATCHES "x86_64" OR
 endif()
 
 set(CMAKE_C_STANDARD 99)
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 set(LIBDDWAF_SHARED_LINKER_FLAGS "-static-libstdc++" CACHE STRING "Shared library extra linker flags")
@@ -44,8 +44,6 @@ if(NOT MSVC)
  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Wall -Wextra -Wno-narrowing")
  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELEASE} -Wall -Wextra -Wno-narrowing -ggdb")
  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-narrowing -ggdb")
-
- add_definitions(-D__STDC_FORMAT_MACROS)
 else()
  set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${LIBDDWAF_MSVC_RUNTIME_LIBRARY}")
  set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELEASE} ${LIBDDWAF_MSVC_RUNTIME_LIBRARY}")
@@ -77,15 +75,14 @@ set(LIBDDWAF_SOURCE
  ${libddwaf_SOURCE_DIR}/src/ruleset_info.cpp
  ${libddwaf_SOURCE_DIR}/src/ip_utils.cpp
  ${libddwaf_SOURCE_DIR}/src/iterator.cpp
- ${libddwaf_SOURCE_DIR}/src/PWTransformer.cpp
  ${libddwaf_SOURCE_DIR}/src/utils.cpp
- ${libddwaf_SOURCE_DIR}/src/utf8.cpp
  ${libddwaf_SOURCE_DIR}/src/log.cpp
  ${libddwaf_SOURCE_DIR}/src/obfuscator.cpp
  ${libddwaf_SOURCE_DIR}/src/waf.cpp
  ${libddwaf_SOURCE_DIR}/src/exclusion/input_filter.cpp
  ${libddwaf_SOURCE_DIR}/src/exclusion/object_filter.cpp
  ${libddwaf_SOURCE_DIR}/src/exclusion/rule_filter.cpp
+ ${libddwaf_SOURCE_DIR}/src/parser/common.cpp
  ${libddwaf_SOURCE_DIR}/src/parser/parser.cpp
  ${libddwaf_SOURCE_DIR}/src/parser/parser_v1.cpp
  ${libddwaf_SOURCE_DIR}/src/parser/parser_v2.cpp
@@ -96,6 +93,24 @@ set(LIBDDWAF_SOURCE
  ${libddwaf_SOURCE_DIR}/src/rule_processor/is_xss.cpp
  ${libddwaf_SOURCE_DIR}/src/rule_processor/ip_match.cpp
  ${libddwaf_SOURCE_DIR}/src/rule_processor/exact_match.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/lowercase.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/compress_whitespace.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/normalize_path.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/manager.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/remove_nulls.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/remove_comments.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/shell_unescape.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/unicode_normalize.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/url_basename.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/url_decode.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/url_querystring.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/url_path.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/base64_decode.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/base64_encode.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/css_decode.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/html_entity_decode.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/js_decode.cpp
+ ${libddwaf_SOURCE_DIR}/src/transformer/common/utf8.cpp
  ${libddwaf_SOURCE_DIR}/src/libcxx-compat/monotonic_buffer_resource.cpp
 )
 add_library(libddwaf_objects OBJECT ${LIBDDWAF_SOURCE})

fuzzing/CMakeLists.txt

@@ -2,7 +2,7 @@ file(GLOB_RECURSE LIBDDWAF_FUZZER_SOURCE src/*.cpp)
 add_executable(fuzzer ${LIBDDWAF_SOURCE} ${LIBDDWAF_FUZZER_SOURCE})
 
 set_target_properties(fuzzer PROPERTIES
- CXX_STANDARD 17
+ CXX_STANDARD 20
  CXX_STANDARD_REQUIRED YES
  CXX_EXTENSIONS NO)
 

fuzzing/scripts/build_corpus.py

@@ -329,28 +329,41 @@ def get_random_transformation_array():
 
  return _get_random_array2(
  [
- "urlDecodeUni",
- "htmlEntityDecode",
- "jsDecode",
- "cssDecode",
- "cmdLine",
- "base64Decode",
- "base64DecodeExt",
- "urlDecode",
+ "keys_only",
+ "lowercase",
+ "remove_nulls",
+ "compress_whitespace",
+ "normalize_path",
+ "normalize_path_win",
+ "url_decode",
+ "url_decode_iis",
+ "css_decode",
+ "js_decode",
+ "html_entity_decode",
+ "base64_decode",
+ "base64_encode",
+ "shell_unescape",
+ "url_basename",
+ "url_path",
+ "url_querystring",
+ "remove_comments",
+ "unicode_normalize",
  "removeNulls",
+ "compressWhiteSpace",
  "normalizePath",
  "normalizePathWin",
- "compressWhiteSpace",
- "lowercase",
- # "length", # no really complex, and will skip a lot of use cases
+ "urlDecode",
+ "urlDecodeUni",
+ "cssDecode",
+ "jsDecode",
+ "htmlEntityDecode",
+ "base64Decode",
  "base64Encode",
+ "cmdLine",
  "_sqr_basename",
  "_sqr_filename",
  "_sqr_querystring",
  "removeComments",
- "numerize",
- "keys_only",
- "unicode_normalize",
  ],
  0,
  self.transformation_max_count,

fuzzing/src/object_builder.cpp

@@ -74,9 +74,9 @@ void pop_string(Data *data, ddwaf_object *object)
 
 bool popBoolean(Data *data)
 {
- bool result = false;
+ uint8_t result = 0;
  popBytes(data, &result, 1);
- return result;
+ return result > 0;
 }
 
 uint64_t popUnsignedInteger(Data *data)