From 93df73a5668413c7f87f6cf41f5cafb2a85f2900 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Thu, 11 Nov 2021 22:53:52 +0100 Subject: [PATCH 01/56] Build infra for external library --- .circleci/config.yml | 4 ++-- .gitignore | 4 ++++ ddtrace/appsec/CMakeLists.txt | 10 ++++++++++ ddtrace/appsec/__init__.py | 0 ddtrace/appsec/_ddwaf.pyx | 13 +++++++++++++ ddtrace/appsec/_libddwaf.pxd | 10 ++++++++++ pyproject.toml | 3 ++- setup.py | 20 +++++++++++++++++++- 8 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 ddtrace/appsec/CMakeLists.txt create mode 100644 ddtrace/appsec/__init__.py create mode 100644 ddtrace/appsec/_ddwaf.pyx create mode 100644 ddtrace/appsec/_libddwaf.pxd diff --git a/.circleci/config.yml b/.circleci/config.yml index ca6b5899574..bf1c24426ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -290,9 +290,9 @@ jobs: steps: - checkout - run: sudo apt-get update - - run: sudo apt-get install --yes clang-format gcc-10 python3 python3-setuptools python3-pip + - run: sudo apt-get install --yes clang-format gcc-10 g++-10 python3 python3-setuptools python3-pip - run: scripts/cformat.sh - - run: DD_COMPILE_DEBUG=1 CC=gcc-10 pip -vvv install . + - run: DD_COMPILE_DEBUG=1 CC=gcc-10 CXX=g++-10 pip -vvv install . coverage_report: executor: python39 diff --git a/.gitignore b/.gitignore index 38ed233de92..1ec18ca81de 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ *$py.class # C extensions +ddtrace/appsec/_ddwaf.cpp ddtrace/profiling/collector/_task.c ddtrace/profiling/collector/_threading.c ddtrace/profiling/collector/_traceback.c @@ -13,6 +14,9 @@ ddtrace/profiling/_build.c ddtrace/internal/_encoding.c ddtrace/internal/_rand.c *.so +*.a +*.cmake +*.h # Cython annotate HTML files ddtrace/**/*.html diff --git a/ddtrace/appsec/CMakeLists.txt b/ddtrace/appsec/CMakeLists.txt new file mode 100644 index 00000000000..b9e8a81e523 --- /dev/null +++ b/ddtrace/appsec/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.14) +project(ddwaf) +include(ExternalProject) + +ExternalProject_Add(libddwaf + GIT_REPOSITORY https://github.com/DataDog/libddwaf.git + GIT_TAG 841cbf703375ae0561b1e9c272a40734830b22fd + INSTALL_DIR ${CMAKE_SOURCE_DIR} + CMAKE_ARGS -DLIBDDWAF_BUILD_SHARED=OFF -DLIBDDWAF_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH= +) diff --git a/ddtrace/appsec/__init__.py b/ddtrace/appsec/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ddtrace/appsec/_ddwaf.pyx b/ddtrace/appsec/_ddwaf.pyx new file mode 100644 index 00000000000..d690aeb4ad7 --- /dev/null +++ b/ddtrace/appsec/_ddwaf.pyx @@ -0,0 +1,13 @@ +# distutils: include_dirs = ddtrace/appsec/include +# distutils: library_dirs = ddtrace/appsec/lib +# distutils: libraries = ddwaf + +import typing +cimport _libddwaf + + +def version(): + # type: () -> typing.Tuple[int, int, int] + cdef _libddwaf.ddwaf_version version + _libddwaf.ddwaf_get_version(&version) + return (version.major, version.minor, version.patch) diff --git a/ddtrace/appsec/_libddwaf.pxd b/ddtrace/appsec/_libddwaf.pxd new file mode 100644 index 00000000000..3284fbc232a --- /dev/null +++ b/ddtrace/appsec/_libddwaf.pxd @@ -0,0 +1,10 @@ +from libc.stdint cimport uint16_t + + +cdef extern from "include/ddwaf.h": + ctypedef struct ddwaf_version: + uint16_t major + uint16_t minor + uint16_t patch + + void ddwaf_get_version(ddwaf_version *version) diff --git a/pyproject.toml b/pyproject.toml index 9f9a2336fd5..4a61656fe01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools >= 40.6.0", "setuptools_scm[toml] >=4,<6.1", "cython"] +requires = ["setuptools >= 40.6.0", "setuptools_scm[toml] >=4,<6.1", "cython", "cmake >= 3.14"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] @@ -23,6 +23,7 @@ exclude = ''' ( .venv* | \.riot/ + | ddtrace/appsec/_ddwaf.pyx$ | ddtrace/internal/_encoding.pyx$ | ddtrace/internal/_rand.pyx$ | ddtrace/profiling/collector/_traceback.pyx$ diff --git a/setup.py b/setup.py index e717b515c20..43d49ff92c8 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,11 @@ import os import platform +import subprocess import sys from setuptools import setup, find_packages, Extension from setuptools.command.test import test as TestCommand +from setuptools.command.build_ext import build_ext as BuildExtCommand # ORDER MATTERS # Import this after setuptools or it will fail @@ -66,6 +68,17 @@ def run_tests(self): sys.exit(errno) +class CMake(BuildExtCommand): + def build_extension(self, ext): + for source in ext.sources: + source_dir = os.path.dirname(source) + if os.path.exists(os.path.join(source_dir, "CMakeLists.txt")): + build_dir = os.path.join(self.build_temp, ext.name) + subprocess.check_call(["cmake", "-S", source_dir, "-B", build_dir]) + subprocess.check_call(["cmake", "--build", build_dir]) + return BuildExtCommand.build_extension(self, ext) + + long_description = """ # dd-trace-py @@ -182,7 +195,7 @@ def get_exts_for(name): }, # plugin tox tests_require=["tox", "flake8"], - cmdclass={"test": Tox}, + cmdclass={"build_ext": CMake, "test": Tox}, entry_points={ "console_scripts": [ "ddtrace-run = ddtrace.commands.ddtrace_run:main", @@ -250,6 +263,11 @@ def get_exts_for(name): sources=["ddtrace/profiling/_build.pyx"], language="c", ), + Cython.Distutils.Extension( + "ddtrace.appsec._ddwaf", + sources=["ddtrace/appsec/_ddwaf.pyx"], + language="c++", + ), ], compile_time_env={ "PY_MAJOR_VERSION": sys.version_info.major, From 74c956370aec5631e4e6b20e4188fe2526621cc8 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Fri, 12 Nov 2021 17:42:46 +0100 Subject: [PATCH 02/56] Use lib instead of lib64 --- ddtrace/appsec/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddtrace/appsec/CMakeLists.txt b/ddtrace/appsec/CMakeLists.txt index b9e8a81e523..1c244cf607d 100644 --- a/ddtrace/appsec/CMakeLists.txt +++ b/ddtrace/appsec/CMakeLists.txt @@ -2,9 +2,9 @@ cmake_minimum_required(VERSION 3.14) project(ddwaf) include(ExternalProject) -ExternalProject_Add(libddwaf +ExternalProject_Add(libddwaf GIT_REPOSITORY https://github.com/DataDog/libddwaf.git GIT_TAG 841cbf703375ae0561b1e9c272a40734830b22fd INSTALL_DIR ${CMAKE_SOURCE_DIR} - CMAKE_ARGS -DLIBDDWAF_BUILD_SHARED=OFF -DLIBDDWAF_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH= + CMAKE_ARGS -DLIBDDWAF_BUILD_SHARED=OFF -DLIBDDWAF_BUILD_STATIC=ON -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX:PATH= ) From 4bbc24a7809a3c949a32c145d73c5432d287030e Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Fri, 12 Nov 2021 19:13:57 +0100 Subject: [PATCH 03/56] Convert Python obj to ddwaf obj --- ddtrace/appsec/_ddwaf.pyx | 70 ++++++++++++++++++++++++++++++++++-- ddtrace/appsec/_libddwaf.pxd | 22 +++++++++++- 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/ddtrace/appsec/_ddwaf.pyx b/ddtrace/appsec/_ddwaf.pyx index d690aeb4ad7..a0a0353671e 100644 --- a/ddtrace/appsec/_ddwaf.pyx +++ b/ddtrace/appsec/_ddwaf.pyx @@ -3,11 +3,75 @@ # distutils: libraries = ddwaf import typing -cimport _libddwaf +from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free + +from _libddwaf cimport ( + ddwaf_object, + ddwaf_object_array, + ddwaf_object_array_add, + ddwaf_object_invalid, + ddwaf_object_map, + ddwaf_object_map_addl, + ddwaf_object_stringl, + ddwaf_object_free, + ddwaf_version, + ddwaf_get_version +) def version(): # type: () -> typing.Tuple[int, int, int] - cdef _libddwaf.ddwaf_version version - _libddwaf.ddwaf_get_version(&version) + cdef ddwaf_version version + ddwaf_get_version(&version) return (version.major, version.minor, version.patch) + + +cdef ddwaf_object* _alloc(): + return PyMem_Malloc(sizeof(ddwaf_object)) + +cdef void _dealloc(ddwaf_object *obj): + if obj != NULL: + ddwaf_object_free(obj) + PyMem_Free(obj) + +# TODO unicode +cdef ddwaf_object* _convert(value): + obj = _alloc() + if obj == NULL: + return NULL + if isinstance(value, unicode): + value = value.encode("utf-8", errors="surrogatepass") + if isinstance(value, bytes): + ddwaf_object_stringl(obj, value, len(value)) + elif isinstance(value, (list, tuple)): + ddwaf_object_array(obj); + for item in value: + item_obj = _convert(item) + ret = ddwaf_object_array_add(obj, item_obj) + if ret is False: + _dealloc(item_obj) + elif isinstance(value, dict): + ddwaf_object_map(obj) + for k, v in value.items(): + item_obj = _convert(v) + if isinstance(k, unicode): + k = k.encode("utf-8", errors="surrogatepass") + if isinstance(k, bytes): + ret = ddwaf_object_map_addl(obj, k, len(k), item_obj) + if ret is False: + _dealloc(item_obj) + else: + ddwaf_object_invalid(obj) + return obj + +cdef class _Wrapper: + cdef ddwaf_object *_ptr + + def __cinit__(self, value): + self._ptr = _convert(value) + if self._ptr == NULL: + raise MemoryError + + def __dealloc__(self): + _dealloc(self._ptr) + self._ptr = NULL diff --git a/ddtrace/appsec/_libddwaf.pxd b/ddtrace/appsec/_libddwaf.pxd index 3284fbc232a..848dfb43715 100644 --- a/ddtrace/appsec/_libddwaf.pxd +++ b/ddtrace/appsec/_libddwaf.pxd @@ -1,4 +1,5 @@ -from libc.stdint cimport uint16_t +from libcpp cimport bool +from libc.stdint cimport uint16_t, int64_t, uint64_t cdef extern from "include/ddwaf.h": @@ -8,3 +9,22 @@ cdef extern from "include/ddwaf.h": uint16_t patch void ddwaf_get_version(ddwaf_version *version) + + ctypedef struct ddwaf_object: + pass + + ddwaf_object* ddwaf_object_invalid(ddwaf_object *obj); + ddwaf_object* ddwaf_object_string(ddwaf_object *obj, const char *val); + ddwaf_object* ddwaf_object_stringl(ddwaf_object *obj, const char *val, size_t length); + ddwaf_object* ddwaf_object_stringl_nc(ddwaf_object *obj, const char *val, size_t length); + ddwaf_object* ddwaf_object_signed(ddwaf_object *obj, int64_t val); + ddwaf_object* ddwaf_object_unsigned(ddwaf_object *obj, uint64_t val); + ddwaf_object* ddwaf_object_signed_force(ddwaf_object *obj, int64_t val); + ddwaf_object* ddwaf_object_unsigned_force(ddwaf_object *obj, uint64_t val); + ddwaf_object* ddwaf_object_array(ddwaf_object *obj); + bool ddwaf_object_array_add(ddwaf_object *array, ddwaf_object *obj); + ddwaf_object* ddwaf_object_map(ddwaf_object *obj); + bool ddwaf_object_map_add(ddwaf_object *map, const char *key, ddwaf_object *obj); + bool ddwaf_object_map_addl(ddwaf_object *map, const char *key, size_t length, ddwaf_object *obj); + bool ddwaf_object_map_addl_nc(ddwaf_object *map, const char *key, size_t length, ddwaf_object *obj); + void ddwaf_object_free(ddwaf_object *obj); From 7c8519451e73a4a8c843753c10f3abc492d5973f Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Fri, 12 Nov 2021 21:11:00 +0100 Subject: [PATCH 04/56] Only run cmake once --- setup.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 43d49ff92c8..5c606063344 100644 --- a/setup.py +++ b/setup.py @@ -70,12 +70,17 @@ def run_tests(self): class CMake(BuildExtCommand): def build_extension(self, ext): + to_build = set() for source in ext.sources: - source_dir = os.path.dirname(source) + source_dir = os.path.dirname(os.path.realpath(source)) if os.path.exists(os.path.join(source_dir, "CMakeLists.txt")): - build_dir = os.path.join(self.build_temp, ext.name) - subprocess.check_call(["cmake", "-S", source_dir, "-B", build_dir]) - subprocess.check_call(["cmake", "--build", build_dir]) + to_build.add(source_dir) + + for idx, source_dir in enumerate(to_build): + build_dir = os.path.join(self.build_temp, "_".join([ext.name, str(idx)])) + subprocess.check_call(["cmake", "-S", source_dir, "-B", build_dir]) + subprocess.check_call(["cmake", "--build", build_dir]) + return BuildExtCommand.build_extension(self, ext) From 5fb7e50d65beb05bc0f87b5f9adcc68dcc438cf3 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Fri, 12 Nov 2021 21:38:08 +0100 Subject: [PATCH 05/56] CMake build type --- ddtrace/appsec/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ddtrace/appsec/CMakeLists.txt b/ddtrace/appsec/CMakeLists.txt index 1c244cf607d..06dd162b450 100644 --- a/ddtrace/appsec/CMakeLists.txt +++ b/ddtrace/appsec/CMakeLists.txt @@ -6,5 +6,10 @@ ExternalProject_Add(libddwaf GIT_REPOSITORY https://github.com/DataDog/libddwaf.git GIT_TAG 841cbf703375ae0561b1e9c272a40734830b22fd INSTALL_DIR ${CMAKE_SOURCE_DIR} - CMAKE_ARGS -DLIBDDWAF_BUILD_SHARED=OFF -DLIBDDWAF_BUILD_STATIC=ON -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX:PATH= + CMAKE_ARGS + -DLIBDDWAF_BUILD_SHARED=OFF + -DLIBDDWAF_BUILD_STATIC=ON + -DCMAKE_INSTALL_LIBDIR=lib + -DCMAKE_BUILD_TYPE=RelWithDebInfo + -DCMAKE_INSTALL_PREFIX:PATH= ) From 5aecb4a1b0a022dc6abcb1e62df0b94fee5f0257 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Sat, 13 Nov 2021 00:37:53 +0100 Subject: [PATCH 06/56] Fast object convertion --- ddtrace/appsec/_ddwaf.pyx | 110 ++++++++++++++++++++--------------- ddtrace/appsec/_libddwaf.pxd | 14 ++--- 2 files changed, 67 insertions(+), 57 deletions(-) diff --git a/ddtrace/appsec/_ddwaf.pyx b/ddtrace/appsec/_ddwaf.pyx index a0a0353671e..d13c755e4db 100644 --- a/ddtrace/appsec/_ddwaf.pyx +++ b/ddtrace/appsec/_ddwaf.pyx @@ -2,18 +2,16 @@ # distutils: library_dirs = ddtrace/appsec/lib # distutils: libraries = ddwaf +import six import typing from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free from _libddwaf cimport ( ddwaf_object, ddwaf_object_array, - ddwaf_object_array_add, ddwaf_object_invalid, ddwaf_object_map, - ddwaf_object_map_addl, - ddwaf_object_stringl, - ddwaf_object_free, + ddwaf_object_stringl_nc, ddwaf_version, ddwaf_get_version ) @@ -26,52 +24,70 @@ def version(): return (version.major, version.minor, version.patch) -cdef ddwaf_object* _alloc(): - return PyMem_Malloc(sizeof(ddwaf_object)) - -cdef void _dealloc(ddwaf_object *obj): - if obj != NULL: - ddwaf_object_free(obj) - PyMem_Free(obj) - -# TODO unicode -cdef ddwaf_object* _convert(value): - obj = _alloc() - if obj == NULL: - return NULL - if isinstance(value, unicode): - value = value.encode("utf-8", errors="surrogatepass") - if isinstance(value, bytes): - ddwaf_object_stringl(obj, value, len(value)) - elif isinstance(value, (list, tuple)): - ddwaf_object_array(obj); - for item in value: - item_obj = _convert(item) - ret = ddwaf_object_array_add(obj, item_obj) - if ret is False: - _dealloc(item_obj) - elif isinstance(value, dict): - ddwaf_object_map(obj) - for k, v in value.items(): - item_obj = _convert(v) - if isinstance(k, unicode): - k = k.encode("utf-8", errors="surrogatepass") - if isinstance(k, bytes): - ret = ddwaf_object_map_addl(obj, k, len(k), item_obj) - if ret is False: - _dealloc(item_obj) - else: - ddwaf_object_invalid(obj) - return obj - cdef class _Wrapper: cdef ddwaf_object *_ptr + cdef public object _strings + cdef public ssize_t _size + cdef public ssize_t _next_idx def __cinit__(self, value): - self._ptr = _convert(value) - if self._ptr == NULL: - raise MemoryError + self._next_idx = 0 + self._size = 0 + self._strings = [] + self._ptr = NULL + self._convert(self._reserve_obj(), value) + + cdef ddwaf_object* _reserve_obj(self, ssize_t n=1): + cdef ssize_t idx, i + cdef ddwaf_object* ptr + + idx = self._next_idx + ptr = self._ptr + if idx + n > self._size: + self._size += n + ((64 - (n % 64)) % 64) + ptr = PyMem_Realloc(self._ptr, self._size * sizeof(ddwaf_object)) + if ptr == NULL: + raise MemoryError + self._ptr = ptr + self._next_idx += n + for i in range(n): + ddwaf_object_invalid(ptr + idx + i) + return ptr + idx + + cdef void _convert(self, ddwaf_object* obj, value): + cdef ssize_t i + + if isinstance(value, six.text_type): + value = value.encode("utf-8", errors="surrogatepass") + + if isinstance(value, bytes): + self._strings.append(value) + ddwaf_object_stringl_nc(obj, value, len(value)) + + elif isinstance(value, (list, tuple)): + ddwaf_object_array(obj); + n = len(value) + items_obj = self._reserve_obj(n) + for i in range(n): + self._convert(items_obj + i, value[i]) + obj.array = items_obj + obj.nbEntries = n + + elif isinstance(value, dict): + ddwaf_object_map(obj) + n = len(value) + items_obj = self._reserve_obj(n) + for i, (k, v) in enumerate(six.iteritems(value)): + if isinstance(k, six.text_type): + k = k.encode("utf-8", errors="surrogatepass") + if isinstance(k, bytes): + item_obj = items_obj + i + self._strings.append(k) + item_obj.parameterName = k + item_obj.parameterNameLength = len(k) + self._convert(item_obj, v) + obj.array = items_obj + obj.nbEntries = n def __dealloc__(self): - _dealloc(self._ptr) - self._ptr = NULL + PyMem_Free(self._ptr) diff --git a/ddtrace/appsec/_libddwaf.pxd b/ddtrace/appsec/_libddwaf.pxd index 848dfb43715..070e9c4cc3e 100644 --- a/ddtrace/appsec/_libddwaf.pxd +++ b/ddtrace/appsec/_libddwaf.pxd @@ -11,20 +11,14 @@ cdef extern from "include/ddwaf.h": void ddwaf_get_version(ddwaf_version *version) ctypedef struct ddwaf_object: - pass + const char* parameterName + uint64_t parameterNameLength + ddwaf_object* array + uint64_t nbEntries ddwaf_object* ddwaf_object_invalid(ddwaf_object *obj); - ddwaf_object* ddwaf_object_string(ddwaf_object *obj, const char *val); - ddwaf_object* ddwaf_object_stringl(ddwaf_object *obj, const char *val, size_t length); ddwaf_object* ddwaf_object_stringl_nc(ddwaf_object *obj, const char *val, size_t length); - ddwaf_object* ddwaf_object_signed(ddwaf_object *obj, int64_t val); - ddwaf_object* ddwaf_object_unsigned(ddwaf_object *obj, uint64_t val); ddwaf_object* ddwaf_object_signed_force(ddwaf_object *obj, int64_t val); ddwaf_object* ddwaf_object_unsigned_force(ddwaf_object *obj, uint64_t val); ddwaf_object* ddwaf_object_array(ddwaf_object *obj); - bool ddwaf_object_array_add(ddwaf_object *array, ddwaf_object *obj); ddwaf_object* ddwaf_object_map(ddwaf_object *obj); - bool ddwaf_object_map_add(ddwaf_object *map, const char *key, ddwaf_object *obj); - bool ddwaf_object_map_addl(ddwaf_object *map, const char *key, size_t length, ddwaf_object *obj); - bool ddwaf_object_map_addl_nc(ddwaf_object *map, const char *key, size_t length, ddwaf_object *obj); - void ddwaf_object_free(ddwaf_object *obj); From df2911954324a3c3b5bb1a4fdbb8f2a0cdae2db6 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Mon, 15 Nov 2021 13:57:25 +0100 Subject: [PATCH 07/56] Pretty print Wrapper --- ddtrace/appsec/_ddwaf.pyx | 27 +++++++++++++++++---------- ddtrace/appsec/_libddwaf.pxd | 2 -- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/ddtrace/appsec/_ddwaf.pyx b/ddtrace/appsec/_ddwaf.pyx index d13c755e4db..3f145015271 100644 --- a/ddtrace/appsec/_ddwaf.pyx +++ b/ddtrace/appsec/_ddwaf.pyx @@ -2,14 +2,15 @@ # distutils: library_dirs = ddtrace/appsec/lib # distutils: libraries = ddwaf +import attr import six import typing from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free from _libddwaf cimport ( ddwaf_object, - ddwaf_object_array, ddwaf_object_invalid, + ddwaf_object_array, ddwaf_object_map, ddwaf_object_stringl_nc, ddwaf_version, @@ -24,20 +25,17 @@ def version(): return (version.major, version.minor, version.patch) -cdef class _Wrapper: +cdef class _Wrapper(object): cdef ddwaf_object *_ptr cdef public object _strings cdef public ssize_t _size cdef public ssize_t _next_idx - def __cinit__(self, value): - self._next_idx = 0 - self._size = 0 + def __init__(self, value): self._strings = [] - self._ptr = NULL self._convert(self._reserve_obj(), value) - cdef ddwaf_object* _reserve_obj(self, ssize_t n=1): + cdef ddwaf_object* _reserve_obj(self, ssize_t n=1) except NULL: cdef ssize_t idx, i cdef ddwaf_object* ptr @@ -50,13 +48,16 @@ cdef class _Wrapper: raise MemoryError self._ptr = ptr self._next_idx += n - for i in range(n): - ddwaf_object_invalid(ptr + idx + i) + for i in range(idx, idx + n): + ddwaf_object_invalid(ptr + i) return ptr + idx - cdef void _convert(self, ddwaf_object* obj, value): + cdef void _convert(self, ddwaf_object* obj, value) except *: cdef ssize_t i + if isinstance(value, (int, float)): + value = str(value) + if isinstance(value, six.text_type): value = value.encode("utf-8", errors="surrogatepass") @@ -89,5 +90,11 @@ cdef class _Wrapper: obj.array = items_obj obj.nbEntries = n + def __repr__(self): + return "<_Wrapper for {0._next_idx} elements>".format(self) + + def __sizeof__(self): + return super(_Wrapper, self).__sizeof__() + self._size + def __dealloc__(self): PyMem_Free(self._ptr) diff --git a/ddtrace/appsec/_libddwaf.pxd b/ddtrace/appsec/_libddwaf.pxd index 070e9c4cc3e..45d6d9ed872 100644 --- a/ddtrace/appsec/_libddwaf.pxd +++ b/ddtrace/appsec/_libddwaf.pxd @@ -18,7 +18,5 @@ cdef extern from "include/ddwaf.h": ddwaf_object* ddwaf_object_invalid(ddwaf_object *obj); ddwaf_object* ddwaf_object_stringl_nc(ddwaf_object *obj, const char *val, size_t length); - ddwaf_object* ddwaf_object_signed_force(ddwaf_object *obj, int64_t val); - ddwaf_object* ddwaf_object_unsigned_force(ddwaf_object *obj, uint64_t val); ddwaf_object* ddwaf_object_array(ddwaf_object *obj); ddwaf_object* ddwaf_object_map(ddwaf_object *obj); From e25278282ecb0b877ff590388614638675bac3c4 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Mon, 15 Nov 2021 18:31:33 +0100 Subject: [PATCH 08/56] Add initialization code --- ddtrace/appsec/_ddwaf.pyx | 36 +++++++++++++++++++++++++++--------- ddtrace/appsec/_libddwaf.pxd | 9 +++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/ddtrace/appsec/_ddwaf.pyx b/ddtrace/appsec/_ddwaf.pyx index 3f145015271..aafcc7b56c3 100644 --- a/ddtrace/appsec/_ddwaf.pyx +++ b/ddtrace/appsec/_ddwaf.pyx @@ -2,12 +2,15 @@ # distutils: library_dirs = ddtrace/appsec/lib # distutils: libraries = ddwaf -import attr +import functools import six import typing from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free from _libddwaf cimport ( + ddwaf_handle, + ddwaf_init, + ddwaf_destroy, ddwaf_object, ddwaf_object_invalid, ddwaf_object_array, @@ -24,15 +27,16 @@ def version(): ddwaf_get_version(&version) return (version.major, version.minor, version.patch) +ctypedef void (*ddwaf_object_applyf)(ddwaf_object *); cdef class _Wrapper(object): cdef ddwaf_object *_ptr - cdef public object _strings - cdef public ssize_t _size - cdef public ssize_t _next_idx + cdef readonly object _string_refs + cdef readonly ssize_t _size + cdef readonly ssize_t _next_idx def __init__(self, value): - self._strings = [] + self._string_refs = [] self._convert(self._reserve_obj(), value) cdef ddwaf_object* _reserve_obj(self, ssize_t n=1) except NULL: @@ -62,8 +66,8 @@ cdef class _Wrapper(object): value = value.encode("utf-8", errors="surrogatepass") if isinstance(value, bytes): - self._strings.append(value) - ddwaf_object_stringl_nc(obj, value, len(value)) + self._string_refs.append(value) + ddwaf_object_stringl_nc(obj, value, len(value)) elif isinstance(value, (list, tuple)): ddwaf_object_array(obj); @@ -83,8 +87,8 @@ cdef class _Wrapper(object): k = k.encode("utf-8", errors="surrogatepass") if isinstance(k, bytes): item_obj = items_obj + i - self._strings.append(k) - item_obj.parameterName = k + self._string_refs.append(k) + item_obj.parameterName = k item_obj.parameterNameLength = len(k) self._convert(item_obj, v) obj.array = items_obj @@ -98,3 +102,17 @@ cdef class _Wrapper(object): def __dealloc__(self): PyMem_Free(self._ptr) + + +cdef class DDWaf(object): + cdef ddwaf_handle _handle + cdef object _rules + + def __init__(self, rules): + cdef ddwaf_object* rule_objects + self._rules = _Wrapper(rules) + rule_objects = (<_Wrapper?>self._rules)._ptr; + self._handle = ddwaf_init(rule_objects, NULL) + + def __dealloc__(self): + ddwaf_destroy(self._handle) diff --git a/ddtrace/appsec/_libddwaf.pxd b/ddtrace/appsec/_libddwaf.pxd index 45d6d9ed872..743b7c995fd 100644 --- a/ddtrace/appsec/_libddwaf.pxd +++ b/ddtrace/appsec/_libddwaf.pxd @@ -20,3 +20,12 @@ cdef extern from "include/ddwaf.h": ddwaf_object* ddwaf_object_stringl_nc(ddwaf_object *obj, const char *val, size_t length); ddwaf_object* ddwaf_object_array(ddwaf_object *obj); ddwaf_object* ddwaf_object_map(ddwaf_object *obj); + + ctypedef struct ddwaf_handle: + pass + + ctypedef struct ddwaf_config: + pass + + ddwaf_handle ddwaf_init(const ddwaf_object* rules, const ddwaf_config* config); + void ddwaf_destroy(ddwaf_handle handle); From 9136956b8d9d7cf2102339b8aeb188ed6ee62758 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Thu, 18 Nov 2021 19:25:49 +0100 Subject: [PATCH 09/56] Use iterative convertion --- ddtrace/appsec/_ddwaf.pyx | 131 +++++++++++++++++++++++------------ ddtrace/appsec/_libddwaf.pxd | 14 ++++ 2 files changed, 99 insertions(+), 46 deletions(-) diff --git a/ddtrace/appsec/_ddwaf.pyx b/ddtrace/appsec/_ddwaf.pyx index aafcc7b56c3..4340e16be1a 100644 --- a/ddtrace/appsec/_ddwaf.pyx +++ b/ddtrace/appsec/_ddwaf.pyx @@ -5,11 +5,19 @@ import functools import six import typing +from collections import deque + from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free +from libc.stdint cimport uint64_t from _libddwaf cimport ( ddwaf_handle, + ddwaf_context, + ddwaf_result, ddwaf_init, + ddwaf_context_init, + ddwaf_run, + ddwaf_context_destroy, ddwaf_destroy, ddwaf_object, ddwaf_object_invalid, @@ -27,7 +35,6 @@ def version(): ddwaf_get_version(&version) return (version.major, version.minor, version.patch) -ctypedef void (*ddwaf_object_applyf)(ddwaf_object *); cdef class _Wrapper(object): cdef ddwaf_object *_ptr @@ -35,64 +42,81 @@ cdef class _Wrapper(object): cdef readonly ssize_t _size cdef readonly ssize_t _next_idx - def __init__(self, value): + def __init__(self, value, max_objects=1024): self._string_refs = [] - self._convert(self._reserve_obj(), value) + self._convert(value, max_objects) - cdef ddwaf_object* _reserve_obj(self, ssize_t n=1) except NULL: + cdef ssize_t _reserve_obj(self, ssize_t n=1) except -1: cdef ssize_t idx, i - cdef ddwaf_object* ptr + cdef ddwaf_object *ptr + cdef ddwaf_object *obj idx = self._next_idx ptr = self._ptr if idx + n > self._size: - self._size += n + ((64 - (n % 64)) % 64) + self._size += n + ((128 - (n % 128)) % 128) ptr = PyMem_Realloc(self._ptr, self._size * sizeof(ddwaf_object)) if ptr == NULL: raise MemoryError + elif ptr != self._ptr: + # we need to patch all array objects because they use pointers to other objects + for i in range(idx): + obj = ptr + i + if obj.array != NULL: + obj.array = obj.array - self._ptr + ptr self._ptr = ptr self._next_idx += n for i in range(idx, idx + n): ddwaf_object_invalid(ptr + i) - return ptr + idx - - cdef void _convert(self, ddwaf_object* obj, value) except *: - cdef ssize_t i - - if isinstance(value, (int, float)): - value = str(value) - - if isinstance(value, six.text_type): - value = value.encode("utf-8", errors="surrogatepass") - - if isinstance(value, bytes): - self._string_refs.append(value) - ddwaf_object_stringl_nc(obj, value, len(value)) - - elif isinstance(value, (list, tuple)): - ddwaf_object_array(obj); - n = len(value) - items_obj = self._reserve_obj(n) - for i in range(n): - self._convert(items_obj + i, value[i]) - obj.array = items_obj - obj.nbEntries = n - - elif isinstance(value, dict): - ddwaf_object_map(obj) - n = len(value) - items_obj = self._reserve_obj(n) - for i, (k, v) in enumerate(six.iteritems(value)): - if isinstance(k, six.text_type): - k = k.encode("utf-8", errors="surrogatepass") - if isinstance(k, bytes): - item_obj = items_obj + i - self._string_refs.append(k) - item_obj.parameterName = k - item_obj.parameterNameLength = len(k) - self._convert(item_obj, v) - obj.array = items_obj - obj.nbEntries = n + return idx + + cdef void _convert(self, value, max_objects) except *: + cdef object stack + cdef ddwaf_object *obj + cdef ddwaf_object *item_obj + cdef ssize_t i, j, idx + + i = 0 + stack = deque([(self._reserve_obj(), value)], maxlen=max_objects) + while len(stack) and (max_objects is None or i < max_objects): + idx, val = stack.popleft() + obj = self._ptr + idx + + if isinstance(val, (int, float)): + val = six.text_type(val) + + if isinstance(val, six.text_type): + val = val.encode("utf-8", errors="surrogatepass") + + if isinstance(val, bytes): + self._string_refs.append(val) + ddwaf_object_stringl_nc(obj, val, len(val)) + + elif isinstance(value, (list, tuple)): + ddwaf_object_array(obj); + n = len(val) + idx = self._reserve_obj(n) + stack.extend([(idx + j, val[j]) for j in range(n)]) + obj.array = self._ptr + idx + obj.nbEntries = n + + elif isinstance(val, dict): + ddwaf_object_map(obj) + n = len(val) + idx = self._reserve_obj(n) + obj.array = self._ptr + idx + obj.nbEntries = n + for j, (k, v) in enumerate(six.iteritems(val)): + if isinstance(k, six.text_type): + k = k.encode("utf-8", errors="surrogatepass") + if isinstance(k, bytes): + item_obj = self._ptr + idx + j + self._string_refs.append(k) + item_obj.parameterName = k + item_obj.parameterNameLength = len(k) + stack.append((idx + j, v)) + + i += 1 def __repr__(self): return "<_Wrapper for {0._next_idx} elements>".format(self) @@ -110,9 +134,24 @@ cdef class DDWaf(object): def __init__(self, rules): cdef ddwaf_object* rule_objects - self._rules = _Wrapper(rules) + self._rules = _Wrapper(rules, max_objects=None) rule_objects = (<_Wrapper?>self._rules)._ptr; self._handle = ddwaf_init(rule_objects, NULL) + if self._handle == NULL: + raise ValueError("invalid rules") + + def run(self, data, timeout_ms=1000): + cdef ddwaf_context ctx + cdef ddwaf_result result + + ctx = ddwaf_context_init(self._handle, NULL) + if ctx == NULL: + raise RuntimeError + try: + wrapper = _Wrapper(data) + ddwaf_run(ctx, (<_Wrapper?>wrapper)._ptr, &result, timeout_ms) + finally: + ddwaf_context_destroy(ctx) def __dealloc__(self): ddwaf_destroy(self._handle) diff --git a/ddtrace/appsec/_libddwaf.pxd b/ddtrace/appsec/_libddwaf.pxd index 743b7c995fd..a42a4a61847 100644 --- a/ddtrace/appsec/_libddwaf.pxd +++ b/ddtrace/appsec/_libddwaf.pxd @@ -27,5 +27,19 @@ cdef extern from "include/ddwaf.h": ctypedef struct ddwaf_config: pass + ctypedef struct ddwaf_context: + pass + + ctypedef struct ddwaf_result: + pass + + ctypedef enum DDWAF_RET_CODE: + pass + + ctypedef void (*ddwaf_object_free_fn)(ddwaf_object *object); + ddwaf_handle ddwaf_init(const ddwaf_object* rules, const ddwaf_config* config); + ddwaf_context ddwaf_context_init(const ddwaf_handle handle, ddwaf_object_free_fn obj_free); + DDWAF_RET_CODE ddwaf_run(ddwaf_context context, ddwaf_object* data, ddwaf_result *result, uint64_t timeout); + void ddwaf_context_destroy(ddwaf_context context); void ddwaf_destroy(ddwaf_handle handle); From fcbb660baae60d1b35c10385bcb46bc8dfdf3d7b Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Sun, 21 Nov 2021 23:29:01 +0100 Subject: [PATCH 10/56] Do not use ddwaf_object_* funcs --- ddtrace/appsec/_ddwaf.pyx | 58 +++++++++++++++++++----------------- ddtrace/appsec/_libddwaf.pxd | 29 ++++++++++++------ 2 files changed, 51 insertions(+), 36 deletions(-) diff --git a/ddtrace/appsec/_ddwaf.pyx b/ddtrace/appsec/_ddwaf.pyx index 4340e16be1a..fd0fbf4be80 100644 --- a/ddtrace/appsec/_ddwaf.pyx +++ b/ddtrace/appsec/_ddwaf.pyx @@ -2,35 +2,35 @@ # distutils: library_dirs = ddtrace/appsec/lib # distutils: libraries = ddwaf -import functools import six -import typing + +from typing import Tuple from collections import deque +from collections.abc import Mapping, Sequence -from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free +from cpython.mem cimport PyMem_Realloc, PyMem_Free +from libc.string cimport memset from libc.stdint cimport uint64_t from _libddwaf cimport ( + DDWAF_OBJ_TYPE, ddwaf_handle, ddwaf_context, ddwaf_result, + ddwaf_object, + ddwaf_version, + ddwaf_get_version, ddwaf_init, - ddwaf_context_init, ddwaf_run, - ddwaf_context_destroy, + ddwaf_result_free, ddwaf_destroy, - ddwaf_object, - ddwaf_object_invalid, - ddwaf_object_array, - ddwaf_object_map, - ddwaf_object_stringl_nc, - ddwaf_version, - ddwaf_get_version + ddwaf_context_init, + ddwaf_context_destroy, ) def version(): - # type: () -> typing.Tuple[int, int, int] + # type: () -> Tuple[int, int, int] cdef ddwaf_version version ddwaf_get_version(&version) return (version.major, version.minor, version.patch) @@ -58,7 +58,8 @@ cdef class _Wrapper(object): ptr = PyMem_Realloc(self._ptr, self._size * sizeof(ddwaf_object)) if ptr == NULL: raise MemoryError - elif ptr != self._ptr: + memset(ptr + idx, 0, (self._size - idx) * sizeof(ddwaf_object)) + if self._ptr != NULL and ptr != self._ptr: # we need to patch all array objects because they use pointers to other objects for i in range(idx): obj = ptr + i @@ -66,8 +67,6 @@ cdef class _Wrapper(object): obj.array = obj.array - self._ptr + ptr self._ptr = ptr self._next_idx += n - for i in range(idx, idx + n): - ddwaf_object_invalid(ptr + i) return idx cdef void _convert(self, value, max_objects) except *: @@ -90,18 +89,12 @@ cdef class _Wrapper(object): if isinstance(val, bytes): self._string_refs.append(val) - ddwaf_object_stringl_nc(obj, val, len(val)) + obj.type = DDWAF_OBJ_TYPE.DDWAF_OBJ_STRING + obj.stringValue = val + obj.nbEntries = len(val) - elif isinstance(value, (list, tuple)): - ddwaf_object_array(obj); - n = len(val) - idx = self._reserve_obj(n) - stack.extend([(idx + j, val[j]) for j in range(n)]) - obj.array = self._ptr + idx - obj.nbEntries = n - - elif isinstance(val, dict): - ddwaf_object_map(obj) + elif isinstance(val, Mapping): + obj.type = DDWAF_OBJ_TYPE.DDWAF_OBJ_MAP n = len(val) idx = self._reserve_obj(n) obj.array = self._ptr + idx @@ -116,6 +109,14 @@ cdef class _Wrapper(object): item_obj.parameterNameLength = len(k) stack.append((idx + j, v)) + elif isinstance(val, Sequence): + obj.type = DDWAF_OBJ_TYPE.DDWAF_OBJ_ARRAY + n = len(val) + idx = self._reserve_obj(n) + stack.extend([(idx + j, val[j]) for j in range(n)]) + obj.array = self._ptr + idx + obj.nbEntries = n + i += 1 def __repr__(self): @@ -150,7 +151,10 @@ cdef class DDWaf(object): try: wrapper = _Wrapper(data) ddwaf_run(ctx, (<_Wrapper?>wrapper)._ptr, &result, timeout_ms) + if result.data != NULL: + return ( result.data).decode("utf-8") finally: + ddwaf_result_free(&result) ddwaf_context_destroy(ctx) def __dealloc__(self): diff --git a/ddtrace/appsec/_libddwaf.pxd b/ddtrace/appsec/_libddwaf.pxd index a42a4a61847..6492e0f5321 100644 --- a/ddtrace/appsec/_libddwaf.pxd +++ b/ddtrace/appsec/_libddwaf.pxd @@ -10,16 +10,18 @@ cdef extern from "include/ddwaf.h": void ddwaf_get_version(ddwaf_version *version) + ctypedef enum DDWAF_OBJ_TYPE: + DDWAF_OBJ_ARRAY + DDWAF_OBJ_MAP + DDWAF_OBJ_STRING + ctypedef struct ddwaf_object: const char* parameterName uint64_t parameterNameLength ddwaf_object* array + const char* stringValue uint64_t nbEntries - - ddwaf_object* ddwaf_object_invalid(ddwaf_object *obj); - ddwaf_object* ddwaf_object_stringl_nc(ddwaf_object *obj, const char *val, size_t length); - ddwaf_object* ddwaf_object_array(ddwaf_object *obj); - ddwaf_object* ddwaf_object_map(ddwaf_object *obj); + DDWAF_OBJ_TYPE type ctypedef struct ddwaf_handle: pass @@ -30,16 +32,25 @@ cdef extern from "include/ddwaf.h": ctypedef struct ddwaf_context: pass - ctypedef struct ddwaf_result: - pass - ctypedef enum DDWAF_RET_CODE: pass + ctypedef struct ddwaf_result: + DDWAF_RET_CODE action + const char* data + ctypedef void (*ddwaf_object_free_fn)(ddwaf_object *object); ddwaf_handle ddwaf_init(const ddwaf_object* rules, const ddwaf_config* config); ddwaf_context ddwaf_context_init(const ddwaf_handle handle, ddwaf_object_free_fn obj_free); - DDWAF_RET_CODE ddwaf_run(ddwaf_context context, ddwaf_object* data, ddwaf_result *result, uint64_t timeout); + DDWAF_RET_CODE ddwaf_run(ddwaf_context context, ddwaf_object* data, ddwaf_result* result, uint64_t timeout); void ddwaf_context_destroy(ddwaf_context context); + void ddwaf_result_free(ddwaf_result* result); void ddwaf_destroy(ddwaf_handle handle); + + ctypedef enum DDWAF_LOG_LEVEL: + DDWAF_LOG_TRACE + + ctypedef void (*ddwaf_log_fn)(DDWAF_LOG_LEVEL level, const char *function, const char *file, unsigned line, const char *message, uint64_t len); + + void ddwaf_set_log_cb(ddwaf_log_fn fn, int level); From 87a0b14c668c08a58895114831495a130c57fb83 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Tue, 23 Nov 2021 18:31:57 +0100 Subject: [PATCH 11/56] Use Ninja to build libddwaf on Linux --- .github/workflows/build_deploy.yml | 6 +++--- ddtrace/appsec/CMakeLists.txt | 4 ++-- ddtrace/appsec/_ddwaf.pyx | 11 ++++++----- pyproject.toml | 2 +- setup.py | 26 ++++++++++++++++++++++---- tests/smoke_test.py | 2 ++ 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_deploy.yml b/.github/workflows/build_deploy.yml index 5e9d0e2990a..c338c2652d7 100644 --- a/.github/workflows/build_deploy.yml +++ b/.github/workflows/build_deploy.yml @@ -25,7 +25,7 @@ jobs: archs: x86_64 i686 - os: ubuntu-18.04 archs: aarch64 - - os: windows-latest + - os: vs2017-win2016 archs: AMD64 x86 - os: macos-latest archs: x86_64 universal2 @@ -75,7 +75,7 @@ jobs: archs: x86_64 i686 - os: ubuntu-18.04 archs: aarch64 - - os: windows-latest + - os: vs2017-win2016 archs: AMD64 x86 - os: macos-latest archs: x86_64 universal2 @@ -150,7 +150,7 @@ jobs: - name: Install build dependencies # Rust + Cargo are needed for Cryptography - run: apk add git gcc musl-dev libffi-dev openssl-dev bash rust cargo + run: apk add git gcc g++ musl-dev libffi-dev openssl-dev bash rust cargo - name: Check source package run: | diff --git a/ddtrace/appsec/CMakeLists.txt b/ddtrace/appsec/CMakeLists.txt index 06dd162b450..d6352428a38 100644 --- a/ddtrace/appsec/CMakeLists.txt +++ b/ddtrace/appsec/CMakeLists.txt @@ -4,12 +4,12 @@ include(ExternalProject) ExternalProject_Add(libddwaf GIT_REPOSITORY https://github.com/DataDog/libddwaf.git - GIT_TAG 841cbf703375ae0561b1e9c272a40734830b22fd + GIT_TAG 8ee1ae3 INSTALL_DIR ${CMAKE_SOURCE_DIR} CMAKE_ARGS -DLIBDDWAF_BUILD_SHARED=OFF -DLIBDDWAF_BUILD_STATIC=ON + -DLIBDDWAF_MSVC_RUNTIME_LIBRARY=/MD -DCMAKE_INSTALL_LIBDIR=lib - -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX:PATH= ) diff --git a/ddtrace/appsec/_ddwaf.pyx b/ddtrace/appsec/_ddwaf.pyx index fd0fbf4be80..75e42205ecd 100644 --- a/ddtrace/appsec/_ddwaf.pyx +++ b/ddtrace/appsec/_ddwaf.pyx @@ -1,12 +1,13 @@ -# distutils: include_dirs = ddtrace/appsec/include -# distutils: library_dirs = ddtrace/appsec/lib -# distutils: libraries = ddwaf - import six +import sys from typing import Tuple from collections import deque -from collections.abc import Mapping, Sequence + +if sys.version_info >= (3, 5): + from typing import Mapping, Sequence +else: + from collections import Mapping, Sequence from cpython.mem cimport PyMem_Realloc, PyMem_Free from libc.string cimport memset diff --git a/pyproject.toml b/pyproject.toml index 4a61656fe01..1fab74819ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools >= 40.6.0", "setuptools_scm[toml] >=4,<6.1", "cython", "cmake >= 3.14"] +requires = ["setuptools >= 40.6.0", "setuptools_scm[toml] >=4,<6.1", "cython", "cmake >= 3.14", "ninja"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] diff --git a/setup.py b/setup.py index 5c606063344..78e897b6f67 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ import os import platform import subprocess +import tempfile import sys from setuptools import setup, find_packages, Extension @@ -77,9 +78,17 @@ def build_extension(self, ext): to_build.add(source_dir) for idx, source_dir in enumerate(to_build): - build_dir = os.path.join(self.build_temp, "_".join([ext.name, str(idx)])) - subprocess.check_call(["cmake", "-S", source_dir, "-B", build_dir]) - subprocess.check_call(["cmake", "--build", build_dir]) + opts = ["-DCMAKE_BUILD_TYPE=RelWithDebInfo"] + if platform.system() == "Windows": + opts.extend(["-A", "x64" if platform.architecture()[0] == "64bit" else "Win32"]) + # self.build_temp is too long on Windows and MSBuild is limited to 255 + # characters file paths. + build_dir = tempfile.mkdtemp() + else: + opts.extend(["-G", "Ninja"]) + build_dir = os.path.join(self.build_temp, "_".join([ext.name, str(idx)])) + subprocess.check_call(["cmake", "-S", source_dir, "-B", build_dir] + opts) + subprocess.check_call(["cmake", "--build", build_dir, "--config", "RelWithDebInfo"]) return BuildExtCommand.build_extension(self, ext) @@ -129,11 +138,13 @@ def get_exts_for(name): encoding_libraries = ["ws2_32"] extra_compile_args = [] debug_compile_args = [] + ddwaf_libraries = ["ddwaf_static"] else: + linux = platform.system() == "Linux" encoding_libraries = [] extra_compile_args = ["-DPy_BUILD_CORE"] if "DD_COMPILE_DEBUG" in os.environ: - if platform.system() == "Linux": + if linux: debug_compile_args = ["-g", "-O0", "-Werror", "-Wall", "-Wextra", "-Wpedantic", "-fanalyzer"] else: debug_compile_args = [ @@ -148,6 +159,10 @@ def get_exts_for(name): ] else: debug_compile_args = [] + if linux: + ddwaf_libraries = ["ddwaf", "rt", "m", "dl", "pthread"] + else: + ddwaf_libraries = ["ddwaf"] if sys.version_info[:2] >= (3, 4): @@ -271,6 +286,9 @@ def get_exts_for(name): Cython.Distutils.Extension( "ddtrace.appsec._ddwaf", sources=["ddtrace/appsec/_ddwaf.pyx"], + include_dirs=["ddtrace/appsec/include"], + library_dirs=["ddtrace/appsec/lib"], + libraries=ddwaf_libraries, language="c++", ), ], diff --git a/tests/smoke_test.py b/tests/smoke_test.py index 75839f8b366..5a3245e58f0 100644 --- a/tests/smoke_test.py +++ b/tests/smoke_test.py @@ -2,6 +2,8 @@ import ddtrace.bootstrap.sitecustomize as module +import ddtrace.appsec._ddwaf + if __name__ == "__main__": sys.exit(0 if module.loaded else 1) From f3d91217f5c48788b48d31805056e5e9d609c23d Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Fri, 26 Nov 2021 18:10:59 +0100 Subject: [PATCH 12/56] Fix --- ddtrace/appsec/_ddwaf.pyx | 130 +++++++++++++++++--------- ddtrace/appsec/_libddwaf.pxd | 7 +- ddtrace/profiling/collector/_task.pyx | 5 +- docs/installation_quickstart.rst | 2 +- setup.py | 12 ++- tests/smoke_test.py | 5 +- 6 files changed, 106 insertions(+), 55 deletions(-) diff --git a/ddtrace/appsec/_ddwaf.pyx b/ddtrace/appsec/_ddwaf.pyx index 75e42205ecd..c88040c38bd 100644 --- a/ddtrace/appsec/_ddwaf.pyx +++ b/ddtrace/appsec/_ddwaf.pyx @@ -1,33 +1,45 @@ -import six +from collections import deque import sys - from typing import Tuple -from collections import deque + +import six + if sys.version_info >= (3, 5): - from typing import Mapping, Sequence + from typing import Mapping + from typing import Sequence else: from collections import Mapping, Sequence -from cpython.mem cimport PyMem_Realloc, PyMem_Free -from libc.string cimport memset +from _libddwaf cimport DDWAF_LOG_LEVEL +from _libddwaf cimport DDWAF_OBJ_TYPE +from _libddwaf cimport ddwaf_context +from _libddwaf cimport ddwaf_context_destroy +from _libddwaf cimport ddwaf_context_init +from _libddwaf cimport ddwaf_destroy +from _libddwaf cimport ddwaf_get_version +from _libddwaf cimport ddwaf_handle +from _libddwaf cimport ddwaf_init +from _libddwaf cimport ddwaf_object +from _libddwaf cimport ddwaf_required_addresses +from _libddwaf cimport ddwaf_result +from _libddwaf cimport ddwaf_result_free +from _libddwaf cimport ddwaf_run +from _libddwaf cimport ddwaf_set_log_cb +from _libddwaf cimport ddwaf_version +from cpython.mem cimport PyMem_Free +from cpython.mem cimport PyMem_Realloc +from libc.stdint cimport uint32_t from libc.stdint cimport uint64_t +from libc.stdint cimport uintptr_t +from libc.string cimport memset + + +cdef void print_trace(DDWAF_LOG_LEVEL level, const char *function, const char *file, unsigned line, const char *message, uint64_t len): + print("[ddwaf] {}".format(message)) + -from _libddwaf cimport ( - DDWAF_OBJ_TYPE, - ddwaf_handle, - ddwaf_context, - ddwaf_result, - ddwaf_object, - ddwaf_version, - ddwaf_get_version, - ddwaf_init, - ddwaf_run, - ddwaf_result_free, - ddwaf_destroy, - ddwaf_context_init, - ddwaf_context_destroy, -) +ddwaf_set_log_cb(print_trace, DDWAF_LOG_LEVEL.DDWAF_LOG_TRACE) def version(): @@ -43,7 +55,7 @@ cdef class _Wrapper(object): cdef readonly ssize_t _size cdef readonly ssize_t _next_idx - def __init__(self, value, max_objects=1024): + def __init__(self, value, max_objects=5000): self._string_refs = [] self._convert(value, max_objects) @@ -53,9 +65,8 @@ cdef class _Wrapper(object): cdef ddwaf_object *obj idx = self._next_idx - ptr = self._ptr if idx + n > self._size: - self._size += n + ((128 - (n % 128)) % 128) + self._size += ((1000 - (n % 1000)) % 1000) ptr = PyMem_Realloc(self._ptr, self._size * sizeof(ddwaf_object)) if ptr == NULL: raise MemoryError @@ -64,23 +75,47 @@ cdef class _Wrapper(object): # we need to patch all array objects because they use pointers to other objects for i in range(idx): obj = ptr + i - if obj.array != NULL: + if (obj.type == DDWAF_OBJ_TYPE.DDWAF_OBJ_MAP or obj.type == DDWAF_OBJ_TYPE.DDWAF_OBJ_ARRAY) and obj.array != NULL: obj.array = obj.array - self._ptr + ptr self._ptr = ptr self._next_idx += n return idx + cdef void _make_string(self, ssize_t idx, const char* val, ssize_t length): + cdef ddwaf_object *obj + obj = self._ptr + idx + obj.type = DDWAF_OBJ_TYPE.DDWAF_OBJ_STRING + obj.stringValue = val + obj.nbEntries = length + + cdef void _make_array(self, ssize_t idx, ssize_t array_idx, ssize_t nb_entries): + cdef ddwaf_object *obj + obj = self._ptr + idx + obj.type = DDWAF_OBJ_TYPE.DDWAF_OBJ_ARRAY + obj.array = self._ptr + array_idx + obj.nbEntries = nb_entries + + cdef void _make_map(self, ssize_t idx, ssize_t array_idx, ssize_t nb_entries): + cdef ddwaf_object *obj + obj = self._ptr + idx + obj.type = DDWAF_OBJ_TYPE.DDWAF_OBJ_MAP + obj.array = self._ptr + array_idx + obj.nbEntries = nb_entries + + cdef void _set_parameter(self, ssize_t idx, const char* name, ssize_t length): + cdef ddwaf_object *obj + obj = self._ptr + idx + obj.parameterName = name + obj.parameterNameLength = length + cdef void _convert(self, value, max_objects) except *: cdef object stack - cdef ddwaf_object *obj - cdef ddwaf_object *item_obj - cdef ssize_t i, j, idx + cdef ssize_t i, j, n, idx, items_idx i = 0 stack = deque([(self._reserve_obj(), value)], maxlen=max_objects) while len(stack) and (max_objects is None or i < max_objects): idx, val = stack.popleft() - obj = self._ptr + idx if isinstance(val, (int, float)): val = six.text_type(val) @@ -90,33 +125,27 @@ cdef class _Wrapper(object): if isinstance(val, bytes): self._string_refs.append(val) - obj.type = DDWAF_OBJ_TYPE.DDWAF_OBJ_STRING - obj.stringValue = val - obj.nbEntries = len(val) + self._make_string(idx, val, len(val)) elif isinstance(val, Mapping): - obj.type = DDWAF_OBJ_TYPE.DDWAF_OBJ_MAP n = len(val) - idx = self._reserve_obj(n) - obj.array = self._ptr + idx - obj.nbEntries = n + items_idx = self._reserve_obj(n) + self._make_map(idx, items_idx, n) + # size of val must not change!! should not happen + # while holding the GIL? for j, (k, v) in enumerate(six.iteritems(val)): if isinstance(k, six.text_type): k = k.encode("utf-8", errors="surrogatepass") if isinstance(k, bytes): - item_obj = self._ptr + idx + j self._string_refs.append(k) - item_obj.parameterName = k - item_obj.parameterNameLength = len(k) - stack.append((idx + j, v)) + self._set_parameter(items_idx + j, k, len(k)) + stack.append((items_idx + j, v)) elif isinstance(val, Sequence): - obj.type = DDWAF_OBJ_TYPE.DDWAF_OBJ_ARRAY n = len(val) - idx = self._reserve_obj(n) - stack.extend([(idx + j, val[j]) for j in range(n)]) - obj.array = self._ptr + idx - obj.nbEntries = n + items_idx = self._reserve_obj(n) + self._make_array(idx, items_idx, n) + stack.extend([(items_idx + j, val[j]) for j in range(n)]) i += 1 @@ -142,6 +171,17 @@ cdef class DDWaf(object): if self._handle == NULL: raise ValueError("invalid rules") + @property + def required_data(self): + cdef uint32_t size + cdef const char* const* ptr + + addresses = [] + ptr = ddwaf_required_addresses(self._handle, &size) + for i in range(size): + addresses.append(( ptr[i]).decode("utf-8")) + return addresses + def run(self, data, timeout_ms=1000): cdef ddwaf_context ctx cdef ddwaf_result result diff --git a/ddtrace/appsec/_libddwaf.pxd b/ddtrace/appsec/_libddwaf.pxd index 6492e0f5321..25ab6f82c7d 100644 --- a/ddtrace/appsec/_libddwaf.pxd +++ b/ddtrace/appsec/_libddwaf.pxd @@ -1,5 +1,8 @@ +from libc.stdint cimport int64_t +from libc.stdint cimport uint16_t +from libc.stdint cimport uint32_t +from libc.stdint cimport uint64_t from libcpp cimport bool -from libc.stdint cimport uint16_t, int64_t, uint64_t cdef extern from "include/ddwaf.h": @@ -14,6 +17,7 @@ cdef extern from "include/ddwaf.h": DDWAF_OBJ_ARRAY DDWAF_OBJ_MAP DDWAF_OBJ_STRING + DDWAF_OBJ_INVALID ctypedef struct ddwaf_object: const char* parameterName @@ -47,6 +51,7 @@ cdef extern from "include/ddwaf.h": void ddwaf_context_destroy(ddwaf_context context); void ddwaf_result_free(ddwaf_result* result); void ddwaf_destroy(ddwaf_handle handle); + const char* const* ddwaf_required_addresses(const ddwaf_handle handle, uint32_t* size); ctypedef enum DDWAF_LOG_LEVEL: DDWAF_LOG_TRACE diff --git a/ddtrace/profiling/collector/_task.pyx b/ddtrace/profiling/collector/_task.pyx index 9b193cce152..3f4798c25e3 100644 --- a/ddtrace/profiling/collector/_task.pyx +++ b/ddtrace/profiling/collector/_task.pyx @@ -5,9 +5,10 @@ from . import _threading try: - from greenlet import settrace, getcurrent - import gevent.thread import gevent.hub + import gevent.thread + from greenlet import getcurrent + from greenlet import settrace except ImportError: _gevent_tracer = None else: diff --git a/docs/installation_quickstart.rst b/docs/installation_quickstart.rst index d14ac307464..08f1b1009ad 100644 --- a/docs/installation_quickstart.rst +++ b/docs/installation_quickstart.rst @@ -31,7 +31,7 @@ Binary distributions are not available for Alpine so build dependencies must be .. code-block:: bash - apk add gcc musl-dev linux-headers + apk add gcc g++ musl-dev linux-headers pip install ddtrace diff --git a/setup.py b/setup.py index 78e897b6f67..00ba936d5b9 100644 --- a/setup.py +++ b/setup.py @@ -78,17 +78,21 @@ def build_extension(self, ext): to_build.add(source_dir) for idx, source_dir in enumerate(to_build): - opts = ["-DCMAKE_BUILD_TYPE=RelWithDebInfo"] + if "DD_COMPILE_DEBUG" in os.environ: + build_type = "RelWithDebInfo" + else: + build_type = "Release" + opts = ["-DCMAKE_BUILD_TYPE={}".format(build_type)] if platform.system() == "Windows": opts.extend(["-A", "x64" if platform.architecture()[0] == "64bit" else "Win32"]) - # self.build_temp is too long on Windows and MSBuild is limited to 255 - # characters file paths. + # self.build_temp is too long on Windows and some Visual Studio versions are + # limited to 255 characters file paths. build_dir = tempfile.mkdtemp() else: opts.extend(["-G", "Ninja"]) build_dir = os.path.join(self.build_temp, "_".join([ext.name, str(idx)])) subprocess.check_call(["cmake", "-S", source_dir, "-B", build_dir] + opts) - subprocess.check_call(["cmake", "--build", build_dir, "--config", "RelWithDebInfo"]) + subprocess.check_call(["cmake", "--build", build_dir, "--config", build_type]) return BuildExtCommand.build_extension(self, ext) diff --git a/tests/smoke_test.py b/tests/smoke_test.py index 5a3245e58f0..87b64547b43 100644 --- a/tests/smoke_test.py +++ b/tests/smoke_test.py @@ -1,9 +1,10 @@ import sys -import ddtrace.bootstrap.sitecustomize as module - import ddtrace.appsec._ddwaf +import ddtrace.bootstrap.sitecustomize as module if __name__ == "__main__": + ddtrace.appsec._ddwaf.version() + sys.exit(0 if module.loaded else 1) From ddcb1f14a8075e69656b03c57a2c2def2f993f2a Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Fri, 26 Nov 2021 18:34:37 +0100 Subject: [PATCH 13/56] Add deps to tox --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index a4bd34c21f1..5758f5a6583 100644 --- a/tox.ini +++ b/tox.ini @@ -93,6 +93,8 @@ extras = deps = cython + cmake + ninja pytest-cov pytest-mock opentracing From 5e23349b99cb96abb56b26fb39ef7c29c192a44a Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Fri, 26 Nov 2021 18:55:28 +0100 Subject: [PATCH 14/56] Try preinstall trick --- tox.ini | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 5758f5a6583..a45ef7f083c 100644 --- a/tox.ini +++ b/tox.ini @@ -71,6 +71,10 @@ envlist = py{37,38}-opentracer_gevent-gevent{13,14} py{39,310}-opentracer_gevent-gevent{209,2012,211} +# trick to enable pre-installation of numpy and cython +indexserver = + preinstall = https://pypi.python.org/simple + isolated_build = true requires = virtualenv<=20.2.1 @@ -92,9 +96,9 @@ extras = profile: profiling deps = - cython - cmake - ninja + :preinstall: cython + :preinstall: cmake + :preinstall: ninja pytest-cov pytest-mock opentracing From 3df25bc586939e6d7c0c9db3831d525d80763af7 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Fri, 26 Nov 2021 19:20:17 +0100 Subject: [PATCH 15/56] Fix tox --- setup.py | 8 +++----- tox.ini | 10 +++------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index 00ba936d5b9..0f8dd8f9ab2 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def build_extension(self, ext): if os.path.exists(os.path.join(source_dir, "CMakeLists.txt")): to_build.add(source_dir) - for idx, source_dir in enumerate(to_build): + for source_dir in to_build: if "DD_COMPILE_DEBUG" in os.environ: build_type = "RelWithDebInfo" else: @@ -85,12 +85,10 @@ def build_extension(self, ext): opts = ["-DCMAKE_BUILD_TYPE={}".format(build_type)] if platform.system() == "Windows": opts.extend(["-A", "x64" if platform.architecture()[0] == "64bit" else "Win32"]) - # self.build_temp is too long on Windows and some Visual Studio versions are - # limited to 255 characters file paths. - build_dir = tempfile.mkdtemp() else: opts.extend(["-G", "Ninja"]) - build_dir = os.path.join(self.build_temp, "_".join([ext.name, str(idx)])) + build_dir = tempfile.mkdtemp() + # TODO remove build_dir subprocess.check_call(["cmake", "-S", source_dir, "-B", build_dir] + opts) subprocess.check_call(["cmake", "--build", build_dir, "--config", build_type]) diff --git a/tox.ini b/tox.ini index a45ef7f083c..5758f5a6583 100644 --- a/tox.ini +++ b/tox.ini @@ -71,10 +71,6 @@ envlist = py{37,38}-opentracer_gevent-gevent{13,14} py{39,310}-opentracer_gevent-gevent{209,2012,211} -# trick to enable pre-installation of numpy and cython -indexserver = - preinstall = https://pypi.python.org/simple - isolated_build = true requires = virtualenv<=20.2.1 @@ -96,9 +92,9 @@ extras = profile: profiling deps = - :preinstall: cython - :preinstall: cmake - :preinstall: ninja + cython + cmake + ninja pytest-cov pytest-mock opentracing From bdc42fbac8888a727602f3523279089a983dce7c Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Fri, 26 Nov 2021 19:37:12 +0100 Subject: [PATCH 16/56] Add a changelog --- ddtrace/profiling/collector/_task.pyx | 5 ++-- .../notes/appsec-64ffb8f244734d44.yaml | 4 ++++ setup.py | 24 +++++++++++-------- 3 files changed, 20 insertions(+), 13 deletions(-) create mode 100644 releasenotes/notes/appsec-64ffb8f244734d44.yaml diff --git a/ddtrace/profiling/collector/_task.pyx b/ddtrace/profiling/collector/_task.pyx index 3f4798c25e3..9b193cce152 100644 --- a/ddtrace/profiling/collector/_task.pyx +++ b/ddtrace/profiling/collector/_task.pyx @@ -5,10 +5,9 @@ from . import _threading try: - import gevent.hub + from greenlet import settrace, getcurrent import gevent.thread - from greenlet import getcurrent - from greenlet import settrace + import gevent.hub except ImportError: _gevent_tracer = None else: diff --git a/releasenotes/notes/appsec-64ffb8f244734d44.yaml b/releasenotes/notes/appsec-64ffb8f244734d44.yaml new file mode 100644 index 00000000000..75c8b887161 --- /dev/null +++ b/releasenotes/notes/appsec-64ffb8f244734d44.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Introduce Datadog Application Security module. diff --git a/setup.py b/setup.py index 0f8dd8f9ab2..e79f1f172f6 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,5 @@ import os import platform -import subprocess -import tempfile import sys from setuptools import setup, find_packages, Extension @@ -71,26 +69,32 @@ def run_tests(self): class CMake(BuildExtCommand): def build_extension(self, ext): + import shutil + import subprocess + import tempfile + to_build = set() for source in ext.sources: source_dir = os.path.dirname(os.path.realpath(source)) if os.path.exists(os.path.join(source_dir, "CMakeLists.txt")): to_build.add(source_dir) + debug_compile = "DD_COMPILE_DEBUG" in os.environ + for source_dir in to_build: - if "DD_COMPILE_DEBUG" in os.environ: - build_type = "RelWithDebInfo" - else: - build_type = "Release" + build_type = "RelWithDebInfo" if debug_compile else "Release" opts = ["-DCMAKE_BUILD_TYPE={}".format(build_type)] if platform.system() == "Windows": opts.extend(["-A", "x64" if platform.architecture()[0] == "64bit" else "Win32"]) else: opts.extend(["-G", "Ninja"]) - build_dir = tempfile.mkdtemp() - # TODO remove build_dir - subprocess.check_call(["cmake", "-S", source_dir, "-B", build_dir] + opts) - subprocess.check_call(["cmake", "--build", build_dir, "--config", build_type]) + try: + build_dir = tempfile.mkdtemp() + subprocess.check_call(["cmake", "-S", source_dir, "-B", build_dir] + opts) + subprocess.check_call(["cmake", "--build", build_dir, "--config", build_type]) + finally: + if not debug_compile: + shutil.rmtree(build_dir, ignore_errors=True) return BuildExtCommand.build_extension(self, ext) From 0ef7363fed6a40b3236772242ad5cd92161085aa Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Sun, 28 Nov 2021 20:19:30 +0100 Subject: [PATCH 17/56] Add a fuzzer target --- riotfile.py | 6 ++++++ tests/appsec/__init__.py | 0 tests/appsec/ddwaf_fuzzer.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 tests/appsec/__init__.py create mode 100644 tests/appsec/ddwaf_fuzzer.py diff --git a/riotfile.py b/riotfile.py index 320806c499b..9b94820c070 100644 --- a/riotfile.py +++ b/riotfile.py @@ -169,6 +169,12 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION): }, command="scripts/build-docs", ), + Venv( + name="appsec", + command="pytest {cmdargs} tests/appsec/ddwaf_fuzzer.py", + venvs=[Venv(pys=select_pys())], + pkgs={"atheris": latest}, + ), Venv( name="benchmarks", pys=select_pys(), diff --git a/tests/appsec/__init__.py b/tests/appsec/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/appsec/ddwaf_fuzzer.py b/tests/appsec/ddwaf_fuzzer.py new file mode 100644 index 00000000000..ba2cb3b1868 --- /dev/null +++ b/tests/appsec/ddwaf_fuzzer.py @@ -0,0 +1,32 @@ +import sys + +import atheris +from hypothesis import given +from hypothesis import strategies as st + + +with atheris.instrument_imports(): + from ddtrace.appsec._ddwaf import _Wrapper + + +PYTHON_OBJECTS = st.recursive( + base=st.one_of(st.none(), st.booleans(), st.integers(), st.floats(), st.text()), + extend=lambda inner: st.lists(inner) | st.dictionaries(st.text(), inner), +) + +WRAPPER_KWARGS = dict( + max_objects=st.integers(min_value=0, max_value=2 ** 63 - 1), +) + + +@given(obj=PYTHON_OBJECTS, kwargs=st.fixed_dictionaries(WRAPPER_KWARGS)) +@atheris.instrument_func +def test_ddwaf_objects_wrapper(obj, kwargs): + obj = _Wrapper(obj, **kwargs) + repr(obj) + del obj + + +if __name__ == "__main__": + atheris.Setup(sys.argv, atheris.instrument_func(test_ddwaf_objects_wrapper.hypothesis.fuzz_one_input)) + atheris.Fuzz() From 021ad0e0ec875c54bffbdef9334c43bc0b33257e Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Sun, 28 Nov 2021 20:51:18 +0100 Subject: [PATCH 18/56] Always link with stdc++ on Linux --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e79f1f172f6..31e5f0902cc 100644 --- a/setup.py +++ b/setup.py @@ -166,7 +166,7 @@ def get_exts_for(name): else: debug_compile_args = [] if linux: - ddwaf_libraries = ["ddwaf", "rt", "m", "dl", "pthread"] + ddwaf_libraries = ["ddwaf", "stdc++", "rt", "m", "dl", "pthread"] else: ddwaf_libraries = ["ddwaf"] From 065edb5aef8d836a2c2b97f7efceff7c44dba3f3 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Sun, 28 Nov 2021 22:42:20 +0100 Subject: [PATCH 19/56] Revert "Always link with stdc++ on Linux" This reverts commit 021ad0e0ec875c54bffbdef9334c43bc0b33257e. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 31e5f0902cc..e79f1f172f6 100644 --- a/setup.py +++ b/setup.py @@ -166,7 +166,7 @@ def get_exts_for(name): else: debug_compile_args = [] if linux: - ddwaf_libraries = ["ddwaf", "stdc++", "rt", "m", "dl", "pthread"] + ddwaf_libraries = ["ddwaf", "rt", "m", "dl", "pthread"] else: ddwaf_libraries = ["ddwaf"] From c054cc5042ab8b3f31db43cc99d47578aafacb3e Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Sun, 28 Nov 2021 22:49:48 +0100 Subject: [PATCH 20/56] -fanalyzer is not compatible with clang --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e79f1f172f6..79d586ad39c 100644 --- a/setup.py +++ b/setup.py @@ -151,7 +151,7 @@ def get_exts_for(name): extra_compile_args = ["-DPy_BUILD_CORE"] if "DD_COMPILE_DEBUG" in os.environ: if linux: - debug_compile_args = ["-g", "-O0", "-Werror", "-Wall", "-Wextra", "-Wpedantic", "-fanalyzer"] + debug_compile_args = ["-g", "-O0", "-Werror", "-Wall", "-Wextra", "-Wpedantic"] else: debug_compile_args = [ "-g", From 118f8880caeeadf8a421f21998bafeb7ebb0ff02 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Sun, 28 Nov 2021 22:51:26 +0100 Subject: [PATCH 21/56] Remove -Werror --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 79d586ad39c..5cff8fd242b 100644 --- a/setup.py +++ b/setup.py @@ -151,12 +151,11 @@ def get_exts_for(name): extra_compile_args = ["-DPy_BUILD_CORE"] if "DD_COMPILE_DEBUG" in os.environ: if linux: - debug_compile_args = ["-g", "-O0", "-Werror", "-Wall", "-Wextra", "-Wpedantic"] + debug_compile_args = ["-g", "-O0", "-Wall", "-Wextra", "-Wpedantic"] else: debug_compile_args = [ "-g", "-O0", - "-Werror", "-Wall", "-Wextra", "-Wpedantic", From f09b40fc4af8b586f093b5c5cb8023b701a916da Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Tue, 7 Dec 2021 21:35:37 +0100 Subject: [PATCH 22/56] Run tests in riot --- riotfile.py | 2 +- tests/appsec/{ddwaf_fuzzer.py => test_ddwaf_fuzz.py} | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) rename tests/appsec/{ddwaf_fuzzer.py => test_ddwaf_fuzz.py} (89%) diff --git a/riotfile.py b/riotfile.py index 9b94820c070..f562910409d 100644 --- a/riotfile.py +++ b/riotfile.py @@ -171,7 +171,7 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION): ), Venv( name="appsec", - command="pytest {cmdargs} tests/appsec/ddwaf_fuzzer.py", + command="pytest {cmdargs} tests/appsec", venvs=[Venv(pys=select_pys())], pkgs={"atheris": latest}, ), diff --git a/tests/appsec/ddwaf_fuzzer.py b/tests/appsec/test_ddwaf_fuzz.py similarity index 89% rename from tests/appsec/ddwaf_fuzzer.py rename to tests/appsec/test_ddwaf_fuzz.py index ba2cb3b1868..6d7e70e488e 100644 --- a/tests/appsec/ddwaf_fuzzer.py +++ b/tests/appsec/test_ddwaf_fuzz.py @@ -4,9 +4,7 @@ from hypothesis import given from hypothesis import strategies as st - -with atheris.instrument_imports(): - from ddtrace.appsec._ddwaf import _Wrapper +from ddtrace.appsec._ddwaf import _Wrapper PYTHON_OBJECTS = st.recursive( From bcc3e8d666072335ea1503abf75f93ea23f5aa0e Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Tue, 7 Dec 2021 21:55:40 +0100 Subject: [PATCH 23/56] Fix riot --- riotfile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/riotfile.py b/riotfile.py index 60d27fcb225..07dc441bc32 100644 --- a/riotfile.py +++ b/riotfile.py @@ -176,7 +176,6 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION): pkgs={"atheris": latest}, ), Venv( - name="benchmarks", pys=select_pys(), pkgs={"pytest-benchmark": latest, "msgpack": latest}, venvs=[ From d0180ce4397e1e8896267a38aed4e949ba953ad4 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Thu, 9 Dec 2021 13:12:24 +0100 Subject: [PATCH 24/56] Bump ddwaf to 1.0.15 --- ddtrace/appsec/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ddtrace/appsec/CMakeLists.txt b/ddtrace/appsec/CMakeLists.txt index d6352428a38..f40fadae98a 100644 --- a/ddtrace/appsec/CMakeLists.txt +++ b/ddtrace/appsec/CMakeLists.txt @@ -4,9 +4,10 @@ include(ExternalProject) ExternalProject_Add(libddwaf GIT_REPOSITORY https://github.com/DataDog/libddwaf.git - GIT_TAG 8ee1ae3 + GIT_TAG 1.0.15 INSTALL_DIR ${CMAKE_SOURCE_DIR} CMAKE_ARGS + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DLIBDDWAF_BUILD_SHARED=OFF -DLIBDDWAF_BUILD_STATIC=ON -DLIBDDWAF_MSVC_RUNTIME_LIBRARY=/MD From 419ad4514e97035a36566961b548592ab5f0a106 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Wed, 22 Dec 2021 14:46:18 +0100 Subject: [PATCH 25/56] Allow cmake extensions to fail --- setup.py | 51 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index a5ab3cadf78..4d9dee49a8f 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,8 @@ HERE = os.path.dirname(os.path.abspath(__file__)) +DEBUG_COMPILE = "DD_COMPILE_DEBUG" in os.environ + def load_module_from_project_file(mod_name, fname): """ @@ -74,29 +76,39 @@ def build_extension(self, ext): import tempfile to_build = set() + # Detect if any source file sits next to a CMakeLists.txt file for source in ext.sources: source_dir = os.path.dirname(os.path.realpath(source)) if os.path.exists(os.path.join(source_dir, "CMakeLists.txt")): to_build.add(source_dir) - debug_compile = "DD_COMPILE_DEBUG" in os.environ - - for source_dir in to_build: - build_type = "RelWithDebInfo" if debug_compile else "Release" - opts = ["-DCMAKE_BUILD_TYPE={}".format(build_type)] - if platform.system() == "Windows": - opts.extend(["-A", "x64" if platform.architecture()[0] == "64bit" else "Win32"]) - else: - opts.extend(["-G", "Ninja"]) - try: - build_dir = tempfile.mkdtemp() - subprocess.check_call(["cmake", "-S", source_dir, "-B", build_dir] + opts) - subprocess.check_call(["cmake", "--build", build_dir, "--config", build_type]) - finally: - if not debug_compile: - shutil.rmtree(build_dir, ignore_errors=True) - - return BuildExtCommand.build_extension(self, ext) + if not to_build: + # Build the extension as usual + BuildExtCommand.build_extension(self, ext) + return + + try: + for source_dir in to_build: + build_type = "RelWithDebInfo" if DEBUG_COMPILE else "Release" + opts = ["-DCMAKE_BUILD_TYPE={}".format(build_type)] + if platform.system() == "Windows": + opts.extend(["-A", "x64" if platform.architecture()[0] == "64bit" else "Win32"]) + else: + opts.extend(["-G", "Ninja"]) + try: + build_dir = tempfile.mkdtemp() + subprocess.check_call(["cmake", "-S", source_dir, "-B", build_dir] + opts) + subprocess.check_call(["cmake", "--build", build_dir, "--config", build_type]) + finally: + if not DEBUG_COMPILE: + shutil.rmtree(build_dir, ignore_errors=True) + + BuildExtCommand.build_extension(self, ext) + except Exception as e: + print("WARNING: building extension \"%s\" failed: %s" % (ext.name, e)) + # Remove this extension from the extension list to avoid errors + # during the install phase. + self.extensions = [item for item in self.extensions if item.name != ext.name] long_description = """ @@ -139,7 +151,6 @@ def get_exts_for(name): else: encoding_macros = [("__LITTLE_ENDIAN__", "1")] - if platform.system() == "Windows": encoding_libraries = ["ws2_32"] extra_compile_args = [] @@ -149,7 +160,7 @@ def get_exts_for(name): linux = platform.system() == "Linux" encoding_libraries = [] extra_compile_args = ["-DPy_BUILD_CORE"] - if "DD_COMPILE_DEBUG" in os.environ: + if DEBUG_COMPILE: if linux: debug_compile_args = ["-g", "-O0", "-Wall", "-Wextra", "-Wpedantic"] else: From 3260fbfe0a3c52710b04110cec906757c00d47df Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Wed, 22 Dec 2021 14:46:43 +0100 Subject: [PATCH 26/56] Bump the WAF timeout --- ddtrace/appsec/_ddwaf.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/appsec/_ddwaf.pyx b/ddtrace/appsec/_ddwaf.pyx index c88040c38bd..4111cf7ea64 100644 --- a/ddtrace/appsec/_ddwaf.pyx +++ b/ddtrace/appsec/_ddwaf.pyx @@ -182,7 +182,7 @@ cdef class DDWaf(object): addresses.append(( ptr[i]).decode("utf-8")) return addresses - def run(self, data, timeout_ms=1000): + def run(self, data, timeout_ms=10000): cdef ddwaf_context ctx cdef ddwaf_result result From 6627f9711e2989a38ffd3654b215def12a8c0d2e Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Wed, 22 Dec 2021 16:06:16 +0100 Subject: [PATCH 27/56] Support for CMAKE_COMMAND --- .circleci/config.yml | 2 +- ddtrace/appsec/CMakeLists.txt | 1 + setup.py | 25 ++++++++++++++++--------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 289cfea9ae0..5cf164a6d04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -283,7 +283,7 @@ jobs: - run: sudo apt-get update - run: sudo apt-get install --yes clang-format gcc-10 g++-10 python3 python3-setuptools python3-pip - run: scripts/cformat.sh - - run: DD_COMPILE_DEBUG=1 CC=gcc-10 CXX=g++-10 pip -vvv install . + - run: DD_COMPILE_DEBUG=1 DD_TESTING_RAISE=1 CC=gcc-10 CXX=g++-10 pip -vvv install . coverage_report: executor: python39 diff --git a/ddtrace/appsec/CMakeLists.txt b/ddtrace/appsec/CMakeLists.txt index f40fadae98a..b9bb77da76a 100644 --- a/ddtrace/appsec/CMakeLists.txt +++ b/ddtrace/appsec/CMakeLists.txt @@ -8,6 +8,7 @@ ExternalProject_Add(libddwaf INSTALL_DIR ${CMAKE_SOURCE_DIR} CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} -DLIBDDWAF_BUILD_SHARED=OFF -DLIBDDWAF_BUILD_STATIC=ON -DLIBDDWAF_MSVC_RUNTIME_LIBRARY=/MD diff --git a/setup.py b/setup.py index 4d9dee49a8f..e05ec49d0e6 100644 --- a/setup.py +++ b/setup.py @@ -88,24 +88,31 @@ def build_extension(self, ext): return try: + cmake_command = os.environ.get("CMAKE_COMMAND", "cmake") + build_type = "RelWithDebInfo" if DEBUG_COMPILE else "Release" + opts = ["-DCMAKE_BUILD_TYPE={}".format(build_type)] + if platform.system() == "Windows": + opts.extend(["-A", "x64" if platform.architecture()[0] == "64bit" else "Win32"]) + else: + opts.extend(["-G", "Ninja"]) + ninja_command = os.environ.get("NINJA_COMMAND", "") + if ninja_command: + opts.append("-DCMAKE_MAKE_PROGRAM={}".format(ninja_command)) + for source_dir in to_build: - build_type = "RelWithDebInfo" if DEBUG_COMPILE else "Release" - opts = ["-DCMAKE_BUILD_TYPE={}".format(build_type)] - if platform.system() == "Windows": - opts.extend(["-A", "x64" if platform.architecture()[0] == "64bit" else "Win32"]) - else: - opts.extend(["-G", "Ninja"]) try: build_dir = tempfile.mkdtemp() - subprocess.check_call(["cmake", "-S", source_dir, "-B", build_dir] + opts) - subprocess.check_call(["cmake", "--build", build_dir, "--config", build_type]) + subprocess.check_call([cmake_command, "-S", source_dir, "-B", build_dir] + opts) + subprocess.check_call([cmake_command, "--build", build_dir, "--config", build_type]) finally: if not DEBUG_COMPILE: shutil.rmtree(build_dir, ignore_errors=True) BuildExtCommand.build_extension(self, ext) except Exception as e: - print("WARNING: building extension \"%s\" failed: %s" % (ext.name, e)) + if "DD_TESTING_RAISE" in os.environ: + raise + print('WARNING: building extension "%s" failed: %s' % (ext.name, e)) # Remove this extension from the extension list to avoid errors # during the install phase. self.extensions = [item for item in self.extensions if item.name != ext.name] From 5b7338840342f29075baccf733f6daa4896dd2fa Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Thu, 23 Dec 2021 15:13:18 +0100 Subject: [PATCH 28/56] Add an AppSec processor --- .circleci/config.yml | 8 + ddtrace/appsec/CMakeLists.txt | 2 +- ddtrace/appsec/__init__.py | 33 + ddtrace/appsec/_ddwaf.pyx | 35 +- ddtrace/appsec/processor.py | 95 + ddtrace/appsec/rules.json | 5683 ++++++++++++++++++++++++++++ ddtrace/bootstrap/sitecustomize.py | 5 + docs/installation_quickstart.rst | 2 +- setup.cfg | 2 +- setup.py | 7 +- tests/appsec/rules-bad.json | 1 + tests/appsec/rules-good.json | 35 + tests/appsec/test_processor.py | 96 + tests/conftest.py | 13 + 14 files changed, 6003 insertions(+), 14 deletions(-) create mode 100644 ddtrace/appsec/processor.py create mode 100644 ddtrace/appsec/rules.json create mode 100644 tests/appsec/rules-bad.json create mode 100644 tests/appsec/rules-good.json create mode 100644 tests/appsec/test_processor.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 5cf164a6d04..423cb8cc440 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -339,6 +339,12 @@ jobs: paths: - "." + appsec: + <<: *contrib_job + steps: + - run_test: + pattern: 'appsec' + tracer: <<: *contrib_job steps: @@ -1020,6 +1026,7 @@ requires_tests: &requires_tests - starlette - test_logging - tracer + - appsec - tornado - urllib3 - vertica @@ -1107,6 +1114,7 @@ workflows: - test_logging: *requires_base_venvs - tornado: *requires_base_venvs - tracer: *requires_base_venvs + - appsec: *requires_base_venvs - urllib3: *requires_base_venvs - vertica: *requires_base_venvs - wsgi: *requires_base_venvs diff --git a/ddtrace/appsec/CMakeLists.txt b/ddtrace/appsec/CMakeLists.txt index b9bb77da76a..bf81d23a9f8 100644 --- a/ddtrace/appsec/CMakeLists.txt +++ b/ddtrace/appsec/CMakeLists.txt @@ -4,7 +4,7 @@ include(ExternalProject) ExternalProject_Add(libddwaf GIT_REPOSITORY https://github.com/DataDog/libddwaf.git - GIT_TAG 1.0.15 + GIT_TAG 1.0.16 INSTALL_DIR ${CMAKE_SOURCE_DIR} CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} diff --git a/ddtrace/appsec/__init__.py b/ddtrace/appsec/__init__.py index e69de29bb2d..88b01c20059 100644 --- a/ddtrace/appsec/__init__.py +++ b/ddtrace/appsec/__init__.py @@ -0,0 +1,33 @@ +from ddtrace.internal.logger import get_logger + + +log = get_logger(__name__) + +_appsec_import_error = None + +try: + from ddtrace.appsec.processor import AppSecProcessor +except ImportError as e: + from ddtrace import config + + if config._raise: + raise + + AppSecProcessor = None # type: ignore + _appsec_import_error = repr(e) + + +def enable(*args, **kwargs): + if AppSecProcessor is None: + log.warning("AppSec was enabled but couldn't start due to the following error:\n%s", _appsec_import_error) + return + AppSecProcessor.enable(*args, **kwargs) + + +def disable(): + if AppSecProcessor is None: + return + AppSecProcessor.disable() + + +__all__ = ["AppSecProcessor", "enable", "disable"] diff --git a/ddtrace/appsec/_ddwaf.pyx b/ddtrace/appsec/_ddwaf.pyx index 4111cf7ea64..463467c95ee 100644 --- a/ddtrace/appsec/_ddwaf.pyx +++ b/ddtrace/appsec/_ddwaf.pyx @@ -35,21 +35,33 @@ from libc.stdint cimport uintptr_t from libc.string cimport memset -cdef void print_trace(DDWAF_LOG_LEVEL level, const char *function, const char *file, unsigned line, const char *message, uint64_t len): - print("[ddwaf] {}".format(message)) - - -ddwaf_set_log_cb(print_trace, DDWAF_LOG_LEVEL.DDWAF_LOG_TRACE) +DEFAULT_DDWAF_TIMEOUT_MS=20 def version(): # type: () -> Tuple[int, int, int] + """Get the version of libddwaf.""" cdef ddwaf_version version ddwaf_get_version(&version) return (version.major, version.minor, version.patch) cdef class _Wrapper(object): + """ + Wrapper to convert Python objects to ddwaf objects. + + libddwaf represents scalar and composite values using ddwaf objects. This + wrapper converts Python objects to ddwaf objects by traversing all values + and their children. By default, the number of objects is limited to avoid + infinite loops. This limitation can be lifted on trusted data by setting + `max_objects` to `None`. + + Under the hood, the wrapper uses an array of `ddwaf_object` allocated as a + single buffer. Objects such as maps or arrays refer to other objects that + are only part of this buffer. Strings are not copied, they live in the + Python heap and are referenced by the wrapper to avoid garbage collection. + """ + cdef ddwaf_object *_ptr cdef readonly object _string_refs cdef readonly ssize_t _size @@ -150,16 +162,21 @@ cdef class _Wrapper(object): i += 1 def __repr__(self): - return "<_Wrapper for {0._next_idx} elements>".format(self) + return "<{0.__class__.__name__} for {0._next_idx} elements>".format(self) def __sizeof__(self): - return super(_Wrapper, self).__sizeof__() + self._size + return super(_Wrapper, self).__sizeof__() + self._size * sizeof(ddwaf_object) def __dealloc__(self): PyMem_Free(self._ptr) cdef class DDWaf(object): + """ + A DDWaf instance performs a matching operation on provided data according + to some rules. + """ + cdef ddwaf_handle _handle cdef object _rules @@ -182,7 +199,7 @@ cdef class DDWaf(object): addresses.append(( ptr[i]).decode("utf-8")) return addresses - def run(self, data, timeout_ms=10000): + def run(self, data, timeout_ms=DEFAULT_DDWAF_TIMEOUT_MS): cdef ddwaf_context ctx cdef ddwaf_result result @@ -191,7 +208,7 @@ cdef class DDWaf(object): raise RuntimeError try: wrapper = _Wrapper(data) - ddwaf_run(ctx, (<_Wrapper?>wrapper)._ptr, &result, timeout_ms) + ddwaf_run(ctx, (<_Wrapper?>wrapper)._ptr, &result, timeout_ms * 1000) if result.data != NULL: return ( result.data).decode("utf-8") finally: diff --git a/ddtrace/appsec/processor.py b/ddtrace/appsec/processor.py new file mode 100644 index 00000000000..eda5b9adb31 --- /dev/null +++ b/ddtrace/appsec/processor.py @@ -0,0 +1,95 @@ +import json +import os.path +from typing import Any +from typing import ClassVar +from typing import Optional +from typing import TYPE_CHECKING + +import attr + +import ddtrace +from ddtrace import config +from ddtrace.appsec._ddwaf import DDWaf +from ddtrace.constants import MANUAL_KEEP_KEY +from ddtrace.internal.logger import get_logger +from ddtrace.utils.formats import get_env + + +if TYPE_CHECKING: + from ddtrace import Span + +ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_RULES = os.path.join(ROOT_DIR, "rules.json") + +log = get_logger(__name__) + + +@attr.s(eq=False) +class AppSecProcessor(object): + + tracer = attr.ib(type=ddtrace.Tracer, default=None) + rules = attr.ib(type=str, default=None) + _ddwaf = attr.ib(type=DDWaf, init=False) + + enabled = False + _instance = None # type: ClassVar[Optional[AppSecProcessor]] + + def __attrs_post_init__(self): + # type: () -> None + if self.tracer is None: + self.tracer = ddtrace.tracer + if self.rules is None: + self.rules = get_env("appsec", "rules", default=DEFAULT_RULES) + if self._ddwaf is None: + with open(self.rules, "r") as f: + rules = json.load(f) + self._ddwaf = DDWaf(rules) + + @classmethod + def enable(cls, *args, **kwargs): + # type: (Any, Any) -> None + if cls._instance is not None: + return + + try: + processor = cls(*args, **kwargs) + except Exception: + log.warning("AppSec module failed to load.", exc_info=True) + if config._raise: + raise + return + + # Automatically enable the AppSec processor on the tracer + processor.tracer.on_start_span(processor._attach_web_span) + cls._instance = processor + cls.enabled = True + log.info("AppSec module is enabled.") + + @classmethod + def disable(cls): + # type: () -> None + if cls._instance is None: + return + # Will only disable AppSec for new spans + cls._instance.tracer.deregister_on_start_span(cls._instance._attach_web_span) + cls._instance = None + cls.enabled = False + + def _attach_web_span(self, span): + # type: (Span) -> None + if span.span_type is not None and span.span_type == "web": + # Insert ourself before the tracer's span processors + span._on_finish_callbacks.insert(0, self.on_span_finish) + + def on_span_finish(self, span): + # type: (Span) -> None + span.set_metric("_dd.appsec.enabled", 1.0) + data = { + "server.request.uri.raw": span.get_tag("http.url"), + "server.response.status": span.get_tag("http.status_code"), + } + res = self._ddwaf.run(data) + if res is not None: + span.set_tag("appsec.event", "true") + span.set_tag("_dd.appsec.json", '{"triggers":%s}' % (res,)) + span.set_tag(MANUAL_KEEP_KEY) diff --git a/ddtrace/appsec/rules.json b/ddtrace/appsec/rules.json new file mode 100644 index 00000000000..d685c7f4008 --- /dev/null +++ b/ddtrace/appsec/rules.json @@ -0,0 +1,5683 @@ +{ + "version": "2.1", + "rules": [ + { + "id": "crs-913-110", + "name": "Found request header associated with Acunetix security scanner", + "tags": { + "type": "security_scanner", + "crs_id": "913110", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies" + } + ], + "list": [ + "acunetix-product", + "(acunetix web vulnerability scanner", + "acunetix-scanning-agreement", + "acunetix-user-agreement" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "crs-913-120", + "name": "Found request filename/argument associated with security scanner", + "tags": { + "type": "security_scanner", + "crs_id": "913120", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + } + ], + "list": [ + "/.adsensepostnottherenonobook", + "/hello.html", + "/actsensepostnottherenonotive", + "/acunetix-wvs-test-for-some-inexistent-file", + "/antidisestablishmentarianism", + "/appscan_fingerprint/mac_address", + "/arachni-", + "/cybercop", + "/nessus_is_probing_you_", + "/nessustest", + "/netsparker-", + "/rfiinc.txt", + "/thereisnowaythat-you-canbethere", + "/w3af/remotefileinclude.html", + "appscan_fingerprint", + "w00tw00t.at.isc.sans.dfind", + "w00tw00t.at.blackhats.romanian.anti-sec" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "crs-920-260", + "name": "Unicode Full/Half Width Abuse Attack Attempt", + "tags": { + "type": "http_protocol_violation", + "crs_id": "920260", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + } + ], + "regex": "\\%u[fF]{2}[0-9a-fA-F]{2}", + "options": { + "case_sensitive": true, + "min_length": 6 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-921-110", + "name": "HTTP Request Smuggling Attack", + "tags": { + "type": "http_protocol_violation", + "crs_id": "921110", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + } + ], + "regex": "(?:get|post|head|options|connect|put|delete|trace|track|patch|propfind|propatch|mkcol|copy|move|lock|unlock)\\s+[^\\s]+\\s+http/\\d", + "options": { + "case_sensitive": true, + "min_length": 12 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "crs-921-140", + "name": "HTTP Header Injection Attack via headers", + "tags": { + "type": "http_protocol_violation", + "crs_id": "921140", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies" + } + ], + "regex": "[\\n\\r]", + "options": { + "case_sensitive": true, + "min_length": 1 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-921-160", + "name": "HTTP Header Injection Attack via payload (CR/LF and header-name detected)", + "tags": { + "type": "http_protocol_violation", + "crs_id": "921160", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.path_params" + } + ], + "regex": "[\\n\\r]+(?:\\s|location|refresh|(?:set-)?cookie|(?:x-)?(?:forwarded-(?:for|host|server)|host|via|remote-ip|remote-addr|originating-IP))\\s*:", + "options": { + "case_sensitive": true, + "min_length": 3 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "crs-930-100", + "name": "Obfuscated Path Traversal Attack (/../)", + "tags": { + "type": "lfi", + "crs_id": "930100", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + }, + { + "address": "server.request.headers.no_cookies" + } + ], + "regex": "(?:\\x5c|(?:%(?:c(?:0%(?:[2aq]f|5c|9v)|1%(?:[19p]c|8s|af))|2(?:5(?:c(?:0%25af|1%259c)|2f|5c)|%46|f)|(?:(?:f(?:8%8)?0%8|e)0%80%a|bg%q)f|%3(?:2(?:%(?:%6|4)6|F)|5%%63)|u(?:221[56]|002f|EFC8|F025)|1u|5c)|0x(?:2f|5c)|/))(?:%(?:(?:f(?:(?:c%80|8)%8)?0%8|e)0%80%ae|2(?:(?:5(?:c0%25a|2))?e|%45)|u(?:(?:002|ff0)e|2024)|%32(?:%(?:%6|4)5|E)|c0(?:%[256aef]e|\\.))|\\.(?:%0[01]|\\?)?|\\?\\.?|0x2e){2}(?:\\x5c|(?:%(?:c(?:0%(?:[2aq]f|5c|9v)|1%(?:[19p]c|8s|af))|2(?:5(?:c(?:0%25af|1%259c)|2f|5c)|%46|f)|(?:(?:f(?:8%8)?0%8|e)0%80%a|bg%q)f|%3(?:2(?:%(?:%6|4)6|F)|5%%63)|u(?:221[56]|002f|EFC8|F025)|1u|5c)|0x(?:2f|5c)|/))", + "options": { + "min_length": 4 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-930-110", + "name": "Simple Path Traversal Attack (/../)", + "tags": { + "type": "lfi", + "crs_id": "930110", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + }, + { + "address": "server.request.headers.no_cookies" + } + ], + "regex": "(?:(?:^|[\\\\/])\\.\\.[\\\\/]|[\\\\/]\\.\\.(?:[\\\\/]|$))", + "options": { + "case_sensitive": true, + "min_length": 3 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "removeNulls" + ] + }, + { + "id": "crs-930-120", + "name": "OS File Access Attempt", + "tags": { + "type": "lfi", + "crs_id": "930120", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "list": [ + ".htaccess", + ".htdigest", + ".htpasswd", + ".addressbook", + ".aptitude/config", + ".bash_config", + ".bash_history", + ".bash_logout", + ".bash_profile", + ".bashrc", + ".cache/notify-osd.log", + ".config/odesk/odesk team.conf", + ".cshrc", + ".dockerignore", + ".drush/", + ".eslintignore", + ".fbcindex", + ".forward", + ".git", + ".gitattributes", + ".gitconfig", + ".gnupg/", + ".hplip/hplip.conf", + ".ksh_history", + ".lesshst", + ".lftp/", + ".lhistory", + ".lldb-history", + ".local/share/mc/", + ".lynx_cookies", + ".my.cnf", + ".mysql_history", + ".nano_history", + ".node_repl_history", + ".pearrc", + ".php_history", + ".pinerc", + ".pki/", + ".proclog", + ".procmailrc", + ".psql_history", + ".python_history", + ".rediscli_history", + ".rhistory", + ".rhosts", + ".sh_history", + ".sqlite_history", + ".ssh/authorized_keys", + ".ssh/config", + ".ssh/id_dsa", + ".ssh/id_dsa.pub", + ".ssh/id_rsa", + ".ssh/id_rsa.pub", + ".ssh/identity", + ".ssh/identity.pub", + ".ssh/known_hosts", + ".subversion/auth", + ".subversion/config", + ".subversion/servers", + ".tconn/tconn.conf", + ".tcshrc", + ".vidalia/vidalia.conf", + ".viminfo", + ".vimrc", + ".www_acl", + ".wwwacl", + ".xauthority", + ".zhistory", + ".zshrc", + ".zsh_history", + ".nsconfig", + "etc/redis.conf", + "etc/redis-sentinel.conf", + "etc/php.ini", + "bin/php.ini", + "etc/httpd/php.ini", + "usr/lib/php.ini", + "usr/lib/php/php.ini", + "usr/local/etc/php.ini", + "usr/local/lib/php.ini", + "usr/local/php/lib/php.ini", + "usr/local/php4/lib/php.ini", + "usr/local/php5/lib/php.ini", + "usr/local/apache/conf/php.ini", + "etc/php4.4/fcgi/php.ini", + "etc/php4/apache/php.ini", + "etc/php4/apache2/php.ini", + "etc/php5/apache/php.ini", + "etc/php5/apache2/php.ini", + "etc/php/php.ini", + "etc/php/php4/php.ini", + "etc/php/apache/php.ini", + "etc/php/apache2/php.ini", + "web/conf/php.ini", + "usr/local/zend/etc/php.ini", + "opt/xampp/etc/php.ini", + "var/local/www/conf/php.ini", + "etc/php/cgi/php.ini", + "etc/php4/cgi/php.ini", + "etc/php5/cgi/php.ini", + "home2/bin/stable/apache/php.ini", + "home/bin/stable/apache/php.ini", + "etc/httpd/conf.d/php.conf", + "php5/php.ini", + "php4/php.ini", + "php/php.ini", + "windows/php.ini", + "winnt/php.ini", + "apache/php/php.ini", + "xampp/apache/bin/php.ini", + "netserver/bin/stable/apache/php.ini", + "volumes/macintosh_hd1/usr/local/php/lib/php.ini", + "etc/mono/1.0/machine.config", + "etc/mono/2.0/machine.config", + "etc/mono/2.0/web.config", + "etc/mono/config", + "usr/local/cpanel/logs/stats_log", + "usr/local/cpanel/logs/access_log", + "usr/local/cpanel/logs/error_log", + "usr/local/cpanel/logs/license_log", + "usr/local/cpanel/logs/login_log", + "var/cpanel/cpanel.config", + "var/log/sw-cp-server/error_log", + "usr/local/psa/admin/logs/httpsd_access_log", + "usr/local/psa/admin/logs/panel.log", + "var/log/sso/sso.log", + "usr/local/psa/admin/conf/php.ini", + "etc/sw-cp-server/applications.d/plesk.conf", + "usr/local/psa/admin/conf/site_isolation_settings.ini", + "usr/local/sb/config", + "etc/sw-cp-server/applications.d/00-sso-cpserver.conf", + "etc/sso/sso_config.ini", + "etc/mysql/conf.d/old_passwords.cnf", + "var/log/mysql/mysql-bin.log", + "var/log/mysql/mysql-bin.index", + "var/log/mysql/data/mysql-bin.index", + "var/log/mysql.log", + "var/log/mysql.err", + "var/log/mysqlderror.log", + "var/log/mysql/mysql.log", + "var/log/mysql/mysql-slow.log", + "var/log/mysql-bin.index", + "var/log/data/mysql-bin.index", + "var/mysql.log", + "var/mysql-bin.index", + "var/data/mysql-bin.index", + "program files/mysql/mysql server 5.0/data/{host}.err", + "program files/mysql/mysql server 5.0/data/mysql.log", + "program files/mysql/mysql server 5.0/data/mysql.err", + "program files/mysql/mysql server 5.0/data/mysql-bin.log", + "program files/mysql/mysql server 5.0/data/mysql-bin.index", + "program files/mysql/data/{host}.err", + "program files/mysql/data/mysql.log", + "program files/mysql/data/mysql.err", + "program files/mysql/data/mysql-bin.log", + "program files/mysql/data/mysql-bin.index", + "mysql/data/{host}.err", + "mysql/data/mysql.log", + "mysql/data/mysql.err", + "mysql/data/mysql-bin.log", + "mysql/data/mysql-bin.index", + "usr/local/mysql/data/mysql.log", + "usr/local/mysql/data/mysql.err", + "usr/local/mysql/data/mysql-bin.log", + "usr/local/mysql/data/mysql-slow.log", + "usr/local/mysql/data/mysqlderror.log", + "usr/local/mysql/data/{host}.err", + "usr/local/mysql/data/mysql-bin.index", + "var/lib/mysql/my.cnf", + "etc/mysql/my.cnf", + "etc/my.cnf", + "program files/mysql/mysql server 5.0/my.ini", + "program files/mysql/mysql server 5.0/my.cnf", + "program files/mysql/my.ini", + "program files/mysql/my.cnf", + "mysql/my.ini", + "mysql/my.cnf", + "mysql/bin/my.ini", + "var/postgresql/log/postgresql.log", + "var/log/postgresql/postgresql.log", + "var/log/postgres/pg_backup.log", + "var/log/postgres/postgres.log", + "var/log/postgresql.log", + "var/log/pgsql/pgsql.log", + "var/log/postgresql/postgresql-8.1-main.log", + "var/log/postgresql/postgresql-8.3-main.log", + "var/log/postgresql/postgresql-8.4-main.log", + "var/log/postgresql/postgresql-9.0-main.log", + "var/log/postgresql/postgresql-9.1-main.log", + "var/log/pgsql8.log", + "var/log/postgresql/postgres.log", + "var/log/pgsql_log", + "var/log/postgresql/main.log", + "var/log/cron/var/log/postgres.log", + "usr/internet/pgsql/data/postmaster.log", + "usr/local/pgsql/data/postgresql.log", + "usr/local/pgsql/data/pg_log", + "postgresql/log/pgadmin.log", + "var/lib/pgsql/data/postgresql.conf", + "var/postgresql/db/postgresql.conf", + "var/nm2/postgresql.conf", + "usr/local/pgsql/data/postgresql.conf", + "usr/local/pgsql/data/pg_hba.conf", + "usr/internet/pgsql/data/pg_hba.conf", + "usr/local/pgsql/data/passwd", + "usr/local/pgsql/bin/pg_passwd", + "etc/postgresql/postgresql.conf", + "etc/postgresql/pg_hba.conf", + "home/postgres/data/postgresql.conf", + "home/postgres/data/pg_version", + "home/postgres/data/pg_ident.conf", + "home/postgres/data/pg_hba.conf", + "program files/postgresql/8.3/data/pg_hba.conf", + "program files/postgresql/8.3/data/pg_ident.conf", + "program files/postgresql/8.3/data/postgresql.conf", + "program files/postgresql/8.4/data/pg_hba.conf", + "program files/postgresql/8.4/data/pg_ident.conf", + "program files/postgresql/8.4/data/postgresql.conf", + "program files/postgresql/9.0/data/pg_hba.conf", + "program files/postgresql/9.0/data/pg_ident.conf", + "program files/postgresql/9.0/data/postgresql.conf", + "program files/postgresql/9.1/data/pg_hba.conf", + "program files/postgresql/9.1/data/pg_ident.conf", + "program files/postgresql/9.1/data/postgresql.conf", + "wamp/logs/access.log", + "wamp/logs/apache_error.log", + "wamp/logs/genquery.log", + "wamp/logs/mysql.log", + "wamp/logs/slowquery.log", + "wamp/bin/apache/apache2.2.22/logs/access.log", + "wamp/bin/apache/apache2.2.22/logs/error.log", + "wamp/bin/apache/apache2.2.21/logs/access.log", + "wamp/bin/apache/apache2.2.21/logs/error.log", + "wamp/bin/mysql/mysql5.5.24/data/mysql-bin.index", + "wamp/bin/mysql/mysql5.5.16/data/mysql-bin.index", + "wamp/bin/apache/apache2.2.21/conf/httpd.conf", + "wamp/bin/apache/apache2.2.22/conf/httpd.conf", + "wamp/bin/apache/apache2.2.21/wampserver.conf", + "wamp/bin/apache/apache2.2.22/wampserver.conf", + "wamp/bin/apache/apache2.2.22/conf/wampserver.conf", + "wamp/bin/mysql/mysql5.5.24/my.ini", + "wamp/bin/mysql/mysql5.5.24/wampserver.conf", + "wamp/bin/mysql/mysql5.5.16/my.ini", + "wamp/bin/mysql/mysql5.5.16/wampserver.conf", + "wamp/bin/php/php5.3.8/php.ini", + "wamp/bin/php/php5.4.3/php.ini", + "xampp/apache/logs/access.log", + "xampp/apache/logs/error.log", + "xampp/mysql/data/mysql-bin.index", + "xampp/mysql/data/mysql.err", + "xampp/mysql/data/{host}.err", + "xampp/sendmail/sendmail.log", + "xampp/apache/conf/httpd.conf", + "xampp/filezillaftp/filezilla server.xml", + "xampp/mercurymail/mercury.ini", + "xampp/php/php.ini", + "xampp/phpmyadmin/config.inc.php", + "xampp/sendmail/sendmail.ini", + "xampp/webalizer/webalizer.conf", + "opt/lampp/etc/httpd.conf", + "xampp/htdocs/aca.txt", + "xampp/htdocs/admin.php", + "xampp/htdocs/leer.txt", + "usr/local/apache/logs/audit_log", + "usr/local/apache2/logs/audit_log", + "logs/security_debug_log", + "logs/security_log", + "usr/local/apache/conf/modsec.conf", + "usr/local/apache2/conf/modsec.conf", + "winnt/system32/logfiles/msftpsvc", + "winnt/system32/logfiles/msftpsvc1", + "winnt/system32/logfiles/msftpsvc2", + "windows/system32/logfiles/msftpsvc", + "windows/system32/logfiles/msftpsvc1", + "windows/system32/logfiles/msftpsvc2", + "etc/logrotate.d/proftpd", + "www/logs/proftpd.system.log", + "var/log/proftpd", + "var/log/proftpd/xferlog.legacy", + "var/log/proftpd.access_log", + "var/log/proftpd.xferlog", + "etc/pam.d/proftpd", + "etc/proftp.conf", + "etc/protpd/proftpd.conf", + "etc/vhcs2/proftpd/proftpd.conf", + "etc/proftpd/modules.conf", + "var/log/vsftpd.log", + "etc/vsftpd.chroot_list", + "etc/logrotate.d/vsftpd.log", + "etc/vsftpd/vsftpd.conf", + "etc/vsftpd.conf", + "etc/chrootusers", + "var/log/xferlog", + "var/adm/log/xferlog", + "etc/wu-ftpd/ftpaccess", + "etc/wu-ftpd/ftphosts", + "etc/wu-ftpd/ftpusers", + "var/log/pure-ftpd/pure-ftpd.log", + "logs/pure-ftpd.log", + "var/log/pureftpd.log", + "usr/sbin/pure-config.pl", + "usr/etc/pure-ftpd.conf", + "etc/pure-ftpd/pure-ftpd.conf", + "usr/local/etc/pure-ftpd.conf", + "usr/local/etc/pureftpd.pdb", + "usr/local/pureftpd/etc/pureftpd.pdb", + "usr/local/pureftpd/sbin/pure-config.pl", + "usr/local/pureftpd/etc/pure-ftpd.conf", + "etc/pure-ftpd.conf", + "etc/pure-ftpd/pure-ftpd.pdb", + "etc/pureftpd.pdb", + "etc/pureftpd.passwd", + "etc/pure-ftpd/pureftpd.pdb", + "usr/ports/ftp/pure-ftpd/pure-ftpd.conf", + "usr/ports/ftp/pure-ftpd/pureftpd.pdb", + "usr/ports/ftp/pure-ftpd/pureftpd.passwd", + "usr/ports/net/pure-ftpd/pure-ftpd.conf", + "usr/ports/net/pure-ftpd/pureftpd.pdb", + "usr/ports/net/pure-ftpd/pureftpd.passwd", + "usr/pkgsrc/net/pureftpd/pure-ftpd.conf", + "usr/pkgsrc/net/pureftpd/pureftpd.pdb", + "usr/pkgsrc/net/pureftpd/pureftpd.passwd", + "usr/ports/contrib/pure-ftpd/pure-ftpd.conf", + "usr/ports/contrib/pure-ftpd/pureftpd.pdb", + "usr/ports/contrib/pure-ftpd/pureftpd.passwd", + "var/log/muddleftpd", + "usr/sbin/mudlogd", + "etc/muddleftpd/mudlog", + "etc/muddleftpd.com", + "etc/muddleftpd/mudlogd.conf", + "etc/muddleftpd/muddleftpd.conf", + "var/log/muddleftpd.conf", + "usr/sbin/mudpasswd", + "etc/muddleftpd/muddleftpd.passwd", + "etc/muddleftpd/passwd", + "var/log/ftp-proxy/ftp-proxy.log", + "var/log/ftp-proxy", + "var/log/ftplog", + "etc/logrotate.d/ftp", + "etc/ftpchroot", + "etc/ftphosts", + "etc/ftpusers", + "var/log/exim_mainlog", + "var/log/exim/mainlog", + "var/log/maillog", + "var/log/exim_paniclog", + "var/log/exim/paniclog", + "var/log/exim/rejectlog", + "var/log/exim_rejectlog", + "winnt/system32/logfiles/smtpsvc", + "winnt/system32/logfiles/smtpsvc1", + "winnt/system32/logfiles/smtpsvc2", + "winnt/system32/logfiles/smtpsvc3", + "winnt/system32/logfiles/smtpsvc4", + "winnt/system32/logfiles/smtpsvc5", + "windows/system32/logfiles/smtpsvc", + "windows/system32/logfiles/smtpsvc1", + "windows/system32/logfiles/smtpsvc2", + "windows/system32/logfiles/smtpsvc3", + "windows/system32/logfiles/smtpsvc4", + "windows/system32/logfiles/smtpsvc5", + "etc/osxhttpd/osxhttpd.conf", + "system/library/webobjects/adaptors/apache2.2/apache.conf", + "etc/apache2/sites-available/default", + "etc/apache2/sites-available/default-ssl", + "etc/apache2/sites-enabled/000-default", + "etc/apache2/sites-enabled/default", + "etc/apache2/apache2.conf", + "etc/apache2/ports.conf", + "usr/local/etc/apache/httpd.conf", + "usr/pkg/etc/httpd/httpd.conf", + "usr/pkg/etc/httpd/httpd-default.conf", + "usr/pkg/etc/httpd/httpd-vhosts.conf", + "etc/httpd/mod_php.conf", + "etc/httpd/extra/httpd-ssl.conf", + "etc/rc.d/rc.httpd", + "usr/local/apache/conf/httpd.conf.default", + "usr/local/apache/conf/access.conf", + "usr/local/apache22/conf/httpd.conf", + "usr/local/apache22/httpd.conf", + "usr/local/etc/apache22/conf/httpd.conf", + "usr/local/apps/apache22/conf/httpd.conf", + "etc/apache22/conf/httpd.conf", + "etc/apache22/httpd.conf", + "opt/apache22/conf/httpd.conf", + "usr/local/etc/apache2/vhosts.conf", + "usr/local/apache/conf/vhosts.conf", + "usr/local/apache2/conf/vhosts.conf", + "usr/local/apache/conf/vhosts-custom.conf", + "usr/local/apache2/conf/vhosts-custom.conf", + "etc/apache/default-server.conf", + "etc/apache2/default-server.conf", + "usr/local/apache2/conf/extra/httpd-ssl.conf", + "usr/local/apache2/conf/ssl.conf", + "etc/httpd/conf.d", + "usr/local/etc/apache22/httpd.conf", + "usr/local/etc/apache2/httpd.conf", + "etc/apache2/httpd2.conf", + "etc/apache2/ssl-global.conf", + "etc/apache2/vhosts.d/00_default_vhost.conf", + "apache/conf/httpd.conf", + "etc/apache/httpd.conf", + "etc/httpd/conf", + "http/httpd.conf", + "usr/local/apache1.3/conf/httpd.conf", + "usr/local/etc/httpd/conf", + "var/apache/conf/httpd.conf", + "var/www/conf", + "www/apache/conf/httpd.conf", + "www/conf/httpd.conf", + "etc/init.d", + "etc/apache/access.conf", + "etc/rc.conf", + "www/logs/freebsddiary-error.log", + "www/logs/freebsddiary-access_log", + "library/webserver/documents/index.html", + "library/webserver/documents/index.htm", + "library/webserver/documents/default.html", + "library/webserver/documents/default.htm", + "library/webserver/documents/index.php", + "library/webserver/documents/default.php", + "var/log/webmin/miniserv.log", + "usr/local/etc/webmin/miniserv.conf", + "etc/webmin/miniserv.conf", + "usr/local/etc/webmin/miniserv.users", + "etc/webmin/miniserv.users", + "winnt/system32/logfiles/w3svc/inetsvn1.log", + "winnt/system32/logfiles/w3svc1/inetsvn1.log", + "winnt/system32/logfiles/w3svc2/inetsvn1.log", + "winnt/system32/logfiles/w3svc3/inetsvn1.log", + "windows/system32/logfiles/w3svc/inetsvn1.log", + "windows/system32/logfiles/w3svc1/inetsvn1.log", + "windows/system32/logfiles/w3svc2/inetsvn1.log", + "windows/system32/logfiles/w3svc3/inetsvn1.log", + "var/log/httpd/access_log", + "var/log/httpd/error_log", + "apache/logs/error.log", + "apache/logs/access.log", + "apache2/logs/error.log", + "apache2/logs/access.log", + "logs/error.log", + "logs/access.log", + "etc/httpd/logs/access_log", + "etc/httpd/logs/access.log", + "etc/httpd/logs/error_log", + "etc/httpd/logs/error.log", + "usr/local/apache/logs/access_log", + "usr/local/apache/logs/access.log", + "usr/local/apache/logs/error_log", + "usr/local/apache/logs/error.log", + "usr/local/apache2/logs/access_log", + "usr/local/apache2/logs/access.log", + "usr/local/apache2/logs/error_log", + "usr/local/apache2/logs/error.log", + "var/www/logs/access_log", + "var/www/logs/access.log", + "var/www/logs/error_log", + "var/www/logs/error.log", + "var/log/httpd/access.log", + "var/log/httpd/error.log", + "var/log/apache/access_log", + "var/log/apache/access.log", + "var/log/apache/error_log", + "var/log/apache/error.log", + "var/log/apache2/access_log", + "var/log/apache2/access.log", + "var/log/apache2/error_log", + "var/log/apache2/error.log", + "var/log/access_log", + "var/log/access.log", + "var/log/error_log", + "var/log/error.log", + "opt/lampp/logs/access_log", + "opt/lampp/logs/error_log", + "opt/xampp/logs/access_log", + "opt/xampp/logs/error_log", + "opt/lampp/logs/access.log", + "opt/lampp/logs/error.log", + "opt/xampp/logs/access.log", + "opt/xampp/logs/error.log", + "program files/apache group/apache/logs/access.log", + "program files/apache group/apache/logs/error.log", + "program files/apache software foundation/apache2.2/logs/error.log", + "program files/apache software foundation/apache2.2/logs/access.log", + "opt/apache/apache.conf", + "opt/apache/conf/apache.conf", + "opt/apache2/apache.conf", + "opt/apache2/conf/apache.conf", + "opt/httpd/apache.conf", + "opt/httpd/conf/apache.conf", + "etc/httpd/apache.conf", + "etc/apache2/apache.conf", + "etc/httpd/conf/apache.conf", + "usr/local/apache/apache.conf", + "usr/local/apache/conf/apache.conf", + "usr/local/apache2/apache.conf", + "usr/local/apache2/conf/apache.conf", + "usr/local/php/apache.conf.php", + "usr/local/php4/apache.conf.php", + "usr/local/php5/apache.conf.php", + "usr/local/php/apache.conf", + "usr/local/php4/apache.conf", + "usr/local/php5/apache.conf", + "private/etc/httpd/apache.conf", + "opt/apache/apache2.conf", + "opt/apache/conf/apache2.conf", + "opt/apache2/apache2.conf", + "opt/apache2/conf/apache2.conf", + "opt/httpd/apache2.conf", + "opt/httpd/conf/apache2.conf", + "etc/httpd/apache2.conf", + "etc/httpd/conf/apache2.conf", + "usr/local/apache/apache2.conf", + "usr/local/apache/conf/apache2.conf", + "usr/local/apache2/apache2.conf", + "usr/local/apache2/conf/apache2.conf", + "usr/local/php/apache2.conf.php", + "usr/local/php4/apache2.conf.php", + "usr/local/php5/apache2.conf.php", + "usr/local/php/apache2.conf", + "usr/local/php4/apache2.conf", + "usr/local/php5/apache2.conf", + "private/etc/httpd/apache2.conf", + "usr/local/apache/conf/httpd.conf", + "usr/local/apache2/conf/httpd.conf", + "etc/httpd/conf/httpd.conf", + "etc/apache/apache.conf", + "etc/apache/conf/httpd.conf", + "etc/apache2/httpd.conf", + "usr/apache2/conf/httpd.conf", + "usr/apache/conf/httpd.conf", + "usr/local/etc/apache/conf/httpd.conf", + "usr/local/apache/httpd.conf", + "usr/local/apache2/httpd.conf", + "usr/local/httpd/conf/httpd.conf", + "usr/local/etc/apache2/conf/httpd.conf", + "usr/local/etc/httpd/conf/httpd.conf", + "usr/local/apps/apache2/conf/httpd.conf", + "usr/local/apps/apache/conf/httpd.conf", + "usr/local/php/httpd.conf.php", + "usr/local/php4/httpd.conf.php", + "usr/local/php5/httpd.conf.php", + "usr/local/php/httpd.conf", + "usr/local/php4/httpd.conf", + "usr/local/php5/httpd.conf", + "etc/apache2/conf/httpd.conf", + "etc/http/conf/httpd.conf", + "etc/httpd/httpd.conf", + "etc/http/httpd.conf", + "etc/httpd.conf", + "opt/apache/conf/httpd.conf", + "opt/apache2/conf/httpd.conf", + "var/www/conf/httpd.conf", + "private/etc/httpd/httpd.conf", + "private/etc/httpd/httpd.conf.default", + "etc/apache2/vhosts.d/default_vhost.include", + "etc/apache2/conf.d/charset", + "etc/apache2/conf.d/security", + "etc/apache2/envvars", + "etc/apache2/mods-available/autoindex.conf", + "etc/apache2/mods-available/deflate.conf", + "etc/apache2/mods-available/dir.conf", + "etc/apache2/mods-available/mem_cache.conf", + "etc/apache2/mods-available/mime.conf", + "etc/apache2/mods-available/proxy.conf", + "etc/apache2/mods-available/setenvif.conf", + "etc/apache2/mods-available/ssl.conf", + "etc/apache2/mods-enabled/alias.conf", + "etc/apache2/mods-enabled/deflate.conf", + "etc/apache2/mods-enabled/dir.conf", + "etc/apache2/mods-enabled/mime.conf", + "etc/apache2/mods-enabled/negotiation.conf", + "etc/apache2/mods-enabled/php5.conf", + "etc/apache2/mods-enabled/status.conf", + "program files/apache group/apache/conf/httpd.conf", + "program files/apache group/apache2/conf/httpd.conf", + "program files/xampp/apache/conf/apache.conf", + "program files/xampp/apache/conf/apache2.conf", + "program files/xampp/apache/conf/httpd.conf", + "program files/apache group/apache/apache.conf", + "program files/apache group/apache/conf/apache.conf", + "program files/apache group/apache2/conf/apache.conf", + "program files/apache group/apache/apache2.conf", + "program files/apache group/apache/conf/apache2.conf", + "program files/apache group/apache2/conf/apache2.conf", + "program files/apache software foundation/apache2.2/conf/httpd.conf", + "volumes/macintosh_hd1/opt/httpd/conf/httpd.conf", + "volumes/macintosh_hd1/opt/apache/conf/httpd.conf", + "volumes/macintosh_hd1/opt/apache2/conf/httpd.conf", + "volumes/macintosh_hd1/usr/local/php/httpd.conf.php", + "volumes/macintosh_hd1/usr/local/php4/httpd.conf.php", + "volumes/macintosh_hd1/usr/local/php5/httpd.conf.php", + "volumes/webbackup/opt/apache2/conf/httpd.conf", + "volumes/webbackup/private/etc/httpd/httpd.conf", + "volumes/webbackup/private/etc/httpd/httpd.conf.default", + "usr/local/etc/apache/vhosts.conf", + "usr/local/jakarta/tomcat/conf/jakarta.conf", + "usr/local/jakarta/tomcat/conf/server.xml", + "usr/local/jakarta/tomcat/conf/context.xml", + "usr/local/jakarta/tomcat/conf/workers.properties", + "usr/local/jakarta/tomcat/conf/logging.properties", + "usr/local/jakarta/dist/tomcat/conf/jakarta.conf", + "usr/local/jakarta/dist/tomcat/conf/server.xml", + "usr/local/jakarta/dist/tomcat/conf/context.xml", + "usr/local/jakarta/dist/tomcat/conf/workers.properties", + "usr/local/jakarta/dist/tomcat/conf/logging.properties", + "usr/share/tomcat6/conf/server.xml", + "usr/share/tomcat6/conf/context.xml", + "usr/share/tomcat6/conf/workers.properties", + "usr/share/tomcat6/conf/logging.properties", + "var/log/tomcat6/catalina.out", + "var/cpanel/tomcat.options", + "usr/local/jakarta/tomcat/logs/catalina.out", + "usr/local/jakarta/tomcat/logs/catalina.err", + "opt/tomcat/logs/catalina.out", + "opt/tomcat/logs/catalina.err", + "usr/share/logs/catalina.out", + "usr/share/logs/catalina.err", + "usr/share/tomcat/logs/catalina.out", + "usr/share/tomcat/logs/catalina.err", + "usr/share/tomcat6/logs/catalina.out", + "usr/share/tomcat6/logs/catalina.err", + "usr/local/apache/logs/mod_jk.log", + "usr/local/jakarta/tomcat/logs/mod_jk.log", + "usr/local/jakarta/dist/tomcat/logs/mod_jk.log", + "opt/[jboss]/server/default/conf/jboss-minimal.xml", + "opt/[jboss]/server/default/conf/jboss-service.xml", + "opt/[jboss]/server/default/conf/jndi.properties", + "opt/[jboss]/server/default/conf/log4j.xml", + "opt/[jboss]/server/default/conf/login-config.xml", + "opt/[jboss]/server/default/conf/standardjaws.xml", + "opt/[jboss]/server/default/conf/standardjboss.xml", + "opt/[jboss]/server/default/conf/server.log.properties", + "opt/[jboss]/server/default/deploy/jboss-logging.xml", + "usr/local/[jboss]/server/default/conf/jboss-minimal.xml", + "usr/local/[jboss]/server/default/conf/jboss-service.xml", + "usr/local/[jboss]/server/default/conf/jndi.properties", + "usr/local/[jboss]/server/default/conf/log4j.xml", + "usr/local/[jboss]/server/default/conf/login-config.xml", + "usr/local/[jboss]/server/default/conf/standardjaws.xml", + "usr/local/[jboss]/server/default/conf/standardjboss.xml", + "usr/local/[jboss]/server/default/conf/server.log.properties", + "usr/local/[jboss]/server/default/deploy/jboss-logging.xml", + "private/tmp/[jboss]/server/default/conf/jboss-minimal.xml", + "private/tmp/[jboss]/server/default/conf/jboss-service.xml", + "private/tmp/[jboss]/server/default/conf/jndi.properties", + "private/tmp/[jboss]/server/default/conf/log4j.xml", + "private/tmp/[jboss]/server/default/conf/login-config.xml", + "private/tmp/[jboss]/server/default/conf/standardjaws.xml", + "private/tmp/[jboss]/server/default/conf/standardjboss.xml", + "private/tmp/[jboss]/server/default/conf/server.log.properties", + "private/tmp/[jboss]/server/default/deploy/jboss-logging.xml", + "tmp/[jboss]/server/default/conf/jboss-minimal.xml", + "tmp/[jboss]/server/default/conf/jboss-service.xml", + "tmp/[jboss]/server/default/conf/jndi.properties", + "tmp/[jboss]/server/default/conf/log4j.xml", + "tmp/[jboss]/server/default/conf/login-config.xml", + "tmp/[jboss]/server/default/conf/standardjaws.xml", + "tmp/[jboss]/server/default/conf/standardjboss.xml", + "tmp/[jboss]/server/default/conf/server.log.properties", + "tmp/[jboss]/server/default/deploy/jboss-logging.xml", + "program files/[jboss]/server/default/conf/jboss-minimal.xml", + "program files/[jboss]/server/default/conf/jboss-service.xml", + "program files/[jboss]/server/default/conf/jndi.properties", + "program files/[jboss]/server/default/conf/log4j.xml", + "program files/[jboss]/server/default/conf/login-config.xml", + "program files/[jboss]/server/default/conf/standardjaws.xml", + "program files/[jboss]/server/default/conf/standardjboss.xml", + "program files/[jboss]/server/default/conf/server.log.properties", + "program files/[jboss]/server/default/deploy/jboss-logging.xml", + "[jboss]/server/default/conf/jboss-minimal.xml", + "[jboss]/server/default/conf/jboss-service.xml", + "[jboss]/server/default/conf/jndi.properties", + "[jboss]/server/default/conf/log4j.xml", + "[jboss]/server/default/conf/login-config.xml", + "[jboss]/server/default/conf/standardjaws.xml", + "[jboss]/server/default/conf/standardjboss.xml", + "[jboss]/server/default/conf/server.log.properties", + "[jboss]/server/default/deploy/jboss-logging.xml", + "opt/[jboss]/server/default/log/server.log", + "opt/[jboss]/server/default/log/boot.log", + "usr/local/[jboss]/server/default/log/server.log", + "usr/local/[jboss]/server/default/log/boot.log", + "private/tmp/[jboss]/server/default/log/server.log", + "private/tmp/[jboss]/server/default/log/boot.log", + "tmp/[jboss]/server/default/log/server.log", + "tmp/[jboss]/server/default/log/boot.log", + "program files/[jboss]/server/default/log/server.log", + "program files/[jboss]/server/default/log/boot.log", + "[jboss]/server/default/log/server.log", + "[jboss]/server/default/log/boot.log", + "var/log/lighttpd.error.log", + "var/log/lighttpd.access.log", + "var/lighttpd.log", + "var/logs/access.log", + "var/log/lighttpd/", + "var/log/lighttpd/error.log", + "var/log/lighttpd/access.www.log", + "var/log/lighttpd/error.www.log", + "var/log/lighttpd/access.log", + "usr/local/apache2/logs/lighttpd.error.log", + "usr/local/apache2/logs/lighttpd.log", + "usr/local/apache/logs/lighttpd.error.log", + "usr/local/apache/logs/lighttpd.log", + "usr/local/lighttpd/log/lighttpd.error.log", + "usr/local/lighttpd/log/access.log", + "var/log/lighttpd/{domain}/access.log", + "var/log/lighttpd/{domain}/error.log", + "usr/home/user/var/log/lighttpd.error.log", + "usr/home/user/var/log/apache.log", + "home/user/lighttpd/lighttpd.conf", + "usr/home/user/lighttpd/lighttpd.conf", + "etc/lighttpd/lighthttpd.conf", + "usr/local/etc/lighttpd.conf", + "usr/local/lighttpd/conf/lighttpd.conf", + "usr/local/etc/lighttpd.conf.new", + "var/www/.lighttpdpassword", + "var/log/nginx/access_log", + "var/log/nginx/error_log", + "var/log/nginx/access.log", + "var/log/nginx/error.log", + "var/log/nginx.access_log", + "var/log/nginx.error_log", + "logs/access_log", + "logs/error_log", + "etc/nginx/nginx.conf", + "usr/local/etc/nginx/nginx.conf", + "usr/local/nginx/conf/nginx.conf", + "usr/local/zeus/web/global.cfg", + "usr/local/zeus/web/log/errors", + "opt/lsws/conf/httpd_conf.xml", + "usr/local/lsws/conf/httpd_conf.xml", + "opt/lsws/logs/error.log", + "opt/lsws/logs/access.log", + "usr/local/lsws/logs/error.log", + "usr/local/logs/access.log", + "usr/local/samba/lib/log.user", + "usr/local/logs/samba.log", + "var/log/samba/log.smbd", + "var/log/samba/log.nmbd", + "var/log/samba.log", + "var/log/samba.log1", + "var/log/samba.log2", + "var/log/log.smb", + "etc/samba/netlogon", + "etc/smbpasswd", + "etc/smb.conf", + "etc/samba/dhcp.conf", + "etc/samba/smb.conf", + "etc/samba/samba.conf", + "etc/samba/smb.conf.user", + "etc/samba/smbpasswd", + "etc/samba/smbusers", + "etc/samba/private/smbpasswd", + "usr/local/etc/smb.conf", + "usr/local/samba/lib/smb.conf.user", + "etc/dhcp3/dhclient.conf", + "etc/dhcp3/dhcpd.conf", + "etc/dhcp/dhclient.conf", + "program files/vidalia bundle/polipo/polipo.conf", + "etc/tor/tor-tsocks.conf", + "etc/stunnel/stunnel.conf", + "etc/tsocks.conf", + "etc/tinyproxy/tinyproxy.conf", + "etc/miredo-server.conf", + "etc/miredo.conf", + "etc/miredo/miredo-server.conf", + "etc/miredo/miredo.conf", + "etc/wicd/dhclient.conf.template.default", + "etc/wicd/manager-settings.conf", + "etc/wicd/wired-settings.conf", + "etc/wicd/wireless-settings.conf", + "var/log/ipfw.log", + "var/log/ipfw", + "var/log/ipfw/ipfw.log", + "var/log/ipfw.today", + "etc/ipfw.rules", + "etc/ipfw.conf", + "etc/firewall.rules", + "winnt/system32/logfiles/firewall/pfirewall.log", + "winnt/system32/logfiles/firewall/pfirewall.log.old", + "windows/system32/logfiles/firewall/pfirewall.log", + "windows/system32/logfiles/firewall/pfirewall.log.old", + "etc/clamav/clamd.conf", + "etc/clamav/freshclam.conf", + "etc/x11/xorg.conf", + "etc/x11/xorg.conf-vesa", + "etc/x11/xorg.conf-vmware", + "etc/x11/xorg.conf.beforevmwaretoolsinstall", + "etc/x11/xorg.conf.orig", + "etc/bluetooth/input.conf", + "etc/bluetooth/main.conf", + "etc/bluetooth/network.conf", + "etc/bluetooth/rfcomm.conf", + "proc/self/environ", + "proc/self/mounts", + "proc/self/stat", + "proc/self/status", + "proc/self/cmdline", + "proc/self/fd/0", + "proc/self/fd/1", + "proc/self/fd/2", + "proc/self/fd/3", + "proc/self/fd/4", + "proc/self/fd/5", + "proc/self/fd/6", + "proc/self/fd/7", + "proc/self/fd/8", + "proc/self/fd/9", + "proc/self/fd/10", + "proc/self/fd/11", + "proc/self/fd/12", + "proc/self/fd/13", + "proc/self/fd/14", + "proc/self/fd/15", + "proc/version", + "proc/devices", + "proc/cpuinfo", + "proc/meminfo", + "proc/net/tcp", + "proc/net/udp", + "etc/bash_completion.d/debconf", + "root/.bash_logout", + "root/.bash_history", + "root/.bash_config", + "root/.bashrc", + "etc/bash.bashrc", + "var/adm/syslog", + "var/adm/sulog", + "var/adm/utmp", + "var/adm/utmpx", + "var/adm/wtmp", + "var/adm/wtmpx", + "var/adm/lastlog/username", + "usr/spool/lp/log", + "var/adm/lp/lpd-errs", + "usr/lib/cron/log", + "var/adm/loginlog", + "var/adm/pacct", + "var/adm/dtmp", + "var/adm/acct/sum/loginlog", + "var/adm/x0msgs", + "var/adm/crash/vmcore", + "var/adm/crash/unix", + "etc/newsyslog.conf", + "var/adm/qacct", + "var/adm/ras/errlog", + "var/adm/ras/bootlog", + "var/adm/cron/log", + "etc/utmp", + "etc/security/lastlog", + "etc/security/failedlogin", + "usr/spool/mqueue/syslog", + "var/adm/messages", + "var/adm/aculogs", + "var/adm/aculog", + "var/adm/vold.log", + "var/adm/log/asppp.log", + "var/log/poplog", + "var/log/authlog", + "var/lp/logs/lpsched", + "var/lp/logs/lpnet", + "var/lp/logs/requests", + "var/cron/log", + "var/saf/_log", + "var/saf/port/log", + "var/log/news.all", + "var/log/news/news.all", + "var/log/news/news.crit", + "var/log/news/news.err", + "var/log/news/news.notice", + "var/log/news/suck.err", + "var/log/news/suck.notice", + "var/log/messages", + "var/log/messages.1", + "var/log/user.log", + "var/log/user.log.1", + "var/log/auth.log", + "var/log/pm-powersave.log", + "var/log/xorg.0.log", + "var/log/daemon.log", + "var/log/daemon.log.1", + "var/log/kern.log", + "var/log/kern.log.1", + "var/log/mail.err", + "var/log/mail.info", + "var/log/mail.warn", + "var/log/ufw.log", + "var/log/boot.log", + "var/log/syslog", + "var/log/syslog.1", + "tmp/access.log", + "etc/sensors.conf", + "etc/sensors3.conf", + "etc/host.conf", + "etc/pam.conf", + "etc/resolv.conf", + "etc/apt/apt.conf", + "etc/inetd.conf", + "etc/syslog.conf", + "etc/sysctl.conf", + "etc/sysctl.d/10-console-messages.conf", + "etc/sysctl.d/10-network-security.conf", + "etc/sysctl.d/10-process-security.conf", + "etc/sysctl.d/wine.sysctl.conf", + "etc/security/access.conf", + "etc/security/group.conf", + "etc/security/limits.conf", + "etc/security/namespace.conf", + "etc/security/pam_env.conf", + "etc/security/sepermit.conf", + "etc/security/time.conf", + "etc/ssh/sshd_config", + "etc/adduser.conf", + "etc/deluser.conf", + "etc/avahi/avahi-daemon.conf", + "etc/ca-certificates.conf", + "etc/ca-certificates.conf.dpkg-old", + "etc/casper.conf", + "etc/chkrootkit.conf", + "etc/debconf.conf", + "etc/dns2tcpd.conf", + "etc/e2fsck.conf", + "etc/esound/esd.conf", + "etc/etter.conf", + "etc/fuse.conf", + "etc/foremost.conf", + "etc/hdparm.conf", + "etc/kernel-img.conf", + "etc/kernel-pkg.conf", + "etc/ld.so.conf", + "etc/ltrace.conf", + "etc/mail/sendmail.conf", + "etc/manpath.config", + "etc/kbd/config", + "etc/ldap/ldap.conf", + "etc/logrotate.conf", + "etc/mtools.conf", + "etc/smi.conf", + "etc/updatedb.conf", + "etc/pulse/client.conf", + "usr/share/adduser/adduser.conf", + "etc/hostname", + "etc/networks", + "etc/timezone", + "etc/modules", + "etc/passwd", + "etc/passwd~", + "etc/passwd-", + "etc/shadow", + "etc/shadow~", + "etc/shadow-", + "etc/fstab", + "etc/motd", + "etc/hosts", + "etc/group", + "etc/group-", + "etc/alias", + "etc/crontab", + "etc/crypttab", + "etc/exports", + "etc/mtab", + "etc/hosts.allow", + "etc/hosts.deny", + "etc/os-release", + "etc/password.master", + "etc/profile", + "etc/default/grub", + "etc/resolvconf/update-libc.d/sendmail", + "etc/inittab", + "etc/issue", + "etc/issue.net", + "etc/login.defs", + "etc/sudoers", + "etc/sysconfig/network-scripts/ifcfg-eth0", + "etc/redhat-release", + "etc/debian_version", + "etc/fedora-release", + "etc/mandrake-release", + "etc/slackware-release", + "etc/suse-release", + "etc/security/group", + "etc/security/passwd", + "etc/security/user", + "etc/security/environ", + "etc/security/limits", + "etc/security/opasswd", + "boot/grub/grub.cfg", + "boot/grub/menu.lst", + "root/.ksh_history", + "root/.xauthority", + "usr/lib/security/mkuser.default", + "var/log/squirrelmail.log", + "var/log/apache2/squirrelmail.log", + "var/log/apache2/squirrelmail.err.log", + "var/lib/squirrelmail/prefs/squirrelmail.log", + "var/log/mail.log", + "etc/squirrelmail/apache.conf", + "etc/squirrelmail/config_local.php", + "etc/squirrelmail/default_pref", + "etc/squirrelmail/index.php", + "etc/squirrelmail/config_default.php", + "etc/squirrelmail/config.php", + "etc/squirrelmail/filters_setup.php", + "etc/squirrelmail/sqspell_config.php", + "etc/squirrelmail/config/config.php", + "etc/httpd/conf.d/squirrelmail.conf", + "usr/share/squirrelmail/config/config.php", + "private/etc/squirrelmail/config/config.php", + "srv/www/htdos/squirrelmail/config/config.php", + "var/www/squirrelmail/config/config.php", + "var/www/html/squirrelmail/config/config.php", + "var/www/html/squirrelmail-1.2.9/config/config.php", + "usr/share/squirrelmail/plugins/squirrel_logger/setup.php", + "usr/local/squirrelmail/www/readme", + "windows/system32/drivers/etc/hosts", + "windows/system32/drivers/etc/lmhosts.sam", + "windows/system32/drivers/etc/networks", + "windows/system32/drivers/etc/protocol", + "windows/system32/drivers/etc/services", + "/boot.ini", + "windows/debug/netsetup.log", + "windows/comsetup.log", + "windows/repair/setup.log", + "windows/setupact.log", + "windows/setupapi.log", + "windows/setuperr.log", + "windows/updspapi.log", + "windows/wmsetup.log", + "windows/windowsupdate.log", + "windows/odbc.ini", + "usr/local/psa/admin/htdocs/domains/databases/phpmyadmin/libraries/config.default.php", + "etc/apache2/conf.d/phpmyadmin.conf", + "etc/phpmyadmin/config.inc.php", + "etc/openldap/ldap.conf", + "etc/cups/acroread.conf", + "etc/cups/cupsd.conf", + "etc/cups/cupsd.conf.default", + "etc/cups/pdftops.conf", + "etc/cups/printers.conf", + "windows/system32/macromed/flash/flashinstall.log", + "windows/system32/macromed/flash/install.log", + "etc/cvs-cron.conf", + "etc/cvs-pserver.conf", + "etc/subversion/config", + "etc/modprobe.d/vmware-tools.conf", + "etc/updatedb.conf.beforevmwaretoolsinstall", + "etc/vmware-tools/config", + "etc/vmware-tools/tpvmlp.conf", + "etc/vmware-tools/vmware-tools-libraries.conf", + "var/log/vmware/hostd.log", + "var/log/vmware/hostd-1.log", + "wp-config.php", + "wp-config.bak", + "wp-config.old", + "wp-config.temp", + "wp-config.tmp", + "wp-config.txt", + "config.yml", + "config_dev.yml", + "config_prod.yml", + "config_test.yml", + "parameters.yml", + "routing.yml", + "security.yml", + "services.yml", + "sites/default/default.settings.php", + "sites/default/settings.php", + "sites/default/settings.local.php", + "app/etc/local.xml", + "sftp-config.json", + "web.config", + "includes/config.php", + "includes/configure.php", + "config.inc.php", + "localsettings.php", + "inc/config.php", + "typo3conf/localconf.php", + "config/app.php", + "config/custom.php", + "config/database.php", + "/configuration.php", + "/config.php", + "var/mail/www-data", + "etc/network/", + "etc/init/", + "inetpub/wwwroot/global.asa", + "system32/inetsrv/config/applicationhost.config", + "system32/inetsrv/config/administration.config", + "system32/inetsrv/config/redirection.config", + "system32/config/default", + "system32/config/sam", + "system32/config/system", + "system32/config/software", + "winnt/repair/sam._", + "package.json", + "package-lock.json", + "gruntfile.js", + "npm-debug.log", + "ormconfig.json", + "tsconfig.json", + "webpack.config.js", + "yarn.lock" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "crs-931-110", + "name": "Possible Remote File Inclusion (RFI) Attack: Common RFI Vulnerable Parameter Name used w/URL Payload", + "tags": { + "type": "rfi", + "crs_id": "931110", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + } + ], + "regex": "(?:\\binclude\\s*\\([^)]*|mosConfig_absolute_path|_CONF\\[path\\]|_SERVER\\[DOCUMENT_ROOT\\]|GALLERY_BASEDIR|path\\[docroot\\]|appserv_root|config\\[root_dir\\])=(?:file|ftps?|https?)://", + "options": { + "min_length": 15 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-931-120", + "name": "Possible Remote File Inclusion (RFI) Attack: URL Payload Used w/Trailing Question Mark Character (?)", + "tags": { + "type": "rfi", + "crs_id": "931120", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + } + ], + "regex": "^(?i:file|ftps?|https?).*?\\?+$", + "options": { + "case_sensitive": true, + "min_length": 4 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-932-160", + "name": "Remote Command Execution: Unix Shell Code Found", + "tags": { + "type": "command_injection", + "crs_id": "932160", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "list": [ + "${cdpath}", + "${dirstack}", + "${home}", + "${hostname}", + "${ifs}", + "${oldpwd}", + "${ostype}", + "${path}", + "${pwd}", + "$cdpath", + "$dirstack", + "$home", + "$hostname", + "$ifs", + "$oldpwd", + "$ostype", + "$path", + "$pwd", + "bin/bash", + "bin/cat", + "bin/csh", + "bin/dash", + "bin/du", + "bin/echo", + "bin/grep", + "bin/less", + "bin/ls", + "bin/mknod", + "bin/more", + "bin/nc", + "bin/ps", + "bin/rbash", + "bin/sh", + "bin/sleep", + "bin/su", + "bin/tcsh", + "bin/uname", + "dev/fd/", + "dev/null", + "dev/stderr", + "dev/stdin", + "dev/stdout", + "dev/tcp/", + "dev/udp/", + "dev/zero", + "etc/group", + "etc/master.passwd", + "etc/passwd", + "etc/pwd.db", + "etc/shadow", + "etc/shells", + "etc/spwd.db", + "proc/self/", + "usr/bin/awk", + "usr/bin/base64", + "usr/bin/cat", + "usr/bin/cc", + "usr/bin/clang", + "usr/bin/clang++", + "usr/bin/curl", + "usr/bin/diff", + "usr/bin/env", + "usr/bin/fetch", + "usr/bin/file", + "usr/bin/find", + "usr/bin/ftp", + "usr/bin/gawk", + "usr/bin/gcc", + "usr/bin/head", + "usr/bin/hexdump", + "usr/bin/id", + "usr/bin/less", + "usr/bin/ln", + "usr/bin/mkfifo", + "usr/bin/more", + "usr/bin/nc", + "usr/bin/ncat", + "usr/bin/nice", + "usr/bin/nmap", + "usr/bin/perl", + "usr/bin/php", + "usr/bin/php5", + "usr/bin/php7", + "usr/bin/php-cgi", + "usr/bin/printf", + "usr/bin/psed", + "usr/bin/python", + "usr/bin/python2", + "usr/bin/python3", + "usr/bin/ruby", + "usr/bin/sed", + "usr/bin/socat", + "usr/bin/tail", + "usr/bin/tee", + "usr/bin/telnet", + "usr/bin/top", + "usr/bin/uname", + "usr/bin/wget", + "usr/bin/who", + "usr/bin/whoami", + "usr/bin/xargs", + "usr/bin/xxd", + "usr/bin/yes", + "usr/local/bin/bash", + "usr/local/bin/curl", + "usr/local/bin/ncat", + "usr/local/bin/nmap", + "usr/local/bin/perl", + "usr/local/bin/php", + "usr/local/bin/python", + "usr/local/bin/python2", + "usr/local/bin/python3", + "usr/local/bin/rbash", + "usr/local/bin/ruby", + "usr/local/bin/wget" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "crs-932-171", + "name": "Remote Command Execution: Shellshock (CVE-2014-6271)", + "tags": { + "type": "command_injection", + "crs_id": "932171", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "^\\(\\s*\\)\\s+{", + "options": { + "case_sensitive": true, + "min_length": 4 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-932-180", + "name": "Restricted File Upload Attempt", + "tags": { + "type": "command_injection", + "crs_id": "932180", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "x-filename" + ] + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "x_filename" + ] + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "x-file-name" + ] + } + ], + "list": [ + ".htaccess", + ".htdigest", + ".htpasswd", + "wp-config.php", + "config.yml", + "config_dev.yml", + "config_prod.yml", + "config_test.yml", + "parameters.yml", + "routing.yml", + "security.yml", + "services.yml", + "default.settings.php", + "settings.php", + "settings.local.php", + "local.xml", + ".env" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "crs-933-111", + "name": "PHP Injection Attack: PHP Script File Upload Found", + "tags": { + "type": "unrestricted_file_upload", + "crs_id": "933111", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "x-filename" + ] + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "x_filename" + ] + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "x.filename" + ] + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "x-file-name" + ] + } + ], + "regex": ".*\\.(?:php\\d*|phtml)\\..*$", + "options": { + "case_sensitive": true, + "min_length": 5 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "crs-933-130", + "name": "PHP Injection Attack: Global Variables Found", + "tags": { + "type": "php_code_injection", + "crs_id": "933130", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "list": [ + "$globals", + "$http_cookie_vars", + "$http_env_vars", + "$http_get_vars", + "$http_post_files", + "$http_post_vars", + "$http_raw_post_data", + "$http_request_vars", + "$http_server_vars", + "$_cookie", + "$_env", + "$_files", + "$_get", + "$_post", + "$_request", + "$_server", + "$_session", + "$argc", + "$argv" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "crs-933-131", + "name": "PHP Injection Attack: HTTP Headers Values Found", + "tags": { + "type": "php_code_injection", + "crs_id": "933131", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?:HTTP_(?:ACCEPT(?:_(?:ENCODING|LANGUAGE|CHARSET))?|(?:X_FORWARDED_FO|REFERE)R|(?:USER_AGEN|HOS)T|CONNECTION|KEEP_ALIVE)|PATH_(?:TRANSLATED|INFO)|ORIG_PATH_INFO|QUERY_STRING|REQUEST_URI|AUTH_TYPE)", + "options": { + "case_sensitive": true, + "min_length": 9 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-933-140", + "name": "PHP Injection Attack: I/O Stream Found", + "tags": { + "type": "php_code_injection", + "crs_id": "933140", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "php://(?:std(?:in|out|err)|(?:in|out)put|fd|memory|temp|filter)", + "options": { + "min_length": 8 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-933-150", + "name": "PHP Injection Attack: High-Risk PHP Function Name Found", + "tags": { + "type": "php_code_injection", + "crs_id": "933150", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "list": [ + "__halt_compiler", + "apache_child_terminate", + "base64_decode", + "bzdecompress", + "call_user_func", + "call_user_func_array", + "call_user_method", + "call_user_method_array", + "convert_uudecode", + "file_get_contents", + "file_put_contents", + "fsockopen", + "get_class_methods", + "get_class_vars", + "get_defined_constants", + "get_defined_functions", + "get_defined_vars", + "gzdecode", + "gzinflate", + "gzuncompress", + "include_once", + "invokeargs", + "pcntl_exec", + "pcntl_fork", + "pfsockopen", + "posix_getcwd", + "posix_getpwuid", + "posix_getuid", + "posix_uname", + "reflectionfunction", + "require_once", + "shell_exec", + "str_rot13", + "sys_get_temp_dir", + "wp_remote_fopen", + "wp_remote_get", + "wp_remote_head", + "wp_remote_post", + "wp_remote_request", + "wp_safe_remote_get", + "wp_safe_remote_head", + "wp_safe_remote_post", + "wp_safe_remote_request", + "zlib_decode" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "crs-933-160", + "name": "PHP Injection Attack: High-Risk PHP Function Call Found", + "tags": { + "type": "php_code_injection", + "crs_id": "933160", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "\\b(?:s(?:e(?:t(?:_(?:e(?:xception|rror)_handler|magic_quotes_runtime|include_path)|defaultstub)|ssion_s(?:et_save_handler|tart))|qlite_(?:(?:(?:unbuffered|single|array)_)?query|create_(?:aggregate|function)|p?open|exec)|tr(?:eam_(?:context_create|socket_client)|ipc?slashes|rev)|implexml_load_(?:string|file)|ocket_c(?:onnect|reate)|h(?:ow_sourc|a1_fil)e|pl_autoload_register|ystem)|p(?:r(?:eg_(?:replace(?:_callback(?:_array)?)?|match(?:_all)?|split)|oc_(?:(?:terminat|clos|nic)e|get_status|open)|int_r)|o(?:six_(?:get(?:(?:e[gu]|g)id|login|pwnam)|mk(?:fifo|nod)|ttyname|kill)|pen)|hp(?:_(?:strip_whitespac|unam)e|version|info)|g_(?:(?:execut|prepar)e|connect|query)|a(?:rse_(?:ini_file|str)|ssthru)|utenv)|r(?:unkit_(?:function_(?:re(?:defin|nam)e|copy|add)|method_(?:re(?:defin|nam)e|copy|add)|constant_(?:redefine|add))|e(?:(?:gister_(?:shutdown|tick)|name)_function|ad(?:(?:gz)?file|_exif_data|dir))|awurl(?:de|en)code)|i(?:mage(?:createfrom(?:(?:jpe|pn)g|x[bp]m|wbmp|gif)|(?:jpe|pn)g|g(?:d2?|if)|2?wbmp|xbm)|s_(?:(?:(?:execut|write?|read)ab|fi)le|dir)|ni_(?:get(?:_all)?|set)|terator_apply|ptcembed)|g(?:et(?:_(?:c(?:urrent_use|fg_va)r|meta_tags)|my(?:[gpu]id|inode)|(?:lastmo|cw)d|imagesize|env)|z(?:(?:(?:defla|wri)t|encod|fil)e|compress|open|read)|lob)|a(?:rray_(?:u(?:intersect(?:_u?assoc)?|diff(?:_u?assoc)?)|intersect_u(?:assoc|key)|diff_u(?:assoc|key)|filter|reduce|map)|ssert(?:_options)?)|h(?:tml(?:specialchars(?:_decode)?|_entity_decode|entities)|(?:ash(?:_(?:update|hmac))?|ighlight)_file|e(?:ader_register_callback|x2bin))|f(?:i(?:le(?:(?:[acm]tim|inod)e|(?:_exist|perm)s|group)?|nfo_open)|tp_(?:nb_(?:ge|pu)|connec|ge|pu)t|(?:unction_exis|pu)ts|write|open)|o(?:b_(?:get_(?:c(?:ontents|lean)|flush)|end_(?:clean|flush)|clean|flush|start)|dbc_(?:result(?:_all)?|exec(?:ute)?|connect)|pendir)|m(?:b_(?:ereg(?:_(?:replace(?:_callback)?|match)|i(?:_replace)?)?|parse_str)|(?:ove_uploaded|d5)_file|ethod_exists|ysql_query|kdir)|e(?:x(?:if_(?:t(?:humbnail|agname)|imagetype|read_data)|ec)|scapeshell(?:arg|cmd)|rror_reporting|val)|c(?:url_(?:file_create|exec|init)|onvert_uuencode|reate_function|hr)|u(?:n(?:serialize|pack)|rl(?:de|en)code|[ak]?sort)|(?:json_(?:de|en)cod|debug_backtrac|tmpfil)e|b(?:(?:son_(?:de|en)|ase64_en)code|zopen)|var_dump)(?:\\s|/\\*.*\\*/|//.*|#.*)*\\(.*\\)", + "options": { + "min_length": 5 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-933-170", + "name": "PHP Injection Attack: Serialized Object Injection", + "tags": { + "type": "php_code_injection", + "crs_id": "933170", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "[oOcC]:\\d+:\\\".+?\\\":\\d+:{[\\W\\w]*}", + "options": { + "case_sensitive": true, + "min_length": 12 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-933-200", + "name": "PHP Injection Attack: Wrapper scheme detected", + "tags": { + "type": "php_code_injection", + "crs_id": "933200", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?i:zlib|glob|phar|ssh2|rar|ogg|expect|zip)://", + "options": { + "case_sensitive": true, + "min_length": 6 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "removeNulls" + ] + }, + { + "id": "crs-934-100", + "name": "Node.js Injection Attack", + "tags": { + "type": "js_code_injection", + "crs_id": "934100", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?:(?:_(?:\\$\\$ND_FUNC\\$\\$_|_js_function)|(?:new\\s+Function|\\beval)\\s*\\(|String\\s*\\.\\s*fromCharCode|function\\s*\\(\\s*\\)\\s*{|this\\.constructor)|module\\.exports\\s*=)", + "options": { + "case_sensitive": true, + "min_length": 5 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-941-100", + "name": "XSS Attack Detected via libinjection", + "tags": { + "type": "xss", + "crs_id": "941100", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "referer" + ] + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ] + }, + "operator": "is_xss" + } + ], + "transformers": [ + "removeNulls" + ] + }, + { + "id": "crs-941-110", + "name": "XSS Filter - Category 1: Script Tag Vector", + "tags": { + "type": "xss", + "crs_id": "941110", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "referer" + ] + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "]*>[\\s\\S]*?", + "options": { + "min_length": 8 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "removeNulls" + ] + }, + { + "id": "crs-941-120", + "name": "XSS Filter - Category 2: Event Handler Vector", + "tags": { + "type": "xss", + "crs_id": "941120", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "referer" + ] + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "[\\s\\\"'`;\\/0-9=\\x0B\\x09\\x0C\\x3B\\x2C\\x28\\x3B]on[a-zA-Z]{3,25}[\\s\\x0B\\x09\\x0C\\x3B\\x2C\\x28\\x3B]*?=[^=]", + "options": { + "min_length": 8 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "removeNulls" + ] + }, + { + "id": "crs-941-140", + "name": "XSS Filter - Category 4: Javascript URI Vector", + "tags": { + "type": "xss", + "crs_id": "941140", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "referer" + ] + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "[a-z]+=(?:[^:=]+:.+;)*?[^:=]+:url\\(javascript", + "options": { + "min_length": 18 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "removeNulls" + ] + }, + { + "id": "crs-941-180", + "name": "Node-Validator Deny List Keywords", + "tags": { + "type": "xss", + "crs_id": "941180", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "list": [ + "document.cookie", + "document.write", + ".parentnode", + ".innerhtml", + "window.location", + "-moz-binding", + "]", + "options": { + "min_length": 8 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "removeNulls" + ] + }, + { + "id": "crs-941-300", + "name": "IE XSS Filters - Attack Detected via object tag", + "tags": { + "type": "xss", + "crs_id": "941300", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": ")|<.*\\+AD4-", + "options": { + "case_sensitive": true, + "min_length": 6 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-941-360", + "name": "JSFuck / Hieroglyphy obfuscation detected", + "tags": { + "type": "xss", + "crs_id": "941360", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "![!+ ]\\[\\]", + "options": { + "case_sensitive": true, + "min_length": 4 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-942-100", + "name": "SQL Injection Attack Detected via libinjection", + "tags": { + "type": "sql_injection", + "crs_id": "942100", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ] + }, + "operator": "is_sqli" + } + ], + "transformers": [ + "removeNulls" + ] + }, + { + "id": "crs-942-140", + "name": "SQL Injection Attack: Common DB Names Detected", + "tags": { + "type": "sql_injection", + "crs_id": "942140", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "\\b(?:(?:m(?:s(?:ys(?:ac(?:cess(?:objects|storage|xml)|es)|(?:relationship|object|querie)s|modules2?)|db)|aster\\.\\.sysdatabases|ysql\\.db)|pg_(?:catalog|toast)|information_schema|northwind|tempdb)\\b|s(?:(?:ys(?:\\.database_name|aux)|qlite(?:_temp)?_master)\\b|chema(?:_name\\b|\\W*\\())|d(?:atabas|b_nam)e\\W*\\()", + "options": { + "min_length": 4 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-942-160", + "name": "Detects blind sqli tests using sleep() or benchmark()", + "tags": { + "type": "sql_injection", + "crs_id": "942160", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?i:sleep\\(\\s*?\\d*?\\s*?\\)|benchmark\\(.*?\\,.*?\\))", + "options": { + "case_sensitive": true, + "min_length": 7 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-942-190", + "name": "Detects MSSQL code execution and information gathering attempts", + "tags": { + "type": "sql_injection", + "crs_id": "942190", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?:\\b(?:(?:c(?:onnection_id|urrent_user)|database)\\s*?\\([^\\)]*?|u(?:nion(?:[\\w(?:\\s]*?select| select @)|ser\\s*?\\([^\\)]*?)|s(?:chema\\s*?\\([^\\)]*?|elect.*?\\w?user\\()|into[\\s+]+(?:dump|out)file\\s*?[\\\"'`]|from\\W+information_schema\\W|exec(?:ute)?\\s+master\\.)|[\\\"'`](?:;?\\s*?(?:union\\b\\s*?(?:(?:distin|sele)ct|all)|having|select)\\b\\s*?[^\\s]|\\s*?!\\s*?[\\\"'`\\w])|\\s*?exec(?:ute)?.*?\\Wxp_cmdshell|\\Wiif\\s*?\\()", + "options": { + "min_length": 3 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-942-220", + "name": "Looking for integer overflow attacks, these are taken from skipfish, except 2.2.2250738585072011e-308 is the \\\"magic number\\\" crash", + "tags": { + "type": "sql_injection", + "crs_id": "942220", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "^(?i:-0000023456|4294967295|4294967296|2147483648|2147483647|0000012345|-2147483648|-2147483649|0000023456|2.2250738585072007e-308|2.2250738585072011e-308|1e309)$", + "options": { + "case_sensitive": true, + "min_length": 5 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-942-240", + "name": "Detects MySQL charset switch and MSSQL DoS attempts", + "tags": { + "type": "sql_injection", + "crs_id": "942240", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?:[\\\"'`](?:;*?\\s*?waitfor\\s+(?:delay|time)\\s+[\\\"'`]|;.*?:\\s*?goto)|alter\\s*?\\w+.*?cha(?:racte)?r\\s+set\\s+\\w+)", + "options": { + "min_length": 7 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-942-250", + "name": "Detects MATCH AGAINST, MERGE and EXECUTE IMMEDIATE injections", + "tags": { + "type": "sql_injection", + "crs_id": "942250", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?i:merge.*?using\\s*?\\(|execute\\s*?immediate\\s*?[\\\"'`]|match\\s*?[\\w(?:),+-]+\\s*?against\\s*?\\()", + "options": { + "case_sensitive": true, + "min_length": 11 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-942-270", + "name": "Looking for basic sql injection. Common attack string for mysql, oracle and others", + "tags": { + "type": "sql_injection", + "crs_id": "942270", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "union.*?select.*?from", + "options": { + "min_length": 15 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-942-280", + "name": "Detects Postgres pg_sleep injection, waitfor delay attacks and database shutdown attempts", + "tags": { + "type": "sql_injection", + "crs_id": "942280", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?:;\\s*?shutdown\\s*?(?:[#;{]|\\/\\*|--)|waitfor\\s*?delay\\s?[\\\"'`]+\\s?\\d|select\\s*?pg_sleep)", + "options": { + "min_length": 10 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-942-290", + "name": "Finds basic MongoDB SQL injection attempts", + "tags": { + "type": "nosql_injection", + "crs_id": "942290", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?i:(?:\\[\\$(?:ne|eq|lte?|gte?|n?in|mod|all|size|exists|type|slice|x?or|div|like|between|and)\\]))", + "options": { + "case_sensitive": true, + "min_length": 5 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-942-360", + "name": "Detects concatenated basic SQL injection and SQLLFI attempts", + "tags": { + "type": "sql_injection", + "crs_id": "942360", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?:^[\\W\\d]+\\s*?(?:alter\\s*(?:a(?:(?:pplication\\s*rol|ggregat)e|s(?:ymmetric\\s*ke|sembl)y|u(?:thorization|dit)|vailability\\s*group)|c(?:r(?:yptographic\\s*provider|edential)|o(?:l(?:latio|um)|nversio)n|ertificate|luster)|s(?:e(?:rv(?:ice|er)|curity|quence|ssion|arch)|y(?:mmetric\\s*key|nonym)|togroup|chema)|m(?:a(?:s(?:ter\\s*key|k)|terialized)|e(?:ssage\\s*type|thod)|odule)|l(?:o(?:g(?:file\\s*group|in)|ckdown)|a(?:ngua|r)ge|ibrary)|t(?:(?:abl(?:espac)?|yp)e|r(?:igger|usted)|hreshold|ext)|p(?:a(?:rtition|ckage)|ro(?:cedur|fil)e|ermission)|d(?:i(?:mension|skgroup)|atabase|efault|omain)|r(?:o(?:l(?:lback|e)|ute)|e(?:sourc|mot)e)|f(?:u(?:lltext|nction)|lashback|oreign)|e(?:xte(?:nsion|rnal)|(?:ndpoi|ve)nt)|in(?:dex(?:type)?|memory|stance)|b(?:roker\\s*priority|ufferpool)|x(?:ml\\s*schema|srobject)|w(?:ork(?:load)?|rapper)|hi(?:erarchy|stogram)|o(?:perator|utline)|(?:nicknam|queu)e|us(?:age|er)|group|java|view)\\b|(?:(?:(?:trunc|cre)at|renam)e|d(?:e(?:lete|sc)|rop)|(?:inser|selec)t|load)\\s+\\w+|u(?:nion\\s*(?:(?:distin|sele)ct|all)\\b|pdate\\s+\\w+))|\\b(?:(?:(?:(?:trunc|cre|upd)at|renam)e|(?:inser|selec)t|de(?:lete|sc)|alter|load)\\s+(?:group_concat|load_file|char)\\b\\s*\\(?|end\\s*?\\);)|[\\\"'`\\w]\\s+as\\b\\s*[\\\"'`\\w]+\\s*\\bfrom|[\\s(?:]load_file\\s*?\\(|[\\\"'`]\\s+regexp\\W)", + "options": { + "min_length": 5 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-942-500", + "name": "MySQL in-line comment detected", + "tags": { + "type": "sql_injection", + "crs_id": "942500", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?i:/\\*[!+](?:[\\w\\s=_\\-(?:)]+)?\\*/)", + "options": { + "case_sensitive": true, + "min_length": 5 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-943-100", + "name": "Possible Session Fixation Attack: Setting Cookie Values in HTML", + "tags": { + "type": "http_protocol_violation", + "crs_id": "943100", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + } + ], + "regex": "(?i:\\.cookie\\b.*?;\\W*?(?:expires|domain)\\W*?=|\\bhttp-equiv\\W+set-cookie\\b)", + "options": { + "case_sensitive": true, + "min_length": 15 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "crs-944-100", + "name": "Remote Command Execution: Suspicious Java class detected", + "tags": { + "type": "java_code_injection", + "crs_id": "944100", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "server.request.cookies" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "java\\.lang\\.(?:runtime|processbuilder)", + "options": { + "case_sensitive": true, + "min_length": 17 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "crs-944-110", + "name": "Remote Command Execution: Java process spawn (CVE-2017-9805)", + "tags": { + "type": "java_code_injection", + "crs_id": "944110", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "server.request.cookies" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?:runtime|processbuilder)", + "options": { + "case_sensitive": true, + "min_length": 7 + } + }, + "operator": "match_regex" + }, + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "server.request.cookies" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?:unmarshaller|base64data|java\\.)", + "options": { + "case_sensitive": true, + "min_length": 5 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "crs-944-130", + "name": "Suspicious Java class detected", + "tags": { + "type": "java_code_injection", + "crs_id": "944130", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "server.request.cookies" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "grpc.server.request.message" + } + ], + "list": [ + "com.opensymphony.xwork2", + "com.sun.org.apache", + "java.io.bufferedinputstream", + "java.io.bufferedreader", + "java.io.bytearrayinputstream", + "java.io.bytearrayoutputstream", + "java.io.chararrayreader", + "java.io.datainputstream", + "java.io.file", + "java.io.fileoutputstream", + "java.io.filepermission", + "java.io.filewriter", + "java.io.filterinputstream", + "java.io.filteroutputstream", + "java.io.filterreader", + "java.io.inputstream", + "java.io.inputstreamreader", + "java.io.linenumberreader", + "java.io.objectoutputstream", + "java.io.outputstream", + "java.io.pipedoutputstream", + "java.io.pipedreader", + "java.io.printstream", + "java.io.pushbackinputstream", + "java.io.reader", + "java.io.stringreader", + "java.lang.class", + "java.lang.integer", + "java.lang.number", + "java.lang.object", + "java.lang.process", + "java.lang.processbuilder", + "java.lang.reflect", + "java.lang.runtime", + "java.lang.string", + "java.lang.stringbuilder", + "java.lang.system", + "javax.script.scriptenginemanager", + "org.apache.commons", + "org.apache.struts", + "org.apache.struts2", + "org.omg.corba", + "java.beans.xmldecode" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "nfd-000-001", + "name": "Detect common directory discovery scans", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.status" + } + ], + "regex": "^404$", + "options": { + "case_sensitive": true + } + } + }, + { + "operator": "phrase_match", + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + } + ], + "list": [ + "/wordpress/", + "/etc/", + "/login.php", + "/install.php", + "/administrator", + "/admin.php", + "/wp-config", + "/phpmyadmin", + "/fckeditor", + "/mysql", + "/manager/html", + ".htaccess", + "/config.php", + "/configuration", + "/cgi-bin/php", + "/search.php", + "/tinymce", + "/tiny_mce", + "/settings.php", + "../../..", + "/install/", + "/download.php", + "/webdav", + "/forum.php", + "/user.php", + "/style.php", + "/jmx-console", + "/modules.php", + "/include.php", + "/default.asp", + "/help.php", + "/database.yml", + "/database.yml.pgsql", + "/database.yml.sqlite3", + "/database.yml.sqlite", + "/database.yml.mysql", + ".%2e/", + "/view.php", + "/header.php", + "/search.asp", + "%5c%5c", + "/server/php/", + "/invoker/jmxinvokerservlet", + "/phpmyadmin/index.php", + "/data/admin/allowurl.txt", + "/verify.php", + "/misc/ajax.js", + "/.idea", + "/module.php", + "/backup.rar", + "/backup.tar", + "/backup.zip", + "/backup.7z", + "/backup.gz", + "/backup.tgz", + "/backup.tar.gz", + "waitfor%20delay", + "/calendar.php", + "/news.php", + "/dompdf.php", + "))))))))))))))))", + "/web.config", + "tree.php", + "/cgi-bin-sdb/printenv", + "/comments.php", + "/detail.asp", + "/license.txt", + "/admin.asp", + "/auth.php", + "/list.php", + "/content.php", + "/mod.php", + "/mini.php", + "/install.pgsql", + "/install.mysql", + "/install.sqlite", + "/install.sqlite3", + "/install.txt", + "/install.md", + "/doku.php", + "/main.asp", + "/myadmin", + "/force-download.php", + "/iisprotect/admin", + "/.gitignore", + "/print.php", + "/common.php", + "/mainfile.php", + "/functions.php", + "/scripts/setup.php", + "/faq.php", + "/op/op.login.php", + "/home.php", + "/includes/hnmain.inc.php3", + "/preview.php", + "/dump.rar", + "/dump.tar", + "/dump.zip", + "/dump.7z", + "/dump.gz", + "/dump.tgz", + "/dump.tar.gz", + "/thumbnail.php", + "/sendcard.php", + "/global.asax", + "/directory.php", + "/footer.php", + "/error.asp", + "/forum.asp", + "/save.php", + "/htmlsax3.php", + "/adm/krgourl.php", + "/includes/converter.inc.php", + "/nucleus/libs/pluginadmin.php", + "/base_qry_common.php", + "/fileadmin", + "/bitrix/admin/", + "/adm.php", + "/util/barcode.php", + "/action.php", + "/rss.asp", + "/downloads.php", + "/page.php", + "/snarf_ajax.php", + "/fck/editor", + "/sendmail.php", + "/detail.php", + "/iframe.php", + "/swfupload.swf", + "/jenkins/login", + "/phpmyadmin/main.php", + "/phpmyadmin/scripts/setup.php", + "/user/index.php", + "/checkout.php", + "/process.php", + "/ks_inc/ajax.js", + "/export.php", + "/register.php", + "/cart.php", + "/console.php", + "/friend.php", + "/readmsg.php", + "/install.asp", + "/dagent/downloadreport.asp", + "/system/index.php", + "/core/changelog.txt", + "/js/util.js", + "/interna.php", + "/gallery.php", + "/links.php", + "/data/admin/ver.txt", + "/language/zh-cn.xml", + "/productdetails.asp", + "/admin/template/article_more/config.htm", + "/components/com_moofaq/includes/file_includer.php", + "/licence.txt", + "/rss.xsl", + "/vtigerservice.php", + "/mysql/main.php", + "/passwiki.php", + "/scr/soustab.php", + "/global.php", + "/email.php", + "/user.asp", + "/msd", + "/products.php", + "/cultbooking.php", + "/cron.php", + "/static/js/admincp.js", + "/comment.php", + "/maintainers", + "/modules/plain/adminpart/addplain.php", + "/wp-content/plugins/ungallery/source_vuln.php", + "/upgrade.txt", + "/category.php", + "/index_logged.php", + "/members.asp", + "/script/html.js", + "/images/ad.js", + "/awstats/awstats.pl", + "/includes/esqueletos/skel_null.php", + "/modules/profile/user.php", + "/window_top.php", + "/openbrowser.php", + "/thread.php", + "tinfoil_xss", + "/includes/include.php", + "/urheber.php", + "/header.inc.php", + "/mysqldumper", + "/display.php", + "/website.php", + "/stats.php", + "/assets/plugins/mp3_id/mp3_id.php", + "/siteminderagent/forms/smpwservices.fcc" + ] + } + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "nfd-000-002", + "name": "Detect failed attempt to fetch readme files", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.status" + } + ], + "regex": "^404$", + "options": { + "case_sensitive": true + } + } + }, + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + } + ], + "regex": "readme\\.[\\.a-z0-9]+$", + "options": { + "case_sensitive": false + } + } + } + ], + "transformers": [] + }, + { + "id": "nfd-000-003", + "name": "Detect failed attempt to fetch Java EE resource files", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.status" + } + ], + "regex": "^404$", + "options": { + "case_sensitive": true + } + } + }, + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + } + ], + "regex": "^(?:.*web\\-inf)(?:.*web\\.xml).*$", + "options": { + "case_sensitive": false + } + } + } + ], + "transformers": [] + }, + { + "id": "nfd-000-004", + "name": "Detect failed attempt to fetch code files", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.status" + } + ], + "regex": "^404$", + "options": { + "case_sensitive": true + } + } + }, + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + } + ], + "regex": "\\.(java|pyc?|rb|class)\\b", + "options": { + "case_sensitive": false + } + } + } + ], + "transformers": [] + }, + { + "id": "nfd-000-005", + "name": "Detect failed attempt to fetch source code archives", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.status" + } + ], + "regex": "^404$", + "options": { + "case_sensitive": true + } + } + }, + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + } + ], + "regex": "\\.(sql|log|ndb|gz|zip|tar\\.gz|tar|regVV|reg|conf|bz2|ini|db|war|bat|inc|btr|server|ds|conf|config|admin|master|sln|bak)\\b(?:[^.]|$)", + "options": { + "case_sensitive": false + } + } + } + ], + "transformers": [] + }, + { + "id": "nfd-000-006", + "name": "Detect failed attempt to fetch sensitive files", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.status" + } + ], + "regex": "^404$", + "options": { + "case_sensitive": true + } + } + }, + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + } + ], + "regex": "\\.(cgi|bat|dll|exe|key|cert|crt|pem|der|pkcs|pkcs|pkcs[0-9]*|nsf|jsa|war|java|class|vb|vba|so|git|svn|hg|cvs)([^a-zA-Z0-9_]|$)", + "options": { + "case_sensitive": false + } + } + } + ], + "transformers": [] + }, + { + "id": "nfd-000-007", + "name": "Detect failed attempt to fetch archives", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.status" + } + ], + "regex": "^404$", + "options": { + "case_sensitive": true + } + } + }, + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + } + ], + "regex": "/[\\d\\-_]*\\.(rar|tar|zip|7z|gz|tgz|tar.gz)", + "options": { + "case_sensitive": false + } + } + } + ], + "transformers": [] + }, + { + "id": "nfd-000-008", + "name": "Detect failed attempt to trigger incorrect application behavior", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.status" + } + ], + "regex": "^404$", + "options": { + "case_sensitive": true + } + } + }, + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + } + ], + "regex": "(/(administrator/components/com.*\\.php|response\\.write\\(.+\\))|select\\(.+\\)from|\\(.*sleep\\(.+\\)|(%[a-zA-Z0-9]{2}[a-zA-Z]{0,1})+\\))", + "options": { + "case_sensitive": false + } + } + } + ], + "transformers": [] + }, + { + "id": "nfd-000-009", + "name": "Detect failed attempt to leak the structure of the application", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.status" + } + ], + "regex": "^404$", + "options": { + "case_sensitive": true + } + } + }, + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + } + ], + "regex": "/(login\\.rol|LICENSE|[\\w-]+\\.(plx|pwd))$", + "options": { + "case_sensitive": false + } + } + } + ], + "transformers": [] + }, + { + "id": "sqr-000-001", + "name": "SSRF: Try to access the credential manager of the main cloud services", + "tags": { + "type": "ssrf", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?i)^\\W*((http|ftp)s?://)?\\W*((::f{4}:)?(169|(0x)?0*a9|0+251)\\.?(254|(0x)?0*fe|0+376)[0-9a-fx\\.:]+|metadata\\.google\\.internal|metadata\\.goog)\\W*/", + "options": { + "min_length": 4 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "removeNulls" + ] + }, + { + "id": "sqr-000-002", + "name": "Server-side Javascript injection: Try to detect obvious JS injection", + "tags": { + "type": "js_code_injection", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "require\\(['\"][\\w\\.]+['\"]\\)|process\\.\\w+\\([\\w\\.]*\\)|\\.toString\\(\\)", + "options": { + "min_length": 4 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "removeNulls" + ] + }, + { + "id": "sqr-000-007", + "name": "NoSQL: Detect common exploitation strategy", + "tags": { + "type": "nosql_injection", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies" + } + ], + "regex": "\\$(eq|ne|lte?|gte?|n?in)\\b" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "sqr-000-008", + "name": "Windows: Detect attempts to exfiltrate .ini files", + "tags": { + "type": "command_injection", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?i)[&|]\\s*type\\s+%\\w+%\\\\+\\w+\\.ini\\s*[&|]" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "sqr-000-009", + "name": "Linux: Detect attempts to exfiltrate passwd files", + "tags": { + "type": "command_injection", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?i)[&|]\\s*cat\\s+\\/etc\\/[\\w\\.\\/]*passwd\\s*[&|]" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "sqr-000-010", + "name": "Windows: Detect attempts to timeout a shell", + "tags": { + "type": "command_injection", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "(?i)[&|]\\s*timeout\\s+/t\\s+\\d+\\s*[&|]" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "sqr-000-011", + "name": "SSRF: Try to access internal OMI service (CVE-2021-38647)", + "tags": { + "type": "ssrf", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "http(s?):\\/\\/([A-Za-z0-9\\.\\-\\_]+|\\[[A-Fa-f0-9\\:]+\\]|):5986\\/wsman", + "options": { + "min_length": 4 + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "sqr-000-012", + "name": "SSRF: Detect SSRF attempt on internal service", + "tags": { + "type": "ssrf", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "^(jar:)?(http|https):\\/\\/([0-9oq]{1,5}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|[0-9]{1,10}|localhost)(:[0-9]{1,5})?(\\/.*|)$" + }, + "operator": "match_regex" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "sqr-000-013", + "name": "SSRF: Detect SSRF attempts using IPv6 or octal/hexdecimal obfuscation", + "tags": { + "type": "ssrf", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "^(jar:)?(http|https):\\/\\/((\\[)?[:0-9a-f\\.x]{2,}(\\])?)(:[0-9]{1,5})?(\\/.*)?$" + }, + "operator": "match_regex" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "sqr-000-014", + "name": "SSRF: Detect SSRF domain redirection bypass", + "tags": { + "type": "ssrf", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "^(http|https):\\/\\/(.*burpcollaborator\\.net|localtest\\.me|mail\\.ebc\\.apple\\.com|bugbounty\\.dod\\.network|.*\\.[nx]ip\\.io)" + }, + "operator": "match_regex" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "sqr-000-015", + "name": "SSRF: Detect SSRF attempt using non HTTP protocol", + "tags": { + "type": "ssrf", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "^(jar:)?((file|netdoc):\\/\\/[\\\\\\/]+|(dict|gopher|ldap|sftp|tftp):\\/\\/.*:[0-9]{1,5})" + }, + "operator": "match_regex" + } + ], + "transformers": [ + "lowercase" + ] + }, + { + "id": "sqr-000-017", + "name": "JNDI: Attempt to exploit log4j CVE-2021-44228", + "tags": { + "type": "java_code_injection", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + }, + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "grpc.server.request.message" + } + ], + "regex": "\\${[^j]*j[^n]*n[^d]*d[^i]*i[^:]*:[^}]*}" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "ua0-600-0xx", + "name": "Joomla exploitation tool", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + } + ], + "regex": "JDatabaseDriverMysqli" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "ua0-600-10x", + "name": "Nessus", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + } + ], + "regex": "(?i)^Nessus(/|([ :]+SOAP))" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "ua0-600-12x", + "name": "Arachni", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + } + ], + "regex": "^Arachni\\/v" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "ua0-600-13x", + "name": "Jorgee", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + } + ], + "regex": "(?i)\\bJorgee\\b" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "ua0-600-14x", + "name": "Probely", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + } + ], + "regex": "(?i)\\bProbely\\b" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "ua0-600-15x", + "name": "Metis", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + } + ], + "regex": "(?i)\\bmetis\\b" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "ua0-600-16x", + "name": "SQL power injector", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + } + ], + "regex": "sql power injector" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "ua0-600-18x", + "name": "N-Stealth", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + } + ], + "regex": "(?i)\\bn-stealth\\b" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "ua0-600-19x", + "name": "Brutus", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + } + ], + "regex": "(?i)\\bbrutus\\b" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "ua0-600-1xx", + "name": "Shellshock exploitation tool", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + } + ], + "regex": "\\(\\) \\{ :; *\\}" + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, + { + "id": "ua0-600-20x", + "name": "Netsparker", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + } + ], + "regex": "(?i)(