feat(appsec): ddwaf extension

.circleci/config.yml

@@ -222,7 +222,7 @@ executors:
  cimg_base:
  docker:
  - image: *cimg_base_image
- resource_class: small
+ resource_class: medium
  python39:
  docker:
  - image: *python39_image
@@ -284,9 +284,9 @@ jobs:
  steps:
  - checkout
  - run: sudo apt-get update
- - run: sudo apt-get install --yes clang-format gcc-10 python3 python3-setuptools python3-pip
+ - run: sudo apt-get install --yes clang-format gcc-10 g++-10 python3 python3-setuptools python3-pip
  - run: scripts/cformat.sh
- - run: DD_COMPILE_DEBUG=1 CC=gcc-10 pip -vvv install .
+ - run: DD_COMPILE_DEBUG=1 DD_TESTING_RAISE=1 CC=gcc-10 CXX=g++-10 pip -vvv install .
 
  coverage_report:
  executor: python39
@@ -342,6 +342,12 @@ jobs:
  paths:
  - "."
 
+ appsec:
+ <<: *contrib_job
+ steps:
+ - run_test:
+ pattern: 'appsec'
+
  tracer:
  <<: *contrib_job
  steps:
@@ -1025,6 +1031,7 @@ requires_tests: &requires_tests
  - starlette
  - test_logging
  - tracer
+ - appsec
  - tornado
  - urllib3
  - vertica
@@ -1112,6 +1119,7 @@ workflows:
  - test_logging: *requires_base_venvs
  - tornado: *requires_base_venvs
  - tracer: *requires_base_venvs
+ - appsec: *requires_base_venvs
  - urllib3: *requires_base_venvs
  - vertica: *requires_base_venvs
  - wsgi: *requires_base_venvs

.github/workflows/build_deploy.yml

@@ -150,7 +150,7 @@ jobs:
 
  - name: Install build dependencies
  # Rust + Cargo are needed for Cryptography
- run: apk add git gcc musl-dev libffi-dev openssl-dev bash rust cargo
+ run: apk add git gcc g++ musl-dev libffi-dev openssl-dev bash rust cargo
 
  - name: Check source package
  run: |

.gitignore

@@ -4,6 +4,7 @@ __pycache__/
 *$py.class
 
 # C extensions
+ddtrace/appsec/_ddwaf.cpp
 ddtrace/profiling/collector/_task.c
 ddtrace/profiling/collector/_threading.c
 ddtrace/profiling/collector/_traceback.c
@@ -14,6 +15,7 @@ ddtrace/internal/_encoding.c
 ddtrace/internal/_rand.c
 ddtrace/internal/_tagset.c
 *.so
+*.a
 
 # Cython annotate HTML files
 ddtrace/**/*.html

ddtrace/appsec/CMakeLists.txt

@@ -0,0 +1,17 @@
+cmake_minimum_required(VERSION 3.14)
+project(ddwaf)
+include(ExternalProject)
+
+ExternalProject_Add(libddwaf
+ GIT_REPOSITORY https://github.com/DataDog/libddwaf.git
+ GIT_TAG 1.0.16
+ INSTALL_DIR ${CMAKE_SOURCE_DIR}
+ CMAKE_ARGS
+ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+ -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
+ -DLIBDDWAF_BUILD_SHARED=OFF
+ -DLIBDDWAF_BUILD_STATIC=ON
+ -DLIBDDWAF_MSVC_RUNTIME_LIBRARY=/MD
+ -DCMAKE_INSTALL_LIBDIR=lib
+ -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+)

ddtrace/appsec/_ddwaf.pyx

@@ -0,0 +1,262 @@
+from collections import deque
+import sys
+from typing import Tuple
+
+import six
+
+
+if six.PY3:
+ from typing import Mapping
+ from typing import Sequence
+else:
+ from collections import Mapping, Sequence
+
+from _libddwaf cimport DDWAF_LOG_LEVEL
+from _libddwaf cimport DDWAF_OBJ_TYPE
+from _libddwaf cimport ddwaf_context
+from _libddwaf cimport ddwaf_context_destroy
+from _libddwaf cimport ddwaf_context_init
+from _libddwaf cimport ddwaf_destroy
+from _libddwaf cimport ddwaf_get_version
+from _libddwaf cimport ddwaf_handle
+from _libddwaf cimport ddwaf_init
+from _libddwaf cimport ddwaf_object
+from _libddwaf cimport ddwaf_required_addresses
+from _libddwaf cimport ddwaf_result
+from _libddwaf cimport ddwaf_result_free
+from _libddwaf cimport ddwaf_run
+from _libddwaf cimport ddwaf_set_log_cb
+from _libddwaf cimport ddwaf_version
+from cpython.bytes cimport PyBytes_AsString
+from cpython.bytes cimport PyBytes_Size
+from cpython.exc cimport PyErr_Clear
+from cpython.exc cimport PyErr_Occurred
+from cpython.mem cimport PyMem_Free
+from cpython.mem cimport PyMem_Realloc
+from cpython.unicode cimport PyUnicode_AsEncodedString
+from libc.stdint cimport uint32_t
+from libc.stdint cimport uint64_t
+from libc.stdint cimport uintptr_t
+from libc.string cimport memset
+
+
+DEFAULT_DDWAF_TIMEOUT_MS=20
+
+
+cdef extern from "Python.h":
+ const char* PyUnicode_AsUTF8AndSize(object o, Py_ssize_t *size)
+
+
+def version():
+ # type: () -> Tuple[int, int, int]
+ """Get the version of libddwaf."""
+ cdef ddwaf_version version
+ ddwaf_get_version(&version)
+ return (version.major, version.minor, version.patch)
+
+
+cdef inline object _string_to_bytes(object string, const char **ptr, ssize_t *length):
+ if isinstance(string, six.binary_type):
+ ptr[0] = PyBytes_AsString(string)
+ length[0] = PyBytes_Size(string)
+ return string
+ elif isinstance(string, six.text_type):
+ IF PY_MAJOR_VERSION >= 3:
+ ptr[0] = PyUnicode_AsUTF8AndSize(string, length)
+ if ptr[0] == NULL and PyErr_Occurred():
+ # ignore exception from this function as we fallback to
+ # PyUnicode_AsEncodedString
+ PyErr_Clear()
+ if ptr[0] == NULL:
+ string = PyUnicode_AsEncodedString(string, "utf-8", "surrogatepass")
+ ptr[0] = PyBytes_AsString(string)
+ length[0] = PyBytes_Size(string)
+ return string
+ raise RuntimeError
+
+
+cdef class _Wrapper(object):
+ """
+ Wrapper to convert Python objects to ddwaf objects.
+
+ libddwaf represents scalar and composite values using ddwaf objects. This
+ wrapper converts Python objects to ddwaf objects by traversing all values
+ and their children. By default, the number of objects is limited to avoid
+ infinite loops. This limitation can be lifted on trusted data by setting
+ `max_objects` to `None`.
+
+ Under the hood, the wrapper uses an array of `ddwaf_object` allocated as a
+ single buffer. Objects such as maps or arrays refer to other objects that
+ are only part of this buffer. Strings are not copied, they live in the
+ Python heap and are referenced by the wrapper to avoid garbage collection.
+ """
+
+ cdef ddwaf_object *_ptr
+ cdef readonly object _string_refs
+ cdef readonly ssize_t _size
+ cdef readonly ssize_t _next_idx
+
+ def __init__(self, value, max_objects=5000):
+ self._string_refs = []
+ self._convert(value, max_objects)
+
+ cdef ssize_t _reserve_obj(self, ssize_t n=1) except -1:
+ """
+ Exponentially grows the size of the memory space used for objects.
+ Will stop if too much memory is allocated.
+ """
+ cdef ssize_t idx, i
+ cdef ddwaf_object *ptr
+ cdef ddwaf_object *obj
+
+ idx = self._next_idx
+ if idx + n > self._size:
+ while idx + n > self._size:
+ # grow 1.5 the previous size + an initial fixed size until
+ # it can accommodate at least n new objects
+ self._size += (self._size >> 1) + 128
+ ptr = <ddwaf_object *> PyMem_Realloc(self._ptr, self._size * sizeof(ddwaf_object))
+ if ptr == NULL:
+ raise MemoryError
+ memset(ptr + idx, 0, (self._size - idx) * sizeof(ddwaf_object))
+ if self._ptr != NULL and ptr != self._ptr:
+ # we need to patch all array objects because they use pointers to other objects
+ for i in range(idx):
+ obj = ptr + i
+ if (obj.type == DDWAF_OBJ_TYPE.DDWAF_OBJ_MAP or obj.type == DDWAF_OBJ_TYPE.DDWAF_OBJ_ARRAY) and obj.array != NULL:
+ obj.array = obj.array - self._ptr + ptr
+ self._ptr = ptr
+ self._next_idx += n
+ return idx
+
+ cdef int _make_string(self, ssize_t idx, object string) except -1:
+ cdef const char * ptr
+ cdef ssize_t length
+ cdef ddwaf_object *obj
+
+ self._string_refs.append(_string_to_bytes(string, &ptr, &length))
+
+ obj = self._ptr + idx
+ obj.type = DDWAF_OBJ_TYPE.DDWAF_OBJ_STRING
+ obj.stringValue = ptr
+ obj.nbEntries = length
+
+ cdef void _make_array(self, ssize_t idx, ssize_t array_idx, ssize_t nb_entries):
+ cdef ddwaf_object *obj
+ obj = self._ptr + idx
+ obj.type = DDWAF_OBJ_TYPE.DDWAF_OBJ_ARRAY
+ obj.array = self._ptr + array_idx
+ obj.nbEntries = nb_entries
+
+ cdef void _make_map(self, ssize_t idx, ssize_t array_idx, ssize_t nb_entries):
+ cdef ddwaf_object *obj
+ obj = self._ptr + idx
+ obj.type = DDWAF_OBJ_TYPE.DDWAF_OBJ_MAP
+ obj.array = self._ptr + array_idx
+ obj.nbEntries = nb_entries
+
+ cdef int _set_parameter(self, ssize_t idx, object string) except -1:
+ cdef const char * ptr
+ cdef ssize_t length
+ cdef ddwaf_object *obj
+
+ self._string_refs.append(_string_to_bytes(string, &ptr, &length))
+
+ obj = self._ptr + idx
+ obj.parameterName = ptr
+ obj.parameterNameLength = length
+
+ cdef void _convert(self, value, max_objects) except *:
+ cdef object stack
+ cdef ssize_t i, j, n, idx, items_idx
+
+ i = 0
+ stack = deque([(self._reserve_obj(), value)], maxlen=max_objects)
+ while len(stack) and (max_objects is None or i < <ssize_t?> max_objects):
+ idx, val = stack.popleft()
+
+ if isinstance(val, (int, float)):
+ val = str(val)
+
+ if isinstance(val, (six.binary_type, six.text_type)):
+ self._make_string(idx, val)
+
+ elif isinstance(val, Mapping):
+ n = len(val)
+ items_idx = self._reserve_obj(n)
+ self._make_map(idx, items_idx, n)
+ # size of val must not change!! should not happen
+ # while holding the GIL?
+ for j, (k, v) in enumerate(six.iteritems(val)):
+ if not isinstance(k, (six.binary_type, six.text_type)):
+ if isinstance(k, (int, float)):
+ k = str(k)
+ else:
+ continue
+ self._set_parameter(items_idx + j, k)
+ stack.append((items_idx + j, v))
+
+ elif isinstance(val, Sequence):
+ n = len(val)
+ items_idx = self._reserve_obj(n)
+ self._make_array(idx, items_idx, n)
+ stack.extend([(items_idx + j, val[j]) for j in range(n)])
+
+ i += 1
+
+ def __repr__(self):
+ return "<{0.__class__.__name__} for {0._next_idx} elements>".format(self)
+
+ def __sizeof__(self):
+ return super(_Wrapper, self).__sizeof__() + self._size * sizeof(ddwaf_object)
+
+ def __dealloc__(self):
+ PyMem_Free(self._ptr)
+
+
+cdef class DDWaf(object):
+ """
+ A DDWaf instance performs a matching operation on provided data according
+ to some rules.
+ """
+
+ cdef ddwaf_handle _handle
+ cdef object _rules
+
+ def __init__(self, rules):
+ cdef ddwaf_object* rule_objects
+ self._rules = _Wrapper(rules, max_objects=None)
+ rule_objects = (<_Wrapper?>self._rules)._ptr;
+ self._handle = ddwaf_init(rule_objects, NULL)
+ if <void *> self._handle == NULL:
+ raise ValueError("invalid rules")
+
+ @property
+ def required_data(self):
+ cdef uint32_t size
+ cdef const char* const* ptr
+
+ addresses = []
+ ptr = ddwaf_required_addresses(self._handle, &size)
+ for i in range(size):
+ addresses.append((<bytes> ptr[i]).decode("utf-8"))
+ return addresses
+
+ def run(self, data, timeout_ms=DEFAULT_DDWAF_TIMEOUT_MS):
+ cdef ddwaf_context ctx
+ cdef ddwaf_result result
+
+ ctx = ddwaf_context_init(self._handle, NULL)
+ if <void *> ctx == NULL:
+ raise RuntimeError
+ try:
+ wrapper = _Wrapper(data)
+ ddwaf_run(ctx, (<_Wrapper?>wrapper)._ptr, &result, <uint64_t?> timeout_ms * 1000)
+ if result.data != NULL:
+ return (<bytes> result.data).decode("utf-8")
+ finally:
+ ddwaf_result_free(&result)
+ ddwaf_context_destroy(ctx)
+
+ def __dealloc__(self):
+ ddwaf_destroy(self._handle)