Schema Extraction Preprocessor (#182)

CMakeLists.txt

@@ -78,13 +78,16 @@ set(LIBDDWAF_SOURCE
  ${libddwaf_SOURCE_DIR}/src/rule.cpp
  ${libddwaf_SOURCE_DIR}/src/ruleset_info.cpp
  ${libddwaf_SOURCE_DIR}/src/ip_utils.cpp
+ ${libddwaf_SOURCE_DIR}/src/preprocessor.cpp
  ${libddwaf_SOURCE_DIR}/src/iterator.cpp
  ${libddwaf_SOURCE_DIR}/src/log.cpp
  ${libddwaf_SOURCE_DIR}/src/obfuscator.cpp
+ ${libddwaf_SOURCE_DIR}/src/utils.cpp
  ${libddwaf_SOURCE_DIR}/src/waf.cpp
  ${libddwaf_SOURCE_DIR}/src/exclusion/input_filter.cpp
  ${libddwaf_SOURCE_DIR}/src/exclusion/object_filter.cpp
  ${libddwaf_SOURCE_DIR}/src/exclusion/rule_filter.cpp
+ ${libddwaf_SOURCE_DIR}/src/generator/extract_schema.cpp
  ${libddwaf_SOURCE_DIR}/src/parser/common.cpp
  ${libddwaf_SOURCE_DIR}/src/parser/parser.cpp
  ${libddwaf_SOURCE_DIR}/src/parser/parser_v1.cpp

include/ddwaf.h

@@ -164,6 +164,8 @@ struct _ddwaf_result
  ddwaf_object events;
  /** Array of actions generated, this is guaranteed to be an array **/
  ddwaf_object actions;
+ /** Map containing all derived objects in the format (address, value) **/
+ ddwaf_object derivatives;
  /** Total WAF runtime in nanoseconds **/
  uint64_t total_runtime;
 };
@@ -332,7 +334,6 @@ ddwaf_object* ddwaf_object_invalid(ddwaf_object *object);
  **/
 ddwaf_object* ddwaf_object_null(ddwaf_object *object);
 
-
 /**
  * ddwaf_object_string
  *

schema/types.json

@@ -0,0 +1,43 @@
+{
+ "title": "Serialized schema types for API Security",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$defs": {
+ "type": {
+ "oneOf": [
+ {
+ "enum": [0, 1, 2, 4, 8, 16],
+ "description": "scalar types"
+ },
+ {
+ "type": "array",
+ "description": "array of types",
+ "items": {
+ "$ref": "#"
+ },
+ "minItems": 1
+ },
+ {
+ "type": "object",
+ "description": "record type",
+ "additionalProperties": {
+ "$ref": "#"
+ }
+ }
+ ]
+ },
+ "metadata": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "pattern": "^[A-Za-z][A-Za-z0-9\\.\\-\\_:\\/]{0,199}$"
+ },
+ "minItems": 1
+ }
+ },
+ "type": "array",
+ "prefixItems": [
+ { "$ref": "#/$defs/type" },
+ { "$ref": "#/$defs/metadata" }
+ ],
+ "minItems": 1
+}

src/context.cpp

@@ -15,15 +15,14 @@
 
 namespace ddwaf {
 
-DDWAF_RET_CODE context::run(
- const ddwaf_object &newParameters, optional_ref<ddwaf_result> res, uint64_t timeout)
+DDWAF_RET_CODE context::run(ddwaf_object &input, optional_ref<ddwaf_result> res, uint64_t timeout)
 {
  if (res.has_value()) {
  ddwaf_result &output = *res;
  output = DDWAF_RESULT_INITIALISER;
  }
 
- if (!store_.insert(newParameters)) {
+ if (!store_.insert(input, ruleset_->free_fn)) {
  DDWAF_WARN("Illegal WAF call: parameter structure invalid!");
  return DDWAF_ERR_INVALID_OBJECT;
  }
@@ -48,8 +47,17 @@ DDWAF_RET_CODE context::run(
 
  const event_serializer serializer(*ruleset_->event_obfuscator);
 
+ optional_ref<ddwaf_object> derived;
+ if (res.has_value()) {
+ ddwaf_result &output = *res;
+ ddwaf_object_map(&output.derivatives);
+ derived.emplace(output.derivatives);
+ }
+
  memory::vector<ddwaf::event> events;
  try {
+ eval_preprocessors(derived, deadline);
+
  const auto &rules_to_exclude = filter_rules(deadline);
  const auto &objects_to_exclude = filter_inputs(rules_to_exclude, deadline);
  events = match(rules_to_exclude, objects_to_exclude, deadline);
@@ -96,6 +104,25 @@ const memory::unordered_map<rule *, filter_mode> &context::filter_rules(ddwaf::t
  return rules_to_exclude_;
 }
 
+void context::eval_preprocessors(optional_ref<ddwaf_object> &derived, ddwaf::timer &deadline)
+{
+ for (const auto &[id, preproc] : ruleset_->preprocessors) {
+ if (deadline.expired()) {
+ DDWAF_INFO("Ran out of time while evaluating preprocessors");
+ throw timeout_exception();
+ }
+
+ auto it = preprocessor_cache_.find(preproc.get());
+ if (it == preprocessor_cache_.end()) {
+ auto [new_it, res] =
+ preprocessor_cache_.emplace(preproc.get(), preprocessor::cache_type{});
+ it = new_it;
+ }
+
+ preproc->eval(store_, derived, it->second, deadline);
+ }
+}
+
 const memory::unordered_map<rule *, context::object_set> &context::filter_inputs(
  const memory::unordered_map<rule *, filter_mode> &rules_to_exclude, ddwaf::timer &deadline)
 {

src/context.hpp

@@ -28,7 +28,7 @@ class context {
 public:
  using object_set = std::unordered_set<const ddwaf_object *>;
 
- explicit context(ruleset::ptr ruleset) : ruleset_(std::move(ruleset)), store_(ruleset_->free_fn)
+ explicit context(ruleset::ptr ruleset) : ruleset_(std::move(ruleset))
  {
  rule_filter_cache_.reserve(ruleset_->rule_filters.size());
  input_filter_cache_.reserve(ruleset_->input_filters.size());
@@ -41,8 +41,9 @@ class context {
  context &operator=(context &&) = delete;
  ~context() = default;
 
- DDWAF_RET_CODE run(const ddwaf_object &, optional_ref<ddwaf_result>, uint64_t);
+ DDWAF_RET_CODE run(ddwaf_object &, optional_ref<ddwaf_result>, uint64_t);
 
+ void eval_preprocessors(optional_ref<ddwaf_object> &derived, ddwaf::timer &deadline);
  // These two functions below return references to internal objects,
  // however using them this way helps with testing
  const memory::unordered_map<rule *, filter_mode> &filter_rules(ddwaf::timer &deadline);
@@ -62,7 +63,9 @@ class context {
  using input_filter = exclusion::input_filter;
  using rule_filter = exclusion::rule_filter;
 
- // Cache of filters and conditions
+ memory::unordered_map<preprocessor *, preprocessor::cache_type> preprocessor_cache_;
+
+ // Caches of filters and conditions
  memory::unordered_map<rule_filter *, rule_filter::cache_type> rule_filter_cache_;
  memory::unordered_map<input_filter *, input_filter::cache_type> input_filter_cache_;
 
@@ -94,7 +97,7 @@ class context_wrapper {
  context_wrapper &operator=(context_wrapper &&) noexcept = delete;
  context_wrapper &operator=(const context_wrapper &) = delete;
 
- DDWAF_RET_CODE run(const ddwaf_object &data, optional_ref<ddwaf_result> res, uint64_t timeout)
+ DDWAF_RET_CODE run(ddwaf_object &data, optional_ref<ddwaf_result> res, uint64_t timeout)
  {
  memory::memory_resource_guard guard(&mr_);
  return ctx_->run(data, res, timeout);

src/context_allocator.hpp

@@ -7,6 +7,7 @@
 #pragma once
 
 #include "memory_resource.hpp"
+#include <list>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -104,4 +105,6 @@ using unordered_map =
 template <class T, class Hash = std::hash<T>, class Pred = std::equal_to<T>>
 using unordered_set = std::unordered_set<T, Hash, Pred, context_allocator<T>>;
 
+template <class T> using list = std::list<T, context_allocator<T>>;
+
 } // namespace ddwaf::memory

src/exclusion/object_filter.cpp

@@ -104,7 +104,7 @@ memory::unordered_set<const ddwaf_object *> object_filter::match(
  continue;
  }
 
- const auto *object = store.get_target(target);
+ auto *object = store.get_target(target);
  if (object == nullptr) {
  continue;
  }

src/expression.cpp

@@ -137,7 +137,7 @@ std::optional<event::match> expression::evaluator::eval_condition(
  continue;
  }
 
- const ddwaf_object *object = store.get_target(target.root);
+ auto *object = store.get_target(target.root);
  if (object == nullptr) {
  continue;
  }

src/generator/base.hpp

@@ -0,0 +1,32 @@
+// Unless explicitly stated otherwise all files in this repository are
+// dual-licensed under the Apache-2.0 License or BSD-3-Clause License.
+//
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include <utils.hpp>
+
+namespace ddwaf::generator {
+
+class base {
+public:
+ using ptr = std::unique_ptr<base>;
+
+ base() = default;
+ virtual ~base() = default;
+ base(const base &) = default;
+ base(base &&) = default;
+ base &operator=(const base &) = default;
+ base &operator=(base &&) = default;
+
+ virtual ddwaf_object generate(const ddwaf_object *input) = 0;
+};
+
+} // namespace ddwaf::generator