diff --git a/cmake/objects.cmake b/cmake/objects.cmake index b793404d7..0c50f21ff 100644 --- a/cmake/objects.cmake +++ b/cmake/objects.cmake @@ -46,6 +46,7 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/parser/scanner_parser.cpp ${libddwaf_SOURCE_DIR}/src/parser/exclusion_parser.cpp ${libddwaf_SOURCE_DIR}/src/processor/extract_schema.cpp + ${libddwaf_SOURCE_DIR}/src/processor/fingerprint.cpp ${libddwaf_SOURCE_DIR}/src/condition/lfi_detector.cpp ${libddwaf_SOURCE_DIR}/src/condition/sqli_detector.cpp ${libddwaf_SOURCE_DIR}/src/condition/ssrf_detector.cpp diff --git a/src/builder/processor_builder.cpp b/src/builder/processor_builder.cpp index b81de6337..7ac0a572c 100644 --- a/src/builder/processor_builder.cpp +++ b/src/builder/processor_builder.cpp @@ -8,6 +8,7 @@ #include "builder/processor_builder.hpp" #include "processor/extract_schema.hpp" +#include "processor/fingerprint.hpp" namespace ddwaf { @@ -42,6 +43,14 @@ template <> struct typed_processor_builder { } }; +template <> struct typed_processor_builder { + std::shared_ptr build(const auto &spec) + { + return std::make_shared( + spec.id, spec.expr, spec.mappings, spec.evaluate, spec.output); + } +}; + template concept has_build_with_scanners = requires(typed_processor_builder b, Spec spec, Scanners scanners) { @@ -70,6 +79,8 @@ template switch (type) { case processor_type::extract_schema: return build_with_type(*this, scanners); + case processor_type::http_endpoint_fingerprint: + return build_with_type(*this, scanners); default: break; } diff --git a/src/builder/processor_builder.hpp b/src/builder/processor_builder.hpp index f2d220e1b..0ab3be0f0 100644 --- a/src/builder/processor_builder.hpp +++ b/src/builder/processor_builder.hpp @@ -19,10 +19,10 @@ namespace ddwaf { enum class processor_type : unsigned { extract_schema, // Reserved - http_fingerprint, + http_endpoint_fingerprint, + http_network_fingerprint, + http_header_fingerprint, session_fingerprint, - network_fingerprint, - header_fingerprint, }; struct processor_builder { diff --git a/src/obfuscator.hpp b/src/obfuscator.hpp index 3dd01ad1d..dfa404a60 100644 --- a/src/obfuscator.hpp +++ b/src/obfuscator.hpp @@ -29,7 +29,7 @@ class obfuscator { static constexpr std::string_view redaction_msg{""}; static constexpr std::string_view default_key_regex_str{ - R"((?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key)|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"}; + R"((?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"}; protected: std::unique_ptr key_regex{nullptr}; diff --git a/src/parser/processor_parser.cpp b/src/parser/processor_parser.cpp index abac7e45d..8b9a9a45e 100644 --- a/src/parser/processor_parser.cpp +++ b/src/parser/processor_parser.cpp @@ -8,13 +8,15 @@ #include "parser/common.hpp" #include "parser/parser.hpp" #include "processor/base.hpp" +#include "processor/extract_schema.hpp" +#include "processor/fingerprint.hpp" #include namespace ddwaf::parser::v2 { namespace { std::vector parse_processor_mappings( - const parameter::vector &root, address_container &addresses) + const parameter::vector &root, address_container &addresses, const auto ¶m_names) { if (root.empty()) { throw ddwaf::parsing_error("empty mappings"); @@ -24,22 +26,26 @@ std::vector parse_processor_mappings( for (const auto &node : root) { auto mapping = static_cast(node); - // TODO support n:1 mappings and key paths - auto inputs = at(mapping, "inputs"); - if (inputs.empty()) { - throw ddwaf::parsing_error("empty processor input mapping"); - } + std::vector parameters; + for (const auto ¶m : param_names) { + // TODO support n:1 mappings and key paths + auto inputs = at(mapping, param); + if (inputs.empty()) { + throw ddwaf::parsing_error("empty processor input mapping"); + } - auto input = static_cast(inputs[0]); - auto input_address = at(input, "address"); - auto output = at(mapping, "output"); + auto input = static_cast(inputs[0]); + auto input_address = at(input, "address"); - addresses.optional.emplace(input_address); + addresses.optional.emplace(input_address); + parameters.emplace_back(processor_parameter{ + {processor_target{get_target_index(input_address), std::move(input_address), {}}}}); + } + + auto output = at(mapping, "output"); mappings.emplace_back(processor_mapping{ - {processor_parameter{ - {processor_target{get_target_index(input_address), std::move(input_address), {}}}}}, - {get_target_index(output), std::move(output), {}}}); + std::move(parameters), {get_target_index(output), std::move(output), {}}}); } return mappings; @@ -71,6 +77,20 @@ processor_container parse_processors( auto generator_id = at(node, "generator"); if (generator_id == "extract_schema") { type = processor_type::extract_schema; + } else if (generator_id == "http_endpoint_fingerprint") { + type = processor_type::http_endpoint_fingerprint; + } else if (generator_id == "http_network_fingerprint") { + type = processor_type::http_network_fingerprint; + // Skip for now + continue; + } else if (generator_id == "http_header_fingerprint") { + type = processor_type::http_header_fingerprint; + // Skip for now + continue; + } else if (generator_id == "session_fingerprint") { + type = processor_type::session_fingerprint; + // Skip for now + continue; } else { DDWAF_WARN("Unknown generator: {}", generator_id); info.add_failed(id, "unknown generator '" + generator_id + "'"); @@ -82,7 +102,14 @@ processor_container parse_processors( auto params = at(node, "parameters"); auto mappings_vec = at(params, "mappings"); - auto mappings = parse_processor_mappings(mappings_vec, addresses); + std::vector mappings; + if (type == processor_type::extract_schema) { + mappings = + parse_processor_mappings(mappings_vec, addresses, extract_schema::param_names); + } else { + mappings = parse_processor_mappings( + mappings_vec, addresses, http_endpoint_fingerprint::param_names); + } std::vector scanners; auto scanners_ref_array = at(params, "scanners", {}); diff --git a/src/processor/fingerprint.cpp b/src/processor/fingerprint.cpp new file mode 100644 index 000000000..774fc728c --- /dev/null +++ b/src/processor/fingerprint.cpp @@ -0,0 +1,301 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "processor/fingerprint.hpp" +#include "sha256.hpp" +#include "transformer/lowercase.hpp" +#include + +namespace ddwaf { +namespace { + +struct string_buffer { + explicit string_buffer(std::size_t length) + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,hicpp-no-malloc,cppcoreguidelines-pro-type-reinterpret-cast) + : buffer(reinterpret_cast(malloc(sizeof(char) * length))), length(length) + { + if (buffer == nullptr) { + throw std::bad_alloc{}; + } + } + + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,hicpp-no-malloc,cppcoreguidelines-pro-type-reinterpret-cast) + ~string_buffer() { free(buffer); } + + string_buffer(const string_buffer &) = delete; + string_buffer(string_buffer &&) = delete; + + string_buffer &operator=(const string_buffer &) = delete; + string_buffer &operator=(string_buffer &&) = delete; + + template + [[nodiscard]] std::span subspan() + requires(N > 0) + { + if ((index + N - 1) >= length) { + throw std::out_of_range("span[index, N) beyond buffer limit"); + } + + std::span res{&buffer[index], N}; + index += N; + return res; + } + + void append(std::string_view str) + { + if (str.empty()) { + return; + } + + if ((index + str.length() - 1) >= length) { + throw std::out_of_range("appending string beyond buffer limit"); + } + memcpy(&buffer[index], str.data(), str.size()); + index += str.size(); + } + + void append_lowercase(std::string_view str) + { + if (str.empty()) { + return; + } + + if ((index + str.length() - 1) >= length) { + throw std::out_of_range("appending string beyond buffer limit"); + } + + for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } + } + + template + void append(std::array str) + requires(N > 0) + { + append(std::string_view{str.data(), N}); + } + + template + void append_lowercase(std::array str) + requires(N > 0) + { + append_lowercase(std::string_view{str.data(), N}); + } + + void append(char c) { append(std::string_view{&c, 1}); } + + std::pair move() + { + auto *ptr = buffer; + buffer = nullptr; + return {ptr, index}; + } + + char *buffer{nullptr}; + std::size_t index{0}; + std::size_t length; +}; + +struct field_generator { + field_generator() = default; + virtual ~field_generator() = default; + field_generator(const field_generator &) = default; + field_generator(field_generator &&) = default; + field_generator &operator=(const field_generator &) = default; + field_generator &operator=(field_generator &&) = default; + + virtual std::size_t length() = 0; + virtual void operator()(string_buffer &output) = 0; +}; + +struct string_field : field_generator { + explicit string_field(std::string_view input) : value(input) {} + ~string_field() override = default; + string_field(const string_field &) = default; + string_field(string_field &&) = default; + string_field &operator=(const string_field &) = default; + string_field &operator=(string_field &&) = default; + + std::size_t length() override { return value.size(); } + void operator()(string_buffer &output) override { output.append_lowercase(value); } + + std::string_view value; +}; + +struct string_hash_field : field_generator { + explicit string_hash_field(std::string_view input) : value(input) {} + ~string_hash_field() override = default; + string_hash_field(const string_hash_field &) = default; + string_hash_field(string_hash_field &&) = default; + string_hash_field &operator=(const string_hash_field &) = default; + string_hash_field &operator=(string_hash_field &&) = default; + + std::size_t length() override { return 8; } + void operator()(string_buffer &output) override; + + std::string_view value; +}; + +struct key_hash_field : field_generator { + explicit key_hash_field(const ddwaf_object &input) : value(input) {} + ~key_hash_field() override = default; + key_hash_field(const key_hash_field &) = default; + key_hash_field(key_hash_field &&) = default; + key_hash_field &operator=(const key_hash_field &) = default; + key_hash_field &operator=(key_hash_field &&) = default; + + std::size_t length() override { return value.type == DDWAF_OBJ_MAP ? 8 : 0; } + void operator()(string_buffer &output) override; + + ddwaf_object value; +}; + +template std::size_t generate_fragment_length(Generators &...generators) +{ + static_assert(sizeof...(generators) > 0, "At least one generator is required"); + return (generators.length() + ...) + sizeof...(generators) - 1; +} + +template +void generate_fragment_field(string_buffer &buffer, T &generator, Rest... rest) +{ + generator(buffer); + if constexpr (sizeof...(rest) > 0) { + buffer.append('-'); + generate_fragment_field(buffer, rest...); + } +} + +template +ddwaf_object generate_fragment(std::string_view header, Generators... generators) +{ + std::size_t total_length = header.size() + 1 + generate_fragment_length(generators...); + + string_buffer buffer{total_length}; + buffer.append_lowercase(header); + buffer.append('-'); + + generate_fragment_field(buffer, generators...); + + ddwaf_object res; + auto [ptr, size] = buffer.move(); + ddwaf_object_stringl_nc(&res, ptr, size); + + return res; +} + +// Return true if the first argument is less than (i.e. is ordered before) the second +bool str_casei_cmp(std::string_view left, std::string_view right) +{ + auto n = std::min(left.size(), right.size()); + for (std::size_t i = 0; i < n; ++i) { + auto lc = ddwaf::tolower(left[i]); + auto rc = ddwaf::tolower(right[i]); + if (lc != rc) { + return lc < rc; + } + } + return left.size() <= right.size(); +} + +void normalize_string(std::string_view key, std::string &buffer, bool trailing_separator) +{ + buffer.clear(); + + if (buffer.capacity() < key.size()) { + // Add space for the extra comma, just in case + buffer.reserve(key.size() + 1); + } + + for (auto c : key) { + if (c == ',') { + buffer.append(R"(\,)"); + } else { + buffer.append(1, ddwaf::tolower(c)); + } + } + + if (trailing_separator) { + buffer.append(1, ','); + } +} + +void string_hash_field::operator()(string_buffer &output) +{ + if (value.empty()) { + return; + } + + cow_string value_lc{value}; + transformer::lowercase::transform(value_lc); + + sha256_hash hasher; + hasher << static_cast(value_lc); + + hasher.write_digest(output.subspan<8>()); +} + +void key_hash_field::operator()(string_buffer &output) +{ + if (value.type != DDWAF_OBJ_MAP or value.nbEntries == 0) { + return; + } + + std::vector keys; + keys.reserve(value.nbEntries); + + for (unsigned i = 0; i < value.nbEntries; ++i) { + const auto &child = value.array[i]; + + std::string_view key{ + child.parameterName, static_cast(child.parameterNameLength)}; + + keys.emplace_back(key); + } + + std::sort(keys.begin(), keys.end(), str_casei_cmp); + + sha256_hash hasher; + std::string normalized; + for (unsigned i = 0; i < keys.size(); ++i) { + normalize_string(keys[i], normalized, (i + 1) < keys.size()); + hasher << normalized; + } + + hasher.write_digest(output.subspan<8>()); +} + +} // namespace + +// NOLINTNEXTLINE(readability-convert-member-functions-to-static) +std::pair http_endpoint_fingerprint::eval_impl( + const unary_argument &method, const unary_argument &uri_raw, + const unary_argument &query, + const unary_argument &body, ddwaf::timer &deadline) const +{ + if (deadline.expired()) { + throw ddwaf::timeout_exception(); + } + + // Strip query parameter from raw URI + auto stripped_uri = uri_raw.value; + auto query_or_frag_idx = stripped_uri.find_first_of("?#"); + if (query_or_frag_idx != std::string_view::npos) { + stripped_uri = stripped_uri.substr(0, query_or_frag_idx); + } + + ddwaf_object res; + ddwaf_object_invalid(&res); + try { + res = generate_fragment("http", string_field{method.value}, string_hash_field{stripped_uri}, + key_hash_field{*query.value}, key_hash_field{*body.value}); + } catch (const std::out_of_range &e) { + DDWAF_WARN("Failed to generate http endpoint fingerprint: {}", e.what()); + } + + return {res, object_store::attribute::none}; +} + +} // namespace ddwaf diff --git a/src/processor/fingerprint.hpp b/src/processor/fingerprint.hpp new file mode 100644 index 000000000..c6fb823bc --- /dev/null +++ b/src/processor/fingerprint.hpp @@ -0,0 +1,81 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include +#include +#include +#include + +#include "processor/base.hpp" +#include "scanner.hpp" +#include "utils.hpp" + +namespace ddwaf { + +class http_endpoint_fingerprint : public structured_processor { +public: + static constexpr std::array param_names{ + "method", "uri_raw", "query", "body"}; + + http_endpoint_fingerprint(std::string id, std::shared_ptr expr, + std::vector mappings, bool evaluate, bool output) + : structured_processor( + std::move(id), std::move(expr), std::move(mappings), evaluate, output) + {} + + std::pair eval_impl( + const unary_argument &method, + const unary_argument &uri_raw, + const unary_argument &query, + const unary_argument &body, ddwaf::timer &deadline) const; +}; + +class http_header_fingerprint : public structured_processor { +public: + static constexpr std::array param_names{"headers"}; + + http_header_fingerprint(std::string id, std::shared_ptr expr, + std::vector mappings, bool evaluate, bool output) + : structured_processor( + std::move(id), std::move(expr), std::move(mappings), evaluate, output) + {} + + std::pair eval_impl( + const unary_argument &headers, ddwaf::timer &deadline) const; +}; + +class http_network_fingerprint : public structured_processor { +public: + static constexpr std::array param_names{"headers"}; + + http_network_fingerprint(std::string id, std::shared_ptr expr, + std::vector mappings, bool evaluate, bool output) + : structured_processor( + std::move(id), std::move(expr), std::move(mappings), evaluate, output) + {} + + std::pair eval_impl( + const unary_argument &headers, ddwaf::timer &deadline) const; +}; + +class session_fingerprint : public structured_processor { +public: + static constexpr std::array param_names{ + "cookies", "session_id", "user_id"}; + + session_fingerprint(std::string id, std::shared_ptr expr, + std::vector mappings, bool evaluate, bool output) + : structured_processor( + std::move(id), std::move(expr), std::move(mappings), evaluate, output) + {} + + std::pair eval_impl( + const unary_argument &cookies, + const unary_argument &session_id, + const unary_argument &user_id, ddwaf::timer &deadline) const; +}; + +} // namespace ddwaf diff --git a/src/sha256.cpp b/src/sha256.cpp index 7ef19a7ee..cabc8bb51 100644 --- a/src/sha256.cpp +++ b/src/sha256.cpp @@ -15,6 +15,9 @@ // #include "sha256.hpp" +#include +#include +#include namespace ddwaf { namespace { @@ -84,9 +87,6 @@ constexpr std::array K256 = {0x428a2f98UL, 0x71374491UL, 0xb5c0fbc 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL}; -// Length of the string representation of the digest -constexpr std::size_t sha_digest_length = 64; - } // namespace sha256_hash &sha256_hash::operator<<(std::string_view str) @@ -145,7 +145,22 @@ sha256_hash &sha256_hash::operator<<(std::string_view str) return *this; } +template std::string sha256_hash::digest() + requires(DigestLength % 8 == 0 && DigestLength <= 64) +{ + std::string final_digest; + final_digest.resize(DigestLength, 0); + write_digest(std::span{final_digest}); + return final_digest; +} + +template std::string sha256_hash::digest<64>(); +template std::string sha256_hash::digest<8>(); + +template +void sha256_hash::write_digest(std::span output) + requires(DigestLength % 8 == 0 && DigestLength <= 64) { auto *p = buffer.data(); size_t n = num; @@ -178,25 +193,25 @@ std::string sha256_hash::digest() num = 0; memset(p, 0, block_size); - std::array final_digest{0}; - for (unsigned int nn = 0; nn < sha_digest_length; nn += 8) { + for (unsigned int nn = 0; nn < DigestLength; nn += 8) { uint32_t ll = hash[nn >> 3]; - final_digest[nn + 0] = UINT8_TO_HEX_CHAR(static_cast((ll >> 28) & 0x0f)); - final_digest[nn + 1] = UINT8_TO_HEX_CHAR(static_cast((ll >> 24) & 0x0f)); - final_digest[nn + 2] = UINT8_TO_HEX_CHAR(static_cast((ll >> 20) & 0x0f)); - final_digest[nn + 3] = UINT8_TO_HEX_CHAR(static_cast((ll >> 16) & 0x0f)); - final_digest[nn + 4] = UINT8_TO_HEX_CHAR(static_cast((ll >> 12) & 0x0f)); - final_digest[nn + 5] = UINT8_TO_HEX_CHAR(static_cast((ll >> 8) & 0x0f)); - final_digest[nn + 6] = UINT8_TO_HEX_CHAR(static_cast((ll >> 4) & 0x0f)); - final_digest[nn + 7] = UINT8_TO_HEX_CHAR(static_cast(ll & 0x0f)); + output[nn + 0] = UINT8_TO_HEX_CHAR(static_cast((ll >> 28) & 0x0f)); + output[nn + 1] = UINT8_TO_HEX_CHAR(static_cast((ll >> 24) & 0x0f)); + output[nn + 2] = UINT8_TO_HEX_CHAR(static_cast((ll >> 20) & 0x0f)); + output[nn + 3] = UINT8_TO_HEX_CHAR(static_cast((ll >> 16) & 0x0f)); + output[nn + 4] = UINT8_TO_HEX_CHAR(static_cast((ll >> 12) & 0x0f)); + output[nn + 5] = UINT8_TO_HEX_CHAR(static_cast((ll >> 8) & 0x0f)); + output[nn + 6] = UINT8_TO_HEX_CHAR(static_cast((ll >> 4) & 0x0f)); + output[nn + 7] = UINT8_TO_HEX_CHAR(static_cast(ll & 0x0f)); } // Reset the hasher and return reset(); - - return std::string{final_digest.data(), 64}; } +template void sha256_hash::write_digest<8>(std::span output); +template void sha256_hash::write_digest<64>(std::span output); + void sha256_hash::sha_block_data_order(const uint8_t *data, size_t len) { unsigned int a; diff --git a/src/sha256.hpp b/src/sha256.hpp index fe03e7c36..59bdcd69a 100644 --- a/src/sha256.hpp +++ b/src/sha256.hpp @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -21,7 +22,13 @@ class sha256_hash { sha256_hash &operator=(sha256_hash &&) noexcept = delete; sha256_hash &operator<<(std::string_view str); - [[nodiscard]] std::string digest(); + template + [[nodiscard]] std::string digest() + requires(N % 8 == 0 && N <= 64); + + template + void write_digest(std::span output) + requires(N % 8 == 0 && N <= 64); void reset() { @@ -38,7 +45,6 @@ class sha256_hash { 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; void sha_block_data_order(const uint8_t *data, size_t len); - std::array hash{initial_hash_values}; uint32_t length_low{0}; uint32_t length_high{0}; diff --git a/tests/integration/processors/ruleset/postprocessor.json b/tests/integration/processors/extract_schema/ruleset/postprocessor.json similarity index 100% rename from tests/integration/processors/ruleset/postprocessor.json rename to tests/integration/processors/extract_schema/ruleset/postprocessor.json diff --git a/tests/integration/processors/ruleset/preprocessor.json b/tests/integration/processors/extract_schema/ruleset/preprocessor.json similarity index 100% rename from tests/integration/processors/ruleset/preprocessor.json rename to tests/integration/processors/extract_schema/ruleset/preprocessor.json diff --git a/tests/integration/processors/ruleset/processor.json b/tests/integration/processors/extract_schema/ruleset/processor.json similarity index 100% rename from tests/integration/processors/ruleset/processor.json rename to tests/integration/processors/extract_schema/ruleset/processor.json diff --git a/tests/integration/processors/ruleset/processor_with_scanner_by_id.json b/tests/integration/processors/extract_schema/ruleset/processor_with_scanner_by_id.json similarity index 100% rename from tests/integration/processors/ruleset/processor_with_scanner_by_id.json rename to tests/integration/processors/extract_schema/ruleset/processor_with_scanner_by_id.json diff --git a/tests/integration/processors/ruleset/processor_with_scanner_by_tags.json b/tests/integration/processors/extract_schema/ruleset/processor_with_scanner_by_tags.json similarity index 100% rename from tests/integration/processors/ruleset/processor_with_scanner_by_tags.json rename to tests/integration/processors/extract_schema/ruleset/processor_with_scanner_by_tags.json diff --git a/tests/integration/processors/test.cpp b/tests/integration/processors/extract_schema/test.cpp similarity index 97% rename from tests/integration/processors/test.cpp rename to tests/integration/processors/extract_schema/test.cpp index 7d6fefd12..fdaed7461 100644 --- a/tests/integration/processors/test.cpp +++ b/tests/integration/processors/extract_schema/test.cpp @@ -4,14 +4,14 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2021 Datadog, Inc. -#include "../../test_utils.hpp" +#include "../../../test_utils.hpp" using namespace ddwaf; namespace { -constexpr std::string_view base_dir = "integration/processors/"; +constexpr std::string_view base_dir = "integration/processors/extract_schema"; -TEST(TestProcessors, Postprocessor) +TEST(TestExtractSchemaIntegration, Postprocessor) { auto rule = read_json_file("postprocessor.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -55,7 +55,7 @@ TEST(TestProcessors, Postprocessor) ddwaf_destroy(handle); } -TEST(TestProcessors, Preprocessor) +TEST(TestExtractSchemaIntegration, Preprocessor) { auto rule = read_json_file("preprocessor.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -108,7 +108,7 @@ TEST(TestProcessors, Preprocessor) ddwaf_destroy(handle); } -TEST(TestProcessors, Processor) +TEST(TestExtractSchemaIntegration, Processor) { auto rule = read_json_file("processor.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -166,7 +166,7 @@ TEST(TestProcessors, Processor) ddwaf_destroy(handle); } -TEST(TestProcessors, ProcessorWithScannerByTags) +TEST(TestExtractSchemaIntegration, ProcessorWithScannerByTags) { auto rule = read_json_file("processor_with_scanner_by_tags.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -203,7 +203,7 @@ TEST(TestProcessors, ProcessorWithScannerByTags) ddwaf_destroy(handle); } -TEST(TestProcessors, ProcessorWithScannerByID) +TEST(TestExtractSchemaIntegration, ProcessorWithScannerByID) { auto rule = read_json_file("processor_with_scanner_by_id.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -241,7 +241,7 @@ TEST(TestProcessors, ProcessorWithScannerByID) ddwaf_destroy(handle); } -TEST(TestProcessors, ProcessorUpdate) +TEST(TestExtractSchemaIntegration, ProcessorUpdate) { auto rule = read_json_file("processor_with_scanner_by_tags.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -318,7 +318,7 @@ TEST(TestProcessors, ProcessorUpdate) ddwaf_destroy(handle); } -TEST(TestProcessors, ScannerUpdate) +TEST(TestExtractSchemaIntegration, ScannerUpdate) { auto rule = read_json_file("processor_with_scanner_by_tags.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -397,7 +397,7 @@ TEST(TestProcessors, ScannerUpdate) ddwaf_destroy(handle); } -TEST(TestProcessors, ProcessorAndScannerUpdate) +TEST(TestExtractSchemaIntegration, ProcessorAndScannerUpdate) { auto rule = read_json_file("processor_with_scanner_by_tags.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -475,7 +475,7 @@ TEST(TestProcessors, ProcessorAndScannerUpdate) ddwaf_destroy(handle); } -TEST(TestProcessors, EmptyScannerUpdate) +TEST(TestExtractSchemaIntegration, EmptyScannerUpdate) { auto rule = read_json_file("processor_with_scanner_by_tags.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -552,7 +552,7 @@ TEST(TestProcessors, EmptyScannerUpdate) ddwaf_destroy(handle); } -TEST(TestProcessors, EmptyProcessorUpdate) +TEST(TestExtractSchemaIntegration, EmptyProcessorUpdate) { auto rule = read_json_file("processor_with_scanner_by_tags.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -627,7 +627,7 @@ TEST(TestProcessors, EmptyProcessorUpdate) ddwaf_destroy(handle); } -TEST(TestProcessors, PostprocessorWithEphemeralMapping) +TEST(TestExtractSchemaIntegration, PostprocessorWithEphemeralMapping) { auto rule = read_json_file("postprocessor.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -694,7 +694,7 @@ TEST(TestProcessors, PostprocessorWithEphemeralMapping) ddwaf_destroy(handle); } -TEST(TestProcessors, PreprocessorWithEphemeralMapping) +TEST(TestExtractSchemaIntegration, PreprocessorWithEphemeralMapping) { auto rule = read_json_file("preprocessor.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -777,7 +777,7 @@ TEST(TestProcessors, PreprocessorWithEphemeralMapping) ddwaf_destroy(handle); } -TEST(TestProcessors, ProcessorEphemeralExpression) +TEST(TestExtractSchemaIntegration, ProcessorEphemeralExpression) { auto rule = read_json_file("processor.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); diff --git a/tests/integration/processors/fingerprint/ruleset/postprocessor.json b/tests/integration/processors/fingerprint/ruleset/postprocessor.json new file mode 100644 index 000000000..4d210d4e3 --- /dev/null +++ b/tests/integration/processors/fingerprint/ruleset/postprocessor.json @@ -0,0 +1,82 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.8.0" + }, + "rules": [ + { + "id": "rule1", + "name": "rule1", + "tags": { + "type": "flow1", + "category": "category1" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.body" + } + ], + "value": 8, + "type": "unsigned" + }, + "operator": "equals" + } + ] + } + ], + "processors": [ + { + "id": "processor-001", + "generator": "http_endpoint_fingerprint", + "conditions": [ + { + "operator": "equals", + "parameters": { + "inputs": [ + { + "address": "waf.context.processor", + "key_path": [ + "fingerprint" + ] + } + ], + "value": true, + "type": "boolean" + } + } + ], + "parameters": { + "mappings": [ + { + "method": [ + { + "address": "server.request.method" + } + ], + "uri_raw": [ + { + "address": "server.request.uri.raw" + } + ], + "body": [ + { + "address": "server.request.body" + } + ], + "query": [ + { + "address": "server.request.query" + } + ], + "output": "_dd.appsec.fp.http.endpoint" + } + ] + }, + "evaluate": false, + "output": true + } + ] +} diff --git a/tests/integration/processors/fingerprint/ruleset/preprocessor.json b/tests/integration/processors/fingerprint/ruleset/preprocessor.json new file mode 100644 index 000000000..d3d4ce0df --- /dev/null +++ b/tests/integration/processors/fingerprint/ruleset/preprocessor.json @@ -0,0 +1,81 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.8.0" + }, + "rules": [ + { + "id": "rule1", + "name": "rule1", + "tags": { + "type": "flow1", + "category": "category1" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "_dd.appsec.fp.http.endpoint" + } + ], + "regex": ".*" + }, + "operator": "match_regex" + } + ] + } + ], + "processors": [ + { + "id": "processor-001", + "generator": "http_endpoint_fingerprint", + "conditions": [ + { + "operator": "equals", + "parameters": { + "inputs": [ + { + "address": "waf.context.processor", + "key_path": [ + "fingerprint" + ] + } + ], + "value": true, + "type": "boolean" + } + } + ], + "parameters": { + "mappings": [ + { + "method": [ + { + "address": "server.request.method" + } + ], + "uri_raw": [ + { + "address": "server.request.uri.raw" + } + ], + "body": [ + { + "address": "server.request.body" + } + ], + "query": [ + { + "address": "server.request.query" + } + ], + "output": "_dd.appsec.fp.http.endpoint" + } + ] + }, + "evaluate": true, + "output": false + } + ] +} diff --git a/tests/integration/processors/fingerprint/ruleset/processor.json b/tests/integration/processors/fingerprint/ruleset/processor.json new file mode 100644 index 000000000..91c86875e --- /dev/null +++ b/tests/integration/processors/fingerprint/ruleset/processor.json @@ -0,0 +1,81 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.8.0" + }, + "rules": [ + { + "id": "rule1", + "name": "rule1", + "tags": { + "type": "flow1", + "category": "category1" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "_dd.appsec.fp.http.endpoint" + } + ], + "regex": ".*" + }, + "operator": "match_regex" + } + ] + } + ], + "processors": [ + { + "id": "processor-001", + "generator": "http_endpoint_fingerprint", + "conditions": [ + { + "operator": "equals", + "parameters": { + "inputs": [ + { + "address": "waf.context.processor", + "key_path": [ + "fingerprint" + ] + } + ], + "value": true, + "type": "boolean" + } + } + ], + "parameters": { + "mappings": [ + { + "method": [ + { + "address": "server.request.method" + } + ], + "uri_raw": [ + { + "address": "server.request.uri.raw" + } + ], + "body": [ + { + "address": "server.request.body" + } + ], + "query": [ + { + "address": "server.request.query" + } + ], + "output": "_dd.appsec.fp.http.endpoint" + } + ] + }, + "evaluate": true, + "output": true + } + ] +} diff --git a/tests/integration/processors/fingerprint/test.cpp b/tests/integration/processors/fingerprint/test.cpp new file mode 100644 index 000000000..5ec090b55 --- /dev/null +++ b/tests/integration/processors/fingerprint/test.cpp @@ -0,0 +1,202 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "../../../test_utils.hpp" + +using namespace ddwaf; + +namespace { +constexpr std::string_view base_dir = "integration/processors/fingerprint"; + +TEST(TestFingerprintIntegration, Postprocessor) +{ + auto rule = read_json_file("postprocessor.json", base_dir); + ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); + ddwaf_handle handle = ddwaf_init(&rule, nullptr, nullptr); + ASSERT_NE(handle, nullptr); + ddwaf_object_free(&rule); + + uint32_t size; + const char *const *addresses = ddwaf_known_addresses(handle, &size); + EXPECT_EQ(size, 5); + std::unordered_set address_set(addresses, addresses + size); + EXPECT_TRUE(address_set.contains("server.request.body")); + EXPECT_TRUE(address_set.contains("server.request.uri.raw")); + EXPECT_TRUE(address_set.contains("server.request.method")); + EXPECT_TRUE(address_set.contains("server.request.query")); + EXPECT_TRUE(address_set.contains("waf.context.processor")); + + ddwaf_context context = ddwaf_context_init(handle); + ASSERT_NE(context, nullptr); + + ddwaf_object tmp; + + ddwaf_object map = DDWAF_OBJECT_MAP; + ddwaf_object settings = DDWAF_OBJECT_MAP; + + ddwaf_object body = DDWAF_OBJECT_MAP; + ddwaf_object_map_add(&body, "key", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&map, "server.request.body", &body); + + ddwaf_object query = DDWAF_OBJECT_MAP; + ddwaf_object_map_add(&query, "key", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&map, "server.request.query", &query); + + ddwaf_object_map_add( + &map, "server.request.uri.raw", ddwaf_object_string(&tmp, "/path/to/resource/?key=")); + ddwaf_object_map_add(&map, "server.request.method", ddwaf_object_string(&tmp, "PuT")); + + ddwaf_object_map_add(&settings, "fingerprint", ddwaf_object_bool(&tmp, true)); + ddwaf_object_map_add(&map, "waf.context.processor", &settings); + + ddwaf_result out; + ASSERT_EQ(ddwaf_run(context, &map, nullptr, &out, LONG_TIME), DDWAF_OK); + EXPECT_FALSE(out.timeout); + + EXPECT_EQ(ddwaf_object_size(&out.derivatives), 1); + + auto json = test::object_to_json(out.derivatives); + EXPECT_STR(json, R"({"_dd.appsec.fp.http.endpoint":"http-put-729d56c3-2c70e12b-2c70e12b"})"); + + ddwaf_result_free(&out); + ddwaf_context_destroy(context); + ddwaf_destroy(handle); +} + +TEST(TestFingerprintIntegration, Preprocessor) +{ + auto rule = read_json_file("preprocessor.json", base_dir); + ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); + ddwaf_handle handle = ddwaf_init(&rule, nullptr, nullptr); + ASSERT_NE(handle, nullptr); + ddwaf_object_free(&rule); + + uint32_t size; + const char *const *addresses = ddwaf_known_addresses(handle, &size); + EXPECT_EQ(size, 6); + std::unordered_set address_set(addresses, addresses + size); + EXPECT_TRUE(address_set.contains("server.request.body")); + EXPECT_TRUE(address_set.contains("server.request.uri.raw")); + EXPECT_TRUE(address_set.contains("server.request.method")); + EXPECT_TRUE(address_set.contains("server.request.query")); + EXPECT_TRUE(address_set.contains("waf.context.processor")); + EXPECT_TRUE(address_set.contains("_dd.appsec.fp.http.endpoint")); + + ddwaf_context context = ddwaf_context_init(handle); + ASSERT_NE(context, nullptr); + + ddwaf_object tmp; + + ddwaf_object map = DDWAF_OBJECT_MAP; + ddwaf_object settings = DDWAF_OBJECT_MAP; + + ddwaf_object body = DDWAF_OBJECT_MAP; + ddwaf_object_map_add(&body, "key", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&map, "server.request.body", &body); + + ddwaf_object query = DDWAF_OBJECT_MAP; + ddwaf_object_map_add(&query, "key", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&map, "server.request.query", &query); + + ddwaf_object_map_add( + &map, "server.request.uri.raw", ddwaf_object_string(&tmp, "/path/to/resource/?key=")); + ddwaf_object_map_add(&map, "server.request.method", ddwaf_object_string(&tmp, "PuT")); + + ddwaf_object_map_add(&settings, "fingerprint", ddwaf_object_bool(&tmp, true)); + ddwaf_object_map_add(&map, "waf.context.processor", &settings); + + ddwaf_result out; + ASSERT_EQ(ddwaf_run(context, &map, nullptr, &out, LONG_TIME), DDWAF_MATCH); + EXPECT_FALSE(out.timeout); + + EXPECT_EVENTS(out, {.id = "rule1", + .name = "rule1", + .tags = {{"type", "flow1"}, {"category", "category1"}}, + .matches = {{.op = "match_regex", + .op_value = ".*", + .highlight = "http-put-729d56c3-2c70e12b-2c70e12b", + .args = {{ + .value = "http-put-729d56c3-2c70e12b-2c70e12b", + .address = "_dd.appsec.fp.http.endpoint", + .path = {}, + }}}}}); + + EXPECT_EQ(ddwaf_object_size(&out.derivatives), 0); + + ddwaf_result_free(&out); + ddwaf_context_destroy(context); + ddwaf_destroy(handle); +} + +TEST(TestFingerprintIntegration, Processor) +{ + auto rule = read_json_file("processor.json", base_dir); + ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); + ddwaf_handle handle = ddwaf_init(&rule, nullptr, nullptr); + ASSERT_NE(handle, nullptr); + ddwaf_object_free(&rule); + + uint32_t size; + const char *const *addresses = ddwaf_known_addresses(handle, &size); + EXPECT_EQ(size, 6); + std::unordered_set address_set(addresses, addresses + size); + EXPECT_TRUE(address_set.contains("server.request.body")); + EXPECT_TRUE(address_set.contains("server.request.uri.raw")); + EXPECT_TRUE(address_set.contains("server.request.method")); + EXPECT_TRUE(address_set.contains("server.request.query")); + EXPECT_TRUE(address_set.contains("waf.context.processor")); + EXPECT_TRUE(address_set.contains("_dd.appsec.fp.http.endpoint")); + + ddwaf_context context = ddwaf_context_init(handle); + ASSERT_NE(context, nullptr); + + ddwaf_object tmp; + + ddwaf_object map = DDWAF_OBJECT_MAP; + ddwaf_object settings = DDWAF_OBJECT_MAP; + + ddwaf_object body = DDWAF_OBJECT_MAP; + ddwaf_object_map_add(&body, "key", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&map, "server.request.body", &body); + + ddwaf_object query = DDWAF_OBJECT_MAP; + ddwaf_object_map_add(&query, "key", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&map, "server.request.query", &query); + + ddwaf_object_map_add( + &map, "server.request.uri.raw", ddwaf_object_string(&tmp, "/path/to/resource/?key=")); + ddwaf_object_map_add(&map, "server.request.method", ddwaf_object_string(&tmp, "PuT")); + + ddwaf_object_map_add(&settings, "fingerprint", ddwaf_object_bool(&tmp, true)); + ddwaf_object_map_add(&map, "waf.context.processor", &settings); + + ddwaf_result out; + ASSERT_EQ(ddwaf_run(context, &map, nullptr, &out, LONG_TIME), DDWAF_MATCH); + EXPECT_FALSE(out.timeout); + + EXPECT_EVENTS(out, {.id = "rule1", + .name = "rule1", + .tags = {{"type", "flow1"}, {"category", "category1"}}, + .matches = {{.op = "match_regex", + .op_value = ".*", + .highlight = "http-put-729d56c3-2c70e12b-2c70e12b", + .args = {{ + .value = "http-put-729d56c3-2c70e12b-2c70e12b", + .address = "_dd.appsec.fp.http.endpoint", + .path = {}, + }}}}}); + + EXPECT_EQ(ddwaf_object_size(&out.derivatives), 1); + + auto json = test::object_to_json(out.derivatives); + EXPECT_STR(json, R"({"_dd.appsec.fp.http.endpoint":"http-put-729d56c3-2c70e12b-2c70e12b"})"); + + ddwaf_result_free(&out); + ddwaf_context_destroy(context); + ddwaf_destroy(handle); +} + +} // namespace diff --git a/tests/processor/fingerprint_test.cpp b/tests/processor/fingerprint_test.cpp new file mode 100644 index 000000000..a22d20c28 --- /dev/null +++ b/tests/processor/fingerprint_test.cpp @@ -0,0 +1,365 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog +// (https://www.datadoghq.com/). Copyright 2023 Datadog, Inc. + +#include "../test_utils.hpp" +#include "ddwaf.h" +#include "matcher/regex_match.hpp" +#include "processor/fingerprint.hpp" + +using namespace ddwaf; +using namespace std::literals; + +namespace { + +TEST(TestHttpEndpointFingerprint, Basic) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_map(&query); + ddwaf_object_map_add(&query, "Key1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "key,3", ddwaf_object_invalid(&tmp)); + + ddwaf_object body; + ddwaf_object_map(&body); + ddwaf_object_map_add(&body, "KEY1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "3", ddwaf_object_invalid(&tmp)); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{ + output.stringValue, static_cast(static_cast(output.nbEntries))}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, EmptyQuery) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_map(&query); + + ddwaf_object body; + ddwaf_object_map(&body); + ddwaf_object_map_add(&body, "KEY1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "3", ddwaf_object_invalid(&tmp)); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; + EXPECT_STRV(output_sv, "http-get-0ede9e60--9798c0e4"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, EmptyBody) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_map(&query); + ddwaf_object_map_add(&query, "Key1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "key,3", ddwaf_object_invalid(&tmp)); + + ddwaf_object body; + ddwaf_object_map(&body); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, EmptyEverything) +{ + ddwaf_object query; + ddwaf_object_map(&query); + + ddwaf_object body; + ddwaf_object_map(&body); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = gen.eval_impl({{}, {}, false, ""}, {{}, {}, false, ""}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; + EXPECT_STRV(output_sv, "http----"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, KeyConsistency) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_map(&query); + ddwaf_object_map_add(&query, "Key1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "key3,Key4", ddwaf_object_invalid(&tmp)); + + ddwaf_object body; + ddwaf_object_map(&body); + ddwaf_object_map_add(&body, "KeY1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "kEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY3", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KeY4", ddwaf_object_invalid(&tmp)); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-ced401fa-ff07216e"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, InvalidQueryType) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_array(&query); + ddwaf_object_array_add(&query, ddwaf_object_string(&tmp, "Key1")); + ddwaf_object_array_add(&query, ddwaf_object_string(&tmp, "KEY2")); + ddwaf_object_array_add(&query, ddwaf_object_string(&tmp, "key,3")); + + ddwaf_object body; + ddwaf_object_map(&body); + ddwaf_object_map_add(&body, "KEY1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "3", ddwaf_object_invalid(&tmp)); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; + EXPECT_STRV(output_sv, "http-get-0ede9e60--9798c0e4"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, InvalidBodyType) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_map(&query); + ddwaf_object_map_add(&query, "Key1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "key,3", ddwaf_object_invalid(&tmp)); + + ddwaf_object body; + ddwaf_object_array(&body); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "KEY1")); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "KEY2")); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "KEY")); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "3")); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, InvalidQueryAndBodyType) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_array(&query); + ddwaf_object_array_add(&query, ddwaf_object_string(&tmp, "Key1")); + ddwaf_object_array_add(&query, ddwaf_object_string(&tmp, "KEY2")); + ddwaf_object_array_add(&query, ddwaf_object_string(&tmp, "key,3")); + + ddwaf_object body; + ddwaf_object_array(&body); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "KEY1")); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "KEY2")); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "KEY")); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "3")); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; + EXPECT_STRV(output_sv, "http-get-0ede9e60--"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, UriRawConsistency) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_map(&query); + ddwaf_object_map_add(&query, "Key1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "key,3", ddwaf_object_invalid(&tmp)); + + ddwaf_object body; + ddwaf_object_map(&body); + ddwaf_object_map_add(&body, "KEY1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "3", ddwaf_object_invalid(&tmp)); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + { + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, + static_cast(static_cast(output.nbEntries))}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); + ddwaf_object_free(&output); + } + + { + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever#fragment"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, + static_cast(static_cast(output.nbEntries))}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); + ddwaf_object_free(&output); + } + + { + ddwaf::timer deadline{2s}; + auto [output, attr] = gen.eval_impl({{}, {}, false, "GET"}, + {{}, {}, false, "/path/to/whatever?param=hello#fragment"}, {{}, {}, false, &query}, + {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, + static_cast(static_cast(output.nbEntries))}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); + ddwaf_object_free(&output); + } + + { + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, + static_cast(static_cast(output.nbEntries))}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); + ddwaf_object_free(&output); + } + + { + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/PaTh/To/WhAtEVER"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, + static_cast(static_cast(output.nbEntries))}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); + ddwaf_object_free(&output); + } + + ddwaf_object_free(&query); + ddwaf_object_free(&body); +} + +} // namespace diff --git a/tools/waf_runner.cpp b/tools/waf_runner.cpp index bd073d5ae..07686bdfd 100644 --- a/tools/waf_runner.cpp +++ b/tools/waf_runner.cpp @@ -47,7 +47,7 @@ auto parse_args(int argc, char *argv[]) } return args; } -const char *key_regex = R"((?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key)|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"; +const char *key_regex = R"((?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"; const char *value_regex = R"((?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key(?:[_-]?id)?|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?|jsessionid|phpsessid|asp\.net(?:[_-]|-)sessionid|sid|jwt)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,})";