diff --git a/docker/otel-collector/config.yaml b/docker/otel-collector/config.yaml index 30632989c..19e006ace 100644 --- a/docker/otel-collector/config.yaml +++ b/docker/otel-collector/config.yaml @@ -27,6 +27,15 @@ receivers: processors: transform: log_statements: + - context: log + error_mode: ignore + statements: + # JSON parsing: Extends log attributes with the fields from structured log body content, either as an OTEL map or + # as a string containing JSON content. + - set(log.cache, ExtractPatterns(log.body, "(?P<0>(\\{.*\\}))")) where IsString(log.body) + - merge_maps(log.attributes, ParseJSON(log.cache["0"]), "upsert") where IsMap(log.cache) + - flatten(log.attributes) where IsMap(log.cache) + - merge_maps(log.attributes, log.body, "upsert") where IsMap(log.body) - context: log error_mode: ignore conditions: diff --git a/smoke-tests/otel-collector/README.md b/smoke-tests/otel-collector/README.md new file mode 100644 index 000000000..cb80376f6 --- /dev/null +++ b/smoke-tests/otel-collector/README.md @@ -0,0 +1,56 @@ +# OpenTelemetry Collector Smoke Tests + +This directory contains smoke tests for validating the OpenTelemetry Collector functionality in HyperDX. + +## Prerequisites + +Before running the tests, ensure you have the following tools installed: + +- [Bats](https://github.com/bats-core/bats-core) - Bash Automated Testing System +- [Docker](https://www.docker.com/) and Docker Compose +- [curl](https://curl.se/) - Command line tool for transferring data +- [ClickHouse client](https://clickhouse.com/docs/en/integrations/sql-clients/clickhouse-client) - Command-line client for ClickHouse + +## Running the Tests + +To run all the tests: + +```bash +cd smoke-tests/otel-collector +bats *.bats +``` + +To run a specific test file: + +```bash +bats hdx-1453-auto-parse-json.bats +``` + +## Test Structure + +- `*.bats` - Test files written in Bats +- `data/` - Test data used by the tests +- `test_helpers/` - Utility functions for the tests +- `docker-compose.yaml` - Docker Compose configuration for the test environment + +## Debugging + +If you need to debug the tests, you can set the `SKIP_CLEANUP` environment variable to prevent the Docker containers from being torn down after the tests complete: + +```bash +SKIP_CLEANUP=1 bats hdx-1453-auto-parse-json.bats +``` + +or + +```bash +SKIP_CLEANUP=true bats hdx-1453-auto-parse-json.bats +``` + +With `SKIP_CLEANUP` enabled, the test containers will remain running after the tests complete, allowing you to inspect logs, connect to the containers, and debug issues. + +To manually clean up the containers after debugging: + +```bash +docker compose down +``` diff --git a/smoke-tests/otel-collector/data/auto-parse/default/assert_query.sql b/smoke-tests/otel-collector/data/auto-parse/default/assert_query.sql new file mode 100644 index 000000000..614ec16f1 --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/default/assert_query.sql @@ -0,0 +1 @@ +SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'default' ORDER BY TimestampTime FORMAT CSV diff --git a/smoke-tests/otel-collector/data/auto-parse/default/expected.snap b/smoke-tests/otel-collector/data/auto-parse/default/expected.snap new file mode 100644 index 000000000..53337dfb0 --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/default/expected.snap @@ -0,0 +1,4 @@ +"{'suite-id':'auto-parse','test-id':'default'}","{}" +"{'suite-id':'auto-parse','test-id':'default'}","{}" +"{'suite-id':'auto-parse','test-id':'default'}","{}" +"{'suite-id':'auto-parse','test-id':'default'}","{}" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/auto-parse/default/input.json b/smoke-tests/otel-collector/data/auto-parse/default/input.json new file mode 100644 index 000000000..4348ece58 --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/default/input.json @@ -0,0 +1,53 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "auto-parse" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "default" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1901999580000000000", + "body": { + "stringValue": "[note] this is very much not JSON even though it starts with an array char" + } + }, + { + "timeUnixNano": "1901999580000000001", + "body": { + "stringValue": "{note} this is very much not JSON even though it starts with an object char" + } + }, + { + "timeUnixNano": "1901999580000000002", + "body": { + "stringValue": "NOTE: this is very much not JSON" + } + }, + { + "timeUnixNano": "1901999580000000003", + "body": { + "stringValue": "this has some {Key {Value { '{' } } invalid JSON in it" + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/auto-parse/json-string/assert_query.sql b/smoke-tests/otel-collector/data/auto-parse/json-string/assert_query.sql new file mode 100644 index 000000000..895efbc50 --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/json-string/assert_query.sql @@ -0,0 +1 @@ +SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'json-string' ORDER BY TimestampTime FORMAT CSV diff --git a/smoke-tests/otel-collector/data/auto-parse/json-string/expected.snap b/smoke-tests/otel-collector/data/auto-parse/json-string/expected.snap new file mode 100644 index 000000000..1bcc43ccf --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/json-string/expected.snap @@ -0,0 +1,5 @@ +"{'suite-id':'auto-parse','test-id':'json-string'}","{'attr.intValue':'1','found':'false','message':'this should be parsed into a map'}" +"{'suite-id':'auto-parse','test-id':'json-string'}","{'bodyAttr':'12345','message':'this has an existing user attribute that should be preserved.','userAttr':'true'}" +"{'suite-id':'auto-parse','test-id':'json-string'}","{'found':'true','position':'trailing'}" +"{'suite-id':'auto-parse','test-id':'json-string'}","{'found':'true','position':'leading'}" +"{'suite-id':'auto-parse','test-id':'json-string'}","{'found':'true','position':'wrapped'}" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/auto-parse/json-string/input.json b/smoke-tests/otel-collector/data/auto-parse/json-string/input.json new file mode 100644 index 000000000..8e0a85370 --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/json-string/input.json @@ -0,0 +1,67 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "auto-parse" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "json-string" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1901999580000000000", + "body": { + "stringValue": "{\"attr\":{\"intValue\": 1},\"found\":false,\"message\":\"this should be parsed into a map\"}" + } + }, + { + "timeUnixNano": "1901999580000000001", + "attributes": [ + { + "key": "userAttr", + "value": { + "boolValue": true + } + } + ], + "body": { + "stringValue": "{\"bodyAttr\":12345,\"message\":\"this has an existing user attribute that should be preserved.\"}" + } + }, + { + "timeUnixNano": "1901999580000000002", + "body": { + "stringValue": "should find the trailing JSON object {\"found\":true,\"position\":\"trailing\"}" + } + }, + { + "timeUnixNano": "1901999580000000003", + "body": { + "stringValue": "{\"found\":true,\"position\":\"leading\"} should find the leading JSON object " + } + }, + { + "timeUnixNano": "1901999580000000004", + "body": { + "stringValue": "should find a wrapped JSON object {\"found\":true,\"position\":\"wrapped\"} between text" + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/auto-parse/otel-map/assert_query.sql b/smoke-tests/otel-collector/data/auto-parse/otel-map/assert_query.sql new file mode 100644 index 000000000..133262650 --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/otel-map/assert_query.sql @@ -0,0 +1 @@ +SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'otel-map' ORDER BY TimestampTime FORMAT CSV diff --git a/smoke-tests/otel-collector/data/auto-parse/otel-map/expected.snap b/smoke-tests/otel-collector/data/auto-parse/otel-map/expected.snap new file mode 100644 index 000000000..ec9bf084b --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/otel-map/expected.snap @@ -0,0 +1 @@ +"{'suite-id':'auto-parse','test-id':'otel-map'}","{'account-id':'550e8400-e29b-41d4-a716-446655440000','message':'data sent as OTEL map should also extend the log attributes','user-id':'1234'}" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/auto-parse/otel-map/input.json b/smoke-tests/otel-collector/data/auto-parse/otel-map/input.json new file mode 100644 index 000000000..8e45c079b --- /dev/null +++ b/smoke-tests/otel-collector/data/auto-parse/otel-map/input.json @@ -0,0 +1,56 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "auto-parse" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "otel-map" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1901999580000000000", + "body": { + "kvlistValue": { + "values": [ + { + "key": "message", + "value": { + "stringValue": "data sent as OTEL map should also extend the log attributes" + } + }, + { + "key": "user-id", + "value": { + "stringValue": "1234" + } + }, + { + "key": "account-id", + "value": { + "stringValue": "550e8400-e29b-41d4-a716-446655440000" + } + } + ] + } + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/smoke-tests/otel-collector/hdx-1453-auto-parse-json.bats b/smoke-tests/otel-collector/hdx-1453-auto-parse-json.bats new file mode 100644 index 000000000..f65ee5cea --- /dev/null +++ b/smoke-tests/otel-collector/hdx-1453-auto-parse-json.bats @@ -0,0 +1,32 @@ +#!/usr/bin/env bats + +load 'test_helpers/utilities.bash' +load 'test_helpers/assertions.bash' + +setup_file() { + validate_env + docker compose up --build --detach + wait_for_ready "otel-collector" "http://localhost:4318" +} + +teardown_file() { + attempt_env_cleanup +} + +@test "JSON string body content should be parsed and stored as log attributes" { + emit_otel_data "http://localhost:4318" "data/auto-parse/json-string" + sleep 1 + assert_test_data "data/auto-parse/json-string" +} + +@test "OTEL map content should be stored as log attributes" { + emit_otel_data "http://localhost:4318" "data/auto-parse/otel-map" + sleep 1 + assert_test_data "data/auto-parse/otel-map" +} + +@test "all other content should skip storing values in log attributes" { + emit_otel_data "http://localhost:4318" "data/auto-parse/default" + sleep 1 + assert_test_data "data/auto-parse/default" +} diff --git a/smoke-tests/otel-collector/test_helpers/utilities.bash b/smoke-tests/otel-collector/test_helpers/utilities.bash index 0232fcc65..d3de3973c 100644 --- a/smoke-tests/otel-collector/test_helpers/utilities.bash +++ b/smoke-tests/otel-collector/test_helpers/utilities.bash @@ -72,3 +72,12 @@ emit_otel_data() { fi return 0 } + +attempt_env_cleanup() { + # Check if we should keep the test containers running + if [[ "${SKIP_CLEANUP}" == "1" ]] || [[ "$(echo "${SKIP_CLEANUP}" | tr '[:upper:]' '[:lower:]')" == "true" ]]; then + echo "🔍 SKIP_CLEANUP is set, skipping container cleanup" >&3 + return 0 + fi + docker compose down +}