feat(ci): Upload test results to Datadog (#11739)

* feat(ci): Upload test results to Datadog Signed-off-by: Spencer Gilbert <spencer.gilbert@datadoghq.com> * actually install the datadog-ci tool Signed-off-by: Spencer Gilbert <spencer.gilbert@datadoghq.com> * fix style Signed-off-by: Spencer Gilbert <spencer.gilbert@datadoghq.com> * try and debug windows failure Signed-off-by: Spencer Gilbert <spencer.gilbert@datadoghq.com> * update nextest config and update shebang Signed-off-by: Spencer Gilbert <spencer.gilbert@datadoghq.com> * more shebang Signed-off-by: Spencer Gilbert <spencer.gilbert@datadoghq.com> * forget shebangs, js is ultimate cross platform language Signed-off-by: Spencer Gilbert <spencer.gilbert@datadoghq.com> * one day this will work Signed-off-by: Spencer Gilbert <spencer.gilbert@datadoghq.com> * skip windows for now, include tags for platform and arch Signed-off-by: Spencer Gilbert <spencer.gilbert@datadoghq.com> * document nextest config Signed-off-by: Spencer Gilbert <spencer.gilbert@datadoghq.com> * link to windows issue in makefile Signed-off-by: Spencer Gilbert <spencer.gilbert@datadoghq.com>
vectordotdev · Mar 10, 2022 · ca84575 · ca84575
1 parent 39f3b18
commit ca84575
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 0 deletions.
diff --git a/.config/nextest.toml b/.config/nextest.toml
@@ -1,2 +1,18 @@
 [profile.default]
+# "retries" defines the number of times a test should be retried. If set to a
+# non-zero value, tests that succeed on a subsequent attempt will be marked as
+# non-flaky.
 retries = 3
+
+# shows all test statuses in output
+status-level = "all"
+
+# output failures as soon as they happen and at the end of the test run
+failure-output = "immediate-final"
+
+# don't cancel the test run on the first failure
+fail-fast = false
+
+[profile.default.junit]
+# output test results at target/nextest/default/junit.xml
+path = "junit.xml"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -11,6 +11,8 @@ env:
   AWS_ACCESS_KEY_ID: "dummy"
   AWS_SECRET_ACCESS_KEY: "dummy"
   CONTAINER_TOOL: "docker"
+  DD_ENV: "ci"
+  DD_API_KEY: ${{ secrets.DD_API_KEY }}
   RUST_BACKTRACE: full
   RUST_TEST_THREADS: 1
   TEST_LOG: vector=debug

diff --git a/Makefile b/Makefile
@@ -297,6 +297,10 @@ target/%/vector.tar.gz: target/%/vector CARGO_HANDLES_FRESHNESS
 .PHONY: test
 test: ## Run the unit test suite
 	${MAYBE_ENVIRONMENT_EXEC} cargo nextest run --workspace --no-fail-fast --no-default-features --features "${DEFAULT_FEATURES}" ${SCOPE}
+# https://github.com/vectordotdev/vector/issues/11762
+ifneq ($(OPERATING_SYSTEM), Windows)
+	@scripts/upload-test-results.sh
+endif
 
 .PHONY: test-docs
 test-docs: ## Run the docs test suite
@@ -346,6 +350,10 @@ ifeq ($(AUTOSPAWN), true)
 	sleep 10 # Many services are very slow... Give them a sec..
 endif
 	${MAYBE_ENVIRONMENT_EXEC} cargo nextest run --no-fail-fast --no-default-features --features nats-integration-tests --lib ::nats::
+# https://github.com/vectordotdev/vector/issues/11762
+ifneq ($(OPERATING_SYSTEM), Windows)
+	@scripts/upload-test-results.sh
+endif
 ifeq ($(AUTODESPAWN), true)
 	@scripts/setup_integration_env.sh nats stop
 endif
@@ -385,6 +393,10 @@ test-shutdown-cleanup:
 .PHONY: test-cli
 test-cli: ## Runs cli tests
 	${MAYBE_ENVIRONMENT_EXEC} cargo nextest run --no-fail-fast --no-default-features --features cli-tests --test cli --test-threads 4
+# https://github.com/vectordotdev/vector/issues/11762
+ifneq ($(OPERATING_SYSTEM), Windows)
+	@scripts/upload-test-results.sh
+endif
 
 ##@ Benching (Supports `ENVIRONMENT=true`)
 

diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh
@@ -17,6 +17,7 @@ cd ..
 # ubuntu 20.04 gives us version 10.19. We can revert once we update the
 # ci image to install a newer version of node.
 sudo npm -g install markdownlint-cli@0.30
+sudo npm -g install @datadog/datadog-ci
 
 pip3 install jsonschema==3.2.0
 pip3 install remarshal==0.11.2

diff --git a/scripts/upload-test-results.sh b/scripts/upload-test-results.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+# upload-test-restults.sh
+#
+# SUMMARY
+#
+#   Upload `cargo-nextest` JUnit output to Datadog
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+set -x
+
+_os_platform="$(uname -s)"
+_os_architecture="$(uname -m)"
+
+export DD_TAGS="os.platform:$_os_platform,os.architecture:$_os_architecture"
+export DD_ENV="${DD_ENV:-"local"}"
+
+datadog-ci junit upload \
+  --service vector \
+  target/nextest/default/junit.xml