diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 933caab0801ae..37a9f8e08cd56 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -125,7 +125,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/run-integration-test.sh int ${{ matrix.service }} + command: cargo vdev integration run ${{ matrix.service }} --build-all --reuse-image e2e-tests: needs: @@ -160,8 +160,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/run-integration-test.sh e2e ${{ matrix.service }} - + command: cargo vdev e2e run ${{ matrix.service }} --build-all --reuse-image update-pr-status: name: Signal result to PR diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 805ee63d39b43..8ab1dd8a77758 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -122,7 +122,7 @@ jobs: # Only install dep if test runs bash scripts/environment/prepare.sh --modules=datadog-ci echo "Running test for ${{ matrix.service }}" - bash scripts/run-integration-test.sh int ${{ matrix.service }} + cargo vdev integration run ${{ matrix.service }} --build-all --reuse-image else echo "Skipping ${{ matrix.service }} test as the value is false or conditions not met." fi @@ -183,7 +183,7 @@ jobs: # Only install dep if test runs bash scripts/environment/prepare.sh --modules=datadog-ci echo "Running test for ${{ matrix.service }}" - bash scripts/run-integration-test.sh e2e ${{ matrix.service }} + cargo vdev e2e run ${{ matrix.service }} --build-all --reuse-image else echo "Skipping ${{ matrix.service }} test as the value is false or conditions not met." fi diff --git a/scripts/e2e/opentelemetry-logs/README.md b/scripts/e2e/opentelemetry-logs/README.md index ce61906248a30..5926eb59d946e 100644 --- a/scripts/e2e/opentelemetry-logs/README.md +++ b/scripts/e2e/opentelemetry-logs/README.md @@ -16,7 +16,7 @@ This end-to-end (E2E) test validates that log events generated in a container ar ```shell # from the repo root directory -./scripts/run-integration-test.sh e2e opentelemetry-logs +cargo vdev e2e run opentelemetry-logs ``` ## Notes diff --git a/scripts/run-integration-test.sh b/scripts/run-integration-test.sh deleted file mode 100755 index f7ea444478c80..0000000000000 --- a/scripts/run-integration-test.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env bash - -# Used in CI to run and stop an integration test and upload the results of it. -# This is useful to allow retrying the integration test at a higher level than -# nextest and reduce code duplication in the workflow file. - -set -u - -if [[ "${ACTIONS_RUNNER_DEBUG:-}" == "true" ]]; then - set -x -fi - -print_compose_logs_on_failure() { - local LAST_RETURN_CODE=$1 - if [[ "$LAST_RETURN_CODE" -ne 0 || "${ACTIONS_RUNNER_DEBUG:-}" == "true" ]]; then - (docker compose --project-name "${TEST_NAME}" logs) || echo "Failed to collect logs" - fi -} - -usage() { - cat >&2 <<'USAGE' -Usage: - scripts/run-integration-test.sh [OPTIONS] (int|e2e) TEST_NAME - -Required positional arguments: - TEST_TYPE One of: int, e2e - TEST_NAME Name of the test/app (used as docker compose project name) - -Options: - -h Show this help and exit - -r Number of retries for the "test" phase (default: 2) - -v Increase verbosity; repeat for more (e.g. -vv or -vvv) - -e One or more environments to run (repeatable or comma-separated). - If provided, these are used as TEST_ENVIRONMENTS instead of auto-discovery. - -b Always build images (disables --reuse-image which is enabled by default) - -Notes: - - All existing two-argument invocations remain compatible: - scripts/run-integration-test.sh int opentelemetry-logs - - Additional options can be added later without breaking callers. -USAGE -} - -# Parse options -# Note: options must come before positional args (standard getopts behavior) -TEST_ENV="" -REUSE_IMAGE="--reuse-image" -while getopts ":hr:v:e:b" opt; do - case "$opt" in - h) - usage - exit 0 - ;; - r) - RETRIES="$OPTARG" - if ! [[ "$RETRIES" =~ ^[0-9]+$ ]] || [[ "$RETRIES" -lt 0 ]]; then - echo "ERROR: -r requires a non-negative integer (got: $RETRIES)" >&2 - exit 2 - fi - ;; - v) - VERBOSITY+="v" - ;; - e) - TEST_ENV="$OPTARG" - ;; - b) - REUSE_IMAGE="" - ;; - \?) - echo "ERROR: unknown option: -$OPTARG" >&2 - usage - exit 2 - ;; - :) - echo "ERROR: option -$OPTARG requires an argument" >&2 - usage - exit 2 - ;; - esac -done -shift $((OPTIND - 1)) - -RETRIES=${RETRIES:-2} -VERBOSITY=${VERBOSITY:-'-v'} - -# Validate required positional args -if [[ $# -ne 2 ]]; then - echo "ERROR: missing required positional arguments" >&2 - usage - exit 1 -fi - -TEST_TYPE=$1 # either "int" or "e2e" -TEST_NAME=$2 - -case "$TEST_TYPE" in - int|e2e) ;; - *) - echo "ERROR: TEST_TYPE must be 'int' or 'e2e' (got: $TEST_TYPE)" >&2 - usage - exit 1 - ;; -esac - -# Determine environments to run -if [[ ${#TEST_ENV} -gt 0 ]]; then - # Use the environments supplied via -e - TEST_ENVIRONMENTS="${TEST_ENV}" -else - # Collect all available environments via auto-discovery - mapfile -t TEST_ENVIRONMENTS < <(cargo vdev "${VERBOSITY}" "${TEST_TYPE}" show -e "${TEST_NAME}") - if [[ ${#TEST_ENVIRONMENTS[@]} -eq 0 ]]; then - echo "ERROR: no environments found for ${TEST_TYPE} test '${TEST_NAME}'" >&2 - exit 1 - fi -fi - -for TEST_ENV in "${TEST_ENVIRONMENTS[@]}"; do - # Execution flow for each environment: - # 1. Clean up previous test output - # 2. Start environment - # 3. If start succeeded: - # - Run tests - # - Upload results to Datadog CI - # 4. If start failed: - # - Skip test phase - # - Exit with error code - # 5. Stop environment (always, best effort) - # 6. Exit if there was a failure - - docker run --rm -v vector_target:/output/"${TEST_NAME}" alpine:3.20 \ - sh -c "rm -rf /output/${TEST_NAME}/*" - - cargo vdev "${VERBOSITY}" "${TEST_TYPE}" start --build-all ${REUSE_IMAGE} "${TEST_NAME}" "${TEST_ENV}" - START_RET=$? - print_compose_logs_on_failure "$START_RET" - - if [[ "$START_RET" -eq 0 ]]; then - cargo vdev "${VERBOSITY}" "${TEST_TYPE}" test --retries "$RETRIES" --build-all ${REUSE_IMAGE} "${TEST_NAME}" "${TEST_ENV}" - RET=$? - print_compose_logs_on_failure "$RET" - - # Upload test results only if the vdev test step ran - ./scripts/upload-test-results.sh - else - echo "Skipping test phase because 'vdev start' failed" - RET=$START_RET - fi - - # Always stop the environment (best effort cleanup) - cargo vdev "${VERBOSITY}" "${TEST_TYPE}" stop --build-all ${REUSE_IMAGE} "${TEST_NAME}" || true - - # Exit early on first failure - if [[ "$RET" -ne 0 ]]; then - exit "$RET" - fi -done - -exit 0 diff --git a/vdev/src/commands/compose_tests/mod.rs b/vdev/src/commands/compose_tests/mod.rs index c289e7dff3d74..55c3063e5f413 100644 --- a/vdev/src/commands/compose_tests/mod.rs +++ b/vdev/src/commands/compose_tests/mod.rs @@ -1,6 +1,7 @@ mod active_projects; pub(crate) mod ci_paths; +pub(crate) mod run; pub(crate) mod show; pub(crate) mod start; pub(crate) mod stop; diff --git a/vdev/src/commands/compose_tests/run.rs b/vdev/src/commands/compose_tests/run.rs new file mode 100644 index 0000000000000..2c839bb8b681d --- /dev/null +++ b/vdev/src/commands/compose_tests/run.rs @@ -0,0 +1,165 @@ +use anyhow::{Context, Result}; +use std::process::Command; + +use crate::testing::{config::ComposeTestConfig, integration::ComposeTestLocalConfig}; + +/// Run a complete test workflow orchestrating start, test, and stop phases +/// +/// This function implements the full test lifecycle used in CI: +/// 1. Clean up previous test output +/// 2. Start the environment +/// 3. Run tests with retries +/// 4. Upload results to Datadog (in CI) +/// 5. Stop the environment (always, as cleanup) +pub fn exec( + local_config: ComposeTestLocalConfig, + test_name: &str, + environments: &[String], + build_all: bool, + reuse_image: bool, + retries: u8, + show_logs: bool, +) -> Result<()> { + let environments = if environments.is_empty() { + // Auto-discover environments + let (_test_dir, config) = ComposeTestConfig::load(local_config.directory, test_name)?; + config.environments().keys().cloned().collect() + } else { + environments.to_vec() + }; + + if environments.is_empty() { + anyhow::bail!("No environments found for test '{test_name}'"); + } + + for environment in &environments { + info!("Running test '{test_name}' in environment '{environment}'"); + + // Clean up previous test output + cleanup_test_output(test_name)?; + + // Start the environment + let start_result = super::start::exec( + local_config, + test_name, + Some(environment), + build_all, + reuse_image, + ); + + if let Err(e) = &start_result { + error!("Failed to start environment: {e}"); + if show_logs || is_debug_mode() { + print_compose_logs(test_name); + } + } + + let test_result = if start_result.is_ok() { + // Run tests + let result = super::test::exec( + local_config, + test_name, + Some(environment), + build_all, + reuse_image, + retries, + &[], + ); + + if let Err(e) = &result { + error!("Tests failed: {e}"); + if show_logs || is_debug_mode() { + print_compose_logs(test_name); + } + } + + // Upload test results (only in CI) + upload_test_results(); + + result + } else { + warn!("Skipping test phase because 'start' failed"); + start_result + }; + + // Always stop the environment (best effort cleanup) + if let Err(e) = super::stop::exec(local_config, test_name, build_all, reuse_image) { + warn!("Failed to stop environment (cleanup): {e}"); + } + + // Exit early on first failure + test_result?; + } + + Ok(()) +} + +/// Check if we're running in debug mode (`ACTIONS_RUNNER_DEBUG` or `RUST_LOG`) +fn is_debug_mode() -> bool { + std::env::var("ACTIONS_RUNNER_DEBUG") + .map(|v| v == "true") + .unwrap_or(false) + || std::env::var("RUST_LOG") + .map(|v| v.contains("debug") || v.contains("trace")) + .unwrap_or(false) +} + +/// Print docker compose logs for debugging +fn print_compose_logs(project_name: &str) { + info!("Collecting docker compose logs for project '{project_name}'..."); + + let result = Command::new("docker") + .args(["compose", "--project-name", project_name, "logs"]) + .status(); + + if let Err(e) = result { + warn!("Failed to collect logs: {e}"); + } +} + +/// Clean up previous test output from the docker volume +fn cleanup_test_output(test_name: &str) -> Result<()> { + debug!("Cleaning up previous test output for '{test_name}'"); + + let status = Command::new("docker") + .args([ + "run", + "--rm", + "-v", + &format!("vector_target:/output/{test_name}"), + "alpine:3.20", + "sh", + "-c", + &format!("rm -rf /output/{test_name}/*"), + ]) + .status() + .context("Failed to run docker cleanup command")?; + + if !status.success() { + warn!("Failed to clean up previous test output (this may be okay if it didn't exist)"); + } + + Ok(()) +} + +/// Upload test results to Datadog (in CI only, no-op locally) +/// +/// The script itself checks for CI environment and handles the logic. +fn upload_test_results() { + // Get the repo root path + let script_path = + std::path::PathBuf::from(crate::app::path()).join("scripts/upload-test-results.sh"); + + // Call the upload script (it checks for CI internally) + let result = Command::new(&script_path).status(); + + match result { + Ok(status) if !status.success() => { + warn!("Upload script exited with non-zero status"); + } + Err(e) => { + warn!("Failed to execute upload script: {e}"); + } + _ => {} // Success or handled by script + } +} diff --git a/vdev/src/commands/e2e/mod.rs b/vdev/src/commands/e2e/mod.rs index 1e84af9c0e96e..7ac8d70e64cca 100644 --- a/vdev/src/commands/e2e/mod.rs +++ b/vdev/src/commands/e2e/mod.rs @@ -9,5 +9,6 @@ These test setups are organized into a set of integrations, located in subdirect mod start, mod stop, mod test, + mod run, mod ci_paths, } diff --git a/vdev/src/commands/e2e/run.rs b/vdev/src/commands/e2e/run.rs new file mode 100644 index 0000000000000..5c087b3d5f7e2 --- /dev/null +++ b/vdev/src/commands/e2e/run.rs @@ -0,0 +1,56 @@ +use anyhow::Result; +use clap::Args; + +use crate::testing::integration::ComposeTestLocalConfig; + +/// Run a complete end-to-end test workflow (CI-style) +/// +/// This command orchestrates the full e2e test lifecycle: +/// 1. Clean up previous test output +/// 2. Start the environment +/// 3. Run tests with retries +/// 4. Upload results to Datadog (in CI) +/// 5. Stop the environment (always, as cleanup) +/// +/// This is useful for CI workflows and local testing that mimics CI behavior. +#[derive(Args, Debug)] +#[command()] +pub struct Cli { + /// The e2e test name + test_name: String, + + /// The desired environment name(s) to run. If omitted, all environments are run. + /// Can be specified multiple times or comma-separated. + #[arg(short = 'e', long = "environment", value_delimiter = ',')] + environments: Vec, + + /// Whether to compile the test runner with all e2e test features + #[arg(short = 'a', long)] + build_all: bool, + + /// Reuse existing test runner image instead of rebuilding (useful in CI) + #[arg(long)] + reuse_image: bool, + + /// Number of retries for the test phase + #[arg(short = 'r', long, default_value = "2")] + retries: u8, + + /// Print docker compose logs on failure or when in debug mode + #[arg(long)] + show_logs: bool, +} + +impl Cli { + pub fn exec(self) -> Result<()> { + crate::commands::compose_tests::run::exec( + ComposeTestLocalConfig::e2e(), + &self.test_name, + &self.environments, + self.build_all, + self.reuse_image, + self.retries, + self.show_logs, + ) + } +} diff --git a/vdev/src/commands/integration/mod.rs b/vdev/src/commands/integration/mod.rs index b517fc475a5db..8e4ae448b017e 100644 --- a/vdev/src/commands/integration/mod.rs +++ b/vdev/src/commands/integration/mod.rs @@ -10,5 +10,6 @@ These test setups are organized into a set of integrations, located in subdirect mod start, mod stop, mod test, + mod run, mod ci_paths, } diff --git a/vdev/src/commands/integration/run.rs b/vdev/src/commands/integration/run.rs new file mode 100644 index 0000000000000..991696927ee71 --- /dev/null +++ b/vdev/src/commands/integration/run.rs @@ -0,0 +1,56 @@ +use anyhow::Result; +use clap::Args; + +use crate::testing::integration::ComposeTestLocalConfig; + +/// Run a complete integration test workflow (CI-style) +/// +/// This command orchestrates the full integration test lifecycle: +/// 1. Clean up previous test output +/// 2. Start the environment +/// 3. Run tests with retries +/// 4. Upload results to Datadog (in CI) +/// 5. Stop the environment (always, as cleanup) +/// +/// This is useful for CI workflows and local testing that mimics CI behavior. +#[derive(Args, Debug)] +#[command()] +pub struct Cli { + /// The integration name + test_name: String, + + /// The desired environment name(s) to run. If omitted, all environments are run. + /// Can be specified multiple times or comma-separated. + #[arg(short = 'e', long = "environment", value_delimiter = ',')] + environments: Vec, + + /// Whether to compile the test runner with all integration test features + #[arg(short = 'a', long)] + build_all: bool, + + /// Reuse existing test runner image instead of rebuilding (useful in CI) + #[arg(long)] + reuse_image: bool, + + /// Number of retries for the test phase + #[arg(short = 'r', long, default_value = "2")] + retries: u8, + + /// Print docker compose logs on failure or when in debug mode + #[arg(long)] + show_logs: bool, +} + +impl Cli { + pub fn exec(self) -> Result<()> { + crate::commands::compose_tests::run::exec( + ComposeTestLocalConfig::integration(), + &self.test_name, + &self.environments, + self.build_all, + self.reuse_image, + self.retries, + self.show_logs, + ) + } +}