From 87e3eeb0b6813cced6194268efdc9a8a3a8fd7fd Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Mon, 6 Oct 2025 15:14:48 +0200 Subject: [PATCH 1/7] Run all unit tests in a single Buildkite job --- .buildkite/commands/run-unit-tests.sh | 104 +++++++++++--------- .buildkite/commands/upload-code-coverage.sh | 2 +- .buildkite/pipeline.yml | 61 ++---------- 3 files changed, 69 insertions(+), 98 deletions(-) diff --git a/.buildkite/commands/run-unit-tests.sh b/.buildkite/commands/run-unit-tests.sh index a9ec43fba860..3f487d4768b4 100755 --- a/.buildkite/commands/run-unit-tests.sh +++ b/.buildkite/commands/run-unit-tests.sh @@ -7,41 +7,24 @@ fi "$(dirname "${BASH_SOURCE[0]}")/restore-cache.sh" -echo "--- ๐Ÿงช Testing" +echo "+++ ๐Ÿงช Testing" set +e -if [ "$1" == "wordpress" ]; then - test_suite="testWordpressVanillaRelease koverXmlReportWordpressVanillaRelease" - test_results_dir="WordPress/build/test-results" - test_log_dir="${test_results_dir}/*/*.xml" - code_coverage_report="WordPress/build/reports/kover/reportWordpressVanillaRelease.xml" -elif [ "$1" == "processors" ]; then - test_suite=":libs:processors:test :libs:processors:koverXmlReport" - test_results_dir="libs/processors/build/test-results" - test_log_dir="${test_results_dir}/test/*.xml" - code_coverage_report="libs/processors/build/reports/kover/report.xml" -elif [ "$1" == "image-editor" ]; then - test_suite=":libs:image-editor:testReleaseUnitTest :libs:image-editor:koverXmlReportRelease" - test_results_dir="libs/image-editor/build/test-results" - test_log_dir="${test_results_dir}/testReleaseUnitTest/*.xml" - code_coverage_report="libs/image-editor/build/reports/kover/reportRelease.xml" -elif [ "$1" == "fluxc" ]; then - test_suite=":libs:fluxc:testReleaseUnitTest :libs:fluxc:koverXmlReportRelease" - test_results_dir="libs/fluxc/build/test-results" - test_log_dir="${test_results_dir}/testReleaseUnitTest/*.xml" - code_coverage_report="libs/fluxc/build/reports/kover/reportRelease.xml" -elif [ "$1" == "login" ]; then - test_suite=":libs:login:testReleaseUnitTest :libs:login:koverXmlReportRelease" - test_results_dir="libs/login/build/test-results" - test_log_dir="${test_results_dir}/testReleaseUnitTest/*.xml" - code_coverage_report="libs/login/build/reports/kover/reportRelease.xml" -else - echo "Invalid Test Suite! Expected 'wordpress', 'processors', or 'image-editor', received '$1' instead" - exit 1 -fi - -./gradlew $test_suite +./gradlew \ + testJetpackJalapenoDebugUnitTest \ + testWordpressJalapenoDebugUnitTest \ + :libs:processors:test \ + :libs:image-editor:testDebugUnitTest \ + :libs:fluxc:testDebugUnitTest \ + :libs:login:testDebugUnitTest \ + koverXmlReportJetpackJalapenoDebug \ + koverXmlReportWordpressJalapenoDebug \ + :libs:processors:koverXmlReportJvm \ + :libs:image-editor:koverXmlReportDebug \ + :libs:fluxc:koverXmlReportDebug \ + :libs:login:koverXmlReportDebug TESTS_EXIT_STATUS=$? set -e +echo "" if [[ "$TESTS_EXIT_STATUS" -ne 0 ]]; then # Keep the (otherwise collapsed) current "Testing" section open in Buildkite logs on error. See https://buildkite.com/docs/pipelines/managing-log-output#collapsing-output @@ -49,23 +32,52 @@ if [[ "$TESTS_EXIT_STATUS" -ne 0 ]]; then echo "Unit Tests failed!" fi -echo "--- ๐Ÿšฆ Report Tests Status" -results_file="$test_results_dir/merged-test-results.xml" +if [[ "$TESTS_EXIT_STATUS" -eq 0 ]]; then + echo "--- โš’๏ธ Uploading code coverage" + .buildkite/commands/upload-code-coverage.sh +fi + +MODULES=(WordPress:jetpack WordPress:wordpress processors image-editor fluxc login) +for module in "${MODULES[@]}"; do + echo "--- ๐Ÿšฆ Report Tests Status (Module: ${module})" -# Merge JUnit results into a single file (for performance reasons with reporting) -merge_junit_reports -d ${test_log_dir%/*} -o $results_file + # Define possible directories for merging JUnit reports + if [[ "$module" == "WordPress:jetpack" ]]; then + junit_test_results_dir="WordPress/build/test-results/testJetpackJalapenoDebugUnitTest" + elif [[ "$module" == "WordPress:wordpress" ]]; then + junit_test_results_dir="WordPress/build/test-results/testWordpressJalapenoDebugUnitTest" + elif [[ "$module" == "processors" ]]; then + junit_test_results_dir="libs/processors/build/test-results/test" + elif [[ "$module" == "image-editor" ]]; then + junit_test_results_dir="libs/image-editor/build/test-results/testDebugUnitTest" + elif [[ "$module" == "fluxc" ]]; then + junit_test_results_dir="libs/fluxc/build/test-results/testDebugUnitTest" + elif [[ "$module" == "login" ]]; then + junit_test_results_dir="libs/login/build/test-results/testDebugUnitTest" + fi -if [[ $BUILDKITE_BRANCH == trunk ]] || [[ $BUILDKITE_BRANCH == release/* ]]; then - annotate_test_failures "$results_file" --slack "build-and-ship" -else - annotate_test_failures "$results_file" -fi + # Determine which directory exists + if [ -d "$junit_test_results_dir" ]; then + merge_dir="$junit_test_results_dir" + else + echo "$junit_test_results_dir does not exist for module $module. Skipping..." + continue + fi + + results_file="${merge_dir}/../merged-test-results.xml" + # Merge JUnit results into a single file (for performance reasons with reporting) + merge_junit_reports -d "$merge_dir" -o "$results_file" -echo "--- ๐Ÿงช Copying test logs for test collector" -mkdir buildkite-test-analytics -cp $results_file buildkite-test-analytics + if [[ $BUILDKITE_BRANCH == trunk ]] || [[ $BUILDKITE_BRANCH == release/* ]]; then + annotate_test_failures "$results_file" --module "$module" --slack "build-and-ship" + else + annotate_test_failures "$results_file" --module "$module" + fi -echo "--- โš’๏ธ Uploading code coverage" -.buildkite/commands/upload-code-coverage.sh $code_coverage_report + echo "--- ๐Ÿงช Copying Test Logs for Test Collector (Module: ${module})" + mkdir -p buildkite-test-analytics + cp "$results_file" "buildkite-test-analytics/${module}-merged-test-results.xml" +done +echo "--- ๐Ÿ“Š Tests Status" exit $TESTS_EXIT_STATUS diff --git a/.buildkite/commands/upload-code-coverage.sh b/.buildkite/commands/upload-code-coverage.sh index 1d3c343f6319..05d40056ad9a 100755 --- a/.buildkite/commands/upload-code-coverage.sh +++ b/.buildkite/commands/upload-code-coverage.sh @@ -7,4 +7,4 @@ curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig gpgv codecov.SHA256SUM.sig codecov.SHA256SUM sha256sum -c codecov.SHA256SUM chmod +x codecov -./codecov -t "$CODECOV_TOKEN" -f "$1" +./codecov -t "$CODECOV_TOKEN" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e9ecd9ca4702..f9aa7254a16a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -101,57 +101,16 @@ steps: ################# # Unit Tests ################# - - group: "๐Ÿ”ฌ Unit Tests" - steps: - - label: "๐Ÿ”ฌ Unit Test WordPress" - command: ".buildkite/commands/run-unit-tests.sh wordpress" - plugins: - - $CI_TOOLKIT - - $TEST_COLLECTOR : - <<: *test_collector_common_params - api-token-env-name: "BUILDKITE_ANALYTICS_TOKEN_UNIT_TESTS_WORDPRESS" - artifact_paths: - - "**/build/test-results/merged-test-results.xml" - - - label: "๐Ÿ”ฌ Unit Test Processors" - command: ".buildkite/commands/run-unit-tests.sh processors" - plugins: - - $CI_TOOLKIT - - $TEST_COLLECTOR : - <<: *test_collector_common_params - api-token-env-name: "BUILDKITE_ANALYTICS_TOKEN_UNIT_TESTS_PROCESSORS" - artifact_paths: - - "**/build/test-results/merged-test-results.xml" - - - label: "๐Ÿ”ฌ Unit Test Image Editor" - command: ".buildkite/commands/run-unit-tests.sh image-editor" - plugins: - - $CI_TOOLKIT - - $TEST_COLLECTOR : - <<: *test_collector_common_params - api-token-env-name: "BUILDKITE_ANALYTICS_TOKEN_UNIT_TESTS_IMAGE_EDITOR" - artifact_paths: - - "**/build/test-results/merged-test-results.xml" - - - label: "๐Ÿ”ฌ Unit Test FluxC" - command: ".buildkite/commands/run-unit-tests.sh fluxc" - plugins: - - $CI_TOOLKIT - - $TEST_COLLECTOR: - <<: *test_collector_common_params - api-token-env-name: "BUILDKITE_ANALYTICS_TOKEN_UNIT_TESTS_FLUXC" - artifact_paths: - - "**/build/test-results/merged-test-results.xml" - - - label: "๐Ÿ”ฌ Unit Test Login" - command: ".buildkite/commands/run-unit-tests.sh login" - plugins: - - $CI_TOOLKIT - - $TEST_COLLECTOR: - <<: *test_collector_common_params - api-token-env-name: "BUILDKITE_ANALYTICS_TOKEN_UNIT_TESTS_LOGIN" - artifact_paths: - - "**/build/test-results/merged-test-results.xml" + - label: "๐Ÿ”ฌ Unit Tests" + command: ".buildkite/commands/run-unit-tests.sh" + plugins: + - $CI_TOOLKIT + - $TEST_COLLECTOR : + <<: *test_collector_common_params + api-token-env-name: "BUILDKITE_ANALYTICS_TOKEN_UNIT_TESTS" + artifact_paths: + - "**/build/test-results/merged-test-results.xml" + - "**/build/reports/kover/*.xml" ################# # Instrumented (aka UI) Tests From 3212163af17fedfd6149cdafc6ca568b1c095c90 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Mon, 6 Oct 2025 16:23:26 +0200 Subject: [PATCH 2/7] Fix results file path --- .buildkite/commands/run-unit-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/commands/run-unit-tests.sh b/.buildkite/commands/run-unit-tests.sh index 3f487d4768b4..87f469c4379d 100755 --- a/.buildkite/commands/run-unit-tests.sh +++ b/.buildkite/commands/run-unit-tests.sh @@ -64,7 +64,7 @@ for module in "${MODULES[@]}"; do continue fi - results_file="${merge_dir}/../merged-test-results.xml" + results_file="${merge_dir}/merged-test-results.xml" # Merge JUnit results into a single file (for performance reasons with reporting) merge_junit_reports -d "$merge_dir" -o "$results_file" From 4b5829bad215d6d3c77f343ea299a980577a79e7 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Mon, 6 Oct 2025 18:58:51 +0200 Subject: [PATCH 3/7] Refactor and fix merging junit reports --- .buildkite/commands/run-unit-tests.sh | 71 ++++++++++++++------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/.buildkite/commands/run-unit-tests.sh b/.buildkite/commands/run-unit-tests.sh index 87f469c4379d..79c8bc85d2ac 100755 --- a/.buildkite/commands/run-unit-tests.sh +++ b/.buildkite/commands/run-unit-tests.sh @@ -37,47 +37,50 @@ if [[ "$TESTS_EXIT_STATUS" -eq 0 ]]; then .buildkite/commands/upload-code-coverage.sh fi -MODULES=(WordPress:jetpack WordPress:wordpress processors image-editor fluxc login) -for module in "${MODULES[@]}"; do - echo "--- ๐Ÿšฆ Report Tests Status (Module: ${module})" +echo "--- ๐Ÿšฆ Collecting Test Results" +# Create temporary directory for collecting all test results +temp_test_results_dir=$(mktemp -d) - # Define possible directories for merging JUnit reports - if [[ "$module" == "WordPress:jetpack" ]]; then - junit_test_results_dir="WordPress/build/test-results/testJetpackJalapenoDebugUnitTest" - elif [[ "$module" == "WordPress:wordpress" ]]; then - junit_test_results_dir="WordPress/build/test-results/testWordpressJalapenoDebugUnitTest" - elif [[ "$module" == "processors" ]]; then - junit_test_results_dir="libs/processors/build/test-results/test" - elif [[ "$module" == "image-editor" ]]; then - junit_test_results_dir="libs/image-editor/build/test-results/testDebugUnitTest" - elif [[ "$module" == "fluxc" ]]; then - junit_test_results_dir="libs/fluxc/build/test-results/testDebugUnitTest" - elif [[ "$module" == "login" ]]; then - junit_test_results_dir="libs/login/build/test-results/testDebugUnitTest" - fi +# Define test result directories for each module +declare -A TEST_RESULT_DIRS=( + ["WordPress:jetpack"]="WordPress/build/test-results/testJetpackJalapenoDebugUnitTest" + ["WordPress:wordpress"]="WordPress/build/test-results/testWordpressJalapenoDebugUnitTest" + ["processors"]="libs/processors/build/test-results/test" + ["image-editor"]="libs/image-editor/build/test-results/testDebugUnitTest" + ["fluxc"]="libs/fluxc/build/test-results/testDebugUnitTest" + ["login"]="libs/login/build/test-results/testDebugUnitTest" +) + +# Copy all XML test results to temporary directory +for module in "${!TEST_RESULT_DIRS[@]}"; do + test_results_dir="${TEST_RESULT_DIRS[$module]}" - # Determine which directory exists - if [ -d "$junit_test_results_dir" ]; then - merge_dir="$junit_test_results_dir" + if [ -d "$test_results_dir" ]; then + echo "Collecting test results from ${module}..." + cp "$test_results_dir"/*.xml "$temp_test_results_dir/" 2>/dev/null || true else - echo "$junit_test_results_dir does not exist for module $module. Skipping..." - continue + echo "Test results directory $test_results_dir does not exist for module $module. Skipping..." fi +done - results_file="${merge_dir}/merged-test-results.xml" - # Merge JUnit results into a single file (for performance reasons with reporting) - merge_junit_reports -d "$merge_dir" -o "$results_file" +echo "--- ๐Ÿšฆ Report Tests Status" +results_file="WordPress/build/test-results/merged-test-results.xml" +# Merge JUnit results into a single file (for performance reasons with reporting) +# See https://github.com/Automattic/a8c-ci-toolkit-buildkite-plugin/pull/103 +merge_junit_reports -d "$temp_test_results_dir" -o "$results_file" - if [[ $BUILDKITE_BRANCH == trunk ]] || [[ $BUILDKITE_BRANCH == release/* ]]; then - annotate_test_failures "$results_file" --module "$module" --slack "build-and-ship" - else - annotate_test_failures "$results_file" --module "$module" - fi +# Clean up temporary directory +rm -rf "$temp_test_results_dir" - echo "--- ๐Ÿงช Copying Test Logs for Test Collector (Module: ${module})" - mkdir -p buildkite-test-analytics - cp "$results_file" "buildkite-test-analytics/${module}-merged-test-results.xml" -done +if [[ $BUILDKITE_BRANCH == trunk ]] || [[ $BUILDKITE_BRANCH == release/* ]]; then + annotate_test_failures "$results_file" --slack "build-and-ship" +else + annotate_test_failures "$results_file" +fi + +echo "--- ๐Ÿงช Copying Test Logs for Test Collector" +mkdir -p buildkite-test-analytics +cp "$results_file" "buildkite-test-analytics/merged-test-results.xml" echo "--- ๐Ÿ“Š Tests Status" exit $TESTS_EXIT_STATUS From 310572536931cc1bade2c332d3f8e8c84fd6b87b Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Mon, 6 Oct 2025 19:24:12 +0200 Subject: [PATCH 4/7] Fix test collector token name --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f9aa7254a16a..9bb2ac013346 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -107,7 +107,7 @@ steps: - $CI_TOOLKIT - $TEST_COLLECTOR : <<: *test_collector_common_params - api-token-env-name: "BUILDKITE_ANALYTICS_TOKEN_UNIT_TESTS" + api-token-env-name: "BUILDKITE_ANALYTICS_TOKEN_UNIT_TESTS_WORDPRESS" artifact_paths: - "**/build/test-results/merged-test-results.xml" - "**/build/reports/kover/*.xml" From 0308568f6f29adf9c6a38b64c0de0ebb3a4c9e50 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Mon, 6 Oct 2025 19:26:19 +0200 Subject: [PATCH 5/7] Remove `Jetpack` variant when runnig unit tests. Use `wasabi` instead of `jalapeno` --- .buildkite/commands/run-unit-tests.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.buildkite/commands/run-unit-tests.sh b/.buildkite/commands/run-unit-tests.sh index 79c8bc85d2ac..7ae8a3a5ebe8 100755 --- a/.buildkite/commands/run-unit-tests.sh +++ b/.buildkite/commands/run-unit-tests.sh @@ -10,14 +10,12 @@ fi echo "+++ ๐Ÿงช Testing" set +e ./gradlew \ - testJetpackJalapenoDebugUnitTest \ - testWordpressJalapenoDebugUnitTest \ + testWordpressWasabiDebugUnitTest \ :libs:processors:test \ :libs:image-editor:testDebugUnitTest \ :libs:fluxc:testDebugUnitTest \ :libs:login:testDebugUnitTest \ - koverXmlReportJetpackJalapenoDebug \ - koverXmlReportWordpressJalapenoDebug \ + koverXmlReportWordpressWasabiDebug \ :libs:processors:koverXmlReportJvm \ :libs:image-editor:koverXmlReportDebug \ :libs:fluxc:koverXmlReportDebug \ @@ -43,8 +41,7 @@ temp_test_results_dir=$(mktemp -d) # Define test result directories for each module declare -A TEST_RESULT_DIRS=( - ["WordPress:jetpack"]="WordPress/build/test-results/testJetpackJalapenoDebugUnitTest" - ["WordPress:wordpress"]="WordPress/build/test-results/testWordpressJalapenoDebugUnitTest" + ["WordPress:wordpress"]="WordPress/build/test-results/testWordpressWasabiDebugUnitTest" ["processors"]="libs/processors/build/test-results/test" ["image-editor"]="libs/image-editor/build/test-results/testDebugUnitTest" ["fluxc"]="libs/fluxc/build/test-results/testDebugUnitTest" From a90bbd01ffc62236420f7cec2fda27d5f47e9d63 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Tue, 7 Oct 2025 09:54:43 +0200 Subject: [PATCH 6/7] Provide Kover reports to Codecov As Codecov have difficulties finding these reports on its own --- .buildkite/commands/run-unit-tests.sh | 8 +++++++- .buildkite/commands/upload-code-coverage.sh | 10 +++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.buildkite/commands/run-unit-tests.sh b/.buildkite/commands/run-unit-tests.sh index 7ae8a3a5ebe8..2583741cda18 100755 --- a/.buildkite/commands/run-unit-tests.sh +++ b/.buildkite/commands/run-unit-tests.sh @@ -32,7 +32,13 @@ fi if [[ "$TESTS_EXIT_STATUS" -eq 0 ]]; then echo "--- โš’๏ธ Uploading code coverage" - .buildkite/commands/upload-code-coverage.sh + # Find all kover XML reports and upload them + coverage_files=$(find . -path "*/build/reports/kover/*.xml" -type f) + if [ -n "$coverage_files" ]; then + .buildkite/commands/upload-code-coverage.sh $coverage_files + else + echo "No coverage files found matching pattern */build/reports/kover/*.xml" + fi fi echo "--- ๐Ÿšฆ Collecting Test Results" diff --git a/.buildkite/commands/upload-code-coverage.sh b/.buildkite/commands/upload-code-coverage.sh index 05d40056ad9a..844becbcdb74 100755 --- a/.buildkite/commands/upload-code-coverage.sh +++ b/.buildkite/commands/upload-code-coverage.sh @@ -7,4 +7,12 @@ curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig gpgv codecov.SHA256SUM.sig codecov.SHA256SUM sha256sum -c codecov.SHA256SUM chmod +x codecov -./codecov -t "$CODECOV_TOKEN" + +# Build arguments with multiple -f flags for each coverage file +coverage_args=() +for coverage_file in "$@"; do + coverage_args+=("-f" "$coverage_file") +done + +# Upload all coverage reports in a single execution +./codecov -t "$CODECOV_TOKEN" "${coverage_args[@]}" From 454737cdcedd4c14928af01429f19a5e0594d5f4 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Wed, 8 Oct 2025 12:44:07 +0200 Subject: [PATCH 7/7] Move `temp_test_results_dir` declaration closer to its usage --- .buildkite/commands/run-unit-tests.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.buildkite/commands/run-unit-tests.sh b/.buildkite/commands/run-unit-tests.sh index 2583741cda18..9b1e5be2c019 100755 --- a/.buildkite/commands/run-unit-tests.sh +++ b/.buildkite/commands/run-unit-tests.sh @@ -42,8 +42,6 @@ if [[ "$TESTS_EXIT_STATUS" -eq 0 ]]; then fi echo "--- ๐Ÿšฆ Collecting Test Results" -# Create temporary directory for collecting all test results -temp_test_results_dir=$(mktemp -d) # Define test result directories for each module declare -A TEST_RESULT_DIRS=( @@ -54,6 +52,9 @@ declare -A TEST_RESULT_DIRS=( ["login"]="libs/login/build/test-results/testDebugUnitTest" ) +# Create temporary directory for collecting all test results +temp_test_results_dir=$(mktemp -d) + # Copy all XML test results to temporary directory for module in "${!TEST_RESULT_DIRS[@]}"; do test_results_dir="${TEST_RESULT_DIRS[$module]}"