diff --git a/actions/setup/sh/download_docker_images.sh b/actions/setup/sh/download_docker_images.sh index 47a0a89237..c22ab9836f 100755 --- a/actions/setup/sh/download_docker_images.sh +++ b/actions/setup/sh/download_docker_images.sh @@ -1,8 +1,16 @@ #!/usr/bin/env bash -# Download Docker images with retry logic +# Download Docker images with retry logic and controlled parallelism # Usage: download_docker_images.sh IMAGE1 [IMAGE2 ...] +# +# This script downloads multiple Docker images in parallel with controlled +# parallelism (max 4 concurrent downloads) to improve performance without +# overwhelming the system. Docker daemon supports concurrent pulls, which can +# provide significant speedup when downloading multiple images. +# +# Each image is pulled with retry logic (3 attempts with exponential backoff). +# The script fails if any image fails to download after all retry attempts. -set -e +set -euo pipefail # Helper function to pull Docker images with retry logic docker_pull_with_retry() { @@ -13,7 +21,7 @@ docker_pull_with_retry() { while [ $attempt -le $max_attempts ]; do echo "Attempt $attempt of $max_attempts: Pulling $image..." - if docker pull --quiet "$image"; then + if docker pull --quiet "$image" 2>&1; then echo "Successfully pulled $image" return 0 fi @@ -30,7 +38,11 @@ docker_pull_with_retry() { done } -# Pull all images passed as arguments -for image in "$@"; do - docker_pull_with_retry "$image" -done +# Export function so xargs can use it +export -f docker_pull_with_retry + +# Pull images with controlled parallelism using xargs +echo "Starting download of ${#@} image(s) with max 4 concurrent downloads..." +printf '%s\n' "$@" | xargs -P 4 -I {} bash -c 'docker_pull_with_retry "$@"' _ {} + +echo "All images downloaded successfully" diff --git a/actions/setup/sh/download_docker_images_test.sh b/actions/setup/sh/download_docker_images_test.sh new file mode 100755 index 0000000000..3c8c22dc3f --- /dev/null +++ b/actions/setup/sh/download_docker_images_test.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Test script for download_docker_images.sh +# Tests concurrent download functionality + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DOWNLOAD_SCRIPT="${SCRIPT_DIR}/download_docker_images.sh" + +# Colors for output +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo "==========================================" +echo "Testing download_docker_images.sh" +echo "==========================================" +echo "" + +# Test 1: Single image download +echo -e "${YELLOW}Test 1: Single image download${NC}" +if bash "$DOWNLOAD_SCRIPT" alpine:3.19 > /tmp/test1.log 2>&1; then + echo -e "${GREEN}✓ PASS${NC}: Single image download succeeded" +else + echo -e "${RED}✗ FAIL${NC}: Single image download failed" + cat /tmp/test1.log + exit 1 +fi +echo "" + +# Test 2: Multiple images concurrent download +echo -e "${YELLOW}Test 2: Multiple images concurrent download${NC}" +if bash "$DOWNLOAD_SCRIPT" alpine:3.18 alpine:3.17 > /tmp/test2.log 2>&1; then + echo -e "${GREEN}✓ PASS${NC}: Multiple images download succeeded" + # Verify concurrent behavior by checking log contains download message + if grep -q "Starting download of 2 image(s) with max 4 concurrent downloads" /tmp/test2.log; then + echo -e "${GREEN}✓ PASS${NC}: Concurrent download mode confirmed" + else + echo -e "${RED}✗ FAIL${NC}: Expected concurrent download message not found" + cat /tmp/test2.log + exit 1 + fi +else + echo -e "${RED}✗ FAIL${NC}: Multiple images download failed" + cat /tmp/test2.log + exit 1 +fi +echo "" + +# Test 3: Already cached images (should be fast) +echo -e "${YELLOW}Test 3: Already cached images${NC}" +START_TIME=$(date +%s) +if bash "$DOWNLOAD_SCRIPT" alpine:3.19 alpine:3.18 > /tmp/test3.log 2>&1; then + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + echo -e "${GREEN}✓ PASS${NC}: Cached images download succeeded (${DURATION}s)" + # Cached images should complete quickly + if [ $DURATION -lt 10 ]; then + echo -e "${GREEN}✓ PASS${NC}: Cached download was fast (<10s)" + else + echo -e "${YELLOW}⚠ WARNING${NC}: Cached download took ${DURATION}s (expected <10s)" + fi +else + echo -e "${RED}✗ FAIL${NC}: Cached images download failed" + cat /tmp/test3.log + exit 1 +fi +echo "" + +# Test 4: Invalid image (should fail gracefully) +echo -e "${YELLOW}Test 4: Invalid image (expected to fail)${NC}" +if bash "$DOWNLOAD_SCRIPT" "nonexistent-registry.invalid/fake-image:v999" > /tmp/test4.log 2>&1; then + echo -e "${RED}✗ FAIL${NC}: Should have failed for invalid image" + exit 1 +else + echo -e "${GREEN}✓ PASS${NC}: Failed as expected for invalid image" + # Check for expected error message + if grep -q "Failed to download" /tmp/test4.log; then + echo -e "${GREEN}✓ PASS${NC}: Error message present" + else + echo -e "${YELLOW}⚠ WARNING${NC}: Expected error message format not found" + fi +fi +echo "" + +# Test 5: Empty arguments (should handle gracefully) +echo -e "${YELLOW}Test 5: No images provided${NC}" +if bash "$DOWNLOAD_SCRIPT" > /tmp/test5.log 2>&1; then + echo -e "${GREEN}✓ PASS${NC}: Handled empty arguments gracefully" +else + # This might fail which is also acceptable behavior + echo -e "${YELLOW}⚠ INFO${NC}: Script exited with error for empty arguments (acceptable)" +fi +echo "" + +echo "==========================================" +echo -e "${GREEN}All tests passed!${NC}" +echo "==========================================" + +# Cleanup +rm -f /tmp/test*.log diff --git a/specs/artifacts.md b/specs/artifacts.md index 9256a3fd34..0079d8e169 100644 --- a/specs/artifacts.md +++ b/specs/artifacts.md @@ -48,7 +48,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Paths**: `${{ env.GH_AW_SAFE_OUTPUTS }}` - **Used in**: 61 workflow(s) - agent-performance-analyzer.md, ai-moderator.md, archie.md, brave.md, breaking-change-checker.md, campaign-generator.md, changeset.md, ci-coach.md, ci-doctor.md, cli-consistency-checker.md, cloclo.md, commit-changes-analyzer.md, copilot-pr-merged-report.md, copilot-pr-nlp-analysis.md, craft.md, daily-choice-test.md, daily-copilot-token-report.md, daily-fact.md, daily-file-diet.md, daily-issues-report.md, daily-news.md, daily-repo-chronicle.md, deep-report.md, dependabot-go-checker.md, dev-hawk.md, dev.md, dictation-prompt.md, github-mcp-structural-analysis.md, glossary-maintainer.md, go-fan.md, go-pattern-detector.md, grumpy-reviewer.md, hourly-ci-cleaner.md, issue-classifier.md, issue-triage-agent.md, layout-spec-maintainer.md, mergefest.md, notion-issue-summary.md, pdf-summary.md, plan.md, playground-org-project-update-issue.md, playground-snapshots-refresh.md, poem-bot.md, pr-nitpick-reviewer.md, python-data-charts.md, q.md, release.md, repository-quality-improver.md, research.md, scout.md, security-compliance.md, slide-deck-maintainer.md, stale-repo-identifier.md, super-linter.md, technical-doc-writer.md, tidy.md, typist.md, video-analyzer.md, weekly-issue-summary.md, workflow-generator.md, workflow-health-manager.md - `safe-outputs-assets` - - **Paths**: `/opt/gh-aw/safeoutputs/assets/` + - **Paths**: `/tmp/gh-aw/safeoutputs/assets/` - **Used in**: 12 workflow(s) - copilot-pr-nlp-analysis.md, daily-copilot-token-report.md, daily-issues-report.md, daily-news.md, daily-repo-chronicle.md, deep-report.md, github-mcp-structural-analysis.md, poem-bot.md, python-data-charts.md, stale-repo-identifier.md, technical-doc-writer.md, weekly-issue-summary.md - `trending-charts` - **Paths**: `/tmp/gh-aw/python/charts/*.png` @@ -158,7 +158,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Download paths**: `/tmp/gh-aw/safeoutputs/` - **Used in**: 12 workflow(s) - copilot-pr-nlp-analysis.md, daily-copilot-token-report.md, daily-issues-report.md, daily-news.md, daily-repo-chronicle.md, deep-report.md, github-mcp-structural-analysis.md, poem-bot.md, python-data-charts.md, stale-repo-identifier.md, technical-doc-writer.md, weekly-issue-summary.md - `safe-outputs-assets` - - **Download paths**: `/opt/gh-aw/safeoutputs/assets/` + - **Download paths**: `/tmp/gh-aw/safeoutputs/assets/` - **Used in**: 12 workflow(s) - copilot-pr-nlp-analysis.md, daily-copilot-token-report.md, daily-issues-report.md, daily-news.md, daily-repo-chronicle.md, deep-report.md, github-mcp-structural-analysis.md, poem-bot.md, python-data-charts.md, stale-repo-identifier.md, technical-doc-writer.md, weekly-issue-summary.md ## Workflows @@ -1030,7 +1030,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Artifact**: `safe-outputs-assets` - **Upload paths**: - - `/opt/gh-aw/safeoutputs/assets/` + - `/tmp/gh-aw/safeoutputs/assets/` - **Artifact**: `agent-artifacts` - **Upload paths**: @@ -1095,7 +1095,7 @@ This section provides an overview of artifacts organized by job name, with dupli **Downloads:** - **Artifact**: `safe-outputs-assets` (by name) - - **Download path**: `/opt/gh-aw/safeoutputs/assets/` + - **Download path**: `/tmp/gh-aw/safeoutputs/assets/` - **Depends on jobs**: [agent detection] - **Artifact**: `agent-output` (by name) @@ -1270,7 +1270,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Artifact**: `safe-outputs-assets` - **Upload paths**: - - `/opt/gh-aw/safeoutputs/assets/` + - `/tmp/gh-aw/safeoutputs/assets/` - **Artifact**: `agent-artifacts` - **Upload paths**: @@ -1335,7 +1335,7 @@ This section provides an overview of artifacts organized by job name, with dupli **Downloads:** - **Artifact**: `safe-outputs-assets` (by name) - - **Download path**: `/opt/gh-aw/safeoutputs/assets/` + - **Download path**: `/tmp/gh-aw/safeoutputs/assets/` - **Depends on jobs**: [agent detection] - **Artifact**: `agent-output` (by name) @@ -1498,7 +1498,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Artifact**: `safe-outputs-assets` - **Upload paths**: - - `/opt/gh-aw/safeoutputs/assets/` + - `/tmp/gh-aw/safeoutputs/assets/` - **Artifact**: `agent-artifacts` - **Upload paths**: @@ -1555,7 +1555,7 @@ This section provides an overview of artifacts organized by job name, with dupli **Downloads:** - **Artifact**: `safe-outputs-assets` (by name) - - **Download path**: `/opt/gh-aw/safeoutputs/assets/` + - **Download path**: `/tmp/gh-aw/safeoutputs/assets/` - **Depends on jobs**: [agent detection] - **Artifact**: `agent-output` (by name) @@ -1600,7 +1600,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Artifact**: `safe-outputs-assets` - **Upload paths**: - - `/opt/gh-aw/safeoutputs/assets/` + - `/tmp/gh-aw/safeoutputs/assets/` - **Artifact**: `agent-artifacts` - **Upload paths**: @@ -1665,7 +1665,7 @@ This section provides an overview of artifacts organized by job name, with dupli **Downloads:** - **Artifact**: `safe-outputs-assets` (by name) - - **Download path**: `/opt/gh-aw/safeoutputs/assets/` + - **Download path**: `/tmp/gh-aw/safeoutputs/assets/` - **Depends on jobs**: [agent detection] - **Artifact**: `agent-output` (by name) @@ -1706,7 +1706,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Artifact**: `safe-outputs-assets` - **Upload paths**: - - `/opt/gh-aw/safeoutputs/assets/` + - `/tmp/gh-aw/safeoutputs/assets/` - **Artifact**: `agent-artifacts` - **Upload paths**: @@ -1763,7 +1763,7 @@ This section provides an overview of artifacts organized by job name, with dupli **Downloads:** - **Artifact**: `safe-outputs-assets` (by name) - - **Download path**: `/opt/gh-aw/safeoutputs/assets/` + - **Download path**: `/tmp/gh-aw/safeoutputs/assets/` - **Depends on jobs**: [agent detection] - **Artifact**: `agent-output` (by name) @@ -1799,7 +1799,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Artifact**: `safe-outputs-assets` - **Upload paths**: - - `/opt/gh-aw/safeoutputs/assets/` + - `/tmp/gh-aw/safeoutputs/assets/` - **Artifact**: `agent-artifacts` - **Upload paths**: @@ -1864,7 +1864,7 @@ This section provides an overview of artifacts organized by job name, with dupli **Downloads:** - **Artifact**: `safe-outputs-assets` (by name) - - **Download path**: `/opt/gh-aw/safeoutputs/assets/` + - **Download path**: `/tmp/gh-aw/safeoutputs/assets/` - **Depends on jobs**: [agent detection] - **Artifact**: `agent-output` (by name) @@ -2206,7 +2206,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Artifact**: `safe-outputs-assets` - **Upload paths**: - - `/opt/gh-aw/safeoutputs/assets/` + - `/tmp/gh-aw/safeoutputs/assets/` - **Artifact**: `agent-artifacts` - **Upload paths**: @@ -2263,7 +2263,7 @@ This section provides an overview of artifacts organized by job name, with dupli **Downloads:** - **Artifact**: `safe-outputs-assets` (by name) - - **Download path**: `/opt/gh-aw/safeoutputs/assets/` + - **Download path**: `/tmp/gh-aw/safeoutputs/assets/` - **Depends on jobs**: [agent detection] - **Artifact**: `agent-output` (by name) @@ -3245,7 +3245,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Artifact**: `safe-outputs-assets` - **Upload paths**: - - `/opt/gh-aw/safeoutputs/assets/` + - `/tmp/gh-aw/safeoutputs/assets/` - **Artifact**: `agent-artifacts` - **Upload paths**: @@ -3307,7 +3307,7 @@ This section provides an overview of artifacts organized by job name, with dupli **Downloads:** - **Artifact**: `safe-outputs-assets` (by name) - - **Download path**: `/opt/gh-aw/safeoutputs/assets/` + - **Download path**: `/tmp/gh-aw/safeoutputs/assets/` - **Depends on jobs**: [agent detection] - **Artifact**: `agent-output` (by name) @@ -3421,7 +3421,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Artifact**: `safe-outputs-assets` - **Upload paths**: - - `/opt/gh-aw/safeoutputs/assets/` + - `/tmp/gh-aw/safeoutputs/assets/` - **Artifact**: `agent-artifacts` - **Upload paths**: @@ -3478,7 +3478,7 @@ This section provides an overview of artifacts organized by job name, with dupli **Downloads:** - **Artifact**: `safe-outputs-assets` (by name) - - **Download path**: `/opt/gh-aw/safeoutputs/assets/` + - **Download path**: `/tmp/gh-aw/safeoutputs/assets/` - **Depends on jobs**: [agent detection] - **Artifact**: `agent-output` (by name) @@ -4029,7 +4029,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Artifact**: `safe-outputs-assets` - **Upload paths**: - - `/opt/gh-aw/safeoutputs/assets/` + - `/tmp/gh-aw/safeoutputs/assets/` - **Artifact**: `agent-artifacts` - **Upload paths**: @@ -4086,7 +4086,7 @@ This section provides an overview of artifacts organized by job name, with dupli **Downloads:** - **Artifact**: `safe-outputs-assets` (by name) - - **Download path**: `/opt/gh-aw/safeoutputs/assets/` + - **Download path**: `/tmp/gh-aw/safeoutputs/assets/` - **Depends on jobs**: [agent detection] - **Artifact**: `agent-output` (by name) @@ -4205,7 +4205,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Artifact**: `safe-outputs-assets` - **Upload paths**: - - `/opt/gh-aw/safeoutputs/assets/` + - `/tmp/gh-aw/safeoutputs/assets/` - **Artifact**: `agent-artifacts` - **Upload paths**: @@ -4267,7 +4267,7 @@ This section provides an overview of artifacts organized by job name, with dupli **Downloads:** - **Artifact**: `safe-outputs-assets` (by name) - - **Download path**: `/opt/gh-aw/safeoutputs/assets/` + - **Download path**: `/tmp/gh-aw/safeoutputs/assets/` - **Depends on jobs**: [agent detection] - **Artifact**: `agent-output` (by name) @@ -4491,7 +4491,7 @@ This section provides an overview of artifacts organized by job name, with dupli - **Artifact**: `safe-outputs-assets` - **Upload paths**: - - `/opt/gh-aw/safeoutputs/assets/` + - `/tmp/gh-aw/safeoutputs/assets/` - **Artifact**: `agent-artifacts` - **Upload paths**: @@ -4548,7 +4548,7 @@ This section provides an overview of artifacts organized by job name, with dupli **Downloads:** - **Artifact**: `safe-outputs-assets` (by name) - - **Download path**: `/opt/gh-aw/safeoutputs/assets/` + - **Download path**: `/tmp/gh-aw/safeoutputs/assets/` - **Depends on jobs**: [agent detection] - **Artifact**: `agent-output` (by name)