feat(windows): generate docker compose from docker bake

jenkinsci · Jun 8, 2024 · 53e511a · 53e511a
1 parent 2985d6f
commit 53e511a
Show file tree

Hide file tree

Showing 10 changed files with 151 additions and 148 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,3 +3,4 @@ bats-core/
 bats/
 target/
 /.vscode/
+docker-compose-*.yaml
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,13 +1,13 @@
 def agentSelector(String imageType) {
-    // Image type running on a Linux agent
+    // Linux agent
     if (imageType == 'linux') {
         return 'linux'
     }
-    // Image types running on a Windows Server Core 2022 agent
+    // Windows Server Core 2022 agent
     if (imageType.contains('2022')) {
         return 'windows-2022'
     }
-    // Remaining image types running on a Windows Server Core 2019 agent: (nanoserver|windowservercore)-(1809|2019)
+    // Windows Server Core 2019 agent (for nanoserver|windowservercore)-(1809|ltsc2019)
     return 'windows-2019'
 }
 
@@ -20,6 +20,9 @@ pipeline {
 
     stages {
         stage('docker-ssh-agent') {
+            environment {
+                DOCKERHUB_ORGANISATION = "${infra.isTrusted() ? 'jenkins' : 'jenkins4eval'}"
+            }
             matrix {
                 axes {
                     axis {
@@ -28,26 +31,45 @@ pipeline {
                     }
                 }
                 stages {
+                    stage('Generate docker compose file') {
+                        agent {
+                            label 'linux'
+                        }
+                        when {
+                            not {
+                                environment name: 'IMAGE_TYPE', value: 'linux'
+                            }
+                        }
+                        steps {
+                            script {
+                                def imageTypeParts = env.IMAGE_TYPE.split('-')
+                                def windowsFlavor = "[\"${imageTypeParts[0]}\"]"
+                                def windowsVersion = "[\"${imageTypeParts[1]}\"]"
+                                def currentVersion = env.TAG_NAME ? env.TAG_NAME : '1.0.0'
+
+                                sh "WINDOWS_FLAVORS_TO_BUILD='$windowsFlavor' WINDOWS_VERSIONS_TO_BUILD='$windowsVersion' VERSION=$currentVersion ON_TAG=true make generate-compose-$IMAGE_TYPE"
+
+                                stash(name: "docker-compose-$IMAGE_TYPE", includes: 'docker-compose-*.yaml')
+                                archiveArtifacts 'docker-compose-*.yaml'
+                            }
+                        }
+                    }
                     stage('Main') {
                         agent {
                             label agentSelector(env.IMAGE_TYPE)
                         }
                         options {
                             timeout(time: 60, unit: 'MINUTES')
                         }
-                        environment {
-                            DOCKERHUB_ORGANISATION = "${infra.isTrusted() ? 'jenkins' : 'jenkins4eval'}"
-                        }
                         stages {
                             stage('Prepare Docker') {
                                 when {
                                     environment name: 'IMAGE_TYPE', value: 'linux'
                                 }
                                 steps {
-                                    sh '''
-                                    docker buildx create --use
-                                    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-                                    '''
+                                    script {
+                                        sh 'make init'
+                                    }
                                 }
                             }
                             stage('Build and Test') {
@@ -61,8 +83,9 @@ pipeline {
                                             sh 'make build'
                                             sh 'make test'
                                             // If the tests are passing for Linux AMD64, then we can build all the CPU architectures
-                                            sh 'docker buildx bake --file docker-bake.hcl linux'
+                                            sh 'make buildall'
                                         } else {
+                                            unstash(name: "docker-compose-$IMAGE_TYPE")
                                             powershell '& ./build.ps1 test'
                                         }
                                     }
@@ -78,18 +101,18 @@ pipeline {
                                 when {
                                     buildingTag()
                                 }
+                                environment {
+                                    ON_TAG = 'true'
+                                    VERSION = "${env.TAG_NAME}"
+                                }
                                 steps {
                                     script {
                                         // This function is defined in the jenkins-infra/pipeline-library
                                         infra.withDockerCredentials {
                                             if (isUnix()) {
-                                                sh """
-                                                export ON_TAG=true
-                                                export VERSION=$TAG_NAME
-                                                docker buildx bake --push --file docker-bake.hcl linux
-                                                """
+                                                sh 'make publish'
                                             } else {
-                                                powershell "& ./build.ps1 -VersionTag ${env.TAG_NAME} publish"
+                                                powershell '& ./build.ps1 publish'
                                             }
                                         }
                                     }

diff --git a/Makefile b/Makefile
@@ -22,7 +22,8 @@ check_cli = type "$(1)" >/dev/null 2>&1 || { echo "Error: command '$(1)' require
 ## Check if a given image exists in the current manifest docker-bake.hcl
 check_image = make --silent list | grep -w '$(1)' >/dev/null 2>&1 || { echo "Error: the image '$(1)' does not exist in manifest for the platform 'linux/$(ARCH)'. Please check the output of 'make list'. Exiting." ; exit 1 ; }
 ## Base "docker buildx base" command to be reused everywhere
-bake_base_cli := docker buildx bake -f docker-bake.hcl --load
+bake_base_cli := docker buildx bake --file docker-bake.hcl
+bake_cli := $(bake_base_cli) --load
 
 .PHONY: build
 .PHONY: test test-alpine test-archlinux test-debian test-jdk11 test-jdk11-alpine
@@ -37,15 +38,28 @@ check-reqs:
 	@$(call check_cli,curl)
 	@$(call check_cli,jq)
 
+init: check-reqs
+	@set -x; docker buildx create --use
+	docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
 build: check-reqs
-	@set -x; $(bake_base_cli) --set '*.platform=linux/$(ARCH)' $(shell make --silent list)
+	@set -x; $(bake_cli) --set '*.platform=linux/$(ARCH)' $(shell make --silent list)
+
+buildall: check-reqs
+	@set -x; $(bake_base_cli) linux
 
 build-%:
 	@$(call check_image,$*)
-	@set -x; $(bake_base_cli) --set '*.platform=linux/$(ARCH)' '$*'
+	@set -x; $(bake_cli) --set '*.platform=linux/$(ARCH)' '$*'
+
+## Generate docker compose file from docker bake definition to build Windows images
+generate-compose-%: check-reqs
+	@set -x; $(bake_cli) windows --print \
+		| yq -P '.target[] | del(.output) | {(. | key): {"image": .tags[0], "build": .}}' \
+		| yq '{"services": .}' > docker-compose-$*.yaml
 
 show:
-	@$(bake_base_cli) linux --print
+	@$(bake_cli) linux --print
 
 list: check-reqs
 	@set -x; make --silent show | jq -r '.target | path(.. | select(.platforms[] | contains("linux/$(ARCH)"))?) | add'
@@ -57,6 +71,9 @@ prepare-test: bats check-reqs
 	git submodule update --init --recursive
 	mkdir -p target
 
+publish:
+	@set -x; $(bake_base_cli) linux --push
+
 ## Define bats options based on environment
 # common flags for all tests
 bats_flags := $(TEST_SUITES)

diff --git a/build-windows.yaml b/build-windows.yaml
diff --git a/build.ps1 b/build.ps1
@@ -3,7 +3,7 @@ Param(
     [Parameter(Position=1)]
     [String] $Target = 'build',
     [String] $Build = '',
-    [String] $VersionTag = '1.0-1',
+    [String] $ImageType = 'nanoserver-ltsc2019',
     [switch] $DryRun = $false,
     # Output debug info for tests. Accepted values:
     # - empty (no additional test output)
@@ -15,43 +15,17 @@ Param(
 $ErrorActionPreference = 'Stop'
 $ProgressPreference = 'SilentlyContinue' # Disable Progress bar for faster downloads
 
-$Repository = 'ssh-agent'
-$Organisation = 'jenkins'
-$ImageType = 'windows-ltsc2019'
-
-$baseDockerCmd = 'docker-compose --file=build-windows.yaml'
-$baseDockerBuildCmd = '{0} build --parallel --pull' -f $baseDockerCmd
-
 if(![String]::IsNullOrWhiteSpace($env:TESTS_DEBUG)) {
-    $ImageType = $env:IMAGE_TYPE
+    $TestsDebug = $env:TESTS_DEBUG
 }
 $env:TESTS_DEBUG = $TestsDebug
 
-if(![String]::IsNullOrWhiteSpace($env:DOCKERHUB_REPO)) {
-    $Repository = $env:DOCKERHUB_REPO
-}
-
-if(![String]::IsNullOrWhiteSpace($env:DOCKERHUB_ORGANISATION)) {
-    $Organisation = $env:DOCKERHUB_ORGANISATION
-}
-
 if(![String]::IsNullOrWhiteSpace($env:IMAGE_TYPE)) {
     $ImageType = $env:IMAGE_TYPE
 }
 
-# Ensure constant env vars used in the docker compose file are defined
-$env:DOCKERHUB_ORGANISATION = "$Organisation"
-$env:DOCKERHUB_REPO = "$Repository"
-$env:VERSION = "$VersionTag"
-
-$items = $ImageType.Split('-')
-$env:WINDOWS_FLAVOR = $items[0]
-$env:WINDOWS_VERSION_TAG = $items[1]
-$env:TOOLS_WINDOWS_VERSION = $items[1]
-if ($items[1] -eq 'ltsc2019') {
-    # There are no eclipse-temurin:*-ltsc2019 or mcr.microsoft.com/powershell:*-ltsc2019 docker images unfortunately, only "1809" ones
-    $env:TOOLS_WINDOWS_VERSION = '1809'
-}
+$baseDockerCmd = 'docker-compose --file=docker-compose-{0}.yaml' -f $ImageType
+$baseDockerBuildCmd = '{0} build --parallel --pull' -f $baseDockerCmd
 
 # Check for required commands
 Function Test-CommandExists {
@@ -84,10 +58,11 @@ function Test-Image {
         $ImageNameAndJavaVersion
     )
 
-    $items = $ImageNameAndJavaVersion.Split("|")
-    $imageName = $items[0]
+    # Ex: docker.io/jenkins/ssh-agent:1.0.0-windowsservercore-ltsc2019-jdk21|21.0.3_9
+    $items = $ImageNameAndJavaVersion.Split('|')
+    $imageName = $items[0] -replace 'docker.io/', ''
     $javaVersion = $items[1]
-    $imageNameItems = $imageName.Split(":")
+    $imageNameItems = $imageName.Split(':')
     $imageTag = $imageNameItems[1]
 
     Write-Host "= TEST: Testing ${ImageName} image"
@@ -116,18 +91,15 @@ function Test-Image {
     return $failed
 }
 
-$baseDockerCmd = 'docker-compose --file=build-windows.yaml'
-$baseDockerBuildCmd = '{0} build --parallel --pull' -f $baseDockerCmd
-
-Write-Host "= PREPARE: List of $Organisation/$env:DOCKERHUB_REPO images and tags to be processed:"
+Write-Host '= PREPARE: List of images and tags to be processed:'
 Invoke-Expression "$baseDockerCmd config"
 
 Write-Host '= BUILD: Building all images...'
-    switch ($DryRun) {
-        $true { Write-Host "(dry-run) $baseDockerBuildCmd" }
-        $false { Invoke-Expression $baseDockerBuildCmd }
-    }
-    Write-Host '= BUILD: Finished building all images.'
+switch ($DryRun) {
+    $true { Write-Host "(dry-run) $baseDockerBuildCmd" }
+    $false { Invoke-Expression $baseDockerBuildCmd }
+}
+Write-Host '= BUILD: Finished building all images.'
 
 if($lastExitCode -ne 0) {
     exit $lastExitCode
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,3 +3,4 @@ bats-core/ @@
     bats/
     target/
     /.vscode/
+    docker-compose-*.yaml