lemeurherve · lemeurherve · Jun 9, 2024 · Jun 10, 2024 · Jun 10, 2024 · Jun 9, 2024
diff --git a/.gitignore b/.gitignore
@@ -3,3 +3,4 @@ bats-core/
 bats/
 target/
 /.vscode/
+build-windows.yaml
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,13 +1,13 @@
 def agentSelector(String imageType) {
-    // Image type running on a Linux agent
+    // Linux agent
     if (imageType == 'linux') {
         return 'linux'
     }
-    // Image types running on a Windows Server Core 2022 agent
+    // Windows Server Core 2022 agent
     if (imageType.contains('2022')) {
         return 'windows-2022'
     }
-    // Remaining image types running on a Windows Server Core 2019 agent: (nanoserver|windowservercore)-(1809|2019)
+    // Windows Server Core 2019 agent (for nanoserver 1809 & ltsc2019 and for windowservercore ltsc2019)
     return 'windows-2019'
 }
 
@@ -20,6 +20,9 @@ pipeline {
 
     stages {
         stage('docker-ssh-agent') {
+            environment {
+                DOCKERHUB_ORGANISATION = "${infra.isTrusted() ? 'jenkins' : 'jenkins4eval'}"
+            }
             matrix {
                 axes {
                     axis {
@@ -35,19 +38,13 @@ pipeline {
                         options {
                             timeout(time: 60, unit: 'MINUTES')
                         }
-                        environment {
-                            DOCKERHUB_ORGANISATION = "${infra.isTrusted() ? 'jenkins' : 'jenkins4eval'}"
-                        }
                         stages {
                             stage('Prepare Docker') {
                                 when {
                                     environment name: 'IMAGE_TYPE', value: 'linux'
                                 }
                                 steps {
-                                    sh '''
-                                    docker buildx create --use
-                                    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-                                    '''
+                                    sh 'make ci-init'
                                 }
                             }
                             stage('Build and Test') {
@@ -61,7 +58,7 @@ pipeline {
                                             sh 'make build'
                                             sh 'make test'
                                             // If the tests are passing for Linux AMD64, then we can build all the CPU architectures
-                                            sh 'docker buildx bake --file docker-bake.hcl linux'
+                                            sh 'make buildall'
                                         } else {
                                             powershell '& ./build.ps1 test'
                                         }
@@ -78,18 +75,18 @@ pipeline {
                                 when {
                                     buildingTag()
                                 }
+                                environment {
+                                    ON_TAG = 'true'
+                                    VERSION = "${env.TAG_NAME}"
+                                }
                                 steps {
                                     script {
                                         // This function is defined in the jenkins-infra/pipeline-library
                                         infra.withDockerCredentials {
                                             if (isUnix()) {
-                                                sh """
-                                                export ON_TAG=true
-                                                export VERSION=$TAG_NAME
-                                                docker buildx bake --push --file docker-bake.hcl linux
-                                                """
+                                                sh 'make publish'
                                             } else {
-                                                powershell "& ./build.ps1 -VersionTag ${env.TAG_NAME} publish"
+                                                powershell '& ./build.ps1 publish'
                                             }
                                         }
                                     }

diff --git a/Makefile b/Makefile
@@ -22,7 +22,8 @@ check_cli = type "$(1)" >/dev/null 2>&1 || { echo "Error: command '$(1)' require
 ## Check if a given image exists in the current manifest docker-bake.hcl
 check_image = make --silent list | grep -w '$(1)' >/dev/null 2>&1 || { echo "Error: the image '$(1)' does not exist in manifest for the platform 'linux/$(ARCH)'. Please check the output of 'make list'. Exiting." ; exit 1 ; }
 ## Base "docker buildx base" command to be reused everywhere
-bake_base_cli := docker buildx bake -f docker-bake.hcl --load
+bake_base_cli := docker buildx bake --file docker-bake.hcl
+bake_cli := $(bake_base_cli) --load
 
 .PHONY: build
 .PHONY: test test-alpine test-archlinux test-debian test-jdk11 test-jdk11-alpine
@@ -37,15 +38,23 @@ check-reqs:
 	@$(call check_cli,curl)
 	@$(call check_cli,jq)
 
+## This function is specific to Jenkins infrastructure and isn't required in other contexts
+ci-init: check-reqs
+	@set -x; docker buildx create --use
+	docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
 build: check-reqs
-	@set -x; $(bake_base_cli) --set '*.platform=linux/$(ARCH)' $(shell make --silent list)
+	@set -x; $(bake_cli) --set '*.platform=linux/$(ARCH)' $(shell make --silent list)
 
 build-%:
 	@$(call check_image,$*)
-	@set -x; $(bake_base_cli) --set '*.platform=linux/$(ARCH)' '$*'
+	@set -x; $(bake_cli) --set '*.platform=linux/$(ARCH)' '$*'
+
+buildall: check-reqs
+	@set -x; $(bake_base_cli) linux
 
 show:
-	@$(bake_base_cli) linux --print
+	@$(bake_cli) linux --print
 
 list: check-reqs
 	@set -x; make --silent show | jq -r '.target | path(.. | select(.platforms[] | contains("linux/$(ARCH)"))?) | add'
@@ -57,6 +66,9 @@ prepare-test: bats check-reqs
 	git submodule update --init --recursive
 	mkdir -p target
 
+publish:
+	@set -x; $(bake_base_cli) linux --push
+
 ## Define bats options based on environment
 # common flags for all tests
 bats_flags := $(TEST_SUITES)

diff --git a/README.md b/README.md
@@ -120,22 +120,23 @@ If you want to build a specific image, you can issue the following command:
 make build-<OS>_<JDK_VERSION>
 ```
 
-That would give for JDK 11 on Alpine Linux:
+That would give for JDK 17 on Alpine Linux:
 
 ```bash
-make build-alpine_jdk11
+make build-alpine_jdk17
 ```
 
-#### Building all images
-Then, you can build all the images by running:
+#### Building images supported by your current architecture
+
+Then, you can build the images supported by your current architecture by running:
 
 ```bash
 make build
 ```
 
 #### Testing all images
 
-If you want to test the images, you can run:
+If you want to test these images, you can run:
 
 ```bash
 make test
@@ -148,10 +149,18 @@ If you want to test a specific image, you can run:
 make test-<OS>_<JDK_VERSION>
 ```
 
-That would give for JDK 11 on Alpine Linux:
+That would give for JDK 17 on Alpine Linux:
+
+```bash
+make test-alpine_jdk17
+```
+
+#### Building all images
+
+You can build all images (even those unsupported by your current architecture) by running:
 
 ```bash
-make test-alpine_jdk11
+make buildall
 ```
 
 #### Other `make` targets
@@ -196,6 +205,10 @@ make bats
 make: 'bats' is up to date.
 ```
 
+`publish` allows the publication of all images targeted by 'linux' to a registry.
+
+`ci-init` is dedicated to Jenkins infrastructure for initialising docker and isn't required in other contexts.
+
 ### Building and testing on Windows
 
 From a Powershell console, set first the `IMAGE_TYPE` environment variable defining the Windows flavor ("nanoserver"/"windowsservercore") and  version you want to build.

diff --git a/build-windows.yaml b/build-windows.yaml
diff --git a/build.ps1 b/build.ps1
@@ -1,14 +1,18 @@
 [CmdletBinding()]
 Param(
     [Parameter(Position=1)]
+    # Default build.ps1 target
     [String] $Target = 'build',
     [String] $Build = '',
-    [String] $VersionTag = '1.0-1',
+    # Image version
+    [String] $VersionTag = '0.0.1',
+    # Windows flavor and windows version to build
+    [String] $ImageType = 'nanoserver-ltsc2019',
+    # Generate a docker compose file even if it already exists
+    [switch] $OverwriteDockerComposeFile = $false,
+    # Print the build and publish command instead of executing them if set
     [switch] $DryRun = $false,
-    # Output debug info for tests. Accepted values:
-    # - empty (no additional test output)
-    # - 'debug' (test cmd & stderr outputed)
-    # - 'verbose' (test cmd, stderr, stdout outputed)
+    # Output debug info for tests: 'empty' (no additional test output), 'debug' (test cmd & stderr outputed), 'verbose' (test cmd, stderr, stdout outputed)
     [String] $TestsDebug = ''
 )
 
@@ -17,7 +21,6 @@ $ProgressPreference = 'SilentlyContinue' # Disable Progress bar for faster downl
 
 $Repository = 'ssh-agent'
 $Organisation = 'jenkins'
-$ImageType = 'windows-ltsc2019'
 
 if(![String]::IsNullOrWhiteSpace($env:TESTS_DEBUG)) {
     $TestsDebug = $env:TESTS_DEBUG
@@ -32,6 +35,10 @@ if(![String]::IsNullOrWhiteSpace($env:DOCKERHUB_ORGANISATION)) {
     $Organisation = $env:DOCKERHUB_ORGANISATION
 }
 
+if(![String]::IsNullOrWhiteSpace($env:VERSION)) {
+    $VersionTag = $env:VERSION
+}
+
 if(![String]::IsNullOrWhiteSpace($env:IMAGE_TYPE)) {
     $ImageType = $env:IMAGE_TYPE
 }
@@ -41,15 +48,6 @@ $env:DOCKERHUB_ORGANISATION = "$Organisation"
 $env:DOCKERHUB_REPO = "$Repository"
 $env:VERSION = "$VersionTag"
 
-$items = $ImageType.Split('-')
-$env:WINDOWS_FLAVOR = $items[0]
-$env:WINDOWS_VERSION_TAG = $items[1]
-$env:TOOLS_WINDOWS_VERSION = $items[1]
-if ($items[1] -eq 'ltsc2019') {
-    # There are no eclipse-temurin:*-ltsc2019 or mcr.microsoft.com/powershell:*-ltsc2019 docker images unfortunately, only "1809" ones
-    $env:TOOLS_WINDOWS_VERSION = '1809'
-}
-
 # Check for required commands
 Function Test-CommandExists {
     # From https://devblogs.microsoft.com/scripting/use-a-powershell-function-to-see-if-a-command-exists/
@@ -60,8 +58,14 @@ Function Test-CommandExists {
     $oldPreference = $ErrorActionPreference
     $ErrorActionPreference = 'stop'
     try {
-        if(Get-Command $command){
+        # Special case to test "docker buildx"
+        if ($command.Contains(" ")) {
+            Invoke-Expression $command | Out-Null
             Write-Debug "$command exists"
+        } else {
+            if(Get-Command $command){
+                Write-Debug "$command exists"
+            }
         }
     }
     Catch {
@@ -74,15 +78,17 @@ Function Test-CommandExists {
 
 Test-CommandExists 'docker'
 Test-CommandExists 'docker-compose'
+Test-CommandExists 'docker buildx'
 Test-CommandExists 'yq'
 
 function Test-Image {
     param (
         $ImageNameAndJavaVersion
     )
 
+    # Ex: docker.io/jenkins/ssh-agent:windowsservercore-ltsc2019-jdk21|21.0.3_9
     $items = $ImageNameAndJavaVersion.Split('|')
-    $imageName = $items[0]
+    $imageName = $items[0] -replace 'docker.io/', ''
     $javaVersion = $items[1]
     $imageNameItems = $imageName.Split(':')
     $imageTag = $imageNameItems[1]
@@ -113,10 +119,55 @@ function Test-Image {
     return $failed
 }
 
-$baseDockerCmd = 'docker-compose --file=build-windows.yaml'
+$dockerBakeFile = 'docker-bake.hcl'
+$dockerComposeFile = 'build-windows.yaml'
+
+$baseDockerBakeCmd = 'docker buildx bake --progress=plain --file={0}' -f $dockerBakeFile
+$baseDockerCmd = 'docker-compose --file={0}' -f $dockerComposeFile
 $baseDockerBuildCmd = '{0} build --parallel --pull' -f $baseDockerCmd
 
-Write-Host "= PREPARE: List of images and tags to be processed:"
+# Generate the docker compose file if it doesn't exists or if the parameter OverwriteDockerComposeFile is set
+$generateDockerCompose = $true
+if ((Test-Path $dockerComposeFile) -and -not $OverwriteDockerComposeFile) {
+    Write-Host "= PREPARE: The docker compose file '$dockerComposeFile' containing the image definitions already exists."
+    $generateDockerCompose = $false
+}
+if ($generateDockerCompose) {
+    Write-Host "= PREPARE: Generating the docker compose file '$dockerComposeFile' containing the image definitions from '$dockerBakeFile'"
+    $items = $ImageType.Split('-')
+    $windowsFlavor = $items[0]
+    $windowsVersion = $items[1]
+
+    # Retrieve the targets from docker buildx bake --print output
+    # Remove the 'output' section (unsupported by docker compose)
+    # For each target name as service key, return a map consisting of:
+    # - 'image' set to the first tag value and
+    # - 'build' set to the content of the bake target
+    $yqMainQuery = '''.target[]' + `
+        ' | del(.output)' + `
+        ' | {(. | key): {\"image\": .tags[0], \"build\": .}}'''
+    # Encapsulate under a top level 'services' map
+    $yqServicesQuery = '''{\"services\": .}'''
+
+    # Override the list of Windows versions to build by the one from image type
+    $env:WINDOWS_VERSION_OVERRIDE = $windowsVersion
+
+    # - Use docker buildx bake to output image definitions from the "<windowsFlavor>" bake target
+    # - Convert with yq to the format expected by docker compose
+    # - Store the result in the docker compose file
+    $generateDockerComposeFileCmd = ' {0} {1} --print' -f $baseDockerBakeCmd, $windowsFlavor + `
+        ' | yq --prettyPrint {0} | yq {1}' -f $yqMainQuery, $yqServicesQuery + `
+        ' | Out-File -FilePath {0}' -f $dockerComposeFile
+
+    Write-Host "= PREPARE: Docker compose file generation command`n$generateDockerComposeFileCmd"
+
+    Invoke-Expression $generateDockerComposeFileCmd
+
+    # Remove override
+    Remove-Item env:\WINDOWS_VERSION_OVERRIDE
+}
+
+Write-Host '= PREPARE: List of images and tags to be processed:'
 Invoke-Expression "$baseDockerCmd config"
 
 Write-Host '= BUILD: Building all images...'
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,3 +3,4 @@ bats-core/ @@
     bats/
     target/
     /.vscode/
+    build-windows.yaml