chore: setup release to reuse CTF from components workflow

.github/config/goreleaser.yaml

@@ -8,7 +8,6 @@ release:
 before:
   hooks:
     - go mod tidy
-    - make CTF_TYPE=directory ctf
 
 builds:
   - <<: &build_defaults

.github/workflows/components.yaml

@@ -3,6 +3,22 @@ name: Components
 on:
   pull_request:
   workflow_call:
+    inputs:
+      effective-version:
+        type: string
+        required: false
+        description: "The version to use for the build"
+        default: ""
+      upload-ctf:
+        type: boolean
+        required: false
+        description: "Whether to upload the final CTF"
+        default: false
+      ref:
+        type: string
+        description: "The ref to use for the component build, defaults to the ref where the workflow was triggered from"
+        required: false
+        default: ""
   push:
     branches:
       - main
@@ -12,8 +28,11 @@ permissions:
   pull-requests: read
 
 env:
+  REF: ${{ inputs.ref == '' && github.ref || inputs.ref }}
   CTF_TYPE: directory
   components: '["ocmcli", "helminstaller", "helmdemo", "subchartsdemo", "ecrplugin"]'
+  IMAGE_PLATFORMS: 'linux/amd64 linux/arm64'
+  PLATFORMS: 'windows/amd64 darwin/arm64 darwin/amd64 linux/amd64 linux/arm64'
 
 jobs:
   define-matrix:
@@ -40,6 +59,8 @@ jobs:
         uses: TooMuch4U/actions-clean@v2.2
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          ref: ${{ env.REF }}
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
@@ -68,7 +89,14 @@ jobs:
       - name: CTF
         run: |
           cd components/${{ matrix.component }}
-          PATH=$PATH:$(go env GOPATH)/bin CTF_TYPE=${{ env.CTF_TYPE }} make ctf descriptor describe
+
+          PATH=$PATH:$(go env GOPATH)/bin \
+          CTF_TYPE=${{ env.CTF_TYPE }} \
+          EFFECTIVE_VERSION=${{ inputs.effective-version }} \
+          PLATFORMS="${{ env.PLATFORMS }}" \
+          IMAGE_PLATFORMS="${{ env.IMAGE_PLATFORMS }}" \
+          make \
+            ctf descriptor describe
       - name: Upload CTF
         uses: actions/upload-artifact@v4
         with:
@@ -89,15 +117,16 @@ jobs:
         uses: TooMuch4U/actions-clean@v2.2
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          ref: ${{ env.REF }}
       - name: Download CTFs
         uses: actions/download-artifact@v4
         with:
           pattern: 'ctf-component-*'
           path: gen/downloaded-ctfs
       - name: Move CTFs into correct directory for aggregation
         run: |
-          IFS=" " read -a COMPONENTS <<< "${{ env.components }}"
-          for i in "${COMPONENTS[@]}"; do
+          for i in ${{ env.components }}; do
             mkdir -p ${{ github.workspace }}/gen/${i}
             mv ${{ github.workspace }}/gen/downloaded-ctfs/ctf-component-${i} ${{ github.workspace }}/gen/${i}/ctf
             ls -R ${{ github.workspace }}/gen/${i}
@@ -130,14 +159,13 @@ jobs:
                 ${{ github.workspace }}/gen/ctf
           done
       - name: Upload aggregated CTF
-        # TODO This is currently permanently disabled,
-        # until we integrate it with the release build, in which it would be reused
-        if: false
+        # only upload the artifact if we are not on a PR
+        if: inputs.upload-ctf
         uses: actions/upload-artifact@v4
         with:
           if-no-files-found: error
           overwrite: true
-          retention-days: 60
+          retention-days: 30
           name: ctf-aggregated
           path: gen/ctf
       - name: Delete old CTFs that lead up to aggregation

.github/workflows/release.yaml

@@ -81,18 +81,22 @@ jobs:
   components:
     name: Component CTF Builds
     uses: ./.github/workflows/components.yaml
-    needs: check
+    needs: [check,release-version]
+    with:
+      effective-version: ${{ needs.release-version.outputs.version_no_prefix }}
+      upload-ctf: true
+      ref: ${{ github.ref }}
     permissions:
       contents: read
       pull-requests: read
 
-
   release:
     needs:
     # run check before actual release to make sure we succeed
     # they will be skipped from the needs check
     - check
     - release-version
+    - components
     name: Release Build
     runs-on: large_runner
     permissions:
@@ -124,6 +128,45 @@ jobs:
     - name: Setup Cosign
       uses: sigstore/cosign-installer@v3.7.0
 
+    - name: Download CTF
+      uses: actions/download-artifact@v4
+      with:
+        pattern: 'ctf-aggregated'
+        path: gen/downloaded-ctfs
+    - name: Move CTF into correct directory to be recognized by the release process
+      run: |
+        mv \
+            ${{ github.workspace }}/gen/downloaded-ctfs/ctf-aggregated \
+            ${{ github.workspace }}/gen/ctf
+
+    # TODO: Remove Go setup once binaries no longer need to be built by goreleaser.
+    - name: Setup Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: '${{ github.workspace }}/go.mod'
+        check-latest: false
+        cache: false
+    - name: Get go environment for use with cache
+      run: |
+        echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
+        echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
+      # This step will only reuse the go mod and build cache from main made during the Build,
+      # see push_ocm.yaml => "ocm-cli-latest" Job
+      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
+      # This is because we have huge storage requirements for our cache because of the mass of dependencies
+    - name: Restore / Reuse Cache from central build
+      id: cache-golang-restore
+      uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
+      with:
+        path: |
+          ${{ env.go_cache }}
+          ${{ env.go_modcache }}
+        key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
+        restore-keys: |
+          ${{ env.cache_name }}-${{ runner.os }}-go-
+      env:
+        cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
+
     - name: Setup git config
       run: |
         git config user.name "GitHub Actions Bot"

Makefile

@@ -3,9 +3,13 @@ REPO_ROOT                                      := $(shell dirname $(realpath $(l
 GITHUBORG                                      ?= open-component-model
 OCMREPO                                        ?= ghcr.io/$(GITHUBORG)/ocm
 VERSION                                        := $(shell go run api/version/generate/release_generate.go print-rc-version $(CANDIDATE))
-EFFECTIVE_VERSION                              := $(VERSION)+$(shell git rev-parse HEAD)
+COMMIT                                         = $(shell git rev-parse --verify HEAD)
+# if EFFECTIVE_VERSION is not set, set it to VERSION+HEAD
+# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string
+ifeq ($(EFFECTIVE_VERSION),)
+EFFECTIVE_VERSION                              := $(VERSION)+$(COMMIT)
+endif
 GIT_TREE_STATE                                 := $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty)
-COMMIT                                         := $(shell git rev-parse --verify HEAD)
 
 CONTROLLER_TOOLS_VERSION ?= v0.14.0
 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen

components/demoplugin/Makefile

@@ -3,13 +3,17 @@ PROVIDER  ?= ocm.software
 GITHUBORG  ?= open-component-model
 COMPONENT = $(PROVIDER)/plugins/$(NAME)
 OCMREPO   ?= ghcr.io/$(GITHUBORG)/ocm
-PLATFORMS = linux/amd64 linux/arm64
+PLATFORMS ?= linux/amd64 linux/arm64
 CTF_TYPE ?= directory
 
 REPO_ROOT                                      := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))/../..
 VERSION                                        = $(shell go run ../../api/version/generate/release_generate.go print-rc-version $(CANDIDATE))
-COMMIT                                         = $(shell git rev-parse HEAD)
-EFFECTIVE_VERSION                              = $(VERSION)+$(COMMIT)
+COMMIT                                         = $(shell git rev-parse --verify HEAD)
+# if EFFECTIVE_VERSION is not set, set it to VERSION+COMMIT
+# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string
+ifeq ($(EFFECTIVE_VERSION),)
+EFFECTIVE_VERSION                              := $(VERSION)+$(COMMIT)
+endif
 GIT_TREE_STATE                                 := $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty)
 
 CMDSRCS=$(shell find $(REPO_ROOT)/cmds/$(NAME) -type f)

components/ecrplugin/Makefile

@@ -3,14 +3,18 @@ PROVIDER  ?= ocm.software
 GITHUBORG  ?= open-component-model
 COMPONENT = $(PROVIDER)/plugins/$(NAME)
 OCMREPO   ?= ghcr.io/$(GITHUBORG)/ocm
-PLATFORMS = linux/amd64 linux/arm64 darwin/amd64 darwin/arm64
+PLATFORMS ?= linux/amd64 linux/arm64 darwin/amd64 darwin/arm64
 CTF_TYPE ?= directory
 
 
 REPO_ROOT                                      := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))/../..
 VERSION                                        = $(shell go run ../../api/version/generate/release_generate.go print-rc-version $(CANDIDATE))
-COMMIT                                         = $(shell git rev-parse HEAD)
-EFFECTIVE_VERSION                              = $(VERSION)+$(COMMIT)
+COMMIT                                         = $(shell git rev-parse --verify HEAD)
+# if EFFECTIVE_VERSION is not set, set it to VERSION+COMMIT
+# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string
+ifeq ($(EFFECTIVE_VERSION),)
+EFFECTIVE_VERSION                              := $(VERSION)+$(COMMIT)
+endif
 GIT_TREE_STATE                                 := $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty)
 
 CMDSRCS=$(shell find $(REPO_ROOT)/cmds/$(NAME) -type f)

components/helmdemo/Makefile

@@ -9,8 +9,12 @@ HELMINSTCOMP = $(PROVIDER)/toi/installers/helminstaller
 
 REPO_ROOT                                     := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))/../..
 VERSION                                        = $(shell go run ../../api/version/generate/release_generate.go print-rc-version $(CANDIDATE))
-COMMIT                                         = $(shell git rev-parse HEAD)
-EFFECTIVE_VERSION                              = $(VERSION)-$(COMMIT)
+COMMIT                                         = $(shell git rev-parse --verify HEAD)
+# if EFFECTIVE_VERSION is not set, set it to VERSION-COMMIT
+# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string
+ifeq ($(EFFECTIVE_VERSION),)
+EFFECTIVE_VERSION                              := $(VERSION)+$(COMMIT)
+endif
 HELMINSTVERSION                               ?= $(VERSION)
 
 CREDS ?=

components/helminstaller/Makefile

@@ -5,13 +5,17 @@ IMAGE     := $(NAME)
 COMPONENT := $(PROVIDER)/toi/installers/$(NAME)
 OCMREPO   ?= ghcr.io/$(GITHUBORG)/ocm
 MULTI     ?= true
-PLATFORMS ?= linux/amd64 linux/arm64
+IMAGE_PLATFORMS ?= linux/amd64 linux/arm64
 CTF_TYPE ?= directory
 
 REPO_ROOT                                      := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))/../..
 VERSION                                        := $(shell go run ../../api/version/generate/release_generate.go print-rc-version $(CANDIDATE))
 COMMIT                                         := $(shell git rev-parse --verify HEAD)
-EFFECTIVE_VERSION                              := $(VERSION)-$(COMMIT)
+# if EFFECTIVE_VERSION is not set, set it to VERSION-COMMIT
+# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string
+ifeq ($(EFFECTIVE_VERSION),)
+EFFECTIVE_VERSION                              := $(VERSION)+$(COMMIT)
+endif
 GIT_TREE_STATE                                 := $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty)
 PLATFORM                                       := $(shell go env GOOS)/$(shell go env GOARCH)
 CACHE_DIR									  := $(shell go env GOCACHE)
@@ -64,7 +68,7 @@ $(GEN)/ctf: $(OCM_BIN) $(GEN)/.exists $(GEN)/image.$(NAME)$(FLAGSUF) component-c
 		PROVIDER="$(PROVIDER)" \
 		COMMIT="$(COMMIT)" \
 		GEN="$(GEN)" \
-		PLATFORMS="$(PLATFORMS)" \
+		IMAGE_PLATFORMS="$(IMAGE_PLATFORMS)" \
 		MULTI="$(MULTI)" \
 		IMAGE="$(IMAGE):$(VERSION)" \
 		component-constructor.yaml
@@ -94,7 +98,7 @@ push-image:
 multi: $(GEN)/image.$(NAME).multi
 
 $(GEN)/image.$(NAME).multi: $(GEN)/.exists Dockerfile $(CMDSRCS) $(OCMSRCS)
-	for i in $(PLATFORMS); do \
+	for i in $(IMAGE_PLATFORMS); do \
 	tag=$$(echo $$i | sed -e s:/:-:g); \
         echo building platform $$i; \
 	docker buildx build --load -t $(IMAGE):$(VERSION)-$$tag --platform $$i --file Dockerfile $(REPO_ROOT) \

components/helminstaller/component-constructor.yaml

@@ -21,5 +21,5 @@ components:
         input:
           type: (( bool(values.MULTI) ? "dockermulti" :"docker" ))
           repository:  (( index(values.IMAGE, ":") >= 0 ? substr(values.IMAGE,0,index(values.IMAGE,":")) :values.IMAGE ))
-          variants: (( bool(values.MULTI) ? map[split(" ", values.PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ ))
+          variants: (( bool(values.MULTI) ? map[split(" ", values.IMAGE_PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ ))
           path: (( !bool(values.MULTI) ? values.IMAGE :~~ ))

components/ocmcli/Makefile

@@ -7,14 +7,18 @@ COMPONENT = $(PROVIDER)/$(NAME)
 OCMREPO   ?= ghcr.io/$(GITHUBORG)/ocm
 MULTI     ?= true
 IMAGE_PLATFORMS ?= linux/amd64 linux/arm64
-PLATFORMS = $(IMAGE_PLATFORMS) darwin/arm64 darwin/amd64 windows/amd64
+PLATFORMS ?= $(IMAGE_PLATFORMS) darwin/arm64 darwin/amd64 windows/amd64
 CTF_TYPE ?= directory
 
 REPO_ROOT                                      := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))../..
 GIT_TREE_STATE                                 = $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty)
 VERSION                                        = $(shell go run ../../api/version/generate/release_generate.go print-rc-version $(CANDIDATE))
-COMMIT                                         = $(shell git rev-parse HEAD)
-EFFECTIVE_VERSION                              = $(VERSION)+$(COMMIT)
+COMMIT                                         = $(shell git rev-parse --verify HEAD)
+# if EFFECTIVE_VERSION is not set, set it to VERSION+COMMIT
+# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string
+ifeq ($(EFFECTIVE_VERSION),)
+EFFECTIVE_VERSION                              := $(VERSION)+$(COMMIT)
+endif
 PLATFORM_OS                                    := $(shell go env GOOS)
 PLATFORM_ARCH                                  := $(shell go env GOARCH)
 

components/subchartsdemo/Makefile

@@ -14,8 +14,12 @@ PODINFO_CHART_VERSION = 6.3.5
 REPO_ROOT                                     := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))/../..
 GIT_TREE_STATE                                 = $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty)
 VERSION                                        = $(shell go run $(REPO_ROOT)/api/version/generate/release_generate.go print-rc-version $(CANDIDATE))
-COMMIT                                         = $(shell git rev-parse HEAD)
-EFFECTIVE_VERSION                              = $(VERSION)-$(COMMIT)
+COMMIT                                         = $(shell git rev-parse --verify HEAD)
+# if EFFECTIVE_VERSION is not set, set it to VERSION-COMMIT
+# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string
+ifeq ($(EFFECTIVE_VERSION),)
+EFFECTIVE_VERSION                              := $(VERSION)+$(COMMIT)
+endif
 HELMINSTVERSION                               ?= $(VERSION)
 
 CREDS ?=

examples/make/Makefile

@@ -3,8 +3,8 @@ OCI_REPO := ghcr.io/mandelsoft/cnudie
 
 REPO_ROOT                                      := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 VERSION                                        = $(shell cat $(REPO_ROOT)/VERSION)
-COMMIT                                         = $(shell git rev-parse HEAD)
-EFFECTIVE_VERSION                              = $(VERSION)-$(COMMIT)
+COMMIT                                         = $(shell git rev-parse --verify HEAD)
+EFFECTIVE_VERSION                              = $(VERSION)+$(COMMIT)
 
 .PHONY: ctf
 ctf: ca ## Create CTF from component archive