From 47572b483868e3b9b03329c28c8d32b32e65bf0c Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Wed, 6 Nov 2024 21:10:32 +0100 Subject: [PATCH 1/3] chore: fix ocm inception during build While building the components we face several layers of what I would call "OCM Inception". This is the problem that while building and packaging OCM, we need OCM. To solve this chicken and egg problem, a small intermediary script is now introduced that reuses the OCM from the built CTF. --- .github/workflows/components.yaml | 29 +++++++++++++-- hack/get_bare_resource_from_ctf.sh | 59 ++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 hack/get_bare_resource_from_ctf.sh diff --git a/.github/workflows/components.yaml b/.github/workflows/components.yaml index ca944d5cfe..8c72077d71 100644 --- a/.github/workflows/components.yaml +++ b/.github/workflows/components.yaml @@ -127,12 +127,33 @@ jobs: mv ${{ github.workspace }}/gen/downloaded-ctfs/ctf-component-${i} ${{ github.workspace }}/gen/${i}/ctf ls -R ${{ github.workspace }}/gen/${i} done + - name: Extract OCM Binary from CTF to avoid OCM Inception + id: extract-ocm + run: | + ocm_binary=$(bash ./hack/get_bare_resource_from_ctf.sh \ + "ocm.software/ocmcli" \ + "" \ + "ocmcli" \ + $(go env GOARCH) \ + $(go env GOOS) \ + "application/octet-stream" \ + ${{ github.workspace }}/gen/ocmcli/ctf) + + new_loc=${{ github.workspace }}/bin/ocm + mkdir -p $(dirname $new_loc) + ln -s $ocm_binary $new_loc + chmod +x $new_loc + echo "OCM binary linked to $new_loc" + echo "binary=$new_loc" >> $GITHUB_OUTPUT - name: Create aggregated CTF run: | - PATH=$PATH:$(go env GOPATH)/bin \ - CTF_TYPE=${{ env.CTF_TYPE }} \ - COMPONENTS="${{ env.components }}" \ - make plain-ctf + for i in ${{ env.components }}; do + echo "transfering component $i..." + ${{ steps.extract-ocm.outputs.binary }} transfer cv \ + --type ${{ env.CTF_TYPE }} -V \ + ${{ github.workspace }}/gen/$i/ctf \ + ${{ github.workspace }}/gen/ctf + done - name: Upload aggregated CTF # only upload the artifact if we are not on a PR if: needs.pr-check.outputs.number != 'null' diff --git a/hack/get_bare_resource_from_ctf.sh b/hack/get_bare_resource_from_ctf.sh new file mode 100644 index 0000000000..242c85da98 --- /dev/null +++ b/hack/get_bare_resource_from_ctf.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -e + +# This script is used to get a bare resource from a CTF file. +# It can be used in case the OCM CLI is not available to extract the resource from a CTF. +# A typical use case for this is the "OCM Inception" in which a CTF containing the CLI needs to be extracted +# to run the CLI to extract the resource. +# +# In this case one can use this script to extract the correct OCM CLI without having to rely on the CLI being +# already available. +# +# By default the script will look for the OCM CLI component with any version (the first encountered will be used) +# and will extract the resource "ocmcli" for amd64/linux as a filepath. This path can then be used to run the CLI, +# but only after allowing to execute it, e.g with `chmod +x `. + +COMPONENT=${1:-"ocm.software/ocmcli"} +COMPONENT_VERSION=${2:-""} +RESOURCE=${3:-"ocmcli"} +ARCHITECTURE=${4:-"amd64"} +OS=${5:-"linux"} +MEDIA_TYPE=${6:-"application/octet-stream"} +PATH_TO_CTF=${7:-"./gen/ctf"} + +INDEX=$( \ +yq -r ".artifacts | filter(.repository == \"component-descriptors/${COMPONENT}\" and (.tag | contains(\"${COMPONENT_VERSION}\")))[0].digest" \ + "${PATH_TO_CTF}"/artifact-index.json | \ + sed 's/:/./g' \ +) + +if [ -z "${INDEX}" ]; then + echo "No index found for ${COMPONENT}" + exit 1 +fi + +RESOURCE=$( \ +yq ".layers | filter( + ( + .annotations.\"software.ocm.artifact\" | + from_json | + .[0] + ) as \$artifact | + ( + \$artifact.identity.name == \"$RESOURCE\" and + \$artifact.identity.architecture == \"$ARCHITECTURE\" and + \$artifact.identity.os == \"$OS\" and + .mediaType == \"$MEDIA_TYPE\" + ) + )[0].digest" "${PATH_TO_CTF}"/blobs/"${INDEX}" | sed 's/:/./g' \ +) + +if [ -z "${RESOURCE}" ]; then + echo "No resource found for ${COMPONENT}" + exit 1 +fi + +RESOURCE=$PATH_TO_CTF/blobs/$RESOURCE + +echo "$RESOURCE" From efda0fdfc760e10961e85d1968ac6dccf430b013 Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Fri, 8 Nov 2024 16:37:34 +0100 Subject: [PATCH 2/3] chore: disable CTF upload until inuse --- .github/workflows/components.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/components.yaml b/.github/workflows/components.yaml index 8c72077d71..732e780299 100644 --- a/.github/workflows/components.yaml +++ b/.github/workflows/components.yaml @@ -155,8 +155,9 @@ jobs: ${{ github.workspace }}/gen/ctf done - name: Upload aggregated CTF - # only upload the artifact if we are not on a PR - if: needs.pr-check.outputs.number != 'null' + # TODO This is currently permanently disabled, + # until we integrate it with the release build, in which it would be reused + if: false uses: actions/upload-artifact@v4 with: if-no-files-found: error From 2e963932ed98467568b13549e0612e3e98e2aed7 Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Fri, 8 Nov 2024 16:40:17 +0100 Subject: [PATCH 3/3] chore: remove need for go in aggregate --- .github/workflows/components.yaml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/.github/workflows/components.yaml b/.github/workflows/components.yaml index 732e780299..ff7f6b9af4 100644 --- a/.github/workflows/components.yaml +++ b/.github/workflows/components.yaml @@ -89,31 +89,6 @@ jobs: uses: TooMuch4U/actions-clean@v2.2 - name: Checkout uses: actions/checkout@v4 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version-file: '${{ github.workspace }}/go.mod' - cache: false - - name: Get go environment for use with cache - run: | - echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV - echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV - # This step will only reuse the go mod and build cache from main made during the Build, - # see push_ocm.yaml => "ocm-cli-latest" Job - # This means it never caches by itself and PRs cannot cause cache pollution / thrashing - # This is because we have huge storage requirements for our cache because of the mass of dependencies - - name: Restore / Reuse Cache from central build - id: cache-golang-restore - uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big) - with: - path: | - ${{ env.go_cache }} - ${{ env.go_modcache }} - key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }} - restore-keys: | - ${{ env.cache_name }}-${{ runner.os }}-go- - env: - cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step - name: Download CTFs uses: actions/download-artifact@v4 with: