diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59f25a95..43c5cedf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,9 +32,6 @@ env: RUSTFLAGS: "-D warnings" RUSTDOCFLAGS: "-D warnings" RUST_LOG: "info" - DEV_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-dev - TEST_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-test - STABLE_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-stable jobs: # Identify unused dependencies @@ -63,47 +60,41 @@ jobs: - uses: stackabletech/cargo-install-action@cargo-udeps - run: cargo udeps --workspace --all-targets - # This job evaluates the github environment to determine why this action is running and selects the appropriate - # target repository for published Helm charts based on this. + # This job evaluates the github environment to determine why this action is running and decides if + # Helm charts are published based on this. # # The following scenarios are identified: - # - all pull requests land in the test repository: + # - all pull requests land are published: # condition: github.event_name == "pull_request" - # repository: test # - # - all tagged releases land in stable: + # - all tagged releases are published: # condition: github.event_name == 'push' & github.ref.startswith('refs/tags/') - # repository: stable # - # - all pushes to main (i.e. PR-merges) and all scheduled/manual workflow runs on main land in dev: + # - all pushes to main (i.e. PR-merges) and all scheduled/manual workflow runs on main land are published: # condition: ( github.event_name == 'push' | github.event_name == 'schedule' | github.event_name == 'workflow_dispatch' ) & github.ref == 'refs/heads/main' - # repository: dev # # Any other scenarios (e.g. when a branch is created/pushed) will cause the publish step to be skipped, most commonly this is expected to happen for the # branches that the GitHub merge queue feature uses internally for which the checks need to run, but we do not want artifacts to be published. - select_helm_repo: - name: Select target helm repository based on action trigger + check_helm_publish: + name: Decide if Helm charts are pushed to the helm repository based on action trigger runs-on: ubuntu-latest outputs: - helm_repository: ${{ steps.selecthelmrepo.outputs.helm_repo }} + skip_helm: ${{ steps.checkhelmpublish.outputs.skip_helm }} steps: - - id: selecthelmrepo + - id: checkhelmpublish env: TRIGGER: ${{ github.event_name }} GITHUB_REF: ${{ github.ref }} run: | if [[ "$TRIGGER" == "pull_request" ]]; then - echo "exporting test as target helm repo: ${{ env.TEST_REPO_HELM_URL }}" - echo "helm_repo=${{ env.TEST_REPO_HELM_URL }}" >> "$GITHUB_OUTPUT" + echo "skip_helm=false" >> "$GITHUB_OUTPUT" elif [[ ( "$TRIGGER" == "push" || "$TRIGGER" == "schedule" || "$TRIGGER" == "workflow_dispatch" ) && "$GITHUB_REF" == "refs/heads/main" ]]; then - echo "exporting dev as target helm repo: ${{ env.DEV_REPO_HELM_URL }}" - echo "helm_repo=${{ env.DEV_REPO_HELM_URL }}" >> "$GITHUB_OUTPUT" + echo "skip_helm=false" >> "$GITHUB_OUTPUT" elif [[ "$TRIGGER" == "push" && $GITHUB_REF == refs/tags/* ]]; then - echo "exporting stable as target helm repo: ${{ env.STABLE_REPO_HELM_URL }}" - echo "helm_repo=${{ env.STABLE_REPO_HELM_URL }}" >> "$GITHUB_OUTPUT" + echo "skip_helm=false" >> "$GITHUB_OUTPUT" else echo "Unknown trigger and ref combination encountered, skipping publish step: $TRIGGER $GITHUB_REF" - echo "helm_repo=skip" >> "$GITHUB_OUTPUT" + echo "skip_helm=true" >> "$GITHUB_OUTPUT" fi run_cargodeny: @@ -317,7 +308,7 @@ jobs: name: Package Charts, Build Docker Image and publish them - ${{ matrix.runner }} needs: - tests_passed - - select_helm_repo + - check_helm_publish strategy: matrix: runner: ["ubuntu-latest", "ubicloud-standard-8-arm"] @@ -326,13 +317,11 @@ jobs: permissions: id-token: write env: - NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} - HELM_REPO: ${{ needs.select_helm_repo.outputs.helm_repository }} OCI_REGISTRY_SDP_PASSWORD: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} OCI_REGISTRY_SDP_USERNAME: "robot$sdp+github-action-build" OCI_REGISTRY_SDP_CHARTS_PASSWORD: ${{ secrets.HARBOR_ROBOT_SDP_CHARTS_GITHUB_ACTION_BUILD_SECRET }} OCI_REGISTRY_SDP_CHARTS_USERNAME: "robot$sdp-charts+github-action-build" - if: needs.select_helm_repo.outputs.helm_repository != 'skip' + if: needs.check_helm_publish.outputs.skip_helm != 'true' outputs: IMAGE_TAG: ${{ steps.printtag.outputs.IMAGE_TAG }} steps: @@ -379,9 +368,7 @@ jobs: PR_VERSION="${MANIFEST_VERSION}-pr${PR_NUMBER}" sed -i "s/version = \"${MANIFEST_VERSION}\"/version = \"${PR_VERSION}\"/" Cargo.toml - # Recreate charts and publish charts and docker image. The "-e" is needed as we want to override the - # default value in the makefile if called from this action, but not otherwise (i.e. when called locally). - # This is needed for the HELM_REPO variable. + # Recreate charts and publish charts and docker image. - name: Install cosign uses: sigstore/cosign-installer@d7d6bc7722e3daa8354c50bcb52f4837da5e9b6a # v3.8.1 - name: Install syft @@ -397,21 +384,21 @@ jobs: sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_arm64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq fi - make -e build + make build - name: Publish Docker image and Helm chart if: ${{ !github.event.pull_request.head.repo.fork }} run: | # We want to publish helmcharts only once as they have a common name, while still publishing both images with architecture specific tags if [ "$(uname -m)" = "x86_64" ]; then - make -e publish + make publish else - make -e docker-publish + make docker-publish fi # Output the name of the published image to the Job output for later use - id: printtag name: Output image name and tag if: ${{ !github.event.pull_request.head.repo.fork }} - run: echo "IMAGE_TAG=$(make -e print-docker-tag)" >> "$GITHUB_OUTPUT" + run: echo "IMAGE_TAG=$(make print-docker-tag)" >> "$GITHUB_OUTPUT" create_manifest_list: name: Build and publish manifest list @@ -422,7 +409,6 @@ jobs: permissions: id-token: write env: - NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} OCI_REGISTRY_SDP_PASSWORD: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} OCI_REGISTRY_SDP_USERNAME: "robot$sdp+github-action-build" OCI_REGISTRY_SDP_CHARTS_PASSWORD: ${{ secrets.HARBOR_ROBOT_SDP_CHARTS_GITHUB_ACTION_BUILD_SECRET }} diff --git a/Makefile b/Makefile index 43785d65..59021bc4 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,10 @@ OPERATOR_NAME := kafka-operator VERSION := $(shell cargo metadata --format-version 1 | jq -r '.packages[] | select(.name=="stackable-${OPERATOR_NAME}") | .version') ARCH := $(shell uname -m | sed -e 's#x86_64#amd64#' | sed -e 's#aarch64#arm64#') -DOCKER_REPO := docker.stackable.tech -ORGANIZATION := stackable OCI_REGISTRY_HOSTNAME := oci.stackable.tech OCI_REGISTRY_PROJECT_IMAGES := sdp OCI_REGISTRY_PROJECT_CHARTS := sdp-charts # This will be overwritten by an environmental variable if called from the github action -HELM_REPO := https://repo.stackable.tech/repository/helm-dev HELM_CHART_NAME := ${OPERATOR_NAME} HELM_CHART_ARTIFACT := target/helm/${OPERATOR_NAME}-${VERSION}.tgz @@ -34,35 +31,9 @@ render-docs: ## Docker related targets docker-build: - docker build --force-rm --build-arg VERSION=${VERSION} -t "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-${ARCH}" -f docker/Dockerfile . - docker tag "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-${ARCH}" "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}" + docker build --force-rm --build-arg VERSION=${VERSION} -t "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}" -f docker/Dockerfile . docker-publish: - # Push to Nexus - echo "${NEXUS_PASSWORD}" | docker login --username github --password-stdin "${DOCKER_REPO}" - DOCKER_OUTPUT=$$(docker push --all-tags "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}");\ - # Obtain the digest of the pushed image from the output of `docker push`, because signing by tag is deprecated and will be removed from cosign in the future\ - REPO_DIGEST_OF_IMAGE=$$(echo "$$DOCKER_OUTPUT" | awk '/^${VERSION}-${ARCH}: digest: sha256:[0-9a-f]{64} size: [0-9]+$$/ { print $$3 }');\ - if [ -z "$$REPO_DIGEST_OF_IMAGE" ]; then\ - echo 'Could not find repo digest for container image: ${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-${ARCH}';\ - exit 1;\ - fi;\ - # This generates a signature and publishes it to the registry, next to the image\ - # Uses the keyless signing flow with Github Actions as identity provider\ - cosign sign -y "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\ - # Generate the SBOM for the operator image, this leverages the already generated SBOM for the operator binary by cargo-cyclonedx\ - syft scan --output cyclonedx-json@1.5=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger,+sbom-cataloger" --scope all-layers --source-name "${OPERATOR_NAME}" --source-version "${VERSION}-${ARCH}" "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\ - # Determine the PURL for the container image\ - URLENCODED_REPO_DIGEST_OF_IMAGE=$$(echo "$$REPO_DIGEST_OF_IMAGE" | sed 's/:/%3A/g');\ - PURL="pkg:oci/${OPERATOR_NAME}@$$URLENCODED_REPO_DIGEST_OF_IMAGE?arch=${ARCH}&repository_url=${DOCKER_REPO}%2F${ORGANIZATION}%2F${OPERATOR_NAME}";\ - # Get metadata from the image\ - IMAGE_DESCRIPTION=$$(docker inspect --format='{{.Config.Labels.description}}' "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-${ARCH}");\ - IMAGE_NAME=$$(docker inspect --format='{{.Config.Labels.name}}' "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-${ARCH}");\ - # Merge the SBOM with the metadata for the operator\ - jq -s '{"metadata":{"component":{"description":"'"$$IMAGE_NAME. $$IMAGE_DESCRIPTION"'","supplier":{"name":"Stackable GmbH","url":["https://stackable.tech/"]},"author":"Stackable GmbH","purl":"'"$$PURL"'","publisher":"Stackable GmbH"}}} * .[0]' sbom.json > sbom.merged.json;\ - # Attest the SBOM to the image\ - cosign attest -y --predicate sbom.merged.json --type cyclonedx "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE" - # Push to Harbor # We need to use "value" here to prevent the variable from being recursively expanded by make (username contains a dollar sign, since it's a Harbor bot) docker login --username '${value OCI_REGISTRY_SDP_USERNAME}' --password '${OCI_REGISTRY_SDP_PASSWORD}' '${OCI_REGISTRY_HOSTNAME}' @@ -91,21 +62,9 @@ docker-publish: # This assumes "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-amd64 and "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-arm64 are built and pushed docker-manifest-list-build: - docker manifest create "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}" --amend "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-amd64" --amend "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-arm64" docker manifest create "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}" --amend "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-amd64" --amend "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-arm64" docker-manifest-list-publish: - # Push to Nexus - echo "${NEXUS_PASSWORD}" | docker login --username github --password-stdin "${DOCKER_REPO}" - # `docker manifest push` directly returns the digest of the manifest list - # As it is an experimental feature, this might change in the future - # Further reading: https://docs.docker.com/reference/cli/docker/manifest/push/ - DIGEST_NEXUS=$$(docker manifest push "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}");\ - # Refer to image via its digest (oci.stackable.tech/sdp/airflow@sha256:0a1b2c...)\ - # This generates a signature and publishes it to the registry, next to the image\ - # Uses the keyless signing flow with Github Actions as identity provider\ - cosign sign -y "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}@$$DIGEST_NEXUS" - # Push to Harbor # We need to use "value" here to prevent the variable from being recursively expanded by make (username contains a dollar sign, since it's a Harbor bot) docker login --username '${value OCI_REGISTRY_SDP_USERNAME}' --password '${OCI_REGISTRY_SDP_PASSWORD}' '${OCI_REGISTRY_HOSTNAME}' @@ -122,9 +81,6 @@ print-docker-tag: @echo "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}" helm-publish: - # Push to Nexus - curl --fail -u "github:${NEXUS_PASSWORD}" --upload-file "${HELM_CHART_ARTIFACT}" "${HELM_REPO}/" - # Push to Harbor # We need to use "value" here to prevent the variable from being recursively expanded by make (username contains a dollar sign, since it's a Harbor bot) helm registry login --username '${value OCI_REGISTRY_SDP_CHARTS_USERNAME}' --password '${OCI_REGISTRY_SDP_CHARTS_PASSWORD}' '${OCI_REGISTRY_HOSTNAME}' @@ -170,7 +126,6 @@ chart-lint: compile-chart clean: chart-clean cargo clean - docker rmi --force "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}" docker rmi --force '${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}' regenerate-charts: chart-clean compile-chart