From bf15ce0295bd20e2a5d58fb00b9c39dc7c181eef Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 29 May 2024 15:20:35 -0700 Subject: [PATCH 1/8] Remove Makefile and build submodule Signed-off-by: Nic Cope --- .gitmodules | 3 -- Makefile | 95 ----------------------------------------------------- build | 1 - 3 files changed, 99 deletions(-) delete mode 100644 .gitmodules delete mode 100644 Makefile delete mode 160000 build diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 8f84209c8..000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "build"] - path = build - url = https://github.com/crossplane/build diff --git a/Makefile b/Makefile deleted file mode 100644 index 4b45a6b99..000000000 --- a/Makefile +++ /dev/null @@ -1,95 +0,0 @@ -# ==================================================================================== -# Setup Project - -PROJECT_NAME := crossplane-runtime -PROJECT_REPO := github.com/crossplane/$(PROJECT_NAME) - -PLATFORMS ?= linux_amd64 linux_arm64 -# -include will silently skip missing files, which allows us -# to load those files with a target in the Makefile. If only -# "include" was used, the make command would fail and refuse -# to run a target until the include commands succeeded. --include build/makelib/common.mk - -# ==================================================================================== -# Setup Images - -# even though this repo doesn't build images (note the no-op img.build target below), -# some of the init is needed for the cross build container, e.g. setting BUILD_REGISTRY --include build/makelib/image.mk -img.build: - -# ==================================================================================== -# Setup Go - -# Set a sane default so that the nprocs calculation below is less noisy on the initial -# loading of this file -NPROCS ?= 1 - -# each of our test suites starts a kube-apiserver and running many test suites in -# parallel can lead to high CPU utilization. by default we reduce the parallelism -# to half the number of CPU cores. -GO_TEST_PARALLEL := $(shell echo $$(( $(NPROCS) / 2 ))) - -GO_LDFLAGS += -X $(GO_PROJECT)/pkg/version.Version=$(VERSION) -GO_SUBDIRS += pkg apis -GO111MODULE = on -GOLANGCILINT_VERSION = 1.56.2 --include build/makelib/golang.mk - -# ==================================================================================== -# Targets - -# run `make help` to see the targets and options - -# We want submodules to be set up the first time `make` is run. -# We manage the build/ folder and its Makefiles as a submodule. -# The first time `make` is run, the includes of build/*.mk files will -# all fail, and this target will be run. The next time, the default as defined -# by the includes will be run instead. -fallthrough: submodules - @echo Initial setup complete. Running make again . . . - @make - - -# NOTE(hasheddan): the build submodule currently overrides XDG_CACHE_HOME in -# order to force the Helm 3 to use the .work/helm directory. This causes Go on -# Linux machines to use that directory as the build cache as well. We should -# adjust this behavior in the build submodule because it is also causing Linux -# users to duplicate their build cache, but for now we just make it easier to -# identify its location in CI so that we cache between builds. -go.cachedir: - @go env GOCACHE - -# Generate a coverage report for cobertura applying exclusions on -# - generated file -cobertura: - @cat $(GO_TEST_OUTPUT)/coverage.txt | \ - grep -v zz_generated.deepcopy | \ - $(GOCOVER_COBERTURA) > $(GO_TEST_OUTPUT)/cobertura-coverage.xml - -# Update the submodules, such as the common build scripts. -submodules: - @git submodule sync - @git submodule update --init --recursive - -.PHONY: cobertura reviewable submodules fallthrough - -# ==================================================================================== -# Special Targets - -define CROSSPLANE_RUNTIME_HELP -Crossplane Runtime Targets: - cobertura Generate a coverage report for cobertura applying exclusions on generated files. - reviewable Ensure a PR is ready for review. - submodules Update the submodules, such as the common build scripts. - -endef -export CROSSPLANE_RUNTIME_HELP - -crossplane-runtime.help: - @echo "$$CROSSPLANE_RUNTIME_HELP" - -help-special: crossplane-runtime.help - -.PHONY: crossplane-runtime.help help-special diff --git a/build b/build deleted file mode 160000 index 231258db2..000000000 --- a/build +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 231258db281237379d8ec0c6e4af9d7c1ae5cc4a From 04814f24f88c920c8884f2e236c9762772315f06 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 29 May 2024 15:20:53 -0700 Subject: [PATCH 2/8] Copy Earthfile and friends from c/crossplane Unmodified for now. I'll strip out everything this repo doesn't need in a follow-up commit. Signed-off-by: Nic Cope --- .github/PULL_REQUEST_TEMPLATE.md | 46 ++- .github/workflows/ci.yml | 468 ++++++++++++++++++++++--------- .golangci.yml | 71 ++--- Earthfile | 432 ++++++++++++++++++++++++++++ 4 files changed, 821 insertions(+), 196 deletions(-) create mode 100644 Earthfile diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7627d1a98..6f84d7d11 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,35 +1,33 @@ ### Description of your changes -Fixes # - -I have: - -- [ ] Read and followed Crossplane's [contribution process]. -- [ ] Run `make reviewable test` to ensure this PR is ready for review. -### How has this code been tested +Fixes # - +I have: -[contribution process]: https://git.io/fj2m9 +- [ ] Read and followed Crossplane's [contribution process]. +- [ ] Run `earthly +reviewable` to ensure this PR is ready for review. +- [ ] Added or updated unit tests. +- [ ] Added or updated e2e tests. +- [ ] Linked a PR or a [docs tracking issue] to [document this change]. +- [ ] Added `backport release-x.y` labels to auto-backport this PR. + +Need help with this checklist? See the [cheat sheet]. + +[contribution process]: https://github.com/crossplane/crossplane/tree/master/contributing +[docs tracking issue]: https://github.com/crossplane/docs/issues/new +[document this change]: https://docs.crossplane.io/contribute/contribute +[cheat sheet]: https://github.com/crossplane/crossplane/tree/master/contributing#checklist-cheat-sheet diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7dea0aa7..0b81498c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,8 +10,18 @@ on: env: # Common versions - GO_VERSION: '1.22.0' - GOLANGCI_VERSION: 'v1.56.2' + EARTHLY_VERSION: '0.8.11' + + # Force Earthly to use color output + FORCE_COLOR: "1" + + # Common users. We can't run a step 'if secrets.AWS_USR != ""' but we can run + # a step 'if env.AWS_USR' != ""', so we copy these to succinctly test whether + # credentials have been provided before trying to run steps that need them. + DOCKER_USR: ${{ secrets.DOCKER_USR }} + AWS_USR: ${{ secrets.AWS_USR }} + UPBOUND_MARKETPLACE_PUSH_ROBOT_USR: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }} + jobs: check-diff: @@ -19,209 +29,391 @@ jobs: steps: - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - submodules: true + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 - - name: Setup Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 + - name: Setup Earthly + uses: earthly/actions-setup@v1 with: - go-version: ${{ env.GO_VERSION }} - - - name: Find the Go Build Cache - id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + github-token: ${{ secrets.GITHUB_TOKEN }} + version: ${{ env.EARTHLY_VERSION }} - - name: Cache the Go Build Cache - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 + - name: Login to DockerHub + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 + if: env.DOCKER_USR != '' with: - path: ${{ steps.go.outputs.cache }} - key: ${{ runner.os }}-build-check-diff-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-check-diff- + username: ${{ secrets.DOCKER_USR }} + password: ${{ secrets.DOCKER_PSW }} - - name: Cache Go Dependencies - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 + - name: Login to GitHub Container Registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 with: - path: .work/pkg - key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-pkg- - - - name: Download Go Modules - run: make modules.download modules.check - - - name: Check Diff - run: make check-diff - - detect-noop: - runs-on: ubuntu-22.04 - outputs: - noop: ${{ steps.noop.outputs.should_skip }} - steps: - - name: Detect No-op Changes - id: noop - uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1 + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Earthly to Push Cache to GitHub Container Registry + if: github.ref == 'refs/heads/master' + run: | + echo "EARTHLY_PUSH=true" >> $GITHUB_ENV + echo "EARTHLY_MAX_REMOTE_CACHE=true" >> $GITHUB_ENV + + - name: Generate Files + run: earthly --strict --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }} +generate + + - name: Count Changed Files + id: changed_files + run: echo "count=$(git status --porcelain | wc -l)" >> $GITHUB_OUTPUT + + - name: Fail if Files Changed + if: steps.changed_files.outputs.count != 0 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 with: - github_token: ${{ secrets.GITHUB_TOKEN }} - paths_ignore: '["**.md", "**.png", "**.jpg"]' - do_not_skip: '["workflow_dispatch", "schedule", "push"]' - concurrent_skipping: false + script: core.setFailed('Found changed files after running earthly +generate.'') lint: runs-on: ubuntu-22.04 - needs: detect-noop - if: needs.detect-noop.outputs.noop != 'true' steps: - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - submodules: true + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 - - name: Setup Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 + - name: Setup Earthly + uses: earthly/actions-setup@v1 with: - go-version: ${{ env.GO_VERSION }} - - - name: Find the Go Build Cache - id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + github-token: ${{ secrets.GITHUB_TOKEN }} + version: ${{ env.EARTHLY_VERSION }} - - name: Cache the Go Build Cache - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 + - name: Login to DockerHub + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 + if: env.DOCKER_USR != '' with: - path: ${{ steps.go.outputs.cache }} - key: ${{ runner.os }}-build-lint-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-lint- + username: ${{ secrets.DOCKER_USR }} + password: ${{ secrets.DOCKER_PSW }} - - name: Cache Go Dependencies - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 + - name: Login to GitHub Container Registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 with: - path: .work/pkg - key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-pkg- + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Download Go Modules - run: make modules.download modules.check + - name: Configure Earthly to Push Cache to GitHub Container Registry + if: github.ref == 'refs/heads/master' + run: | + echo "EARTHLY_PUSH=true" >> $GITHUB_ENV + echo "EARTHLY_MAX_REMOTE_CACHE=true" >> $GITHUB_ENV - # We could run 'make lint' to ensure our desired Go version, but we prefer - # this action because it leaves 'annotations' (i.e. it comments on PRs to - # point out linter violations). - name: Lint - uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3 - with: - version: ${{ env.GOLANGCI_VERSION }} - skip-cache: true # We do our own caching. + run: earthly --strict --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }} +lint codeql: runs-on: ubuntu-22.04 - needs: detect-noop - if: needs.detect-noop.outputs.noop != 'true' steps: - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - submodules: true + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 - - name: Setup Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 + - name: Setup Earthly + uses: earthly/actions-setup@v1 with: - go-version: ${{ env.GO_VERSION }} + github-token: ${{ secrets.GITHUB_TOKEN }} + version: ${{ env.EARTHLY_VERSION }} - - name: Find the Go Build Cache - id: go - run: echo "::set-output name=cache::$(make go.cachedir)" - - - name: Cache the Go Build Cache - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 + - name: Login to DockerHub + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 + if: env.DOCKER_USR != '' with: - path: ${{ steps.go.outputs.cache }} - key: ${{ runner.os }}-build-check-diff-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-check-diff- + username: ${{ secrets.DOCKER_USR }} + password: ${{ secrets.DOCKER_PSW }} - - name: Cache Go Dependencies - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 + - name: Login to GitHub Container Registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 with: - path: .work/pkg - key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-pkg- + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Earthly to Push Cache to GitHub Container Registry + if: github.ref == 'refs/heads/master' + run: | + echo "EARTHLY_PUSH=true" >> $GITHUB_ENV + echo "EARTHLY_MAX_REMOTE_CACHE=true" >> $GITHUB_ENV - - name: Download Go Modules - run: make modules.download modules.check + - name: Run CodeQL + run: earthly --strict --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }} +ci-codeql - - name: Initialize CodeQL - uses: github/codeql-action/init@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3 + - name: Upload CodeQL Results to GitHub + uses: github/codeql-action/upload-sarif@9fdb3e49720b44c48891d036bb502feb25684276 # v3 with: - languages: go + sarif_file: '_output/codeql/go.sarif' - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3 trivy-scan-fs: runs-on: ubuntu-22.04 - needs: detect-noop - if: needs.detect-noop.outputs.noop != 'true' steps: - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - submodules: true + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 - name: Run Trivy vulnerability scanner in fs mode - uses: aquasecurity/trivy-action@84384bd6e777ef152729993b8145ea352e9dd3ef # 0.17.0 + uses: aquasecurity/trivy-action@fd25fed6972e341ff0007ddb61f77e88103953c2 # 0.21.0 with: scan-type: 'fs' ignore-unfixed: true skip-dirs: design scan-ref: '.' - exit-code: '1' severity: 'CRITICAL,HIGH' + format: sarif + output: 'trivy-results.sarif' + + - name: Upload Trivy Results to GitHub + uses: github/codeql-action/upload-sarif@9fdb3e49720b44c48891d036bb502feb25684276 # v3 + with: + sarif_file: 'trivy-results.sarif' unit-tests: runs-on: ubuntu-22.04 - needs: detect-noop - if: needs.detect-noop.outputs.noop != 'true' steps: - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - submodules: true - - - name: Fetch History - run: git fetch --prune --unshallow + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 - - name: Setup Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 + - name: Setup Earthly + uses: earthly/actions-setup@v1 with: - go-version: ${{ env.GO_VERSION }} + github-token: ${{ secrets.GITHUB_TOKEN }} + version: ${{ env.EARTHLY_VERSION }} - - name: Find the Go Build Cache - id: go - run: echo "::set-output name=cache::$(make go.cachedir)" - - - name: Cache the Go Build Cache - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 + - name: Login to DockerHub + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 + if: env.DOCKER_USR != '' with: - path: ${{ steps.go.outputs.cache }} - key: ${{ runner.os }}-build-unit-tests-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-unit-tests- + username: ${{ secrets.DOCKER_USR }} + password: ${{ secrets.DOCKER_PSW }} - - name: Cache Go Dependencies - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 + - name: Login to GitHub Container Registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 with: - path: .work/pkg - key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-pkg- + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Download Go Modules - run: make modules.download modules.check + - name: Configure Earthly to Push Cache to GitHub Container Registry + if: github.ref == 'refs/heads/master' + run: | + echo "EARTHLY_PUSH=true" >> $GITHUB_ENV + echo "EARTHLY_MAX_REMOTE_CACHE=true" >> $GITHUB_ENV - name: Run Unit Tests - run: make -j2 test + run: earthly --strict --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }} +test - name: Publish Unit Test Coverage - uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4 + uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c # v4 with: flags: unittests - file: _output/tests/linux_amd64/coverage.txt + file: _output/tests/coverage.txt + token: ${{ secrets.CODECOV_TOKEN }} + + e2e-tests: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + test-suite: + - base + - environment-configs + - usage + - ssa-claims + - realtime-compositions + + steps: + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + + - name: Setup Earthly + uses: earthly/actions-setup@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + version: ${{ env.EARTHLY_VERSION }} + + - name: Login to DockerHub + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 + if: env.DOCKER_USR != '' + with: + username: ${{ secrets.DOCKER_USR }} + password: ${{ secrets.DOCKER_PSW }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Earthly to Push Cache to GitHub Container Registry + if: github.ref == 'refs/heads/master' + run: | + echo "EARTHLY_PUSH=true" >> $GITHUB_ENV + echo "EARTHLY_MAX_REMOTE_CACHE=true" >> $GITHUB_ENV + + - name: Run E2E Tests + run: | + earthly --strict --allow-privileged --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }}-${{ matrix.test-suite}} \ + +e2e --FLAGS="-test.failfast -fail-fast --test-suite ${{ matrix.test-suite }}" + + - name: Publish E2E Test Flakes + if: '!cancelled()' + uses: buildpulse/buildpulse-action@d0d30f53585cf16b2e01811a5a753fd47968654a # v0.11.0 + with: + account: 45158470 + repository: 147886080 + key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }} + secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }} + path: _output/tests/e2e-tests.xml + + publish-artifacts: + runs-on: ubuntu-22.04 + + steps: + - name: Cleanup Disk + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + android: true + dotnet: true + haskell: true + tool-cache: true + swap-storage: false + # This works, and saves ~5GiB, but takes ~2 minutes to do it. + large-packages: false + # TODO(negz): Does having these around avoid Earthly needing to pull + # large images like golang? + docker-images: false + + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + with: + fetch-depth: 0 + + - name: Setup Earthly + uses: earthly/actions-setup@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + version: ${{ env.EARTHLY_VERSION }} + + - name: Login to DockerHub + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 + if: env.DOCKER_USR != '' + with: + username: ${{ secrets.DOCKER_USR }} + password: ${{ secrets.DOCKER_PSW }} + + - name: Login to Upbound + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 + if: env.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR != '' + with: + registry: xpkg.upbound.io + username: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }} + password: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Earthly to Push Cache to GitHub Container Registry + if: github.ref == 'refs/heads/master' + run: echo "EARTHLY_MAX_REMOTE_CACHE=true" >> $GITHUB_ENV + + - name: Configure Earthly to Push Artifacts + if: env.DOCKER_USR != '' && env.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR != '' && env.AWS_USR != '' + run: echo "EARTHLY_PUSH=true" >> $GITHUB_ENV + + - name: Set CROSSPLANE_VERSION GitHub Environment Variable + run: earthly +ci-version + + - name: Build and Push Artifacts + run: earthly --strict --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }} +ci-artifacts --CROSSPLANE_VERSION=${CROSSPLANE_VERSION} + + - name: Push Artifacts to https://releases.crossplane.io/build/ + if: env.AWS_USR != '' + run: | + earthly --strict \ + --secret=AWS_ACCESS_KEY_ID=${{ secrets.AWS_USR }} \ + --secret=AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_PSW }} \ + +ci-push-build-artifacts --AWS_DEFAULT_REGION=us-east-1 --CROSSPLANE_VERSION=${CROSSPLANE_VERSION} + + - name: Push Artifacts to https://releases.crossplane.io/master/ and https://charts.crossplane.io/master + if: env.AWS_USR != '' && github.ref == 'refs/heads/master' + run: | + earthly --strict \ + --secret=AWS_ACCESS_KEY_ID=${{ secrets.AWS_USR }} \ + --secret=AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_PSW }} \ + +ci-promote-build-artifacts --AWS_DEFAULT_REGION=us-east-1 --CROSSPLANE_VERSION=${CROSSPLANE_VERSION} --CHANNEL=master + + - name: Upload Artifacts to GitHub + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 + with: + name: output + path: _output/** + + fuzz-test: + runs-on: ubuntu-22.04 + + steps: + # TODO(negz): Can we make this use our Go build and dependency cache? It + # seems to build Crossplane inside of a Docker image. + - name: Build Fuzzers + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: "crossplane" + language: go + + - name: Run Fuzzers + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + oss-fuzz-project-name: "crossplane" + fuzz-seconds: 300 + language: go + + - name: Upload Crash + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 + if: failure() && steps.build.outcome == 'success' + with: + name: artifacts + path: ./out/artifacts + + protobuf-schemas: + runs-on: ubuntu-22.04 + + steps: + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + + - name: Setup Buf + uses: bufbuild/buf-setup-action@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Lint Protocol Buffers + uses: bufbuild/buf-lint-action@v1 + with: + input: apis + + # buf-breaking-action doesn't support branches + # https://github.com/bufbuild/buf-push-action/issues/34 + - name: Detect Breaking Changes in Protocol Buffers + uses: bufbuild/buf-breaking-action@a074e988ee34efcd4927079e79c611f428354c01 # v1 + # We want to run this for the master branch, and PRs against master. + if: ${{ github.ref == 'refs/heads/master' || github.base_ref == 'master' }} + with: + input: apis + against: "https://github.com/${GITHUB_REPOSITORY}.git#branch=master,subdir=apis" + + - name: Push Protocol Buffers to Buf Schema Registry + if: ${{ github.repository == 'crossplane/crossplane' && github.ref == 'refs/heads/master' }} + uses: bufbuild/buf-push-action@v1 + with: + input: apis + buf_token: ${{ secrets.BUF_TOKEN }} diff --git a/.golangci.yml b/.golangci.yml index 77d7c13a8..90f918be7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,30 +1,20 @@ run: timeout: 10m - skip-files: - - "zz_generated\\..+\\.go$" - output: # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" - format: colored-line-number + formats: + - format: colored-line-number + path: stderr linters: enable-all: true fast: false - + disable: # These linters are all deprecated. We disable them explicitly to avoid the # linter logging deprecation warnings. - - deadcode - - varcheck - - scopelint - - structcheck - - interfacer - - exhaustivestruct - - ifshort - - golint - - maligned - - nosnakecase + - execinquery # These are linters we'd like to enable, but that will be labor intensive to # make existing code compliant. @@ -41,7 +31,7 @@ linters: # These linters add whitespace in an attempt to make code more readable. # This isn't a widely accepted Go best practice, and would be laborious to # apply to existing code. - - wsl + - wsl - nlreturn # Warns about uses of fmt.Sprintf that are less performant than alternatives @@ -58,7 +48,7 @@ linters: # 1 is covered by other linters. 2 is covered by wrapcheck, which can also # handle our use of crossplane-runtime's errors package. 3 is more strict # than we need. Not every error needs to be tested for equality. - - goerr113 + - err113 # These linters duplicate gocognit, but calculate complexity differently. - gocyclo @@ -93,7 +83,7 @@ linters: # Warns about returning interfaces rather than concrete types. We do think # it's best to avoid returning interfaces where possible. However, at the # time of writing enabling this linter would only catch the (many) cases - # where we must return an interface. + # where we must return an interface. - ireturn # Warns about returning named variables. We do think it's best to avoid @@ -104,6 +94,14 @@ linters: # to communicate what the bool means. - nonamedreturns + # Warns about taking the address of a range variable. This isn't an issue in + # Go v1.22 and above: https://tip.golang.org/doc/go1.22 + - exportloopref + + # Warns about using magic numbers. We do think it's best to avoid magic + # numbers, but we should not be strict about it. + - mnd + linters-settings: errcheck: # report about not checking of errors in type assetions: `a := b.(MyStruct)`; @@ -114,14 +112,10 @@ linters-settings: # default is false: such cases aren't reported by default. check-blank: false - # [deprecated] comma-separated list of pairs of the form pkg:regex - # the regex is used to ignore names within pkg. (default "fmt:.*"). - # see https://github.com/kisielk/errcheck#the-deprecated-method for details - ignore: fmt:.*,io/ioutil:^Read.* - govet: # report about shadowed variables - check-shadowing: false + disable: + - shadow gofmt: # simplify code: gofmt with `-s` option, true by default @@ -133,13 +127,10 @@ linters-settings: - standard - default - prefix(github.com/crossplane/crossplane-runtime) + - prefix(github.com/crossplane/crossplane) - blank - dot - maligned: - # print struct with more effective memory layout or not, false by default - suggest-new: true - dupl: # tokens count to trigger issue, 150 by default threshold: 100 @@ -159,7 +150,8 @@ linters-settings: # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: # if it's called for subdir of a project it can't find funcs usages. All text editor integrations # with golangci-lint call it on a directory with the changed file. - check-exported: false + exported-is-used: true + exported-fields-are-used: true unparam: # Inspect exported functions, default is false. Set to true if no external program/library imports your code. @@ -197,7 +189,7 @@ linters-settings: nolintlint: require-explanation: true require-specific: true - + depguard: rules: no_third_party_test_libraries: @@ -214,14 +206,18 @@ linters-settings: interfacebloat: max: 5 - + tagliatelle: case: rules: json: goCamel issues: - # Excluding configuration per-path and per-linter + # Excluding generated files. + exclude-files: + - "zz_generated\\..+\\.go$" + - ".+\\.pb.go$" + # Excluding configuration per-path and per-linter. exclude-rules: # Exclude some linters from running on tests files. - path: _test(ing)?\.go @@ -241,7 +237,7 @@ issues: text: "(unnamedResult|exitAfterDefer)" linters: - gocritic - + # It's idiomatic to register Kubernetes types with a package scoped # SchemeBuilder using an init function. - path: apis/ @@ -280,6 +276,13 @@ issues: - gosec - gas + # This is about implicit memory aliasing in a range loop. + # This is a false positive with Go v1.22 and above. + - text: "G601:" + linters: + - gosec + - gas + # Some k8s dependencies do not have JSON tags on all fields in structs. - path: k8s.io/ linters: @@ -300,7 +303,7 @@ issues: new: false # Maximum issues count per one linter. Set to 0 to disable. Default is 50. - max-per-linter: 0 + max-issues-per-linter: 0 # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. max-same-issues: 0 diff --git a/Earthfile b/Earthfile new file mode 100644 index 000000000..2c14638dc --- /dev/null +++ b/Earthfile @@ -0,0 +1,432 @@ +# See https://docs.earthly.dev/docs/earthfile/features +VERSION --try --raw-output 0.8 + +PROJECT crossplane/crossplane + +ARG --global GO_VERSION=1.22.3 + +# reviewable checks that a branch is ready for review. Run it before opening a +# pull request. It will catch a lot of the things our CI workflow will catch. +reviewable: + WAIT + BUILD +generate + END + BUILD +lint + BUILD +test + +# test runs unit tests. +test: + BUILD +go-test + +# lint runs linters. +lint: + BUILD +go-lint + BUILD +helm-lint + +# build builds Crossplane for your native OS and architecture. +build: + BUILD +image + BUILD +helm-build + +# multiplatform-build builds Crossplane for all supported OS and architectures. +multiplatform-build: + BUILD +go-multiplatform-build + BUILD +multiplatform-image + BUILD +helm-build + +# generate runs code generation. To keep builds fast, it doesn't run as part of +# the build target. It's important to run it explicitly when code needs to be +# generated, for example when you update an API type. +generate: + BUILD +go-modules-tidy + BUILD +go-generate + BUILD +helm-generate + +# e2e runs end-to-end tests. See test/e2e/README.md for details. +e2e: + ARG FLAGS="-test-suite=base" + # Docker installs faster on Alpine, and we only need Go for go tool test2json. + FROM golang:${GO_VERSION}-alpine3.20 + RUN apk add --no-cache docker jq + COPY +helm-setup/helm /usr/local/bin/helm + COPY +kind-setup/kind /usr/local/bin/kind + COPY +gotestsum-setup/gotestsum /usr/local/bin/gotestsum + COPY +go-build-e2e/e2e . + COPY --dir cluster test . + # Using a static CROSSPLANE_VERSION allows Earthly to cache E2E runs as long + # as no code changed. If the version contains a git commit (the default) the + # build layer cache is invalidated on every commit. + WITH DOCKER --load crossplane-e2e/crossplane:latest=(+image --CROSSPLANE_VERSION=v0.0.0-e2e) + TRY + # TODO(negz:) Set GITHUB_ACTIONS=true and use RUN --raw-output when + # https://github.com/earthly/earthly/issues/4143 is fixed. + RUN gotestsum --no-color=false --format testname --junitfile e2e-tests.xml --raw-command go tool test2json -t -p E2E ./e2e -test.v ${FLAGS} + FINALLY + SAVE ARTIFACT --if-exists e2e-tests.xml AS LOCAL _output/tests/e2e-tests.xml + END + END + +# hack builds Crossplane, and deploys it to a kind cluster. It runs in your +# local environment, not a container. The kind cluster will keep running until +# you run the unhack target. Run hack again to rebuild Crossplane and restart +# the kind cluster with the new build. +hack: + # TODO(negz): This could run an interactive shell inside a temporary container + # once https://github.com/earthly/earthly/issues/3206 is fixed. + ARG USERPLATFORM + LOCALLY + WAIT + BUILD +unhack + END + COPY --platform=${USERPLATFORM} +helm-setup/helm .hack/helm + COPY --platform=${USERPLATFORM} +kind-setup/kind .hack/kind + COPY (+helm-build/output --CROSSPLANE_VERSION=v0.0.0-hack) .hack/charts + WITH DOCKER --load crossplane-hack/crossplane:hack=+image + RUN \ + .hack/kind create cluster --name crossplane-hack && \ + .hack/kind load docker-image --name crossplane-hack crossplane-hack/crossplane:hack && \ + .hack/helm install --create-namespace --namespace crossplane-system crossplane .hack/charts/crossplane-0.0.0-hack.tgz \ + --set "image.pullPolicy=Never,image.repository=crossplane-hack/crossplane,image.tag=hack" \ + --set "args={--debug}" + END + RUN docker image rm crossplane-hack/crossplane:hack + RUN rm -rf .hack + +# unhack deletes the kind cluster created by the hack target. +unhack: + ARG USERPLATFORM + LOCALLY + COPY --platform=${USERPLATFORM} +kind-setup/kind .hack/kind + RUN .hack/kind delete cluster --name crossplane-hack + RUN rm -rf .hack + +# go-modules downloads Crossplane's go modules. It's the base target of most Go +# related target (go-build, etc). +go-modules: + ARG NATIVEPLATFORM + FROM --platform=${NATIVEPLATFORM} golang:${GO_VERSION} + WORKDIR /crossplane + CACHE --id go-build --sharing shared /root/.cache/go-build + COPY go.mod go.sum ./ + RUN go mod download + SAVE ARTIFACT go.mod AS LOCAL go.mod + SAVE ARTIFACT go.sum AS LOCAL go.sum + +# go-modules-tidy tidies and verifies go.mod and go.sum. +go-modules-tidy: + FROM +go-modules + CACHE --id go-build --sharing shared /root/.cache/go-build + COPY --dir apis/ cmd/ internal/ pkg/ test/ . + RUN go mod tidy + RUN go mod verify + SAVE ARTIFACT go.mod AS LOCAL go.mod + SAVE ARTIFACT go.sum AS LOCAL go.sum + +# go-generate runs Go code generation. +go-generate: + FROM +go-modules + CACHE --id go-build --sharing shared /root/.cache/go-build + COPY +kubectl-setup/kubectl /usr/local/bin/kubectl + COPY --dir cluster/crd-patches cluster/crd-patches + COPY --dir hack/ apis/ internal/ . + RUN go generate -tags 'generate' ./apis/... + # TODO(negz): Can this move into generate.go? Ideally it would live there with + # the code that actually generates the CRDs, but it depends on kubectl. + RUN kubectl patch --local --type=json \ + --patch-file cluster/crd-patches/pkg.crossplane.io_deploymentruntimeconfigs.yaml \ + --filename cluster/crds/pkg.crossplane.io_deploymentruntimeconfigs.yaml \ + --output=yaml > /tmp/patched.yaml \ + && mv /tmp/patched.yaml cluster/crds/pkg.crossplane.io_deploymentruntimeconfigs.yaml + SAVE ARTIFACT apis/ AS LOCAL apis + SAVE ARTIFACT cluster/crds AS LOCAL cluster/crds + +# go-build builds Crossplane binaries for your native OS and architecture. +go-build: + ARG EARTHLY_GIT_SHORT_HASH + ARG EARTHLY_GIT_COMMIT_TIMESTAMP + ARG CROSSPLANE_VERSION=v0.0.0-${EARTHLY_GIT_COMMIT_TIMESTAMP}-${EARTHLY_GIT_SHORT_HASH} + ARG TARGETARCH + ARG TARGETOS + ARG GOARCH=${TARGETARCH} + ARG GOOS=${TARGETOS} + ARG GOFLAGS="-ldflags=-X=github.com/crossplane/crossplane/internal/version.version=${CROSSPLANE_VERSION}" + ARG CGO_ENABLED=0 + FROM +go-modules + CACHE --id go-build --sharing shared /root/.cache/go-build + COPY --dir apis/ cmd/ internal/ pkg/ . + RUN go build -o crossplane ./cmd/crossplane + RUN go build -o crank ./cmd/crank + SAVE ARTIFACT crossplane AS LOCAL _output/bin/${GOOS}_${GOARCH}/crossplane + SAVE ARTIFACT crank AS LOCAL _output/bin/${GOOS}_${GOARCH}/crank + +# go-multiplatform-build builds Crossplane binaries for all supported OS +# and architectures. +go-multiplatform-build: + BUILD \ + --platform=linux/amd64 \ + --platform=linux/arm64 \ + --platform=linux/arm \ + --platform=linux/ppc64le \ + --platform=darwin/arm64 \ + --platform=darwin/amd64 \ + --platform=windows/amd64 \ + +go-build + +# go-build-e2e builds Crossplane's end-to-end tests. +go-build-e2e: + ARG CGO_ENABLED=0 + FROM +go-modules + CACHE --id go-build --sharing shared /root/.cache/go-build + COPY --dir apis/ internal/ test/ . + RUN go test -c -o e2e ./test/e2e + SAVE ARTIFACT e2e + +# go-test runs Go unit tests. +go-test: + FROM +go-modules + CACHE --id go-build --sharing shared /root/.cache/go-build + COPY --dir apis/ cmd/ internal/ pkg/ . + RUN go test -covermode=count -coverprofile=coverage.txt ./... + SAVE ARTIFACT coverage.txt AS LOCAL _output/tests/coverage.txt + +# go-lint lints Go code. +go-lint: + ARG GOLANGCI_LINT_VERSION=v1.59.0 + FROM +go-modules + # This cache is private because golangci-lint doesn't support concurrent runs. + CACHE --id go-lint --sharing private /root/.cache/golangci-lint + CACHE --id go-build --sharing shared /root/.cache/go-build + RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin ${GOLANGCI_LINT_VERSION} + COPY .golangci.yml . + COPY --dir apis/ cmd/ internal/ pkg/ test/ . + RUN golangci-lint run --fix + SAVE ARTIFACT apis AS LOCAL apis + SAVE ARTIFACT cmd AS LOCAL cmd + SAVE ARTIFACT internal AS LOCAL internal + SAVE ARTIFACT pkg AS LOCAL pkg + SAVE ARTIFACT test AS LOCAL test + +# image builds the Crossplane OCI image for your native architecture. +image: + ARG EARTHLY_GIT_BRANCH + ARG EARTHLY_GIT_SHORT_HASH + ARG EARTHLY_GIT_COMMIT_TIMESTAMP + ARG CROSSPLANE_REPO=build-${EARTHLY_GIT_SHORT_HASH}/crossplane + ARG CROSSPLANE_VERSION=v0.0.0-${EARTHLY_GIT_COMMIT_TIMESTAMP}-${EARTHLY_GIT_SHORT_HASH} + ARG NATIVEPLATFORM + ARG TARGETPLATFORM + ARG TARGETARCH + ARG TARGETOS + FROM --platform=${TARGETPLATFORM} gcr.io/distroless/static@sha256:41972110a1c1a5c0b6adb283e8aa092c43c31f7c5d79b8656fbffff2c3e61f05 + COPY --platform=${NATIVEPLATFORM} (+go-build/crossplane --GOOS=${TARGETOS} --GOARCH=${TARGETARCH}) /usr/local/bin/ + COPY --dir cluster/crds/ /crds + COPY --dir cluster/webhookconfigurations/ /webhookconfigurations + EXPOSE 8080 + USER 65532 + ENTRYPOINT ["crossplane"] + SAVE IMAGE --push ${CROSSPLANE_REPO}:${CROSSPLANE_VERSION} + SAVE IMAGE --push ${CROSSPLANE_REPO}:${EARTHLY_GIT_BRANCH} + +# multiplatform-image builds the Crossplane OCI image for all supported +# architectures. +multiplatform-image: + BUILD \ + --platform=linux/amd64 \ + --platform=linux/arm64 \ + --platform=linux/arm \ + --platform=linux/ppc64le \ + +image + +# helm-lint lints the Crossplane Helm chart. +helm-lint: + FROM alpine:3.20 + WORKDIR /chart + COPY +helm-setup/helm /usr/local/bin/helm + COPY cluster/charts/crossplane/ . + RUN --entrypoint helm lint + +# helm-generate runs Helm code generation - specifically helm-docs. +helm-generate: + FROM alpine:3.20 + WORKDIR /chart + COPY +helm-docs-setup/helm-docs /usr/local/bin/helm-docs + COPY cluster/charts/crossplane/ . + RUN helm-docs + SAVE ARTIFACT . AS LOCAL cluster/charts/crossplane + +# helm-build packages the Crossplane Helm chart. +helm-build: + ARG EARTHLY_GIT_SHORT_HASH + ARG EARTHLY_GIT_COMMIT_TIMESTAMP + ARG CROSSPLANE_VERSION=v0.0.0-${EARTHLY_GIT_COMMIT_TIMESTAMP}-${EARTHLY_GIT_SHORT_HASH} + FROM alpine:3.20 + WORKDIR /chart + COPY +helm-setup/helm /usr/local/bin/helm + COPY cluster/charts/crossplane/ . + # We strip the leading v from Helm chart versions. + LET CROSSPLANE_CHART_VERSION=$(echo ${CROSSPLANE_VERSION}|sed -e 's/^v//') + RUN helm dependency update + RUN helm package --version ${CROSSPLANE_CHART_VERSION} --app-version ${CROSSPLANE_CHART_VERSION} -d output . + SAVE ARTIFACT output AS LOCAL _output/charts + +# kubectl-setup is used by other targets to setup kubectl. +kubectl-setup: + ARG KUBECTL_VERSION=v1.30.1 + ARG NATIVEPLATFORM + ARG TARGETOS + ARG TARGETARCH + FROM --platform=${NATIVEPLATFORM} curlimages/curl:8.8.0 + RUN curl -fsSL https://dl.k8s.io/${KUBECTL_VERSION}/kubernetes-client-${TARGETOS}-${TARGETARCH}.tar.gz|tar zx + SAVE ARTIFACT kubernetes/client/bin/kubectl + +# kind-setup is used by other targets to setup kind. +kind-setup: + ARG KIND_VERSION=v0.21.0 + ARG NATIVEPLATFORM + ARG TARGETOS + ARG TARGETARCH + FROM --platform=${NATIVEPLATFORM} curlimages/curl:8.8.0 + RUN curl -fsSLo kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${TARGETOS}-${TARGETARCH}&&chmod +x kind + SAVE ARTIFACT kind + +# gotestsum-setup is used by other targets to setup gotestsum. +gotestsum-setup: + ARG GOTESTSUM_VERSION=1.11.0 + ARG NATIVEPLATFORM + ARG TARGETOS + ARG TARGETARCH + FROM --platform=${NATIVEPLATFORM} curlimages/curl:8.8.0 + RUN curl -fsSL https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/gotestsum_${GOTESTSUM_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz|tar zx>gotestsum + SAVE ARTIFACT gotestsum + +# helm-docs-setup is used by other targets to setup helm-docs. +helm-docs-setup: + ARG HELM_DOCS_VERSION=1.11.0 + ARG NATIVEPLATFORM + ARG TARGETOS + ARG TARGETARCH + FROM --platform=${NATIVEPLATFORM} curlimages/curl:8.8.0 + IF [ "${TARGETARCH}" = "amd64" ] + LET ARCH=x86_64 + ELSE + LET ARCH=${TARGETARCH} + END + RUN curl -fsSL https://github.com/norwoodj/helm-docs/releases/download/v${HELM_DOCS_VERSION}/helm-docs_${HELM_DOCS_VERSION}_${TARGETOS}_${ARCH}.tar.gz|tar zx>helm-docs + SAVE ARTIFACT helm-docs + +# helm-setup is used by other targets to setup helm. +helm-setup: + ARG HELM_VERSION=v3.15.1 + ARG NATIVEPLATFORM + ARG TARGETOS + ARG TARGETARCH + FROM --platform=${NATIVEPLATFORM} curlimages/curl:8.8.0 + RUN curl -fsSL https://get.helm.sh/helm-${HELM_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz|tar zx --strip-components=1 + SAVE ARTIFACT helm + +# Targets below this point are intended only for use in GitHub Actions CI. They +# may not work outside of that environment. For example they may depend on +# secrets that are only availble in the CI environment. Targets below this point +# must be prefixed with ci-. + +# TODO(negz): Is there a better way to determine the Crossplane version? +# This versioning approach maintains compatibility with the build submodule. See +# https://github.com/crossplane/build/blob/231258/makelib/common.mk#L205. This +# approach is problematic in Earthly because computing it inside a containerized +# target requires copying the entire git repository into the container. Doing so +# would invalidate all dependent target caches any time any file in git changed. + +# ci-version is used by CI to set the CROSSPLANE_VERSION environment variable. +ci-version: + LOCALLY + RUN echo "CROSSPLANE_VERSION=$(git describe --dirty --always --tags|sed -e 's/-/./2g')" > $GITHUB_ENV + +# ci-artifacts is used by CI to build and push the Crossplane image, chart, and +# binaries. +ci-artifacts: + BUILD +multiplatform-build \ + --CROSSPLANE_REPO=index.docker.io/crossplane/crossplane \ + --CROSSPLANE_REPO=xpkg.upbound.io/crossplane/crossplane + +# ci-codeql-setup sets up CodeQL for the ci-codeql target. +ci-codeql-setup: + ARG CODEQL_VERSION=v2.17.3 + FROM curlimages/curl:8.8.0 + RUN curl -fsSL https://github.com/github/codeql-action/releases/download/codeql-bundle-${CODEQL_VERSION}/codeql-bundle-linux64.tar.gz|tar zx + SAVE ARTIFACT codeql + +# ci-codeql is used by CI to build Crossplane with CodeQL scanning enabled. +ci-codeql: + ARG CGO_ENABLED=0 + ARG TARGETOS + ARG TARGETARCH + # Using a static CROSSPLANE_VERSION allows Earthly to cache E2E runs as long + # as no code changed. If the version contains a git commit (the default) the + # build layer cache is invalidated on every commit. + FROM +go-modules --CROSSPLANE_VERSION=v0.0.0-codeql + IF [ "${TARGETARCH}" = "arm64" ] && [ "${TARGETOS}" = "linux" ] + RUN --no-cache echo "CodeQL doesn't support Linux on Apple Silicon" && false + END + COPY --dir +ci-codeql-setup/codeql /codeql + CACHE --id go-build --sharing shared /root/.cache/go-build + COPY --dir apis/ cmd/ internal/ pkg/ . + RUN /codeql/codeql database create /codeqldb --language=go + RUN /codeql/codeql database analyze /codeqldb --threads=0 --format=sarif-latest --output=go.sarif --sarif-add-baseline-file-info + SAVE ARTIFACT go.sarif AS LOCAL _output/codeql/go.sarif + +# ci-promote-image is used by CI to promote a Crossplane image to a channel. +# In practice, this means creating a new channel tag (e.g. master or stable) +# that points to the supplied version. +ci-promote-image: + ARG --required CROSSPLANE_REPO + ARG --required CROSSPLANE_VERSION + ARG --required CHANNEL + FROM alpine:3.20 + RUN apk add docker + RUN --secret DOCKER_USER --secret DOCKER_PASSWORD docker login -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} + RUN --push docker buildx imagetools create \ + --tag ${CROSSPLANE_REPO}:${CHANNEL} \ + --tag ${CROSSPLANE_REPO}:${CROSSPLANE_VERSION}-${CHANNEL} \ + ${CROSSPLANE_REPO}:${CROSSPLANE_VERSION} + +# TODO(negz): Ideally ci-push-build-artifacts would be merged into ci-artifacts, +# i.e. just build and push them all in the same target. Currently we're relying +# on the fact that ci-artifacts does a bunch of SAVE ARTIFACT AS LOCAL, which +# ci-push-build-artifacts then loads. That's an anti-pattern in Earthly. We're +# supposed to use COPY instead, but I'm not sure how to COPY artifacts from a +# matrix build. + +# ci-push-build-artifacts is used by CI to push binary artifacts to S3. +ci-push-build-artifacts: + ARG --required CROSSPLANE_VERSION + ARG ARTIFACTS_DIR=_output + ARG EARTHLY_GIT_BRANCH + ARG BUCKET_RELEASES=crossplane.releases + ARG AWS_DEFAULT_REGION + FROM amazon/aws-cli:2.15.57 + COPY --dir ${ARTIFACTS_DIR} artifacts + RUN --push --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 sync --delete --only-show-errors artifacts s3://${BUCKET_RELEASES}/build/${EARTHLY_GIT_BRANCH}/${CROSSPLANE_VERSION} + +# ci-promote-build-artifacts is used by CI to promote binary artifacts and Helm +# charts to a channel. In practice, this means copying them from one S3 +# directory to another. +ci-promote-build-artifacts: + ARG --required CROSSPLANE_VERSION + ARG --required CHANNEL + ARG HELM_REPO_URL=https://charts.crossplane.io + ARG EARTHLY_GIT_BRANCH + ARG BUCKET_RELEASES=crossplane.releases + ARG BUCKET_CHARTS=crossplane.charts + ARG PRERELEASE=false + ARG AWS_DEFAULT_REGION + FROM amazon/aws-cli:2.15.57 + COPY +helm-setup/helm /usr/local/bin/helm + RUN --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 sync --only-show-errors s3://${BUCKET_CHARTS}/${CHANNEL} repo + RUN --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 sync --only-show-errors s3://${BUCKET_RELEASES}/build/${EARTHLY_GIT_BRANCH}/${CROSSPLANE_VERSION}/charts repo + RUN helm repo index --url ${HELM_REPO_URL} repo + RUN --push --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 sync --delete --only-show-errors repo s3://${BUCKET_CHARTS}/${CHANNEL} + RUN --push --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 cp --only-show-errors --cache-control "private, max-age=0, no-transform" repo/index.yaml s3://${BUCKET_CHARTS}/${CHANNEL}/index.yaml + RUN --push --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 sync --delete --only-show-errors s3://${BUCKET_RELEASES}/build/${EARTHLY_GIT_BRANCH}/${CROSSPLANE_VERSION} s3://${BUCKET_RELEASES}/${CHANNEL}/${CROSSPLANE_VERSION} + IF [ "${PRERELEASE}" = "false" ] + RUN --push --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 sync --delete --only-show-errors s3://${BUCKET_RELEASES}/build/${EARTHLY_GIT_BRANCH}/${CROSSPLANE_VERSION} s3://${BUCKET_RELEASES}/${CHANNEL}/current + END From 4b32f0e0405d790b9c2f8a801a6c734d878a6e32 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 29 May 2024 15:32:16 -0700 Subject: [PATCH 3/8] Strip down build and CI to only things runtime needs Removes all jobs and targets that are specific to crossplane/crossplane. Signed-off-by: Nic Cope --- .github/PULL_REQUEST_TEMPLATE.md | 1 - .github/workflows/ci.yml | 184 +------------------ .github/workflows/promote.yml | 48 ----- Earthfile | 297 +------------------------------ 4 files changed, 14 insertions(+), 516 deletions(-) delete mode 100644 .github/workflows/promote.yml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6f84d7d11..1d3b7e5fd 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -21,7 +21,6 @@ I have: - [ ] Read and followed Crossplane's [contribution process]. - [ ] Run `earthly +reviewable` to ensure this PR is ready for review. - [ ] Added or updated unit tests. -- [ ] Added or updated e2e tests. - [ ] Linked a PR or a [docs tracking issue] to [document this change]. - [ ] Added `backport release-x.y` labels to auto-backport this PR. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b81498c9..344ff66a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,9 +19,6 @@ env: # a step 'if env.AWS_USR' != ""', so we copy these to succinctly test whether # credentials have been provided before trying to run steps that need them. DOCKER_USR: ${{ secrets.DOCKER_USR }} - AWS_USR: ${{ secrets.AWS_USR }} - UPBOUND_MARKETPLACE_PUSH_ROBOT_USR: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }} - jobs: check-diff: @@ -58,7 +55,7 @@ jobs: echo "EARTHLY_MAX_REMOTE_CACHE=true" >> $GITHUB_ENV - name: Generate Files - run: earthly --strict --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }} +generate + run: earthly --strict --remote-cache ghcr.io/crossplane/crossplane-runtime-earthly-cache:${{ github.job }} +generate - name: Count Changed Files id: changed_files @@ -104,7 +101,7 @@ jobs: echo "EARTHLY_MAX_REMOTE_CACHE=true" >> $GITHUB_ENV - name: Lint - run: earthly --strict --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }} +lint + run: earthly --strict --remote-cache ghcr.io/crossplane/crossplane-runtime-earthly-cache:${{ github.job }} +lint codeql: runs-on: ubuntu-22.04 @@ -140,7 +137,7 @@ jobs: echo "EARTHLY_MAX_REMOTE_CACHE=true" >> $GITHUB_ENV - name: Run CodeQL - run: earthly --strict --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }} +ci-codeql + run: earthly --strict --remote-cache ghcr.io/crossplane/crossplane-runtime-earthly-cache:${{ github.job }} +ci-codeql - name: Upload CodeQL Results to GitHub uses: github/codeql-action/upload-sarif@9fdb3e49720b44c48891d036bb502feb25684276 # v3 @@ -204,7 +201,7 @@ jobs: echo "EARTHLY_MAX_REMOTE_CACHE=true" >> $GITHUB_ENV - name: Run Unit Tests - run: earthly --strict --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }} +test + run: earthly --strict --remote-cache ghcr.io/crossplane/crossplane-runtime-earthly-cache:${{ github.job }} +test - name: Publish Unit Test Coverage uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c # v4 @@ -213,177 +210,6 @@ jobs: file: _output/tests/coverage.txt token: ${{ secrets.CODECOV_TOKEN }} - e2e-tests: - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - test-suite: - - base - - environment-configs - - usage - - ssa-claims - - realtime-compositions - - steps: - - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 - - - name: Setup Earthly - uses: earthly/actions-setup@v1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - version: ${{ env.EARTHLY_VERSION }} - - - name: Login to DockerHub - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 - if: env.DOCKER_USR != '' - with: - username: ${{ secrets.DOCKER_USR }} - password: ${{ secrets.DOCKER_PSW }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Configure Earthly to Push Cache to GitHub Container Registry - if: github.ref == 'refs/heads/master' - run: | - echo "EARTHLY_PUSH=true" >> $GITHUB_ENV - echo "EARTHLY_MAX_REMOTE_CACHE=true" >> $GITHUB_ENV - - - name: Run E2E Tests - run: | - earthly --strict --allow-privileged --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }}-${{ matrix.test-suite}} \ - +e2e --FLAGS="-test.failfast -fail-fast --test-suite ${{ matrix.test-suite }}" - - - name: Publish E2E Test Flakes - if: '!cancelled()' - uses: buildpulse/buildpulse-action@d0d30f53585cf16b2e01811a5a753fd47968654a # v0.11.0 - with: - account: 45158470 - repository: 147886080 - key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }} - secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }} - path: _output/tests/e2e-tests.xml - - publish-artifacts: - runs-on: ubuntu-22.04 - - steps: - - name: Cleanup Disk - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - android: true - dotnet: true - haskell: true - tool-cache: true - swap-storage: false - # This works, and saves ~5GiB, but takes ~2 minutes to do it. - large-packages: false - # TODO(negz): Does having these around avoid Earthly needing to pull - # large images like golang? - docker-images: false - - - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 - with: - fetch-depth: 0 - - - name: Setup Earthly - uses: earthly/actions-setup@v1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - version: ${{ env.EARTHLY_VERSION }} - - - name: Login to DockerHub - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 - if: env.DOCKER_USR != '' - with: - username: ${{ secrets.DOCKER_USR }} - password: ${{ secrets.DOCKER_PSW }} - - - name: Login to Upbound - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 - if: env.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR != '' - with: - registry: xpkg.upbound.io - username: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }} - password: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Configure Earthly to Push Cache to GitHub Container Registry - if: github.ref == 'refs/heads/master' - run: echo "EARTHLY_MAX_REMOTE_CACHE=true" >> $GITHUB_ENV - - - name: Configure Earthly to Push Artifacts - if: env.DOCKER_USR != '' && env.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR != '' && env.AWS_USR != '' - run: echo "EARTHLY_PUSH=true" >> $GITHUB_ENV - - - name: Set CROSSPLANE_VERSION GitHub Environment Variable - run: earthly +ci-version - - - name: Build and Push Artifacts - run: earthly --strict --remote-cache ghcr.io/crossplane/earthly-cache:${{ github.job }} +ci-artifacts --CROSSPLANE_VERSION=${CROSSPLANE_VERSION} - - - name: Push Artifacts to https://releases.crossplane.io/build/ - if: env.AWS_USR != '' - run: | - earthly --strict \ - --secret=AWS_ACCESS_KEY_ID=${{ secrets.AWS_USR }} \ - --secret=AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_PSW }} \ - +ci-push-build-artifacts --AWS_DEFAULT_REGION=us-east-1 --CROSSPLANE_VERSION=${CROSSPLANE_VERSION} - - - name: Push Artifacts to https://releases.crossplane.io/master/ and https://charts.crossplane.io/master - if: env.AWS_USR != '' && github.ref == 'refs/heads/master' - run: | - earthly --strict \ - --secret=AWS_ACCESS_KEY_ID=${{ secrets.AWS_USR }} \ - --secret=AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_PSW }} \ - +ci-promote-build-artifacts --AWS_DEFAULT_REGION=us-east-1 --CROSSPLANE_VERSION=${CROSSPLANE_VERSION} --CHANNEL=master - - - name: Upload Artifacts to GitHub - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 - with: - name: output - path: _output/** - - fuzz-test: - runs-on: ubuntu-22.04 - - steps: - # TODO(negz): Can we make this use our Go build and dependency cache? It - # seems to build Crossplane inside of a Docker image. - - name: Build Fuzzers - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: "crossplane" - language: go - - - name: Run Fuzzers - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - oss-fuzz-project-name: "crossplane" - fuzz-seconds: 300 - language: go - - - name: Upload Crash - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 - if: failure() && steps.build.outcome == 'success' - with: - name: artifacts - path: ./out/artifacts - protobuf-schemas: runs-on: ubuntu-22.04 @@ -412,7 +238,7 @@ jobs: against: "https://github.com/${GITHUB_REPOSITORY}.git#branch=master,subdir=apis" - name: Push Protocol Buffers to Buf Schema Registry - if: ${{ github.repository == 'crossplane/crossplane' && github.ref == 'refs/heads/master' }} + if: ${{ github.repository == 'crossplane/crossplane-runtime' && github.ref == 'refs/heads/master' }} uses: bufbuild/buf-push-action@v1 with: input: apis diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml deleted file mode 100644 index c79a900f8..000000000 --- a/.github/workflows/promote.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Promote - -on: - workflow_dispatch: - inputs: - version: - description: 'Release version (e.g. v0.1.0)' - required: true - channel: - description: 'Release channel' - required: true - default: 'alpha' - -env: - # Common users. We can't run a step 'if secrets.AWS_USR != ""' but we can run - # a step 'if env.AWS_USR' != ""', so we copy these to succinctly test whether - # credentials have been provided before trying to run steps that need them. - DOCKER_USR: ${{ secrets.DOCKER_USR }} - AWS_USR: ${{ secrets.AWS_USR }} - -jobs: - promote-artifacts: - runs-on: ubuntu-22.04 - - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - submodules: true - - - name: Fetch History - run: git fetch --prune --unshallow - - - name: Login to Docker - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3 - if: env.DOCKER_USR != '' - with: - username: ${{ secrets.DOCKER_USR }} - password: ${{ secrets.DOCKER_PSW }} - - - name: Promote Artifacts in S3 and Docker Hub - if: env.AWS_USR != '' && env.DOCKER_USR != '' - run: make -j2 promote BRANCH_NAME=${GITHUB_REF##*/} - env: - VERSION: ${{ github.event.inputs.version }} - CHANNEL: ${{ github.event.inputs.channel }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_USR }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PSW }} diff --git a/Earthfile b/Earthfile index 2c14638dc..77c6e5539 100644 --- a/Earthfile +++ b/Earthfile @@ -1,7 +1,7 @@ # See https://docs.earthly.dev/docs/earthfile/features VERSION --try --raw-output 0.8 -PROJECT crossplane/crossplane +PROJECT crossplane/crossplane-runtime ARG --global GO_VERSION=1.22.3 @@ -21,18 +21,14 @@ test: # lint runs linters. lint: BUILD +go-lint - BUILD +helm-lint # build builds Crossplane for your native OS and architecture. build: - BUILD +image - BUILD +helm-build + BUILD +go-build # multiplatform-build builds Crossplane for all supported OS and architectures. multiplatform-build: BUILD +go-multiplatform-build - BUILD +multiplatform-image - BUILD +helm-build # generate runs code generation. To keep builds fast, it doesn't run as part of # the build target. It's important to run it explicitly when code needs to be @@ -40,65 +36,6 @@ multiplatform-build: generate: BUILD +go-modules-tidy BUILD +go-generate - BUILD +helm-generate - -# e2e runs end-to-end tests. See test/e2e/README.md for details. -e2e: - ARG FLAGS="-test-suite=base" - # Docker installs faster on Alpine, and we only need Go for go tool test2json. - FROM golang:${GO_VERSION}-alpine3.20 - RUN apk add --no-cache docker jq - COPY +helm-setup/helm /usr/local/bin/helm - COPY +kind-setup/kind /usr/local/bin/kind - COPY +gotestsum-setup/gotestsum /usr/local/bin/gotestsum - COPY +go-build-e2e/e2e . - COPY --dir cluster test . - # Using a static CROSSPLANE_VERSION allows Earthly to cache E2E runs as long - # as no code changed. If the version contains a git commit (the default) the - # build layer cache is invalidated on every commit. - WITH DOCKER --load crossplane-e2e/crossplane:latest=(+image --CROSSPLANE_VERSION=v0.0.0-e2e) - TRY - # TODO(negz:) Set GITHUB_ACTIONS=true and use RUN --raw-output when - # https://github.com/earthly/earthly/issues/4143 is fixed. - RUN gotestsum --no-color=false --format testname --junitfile e2e-tests.xml --raw-command go tool test2json -t -p E2E ./e2e -test.v ${FLAGS} - FINALLY - SAVE ARTIFACT --if-exists e2e-tests.xml AS LOCAL _output/tests/e2e-tests.xml - END - END - -# hack builds Crossplane, and deploys it to a kind cluster. It runs in your -# local environment, not a container. The kind cluster will keep running until -# you run the unhack target. Run hack again to rebuild Crossplane and restart -# the kind cluster with the new build. -hack: - # TODO(negz): This could run an interactive shell inside a temporary container - # once https://github.com/earthly/earthly/issues/3206 is fixed. - ARG USERPLATFORM - LOCALLY - WAIT - BUILD +unhack - END - COPY --platform=${USERPLATFORM} +helm-setup/helm .hack/helm - COPY --platform=${USERPLATFORM} +kind-setup/kind .hack/kind - COPY (+helm-build/output --CROSSPLANE_VERSION=v0.0.0-hack) .hack/charts - WITH DOCKER --load crossplane-hack/crossplane:hack=+image - RUN \ - .hack/kind create cluster --name crossplane-hack && \ - .hack/kind load docker-image --name crossplane-hack crossplane-hack/crossplane:hack && \ - .hack/helm install --create-namespace --namespace crossplane-system crossplane .hack/charts/crossplane-0.0.0-hack.tgz \ - --set "image.pullPolicy=Never,image.repository=crossplane-hack/crossplane,image.tag=hack" \ - --set "args={--debug}" - END - RUN docker image rm crossplane-hack/crossplane:hack - RUN rm -rf .hack - -# unhack deletes the kind cluster created by the hack target. -unhack: - ARG USERPLATFORM - LOCALLY - COPY --platform=${USERPLATFORM} +kind-setup/kind .hack/kind - RUN .hack/kind delete cluster --name crossplane-hack - RUN rm -rf .hack # go-modules downloads Crossplane's go modules. It's the base target of most Go # related target (go-build, etc). @@ -116,7 +53,7 @@ go-modules: go-modules-tidy: FROM +go-modules CACHE --id go-build --sharing shared /root/.cache/go-build - COPY --dir apis/ cmd/ internal/ pkg/ test/ . + COPY --dir apis/ pkg/ . RUN go mod tidy RUN go mod verify SAVE ARTIFACT go.mod AS LOCAL go.mod @@ -126,38 +63,21 @@ go-modules-tidy: go-generate: FROM +go-modules CACHE --id go-build --sharing shared /root/.cache/go-build - COPY +kubectl-setup/kubectl /usr/local/bin/kubectl - COPY --dir cluster/crd-patches cluster/crd-patches - COPY --dir hack/ apis/ internal/ . + COPY --dir apis/ hack/ . RUN go generate -tags 'generate' ./apis/... - # TODO(negz): Can this move into generate.go? Ideally it would live there with - # the code that actually generates the CRDs, but it depends on kubectl. - RUN kubectl patch --local --type=json \ - --patch-file cluster/crd-patches/pkg.crossplane.io_deploymentruntimeconfigs.yaml \ - --filename cluster/crds/pkg.crossplane.io_deploymentruntimeconfigs.yaml \ - --output=yaml > /tmp/patched.yaml \ - && mv /tmp/patched.yaml cluster/crds/pkg.crossplane.io_deploymentruntimeconfigs.yaml SAVE ARTIFACT apis/ AS LOCAL apis - SAVE ARTIFACT cluster/crds AS LOCAL cluster/crds # go-build builds Crossplane binaries for your native OS and architecture. go-build: - ARG EARTHLY_GIT_SHORT_HASH - ARG EARTHLY_GIT_COMMIT_TIMESTAMP - ARG CROSSPLANE_VERSION=v0.0.0-${EARTHLY_GIT_COMMIT_TIMESTAMP}-${EARTHLY_GIT_SHORT_HASH} ARG TARGETARCH ARG TARGETOS ARG GOARCH=${TARGETARCH} ARG GOOS=${TARGETOS} - ARG GOFLAGS="-ldflags=-X=github.com/crossplane/crossplane/internal/version.version=${CROSSPLANE_VERSION}" ARG CGO_ENABLED=0 FROM +go-modules CACHE --id go-build --sharing shared /root/.cache/go-build - COPY --dir apis/ cmd/ internal/ pkg/ . - RUN go build -o crossplane ./cmd/crossplane - RUN go build -o crank ./cmd/crank - SAVE ARTIFACT crossplane AS LOCAL _output/bin/${GOOS}_${GOARCH}/crossplane - SAVE ARTIFACT crank AS LOCAL _output/bin/${GOOS}_${GOARCH}/crank + COPY --dir apis/ pkg/ . + RUN go build ./... # go-multiplatform-build builds Crossplane binaries for all supported OS # and architectures. @@ -172,20 +92,11 @@ go-multiplatform-build: --platform=windows/amd64 \ +go-build -# go-build-e2e builds Crossplane's end-to-end tests. -go-build-e2e: - ARG CGO_ENABLED=0 - FROM +go-modules - CACHE --id go-build --sharing shared /root/.cache/go-build - COPY --dir apis/ internal/ test/ . - RUN go test -c -o e2e ./test/e2e - SAVE ARTIFACT e2e - # go-test runs Go unit tests. go-test: FROM +go-modules CACHE --id go-build --sharing shared /root/.cache/go-build - COPY --dir apis/ cmd/ internal/ pkg/ . + COPY --dir apis/ pkg/ . RUN go test -covermode=count -coverprofile=coverage.txt ./... SAVE ARTIFACT coverage.txt AS LOCAL _output/tests/coverage.txt @@ -198,131 +109,10 @@ go-lint: CACHE --id go-build --sharing shared /root/.cache/go-build RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin ${GOLANGCI_LINT_VERSION} COPY .golangci.yml . - COPY --dir apis/ cmd/ internal/ pkg/ test/ . + COPY --dir apis/ pkg/ . RUN golangci-lint run --fix SAVE ARTIFACT apis AS LOCAL apis - SAVE ARTIFACT cmd AS LOCAL cmd - SAVE ARTIFACT internal AS LOCAL internal SAVE ARTIFACT pkg AS LOCAL pkg - SAVE ARTIFACT test AS LOCAL test - -# image builds the Crossplane OCI image for your native architecture. -image: - ARG EARTHLY_GIT_BRANCH - ARG EARTHLY_GIT_SHORT_HASH - ARG EARTHLY_GIT_COMMIT_TIMESTAMP - ARG CROSSPLANE_REPO=build-${EARTHLY_GIT_SHORT_HASH}/crossplane - ARG CROSSPLANE_VERSION=v0.0.0-${EARTHLY_GIT_COMMIT_TIMESTAMP}-${EARTHLY_GIT_SHORT_HASH} - ARG NATIVEPLATFORM - ARG TARGETPLATFORM - ARG TARGETARCH - ARG TARGETOS - FROM --platform=${TARGETPLATFORM} gcr.io/distroless/static@sha256:41972110a1c1a5c0b6adb283e8aa092c43c31f7c5d79b8656fbffff2c3e61f05 - COPY --platform=${NATIVEPLATFORM} (+go-build/crossplane --GOOS=${TARGETOS} --GOARCH=${TARGETARCH}) /usr/local/bin/ - COPY --dir cluster/crds/ /crds - COPY --dir cluster/webhookconfigurations/ /webhookconfigurations - EXPOSE 8080 - USER 65532 - ENTRYPOINT ["crossplane"] - SAVE IMAGE --push ${CROSSPLANE_REPO}:${CROSSPLANE_VERSION} - SAVE IMAGE --push ${CROSSPLANE_REPO}:${EARTHLY_GIT_BRANCH} - -# multiplatform-image builds the Crossplane OCI image for all supported -# architectures. -multiplatform-image: - BUILD \ - --platform=linux/amd64 \ - --platform=linux/arm64 \ - --platform=linux/arm \ - --platform=linux/ppc64le \ - +image - -# helm-lint lints the Crossplane Helm chart. -helm-lint: - FROM alpine:3.20 - WORKDIR /chart - COPY +helm-setup/helm /usr/local/bin/helm - COPY cluster/charts/crossplane/ . - RUN --entrypoint helm lint - -# helm-generate runs Helm code generation - specifically helm-docs. -helm-generate: - FROM alpine:3.20 - WORKDIR /chart - COPY +helm-docs-setup/helm-docs /usr/local/bin/helm-docs - COPY cluster/charts/crossplane/ . - RUN helm-docs - SAVE ARTIFACT . AS LOCAL cluster/charts/crossplane - -# helm-build packages the Crossplane Helm chart. -helm-build: - ARG EARTHLY_GIT_SHORT_HASH - ARG EARTHLY_GIT_COMMIT_TIMESTAMP - ARG CROSSPLANE_VERSION=v0.0.0-${EARTHLY_GIT_COMMIT_TIMESTAMP}-${EARTHLY_GIT_SHORT_HASH} - FROM alpine:3.20 - WORKDIR /chart - COPY +helm-setup/helm /usr/local/bin/helm - COPY cluster/charts/crossplane/ . - # We strip the leading v from Helm chart versions. - LET CROSSPLANE_CHART_VERSION=$(echo ${CROSSPLANE_VERSION}|sed -e 's/^v//') - RUN helm dependency update - RUN helm package --version ${CROSSPLANE_CHART_VERSION} --app-version ${CROSSPLANE_CHART_VERSION} -d output . - SAVE ARTIFACT output AS LOCAL _output/charts - -# kubectl-setup is used by other targets to setup kubectl. -kubectl-setup: - ARG KUBECTL_VERSION=v1.30.1 - ARG NATIVEPLATFORM - ARG TARGETOS - ARG TARGETARCH - FROM --platform=${NATIVEPLATFORM} curlimages/curl:8.8.0 - RUN curl -fsSL https://dl.k8s.io/${KUBECTL_VERSION}/kubernetes-client-${TARGETOS}-${TARGETARCH}.tar.gz|tar zx - SAVE ARTIFACT kubernetes/client/bin/kubectl - -# kind-setup is used by other targets to setup kind. -kind-setup: - ARG KIND_VERSION=v0.21.0 - ARG NATIVEPLATFORM - ARG TARGETOS - ARG TARGETARCH - FROM --platform=${NATIVEPLATFORM} curlimages/curl:8.8.0 - RUN curl -fsSLo kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${TARGETOS}-${TARGETARCH}&&chmod +x kind - SAVE ARTIFACT kind - -# gotestsum-setup is used by other targets to setup gotestsum. -gotestsum-setup: - ARG GOTESTSUM_VERSION=1.11.0 - ARG NATIVEPLATFORM - ARG TARGETOS - ARG TARGETARCH - FROM --platform=${NATIVEPLATFORM} curlimages/curl:8.8.0 - RUN curl -fsSL https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/gotestsum_${GOTESTSUM_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz|tar zx>gotestsum - SAVE ARTIFACT gotestsum - -# helm-docs-setup is used by other targets to setup helm-docs. -helm-docs-setup: - ARG HELM_DOCS_VERSION=1.11.0 - ARG NATIVEPLATFORM - ARG TARGETOS - ARG TARGETARCH - FROM --platform=${NATIVEPLATFORM} curlimages/curl:8.8.0 - IF [ "${TARGETARCH}" = "amd64" ] - LET ARCH=x86_64 - ELSE - LET ARCH=${TARGETARCH} - END - RUN curl -fsSL https://github.com/norwoodj/helm-docs/releases/download/v${HELM_DOCS_VERSION}/helm-docs_${HELM_DOCS_VERSION}_${TARGETOS}_${ARCH}.tar.gz|tar zx>helm-docs - SAVE ARTIFACT helm-docs - -# helm-setup is used by other targets to setup helm. -helm-setup: - ARG HELM_VERSION=v3.15.1 - ARG NATIVEPLATFORM - ARG TARGETOS - ARG TARGETARCH - FROM --platform=${NATIVEPLATFORM} curlimages/curl:8.8.0 - RUN curl -fsSL https://get.helm.sh/helm-${HELM_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz|tar zx --strip-components=1 - SAVE ARTIFACT helm # Targets below this point are intended only for use in GitHub Actions CI. They # may not work outside of that environment. For example they may depend on @@ -336,18 +126,6 @@ helm-setup: # target requires copying the entire git repository into the container. Doing so # would invalidate all dependent target caches any time any file in git changed. -# ci-version is used by CI to set the CROSSPLANE_VERSION environment variable. -ci-version: - LOCALLY - RUN echo "CROSSPLANE_VERSION=$(git describe --dirty --always --tags|sed -e 's/-/./2g')" > $GITHUB_ENV - -# ci-artifacts is used by CI to build and push the Crossplane image, chart, and -# binaries. -ci-artifacts: - BUILD +multiplatform-build \ - --CROSSPLANE_REPO=index.docker.io/crossplane/crossplane \ - --CROSSPLANE_REPO=xpkg.upbound.io/crossplane/crossplane - # ci-codeql-setup sets up CodeQL for the ci-codeql target. ci-codeql-setup: ARG CODEQL_VERSION=v2.17.3 @@ -369,64 +147,7 @@ ci-codeql: END COPY --dir +ci-codeql-setup/codeql /codeql CACHE --id go-build --sharing shared /root/.cache/go-build - COPY --dir apis/ cmd/ internal/ pkg/ . + COPY --dir apis/ pkg/ . RUN /codeql/codeql database create /codeqldb --language=go RUN /codeql/codeql database analyze /codeqldb --threads=0 --format=sarif-latest --output=go.sarif --sarif-add-baseline-file-info SAVE ARTIFACT go.sarif AS LOCAL _output/codeql/go.sarif - -# ci-promote-image is used by CI to promote a Crossplane image to a channel. -# In practice, this means creating a new channel tag (e.g. master or stable) -# that points to the supplied version. -ci-promote-image: - ARG --required CROSSPLANE_REPO - ARG --required CROSSPLANE_VERSION - ARG --required CHANNEL - FROM alpine:3.20 - RUN apk add docker - RUN --secret DOCKER_USER --secret DOCKER_PASSWORD docker login -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} - RUN --push docker buildx imagetools create \ - --tag ${CROSSPLANE_REPO}:${CHANNEL} \ - --tag ${CROSSPLANE_REPO}:${CROSSPLANE_VERSION}-${CHANNEL} \ - ${CROSSPLANE_REPO}:${CROSSPLANE_VERSION} - -# TODO(negz): Ideally ci-push-build-artifacts would be merged into ci-artifacts, -# i.e. just build and push them all in the same target. Currently we're relying -# on the fact that ci-artifacts does a bunch of SAVE ARTIFACT AS LOCAL, which -# ci-push-build-artifacts then loads. That's an anti-pattern in Earthly. We're -# supposed to use COPY instead, but I'm not sure how to COPY artifacts from a -# matrix build. - -# ci-push-build-artifacts is used by CI to push binary artifacts to S3. -ci-push-build-artifacts: - ARG --required CROSSPLANE_VERSION - ARG ARTIFACTS_DIR=_output - ARG EARTHLY_GIT_BRANCH - ARG BUCKET_RELEASES=crossplane.releases - ARG AWS_DEFAULT_REGION - FROM amazon/aws-cli:2.15.57 - COPY --dir ${ARTIFACTS_DIR} artifacts - RUN --push --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 sync --delete --only-show-errors artifacts s3://${BUCKET_RELEASES}/build/${EARTHLY_GIT_BRANCH}/${CROSSPLANE_VERSION} - -# ci-promote-build-artifacts is used by CI to promote binary artifacts and Helm -# charts to a channel. In practice, this means copying them from one S3 -# directory to another. -ci-promote-build-artifacts: - ARG --required CROSSPLANE_VERSION - ARG --required CHANNEL - ARG HELM_REPO_URL=https://charts.crossplane.io - ARG EARTHLY_GIT_BRANCH - ARG BUCKET_RELEASES=crossplane.releases - ARG BUCKET_CHARTS=crossplane.charts - ARG PRERELEASE=false - ARG AWS_DEFAULT_REGION - FROM amazon/aws-cli:2.15.57 - COPY +helm-setup/helm /usr/local/bin/helm - RUN --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 sync --only-show-errors s3://${BUCKET_CHARTS}/${CHANNEL} repo - RUN --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 sync --only-show-errors s3://${BUCKET_RELEASES}/build/${EARTHLY_GIT_BRANCH}/${CROSSPLANE_VERSION}/charts repo - RUN helm repo index --url ${HELM_REPO_URL} repo - RUN --push --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 sync --delete --only-show-errors repo s3://${BUCKET_CHARTS}/${CHANNEL} - RUN --push --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 cp --only-show-errors --cache-control "private, max-age=0, no-transform" repo/index.yaml s3://${BUCKET_CHARTS}/${CHANNEL}/index.yaml - RUN --push --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 sync --delete --only-show-errors s3://${BUCKET_RELEASES}/build/${EARTHLY_GIT_BRANCH}/${CROSSPLANE_VERSION} s3://${BUCKET_RELEASES}/${CHANNEL}/${CROSSPLANE_VERSION} - IF [ "${PRERELEASE}" = "false" ] - RUN --push --secret=AWS_ACCESS_KEY_ID --secret=AWS_SECRET_ACCESS_KEY aws s3 sync --delete --only-show-errors s3://${BUCKET_RELEASES}/build/${EARTHLY_GIT_BRANCH}/${CROSSPLANE_VERSION} s3://${BUCKET_RELEASES}/${CHANNEL}/current - END From ef0fdfa63dd1e6d535804f189fc0761baba3bd47 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 29 May 2024 15:35:37 -0700 Subject: [PATCH 4/8] Fix linter issues surfaced by new linter config Signed-off-by: Nic Cope --- pkg/fieldpath/merge.go | 2 +- pkg/password/password.go | 2 +- pkg/reference/reference.go | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/fieldpath/merge.go b/pkg/fieldpath/merge.go index 2e2377711..dab88d4a1 100644 --- a/pkg/fieldpath/merge.go +++ b/pkg/fieldpath/merge.go @@ -93,7 +93,7 @@ func removeSourceDuplicates(dst, src any) any { } result := reflect.New(sliceSrc.Type()).Elem() // we will not modify src - for i := 0; i < sliceSrc.Len(); i++ { + for i := range sliceSrc.Len() { itemSrc := sliceSrc.Index(i) found := false for j := 0; j < sliceDst.Len() && !found; j++ { diff --git a/pkg/password/password.go b/pkg/password/password.go index 1e7450135..bdf31a29e 100644 --- a/pkg/password/password.go +++ b/pkg/password/password.go @@ -48,7 +48,7 @@ func Generate() (string, error) { // Generate a password. func (s Settings) Generate() (string, error) { pw := make([]byte, s.Length) - for i := 0; i < s.Length; i++ { + for i := range s.Length { n, err := rand.Int(rand.Reader, big.NewInt(int64(len(s.CharacterSet)))) if err != nil { return "", err diff --git a/pkg/reference/reference.go b/pkg/reference/reference.go index 56213b384..3248e21fa 100644 --- a/pkg/reference/reference.go +++ b/pkg/reference/reference.go @@ -107,7 +107,7 @@ func ToIntPtrValue(v string) *int64 { // string pointers and need to be resolved as part of `ResolveMultiple`. func FromPtrValues(v []*string) []string { res := make([]string, len(v)) - for i := 0; i < len(v); i++ { + for i := range len(v) { res[i] = FromPtrValue(v[i]) } return res @@ -116,7 +116,7 @@ func FromPtrValues(v []*string) []string { // FromFloatPtrValues adapts a slice of float64 pointer fields for use as CurrentValues. func FromFloatPtrValues(v []*float64) []string { res := make([]string, len(v)) - for i := 0; i < len(v); i++ { + for i := range len(v) { res[i] = FromFloatPtrValue(v[i]) } return res @@ -125,7 +125,7 @@ func FromFloatPtrValues(v []*float64) []string { // FromIntPtrValues adapts a slice of int64 pointer fields for use as CurrentValues. func FromIntPtrValues(v []*int64) []string { res := make([]string, len(v)) - for i := 0; i < len(v); i++ { + for i := range len(v) { res[i] = FromIntPtrValue(v[i]) } return res @@ -138,7 +138,7 @@ func FromIntPtrValues(v []*int64) []string { // string pointers and need to be resolved as part of `ResolveMultiple`. func ToPtrValues(v []string) []*string { res := make([]*string, len(v)) - for i := 0; i < len(v); i++ { + for i := range len(v) { res[i] = ToPtrValue(v[i]) } return res @@ -147,7 +147,7 @@ func ToPtrValues(v []string) []*string { // ToFloatPtrValues adapts ResolvedValues for use as a slice of float64 pointer fields. func ToFloatPtrValues(v []string) []*float64 { res := make([]*float64, len(v)) - for i := 0; i < len(v); i++ { + for i := range len(v) { res[i] = ToFloatPtrValue(v[i]) } return res @@ -156,7 +156,7 @@ func ToFloatPtrValues(v []string) []*float64 { // ToIntPtrValues adapts ResolvedValues for use as a slice of int64 pointer fields. func ToIntPtrValues(v []string) []*int64 { res := make([]*int64, len(v)) - for i := 0; i < len(v); i++ { + for i := range len(v) { res[i] = ToIntPtrValue(v[i]) } return res From cf4876673ca51bfc8c6c7a4130d8d50cad83e0fb Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 29 May 2024 15:41:14 -0700 Subject: [PATCH 5/8] Port renovate config from c/c too This implies switching from the Renovate app to the Renovate action. Signed-off-by: Nic Cope --- .github/renovate.json5 | 231 +++++++++++++++++++++++++-------- .github/workflows/renovate.yml | 57 ++++++++ 2 files changed, 232 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/renovate.yml diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 9e05c1b65..deab5d453 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -1,63 +1,172 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ - "config:base", - "helpers:pinGitHubActionDigests" + "config:recommended", + "helpers:pinGitHubActionDigests", + ":semanticCommits" ], -// We only want renovate to rebase PRs when they have conflicts, -// default "auto" mode is not required. + // We only want renovate to rebase PRs when they have conflicts, default + // "auto" mode is not required. "rebaseWhen": "conflicted", -// The maximum number of PRs to be created in parallel + // The maximum number of PRs to be created in parallel "prConcurrentLimit": 5, - "postUpdateOptions": ["gomodTidy"], -// By default renovate will auto detect whether semantic commits have been used -// in the recent history and comply with that, we explicitly disable it - "semanticCommits": "disabled", -// All PRs should have a label - "labels": ["automated"], - "regexManagers": [ + // The branches renovate should target + // PLEASE UPDATE THIS WHEN RELEASING. + "baseBranches": [ + "master", + "release-1.14", + "release-1.15", + "release-1.16" + ], + "ignorePaths": [ + "design/**", + // We test upgrades, so leave it on an older version on purpose. + "test/e2e/manifests/pkg/provider/provider-initial.yaml", + ], + "postUpdateOptions": [ + "gomodTidy" + ], + // All PRs should have a label + "labels": [ + "automated" + ], + "customManagers": [ { - "description": "Bump Go version ued in workflows", - "fileMatch": ["^\\.github\\/workflows\\/[^/]+\\.ya?ml$"], + "customType": "regex", + "description": "Bump Earthly version in GitHub workflows", + "fileMatch": [ + "^\\.github\\/workflows\\/[^/]+\\.ya?ml$" + ], "matchStrings": [ - "GO_VERSION: '(?.*?)'\\n" + "EARTHLY_VERSION '(?.*?)'\\n" + ], + "datasourceTemplate": "github-tags", + "depNameTemplate": "earthly/earthly", + "extractVersionTemplate": "^v(?.*)$" + }, + { + "customType": "regex", + "description": "Bump Go version in Earthfile", + "fileMatch": [ + "^Earthfile$" + ], + "matchStrings": [ + "ARG --global GO_VERSION=(?.*?)\\n" ], "datasourceTemplate": "golang-version", "depNameTemplate": "golang" - }, { - "description": "Bump golangci-lint version in workflows and the Makefile", - "fileMatch": ["^\\.github\\/workflows\\/[^/]+\\.ya?ml$","^Makefile$"], + }, + { + "customType": "regex", + "description": "Bump golangci-lint version in the Earthfile", + "fileMatch": [ + "^Earthfile$" + ], "matchStrings": [ - "GOLANGCI_VERSION: 'v(?.*?)'\\n", - "GOLANGCILINT_VERSION = (?.*?)\\n" + "ARG GOLANGCI_LINT_VERSION=(?.*?)\\n" ], "datasourceTemplate": "github-tags", - "depNameTemplate": "golangci/golangci-lint", - "extractVersionTemplate": "^v(?.*)$" - }, { - "description": "Bump Go required version in workflows and the Makefile", - "fileMatch": ["^\\.github\\/workflows\\/[^/]+\\.ya?ml$", "^Makefile$"], + "depNameTemplate": "golangci/golangci-lint" + }, + { + "customType": "regex", + "description": "Bump codeql version in the Earthfile", + "fileMatch": [ + "^Earthfile$" + ], "matchStrings": [ - "GO_REQUIRED_VERSION = (?.*?)\\n", + "ARG CODEQL_VERSION=(?.*?)\\n" ], - "datasourceTemplate": "golang-version", - "depNameTemplate": "golang", - "versioningTemplate": "loose", - "extractVersionTemplate": "^(?\\d+\\.\\d+)" - } + "datasourceTemplate": "github-tags", + "depNameTemplate": "github/codeql-action", + "extractVersionTemplate": "^codeql-bundle-(?.*)$" + }, ], -// PackageRules disabled below should be enabled in case of vulnerabilities + // Renovate doesn't have native Earthfile support, but because Earthfile + // syntax is a superset of Dockerfile syntax this works to update FROM images. + // https://github.com/renovatebot/renovate/issues/15975 + "dockerfile": { + "fileMatch": [ + "(^|/)Earthfile$" + ] + }, + // PackageRules disabled below should be enabled in case of vulnerabilities "vulnerabilityAlerts": { "enabled": true }, "osvVulnerabilityAlerts": true, + // Renovate evaluates all packageRules in order, so low priority rules should + // be at the beginning, high priority at the end "packageRules": [ { - "description": "Only get docker image updates every 2 weeks to reduce noise", - "matchDatasources": ["docker"], - "schedule": ["every 2 week on monday"], - "enabled": true, - }, { + "description": "Generate code after upgrading go dependencies", + "matchDatasources": [ + "go" + ], + postUpgradeTasks: { + // Post-upgrade tasks that are executed before a commit is made by Renovate. + "commands": [ + "earthly --strict +go-generate", + ], + fileFilters: [ + "**/*" + ], + executionMode: "update", + }, + }, + { + "description": "Lint code after upgrading golangci-lint", + "matchDepNames": [ + "golangci/golangci-lint" + ], + postUpgradeTasks: { + // Post-upgrade tasks that are executed before a commit is made by Renovate. + "commands": [ + "earthly --strict +go-lint", + ], + fileFilters: [ + "**/*" + ], + executionMode: "update", + }, + }, + { + "matchManagers": [ + "crossplane" + ], + "matchFileNames": [ + "test/e2e/**" + ], + "groupName": "e2e-manifests", + }, + { + "description": "Ignore non-security related updates to release branches", + matchBaseBranches: [ + "/^release-.*/" + ], + enabled: false, + }, + { + "description": "Still update Docker images on release branches though", + "matchDatasources": [ + "docker" + ], + matchBaseBranches: [ + "/^release-.*/" + ], + enabled: true, + }, + { + "description": "Only get Docker image updates every 2 weeks to reduce noise", + "matchDatasources": [ + "docker" + ], + "schedule": [ + "every 2 week on monday" + ], + enabled: true, + }, + { "description": "Ignore k8s.io/client-go older versions, they switched to semantic version and old tags are still available in the repo", "matchDatasources": [ "go" @@ -65,32 +174,35 @@ "matchDepNames": [ "k8s.io/client-go" ], - "allowedVersions": "<1.0" - }, { - "description": "Only get dependency digest updates every month to reduce noise", + "allowedVersions": "<1.0", + }, + { + "description": "Ignore k8s dependencies, should be updated on crossplane-runtime", "matchDatasources": [ "go" ], - "matchUpdateTypes": [ - "digest", + "matchPackagePrefixes": [ + "k8s.io", + "sigs.k8s.io" + ], + "enabled": false, + }, + { + "description": "Only get dependency digest updates every month to reduce noise, except crossplane-runtime", + "excludePackageNames": [ + "github.com/crossplane/crossplane-runtime" ], - "extends": ["schedule:monthly"], - }, { - "description": "Single PR for all kubernetes dependency updates, as they usually are all linked", "matchDatasources": [ "go" ], - "groupName": "kubernetes deps", "matchUpdateTypes": [ - "major", - "minor", - "patch" + "digest", ], - "matchPackagePrefixes": [ - "k8s.io", - "sigs.k8s.io" - ] - }, { + "extends": [ + "schedule:monthly" + ], + }, + { "description": "Ignore oss-fuzz, it's not using tags, we'll stick to master", "matchDepTypes": [ "action" @@ -99,6 +211,13 @@ "google/oss-fuzz" ], "enabled": false + }, + { + "description": "Group all go version updates", + "matchDatasources": [ + "golang-version" + ], + "groupName": "golang version", } - ] -} + ], +} \ No newline at end of file diff --git a/.github/workflows/renovate.yml b/.github/workflows/renovate.yml new file mode 100644 index 000000000..7316734d2 --- /dev/null +++ b/.github/workflows/renovate.yml @@ -0,0 +1,57 @@ +name: Renovate +on: + # Allows manual/automated trigger for debugging purposes + workflow_dispatch: + inputs: + logLevel: + description: "Renovate's log level" + required: true + default: "info" + type: string + schedule: + - cron: '0 8 * * *' + +env: + # Common versions + EARTHLY_VERSION: '0.8.11' + + LOG_LEVEL: "info" + +jobs: + renovate: + runs-on: ubuntu-latest + if: | + !github.event.repository.fork && + !github.event.pull_request.head.repo.fork + steps: + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + + - name: Setup Earthly + uses: earthly/actions-setup@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + version: ${{ env.EARTHLY_VERSION }} + + # Don't waste time starting Renovate if JSON is invalid + - name: Validate Renovate JSON + run: npx --yes --package renovate -- renovate-config-validator + + - name: Get token + id: get-github-app-token + uses: actions/create-github-app-token@a0de6af83968303c8c955486bf9739a57d23c7f1 # v1 + with: + app-id: ${{ secrets.RENOVATE_GITHUB_APP_ID }} + private-key: ${{ secrets.RENOVATE_GITHUB_APP_PRIVATE_KEY }} + + - name: Self-hosted Renovate + uses: renovatebot/github-action@063e0c946b9c1af35ef3450efc44114925d6e8e6 # v40.1.11 + env: + RENOVATE_REPOSITORIES: ${{ github.repository }} + # Use GitHub API to create commits + RENOVATE_PLATFORM_COMMIT: "true" + LOG_LEVEL: ${{ github.event.inputs.logLevel || env.LOG_LEVEL }} + RENOVATE_ALLOWED_POST_UPGRADE_COMMANDS: '["^earthly .+"]' + with: + configurationFile: .github/renovate.json5 + token: '${{ steps.get-github-app-token.outputs.token }}' From 703577426fe19bdefd1ea5dc94849ceb20c5a680 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 29 May 2024 15:43:46 -0700 Subject: [PATCH 6/8] Update CODEOWNERS to reflect current state This syncs with what we currently have in c/c. In practice we've dropped reviewers because everything blocks on reviewers. Signed-off-by: Nic Cope --- CODEOWNERS | 2 +- OWNERS.md | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index a9adb3c7d..775aa5422 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -22,7 +22,7 @@ # See also OWNERS.md for governance details # Fallback owners -* @crossplane/crossplane-maintainers @crossplane/crossplane-reviewers +* @crossplane/crossplane-maintainers # Governance owners - steering committee /README.md @crossplane/steering-committee diff --git a/OWNERS.md b/OWNERS.md index b8a0824c7..beb91f1b8 100644 --- a/OWNERS.md +++ b/OWNERS.md @@ -14,17 +14,15 @@ See [CODEOWNERS](CODEOWNERS) for automatic PR assignment. ## Maintainers * Nic Cope ([negz](https://github.com/negz)) -* Daniel Mangum ([hasheddan](https://github.com/hasheddan)) -* Muvaffak Onus ([muvaf](https://github.com/muvaf)) * Hasan Turken ([turkenh](https://github.com/turkenh)) +* Bob Haddleton ([bobh66](https://github.com/bobh66)) +* Philippe Scorsolini ([phisco](https://github.com/phisco)) ## Reviewers -* Bob Haddleton ([bobh66](https://github.com/bobh66)) * Yury Tsarev ([ytsarev](https://github.com/ytsarev)) * Ezgi Demirel ([ezgidemirel](https://github.com/ezgidemirel)) * Max Blatt ([MisterMX](https://github.com/MisterMX)) -* Philippe Scorsolini ([phisco](https://github.com/phisco)) ## Emeritus maintainers From f50dd7dabcfcc6c23a411a4b6f02d9f5994cbc3e Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 29 May 2024 15:54:09 -0700 Subject: [PATCH 7/8] Ignore buf linter warning about incorrect package Changing this package would be a breaking change. Signed-off-by: Nic Cope --- apis/buf.yaml | 3 +++ apis/proto/v1alpha1/ess.pb.go | 2 ++ apis/proto/v1alpha1/ess.proto | 1 + apis/proto/v1alpha1/ess_grpc.pb.go | 2 ++ 4 files changed, 8 insertions(+) create mode 100644 apis/buf.yaml diff --git a/apis/buf.yaml b/apis/buf.yaml new file mode 100644 index 000000000..0061b9f5f --- /dev/null +++ b/apis/buf.yaml @@ -0,0 +1,3 @@ +version: v1 +lint: + allow_comment_ignores: true \ No newline at end of file diff --git a/apis/proto/v1alpha1/ess.pb.go b/apis/proto/v1alpha1/ess.pb.go index 12afeeee2..b15dd8b4b 100644 --- a/apis/proto/v1alpha1/ess.pb.go +++ b/apis/proto/v1alpha1/ess.pb.go @@ -16,6 +16,8 @@ // protoc (unknown) // source: proto/v1alpha1/ess.proto +// buf:lint:ignore PACKAGE_DIRECTORY_MATCH + package v1alpha1 import ( diff --git a/apis/proto/v1alpha1/ess.proto b/apis/proto/v1alpha1/ess.proto index 43588fabf..4494f3fa1 100644 --- a/apis/proto/v1alpha1/ess.proto +++ b/apis/proto/v1alpha1/ess.proto @@ -13,6 +13,7 @@ limitations under the License. syntax = "proto3"; +// buf:lint:ignore PACKAGE_DIRECTORY_MATCH package ess.proto.v1alpha1; option go_package = "github.com/crossplane/crossplane-runtime/apis/proto/v1alpha1"; diff --git a/apis/proto/v1alpha1/ess_grpc.pb.go b/apis/proto/v1alpha1/ess_grpc.pb.go index d9a874a79..5760743f5 100644 --- a/apis/proto/v1alpha1/ess_grpc.pb.go +++ b/apis/proto/v1alpha1/ess_grpc.pb.go @@ -16,6 +16,8 @@ // - protoc (unknown) // source: proto/v1alpha1/ess.proto +// buf:lint:ignore PACKAGE_DIRECTORY_MATCH + package v1alpha1 import ( From e977812f6d412b6163a3daabda6983dc2fc47e38 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 29 May 2024 15:58:14 -0700 Subject: [PATCH 8/8] Fix extra ' in check-diff job Signed-off-by: Nic Cope --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 344ff66a8..ee501bfc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,7 @@ jobs: if: steps.changed_files.outputs.count != 0 uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 with: - script: core.setFailed('Found changed files after running earthly +generate.'') + script: core.setFailed('Found changed files after running earthly +generate.') lint: runs-on: ubuntu-22.04