From d3f3d52df2d58f95fe71753c07f8e5f73d783806 Mon Sep 17 00:00:00 2001 From: galargh Date: Sun, 16 Jun 2024 13:52:18 +0200 Subject: [PATCH 01/20] feat: ci: automate the new release process --- .github/workflows/docker.yml | 27 +++--- .github/workflows/release.yml | 161 ++++++++++++++++++++++------------ .goreleaser.yaml | 25 ++++-- cmd/release/main.go | 69 +++++++++++++++ scripts/publish-checksums.sh | 39 +------- scripts/version-check.sh | 39 -------- 6 files changed, 206 insertions(+), 154 deletions(-) create mode 100644 cmd/release/main.go delete mode 100755 scripts/version-check.sh diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cfde8bb42e1..a5c154be531 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,19 +1,17 @@ name: Docker +# TODO(p2): Restore worfklow_dispatch trigger on: push: branches: - master - - release/* tags: - v* + pull_request: + branches: + - master schedule: - cron: '0 0 * * *' - workflow_dispatch: - inputs: - ref: - description: The GitHub ref (e.g. refs/tags/v1.0.0) to release - required: false defaults: run: @@ -24,7 +22,7 @@ permissions: jobs: docker: - name: Docker (${{ matrix.image }} / ${{ matrix.network }}) [publish=${{ (inputs.ref || github.ref) == 'refs/heads/master' || startsWith(inputs.ref || github.ref, 'refs/tags/') }}] + name: Docker (${{ matrix.image }} / ${{ matrix.network }}) [publish=${{ github.event_name != 'pull_request' }}] runs-on: ubuntu-latest strategy: fail-fast: false @@ -40,13 +38,13 @@ jobs: - image: lotus network: mainnet env: - PUBLISH: ${{ github.ref == 'refs/heads/master' || startsWith(inputs.ref || github.ref, 'refs/tags/') }} + PUBLISH: ${{ github.event_name != 'pull_request' }} steps: - id: channel env: - IS_MASTER: ${{ (inputs.ref || github.ref) == 'refs/heads/master' }} - IS_TAG: ${{ startsWith(inputs.ref || github.ref, 'refs/tags/') }} - IS_RC: ${{ contains(inputs.ref || github.ref, '-rc') }} + IS_MASTER: ${{ github.ref == 'refs/heads/master' }} + IS_TAG: ${{ startsWith(github.ref, 'refs/tags/') }} + IS_RC: ${{ contains(github.ref, '-rc') }} IS_SCHEDULED: ${{ github.event_name == 'schedule' }} run: | channel='' @@ -67,12 +65,9 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'recursive' - ref: ${{ inputs.ref || github.ref }} - id: git - env: - REF: ${{ inputs.ref || github.ref }} run: | - ref="${REF#refs/heads/}" + ref="${GITHUB_REF#refs/heads/}" ref="${ref#refs/tags/}" sha="$(git rev-parse --short HEAD)" echo "ref=$ref" | tee -a "$GITHUB_OUTPUT" @@ -86,7 +81,7 @@ jobs: images: filecoin/${{ matrix.image }} tags: | type=raw,enable=${{ steps.channel.outputs.channel != '' }},value=${{ steps.channel.outputs.channel }} - type=raw,enable=${{ startsWith(inputs.ref || github.ref, 'refs/tags/') }},value=${{ steps.git.outputs.ref }} + type=raw,enable=${{ startsWith(github.ref, 'refs/tags/') }},value=${{ steps.git.outputs.ref }} type=raw,value=${{ steps.git.outputs.sha }} flavor: | latest=false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 12ea6e3f90e..7ebc49b13e2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,17 +1,17 @@ name: Release +# TODO(p2): Restore worfklow_dispatch trigger on: push: branches: - - ci/* - - release/* - tags: - - v* - workflow_dispatch: - inputs: - ref: - description: The GitHub ref (e.g. refs/tags/v1.0.0) to release - required: false + - master + paths: + - build/version.go + pull_request: + branches: + - master + paths: + - build/version.go defaults: run: @@ -21,12 +21,42 @@ permissions: contents: read jobs: + check: + name: Check which projects need to be built + runs-on: ubuntu-latest + outputs: + projects: ${{ steps.projects.outputs.projects }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: projects + run: | + projects=() + node_current_version=$(go run cmd/release/main.go --json node version | jq -r .msg) + node_current_version_exists=(git tag -l 'v*' | grep -q $node_current_version) + echo "node_current_version=$node_current_version" + echo "node_current_version_exists=$node_current_version_exists" + if [[ ! $node_current_version_exists ]]; then + projects+=('node') + fi + miner_current_version=$(go run cmd/release/main.go --json miner version | jq -r .msg) + miner_current_version_exists=(git tag -l 'miner/v*' | grep -q $miner_current_version) + echo "miner_current_version=$miner_current_version" + echo "miner_current_version_exists=$miner_current_version_exists" + if [[ ! $miner_current_version_exists ]]; then + projects+=('miner') + fi + echo "projects=$(echo "${projects[@]}" | jq -c .)" | tee -a $GITHUB_OUTPUT build: - name: Build (${{ matrix.os }}/${{ matrix.arch }}) + needs: [check] + if: needs.check.outputs.projects != '[]' + name: Build ${{ matrix.project }} (${{ matrix.os }}/${{ matrix.arch }}) runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: + project: ${{ fromJSON(needs.check.outputs.projects) }} include: - runner: ubuntu-latest os: Linux @@ -46,96 +76,117 @@ jobs: echo "::error title=Unexpected Runner::Expected $OS/$ARCH, got $RUNNER_OS/$RUNNER_ARCH" exit 1 fi - - uses: actions/checkout@v4 - with: - path: actions - uses: actions/checkout@v4 with: submodules: 'recursive' - ref: ${{ inputs.ref || github.ref }} - path: lotus - - uses: ./actions/.github/actions/install-system-dependencies - - uses: ./actions/.github/actions/install-go - with: - working-directory: lotus + - uses: ./.github/actions/install-system-dependencies + - uses: ./.github/actions/install-go - env: GITHUB_TOKEN: ${{ github.token }} - run: make deps lotus lotus-miner lotus-worker - working-directory: lotus - - if: runner.os == 'macOS' + run: make deps + - if: matrix.project == 'node' + env: + GITHUB_TOKEN: ${{ github.token }} + run: make lotus + - if: matrix.project == 'miner' + env: + GITHUB_TOKEN: ${{ github.token }} + run: make lotus-miner lotus-worker + - if: matrix.project == 'lotus' && runner.os == 'macOS' run: otool -hv lotus - working-directory: lotus - - env: - INPUTS_REF: ${{ inputs.ref }} - run: | - export GITHUB_REF=${INPUTS_REF:-$GITHUB_REF} - ../actions/scripts/version-check.sh ./lotus - working-directory: lotus - uses: actions/upload-artifact@v4 with: - name: lotus-${{ matrix.os }}-${{ matrix.arch }} + name: lotus-${{ matrix.project }}-${{ matrix.os }}-${{ matrix.arch }} path: | lotus/lotus lotus/lotus-miner lotus/lotus-worker release: - name: Release [publish=${{ startsWith(inputs.ref || github.ref, 'refs/tags/') }}] + needs: [check, build] + if: needs.check.outputs.projects != '[]' + name: Release [publish=${{ github.event_name != 'pull_request' }}] permissions: # This enables the job to create and/or update GitHub releases contents: write runs-on: ubuntu-latest - needs: [build] + strategy: + fail-fast: false + matrix: + project: ${{ fromJSON(needs.check.outputs.projects) }} env: - PUBLISH: ${{ startsWith(inputs.ref || github.ref, 'refs/tags/') }} + PUBLISH: ${{ github.event_name != 'pull_request' }} steps: - - uses: actions/checkout@v4 - with: - path: actions - uses: actions/checkout@v4 with: submodules: 'recursive' fetch-depth: 0 - path: lotus - ref: ${{ inputs.ref || github.ref }} - uses: actions/download-artifact@v4 with: - name: lotus-Linux-X64 + name: lotus-${{ matrix.project }}-Linux-X64 path: linux_amd64_v1 - uses: actions/download-artifact@v4 with: - name: lotus-macOS-X64 + name: lotus-${{ matrix.project }}-macOS-X64 path: darwin_amd64_v1 - uses: actions/download-artifact@v4 with: - name: lotus-macOS-ARM64 + name: lotus-${{ matrix.project }}-macOS-ARM64 path: darwin_arm64 - - uses: ./actions/.github/actions/install-go - with: - working-directory: lotus + - uses: ./.github/actions/install-go - uses: ipfs/download-ipfs-distribution-action@v1 with: name: kubo version: v0.16.0 + - id: meta + run: | + version="$(go run cmd/release/main.go --json ${{ matrix.project }} version | jq -r .msg)" + if [[ "${{ matrix.project}}" == "miner" ]]; then + tag="miner/$version" + latest="$(git tag -l 'miner/v*' | sort -V | tail -n1)" + elif [[ "${{ matrix.project}}" == "node" ]]; then + tag="$version" + latest="$(git tag -l 'v*' | sort -V | tail -n1)" + else + echo "::error title=Unknown Project::${{ matrix.project }}" + exit 1 + fi + latest="$(echo -e "$latest\n$version" | sort -V | tail -n1)" + echo "version=$version" | tee -a $GITHUB_OUTPUT + echo "latest=$latest" | tee -a $GITHUB_OUTPUT + echo "tag=$tag" | tee -a $GITHUB_OUTPUT + - env: + expected: ${{ steps.meta.outputs.version }} + run: | + for dir in linux_amd64_v1 darwin_amd64_v1 darwin_arm64; do + for bin in lotus lotus-miner lotus-worker; do + if [[ -f $dir/$bin ]]; then + actual=$(./$dir/$bin --version) + if [[ "$actual" != "$expected" ]]; then + echo "::error title=Version Mismatch::Expected $expected, got $actual ($dir/$bin)" + exit 1 + fi + fi + done + done - uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0 with: distribution: goreleaser-pro version: 2.0.1 args: release --clean ${{ env.PUBLISH == 'false' && '--snapshot' || '' }} - workdir: lotus env: GITHUB_TOKEN: ${{ env.PUBLISH == 'true' && secrets.GORELEASER_GITUB_TOKEN || github.token || '' }} GORELEASER_KEY: ${{ env.PUBLISH == 'true' && secrets.GORELEASER_KEY || '' }} - - env: - INPUTS_REF: ${{ inputs.ref }} - run: | - export GITHUB_REF=${INPUTS_REF:-$GITHUB_REF} - ../actions/scripts/generate-checksums.sh - working-directory: lotus + TAG: ${{ steps.meta.outputs.tag }} + VERSION: ${{ steps.meta.outputs.version }} + IS_LATEST: ${{ steps.meta.outputs.latest == steps.meta.outputs.version }} + IS_DRAFT: ${{ github.event_name == 'pull_request' }} + TARGET_COMMITISH: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.sha }} + HEADER: '' # TODO(p3): Extract release notes from CHANGELOG.md + - run: | + ./scripts/generate-checksums.sh - if: env.PUBLISH == 'true' env: GITHUB_TOKEN: ${{ github.token }} - INPUTS_REF: ${{ inputs.ref }} + TAG: ${{ steps.meta.outputs.tag }} run: | - export GITHUB_REF=${INPUTS_REF:-$GITHUB_REF} - ../actions/scripts/publish-checksums.sh - working-directory: lotus + ./scripts/publish-checksums.sh diff --git a/.goreleaser.yaml b/.goreleaser.yaml index bf620071457..684d3cef4d5 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,15 +1,10 @@ +# TODO(p0): Test if templating on boolean values is supported project_name: lotus universal_binaries: - id: lotus replace: true name_template: lotus - - id: lotus-miner - replace: true - name_template: lotus-miner - - id: lotus-worker - replace: true - name_template: lotus-worker builds: - id: lotus @@ -28,6 +23,7 @@ builds: goarch: arm64 prebuilt: path: '{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus' + skip: '{{ .Env.PROJECT != "node" }}' - id: lotus-miner binary: lotus-miner builder: prebuilt @@ -44,6 +40,7 @@ builds: goarch: arm64 prebuilt: path: '{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-miner' + skip: '{{ .Env.PROJECT != "miner" }}' - id: lotus-worker binary: lotus-worker builder: prebuilt @@ -60,12 +57,13 @@ builds: goarch: arm64 prebuilt: path: '{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-worker' + skip: '{{ .Env.PROJECT != "miner" }}' archives: - id: primary format: tar.gz wrap_in_directory: true - name_template: "{{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}" + name_template: "{{ .ProjectName }}_{{ .Env.VERSION }}_{{ .Os }}_{{ .Arch }}" files: # this is a dumb but required hack so it doesn't include the default files # https://github.com/goreleaser/goreleaser/issues/602 @@ -75,9 +73,18 @@ release: github: owner: filecoin-project name: lotus + draft: "{{ .Env.IS_DRAFT }}" + header: "{{ .Env.HEADER }}" + make_latest: "${{ .Env.IS_LATEST }}" + name_template: "{{ .Env.VERSION }}" prerelease: auto - name_template: "v{{.Version}}" + replace_existing_artifacts: true + # TODO(p0): Test if not replacing existing drafts updates them instead of failing the release + replace_existing_draft: false + tag: "{{ .Env.TAG }}" + target_commitish: "{{ .Env.TARGET_COMMITISH }}" +# TODO(p0): Figure out how to perform partial Homebrew updates brews: - tap: owner: filecoin-project @@ -106,4 +113,4 @@ checksum: disable: true snapshot: - name_template: "{{ .Version }}" + name_template: "{{ .Env.VERSION }}" diff --git a/cmd/release/main.go b/cmd/release/main.go new file mode 100644 index 00000000000..6462232cbc9 --- /dev/null +++ b/cmd/release/main.go @@ -0,0 +1,69 @@ +package main + +import ( + "os" + + log "github.com/sirupsen/logrus" + + "github.com/urfave/cli/v2" + "github.com/filecoin-project/lotus/build" +) + +func main() { + app := &cli.App{ + Name: "release", + Usage: "Lotus release tool", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "json", + Usage: "Format output as JSON", + }, + }, + Before: func(c *cli.Context) error { + if c.Bool("json") { + log.SetFormatter(&log.JSONFormatter{}) + } else { + log.SetFormatter(&log.TextFormatter{ + TimestampFormat: "2006-01-02 15:04:05", + FullTimestamp: true, + }) + } + log.SetOutput(os.Stdout) + return nil + }, + Commands: []*cli.Command{ + { + Name: "node", + Usage: "Commands related to the Lotus Node", + Subcommands: []*cli.Command{ + { + Name: "version", + Usage: "Print the Lotus Node version", + Action: func(c *cli.Context) error { + log.Info(build.NodeUserVersion()) + return nil + }, + }, + }, + }, + { + Name: "miner", + Usage: "Commands related to the Lotus Miner", + Subcommands: []*cli.Command{ + { + Name: "version", + Usage: "Print the Lotus Miner version", + Action: func(c *cli.Context) error { + log.Info(build.MinerUserVersion()) + return nil + }, + }, + }, + }, + }, + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} diff --git a/scripts/publish-checksums.sh b/scripts/publish-checksums.sh index ff80f388bdc..1dc2e41de84 100755 --- a/scripts/publish-checksums.sh +++ b/scripts/publish-checksums.sh @@ -9,8 +9,8 @@ if [ -z "${GITHUB_TOKEN}" ]; then exit 1 fi -if [[ "$GITHUB_REF" != refs/tags/* ]]; then - echo "$GITHUB_REF is not a tag, publish failed" +if [ -z "${TAG}" ]; then + echo "\${TAG} not set, publish failed" exit 1 fi @@ -23,48 +23,17 @@ do command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed" done -GITHUB_TAG="${GITHUB_REF#refs/tags/}" - #see if the release already exists by tag RELEASE_RESPONSE=` curl \ --fail \ --header "Authorization: token ${GITHUB_TOKEN}" \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${GITHUB_TAG}" + "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" ` RELEASE_ID=`echo "${RELEASE_RESPONSE}" | jq '.id'` if [ "${RELEASE_ID}" = "null" ]; then - echo "creating release" - - COND_CREATE_DISCUSSION="" - PRERELEASE=true - if [[ ${GITHUB_TAG} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - COND_CREATE_DISCUSSION="\"discussion_category_name\": \"announcement\"," - PRERELEASE=false - fi - - RELEASE_DATA="{ - \"tag_name\": \"${GITHUB_TAG}\", - \"target_commitish\": \"${GITHUB_SHA}\", - ${COND_CREATE_DISCUSSION} - \"name\": \"${GITHUB_TAG}\", - \"body\": \"\", - \"prerelease\": ${PRERELEASE} - }" - - # create it if it doesn't exist yet - RELEASE_RESPONSE=` - curl \ - --fail \ - --request POST \ - --header "Authorization: token ${GITHUB_TOKEN}" \ - --header "Content-Type: application/json" \ - --data "${RELEASE_DATA}" \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases" - ` -else - echo "release already exists" + echo "https://github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${TAG} does not exist, publish failed" fi RELEASE_UPLOAD_URL=`echo "${RELEASE_RESPONSE}" | jq -r '.upload_url' | cut -d'{' -f1` diff --git a/scripts/version-check.sh b/scripts/version-check.sh deleted file mode 100755 index 4099c800ebb..00000000000 --- a/scripts/version-check.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -set -ex - -# Validate lotus version matches the current tag -# $1 - lotus path to execute -# $2 - lotus git tag for this release -function validate_lotus_version_matches_tag(){ - # sanity checks - if [[ $# != 2 ]]; then - echo "expected 2 args for validate_lotus_version, got ${$#}" - exit 100 - fi - - # extract version from `lotus --version` response - lotus_path=$1 - # get version - lotus_raw_version=`${lotus_path} --version` - # grep for version string - lotus_actual_version=`echo ${lotus_raw_version} | grep -oE '[0-9]+\.[0-9]+\.[0-9]+'` - - # trim leading 'v' - tag=${2#v} - # trim possible -rc[0-9] - expected_version=${tag%-*} - - # check the versions are consistent - if [[ ${expected_version} != ${lotus_actual_version} ]]; then - echo "lotus version does not match build tag" - exit 101 - fi -} - -_lotus_path=$1 - -if [[ "$GITHUB_REF" == refs/tags/* ]]; then - validate_lotus_version_matches_tag "${_lotus_path}" "${GITHUB_REF#refs/tags/}" -else - echo "$GITHUB_REF is not a tag, skipping version check" -fi From c78b802975e461b3b851ab998c482eb8d1025aa8 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 21 Jun 2024 11:55:42 +0100 Subject: [PATCH 02/20] docs: document the usage and purpose of the release cmd --- cmd/release/README.md | 53 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 cmd/release/README.md diff --git a/cmd/release/README.md b/cmd/release/README.md new file mode 100644 index 00000000000..e00187643c6 --- /dev/null +++ b/cmd/release/README.md @@ -0,0 +1,53 @@ +# Lotus Release Tool + +The Lotus Release Tool is a CLI (Command Line Interface) utility designed to facilitate interactions with the Lotus Node and Miner metadata. This tool allows users to retrieve version information in either JSON or text format. The Lotus Release Tool was developed as a part of the release process automation initiative and is used in the GitHub Actions workflows related to the release process. + +## Features + +- Retrieve version information for the Lotus Node. +- Retrieve version information for the Lotus Miner. +- Output logs in JSON or text format. + +## Installation + +To install the Lotus Release Tool, you need to have Go installed on your system. + +1. Build the tool: + ```sh + go build -o release ./cmd/release + ``` + +## Usage + +The `release` tool provides several commands and options to interact with the Lotus Node and Miner. + +### Basic Commands + +- **Node Version**: Retrieve the Lotus Node version. + ```sh + ./release node version + ``` + +- **Miner Version**: Retrieve the Lotus Miner version. + ```sh + ./release miner version + ``` + +### Options + +- **--json**: Format output as JSON. + ```sh + ./release --json node version + ``` + +## Example + +Retrieve the Lotus Node version with JSON formatted output: +```sh +./release --json node version +``` + +Retrieve the Lotus Miner version with text formatted output: +```sh +./release miner version +``` From f0abec879667f088dc02cb8e207e51cf9d2ab34a Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 21 Jun 2024 12:57:38 +0100 Subject: [PATCH 03/20] docs: update the RELEASE_ISSUE_TEMPLATE to reflect the automation addition --- documentation/misc/RELEASE_ISSUE_TEMPLATE.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md index e26e06b326d..80bad63b7e0 100644 --- a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md +++ b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md @@ -22,9 +22,9 @@ - [ ] run `make gen && make docsgen-cli` - [ ] Generate changelog using the script at scripts/mkreleaselog - [ ] Add contents of generated text to lotus/CHANGELOG.md in addition to other details -- [ ] Commit using PR -- [ ] tag commit with `vX.Y.Z-rcN` -- [ ] cut a pre-release [here](https://github.com/filecoin-project/lotus/releases/new?prerelease=true) +- [ ] Commit using **PR** targetting `release/vX.Y.Z` branch + - Opening a PR should trigger a CI run that will build the release and create a draft release on GitHub + - Merging the PR should trigger a CI run that will publish the release to GitHub **Testing** @@ -32,13 +32,13 @@ Test the release candidate thoroughly, including automated and manual tests to e **Stable Release** - [ ] Final preparation - - [ ] Verify that version string in [`version.go`](https://github.com/filecoin-project/lotus/blob/master/build/version.go) has been updated. + - [ ] Verify that version string in [`version.go`](https://github.com/filecoin-project/lotus/blob/master/build/version.go) has been updated **NOT** to end with '-rcX' (in the `release/vX.Y.Z` branch). - [ ] Verify that codegen is up to date (`make gen && make docsgen-cli`) - [ ] Ensure that [CHANGELOG.md](https://github.com/filecoin-project/lotus/blob/master/CHANGELOG.md) is up to date - - [ ] Open a pull request against the `releases` branch with a merge of `release-vX.Y.Z`. - - [ ] Cut the release [here](https://github.com/filecoin-project/lotus/releases/new?prerelease=false&target=releases). - - The target should be the `releases` branch. - - Either make the tag locally and push to the `releases` branch, or allow GitHub to create a new tag via the UI when the release is published. + - [ ] Commit using **PR** targetting `release/vX.Y.Z` branch or directly to the branch + - [ ] Open a PR against the `releases` branch (base) from the `release/vX.Y.Z` branch (head) + - Opening a PR should trigger a CI run that will build the release and create a draft release on GitHub + - Merging the PR should trigger a CI run that will publish the release to GitHub **Post-Release** - [ ] Open a pull request against the `master` branch with a merge of the `releases` branch. Conflict resolution should ignore the changes to `version.go` (keep the `-dev` version from master). Do NOT delete the `releases` branch when doing so! From 97fb59501b9e92b7bbe6455676fa483406313685 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 21 Jun 2024 12:58:37 +0100 Subject: [PATCH 04/20] fix: setting the prerelease checkbox in goreleaser --- .goreleaser.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index af7fde06706..bfc99941c8d 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -81,7 +81,7 @@ release: header: "{{ .Env.HEADER }}" make_latest: "${{ .Env.IS_LATEST }}" name_template: "{{ .Env.VERSION }}" - prerelease: auto + prerelease: "${{ .Env.VERSION | strings.hasSuffix '-rc' }}" replace_existing_artifacts: true # TODO(p0): Test if not replacing existing drafts updates them instead of failing the release replace_existing_draft: false From e90f027a347aa8360cd15b899111b632be63b5d4 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 21 Jun 2024 12:58:58 +0100 Subject: [PATCH 05/20] fix: when the release workflow is triggered automatically --- .github/workflows/release.yml | 44 ++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7ebc49b13e2..a4d9a2ddf01 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,12 +4,14 @@ name: Release on: push: branches: - - master + - release/v* + - releases paths: - build/version.go pull_request: branches: - - master + - release/v* + - releases paths: - build/version.go @@ -31,22 +33,32 @@ jobs: with: fetch-depth: 0 - id: projects + env: + TARGET_REF: ${{ github.base_ref || github.ref }} run: | projects=() - node_current_version=$(go run cmd/release/main.go --json node version | jq -r .msg) - node_current_version_exists=(git tag -l 'v*' | grep -q $node_current_version) - echo "node_current_version=$node_current_version" - echo "node_current_version_exists=$node_current_version_exists" - if [[ ! $node_current_version_exists ]]; then - projects+=('node') - fi - miner_current_version=$(go run cmd/release/main.go --json miner version | jq -r .msg) - miner_current_version_exists=(git tag -l 'miner/v*' | grep -q $miner_current_version) - echo "miner_current_version=$miner_current_version" - echo "miner_current_version_exists=$miner_current_version_exists" - if [[ ! $miner_current_version_exists ]]; then - projects+=('miner') - fi + for project in node miner; do + echo "project=$project" + current_version=$(go run cmd/release/main.go --json $project version | jq -r .msg) + echo "${project}_current_version=$current_version" + if [[ $current_version =~ -dev$ ]]; then + echo "Skipping $project because it's a dev version" + continue + fi + if [[ $current_version =~ -rc$ && "$TARGET_REF" == 'refs/heads/releases' ]]; then + echo "Skipping $project because it's an RC version on the releases branch" + continue + fi + if [[ "$project" == "node" ]]; then + current_version_exists=(git tag -l 'v*' | grep -q $current_version) + else + current_version_exists=(git tag -l '$project/v*' | grep -q $current_version) + fi + echo "${project}_current_version_exists=$current_version_exists" + if [[ ! $current_version_exists ]]; then + projects+=($project) + fi + done echo "projects=$(echo "${projects[@]}" | jq -c .)" | tee -a $GITHUB_OUTPUT build: needs: [check] From ba0678470f6570fe97897c40d0c9236d13771826 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 21 Jun 2024 13:00:51 +0100 Subject: [PATCH 06/20] fix: docker workflow triggers --- .github/workflows/docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a5c154be531..a0bc21ffb63 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,6 +10,8 @@ on: pull_request: branches: - master + - release/v* + - releases schedule: - cron: '0 0 * * *' From e960dfc689068d2bef59bee3457fbf0dca7d6f9a Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 21 Jun 2024 13:03:30 +0100 Subject: [PATCH 07/20] docs: update the release issue template --- documentation/misc/RELEASE_ISSUE_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md index 80bad63b7e0..6615a9986e5 100644 --- a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md +++ b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md @@ -23,7 +23,7 @@ - [ ] Generate changelog using the script at scripts/mkreleaselog - [ ] Add contents of generated text to lotus/CHANGELOG.md in addition to other details - [ ] Commit using **PR** targetting `release/vX.Y.Z` branch - - Opening a PR should trigger a CI run that will build the release and create a draft release on GitHub + - Opening a PR should trigger a CI run that will build the release and run goreleaser in a dry/snapshot mode - Merging the PR should trigger a CI run that will publish the release to GitHub **Testing** @@ -37,7 +37,7 @@ Test the release candidate thoroughly, including automated and manual tests to e - [ ] Ensure that [CHANGELOG.md](https://github.com/filecoin-project/lotus/blob/master/CHANGELOG.md) is up to date - [ ] Commit using **PR** targetting `release/vX.Y.Z` branch or directly to the branch - [ ] Open a PR against the `releases` branch (base) from the `release/vX.Y.Z` branch (head) - - Opening a PR should trigger a CI run that will build the release and create a draft release on GitHub + - Opening a PR should trigger a CI run that will build the release and run goreleaser in a dry/snapshot mode - Merging the PR should trigger a CI run that will publish the release to GitHub **Post-Release** From 46bc7071e0cf57e5b2414451b8459c328df9a30e Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 21 Jun 2024 13:43:12 +0100 Subject: [PATCH 08/20] fix: go fmt and go mod tidy --- cmd/release/main.go | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/release/main.go b/cmd/release/main.go index 6462232cbc9..454c9386573 100644 --- a/cmd/release/main.go +++ b/cmd/release/main.go @@ -5,8 +5,8 @@ import ( log "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/build" + "github.com/urfave/cli/v2" ) func main() { diff --git a/go.mod b/go.mod index 350b7333a0e..6d043476a9b 100644 --- a/go.mod +++ b/go.mod @@ -127,6 +127,7 @@ require ( github.com/raulk/clock v1.1.0 github.com/raulk/go-watchdog v1.3.0 github.com/samber/lo v1.39.0 + github.com/sirupsen/logrus v1.9.2 github.com/stretchr/testify v1.9.0 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/triplewz/poseidon v0.0.0-20230828015038-79d8165c88ed @@ -300,7 +301,6 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v2.18.12+incompatible // indirect - github.com/sirupsen/logrus v1.9.2 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/tidwall/gjson v1.14.4 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect From 2428f629f22165e47242775dc34e9f0fcc900983 Mon Sep 17 00:00:00 2001 From: galargh Date: Wed, 26 Jun 2024 14:01:21 +0100 Subject: [PATCH 09/20] chore: update the release automation after initial testing --- .github/workflows/release.yml | 160 +++++++++++++++------------------- .goreleaser.yaml | 47 +++++----- cmd/release/main.go | 116 ++++++++++++++++++------ go.mod | 2 +- 4 files changed, 183 insertions(+), 142 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a4d9a2ddf01..599de3e90e2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,58 +36,35 @@ jobs: env: TARGET_REF: ${{ github.base_ref || github.ref }} run: | - projects=() - for project in node miner; do - echo "project=$project" - current_version=$(go run cmd/release/main.go --json $project version | jq -r .msg) - echo "${project}_current_version=$current_version" - if [[ $current_version =~ -dev$ ]]; then - echo "Skipping $project because it's a dev version" - continue - fi - if [[ $current_version =~ -rc$ && "$TARGET_REF" == 'refs/heads/releases' ]]; then - echo "Skipping $project because it's an RC version on the releases branch" - continue - fi - if [[ "$project" == "node" ]]; then - current_version_exists=(git tag -l 'v*' | grep -q $current_version) - else - current_version_exists=(git tag -l '$project/v*' | grep -q $current_version) - fi - echo "${project}_current_version_exists=$current_version_exists" - if [[ ! $current_version_exists ]]; then - projects+=($project) - fi - done - echo "projects=$(echo "${projects[@]}" | jq -c .)" | tee -a $GITHUB_OUTPUT + go run cmd/release/main.go --json list-projects | jq -r '.msg' | + jq 'map(select(.version | endswith("-dev") | not))' | + jq 'map(select(.released | not))' | + jq 'map(select(.prerelease != (env.TARGET_REF == "refs/heads/releases")))' | + jq -c '.' | xargs -I {} -0 echo "projects={}" | + tee -a $GITHUB_OUTPUT build: needs: [check] if: needs.check.outputs.projects != '[]' - name: Build ${{ matrix.project }} (${{ matrix.os }}/${{ matrix.arch }}) + name: Build ${{ matrix.project }} (${{ matrix.runner }}) runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - project: ${{ fromJSON(needs.check.outputs.projects) }} - include: - - runner: ubuntu-latest - os: Linux - arch: X64 - - runner: macos-13 - os: macOS - arch: X64 - - runner: macos-14 - os: macOS - arch: ARM64 + project: ${{ fromJSON(needs.check.outputs.projects).*.name }} + runner: + - ubuntu-latest # Linux X64 + - macos-13 # MacOs X64 + - macos-14 # MacOS ARM64 steps: - - env: - OS: ${{ matrix.os }} - ARCH: ${{ matrix.arch }} + - run: echo "Building on $RUNNER_OS/$RUNNER_ARCH" + - id: project + env: + projects: ${{ needs.check.outputs.projects }} + name: ${{ matrix.project }} run: | - if [[ "$OS" != "$RUNNER_OS" || "$ARCH" != "$RUNNER_ARCH" ]]; then - echo "::error title=Unexpected Runner::Expected $OS/$ARCH, got $RUNNER_OS/$RUNNER_ARCH" - exit 1 - fi + jq -nc 'env.projects | fromjson | map(select(.name == env.name)) | .[0]' | + xargs -I {} -0 echo "config={}" | + tee -a $GITHUB_OUTPUT - uses: actions/checkout@v4 with: submodules: 'recursive' @@ -104,15 +81,29 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: make lotus-miner lotus-worker - - if: matrix.project == 'lotus' && runner.os == 'macOS' - run: otool -hv lotus + - if: runner.os == 'macOS' + run: if [[ -f lotus ]]; then otool -hv lotus; fi + - env: + LOTUS_VERSION_IGNORE_COMMIT: 1 + expected: ${{ fromJSON(steps.project.outputs.config).version }} + run: | + for bin in lotus lotus-miner lotus-worker; do + if [[ -f ./$bin ]]; then + chmod +x ./$bin + actual=$(./$bin --version | cut -d' ' -f3) + if [[ "$actual" != "$expected" ]]; then + echo "::error title=Version Mismatch::Expected $expected, got $actual (./$bin)" + exit 1 + fi + fi + done - uses: actions/upload-artifact@v4 with: - name: lotus-${{ matrix.project }}-${{ matrix.os }}-${{ matrix.arch }} + name: lotus-${{ matrix.project }}-${{ runner.os }}-${{ runner.arch }} path: | - lotus/lotus - lotus/lotus-miner - lotus/lotus-worker + lotus + lotus-miner + lotus-worker release: needs: [check, build] if: needs.check.outputs.projects != '[]' @@ -124,10 +115,18 @@ jobs: strategy: fail-fast: false matrix: - project: ${{ fromJSON(needs.check.outputs.projects) }} + project: ${{ fromJSON(needs.check.outputs.projects).*.name }} env: PUBLISH: ${{ github.event_name != 'pull_request' }} steps: + - id: project + env: + projects: ${{ needs.check.outputs.projects }} + name: ${{ matrix.project }} + run: | + jq -nc 'env.projects | fromjson | map(select(.name == env.name)) | .[0]' | + xargs -I {} -0 echo "config={}" | + tee -a $GITHUB_OUTPUT - uses: actions/checkout@v4 with: submodules: 'recursive' @@ -149,56 +148,37 @@ jobs: with: name: kubo version: v0.16.0 - - id: meta - run: | - version="$(go run cmd/release/main.go --json ${{ matrix.project }} version | jq -r .msg)" - if [[ "${{ matrix.project}}" == "miner" ]]; then - tag="miner/$version" - latest="$(git tag -l 'miner/v*' | sort -V | tail -n1)" - elif [[ "${{ matrix.project}}" == "node" ]]; then - tag="$version" - latest="$(git tag -l 'v*' | sort -V | tail -n1)" - else - echo "::error title=Unknown Project::${{ matrix.project }}" - exit 1 - fi - latest="$(echo -e "$latest\n$version" | sort -V | tail -n1)" - echo "version=$version" | tee -a $GITHUB_OUTPUT - echo "latest=$latest" | tee -a $GITHUB_OUTPUT - echo "tag=$tag" | tee -a $GITHUB_OUTPUT - - env: - expected: ${{ steps.meta.outputs.version }} - run: | - for dir in linux_amd64_v1 darwin_amd64_v1 darwin_arm64; do - for bin in lotus lotus-miner lotus-worker; do - if [[ -f $dir/$bin ]]; then - actual=$(./$dir/$bin --version) - if [[ "$actual" != "$expected" ]]; then - echo "::error title=Version Mismatch::Expected $expected, got $actual ($dir/$bin)" - exit 1 - fi - fi - done - done + - if: github.event_name != 'pull_request' + uses: mikefarah/yq@f15500b20a1c991c8729870ba60a4dc3524b6a94 # v4.44.2 + with: + cmd: yq -i '.release.draft = false' '.goreleaser.yaml' + - if: matrix.project == 'node' + uses: mikefarah/yq@f15500b20a1c991c8729870ba60a4dc3524b6a94 # v4.44.2 + with: + cmd: yq -i '(.builds[] | select(.id == "lotus")).skip = false' '.goreleaser.yaml' + - if: matrix.project == 'miner' + uses: mikefarah/yq@f15500b20a1c991c8729870ba60a4dc3524b6a94 # v4.44.2 + with: + cmd: yq -i '(.builds[] | select(.id == "lotus-miner" or .id == "lotus-worker")).skip = false' '.goreleaser.yaml' - uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0 with: distribution: goreleaser-pro version: 2.0.1 args: release --clean ${{ env.PUBLISH == 'false' && '--snapshot' || '' }} env: - GITHUB_TOKEN: ${{ env.PUBLISH == 'true' && secrets.GORELEASER_GITUB_TOKEN || github.token || '' }} + PROJECT: ${{ matrix.project }} + GITHUB_TOKEN: ${{ env.PUBLISH == 'true' && github.token || '' }} GORELEASER_KEY: ${{ env.PUBLISH == 'true' && secrets.GORELEASER_KEY || '' }} - TAG: ${{ steps.meta.outputs.tag }} - VERSION: ${{ steps.meta.outputs.version }} - IS_LATEST: ${{ steps.meta.outputs.latest == steps.meta.outputs.version }} + TAG: ${{ fromJSON(steps.project.outputs.config).tag }} + VERSION: ${{ fromJSON(steps.project.outputs.config).version }} + IS_LATEST: ${{ fromJSON(steps.project.outputs.config).latest }} IS_DRAFT: ${{ github.event_name == 'pull_request' }} + IS_PRERELEASE: ${{ fromJSON(steps.project.outputs.config).prerelease }} TARGET_COMMITISH: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.sha }} HEADER: '' # TODO(p3): Extract release notes from CHANGELOG.md - - run: | - ./scripts/generate-checksums.sh + - run: ./scripts/generate-checksums.sh - if: env.PUBLISH == 'true' env: GITHUB_TOKEN: ${{ github.token }} - TAG: ${{ steps.meta.outputs.tag }} - run: | - ./scripts/publish-checksums.sh + TAG: ${{ fromJSON(steps.project.outputs.config).tag }} + run: ./scripts/publish-checksums.sh diff --git a/.goreleaser.yaml b/.goreleaser.yaml index bfc99941c8d..e6f3586558e 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -2,14 +2,11 @@ # TODO(p0): Test if templating on boolean values is supported version: 2 - project_name: lotus - universal_binaries: - id: lotus replace: true name_template: lotus - builds: - id: lotus binary: lotus @@ -27,7 +24,7 @@ builds: goarch: arm64 prebuilt: path: '{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus' - skip: '{{ .Env.PROJECT != "node" }}' + skip: true - id: lotus-miner binary: lotus-miner builder: prebuilt @@ -44,7 +41,7 @@ builds: goarch: arm64 prebuilt: path: '{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-miner' - skip: '{{ .Env.PROJECT != "miner" }}' + skip: true - id: lotus-worker binary: lotus-worker builder: prebuilt @@ -61,60 +58,58 @@ builds: goarch: arm64 prebuilt: path: '{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-worker' - skip: '{{ .Env.PROJECT != "miner" }}' - + skip: true archives: - id: primary format: tar.gz wrap_in_directory: true - name_template: "{{ .ProjectName }}_{{ .Env.VERSION }}_{{ .Os }}_{{ .Arch }}" + name_template: "{{ .ProjectName }}_{{ .Env.TAG }}_{{ .Os }}_{{ .Arch }}" files: # this is a dumb but required hack so it doesn't include the default files # https://github.com/goreleaser/goreleaser/issues/602 - _n_o_n_e_* - release: github: owner: filecoin-project name: lotus - draft: "{{ .Env.IS_DRAFT }}" + draft: true header: "{{ .Env.HEADER }}" - make_latest: "${{ .Env.IS_LATEST }}" - name_template: "{{ .Env.VERSION }}" - prerelease: "${{ .Env.VERSION | strings.hasSuffix '-rc' }}" + make_latest: "{{ .Env.IS_LATEST }}" + name_template: "{{ .Env.TAG }}" + prerelease: "{{ .Env.IS_PRERELEASE }}" replace_existing_artifacts: true # TODO(p0): Test if not replacing existing drafts updates them instead of failing the release replace_existing_draft: false tag: "{{ .Env.TAG }}" target_commitish: "{{ .Env.TARGET_COMMITISH }}" - -# TODO(p0): Figure out how to perform partial Homebrew updates brews: - - repository: + - name: | + {{ if eq .Env.PROJECT "node" }}lotus{{ else }}lotus-{{ .Env.PROJECT }}{{ end }}" + repository: owner: filecoin-project name: homebrew-lotus branch: master ids: - primary + custom_block: "" install: | - bin.install "lotus" - bin.install "lotus-miner" - bin.install "lotus-worker" + {{ if eq .Env.PROJECT "node" }}bin.install "lotus"{{ end }} + {{ if eq .Env.PROJECT "miner" }}bin.install "lotus-miner"{{ end }} + {{ if eq .Env.PROJECT "miner" }}bin.install "lotus-worker"{{ end }} test: | - system "#{bin}/lotus --version" - system "#{bin}/lotus-miner --version" - system "#{bin}/lotus-worker --version" + {{ if eq .Env.PROJECT "node" }}system "#{bin}/lotus --version"{{ end }} + {{ if eq .Env.PROJECT "miner" }}system "#{bin}/lotus-miner --version"{{ end }} + {{ if eq .Env.PROJECT "miner" }}system "#{bin}/lotus-worker --version"{{ end }} directory: Formula homepage: "https://filecoin.io" description: "A homebrew cask for installing filecoin-project/lotus on MacOS" license: MIT - skip_upload: auto + skip_upload: | + {{ if and (eq .Env.IS_DRAFT "false") (eq .Env.IS_LATEST "true") }}false{{ else }}true{{ end }} dependencies: - name: hwloc - # produced manually so we can include cid checksums checksum: disable: true - snapshot: - name_template: "{{ .Env.VERSION }}" + name_template: "{{ .Env.TAG }}" diff --git a/cmd/release/main.go b/cmd/release/main.go index 454c9386573..41106e6570a 100644 --- a/cmd/release/main.go +++ b/cmd/release/main.go @@ -1,14 +1,92 @@ package main import ( + "encoding/json" "os" + "os/exec" + "strings" log "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" "github.com/filecoin-project/lotus/build" "github.com/urfave/cli/v2" ) +var _tags []string + +func getTags() []string { + if _tags == nil { + output, err := exec.Command("git", "tag").Output() + if err != nil { + log.Fatal(err) + } + _tags = strings.Split(string(output), "\n") + } + return _tags +} + +func isPrerelease(version string) bool { + return semver.Prerelease("v"+version) != "" +} + +func isLatest(name, version string) bool { + if isPrerelease(version) { + return false + } + prefix := getPrefix(name) + tags := getTags() + for _, t := range tags { + if strings.HasPrefix(t, prefix) { + v := strings.TrimPrefix(t, prefix) + if !isPrerelease(v) { + if semver.Compare("v"+v, "v"+version) > 0 { + return false + } + } + } + } + return true +} + +func isReleased(tag string) bool { + tags := getTags() + for _, t := range tags { + if t == tag { + return true + } + } + return false +} + +func getPrefix(name string) string { + if name == "node" { + return "v" + } + return name + "/v" +} + +type project struct { + Name string `json:"name"` + Version string `json:"version"` + Tag string `json:"tag"` + Latest bool `json:"latest"` + Prerelease bool `json:"prerelease"` + Released bool `json:"released"` +} + +func getProject(name, version string) project { + tag := getPrefix(name) + version + return project{ + Name: name, + Version: version, + Tag: getPrefix(name) + version, + Latest: isLatest(name, version), + Prerelease: isPrerelease(version), + Released: isReleased(tag), + } +} + func main() { app := &cli.App{ Name: "release", @@ -33,31 +111,19 @@ func main() { }, Commands: []*cli.Command{ { - Name: "node", - Usage: "Commands related to the Lotus Node", - Subcommands: []*cli.Command{ - { - Name: "version", - Usage: "Print the Lotus Node version", - Action: func(c *cli.Context) error { - log.Info(build.NodeUserVersion()) - return nil - }, - }, - }, - }, - { - Name: "miner", - Usage: "Commands related to the Lotus Miner", - Subcommands: []*cli.Command{ - { - Name: "version", - Usage: "Print the Lotus Miner version", - Action: func(c *cli.Context) error { - log.Info(build.MinerUserVersion()) - return nil - }, - }, + Name: "list-projects", + Usage: "List all projects", + Action: func(c *cli.Context) error { + projects := []project{ + getProject("node", build.NodeBuildVersion), + getProject("miner", build.MinerBuildVersion), + } + b, err := json.MarshalIndent(projects, "", " ") + if err != nil { + log.Fatal(err) + } + log.Info(string(b)) + return nil }, }, }, diff --git a/go.mod b/go.mod index 68893943762..77cf1182156 100644 --- a/go.mod +++ b/go.mod @@ -150,6 +150,7 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.23.0 + golang.org/x/mod v0.17.0 golang.org/x/net v0.25.0 golang.org/x/sync v0.7.0 golang.org/x/sys v0.20.0 @@ -321,7 +322,6 @@ require ( go.uber.org/mock v0.4.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect - golang.org/x/mod v0.17.0 // indirect golang.org/x/text v0.15.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect From 21020918e33f6045c6deb52af31fffa9b07a035a Mon Sep 17 00:00:00 2001 From: galargh Date: Wed, 26 Jun 2024 14:12:21 +0100 Subject: [PATCH 10/20] docs: update README of the release tool --- cmd/release/README.md | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/cmd/release/README.md b/cmd/release/README.md index e00187643c6..92b55507608 100644 --- a/cmd/release/README.md +++ b/cmd/release/README.md @@ -1,11 +1,10 @@ # Lotus Release Tool -The Lotus Release Tool is a CLI (Command Line Interface) utility designed to facilitate interactions with the Lotus Node and Miner metadata. This tool allows users to retrieve version information in either JSON or text format. The Lotus Release Tool was developed as a part of the release process automation initiative and is used in the GitHub Actions workflows related to the release process. +The Lotus Release Tool is a CLI (Command Line Interface) utility designed to facilitate interactions with the Lotus Node and Miner metadata. This tool allows users to retrieve version information in either JSON or text format. The Lotus Release Tool was developed as a part of the [2024Q2 release process automation initiative](https://github.com/filecoin-project/lotus/issues/12010) and is used in the GitHub Actions workflows related to the release process. ## Features -- Retrieve version information for the Lotus Node. -- Retrieve version information for the Lotus Miner. +- List all projects with their expected version information. - Output logs in JSON or text format. ## Installation @@ -21,33 +20,23 @@ To install the Lotus Release Tool, you need to have Go installed on your system. The `release` tool provides several commands and options to interact with the Lotus Node and Miner. -### Basic Commands +### Commands -- **Node Version**: Retrieve the Lotus Node version. +- **List Projects**: List all projects with their version information. ```sh - ./release node version - ``` - -- **Miner Version**: Retrieve the Lotus Miner version. - ```sh - ./release miner version + ./release list-projects ``` ### Options - **--json**: Format output as JSON. ```sh - ./release --json node version + ./release --json list-projects ``` ## Example -Retrieve the Lotus Node version with JSON formatted output: -```sh -./release --json node version -``` - -Retrieve the Lotus Miner version with text formatted output: +List Lotus Node and Lotus Miner version information with JSON formatted output: ```sh -./release miner version +./release --json list-projects ``` From 253ab88dc46e9e8bfe67e87a8415be2f9b5dd1ec Mon Sep 17 00:00:00 2001 From: galargh Date: Wed, 26 Jun 2024 16:19:57 +0100 Subject: [PATCH 11/20] chore: fix go imports --- cmd/release/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/release/main.go b/cmd/release/main.go index 41106e6570a..711ab1356b2 100644 --- a/cmd/release/main.go +++ b/cmd/release/main.go @@ -7,10 +7,10 @@ import ( "strings" log "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" "golang.org/x/mod/semver" "github.com/filecoin-project/lotus/build" - "github.com/urfave/cli/v2" ) var _tags []string From 72254f81f4b9f0b7b66d3781a4be6e79f0e2796c Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 27 Jun 2024 14:30:51 +0100 Subject: [PATCH 12/20] fix: apply fixes from the testing branches --- .github/workflows/docker.yml | 11 +++-- .github/workflows/release.yml | 76 +++++++++++++++++++++++++---------- .gitignore | 3 ++ .goreleaser.yaml | 36 ++++++++--------- 4 files changed, 83 insertions(+), 43 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a0bc21ffb63..6df6d0e4a4b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,6 +1,5 @@ name: Docker -# TODO(p2): Restore worfklow_dispatch trigger on: push: branches: @@ -14,6 +13,12 @@ on: - releases schedule: - cron: '0 0 * * *' + workflow_dispatch: + inputs: + publish: + description: 'Publish the Docker image' + required: false + default: 'false' defaults: run: @@ -24,7 +29,7 @@ permissions: jobs: docker: - name: Docker (${{ matrix.image }} / ${{ matrix.network }}) [publish=${{ github.event_name != 'pull_request' }}] + name: Docker (${{ matrix.image }} / ${{ matrix.network }}) [publish=${{ inputs.publish == 'true' || github.event_name != 'pull_request' }}] runs-on: ubuntu-latest strategy: fail-fast: false @@ -40,7 +45,7 @@ jobs: - image: lotus network: mainnet env: - PUBLISH: ${{ github.event_name != 'pull_request' }} + PUBLISH: ${{ inputs.publish == 'true' || github.event_name != 'pull_request' }} steps: - id: channel env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 599de3e90e2..25612cb56d8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,5 @@ name: Release -# TODO(p2): Restore worfklow_dispatch trigger on: push: branches: @@ -14,7 +13,20 @@ on: - releases paths: - build/version.go - + workflow_dispatch: + inputs: + publish: + description: 'Publish the release' + required: false + default: 'false' + draft: + description: 'Create a draft release' + required: false + default: 'true' + target_commitish: + description: 'The commitish value that determines where the Git tag is created from' + required: false + default: '' defaults: run: shell: bash @@ -39,7 +51,7 @@ jobs: go run cmd/release/main.go --json list-projects | jq -r '.msg' | jq 'map(select(.version | endswith("-dev") | not))' | jq 'map(select(.released | not))' | - jq 'map(select(.prerelease != (env.TARGET_REF == "refs/heads/releases")))' | + jq 'map(select(.prerelease != ((env.TARGET_REF | sub("^refs/heads/"; "")) == "releases")))' | jq -c '.' | xargs -I {} -0 echo "projects={}" | tee -a $GITHUB_OUTPUT build: @@ -107,7 +119,7 @@ jobs: release: needs: [check, build] if: needs.check.outputs.projects != '[]' - name: Release [publish=${{ github.event_name != 'pull_request' }}] + name: Release ${{ matrix.project }} [publish=${{ inputs.publish == 'true' || github.event_name != 'pull_request' }}] permissions: # This enables the job to create and/or update GitHub releases contents: write @@ -117,7 +129,8 @@ jobs: matrix: project: ${{ fromJSON(needs.check.outputs.projects).*.name }} env: - PUBLISH: ${{ github.event_name != 'pull_request' }} + PUBLISH: ${{ inputs.publish == 'true' || github.event_name != 'pull_request' }} + IS_DRAFT: ${{ inputs.draft == 'true' || github.event_name == 'pull_request' }} steps: - id: project env: @@ -148,36 +161,57 @@ jobs: with: name: kubo version: v0.16.0 - - if: github.event_name != 'pull_request' - uses: mikefarah/yq@f15500b20a1c991c8729870ba60a4dc3524b6a94 # v4.44.2 - with: - cmd: yq -i '.release.draft = false' '.goreleaser.yaml' + cache: false + - name: Install yq + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + gh release --repo mikefarah/yq download v4.44.2 -p yq_linux_amd64 + sudo mv yq_linux_amd64 /usr/bin/yq + sudo chmod +x /usr/bin/yq + - if: env.IS_DRAFT == 'false' + run: yq -i '.release.draft = false' '.goreleaser.yaml' + - if: env.PUBLISH == 'true' && env.IS_DRAFT == 'false' && fromJSON(steps.project.outputs.config).latest && !fromJSON(steps.project.outputs.config).prerelease + run: yq -i '.brews.[0].skip_upload = false' '.goreleaser.yaml' - if: matrix.project == 'node' - uses: mikefarah/yq@f15500b20a1c991c8729870ba60a4dc3524b6a94 # v4.44.2 - with: - cmd: yq -i '(.builds[] | select(.id == "lotus")).skip = false' '.goreleaser.yaml' + env: + BREW_INSTALL: | + bin.install "lotus" + BREW_TEST: | + system "#{bin}/lotus --version" + run: | + yq -i '(.builds[] | select(.id == "lotus")).skip = false' '.goreleaser.yaml' + yq -i '.brews[0].install = strenv(BREW_INSTALL)' '.goreleaser.yaml' + yq -i '.brews[0].test = strenv(BREW_TEST)' '.goreleaser.yaml' - if: matrix.project == 'miner' - uses: mikefarah/yq@f15500b20a1c991c8729870ba60a4dc3524b6a94 # v4.44.2 - with: - cmd: yq -i '(.builds[] | select(.id == "lotus-miner" or .id == "lotus-worker")).skip = false' '.goreleaser.yaml' + env: + BREW_INSTALL: | + bin.install "lotus-miner" + bin.install "lotus-worker" + BREW_TEST: | + system "#{bin}/lotus-miner --version" + system "#{bin}/lotus-worker --version" + run: | + yq -i '(.builds[] | select(.id == "lotus-miner" or .id == "lotus-worker")).skip = false' '.goreleaser.yaml' + yq -i '.brews[0].install = strenv(BREW_INSTALL)' '.goreleaser.yaml' + yq -i '.brews[0].test = strenv(BREW_TEST)' '.goreleaser.yaml' - uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0 with: distribution: goreleaser-pro version: 2.0.1 - args: release --clean ${{ env.PUBLISH == 'false' && '--snapshot' || '' }} + args: release --clean --skip=validate ${{ env.PUBLISH == 'false' && '--snapshot' || '' }} env: - PROJECT: ${{ matrix.project }} - GITHUB_TOKEN: ${{ env.PUBLISH == 'true' && github.token || '' }} + PROJECT_NAME: ${{ matrix.project == 'node' && 'lotus' || format('lotus-{0}', matrix.project) }} + GITHUB_TOKEN: ${{ env.PUBLISH == 'true' && (secrets.GORELEASER_GITHUB_TOKEN || github.token) || '' }} GORELEASER_KEY: ${{ env.PUBLISH == 'true' && secrets.GORELEASER_KEY || '' }} TAG: ${{ fromJSON(steps.project.outputs.config).tag }} VERSION: ${{ fromJSON(steps.project.outputs.config).version }} IS_LATEST: ${{ fromJSON(steps.project.outputs.config).latest }} - IS_DRAFT: ${{ github.event_name == 'pull_request' }} IS_PRERELEASE: ${{ fromJSON(steps.project.outputs.config).prerelease }} - TARGET_COMMITISH: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.sha }} + TARGET_COMMITISH: ${{ inputs.target_commitish || (github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.sha) }} HEADER: '' # TODO(p3): Extract release notes from CHANGELOG.md - run: ./scripts/generate-checksums.sh - - if: env.PUBLISH == 'true' + - if: env.PUBLISH == 'true' && env.IS_DRAFT == 'false' env: GITHUB_TOKEN: ${{ github.token }} TAG: ${{ fromJSON(steps.project.outputs.config).tag }} diff --git a/.gitignore b/.gitignore index 93b79479a7e..0851a5a5adb 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,9 @@ build/builtin-actors/v* build/builtin-actors/*.car dist/ +darwin_amd64_v1/ +darwin_arm64/ +linux_amd64_v1/ # The following files are checked into git and result diff --git a/.goreleaser.yaml b/.goreleaser.yaml index e6f3586558e..98658059bf6 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -2,11 +2,17 @@ # TODO(p0): Test if templating on boolean values is supported version: 2 -project_name: lotus +project_name: "{{ .Env.PROJECT_NAME }}" universal_binaries: - id: lotus replace: true name_template: lotus + - id: lotus-miner + replace: true + name_template: lotus-miner + - id: lotus-worker + replace: true + name_template: lotus-worker builds: - id: lotus binary: lotus @@ -23,7 +29,7 @@ builds: - goos: linux goarch: arm64 prebuilt: - path: '{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus' + path: "{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus" skip: true - id: lotus-miner binary: lotus-miner @@ -40,7 +46,7 @@ builds: - goos: linux goarch: arm64 prebuilt: - path: '{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-miner' + path: "{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-miner" skip: true - id: lotus-worker binary: lotus-worker @@ -57,13 +63,13 @@ builds: - goos: linux goarch: arm64 prebuilt: - path: '{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-worker' + path: "{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-worker" skip: true archives: - id: primary format: tar.gz wrap_in_directory: true - name_template: "{{ .ProjectName }}_{{ .Env.TAG }}_{{ .Os }}_{{ .Arch }}" + name_template: "{{ .Env.PROJECT_NAME }}_v{{ .Env.VERSION }}_{{ .Os }}_{{ .Arch }}" files: # this is a dumb but required hack so it doesn't include the default files # https://github.com/goreleaser/goreleaser/issues/602 @@ -83,33 +89,25 @@ release: tag: "{{ .Env.TAG }}" target_commitish: "{{ .Env.TARGET_COMMITISH }}" brews: - - name: | - {{ if eq .Env.PROJECT "node" }}lotus{{ else }}lotus-{{ .Env.PROJECT }}{{ end }}" + - name: "{{ .Env.PROJECT_NAME }}" repository: owner: filecoin-project name: homebrew-lotus branch: master ids: - primary - custom_block: "" - install: | - {{ if eq .Env.PROJECT "node" }}bin.install "lotus"{{ end }} - {{ if eq .Env.PROJECT "miner" }}bin.install "lotus-miner"{{ end }} - {{ if eq .Env.PROJECT "miner" }}bin.install "lotus-worker"{{ end }} - test: | - {{ if eq .Env.PROJECT "node" }}system "#{bin}/lotus --version"{{ end }} - {{ if eq .Env.PROJECT "miner" }}system "#{bin}/lotus-miner --version"{{ end }} - {{ if eq .Env.PROJECT "miner" }}system "#{bin}/lotus-worker --version"{{ end }} + install: "" + test: "" directory: Formula homepage: "https://filecoin.io" description: "A homebrew cask for installing filecoin-project/lotus on MacOS" license: MIT - skip_upload: | - {{ if and (eq .Env.IS_DRAFT "false") (eq .Env.IS_LATEST "true") }}false{{ else }}true{{ end }} + skip_upload: true dependencies: - name: hwloc + commit_msg_template: "Brew formula update for {{ .Env.PROJECT_NAME }} version {{ .Env.TAG }}" # produced manually so we can include cid checksums checksum: disable: true snapshot: - name_template: "{{ .Env.TAG }}" + name_template: "{{ .Env.PROJECT_NAME }}_v{{ .Env.VERSION }}" From 2df032e823672026d58197db9003035b03dc2e35 Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 27 Jun 2024 14:45:27 +0100 Subject: [PATCH 13/20] docs: update the release issue template --- documentation/misc/RELEASE_ISSUE_TEMPLATE.md | 44 ++++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md index 6615a9986e5..b35c26027b8 100644 --- a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md +++ b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md @@ -18,32 +18,40 @@ **Prepping an RC**: -- [ ] version string in `build/version.go` needs to be updated to end with '-rcX' (in the `release/vX.Y.Z` branch) -- [ ] run `make gen && make docsgen-cli` -- [ ] Generate changelog using the script at scripts/mkreleaselog -- [ ] Add contents of generated text to lotus/CHANGELOG.md in addition to other details -- [ ] Commit using **PR** targetting `release/vX.Y.Z` branch - - Opening a PR should trigger a CI run that will build the release and run goreleaser in a dry/snapshot mode - - Merging the PR should trigger a CI run that will publish the release to GitHub +Perform the following changes to the `release/vX.Y.Z` branch through a PR: +- [ ] update the version string in `build/version.go` to one ending with '-rcX' +- [ ] run `make gen && make docsgen-cli` to generate documentation +- [ ] run `scripts/mkreleaselog` to generate changelog + - [ ] add the contents of generated changelog to `CHANGELOG.md` + - [ ] add any other details about the release to `CHANGELOG.md` +- [ ] create a **PR** targetting `release/vX.Y.Z` branch + - Opening a PR will trigger a CI run that will build the release and run goreleaser in a dry/snapshot mode + - Merging the PR will trigger a CI run that will publish the release to GitHub (including the creation of a GitHub release and a tag) +- [ ] find the newly created [GitHub release](https://github.com/filecoin-project/lotus/releases) and update its' description accordingly **Testing** Test the release candidate thoroughly, including automated and manual tests to ensure stability and functionality across various environments and scenarios. **Stable Release** - - [ ] Final preparation - - [ ] Verify that version string in [`version.go`](https://github.com/filecoin-project/lotus/blob/master/build/version.go) has been updated **NOT** to end with '-rcX' (in the `release/vX.Y.Z` branch). - - [ ] Verify that codegen is up to date (`make gen && make docsgen-cli`) - - [ ] Ensure that [CHANGELOG.md](https://github.com/filecoin-project/lotus/blob/master/CHANGELOG.md) is up to date - - [ ] Commit using **PR** targetting `release/vX.Y.Z` branch or directly to the branch - - [ ] Open a PR against the `releases` branch (base) from the `release/vX.Y.Z` branch (head) - - Opening a PR should trigger a CI run that will build the release and run goreleaser in a dry/snapshot mode - - Merging the PR should trigger a CI run that will publish the release to GitHub + +Perform the following changes to the `release/vX.Y.Z` branch (optionally, through a PR): +- [ ] update the version string in `build/version.go` to one **NOT** ending with '-rcX' +- [ ] run `make gen && make docsgen-cli` to generate documentation +- [ ] ensure `CHANGELOG.md` is up to date (run `scripts/mkreleaselog` and update `CHANGELOG.md` if needed) +- [ ] either commit the changes directly or open a PR against the `release/vX.Y.Z` branch + +Perform the following changes to the `releases` branch through a PR: +- [ ] create a **PR** targetting `releases` branch (base) from `release/vX.Y.Z` branch (head) + - Opening a PR will trigger a CI run that will build the release and run goreleaser in a dry/snapshot mode + - Merging the PR will trigger a CI run that will publish the release to GitHub (including the creation of a GitHub release and a tag) +- [ ] find the newly created [GitHub release](https://github.com/filecoin-project/lotus/releases) and update its' description accordingly **Post-Release** - - [ ] Open a pull request against the `master` branch with a merge of the `releases` branch. Conflict resolution should ignore the changes to `version.go` (keep the `-dev` version from master). Do NOT delete the `releases` branch when doing so! - - [ ] Update [RELEASE_ISSUE_TEMPLATE.md](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md) with any improvements determined from this latest release iteration. - - [ ] Create an issue using [RELEASE_ISSUE_TEMPLATE.md](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md) for the _next_ release. + +- [ ] Open a pull request against the `master` branch with a merge of the `releases` branch. Conflict resolution should ignore the changes to `version.go` (keep the `-dev` version from master). Do NOT delete the `releases` branch when doing so! +- [ ] Update [RELEASE_ISSUE_TEMPLATE.md](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md) with any improvements determined from this latest release iteration. +- [ ] Create an issue using [RELEASE_ISSUE_TEMPLATE.md](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md) for the _next_ release. ## ❤️ Contributors From 438ecba9583fd404804966505a0695849cf28635 Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 27 Jun 2024 15:04:15 +0100 Subject: [PATCH 14/20] chore: resolve release automation todos --- .github/workflows/release.yml | 2 +- .goreleaser.yaml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 25612cb56d8..9777542e0c8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -209,7 +209,7 @@ jobs: IS_LATEST: ${{ fromJSON(steps.project.outputs.config).latest }} IS_PRERELEASE: ${{ fromJSON(steps.project.outputs.config).prerelease }} TARGET_COMMITISH: ${{ inputs.target_commitish || (github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.sha) }} - HEADER: '' # TODO(p3): Extract release notes from CHANGELOG.md + HEADER: '' - run: ./scripts/generate-checksums.sh - if: env.PUBLISH == 'true' && env.IS_DRAFT == 'false' env: diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 98658059bf6..138f37a5f3e 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,5 +1,4 @@ # yaml-language-server: $schema=https://goreleaser.com/static/schema-pro.json -# TODO(p0): Test if templating on boolean values is supported version: 2 project_name: "{{ .Env.PROJECT_NAME }}" @@ -84,7 +83,6 @@ release: name_template: "{{ .Env.TAG }}" prerelease: "{{ .Env.IS_PRERELEASE }}" replace_existing_artifacts: true - # TODO(p0): Test if not replacing existing drafts updates them instead of failing the release replace_existing_draft: false tag: "{{ .Env.TAG }}" target_commitish: "{{ .Env.TARGET_COMMITISH }}" From 86f67eacd9d6d8acfcf81194e07abca064f91480 Mon Sep 17 00:00:00 2001 From: galargh Date: Tue, 9 Jul 2024 16:27:40 +0100 Subject: [PATCH 15/20] feat: ci: remove gorelease, create drafts and automate release notes --- .github/workflows/docker.yml | 5 +- .github/workflows/release.yml | 278 +++++++++++++------ .gitignore | 3 + .goreleaser.yaml | 111 -------- cmd/release/main.go | 49 +++- documentation/misc/RELEASE_ISSUE_TEMPLATE.md | 24 +- scripts/publish-checksums.sh | 56 ---- 7 files changed, 251 insertions(+), 275 deletions(-) delete mode 100644 .goreleaser.yaml delete mode 100755 scripts/publish-checksums.sh diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6df6d0e4a4b..da17456d22b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,6 +10,7 @@ on: branches: - master - release/v* + - release/miner/v* - releases schedule: - cron: '0 0 * * *' @@ -29,7 +30,7 @@ permissions: jobs: docker: - name: Docker (${{ matrix.image }} / ${{ matrix.network }}) [publish=${{ inputs.publish == 'true' || github.event_name != 'pull_request' }}] + name: Docker (${{ matrix.image }} / ${{ matrix.network }}) [publish=${{ github.event.inputs.publish == 'true' || github.event_name != 'pull_request' }}] runs-on: ubuntu-latest strategy: fail-fast: false @@ -45,7 +46,7 @@ jobs: - image: lotus network: mainnet env: - PUBLISH: ${{ inputs.publish == 'true' || github.event_name != 'pull_request' }} + PUBLISH: ${{ github.event.inputs.publish == 'true' || github.event_name != 'pull_request' }} steps: - id: channel env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9777542e0c8..9aee2b8688a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,29 +4,32 @@ on: push: branches: - release/v* + - release/miner/v* - releases paths: - build/version.go pull_request: branches: - release/v* + - release/miner/v* - releases paths: - build/version.go workflow_dispatch: inputs: + projects: + description: 'Projects to release (JSON array)' + required: false + default: '[]' + ref: + description: 'Ref to buuild the binaries from' + required: false + default: '' publish: description: 'Publish the release' required: false default: 'false' - draft: - description: 'Create a draft release' - required: false - default: 'true' - target_commitish: - description: 'The commitish value that determines where the Git tag is created from' - required: false - default: '' + defaults: run: shell: bash @@ -36,22 +39,29 @@ permissions: jobs: check: - name: Check which projects need to be built + name: Check which projects need to be released runs-on: ubuntu-latest outputs: projects: ${{ steps.projects.outputs.projects }} steps: - - uses: actions/checkout@v4 + - name: Check out lotus + uses: actions/checkout@v4 with: fetch-depth: 0 - - id: projects + ref: ${{ github.event.inputs.ref }} + - name: Find projects that need to be released + id: projects env: - TARGET_REF: ${{ github.base_ref || github.ref }} + projects: ${{ github.event.inputs.projects }} run: | go run cmd/release/main.go --json list-projects | jq -r '.msg' | jq 'map(select(.version | endswith("-dev") | not))' | jq 'map(select(.released | not))' | - jq 'map(select(.prerelease != ((env.TARGET_REF | sub("^refs/heads/"; "")) == "releases")))' | + jq 'map(select( + (env.GITHUB_EVENT_NAME == "push" and ((env.GITHUB_REF == "refs/heads/releases" and .prerelease == false) or (env.GITHUB_REF != "refs/heads/releases" and .prerelease == true))) or + (env.GITHUB_EVENT_NAME == "pull_request" and ((env.GITHUB_BASE_REF == "releases" and .prerelease == false) or (env.GITHUB_BASE_REF != "releases" and .prerelease == true))) or + (env.GITHUB_EVENT_NAME == "workflow_dispatch" and (.name as $name | env.projects | fromjson | index($name) != null)) + ))' | jq -c '.' | xargs -I {} -0 echo "projects={}" | tee -a $GITHUB_OUTPUT build: @@ -68,8 +78,10 @@ jobs: - macos-13 # MacOs X64 - macos-14 # MacOS ARM64 steps: - - run: echo "Building on $RUNNER_OS/$RUNNER_ARCH" - - id: project + - name: Print runner information + run: echo "Building on $RUNNER_OS/$RUNNER_ARCH" + - name: Make project config available + id: project env: projects: ${{ needs.check.outputs.projects }} name: ${{ matrix.project }} @@ -77,25 +89,36 @@ jobs: jq -nc 'env.projects | fromjson | map(select(.name == env.name)) | .[0]' | xargs -I {} -0 echo "config={}" | tee -a $GITHUB_OUTPUT - - uses: actions/checkout@v4 + - name: Check out lotus + uses: actions/checkout@v4 with: submodules: 'recursive' - - uses: ./.github/actions/install-system-dependencies - - uses: ./.github/actions/install-go - - env: - GITHUB_TOKEN: ${{ github.token }} - run: make deps - - if: matrix.project == 'node' - env: - GITHUB_TOKEN: ${{ github.token }} - run: make lotus - - if: matrix.project == 'miner' + - name: Install system dependencies + uses: ./.github/actions/install-system-dependencies + - name: Install Go + uses: ./.github/actions/install-go + - if: github.event.inputs.ref + uses: actions/checkout@v4 + with: + submodules: 'recursive' + ref: ${{ github.event.inputs.ref }} + - name: Build binaries env: GITHUB_TOKEN: ${{ github.token }} - run: make lotus-miner lotus-worker - - if: runner.os == 'macOS' + binaries: ${{ toJSON(fromJSON(steps.project.outputs.config).binaries) }} + run: | + make deps + while read -r binary; do + if [[ -z "$binary" ]]; then + continue + fi + make $binary + done <<< "$(jq -r '.[]' <<< "$binaries")" + - name: Run otool + if: runner.os == 'macOS' run: if [[ -f lotus ]]; then otool -hv lotus; fi - - env: + - name: Verify binary versions + env: LOTUS_VERSION_IGNORE_COMMIT: 1 expected: ${{ fromJSON(steps.project.outputs.config).version }} run: | @@ -109,7 +132,8 @@ jobs: fi fi done - - uses: actions/upload-artifact@v4 + - name: Upload artifacts + uses: actions/upload-artifact@v4 with: name: lotus-${{ matrix.project }}-${{ runner.os }}-${{ runner.arch }} path: | @@ -119,7 +143,7 @@ jobs: release: needs: [check, build] if: needs.check.outputs.projects != '[]' - name: Release ${{ matrix.project }} [publish=${{ inputs.publish == 'true' || github.event_name != 'pull_request' }}] + name: Release ${{ matrix.project }} permissions: # This enables the job to create and/or update GitHub releases contents: write @@ -128,11 +152,9 @@ jobs: fail-fast: false matrix: project: ${{ fromJSON(needs.check.outputs.projects).*.name }} - env: - PUBLISH: ${{ inputs.publish == 'true' || github.event_name != 'pull_request' }} - IS_DRAFT: ${{ inputs.draft == 'true' || github.event_name == 'pull_request' }} steps: - - id: project + - name: Make project config available + id: project env: projects: ${{ needs.check.outputs.projects }} name: ${{ matrix.project }} @@ -140,79 +162,165 @@ jobs: jq -nc 'env.projects | fromjson | map(select(.name == env.name)) | .[0]' | xargs -I {} -0 echo "config={}" | tee -a $GITHUB_OUTPUT - - uses: actions/checkout@v4 + - name: Check out lotus + uses: actions/checkout@v4 with: submodules: 'recursive' fetch-depth: 0 - - uses: actions/download-artifact@v4 + - name: Download Linux X64 binaries + uses: actions/download-artifact@v4 with: name: lotus-${{ matrix.project }}-Linux-X64 path: linux_amd64_v1 - - uses: actions/download-artifact@v4 + - name: Download macOS X64 binaries + uses: actions/download-artifact@v4 with: name: lotus-${{ matrix.project }}-macOS-X64 path: darwin_amd64_v1 - - uses: actions/download-artifact@v4 + - name: Download macOS ARM64 binaries + uses: actions/download-artifact@v4 with: name: lotus-${{ matrix.project }}-macOS-ARM64 path: darwin_arm64 - - uses: ./.github/actions/install-go - - uses: ipfs/download-ipfs-distribution-action@v1 + - name: Install Go (release body & archives generation dependency) + uses: ./.github/actions/install-go + - name: Install makefat (archives generation dependency) + run: go install github.com/randall77/makefat@7ddd0e42c8442593c87c1705a5545099604008e5 + - name: Generate archives + env: + prefix: ${{ matrix.project == 'node' && 'lotus' || format('lotus-{0}', matrix.project) }} + version: ${{ fromJSON(steps.project.outputs.config).version }} + binaries: ${{ toJSON(fromJSON(steps.project.outputs.config).binaries) }} + run: | + mkdir darwin_all + while read -r binary; do + if [[ -z "$binary" ]]; then + continue + fi + makefat ./darwin_all/$binary ./darwin_amd64_v1/$binary ./darwin_arm64/$binary + done <<< "$(jq -r '.[]' <<< "$binaries")" + mkdir dist + pushd dist + for directory in darwin_all linux_amd64_v1; do + archive_name="${prefix}_v${version}_${directory}" + cp -r ../$directory $archive_name + tar -czf $archive_name.tar.gz $archive_name + rm -r $archive_name + done + popd + ls -la dist + - name: Install Kubo (checksums generation dependency) + uses: ipfs/download-ipfs-distribution-action@v1 with: name: kubo version: v0.16.0 cache: false - - name: Install yq + - name: Generate checksums + run: | + ./scripts/generate-checksums.sh + ls -la dist + - name: Install zsh (release body generation dependency) + run: sudo apt update && sudo apt install -y zsh + - name: Generate release body env: + tag: ${{ fromJSON(steps.project.outputs.config).tag }} + previous: ${{ fromJSON(steps.project.outputs.config).previous }} + current: ${{ github.event.inputs.ref || github.sha }} + run: | + echo "# $tag" > release_body.md + echo "" >> release_body.md + csplit --digits=4 --quiet --elide-empty-files CHANGELOG.md '/^# /' '{*}' + # Checking the files in reverse order to get to the UNRELEASED section last + for file in $(ls -r xx*); do + if grep -q "^# $tag " $file || grep -q "^# UNRELEASED" $file; then + tail -n +3 $file >> release_body.md + break + fi + done + if [[ "$previous" != '' ]]; then + mkdir -p "$(go env GOPATH)/src/github.com/filecoin-project/lotus" + rm -rf "$(go env GOPATH)/src/github.com/filecoin-project/lotus" + ln -s "$(pwd)" "$(go env GOPATH)/src/github.com/filecoin-project/lotus" + ./scripts/mkreleaselog "$previous" "$current" >> release_body.md + fi + cat release_body.md + - name: Find release + id: before + env: + tag: ${{ fromJSON(steps.project.outputs.config).tag }} GITHUB_TOKEN: ${{ github.token }} run: | - gh release --repo mikefarah/yq download v4.44.2 -p yq_linux_amd64 - sudo mv yq_linux_amd64 /usr/bin/yq - sudo chmod +x /usr/bin/yq - - if: env.IS_DRAFT == 'false' - run: yq -i '.release.draft = false' '.goreleaser.yaml' - - if: env.PUBLISH == 'true' && env.IS_DRAFT == 'false' && fromJSON(steps.project.outputs.config).latest && !fromJSON(steps.project.outputs.config).prerelease - run: yq -i '.brews.[0].skip_upload = false' '.goreleaser.yaml' - - if: matrix.project == 'node' + echo "release< 0 { + previous = v + } + } + } + } + } + if previous == "" { + return "" + } + return prefix + previous +} + +func getBinaries(name string) []string { + if name == "node" { + return []string{"lotus"} + } + if name == "miner" { + return []string{"lotus-miner", "lotus-worker"} + } + return nil +} + func isReleased(tag string) bool { tags := getTags() for _, t := range tags { @@ -67,12 +100,14 @@ func getPrefix(name string) string { } type project struct { - Name string `json:"name"` - Version string `json:"version"` - Tag string `json:"tag"` - Latest bool `json:"latest"` - Prerelease bool `json:"prerelease"` - Released bool `json:"released"` + Name string `json:"name"` + Version string `json:"version"` + Tag string `json:"tag"` + Previous string `json:"previous"` + Latest bool `json:"latest"` + Prerelease bool `json:"prerelease"` + Released bool `json:"released"` + Binaries []string `json:"binaries"` } func getProject(name, version string) project { @@ -81,9 +116,11 @@ func getProject(name, version string) project { Name: name, Version: version, Tag: getPrefix(name) + version, + Previous: getPrevious(name, version), Latest: isLatest(name, version), Prerelease: isPrerelease(version), Released: isReleased(tag), + Binaries: getBinaries(name), } } diff --git a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md index b35c26027b8..4fa59b9b4b7 100644 --- a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md +++ b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md @@ -13,21 +13,17 @@ **Note for whoever is owning the release:** please capture notes as comments in this issue for anything you noticed that could be improved for future releases. There is a *Post Release* step below for incorporating changes back into the [RELEASE_ISSUE_TEMPLATE](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md), and this is easier done by collecting notes from along the way rather than just thinking about it at the end. -- [ ] Fork a new branch (`release/vX.Y.Z`) from `master` and make any further release related changes to this branch. If any "non-trivial" changes get added to the release, uncheck all the checkboxes and return to this stage. +- [ ] Fork a new branch (`release/vX.Y.Z` or `release/miner/vX.Y.Z`) from `master` and make any further release related changes to this branch. If any "non-trivial" changes get added to the release, uncheck all the checkboxes and return to this stage. - [ ] Bump the version in `build/version.go` in the `master` branch to `vX.Y.(Z+1)-dev` (bump from feature release) or `vX.(Y+1).0-dev` (bump from mandatory release). Run make gen and make docsgen-cli before committing changes **Prepping an RC**: -Perform the following changes to the `release/vX.Y.Z` branch through a PR: +Perform the following changes to the `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch through a PR: - [ ] update the version string in `build/version.go` to one ending with '-rcX' - [ ] run `make gen && make docsgen-cli` to generate documentation -- [ ] run `scripts/mkreleaselog` to generate changelog - - [ ] add the contents of generated changelog to `CHANGELOG.md` - - [ ] add any other details about the release to `CHANGELOG.md` -- [ ] create a **PR** targetting `release/vX.Y.Z` branch - - Opening a PR will trigger a CI run that will build the release and run goreleaser in a dry/snapshot mode - - Merging the PR will trigger a CI run that will publish the release to GitHub (including the creation of a GitHub release and a tag) -- [ ] find the newly created [GitHub release](https://github.com/filecoin-project/lotus/releases) and update its' description accordingly +- [ ] create a **PR** targetting `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch + - Opening a PR will trigger a CI run that will build the release and publish it to GitHub as a draft + - Merging the PR will trigger a CI run that will publish the GitHub release (it will also create a git tag) **Testing** @@ -38,14 +34,12 @@ Test the release candidate thoroughly, including automated and manual tests to e Perform the following changes to the `release/vX.Y.Z` branch (optionally, through a PR): - [ ] update the version string in `build/version.go` to one **NOT** ending with '-rcX' - [ ] run `make gen && make docsgen-cli` to generate documentation -- [ ] ensure `CHANGELOG.md` is up to date (run `scripts/mkreleaselog` and update `CHANGELOG.md` if needed) -- [ ] either commit the changes directly or open a PR against the `release/vX.Y.Z` branch +- [ ] either commit the changes directly or open a PR against the `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch Perform the following changes to the `releases` branch through a PR: -- [ ] create a **PR** targetting `releases` branch (base) from `release/vX.Y.Z` branch (head) - - Opening a PR will trigger a CI run that will build the release and run goreleaser in a dry/snapshot mode - - Merging the PR will trigger a CI run that will publish the release to GitHub (including the creation of a GitHub release and a tag) -- [ ] find the newly created [GitHub release](https://github.com/filecoin-project/lotus/releases) and update its' description accordingly +- [ ] create a **PR** targetting `releases` branch (base) from `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch (head) + - Opening a PR will trigger a CI run that will build the release and publish it to GitHub as a draft + - Merging the PR will trigger a CI run that will publish the GitHub release (it will also create a git tag) **Post-Release** diff --git a/scripts/publish-checksums.sh b/scripts/publish-checksums.sh deleted file mode 100755 index 1dc2e41de84..00000000000 --- a/scripts/publish-checksums.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -set -exo - -pushd dist - -# make sure we have a token set, api requests won't work otherwise -if [ -z "${GITHUB_TOKEN}" ]; then - echo "\${GITHUB_TOKEN} not set, publish failed" - exit 1 -fi - -if [ -z "${TAG}" ]; then - echo "\${TAG} not set, publish failed" - exit 1 -fi - -REQUIRED=( - "jq" - "curl" -) -for REQUIRE in "${REQUIRED[@]}" -do - command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed" -done - -#see if the release already exists by tag -RELEASE_RESPONSE=` - curl \ - --fail \ - --header "Authorization: token ${GITHUB_TOKEN}" \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" -` -RELEASE_ID=`echo "${RELEASE_RESPONSE}" | jq '.id'` - -if [ "${RELEASE_ID}" = "null" ]; then - echo "https://github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${TAG} does not exist, publish failed" -fi - -RELEASE_UPLOAD_URL=`echo "${RELEASE_RESPONSE}" | jq -r '.upload_url' | cut -d'{' -f1` -echo "Preparing to send artifacts to ${RELEASE_UPLOAD_URL}" - -for CHECKSUM_FILE in *.{cid,sha512} -do - echo "Uploading ${CHECKSUM_FILE}..." - curl \ - --fail \ - --request POST \ - --header "Authorization: token ${GITHUB_TOKEN}" \ - --header "Content-Type: application/octet-stream" \ - --data-binary "@${CHECKSUM_FILE}" \ - "$RELEASE_UPLOAD_URL?name=$(basename "${CHECKSUM_FILE}")" - - echo "Uploaded ${CHECKSUM_FILE}" -done - -popd From aad564f7d68a49515d9d68a12446b58bd936b24f Mon Sep 17 00:00:00 2001 From: galargh Date: Mon, 15 Jul 2024 12:06:08 +0100 Subject: [PATCH 16/20] chore: ci: allow using pat for tag creation during release --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9aee2b8688a..4792f8b2f50 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -318,7 +318,7 @@ jobs: env: release_id: ${{ fromJSON(steps.after.outputs.release).id }} make_latest: ${{ fromJSON(steps.project.outputs.config).latest }} - GITHUB_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ secrets.TAG_CREATE_GITHUB_TOKEN || github.token }} run: | gh api -X PATCH /repos/$GITHUB_REPOSITORY/releases/$release_id \ --field draft=false \ From 1bda8d95d2ac8b4c695b6f614f27b826470d2955 Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Mon, 15 Jul 2024 13:14:48 +0200 Subject: [PATCH 17/20] chore: docs: apply suggestions from code review Co-authored-by: Phi-rjan --- documentation/misc/RELEASE_ISSUE_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md index 4fa59b9b4b7..e55eca0869f 100644 --- a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md +++ b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md @@ -19,7 +19,7 @@ **Prepping an RC**: Perform the following changes to the `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch through a PR: -- [ ] update the version string in `build/version.go` to one ending with '-rcX' +- [ ] Update the version string in `build/version.go` to one ending with '-rcX'. Ensure you update the appropriate version string based on whether you are creating a node release (`NodeBuildVersion`), a miner release (`MinerBuildVersion`) or both. - [ ] run `make gen && make docsgen-cli` to generate documentation - [ ] create a **PR** targetting `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch - Opening a PR will trigger a CI run that will build the release and publish it to GitHub as a draft From c1d8638f2ce686d58156437e04988a7fa68e4731 Mon Sep 17 00:00:00 2001 From: galargh Date: Wed, 17 Jul 2024 13:47:27 +0100 Subject: [PATCH 18/20] chore: ci: stop using the releases branch --- .github/workflows/docker.yml | 1 - .github/workflows/release.yml | 6 ++---- documentation/misc/RELEASE_ISSUE_TEMPLATE.md | 7 ++----- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index da17456d22b..5ddafb63dce 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,7 +11,6 @@ on: - master - release/v* - release/miner/v* - - releases schedule: - cron: '0 0 * * *' workflow_dispatch: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4792f8b2f50..7a83c520588 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,14 +5,12 @@ on: branches: - release/v* - release/miner/v* - - releases paths: - build/version.go pull_request: branches: - release/v* - release/miner/v* - - releases paths: - build/version.go workflow_dispatch: @@ -58,8 +56,8 @@ jobs: jq 'map(select(.version | endswith("-dev") | not))' | jq 'map(select(.released | not))' | jq 'map(select( - (env.GITHUB_EVENT_NAME == "push" and ((env.GITHUB_REF == "refs/heads/releases" and .prerelease == false) or (env.GITHUB_REF != "refs/heads/releases" and .prerelease == true))) or - (env.GITHUB_EVENT_NAME == "pull_request" and ((env.GITHUB_BASE_REF == "releases" and .prerelease == false) or (env.GITHUB_BASE_REF != "releases" and .prerelease == true))) or + (env.GITHUB_EVENT_NAME == "push") or + (env.GITHUB_EVENT_NAME == "pull_request") or (env.GITHUB_EVENT_NAME == "workflow_dispatch" and (.name as $name | env.projects | fromjson | index($name) != null)) ))' | jq -c '.' | xargs -I {} -0 echo "projects={}" | diff --git a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md index ad24e6720c3..639750faf9d 100644 --- a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md +++ b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md @@ -32,13 +32,10 @@ Test the release candidate thoroughly, including automated and manual tests to e **Stable Release** -Perform the following changes to the `release/vX.Y.Z` branch (optionally, through a PR): +Perform the following changes to the `release/vX.Y.Z` branch through a PR: - [ ] update the version string in `build/version.go` to one **NOT** ending with '-rcX' - [ ] run `make gen && make docsgen-cli` to generate documentation -- [ ] either commit the changes directly or open a PR against the `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch - -Perform the following changes to the `releases` branch through a PR: -- [ ] create a **PR** targetting `releases` branch (base) from `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch (head) +- [ ] create a **PR** targetting `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch - Opening a PR will trigger a CI run that will build the release and publish it to GitHub as a draft - Merging the PR will trigger a CI run that will publish the GitHub release (it will also create a git tag) From 55d83f067f9d55421e989e5ced31afa36537c9f7 Mon Sep 17 00:00:00 2001 From: galargh Date: Wed, 17 Jul 2024 13:54:31 +0100 Subject: [PATCH 19/20] chore: ci: preserve release body --- .github/workflows/release.yml | 51 ++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a83c520588..e2f387b4c7f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -219,29 +219,6 @@ jobs: ls -la dist - name: Install zsh (release body generation dependency) run: sudo apt update && sudo apt install -y zsh - - name: Generate release body - env: - tag: ${{ fromJSON(steps.project.outputs.config).tag }} - previous: ${{ fromJSON(steps.project.outputs.config).previous }} - current: ${{ github.event.inputs.ref || github.sha }} - run: | - echo "# $tag" > release_body.md - echo "" >> release_body.md - csplit --digits=4 --quiet --elide-empty-files CHANGELOG.md '/^# /' '{*}' - # Checking the files in reverse order to get to the UNRELEASED section last - for file in $(ls -r xx*); do - if grep -q "^# $tag " $file || grep -q "^# UNRELEASED" $file; then - tail -n +3 $file >> release_body.md - break - fi - done - if [[ "$previous" != '' ]]; then - mkdir -p "$(go env GOPATH)/src/github.com/filecoin-project/lotus" - rm -rf "$(go env GOPATH)/src/github.com/filecoin-project/lotus" - ln -s "$(pwd)" "$(go env GOPATH)/src/github.com/filecoin-project/lotus" - ./scripts/mkreleaselog "$previous" "$current" >> release_body.md - fi - cat release_body.md - name: Find release id: before env: @@ -250,8 +227,34 @@ jobs: run: | echo "release<> release_body.md + else + csplit --digits=4 --quiet --elide-empty-files CHANGELOG.md '/^# /' '{*}' + # Checking the files in reverse order to get to the UNRELEASED section last + for file in $(ls -r xx*); do + if grep -q "^# $tag " $file || grep -q "^# UNRELEASED" $file; then + tail -n +3 $file >> release_body.md + break + fi + done + if [[ "$previous" != '' ]]; then + mkdir -p "$(go env GOPATH)/src/github.com/filecoin-project/lotus" + rm -rf "$(go env GOPATH)/src/github.com/filecoin-project/lotus" + ln -s "$(pwd)" "$(go env GOPATH)/src/github.com/filecoin-project/lotus" + ./scripts/mkreleaselog "$previous" "$current" >> release_body.md + fi + fi + cat release_body.md - name: Create or update release id: after env: From 914b17bdf5163ff7568ed04bc8a6ab7311a8288a Mon Sep 17 00:00:00 2001 From: galargh Date: Wed, 17 Jul 2024 16:11:35 +0100 Subject: [PATCH 20/20] chore: ci: apply fixes discovered through testing in a fork --- .github/workflows/release.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e2f387b4c7f..92ac2a06bee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -233,10 +233,10 @@ jobs: env: tag: ${{ fromJSON(steps.project.outputs.config).tag }} previous: ${{ fromJSON(steps.project.outputs.config).previous }} - body: ${{ fromJSON(steps.before.outputs.release).body }} + body: ${{ steps.before.outputs.release == 'null' && '' || fromJSON(steps.before.outputs.release).body }} current: ${{ github.event.inputs.ref || github.sha }} run: | - if [[ -z "$body" ]]; then + if [[ -n "$body" ]]; then echo "$body" >> release_body.md else csplit --digits=4 --quiet --elide-empty-files CHANGELOG.md '/^# /' '{*}' @@ -251,6 +251,8 @@ jobs: mkdir -p "$(go env GOPATH)/src/github.com/filecoin-project/lotus" rm -rf "$(go env GOPATH)/src/github.com/filecoin-project/lotus" ln -s "$(pwd)" "$(go env GOPATH)/src/github.com/filecoin-project/lotus" + echo "## Release Log" >> release_body.md + echo "" >> release_body.md ./scripts/mkreleaselog "$previous" "$current" >> release_body.md fi fi @@ -265,7 +267,7 @@ jobs: name: ${{ fromJSON(steps.project.outputs.config).tag }} draft: ${{ steps.before.outputs.release == 'null' && true || fromJSON(steps.before.outputs.release).draft }} prerelease: ${{ fromJSON(steps.project.outputs.config).prerelease }} - make_latest: ${{ fromJSON(steps.project.outputs.config).latest }} + make_latest: ${{ matrix.project == 'node' && fromJSON(steps.project.outputs.config).latest || false }} discussion_category_name: "" GITHUB_TOKEN: ${{ github.token }} run: | @@ -318,7 +320,7 @@ jobs: if: github.event.inputs.publish == 'true' || github.event_name == 'push' env: release_id: ${{ fromJSON(steps.after.outputs.release).id }} - make_latest: ${{ fromJSON(steps.project.outputs.config).latest }} + make_latest: ${{ matrix.project == 'node' && fromJSON(steps.project.outputs.config).latest || false }} GITHUB_TOKEN: ${{ secrets.TAG_CREATE_GITHUB_TOKEN || github.token }} run: | gh api -X PATCH /repos/$GITHUB_REPOSITORY/releases/$release_id \