From bdd8ca02e6bdd2291f2659268d2d1a98f6be63b8 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Thu, 22 Apr 2021 22:54:54 -0700 Subject: [PATCH] Release process improvements Closes #1175 Closes #1134 --- .github/ci.sh | 9 +- .github/workflows/ci.yml | 186 ++++++++++++++------------------------- 2 files changed, 66 insertions(+), 129 deletions(-) diff --git a/.github/ci.sh b/.github/ci.sh index f48171fe4..bd2148940 100755 --- a/.github/ci.sh +++ b/.github/ci.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -Eeuo pipefail +set -xEeuo pipefail [[ "$RUNNER_OS" == 'Windows' ]] && IS_WIN=true || IS_WIN=false BIN=bin @@ -38,12 +38,10 @@ retry() { } setup_external_tools() { - is_exe "$BIN" "test-runner" && return cabal v2-install --install-method=copy --installdir="$BIN" test-lib } setup_dist_bins() { - is_exe "dist/bin" "cryptol" && is_exe "dist/bin" "cryptol-html" && return extract_exe "cryptol" "dist/bin" extract_exe "cryptol-html" "dist/bin" extract_exe "cryptol-remote-api" "dist/bin" @@ -52,8 +50,6 @@ setup_dist_bins() { } install_z3() { - is_exe "$BIN" "z3" && return - case "$RUNNER_OS" in Linux) file="ubuntu-16.04" ;; macOS) file="osx-10.14.6" ;; @@ -69,7 +65,6 @@ install_z3() { } install_cvc4() { - is_exe "$BIN" "cvc4" && return version="${CVC4_VERSION#4.}" # 4.y.z -> y.z case "$RUNNER_OS" in @@ -89,7 +84,6 @@ install_cvc4() { } install_yices() { - is_exe "$BIN" "yices" && return ext=".tar.gz" case "$RUNNER_OS" in Linux) file="pc-linux-gnu-static-gmp.tar.gz" ;; @@ -132,7 +126,6 @@ install_system_deps() { } test_dist() { - setup_dist_bins setup_external_tools echo "test-runner version: $($BIN/test-runner --version)" $BIN/test-runner --ext=.icry -F -b --exe=dist/bin/cryptol tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ce62a4db..a88abfa00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,17 +1,12 @@ name: Cryptol on: push: - tags: ["v?[0-9]+.[0-9]+(.[0-9]+)?"] + tags: ["?[0-9]+.[0-9]+(.[0-9]+)?"] branches: [master, "release-**"] pull_request: schedule: - - cron: "0 0 * * *" + - cron: "0 10 * * *" # 10am UTC -> 2/3am PST workflow_dispatch: - inputs: - publish: - description: "Publish release artifacts" - required: false - default: "false" env: Z3_VERSION: "4.8.8" @@ -22,50 +17,35 @@ jobs: config: runs-on: ubuntu-latest outputs: - changed: ${{ steps.getconf.outputs.changed-files }} - cryptol-version: ${{ steps.getconf.outputs.cryptol-version }} - name: ${{ steps.getconf.outputs.name }} - publish: ${{ steps.getconf-publish.outputs.publish }} - release: ${{ steps.getconf-release.outputs.release }} - retention-days: ${{ steps.getconf-retention.outputs.retention-days }} + name: ${{ steps.config.outputs.name }} + version: ${{ steps.config.outputs.version }} + event-tag: ${{ steps.config.outputs.tag }} + event-schedule: ${{ steps.config.outputs.schedule }} + release: ${{ steps.config.outputs.release }} + retention-days: ${{ steps.config.outputs.retention-days }} steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: getconf - id: getconf - run: | - set -x - .github/ci.sh set_files ${{ github.sha }} - .github/ci.sh set_version - .github/ci.sh output name cryptol-$(.github/ci.sh ver) - - name: getconf-publish - id: getconf-publish - if: | - (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')) || - (github.event_name == 'schedule') || - (github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true') - run: | - set -x - .github/ci.sh output publish true - - name: getconf-release - id: getconf-release - if: startsWith(github.event.ref, 'refs/heads/release-') - run: | - set -x - .github/ci.sh output release true - - name: getconf-retention - id: getconf-retention + + - name: config + id: config env: - RELEASE: ${{ steps.getconf-release.outputs.release }} - shell: bash + EVENT_TAG: ${{ startsWith(github.event.ref, 'refs/tags/') }} + EVENT_SCHEDULE: ${{ github.event_name == 'schedule' }} + EVENT_DISPATCH: ${{ github.event_name == 'workflow_dispatch' }} run: | set -x - if [[ "$RELEASE" == "true" ]]; then - .github/ci.sh output retention-days 90 - else - .github/ci.sh output retention-days 5 - fi + .github/ci.sh output name cryptol-$(.github/ci.sh ver) + .github/ci.sh output version $(.github/ci.sh ver) + .github/ci.sh output tag $EVENT_TAG + .github/ci.sh output schedule $EVENT_SCHEDULE + RELEASE=$( \ + [[ "refs/heads/release-$(.github/ci.sh ver)" == "${{ github.event.ref }}" ]] && \ + [[ "refs/heads/release-$(git describe --tags --abbrev=0)" == "${{ github.event.ref }}" ]] && \ + echo true || echo false) + .github/ci.sh output release $RELEASE + .github/ci.sh output retention-days $($RELEASE && echo 90 || echo 5) build: runs-on: ${{ matrix.os }} @@ -79,6 +59,9 @@ jobs: # https://gitlab.haskell.org/ghc/ghc/-/issues/18550 - os: windows-latest ghc: 8.10.2 + env: + VERSION: ${{ needs.config.outputs.version }} + RELEASE: ${{ needs.config.outputs.release }} steps: - uses: actions/checkout@v2 with: @@ -106,39 +89,20 @@ jobs: - shell: bash run: .github/ci.sh install_system_deps - - shell: bash - run: .github/ci.sh setup_external_tools - - shell: bash env: RELEASE: ${{ needs.config.outputs.release }} run: .github/ci.sh build - shell: bash - run: .github/ci.sh check_docs - if: runner.os != 'Windows' - - - shell: bash - run: .github/ci.sh test_dist + run: .github/ci.sh setup_dist_bins - # TODO: get Python client to work on Windows - shell: bash - run: .github/ci.sh test_rpc + run: .github/ci.sh check_docs if: runner.os != 'Windows' - - if: matrix.ghc == '8.8.4' - uses: actions/upload-artifact@v2 - with: - path: dist/bin - name: ${{ runner.os }}-bins - retention-days: ${{ needs.config.outputs.retention-days }} - - build-docs: - runs-on: ubuntu-latest - needs: [config] - steps: - - uses: actions/checkout@v2 - - uses: docker://pandoc/latex:2.9.2 + - if: runner.os == 'Linux' + uses: docker://pandoc/latex:2.9.2 with: args: >- sh -c @@ -148,34 +112,6 @@ jobs: cd docs && make " - - uses: actions/upload-artifact@v2 - with: - path: docs - name: docs - retention-days: ${{ needs.config.outputs.retention-days }} - - bundle: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - needs: [config, build-docs, build] - env: - VERSION: ${{ needs.config.outputs.cryptol-version }} - RELEASE: ${{ needs.config.outputs.release }} - steps: - - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v2 - with: - path: dist/bin - name: ${{ runner.os }}-bins - - - uses: actions/download-artifact@v2 - with: - path: docs - name: docs - shell: bash run: .github/ci.sh bundle_files @@ -190,11 +126,6 @@ jobs: SIGNING_KEY: ${{ secrets.SIGNING_KEY }} run: .github/ci.sh sign cryptol.msi - - shell: bash - run: | - echo "NAME=${{ needs.config.outputs.name }}-${{ runner.os }}-x86_64" >> $GITHUB_ENV - .github/ci.sh zip_dist $NAME - - if: needs.config.outputs.release == 'true' shell: bash env: @@ -202,6 +133,11 @@ jobs: SIGNING_KEY: ${{ secrets.SIGNING_KEY }} run: .github/ci.sh sign ${NAME}.tar.gz + - shell: bash + run: | + echo "NAME=${{ needs.config.outputs.name }}-${{ runner.os }}-x86_64" >> $GITHUB_ENV + .github/ci.sh zip_dist $NAME + - uses: actions/upload-artifact@v2 with: name: ${{ env.NAME }} @@ -217,6 +153,14 @@ jobs: if-no-files-found: error retention-days: ${{ needs.config.outputs.retention-days }} + - shell: bash + run: .github/ci.sh test_dist + + # TODO: get Python client to work on Windows + - shell: bash + run: .github/ci.sh test_rpc + if: runner.os != 'Windows' + build-push-image: runs-on: ubuntu-latest needs: [config] @@ -254,45 +198,34 @@ jobs: with: images: ${{ matrix.image }} - - if: needs.config.outputs.publish == 'true' - uses: crazy-max/ghaction-docker-meta@v1 - name: Tags - id: tags - with: - images: ${{ matrix.image }} - tag-semver: | - ${{ steps.prefix.outputs.prefix }}{{version}} - ${{ steps.prefix.outputs.prefix }}{{major}}.{{minor}} - tag-schedule: | - ${{ steps.prefix.outputs.prefix }}nightly - - uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ github.actor }} - password: ${{ secrets.CR_PAT }} + password: ${{ secrets.GITHUB_TOKEN }} - uses: docker/build-push-action@v2 with: - tags: | - ${{ matrix.image }}:${{ steps.common-tag.outputs.common-tag }} - ${{ steps.tags.outputs.tags }} + tags: ${{ matrix.image }}:${{ steps.common-tag.outputs.common-tag }} labels: ${{ steps.labels.outputs.labels }} load: true push: false file: ${{ matrix.file }} build-args: ${{ matrix.build-args }} cache-from: | - type=registry,ref=${{ matrix.cache }}:${{ steps.prefix.outputs.prefix }}master - type=registry,ref=${{ matrix.cache }}:${{ steps.common-tag.outputs.common-tag }} + type=registry,ref=${{ matrix.cache }}:cache-${{ steps.prefix.outputs.prefix }}master + type=registry,ref=${{ matrix.cache }}:cache-${{ steps.common-tag.outputs.common-tag }} - name: Cache image build uses: docker/build-push-action@v2 continue-on-error: true # Tolerate cache upload failures - this should be handled better with: + tags: ${{ matrix.cache }}:${{ steps.common-tag.outputs.common-tag }} + labels: ${{ steps.labels.outputs.labels }} + push: true file: ${{ matrix.file }} build-args: ${{ matrix.build-args }} - cache-to: type=registry,ref=${{ matrix.cache }}:${{ steps.common-tag.outputs.common-tag }},mode=max + cache-to: type=registry,ref=${{ matrix.cache }}:cache-${{ steps.common-tag.outputs.common-tag }},mode=max - if: matrix.image == 'ghcr.io/galoisinc/cryptol-remote-api' uses: actions/checkout@v2 @@ -343,5 +276,16 @@ jobs: --restart=Never \ -- socket cra-socket-cryptol-remote-api 8080 - - if: needs.config.outputs.publish == 'true' - run: docker push --all-tags ${{ matrix.image }} + - if: needs.config.outputs.event-schedule == 'true' + name: ${{ matrix.image }}:nightly + run: | + docker tag ${{ matrix.image }}:$COMMON_TAG ${{ matrix.image }}:nightly + docker push ${{ matrix.image }}:nightly + + - if: needs.config.outputs.release == 'true' + name: ${{ matrix.image }}:${{ needs.config.outputs.version }} + run: | + docker tag ${{ matrix.image }}:$COMMON_TAG ${{ matrix.image }}:${{ needs.config.outputs.version }} + docker push ${{ matrix.image }}:${{ needs.config.outputs.version }} + docker tag ${{ matrix.image }}:$COMMON_TAG ${{ matrix.image }}:latest + docker push ${{ matrix.image }}:latest