Merge pull request #534 from DataDog/amaan.qureshi/STAL-3059

[STAL-3059] ci: rework release workflow
DataDog · Oct 29, 2024 · e7fd707 · e7fd707
2 parents 3228c65 + 597f820
commit e7fd707
Show file tree

Hide file tree

Showing 11 changed files with 196 additions and 182 deletions.
diff --git a/.github/actions/rust-cache/action.yaml b/.github/actions/rust-cache/action.yaml
diff --git a/.github/workflows/check-regressions.yml b/.github/workflows/check-regressions.yml
@@ -1,5 +1,7 @@
 on:
   push:
+    branches:
+      - "**"
 
 name: Check for regressions in static analysis kernel
 
@@ -37,13 +39,8 @@ jobs:
         with:
           ref: main
 
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-
-      - uses: ./.github/actions/rust-cache
-        with:
-          cargo-target: x86_64-unknown-linux-gnu
-          cargo-cache-key: regression-test
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
 
       - name: Checkout test repositories
         uses: actions/checkout@v4

diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml
@@ -1,10 +1,13 @@
 name: Publish to GHCR
 
 on:
-  release:
-    types:
-      - published
-  workflow_dispatch:
+  workflow_call:
+    inputs:
+      release:
+        description: "Whether this is a release build or not"
+        required: true
+        default: false
+        type: boolean
 
 env:
   REGISTRY: ghcr.io
@@ -42,9 +45,9 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           flavor: |
-            latest=false
+            latest=${{ inputs.release }}
           tags: |
-            type=raw,value=latest,enable=${{ !github.event.release.prerelease && github.event_name == 'release' }}
+            type=raw,value=latest,enable=${{ inputs.release }}
             type=ref,event=tag
             type=sha
 

diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml
@@ -1,10 +1,15 @@
+name: Run Integration tests
+
 on:
   push:
+    branches:
+      - "**"
   pull_request:
   schedule:
     # run every day at 9am UTC
-    - cron:  '0 9 * * *'
-name: Run Integration tests
+    - cron: '0 9 * * *'
+  workflow_call:
+
 jobs:
   test_scripts:
     runs-on: ubuntu-latest
@@ -27,14 +32,7 @@ jobs:
     name: Run integration test - ${{ matrix.scripts.gha_alias }}
     steps:
       - uses: actions/checkout@v4
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          toolchain: stable
-          profile: default
-      - uses: ./.github/actions/rust-cache
-        with:
-          cargo-target: x86_64-unknown-linux-gnu
-          cargo-cache-key: release-dev
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
       - name: Execute script
         run: ${{ matrix.scripts.file }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,41 +1,38 @@
 name: Release new version
 
 on:
-  release:
-    types: [created]
+  push:
+    tags:
+      - '**'
 
 permissions:
   contents: write
 
 jobs:
-  upload-versions:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-      - name: Upload versions
-        uses: softprops/action-gh-release@v2
-        with:
-          files: versions.json
-  upload-assets:
+  test-rules:
+    uses: './.github/workflows/test-rules.yaml'
+
+  integration-tests:
+    uses: './.github/workflows/integration-tests.yaml'
+    secrets: inherit
+
+  verify-schema:
+    uses: './.github/workflows/verify-schema.yaml'
+
+  versions-check:
+    uses: './.github/workflows/versions-check.yaml'
+
+  build:
     strategy:
+      fail-fast: false
       matrix:
         include:
-          # Ubuntu
-          - target: aarch64-unknown-linux-gnu
-            os: ubuntu-latest
-            image: ubuntu:20.04
-          - target: x86_64-unknown-linux-gnu
-            os: ubuntu-latest
-            image: ubuntu:20.04
-          # Mac OS
-          - target: aarch64-apple-darwin
-            os: macos-latest
-          - target: x86_64-apple-darwin
-            os: macos-13
-          # Windows
-          - target: x86_64-pc-windows-msvc
-            os: windows-latest
+          - { target: aarch64-unknown-linux-gnu, os: ubuntu-latest, image: 'ubuntu:20.04' }
+          - { target: x86_64-unknown-linux-gnu,  os: ubuntu-latest, image: 'ubuntu:20.04' }
+          - { target: aarch64-apple-darwin,      os: macos-latest                         }
+          - { target: x86_64-apple-darwin,       os: macos-13                             }
+          - { target: x86_64-pc-windows-msvc,    os: windows-latest                       }
+    name: Build binaries for ${{ matrix.target }}
     runs-on: ${{ matrix.os }}
     container:
       image: ${{ matrix.image || '' }}
@@ -46,79 +43,124 @@ jobs:
         run: |
           apt-get update
           apt-get --no-install-recommends install -y build-essential ca-certificates curl git jq wget zip
-          mkdir -p -m 755 /etc/apt/keyrings
-          wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
-          chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
-          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null
-          apt-get update
-          apt-get install gh -y
           git config --global --add safe.directory $GITHUB_WORKSPACE
-      - uses: actions/checkout@v4
-      - name: Install Stable + Rustfmt + Clippy
-        uses: dtolnay/rust-toolchain@stable
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
         with:
-          toolchain: stable
-          components: rustfmt, clippy
-        # Set the current SHA as the version so that it's exposed on the server.
+          target: ${{ matrix.target }}
+
       - name: Install cross-compilation tools
         uses: taiki-e/setup-cross-toolchain-action@v1
         with:
           target: ${{ matrix.target }}
-        if: startsWith(matrix.os, 'ubuntu')
+
+      # Set the current SHA as the version so that it's exposed on the server.
       - name: Set the version
         shell: bash
-        run: sed  "s/development/$GITHUB_SHA/g" crates/static-analysis-kernel/src/constants.rs > bla && rm crates/static-analysis-kernel/src/constants.rs && mv bla crates/static-analysis-kernel/src/constants.rs
-      - name: Release the CLI
-        uses: taiki-e/upload-rust-binary-action@v1
+        run: sed "s/development/$GITHUB_SHA/g" crates/static-analysis-kernel/src/constants.rs > bla && rm crates/static-analysis-kernel/src/constants.rs && mv bla crates/static-analysis-kernel/src/constants.rs
+
+      - name: Build Rust binaries
+        run: |
+          cargo build --release --target ${{ matrix.target }} --bin datadog-static-analyzer
+          cargo build --release --target ${{ matrix.target }} --bin datadog-static-analyzer-git-hook
+          cargo build --release --target ${{ matrix.target }} --bin datadog-static-analyzer-server
+
+      - name: Zip Rust binaries (Unix)
+        if: ${{ !startsWith(matrix.os, 'windows') }}
+        run: |
+          zip -j datadog-static-analyzer-${{ matrix.target }}.zip target/${{ matrix.target }}/release/datadog-static-analyzer
+          zip -j datadog-static-analyzer-git-hook-${{ matrix.target }}.zip target/${{ matrix.target }}/release/datadog-static-analyzer-git-hook
+          zip -j datadog-static-analyzer-server-${{ matrix.target }}.zip target/${{ matrix.target }}/release/datadog-static-analyzer-server
+
+      - name: Zip Rust binaries (Windows)
+        if: startsWith(matrix.os, 'windows')
+        run: |
+          7z a datadog-static-analyzer-${{ matrix.target }}.zip target/${{ matrix.target }}/release/datadog-static-analyzer.exe
+          7z a datadog-static-analyzer-git-hook-${{ matrix.target }}.zip target/${{ matrix.target }}/release/datadog-static-analyzer-git-hook.exe
+          7z a datadog-static-analyzer-server-${{ matrix.target }}.zip target/${{ matrix.target }}/release/datadog-static-analyzer-server.exe
+
+      - name: Upload assets
+        uses: actions/upload-artifact@v4
         with:
-          # (required) Comma-separated list of binary names (non-extension portion of filename) to build and upload.
-          # Note that glob pattern is not supported yet.
-          bin: datadog-static-analyzer
-          # (optional) On which platform to distribute the `.tar.gz` file.
-          # [default value: unix]
-          # [possible values: all, unix, windows, none]
-          tar: none
-          # (optional) On which platform to distribute the `.zip` file.
-          # [default value: windows]
-          # [possible values: all, unix, windows, none]
-          zip: all
-          target: ${{ matrix.target }}
-          # (required) GitHub token for uploading assets to GitHub Releases.
-          token: ${{ secrets.GITHUB_TOKEN }}
-          archive: 'datadog-static-analyzer-$target'
-      - name: Release datadog-static-analyzer-git-hook
-        uses: taiki-e/upload-rust-binary-action@v1
+          name: ${{ matrix.target }}
+          path: |
+            datadog-static-analyzer-${{ matrix.target }}.zip
+            datadog-static-analyzer-git-hook-${{ matrix.target }}.zip
+            datadog-static-analyzer-server-${{ matrix.target }}.zip
+          if-no-files-found: error
+          retention-days: 7
+
+  release:
+    name: Release on GitHub
+    needs:
+      - test-rules
+      - integration-tests
+      - verify-schema
+      - versions-check
+      - build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    outputs:
+      release: ${{ steps.set-release.outputs.release }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
         with:
-          # (required) Comma-separated list of binary names (non-extension portion of filename) to build and upload.
-          # Note that glob pattern is not supported yet.
-          bin: datadog-static-analyzer-git-hook
-          # (optional) On which platform to distribute the `.tar.gz` file.
-          # [default value: unix]
-          # [possible values: all, unix, windows, none]
-          tar: none
-          # (optional) On which platform to distribute the `.zip` file.
-          # [default value: windows]
-          # [possible values: all, unix, windows, none]
-          zip: all
-          target: ${{ matrix.target }}
-          # (required) GitHub token for uploading assets to GitHub Releases.
-          token: ${{ secrets.GITHUB_TOKEN }}
-          archive: 'datadog-static-analyzer-git-hook-$target'
-      - name: Release the server
-        uses: taiki-e/upload-rust-binary-action@v1
+          fetch-depth: 0
+
+      - name: Set release variable
+        id: set-release
+        run: |
+          mainCount=$(git branch -r --contains ${{ github.ref }} --format "%(refname:lstrip=3)" | grep -xc main)
+          if [[ $mainCount -eq 0 ]]; then
+            echo "Tag was not pushed onto main branch, exiting"
+            exit 1
+          elif [[ $GITHUB_REF =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "release=true" >> $GITHUB_OUTPUT
+          else
+            echo "release=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
         with:
-          # (required) Comma-separated list of binary names (non-extension portion of filename) to build and upload.
-          # Note that glob pattern is not supported yet.
-          bin: datadog-static-analyzer-server
-          # (optional) On which platform to distribute the `.tar.gz` file.
-          # [default value: unix]
-          # [possible values: all, unix, windows, none]
-          tar: none
-          # (optional) On which platform to distribute the `.zip` file.
-          # [default value: windows]
-          # [possible values: all, unix, windows, none]
-          zip: all
-          target: ${{ matrix.target }}
-          # (required) GitHub token for uploading assets to GitHub Releases.
-          token: ${{ secrets.GITHUB_TOKEN }}
-          archive: 'datadog-static-analyzer-server-$target'
+          path: artifacts
+
+      - name: Display structure of downloaded files
+        run: ls -lR
+        working-directory: artifacts
+
+      - name: Create release
+        if: ${{ steps.set-release.outputs.release == 'true' }}
+        run: |-
+          gh release create --generate-notes  \
+            ${{ github.ref_name }} \
+            versions.json \
+            artifacts/*/datadog-static-analyzer*.zip
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Create pre-release
+        if: ${{ steps.set-release.outputs.release == 'false' }}
+        run: |-
+          gh release create --generate-notes --prerelease \
+            ${{ github.ref_name }} \
+            versions.json \
+            artifacts/*/datadog-static-analyzer*.zip
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+  ghcr:
+    needs: release
+    uses: './.github/workflows/ghcr.yml'
+    with:
+      release: ${{ needs.release.outputs.release == 'true' }}
+    permissions:
+      contents: read
+      packages: write
+      id-token: write