build.yml

name: Build

on:
  push:
    branches:
      - 'master'
    paths-ignore:
      - 'docs/**'
      - 'mkdocs.yml'
  pull_request:
    branches:
      - 'master'
  workflow_dispatch:
    inputs:
      intergation-tests:
        type: boolean
        required: true
        default: false

env:
  BUILD_INCREMENT: 150
  PIP_DISABLE_PIP_VERSION_CHECK: on
  PIP_DEFAULT_TIMEOUT: 10
  PIP_PROGRESS_BAR: off

jobs:
  python-lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - run: python -m pip install pre-commit
      - run: pre-commit run -a --show-diff-on-failure

  frontend-build:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: frontend
    steps:
      - uses: actions/checkout@v4
      - name: Set up Node
        uses: actions/setup-node@v4
        with:
          node-version: 18
      - name: Install packages
        run: npm install
      - name: Build dist
        run: npm run build
      - name: Upload dist
        uses: actions/upload-artifact@v4
        with:
          name: frontend-build
          path: frontend/build

  python-test:
    needs: [ python-lint, frontend-build ]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ macos-latest, ubuntu-latest, windows-latest ]
        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: pip install -U '.[all]' -r requirements_dev.txt
      - name: Download frontend build
        uses: actions/download-artifact@v4
        with:
          name: frontend-build
          path: src/dstack/_internal/server/statics
      - name: Run pytest on POSIX
        if: matrix.os != 'windows-latest'
        # Skip Postgres tests on macos since macos runner doesn't have Docker.
        # Skip Postgres tests for Python 3.8 since testcontainers<4 doesn't support asyncpg correctly.
        run: |
          RUNPOSTGRES=""
          if [ "${{ matrix.os }}" != "macos-latest" ] && [ "${{ matrix.python-version }}" != "3.8" ]; then
            RUNPOSTGRES="--runpostgres"
          fi
          pytest src/tests --runui $RUNPOSTGRES
      - name: Run pytest on Windows
        if: matrix.os == 'windows-latest'
        run: |
          pytest src/tests --runui --runpostgres

  update-get-dstack:
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    needs: [ python-test ]
    runs-on: ubuntu-latest
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
    steps:
      - name: Install AWS
        run: pip install awscli
      - run: |
          VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }}))
          echo $VERSION | aws s3 cp - s3://get-dstack/stgn-cli/latest-version --acl public-read

  runner-test:
    defaults:
      run:
        working-directory: runner
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version-file: runner/go.mod
          cache-dependency-path: runner/go.sum
      - name: Run golangci-lint
        uses: golangci/golangci-lint-action@v6
        with:
          version: v1.58
          args: --timeout=20m
          working-directory: runner
      - name: Test
        # Do not run slow integration tests automatically.
        # Slow tests can be run manually via workflow_dispatch when required.
        run: |
          SHORT="-short"
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            if [[ "${{ github.event.inputs.intergation-tests }}" == "true" ]]; then
              SHORT=""
            fi
          fi
          go version
          go fmt $(go list ./... | grep -v /vendor/)
          go vet $(go list ./... | grep -v /vendor/)
          go test $SHORT -race $(go list ./... | grep -v /vendor/)

  runner-compile:
    needs: [runner-test]
    defaults:
      run:
        working-directory: runner
    env:
      REPO_NAME: github.com/dstackai/dstack
    strategy:
      matrix:
        include:
          - { goos: "linux", goarch: "amd64", runson: "ubuntu-latest" }
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version-file: runner/go.mod
          cache-dependency-path: runner/go.sum
      - name: build
        env:
          GOOS: ${{ matrix.goos }}
          GOARCH: ${{ matrix.goarch }}
          ACTIONSOS: ${{ matrix.runson }}
          CGO_ENABLED: 0
        run: |
          VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }}))
          go build -ldflags "-X 'main.Version=$VERSION' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner
          go build -ldflags "-X 'main.Version=$VERSION' -extldflags '-static'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim
          echo $VERSION
      - uses: actions/upload-artifact@v4
        with:
          name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
          path: |
            runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
            runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }}
          retention-days: 1

  runner-upload:
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    needs: [runner-compile]
    runs-on: ubuntu-latest
    steps:
      - name: Install AWS
        run: pip install awscli
      - name: Download Runner
        uses: actions/download-artifact@v4
        with:
          pattern: dstack-runner-*
          merge-multiple: true
          path: runner
      - name: Upload to S3
        working-directory: runner
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }}))
          aws s3 cp . "s3://dstack-runner-downloads-stgn/$VERSION/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read
          aws s3 cp . "s3://dstack-runner-downloads-stgn/latest/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read

  generate-json-schema:
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    needs: [ python-test ]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install AWS
        run: pip install awscli
      - name: Install dstack
        run: pip install .
      - name: Generate json schema
        run: |
          python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json(indent=2))" > configuration.json
          python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json(indent=2))" > profiles.json
      - name: Upload json schema to S3
        run: |
          VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }}))
          aws s3 cp configuration.json "s3://dstack-runner-downloads-stgn/$VERSION/schemas/configuration.json" --acl public-read
          aws s3 cp configuration.json "s3://dstack-runner-downloads-stgn/latest/schemas/configuration.json" --acl public-read
          aws s3 cp profiles.json "s3://dstack-runner-downloads-stgn/$VERSION/schemas/profiles.json" --acl public-read
          aws s3 cp profiles.json "s3://dstack-runner-downloads-stgn/latest/schemas/profiles.json" --acl public-read

  gateway-build:
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: gateway
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install AWS
        run: pip install awscli
      - name: Install dependencies
        run: pip install wheel build
      - name: Compute version
        run: echo VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) > $GITHUB_ENV
      - name: Build package
        run: |
          echo "__version__ = \"${{ env.VERSION }}\"" > src/dstack/gateway/version.py
          python -m build .
      - name: Upload to S3
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          WHEEL=dstack_gateway-${{ env.VERSION }}-py3-none-any.whl
          aws s3 cp dist/$WHEEL "s3://dstack-gateway-downloads/stgn/$WHEEL"
          echo "${{ env.VERSION }}" | aws s3 cp - "s3://dstack-gateway-downloads/stgn/latest-version"