Fix shim download error on instance provisioning (#1698) #3097
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| push: | |
| branches: | |
| - 'master' | |
| paths-ignore: | |
| - 'docs/**' | |
| - 'mkdocs.yml' | |
| pull_request: | |
| branches: | |
| - 'master' | |
| workflow_dispatch: | |
| inputs: | |
| intergation-tests: | |
| type: boolean | |
| required: true | |
| default: false | |
| env: | |
| BUILD_INCREMENT: 150 | |
| PIP_DISABLE_PIP_VERSION_CHECK: on | |
| PIP_DEFAULT_TIMEOUT: 10 | |
| PIP_PROGRESS_BAR: off | |
| jobs: | |
| python-lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - run: python -m pip install pre-commit | |
| - run: pre-commit run -a --show-diff-on-failure | |
| frontend-build: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: frontend | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18 | |
| - name: Install packages | |
| run: npm install | |
| - name: Build dist | |
| run: npm run build | |
| - name: Upload dist | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: frontend-build | |
| path: frontend/build | |
| python-test: | |
| needs: [ python-lint, frontend-build ] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ macos-latest, ubuntu-latest, windows-latest ] | |
| python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: pip install -U '.[all]' -r requirements_dev.txt | |
| - name: Download frontend build | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: frontend-build | |
| path: src/dstack/_internal/server/statics | |
| - name: Run pytest on POSIX | |
| if: matrix.os != 'windows-latest' | |
| # Skip Postgres tests on macos since macos runner doesn't have Docker. | |
| # Skip Postgres tests for Python 3.8 since testcontainers<4 doesn't support asyncpg correctly. | |
| run: | | |
| RUNPOSTGRES="" | |
| if [ "${{ matrix.os }}" != "macos-latest" ] && [ "${{ matrix.python-version }}" != "3.8" ]; then | |
| RUNPOSTGRES="--runpostgres" | |
| fi | |
| pytest src/tests --runui $RUNPOSTGRES | |
| - name: Run pytest on Windows | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| pytest src/tests --runui --runpostgres | |
| update-get-dstack: | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| needs: [ python-test ] | |
| runs-on: ubuntu-latest | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| steps: | |
| - name: Install AWS | |
| run: pip install awscli | |
| - run: | | |
| VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) | |
| echo $VERSION | aws s3 cp - s3://get-dstack/stgn-cli/latest-version --acl public-read | |
| runner-test: | |
| defaults: | |
| run: | |
| working-directory: runner | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: runner/go.mod | |
| cache-dependency-path: runner/go.sum | |
| - name: Run golangci-lint | |
| uses: golangci/golangci-lint-action@v6 | |
| with: | |
| version: v1.58 | |
| args: --timeout=20m | |
| working-directory: runner | |
| - name: Test | |
| # Do not run slow integration tests automatically. | |
| # Slow tests can be run manually via workflow_dispatch when required. | |
| run: | | |
| SHORT="-short" | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| if [[ "${{ github.event.inputs.intergation-tests }}" == "true" ]]; then | |
| SHORT="" | |
| fi | |
| fi | |
| go version | |
| go fmt $(go list ./... | grep -v /vendor/) | |
| go vet $(go list ./... | grep -v /vendor/) | |
| go test $SHORT -race $(go list ./... | grep -v /vendor/) | |
| runner-compile: | |
| needs: [runner-test] | |
| defaults: | |
| run: | |
| working-directory: runner | |
| env: | |
| REPO_NAME: github.com/dstackai/dstack | |
| strategy: | |
| matrix: | |
| include: | |
| - { goos: "linux", goarch: "amd64", runson: "ubuntu-latest" } | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: runner/go.mod | |
| cache-dependency-path: runner/go.sum | |
| - name: build | |
| env: | |
| GOOS: ${{ matrix.goos }} | |
| GOARCH: ${{ matrix.goarch }} | |
| ACTIONSOS: ${{ matrix.runson }} | |
| CGO_ENABLED: 0 | |
| run: | | |
| VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) | |
| go build -ldflags "-X 'main.Version=$VERSION' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner | |
| go build -ldflags "-X 'main.Version=$VERSION' -extldflags '-static'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim | |
| echo $VERSION | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} | |
| path: | | |
| runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} | |
| runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }} | |
| retention-days: 1 | |
| runner-upload: | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| needs: [runner-compile] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install AWS | |
| run: pip install awscli | |
| - name: Download Runner | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: dstack-runner-* | |
| merge-multiple: true | |
| path: runner | |
| - name: Upload to S3 | |
| working-directory: runner | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) | |
| aws s3 cp . "s3://dstack-runner-downloads-stgn/$VERSION/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read | |
| aws s3 cp . "s3://dstack-runner-downloads-stgn/latest/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read | |
| generate-json-schema: | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| needs: [ python-test ] | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Install AWS | |
| run: pip install awscli | |
| - name: Install dstack | |
| run: pip install . | |
| - name: Generate json schema | |
| run: | | |
| python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json(indent=2))" > configuration.json | |
| python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json(indent=2))" > profiles.json | |
| - name: Upload json schema to S3 | |
| run: | | |
| VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) | |
| aws s3 cp configuration.json "s3://dstack-runner-downloads-stgn/$VERSION/schemas/configuration.json" --acl public-read | |
| aws s3 cp configuration.json "s3://dstack-runner-downloads-stgn/latest/schemas/configuration.json" --acl public-read | |
| aws s3 cp profiles.json "s3://dstack-runner-downloads-stgn/$VERSION/schemas/profiles.json" --acl public-read | |
| aws s3 cp profiles.json "s3://dstack-runner-downloads-stgn/latest/schemas/profiles.json" --acl public-read | |
| gateway-build: | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: gateway | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Install AWS | |
| run: pip install awscli | |
| - name: Install dependencies | |
| run: pip install wheel build | |
| - name: Compute version | |
| run: echo VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) > $GITHUB_ENV | |
| - name: Build package | |
| run: | | |
| echo "__version__ = \"${{ env.VERSION }}\"" > src/dstack/gateway/version.py | |
| python -m build . | |
| - name: Upload to S3 | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| WHEEL=dstack_gateway-${{ env.VERSION }}-py3-none-any.whl | |
| aws s3 cp dist/$WHEEL "s3://dstack-gateway-downloads/stgn/$WHEEL" | |
| echo "${{ env.VERSION }}" | aws s3 cp - "s3://dstack-gateway-downloads/stgn/latest-version" |