From d07999231ccc699bba4281b89b4c0e15151c5885 Mon Sep 17 00:00:00 2001 From: Erik Sipsma Date: Thu, 15 Dec 2022 20:22:04 -0800 Subject: [PATCH] Add cli publish from main and tests against it. Signed-off-by: Erik Sipsma --- .github/workflows/provision.yml | 62 ------ .github/workflows/publish.yml | 222 ++++++++++++++++++- .goreleaser.common.yml | 27 +++ .goreleaser.nightly.yml | 22 ++ .goreleaser.yml | 30 +-- internal/engine/version.go | 22 +- internal/mage/cli.go | 27 ++- internal/mage/sdk/go.go | 2 + internal/mage/sdk/nodejs.go | 5 +- internal/mage/sdk/python.go | 2 + sdk/go/internal/engineconn/cli.go | 47 ++-- sdk/go/provision_test.go | 32 +-- sdk/nodejs/provisioning/bin.ts | 36 +-- sdk/nodejs/test/connect.spec.ts | 32 ++- sdk/python/src/dagger/engine/bin.py | 18 +- sdk/python/tests/engine/test_download_bin.py | 27 ++- 16 files changed, 445 insertions(+), 168 deletions(-) delete mode 100644 .github/workflows/provision.yml create mode 100644 .goreleaser.common.yml create mode 100644 .goreleaser.nightly.yml diff --git a/.github/workflows/provision.yml b/.github/workflows/provision.yml deleted file mode 100644 index 917ec4116a..0000000000 --- a/.github/workflows/provision.yml +++ /dev/null @@ -1,62 +0,0 @@ -# name: provision -# on: -# # Enable manual trigger for easy debugging -# # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_dispatchinputs -# workflow_dispatch: - -# # We want to trigger this workflow when our engine package gets updated: -# # https://github.com/dagger/dagger/pkgs/container/engine -# registry_package: -# # ⚠️ Differs to the official docs: -# # ✅ https://github.com/orgs/community/discussions/25123 -# # ❌ https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#registry_package -# action: ["updated"] - -# jobs: -# macos: -# runs-on: macos-latest -# steps: -# - name: "Install Docker" -# run: | -# echo "Install docker CLI..." -# brew install docker -# echo "Start Docker daemon via Colima..." -# echo "⚠️ Use mount-type 9p so that launched containers can chown: https://github.com/abiosoft/colima/issues/54#issuecomment-1250217077" -# colima start --mount-type 9p - -# - uses: actions/checkout@v3 - -# - uses: actions/setup-go@v3 -# with: -# go-version: "1.19" - -# - name: "Test Engine provisioning for Go SDK" -# run: | -# cd sdk/go -# go test -run="TestImageProvision" -v ./... - -# - uses: actions/setup-python@v4 -# with: -# python-version: "3.10" - -# # https://github.com/python-poetry/poetry/blob/dcd48c8df6d22246c21c0243fd387e3a9b189f93/.github/workflows/main.yml -# - name: "Bootstrap Poetry" -# run: | -# curl -sL https://install.python-poetry.org | python - -y - -# - name: "Add Poetry & deps to PATH" -# run: echo "$HOME/.local/bin" >> $GITHUB_PATH - -# - name: "Test Engine provisioning for Python SDK" -# run: | -# cd sdk/python -# poetry install -# poetry run poe test -xm provision - -# - name: "ALWAYS print engine logs - especially useful on failure" -# if: always() -# run: docker logs $(docker ps -q --filter name=dagger-engine) - -# - name: "ALWAYS print kernel logs - especially useful on failure" -# if: always() -# run: sudo dmesg diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5cf578cead..903354c608 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,11 +1,21 @@ -name: publish +name: Publish CLI & Engine on: push: branches: ["main"] tags: ["v**"] + + # Run tests in a PR when an SDK has a default CLI version bump + pull_request: + paths: + - sdk/go/internal/engineconn/version.gen.go + - sdk/python/src/dagger/_version.py + - sdk/nodejs/provisioning/default.ts + jobs: publish: runs-on: ubuntu-latest + # only run this on push events, not in PRs + if: github.event_name == 'push' steps: - uses: actions/checkout@v3 @@ -29,6 +39,7 @@ jobs: AWS_BUCKET: ${{ secrets.RELEASE_AWS_BUCKET }} ARTEFACTS_FQDN: ${{ secrets.RELEASE_FQDN }} HOMEBREW_TAP_OWNER: ${{ secrets.RELEASE_HOMEBREW_TAP_OWNER }} + GORELEASER_KEY: ${{ secrets.GORELEASER_PRO_LICENSE_KEY }} run: ./hack/make dagger:publish ${{ github.ref_name }} - name: "Bump SDK Engine Dependencies" @@ -46,3 +57,212 @@ jobs: This PR was auto-generated. delete-branch: true branch: bump-engine + + test-provision-macos: + name: "Test SDK Provision / macos" + # We want to test the SDKs in a CLI dependency bump PR, in which case publish + # has to be skipped, AND after every push to main/tags, in which case publish + # must run first. This is unfortunately quite annoying to express in yaml... + # https://github.com/actions/runner/issues/491#issuecomment-850884422 + needs: [publish] + if: | + always() && + (needs.publish.result == 'success' || needs.publish.result == 'skipped') + runs-on: macos-latest + env: + _INTERNAL_DAGGER_TEST_CLI_URL: ${{ needs.publish.outputs.baseurl }}/dagger_${{ needs.publish.outputs.version }}_darwin_amd64.tar.gz + steps: + - name: "Set CLI Test URL" + run: | + if [ ${{ github.event_name }} == 'push' ]; then + if [ $GITHUB_REF_NAME == 'main' ]; then + CLI_VERSION=$GITHUB_SHA + ENGINE_VERSION='main' + else + # this is a tag push, trim the leading v from vx.y.z + CLI_VERSION=${GITHUB_REF_NAME:1} + ENGINE_VERSION=$GITHUB_REF_NAME + fi + echo "_INTERNAL_DAGGER_TEST_CLI_URL=https://${{ secrets.RELEASE_FQDN }}/dagger/releases/${CLI_VERSION}/dagger_${CLI_VERSION}_darwin_amd64.tar.gz" >> $GITHUB_ENV + echo "_INTERNAL_DAGGER_TEST_CLI_CHECKSUMS_URL=https://${{ secrets.RELEASE_FQDN }}/dagger/releases/${CLI_VERSION}/checksums.txt" >> $GITHUB_ENV + echo "_EXPERIMENTAL_DAGGER_RUNNER_HOST=docker-image://${{ secrets.RELEASE_DAGGER_ENGINE_IMAGE }}:${ENGINE_VERSION}" >> $GITHUB_ENV + fi + shell: bash + - name: "Install Docker" + run: | + echo "Install docker CLI..." + brew install docker + echo "Start Docker daemon via Colima..." + echo "⚠️ Use mount-type 9p so that launched containers can chown: https://github.com/abiosoft/colima/issues/54#issuecomment-1250217077" + colima start --mount-type 9p + + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v3 + with: + go-version: "1.19" + + - name: "Test Go SDK" + run: | + cd sdk/go + go test -v -run TestProvision ./... + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + # https://github.com/python-poetry/poetry/blob/dcd48c8df6d22246c21c0243fd387e3a9b189f93/.github/workflows/main.yml + - name: "Bootstrap Poetry" + run: | + curl -sL https://install.python-poetry.org | python - -y + + - name: "Add Poetry & deps to PATH" + run: echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: "Test Python SDK" + run: | + cd sdk/python + poetry install + poetry run poe test -xm provision + + - uses: actions/setup-node@v2 + with: + node-version: 16 + + - name: "Test NodeJS SDK" + run: | + cd sdk/nodejs + yarn install + yarn test -g 'Automatic Provisioned CLI Binary' + + - name: "ALWAYS print engine logs - especially useful on failure" + if: always() + run: docker logs $(docker ps -q --filter name=dagger-engine) + + - name: "ALWAYS print kernel logs - especially useful on failure" + if: always() + run: sudo dmesg + + test-provision-go-linux: + name: "Test SDK Provision / go / linux" + needs: [publish] + if: | + always() && + (needs.publish.result == 'success' || needs.publish.result == 'skipped') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: "Set CLI Test URL" + run: | + if [ ${{ github.event_name }} == 'push' ]; then + if [ $GITHUB_REF_NAME == 'main' ]; then + CLI_VERSION=$GITHUB_SHA + ENGINE_VERSION='main' + else + # this is a tag push, trim the leading v from vx.y.z + CLI_VERSION=${GITHUB_REF_NAME:1} + ENGINE_VERSION=$GITHUB_REF_NAME + fi + echo "_INTERNAL_DAGGER_TEST_CLI_URL=https://${{ secrets.RELEASE_FQDN }}/dagger/releases/${CLI_VERSION}/dagger_${CLI_VERSION}_linux_amd64.tar.gz" >> $GITHUB_ENV + echo "_INTERNAL_DAGGER_TEST_CLI_CHECKSUMS_URL=https://${{ secrets.RELEASE_FQDN }}/dagger/releases/${CLI_VERSION}/checksums.txt" >> $GITHUB_ENV + echo "_EXPERIMENTAL_DAGGER_RUNNER_HOST=docker-image://${{ secrets.RELEASE_DAGGER_ENGINE_IMAGE }}:${ENGINE_VERSION}" >> $GITHUB_ENV + fi + shell: bash + - uses: actions/setup-go@v3 + with: + go-version: 1.19 + - name: "Test Go SDK" + run: | + cd sdk/go + go test -v -run TestProvision ./... + - name: "ALWAYS print engine logs - especially useful on failure" + if: always() + run: docker logs $(docker ps -q --filter name=dagger-engine) + - name: "ALWAYS print kernel logs - especially useful on failure" + if: always() + run: sudo dmesg + + test-provision-python-linux: + name: "Test SDK Provision / python / linux" + needs: [publish] + if: | + always() && + (needs.publish.result == 'success' || needs.publish.result == 'skipped') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: "Set CLI Test URL" + run: | + if [ ${{ github.event_name }} == 'push' ]; then + if [ $GITHUB_REF_NAME == 'main' ]; then + CLI_VERSION=$GITHUB_SHA + ENGINE_VERSION='main' + else + # this is a tag push, trim the leading v from vx.y.z + CLI_VERSION=${GITHUB_REF_NAME:1} + ENGINE_VERSION=$GITHUB_REF_NAME + fi + echo "_INTERNAL_DAGGER_TEST_CLI_URL=https://${{ secrets.RELEASE_FQDN }}/dagger/releases/${CLI_VERSION}/dagger_${CLI_VERSION}_linux_amd64.tar.gz" >> $GITHUB_ENV + echo "_INTERNAL_DAGGER_TEST_CLI_CHECKSUMS_URL=https://${{ secrets.RELEASE_FQDN }}/dagger/releases/${CLI_VERSION}/checksums.txt" >> $GITHUB_ENV + echo "_EXPERIMENTAL_DAGGER_RUNNER_HOST=docker-image://${{ secrets.RELEASE_DAGGER_ENGINE_IMAGE }}:${ENGINE_VERSION}" >> $GITHUB_ENV + fi + shell: bash + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: "Bootstrap Poetry" + run: | + curl -sL https://install.python-poetry.org | python - -y + - name: "Add Poetry & deps to PATH" + run: echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: "Test Python SDK" + run: | + cd sdk/python + poetry install + poetry run poe test -xm provision + - name: "ALWAYS print engine logs - especially useful on failure" + if: always() + run: docker logs $(docker ps -q --filter name=dagger-engine) + - name: "ALWAYS print kernel logs - especially useful on failure" + if: always() + run: sudo dmesg + + test-provision-nodejs-linux: + name: "Test SDK Provision / nodejs / linux" + needs: [publish] + if: | + always() && + (needs.publish.result == 'success' || needs.publish.result == 'skipped') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: "Set CLI Test URL" + run: | + if [ ${{ github.event_name }} == 'push' ]; then + if [ $GITHUB_REF_NAME == 'main' ]; then + CLI_VERSION=$GITHUB_SHA + ENGINE_VERSION='main' + else + # this is a tag push, trim the leading v from vx.y.z + CLI_VERSION=${GITHUB_REF_NAME:1} + ENGINE_VERSION=$GITHUB_REF_NAME + fi + echo "_INTERNAL_DAGGER_TEST_CLI_URL=https://${{ secrets.RELEASE_FQDN }}/dagger/releases/${CLI_VERSION}/dagger_${CLI_VERSION}_linux_amd64.tar.gz" >> $GITHUB_ENV + echo "_INTERNAL_DAGGER_TEST_CLI_CHECKSUMS_URL=https://${{ secrets.RELEASE_FQDN }}/dagger/releases/${CLI_VERSION}/checksums.txt" >> $GITHUB_ENV + echo "_EXPERIMENTAL_DAGGER_RUNNER_HOST=docker-image://${{ secrets.RELEASE_DAGGER_ENGINE_IMAGE }}:${ENGINE_VERSION}" >> $GITHUB_ENV + fi + shell: bash + - uses: actions/setup-node@v2 + with: + node-version: 16 + - name: "Test NodeJS SDK" + run: | + cd sdk/nodejs + yarn install + yarn test -g 'Automatic Provisioned CLI Binary' + - name: "ALWAYS print engine logs - especially useful on failure" + if: always() + run: docker logs $(docker ps -q --filter name=dagger-engine) + - name: "ALWAYS print kernel logs - especially useful on failure" + if: always() + run: sudo dmesg diff --git a/.goreleaser.common.yml b/.goreleaser.common.yml new file mode 100644 index 0000000000..00eb12177f --- /dev/null +++ b/.goreleaser.common.yml @@ -0,0 +1,27 @@ +project_name: dagger + +before: + hooks: + - go mod download + +builds: + - env: + - CGO_ENABLED=0 + main: ./cmd/dagger + binary: dagger + ldflags: + - -s -w + - -X github.com/dagger/dagger/internal/engine.Version={{.Version}} + goos: + - linux + - windows + - darwin + goarch: + - amd64 + - arm64 + - arm + goarm: + - "7" + +checksum: + name_template: "checksums.txt" diff --git a/.goreleaser.nightly.yml b/.goreleaser.nightly.yml new file mode 100644 index 0000000000..0685d1dba6 --- /dev/null +++ b/.goreleaser.nightly.yml @@ -0,0 +1,22 @@ +includes: + - from_file: + path: ./.goreleaser.common.yml + +nightly: + # name_template will override .Version for nightly builds: + # https://goreleaser.com/customization/nightlies/#how-it-works + name_template: "{{ .FullCommit }}" + +archives: + - name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ if .Mips }}_{{ .Mips }}{{ end }}" + files: + - LICENSE + format_overrides: + - goos: windows + format: zip + +blobs: + - provider: s3 + region: "{{ .Env.AWS_REGION }}" + bucket: "{{ .Env.AWS_BUCKET }}" + folder: "dagger/releases/{{ .Version }}" diff --git a/.goreleaser.yml b/.goreleaser.yml index 17051e1443..e576bdfacc 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,27 +1,6 @@ -project_name: dagger - -before: - hooks: - - go mod download - -builds: - - env: - - CGO_ENABLED=0 - main: ./cmd/dagger - binary: dagger - ldflags: - - -s -w - - -X github.com/dagger/dagger/internal/engine.Version={{.Version}} - goos: - - linux - - windows - - darwin - goarch: - - amd64 - - arm64 - - arm - goarm: - - "7" +includes: + - from_file: + path: ./.goreleaser.common.yml archives: - name_template: "{{ .ProjectName }}_{{ .Tag }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ if .Mips }}_{{ .Mips }}{{ end }}" @@ -31,9 +10,6 @@ archives: - goos: windows format: zip -checksum: - name_template: "checksums.txt" - snapshot: name_template: "{{ .Tag }}-next" diff --git a/internal/engine/version.go b/internal/engine/version.go index 9f3d6483e5..b52f062889 100644 --- a/internal/engine/version.go +++ b/internal/engine/version.go @@ -3,6 +3,8 @@ package engine import ( "fmt" "os" + + "golang.org/x/mod/semver" ) const ( @@ -14,12 +16,22 @@ const ( var Version = DevelopmentVersion func ImageRef() string { - // if this is a release, use the release image - if Version != DevelopmentVersion { - return fmt.Sprintf("%s:v%s", EngineImageRepo, Version) + // If "devel" is set, then this is a local build. Normally _EXPERIMENTAL_DAGGER_RUNNER_HOST + // should be set to point to a runner built from local code, but we default to using "main" + // in case it's not. + if Version == DevelopmentVersion { + return fmt.Sprintf("%s:main", EngineImageRepo) + } + + // If Version is set to something besides a semver tag, then it's a build off our main branch. + // For now, this also defaults to using the "main" tag, but in the future if we tag engine + // images with git sha then we could use that instead + if semver.IsValid(Version) { + return fmt.Sprintf("%s:main", EngineImageRepo) } - // fallback to using the latest image from main - return fmt.Sprintf("%s:main", EngineImageRepo) + + // Version is a semver tag, so use the engine image at that tag + return fmt.Sprintf("%s:v%s", EngineImageRepo, Version) } func RunnerHost() string { diff --git a/internal/mage/cli.go b/internal/mage/cli.go index f21d54af49..e02097dbe6 100644 --- a/internal/mage/cli.go +++ b/internal/mage/cli.go @@ -2,7 +2,6 @@ package mage import ( "context" - "fmt" "os" "dagger.io/dagger" @@ -15,10 +14,8 @@ type Cli mg.Namespace // Publish publishes dagger CLI using GoReleaser func (cl Cli) Publish(ctx context.Context, version string) error { - if !semver.IsValid(version) { - fmt.Printf("'%s' is not a semver version, skipping CLI publish", version) - return nil - } + // if this isn't an official semver version, do a nightly release + nightly := !semver.IsValid(version) c, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr)) if err != nil { @@ -28,13 +25,13 @@ func (cl Cli) Publish(ctx context.Context, version string) error { wd := c.Host().Directory(".") container := c.Container(). - From("ghcr.io/goreleaser/goreleaser:v1.12.3"). + From("ghcr.io/goreleaser/goreleaser-pro:v1.12.3-pro"). WithEntrypoint([]string{}). WithExec([]string{"apk", "add", "aws-cli"}). - WithEntrypoint([]string{"/sbin/tini", "--", "/entrypoint.sh"}). WithWorkdir("/app"). WithMountedDirectory("/app", wd). WithSecretVariable("GITHUB_TOKEN", util.WithSetHostVar(ctx, c.Host(), "GITHUB_TOKEN").Secret()). + WithSecretVariable("GORELEASER_KEY", util.WithSetHostVar(ctx, c.Host(), "GORELEASER_KEY").Secret()). WithSecretVariable("AWS_ACCESS_KEY_ID", util.WithSetHostVar(ctx, c.Host(), "AWS_ACCESS_KEY_ID").Secret()). WithSecretVariable("AWS_SECRET_ACCESS_KEY", util.WithSetHostVar(ctx, c.Host(), "AWS_SECRET_ACCESS_KEY").Secret()). WithSecretVariable("AWS_REGION", util.WithSetHostVar(ctx, c.Host(), "AWS_REGION").Secret()). @@ -42,8 +39,22 @@ func (cl Cli) Publish(ctx context.Context, version string) error { WithSecretVariable("ARTEFACTS_FQDN", util.WithSetHostVar(ctx, c.Host(), "ARTEFACTS_FQDN").Secret()). WithSecretVariable("HOMEBREW_TAP_OWNER", util.WithSetHostVar(ctx, c.Host(), "HOMEBREW_TAP_OWNER").Secret()) + if nightly { + // goreleaser refuses to run if there isn't a tag, so set it to a dummy but valid semver + container = container.WithExec([]string{"git", "tag", "0.0.0"}) + } + + args := []string{"release", "--rm-dist", "--skip-validate", "--debug"} + if nightly { + args = append(args, + "--nightly", + "--config", ".goreleaser.nightly.yml", + ) + } + _, err = container. - WithExec([]string{"release", "--rm-dist", "--skip-validate", "--debug"}). + WithEntrypoint([]string{"/sbin/tini", "--", "/entrypoint.sh"}). + WithExec(args). ExitCode(ctx) return err } diff --git a/internal/mage/sdk/go.go b/internal/mage/sdk/go.go index 76b31c3c49..0737fd9e6c 100644 --- a/internal/mage/sdk/go.go +++ b/internal/mage/sdk/go.go @@ -160,5 +160,7 @@ package engineconn const CLIVersion = %q `, version) + // NOTE: if you change this path, be sure to update .github/workflows/publish.yml so that + // provision tests run whenever this file changes. return os.WriteFile("sdk/go/internal/engineconn/version.gen.go", []byte(versionFile), 0o600) } diff --git a/internal/mage/sdk/nodejs.go b/internal/mage/sdk/nodejs.go index d9d4a7c724..b983f74d62 100644 --- a/internal/mage/sdk/nodejs.go +++ b/internal/mage/sdk/nodejs.go @@ -110,9 +110,10 @@ always-auth=true`, token) // Bump the Node.js SDK's Engine dependency func (t Nodejs) Bump(ctx context.Context, version string) error { engineReference := fmt.Sprintf("// Code generated by dagger. DO NOT EDIT.\n"+ - "const CLI_VERSION =\n"+ - " %q\n\n", version) + "export const CLI_VERSION = %q\n", version) + // NOTE: if you change this path, be sure to update .github/workflows/publish.yml so that + // provision tests run whenever this file changes. return os.WriteFile("sdk/nodejs/provisioning/default.ts", []byte(engineReference), 0o600) } diff --git a/internal/mage/sdk/python.go b/internal/mage/sdk/python.go index 4599936048..6fb7d731dc 100644 --- a/internal/mage/sdk/python.go +++ b/internal/mage/sdk/python.go @@ -161,6 +161,8 @@ func (t Python) Bump(ctx context.Context, version string) error { CLI_VERSION = %q `, version) + // NOTE: if you change this path, be sure to update .github/workflows/publish.yml so that + // provision tests run whenever this file changes. return os.WriteFile("sdk/python/src/dagger/_version.py", []byte(engineReference), 0o600) } diff --git a/sdk/go/internal/engineconn/cli.go b/sdk/go/internal/engineconn/cli.go index 2bf0ca1d2a..c13471ca2e 100644 --- a/sdk/go/internal/engineconn/cli.go +++ b/sdk/go/internal/engineconn/cli.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "os/exec" "path/filepath" @@ -21,13 +22,14 @@ import ( const ( daggerCLIBinPrefix = "dagger-" + defaultCLIHost = "dl.dagger.io" ) var ( // Only modified by tests, not changeable by outside users due to being in // an internal package - DefaultCLIHost = "dl.dagger.io" - DefaultCLIScheme = "https" + OverrideCLIArchiveURL string + OverrideChecksumsURL string ) func FromLocalCLI(ctx context.Context, cfg *Config) (EngineConn, bool, error) { @@ -128,20 +130,20 @@ func checksumMap(ctx context.Context) (map[string]string, error) { checksums := make(map[string]string) checksumFileContents := bytes.NewBuffer(nil) - checksumReq, err := http.NewRequestWithContext(ctx, http.MethodGet, defaultChecksumsURL(), nil) + checksumReq, err := http.NewRequestWithContext(ctx, http.MethodGet, checksumsURL(), nil) if err != nil { return nil, fmt.Errorf("failed to create checksums request: %w", err) } resp, err := http.DefaultClient.Do(checksumReq) if err != nil { - return nil, fmt.Errorf("failed to download checksums: %w", err) + return nil, fmt.Errorf("failed to download checksums from %s: %w", checksumsURL(), err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to download checksums: %s", resp.Status) + return nil, fmt.Errorf("failed to download checksums from %s: %s", checksumsURL(), resp.Status) } if _, err := io.Copy(checksumFileContents, resp.Body); err != nil { - return nil, fmt.Errorf("failed to download checksums: %w", err) + return nil, fmt.Errorf("failed to download checksums from %s: %w", checksumsURL(), err) } scanner := bufio.NewScanner(checksumFileContents) @@ -173,7 +175,7 @@ func expectedChecksum(ctx context.Context) (string, error) { // Download the CLI archive and extract the CLI from it into the provided dest. // Returns the sha256 hash of the whole archive as read during download. func extractCLI(ctx context.Context, dest io.Writer) (string, error) { - archiveReq, err := http.NewRequestWithContext(ctx, http.MethodGet, defaultCLIArchiveURL(), nil) + archiveReq, err := http.NewRequestWithContext(ctx, http.MethodGet, cliArchiveURL(), nil) if err != nil { return "", fmt.Errorf("failed to create archive request: %w", err) } @@ -183,7 +185,7 @@ func extractCLI(ctx context.Context, dest io.Writer) (string, error) { } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("failed to download CLI archive: %s", resp.Status) + return "", fmt.Errorf("failed to download CLI archive from %s: %s", cliArchiveURL(), resp.Status) } // the body is a tar.gz file, untar it and extract the dagger binary @@ -223,27 +225,38 @@ func extractCLI(ctx context.Context, dest io.Writer) (string, error) { } func defaultCLIArchiveName() string { + if OverrideCLIArchiveURL != "" { + url, err := url.Parse(OverrideCLIArchiveURL) + if err != nil { + panic(err) + } + return filepath.Base(url.Path) + } // TODO:(sipsma) fix this for windows - return fmt.Sprintf("dagger_v%s_%s_%s.tar.gz", + return fmt.Sprintf("dagger_%s_%s_%s.tar.gz", CLIVersion, runtime.GOOS, runtime.GOARCH, ) } -func defaultCLIArchiveURL() string { - return fmt.Sprintf("%s://%s/dagger/releases/%s/%s", - DefaultCLIScheme, - DefaultCLIHost, +func cliArchiveURL() string { + if OverrideCLIArchiveURL != "" { + return OverrideCLIArchiveURL + } + return fmt.Sprintf("https://%s/dagger/releases/%s/%s", + defaultCLIHost, CLIVersion, defaultCLIArchiveName(), ) } -func defaultChecksumsURL() string { - return fmt.Sprintf("%s://%s/dagger/releases/%s/checksums.txt", - DefaultCLIScheme, - DefaultCLIHost, +func checksumsURL() string { + if OverrideChecksumsURL != "" { + return OverrideChecksumsURL + } + return fmt.Sprintf("https://%s/dagger/releases/%s/checksums.txt", + defaultCLIHost, CLIVersion, ) } diff --git a/sdk/go/provision_test.go b/sdk/go/provision_test.go index e094ad54fb..2c48623c8a 100644 --- a/sdk/go/provision_test.go +++ b/sdk/go/provision_test.go @@ -47,31 +47,35 @@ func TestProvision(t *testing.T) { } os.Unsetenv("DAGGER_SESSION_URL") - // Setup a test server if _EXPERIMENTAL_DAGGER_CLI_BIN is set - binPath, ok := os.LookupEnv("_EXPERIMENTAL_DAGGER_CLI_BIN") - if ok { + if cliURL := os.Getenv("_INTERNAL_DAGGER_TEST_CLI_URL"); cliURL != "" { + // If explicitly requested to test against a certain URL, use that + engineconn.OverrideCLIArchiveURL = cliURL + engineconn.OverrideChecksumsURL = os.Getenv("_INTERNAL_DAGGER_TEST_CLI_CHECKSUMS_URL") + defer func() { + engineconn.OverrideCLIArchiveURL = "" + engineconn.OverrideChecksumsURL = "" + }() + } else if binPath, ok := os.LookupEnv("_EXPERIMENTAL_DAGGER_CLI_BIN"); ok { + // Otherwise if _EXPERIMENTAL_DAGGER_CLI_BIN is set, create a mock http server for it defer os.Setenv("_EXPERIMENTAL_DAGGER_CLI_BIN", binPath) os.Unsetenv("_EXPERIMENTAL_DAGGER_CLI_BIN") - originalBaseURL := engineconn.DefaultCLIHost - defer func() { - engineconn.DefaultCLIHost = originalBaseURL - }() - originalScheme := engineconn.DefaultCLIScheme - defer func() { - engineconn.DefaultCLIScheme = originalScheme - }() - engineconn.DefaultCLIScheme = "http" + archiveName := fmt.Sprintf("dagger_v%s_%s_%s.tar.gz", engineconn.CLIVersion, runtime.GOOS, runtime.GOARCH) l, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) defer l.Close() - engineconn.DefaultCLIHost = l.Addr().String() + + engineconn.OverrideCLIArchiveURL = fmt.Sprintf("http://%s/dagger/releases/%s/%s", l.Addr().String(), engineconn.CLIVersion, archiveName) + engineconn.OverrideChecksumsURL = fmt.Sprintf("http://%s/dagger/releases/%s/checksums.txt", l.Addr().String(), engineconn.CLIVersion) + defer func() { + engineconn.OverrideCLIArchiveURL = "" + engineconn.OverrideChecksumsURL = "" + }() basePath := fmt.Sprintf("dagger/releases/%s/", engineconn.CLIVersion) archiveBytes := createCLIArchive(t, binPath) - archiveName := fmt.Sprintf("dagger_v%s_%s_%s.tar.gz", engineconn.CLIVersion, runtime.GOOS, runtime.GOARCH) archivePath := path.Join(basePath, archiveName) checksum := sha256.Sum256(archiveBytes.Bytes()) diff --git a/sdk/nodejs/provisioning/bin.ts b/sdk/nodejs/provisioning/bin.ts index 0231979543..9411cae72d 100644 --- a/sdk/nodejs/provisioning/bin.ts +++ b/sdk/nodejs/provisioning/bin.ts @@ -14,8 +14,9 @@ import { } from "../common/errors/index.js" import fetch from "node-fetch" -let CLI_HOST = "dl.dagger.io" -let CLI_SCHEME = "https" +const CLI_HOST = "dl.dagger.io" +let OVERRIDE_CLI_URL: string +let OVERRIDE_CHECKSUMS_URL: string /** * Bin runs an engine session from a specified binary @@ -235,19 +236,28 @@ export class Bin implements EngineConn { } private cliArchiveName(): string { - return `dagger_v${ + if (OVERRIDE_CLI_URL) { + return path.basename(new URL(OVERRIDE_CLI_URL).pathname) + } + return `dagger_${ this.cliVersion }_${this.normalizedOS()}_${this.normalizedArch()}.tar.gz` } private cliArchiveURL(): string { - return `${CLI_SCHEME}://${CLI_HOST}/dagger/releases/${ + if (OVERRIDE_CLI_URL) { + return OVERRIDE_CLI_URL + } + return `https://${CLI_HOST}/dagger/releases/${ this.cliVersion }/${this.cliArchiveName()}` } private cliChecksumURL(): string { - return `${CLI_SCHEME}://${CLI_HOST}/dagger/releases/${this.cliVersion}/checksums.txt` + if (OVERRIDE_CHECKSUMS_URL) { + return OVERRIDE_CHECKSUMS_URL + } + return `https://${CLI_HOST}/dagger/releases/${this.cliVersion}/checksums.txt` } private async checksumMap(): Promise> { @@ -323,18 +333,10 @@ export class Bin implements EngineConn { } // Only meant for tests -export function _cliHost(): string { - return CLI_HOST -} -// Only meant for tests -export function _overrideCLIHost(host: string): void { - CLI_HOST = host -} -// Only meant for tests -export function _cliScheme(): string { - return CLI_SCHEME +export function _overrideCLIURL(url: string): void { + OVERRIDE_CLI_URL = url } // Only meant for tests -export function _overrideCLIScheme(scheme: string): void { - CLI_SCHEME = scheme +export function _overrideCLIChecksumsURL(url: string): void { + OVERRIDE_CHECKSUMS_URL = url } diff --git a/sdk/nodejs/test/connect.spec.ts b/sdk/nodejs/test/connect.spec.ts index 5ce1a83b2d..6494694ac5 100644 --- a/sdk/nodejs/test/connect.spec.ts +++ b/sdk/nodejs/test/connect.spec.ts @@ -51,15 +51,11 @@ describe("NodeJS sdk Connect", function () { describe("Automatic Provisioned CLI Binary", function () { let oldEnv: string - let oldCLIHost: string - let oldCLIScheme: string let tempDir: string let cacheDir: string before(() => { oldEnv = JSON.stringify(process.env) - oldCLIHost = bin._cliHost() - oldCLIScheme = bin._cliScheme() tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "dagger-test-")) cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), "dagger-test-cache")) process.env.XDG_CACHE_HOME = cacheDir @@ -71,8 +67,22 @@ describe("NodeJS sdk Connect", function () { // ignore DAGGER_SESSION_URL delete process.env.DAGGER_SESSION_URL + // If explicitly requested to test against a certain URL, use that + const cliURL = process.env._INTERNAL_DAGGER_TEST_CLI_URL + if (cliURL) { + bin._overrideCLIURL(cliURL) + const checksumsUrl = process.env._INTERNAL_DAGGER_TEST_CLI_CHECKSUMS_URL + if (!checksumsUrl) { + throw new Error( + "Missing override checksums URL when overriding CLI URL" + ) + } + bin._overrideCLIChecksumsURL(checksumsUrl) + } + + // Otherwise if _EXPERIMENTAL_DAGGER_CLI_BIN is set, create a mock http server for it const cliBin = process.env._EXPERIMENTAL_DAGGER_CLI_BIN - if (cliBin) { + if (cliBin && !cliURL) { delete process.env._EXPERIMENTAL_DAGGER_CLI_BIN // create a temporary dir and write a tar.gz with the cli_bin in it const tempArchivePath = path.join(tempDir, "cli.tar.gz") @@ -118,8 +128,12 @@ describe("NodeJS sdk Connect", function () { server .listen(0, "127.0.0.1", () => { const addr = server.address() as AddressInfo - bin._overrideCLIHost(addr.address + ":" + addr.port) - bin._overrideCLIScheme("http") + bin._overrideCLIURL( + `http://${addr.address}:${addr.port}/${basePath}/${archiveName}` + ) + bin._overrideCLIChecksumsURL( + `http://${addr.address}:${addr.port}/${basePath}/checksums.txt` + ) resolve() }) .unref() @@ -136,8 +150,8 @@ describe("NodeJS sdk Connect", function () { after(() => { process.env = JSON.parse(oldEnv) - bin._overrideCLIHost(oldCLIHost) - bin._overrideCLIScheme(oldCLIScheme) + bin._overrideCLIURL("") + bin._overrideCLIChecksumsURL("") fs.rmSync(tempDir, { recursive: true }) fs.rmSync(cacheDir, { recursive: true }) }) diff --git a/sdk/python/src/dagger/engine/bin.py b/sdk/python/src/dagger/engine/bin.py index a994d12aaa..42f745ca2a 100644 --- a/sdk/python/src/dagger/engine/bin.py +++ b/sdk/python/src/dagger/engine/bin.py @@ -9,6 +9,7 @@ import tempfile import time import urllib.request +import urllib.parse from pathlib import Path from typing import IO @@ -25,7 +26,8 @@ DAGGER_CLI_BIN_PREFIX = "dagger-" CLI_HOST = "dl.dagger.io" -CLI_SCHEME = "https" +OVERRIDE_CLI_ARCHIVE_URL = "" +OVERRIDE_CLI_CHECKSUMS_URL = "" @register_engine("bin") @@ -200,19 +202,27 @@ def get_platform() -> tuple[str, str]: def cli_archive_name(cli_version: str): + if OVERRIDE_CLI_ARCHIVE_URL: + return Path(urllib.parse.urlparse(OVERRIDE_CLI_ARCHIVE_URL).path).name os_, arch = get_platform() - return f"dagger_v{cli_version}_{os_}_{arch}.tar.gz" + return f"dagger_{cli_version}_{os_}_{arch}.tar.gz" def cli_archive_url(cli_version: str): + if OVERRIDE_CLI_ARCHIVE_URL: + return OVERRIDE_CLI_ARCHIVE_URL + return ( - f"{CLI_SCHEME}://{CLI_HOST}/dagger/releases/{cli_version}" + f"https://{CLI_HOST}/dagger/releases/{cli_version}" f"/{cli_archive_name(cli_version)}" ) def cli_checksum_url(cli_version: str): - return f"{CLI_SCHEME}://{CLI_HOST}/dagger/releases/{cli_version}/checksums.txt" + if OVERRIDE_CLI_CHECKSUMS_URL: + return OVERRIDE_CLI_CHECKSUMS_URL + + return f"https://{CLI_HOST}/dagger/releases/{cli_version}/checksums.txt" # returns a dict of CLI archive name -> checksum for that archive diff --git a/sdk/python/tests/engine/test_download_bin.py b/sdk/python/tests/engine/test_download_bin.py index 2ac349ee76..ba048932a5 100644 --- a/sdk/python/tests/engine/test_download_bin.py +++ b/sdk/python/tests/engine/test_download_bin.py @@ -27,6 +27,19 @@ def cache_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): async def temporary_cli_server(monkeypatch: pytest.MonkeyPatch): # ignore DAGGER_SESSION_URL monkeypatch.delenv("DAGGER_SESSION_URL", raising=False) + + # If explicitly requested to test against a certain URL, use that + override_cli_url = os.environ.get("_INTERNAL_DAGGER_TEST_CLI_URL") + if override_cli_url: + monkeypatch.setattr(bin, "OVERRIDE_CLI_ARCHIVE_URL", override_cli_url) + monkeypatch.setattr( + bin, + "OVERRIDE_CLI_CHECKSUMS_URL", + os.environ.get("_INTERNAL_DAGGER_TEST_CLI_CHECKSUMS_URL"), + ) + yield + return + # if _EXPERIMENTAL_DAGGER_CLI_BIN is set, create a temporary http server for it cli_bin = os.environ.get("_EXPERIMENTAL_DAGGER_CLI_BIN") if cli_bin: @@ -71,14 +84,24 @@ def do_GET(self): # create a listener on a random localhost port and start the server httpd = HTTPServer(("127.0.0.1", 0), RequestHandler) address = httpd.socket.getsockname() - monkeypatch.setattr(bin, "CLI_HOST", f"{address[0]}:{address[1]}") - monkeypatch.setattr(bin, "CLI_SCHEME", "http") + monkeypatch.setattr( + bin, + "OVERRIDE_CLI_ARCHIVE_URL", + f"http://{address[0]}:{address[1]}/{base_path}/{archive_name}", + ) + monkeypatch.setattr( + bin, + "OVERRIDE_CLI_CHECKSUMS_URL", + f"http://{address[0]}:{address[1]}/{base_path}/checksums.txt", + ) with anyio.start_blocking_portal() as portal: server = portal.start_task_soon(httpd.serve_forever) yield httpd.shutdown() server.cancel() + yield + @pytest.mark.anyio @pytest.mark.slow