GitHub Actions e2e tests

.github/workflows/build.yml

@@ -50,3 +50,18 @@ jobs:
           name: ${{ matrix.target }}
           path: ./build/*
           if-no-files-found: error
+
+  plugins:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Build plugins
+        uses: docker/bake-action@v1
+        with:
+          targets: plugins-cross

.github/workflows/e2e.yml

@@ -0,0 +1,54 @@
+name: e2e
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'master'
+      - '[0-9]+.[0-9]{2}'
+    tags:
+      - 'v*'
+  pull_request:
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - non-experimental
+          - experimental
+          - connhelper-ssh
+        base:
+          - alpine
+          - buster
+        engine-version:
+#          - 20.10-dind # FIXME: Fails on 20.10
+          - stable-dind # TODO: Use 20.10-dind, stable-dind is deprecated
+        include:
+          - target: non-experimental
+            engine-version: 19.03-dind
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Update daemon.json
+        run: |
+          sudo jq '.experimental = true' < /etc/docker/daemon.json > /tmp/docker.json
+          sudo mv /tmp/docker.json /etc/docker/daemon.json
+          sudo cat /etc/docker/daemon.json
+          sudo service docker restart
+          docker version
+          docker info
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Run ${{ matrix.target }}
+        run: |
+          make -f docker.Makefile test-e2e-${{ matrix.target }}
+        env:
+          BASE_VARIANT: ${{ matrix.base }}
+          E2E_ENGINE_VERSION: ${{ matrix.engine-version }}

Dockerfile

@@ -20,7 +20,7 @@ FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 
 FROM go-${TARGETOS} AS build-base-alpine
 COPY --from=xx / /
-RUN apk add --no-cache clang lld llvm file git
+RUN apk add --no-cache bash clang lld llvm file git
 WORKDIR /go/src/github.com/docker/cli
 
 FROM build-base-alpine AS build-alpine
@@ -30,7 +30,7 @@ RUN xx-apk add --no-cache musl-dev gcc
 
 FROM go-${TARGETOS} AS build-base-buster
 COPY --from=xx / /
-RUN apt-get update && apt-get install --no-install-recommends -y clang lld file
+RUN apt-get update && apt-get install --no-install-recommends -y bash clang lld file
 WORKDIR /go/src/github.com/docker/cli
 
 FROM build-base-buster AS build-buster
@@ -65,8 +65,51 @@ RUN --mount=type=bind,target=.,ro \
     TARGET=/out ./scripts/build/binary && \
     xx-verify $([ "$GO_LINKMODE" = "static" ] && echo "--static") /out/docker
 
-FROM build-base-${BASE_VARIANT} AS dev 
+FROM build-${BASE_VARIANT} AS build-plugins
+ARG GO_LINKMODE=static
+ARG GO_BUILDTAGS
+ARG GO_STRIP
+ARG CGO_ENABLED
+ARG VERSION
+RUN --mount=ro --mount=type=cache,target=/root/.cache \
+    --mount=from=dockercore/golang-cross:xx-sdk-extras,target=/xx-sdk,src=/xx-sdk \
+    xx-go --wrap && \
+    TARGET=/out ./scripts/build/plugins e2e/cli-plugins/plugins/*
+
+FROM build-base-alpine AS e2e-base-alpine
+RUN apk add --no-cache build-base curl docker-compose openssl openssh-client
+
+FROM build-base-buster AS e2e-base-buster
+RUN apt-get update && apt-get install -y build-essential curl openssl openssh-client
+ARG COMPOSE_VERSION=1.29.2
+RUN curl -fsSL https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose && \
+    chmod +x /usr/local/bin/docker-compose
+
+FROM build-${BASE_VARIANT} AS gotestsum
+ARG GOTESTSUM_VERSION=v1.7.0
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    GOBIN=/out GO111MODULE=on go install "gotest.tools/gotestsum@${GOTESTSUM_VERSION}" \
+    && /out/gotestsum --version
+
+FROM e2e-base-${BASE_VARIANT} AS e2e
+ARG NOTARY_VERSION=v0.6.1
+ADD --chmod=0755 https://github.com/theupdateframework/notary/releases/download/${NOTARY_VERSION}/notary-Linux-amd64 /usr/local/bin/notary
+COPY e2e/testdata/notary/root-ca.cert /usr/share/ca-certificates/notary.cert
+RUN echo 'notary.cert' >> /etc/ca-certificates.conf && update-ca-certificates
+COPY --from=gotestsum /out/gotestsum /usr/bin/gotestsum
+COPY --from=build /out ./build/
+COPY --from=build-plugins /out ./build/
+COPY . .
+ENV DOCKER_BUILDKIT=1
+ENV PATH=/go/src/github.com/docker/cli/build:$PATH
+CMD ./scripts/test/e2e/entry
+
+FROM build-base-${BASE_VARIANT} AS dev
 COPY . .
 
 FROM scratch AS binary
 COPY --from=build /out .
+
+FROM scratch AS plugins
+COPY --from=build-plugins /out .

Makefile

@@ -48,14 +48,6 @@ dynbinary: ## build dynamically linked binary
 plugins: ## build example CLI plugins
 	./scripts/build/plugins
 
-.PHONY: plugins-windows
-plugins-windows: ## build example CLI plugins for Windows
-	./scripts/build/plugins-windows
-
-.PHONY: plugins-osx
-plugins-osx: ## build example CLI plugins for macOS
-	./scripts/build/plugins-osx
-
 vendor: vendor.conf ## check that vendor matches vendor.conf
 	rm -rf vendor
 	bash -c 'vndr |& grep -v -i clone | tee ./vndr.log'

README.md

@@ -3,7 +3,6 @@
 [![PkgGoDev](https://img.shields.io/badge/go.dev-docs-007d9c?logo=go&logoColor=white)](https://pkg.go.dev/github.com/docker/cli)
 [![Build Status](https://img.shields.io/github/workflow/status/docker/cli/build?logo=github)](https://github.com/docker/cli/actions?query=workflow%3Abuild)
 [![CircleCI Status](https://img.shields.io/circleci/build/github/docker/cli/master?logo=circleci)](https://circleci.com/gh/docker/cli/tree/master)
-[![Jenkins Status](https://img.shields.io/jenkins/build?jobUrl=https%3A%2F%2Fci.docker.com%2Fpublic%2Fjob%2Fcli%2Fjob%2Fmaster&logo=jenkins)](https://ci.docker.com/public/job/cli/job/master)
 [![Go Report Card](https://goreportcard.com/badge/github.com/docker/cli)](https://goreportcard.com/report/github.com/docker/cli)
 [![Codecov](https://codecov.io/gh/docker/cli/branch/master/graph/badge.svg)](https://codecov.io/gh/docker/cli)
 

docker-bake.hcl

@@ -1,14 +1,15 @@
 variable "VERSION" {
     default = ""
 }
-
 variable "USE_GLIBC" {
     default = ""
 }
-
 variable "STRIP_TARGET" {
     default = ""
 }
+variable "IMAGE_NAME" {
+    default = "docker-cli"
+}
 
 # Sets the name of the company that produced the windows binary.
 variable "COMPANY_NAME" {
@@ -38,6 +39,17 @@ target "dynbinary" {
     }
 }
 
+target "plugins" {
+    target = "plugins"
+    platforms = ["local"]
+    output = ["build"]
+    args = {
+        BASE_VARIANT = USE_GLIBC != "" ? "buster" : "alpine"
+        VERSION = VERSION
+        GO_STRIP = STRIP_TARGET
+    }
+}
+
 target "platforms" {
     platforms = concat(["linux/amd64", "linux/386", "linux/arm64", "linux/arm", "linux/ppc64le", "linux/s390x", "darwin/amd64", "darwin/arm64", "windows/amd64", "windows/arm", "windows/386"], USE_GLIBC!=""?[]:["windows/arm64"])
 }
@@ -50,6 +62,10 @@ target "dynbinary-cross" {
     inherits = ["dynbinary", "platforms"]
 }
 
+target "plugins-cross" {
+    inherits = ["plugins", "platforms"]
+}
+
 target "lint" {
     dockerfile = "./dockerfiles/Dockerfile.lint"
     target = "lint"
@@ -61,3 +77,13 @@ target "shellcheck" {
     target = "shellcheck"
     output = ["type=cacheonly"]
 }
+
+target "e2e-image" {
+    target = "e2e"
+    output = ["type=docker"]
+    tags = ["${IMAGE_NAME}"]
+    args = {
+        BASE_VARIANT = USE_GLIBC != "" ? "buster" : "alpine"
+        VERSION = VERSION
+    }
+}

docker.Makefile

@@ -13,9 +13,7 @@ DOCKER_CLI_GO_BUILD_CACHE ?= y
 COMPANY_NAME ?=
 
 DEV_DOCKER_IMAGE_NAME = docker-cli-dev$(IMAGE_TAG)
-BINARY_NATIVE_IMAGE_NAME = docker-cli-native$(IMAGE_TAG)
-CROSS_IMAGE_NAME = docker-cli-cross$(IMAGE_TAG)
-E2E_IMAGE_NAME = docker-cli-e2e$(IMAGE_TAG)
+E2E_IMAGE_NAME = docker-cli-e2e
 E2E_ENGINE_VERSION ?=
 CACHE_VOLUME_NAME := docker-cli-dev-cache
 ifeq ($(DOCKER_CLI_GO_BUILD_CACHE),y)
@@ -33,15 +31,6 @@ build_docker_image:
 	# build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment
 	cat ./dockerfiles/Dockerfile.dev | docker build ${DOCKER_BUILD_ARGS} --build-arg=GO_VERSION -t $(DEV_DOCKER_IMAGE_NAME) -
 
-.PHONY: build_binary_native_image
-build_binary_native_image:
-	# build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment
-	cat ./dockerfiles/Dockerfile.binary-native | docker build --build-arg=GO_VERSION -t $(BINARY_NATIVE_IMAGE_NAME) -
-
-.PHONY: build_e2e_image
-build_e2e_image:
-	docker build -t $(E2E_IMAGE_NAME) --build-arg=GO_VERSION --build-arg VERSION=$(VERSION) --build-arg GITCOMMIT=$(GITCOMMIT) -f ./dockerfiles/Dockerfile.e2e .
-
 DOCKER_RUN_NAME_OPTION := $(if $(DOCKER_CLI_CONTAINER_NAME),--name $(DOCKER_CLI_CONTAINER_NAME),)
 DOCKER_RUN := docker run --rm $(ENVVARS) $(DOCKER_CLI_MOUNTS) $(DOCKER_RUN_NAME_OPTION)
 
@@ -51,8 +40,11 @@ binary:
 
 build: binary ## alias for binary
 
-plugins: build_binary_native_image ## build the CLI plugin examples
-	$(DOCKER_RUN) $(BINARY_NATIVE_IMAGE_NAME) ./scripts/build/plugins
+plugins: ## build the CLI plugin examples
+	docker buildx bake plugins
+
+plugins-cross: ## build the CLI plugin examples for all platforms
+	docker buildx bake plugins-cross
 
 .PHONY: clean
 clean: build_docker_image ## clean build artifacts
@@ -70,14 +62,6 @@ test: test-unit test-e2e
 cross:
 	COMPANY_NAME=$(COMPANY_NAME) docker buildx bake cross
 
-.PHONY: plugins-windows
-plugins-windows: build_cross_image ## build the example CLI plugins for Windows
-	$(DOCKER_RUN) $(CROSS_IMAGE_NAME) make $@
-
-.PHONY: plugins-osx
-plugins-osx: build_cross_image ## build the example CLI plugins for macOS
-	$(DOCKER_RUN) $(CROSS_IMAGE_NAME) make $@
-
 .PHONY: dynbinary
 dynbinary: ## build dynamically linked binary
 	USE_GLIBC=1 COMPANY_NAME=$(COMPANY_NAME)  docker buildx bake dynbinary
@@ -118,20 +102,24 @@ manpages: build_docker_image ## generate man pages from go source and markdown
 yamldocs: build_docker_image ## generate documentation YAML files consumed by docs repo
 	$(DOCKER_RUN) -it $(DEV_DOCKER_IMAGE_NAME) make yamldocs
 
+.PHONY: build-e2e-image
+build-e2e-image:
+	IMAGE_NAME=$(E2E_IMAGE_NAME) VERSION=$(VERSION) docker buildx bake e2e-image
+
 .PHONY: test-e2e
 test-e2e: test-e2e-non-experimental test-e2e-experimental test-e2e-connhelper-ssh ## run all e2e tests
 
 .PHONY: test-e2e-experimental
-test-e2e-experimental: build_e2e_image # run experimental e2e tests
-	docker run --rm --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 $(E2E_IMAGE_NAME)
+test-e2e-experimental: build-e2e-image # run experimental e2e tests
+	docker run --rm --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) $(E2E_IMAGE_NAME)
 
 .PHONY: test-e2e-non-experimental
-test-e2e-non-experimental: build_e2e_image # run non-experimental e2e tests
+test-e2e-non-experimental: build-e2e-image # run non-experimental e2e tests
 	docker run --rm --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock $(ENVVARS) -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) $(E2E_IMAGE_NAME)
 
 .PHONY: test-e2e-connhelper-ssh
-test-e2e-connhelper-ssh: build_e2e_image # run experimental SSH-connection helper e2e tests
-	docker run --rm --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 -e TEST_CONNHELPER=ssh $(E2E_IMAGE_NAME)
+test-e2e-connhelper-ssh: build-e2e-image # run experimental SSH-connection helper e2e tests
+	docker run --rm --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) -e TEST_CONNHELPER=ssh $(E2E_IMAGE_NAME)
 
 .PHONY: help
 help: ## print this help