GitHub Actions e2e tests

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>

.github/workflows/build.yml

@@ -48,3 +48,18 @@ jobs:
           name: ${{ matrix.target }}
           path: ./build/*
           if-no-files-found: error
+
+  plugins:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Build plugins
+        uses: docker/bake-action@v1
+        with:
+          targets: plugins

.github/workflows/e2e.yml

@@ -0,0 +1,40 @@
+name: e2e
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'master'
+      - '[0-9]+.[0-9]{2}'
+    tags:
+      - 'v*'
+  pull_request:
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - non-experimental
+          - experimental
+          - connhelper-ssh
+        engine-version:
+          - stable-dind
+        include:
+          - target: non-experimental
+            engine-version: 19.03-dind
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Run ${{ matrix.target }}
+        run: |
+          make -f docker.Makefile test-e2e-${{ matrix.target }}
+        env:
+          E2E_ENGINE_VERSION: ${{ matrix.engine-version }}

Dockerfile

@@ -19,7 +19,7 @@ FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 
 FROM go-${TARGETOS} AS build-base-alpine
 COPY --from=xx / /
-RUN apk add --no-cache clang lld llvm file git
+RUN apk add --no-cache bash clang lld llvm file git
 WORKDIR /go/src/github.com/docker/cli
 
 FROM build-base-alpine AS build-alpine
@@ -29,7 +29,7 @@ RUN xx-apk add --no-cache musl-dev gcc
 
 FROM go-${TARGETOS} AS build-base-buster
 COPY --from=xx / /
-RUN apt-get update && apt-get install --no-install-recommends -y clang lld file
+RUN apt-get update && apt-get install --no-install-recommends -y bash clang lld file
 WORKDIR /go/src/github.com/docker/cli
 
 FROM build-base-buster AS build-buster
@@ -55,8 +55,47 @@ RUN --mount=ro --mount=type=cache,target=/root/.cache \
     TARGET=/out ./scripts/build/binary && \
     xx-verify $([ "$GO_LINKMODE" = "static" ] && echo "--static") /out/docker
 
-FROM build-base-${BASE_VARIANT} AS dev 
+FROM build-${BASE_VARIANT} AS build-plugins
+ARG CGO_ENABLED=0
+ARG VERSION
+RUN --mount=ro --mount=type=cache,target=/root/.cache \
+    --mount=from=dockercore/golang-cross:xx-sdk-extras,target=/xx-sdk,src=/xx-sdk \
+    --mount=type=tmpfs,target=cli/winresources \
+    xx-go --wrap && \
+    TARGET=/out ./scripts/build/plugins e2e/cli-plugins/plugins/*
+
+FROM build-base-alpine AS e2e-base-alpine
+RUN apk add --no-cache build-base curl openssl openssh-client
+
+FROM build-base-buster AS e2e-base-buster
+RUN apt-get update && apt-get install --no-install-recommends -y build-essential curl openssl openssh-client
+
+FROM e2e-base-${BASE_VARIANT} AS e2e
+ARG COMPOSE_VERSION=1.29.2
+RUN curl -fsSL https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose && \
+    chmod +x /usr/local/bin/docker-compose
+ARG NOTARY_VERSION=v0.6.1
+RUN curl -fsSL https://github.com/theupdateframework/notary/releases/download/${NOTARY_VERSION}/notary-Linux-amd64 -o /usr/local/bin/notary && \
+    chmod +x /usr/local/bin/notary
+COPY e2e/testdata/notary/root-ca.cert /usr/share/ca-certificates/notary.cert
+RUN echo 'notary.cert' >> /etc/ca-certificates.conf && update-ca-certificates
+ARG GOTESTSUM_VERSION=1.7.0
+RUN curl -fsSL https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/gotestsum_${GOTESTSUM_VERSION}_linux_amd64.tar.gz -o gotestsum.tar.gz && \
+    tar -xf gotestsum.tar.gz gotestsum && \
+    mv gotestsum /usr/local/bin/gotestsum && \
+    rm gotestsum.tar.gz
+COPY --from=build /out ./build/
+COPY --from=build-plugins /out ./build/
+COPY . .
+ENV DOCKER_BUILDKIT=1
+ENV PATH=/go/src/github.com/docker/cli/build:$PATH
+CMD ./scripts/test/e2e/entry
+
+FROM build-base-${BASE_VARIANT} AS dev
 COPY . .
 
 FROM scratch AS binary
 COPY --from=build /out .
+
+FROM scratch AS plugins
+COPY --from=build-plugins /out .

Makefile

@@ -44,14 +44,6 @@ dynbinary: ## build dynamically linked binary
 plugins: ## build example CLI plugins
 	./scripts/build/plugins
 
-.PHONY: plugins-windows
-plugins-windows: ## build example CLI plugins for Windows
-	./scripts/build/plugins-windows
-
-.PHONY: plugins-osx
-plugins-osx: ## build example CLI plugins for macOS
-	./scripts/build/plugins-osx
-
 vendor: vendor.conf ## check that vendor matches vendor.conf
 	rm -rf vendor
 	bash -c 'vndr |& grep -v -i clone | tee ./vndr.log'

README.md

@@ -3,7 +3,6 @@
 [![PkgGoDev](https://img.shields.io/badge/go.dev-docs-007d9c?logo=go&logoColor=white)](https://pkg.go.dev/github.com/docker/cli)
 [![Build Status](https://img.shields.io/github/workflow/status/docker/cli/build?logo=github)](https://github.com/docker/cli/actions?query=workflow%3Abuild)
 [![CircleCI Status](https://img.shields.io/circleci/build/github/docker/cli/master?logo=circleci)](https://circleci.com/gh/docker/cli/tree/master)
-[![Jenkins Status](https://img.shields.io/jenkins/build?jobUrl=https%3A%2F%2Fci.docker.com%2Fpublic%2Fjob%2Fcli%2Fjob%2Fmaster&logo=jenkins)](https://ci.docker.com/public/job/cli/job/master)
 [![Go Report Card](https://goreportcard.com/badge/github.com/docker/cli)](https://goreportcard.com/report/github.com/docker/cli)
 [![Codecov](https://codecov.io/gh/docker/cli/branch/master/graph/badge.svg)](https://codecov.io/gh/docker/cli)
 

docker-bake.hcl

@@ -1,14 +1,15 @@
 variable "VERSION" {
     default = ""
 }
-
 variable "USE_GLIBC" {
     default = ""
 }
-
 variable "STRIP_TARGET" {
     default = ""
 }
+variable "IMAGE_NAME" {
+    default = "docker-cli"
+}
 
 group "default" {
     targets = ["binary"]
@@ -32,6 +33,17 @@ target "dynbinary" {
     }
 }
 
+target "plugins" {
+    target = "plugins"
+    platforms = ["local"]
+    output = ["build"]
+    args = {
+        BASE_VARIANT = USE_GLIBC != "" ? "buster" : "alpine"
+        VERSION = VERSION
+        GO_STRIP = STRIP_TARGET
+    }
+}
+
 target "platforms" {
     platforms = concat(["linux/amd64", "linux/386", "linux/arm64", "linux/arm", "linux/ppc64le", "linux/s390x", "darwin/amd64", "darwin/arm64", "windows/amd64", "windows/arm", "windows/386"], USE_GLIBC!=""?[]:["windows/arm64"])
 }
@@ -55,3 +67,13 @@ target "shellcheck" {
     target = "shellcheck"
     output = ["type=cacheonly"]
 }
+
+target "e2e-image" {
+    target = "e2e"
+    output = ["type=docker"]
+    tags = ["${IMAGE_NAME}"]
+    args = {
+        BASE_VARIANT = "buster"
+        VERSION = VERSION
+    }
+}

docker.Makefile

@@ -10,9 +10,7 @@ DOCKER_CLI_CONTAINER_NAME ?=
 DOCKER_CLI_GO_BUILD_CACHE ?= y
 
 DEV_DOCKER_IMAGE_NAME = docker-cli-dev$(IMAGE_TAG)
-BINARY_NATIVE_IMAGE_NAME = docker-cli-native$(IMAGE_TAG)
-CROSS_IMAGE_NAME = docker-cli-cross$(IMAGE_TAG)
-E2E_IMAGE_NAME = docker-cli-e2e$(IMAGE_TAG)
+E2E_IMAGE_NAME = docker-cli-e2e
 E2E_ENGINE_VERSION ?=
 CACHE_VOLUME_NAME := docker-cli-dev-cache
 ifeq ($(DOCKER_CLI_GO_BUILD_CACHE),y)
@@ -30,15 +28,6 @@ build_docker_image:
 	# build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment
 	cat ./dockerfiles/Dockerfile.dev | docker build ${DOCKER_BUILD_ARGS} --build-arg=GO_VERSION -t $(DEV_DOCKER_IMAGE_NAME) -
 
-.PHONY: build_binary_native_image
-build_binary_native_image:
-	# build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment
-	cat ./dockerfiles/Dockerfile.binary-native | docker build --build-arg=GO_VERSION -t $(BINARY_NATIVE_IMAGE_NAME) -
-
-.PHONY: build_e2e_image
-build_e2e_image:
-	docker build -t $(E2E_IMAGE_NAME) --build-arg=GO_VERSION --build-arg VERSION=$(VERSION) --build-arg GITCOMMIT=$(GITCOMMIT) -f ./dockerfiles/Dockerfile.e2e .
-
 DOCKER_RUN_NAME_OPTION := $(if $(DOCKER_CLI_CONTAINER_NAME),--name $(DOCKER_CLI_CONTAINER_NAME),)
 DOCKER_RUN := docker run --rm $(ENVVARS) $(DOCKER_CLI_MOUNTS) $(DOCKER_RUN_NAME_OPTION)
 
@@ -48,8 +37,8 @@ binary:
 
 build: binary ## alias for binary
 
-plugins: build_binary_native_image ## build the CLI plugin examples
-	$(DOCKER_RUN) $(BINARY_NATIVE_IMAGE_NAME) ./scripts/build/plugins
+plugins: ## build the CLI plugin examples
+	docker buildx bake plugins
 
 .PHONY: clean
 clean: build_docker_image ## clean build artifacts
@@ -67,14 +56,6 @@ test: test-unit test-e2e
 cross:
 	docker buildx bake cross
 
-.PHONY: plugins-windows
-plugins-windows: build_cross_image ## build the example CLI plugins for Windows
-	$(DOCKER_RUN) $(CROSS_IMAGE_NAME) make $@
-
-.PHONY: plugins-osx
-plugins-osx: build_cross_image ## build the example CLI plugins for macOS
-	$(DOCKER_RUN) $(CROSS_IMAGE_NAME) make $@
-
 .PHONY: dynbinary
 dynbinary: ## build dynamically linked binary
 	USE_GLIBC=1 docker buildx bake dynbinary
@@ -115,20 +96,24 @@ manpages: build_docker_image ## generate man pages from go source and markdown
 yamldocs: build_docker_image ## generate documentation YAML files consumed by docs repo
 	$(DOCKER_RUN) -it $(DEV_DOCKER_IMAGE_NAME) make yamldocs
 
+.PHONY: build-e2e-image
+build-e2e-image:
+	IMAGE_NAME=$(E2E_IMAGE_NAME) VERSION=$(VERSION) docker buildx bake e2e-image
+
 .PHONY: test-e2e
 test-e2e: test-e2e-non-experimental test-e2e-experimental test-e2e-connhelper-ssh ## run all e2e tests
 
 .PHONY: test-e2e-experimental
-test-e2e-experimental: build_e2e_image # run experimental e2e tests
-	docker run --rm --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 $(E2E_IMAGE_NAME)
+test-e2e-experimental: build-e2e-image # run experimental e2e tests
+	docker run --rm --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) $(E2E_IMAGE_NAME)
 
 .PHONY: test-e2e-non-experimental
-test-e2e-non-experimental: build_e2e_image # run non-experimental e2e tests
+test-e2e-non-experimental: build-e2e-image # run non-experimental e2e tests
 	docker run --rm --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock $(ENVVARS) -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) $(E2E_IMAGE_NAME)
 
 .PHONY: test-e2e-connhelper-ssh
-test-e2e-connhelper-ssh: build_e2e_image # run experimental SSH-connection helper e2e tests
-	docker run --rm --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 -e TEST_CONNHELPER=ssh $(E2E_IMAGE_NAME)
+test-e2e-connhelper-ssh: build-e2e-image # run experimental SSH-connection helper e2e tests
+	docker run --rm --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock $(ENVVARS) -e DOCKERD_EXPERIMENTAL=1 -e TEST_ENGINE_VERSION=$(E2E_ENGINE_VERSION) -e TEST_CONNHELPER=ssh $(E2E_IMAGE_NAME)
 
 .PHONY: help
 help: ## print this help