ci: Add a linkerd install workflow (#2268)

We don't currently test that proxy changes are at least usable in a
basic Linkerd installation. This adds a CI workflow to test all Linkerd
code changes against a real k8s cluster with the control plane from the
most recent edge release.

Author: olix0r

.github/workflows/docker.yml

@@ -0,0 +1,43 @@
+name: k8s
+
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    paths:
+      - Cargo.lock
+      - Dockerfile
+      - "**/*.rs"
+      - "**/*.toml"
+      - justfile
+      - .github/workflows/k8s.yml
+
+jobs:
+  k3d-linkerd-install:
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: linkerd/dev/actions/setup-tools@v39
+
+      - name: Install linkerd CLI (edge)
+        id: linkerd
+        run: |
+          scurl https://run.linkerd.io/install-edge | sh
+          echo "PATH=$PATH:$HOME/.linkerd2/bin" >> "$GITHUB_ENV"
+          export PATH="$PATH:$HOME/.linkerd2/bin"
+          tag=$(linkerd version --client --short)
+          echo "linkerd $tag"
+          echo "LINKERD_TAG=$tag" >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+      - run: just docker
+
+      - run: just-k3d create
+      - run: just k3d-load-linkerd
+
+      - run: just linkerd-install
+      - run: just linkerd-check-contol-plane-proxy
+        env:
+          TMPDIR: ${{ runner.temp }}

.github/workflows/k8s.yml

@@ -1,30 +1,15 @@
-# syntax=docker/dockerfile:experimental
+# syntax=docker/dockerfile:1.4
 
-# Proxy build and runtime
-#
 # This is intended **DEVELOPMENT ONLY**, i.e. so that proxy developers can
 # easily test the proxy in the context of the larger `linkerd2` project.
-#
-# This Dockerfile requires expirmental features to be enabled in both the
-# Docker client and daemon. You MUST use buildkit to build this image. The
-# simplest way to do this is to set the `DOCKER_BUILDKIT` environment variable:
-#
-#     :; DOCKER_BUILDKIT=1 docker build .
-#
-# Alternatively, you can use `buildx`, which supports more complicated build
-# configurations:
-#
-#     :; docker buildx build . --load
 
 ARG RUST_IMAGE=ghcr.io/linkerd/dev:v39-rust
 
 # Use an arbitrary ~recent edge release image to get the proxy
 # identity-initializing and linkerd-await wrappers.
 ARG RUNTIME_IMAGE=ghcr.io/linkerd/proxy:edge-22.12.1
 
-# Build the proxy, leveraging (new, experimental) cache mounting.
-#
-# See: https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md#run---mounttypecache
+# Build the proxy.
 FROM --platform=$BUILDPLATFORM $RUST_IMAGE as build
 
 ARG PROXY_FEATURES=""
@@ -35,6 +20,11 @@ RUN apt-get update && \
     fi && \
     rm -rf /var/lib/apt/lists/*
 
+ENV CARGO_INCREMENTAL=0
+ENV CARGO_NET_RETRY=10
+ENV RUSTFLAGS="-D warnings -A deprecated"
+ENV RUSTUP_MAX_RETRIES=10
+
 WORKDIR /usr/src/linkerd2-proxy
 COPY . .
 RUN --mount=type=cache,id=cargo,target=/usr/local/cargo/registry \
@@ -43,9 +33,9 @@ ARG TARGETARCH="amd64"
 ARG PROFILE="release"
 RUN --mount=type=cache,id=target,target=target \
     --mount=type=cache,id=cargo,target=/usr/local/cargo/registry \
-    just arch=$TARGETARCH features=$PROXY_FEATURES profile=$PROFILE build && \
-    bin=$(just --evaluate profile="$PROFILE" _target_bin) ; \
-    mkdir -p /out && mv $bin /out/linkerd2-proxy
+    just arch="$TARGETARCH" features="$PROXY_FEATURES" profile="$PROFILE" build && \
+    mkdir -p /out && \
+    mv $(just --evaluate profile="$PROFILE" _target_bin) /out/linkerd2-proxy
 
 ## Install the proxy binary into the base runtime image.
 FROM $RUNTIME_IMAGE as runtime

Dockerfile

@@ -19,9 +19,9 @@ features := ""
 package_version := `git rev-parse --short HEAD`
 
 # Docker image name & tag.
-docker_repo := "localhost/linkerd/proxy"
-docker_tag := `git rev-parse --abbrev-ref HEAD | sed 's|/|.|'` + "." + `git rev-parse --short HEAD`
-docker_image := docker_repo + ":" + docker_tag
+docker-repo := "localhost/linkerd/proxy"
+docker-tag := `git rev-parse --abbrev-ref HEAD | sed 's|/|.|'` + "." + `git rev-parse --short HEAD`
+docker-image := docker-repo + ":" + docker-tag
 
 # The architecture name to use for packages. Either 'amd64', 'arm64', or 'arm'.
 arch := "amd64"
@@ -167,15 +167,22 @@ fuzzers:
         )
     done
 
+export DOCKER_BUILDX_CACHE_DIR := env_var_or_default('DOCKER_BUILDX_CACHE_DIR', '')
+
 # Build a docker image (FOR TESTING ONLY)
-docker *args='--output=type=docker':
+docker *args='--output=type=docker': && _clean-cache
     docker buildx build . \
         --pull \
-        --tag={{ docker_image }} \
+        --tag={{ docker-image }} \
         --build-arg PROFILE='{{ profile }}' \
+        {{ if linkerd-tag == '' { '' } else { '--build-arg=RUNTIME_IMAGE=ghcr.io/linkerd/proxy:' + linkerd-tag } }} \
         {{ if features != "" { "--build-arg PROXY_FEATURES=" + features } else { "" } }} \
+        {{ if DOCKER_BUILDX_CACHE_DIR == '' { '' } else { '--cache-from=type=local,src=' + DOCKER_BUILDX_CACHE_DIR + ' --cache-to=type=local,dest=' + DOCKER_BUILDX_CACHE_DIR } }} \
         {{ args }}
 
+_clean-cache:
+    @{{ if DOCKER_BUILDX_CACHE_DIR == '' { 'true' } else { 'just-dev prune-action-cache ' + DOCKER_BUILDX_CACHE_DIR } }}
+
 # Lints all shell scripts in the repo.
 sh-lint:
     @just-sh
@@ -189,3 +196,88 @@ action-lint:
 
 action-dev-check:
     @just-dev check-action-images
+
+##
+## Linkerd
+##
+
+linkerd-tag := env_var_or_default('LINKERD_TAG', '')
+_controller-image := 'ghcr.io/linkerd/controller'
+_policy-image := 'ghcr.io/linkerd/policy-controller'
+_init-image := 'ghcr.io/linkerd/proxy-init'
+_init-tag := 'v2.2.0'
+
+_kubectl := 'just-k3d kubectl'
+
+_tag-set:
+    #!/usr/bin/env bash
+    if [ -z '{{ linkerd-tag }}' ]; then
+        echo "linkerd-tag must be set" >&2
+        exit 1
+    fi
+
+_k3d-ready:
+    @just-k3d ready
+
+k3d-load-linkerd: _tag-set _k3d-ready
+    for i in \
+        '{{ _controller-image }}:{{ linkerd-tag }}' \
+        '{{ _policy-image }}:{{ linkerd-tag }}' \
+        '{{ _init-image }}:{{ _init-tag }}' \
+    ; do \
+        docker pull -q "$i" ; \
+    done
+    @just-k3d import \
+        '{{ docker-image }}' \
+        '{{ _controller-image }}:{{ linkerd-tag }}' \
+        '{{ _policy-image }}:{{ linkerd-tag }}' \
+        '{{ _init-image }}:{{ _init-tag }}'
+
+# Install crds on the test cluster.
+_linkerd-crds-install: _k3d-ready
+    linkerd install --crds \
+        | {{ _kubectl }} apply -f -
+    {{ _kubectl }} wait crd --for condition=established \
+        --selector='linkerd.io/control-plane-ns' \
+        --timeout=1m
+
+# Install linkerd on the test cluster using test images.
+linkerd-install *args='': _tag-set k3d-load-linkerd _linkerd-crds-install && _linkerd-ready
+    linkerd install \
+            --set='imagePullPolicy=Never' \
+            --set='controllerImage={{ _controller-image }}' \
+            --set='linkerdVersion={{ linkerd-tag }}' \
+            --set='policyController.image.name={{ _policy-image }}' \
+            --set='policyController.image.version={{ linkerd-tag }}' \
+            --set='proxy.image.name={{ docker-repo }}' \
+            --set='proxy.image.version={{ docker-tag }}' \
+            --set='proxy.logLevel=linkerd=debug\,info' \
+            --set='proxyInit.image.name={{ _init-image }}' \
+            --set='proxyInit.image.version={{ _init-tag }}' \
+            {{ args }} \
+        | {{ _kubectl }} apply -f -
+
+linkerd-uninstall:
+    linkerd uninstall \
+        | {{ _kubectl }} delete -f -
+
+linkerd-check-contol-plane-proxy:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    check=$(mktemp --tmpdir check-XXXX.json)
+    linkerd check -o json > "$check"
+    result=$(jq -r \
+        '.categories[] | select(.categoryName == "linkerd-control-plane-proxy") | .checks[] | select(.description == "control plane proxies are healthy") | .result' \
+        "$check")
+    if [ "$result" != "success" ]; then
+        jq '.categories[] | .checks[] | select(.result != "success") | select(.hint | contains("-version") | not)' \
+            "$check" >&2
+        {{ _kubectl }} describe po -n linkerd >&2
+        exit 1
+    fi
+    rm "$check"
+
+_linkerd-ready:
+    {{ _kubectl }} wait pod --for=condition=ready \
+        --namespace=linkerd --selector='linkerd.io/control-plane-component' \
+        --timeout=1m