add support for kind clusters

.envrc

@@ -1,3 +1,4 @@
 layout go
 dotenv_if_exists
+export KUBECONFIG=$(direnv_layout_dir)/kubeconfig
 use flake

Makefile

@@ -1,3 +1,4 @@
+REGISTRY ?= localhost:5001
 
 # Image URL to use all building/pushing image targets
 IMG ?= kubernetes-ingress-controller
@@ -94,11 +95,11 @@ run: manifests generate fmt vet ## Run a controller from your host.
 
 .PHONY: docker-build
 docker-build: test ## Build docker image with the manager.
-	DOCKER_BUILDKIT=1 docker build -t ${IMG} .
+	DOCKER_BUILDKIT=1 docker build -t ${REGISTRY}/${IMG} .
 
 .PHONY: docker-push
-docker-push: ## Push docker image with the manager.
-	docker push ${IMG}
+docker-push: docker-build ## Push docker image with the manager.
+	docker push ${REGISTRY}/${IMG}
 
 ##@ Deployment
 
@@ -115,12 +116,14 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 	$(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
 
 .PHONY: deploy
-deploy: _deploy-check-env-vars docker-build manifests kustomize _helm_setup ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+deploy: _deploy-check-env-vars docker-push manifests kustomize _helm_setup ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	helm upgrade ngrok-ingress-controller $(HELM_CHART_DIR) --install \
 		--namespace ngrok-ingress-controller \
 		--create-namespace \
+		--set image.registry=$(REGISTRY) \
 		--set image.repository=$(IMG) \
 		--set image.tag="latest" \
+		--set image.pullPolicy="Always" \
 		--set podAnnotations."k8s\.ngrok\.com/test"="\{\"env\": \"local\"\}" \
 		--set credentials.apiKey=$(NGROK_API_KEY) \
 		--set credentials.authtoken=$(NGROK_AUTHTOKEN) \

docs/developer-guide/README.md

@@ -11,10 +11,13 @@ Have a look at the architecture guide on the internal workings of the ingress co
 - [Go 1.20](https://go.dev/dl/)
 - [Helm](https://helm.sh/docs/intro/install/)
 
-Both of these can be obtained via [nix-direnv](https://github.com/nix-community/nix-direnv), which will automatically configure your shell for you.
+Both of these can be obtained via [nix-direnv](https://github.com/nix-community/nix-direnv), which will automatically configure your shell for you. Note that it will also set `KUBECONFIG` to a file "owned" by direnv, so as not to pollute the configuration in your `$HOME`.
 
-- A k8s cluster is available via your kubectl client. This can be a remote cluster or a local cluster like [minikube](https://minikube.sigs.k8s.io/docs/start/)
-  - NOTE: Depending on your cluster, you may have to take additional steps to make the image available. For example with minikube, you may need to run `eval $(minikube docker-env)` in each terminal session to make the image from `make deploy` available to the cluster.
+A k8s cluster is available via your kubectl client. This can be a remote cluster or a local cluster like [kind](https://kind.sigs.k8s.io/) or [minikube](https://minikube.sigs.k8s.io/docs/start/).
+
+Depending on your cluster, you may have to take additional steps to make the image available. For example with minikube, you may need to run `eval $(minikube docker-env)` in each terminal session to make the image from `make deploy` available to the cluster.
+
+Support for `kind` is provided out of the box. `scripts/kind-up.sh` will create a kind cluster and an accompanying docker registry, and `scripts/kind-down.sh` will tear both down. The `docker-push` and `deploy` tasks in the provided `Makefile` are configured to use this registry.
 
 ### Setup
 

flake.nix

@@ -15,6 +15,8 @@
           gotools
           go-tools
           golangci-lint
+          kind
+          kubectl
           kubernetes-helm
           kubebuilder
           jq

scripts/kind-down.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+kind delete cluster --name ngrok-ingress-controller
+docker stop kind-registry | xargs docker rm

scripts/kind-up.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+set -o errexit
+
+cluster_name=ngrok-ingress-controller
+
+# 1. Create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
+    registry:2
+fi
+
+# 2. Create kind cluster with containerd registry config dir enabled
+# TODO: kind will eventually enable this by default and this patch will
+# be unnecessary.
+#
+# See:
+# https://github.com/kubernetes-sigs/kind/issues/2875
+# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration
+# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
+cat <<EOF | kind create cluster --name $cluster_name --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry]
+    config_path = "/etc/containerd/certs.d"
+EOF
+
+# 3. Add the registry config to the nodes
+#
+# This is necessary because localhost resolves to loopback addresses that are
+# network-namespace local.
+# In other words: localhost in the container is not localhost on the host.
+#
+# We want a consistent name that works from both ends, so we tell containerd to
+# alias localhost:${reg_port} to the registry container when pulling images
+REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
+for node in $(kind get nodes --name $cluster_name); do
+  docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
+  cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
+[host."http://${reg_name}:5000"]
+EOF
+done
+
+# 4. Connect the registry to the cluster network if not already connected
+# This allows kind to bootstrap the network but ensures they're on the same network
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+  docker network connect "kind" "${reg_name}"
+fi
+
+# 5. Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
+
+kubectl config set-context --current --namespace=ngrok-ingress-controller