Add a validation init container

.github/workflows/create-release.yml

@@ -14,6 +14,7 @@ env:
  # Common versions
  GO_VERSION: '1.18'
  IMAGE_NAME: 'virtual-kubelet'
+ INIT_IMAGE_NAME: 'init-validation'
 
 jobs:
  create-release:
@@ -49,6 +50,7 @@ jobs:
  needs: export-registry
  env:
  REGISTRY: ${{ needs.export-registry.outputs.registry }}
+ INIT_IMG_TAG : 0.1.0
  runs-on: ubuntu-20.04
  steps:
  - uses: actions/checkout@v3
@@ -70,6 +72,7 @@ jobs:
  - name: Build and push image
  run: |
  OUTPUT_TYPE=type=registry make docker-build-image
+ OUTPUT_TYPE=type=registry make docker-build-init-image
  env:
  VERSION: ${{ env.IMG_TAG }}
 
@@ -86,3 +89,17 @@ jobs:
  env:
  TRIVY_USERNAME: ${{ github.actor }}
  TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Scan ${{ env.REGISTRY }}/${{ env.INIT_IMAGE_NAME }}:${{ env.INIT_IMG_TAG }}
+ uses: aquasecurity/trivy-action@master
+ with:
+ image-ref: ${{ env.REGISTRY }}/${{ env.INIT_IMAGE_NAME }}:${{ env.INIT_IMG_TAG }}
+ format: 'table'
+ exit-code: '1'
+ ignore-unfixed: true
+ vuln-type: 'os,library'
+ severity: 'CRITICAL,HIGH'
+ timeout: '5m0s'
+ env:
+ TRIVY_USERNAME: ${{ github.actor }}
+ TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}

Makefile

@@ -18,14 +18,18 @@ TEST_CREDENTIALS_JSON ?= $(TEST_CREDENTIALS_DIR)/credentials.json
 TEST_LOGANALYTICS_JSON ?= $(TEST_CREDENTIALS_DIR)/loganalytics.json
 export TEST_CREDENTIALS_JSON TEST_LOGANALYTICS_JSON
 
+REGISTRY ?= ghcr.io
 IMG_NAME ?= virtual-kubelet
+INIT_IMG_NAME ?= init-validation
 IMAGE ?= $(REGISTRY)/$(IMG_NAME)
+INIT_IMAGE ?= $(REGISTRY)/$(INIT_IMG_NAME)
 LOCATION := $(E2E_REGION)
 E2E_CLUSTER_NAME := $(CLUSTER_NAME)
 
 OUTPUT_TYPE ?= type=docker
 BUILDPLATFORM ?= linux/amd64
 IMG_TAG ?= $(subst v,,$(VERSION))
+INIT_IMG_TAG ?= 0.1.0
 
 
 ## --------------------------------------
@@ -35,13 +39,6 @@ IMG_TAG ?= $(subst v,,$(VERSION))
 $(GOLANGCI_LINT):
  GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) github.com/golangci/golangci-lint/cmd/golangci-lint $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER)
 
-.PHONY: safebuild
-# docker build
-safebuild:
- @echo "Building image..."
- docker build -t $(DOCKER_IMAGE):$(VERSION) .
-
-
 BUILDX_BUILDER_NAME ?= img-builder
 QEMU_VERSION ?= 5.2.0-2
 
@@ -56,12 +53,21 @@ docker-buildx-builder:
 .PHONY: docker-build-image
 docker-build-image: docker-buildx-builder
  docker buildx build \
- --file Dockerfile \
+ --file docker/virtual-kubelet/Dockerfile \
  --output=$(OUTPUT_TYPE) \
  --platform="$(BUILDPLATFORM)" \
  --pull \
  --tag $(IMAGE):$(IMG_TAG) .
 
+.PHONY: docker-build-init-image
+docker-build-init-image: docker-buildx-builder
+ docker buildx build \
+ --file docker/init-container/Dockerfile \
+ --output=$(OUTPUT_TYPE) \
+ --platform="$(BUILDPLATFORM)" \
+ --pull \
+ --tag $(INIT_IMAGE):$(INIT_IMG_TAG) .
+
 .PHONY: build
 build: bin/virtual-kubelet
 

charts/virtual-kubelet/README.md

@@ -55,6 +55,9 @@ The following table lists the configurable parameters of the azure-aci chart and
 | image.name | Image name. | `oss/virtual-kubelet/virtual-kubelet` |
 | image.tag | Image release version/tag. | `1.4.7` |
 | image.pullPolicy | Image pull policy. | `Always` | 
+| initImage.name | Init container image name. | `oss/virtual-kubelet/init-validation` |
+| initImage.tag | Init container image release version/tag. | `0.1.0` |
+| initImage.pullPolicy | Init container image pull policy. | `Always` |
 | nodeName | The node name that will be assigned to be the VK one. | `virtual-node-aci-linux-helm` |
 | nodeOsType | The node/VM type. Values should be `Windows` or `Linux`. | `Linux` |
 | monitoredNamespace | Kubernetes namespace. default values means monitor `all` | `""` |

charts/virtual-kubelet/templates/deployment.yaml

@@ -19,6 +19,36 @@ spec:
  labels:
  app: {{ template "vk.fullname" . }}
  spec:
+ {{- if .Values.useVKVersion2}}
+ initContainers:
+ - name: init-validation
+ image: "{{ .Values.image.repository }}/{{ .Values.initImage.name }}:{{ .Values.initImage.tag }}"
+ imagePullPolicy: {{ .Values.initImage.pullPolicy }}
+ env:
+ - name: VIRTUALNODE_USER_IDENTITY_CLIENTID
+ value: {{ .managedIdentityID }}
+ - name: AZURE_CLIENT_ID
+ value: {{ .clientId }}
+ - name: AZURE_CLIENT_SECRET
+ valueFrom:
+ secretKeyRef:
+ name: {{ template "vk.fullname" $ }}
+ key: clientSecret
+ - name: AKS_CREDENTIAL_LOCATION
+ value: /etc/aks/azure.json
+ - name: AZURE_AUTH_LOCATION
+ value: /etc/virtual-kubelet/credentials.json
+ volumeMounts:
+ - name: credentials
+ mountPath: "/etc/virtual-kubelet"
+ - name: certificates
+ mountPath: /etc/kubernetes/certs
+ readOnly: true
+ {{- if .Values.providers.azure.targetAKS }}
+ - name: aks-credential
+ mountPath: "/etc/aks/azure.json"
+ {{- end }}
+ {{- end }}
  containers:
  - name: {{ template "vk.fullname" . }}
  image: "{{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag }}"

charts/virtual-kubelet/values.yaml

@@ -5,7 +5,13 @@ image:
  tag: 1.4.7
  pullPolicy: Always
 
+initImage:
+ name: init-validation
+ tag: 0.1.0
+ pullPolicy: Always
+
 namespace: vk-azure-aci
+
 useVKVersion2: true
 
 nodeName: "virtual-node-aci-linux-helm"

cmd/init-container/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+ "context"
+ "os"
+ "strconv"
+
+ "github.com/sirupsen/logrus"
+ "github.com/virtual-kubelet/azure-aci/pkg/auth"
+ "github.com/virtual-kubelet/azure-aci/pkg/initcontainer"
+ cli "github.com/virtual-kubelet/node-cli"
+ "github.com/virtual-kubelet/virtual-kubelet/log"
+ logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus"
+)
+
+func main() {
+ logger := logrus.StandardLogger()
+ log.L = logruslogger.FromLogrus(logrus.NewEntry(logger))
+
+ ctx := cli.ContextWithCancelOnSignal(context.Background())
+
+ vkVersion, err := strconv.ParseBool(os.Getenv("USE_VK_VERSION_2"))
+ if err != nil {
+ log.G(ctx).Warn("cannot get USE_VK_VERSION_2 environment variable, the provider will use VK version 1. Skipping init container checks")
+ return
+ }
+
+ azConfig := auth.Config{}
+
+ if vkVersion {
+ //Setup config
+ err = azConfig.SetAuthConfig()
+ if err != nil {
+ log.G(ctx).Fatal(err)
+ }
+ }
+ // check role assignments
+ err = initcontainer.CheckRoleAssignments(ctx, azConfig)
+ if err != nil {
+ log.G(ctx).Fatal(err, "role assignments needed for virtual kubelet are not set.")
+ os.Exit(1)
+ }
+
+ err = initcontainer.CheckSubnetConfiguration(ctx, azConfig, "", "")
+ if err != nil {
+ log.G(ctx).Fatal(err, "virtual kubelet subnet is not configured with the user defiled route.")
+ os.Exit(1)
+ }
+}

deploy/deployment.yaml

@@ -28,6 +28,31 @@ spec:
  operator: In
  values:
  - linux
+ initContainers:
+ - name: init-validation
+ image: TEST_INIT_IMAGE
+ imagePullPolicy: Always
+ env:
+ - name: VIRTUALNODE_USER_IDENTITY_CLIENTID
+ valueFrom:
+ configMapKeyRef:
+ name: test-vars
+ key: aci_user_identity
+ - name: AZURE_CLIENT_SECRET
+ valueFrom:
+ secretKeyRef:
+ name: aci-connector-linux
+ key: clientSecret
+ - name: AKS_CREDENTIAL_LOCATION
+ value: /etc/aks/azure.json
+ volumeMounts:
+ - name: credentials
+ mountPath: "/etc/virtual-kubelet"
+ - name: certificates
+ mountPath: /etc/kubernetes/certs
+ readOnly: true
+ - name: aks-credential
+ mountPath: "/etc/aks/azure.json"
  containers:
  - name: vk-azure-aci
  image: TEST_IMAGE

docker/.dockerignore

@@ -0,0 +1,21 @@
+.dockerignore
+Dockerfile
+bin/
+helm/
+hack/
+charts/
+.circleci/
+README.md
+.gitignore
+
+# binary output
+bin/
+hack/tools/bin/
+
+.azure
+
+.DS_Store
+.vscode/
+.idea/
+.run/
+dist/

docker/init-container/Dockerfile

@@ -0,0 +1,30 @@
+FROM --platform=$BUILDPLATFORM golang:1.18 as builder
+ARG TARGETARCH
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+ENV GOCACHE=/root/gocache
+RUN \
+ --mount=type=cache,target=${GOCACHE} \
+ --mount=type=cache,target=/go/pkg/mod \
+ go mod download
+
+# Copy the go source
+COPY cmd/init-container/main.go main.go
+COPY pkg/auth pkg/auth
+COPY pkg/initcontainer pkg/initcontainer
+
+RUN --mount=type=cache,target=${GOCACHE} \
+ --mount=type=cache,id=vk-azure-aci-init,sharing=locked,target=/go/pkg/mod \
+ CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -o initcontainer main.go
+
+FROM gcr.io/distroless/static:nonroot
+WORKDIR /
+COPY --from=builder /workspace/initcontainer .
+USER 65532:65532
+
+ENTRYPOINT [ "/initcontainer" ]

goreleaser.yml

@@ -29,6 +29,26 @@ builds:
  - -X {{.ModulePath}}/pkg/version.Vcs={{.ShortCommit}}
  env:
  - CGO_ENABLED=0
+ - id: init-validation
+ dir: cmd/init-container
+ binary: bin/init-container
+ goos:
+ - linux
+ goarch:
+ - amd64
+ - arm64
+ goarm:
+ - 5
+ - 6
+ - 7
+ goamd64:
+ - v2
+ - v3
+ flags:
+ - -mod=vendor
+ env:
+ - CGO_ENABLED=0
+ - GO111MODULE=on
 
 release:
  prerelease: auto
@@ -37,6 +57,9 @@ release:
 changelog:
  skip: false
  groups:
+ - title: Features 🌈
+ regexp: ^.*feat[(\\w)]*:+.*$
+
  - title: Bug Fixes 🐞
  regexp: ^.*fix[(\\w)]*:+.*$
  - title: Build 🏭
@@ -49,8 +72,6 @@ changelog:
  regexp: ^.*ci[(\\w)]*:+.*$
  - title: Documentation 📘
  regexp: ^.*docs[(\\w)]*:+.*$
- - title: Features 🌈
- regexp: ^.*feat[(\\w)]*:+.*$
  - title: Maintenance 🔧
  regexp: ^.*chore[(\\w)]*:+.*$
  - title: Performance Improvements 🚀

hack/e2e/aks-addon.sh

@@ -27,6 +27,7 @@ fi
 : "${TEST_NODE_NAME:=vk-aci-test-aks}"
 : "${IMG_REPO:=oss/virtual-kubelet/virtual-kubelet}"
 : "${IMG_URL:=mcr.microsoft.com}"
+: "${INIT_IMG_TAG:=0.1.0}"
 : "${VNET_RANGE=10.0.0.0/8}"
 : "${CLUSTER_SUBNET_CIDR=10.240.0.0/16}"
 : "${ACI_SUBNET_CIDR=10.241.0.0/16}"
@@ -79,7 +80,10 @@ if [ "$E2E_TARGET" = "pr" ]; then
  az acr login --name "$ACR_NAME"
  IMG_URL=$ACR_NAME.azurecr.io
  IMG_REPO="virtual-kubelet"
+ INIT_IMG_REPO="init-validation"
  OUTPUT_TYPE=type=registry IMG_TAG=$IMG_TAG IMAGE=$ACR_NAME.azurecr.io/$IMG_REPO make docker-build-image
+ OUTPUT_TYPE=type=registry INIT_IMG_TAG=$INIT_IMG_TAG INIT_IMAGE=$ACR_NAME.azurecr.io/$INIT_IMG_REPO make docker-build-init-image
+
 fi
 
 TMPDIR="$(mktemp -d)"
@@ -151,7 +155,7 @@ kubectl create configmap test-vars -n kube-system \
  --from-literal=cluster_subnet_cidr="$CLUSTER_SUBNET_CIDR" \
  --from-literal=aci_subnet_name="$ACI_SUBNET_NAME"
 
-sed -e "s|TEST_IMAGE|$ACR_NAME.azurecr.io/$IMG_REPO:$IMG_TAG|g" deploy/deployment.yaml | kubectl apply -n kube-system -f -
+sed -e "s|TEST_INIT_IMAGE|$ACR_NAME.azurecr.io/$INIT_IMG_REPO:$INIT_IMG_TAG|g" -e "s|TEST_IMAGE|$ACR_NAME.azurecr.io/$IMG_REPO:$IMG_TAG|g" deploy/deployment.yaml | kubectl apply -n kube-system -f -
 
 kubectl wait --for=condition=available deploy "virtual-kubelet-azure-aci" -n kube-system --timeout=300s