feat: Add a validation init container

.github/workflows/create-release.yml

@@ -17,6 +17,7 @@ env:
   # Common versions
   GO_VERSION: '1.18'
   IMAGE_NAME: 'virtual-kubelet'
+  INIT_IMAGE_NAME: 'init-validation'
 
 jobs:
   create-release:
@@ -63,6 +64,7 @@ jobs:
     - create-release
     env:
       REGISTRY: ${{ needs.export-registry.outputs.registry }}
+      INIT_IMG_TAG : 0.1.0
     runs-on: ubuntu-20.04
     steps:
       - id: get-tag
@@ -86,6 +88,7 @@ jobs:
       - name: Build and push image
         run: |
           OUTPUT_TYPE=type=registry make docker-build-image
+          OUTPUT_TYPE=type=registry make docker-build-init-image
         env:
           VERSION: ${{ env.IMG_TAG }}
 
@@ -102,3 +105,17 @@ jobs:
         env:
           TRIVY_USERNAME: ${{ github.actor }}
           TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Scan ${{ env.REGISTRY }}/${{ env.INIT_IMAGE_NAME }}:${{ env.INIT_IMG_TAG }}
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ${{ env.REGISTRY }}/${{ env.INIT_IMAGE_NAME }}:${{ env.INIT_IMG_TAG }}
+          format: 'table'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'
+          timeout: '5m0s'
+        env:
+          TRIVY_USERNAME: ${{ github.actor }}
+          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}

Makefile

@@ -23,14 +23,18 @@ TEST_LOGANALYTICS_JSON ?= $(TEST_CREDENTIALS_DIR)/loganalytics.json
 export TEST_CREDENTIALS_JSON TEST_LOGANALYTICS_JSON
 
 VERSION ?= v1.4.9
+REGISTRY ?= ghcr.io
 IMG_NAME ?= virtual-kubelet
+INIT_IMG_NAME ?= init-validation
 IMAGE ?= $(REGISTRY)/$(IMG_NAME)
+INIT_IMAGE ?= $(REGISTRY)/$(INIT_IMG_NAME)
 LOCATION := $(E2E_REGION)
 E2E_CLUSTER_NAME := $(CLUSTER_NAME)
 
 OUTPUT_TYPE ?= type=docker
 BUILDPLATFORM ?= linux/amd64
 IMG_TAG ?= $(subst v,,$(VERSION))
+INIT_IMG_TAG ?= 0.1.0
 
 
 ## --------------------------------------
@@ -58,12 +62,21 @@ docker-buildx-builder:
 .PHONY: docker-build-image
 docker-build-image: docker-buildx-builder
 	docker buildx build \
-		--file Dockerfile \
+		--file docker/virtual-kubelet/Dockerfile \
 		--output=$(OUTPUT_TYPE) \
 		--platform="$(BUILDPLATFORM)" \
 		--pull \
 		--tag $(IMAGE):$(IMG_TAG) .
 
+.PHONY: docker-build-init-image
+docker-build-init-image: docker-buildx-builder
+	docker buildx build \
+		--file docker/init-container/Dockerfile \
+		--output=$(OUTPUT_TYPE) \
+		--platform="$(BUILDPLATFORM)" \
+		--pull \
+		--tag $(INIT_IMAGE):$(INIT_IMG_TAG) .
+
 .PHONY: build
 build: bin/virtual-kubelet
 

charts/virtual-kubelet/README.md

@@ -61,6 +61,9 @@ The following table lists the configurable parameters of the azure-aci chart and
 | image.name                                     | Image name.                                                                                                           | `oss/virtual-kubelet/virtual-kubelet` |
 | image.tag                                      | Image release version/tag.                                                                                            | `latest`                              |
 | image.pullPolicy                               | Image pull policy.                                                                                                    | `Always`                              | 
+| initImage.name                                 | Init container image name.                                                                                            | `oss/virtual-kubelet/init-validation` |
+| initImage.tag                                  | Init container image release version/tag.                                                                             | `0.1.0`                               |
+| initImage.pullPolicy                           | Init container image pull policy.                                                                                     | `Always`                              |
 | nodeName                                       | The node name that will be assigned to be the VK one.                                                                 | `virtual-node-aci-linux-helm`         |
 | nodeOsType                                     | The node/VM type. Values should be `Windows` or `Linux`.                                                              | `Linux`                               |
 | monitoredNamespace                             | Kubernetes namespace. default values means monitor `all`                                                              | `""`                                  |

charts/virtual-kubelet/templates/deployment.yaml

@@ -19,6 +19,51 @@ spec:
       labels:
         app: {{ template "vk.fullname" . }}
     spec:
+    {{- if .Values.useVKVersion2}}
+      initContainers:
+        - name: init-validation
+          image: "{{ .Values.initImage.repository }}/{{ .Values.initImage.name }}:{{ .Values.initImage.tag }}"
+          imagePullPolicy: {{ .Values.initImage.pullPolicy }}
+          env:
+            - name: VIRTUALNODE_USER_IDENTITY_CLIENTID
+              value: {{ .managedIdentityID }}
+            - name: AZURE_CLIENT_ID
+              value: {{ .clientId }}
+            - name: AZURE_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "vk.fullname" $ }}
+                  key: clientSecret
+            - name: AKS_CREDENTIAL_LOCATION
+              value: /etc/aks/azure.json
+
+ {{- if .Values.providers.azure.vnet.enabled }}
+            - name: ACI_SUBNET_NAME
+              value: {{ required "subnetName is required" .vnet.subnetName }}
+            - name: KUBE_DNS_IP
+              value:{{ .vnet.kubeDnsIp }}
+ {{- end }}
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: USE_VK_VERSION_2
+              value: "true"
+          volumeMounts:
+            - name: credentials
+              mountPath: "/etc/virtual-kubelet"
+            - name: certificates
+              mountPath: /etc/kubernetes/certs
+              readOnly: true
+          {{- if .Values.providers.azure.targetAKS }}
+            - name: aks-credential
+              mountPath: "/etc/aks/azure.json"
+          {{- end }}
+    {{- end }}
       containers:
       - name: {{ template "vk.fullname" . }}
         image: "{{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag }}"

charts/virtual-kubelet/values.yaml

@@ -5,7 +5,14 @@ image:
   tag: 1.4.9
   pullPolicy: Always
 
+initImage:
+  repository: mcr.microsoft.com
+  name: init-validation
+  tag: 0.1.0
+  pullPolicy: Always
+
 namespace: vk-azure-aci
+
 useVKVersion2: true
 
 nodeName: "virtual-node-aci-linux-helm"

cmd/init-container/main.go

@@ -0,0 +1,117 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the Apache 2.0 license.
+*/
+package main
+
+import (
+	"context"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/virtual-kubelet/azure-aci/pkg/auth"
+	"github.com/virtual-kubelet/azure-aci/pkg/network"
+	"github.com/virtual-kubelet/azure-aci/pkg/util"
+	cli "github.com/virtual-kubelet/node-cli"
+	logruscli "github.com/virtual-kubelet/node-cli/logrus"
+	"github.com/virtual-kubelet/virtual-kubelet/log"
+	logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
+)
+
+func main() {
+	ctx := cli.ContextWithCancelOnSignal(context.Background())
+
+	logger := logrus.StandardLogger()
+	log.L = logruslogger.FromLogrus(logrus.NewEntry(logger))
+	_ = logruscli.Config{LogLevel: "debug"}
+
+	log.G(ctx).Debug("Init container started")
+
+	podName := os.Getenv("POD_NAME")
+	podNamespace := os.Getenv("NAMESPACE")
+
+	if podName == "" || podNamespace == "" {
+		log.G(ctx).Fatal("an error has occurred while retrieve the pod info ")
+	}
+
+	config, err := clientcmd.BuildConfigFromFlags("", "")
+	if err != nil {
+		log.G(ctx).Fatal("an error has occurred while creating client ", err)
+	}
+
+	kubeClient := kubernetes.NewForConfigOrDie(config)
+	eventBroadcast := util.NewRecorder(ctx, kubeClient)
+	defer eventBroadcast.Shutdown()
+
+	recorder := eventBroadcast.NewRecorder(scheme.Scheme, v1.EventSource{Component: "virtual kubelet"})
+	vkVersion, err := strconv.ParseBool(os.Getenv("USE_VK_VERSION_2"))
+	if err != nil {
+		log.G(ctx).Warn("cannot get USE_VK_VERSION_2 environment variable, the provider will use VK version 1. Skipping init container checks")
+		return
+	}
+
+	if !vkVersion {
+		log.G(ctx).Warn("the provider will use VK version 1. Skipping init container checks")
+		return
+	}
+
+	setupBackoff := wait.Backoff{
+		Steps:    50,
+		Duration: time.Minute,
+		Factor:   0,
+		Jitter:   0.01,
+	}
+	azConfig := auth.Config{}
+
+	//Setup config
+	err = azConfig.SetAuthConfig(ctx)
+	if err != nil {
+		log.G(ctx).Fatalf("cannot setup the auth configuration. Retrying, ", err)
+	}
+
+	err = retry.OnError(setupBackoff,
+		func(err error) bool {
+			return true
+		}, func() error {
+			var providerNetwork network.ProviderNetwork
+			if azConfig.AKSCredential != nil {
+				providerNetwork.VnetName = azConfig.AKSCredential.VNetName
+				if azConfig.AKSCredential.VNetResourceGroup != "" {
+					providerNetwork.VnetResourceGroup = azConfig.AKSCredential.VNetResourceGroup
+				} else {
+					providerNetwork.VnetResourceGroup = azConfig.AKSCredential.ResourceGroup
+				}
+			}
+			// Check or set up a network for VK
+			log.G(ctx).Debug("setting up the network configuration")
+			err = providerNetwork.SetVNETConfig(ctx, &azConfig)
+			if err != nil {
+				log.G(ctx).Errorf("cannot setup the VNet configuration. Retrying", err)
+				return err
+			}
+			return nil
+		})
+
+	if err != nil {
+		recorder.Eventf(&v1.ObjectReference{
+			Kind:      "Pod",
+			Name:      podName,
+			Namespace: podNamespace,
+		}, v1.EventTypeWarning, "InitFailed", "VNet config setup failed")
+		log.G(ctx).Fatal("cannot setup the VNet configuration ", err)
+	}
+	recorder.Eventf(&v1.ObjectReference{
+		Kind:      "Pod",
+		Name:      podName,
+		Namespace: podNamespace,
+	}, v1.EventTypeNormal, "InitSuccess", "initial setup for virtual kubelet Azure ACI is successful")
+	log.G(ctx).Info("initial setup for virtual kubelet Azure ACI is successful")
+}

cmd/virtual-kubelet/main.go

@@ -77,7 +77,7 @@ func main() {
 
 	if vkVersion {
 		//Setup config
-		err = azConfig.SetAuthConfig()
+		err = azConfig.SetAuthConfig(ctx)
 		if err != nil {
 			log.G(ctx).Fatal(err)
 		}

deploy/deployment.yaml

@@ -28,6 +28,53 @@ spec:
                     operator: In
                     values:
                       - linux
+      initContainers:
+      - name: init-validation
+        image: TEST_INIT_IMAGE
+        imagePullPolicy: Always
+        env:
+        - name: AKS_CREDENTIAL_LOCATION
+          value: /etc/aks/azure.json
+        - name: AZURE_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: aci-connector-linux
+              key: clientSecret
+        - name: ACI_EXTRA_USER_AGENT
+          value: "deploy/aks/azure-aci/e2e-test"
+        - name: ACI_SUBNET_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: test-vars
+              key: aci_subnet_name
+        - name: KUBE_DNS_IP
+          valueFrom:
+            configMapKeyRef:
+              name: test-vars
+              key: kube_dns_ip
+        - name: USE_VK_VERSION_2
+          value: "true"
+        - name: VIRTUALNODE_USER_IDENTITY_CLIENTID
+          valueFrom:
+            configMapKeyRef:
+              name: test-vars
+              key: aci_user_identity
+        - name: POD_NAME
+          valueFrom:
+           fieldRef:
+            fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: certificates
+          mountPath: /etc/kubernetes/certs
+          readOnly: true
+        - name: credentials
+          mountPath: "/etc/virtual-kubelet"
+        - name: aks-credential
+          mountPath: "/etc/aks/azure.json"
       containers:
       - name: vk-azure-aci
         image: TEST_IMAGE
@@ -50,7 +97,6 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
-
         - name: ACI_EXTRA_USER_AGENT
           value: "deploy/aks/azure-aci/e2e-test"
         - name: ACI_SUBNET_NAME
@@ -83,13 +129,13 @@ spec:
               name: test-vars
               key: aci_user_identity
         volumeMounts:
-          - name: certificates
-            mountPath: /etc/kubernetes/certs
-            readOnly: true
-          - name: credentials
-            mountPath: "/etc/virtual-kubelet"
-          - name: aks-credential
-            mountPath: "/etc/aks/azure.json"
+        - name: certificates
+          mountPath: /etc/kubernetes/certs
+          readOnly: true
+        - name: credentials
+          mountPath: "/etc/virtual-kubelet"
+        - name: aks-credential
+          mountPath: "/etc/aks/azure.json"
         command: ["virtual-kubelet"]
         args: [
           "--provider", "azure",