Implement events for the init container

Signed-off-by: Heba Elayoty <hebaelayoty@gmail.com>

charts/virtual-kubelet/templates/deployment.yaml

@@ -22,7 +22,7 @@ spec:
     {{- if .Values.useVKVersion2}}
       initContainers:
         - name: init-validation
-          image: "{{ .Values.image.repository }}/{{ .Values.initImage.name }}:{{ .Values.initImage.tag }}"
+          image: "{{ .Values.initImage.repository }}/{{ .Values.initImage.name }}:{{ .Values.initImage.tag }}"
           imagePullPolicy: {{ .Values.initImage.pullPolicy }}
           env:
             - name: VIRTUALNODE_USER_IDENTITY_CLIENTID
@@ -36,8 +36,23 @@ spec:
                   key: clientSecret
             - name: AKS_CREDENTIAL_LOCATION
               value: /etc/aks/azure.json
-            - name: AZURE_AUTH_LOCATION
-              value: /etc/virtual-kubelet/credentials.json
+
+ {{- if .Values.providers.azure.vnet.enabled }}
+            - name: ACI_SUBNET_NAME
+              value: {{ required "subnetName is required" .vnet.subnetName }}
+            - name: KUBE_DNS_IP
+              value:{{ .vnet.kubeDnsIp }}
+ {{- end }}
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: USE_VK_VERSION_2
+              value: "true"
           volumeMounts:
             - name: credentials
               mountPath: "/etc/virtual-kubelet"

charts/virtual-kubelet/values.yaml

@@ -6,6 +6,7 @@ image:
   pullPolicy: Always
 
 initImage:
+  repository: mcr.microsoft.com
   name: init-validation
   tag: 0.1.0
   pullPolicy: Always

cmd/init-container/main.go

@@ -1,3 +1,7 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the Apache 2.0 license.
+*/
 package main
 
 import (
@@ -9,44 +13,97 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/virtual-kubelet/azure-aci/pkg/auth"
 	"github.com/virtual-kubelet/azure-aci/pkg/network"
+	"github.com/virtual-kubelet/azure-aci/pkg/util"
 	cli "github.com/virtual-kubelet/node-cli"
+	logruscli "github.com/virtual-kubelet/node-cli/logrus"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
 )
 
 func main() {
+	ctx := cli.ContextWithCancelOnSignal(context.Background())
+
 	logger := logrus.StandardLogger()
 	log.L = logruslogger.FromLogrus(logrus.NewEntry(logger))
+	_ = logruscli.Config{LogLevel: "debug"}
 
-	ctx := cli.ContextWithCancelOnSignal(context.Background())
+	log.G(ctx).Debug("Init container started")
+
+	podName := os.Getenv("POD_NAME")
+	podNamespace := os.Getenv("NAMESPACE")
+
+	if podName == "" || podNamespace == "" {
+		log.G(ctx).Fatal("an error has occurred while retrieve the pod info ")
+	}
+
+	config, err := clientcmd.BuildConfigFromFlags("", "")
+	if err != nil {
+		log.G(ctx).Fatal("an error has occurred while creating client ", err)
+	}
+
+	kubeClient := kubernetes.NewForConfigOrDie(config)
+	eventBroadcast := util.NewRecorder(ctx, kubeClient)
+	defer eventBroadcast.Shutdown()
 
+	recorder := eventBroadcast.NewRecorder(scheme.Scheme, v1.EventSource{Component: "k8s.io.api.core.v1.Pod"})
 	vkVersion, err := strconv.ParseBool(os.Getenv("USE_VK_VERSION_2"))
 	if err != nil {
-		log.G(ctx).Warn("init container: cannot get USE_VK_VERSION_2 environment variable, the provider will use VK version 1. Skipping init container checks")
+		log.G(ctx).Warn("cannot get USE_VK_VERSION_2 environment variable, the provider will use VK version 1. Skipping init container checks")
 		return
 	}
 
-	wait.Until(func() {
-		azConfig := auth.Config{}
+	if !vkVersion {
+		log.G(ctx).Warn("the provider will use VK version 1. Skipping init container checks")
+		return
+	}
 
-		if vkVersion {
-			//Setup config
-			err = azConfig.SetAuthConfig(ctx)
+	setupBackoff := wait.Backoff{
+		Steps:    50,
+		Duration: time.Minute,
+		Factor:   0,
+		Jitter:   0.01,
+	}
+	azConfig := auth.Config{}
+
+	//Setup config
+	err = azConfig.SetAuthConfig(ctx)
+	if err != nil {
+		log.G(ctx).Fatalf("cannot setup the auth configuration. Retrying, ", err)
+	}
+
+	err = retry.OnError(setupBackoff,
+		func(err error) bool {
+			return true
+		}, func() error {
+			var providerNetwork network.ProviderNetwork
+			// Check or set up a network for VK
+			log.G(ctx).Debug("setting up the network configuration")
+			err = providerNetwork.SetVNETConfig(ctx, &azConfig)
 			if err != nil {
-				log.G(ctx).Fatalf("init container: cannot setup the auth configuration ", err)
+				log.G(ctx).Errorf("cannot setup the VNet configuration. Retrying", err)
+				return err
 			}
-		}
-
-		var providerNetwork network.ProviderNetwork
-		// Check or set up a network for VK
-		log.G(ctx).Info("init container: setting up the network configuration")
-		err = providerNetwork.SetVNETConfig(ctx, &azConfig)
-		if err != nil {
-			log.G(ctx).Fatalf("init container: cannot setup the VNet configuration ", err)
-		}
-	},
-		1*time.Minute, ctx.Done())
+			return nil
+		})
 
+	if err != nil {
+		recorder.Eventf(&v1.ObjectReference{
+			Kind:      "Pod",
+			Name:      podName,
+			Namespace: podNamespace,
+		}, v1.EventTypeWarning, "InitFailed", "VNet config setup failed")
+		log.G(ctx).Fatal("cannot setup the VNet configuration ", err)
+	}
+	recorder.Eventf(&v1.ObjectReference{
+		Kind:      "Pod",
+		Name:      podName,
+		Namespace: podNamespace,
+	}, v1.EventTypeNormal, "InitSuccess", "initial setup for virtual kubelet Azure ACI is successful")
 	log.G(ctx).Info("initial setup for virtual kubelet Azure ACI is successful")
 }

deploy/deployment.yaml

@@ -29,30 +29,52 @@ spec:
                     values:
                       - linux
       initContainers:
-        - name: init-validation
-          image: TEST_INIT_IMAGE
-          imagePullPolicy: Always
-          env:
-            - name: VIRTUALNODE_USER_IDENTITY_CLIENTID
-              valueFrom:
-                configMapKeyRef:
-                  name: test-vars
-                  key: aci_user_identity
-            - name: AZURE_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: aci-connector-linux
-                  key: clientSecret
-            - name: AKS_CREDENTIAL_LOCATION
-              value: /etc/aks/azure.json
-          volumeMounts:
-            - name: credentials
-              mountPath: "/etc/virtual-kubelet"
-            - name: certificates
-              mountPath: /etc/kubernetes/certs
-              readOnly: true
-            - name: aks-credential
-              mountPath: "/etc/aks/azure.json"
+      - name: init-validation
+        image: TEST_INIT_IMAGE
+        imagePullPolicy: Always
+        env:
+        - name: AKS_CREDENTIAL_LOCATION
+          value: /etc/aks/azure.json
+        - name: AZURE_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: aci-connector-linux
+              key: clientSecret
+        - name: ACI_EXTRA_USER_AGENT
+          value: "deploy/aks/azure-aci/e2e-test"
+        - name: ACI_SUBNET_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: test-vars
+              key: aci_subnet_name
+        - name: KUBE_DNS_IP
+          valueFrom:
+            configMapKeyRef:
+              name: test-vars
+              key: kube_dns_ip
+        - name: USE_VK_VERSION_2
+          value: "true"
+        - name: VIRTUALNODE_USER_IDENTITY_CLIENTID
+          valueFrom:
+            configMapKeyRef:
+              name: test-vars
+              key: aci_user_identity
+        - name: POD_NAME
+          valueFrom:
+           fieldRef:
+            fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: certificates
+          mountPath: /etc/kubernetes/certs
+          readOnly: true
+        - name: credentials
+          mountPath: "/etc/virtual-kubelet"
+        - name: aks-credential
+          mountPath: "/etc/aks/azure.json"
       containers:
       - name: vk-azure-aci
         image: TEST_IMAGE
@@ -75,7 +97,6 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
-
         - name: ACI_EXTRA_USER_AGENT
           value: "deploy/aks/azure-aci/e2e-test"
         - name: ACI_SUBNET_NAME
@@ -108,13 +129,13 @@ spec:
               name: test-vars
               key: aci_user_identity
         volumeMounts:
-          - name: certificates
-            mountPath: /etc/kubernetes/certs
-            readOnly: true
-          - name: credentials
-            mountPath: "/etc/virtual-kubelet"
-          - name: aks-credential
-            mountPath: "/etc/aks/azure.json"
+        - name: certificates
+          mountPath: /etc/kubernetes/certs
+          readOnly: true
+        - name: credentials
+          mountPath: "/etc/virtual-kubelet"
+        - name: aks-credential
+          mountPath: "/etc/aks/azure.json"
         command: ["virtual-kubelet"]
         args: [
           "--provider", "azure",

docker/init-container/Dockerfile

@@ -21,9 +21,8 @@ RUN --mount=type=cache,target=${GOCACHE} \
     --mount=type=cache,id=vk-azure-aci-init,sharing=locked,target=/go/pkg/mod \
     CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -o initcontainer main.go
 
-FROM gcr.io/distroless/static:nonroot
+FROM --platform=$BUILDPLATFORM gcr.io/distroless/static
 WORKDIR /
 COPY --from=builder  /workspace/initcontainer .
-USER 65532:65532
 
-ENTRYPOINT [ "/initcontainer" ]
+ENTRYPOINT ["/initcontainer"]

go.mod

@@ -83,7 +83,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect
 	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.8 // indirect
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
 	google.golang.org/api v0.15.1 // indirect
 	google.golang.org/appengine v1.6.6 // indirect

go.sum

@@ -632,8 +632,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

hack/e2e/aks-addon.sh

@@ -53,10 +53,10 @@ fi
 TMPDIR=""
 
 cleanup() {
-  az group delete --name "$RESOURCE_GROUP" --yes --no-wait || true
-  if [ -n "$TMPDIR" ]; then
-      rm -rf "$TMPDIR"
-  fi
+ az group delete --name "$RESOURCE_GROUP" --yes --no-wait || true
+ if [ -n "$TMPDIR" ]; then
+     rm -rf "$TMPDIR"
+ fi
 }
 trap 'cleanup' EXIT
 
@@ -177,15 +177,18 @@ helm install \
     --kubeconfig="${KUBECONFIG}" \
     --set nodeOsType=Windows \
     --set "image.repository=${IMG_URL}"  \
-    --set "image.name=${IMG_REPO}" \
     --set "image.tag=${IMG_TAG}" \
+    --set "image.name=${IMG_REPO}" \
+    --set "initImage.repository=${IMG_URL}"  \
+    --set "initImage.name=${INIT_IMG_REPO}" \
+    --set "initImage.tag=${INIT_IMG_TAG}" \
     --set "nodeName=${TEST_WINDOWS_NODE_NAME}" \
     --set "providers.azure.masterUri=$MASTER_URI" \
     --set "providers.azure.managedIdentityID=$ACI_USER_IDENTITY" \
     "$WIN_CHART_NAME" \
     ./charts/virtual-kubelet
 
-kubectl wait --for=condition=available deploy "${TEST_WINDOWS_NODE_NAME}-virtual-kubelet-azure-aci" -n vk-azure-aci --timeout=300s
+kubectl wait --for=condition=available deploy "${TEST_WINDOWS_NODE_NAME}-virtual-kubelet-azure-aci" -n vk-azure-aci --timeout=500s
 
 while true; do
     kubectl get node "$TEST_WINDOWS_NODE_NAME" &> /dev/null && break

hack/e2e/aks.sh

@@ -186,6 +186,7 @@ helm install \
     --set "image.repository=${IMG_URL}"  \
     --set "image.name=${IMG_REPO}" \
     --set "image.tag=${IMG_TAG}" \
+    --set "initImage.repository=${IMG_URL}"  \
     --set "initImage.name=${INIT_IMG_REPO}" \
     --set "initImage.tag=${INIT_IMG_TAG}" \
     --set "nodeName=${TEST_NODE_NAME}" \
@@ -216,6 +217,9 @@ helm install \
     --set "image.repository=${IMG_URL}"  \
     --set "image.name=${IMG_REPO}" \
     --set "image.tag=${IMG_TAG}" \
+    --set "initImage.repository=${IMG_URL}"  \
+    --set "initImage.name=${INIT_IMG_REPO}" \
+    --set "initImage.tag=${INIT_IMG_TAG}" \
     --set "nodeName=${TEST_WINDOWS_NODE_NAME}" \
     --set "providers.azure.masterUri=$MASTER_URI" \
     "$WIN_CHART_NAME" \

pkg/network/aci_network.go

@@ -13,12 +13,12 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/virtual-kubelet/azure-aci/pkg/util"
+	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
 	"github.com/virtual-kubelet/virtual-kubelet/trace"
 	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
 
 	azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance"
 	aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network"
-	"github.com/virtual-kubelet/azure-aci/client/network"
 	"github.com/virtual-kubelet/azure-aci/pkg/auth"
 	client2 "github.com/virtual-kubelet/azure-aci/pkg/client"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
@@ -46,6 +46,10 @@ func (pn *ProviderNetwork) SetVNETConfig(ctx context.Context, azConfig *auth.Con
 	ctx, span := trace.StartSpan(ctx, "network.SetVNETConfig")
 	defer span.End()
 
+	if azConfig.AKSCredential != nil {
+		pn.VnetName = azConfig.AKSCredential.VNetName
+		pn.VnetResourceGroup = azConfig.AKSCredential.VNetResourceGroup
+	}
 	// the VNET subscription ID by default is authentication subscription ID.
 	// We need to override when using cross subscription virtual network resource
 	pn.VnetSubscriptionID = azConfig.AuthConfig.SubscriptionID
@@ -108,10 +112,10 @@ func (pn *ProviderNetwork) setupNetwork(ctx context.Context, azConfig *auth.Conf
 
 	createSubnet := true
 	subnet, err := c.Get(ctx, pn.VnetResourceGroup, pn.VnetName, pn.SubnetName, "")
-	if err != nil && !network.IsNotFound(err) {
+	if err != nil && !errdefs.IsNotFound(err) {
 		return fmt.Errorf("error while looking up subnet: %v", err)
 	}
-	if network.IsNotFound(err) && pn.SubnetCIDR == "" {
+	if errdefs.IsNotFound(err) && pn.SubnetCIDR == "" {
 		return fmt.Errorf("subnet '%s' is not found in vnet '%s' in resource group '%s' and subscription '%s' and subnet CIDR is not specified", pn.SubnetName, pn.VnetName, pn.VnetResourceGroup, pn.VnetSubscriptionID)
 	}
 	if err == nil {