diff --git a/charts/virtual-kubelet/templates/deployment.yaml b/charts/virtual-kubelet/templates/deployment.yaml index a3a6680f..e610bda4 100644 --- a/charts/virtual-kubelet/templates/deployment.yaml +++ b/charts/virtual-kubelet/templates/deployment.yaml @@ -22,7 +22,7 @@ spec: {{- if .Values.useVKVersion2}} initContainers: - name: init-validation - image: "{{ .Values.image.repository }}/{{ .Values.initImage.name }}:{{ .Values.initImage.tag }}" + image: "{{ .Values.initImage.repository }}/{{ .Values.initImage.name }}:{{ .Values.initImage.tag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} env: - name: VIRTUALNODE_USER_IDENTITY_CLIENTID @@ -36,8 +36,23 @@ spec: key: clientSecret - name: AKS_CREDENTIAL_LOCATION value: /etc/aks/azure.json - - name: AZURE_AUTH_LOCATION - value: /etc/virtual-kubelet/credentials.json + + {{- if .Values.providers.azure.vnet.enabled }} + - name: ACI_SUBNET_NAME + value: {{ required "subnetName is required" .vnet.subnetName }} + - name: KUBE_DNS_IP + value:{{ .vnet.kubeDnsIp }} + {{- end }} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: USE_VK_VERSION_2 + value: "true" volumeMounts: - name: credentials mountPath: "/etc/virtual-kubelet" diff --git a/charts/virtual-kubelet/values.yaml b/charts/virtual-kubelet/values.yaml index 91eab955..96a7ac85 100644 --- a/charts/virtual-kubelet/values.yaml +++ b/charts/virtual-kubelet/values.yaml @@ -6,6 +6,7 @@ image: pullPolicy: Always initImage: + repository: mcr.microsoft.com name: init-validation tag: 0.1.0 pullPolicy: Always diff --git a/cmd/init-container/main.go b/cmd/init-container/main.go index 552e299e..7600d3a1 100644 --- a/cmd/init-container/main.go +++ b/cmd/init-container/main.go @@ -1,3 +1,7 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the Apache 2.0 license. +*/ package main import ( @@ -9,44 +13,96 @@ import ( "github.com/sirupsen/logrus" "github.com/virtual-kubelet/azure-aci/pkg/auth" "github.com/virtual-kubelet/azure-aci/pkg/network" + "github.com/virtual-kubelet/azure-aci/pkg/util" cli "github.com/virtual-kubelet/node-cli" + logruscli "github.com/virtual-kubelet/node-cli/logrus" "github.com/virtual-kubelet/virtual-kubelet/log" logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/retry" ) func main() { + ctx := cli.ContextWithCancelOnSignal(context.Background()) + logger := logrus.StandardLogger() log.L = logruslogger.FromLogrus(logrus.NewEntry(logger)) + _ = logruscli.Config{LogLevel: "debug"} - ctx := cli.ContextWithCancelOnSignal(context.Background()) + log.G(ctx).Debug("Init container started") + + podName := os.Getenv("POD_NAME") + podNamespace := os.Getenv("NAMESPACE") + + if podName == "" || podNamespace == "" { + log.G(ctx).Fatal("an error has occurred while retrieve the pod info ") + } + + config, err := clientcmd.BuildConfigFromFlags("", "") + if err != nil { + log.G(ctx).Fatal("an error has occurred while creating client ", err) + } + + kubeClient := kubernetes.NewForConfigOrDie(config) + eventBroadcast := util.NewRecorder(ctx, kubeClient) + defer eventBroadcast.Shutdown() + recorder := eventBroadcast.NewRecorder(scheme.Scheme, v1.EventSource{Component: "k8s.io.api.core.v1.Pod"}) vkVersion, err := strconv.ParseBool(os.Getenv("USE_VK_VERSION_2")) if err != nil { log.G(ctx).Warn("init container: cannot get USE_VK_VERSION_2 environment variable, the provider will use VK version 1. Skipping init container checks") return } - wait.Until(func() { - azConfig := auth.Config{} + setupBackoff := wait.Backoff{ + Steps: 50, + Duration: time.Minute, + Factor: 0, + Jitter: 0.01, + } + + err = retry.OnError(setupBackoff, + func(err error) bool { + return true + }, func() error { + azConfig := auth.Config{} - if vkVersion { - //Setup config - err = azConfig.SetAuthConfig(ctx) + if vkVersion { + //Setup config + err = azConfig.SetAuthConfig(ctx) + if err != nil { + log.G(ctx).Errorf("init container: cannot setup the auth configuration. Retrying, ", err) + return err + } + } + + var providerNetwork network.ProviderNetwork + // Check or set up a network for VK + log.G(ctx).Debug("init container: setting up the network configuration") + err = providerNetwork.SetVNETConfig(ctx, &azConfig) if err != nil { - log.G(ctx).Fatalf("init container: cannot setup the auth configuration ", err) + log.G(ctx).Errorf("init container: cannot setup the VNet configuration. Retrying", err) + return err } - } - - var providerNetwork network.ProviderNetwork - // Check or set up a network for VK - log.G(ctx).Info("init container: setting up the network configuration") - err = providerNetwork.SetVNETConfig(ctx, &azConfig) - if err != nil { - log.G(ctx).Fatalf("init container: cannot setup the VNet configuration ", err) - } - }, - 1*time.Minute, ctx.Done()) + return nil + }) + if err != nil { + recorder.Eventf(&v1.ObjectReference{ + Kind: "Pod", + Name: podName, + Namespace: podNamespace, + }, v1.EventTypeWarning, "InitFailed", "VNet config setup failed") + log.G(ctx).Fatal("init container: cannot setup the VNet configuration ", err) + } + recorder.Eventf(&v1.ObjectReference{ + Kind: "Pod", + Name: podName, + Namespace: podNamespace, + }, v1.EventTypeNormal, "InitSuccess", "initial setup for virtual kubelet Azure ACI is successful") log.G(ctx).Info("initial setup for virtual kubelet Azure ACI is successful") } diff --git a/deploy/deployment.yaml b/deploy/deployment.yaml index 2309b8fd..effbb907 100644 --- a/deploy/deployment.yaml +++ b/deploy/deployment.yaml @@ -29,30 +29,52 @@ spec: values: - linux initContainers: - - name: init-validation - image: TEST_INIT_IMAGE - imagePullPolicy: Always - env: - - name: VIRTUALNODE_USER_IDENTITY_CLIENTID - valueFrom: - configMapKeyRef: - name: test-vars - key: aci_user_identity - - name: AZURE_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: aci-connector-linux - key: clientSecret - - name: AKS_CREDENTIAL_LOCATION - value: /etc/aks/azure.json - volumeMounts: - - name: credentials - mountPath: "/etc/virtual-kubelet" - - name: certificates - mountPath: /etc/kubernetes/certs - readOnly: true - - name: aks-credential - mountPath: "/etc/aks/azure.json" + - name: init-validation + image: TEST_INIT_IMAGE + imagePullPolicy: Always + env: + - name: AKS_CREDENTIAL_LOCATION + value: /etc/aks/azure.json + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: aci-connector-linux + key: clientSecret + - name: ACI_EXTRA_USER_AGENT + value: "deploy/aks/azure-aci/e2e-test" + - name: ACI_SUBNET_NAME + valueFrom: + configMapKeyRef: + name: test-vars + key: aci_subnet_name + - name: KUBE_DNS_IP + valueFrom: + configMapKeyRef: + name: test-vars + key: kube_dns_ip + - name: USE_VK_VERSION_2 + value: "true" + - name: VIRTUALNODE_USER_IDENTITY_CLIENTID + valueFrom: + configMapKeyRef: + name: test-vars + key: aci_user_identity + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: certificates + mountPath: /etc/kubernetes/certs + readOnly: true + - name: credentials + mountPath: "/etc/virtual-kubelet" + - name: aks-credential + mountPath: "/etc/aks/azure.json" containers: - name: vk-azure-aci image: TEST_IMAGE @@ -75,7 +97,6 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - - name: ACI_EXTRA_USER_AGENT value: "deploy/aks/azure-aci/e2e-test" - name: ACI_SUBNET_NAME @@ -108,13 +129,13 @@ spec: name: test-vars key: aci_user_identity volumeMounts: - - name: certificates - mountPath: /etc/kubernetes/certs - readOnly: true - - name: credentials - mountPath: "/etc/virtual-kubelet" - - name: aks-credential - mountPath: "/etc/aks/azure.json" + - name: certificates + mountPath: /etc/kubernetes/certs + readOnly: true + - name: credentials + mountPath: "/etc/virtual-kubelet" + - name: aks-credential + mountPath: "/etc/aks/azure.json" command: ["virtual-kubelet"] args: [ "--provider", "azure", diff --git a/docker/init-container/Dockerfile b/docker/init-container/Dockerfile index 587793ad..8b4728ee 100644 --- a/docker/init-container/Dockerfile +++ b/docker/init-container/Dockerfile @@ -21,9 +21,8 @@ RUN --mount=type=cache,target=${GOCACHE} \ --mount=type=cache,id=vk-azure-aci-init,sharing=locked,target=/go/pkg/mod \ CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -o initcontainer main.go -FROM gcr.io/distroless/static:nonroot +FROM --platform=$BUILDPLATFORM gcr.io/distroless/static WORKDIR / COPY --from=builder /workspace/initcontainer . -USER 65532:65532 -ENTRYPOINT [ "/initcontainer" ] +ENTRYPOINT ["/initcontainer"] diff --git a/go.mod b/go.mod index bb4c1b91..026cafb0 100644 --- a/go.mod +++ b/go.mod @@ -83,7 +83,7 @@ require ( golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.8 // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect google.golang.org/api v0.15.1 // indirect google.golang.org/appengine v1.6.6 // indirect diff --git a/go.sum b/go.sum index a225d4bf..33204a28 100644 --- a/go.sum +++ b/go.sum @@ -632,8 +632,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index 072272bf..3a57ea1f 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -53,10 +53,10 @@ fi TMPDIR="" cleanup() { - az group delete --name "$RESOURCE_GROUP" --yes --no-wait || true - if [ -n "$TMPDIR" ]; then - rm -rf "$TMPDIR" - fi + az group delete --name "$RESOURCE_GROUP" --yes --no-wait || true + if [ -n "$TMPDIR" ]; then + rm -rf "$TMPDIR" + fi } trap 'cleanup' EXIT @@ -177,8 +177,11 @@ helm install \ --kubeconfig="${KUBECONFIG}" \ --set nodeOsType=Windows \ --set "image.repository=${IMG_URL}" \ - --set "image.name=${IMG_REPO}" \ --set "image.tag=${IMG_TAG}" \ + --set "image.name=${IMG_REPO}" \ + --set "initImage.repository=${IMG_URL}" \ + --set "initImage.name=${INIT_IMG_REPO}" \ + --set "initImage.tag=${INIT_IMG_TAG}" \ --set "nodeName=${TEST_WINDOWS_NODE_NAME}" \ --set "providers.azure.masterUri=$MASTER_URI" \ --set "providers.azure.managedIdentityID=$ACI_USER_IDENTITY" \ diff --git a/hack/e2e/aks.sh b/hack/e2e/aks.sh index 0c512e2d..2ee2f901 100755 --- a/hack/e2e/aks.sh +++ b/hack/e2e/aks.sh @@ -186,6 +186,7 @@ helm install \ --set "image.repository=${IMG_URL}" \ --set "image.name=${IMG_REPO}" \ --set "image.tag=${IMG_TAG}" \ + --set "initImage.repository=${IMG_URL}" \ --set "initImage.name=${INIT_IMG_REPO}" \ --set "initImage.tag=${INIT_IMG_TAG}" \ --set "nodeName=${TEST_NODE_NAME}" \ @@ -216,6 +217,9 @@ helm install \ --set "image.repository=${IMG_URL}" \ --set "image.name=${IMG_REPO}" \ --set "image.tag=${IMG_TAG}" \ + --set "initImage.repository=${IMG_URL}" \ + --set "initImage.name=${INIT_IMG_REPO}" \ + --set "initImage.tag=${INIT_IMG_TAG}" \ --set "nodeName=${TEST_WINDOWS_NODE_NAME}" \ --set "providers.azure.masterUri=$MASTER_URI" \ "$WIN_CHART_NAME" \ diff --git a/pkg/network/aci_network.go b/pkg/network/aci_network.go index c828dda0..f6245c82 100644 --- a/pkg/network/aci_network.go +++ b/pkg/network/aci_network.go @@ -13,12 +13,12 @@ import ( "github.com/pkg/errors" "github.com/virtual-kubelet/azure-aci/pkg/util" + "github.com/virtual-kubelet/virtual-kubelet/errdefs" "github.com/virtual-kubelet/virtual-kubelet/trace" utilvalidation "k8s.io/apimachinery/pkg/util/validation" azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" - "github.com/virtual-kubelet/azure-aci/client/network" "github.com/virtual-kubelet/azure-aci/pkg/auth" client2 "github.com/virtual-kubelet/azure-aci/pkg/client" "github.com/virtual-kubelet/virtual-kubelet/log" @@ -46,6 +46,10 @@ func (pn *ProviderNetwork) SetVNETConfig(ctx context.Context, azConfig *auth.Con ctx, span := trace.StartSpan(ctx, "network.SetVNETConfig") defer span.End() + if azConfig.AKSCredential != nil { + pn.VnetName = azConfig.AKSCredential.VNetName + pn.VnetResourceGroup = azConfig.AKSCredential.VNetResourceGroup + } // the VNET subscription ID by default is authentication subscription ID. // We need to override when using cross subscription virtual network resource pn.VnetSubscriptionID = azConfig.AuthConfig.SubscriptionID @@ -108,10 +112,10 @@ func (pn *ProviderNetwork) setupNetwork(ctx context.Context, azConfig *auth.Conf createSubnet := true subnet, err := c.Get(ctx, pn.VnetResourceGroup, pn.VnetName, pn.SubnetName, "") - if err != nil && !network.IsNotFound(err) { + if err != nil && !errdefs.IsNotFound(err) { return fmt.Errorf("error while looking up subnet: %v", err) } - if network.IsNotFound(err) && pn.SubnetCIDR == "" { + if errdefs.IsNotFound(err) && pn.SubnetCIDR == "" { return fmt.Errorf("subnet '%s' is not found in vnet '%s' in resource group '%s' and subscription '%s' and subnet CIDR is not specified", pn.SubnetName, pn.VnetName, pn.VnetResourceGroup, pn.VnetSubscriptionID) } if err == nil { diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index 3ac8a0e8..7c57b93b 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -193,8 +193,7 @@ func NewACIProvider(ctx context.Context, config string, azConfig auth.Config, az if azConfig.AKSCredential != nil { p.resourceGroup = azConfig.AKSCredential.ResourceGroup p.region = azConfig.AKSCredential.Region - p.ProviderNetwork.VnetName = azConfig.AKSCredential.VNetName - p.ProviderNetwork.VnetResourceGroup = azConfig.AKSCredential.VNetResourceGroup + } if p.ProviderNetwork.VnetResourceGroup == "" { diff --git a/pkg/util/events.go b/pkg/util/events.go new file mode 100644 index 00000000..a594523b --- /dev/null +++ b/pkg/util/events.go @@ -0,0 +1,22 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the Apache 2.0 license. +*/ +package util + +import ( + "context" + + "k8s.io/client-go/kubernetes" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" +) + +func NewRecorder(ctx context.Context, kubeClient *kubernetes.Clientset) record.EventBroadcaster { + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartStructuredLogging(3) + if eventBroadcaster != nil && kubeClient != nil { + eventBroadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) + } + return eventBroadcaster +}