From 9c109811db66aba85458924f0b63cbef1dd78a78 Mon Sep 17 00:00:00 2001 From: Heba Elayoty <31887807+helayoty@users.noreply.github.com> Date: Wed, 25 Jan 2023 18:26:41 -0800 Subject: [PATCH] ci: Update helm chart and e2e (#433) Signed-off-by: Heba Elayoty --- .github/workflows/e2e-tests.yml | 4 +- .../virtual-kubelet/templates/deployment.yaml | 127 ++++++++++++------ charts/virtual-kubelet/values.yaml | 5 +- hack/e2e/aks-addon.sh | 9 +- hack/e2e/aks.sh | 7 +- 5 files changed, 102 insertions(+), 50 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 04b82cd9..88a75420 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -5,10 +5,11 @@ concurrency: cancel-in-progress: true on: + workflow_dispatch: pull_request_target: branches: + - master - release-** - types: [opened] paths: - '.github/workflows/chart.yml' - 'charts/**' @@ -24,6 +25,7 @@ env: jobs: export-registry: + if: contains(github.event.pull_request.title, 'update manifest and helm charts') || (contains(github.event.pull_request.title, 'helm')) runs-on: ubuntu-20.04 outputs: registry: ${{ steps.export.outputs.registry }} diff --git a/charts/virtual-kubelet/templates/deployment.yaml b/charts/virtual-kubelet/templates/deployment.yaml index e610bda4..80767c29 100644 --- a/charts/virtual-kubelet/templates/deployment.yaml +++ b/charts/virtual-kubelet/templates/deployment.yaml @@ -19,50 +19,99 @@ spec: labels: app: {{ template "vk.fullname" . }} spec: - {{- if .Values.useVKVersion2}} + {{- if .Values.useVKVersion2 }} initContainers: - name: init-validation image: "{{ .Values.initImage.repository }}/{{ .Values.initImage.name }}:{{ .Values.initImage.tag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} env: - - name: VIRTUALNODE_USER_IDENTITY_CLIENTID - value: {{ .managedIdentityID }} - - name: AZURE_CLIENT_ID - value: {{ .clientId }} - - name: AZURE_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ template "vk.fullname" $ }} - key: clientSecret - - name: AKS_CREDENTIAL_LOCATION - value: /etc/aks/azure.json - - {{- if .Values.providers.azure.vnet.enabled }} - - name: ACI_SUBNET_NAME - value: {{ required "subnetName is required" .vnet.subnetName }} - - name: KUBE_DNS_IP - value:{{ .vnet.kubeDnsIp }} - {{- end }} - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: USE_VK_VERSION_2 - value: "true" + - name: USE_VK_VERSION_2 + value: "true" + - name: KUBELET_PORT + value: "10250" + - name: APISERVER_CERT_LOCATION + value: /etc/virtual-kubelet/cert.pem + - name: APISERVER_KEY_LOCATION + value: /etc/virtual-kubelet/key.pem + - name: VKUBELET_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{- with .Values.providers.azure }} +{{- if .managedIdentityID }} + - name: VIRTUALNODE_USER_IDENTITY_CLIENTID + value: {{ .managedIdentityID }} +{{- end }} +{{- if .targetAKS }} + - name: AKS_CREDENTIAL_LOCATION + value: /etc/aks/azure.json + - name: AZURE_TENANT_ID + value: {{ .tenantId }} + - name: AZURE_SUBSCRIPTION_ID + value: {{ .subscriptionId }} + - name: AZURE_CLIENT_ID + value: {{ .clientId }} + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ template "vk.fullname" $ }} + key: clientSecret + - name: ACI_RESOURCE_GROUP + value: {{ .aciResourceGroup }} + - name: ACI_REGION + value: {{ .aciRegion }} + - name: ACI_EXTRA_USER_AGENT + value: {{ printf "helm-chart/aks/%s/%s" $.Chart.Name $.Chart.Version }} +{{- else }} + - name: AZURE_AUTH_LOCATION + value: /etc/virtual-kubelet/credentials.json + - name: ACI_RESOURCE_GROUP + value: {{ required "aciResourceGroup is required" .aciResourceGroup }} + - name: ACI_REGION + value: {{ required "aciRegion is required" .aciRegion }} + - name: ACI_EXTRA_USER_AGENT + value: {{ printf "helm-chart/other/%s/%s" $.Chart.Name $.Chart.Version }} +{{- end }} +{{- if .vnet.enabled }} + - name: ACI_VNET_SUBSCRIPTION_ID + value: {{ .vnet.vnetSubscriptionID}} + - name: ACI_VNET_RESOURCE_GROUP + value: {{ .vnet.vnetResourceGroup}} + - name: ACI_VNET_NAME + value: {{ .vnet.vnetName}} + - name: ACI_SUBNET_NAME + value: {{ required "subnetName is required" .vnet.subnetName }} + - name: ACI_SUBNET_CIDR + value: {{ .vnet.subnetCidr }} + - name: MASTER_URI + value: {{ required "masterUri is required" .masterUri | quote }} + - name: CLUSTER_CIDR + value: {{ .vnet.clusterCidr }} + - name: KUBE_DNS_IP + value: {{ .vnet.kubeDnsIp }} +{{- else }} + - name: MASTER_URI + value: {{ .masterUri | quote }} +{{- end }} +{{- end }} volumeMounts: - - name: credentials - mountPath: "/etc/virtual-kubelet" - - name: certificates - mountPath: /etc/kubernetes/certs - readOnly: true - {{- if .Values.providers.azure.targetAKS }} - - name: aks-credential - mountPath: "/etc/aks/azure.json" - {{- end }} + - name: credentials + mountPath: "/etc/virtual-kubelet" + - name: certificates + mountPath: /etc/kubernetes/certs + readOnly: true + {{- if .Values.providers.azure.targetAKS }} + - name: aks-credential + mountPath: "/etc/aks/azure.json" + {{- end }} {{- end }} containers: - name: {{ template "vk.fullname" . }} @@ -148,10 +197,12 @@ spec: - name: MASTER_URI value: {{ .masterUri | quote }} {{- end }} +{{- if .enableRealTimeMetrics }} - name: ENABLE_REAL_TIME_METRICS value: "true" {{- end }} {{- end }} +{{- end }} {{- if .Values.useVKVersion2 }} - name: USE_VK_VERSION_2 value: "true" diff --git a/charts/virtual-kubelet/values.yaml b/charts/virtual-kubelet/values.yaml index df0a5f9c..575fc0a9 100644 --- a/charts/virtual-kubelet/values.yaml +++ b/charts/virtual-kubelet/values.yaml @@ -7,8 +7,8 @@ image: initImage: repository: mcr.microsoft.com - name: init-validation - tag: 1.4.10 + name: oss/virtual-kubelet/init-validation + tag: 0.1.0 pullPolicy: Always namespace: vk-azure-aci @@ -38,6 +38,7 @@ trace: providers: azure: + enableRealTimeMetrics: true ## Set to true if deploying to Azure Kubernetes Service (AKS), otherwise false targetAKS: true clientId: diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index ae8f9b8a..81728f73 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -27,6 +27,7 @@ fi : "${WIN_CHART_NAME:=vk-aci-test-win-aks}" : "${TEST_NODE_NAME:=vk-aci-test-aks}" : "${TEST_WINDOWS_NODE_NAME:=vk-aci-test-win-aks}" +: "${INIT_IMG_REPO:=oss/virtual-kubelet/init-validation}" : "${IMG_REPO:=oss/virtual-kubelet/virtual-kubelet}" : "${IMG_URL:=mcr.microsoft.com}" : "${INIT_IMG_TAG:=0.1.0}" @@ -81,10 +82,8 @@ if [ "$E2E_TARGET" = "pr" ]; then az acr login --name "$ACR_NAME" IMG_URL=$ACR_NAME.azurecr.io - IMG_REPO="virtual-kubelet" - INIT_IMG_REPO="init-validation" - OUTPUT_TYPE=type=registry IMG_TAG=$IMG_TAG IMAGE=$ACR_NAME.azurecr.io/$IMG_REPO make docker-build-image - OUTPUT_TYPE=type=registry INIT_IMG_TAG=$INIT_IMG_TAG INIT_IMAGE=$ACR_NAME.azurecr.io/$INIT_IMG_REPO make docker-build-init-image + OUTPUT_TYPE=type=registry IMG_TAG=$IMG_TAG IMAGE=$IMG_URL/$IMG_REPO make docker-build-image + OUTPUT_TYPE=type=registry INIT_IMG_TAG=$INIT_IMG_TAG INIT_IMAGE=$IMG_URL/$INIT_IMG_REPO make docker-build-init-image fi @@ -159,7 +158,7 @@ kubectl create configmap test-vars -n kube-system \ --from-literal=cluster_subnet_cidr="$CLUSTER_SUBNET_CIDR" \ --from-literal=aci_subnet_name="$ACI_SUBNET_NAME" -sed -e "s|TEST_INIT_IMAGE|$ACR_NAME.azurecr.io/$INIT_IMG_REPO:$INIT_IMG_TAG|g" -e "s|TEST_IMAGE|$ACR_NAME.azurecr.io/$IMG_REPO:$IMG_TAG|g" deploy/deployment.yaml | kubectl apply -n kube-system -f - +sed -e "s|TEST_INIT_IMAGE|$IMG_URL/$INIT_IMG_REPO:$INIT_IMG_TAG|g" -e "s|TEST_IMAGE|$IMG_URL/$IMG_REPO:$IMG_TAG|g" deploy/deployment.yaml | kubectl apply -n kube-system -f - kubectl wait --for=condition=available deploy "virtual-kubelet-azure-aci" -n kube-system --timeout=300s diff --git a/hack/e2e/aks.sh b/hack/e2e/aks.sh index 2ee2f901..114e56cf 100755 --- a/hack/e2e/aks.sh +++ b/hack/e2e/aks.sh @@ -27,6 +27,7 @@ fi : "${WIN_CHART_NAME:=vk-aci-test-win-aks}" : "${TEST_NODE_NAME:=vk-aci-test-aks}" : "${TEST_WINDOWS_NODE_NAME:=vk-aci-test-win-aks}" +: "${INIT_IMG_REPO:=oss/virtual-kubelet/init-validation}" : "${IMG_REPO:=oss/virtual-kubelet/virtual-kubelet}" : "${IMG_URL:=mcr.microsoft.com}" : "${INIT_IMG_TAG:=0.1.0}" @@ -81,10 +82,8 @@ if [ "$E2E_TARGET" = "pr" ]; then az acr login --name "$ACR_NAME" IMG_URL=$ACR_NAME.azurecr.io - IMG_REPO="virtual-kubelet" - INIT_IMG_REPO="init-validation" - OUTPUT_TYPE=type=registry IMG_TAG=$IMG_TAG IMAGE=$ACR_NAME.azurecr.io/$IMG_REPO make docker-build-image - OUTPUT_TYPE=type=registry INIT_IMG_TAG=$INIT_IMG_TAG INIT_IMAGE=$ACR_NAME.azurecr.io/$INIT_IMG_REPO make docker-build-init-image + OUTPUT_TYPE=type=registry IMG_TAG=$IMG_TAG IMAGE=$IMG_URL/$IMG_REPO make docker-build-image + OUTPUT_TYPE=type=registry INIT_IMG_TAG=$INIT_IMG_TAG INIT_IMAGE=$IMG_URL/$INIT_IMG_REPO make docker-build-init-image fi