Skip to content

Commit

Permalink
Add a validation init container
Browse files Browse the repository at this point in the history
  • Loading branch information
helayoty committed Dec 13, 2022
1 parent bbe1aa4 commit 1dc7435
Show file tree
Hide file tree
Showing 19 changed files with 274 additions and 50 deletions.
9 changes: 0 additions & 9 deletions .dockerignore

This file was deleted.

17 changes: 17 additions & 0 deletions .github/workflows/create-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ env:
# Common versions
GO_VERSION: '1.18'
IMAGE_NAME: 'virtual-kubelet'
INIT_IMAGE_NAME: 'init-validation'

jobs:
create-release:
Expand Down Expand Up @@ -49,6 +50,7 @@ jobs:
needs: export-registry
env:
REGISTRY: ${{ needs.export-registry.outputs.registry }}
INIT_IMG_TAG : 0.1.0
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
Expand All @@ -70,6 +72,7 @@ jobs:
- name: Build and push image
run: |
OUTPUT_TYPE=type=registry make docker-build-image
OUTPUT_TYPE=type=registry make docker-build-init-image
env:
VERSION: ${{ env.IMG_TAG }}

Expand All @@ -86,3 +89,17 @@ jobs:
env:
TRIVY_USERNAME: ${{ github.actor }}
TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}

- name: Scan ${{ env.REGISTRY }}/${{ env.INIT_IMAGE_NAME }}:${{ env.INIT_IMG_TAG }}
uses: aquasecurity/trivy-action@master
with:
image-ref: ${{ env.REGISTRY }}/${{ env.INIT_IMAGE_NAME }}:${{ env.INIT_IMG_TAG }}
format: 'table'
exit-code: '1'
ignore-unfixed: true
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'
timeout: '5m0s'
env:
TRIVY_USERNAME: ${{ github.actor }}
TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
22 changes: 14 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,18 @@ TEST_CREDENTIALS_JSON ?= $(TEST_CREDENTIALS_DIR)/credentials.json
TEST_LOGANALYTICS_JSON ?= $(TEST_CREDENTIALS_DIR)/loganalytics.json
export TEST_CREDENTIALS_JSON TEST_LOGANALYTICS_JSON

REGISTRY ?= ghcr.io
IMG_NAME ?= virtual-kubelet
INIT_IMG_NAME ?= init-validation
IMAGE ?= $(REGISTRY)/$(IMG_NAME)
INIT_IMAGE ?= $(REGISTRY)/$(INIT_IMG_NAME)
LOCATION := $(E2E_REGION)
E2E_CLUSTER_NAME := $(CLUSTER_NAME)

OUTPUT_TYPE ?= type=docker
BUILDPLATFORM ?= linux/amd64
IMG_TAG ?= $(subst v,,$(VERSION))
INIT_IMG_TAG ?= 0.1.0


## --------------------------------------
Expand All @@ -35,13 +39,6 @@ IMG_TAG ?= $(subst v,,$(VERSION))
$(GOLANGCI_LINT):
GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) github.com/golangci/golangci-lint/cmd/golangci-lint $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER)

.PHONY: safebuild
# docker build
safebuild:
@echo "Building image..."
docker build -t $(DOCKER_IMAGE):$(VERSION) .


BUILDX_BUILDER_NAME ?= img-builder
QEMU_VERSION ?= 5.2.0-2

Expand All @@ -56,12 +53,21 @@ docker-buildx-builder:
.PHONY: docker-build-image
docker-build-image: docker-buildx-builder
docker buildx build \
--file Dockerfile \
--file docker/virtual-kubelet/Dockerfile \
--output=$(OUTPUT_TYPE) \
--platform="$(BUILDPLATFORM)" \
--pull \
--tag $(IMAGE):$(IMG_TAG) .

.PHONY: docker-build-init-image
docker-build-init-image: docker-buildx-builder
docker buildx build \
--file docker/init-container/Dockerfile \
--output=$(OUTPUT_TYPE) \
--platform="$(BUILDPLATFORM)" \
--pull \
--tag $(INIT_IMAGE):$(INIT_IMG_TAG) .

.PHONY: build
build: bin/virtual-kubelet

Expand Down
3 changes: 3 additions & 0 deletions charts/virtual-kubelet/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ The following table lists the configurable parameters of the azure-aci chart and
| image.name | Image name. | `oss/virtual-kubelet/virtual-kubelet` |
| image.tag | Image release version/tag. | `1.4.7` |
| image.pullPolicy | Image pull policy. | `Always` |
| initImage.name | Init container image name. | `oss/virtual-kubelet/init-validation` |
| initImage.tag | Init container image release version/tag. | `0.1.0` |
| initImage.pullPolicy | Init container image pull policy. | `Always` |
| nodeName | The node name that will be assigned to be the VK one. | `virtual-node-aci-linux-helm` |
| nodeOsType | The node/VM type. Values should be `Windows` or `Linux`. | `Linux` |
| monitoredNamespace | Kubernetes namespace. default values means monitor `all` | `""` |
Expand Down
34 changes: 34 additions & 0 deletions charts/virtual-kubelet/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,40 @@ spec:
labels:
app: {{ template "vk.fullname" . }}
spec:
{{- if .Values.useVKVersion2}}
initContainers:
- name: init-validation
image:"{{ .Values.image.repository }}/{{ .Values.initImage.name }}:{{ .Values.initImage.tag }}"
imagePullPolicy: {{ .Values.initImage.pullPolicy }}
env:
- name: VIRTUALNODE_USER_IDENTITY_CLIENTID
value: {{ .managedIdentityID }}
- name: AZURE_CLIENT_ID
value: {{ .clientId }}
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: {{ template "vk.fullname" $ }}
key: clientSecret
- name: AZURE_TENANT_ID
value: {{ .tenantId }}
- name: AZURE_SUBSCRIPTION_ID
value: {{ .subscriptionId }}
- name: AKS_CREDENTIAL_LOCATION
value: /etc/aks/azure.json
- name: AZURE_AUTH_LOCATION
value: /etc/virtual-kubelet/credentials.json
volumeMounts:
- name: credentials
mountPath: "/etc/virtual-kubelet"
- name: certificates
mountPath: /etc/kubernetes/certs
readOnly: true
{{- if .Values.providers.azure.targetAKS }}
- name: aks-credential
mountPath: "/etc/aks/azure.json"
{{- end }}
{{- end }}
containers:
- name: {{ template "vk.fullname" . }}
image: "{{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag }}"
Expand Down
6 changes: 6 additions & 0 deletions charts/virtual-kubelet/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,13 @@ image:
tag: 1.4.7
pullPolicy: Always

initImage:
name: init-validation
tag: 0.1.0
pullPolicy: Always

namespace: vk-azure-aci

useVKVersion2: true

nodeName: "virtual-node-aci-linux-helm"
Expand Down
49 changes: 49 additions & 0 deletions cmd/init-container/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package main

import (
"context"
"os"
"strconv"

"github.com/sirupsen/logrus"
"github.com/virtual-kubelet/azure-aci/pkg/auth"
"github.com/virtual-kubelet/azure-aci/pkg/initcontainer"
cli "github.com/virtual-kubelet/node-cli"
"github.com/virtual-kubelet/virtual-kubelet/log"
logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus"
)

func main() {
logger := logrus.StandardLogger()
log.L = logruslogger.FromLogrus(logrus.NewEntry(logger))

ctx := cli.ContextWithCancelOnSignal(context.Background())

vkVersion, err := strconv.ParseBool(os.Getenv("USE_VK_VERSION_2"))
if err != nil {
log.G(ctx).Warn("cannot get USE_VK_VERSION_2 environment variable, the provider will use VK version 1. Skipping init container checks")
return
}

azConfig := auth.Config{}

if vkVersion {
//Setup config
err = azConfig.SetAuthConfig()
if err != nil {
log.G(ctx).Fatal(err)
}
}
// check role assignments
err = initcontainer.CheckRoleAssignments(ctx, azConfig)
if err != nil {
log.G(ctx).Fatal(err, "role assignments needed for virtual kubelet are not set.")
os.Exit(1)
}

err = initcontainer.CheckSubnetConfiguration(ctx, azConfig, "", "")
if err != nil {
log.G(ctx).Fatal(err, "virtual kubelet subnet is not configured with the user defiled route.")
os.Exit(1)
}
}
25 changes: 25 additions & 0 deletions deploy/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,31 @@ spec:
operator: In
values:
- linux
initContainers:
- name: init-validation
image: TEST_INIT_IMAGE
imagePullPolicy: Always
env:
- name: VIRTUALNODE_USER_IDENTITY_CLIENTID
valueFrom:
configMapKeyRef:
name: test-vars
key: aci_user_identity
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: aci-connector-linux
key: clientSecret
- name: AKS_CREDENTIAL_LOCATION
value: /etc/aks/azure.json
volumeMounts:
- name: credentials
mountPath: "/etc/virtual-kubelet"
- name: certificates
mountPath: /etc/kubernetes/certs
readOnly: true
- name: aks-credential
mountPath: "/etc/aks/azure.json"
containers:
- name: vk-azure-aci
image: TEST_IMAGE
Expand Down
21 changes: 21 additions & 0 deletions docker/.dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
.dockerignore
Dockerfile
bin/
helm/
hack/
charts/
.circleci/
README.md
.gitignore

# binary output
bin/
hack/tools/bin/

.azure

.DS_Store
.vscode/
.idea/
.run/
dist/
30 changes: 30 additions & 0 deletions docker/init-container/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
FROM --platform=$BUILDPLATFORM golang:1.18 as builder
ARG TARGETARCH

WORKDIR /workspace
# Copy the Go Modules manifests
COPY go.mod go.mod
COPY go.sum go.sum
# cache deps before building and copying source so that we don't need to re-download as much
# and so that source changes don't invalidate our downloaded layer
ENV GOCACHE=/root/gocache
RUN \
--mount=type=cache,target=${GOCACHE} \
--mount=type=cache,target=/go/pkg/mod \
go mod download

# Copy the go source
COPY cmd/init-container/main.go main.go
COPY pkg/auth pkg/auth
COPY pkg/initcontainer pkg/initcontainer

RUN --mount=type=cache,target=${GOCACHE} \
--mount=type=cache,id=vk-azure-aci-init,sharing=locked,target=/go/pkg/mod \
CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -o initcontainer main.go

FROM gcr.io/distroless/static:nonroot
WORKDIR /
COPY --from=builder /workspace/initcontainer .
USER 65532:65532

ENTRYPOINT [ "/initcontainer" ]
File renamed without changes.
7 changes: 3 additions & 4 deletions e2e/initcontainer_test.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@

package e2e

import (
"testing"
"time"
"io/ioutil"
"os/exec"
"os"
"os/exec"
"testing"
"time"

"gotest.tools/assert"
)
Expand Down
25 changes: 23 additions & 2 deletions goreleaser.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,26 @@ builds:
- -X {{.ModulePath}}/pkg/version.Vcs={{.ShortCommit}}
env:
- CGO_ENABLED=0
- id: init-validation
dir: cmd/init-container
binary: bin/init-container
goos:
- linux
goarch:
- amd64
- arm64
goarm:
- 5
- 6
- 7
goamd64:
- v2
- v3
flags:
- -mod=vendor
env:
- CGO_ENABLED=0
- GO111MODULE=on

release:
prerelease: auto
Expand All @@ -37,6 +57,9 @@ release:
changelog:
skip: false
groups:
- title: Features 🌈
regexp: ^.*feat[(\\w)]*:+.*$

- title: Bug Fixes 🐞
regexp: ^.*fix[(\\w)]*:+.*$
- title: Build 🏭
Expand All @@ -49,8 +72,6 @@ changelog:
regexp: ^.*ci[(\\w)]*:+.*$
- title: Documentation 📘
regexp: ^.*docs[(\\w)]*:+.*$
- title: Features 🌈
regexp: ^.*feat[(\\w)]*:+.*$
- title: Maintenance 🔧
regexp: ^.*chore[(\\w)]*:+.*$
- title: Performance Improvements 🚀
Expand Down
6 changes: 5 additions & 1 deletion hack/e2e/aks-addon.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ fi
: "${TEST_NODE_NAME:=vk-aci-test-aks}"
: "${IMG_REPO:=oss/virtual-kubelet/virtual-kubelet}"
: "${IMG_URL:=mcr.microsoft.com}"
: "${INIT_IMG_TAG:=0.1.0}"
: "${VNET_RANGE=10.0.0.0/8}"
: "${CLUSTER_SUBNET_CIDR=10.240.0.0/16}"
: "${ACI_SUBNET_CIDR=10.241.0.0/16}"
Expand Down Expand Up @@ -79,7 +80,10 @@ if [ "$E2E_TARGET" = "pr" ]; then
az acr login --name "$ACR_NAME"
IMG_URL=$ACR_NAME.azurecr.io
IMG_REPO="virtual-kubelet"
INIT_IMG_REPO="init-validation"
OUTPUT_TYPE=type=registry IMG_TAG=$IMG_TAG IMAGE=$ACR_NAME.azurecr.io/$IMG_REPO make docker-build-image
OUTPUT_TYPE=type=registry INIT_IMG_TAG=$INIT_IMG_TAG INIT_IMAGE=$ACR_NAME.azurecr.io/$INIT_IMG_REPO make docker-build-init-image

fi

TMPDIR="$(mktemp -d)"
Expand Down Expand Up @@ -151,7 +155,7 @@ kubectl create configmap test-vars -n kube-system \
--from-literal=cluster_subnet_cidr="$CLUSTER_SUBNET_CIDR" \
--from-literal=aci_subnet_name="$ACI_SUBNET_NAME"

sed -e "s|TEST_IMAGE|$ACR_NAME.azurecr.io/$IMG_REPO:$IMG_TAG|g" deploy/deployment.yaml | kubectl apply -n kube-system -f -
sed -e "s|TEST_INIT_IMAGE|$ACR_NAME.azurecr.io/$INIT_IMG_REPO:$INIT_IMG_TAG|g" "s|TEST_IMAGE|$ACR_NAME.azurecr.io/$IMG_REPO:$IMG_TAG|g" deploy/deployment.yaml | kubectl apply -n kube-system -f -

kubectl wait --for=condition=available deploy "virtual-kubelet-azure-aci" -n kube-system --timeout=300s

Expand Down
Loading

0 comments on commit 1dc7435

Please sign in to comment.