Skip to content

Commit

Permalink
Add a validation init container
Browse files Browse the repository at this point in the history
  • Loading branch information
helayoty committed Dec 13, 2022
1 parent bbe1aa4 commit b0cc30a
Show file tree
Hide file tree
Showing 22 changed files with 288 additions and 52 deletions.
9 changes: 0 additions & 9 deletions .dockerignore

This file was deleted.

17 changes: 17 additions & 0 deletions .github/workflows/create-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ env:
# Common versions
GO_VERSION: '1.18'
IMAGE_NAME: 'virtual-kubelet'
INIT_IMAGE_NAME: 'init-validation'

jobs:
create-release:
Expand Down Expand Up @@ -49,6 +50,7 @@ jobs:
needs: export-registry
env:
REGISTRY: ${{ needs.export-registry.outputs.registry }}
INIT_IMG_TAG : 0.1.0
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
Expand All @@ -70,6 +72,7 @@ jobs:
- name: Build and push image
run: |
OUTPUT_TYPE=type=registry make docker-build-image
OUTPUT_TYPE=type=registry make docker-build-init-image
env:
VERSION: ${{ env.IMG_TAG }}

Expand All @@ -86,3 +89,17 @@ jobs:
env:
TRIVY_USERNAME: ${{ github.actor }}
TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}

- name: Scan ${{ env.REGISTRY }}/${{ env.INIT_IMAGE_NAME }}:${{ env.INIT_IMG_TAG }}
uses: aquasecurity/trivy-action@master
with:
image-ref: ${{ env.REGISTRY }}/${{ env.INIT_IMAGE_NAME }}:${{ env.INIT_IMG_TAG }}
format: 'table'
exit-code: '1'
ignore-unfixed: true
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'
timeout: '5m0s'
env:
TRIVY_USERNAME: ${{ github.actor }}
TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
5 changes: 4 additions & 1 deletion .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ env:

jobs:
lint:
name: "Lint"
name: "Lint & Format"
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
Expand All @@ -39,5 +39,8 @@ jobs:
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}

- name: Format
run: make fmt

- name: golangci-lint
run: make lint
33 changes: 26 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ GOLANGCI_LINT_VER := v1.49.0
GOLANGCI_LINT_BIN := golangci-lint
GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER))

GOIMPORTS_VER := latest
GOIMPORTS_BIN := goimports
GOIMPORTS := $(abspath $(TOOLS_BIN_DIR)/$(GOIMPORTS_BIN)-$(GOIMPORTS_VER))

# Scripts
GO_INSTALL := ./hack/go-install.sh
AKS_E2E_SCRIPT := ./hack/e2e/aks.sh
Expand All @@ -18,14 +22,18 @@ TEST_CREDENTIALS_JSON ?= $(TEST_CREDENTIALS_DIR)/credentials.json
TEST_LOGANALYTICS_JSON ?= $(TEST_CREDENTIALS_DIR)/loganalytics.json
export TEST_CREDENTIALS_JSON TEST_LOGANALYTICS_JSON

REGISTRY ?= ghcr.io
IMG_NAME ?= virtual-kubelet
INIT_IMG_NAME ?= init-validation
IMAGE ?= $(REGISTRY)/$(IMG_NAME)
INIT_IMAGE ?= $(REGISTRY)/$(INIT_IMG_NAME)
LOCATION := $(E2E_REGION)
E2E_CLUSTER_NAME := $(CLUSTER_NAME)

OUTPUT_TYPE ?= type=docker
BUILDPLATFORM ?= linux/amd64
IMG_TAG ?= $(subst v,,$(VERSION))
INIT_IMG_TAG ?= 0.1.0


## --------------------------------------
Expand All @@ -35,12 +43,9 @@ IMG_TAG ?= $(subst v,,$(VERSION))
$(GOLANGCI_LINT):
GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) github.com/golangci/golangci-lint/cmd/golangci-lint $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER)

.PHONY: safebuild
# docker build
safebuild:
@echo "Building image..."
docker build -t $(DOCKER_IMAGE):$(VERSION) .

# GOIMPORTS
$(GOIMPORTS):
GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) golang.org/x/tools/cmd/goimports $(GOIMPORTS_BIN) $(GOIMPORTS_VER)

BUILDX_BUILDER_NAME ?= img-builder
QEMU_VERSION ?= 5.2.0-2
Expand All @@ -56,12 +61,21 @@ docker-buildx-builder:
.PHONY: docker-build-image
docker-build-image: docker-buildx-builder
docker buildx build \
--file Dockerfile \
--file docker/virtual-kubelet/Dockerfile \
--output=$(OUTPUT_TYPE) \
--platform="$(BUILDPLATFORM)" \
--pull \
--tag $(IMAGE):$(IMG_TAG) .

.PHONY: docker-build-init-image
docker-build-init-image: docker-buildx-builder
docker buildx build \
--file docker/init-container/Dockerfile \
--output=$(OUTPUT_TYPE) \
--platform="$(BUILDPLATFORM)" \
--pull \
--tag $(INIT_IMAGE):$(INIT_IMG_TAG) .

.PHONY: build
build: bin/virtual-kubelet

Expand Down Expand Up @@ -111,6 +125,11 @@ check-mod: # verifies that module changes for go.mod and go.sum are checked in
mod:
@go mod tidy

.PHONY: fmt
fmt: $(GOIMPORTS) ## Run go fmt against code.
go fmt ./...
$(GOIMPORTS) -w $$(go list -f {{.Dir}} ./...)

.PHONY: testauth
testauth: test-cred-json test-loganalytics-json

Expand Down
3 changes: 3 additions & 0 deletions charts/virtual-kubelet/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ The following table lists the configurable parameters of the azure-aci chart and
| image.name | Image name. | `oss/virtual-kubelet/virtual-kubelet` |
| image.tag | Image release version/tag. | `1.4.7` |
| image.pullPolicy | Image pull policy. | `Always` |
| initImage.name | Init container image name. | `oss/virtual-kubelet/init-validation` |
| initImage.tag | Init container image release version/tag. | `0.1.0` |
| initImage.pullPolicy | Init container image pull policy. | `Always` |
| nodeName | The node name that will be assigned to be the VK one. | `virtual-node-aci-linux-helm` |
| nodeOsType | The node/VM type. Values should be `Windows` or `Linux`. | `Linux` |
| monitoredNamespace | Kubernetes namespace. default values means monitor `all` | `""` |
Expand Down
30 changes: 30 additions & 0 deletions charts/virtual-kubelet/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,36 @@ spec:
labels:
app: {{ template "vk.fullname" . }}
spec:
{{- if .Values.useVKVersion2}}
initContainers:
- name: init-validation
image: "{{ .Values.image.repository }}/{{ .Values.initImage.name }}:{{ .Values.initImage.tag }}"
imagePullPolicy: {{ .Values.initImage.pullPolicy }}
env:
- name: VIRTUALNODE_USER_IDENTITY_CLIENTID
value: {{ .managedIdentityID }}
- name: AZURE_CLIENT_ID
value: {{ .clientId }}
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: {{ template "vk.fullname" $ }}
key: clientSecret
- name: AKS_CREDENTIAL_LOCATION
value: /etc/aks/azure.json
- name: AZURE_AUTH_LOCATION
value: /etc/virtual-kubelet/credentials.json
volumeMounts:
- name: credentials
mountPath: "/etc/virtual-kubelet"
- name: certificates
mountPath: /etc/kubernetes/certs
readOnly: true
{{- if .Values.providers.azure.targetAKS }}
- name: aks-credential
mountPath: "/etc/aks/azure.json"
{{- end }}
{{- end }}
containers:
- name: {{ template "vk.fullname" . }}
image: "{{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag }}"
Expand Down
6 changes: 6 additions & 0 deletions charts/virtual-kubelet/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,13 @@ image:
tag: 1.4.7
pullPolicy: Always

initImage:
name: init-validation
tag: 0.1.0
pullPolicy: Always

namespace: vk-azure-aci

useVKVersion2: true

nodeName: "virtual-node-aci-linux-helm"
Expand Down
3 changes: 2 additions & 1 deletion client/api/api_test.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package api

import (
. "github.com/onsi/gomega"
"io"
"math/rand"
"net/http"
"net/url"
"testing"

. "github.com/onsi/gomega"
)

func TestErrorWithNoMessage(t *testing.T) {
Expand Down
49 changes: 49 additions & 0 deletions cmd/init-container/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package main

import (
"context"
"os"
"strconv"

"github.com/sirupsen/logrus"
"github.com/virtual-kubelet/azure-aci/pkg/auth"
"github.com/virtual-kubelet/azure-aci/pkg/initcontainer"
cli "github.com/virtual-kubelet/node-cli"
"github.com/virtual-kubelet/virtual-kubelet/log"
logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus"
)

func main() {
logger := logrus.StandardLogger()
log.L = logruslogger.FromLogrus(logrus.NewEntry(logger))

ctx := cli.ContextWithCancelOnSignal(context.Background())

vkVersion, err := strconv.ParseBool(os.Getenv("USE_VK_VERSION_2"))
if err != nil {
log.G(ctx).Warn("cannot get USE_VK_VERSION_2 environment variable, the provider will use VK version 1. Skipping init container checks")
return
}

azConfig := auth.Config{}

if vkVersion {
//Setup config
err = azConfig.SetAuthConfig()
if err != nil {
log.G(ctx).Fatal(err)
}
}
// check role assignments
err = initcontainer.CheckRoleAssignments(ctx, azConfig)
if err != nil {
log.G(ctx).Fatal(err, "role assignments needed for virtual kubelet are not set.")
os.Exit(1)
}

err = initcontainer.CheckSubnetConfiguration(ctx, azConfig, "", "")
if err != nil {
log.G(ctx).Fatal(err, "virtual kubelet subnet is not configured with the user defiled route.")
os.Exit(1)
}
}
25 changes: 25 additions & 0 deletions deploy/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,31 @@ spec:
operator: In
values:
- linux
initContainers:
- name: init-validation
image: TEST_INIT_IMAGE
imagePullPolicy: Always
env:
- name: VIRTUALNODE_USER_IDENTITY_CLIENTID
valueFrom:
configMapKeyRef:
name: test-vars
key: aci_user_identity
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: aci-connector-linux
key: clientSecret
- name: AKS_CREDENTIAL_LOCATION
value: /etc/aks/azure.json
volumeMounts:
- name: credentials
mountPath: "/etc/virtual-kubelet"
- name: certificates
mountPath: /etc/kubernetes/certs
readOnly: true
- name: aks-credential
mountPath: "/etc/aks/azure.json"
containers:
- name: vk-azure-aci
image: TEST_IMAGE
Expand Down
21 changes: 21 additions & 0 deletions docker/.dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
.dockerignore
Dockerfile
bin/
helm/
hack/
charts/
.circleci/
README.md
.gitignore

# binary output
bin/
hack/tools/bin/

.azure

.DS_Store
.vscode/
.idea/
.run/
dist/
30 changes: 30 additions & 0 deletions docker/init-container/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
FROM --platform=$BUILDPLATFORM golang:1.18 as builder
ARG TARGETARCH

WORKDIR /workspace
# Copy the Go Modules manifests
COPY go.mod go.mod
COPY go.sum go.sum
# cache deps before building and copying source so that we don't need to re-download as much
# and so that source changes don't invalidate our downloaded layer
ENV GOCACHE=/root/gocache
RUN \
--mount=type=cache,target=${GOCACHE} \
--mount=type=cache,target=/go/pkg/mod \
go mod download

# Copy the go source
COPY cmd/init-container/main.go main.go
COPY pkg/auth pkg/auth
COPY pkg/initcontainer pkg/initcontainer

RUN --mount=type=cache,target=${GOCACHE} \
--mount=type=cache,id=vk-azure-aci-init,sharing=locked,target=/go/pkg/mod \
CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -o initcontainer main.go

FROM gcr.io/distroless/static:nonroot
WORKDIR /
COPY --from=builder /workspace/initcontainer .
USER 65532:65532

ENTRYPOINT [ "/initcontainer" ]
File renamed without changes.
7 changes: 3 additions & 4 deletions e2e/initcontainer_test.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@

package e2e

import (
"testing"
"time"
"io/ioutil"
"os/exec"
"os"
"os/exec"
"testing"
"time"

"gotest.tools/assert"
)
Expand Down
1 change: 0 additions & 1 deletion e2e/pods_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,4 +106,3 @@ func TestPodLifecycle(t *testing.T) {
t.Fatal(string(out))
}
}

Loading

0 comments on commit b0cc30a

Please sign in to comment.