injector: split sidecar-injector from osm-controller

This change splits the sidecar injector component from osm-controller for scalability purpose as a part of openservicemesh#1939. Summary of changes: - injector is moved to a separate osm-injector app - Helm chart changes to deploy osm-injector - osm-controller changes to remove injector initialization - utility method on `providers.Config` to watch CA bundle secret - Makefile, dockerfile changes to build osm-injector - cleanup of unused options for mutating webhook's reconciler - update e2e tests to wait on osm-injector pod Signed-off-by: Shashank Ram <shashr2204@gmail.com>
shashankram · Feb 11, 2021 · a78d2d4 · a78d2d4
1 parent 7930ef0
commit a78d2d4
Show file tree

Hide file tree

Showing 29 changed files with 495 additions and 183 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -177,7 +177,7 @@ jobs:
         with:
           go-version: 1.15
       - name: Build test dependencies
-        run: make docker-build-osm-controller docker-build-init build-osm docker-build-tcp-echo-server
+        run: make docker-build-osm-controller docker-build-osm-injector docker-build-init build-osm docker-build-tcp-echo-server
       - name: Run PR tests
         if: ${{ github.event_name == 'pull_request' }}
         run: go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -installType=KindCluster -test.timeout 0 -test.failfast -ginkgo.failFast -ginkgo.focus='\[Tier 1\]\[Bucket ${{ matrix.bucket }}\]' -ginkgo.skip='Upgrade'

diff --git a/Makefile b/Makefile
@@ -40,13 +40,21 @@ clean-cert:
 clean-osm-controller:
 	@rm -rf bin/osm-controller
 
+.PHONY: clean-osm-injector
+clean-osm-injector:
+	@rm -rf bin/osm-injector
+
 .PHONY: build
-build: build-osm-controller
+build: build-osm-controller build-osm-injector
 
 .PHONY: build-osm-controller
 build-osm-controller: clean-osm-controller
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./bin/osm-controller/osm-controller -ldflags "-X $(BUILD_DATE_VAR)=$(BUILD_DATE) -X $(BUILD_VERSION_VAR)=$(VERSION) -X $(BUILD_GITCOMMIT_VAR)=$(GIT_SHA) -s -w" ./cmd/osm-controller
 
+.PHONY: build-osm-injector
+build-osm-injector: clean-osm-injector
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./bin/osm-injector/osm-injector -ldflags "-X $(BUILD_DATE_VAR)=$(BUILD_DATE) -X $(BUILD_VERSION_VAR)=$(VERSION) -X $(BUILD_GITCOMMIT_VAR)=$(GIT_SHA) -s -w" ./cmd/osm-injector
+
 .PHONY: build-osm
 build-osm:
 	go run scripts/generate_chart/generate_chart.go | CGO_ENABLED=0  go build -v -o ./bin/osm -ldflags ${LDFLAGS} ./cmd/cli
@@ -99,7 +107,7 @@ kind-reset:
 	kind delete cluster --name osm
 
 .PHONY: test-e2e
-test-e2e: docker-build-osm-controller docker-build-init build-osm docker-build-tcp-echo-server
+test-e2e: docker-build-osm-controller docker-build-osm-injector docker-build-init build-osm docker-build-tcp-echo-server
 	go test ./tests/e2e $(E2E_FLAGS_DEFAULT) $(E2E_FLAGS)
 
 .env:
@@ -120,7 +128,7 @@ $(DEMO_BUILD_TARGETS):
 	@if [ -f demo/$(NAME).html.template ]; then cp demo/$(NAME).html.template demo/bin/$(NAME); fi
 
 .PHONY: demo-build
-demo-build: $(DEMO_BUILD_TARGETS) build-osm-controller
+demo-build: $(DEMO_BUILD_TARGETS) build-osm-controller build-osm-injector
 
 # docker-build-bookbuyer, etc
 DOCKER_DEMO_TARGETS = $(addprefix docker-build-, $(DEMO_TARGETS))
@@ -136,11 +144,14 @@ docker-build-init:
 docker-build-osm-controller: build-osm-controller
 	docker build -t $(CTR_REGISTRY)/osm-controller:$(CTR_TAG) -f dockerfiles/Dockerfile.osm-controller bin/osm-controller
 
+docker-build-osm-injector: build-osm-injector
+	docker build -t $(CTR_REGISTRY)/osm-injector:$(CTR_TAG) -f dockerfiles/Dockerfile.osm-injector bin/osm-injector
+
 .PHONY: docker-build
-docker-build: $(DOCKER_DEMO_TARGETS) docker-build-init docker-build-osm-controller
+docker-build: $(DOCKER_DEMO_TARGETS) docker-build-init docker-build-osm-controller docker-build-osm-injector
 
 # docker-push-bookbuyer, etc
-DOCKER_PUSH_TARGETS = $(addprefix docker-push-, $(DEMO_TARGETS) init osm-controller)
+DOCKER_PUSH_TARGETS = $(addprefix docker-push-, $(DEMO_TARGETS) init osm-controller osm-injector)
 VERIFY_TAGS = 0
 .PHONY: $(DOCKER_PUSH_TARGETS)
 $(DOCKER_PUSH_TARGETS): NAME=$(@:docker-push-%=%)

diff --git a/charts/osm/README.md b/charts/osm/README.md
@@ -94,6 +94,7 @@ The following table lists the configurable parameters of the osm chart and their
 | OpenServiceMesh.image.registry | string | `"openservicemesh"` | `osm-controller` image registry |
 | OpenServiceMesh.image.tag | string | `"v0.7.0"` | `osm-controller` image tag |
 | OpenServiceMesh.imagePullSecrets | list | `[]` | `osm-controller` image pull secret |
+| OpenServiceMesh.injector | object | `{"replicaCount":1,"resource":{"limits":{"cpu":"0.5","memory":"32M"},"requests":{"cpu":"0.5","memory":"32M"}}}` | Sidecar injector configuration |
 | OpenServiceMesh.meshName | string | `"osm"` | Name for the new control plane instance |
 | OpenServiceMesh.osmNamespace | string | `""` | Optional parameter. If not specified, the release namespace is used to deploy the osm components. |
 | OpenServiceMesh.osmcontroller.resource.limits.cpu | string | `"1.5"` |  |

diff --git a/charts/osm/templates/mutatingwebhook.yaml b/charts/osm/templates/mutatingwebhook.yaml
@@ -8,7 +8,7 @@ webhooks:
 - name: osm-inject.k8s.io
   clientConfig:
     service:
-      name: osm-controller
+      name: osm-injector
       namespace: {{ include "osm.namespace" . }}
       path: /mutate-pod-creation
       port: 9090

diff --git a/charts/osm/templates/osm-deployment.yaml b/charts/osm/templates/osm-deployment.yaml
@@ -37,8 +37,6 @@ spec:
             "--verbosity", "{{.Values.OpenServiceMesh.controllerLogLevel}}",
             "--osm-namespace", "{{ include "osm.namespace" . }}",
             "--mesh-name", "{{.Values.OpenServiceMesh.meshName}}",
-            "--init-container-image", "{{.Values.OpenServiceMesh.image.registry}}/init:{{ .Values.OpenServiceMesh.image.tag }}",
-            "--sidecar-image", "{{.Values.OpenServiceMesh.sidecarImage}}",
             "--webhook-config-name", "{{.Values.OpenServiceMesh.webhookConfigNamePrefix}}-{{.Values.OpenServiceMesh.meshName}}",
             "--ca-bundle-secret-name", "{{.Values.OpenServiceMesh.caBundleSecretName}}",
             "--certificate-manager", "{{.Values.OpenServiceMesh.certificateManager}}",

diff --git a/charts/osm/templates/osm-injector-deployment.yaml b/charts/osm/templates/osm-injector-deployment.yaml
@@ -0,0 +1,81 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: osm-injector
+  namespace: {{ include "osm.namespace" . }}
+  labels:
+    app: osm-injector
+    meshName: {{ .Values.OpenServiceMesh.meshName }}
+spec:
+  replicas: {{ .Values.OpenServiceMesh.injector.replicaCount }}
+  selector:
+    matchLabels:
+      app: osm-injector
+  template:
+    metadata:
+      labels:
+        app: osm-injector
+      annotations:
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: '9091'
+    spec:
+      serviceAccountName: {{ .Release.Name }}
+      containers:
+        - name: osm-injector
+          image: "{{ .Values.OpenServiceMesh.image.registry }}/osm-injector:{{ .Values.OpenServiceMesh.image.tag }}"
+          imagePullPolicy: {{ .Values.OpenServiceMesh.image.pullPolicy }}
+          ports:
+            - name: "sidecar-inject"
+              containerPort: 9090
+            - name: "metrics"
+              containerPort: 9091
+          command: ['/osm-injector']
+          args: [
+            "--verbosity", "{{.Values.OpenServiceMesh.controllerLogLevel}}",
+            "--osm-namespace", "{{ include "osm.namespace" . }}",
+            "--mesh-name", "{{.Values.OpenServiceMesh.meshName}}",
+            "--init-container-image", "{{.Values.OpenServiceMesh.image.registry}}/init:{{ .Values.OpenServiceMesh.image.tag }}",
+            "--sidecar-image", "{{.Values.OpenServiceMesh.sidecarImage}}",
+            "--webhook-config-name", "{{.Values.OpenServiceMesh.webhookConfigNamePrefix}}-{{.Values.OpenServiceMesh.meshName}}",
+            "--ca-bundle-secret-name", "{{.Values.OpenServiceMesh.caBundleSecretName}}",
+            "--certificate-manager", "{{.Values.OpenServiceMesh.certificateManager}}",
+            {{ if eq .Values.OpenServiceMesh.certificateManager "vault" }}
+            "--vault-host", "{{.Values.OpenServiceMesh.vault.host}}",
+            "--vault-protocol", "{{.Values.OpenServiceMesh.vault.protocol}}",
+            "--vault-token", "{{.Values.OpenServiceMesh.vault.token}}",
+            {{- end }}
+            "--cert-manager-issuer-name", "{{.Values.OpenServiceMesh.certmanager.issuerName}}",
+            "--cert-manager-issuer-kind", "{{.Values.OpenServiceMesh.certmanager.issuerKind}}",
+            "--cert-manager-issuer-group", "{{.Values.OpenServiceMesh.certmanager.issuerGroup}}",
+          ]
+          resources:
+            limits:
+              cpu: "{{.Values.OpenServiceMesh.injector.resource.limits.cpu}}"
+              memory: "{{.Values.OpenServiceMesh.injector.resource.limits.memory}}"
+            requests:
+              cpu: "{{.Values.OpenServiceMesh.injector.resource.requests.cpu}}"
+              memory: "{{.Values.OpenServiceMesh.injector.resource.requests.memory}}"
+          readinessProbe:
+            initialDelaySeconds: 1
+            timeoutSeconds: 5
+            httpGet:
+              scheme: HTTPS
+              path: /healthz
+              port: 9090
+          livenessProbe:
+            initialDelaySeconds: 1
+            timeoutSeconds: 5
+            httpGet:
+              scheme: HTTPS
+              path: /healthz
+              port: 9090
+          env:
+            # The INJECTOR_POD_NAME env variable sets pod name dynamically, used by osm-injector to register events
+            - name: INJECTOR_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+    {{- if .Values.OpenServiceMesh.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml .Values.OpenServiceMesh.imagePullSecrets | indent 8 }}
+    {{- end }}
diff --git a/charts/osm/templates/osm-injector-service.yaml b/charts/osm/templates/osm-injector-service.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: osm-injector
+  namespace: {{ include "osm.namespace" . }}
+  labels:
+    app: osm-injector
+spec:
+  ports:
+    - name: sidecar-inject
+      port: 9090
+  selector:
+    app: osm-injector
diff --git a/charts/osm/templates/osm-service.yaml b/charts/osm/templates/osm-service.yaml
@@ -10,9 +10,6 @@ spec:
     - name: osm-port
       port: 15128
       targetPort: 15128
-    - name: sidecar-injector
-      port: 9090
-      targetPort: 9090
     - name: debug-port
       port: 9092
       targetPort: 9092

diff --git a/charts/osm/values.yaml b/charts/osm/values.yaml
@@ -137,3 +137,14 @@ OpenServiceMesh:
   # -- Optional parameter to specify a global list of IP ranges to exclude from outbound traffic interception by the sidecar proxy.
   # If specified, must be a list of IP ranges of the form a.b.c.d/x.
   outboundIPRangeExclusionList: []
+
+  # -- Sidecar injector configuration
+  injector:
+    replicaCount: 1
+    resource:
+      limits:
+        cpu: "0.5"
+        memory: "32M"
+      requests:
+        cpu: "0.5"
+        memory: "32M"