injector: split sidecar-injector from osm-controller

This change splits the sidecar injector component from osm-controller for scalability purpose as a part of openservicemesh#1939. Summary of changes: - injector is moved to a separate osm-injector app - Helm chart changes to deploy osm-injector - osm-controller changes to remove injector initialization - utility method on `providers.Config` to watch CA bundle secret - Makefile, dockerfile changes to build osm-injector - cleanup of unused options for mutating webhook's reconciler - update e2e tests to wait on osm-injector pod Signed-off-by: Shashank Ram <shashr2204@gmail.com>
shashankram · Feb 10, 2021 · a14e4ef · a14e4ef
1 parent a8332dc
commit a14e4ef
Show file tree

Hide file tree

Showing 27 changed files with 493 additions and 182 deletions.
diff --git a/Makefile b/Makefile
@@ -40,13 +40,21 @@ clean-cert:
 clean-osm-controller:
 	@rm -rf bin/osm-controller
 
+.PHONY: clean-osm-injector
+clean-osm-injector:
+	@rm -rf bin/osm-injector
+
 .PHONY: build
-build: build-osm-controller
+build: build-osm-controller build-osm-injector
 
 .PHONY: build-osm-controller
 build-osm-controller: clean-osm-controller
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./bin/osm-controller/osm-controller -ldflags "-X $(BUILD_DATE_VAR)=$(BUILD_DATE) -X $(BUILD_VERSION_VAR)=$(VERSION) -X $(BUILD_GITCOMMIT_VAR)=$(GIT_SHA) -s -w" ./cmd/osm-controller
 
+.PHONY: build-osm-injector
+build-osm-injector: clean-osm-injector
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./bin/osm-injector/osm-injector -ldflags "-X $(BUILD_DATE_VAR)=$(BUILD_DATE) -X $(BUILD_VERSION_VAR)=$(VERSION) -X $(BUILD_GITCOMMIT_VAR)=$(GIT_SHA) -s -w" ./cmd/osm-injector
+
 .PHONY: build-osm
 build-osm:
 	go run scripts/generate_chart/generate_chart.go | CGO_ENABLED=0  go build -v -o ./bin/osm -ldflags ${LDFLAGS} ./cmd/cli
@@ -95,7 +103,7 @@ kind-reset:
 	kind delete cluster --name osm
 
 .PHONY: test-e2e
-test-e2e: docker-build-osm-controller docker-build-init build-osm docker-build-tcp-echo-server
+test-e2e: docker-build-osm-controller docker-build-osm-injector docker-build-init build-osm docker-build-tcp-echo-server
 	go test ./tests/e2e $(E2E_FLAGS_DEFAULT) $(E2E_FLAGS)
 
 .env:
@@ -116,7 +124,7 @@ $(DEMO_BUILD_TARGETS):
 	@if [ -f demo/$(NAME).html.template ]; then cp demo/$(NAME).html.template demo/bin/$(NAME); fi
 
 .PHONY: demo-build
-demo-build: $(DEMO_BUILD_TARGETS) build-osm-controller
+demo-build: $(DEMO_BUILD_TARGETS) build-osm-controller build-osm-injector
 
 # docker-build-bookbuyer, etc
 DOCKER_DEMO_TARGETS = $(addprefix docker-build-, $(DEMO_TARGETS))
@@ -132,11 +140,14 @@ docker-build-init:
 docker-build-osm-controller: build-osm-controller
 	docker build -t $(CTR_REGISTRY)/osm-controller:$(CTR_TAG) -f dockerfiles/Dockerfile.osm-controller bin/osm-controller
 
+docker-build-osm-injector: build-osm-injector
+	docker build -t $(CTR_REGISTRY)/osm-injector:$(CTR_TAG) -f dockerfiles/Dockerfile.osm-injector bin/osm-injector
+
 .PHONY: docker-build
-docker-build: $(DOCKER_DEMO_TARGETS) docker-build-init docker-build-osm-controller
+docker-build: $(DOCKER_DEMO_TARGETS) docker-build-init docker-build-osm-controller docker-build-osm-injector
 
 # docker-push-bookbuyer, etc
-DOCKER_PUSH_TARGETS = $(addprefix docker-push-, $(DEMO_TARGETS) init osm-controller)
+DOCKER_PUSH_TARGETS = $(addprefix docker-push-, $(DEMO_TARGETS) init osm-controller osm-injector)
 VERIFY_TAGS = 0
 .PHONY: $(DOCKER_PUSH_TARGETS)
 $(DOCKER_PUSH_TARGETS): NAME=$(@:docker-push-%=%)

diff --git a/charts/osm/README.md b/charts/osm/README.md
@@ -94,6 +94,7 @@ The following table lists the configurable parameters of the osm chart and their
 | OpenServiceMesh.image.registry | string | `"openservicemesh"` | `osm-controller` image registry |
 | OpenServiceMesh.image.tag | string | `"v0.7.0"` | `osm-controller` image tag |
 | OpenServiceMesh.imagePullSecrets | list | `[]` | `osm-controller` image pull secret |
+| OpenServiceMesh.injector | object | `{"replicaCount":1,"resource":{"limits":{"cpu":"1.0","memory":"256M"},"requests":{"cpu":"0.5","memory":"32M"}}}` | Sidecar injector configuration |
 | OpenServiceMesh.meshName | string | `"osm"` | Name for the new control plane instance |
 | OpenServiceMesh.osmNamespace | string | `""` | Optional parameter. If not specified, the release namespace is used to deploy the osm components. |
 | OpenServiceMesh.osmcontroller.resource.limits.cpu | string | `"1.5"` |  |

diff --git a/charts/osm/templates/mutatingwebhook.yaml b/charts/osm/templates/mutatingwebhook.yaml
@@ -8,7 +8,7 @@ webhooks:
 - name: osm-inject.k8s.io
   clientConfig:
     service:
-      name: osm-controller
+      name: osm-injector
       namespace: {{ include "osm.namespace" . }}
       path: /mutate-pod-creation
       port: 9090

diff --git a/charts/osm/templates/osm-deployment.yaml b/charts/osm/templates/osm-deployment.yaml
@@ -37,8 +37,6 @@ spec:
             "--verbosity", "{{.Values.OpenServiceMesh.controllerLogLevel}}",
             "--osm-namespace", "{{ include "osm.namespace" . }}",
             "--mesh-name", "{{.Values.OpenServiceMesh.meshName}}",
-            "--init-container-image", "{{.Values.OpenServiceMesh.image.registry}}/init:{{ .Values.OpenServiceMesh.image.tag }}",
-            "--sidecar-image", "{{.Values.OpenServiceMesh.sidecarImage}}",
             "--webhook-config-name", "{{.Values.OpenServiceMesh.webhookConfigNamePrefix}}-{{.Values.OpenServiceMesh.meshName}}",
             "--ca-bundle-secret-name", "{{.Values.OpenServiceMesh.caBundleSecretName}}",
             "--certificate-manager", "{{.Values.OpenServiceMesh.certificateManager}}",

diff --git a/charts/osm/templates/osm-injector-deployment.yaml b/charts/osm/templates/osm-injector-deployment.yaml
@@ -0,0 +1,81 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: osm-injector
+  namespace: {{ include "osm.namespace" . }}
+  labels:
+    app: osm-injector
+    meshName: {{ .Values.OpenServiceMesh.meshName }}
+spec:
+  replicas: {{ .Values.OpenServiceMesh.injector.replicaCount }}
+  selector:
+    matchLabels:
+      app: osm-injector
+  template:
+    metadata:
+      labels:
+        app: osm-injector
+      annotations:
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: '9091'
+    spec:
+      serviceAccountName: {{ .Release.Name }}
+      containers:
+        - name: osm-injector
+          image: "{{ .Values.OpenServiceMesh.image.registry }}/osm-injector:{{ .Values.OpenServiceMesh.image.tag }}"
+          imagePullPolicy: {{ .Values.OpenServiceMesh.image.pullPolicy }}
+          ports:
+            - name: "sidecar-inject"
+              containerPort: 9090
+            - name: "metrics"
+              containerPort: 9091
+          command: ['/osm-injector']
+          args: [
+            "--verbosity", "{{.Values.OpenServiceMesh.controllerLogLevel}}",
+            "--osm-namespace", "{{ include "osm.namespace" . }}",
+            "--mesh-name", "{{.Values.OpenServiceMesh.meshName}}",
+            "--init-container-image", "{{.Values.OpenServiceMesh.image.registry}}/init:{{ .Values.OpenServiceMesh.image.tag }}",
+            "--sidecar-image", "{{.Values.OpenServiceMesh.sidecarImage}}",
+            "--webhook-config-name", "{{.Values.OpenServiceMesh.webhookConfigNamePrefix}}-{{.Values.OpenServiceMesh.meshName}}",
+            "--ca-bundle-secret-name", "{{.Values.OpenServiceMesh.caBundleSecretName}}",
+            "--certificate-manager", "{{.Values.OpenServiceMesh.certificateManager}}",
+            {{ if eq .Values.OpenServiceMesh.certificateManager "vault" }}
+            "--vault-host", "{{.Values.OpenServiceMesh.vault.host}}",
+            "--vault-protocol", "{{.Values.OpenServiceMesh.vault.protocol}}",
+            "--vault-token", "{{.Values.OpenServiceMesh.vault.token}}",
+            {{- end }}
+            "--cert-manager-issuer-name", "{{.Values.OpenServiceMesh.certmanager.issuerName}}",
+            "--cert-manager-issuer-kind", "{{.Values.OpenServiceMesh.certmanager.issuerKind}}",
+            "--cert-manager-issuer-group", "{{.Values.OpenServiceMesh.certmanager.issuerGroup}}",
+          ]
+          resources:
+            limits:
+              cpu: "{{.Values.OpenServiceMesh.injector.resource.limits.cpu}}"
+              memory: "{{.Values.OpenServiceMesh.injector.resource.limits.memory}}"
+            requests:
+              cpu: "{{.Values.OpenServiceMesh.injector.resource.requests.cpu}}"
+              memory: "{{.Values.OpenServiceMesh.injector.resource.requests.memory}}"
+          readinessProbe:
+            initialDelaySeconds: 1
+            timeoutSeconds: 5
+            httpGet:
+              scheme: HTTPS
+              path: /healthz
+              port: 9090
+          livenessProbe:
+            initialDelaySeconds: 1
+            timeoutSeconds: 5
+            httpGet:
+              scheme: HTTPS
+              path: /healthz
+              port: 9090
+          env:
+            # The INJECTOR_POD_NAME env variable sets pod name dynamically, used by osm-injector to register events
+            - name: INJECTOR_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+    {{- if .Values.OpenServiceMesh.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml .Values.OpenServiceMesh.imagePullSecrets | indent 8 }}
+    {{- end }}
diff --git a/charts/osm/templates/osm-injector-service.yaml b/charts/osm/templates/osm-injector-service.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: osm-injector
+  namespace: {{ include "osm.namespace" . }}
+  labels:
+    app: osm-injector
+spec:
+  ports:
+    - name: sidecar-inject
+      port: 9090
+  selector:
+    app: osm-injector
diff --git a/charts/osm/templates/osm-service.yaml b/charts/osm/templates/osm-service.yaml
@@ -10,9 +10,6 @@ spec:
     - name: osm-port
       port: 15128
       targetPort: 15128
-    - name: sidecar-injector
-      port: 9090
-      targetPort: 9090
     - name: debug-port
       port: 9092
       targetPort: 9092

diff --git a/charts/osm/values.yaml b/charts/osm/values.yaml
@@ -137,3 +137,14 @@ OpenServiceMesh:
   # -- Optional parameter to specify a global list of IP ranges to exclude from outbound traffic interception by the sidecar proxy.
   # If specified, must be a list of IP ranges of the form a.b.c.d/x.
   outboundIPRangeExclusionList: []
+
+  # -- Sidecar injector configuration
+  injector:
+    replicaCount: 1
+    resource:
+      limits:
+        cpu: "1.0"
+        memory: "256M"
+      requests:
+        cpu: "0.5"
+        memory: "32M"
diff --git a/cmd/osm-controller/osm-controller.go b/cmd/osm-controller/osm-controller.go
@@ -22,7 +22,6 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"k8s.io/client-go/tools/clientcmd"
-	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/openservicemesh/osm/pkg/catalog"
 	"github.com/openservicemesh/osm/pkg/certificate"
@@ -37,12 +36,10 @@ import (
 	"github.com/openservicemesh/osm/pkg/health"
 	"github.com/openservicemesh/osm/pkg/httpserver"
 	"github.com/openservicemesh/osm/pkg/ingress"
-	"github.com/openservicemesh/osm/pkg/injector"
 	k8s "github.com/openservicemesh/osm/pkg/kubernetes"
 	"github.com/openservicemesh/osm/pkg/kubernetes/events"
 	"github.com/openservicemesh/osm/pkg/logger"
 	"github.com/openservicemesh/osm/pkg/metricsstore"
-	reconciler "github.com/openservicemesh/osm/pkg/reconciler/mutatingwebhook"
 	"github.com/openservicemesh/osm/pkg/signals"
 	"github.com/openservicemesh/osm/pkg/smi"
 	"github.com/openservicemesh/osm/pkg/version"
@@ -53,17 +50,13 @@ const (
 )
 
 var (
-	verbosity            string
-	meshName             string // An ID that uniquely identifies an OSM instance
-	kubeConfigFile       string
-	osmNamespace         string
-	webhookConfigName    string
-	caBundleSecretName   string
-	osmConfigMapName     string
-	metricsAddr          string
-	enableLeaderElection bool
-
-	injectorConfig injector.Config
+	verbosity          string
+	meshName           string // An ID that uniquely identifies an OSM instance
+	kubeConfigFile     string
+	osmNamespace       string
+	webhookConfigName  string
+	caBundleSecretName string
+	osmConfigMapName   string
 
 	certProviderKind string
 
@@ -91,11 +84,6 @@ func init() {
 	flags.StringVar(&webhookConfigName, "webhook-config-name", "", "Name of the MutatingWebhookConfiguration to be configured by osm-controller")
 	flags.StringVar(&osmConfigMapName, "osm-configmap-name", "osm-config", "Name of the OSM ConfigMap")
 
-	// sidecar injector options
-	flags.IntVar(&injectorConfig.ListenPort, "webhook-port", constants.InjectorWebhookPort, "Webhook port for sidecar-injector")
-	flags.StringVar(&injectorConfig.InitContainerImage, "init-container-image", "", "InitContainer image")
-	flags.StringVar(&injectorConfig.SidecarImage, "sidecar-image", "", "Sidecar proxy Container image")
-
 	// Generic certificate manager/provider options
 	flags.StringVar(&certProviderKind, "certificate-manager", providers.TresorKind.String(), fmt.Sprintf("Certificate manager, one of [%v]", providers.ValidCertificateProviders))
 	flags.StringVar(&caBundleSecretName, "ca-bundle-secret-name", "", "Name of the Kubernetes Secret for the OSM CA bundle")
@@ -116,15 +104,6 @@ func init() {
 	flags.BoolVar(&optionalFeatures.Backpressure, "enable-backpressure-experimental", false, "Enable experimental backpressure feature")
 	flags.BoolVar(&optionalFeatures.RoutesV2, "enable-routes-v2-experimental", false, "Enable experimental routes v2 feature")
 
-	// k8s controller manager options
-	// a k8s controller provided by the package "sigs.k8s.io/controller-runtime" helps to ensure that the state of a given k8s object is as per its desired state
-	// a controller manager is responsible for running controllers, managing the life cycle of the controller and setting up common dependencies
-	// metrics-addr is the endpoint for the performance metrics generated by the controller
-	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
-	// enable-leader-election is a flag to ensure there's only one manager for the controller
-	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
-		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
-
 	_ = clientgoscheme.AddToScheme(scheme)
 	_ = v1beta1.AddToScheme(scheme)
 }
@@ -174,7 +153,7 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	// Start the default metrics store and start it.
+	// Start the default metrics store
 	metricsstore.DefaultMetricsStore.Start()
 
 	// This component will be watching the OSM ConfigMap and will make it
@@ -230,11 +209,6 @@ func main() {
 		cfg,
 		endpointsProviders...)
 
-	// Create the sidecar-injector webhook
-	if err := injector.NewMutatingWebhook(injectorConfig, kubeClient, certManager, meshCatalog, kubernetesClient, meshName, osmNamespace, webhookConfigName, stop, cfg); err != nil {
-		events.GenericEventRecorder().FatalEvent(err, events.InitializationError, "Error creating sidecar injector webhook")
-	}
-
 	// Create the configMap validating webhook
 	if err := configurator.NewValidatingWebhook(kubeClient, certManager, osmNamespace, webhookConfigName, stop); err != nil {
 		events.GenericEventRecorder().FatalEvent(err, events.InitializationError, "Error creating osm-config validating webhook")
@@ -251,10 +225,6 @@ func main() {
 		events.GenericEventRecorder().FatalEvent(err, events.InitializationError, "Error initializing ADS server")
 	}
 
-	if err := createControllerManagerForOSMResources(certManager); err != nil {
-		events.GenericEventRecorder().FatalEvent(err, events.InitializationError, "Error creating controller manager to reconcile OSM resources")
-	}
-
 	// Initialize OSM's http service server
 	httpServer := httpserver.NewHTTPServer(constants.OSMHTTPServerPort)
 
@@ -284,15 +254,16 @@ func main() {
 	log.Info().Msgf("Stopping osm-controller %s; %s; %s", version.Version, version.GitCommit, version.BuildDate)
 }
 
+// getHTTPHealthProbes returns the HTTP health probes served by OSM controller
 func getHTTPHealthProbes() []health.HTTPProbe {
-	return []health.HTTPProbe{
-		{
-			// HTTP probe on the sidecar injector webhook's port
-			URL: joinURL(fmt.Sprintf("https://%s:%d", constants.LocalhostIPAddress, injectorConfig.ListenPort),
-				injector.WebhookHealthPath),
-			Protocol: health.ProtocolHTTPS,
-		},
-	}
+	// Example:
+	// return []health.HTTPProbe{
+	// 	{
+	// 		URL: "https://127.0.0.1:<local-port>",
+	// 		Protocol: health.ProtocolHTTPS,
+	// 	},
+	// }
+	return nil
 }
 
 func parseFlags() error {
@@ -304,11 +275,6 @@ func parseFlags() error {
 }
 
 func createOrUpdateCABundleKubernetesSecret(kubeClient clientset.Interface, certManager certificate.Manager, namespace, caBundleSecretName string) error {
-	if caBundleSecretName == "" {
-		log.Info().Msg("No name provided for CA bundle k8s secret. Skip creation of secret")
-		return nil
-	}
-
 	ca, err := certManager.GetRootCertificate()
 	if err != nil {
 		log.Error().Err(err).Msgf("Error getting root certificate")
@@ -389,44 +355,3 @@ func getOSMControllerPod(kubeClient kubernetes.Interface) (*corev1.Pod, error) {
 
 	return pod, nil
 }
-
-//Setting up k8s controller manager to reconcile OSM resources
-func createControllerManagerForOSMResources(certManager certificate.Manager) error {
-	log.Info().Msg("Setting up controller manager to reconcile OSM resources")
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:             scheme,
-		MetricsBindAddress: metricsAddr,
-		LeaderElection:     enableLeaderElection,
-		Namespace:          osmNamespace,
-	})
-	if err != nil {
-		log.Error().Err(err).Msg("Error starting up controller manager")
-		return err
-	}
-
-	log.Info().Msg("Successfully setup controller for resource reconciliation")
-	log.Info().Msg("Setting up mutatingWebhookConfiguration reconciler")
-
-	// controller logic is implemented by reconciler
-	// Adding a reconciler for OSM's mutatingwehbookconfiguration
-	if err = (&reconciler.MutatingWebhookConfigurationReconciler{
-		Client:       mgr.GetClient(),
-		Scheme:       mgr.GetScheme(),
-		OsmWebhook:   fmt.Sprintf("osm-webhook-%s", meshName),
-		OsmNamespace: osmNamespace,
-		CertManager:  certManager,
-	}).SetupWithManager(mgr); err != nil {
-		log.Error().Err(err).Msg("Error creating reconcile controller for MutatingWebhookConfiguration")
-		return err
-	}
-
-	go func() {
-		// mgr.Start() below will block until stopped
-		// See: https://github.com/kubernetes-sigs/controller-runtime/blob/release-0.6/pkg/manager/internal.go#L507-L514
-		if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
-			log.Error().Err(err).Msg("problem running manager for controller")
-		}
-	}()
-
-	return nil
-}