feat(upgrade): Add new http server to crd converter

Add another http server to the crd converter which servers as the health endpoint which is used to sync the readiness of osm-controller based on this port. Update the cleanup hook as pre-delete to ensure propoer deletion order of CRD's and services via Helm. Part of openservicemesh#3396 Signed-off-by: Sneha Chhabria <snchh@microsoft.com>
snehachhabria · Jul 19, 2021 · e2ab88a · e2ab88a
1 parent 7ecbf42
commit e2ab88a
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 17 deletions.
diff --git a/charts/osm/templates/cleanup-hook.yaml b/charts/osm/templates/cleanup-hook.yaml
@@ -9,7 +9,7 @@ metadata:
     seccomp.security.alpha.kubernetes.io/defaultProfileName:  'runtime/default'
     apparmor.security.beta.kubernetes.io/defaultProfileName:  'runtime/default'
     helm.sh/hook-weight: "-1"
-    helm.sh/hook: post-delete
+    helm.sh/hook: pre-delete
     helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 spec:
   privileged: false
@@ -60,7 +60,7 @@ metadata:
   labels:
     {{- include "osm.labels" . | nindent 4 }}
   annotations:
-    helm.sh/hook: post-delete
+    helm.sh/hook: pre-delete
     helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 rules:
   - apiGroups: ["config.openservicemesh.io"]
@@ -85,7 +85,7 @@ metadata:
   labels:
     {{- include "osm.labels" . | nindent 4 }}
   annotations:
-    helm.sh/hook: post-delete
+    helm.sh/hook: pre-delete
     helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 subjects:
   - kind: ServiceAccount
@@ -104,7 +104,7 @@ metadata:
   labels:
     {{- include "osm.labels" . | nindent 4 }}
   annotations:
-    helm.sh/hook: post-delete
+    helm.sh/hook: pre-delete
     helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 ---
 apiVersion: v1
@@ -115,7 +115,7 @@ metadata:
   labels:
     {{- include "osm.labels" . | nindent 4 }}
   annotations:
-    helm.sh/hook: post-delete
+    helm.sh/hook: pre-delete
     helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 spec:
   serviceAccountName: {{ .Release.Name }}-cleanup

diff --git a/charts/osm/templates/osm-crd-converter-deployment.yaml b/charts/osm/templates/osm-crd-converter-deployment.yaml
@@ -41,6 +41,8 @@ spec:
               containerPort: 443
             - name: "metrics"
               containerPort: 9091
+            - name: "health"
+              containerPort: 9095
           command: ['/osm-crd-converter']
           args: [
             "--verbosity", "{{.Values.OpenServiceMesh.controllerLogLevel}}",
@@ -67,16 +69,16 @@ spec:
             initialDelaySeconds: 5
             timeoutSeconds: 5
             httpGet:
-              scheme: HTTPS
+              scheme: HTTP
               path: /healthz
-              port: 443
+              port: 9095
           livenessProbe:
             initialDelaySeconds: 5
             timeoutSeconds: 5
             httpGet:
-              scheme: HTTPS
+              scheme: HTTP
               path: /healthz
-              port: 443
+              port: 9095
           env:
             # The CRD_CONVERTER_POD_NAME env variable sets pod name dynamically, used by osm-crd-converter to register events
             - name: CRD_CONVERTER_POD_NAME

diff --git a/charts/osm/templates/osm-crd-converter-service.yaml b/charts/osm/templates/osm-crd-converter-service.yaml
@@ -12,6 +12,8 @@ spec:
     - name: tls
       port: 443
       targetPort: tls
+    - name: health
+      port: 9095
   selector:
     app: osm-crd-converter
 {{- end }}

diff --git a/charts/osm/templates/osm-deployment.yaml b/charts/osm/templates/osm-deployment.yaml
@@ -35,6 +35,18 @@ spec:
         kubernetes.io/arch: amd64
         kubernetes.io/os: linux
       initContainers:
+        {{- if .Values.OpenServiceMesh.featureFlags.enableCRDConverter }}
+        - name: init-osm-controller-crd
+          image: curlimages/curl
+          args:
+          - /bin/sh
+          - -c
+          - >
+            set -x;
+            while [ $(curl -sw '%{http_code}' "http://osm-crd-converter.{{ include "osm.namespace" . }}.svc.cluster.local:9095/healthz" -o /dev/null) -ne 200 ]; do
+              sleep 10;
+            done
+        {{- end }}
         - name: init-osm-controller
           image: "{{ .Values.OpenServiceMesh.image.registry }}/init-osm-controller:{{ .Values.OpenServiceMesh.image.tag }}"
           imagePullPolicy: {{ .Values.OpenServiceMesh.image.pullPolicy }}

diff --git a/pkg/crdconversion/crdconversion.go b/pkg/crdconversion/crdconversion.go
@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"net/http"
+	"sync"
 
 	"github.com/pkg/errors"
 	apiv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -23,6 +24,9 @@ const (
 
 	// crdConverterServiceName is the name of the OSM crd converter webhook service
 	crdConverterServiceName = "osm-crd-converter"
+
+	// healthPort is the port on which the '/healthz` requests are served
+	healthPort = 9095
 )
 
 var crdConversionWebhookConfiguration = map[string]string{
@@ -75,15 +79,16 @@ func (crdWh *crdConversionWebhook) run(stop <-chan struct{}) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	mux := http.NewServeMux()
+	wg := new(sync.WaitGroup)
+	wg.Add(2)
 
-	mux.HandleFunc(webhookHealthPath, healthHandler)
+	webhookMux := http.NewServeMux()
 
 	// TODO (snchh): add handler and logic for conversion stratergy of each CRD in OSM
 
-	server := &http.Server{
+	webhookServer := &http.Server{
 		Addr:    fmt.Sprintf(":%d", crdWh.config.ListenPort),
-		Handler: mux,
+		Handler: webhookMux,
 	}
 
 	log.Info().Msgf("Starting conversion webhook server on port: %v", crdWh.config.ListenPort)
@@ -96,25 +101,49 @@ func (crdWh *crdConversionWebhook) run(stop <-chan struct{}) {
 		}
 
 		// #nosec G402
-		server.TLSConfig = &tls.Config{
+		webhookServer.TLSConfig = &tls.Config{
 			Certificates: []tls.Certificate{cert},
 		}
 
-		if err := server.ListenAndServeTLS("", ""); err != nil {
+		if err := webhookServer.ListenAndServeTLS("", ""); err != nil {
 			log.Error().Err(err).Msg("crd-converter webhook HTTP server failed to start")
 			return
 		}
+		wg.Done()
+	}()
+
+	healthMux := http.NewServeMux()
+
+	healthMux.HandleFunc(webhookHealthPath, healthHandler)
+
+	healthServer := &http.Server{
+		Addr:    fmt.Sprintf(":%d", healthPort),
+		Handler: healthMux,
+	}
+
+	go func() {
+		if err := healthServer.ListenAndServe(); err != nil {
+			log.Error().Err(err).Msg("crd-converter health server failed to start")
+			return
+		}
+		wg.Done()
 	}()
 
 	// Wait on exit signals
 	<-stop
 
-	// Stop the server
-	if err := server.Shutdown(ctx); err != nil {
+	// Stop the servers
+	if err := webhookServer.Shutdown(ctx); err != nil {
 		log.Error().Err(err).Msg("Error shutting down crd-conversion webhook HTTP server")
 	} else {
 		log.Info().Msg("Done shutting down crd-conversion webhook HTTP server")
 	}
+
+	if err := healthServer.Shutdown(ctx); err != nil {
+		log.Error().Err(err).Msg("Error shutting down crd-conversion health server")
+	} else {
+		log.Info().Msg("Done shutting down crd-conversion health server")
+	}
 }
 
 func healthHandler(w http.ResponseWriter, _ *http.Request) {