remove all k8s services from consul

.changelog/4255.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+sync-catalog: After running `helm upgrade` with `syncCatalog` disabled, the registered K8S services should be removed from consul.
+```

charts/consul/templates/gateway-resources-job.yaml

@@ -17,7 +17,7 @@ metadata:
   annotations:
     "helm.sh/hook": post-install,post-upgrade
     "helm.sh/hook-weight": "0"
-    "helm.sh/hook-delete-policy": hook-succeeded
+    "helm.sh/hook-delete-policy": hook-succeeded,hook-failed
 spec:
   template:
     metadata:

charts/consul/templates/sync-catalog-cleanup-k8s-svc-in-consul-job.yaml

@@ -0,0 +1,112 @@
+{{- if not .Values.syncCatalog.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "consul.fullname" . }}-sync-catalog-cleanup
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: sync-catalog
+    {{- if .Values.global.extraLabels }}
+      {{- toYaml .Values.global.extraLabels | nindent 4 }}
+    {{- end }}
+  annotations:
+    "helm.sh/hook": pre-delete,post-upgrade
+    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-delete-policy": hook-succeeded,hook-failed,before-hook-creation
+spec:
+  template:
+    metadata:
+      name: {{ template "consul.fullname" . }}-sync-catalog-cleanup
+      labels:
+        app: {{ template "consul.name" . }}
+        chart: {{ template "consul.chart" . }}
+        release: {{ .Release.Name }}
+        component: sync-catalog
+        {{- if .Values.global.extraLabels }}
+          {{- toYaml .Values.global.extraLabels | nindent 8 }}
+        {{- end }}
+      annotations:
+        "consul.hashicorp.com/connect-inject": "false"
+        "consul.hashicorp.com/mesh-inject": "false"
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ template "consul.fullname" . }}-sync-catalog
+      volumes:
+      {{- if .Values.global.tls.enabled }}
+      {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }}
+      - name: consul-ca-cert
+        secret:
+          {{- if .Values.global.tls.caCert.secretName }}
+          secretName: {{ .Values.global.tls.caCert.secretName }}
+          {{- else }}
+          secretName: {{ template "consul.fullname" . }}-ca-cert
+          {{- end }}
+          items:
+          - key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }}
+            path: tls.crt
+      {{- end }}
+      {{- end }}
+      containers:
+      - name: sync-catalog
+        image: "{{ default .Values.global.imageK8S .Values.syncCatalog.image }}"
+        {{ template "consul.imagePullPolicy" . }}
+        {{- include "consul.restrictedSecurityContext" . | nindent 8 }}
+        env:
+        {{- include "consul.consulK8sConsulServerEnvVars" . | nindent 8 }}
+        {{- if .Values.global.acls.manageSystemACLs }}
+        - name: CONSUL_LOGIN_AUTH_METHOD
+          {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }}
+          value: {{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }}
+          {{- else }}
+          value: {{ template "consul.fullname" . }}-k8s-component-auth-method
+          {{- end }}
+        - name: CONSUL_LOGIN_DATACENTER
+          {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }}
+          value: {{ .Values.global.federation.primaryDatacenter }}
+          {{- else }}
+          value: {{ .Values.global.datacenter }}
+          {{- end }}
+        - name: CONSUL_LOGIN_META
+          value: "component=sync-catalog,pod=$(NAMESPACE)/$(POD_NAME)"
+        {{- end }}
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        {{- if (and .Values.syncCatalog.aclSyncToken.secretName .Values.syncCatalog.aclSyncToken.secretKey) }}
+        - name: CONSUL_ACL_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.syncCatalog.aclSyncToken.secretName }}
+              key: {{ .Values.syncCatalog.aclSyncToken.secretKey }}
+        {{- end }}
+        volumeMounts:
+        {{- if .Values.global.tls.enabled }}
+        {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }}
+          - name: consul-ca-cert
+            mountPath: /consul/tls/ca
+            readOnly: true
+        {{- end }}
+        {{- end }}
+        command:
+          - consul-k8s-control-plane
+        args:
+          - sync-catalog
+          - -to-consul=false
+          - -to-k8s=false
+        resources:
+          requests:
+            memory: "50Mi"
+            cpu: "50m"
+          limits:
+            memory: "50Mi"
+            cpu: "50m"
+      {{- if .Values.global.acls.tolerations }}
+      tolerations:
+        {{ tpl .Values.global.acls.tolerations . | indent 8 | trim }}
+      {{- end }}
+{{- end }}

charts/consul/templates/sync-catalog-serviceaccount.yaml

@@ -1,5 +1,3 @@
-{{- $syncEnabled := (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }}
-{{- if $syncEnabled }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -21,4 +19,3 @@ imagePullSecrets:
   - name: {{ .name }}
 {{- end }}
 {{- end }}
-{{- end }}

control-plane/catalog/registration/registrations_controller.go

@@ -49,7 +49,7 @@ type RegistrationsController struct {
 
 func (r *RegistrationsController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := r.Log.V(1).WithValues("registration", req.NamespacedName)
-	log.Info("Reconciling Registaration")
+	log.Info("Reconciling Registration")
 
 	registration := &v1alpha1.Registration{}
 	// get the registration

control-plane/subcommand/sync-catalog/command.go

@@ -17,6 +17,7 @@ import (
 
 	mapset "github.com/deckarep/golang-set"
 	"github.com/hashicorp/consul-server-connection-manager/discovery"
+	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/go-hclog"
 	"github.com/mitchellh/cli"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -251,6 +252,13 @@ func (c *Command) Run(args []string) int {
 	}
 	c.ready = true
 
+	if !c.flagToConsul && !c.flagToK8S {
+		if err := c.removeAllK8SServicesFromConsul(consulConfig); err != nil {
+			return 1
+		}
+		return 0
+	}
+
 	// Convert allow/deny lists to sets
 	allowSet := flags.ToSet(c.flagAllowK8sNamespacesList)
 	denySet := flags.ToSet(c.flagDenyK8sNamespacesList)
@@ -393,6 +401,22 @@ func (c *Command) Run(args []string) int {
 	}
 }
 
+// remove all k8s services from Consul.
+func (c *Command) removeAllK8SServicesFromConsul(consulConfig *consul.Config) error {
+	consulClient, err := consul.NewClientFromConnMgr(consulConfig, c.connMgr)
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("unable to create Consul API client: %s", err))
+		return err
+	}
+	_, err = consulClient.Catalog().Deregister(&api.CatalogDeregistration{Node: c.flagConsulNodeName}, nil)
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("unable to deregister services from Consul: %s", err))
+		return err
+	}
+	c.logger.Info("All K8S services were deregistered from Consul")
+	return nil
+}
+
 func (c *Command) handleReady(rw http.ResponseWriter, _ *http.Request) {
 	if !c.ready {
 		c.UI.Error("[GET /health/ready] sync catalog controller is not yet ready")

control-plane/subcommand/sync-catalog/command_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/hashicorp/go-hclog"
 	"github.com/mitchellh/cli"
@@ -571,6 +572,69 @@ func TestRun_ToConsulChangingFlags(t *testing.T) {
 	}
 }
 
+// Test services could be de-registered from Consul.
+func TestRemoveAllK8SServicesFromConsul(t *testing.T) {
+	t.Parallel()
+
+	k8s, testClient := completeSetup(t)
+
+	consulClient := testClient.APIClient
+
+	// Run the command.
+	ui := cli.NewMockUi()
+	cmd := Command{
+		UI:        ui,
+		clientset: k8s,
+		logger: hclog.New(&hclog.LoggerOptions{
+			Name:  t.Name(),
+			Level: hclog.Debug,
+		}),
+		flagAllowK8sNamespacesList: []string{"*"},
+		connMgr:                    testClient.Watcher,
+	}
+
+	// create two services in k8s
+	_, err := k8s.CoreV1().Services("bar").Create(context.Background(), lbService("foo", "1.1.1.1"), metav1.CreateOptions{})
+	require.NoError(t, err)
+
+	_, err = k8s.CoreV1().Services("baz").Create(context.Background(), lbService("foo", "2.2.2.2"), metav1.CreateOptions{})
+	require.NoError(t, err)
+
+	longRunningChan := runCommandAsynchronously(&cmd, []string{
+		"-addresses", "127.0.0.1",
+		"-http-port", strconv.Itoa(testClient.Cfg.HTTPPort),
+		"-consul-write-interval", "100ms",
+		"-add-k8s-namespace-suffix",
+	})
+	defer stopCommand(t, &cmd, longRunningChan)
+
+	// check that the name of the service is namespaced
+	retry.Run(t, func(r *retry.R) {
+		svc, _, err := consulClient.Catalog().Service("foo-bar", "k8s", nil)
+		require.NoError(r, err)
+		require.Len(r, svc, 1)
+		require.Equal(r, "1.1.1.1", svc[0].ServiceAddress)
+		svc, _, err = consulClient.Catalog().Service("foo-baz", "k8s", nil)
+		require.NoError(r, err)
+		require.Len(r, svc, 1)
+		require.Equal(r, "2.2.2.2", svc[0].ServiceAddress)
+	})
+
+	exitChan := runCommandAsynchronously(&cmd, []string{
+		"-addresses", "127.0.0.1",
+		"-http-port", strconv.Itoa(testClient.Cfg.HTTPPort),
+		"-to-consul=false",
+		"-to-k8s=false",
+	})
+	stopCommand(t, &cmd, exitChan)
+
+	retry.Run(t, func(r *retry.R) {
+		serviceList, _, err := consulClient.Catalog().NodeServiceList("k8s-sync", &api.QueryOptions{AllowStale: false})
+		require.NoError(r, err)
+		require.Len(r, serviceList.Services, 0)
+	})
+}
+
 // Set up test consul agent and fake kubernetes cluster client.
 func completeSetup(t *testing.T) (*fake.Clientset, *test.TestServerClient) {
 	k8s := fake.NewSimpleClientset()