feat(kuma-cp): add option to disable KDS traces (#11847)

## Motivation

These traces are potentially not useful and too long-lived.

## Implementation information

Added option to leave the status quo or completely disable KDS traces.
Note that Envoy admin operations should not be affected.

Signed-off-by: Mike Beaumont <mjboamail@gmail.com>

docs/generated/raw/kuma-cp.yaml

@@ -516,6 +516,10 @@ multizone:
       # Response backoff is a time Global CP waits before sending ACK/NACK.
       # This is a way to slow down Zone CP from sending resources too often.
       responseBackoff: 0s # ENV: KUMA_MULTIZONE_GLOBAL_KDS_RESPONSE_BACKOFF
+      tracing:
+        # Defines whether tracing is enabled for all gRPC methods
+        # of GlobalKDSServer and KDSSyncService or completely disabled
+        enabled: true # ENV: KUMA_MULTIZONE_GLOBAL_KDS_TRACING_ENABLED
   zone:
     # Kuma Zone name used to mark the zone dataplane resources
     name: "default" # ENV: KUMA_MULTIZONE_ZONE_NAME

pkg/config/app/kuma-cp/kuma-cp.defaults.yaml

@@ -516,6 +516,10 @@ multizone:
       # Response backoff is a time Global CP waits before sending ACK/NACK.
       # This is a way to slow down Zone CP from sending resources too often.
       responseBackoff: 0s # ENV: KUMA_MULTIZONE_GLOBAL_KDS_RESPONSE_BACKOFF
+      tracing:
+        # Defines whether tracing is enabled for all gRPC methods
+        # of GlobalKDSServer and KDSSyncService or completely disabled
+        enabled: true # ENV: KUMA_MULTIZONE_GLOBAL_KDS_TRACING_ENABLED
   zone:
     # Kuma Zone name used to mark the zone dataplane resources
     name: "default" # ENV: KUMA_MULTIZONE_ZONE_NAME

pkg/config/loader_test.go

@@ -266,6 +266,7 @@ var _ = Describe("Config loader", func() {
 			Expect(cfg.Multizone.Global.KDS.ResponseBackoff.Duration).To(Equal(time.Second))
 			Expect(cfg.Multizone.Global.KDS.ZoneHealthCheck.PollInterval.Duration).To(Equal(11 * time.Second))
 			Expect(cfg.Multizone.Global.KDS.ZoneHealthCheck.Timeout.Duration).To(Equal(110 * time.Second))
+			Expect(cfg.Multizone.Global.KDS.Tracing.Enabled).To(BeFalse())
 			Expect(cfg.Multizone.Zone.GlobalAddress).To(Equal("grpc://1.1.1.1:5685"))
 			Expect(cfg.Multizone.Zone.Name).To(Equal("zone-1"))
 			Expect(cfg.Multizone.Zone.KDS.RootCAFile).To(Equal("/rootCa"))
@@ -621,6 +622,8 @@ multizone:
       zoneHealthCheck:
         pollInterval: 11s
         timeout: 110s
+      tracing:
+        enabled: false
   zone:
     globalAddress: "grpc://1.1.1.1:5685"
     name: "zone-1"
@@ -964,6 +967,7 @@ meshService:
 				"KUMA_MULTIZONE_GLOBAL_KDS_RESPONSE_BACKOFF":                                               "1s",
 				"KUMA_MULTIZONE_GLOBAL_KDS_ZONE_HEALTH_CHECK_POLL_INTERVAL":                                "11s",
 				"KUMA_MULTIZONE_GLOBAL_KDS_ZONE_HEALTH_CHECK_TIMEOUT":                                      "110s",
+				"KUMA_MULTIZONE_GLOBAL_KDS_TRACING_ENABLED":                                                "false",
 				"KUMA_MULTIZONE_ZONE_GLOBAL_ADDRESS":                                                       "grpc://1.1.1.1:5685",
 				"KUMA_MULTIZONE_ZONE_NAME":                                                                 "zone-1",
 				"KUMA_MULTIZONE_ZONE_KDS_ROOT_CA_FILE":                                                     "/rootCa",

pkg/config/multizone/kds.go

@@ -42,6 +42,7 @@ type KdsServerConfig struct {
 	ResponseBackoff config_types.Duration `json:"responseBackoff" envconfig:"kuma_multizone_global_kds_response_backoff"`
 	// ZoneHealthCheck holds config for ensuring zones are online
 	ZoneHealthCheck ZoneHealthCheckConfig `json:"zoneHealthCheck"`
+	Tracing         KDSServerTracing      `json:"tracing"`
 }
 
 var _ config.Config = &KdsServerConfig{}

pkg/config/multizone/multicluster.go

@@ -46,6 +46,9 @@ func DefaultGlobalConfig() *GlobalConfig {
 			TlsMinVersion:            "TLSv1_2",
 			TlsCipherSuites:          []string{},
 			NackBackoff:              config_types.Duration{Duration: 5 * time.Second},
+			Tracing: KDSServerTracing{
+				Enabled: true,
+			},
 		},
 	}
 }

pkg/config/multizone/tracing.go

@@ -0,0 +1,7 @@
+package multizone
+
+type KDSServerTracing struct {
+	// Defines whether tracing is enabled for all gRPC methods
+	// of GlobalKDSServer and KDSSyncService or completely disabled
+	Enabled bool `json:"enabled,omitempty" envconfig:"kuma_multizone_global_kds_tracing_enabled"`
+}

pkg/kds/mux/server.go

@@ -110,8 +110,11 @@ func (s *server) Start(stop <-chan struct{}) error {
 	grpcOptions = append(
 		grpcOptions,
 		grpc.ChainUnaryInterceptor(s.unaryInterceptors...),
-		grpc.StatsHandler(otelgrpc.NewServerHandler()),
 	)
+	if s.config.Tracing.Enabled {
+		grpcOptions = append(grpcOptions, grpc.StatsHandler(otelgrpc.NewServerHandler()))
+	}
+
 	grpcServer := grpc.NewServer(grpcOptions...)
 
 	mesh_proto.RegisterGlobalKDSServiceServer(grpcServer, s.serviceServer)