Merge pull request #604 from KubeKyrie/add-otel-trace

add OTEL trace for clusterpedia-apiserver
clusterpedia-io · Dec 5, 2023 · ed85cfc · ed85cfc
2 parents 84c7402 + 2e13a70
commit ed85cfc
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 3 deletions.
diff --git a/cmd/apiserver/app/options/options.go b/cmd/apiserver/app/options/options.go
@@ -39,7 +39,7 @@ type ClusterPediaServerOptions struct {
 	CoreAPI        *genericoptions.CoreAPIOptions
 	FeatureGate    featuregate.FeatureGate
 	Admission      *genericoptions.AdmissionOptions
-	//      Traces         *genericoptions.TracingOptions
+	Traces         *genericoptions.TracingOptions
 
 	Storage *storageoptions.StorageOptions
 }
@@ -66,7 +66,7 @@ func NewServerOptions() *ClusterPediaServerOptions {
 		CoreAPI:        genericoptions.NewCoreAPIOptions(),
 		FeatureGate:    feature.DefaultFeatureGate,
 		Admission:      genericoptions.NewAdmissionOptions(),
-		//      Traces:         genericoptions.NewTracingOptions(),
+		Traces:         genericoptions.NewTracingOptions(),
 
 		Storage: storageoptions.NewStorageOptions(),
 	}
@@ -154,6 +154,9 @@ func (o *ClusterPediaServerOptions) genericOptionsApplyTo(config *genericapiserv
 	if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, client, dynamicClient, o.FeatureGate); err != nil {
 		return err
 	}
+	if err := o.Traces.ApplyTo(nil, &config.Config); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -176,7 +179,7 @@ func (o *ClusterPediaServerOptions) Flags() cliflag.NamedFlagSets {
 	logsapi.AddFlags(o.Logs, fss.FlagSet("logs"))
 
 	// o.Admission.AddFlags(fss.FlagSet("admission"))
-	// o.Traces.AddFlags(fss.FlagSet("traces"))
+	o.Traces.AddFlags(fss.FlagSet("traces"))
 
 	o.Storage.AddFlags(fss.FlagSet("storage"))
 	return fss

diff --git a/deploy/clusterpedia_apiserver_deployment.yaml b/deploy/clusterpedia_apiserver_deployment.yaml
@@ -41,6 +41,7 @@ spec:
         - /usr/local/bin/apiserver
         - --secure-port=443
         - --storage-config=/etc/clusterpedia/storage/internalstorage-config.yaml
+        - --tracing-config-file=/etc/clusterpedia/trace/tracing-config.yaml
         - -v=3
         env:
         - name: DB_PASSWORD
@@ -52,8 +53,27 @@ spec:
         - name: internalstorage-config
           mountPath: /etc/clusterpedia/storage
           readOnly: true
+        - name: tracing-config
+          mountPath: /etc/clusterpedia/trace
+          readOnly: true
       serviceAccountName: clusterpedia-apiserver
       volumes:
       - name: internalstorage-config
         configMap:
           name: clusterpedia-internalstorage
+      - name: tracing-config
+        configMap:
+          name: clusterpedia-tracing-config
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: clusterpedia-tracing-config
+  namespace: clusterpedia-system
+data:
+  tracing-config.yaml: |
+    apiVersion: apiserver.config.k8s.io/v1beta1
+    kind: TracingConfiguration
+    # default
+    # endpoint: localhost:4317
+    samplingRatePerMillion: 1000000
diff --git a/kustomize/apiserver/clusterpedia_apiserver_deployment.yaml b/kustomize/apiserver/clusterpedia_apiserver_deployment.yaml
@@ -41,6 +41,7 @@ spec:
         - /usr/local/bin/apiserver
         - --secure-port=443
         - --storage-config=/etc/clusterpedia/storage/internalstorage-config.yaml
+        - --tracing-config-file=/etc/clusterpedia/trace/tracing-config.yaml
         - -v=3
         env:
         - name: DB_PASSWORD
@@ -52,8 +53,27 @@ spec:
         - name: internalstorage-config
           mountPath: /etc/clusterpedia/storage
           readOnly: true
+        - name: tracing-config
+          mountPath: /etc/clusterpedia/trace
+          readOnly: true
       serviceAccountName: clusterpedia-apiserver
       volumes:
       - name: internalstorage-config
         configMap:
           name: clusterpedia-internalstorage
+      - name: tracing-config
+        configMap:
+          name: clusterpedia-tracing-config
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: clusterpedia-tracing-config
+  namespace: clusterpedia-system
+data:
+  tracing-config.yaml: |
+    apiVersion: apiserver.config.k8s.io/v1beta1
+    kind: TracingConfiguration
+    # default
+    # endpoint: localhost:4317
+    samplingRatePerMillion: 1000000
diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go
@@ -124,6 +124,7 @@ func (config completedConfig) New() (*ClusterPediaServer, error) {
 	resourceServerConfig := kubeapiserver.NewDefaultConfig()
 	resourceServerConfig.GenericConfig.ExternalAddress = config.GenericConfig.ExternalAddress
 	resourceServerConfig.GenericConfig.LoopbackClientConfig = config.GenericConfig.LoopbackClientConfig
+	resourceServerConfig.GenericConfig.TracerProvider = config.GenericConfig.TracerProvider
 	resourceServerConfig.ExtraConfig = kubeapiserver.ExtraConfig{
 		InformerFactory:          clusterpediaInformerFactory,
 		StorageFactory:           config.StorageFactory,

diff --git a/pkg/kubeapiserver/apiserver.go b/pkg/kubeapiserver/apiserver.go
@@ -11,9 +11,11 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	genericrequest "k8s.io/apiserver/pkg/endpoints/request"
+	genericfeatures "k8s.io/apiserver/pkg/features"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericfilters "k8s.io/apiserver/pkg/server/filters"
 	"k8s.io/apiserver/pkg/server/healthz"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/restmapper"
 
 	informers "github.com/clusterpedia-io/clusterpedia/pkg/generated/informers/externalversions"
@@ -145,6 +147,10 @@ func BuildHandlerChain(apiHandler http.Handler, c *genericapiserver.Config) http
 	// https://github.com/clusterpedia-io/clusterpedia/issues/54
 	handler = filters.RemoveFieldSelectorFromRequest(handler)
 
+	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
+		handler = genericapifilters.WithTracing(handler, c.TracerProvider)
+	}
+
 	/* used for debugging
 	handler = genericapifilters.WithWarningRecorder(handler)
 	handler = WithClusterName(handler, "cluster-1")