diff --git a/README.md b/README.md index 870b9228..c44c093f 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ Steps: 7. While developing and making changes to `k8s-dqlite`, just restart k8s-dqlite -Note: When develop k8s-dqlite against Canonical Kubernetes use the following flags: +Note: When developing k8s-dqlite against Canonical Kubernetes use the following flags: - `--storage-dir /var/snap/k8s/common/var/lib/k8s-dqlite` - `--listen unix:///var/snap/k8s/common/var/lib/k8s-dqlite/k8s-dqlite.sock` diff --git a/cmd/root.go b/cmd/root.go index 1b061e87..152a1cce 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "errors" "net/http" _ "net/http/pprof" "os" @@ -28,6 +29,8 @@ var ( minTLSVersion string metrics bool metricsAddress string + otel bool + otelAddress string watchAvailableStorageInterval time.Duration watchAvailableStorageMinBytes uint64 @@ -58,6 +61,17 @@ var ( }() } + var otelShutdown func(context.Context) error + + if rootCmdOpts.otel { + var err error + logrus.WithField("address", rootCmdOpts.otelAddress).Print("Enable otel endpoint") + otelShutdown, err = setupOTelSDK(cmd.Context(), rootCmdOpts.otelAddress) + if err != nil { + logrus.WithError(err).Warning("Failed to setup OpenTelemetry SDK") + } + } + if rootCmdOpts.metrics { go func() { logrus.WithField("address", rootCmdOpts.metricsAddress).Print("Enable metrics endpoint") @@ -110,6 +124,12 @@ var ( if err := instance.Shutdown(stopCtx); err != nil { logrus.WithError(err).Fatal("Failed to shutdown server") } + if rootCmdOpts.otel && otelShutdown != nil { + err = errors.Join(err, otelShutdown(stopCtx)) + if err != nil { + logrus.WithError(err).Warning("Failed to shutdown OpenTelemetry SDK") + } + } }, } ) @@ -133,6 +153,8 @@ func init() { rootCmd.Flags().UintVar(&rootCmdOpts.clientSessionCacheSize, "tls-client-session-cache-size", 0, "ClientCacheSession size for dial TLS config") rootCmd.Flags().StringVar(&rootCmdOpts.minTLSVersion, "min-tls-version", "tls12", "Minimum TLS version for dqlite endpoint (tls10|tls11|tls12|tls13). Default is tls12") rootCmd.Flags().BoolVar(&rootCmdOpts.metrics, "metrics", true, "enable metrics endpoint") + rootCmd.Flags().BoolVar(&rootCmdOpts.otel, "otel", false, "enable traces endpoint") + rootCmd.Flags().StringVar(&rootCmdOpts.otelAddress, "otel-listen", "127.0.0.1:4317", "listen address for OpenTelemetry endpoint") rootCmd.Flags().StringVar(&rootCmdOpts.metricsAddress, "metrics-listen", "127.0.0.1:9042", "listen address for metrics endpoint") rootCmd.Flags().DurationVar(&rootCmdOpts.watchAvailableStorageInterval, "watch-storage-available-size-interval", 5*time.Second, "Interval to check if the disk is running low on space. Set to 0 to disable the periodic disk size check") rootCmd.Flags().Uint64Var(&rootCmdOpts.watchAvailableStorageMinBytes, "watch-storage-available-size-min-bytes", 10*1024*1024, "Minimum required available disk size (in bytes) to continue operation. If available disk space gets below this threshold, then the --low-available-storage-action is performed") diff --git a/cmd/tracing.go b/cmd/tracing.go new file mode 100644 index 00000000..b6eabb82 --- /dev/null +++ b/cmd/tracing.go @@ -0,0 +1,134 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +var ( + resourceName = "k8s-dqlite" +) + +// setupOTelSDK bootstraps the OpenTelemetry pipeline. +// If it does not return an error, make sure to call shutdown for proper cleanup. +func setupOTelSDK(ctx context.Context, otelEndpoint string) (shutdown func(context.Context) error, err error) { + conn, err := initConn(otelEndpoint) + if err != nil { + return nil, fmt.Errorf("failed to create gRPC connection to collector: %w", err) + } + + res, err := resource.New(ctx, + resource.WithAttributes( + semconv.ServiceNameKey.String(resourceName), + ), + ) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create resource") + } + + traceExporter, err := newTraceExporter(ctx, conn) + if err != nil { + connErr := conn.Close() + if connErr != nil { + logrus.WithError(connErr).Warning("Failed to shut down otel gRPC connection") + } + return nil, fmt.Errorf("failed to create trace exporter: %w", err) + } + + tracerProvider := newTraceProvider(traceExporter, res) + otel.SetTracerProvider(tracerProvider) + + meterProvider, err := newMeterProvider(res) + if err != nil { + var shutdownErrs error + shutdownErr := tracerProvider.Shutdown(ctx) + if shutdownErr != nil { + shutdownErrs = errors.Join(shutdownErrs, shutdownErr) + } + shutdownErr = conn.Close() + if shutdownErr != nil { + shutdownErrs = errors.Join(shutdownErrs, shutdownErr) + } + if shutdownErrs != nil { + logrus.WithError(shutdownErrs).Warning("Failed to shutdown OpenTelemetry SDK") + return nil, fmt.Errorf("failed to create meter provider: %w", err) + } + } + otel.SetMeterProvider(meterProvider) + + shutdown = func(ctx context.Context) error { + var shutdownErrs error + err = meterProvider.Shutdown(ctx) + if err != nil { + shutdownErrs = errors.Join(shutdownErrs, err) + } + err = tracerProvider.Shutdown(ctx) + if err != nil { + shutdownErrs = errors.Join(shutdownErrs, err) + } + err = conn.Close() + if err != nil { + shutdownErrs = errors.Join(shutdownErrs, err) + } + return shutdownErrs + } + return shutdown, nil +} + +func initConn(otelEndpoint string) (*grpc.ClientConn, error) { + // It connects the OpenTelemetry Collector through local gRPC connection. + conn, err := grpc.NewClient(otelEndpoint, + // Note the use of insecure transport here. TLS is recommended in production. + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + return nil, fmt.Errorf("failed to create gRPC connection to collector: %w", err) + } + + return conn, nil +} + +func newTraceExporter(ctx context.Context, conn *grpc.ClientConn) (trace.SpanExporter, error) { + exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn)) + if err != nil { + return nil, fmt.Errorf("failed to create trace exporter: %w", err) + } + return exporter, nil +} + +func newTraceProvider(traceExporter trace.SpanExporter, res *resource.Resource) *trace.TracerProvider { + traceProvider := trace.NewTracerProvider( + trace.WithBatcher(traceExporter, + trace.WithBatchTimeout(time.Second), + ), + trace.WithResource(res), + ) + return traceProvider +} + +func newMeterProvider(res *resource.Resource) (*metric.MeterProvider, error) { + metricExporter, err := stdoutmetric.New() + if err != nil { + return nil, err + } + + meterProvider := metric.NewMeterProvider( + metric.WithReader(metric.NewPeriodicReader(metricExporter, + metric.WithInterval(30*time.Second))), + metric.WithResource(res), + ) + return meterProvider, nil +} diff --git a/go.mod b/go.mod index 7a640896..37acc898 100644 --- a/go.mod +++ b/go.mod @@ -15,20 +15,27 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.12 go.etcd.io/etcd/client/v3 v3.5.12 go.etcd.io/etcd/server/v3 v3.5.12 - golang.org/x/sync v0.6.0 - golang.org/x/sys v0.18.0 - google.golang.org/grpc v1.62.1 + go.opentelemetry.io/otel v1.28.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 + go.opentelemetry.io/otel/metric v1.28.0 + go.opentelemetry.io/otel/sdk v1.28.0 + go.opentelemetry.io/otel/sdk/metric v1.28.0 + go.opentelemetry.io/otel/trace v1.28.0 + golang.org/x/sync v0.7.0 + golang.org/x/sys v0.21.0 + google.golang.org/grpc v1.65.0 gopkg.in/yaml.v2 v2.4.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect @@ -36,11 +43,12 @@ require ( github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/renameio v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -58,23 +66,18 @@ require ( go.etcd.io/etcd/pkg/v3 v3.5.12 // indirect go.etcd.io/etcd/raft/v3 v3.5.12 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/sdk v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect - go.opentelemetry.io/proto/otlp v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 7e58cc53..4082c078 100644 --- a/go.sum +++ b/go.sum @@ -53,11 +53,11 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/canonical/go-dqlite v1.20.0 h1:pnkn0oS0hPXWeODjvjWONKGb5KYh8kK0aruDPzZLwmU= github.com/canonical/go-dqlite v1.20.0/go.mod h1:Uvy943N8R4CFUAs59A1NVaziWY9nJ686lScY7ywurfg= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -96,8 +96,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -178,6 +178,8 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU= github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -190,8 +192,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -287,8 +289,8 @@ github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43Z github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -319,8 +321,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= @@ -359,20 +361,24 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 h1:Mw5xcxMwlqoJd97vwPxA8isEaIoxsta9/Q51+TTJLGE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0/go.mod h1:CQNu9bj7o7mC6U7+CA/schKEYakYXWr79ucDHTMGhCM= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= -go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= -go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= -go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 h1:BJee2iLkfRfl9lc7aFmBwkWxY/RI1RDdXepSF6y8TPE= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0/go.mod h1:DIzlHs3DRscCIBU3Y9YSzPfScwnYnzfnCd4g8zA7bZc= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnCQArXCKlg08= +go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -391,8 +397,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -466,8 +472,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -491,8 +497,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -540,8 +546,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -551,8 +557,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -610,8 +616,8 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -689,10 +695,10 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhlOkSocDLfG9rrNHH7w7uoKWPaWZ8s= google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U= -google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 h1:oqta3O3AnlWbmIE3bFnWbu4bRxZjfbWCp0cKSuZh01E= -google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -713,8 +719,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -727,8 +733,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/hack/jaeger.sh b/hack/jaeger.sh new file mode 100755 index 00000000..a0817986 --- /dev/null +++ b/hack/jaeger.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# This script sets up a local jaeger instance for the otel tracing logs +# 4317 HTTP (OTLP) over gRPC +# 4318 HTTP (OTLP) over HTTP +# connect via http://localhost:16686 +docker run --name jaeger \ + -e COLLECTOR_OTLP_ENABLED=true \ + -p 16686:16686 \ + -p 4317:4317 \ + -p 4318:4318 \ + jaegertracing/all-in-one:1.35 + diff --git a/pkg/kine/drivers/generic/generic.go b/pkg/kine/drivers/generic/generic.go index 309b33bb..f43850fc 100644 --- a/pkg/kine/drivers/generic/generic.go +++ b/pkg/kine/drivers/generic/generic.go @@ -13,8 +13,51 @@ import ( "github.com/Rican7/retry/jitter" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" ) +const otelName = "generic" + +var ( + otelTracer trace.Tracer + otelMeter metric.Meter + setCompactRevCnt metric.Int64Counter + getRevisionCnt metric.Int64Counter + deleteRevCnt metric.Int64Counter + currentRevCnt metric.Int64Counter + getCompactRevCnt metric.Int64Counter +) + +func init() { + var err error + otelTracer = otel.Tracer(otelName) + otelMeter = otel.Meter(otelName) + setCompactRevCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.compact", otelName), metric.WithDescription("Number of compact requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + getRevisionCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.get_revision", otelName), metric.WithDescription("Number of get revision requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + deleteRevCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.delete_revision", otelName), metric.WithDescription("Number of delete revision requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + currentRevCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.current_revision", otelName), metric.WithDescription("Current revision")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + getCompactRevCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.get_compact_revision", otelName), metric.WithDescription("Get compact revision")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + +} + var ( columns = "kv.id as theid, kv.name, kv.created, kv.deleted, kv.create_revision, kv.prev_revision, kv.lease, kv.value, kv.old_value" @@ -477,6 +520,8 @@ func (d *Generic) executePrepared(ctx context.Context, txName, sql string, prepa } func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) { + getCompactRevCnt.Add(ctx, 1) + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.get_compact_revision", otelName)) var compact, target sql.NullInt64 start := time.Now() var err error @@ -484,8 +529,10 @@ func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) if err == sql.ErrNoRows { err = nil } + span.RecordError(err) recordOpResult("revision_interval_sql", err, start) recordTxResult("revision_interval_sql", err) + span.End() }() done, err := d.AdmissionControlPolicy.Admit(ctx, "revision_interval_sql") @@ -511,21 +558,49 @@ func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) if err == sql.ErrNoRows { return 0, 0, nil } - + span.SetAttributes(attribute.Int64("compact", compact.Int64), attribute.Int64("target", target.Int64)) return compact.Int64, target.Int64, err } func (d *Generic) SetCompactRevision(ctx context.Context, revision int64) error { - _, err := d.executePrepared(ctx, "update_compact_sql", d.UpdateCompactSQL, d.updateCompactSQLPrepared, revision) + var err error + setCompactRevCnt.Add(ctx, 1) + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.set_compact_revision", otelName)) + defer func() { + span.RecordError(err) + span.End() + }() + span.SetAttributes(attribute.Int64("revision", revision)) + + _, err = d.executePrepared(ctx, "update_compact_sql", d.UpdateCompactSQL, d.updateCompactSQLPrepared, revision) return err } func (d *Generic) GetRevision(ctx context.Context, revision int64) (*sql.Rows, error) { - return d.queryPrepared(ctx, "get_revision_sql", d.GetRevisionSQL, d.getRevisionSQLPrepared, revision) + var err error + getRevisionCnt.Add(ctx, 1) + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.get_revision", otelName)) + defer func() { + span.RecordError(err) + span.End() + }() + span.SetAttributes(attribute.Int64("revision", revision)) + + result, err := d.queryPrepared(ctx, "get_revision_sql", d.GetRevisionSQL, d.getRevisionSQLPrepared, revision) + return result, err } func (d *Generic) DeleteRevision(ctx context.Context, revision int64) error { - _, err := d.executePrepared(ctx, "delete_sql", d.DeleteSQL, d.deleteSQLPrepared, revision) + var err error + deleteRevCnt.Add(ctx, 1) + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.delete_revision", otelName)) + defer func() { + span.RecordError(err) + span.End() + }() + span.SetAttributes(attribute.Int64("revision", revision)) + + _, err = d.executePrepared(ctx, "delete_sql", d.DeleteSQL, d.deleteSQLPrepared, revision) return err } @@ -565,6 +640,13 @@ func (d *Generic) CurrentRevision(ctx context.Context) (int64, error) { var id int64 var err error + currentRevCnt.Add(ctx, 1) + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.current_revision", otelName)) + defer func() { + span.RecordError(err) + span.End() + }() + done, err := d.AdmissionControlPolicy.Admit(ctx, "rev_sql") if err != nil { return 0, fmt.Errorf("denied: %w", err) @@ -574,8 +656,10 @@ func (d *Generic) CurrentRevision(ctx context.Context) (int64, error) { done() err = row.Scan(&id) if err == sql.ErrNoRows { + span.AddEvent("no rows") return 0, nil } + span.SetAttributes(attribute.Int64("id", id)) return id, err } diff --git a/pkg/kine/logstructured/logstructured.go b/pkg/kine/logstructured/logstructured.go index c3f59a92..0f6ff105 100644 --- a/pkg/kine/logstructured/logstructured.go +++ b/pkg/kine/logstructured/logstructured.go @@ -2,14 +2,28 @@ package logstructured import ( "context" + "fmt" "sync" "sync/atomic" "time" "github.com/canonical/k8s-dqlite/pkg/kine/server" "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) +const otelName = "logstructured" + +var ( + otelTracer trace.Tracer +) + +func init() { + otelTracer = otel.Tracer(otelName) +} + type Log interface { Start(ctx context.Context) error Wait() @@ -57,9 +71,19 @@ func (l *LogStructured) Wait() { } func (l *LogStructured) Get(ctx context.Context, key, rangeEnd string, limit, revision int64) (revRet int64, kvRet *server.KeyValue, errRet error) { + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Get", otelName)) + span.SetAttributes( + attribute.String("key", key), + attribute.String("rangeEnd", rangeEnd), + attribute.Int64("limit", limit), + attribute.Int64("revision", revision), + ) defer func() { l.adjustRevision(ctx, &revRet) logrus.Debugf("GET %s, rev=%d => rev=%d, kv=%v, err=%v", key, revision, revRet, kvRet != nil, errRet) + span.SetAttributes(attribute.Int64("adjusted-revision", revRet)) + span.RecordError(errRet) + span.End() }() rev, event, err := l.get(ctx, key, rangeEnd, limit, revision, false) @@ -70,8 +94,22 @@ func (l *LogStructured) Get(ctx context.Context, key, rangeEnd string, limit, re } func (l *LogStructured) get(ctx context.Context, key, rangeEnd string, limit, revision int64, includeDeletes bool) (int64, *server.Event, error) { + var err error + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.get", otelName)) + defer func() { + span.RecordError(err) + span.End() + }() + span.SetAttributes( + attribute.String("key", key), + attribute.String("rangeEnd", rangeEnd), + attribute.Int64("limit", limit), + attribute.Int64("revision", revision), + attribute.Bool("includeDeletes", includeDeletes), + ) rev, events, err := l.log.List(ctx, key, rangeEnd, limit, revision, includeDeletes) if err == server.ErrCompacted { + span.AddEvent("key already compacted") // ignore compacted when getting by revision err = nil } @@ -98,9 +136,19 @@ func (l *LogStructured) adjustRevision(ctx context.Context, rev *int64) { } func (l *LogStructured) Create(ctx context.Context, key string, value []byte, lease int64) (revRet int64, errRet error) { + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Create", otelName)) + span.SetAttributes( + attribute.String("key", key), + attribute.Int64("lease", lease), + attribute.Int64("value-size", int64(len(value))), + ) defer func() { l.adjustRevision(ctx, &revRet) logrus.Debugf("CREATE %s, size=%d, lease=%d => rev=%d, err=%v", key, len(value), lease, revRet, errRet) + span.SetAttributes(attribute.Int64("returned-revision", revRet)) + span.RecordError(errRet) + span.End() + }() rev, prevEvent, err := l.get(ctx, key, "", 1, 0, true) @@ -130,13 +178,25 @@ func (l *LogStructured) Create(ctx context.Context, key string, value []byte, le } func (l *LogStructured) Delete(ctx context.Context, key string, revision int64) (revRet int64, kvRet *server.KeyValue, deletedRet bool, errRet error) { + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Delete", otelName)) + defer func() { l.adjustRevision(ctx, &revRet) logrus.Debugf("DELETE %s, rev=%d => rev=%d, kv=%v, deleted=%v, err=%v", key, revision, revRet, kvRet != nil, deletedRet, errRet) + span.SetAttributes( + attribute.String("key", key), + attribute.Int64("revision", revision), + attribute.Int64("adjusted-revision", revRet), + attribute.Bool("deleted", deletedRet), + attribute.Bool("kv-found", kvRet != nil), + ) + span.RecordError(errRet) + span.End() }() rev, event, err := l.get(ctx, key, "", 1, 0, true) if err != nil { + span.RecordError(err) return 0, nil, false, err } @@ -162,8 +222,12 @@ func (l *LogStructured) Delete(ctx context.Context, key string, revision int64) if err != nil { // If error on Append we assume it's a UNIQUE constraint error, so we fetch the latest (if we can) // and return that the delete failed + span.AddEvent("Failed to append delete event") + span.RecordError(err) latestRev, latestEvent, latestErr := l.get(ctx, key, "", 1, 0, true) if latestErr != nil || latestEvent == nil { + span.RecordError(latestErr) + span.SetAttributes(attribute.Bool("latest-event-found", latestEvent != nil)) return rev, event.KV, false, nil } return latestRev, latestEvent.KV, false, nil @@ -172,8 +236,20 @@ func (l *LogStructured) Delete(ctx context.Context, key string, revision int64) } func (l *LogStructured) List(ctx context.Context, prefix, startKey string, limit, revision int64) (revRet int64, kvRet []*server.KeyValue, errRet error) { + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.List", otelName)) + defer func() { logrus.Debugf("LIST %s, start=%s, limit=%d, rev=%d => rev=%d, kvs=%d, err=%v", prefix, startKey, limit, revision, revRet, len(kvRet), errRet) + span.SetAttributes( + attribute.String("prefix", prefix), + attribute.String("startKey", startKey), + attribute.Int64("limit", limit), + attribute.Int64("revision", revision), + attribute.Int64("adjusted-revision", revRet), + attribute.Int64("kv-count", int64(len(kvRet))), + ) + span.RecordError(errRet) + span.End() }() rev, events, err := l.log.List(ctx, prefix, startKey, limit, revision, false) @@ -202,8 +278,18 @@ func (l *LogStructured) List(ctx context.Context, prefix, startKey string, limit } func (l *LogStructured) Count(ctx context.Context, prefix, startKey string, revision int64) (revRet int64, count int64, err error) { + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Count", otelName)) defer func() { logrus.Debugf("COUNT prefix=%s startKey=%s => rev=%d, count=%d, err=%v", prefix, startKey, revRet, count, err) + span.SetAttributes( + attribute.String("prefix", prefix), + attribute.String("startKey", startKey), + attribute.Int64("revision", revision), + attribute.Int64("adjusted-revision", revRet), + attribute.Int64("count", count), + ) + span.RecordError(err) + span.End() }() rev, count, err := l.log.Count(ctx, prefix, startKey, revision) if err != nil { @@ -223,6 +309,7 @@ func (l *LogStructured) Count(ctx context.Context, prefix, startKey string, revi } func (l *LogStructured) Update(ctx context.Context, key string, value []byte, revision, lease int64) (revRet int64, kvRet *server.KeyValue, updateRet bool, errRet error) { + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Update", otelName)) defer func() { l.adjustRevision(ctx, &revRet) kvRev := int64(0) @@ -230,6 +317,16 @@ func (l *LogStructured) Update(ctx context.Context, key string, value []byte, re kvRev = kvRet.ModRevision } logrus.Debugf("UPDATE %s, value=%d, rev=%d, lease=%v => rev=%d, kvrev=%d, updated=%v, err=%v", key, len(value), revision, lease, revRet, kvRev, updateRet, errRet) + span.SetAttributes( + attribute.String("key", key), + attribute.Int64("revision", revision), + attribute.Int64("lease", lease), + attribute.Int64("value-size", int64(len(value))), + attribute.Int64("adjusted-revision", revRet), + attribute.Int64("kv-mod-revision", kvRev), + attribute.Bool("updated", updateRet), + ) + span.End() }() rev, event, err := l.get(ctx, key, "", 1, 0, false) @@ -335,6 +432,12 @@ func (l *LogStructured) ttl(ctx context.Context) { func (l *LogStructured) Watch(ctx context.Context, prefix string, revision int64) <-chan []*server.Event { logrus.Debugf("WATCH %s, revision=%d", prefix, revision) + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Watch", otelName)) + defer span.End() + span.SetAttributes( + attribute.String("prefix", prefix), + attribute.Int64("revision", revision), + ) // starting watching right away so we don't miss anything ctx, cancel := context.WithCancel(ctx) @@ -350,10 +453,14 @@ func (l *LogStructured) Watch(ctx context.Context, prefix string, revision int64 rev, kvs, err := l.log.After(ctx, prefix, revision, 0) if err != nil { logrus.Errorf("failed to list %s for revision %d", prefix, revision) + msg := fmt.Sprintf("failed to list %s for revision %d", prefix, revision) + span.AddEvent(msg) + logrus.Errorf(msg) cancel() } logrus.Debugf("WATCH LIST key=%s rev=%d => rev=%d kvs=%d", prefix, revision, rev, len(kvs)) + span.SetAttributes(attribute.Int64("current-revision", rev), attribute.Int64("kvs-count", int64(len(kvs)))) l.wg.Add(1) go func() { diff --git a/pkg/kine/logstructured/sqllog/sql.go b/pkg/kine/logstructured/sqllog/sql.go index c688a594..678fc0ca 100644 --- a/pkg/kine/logstructured/sqllog/sql.go +++ b/pkg/kine/logstructured/sqllog/sql.go @@ -11,9 +11,33 @@ import ( "github.com/canonical/k8s-dqlite/pkg/kine/broadcaster" "github.com/canonical/k8s-dqlite/pkg/kine/server" "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" ) -const SupersededCount = 100 +const ( + SupersededCount = 100 + otelName = "sqllog" +) + +var ( + otelTracer trace.Tracer + otelMeter metric.Meter + compactCnt metric.Int64Counter +) + +func init() { + var err error + otelTracer = otel.Tracer(otelName) + otelMeter = otel.Meter(otelName) + + compactCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.compact", otelName), metric.WithDescription("Number of compact requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } +} type SQLLog struct { d Dialect @@ -124,7 +148,16 @@ func (s *SQLLog) DoCompact(ctx context.Context) error { } func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { + var err error + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.compactor", otelName)) + defer func() { + span.RecordError(err) + span.End() + }() + span.SetAttributes(attribute.Int64("nextEnd", nextEnd)) + currentRev, err := s.d.CurrentRevision(ctx) + span.SetAttributes(attribute.Int64("currentRev", currentRev)) if err != nil { logrus.Errorf("failed to get current revision: %v", err) return nextEnd, fmt.Errorf("failed to get current revision: %v", err) @@ -135,6 +168,7 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { logrus.Errorf("failed to get compact revision: %v", err) return nextEnd, fmt.Errorf("failed to get compact revision: %v", err) } + span.SetAttributes(attribute.Int64("cursor", cursor)) end := nextEnd nextEnd = currentRev @@ -148,6 +182,8 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { savedCursor := cursor // Purposefully start at the current and redo the current as // it could have failed before actually compacting + compactCnt.Add(ctx, 1) + span.AddEvent(fmt.Sprintf("start compaction from %d to %d", cursor, end)) for ; cursor <= end; cursor++ { rows, err := s.d.GetRevision(ctx, cursor) if err != nil { @@ -168,6 +204,7 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { event := events[0] if event.KV.Key == "compact_rev_key" { + span.AddEvent("skip compact_rev_key") // don't compact the compact key continue } @@ -176,6 +213,7 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { if event.PrevKV != nil && event.PrevKV.ModRevision != 0 { if savedCursor != cursor { if err := s.d.SetCompactRevision(ctx, cursor); err != nil { + span.AddEvent(fmt.Sprintf("failed to record compact revision: %v", err)) logrus.Errorf("failed to record compact revision: %v", err) return nextEnd, fmt.Errorf("failed to record compact revision: %v", err) } @@ -184,6 +222,7 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { } if err := s.d.DeleteRevision(ctx, event.PrevKV.ModRevision); err != nil { + span.AddEvent(fmt.Sprintf("failed to delete revision %d", event.PrevKV.ModRevision)) logrus.Errorf("failed to delete revision %d: %v", event.PrevKV.ModRevision, err) return nextEnd, fmt.Errorf("failed to delete revision %d: %v", event.PrevKV.ModRevision, err) } @@ -211,13 +250,12 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { return nextEnd, fmt.Errorf("failed to record compact revision: %v", err) } } - + span.SetAttributes(attribute.Int64("new-nextEnd", nextEnd), attribute.Int64("cursor-ended:", cursor)) return nextEnd, nil } func (s *SQLLog) compact() { var nextEnd int64 - t := time.NewTicker(s.d.GetCompactInterval()) nextEnd, _ = s.d.CurrentRevision(s.ctx) diff --git a/pkg/kine/server/create.go b/pkg/kine/server/create.go index af6f43c1..2779a0d3 100644 --- a/pkg/kine/server/create.go +++ b/pkg/kine/server/create.go @@ -2,8 +2,10 @@ package server import ( "context" + "fmt" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.opentelemetry.io/otel/attribute" ) func isCreate(txn *etcdserverpb.TxnRequest) *etcdserverpb.PutRequest { @@ -20,6 +22,18 @@ func isCreate(txn *etcdserverpb.TxnRequest) *etcdserverpb.PutRequest { } func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest, txn *etcdserverpb.TxnRequest) (*etcdserverpb.TxnResponse, error) { + var err error + createCnt.Add(ctx, 1) + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.create", otelName)) + defer func() { + span.RecordError(err) + span.End() + }() + span.SetAttributes( + attribute.String("key", string(put.Key)), + attribute.Int64("lease", put.Lease), + ) + if put.IgnoreLease { return nil, unsupported("ignoreLease") } else if put.IgnoreValue { @@ -29,7 +43,9 @@ func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest } rev, err := l.backend.Create(ctx, string(put.Key), put.Value, put.Lease) + span.SetAttributes(attribute.Int64("revision", rev)) if err == ErrKeyExists { + span.AddEvent("key exists") return &etcdserverpb.TxnResponse{ Header: txnHeader(rev), Succeeded: false, diff --git a/pkg/kine/server/delete.go b/pkg/kine/server/delete.go index 03cace71..0a5e63b3 100644 --- a/pkg/kine/server/delete.go +++ b/pkg/kine/server/delete.go @@ -2,8 +2,10 @@ package server import ( "context" + "fmt" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.opentelemetry.io/otel/attribute" ) func isDelete(txn *etcdserverpb.TxnRequest) (int64, string, bool) { @@ -28,10 +30,23 @@ func isDelete(txn *etcdserverpb.TxnRequest) (int64, string, bool) { } func (l *LimitedServer) delete(ctx context.Context, key string, revision int64) (*etcdserverpb.TxnResponse, error) { + var err error + deleteCnt.Add(ctx, 1) + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.delete", otelName)) + defer func() { + span.RecordError(err) + span.End() + }() + span.SetAttributes( + attribute.String("key", key), + attribute.Int64("revision", revision), + ) + rev, kv, ok, err := l.backend.Delete(ctx, key, revision) if err != nil { return nil, err } + span.SetAttributes(attribute.Bool("ok", ok)) if !ok { return &etcdserverpb.TxnResponse{ diff --git a/pkg/kine/server/get.go b/pkg/kine/server/get.go index 2a126742..2f1ce6d5 100644 --- a/pkg/kine/server/get.go +++ b/pkg/kine/server/get.go @@ -5,11 +5,27 @@ import ( "fmt" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.opentelemetry.io/otel/attribute" ) func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { + var err error + getCnt.Add(ctx, 1) + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.get", otelName)) + defer func() { + span.RecordError(err) + span.End() + }() + + span.SetAttributes( + attribute.String("key", string(r.Key)), + attribute.String("rangeEnd", string(r.RangeEnd)), + attribute.Int64("limit", r.Limit), + attribute.Int64("revision", r.Revision), + ) if r.Limit != 0 && len(r.RangeEnd) != 0 { - return nil, fmt.Errorf("invalid combination of rangeEnd and limit, limit should be 0 got %d", r.Limit) + err := fmt.Errorf("invalid combination of rangeEnd and limit, limit should be 0 got %d", r.Limit) + return nil, err } rev, kv, err := l.backend.Get(ctx, string(r.Key), string(r.RangeEnd), r.Limit, r.Revision) diff --git a/pkg/kine/server/limited.go b/pkg/kine/server/limited.go index 2d362fc3..476f28fa 100644 --- a/pkg/kine/server/limited.go +++ b/pkg/kine/server/limited.go @@ -12,6 +12,8 @@ type LimitedServer struct { } func (l *LimitedServer) Range(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Range", otelName)) + defer span.End() if len(r.RangeEnd) == 0 { return l.get(ctx, r) } diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index 07802060..2d7f770d 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -8,9 +8,22 @@ import ( "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.opentelemetry.io/otel/attribute" ) func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { + var err error + listCnt.Add(ctx, 1) + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.list", otelName)) + defer func() { + span.RecordError(err) + span.End() + }() + + span.SetAttributes( + attribute.String("key", string(r.Key)), + attribute.String("rangeEnd", string(r.RangeEnd)), + ) if len(r.RangeEnd) == 0 { return nil, fmt.Errorf("invalid range end length of 0") } @@ -21,12 +34,19 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) } start := string(bytes.TrimRight(r.Key, "\x00")) revision := r.Revision + span.SetAttributes( + attribute.String("prefix", prefix), + attribute.String("start", start), + attribute.Int64("revision", revision), + ) if r.CountOnly { rev, count, err := l.backend.Count(ctx, prefix, start, revision) if err != nil { return nil, err } + span.SetAttributes(attribute.Int64("count", count)) + logrus.Tracef("LIST COUNT key=%s, end=%s, revision=%d, currentRev=%d count=%d", r.Key, r.RangeEnd, revision, rev, count) return &RangeResponse{ Header: txnHeader(rev), @@ -38,6 +58,7 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) if limit > 0 { limit++ } + span.SetAttributes(attribute.Int64("limit", limit)) rev, kvs, err := l.backend.List(ctx, prefix, start, limit, revision) if err != nil { @@ -49,6 +70,7 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) Count: int64(len(kvs)), Kvs: kvs, } + span.SetAttributes(attribute.Int64("list-count", resp.Count)) // count the actual number of results if there are more items in the db. if limit > 0 && resp.Count > r.Limit { @@ -64,6 +86,8 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) if err != nil { return nil, err } + + span.SetAttributes(attribute.Int64("count", resp.Count)) logrus.Tracef("LIST COUNT key=%s, end=%s, revision=%d, currentRev=%d count=%d", r.Key, r.RangeEnd, revision, rev, resp.Count) resp.Header = txnHeader(rev) } diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go new file mode 100644 index 00000000..9737a3fb --- /dev/null +++ b/pkg/kine/server/otel.go @@ -0,0 +1,49 @@ +package server + +import ( + "fmt" + + "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +const otelName = "limited-server" + +var ( + otelTracer trace.Tracer + otelMeter metric.Meter + createCnt metric.Int64Counter + deleteCnt metric.Int64Counter + getCnt metric.Int64Counter + listCnt metric.Int64Counter + updateCnt metric.Int64Counter +) + +func init() { + var err error + otelTracer = otel.Tracer(otelName) + otelMeter = otel.Meter(otelName) + + createCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.create", otelName), metric.WithDescription("Number of create requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + deleteCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.delete", otelName), metric.WithDescription("Number of delete requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create delete counter") + } + getCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.get", otelName), metric.WithDescription("Number of get requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create get counter") + } + listCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.list", otelName), metric.WithDescription("Number of list requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create list counter") + } + updateCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.update", otelName), metric.WithDescription("Number of update requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create update counter") + } +} diff --git a/pkg/kine/server/update.go b/pkg/kine/server/update.go index 52ba5993..94acfd49 100644 --- a/pkg/kine/server/update.go +++ b/pkg/kine/server/update.go @@ -2,8 +2,10 @@ package server import ( "context" + "fmt" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.opentelemetry.io/otel/attribute" ) func isUpdate(txn *etcdserverpb.TxnRequest) (int64, string, []byte, int64, bool) { @@ -29,12 +31,30 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value ok bool err error ) + updateCnt.Add(ctx, 1) + + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.update", otelName)) + defer func() { + span.RecordError(err) + span.End() + }() + span.SetAttributes( + attribute.String("key", key), + attribute.Int64("lease", lease), + attribute.Int64("revision", rev), + ) if rev == 0 { rev, err = l.backend.Create(ctx, key, value, lease) ok = true + + span.SetAttributes( + attribute.Int64("revision", rev), + attribute.Bool("ok", ok), + ) } else { rev, kv, ok, err = l.backend.Update(ctx, key, value, rev, lease) + span.SetAttributes(attribute.Bool("ok", ok)) } if err != nil { return nil, err