From cbdda80590be12dd2a6a841892dfd2420c6093e8 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 8 Jul 2024 12:56:52 +0200 Subject: [PATCH 01/43] initial tracing try with background.get --- README.md | 26 ++++----- go.mod | 20 +++++-- go.sum | 44 ++++++++++----- pkg/kine/server/get.go | 36 ++++++++++++ pkg/server/server.go | 9 +++ pkg/server/tracing.go | 122 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 224 insertions(+), 33 deletions(-) create mode 100644 pkg/server/tracing.go diff --git a/README.md b/README.md index 2721879b..ecad4fa2 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ This project is a drop in replacement for etcd on Kubernetes. If you are looking for an in-memory database to replace etcd in your K8s cluster dqlite might be just right for you. Here is what you are getting: - - distributed sqlite - - in-memory - - high availability - - almost zero ops +- distributed sqlite +- in-memory +- high availability +- almost zero ops ## Building @@ -30,8 +30,8 @@ make dynamic Prepare a directory containing: - - `init.yaml` with `Address: :` - - `cluster.crt` and `cluster.key` pair. +- `init.yaml` with `Address: :` +- `cluster.crt` and `cluster.key` pair. Here is a script to build you can use: @@ -121,7 +121,7 @@ Steps: ## Develop k8s-dqlite against an active MicroK8s instance -1. Install development tools: +1. Install development tools: ```bash sudo snap install go --classic @@ -129,31 +129,31 @@ Steps: sudo apt install build-essential -y ``` -2. Clone k8s-dqlite repository: +2. Clone k8s-dqlite repository: ```bash git clone https://github.com/canonical/k8s-dqlite ``` -3. Install MicroK8s on the machine: +3. Install MicroK8s on the machine: ```bash sudo snap install microk8s --classic ``` -4. Wait for MicroK8s to come up: +4. Wait for MicroK8s to come up: ```bash sudo microk8s status --wait-ready ``` -5. Stop k8s-dqlite included in the snap: +5. Stop k8s-dqlite included in the snap: ```bash sudo snap stop microk8s.daemon-k8s-dqlite --disable ``` -6. Run k8s-dqlite from this repository: +6. Run k8s-dqlite from this repository: ```bash cd k8s-dqlite @@ -165,7 +165,7 @@ Steps: 7. While developing and making changes to `k8s-dqlite`, just restart k8s-dqlite -Note: When develop k8s-dqlite against Canonical Kubernetes use the following flags: +Note: When developing k8s-dqlite against Canonical Kubernetes use the following flags: - `--storage-dir /var/snap/k8s/common/var/lib/k8s-dqlite` - `--listen unix:///var/snap/k8s/common/var/lib/k8s-dqlite/k8s-dqlite.sock` diff --git a/go.mod b/go.mod index 7a640896..98e8ef36 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,17 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.12 go.etcd.io/etcd/client/v3 v3.5.12 go.etcd.io/etcd/server/v3 v3.5.12 + go.opentelemetry.io/otel v1.28.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 + go.opentelemetry.io/otel/log v0.4.0 + go.opentelemetry.io/otel/metric v1.28.0 + go.opentelemetry.io/otel/sdk v1.28.0 + go.opentelemetry.io/otel/sdk/log v0.4.0 + go.opentelemetry.io/otel/sdk/metric v1.28.0 golang.org/x/sync v0.6.0 - golang.org/x/sys v0.18.0 + golang.org/x/sys v0.21.0 google.golang.org/grpc v1.62.1 gopkg.in/yaml.v2 v2.4.0 ) @@ -28,7 +37,7 @@ require ( github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect @@ -36,6 +45,7 @@ require ( github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/renameio v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect @@ -57,13 +67,11 @@ require ( go.etcd.io/etcd/client/v2 v2.305.12 // indirect go.etcd.io/etcd/pkg/v3 v3.5.12 // indirect go.etcd.io/etcd/raft/v3 v3.5.12 // indirect + go.opentelemetry.io/contrib/bridges/otelslog v0.3.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/sdk v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect diff --git a/go.sum b/go.sum index 7e58cc53..bb3c37e9 100644 --- a/go.sum +++ b/go.sum @@ -96,8 +96,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -178,6 +178,8 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU= github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -319,8 +321,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= @@ -357,20 +359,34 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib/bridges/otelslog v0.3.0 h1:Kf8NK4WW/pn3f9Gwx6XJAB2zlaW2M3VLQ4sQ3TKJhA8= +go.opentelemetry.io/contrib/bridges/otelslog v0.3.0/go.mod h1:JV00+So1cv6GIYNUeO0xFfl/qE+DUtS3hpBlLIyOFUE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 h1:Mw5xcxMwlqoJd97vwPxA8isEaIoxsta9/Q51+TTJLGE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0/go.mod h1:CQNu9bj7o7mC6U7+CA/schKEYakYXWr79ucDHTMGhCM= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= -go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 h1:0MH3f8lZrflbUWXVxyBg/zviDFdGE062uKh5+fu8Vv0= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0/go.mod h1:Vh68vYiHY5mPdekTr0ox0sALsqjoVy0w3Os278yX5SQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 h1:BJee2iLkfRfl9lc7aFmBwkWxY/RI1RDdXepSF6y8TPE= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0/go.mod h1:DIzlHs3DRscCIBU3Y9YSzPfScwnYnzfnCd4g8zA7bZc= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= +go.opentelemetry.io/otel/log v0.4.0 h1:/vZ+3Utqh18e8TPjuc3ecg284078KWrR8BRz+PQAj3o= +go.opentelemetry.io/otel/log v0.4.0/go.mod h1:DhGnQvky7pHy82MIRV43iXh3FlKN8UUKftn0KbLOq6I= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/sdk/log v0.4.0 h1:1mMI22L82zLqf6KtkjrRy5BbagOTWdJsqMY/HSqILAA= +go.opentelemetry.io/otel/sdk/log v0.4.0/go.mod h1:AYJ9FVF0hNOgAVzUG/ybg/QttnXhUePWAupmCqtdESo= +go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnCQArXCKlg08= +go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -540,8 +556,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pkg/kine/server/get.go b/pkg/kine/server/get.go index 2a126742..47478551 100644 --- a/pkg/kine/server/get.go +++ b/pkg/kine/server/get.go @@ -5,6 +5,18 @@ import ( "fmt" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.opentelemetry.io/contrib/bridges/otelslog" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +const name = "kine" + +var ( + tracer = otel.Tracer(name) + meter = otel.Meter(name) + logger = otelslog.NewLogger(name) + getCnt metric.Int64Counter ) func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { @@ -12,8 +24,32 @@ func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) ( return nil, fmt.Errorf("invalid combination of rangeEnd and limit, limit should be 0 got %d", r.Limit) } + ctx, span := tracer.Start( + ctx, + "Backend.get", + // trace.WithAttributes( + // trace.StringAttribute("key", string(r.Key)), + // trace.StringAttribute("rangeEnd", string(r.RangeEnd)), + // trace.Int64Attribute("limit", int64(r.Limit)), + // trace.Int64Attribute("revision", r.Revision), + // ), + ) + + defer span.End() + + getCnt, err := meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) // TODO: move this init out of the function + if err != nil { + span.RecordError(err) + } + + span.SetAttributes() + + getCnt.Add(ctx, 1) + logger.InfoContext(ctx, fmt.Sprintf("get request number %d", getCnt)) + rev, kv, err := l.backend.Get(ctx, string(r.Key), string(r.RangeEnd), r.Limit, r.Revision) if err != nil { + span.RecordError(err) return nil, err } diff --git a/pkg/server/server.go b/pkg/server/server.go index 6d3397de..4370f85a 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -336,6 +336,15 @@ func (s *Server) Start(ctx context.Context) error { if err := s.app.Ready(ctx); err != nil { return fmt.Errorf("failed to start dqlite app: %w", err) } + + // otel setup + otelShutdown, err := setupOTelSDK(ctx) + if err != nil { + return err + } + + defer otelShutdown(ctx) + logrus.WithFields(logrus.Fields{"id": s.app.ID(), "address": s.app.Address()}).Print("Started dqlite") logrus.WithField("config", s.kineConfig).Debug("Starting kine") diff --git a/pkg/server/tracing.go b/pkg/server/tracing.go new file mode 100644 index 00000000..04f60110 --- /dev/null +++ b/pkg/server/tracing.go @@ -0,0 +1,122 @@ +package server + +import ( + "context" + "errors" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" + "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/log/global" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/trace" +) + +// setupOTelSDK bootstraps the OpenTelemetry pipeline. +// If it does not return an error, make sure to call shutdown for proper cleanup. +func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, err error) { + var shutdownFuncs []func(context.Context) error + + // shutdown calls cleanup functions registered via shutdownFuncs. + // The errors from the calls are joined. + // Each registered cleanup will be invoked once. + shutdown = func(ctx context.Context) error { + var err error + for _, fn := range shutdownFuncs { + err = errors.Join(err, fn(ctx)) + } + shutdownFuncs = nil + return err + } + + // handleErr calls shutdown for cleanup and makes sure that all errors are returned. + handleErr := func(inErr error) { + err = errors.Join(inErr, shutdown(ctx)) + } + + // Set up propagator. + prop := newPropagator() + otel.SetTextMapPropagator(prop) + + // Set up trace provider. + tracerProvider, err := newTraceProvider() + if err != nil { + handleErr(err) + return + } + shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown) + otel.SetTracerProvider(tracerProvider) + + // Set up meter provider. + meterProvider, err := newMeterProvider() + if err != nil { + handleErr(err) + return + } + shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown) + otel.SetMeterProvider(meterProvider) + + // Set up logger provider. + loggerProvider, err := newLoggerProvider() + if err != nil { + handleErr(err) + return + } + shutdownFuncs = append(shutdownFuncs, loggerProvider.Shutdown) + global.SetLoggerProvider(loggerProvider) + + return +} + +func newPropagator() propagation.TextMapPropagator { + return propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) +} + +func newTraceProvider() (*trace.TracerProvider, error) { + // TODO: Replace with exporter such as Jaeger + traceExporter, err := stdouttrace.New( + stdouttrace.WithPrettyPrint()) + if err != nil { + return nil, err + } + + traceProvider := trace.NewTracerProvider( + trace.WithBatcher(traceExporter, + // Default is 5s. Set to 1s for demonstrative purposes. + trace.WithBatchTimeout(time.Second)), + ) + return traceProvider, nil +} + +func newMeterProvider() (*metric.MeterProvider, error) { + metricExporter, err := stdoutmetric.New() + if err != nil { + return nil, err + } + + meterProvider := metric.NewMeterProvider( + metric.WithReader(metric.NewPeriodicReader(metricExporter, + // Default is 1m. Set to 3s for demonstrative purposes. + metric.WithInterval(3*time.Second))), + ) + return meterProvider, nil +} + +func newLoggerProvider() (*log.LoggerProvider, error) { + logExporter, err := stdoutlog.New() + if err != nil { + return nil, err + } + + loggerProvider := log.NewLoggerProvider( + log.WithProcessor(log.NewBatchProcessor(logExporter)), + ) + return loggerProvider, nil +} From ecda7a8cc819ce2750f3e6a1f6085b2e38c09702 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 9 Jul 2024 12:56:28 +0200 Subject: [PATCH 02/43] more updates backend.get --- pkg/kine/server/get.go | 39 +++++++++----------------------- pkg/kine/server/list.go | 12 ++++++++++ pkg/kine/server/otel.go | 32 ++++++++++++++++++++++++++ pkg/{ => kine}/server/tracing.go | 0 pkg/server/server.go | 8 ------- 5 files changed, 55 insertions(+), 36 deletions(-) create mode 100644 pkg/kine/server/otel.go rename pkg/{ => kine}/server/tracing.go (100%) diff --git a/pkg/kine/server/get.go b/pkg/kine/server/get.go index 47478551..5ac93918 100644 --- a/pkg/kine/server/get.go +++ b/pkg/kine/server/get.go @@ -5,18 +5,7 @@ import ( "fmt" "go.etcd.io/etcd/api/v3/etcdserverpb" - "go.opentelemetry.io/contrib/bridges/otelslog" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/metric" -) - -const name = "kine" - -var ( - tracer = otel.Tracer(name) - meter = otel.Meter(name) - logger = otelslog.NewLogger(name) - getCnt metric.Int64Counter + "go.opentelemetry.io/otel/attribute" ) func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { @@ -26,26 +15,17 @@ func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) ( ctx, span := tracer.Start( ctx, - "Backend.get", - // trace.WithAttributes( - // trace.StringAttribute("key", string(r.Key)), - // trace.StringAttribute("rangeEnd", string(r.RangeEnd)), - // trace.Int64Attribute("limit", int64(r.Limit)), - // trace.Int64Attribute("revision", r.Revision), - // ), + "backend.get", ) - defer span.End() - - getCnt, err := meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) // TODO: move this init out of the function - if err != nil { - span.RecordError(err) - } - - span.SetAttributes() + span.SetAttributes( + attribute.String("key", string(r.Key)), + attribute.Int64("limit", int64(r.Limit)), + attribute.Int64("revision", r.Revision), + ) + defer span.End() getCnt.Add(ctx, 1) - logger.InfoContext(ctx, fmt.Sprintf("get request number %d", getCnt)) rev, kv, err := l.backend.Get(ctx, string(r.Key), string(r.RangeEnd), r.Limit, r.Revision) if err != nil { @@ -53,6 +33,9 @@ func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) ( return nil, err } + span.AddEvent("get request is completed") + span.End() + resp := &RangeResponse{ Header: txnHeader(rev), } diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index 07802060..6f1122e7 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -39,10 +39,22 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) limit++ } + ctx, span := tracer.Start( + ctx, + "backend.list", + ) + + defer span.End() + + span.SetAttributes() + rev, kvs, err := l.backend.List(ctx, prefix, start, limit, revision) if err != nil { + span.RecordError(err) return nil, err } + listCnt.Add(ctx, 1) + span.End() resp := &RangeResponse{ Header: txnHeader(rev), diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go new file mode 100644 index 00000000..9af49f41 --- /dev/null +++ b/pkg/kine/server/otel.go @@ -0,0 +1,32 @@ +package server + +import ( + "context" + "fmt" + + "go.opentelemetry.io/contrib/bridges/otelslog" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +// TODO: This can probably get moved into tracing.go +const name = "kine" + +var ( + tracer = otel.Tracer(name) + meter = otel.Meter(name) + logger = otelslog.NewLogger(name) + getCnt metric.Int64Counter + listCnt metric.Int64Counter +) + +func init() { + ctx := context.Background() + // otel setup + setupOTelSDK(ctx) + + //TODO: handle shutdown + + listCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) + getCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) +} diff --git a/pkg/server/tracing.go b/pkg/kine/server/tracing.go similarity index 100% rename from pkg/server/tracing.go rename to pkg/kine/server/tracing.go diff --git a/pkg/server/server.go b/pkg/server/server.go index 4370f85a..212cff47 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -337,14 +337,6 @@ func (s *Server) Start(ctx context.Context) error { return fmt.Errorf("failed to start dqlite app: %w", err) } - // otel setup - otelShutdown, err := setupOTelSDK(ctx) - if err != nil { - return err - } - - defer otelShutdown(ctx) - logrus.WithFields(logrus.Fields{"id": s.app.ID(), "address": s.app.Address()}).Print("Started dqlite") logrus.WithField("config", s.kineConfig).Debug("Starting kine") From 6b95d6ce7e6bcbf33bbad3fc0121a3e9f6dcce72 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 9 Jul 2024 14:29:10 +0200 Subject: [PATCH 03/43] parent spans --- pkg/kine/server/get.go | 2 +- pkg/kine/server/limited.go | 4 ++++ pkg/kine/server/list.go | 2 +- pkg/kine/server/otel.go | 14 +++++++------- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/kine/server/get.go b/pkg/kine/server/get.go index 5ac93918..a340cf64 100644 --- a/pkg/kine/server/get.go +++ b/pkg/kine/server/get.go @@ -25,7 +25,7 @@ func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) ( ) defer span.End() - getCnt.Add(ctx, 1) + backendGetCnt.Add(ctx, 1) rev, kv, err := l.backend.Get(ctx, string(r.Key), string(r.RangeEnd), r.Limit, r.Revision) if err != nil { diff --git a/pkg/kine/server/limited.go b/pkg/kine/server/limited.go index 2d362fc3..8b2f8ebb 100644 --- a/pkg/kine/server/limited.go +++ b/pkg/kine/server/limited.go @@ -13,8 +13,12 @@ type LimitedServer struct { func (l *LimitedServer) Range(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { if len(r.RangeEnd) == 0 { + ctx, span := tracer.Start(ctx, "limited.get") + defer span.End() return l.get(ctx, r) } + ctx, span := tracer.Start(ctx, "limited.list") + defer span.End() return l.list(ctx, r) } diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index 6f1122e7..0a83d393 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -53,7 +53,7 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) span.RecordError(err) return nil, err } - listCnt.Add(ctx, 1) + backendListCnt.Add(ctx, 1) span.End() resp := &RangeResponse{ diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go index 9af49f41..c6988c8f 100644 --- a/pkg/kine/server/otel.go +++ b/pkg/kine/server/otel.go @@ -13,11 +13,11 @@ import ( const name = "kine" var ( - tracer = otel.Tracer(name) - meter = otel.Meter(name) - logger = otelslog.NewLogger(name) - getCnt metric.Int64Counter - listCnt metric.Int64Counter + tracer = otel.Tracer(name) + meter = otel.Meter(name) + logger = otelslog.NewLogger(name) + backendGetCnt metric.Int64Counter + backendListCnt metric.Int64Counter ) func init() { @@ -27,6 +27,6 @@ func init() { //TODO: handle shutdown - listCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) - getCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) + backendListCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) + backendGetCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) } From 38e3b21a3bdba3b4c75fc06aa3faff98cec567af Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 9 Jul 2024 21:49:29 +0200 Subject: [PATCH 04/43] logs to jaeger --- Makefile | 13 ++++++++ go.mod | 31 +++++++++--------- go.sum | 66 ++++++++++++++++++-------------------- pkg/kine/server/tracing.go | 40 ++++++++++++++++++++--- 4 files changed, 95 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index da8bf958..4558ddfa 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,19 @@ bin/dynamic/dqlite: bin/dynamic/lib/libdqlite.so mkdir -p bin/dynamic GOBIN=$(shell pwd)/bin/dynamic $(DQLITE_BUILD_SCRIPTS_DIR)/dynamic-go-install.sh github.com/canonical/go-dqlite/cmd/dqlite@v1.20.0 +# Tracing logs export to local jaeger instance +# 4317 HTTP (OTLP) over gRPC +# 4318 HTTP (OTLP) over HTTP +# connect http://localhost:16686 +jaeger: + docker run --name jaeger \ + -e COLLECTOR_OTLP_ENABLED=true \ + -p 16686:16686 \ + -p 4317:4317 \ + -p 4318:4318 \ + jaegertracing/all-in-one:1.35 + + ## Cleanup clean: rm -rf bin hack/.build hack/.deps diff --git a/go.mod b/go.mod index 98e8ef36..dc1d23a2 100644 --- a/go.mod +++ b/go.mod @@ -15,25 +15,26 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.12 go.etcd.io/etcd/client/v3 v3.5.12 go.etcd.io/etcd/server/v3 v3.5.12 + go.opentelemetry.io/contrib/bridges/otelslog v0.3.0 go.opentelemetry.io/otel v1.28.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 go.opentelemetry.io/otel/log v0.4.0 go.opentelemetry.io/otel/metric v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/log v0.4.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 - golang.org/x/sync v0.6.0 + golang.org/x/sync v0.7.0 golang.org/x/sys v0.21.0 - google.golang.org/grpc v1.62.1 + google.golang.org/grpc v1.65.0 gopkg.in/yaml.v2 v2.4.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -50,7 +51,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -67,22 +68,20 @@ require ( go.etcd.io/etcd/client/v2 v2.305.12 // indirect go.etcd.io/etcd/pkg/v3 v3.5.12 // indirect go.etcd.io/etcd/raft/v3 v3.5.12 // indirect - go.opentelemetry.io/contrib/bridges/otelslog v0.3.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect - go.opentelemetry.io/proto/otlp v1.1.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index bb3c37e9..e8f129d3 100644 --- a/go.sum +++ b/go.sum @@ -53,11 +53,11 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/canonical/go-dqlite v1.20.0 h1:pnkn0oS0hPXWeODjvjWONKGb5KYh8kK0aruDPzZLwmU= github.com/canonical/go-dqlite v1.20.0/go.mod h1:Uvy943N8R4CFUAs59A1NVaziWY9nJ686lScY7ywurfg= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -192,8 +192,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -289,8 +289,8 @@ github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43Z github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -365,16 +365,14 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.4 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 h1:Mw5xcxMwlqoJd97vwPxA8isEaIoxsta9/Q51+TTJLGE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0/go.mod h1:CQNu9bj7o7mC6U7+CA/schKEYakYXWr79ucDHTMGhCM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 h1:0MH3f8lZrflbUWXVxyBg/zviDFdGE062uKh5+fu8Vv0= go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0/go.mod h1:Vh68vYiHY5mPdekTr0ox0sALsqjoVy0w3Os278yX5SQ= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 h1:BJee2iLkfRfl9lc7aFmBwkWxY/RI1RDdXepSF6y8TPE= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0/go.mod h1:DIzlHs3DRscCIBU3Y9YSzPfScwnYnzfnCd4g8zA7bZc= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= go.opentelemetry.io/otel/log v0.4.0 h1:/vZ+3Utqh18e8TPjuc3ecg284078KWrR8BRz+PQAj3o= go.opentelemetry.io/otel/log v0.4.0/go.mod h1:DhGnQvky7pHy82MIRV43iXh3FlKN8UUKftn0KbLOq6I= go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= @@ -387,8 +385,8 @@ go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnC go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg= go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= -go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= -go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -407,8 +405,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -482,8 +480,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -507,8 +505,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -567,8 +565,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -626,8 +624,8 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -705,10 +703,10 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhlOkSocDLfG9rrNHH7w7uoKWPaWZ8s= google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U= -google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 h1:oqta3O3AnlWbmIE3bFnWbu4bRxZjfbWCp0cKSuZh01E= -google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -729,8 +727,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -743,8 +741,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/pkg/kine/server/tracing.go b/pkg/kine/server/tracing.go index 04f60110..4321cb39 100644 --- a/pkg/kine/server/tracing.go +++ b/pkg/kine/server/tracing.go @@ -3,17 +3,20 @@ package server import ( "context" "errors" + "fmt" "time" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" - "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" "go.opentelemetry.io/otel/log/global" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/trace" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) // setupOTelSDK bootstraps the OpenTelemetry pipeline. @@ -43,7 +46,7 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er otel.SetTextMapPropagator(prop) // Set up trace provider. - tracerProvider, err := newTraceProvider() + tracerProvider, err := newTraceProvider(ctx) if err != nil { handleErr(err) return @@ -79,10 +82,37 @@ func newPropagator() propagation.TextMapPropagator { ) } -func newTraceProvider() (*trace.TracerProvider, error) { +// Initialize a gRPC connection to be used by both the tracer and meter +// providers. +func initConn() (*grpc.ClientConn, error) { + // It connects the OpenTelemetry Collector through local gRPC connection. + // You may replace `localhost:4317` with your endpoint. + conn, err := grpc.NewClient("localhost:4317", + // Note the use of insecure transport here. TLS is recommended in production. + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + return nil, fmt.Errorf("failed to create gRPC connection to collector: %w", err) + } + + return conn, err +} + +func newExporter(ctx context.Context) (trace.SpanExporter, error) { + conn, _ := initConn() + + traceExporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn)) + if err != nil { + return nil, fmt.Errorf("failed to create trace exporter: %w", err) + } + return traceExporter, nil +} + +func newTraceProvider(ctx context.Context) (*trace.TracerProvider, error) { // TODO: Replace with exporter such as Jaeger - traceExporter, err := stdouttrace.New( - stdouttrace.WithPrettyPrint()) + // traceExporter, err := stdouttrace.New( + // stdouttrace.WithPrettyPrint()) + traceExporter, err := newExporter(ctx) if err != nil { return nil, err } From 85148498eae5af2c05f4b6b7557f24268cbfe12d Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 9 Jul 2024 22:02:35 +0200 Subject: [PATCH 05/43] svc name --- pkg/kine/server/tracing.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/pkg/kine/server/tracing.go b/pkg/kine/server/tracing.go index 4321cb39..f20522eb 100644 --- a/pkg/kine/server/tracing.go +++ b/pkg/kine/server/tracing.go @@ -14,7 +14,9 @@ import ( "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -41,12 +43,20 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er err = errors.Join(inErr, shutdown(ctx)) } + // Set up resource. + res, err := resource.New(ctx, + resource.WithAttributes( + // The service name used to display traces in backends + semconv.ServiceNameKey.String("k8s-dqlite-service"), + ), + ) + // Set up propagator. prop := newPropagator() otel.SetTextMapPropagator(prop) // Set up trace provider. - tracerProvider, err := newTraceProvider(ctx) + tracerProvider, err := newTraceProvider(ctx, res) if err != nil { handleErr(err) return @@ -75,7 +85,7 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er return } -func newPropagator() propagation.TextMapPropagator { +func newPropagator() propagation.TextMapPropagator { //TODO: is this needed return propagation.NewCompositeTextMapPropagator( propagation.TraceContext{}, propagation.Baggage{}, @@ -108,7 +118,7 @@ func newExporter(ctx context.Context) (trace.SpanExporter, error) { return traceExporter, nil } -func newTraceProvider(ctx context.Context) (*trace.TracerProvider, error) { +func newTraceProvider(ctx context.Context, res *resource.Resource) (*trace.TracerProvider, error) { // TODO: Replace with exporter such as Jaeger // traceExporter, err := stdouttrace.New( // stdouttrace.WithPrettyPrint()) @@ -120,7 +130,9 @@ func newTraceProvider(ctx context.Context) (*trace.TracerProvider, error) { traceProvider := trace.NewTracerProvider( trace.WithBatcher(traceExporter, // Default is 5s. Set to 1s for demonstrative purposes. - trace.WithBatchTimeout(time.Second)), + trace.WithBatchTimeout(time.Second), + ), + trace.WithResource(res), ) return traceProvider, nil } @@ -134,7 +146,7 @@ func newMeterProvider() (*metric.MeterProvider, error) { meterProvider := metric.NewMeterProvider( metric.WithReader(metric.NewPeriodicReader(metricExporter, // Default is 1m. Set to 3s for demonstrative purposes. - metric.WithInterval(3*time.Second))), + metric.WithInterval(30*time.Second))), ) return meterProvider, nil } From 0be0a104877b8c541f9a82bce77cf9a9bee1d674 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 10 Jul 2024 08:50:47 +0200 Subject: [PATCH 06/43] traces mostly list --- pkg/kine/server/get.go | 12 ++++------- pkg/kine/server/limited.go | 8 +++++++ pkg/kine/server/list.go | 44 ++++++++++++++++++++++++++++++++------ pkg/kine/server/otel.go | 24 +++++++++++++++------ 4 files changed, 66 insertions(+), 22 deletions(-) diff --git a/pkg/kine/server/get.go b/pkg/kine/server/get.go index a340cf64..e5ecfeb4 100644 --- a/pkg/kine/server/get.go +++ b/pkg/kine/server/get.go @@ -13,18 +13,15 @@ func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) ( return nil, fmt.Errorf("invalid combination of rangeEnd and limit, limit should be 0 got %d", r.Limit) } - ctx, span := tracer.Start( - ctx, - "backend.get", - ) + ctx, span := tracer.Start(ctx, "backend.get") + defer span.End() span.SetAttributes( attribute.String("key", string(r.Key)), - attribute.Int64("limit", int64(r.Limit)), + attribute.String("rangeEnd", string(r.RangeEnd)), + attribute.Int64("limit", r.Limit), attribute.Int64("revision", r.Revision), ) - - defer span.End() backendGetCnt.Add(ctx, 1) rev, kv, err := l.backend.Get(ctx, string(r.Key), string(r.RangeEnd), r.Limit, r.Revision) @@ -33,7 +30,6 @@ func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) ( return nil, err } - span.AddEvent("get request is completed") span.End() resp := &RangeResponse{ diff --git a/pkg/kine/server/limited.go b/pkg/kine/server/limited.go index 8b2f8ebb..745dd12a 100644 --- a/pkg/kine/server/limited.go +++ b/pkg/kine/server/limited.go @@ -30,15 +30,23 @@ func txnHeader(rev int64) *etcdserverpb.ResponseHeader { func (l *LimitedServer) Txn(ctx context.Context, txn *etcdserverpb.TxnRequest) (*etcdserverpb.TxnResponse, error) { if put := isCreate(txn); put != nil { + ctx, span := tracer.Start(ctx, "limited.create") + defer span.End() return l.create(ctx, put, txn) } if rev, key, ok := isDelete(txn); ok { + ctx, span := tracer.Start(ctx, "limited.delete") + defer span.End() return l.delete(ctx, key, rev) } if rev, key, value, lease, ok := isUpdate(txn); ok { + ctx, span := tracer.Start(ctx, "limited.update") + defer span.End() return l.update(ctx, rev, key, value, lease) } if isCompact(txn) { + ctx, span := tracer.Start(ctx, "limited.compact") + defer span.End() return l.compact(ctx) } return nil, fmt.Errorf("unsupported transaction: %v", txn) diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index 0a83d393..ff7a41bb 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -8,6 +8,7 @@ import ( "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.opentelemetry.io/otel/attribute" ) func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { @@ -23,10 +24,23 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) revision := r.Revision if r.CountOnly { + ctx, span := tracer.Start(ctx, "backend.count") + defer span.End() + span.SetAttributes( + attribute.String("key", string(r.Key)), + attribute.String("rangeEnd", string(r.RangeEnd)), + attribute.String("prefix", prefix), + attribute.String("start", start), + attribute.Int64("revision", revision), + ) + backendCountCnt.Add(ctx, 1) rev, count, err := l.backend.Count(ctx, prefix, start, revision) if err != nil { + span.RecordError(err) return nil, err } + span.SetAttributes(attribute.Int64("count", count)) + logrus.Tracef("LIST COUNT key=%s, end=%s, revision=%d, currentRev=%d count=%d", r.Key, r.RangeEnd, revision, rev, count) return &RangeResponse{ Header: txnHeader(rev), @@ -39,14 +53,18 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) limit++ } - ctx, span := tracer.Start( - ctx, - "backend.list", - ) - + ctx, span := tracer.Start(ctx, "backend.list") defer span.End() - span.SetAttributes() + span.SetAttributes( + attribute.String("key", string(r.Key)), + attribute.String("rangeEnd", string(r.RangeEnd)), + attribute.String("prefix", prefix), + attribute.String("start", start), + attribute.Int64("limit", limit), + attribute.Int64("revision", revision), + ) + backendListCnt.Add(ctx, 1) rev, kvs, err := l.backend.List(ctx, prefix, start, limit, revision) if err != nil { @@ -70,12 +88,24 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) if revision == 0 { revision = rev } - // count the actual number of results if there are more items in the db. + ctx, span := tracer.Start(ctx, "backend.count") + defer span.End() + span.SetAttributes( + attribute.String("key", string(r.Key)), + attribute.String("rangeEnd", string(r.RangeEnd)), + attribute.String("prefix", prefix), + attribute.String("start", start), + attribute.Int64("revision", revision), + ) + backendCountCnt.Add(ctx, 1) rev, resp.Count, err = l.backend.Count(ctx, prefix, start, revision) if err != nil { + span.RecordError(err) return nil, err } + span.SetAttributes(attribute.Int64("count", resp.Count)) + logrus.Tracef("LIST COUNT key=%s, end=%s, revision=%d, currentRev=%d count=%d", r.Key, r.RangeEnd, revision, rev, resp.Count) resp.Header = txnHeader(rev) } diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go index c6988c8f..5f287cdf 100644 --- a/pkg/kine/server/otel.go +++ b/pkg/kine/server/otel.go @@ -10,14 +10,19 @@ import ( ) // TODO: This can probably get moved into tracing.go -const name = "kine" +const name = "k8s-dqlite" var ( - tracer = otel.Tracer(name) - meter = otel.Meter(name) - logger = otelslog.NewLogger(name) - backendGetCnt metric.Int64Counter - backendListCnt metric.Int64Counter + tracer = otel.Tracer(name) + meter = otel.Meter(name) + logger = otelslog.NewLogger(name) + backendGetCnt metric.Int64Counter + backendListCnt metric.Int64Counter + backendCountCnt metric.Int64Counter + backendCreateCnt metric.Int64Counter + backendDeleteCnt metric.Int64Counter + backendUpdateCnt metric.Int64Counter + backendCompactCnt metric.Int64Counter ) func init() { @@ -25,8 +30,13 @@ func init() { // otel setup setupOTelSDK(ctx) - //TODO: handle shutdown + //TODO: handle shutdown setupOTelSDK backendListCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) backendGetCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) + backendCountCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.count", name), metric.WithDescription("Number of count requests")) + backendCreateCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.create", name), metric.WithDescription("Number of create requests")) + backendDeleteCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.delete", name), metric.WithDescription("Number of delete requests")) + backendUpdateCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.update", name), metric.WithDescription("Number of update requests")) + backendCompactCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) } From 87e5548bcced9e3beace0ea300d3bb7344179b1f Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 10 Jul 2024 09:24:24 +0200 Subject: [PATCH 07/43] instrument create --- pkg/kine/server/create.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/kine/server/create.go b/pkg/kine/server/create.go index af6f43c1..4c9a92ce 100644 --- a/pkg/kine/server/create.go +++ b/pkg/kine/server/create.go @@ -4,6 +4,7 @@ import ( "context" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.opentelemetry.io/otel/attribute" ) func isCreate(txn *etcdserverpb.TxnRequest) *etcdserverpb.PutRequest { @@ -28,13 +29,27 @@ func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest return nil, unsupported("prevKv") } + ctx, span := tracer.Start(ctx, "backend.create") + defer span.End() + + span.SetAttributes( + + attribute.String("key", string(put.Key)), + //TODO: get key and value in utf-8 string + // attribute.String("value", string(put.Value)), + attribute.Int64("lease", put.Lease), + ) + backendCreateCnt.Add(ctx, 1) + rev, err := l.backend.Create(ctx, string(put.Key), put.Value, put.Lease) if err == ErrKeyExists { + span.RecordError(err) return &etcdserverpb.TxnResponse{ Header: txnHeader(rev), Succeeded: false, }, nil } else if err != nil { + span.RecordError(err) return nil, err } From 4e0dd87672070359e74762735b5a3cca43bc45c8 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 10 Jul 2024 09:38:11 +0200 Subject: [PATCH 08/43] instrument delete and update --- pkg/kine/server/delete.go | 10 ++++++++++ pkg/kine/server/update.go | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/pkg/kine/server/delete.go b/pkg/kine/server/delete.go index 03cace71..439567b1 100644 --- a/pkg/kine/server/delete.go +++ b/pkg/kine/server/delete.go @@ -4,6 +4,7 @@ import ( "context" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.opentelemetry.io/otel/attribute" ) func isDelete(txn *etcdserverpb.TxnRequest) (int64, string, bool) { @@ -28,10 +29,19 @@ func isDelete(txn *etcdserverpb.TxnRequest) (int64, string, bool) { } func (l *LimitedServer) delete(ctx context.Context, key string, revision int64) (*etcdserverpb.TxnResponse, error) { + ctx, span := tracer.Start(ctx, "backend.delete") + defer span.End() + span.SetAttributes( + attribute.String("key", key), + attribute.Int64("revision", revision), + ) + backendDeleteCnt.Add(ctx, 1) rev, kv, ok, err := l.backend.Delete(ctx, key, revision) if err != nil { + span.RecordError(err) return nil, err } + span.SetAttributes(attribute.Bool("ok", ok)) if !ok { return &etcdserverpb.TxnResponse{ diff --git a/pkg/kine/server/update.go b/pkg/kine/server/update.go index 52ba5993..4b4da624 100644 --- a/pkg/kine/server/update.go +++ b/pkg/kine/server/update.go @@ -4,6 +4,7 @@ import ( "context" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.opentelemetry.io/otel/attribute" ) func isUpdate(txn *etcdserverpb.TxnRequest) (int64, string, []byte, int64, bool) { @@ -31,10 +32,28 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value ) if rev == 0 { + ctx, span := tracer.Start(ctx, "backend.create") + defer span.End() + span.SetAttributes( + attribute.String("key", key), + attribute.Int64("lease", lease), + ) rev, err = l.backend.Create(ctx, key, value, lease) + //TODO: why are here no error checks? ok = true + span.SetAttributes(attribute.Bool("ok", ok)) + backendCreateCnt.Add(ctx, 1) } else { + ctx, span := tracer.Start(ctx, "backend.update") + defer span.End() + span.SetAttributes( + attribute.String("key", key), + attribute.Int64("revision", rev), + attribute.Int64("lease", lease), + ) rev, kv, ok, err = l.backend.Update(ctx, key, value, rev, lease) + span.SetAttributes(attribute.Bool("ok", ok)) + backendUpdateCnt.Add(ctx, 1) } if err != nil { return nil, err From 089c4d3c7fb48dcbc8c40f6261f15041389f5c0d Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 10 Jul 2024 09:43:17 +0200 Subject: [PATCH 09/43] dbsize --- pkg/kine/server/maintenance.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/kine/server/maintenance.go b/pkg/kine/server/maintenance.go index bc05aa83..ff517272 100644 --- a/pkg/kine/server/maintenance.go +++ b/pkg/kine/server/maintenance.go @@ -14,8 +14,11 @@ func (s *KVServerBridge) Alarm(context.Context, *etcdserverpb.AlarmRequest) (*et } func (s *KVServerBridge) Status(ctx context.Context, r *etcdserverpb.StatusRequest) (*etcdserverpb.StatusResponse, error) { + ctx, span := tracer.Start(ctx, "backend.dbSize") + defer span.End() size, err := s.limited.dbSize(ctx) if err != nil { + span.RecordError(err) return nil, err } return &etcdserverpb.StatusResponse{ From 3be9a844f9409f19bcba858cec9d7914b3df31e3 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 10 Jul 2024 09:56:12 +0200 Subject: [PATCH 10/43] cleanup --- pkg/kine/server/create.go | 2 -- pkg/kine/server/otel.go | 7 ++----- pkg/kine/server/tracing.go | 39 -------------------------------------- 3 files changed, 2 insertions(+), 46 deletions(-) diff --git a/pkg/kine/server/create.go b/pkg/kine/server/create.go index 4c9a92ce..a9ab3bbe 100644 --- a/pkg/kine/server/create.go +++ b/pkg/kine/server/create.go @@ -35,8 +35,6 @@ func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest span.SetAttributes( attribute.String("key", string(put.Key)), - //TODO: get key and value in utf-8 string - // attribute.String("value", string(put.Value)), attribute.Int64("lease", put.Lease), ) backendCreateCnt.Add(ctx, 1) diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go index 5f287cdf..90748486 100644 --- a/pkg/kine/server/otel.go +++ b/pkg/kine/server/otel.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "go.opentelemetry.io/contrib/bridges/otelslog" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" ) @@ -15,7 +14,6 @@ const name = "k8s-dqlite" var ( tracer = otel.Tracer(name) meter = otel.Meter(name) - logger = otelslog.NewLogger(name) backendGetCnt metric.Int64Counter backendListCnt metric.Int64Counter backendCountCnt metric.Int64Counter @@ -27,10 +25,9 @@ var ( func init() { ctx := context.Background() - // otel setup - setupOTelSDK(ctx) - //TODO: handle shutdown setupOTelSDK + setupOTelSDK(ctx) + //TODO: move this and handle shutdown setupOTelSDK backendListCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) backendGetCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) diff --git a/pkg/kine/server/tracing.go b/pkg/kine/server/tracing.go index f20522eb..64c87b66 100644 --- a/pkg/kine/server/tracing.go +++ b/pkg/kine/server/tracing.go @@ -8,11 +8,7 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" - "go.opentelemetry.io/otel/log/global" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" @@ -51,10 +47,6 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er ), ) - // Set up propagator. - prop := newPropagator() - otel.SetTextMapPropagator(prop) - // Set up trace provider. tracerProvider, err := newTraceProvider(ctx, res) if err != nil { @@ -73,25 +65,9 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown) otel.SetMeterProvider(meterProvider) - // Set up logger provider. - loggerProvider, err := newLoggerProvider() - if err != nil { - handleErr(err) - return - } - shutdownFuncs = append(shutdownFuncs, loggerProvider.Shutdown) - global.SetLoggerProvider(loggerProvider) - return } -func newPropagator() propagation.TextMapPropagator { //TODO: is this needed - return propagation.NewCompositeTextMapPropagator( - propagation.TraceContext{}, - propagation.Baggage{}, - ) -} - // Initialize a gRPC connection to be used by both the tracer and meter // providers. func initConn() (*grpc.ClientConn, error) { @@ -119,9 +95,6 @@ func newExporter(ctx context.Context) (trace.SpanExporter, error) { } func newTraceProvider(ctx context.Context, res *resource.Resource) (*trace.TracerProvider, error) { - // TODO: Replace with exporter such as Jaeger - // traceExporter, err := stdouttrace.New( - // stdouttrace.WithPrettyPrint()) traceExporter, err := newExporter(ctx) if err != nil { return nil, err @@ -150,15 +123,3 @@ func newMeterProvider() (*metric.MeterProvider, error) { ) return meterProvider, nil } - -func newLoggerProvider() (*log.LoggerProvider, error) { - logExporter, err := stdoutlog.New() - if err != nil { - return nil, err - } - - loggerProvider := log.NewLoggerProvider( - log.WithProcessor(log.NewBatchProcessor(logExporter)), - ) - return loggerProvider, nil -} From 478cbb78f8d2c2a0ffba317e375a2dc98cab646e Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 10 Jul 2024 09:57:26 +0200 Subject: [PATCH 11/43] cleanup 2 --- go.mod | 4 ---- go.sum | 8 -------- 2 files changed, 12 deletions(-) diff --git a/go.mod b/go.mod index dc1d23a2..0c166c9a 100644 --- a/go.mod +++ b/go.mod @@ -15,15 +15,11 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.12 go.etcd.io/etcd/client/v3 v3.5.12 go.etcd.io/etcd/server/v3 v3.5.12 - go.opentelemetry.io/contrib/bridges/otelslog v0.3.0 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 - go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 - go.opentelemetry.io/otel/log v0.4.0 go.opentelemetry.io/otel/metric v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 - go.opentelemetry.io/otel/sdk/log v0.4.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 golang.org/x/sync v0.7.0 golang.org/x/sys v0.21.0 diff --git a/go.sum b/go.sum index e8f129d3..4082c078 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,6 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib/bridges/otelslog v0.3.0 h1:Kf8NK4WW/pn3f9Gwx6XJAB2zlaW2M3VLQ4sQ3TKJhA8= -go.opentelemetry.io/contrib/bridges/otelslog v0.3.0/go.mod h1:JV00+So1cv6GIYNUeO0xFfl/qE+DUtS3hpBlLIyOFUE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= @@ -369,18 +367,12 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9RO go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= -go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0 h1:0MH3f8lZrflbUWXVxyBg/zviDFdGE062uKh5+fu8Vv0= -go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.4.0/go.mod h1:Vh68vYiHY5mPdekTr0ox0sALsqjoVy0w3Os278yX5SQ= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 h1:BJee2iLkfRfl9lc7aFmBwkWxY/RI1RDdXepSF6y8TPE= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0/go.mod h1:DIzlHs3DRscCIBU3Y9YSzPfScwnYnzfnCd4g8zA7bZc= -go.opentelemetry.io/otel/log v0.4.0 h1:/vZ+3Utqh18e8TPjuc3ecg284078KWrR8BRz+PQAj3o= -go.opentelemetry.io/otel/log v0.4.0/go.mod h1:DhGnQvky7pHy82MIRV43iXh3FlKN8UUKftn0KbLOq6I= go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/sdk/log v0.4.0 h1:1mMI22L82zLqf6KtkjrRy5BbagOTWdJsqMY/HSqILAA= -go.opentelemetry.io/otel/sdk/log v0.4.0/go.mod h1:AYJ9FVF0hNOgAVzUG/ybg/QttnXhUePWAupmCqtdESo= go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnCQArXCKlg08= go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg= go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= From 755e113ea55acf85a5e3d473e29fd4293438026a Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 10 Jul 2024 10:07:34 +0200 Subject: [PATCH 12/43] cleanup 3 --- pkg/kine/server/compact.go | 1 + pkg/kine/server/create.go | 2 +- pkg/kine/server/delete.go | 2 +- pkg/kine/server/get.go | 2 +- pkg/kine/server/list.go | 8 ++++---- pkg/kine/server/otel.go | 32 ++++++++++++++++---------------- pkg/kine/server/update.go | 4 ++-- 7 files changed, 26 insertions(+), 25 deletions(-) diff --git a/pkg/kine/server/compact.go b/pkg/kine/server/compact.go index f1b82976..104d2ab5 100644 --- a/pkg/kine/server/compact.go +++ b/pkg/kine/server/compact.go @@ -20,6 +20,7 @@ func isCompact(txn *etcdserverpb.TxnRequest) bool { } func (l *LimitedServer) compact(ctx context.Context) (*etcdserverpb.TxnResponse, error) { + compactCnt.Add(ctx, 1) // return comparison failure so that the apiserver does not bother compacting return &etcdserverpb.TxnResponse{ Header: &etcdserverpb.ResponseHeader{}, diff --git a/pkg/kine/server/create.go b/pkg/kine/server/create.go index a9ab3bbe..9c9838d6 100644 --- a/pkg/kine/server/create.go +++ b/pkg/kine/server/create.go @@ -37,7 +37,7 @@ func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest attribute.String("key", string(put.Key)), attribute.Int64("lease", put.Lease), ) - backendCreateCnt.Add(ctx, 1) + createCnt.Add(ctx, 1) rev, err := l.backend.Create(ctx, string(put.Key), put.Value, put.Lease) if err == ErrKeyExists { diff --git a/pkg/kine/server/delete.go b/pkg/kine/server/delete.go index 439567b1..e1caa96f 100644 --- a/pkg/kine/server/delete.go +++ b/pkg/kine/server/delete.go @@ -35,7 +35,7 @@ func (l *LimitedServer) delete(ctx context.Context, key string, revision int64) attribute.String("key", key), attribute.Int64("revision", revision), ) - backendDeleteCnt.Add(ctx, 1) + deleteCnt.Add(ctx, 1) rev, kv, ok, err := l.backend.Delete(ctx, key, revision) if err != nil { span.RecordError(err) diff --git a/pkg/kine/server/get.go b/pkg/kine/server/get.go index e5ecfeb4..08920dc4 100644 --- a/pkg/kine/server/get.go +++ b/pkg/kine/server/get.go @@ -22,7 +22,7 @@ func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) ( attribute.Int64("limit", r.Limit), attribute.Int64("revision", r.Revision), ) - backendGetCnt.Add(ctx, 1) + getCnt.Add(ctx, 1) rev, kv, err := l.backend.Get(ctx, string(r.Key), string(r.RangeEnd), r.Limit, r.Revision) if err != nil { diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index ff7a41bb..2a49e6c5 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -33,7 +33,7 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) attribute.String("start", start), attribute.Int64("revision", revision), ) - backendCountCnt.Add(ctx, 1) + countCnt.Add(ctx, 1) rev, count, err := l.backend.Count(ctx, prefix, start, revision) if err != nil { span.RecordError(err) @@ -64,14 +64,14 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) attribute.Int64("limit", limit), attribute.Int64("revision", revision), ) - backendListCnt.Add(ctx, 1) + listCnt.Add(ctx, 1) rev, kvs, err := l.backend.List(ctx, prefix, start, limit, revision) if err != nil { span.RecordError(err) return nil, err } - backendListCnt.Add(ctx, 1) + listCnt.Add(ctx, 1) span.End() resp := &RangeResponse{ @@ -98,7 +98,7 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) attribute.String("start", start), attribute.Int64("revision", revision), ) - backendCountCnt.Add(ctx, 1) + countCnt.Add(ctx, 1) rev, resp.Count, err = l.backend.Count(ctx, prefix, start, revision) if err != nil { span.RecordError(err) diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go index 90748486..3ce4c9f0 100644 --- a/pkg/kine/server/otel.go +++ b/pkg/kine/server/otel.go @@ -12,15 +12,15 @@ import ( const name = "k8s-dqlite" var ( - tracer = otel.Tracer(name) - meter = otel.Meter(name) - backendGetCnt metric.Int64Counter - backendListCnt metric.Int64Counter - backendCountCnt metric.Int64Counter - backendCreateCnt metric.Int64Counter - backendDeleteCnt metric.Int64Counter - backendUpdateCnt metric.Int64Counter - backendCompactCnt metric.Int64Counter + tracer = otel.Tracer(name) + meter = otel.Meter(name) + getCnt metric.Int64Counter + listCnt metric.Int64Counter + countCnt metric.Int64Counter + createCnt metric.Int64Counter + deleteCnt metric.Int64Counter + updateCnt metric.Int64Counter + compactCnt metric.Int64Counter ) func init() { @@ -29,11 +29,11 @@ func init() { setupOTelSDK(ctx) //TODO: move this and handle shutdown setupOTelSDK - backendListCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) - backendGetCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) - backendCountCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.count", name), metric.WithDescription("Number of count requests")) - backendCreateCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.create", name), metric.WithDescription("Number of create requests")) - backendDeleteCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.delete", name), metric.WithDescription("Number of delete requests")) - backendUpdateCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.update", name), metric.WithDescription("Number of update requests")) - backendCompactCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) + listCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) + getCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) + countCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.count", name), metric.WithDescription("Number of count requests")) + createCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.create", name), metric.WithDescription("Number of create requests")) + deleteCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.delete", name), metric.WithDescription("Number of delete requests")) + updateCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.update", name), metric.WithDescription("Number of update requests")) + compactCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) } diff --git a/pkg/kine/server/update.go b/pkg/kine/server/update.go index 4b4da624..2ed4c56d 100644 --- a/pkg/kine/server/update.go +++ b/pkg/kine/server/update.go @@ -42,7 +42,7 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value //TODO: why are here no error checks? ok = true span.SetAttributes(attribute.Bool("ok", ok)) - backendCreateCnt.Add(ctx, 1) + createCnt.Add(ctx, 1) } else { ctx, span := tracer.Start(ctx, "backend.update") defer span.End() @@ -53,7 +53,7 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value ) rev, kv, ok, err = l.backend.Update(ctx, key, value, rev, lease) span.SetAttributes(attribute.Bool("ok", ok)) - backendUpdateCnt.Add(ctx, 1) + updateCnt.Add(ctx, 1) } if err != nil { return nil, err From 10ac5db51acefc04e5bc79f3fe736227ed4ab123 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 10 Jul 2024 10:52:41 +0200 Subject: [PATCH 13/43] add service name to meterprovider --- pkg/kine/server/tracing.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/kine/server/tracing.go b/pkg/kine/server/tracing.go index 64c87b66..cf39404e 100644 --- a/pkg/kine/server/tracing.go +++ b/pkg/kine/server/tracing.go @@ -43,7 +43,7 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er res, err := resource.New(ctx, resource.WithAttributes( // The service name used to display traces in backends - semconv.ServiceNameKey.String("k8s-dqlite-service"), + semconv.ServiceNameKey.String("k8s-dqlite"), ), ) @@ -57,7 +57,7 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er otel.SetTracerProvider(tracerProvider) // Set up meter provider. - meterProvider, err := newMeterProvider() + meterProvider, err := newMeterProvider(res) if err != nil { handleErr(err) return @@ -110,7 +110,7 @@ func newTraceProvider(ctx context.Context, res *resource.Resource) (*trace.Trace return traceProvider, nil } -func newMeterProvider() (*metric.MeterProvider, error) { +func newMeterProvider(res *resource.Resource) (*metric.MeterProvider, error) { metricExporter, err := stdoutmetric.New() if err != nil { return nil, err @@ -120,6 +120,7 @@ func newMeterProvider() (*metric.MeterProvider, error) { metric.WithReader(metric.NewPeriodicReader(metricExporter, // Default is 1m. Set to 3s for demonstrative purposes. metric.WithInterval(30*time.Second))), + metric.WithResource(res), ) return meterProvider, nil } From 69134874d7ff4b828805ae6382bf52bb36d2dbe4 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 10 Jul 2024 10:56:41 +0200 Subject: [PATCH 14/43] cleanup --- pkg/server/server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index f82cb28d..610df044 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -334,7 +334,6 @@ func (s *Server) Start(ctx context.Context) error { if err := s.app.Ready(ctx); err != nil { return fmt.Errorf("failed to start dqlite app: %w", err) } - logrus.WithFields(logrus.Fields{"id": s.app.ID(), "address": s.app.Address()}).Print("Started dqlite") logrus.WithField("config", s.kineConfig).Debug("Starting kine") From 778a82c60a74517af606324ec8f049642e385dba Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 10 Jul 2024 12:28:01 +0200 Subject: [PATCH 15/43] comments --- pkg/kine/server/create.go | 1 - pkg/kine/server/list.go | 1 - pkg/kine/server/tracing.go | 2 +- pkg/kine/server/update.go | 6 ++++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/kine/server/create.go b/pkg/kine/server/create.go index 9c9838d6..ea5b60e9 100644 --- a/pkg/kine/server/create.go +++ b/pkg/kine/server/create.go @@ -33,7 +33,6 @@ func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest defer span.End() span.SetAttributes( - attribute.String("key", string(put.Key)), attribute.Int64("lease", put.Lease), ) diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index 2a49e6c5..950073d8 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -71,7 +71,6 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) span.RecordError(err) return nil, err } - listCnt.Add(ctx, 1) span.End() resp := &RangeResponse{ diff --git a/pkg/kine/server/tracing.go b/pkg/kine/server/tracing.go index cf39404e..6d097893 100644 --- a/pkg/kine/server/tracing.go +++ b/pkg/kine/server/tracing.go @@ -43,7 +43,7 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er res, err := resource.New(ctx, resource.WithAttributes( // The service name used to display traces in backends - semconv.ServiceNameKey.String("k8s-dqlite"), + semconv.ServiceNameKey.String(name), ), ) diff --git a/pkg/kine/server/update.go b/pkg/kine/server/update.go index 2ed4c56d..b9b8609c 100644 --- a/pkg/kine/server/update.go +++ b/pkg/kine/server/update.go @@ -38,11 +38,12 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value attribute.String("key", key), attribute.Int64("lease", lease), ) + createCnt.Add(ctx, 1) + rev, err = l.backend.Create(ctx, key, value, lease) //TODO: why are here no error checks? ok = true span.SetAttributes(attribute.Bool("ok", ok)) - createCnt.Add(ctx, 1) } else { ctx, span := tracer.Start(ctx, "backend.update") defer span.End() @@ -51,9 +52,10 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value attribute.Int64("revision", rev), attribute.Int64("lease", lease), ) + updateCnt.Add(ctx, 1) + rev, kv, ok, err = l.backend.Update(ctx, key, value, rev, lease) span.SetAttributes(attribute.Bool("ok", ok)) - updateCnt.Add(ctx, 1) } if err != nil { return nil, err From 67dc71f46fcf07aca9afefa97ad88241eae2155f Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 10 Jul 2024 13:19:43 +0200 Subject: [PATCH 16/43] move otel setup --- cmd/root.go | 11 +++++++++++ {pkg/kine/server => cmd}/tracing.go | 16 ++-------------- pkg/kine/server/otel.go | 6 ------ 3 files changed, 13 insertions(+), 20 deletions(-) rename {pkg/kine/server => cmd}/tracing.go (82%) diff --git a/cmd/root.go b/cmd/root.go index 1b061e87..6e50c2f4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "errors" "net/http" _ "net/http/pprof" "os" @@ -60,6 +61,16 @@ var ( if rootCmdOpts.metrics { go func() { + otelShutdown, err := setupOTelSDK(cmd.Context()) + if err != nil { + logrus.WithError(err).Warning("Failed to setup OpenTelemetry SDK") + } + defer func() { + err = errors.Join(err, otelShutdown(cmd.Context())) + if err != nil { + logrus.WithError(err).Warning("Failed to shutdown OpenTelemetry SDK") + } + }() logrus.WithField("address", rootCmdOpts.metricsAddress).Print("Enable metrics endpoint") mux := http.NewServeMux() mux.Handle("/metrics", promhttp.Handler()) diff --git a/pkg/kine/server/tracing.go b/cmd/tracing.go similarity index 82% rename from pkg/kine/server/tracing.go rename to cmd/tracing.go index 6d097893..15fb527c 100644 --- a/pkg/kine/server/tracing.go +++ b/cmd/tracing.go @@ -1,4 +1,4 @@ -package server +package cmd import ( "context" @@ -22,9 +22,6 @@ import ( func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, err error) { var shutdownFuncs []func(context.Context) error - // shutdown calls cleanup functions registered via shutdownFuncs. - // The errors from the calls are joined. - // Each registered cleanup will be invoked once. shutdown = func(ctx context.Context) error { var err error for _, fn := range shutdownFuncs { @@ -34,20 +31,16 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er return err } - // handleErr calls shutdown for cleanup and makes sure that all errors are returned. handleErr := func(inErr error) { err = errors.Join(inErr, shutdown(ctx)) } - // Set up resource. res, err := resource.New(ctx, resource.WithAttributes( - // The service name used to display traces in backends - semconv.ServiceNameKey.String(name), + semconv.ServiceNameKey.String("k8s-dqlite"), ), ) - // Set up trace provider. tracerProvider, err := newTraceProvider(ctx, res) if err != nil { handleErr(err) @@ -56,7 +49,6 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown) otel.SetTracerProvider(tracerProvider) - // Set up meter provider. meterProvider, err := newMeterProvider(res) if err != nil { handleErr(err) @@ -68,8 +60,6 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er return } -// Initialize a gRPC connection to be used by both the tracer and meter -// providers. func initConn() (*grpc.ClientConn, error) { // It connects the OpenTelemetry Collector through local gRPC connection. // You may replace `localhost:4317` with your endpoint. @@ -102,7 +92,6 @@ func newTraceProvider(ctx context.Context, res *resource.Resource) (*trace.Trace traceProvider := trace.NewTracerProvider( trace.WithBatcher(traceExporter, - // Default is 5s. Set to 1s for demonstrative purposes. trace.WithBatchTimeout(time.Second), ), trace.WithResource(res), @@ -118,7 +107,6 @@ func newMeterProvider(res *resource.Resource) (*metric.MeterProvider, error) { meterProvider := metric.NewMeterProvider( metric.WithReader(metric.NewPeriodicReader(metricExporter, - // Default is 1m. Set to 3s for demonstrative purposes. metric.WithInterval(30*time.Second))), metric.WithResource(res), ) diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go index 3ce4c9f0..dcb8d998 100644 --- a/pkg/kine/server/otel.go +++ b/pkg/kine/server/otel.go @@ -1,7 +1,6 @@ package server import ( - "context" "fmt" "go.opentelemetry.io/otel" @@ -24,11 +23,6 @@ var ( ) func init() { - ctx := context.Background() - - setupOTelSDK(ctx) - //TODO: move this and handle shutdown setupOTelSDK - listCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) getCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) countCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.count", name), metric.WithDescription("Number of count requests")) From 988decaf3e71a3cefeb7af72e150fa25d4c4bd0a Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 10 Jul 2024 13:25:09 +0200 Subject: [PATCH 17/43] cleanup --- pkg/kine/server/otel.go | 1 - pkg/kine/server/update.go | 1 - 2 files changed, 2 deletions(-) diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go index dcb8d998..7494c3da 100644 --- a/pkg/kine/server/otel.go +++ b/pkg/kine/server/otel.go @@ -7,7 +7,6 @@ import ( "go.opentelemetry.io/otel/metric" ) -// TODO: This can probably get moved into tracing.go const name = "k8s-dqlite" var ( diff --git a/pkg/kine/server/update.go b/pkg/kine/server/update.go index b9b8609c..b129b5ff 100644 --- a/pkg/kine/server/update.go +++ b/pkg/kine/server/update.go @@ -41,7 +41,6 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value createCnt.Add(ctx, 1) rev, err = l.backend.Create(ctx, key, value, lease) - //TODO: why are here no error checks? ok = true span.SetAttributes(attribute.Bool("ok", ok)) } else { From 213b745543104737c76e8c79ff7b6a92dfcabf2c Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Fri, 12 Jul 2024 10:25:05 +0200 Subject: [PATCH 18/43] remove wrong place compact logs --- pkg/kine/server/compact.go | 1 - pkg/kine/server/create.go | 1 + pkg/kine/server/get.go | 8 ++++---- pkg/kine/server/limited.go | 4 ---- pkg/kine/server/maintenance.go | 2 ++ pkg/kine/server/otel.go | 18 ++++++++---------- 6 files changed, 15 insertions(+), 19 deletions(-) diff --git a/pkg/kine/server/compact.go b/pkg/kine/server/compact.go index 104d2ab5..f1b82976 100644 --- a/pkg/kine/server/compact.go +++ b/pkg/kine/server/compact.go @@ -20,7 +20,6 @@ func isCompact(txn *etcdserverpb.TxnRequest) bool { } func (l *LimitedServer) compact(ctx context.Context) (*etcdserverpb.TxnResponse, error) { - compactCnt.Add(ctx, 1) // return comparison failure so that the apiserver does not bother compacting return &etcdserverpb.TxnResponse{ Header: &etcdserverpb.ResponseHeader{}, diff --git a/pkg/kine/server/create.go b/pkg/kine/server/create.go index ea5b60e9..9ba6d06c 100644 --- a/pkg/kine/server/create.go +++ b/pkg/kine/server/create.go @@ -39,6 +39,7 @@ func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest createCnt.Add(ctx, 1) rev, err := l.backend.Create(ctx, string(put.Key), put.Value, put.Lease) + span.SetAttributes(attribute.Int64("revision", rev)) if err == ErrKeyExists { span.RecordError(err) return &etcdserverpb.TxnResponse{ diff --git a/pkg/kine/server/get.go b/pkg/kine/server/get.go index 08920dc4..cde9e509 100644 --- a/pkg/kine/server/get.go +++ b/pkg/kine/server/get.go @@ -9,10 +9,6 @@ import ( ) func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { - if r.Limit != 0 && len(r.RangeEnd) != 0 { - return nil, fmt.Errorf("invalid combination of rangeEnd and limit, limit should be 0 got %d", r.Limit) - } - ctx, span := tracer.Start(ctx, "backend.get") defer span.End() @@ -22,6 +18,10 @@ func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) ( attribute.Int64("limit", r.Limit), attribute.Int64("revision", r.Revision), ) + if r.Limit != 0 && len(r.RangeEnd) != 0 { + span.RecordError(fmt.Errorf("invalid combination of rangeEnd and limit, limit should be 0 got %d", r.Limit)) + return nil, fmt.Errorf("invalid combination of rangeEnd and limit, limit should be 0 got %d", r.Limit) + } getCnt.Add(ctx, 1) rev, kv, err := l.backend.Get(ctx, string(r.Key), string(r.RangeEnd), r.Limit, r.Revision) diff --git a/pkg/kine/server/limited.go b/pkg/kine/server/limited.go index 745dd12a..20753061 100644 --- a/pkg/kine/server/limited.go +++ b/pkg/kine/server/limited.go @@ -40,13 +40,9 @@ func (l *LimitedServer) Txn(ctx context.Context, txn *etcdserverpb.TxnRequest) ( return l.delete(ctx, key, rev) } if rev, key, value, lease, ok := isUpdate(txn); ok { - ctx, span := tracer.Start(ctx, "limited.update") - defer span.End() return l.update(ctx, rev, key, value, lease) } if isCompact(txn) { - ctx, span := tracer.Start(ctx, "limited.compact") - defer span.End() return l.compact(ctx) } return nil, fmt.Errorf("unsupported transaction: %v", txn) diff --git a/pkg/kine/server/maintenance.go b/pkg/kine/server/maintenance.go index ff517272..b789b8bc 100644 --- a/pkg/kine/server/maintenance.go +++ b/pkg/kine/server/maintenance.go @@ -5,6 +5,7 @@ import ( "fmt" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.opentelemetry.io/otel/attribute" ) var _ etcdserverpb.MaintenanceServer = (*KVServerBridge)(nil) @@ -21,6 +22,7 @@ func (s *KVServerBridge) Status(ctx context.Context, r *etcdserverpb.StatusReque span.RecordError(err) return nil, err } + span.SetAttributes(attribute.Int64("size", size)) return &etcdserverpb.StatusResponse{ Header: &etcdserverpb.ResponseHeader{}, DbSize: size, diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go index 7494c3da..faed621c 100644 --- a/pkg/kine/server/otel.go +++ b/pkg/kine/server/otel.go @@ -10,15 +10,14 @@ import ( const name = "k8s-dqlite" var ( - tracer = otel.Tracer(name) - meter = otel.Meter(name) - getCnt metric.Int64Counter - listCnt metric.Int64Counter - countCnt metric.Int64Counter - createCnt metric.Int64Counter - deleteCnt metric.Int64Counter - updateCnt metric.Int64Counter - compactCnt metric.Int64Counter + tracer = otel.Tracer(name) + meter = otel.Meter(name) + getCnt metric.Int64Counter + listCnt metric.Int64Counter + countCnt metric.Int64Counter + createCnt metric.Int64Counter + deleteCnt metric.Int64Counter + updateCnt metric.Int64Counter ) func init() { @@ -28,5 +27,4 @@ func init() { createCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.create", name), metric.WithDescription("Number of create requests")) deleteCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.delete", name), metric.WithDescription("Number of delete requests")) updateCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.update", name), metric.WithDescription("Number of update requests")) - compactCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) } From 85af5c36af73012ef359826d8a9b8e729c7b1fa9 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Fri, 12 Jul 2024 10:33:36 +0200 Subject: [PATCH 19/43] add traces to compaction --- pkg/kine/logstructured/sqllog/otel.go | 20 ++++++++++++++++++++ pkg/kine/logstructured/sqllog/sql.go | 27 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 pkg/kine/logstructured/sqllog/otel.go diff --git a/pkg/kine/logstructured/sqllog/otel.go b/pkg/kine/logstructured/sqllog/otel.go new file mode 100644 index 00000000..e7c87999 --- /dev/null +++ b/pkg/kine/logstructured/sqllog/otel.go @@ -0,0 +1,20 @@ +package sqllog + +import ( + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +const name = "sqllog" + +var ( + tracer = otel.Tracer(name) + meter = otel.Meter(name) + compactCnt metric.Int64Counter +) + +func init() { + compactCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) +} diff --git a/pkg/kine/logstructured/sqllog/sql.go b/pkg/kine/logstructured/sqllog/sql.go index c688a594..9fcfcdf6 100644 --- a/pkg/kine/logstructured/sqllog/sql.go +++ b/pkg/kine/logstructured/sqllog/sql.go @@ -11,6 +11,7 @@ import ( "github.com/canonical/k8s-dqlite/pkg/kine/broadcaster" "github.com/canonical/k8s-dqlite/pkg/kine/server" "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/attribute" ) const SupersededCount = 100 @@ -124,17 +125,26 @@ func (s *SQLLog) DoCompact(ctx context.Context) error { } func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { + spanCtx, span := tracer.Start(s.ctx, "sqllog.compactor") + s.ctx = spanCtx + defer span.End() + span.AddEvent(fmt.Sprintf("nextEnd: %d", nextEnd)) + currentRev, err := s.d.CurrentRevision(ctx) + span.AddEvent(fmt.Sprintf("get current revision: %d", currentRev)) if err != nil { + span.RecordError(err) logrus.Errorf("failed to get current revision: %v", err) return nextEnd, fmt.Errorf("failed to get current revision: %v", err) } cursor, _, err := s.d.GetCompactRevision(ctx) if err != nil { + span.RecordError(err) logrus.Errorf("failed to get compact revision: %v", err) return nextEnd, fmt.Errorf("failed to get compact revision: %v", err) } + span.AddEvent(fmt.Sprintf("get compact revision: %d", cursor)) end := nextEnd nextEnd = currentRev @@ -148,15 +158,19 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { savedCursor := cursor // Purposefully start at the current and redo the current as // it could have failed before actually compacting + compactCnt.Add(ctx, 1) + span.AddEvent(fmt.Sprintf("start compaction from %d to %d", cursor, end)) for ; cursor <= end; cursor++ { rows, err := s.d.GetRevision(ctx, cursor) if err != nil { + span.RecordError(err) logrus.Errorf("failed to get revision %d: %v", cursor, err) return nextEnd, fmt.Errorf("failed to get revision %d: %v", cursor, err) } events, err := RowsToEvents(rows) if err != nil { + span.RecordError(err) logrus.Errorf("failed to convert to events: %v", err) return nextEnd, fmt.Errorf("failed to convert to events: %v", err) } @@ -168,6 +182,7 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { event := events[0] if event.KV.Key == "compact_rev_key" { + span.AddEvent("skip compact_rev_key") // don't compact the compact key continue } @@ -176,6 +191,8 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { if event.PrevKV != nil && event.PrevKV.ModRevision != 0 { if savedCursor != cursor { if err := s.d.SetCompactRevision(ctx, cursor); err != nil { + span.RecordError(err) + span.AddEvent(fmt.Sprintf("failed to record compact revision: %v", err)) logrus.Errorf("failed to record compact revision: %v", err) return nextEnd, fmt.Errorf("failed to record compact revision: %v", err) } @@ -184,6 +201,8 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { } if err := s.d.DeleteRevision(ctx, event.PrevKV.ModRevision); err != nil { + span.RecordError(err) + span.AddEvent(fmt.Sprintf("failed to delete revision %d", event.PrevKV.ModRevision)) logrus.Errorf("failed to delete revision %d: %v", event.PrevKV.ModRevision, err) return nextEnd, fmt.Errorf("failed to delete revision %d: %v", event.PrevKV.ModRevision, err) } @@ -192,6 +211,7 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { if event.Delete { if !setRev && savedCursor != cursor { if err := s.d.SetCompactRevision(ctx, cursor); err != nil { + span.RecordError(err) logrus.Errorf("failed to record compact revision: %v", err) return nextEnd, fmt.Errorf("failed to record compact revision: %v", err) } @@ -199,6 +219,7 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { } if err := s.d.DeleteRevision(ctx, cursor); err != nil { + span.RecordError(err) logrus.Errorf("failed to delete current revision %d: %v", cursor, err) return nextEnd, fmt.Errorf("failed to delete current revision %d: %v", cursor, err) } @@ -207,6 +228,7 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { if savedCursor != cursor { if err := s.d.SetCompactRevision(ctx, cursor); err != nil { + span.RecordError(err) logrus.Errorf("failed to record compact revision: %v", err) return nextEnd, fmt.Errorf("failed to record compact revision: %v", err) } @@ -216,14 +238,19 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { } func (s *SQLLog) compact() { + ctx, span := tracer.Start(s.ctx, "sqllog.compact") + s.ctx = ctx // yes? no? + defer span.End() var nextEnd int64 t := time.NewTicker(s.d.GetCompactInterval()) nextEnd, _ = s.d.CurrentRevision(s.ctx) + span.SetAttributes(attribute.Int64("nextEnd", nextEnd)) for { select { case <-s.ctx.Done(): + span.AddEvent("context done") return case <-t.C: } From 0534bef8ed0784f6b245cb30ac0dcabcc07886dd Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Fri, 12 Jul 2024 10:35:05 +0200 Subject: [PATCH 20/43] on update's create if errkeyexists do a get --- pkg/kine/server/update.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pkg/kine/server/update.go b/pkg/kine/server/update.go index b129b5ff..4523cb44 100644 --- a/pkg/kine/server/update.go +++ b/pkg/kine/server/update.go @@ -2,6 +2,7 @@ package server import ( "context" + "fmt" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.opentelemetry.io/otel/attribute" @@ -30,6 +31,13 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value ok bool err error ) + ctx, updateSpan := tracer.Start(ctx, "limited.update") + defer updateSpan.End() + updateSpan.SetAttributes( + attribute.String("key", key), + attribute.Int64("revision", rev), + attribute.Int64("lease", lease), + ) if rev == 0 { ctx, span := tracer.Start(ctx, "backend.create") @@ -43,6 +51,26 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value rev, err = l.backend.Create(ctx, key, value, lease) ok = true span.SetAttributes(attribute.Bool("ok", ok)) + span.RecordError(err) + span.End() + + ctx, span = tracer.Start(ctx, "backend.get") + defer span.End() + span.SetAttributes( + attribute.String("key", key), + attribute.Int64("revision", rev), + ) + if err == ErrKeyExists { + span.AddEvent("key exists") + rev, kv, err = l.backend.Get(ctx, key, "", 1, rev) + if err != nil { + span.RecordError(err) + } + span.AddEvent(fmt.Sprintf("got revision %d", rev)) + } else { + ok = true + } + span.End() } else { ctx, span := tracer.Start(ctx, "backend.update") defer span.End() @@ -54,6 +82,7 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value updateCnt.Add(ctx, 1) rev, kv, ok, err = l.backend.Update(ctx, key, value, rev, lease) + span.RecordError(err) span.SetAttributes(attribute.Bool("ok", ok)) } if err != nil { From 81ef2bb1d4da9f19019ca06e300671e71dac4789 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Fri, 12 Jul 2024 10:41:04 +0200 Subject: [PATCH 21/43] trace names --- pkg/kine/server/create.go | 14 +++++++------- pkg/kine/server/limited.go | 8 ++------ pkg/kine/server/list.go | 14 ++++++++------ pkg/kine/server/update.go | 2 +- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/pkg/kine/server/create.go b/pkg/kine/server/create.go index 9ba6d06c..b5070382 100644 --- a/pkg/kine/server/create.go +++ b/pkg/kine/server/create.go @@ -21,6 +21,13 @@ func isCreate(txn *etcdserverpb.TxnRequest) *etcdserverpb.PutRequest { } func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest, txn *etcdserverpb.TxnRequest) (*etcdserverpb.TxnResponse, error) { + ctx, span := tracer.Start(ctx, "backend.create") + defer span.End() + span.SetAttributes( + attribute.String("key", string(put.Key)), + attribute.Int64("lease", put.Lease), + ) + if put.IgnoreLease { return nil, unsupported("ignoreLease") } else if put.IgnoreValue { @@ -29,13 +36,6 @@ func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest return nil, unsupported("prevKv") } - ctx, span := tracer.Start(ctx, "backend.create") - defer span.End() - - span.SetAttributes( - attribute.String("key", string(put.Key)), - attribute.Int64("lease", put.Lease), - ) createCnt.Add(ctx, 1) rev, err := l.backend.Create(ctx, string(put.Key), put.Value, put.Lease) diff --git a/pkg/kine/server/limited.go b/pkg/kine/server/limited.go index 20753061..61e0f298 100644 --- a/pkg/kine/server/limited.go +++ b/pkg/kine/server/limited.go @@ -13,11 +13,11 @@ type LimitedServer struct { func (l *LimitedServer) Range(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { if len(r.RangeEnd) == 0 { - ctx, span := tracer.Start(ctx, "limited.get") + ctx, span := tracer.Start(ctx, "range.get") defer span.End() return l.get(ctx, r) } - ctx, span := tracer.Start(ctx, "limited.list") + ctx, span := tracer.Start(ctx, "range.list") defer span.End() return l.list(ctx, r) } @@ -30,13 +30,9 @@ func txnHeader(rev int64) *etcdserverpb.ResponseHeader { func (l *LimitedServer) Txn(ctx context.Context, txn *etcdserverpb.TxnRequest) (*etcdserverpb.TxnResponse, error) { if put := isCreate(txn); put != nil { - ctx, span := tracer.Start(ctx, "limited.create") - defer span.End() return l.create(ctx, put, txn) } if rev, key, ok := isDelete(txn); ok { - ctx, span := tracer.Start(ctx, "limited.delete") - defer span.End() return l.delete(ctx, key, rev) } if rev, key, value, lease, ok := isUpdate(txn); ok { diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index 950073d8..965019a9 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -12,6 +12,13 @@ import ( ) func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { + ctx, span := tracer.Start(ctx, "list") + defer span.End() + + span.SetAttributes( + attribute.String("key", string(r.Key)), + attribute.String("rangeEnd", string(r.RangeEnd)), + ) if len(r.RangeEnd) == 0 { return nil, fmt.Errorf("invalid range end length of 0") } @@ -53,17 +60,12 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) limit++ } - ctx, span := tracer.Start(ctx, "backend.list") - defer span.End() - span.SetAttributes( - attribute.String("key", string(r.Key)), - attribute.String("rangeEnd", string(r.RangeEnd)), - attribute.String("prefix", prefix), attribute.String("start", start), attribute.Int64("limit", limit), attribute.Int64("revision", revision), ) + listCnt.Add(ctx, 1) rev, kvs, err := l.backend.List(ctx, prefix, start, limit, revision) diff --git a/pkg/kine/server/update.go b/pkg/kine/server/update.go index 4523cb44..72312532 100644 --- a/pkg/kine/server/update.go +++ b/pkg/kine/server/update.go @@ -31,7 +31,7 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value ok bool err error ) - ctx, updateSpan := tracer.Start(ctx, "limited.update") + ctx, updateSpan := tracer.Start(ctx, "update") defer updateSpan.End() updateSpan.SetAttributes( attribute.String("key", key), From 9143cc93e4b76595f3b941eb8dbe3937e7985982 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Fri, 12 Jul 2024 11:04:23 +0200 Subject: [PATCH 22/43] ctx --- pkg/kine/logstructured/sqllog/sql.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/kine/logstructured/sqllog/sql.go b/pkg/kine/logstructured/sqllog/sql.go index 9fcfcdf6..790c6124 100644 --- a/pkg/kine/logstructured/sqllog/sql.go +++ b/pkg/kine/logstructured/sqllog/sql.go @@ -125,8 +125,7 @@ func (s *SQLLog) DoCompact(ctx context.Context) error { } func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { - spanCtx, span := tracer.Start(s.ctx, "sqllog.compactor") - s.ctx = spanCtx + _, span := tracer.Start(s.ctx, "sqllog.compactor") defer span.End() span.AddEvent(fmt.Sprintf("nextEnd: %d", nextEnd)) @@ -238,8 +237,7 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { } func (s *SQLLog) compact() { - ctx, span := tracer.Start(s.ctx, "sqllog.compact") - s.ctx = ctx // yes? no? + _, span := tracer.Start(s.ctx, "sqllog.compact") defer span.End() var nextEnd int64 From 2681a0eea49ded5929be3e8b46a9633cb069f6fa Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Fri, 12 Jul 2024 11:44:24 +0200 Subject: [PATCH 23/43] ctx --- pkg/kine/logstructured/sqllog/sql.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pkg/kine/logstructured/sqllog/sql.go b/pkg/kine/logstructured/sqllog/sql.go index 790c6124..cb560cb9 100644 --- a/pkg/kine/logstructured/sqllog/sql.go +++ b/pkg/kine/logstructured/sqllog/sql.go @@ -11,7 +11,6 @@ import ( "github.com/canonical/k8s-dqlite/pkg/kine/broadcaster" "github.com/canonical/k8s-dqlite/pkg/kine/server" "github.com/sirupsen/logrus" - "go.opentelemetry.io/otel/attribute" ) const SupersededCount = 100 @@ -125,7 +124,7 @@ func (s *SQLLog) DoCompact(ctx context.Context) error { } func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { - _, span := tracer.Start(s.ctx, "sqllog.compactor") + _, span := tracer.Start(ctx, "sqllog.compactor") defer span.End() span.AddEvent(fmt.Sprintf("nextEnd: %d", nextEnd)) @@ -237,18 +236,13 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { } func (s *SQLLog) compact() { - _, span := tracer.Start(s.ctx, "sqllog.compact") - defer span.End() var nextEnd int64 - t := time.NewTicker(s.d.GetCompactInterval()) nextEnd, _ = s.d.CurrentRevision(s.ctx) - span.SetAttributes(attribute.Int64("nextEnd", nextEnd)) for { select { case <-s.ctx.Done(): - span.AddEvent("context done") return case <-t.C: } From 38c31b3c0fa4357d5fa089bcc8d7329d89a4e240 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Fri, 12 Jul 2024 17:10:49 +0200 Subject: [PATCH 24/43] move run jaeger instance to script --- Makefile | 13 ------------- hack/jaeger.sh | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) create mode 100755 hack/jaeger.sh diff --git a/Makefile b/Makefile index e52062dd..be021b13 100644 --- a/Makefile +++ b/Makefile @@ -46,19 +46,6 @@ bin/dynamic/dqlite: bin/dynamic/lib/libdqlite.so mkdir -p bin/dynamic GOBIN=$(shell pwd)/bin/dynamic $(DQLITE_BUILD_SCRIPTS_DIR)/dynamic-go-install.sh github.com/canonical/go-dqlite/cmd/dqlite@v1.20.0 -# Tracing logs export to local jaeger instance -# 4317 HTTP (OTLP) over gRPC -# 4318 HTTP (OTLP) over HTTP -# connect http://localhost:16686 -jaeger: - docker run --name jaeger \ - -e COLLECTOR_OTLP_ENABLED=true \ - -p 16686:16686 \ - -p 4317:4317 \ - -p 4318:4318 \ - jaegertracing/all-in-one:1.35 - - ## Cleanup clean: rm -rf bin hack/.build hack/.deps diff --git a/hack/jaeger.sh b/hack/jaeger.sh new file mode 100755 index 00000000..a0817986 --- /dev/null +++ b/hack/jaeger.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# This script sets up a local jaeger instance for the otel tracing logs +# 4317 HTTP (OTLP) over gRPC +# 4318 HTTP (OTLP) over HTTP +# connect via http://localhost:16686 +docker run --name jaeger \ + -e COLLECTOR_OTLP_ENABLED=true \ + -p 16686:16686 \ + -p 4317:4317 \ + -p 4318:4318 \ + jaegertracing/all-in-one:1.35 + From 0febbb9dad2f25c5a34095f020293d5f16af584d Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 16 Jul 2024 14:58:09 +0200 Subject: [PATCH 25/43] comments part 1 --- cmd/tracing.go | 47 +++++++++++++++++----------- pkg/kine/logstructured/sqllog/sql.go | 9 +++--- pkg/kine/server/create.go | 9 +++--- pkg/kine/server/delete.go | 5 +-- pkg/kine/server/get.go | 9 +++--- pkg/kine/server/list.go | 37 +++++----------------- pkg/kine/server/maintenance.go | 5 --- pkg/kine/server/otel.go | 41 +++++++++++++++++++----- pkg/kine/server/update.go | 34 +++++--------------- 9 files changed, 94 insertions(+), 102 deletions(-) diff --git a/cmd/tracing.go b/cmd/tracing.go index 15fb527c..955ca1ae 100644 --- a/cmd/tracing.go +++ b/cmd/tracing.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" @@ -17,9 +18,14 @@ import ( "google.golang.org/grpc/credentials/insecure" ) +var ( + otelEndpoint = "localhost:4317" +) + // setupOTelSDK bootstraps the OpenTelemetry pipeline. // If it does not return an error, make sure to call shutdown for proper cleanup. func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, err error) { + logrus.SetLevel(logrus.TraceLevel) var shutdownFuncs []func(context.Context) error shutdown = func(ctx context.Context) error { @@ -40,6 +46,9 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er semconv.ServiceNameKey.String("k8s-dqlite"), ), ) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create resource") + } tracerProvider, err := newTraceProvider(ctx, res) if err != nil { @@ -60,18 +69,19 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er return } -func initConn() (*grpc.ClientConn, error) { - // It connects the OpenTelemetry Collector through local gRPC connection. - // You may replace `localhost:4317` with your endpoint. - conn, err := grpc.NewClient("localhost:4317", - // Note the use of insecure transport here. TLS is recommended in production. - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) +func newTraceProvider(ctx context.Context, res *resource.Resource) (*trace.TracerProvider, error) { + traceExporter, err := newExporter(ctx) if err != nil { - return nil, fmt.Errorf("failed to create gRPC connection to collector: %w", err) + return nil, err } - return conn, err + traceProvider := trace.NewTracerProvider( + trace.WithBatcher(traceExporter, + trace.WithBatchTimeout(time.Second), + ), + trace.WithResource(res), + ) + return traceProvider, nil } func newExporter(ctx context.Context) (trace.SpanExporter, error) { @@ -84,19 +94,18 @@ func newExporter(ctx context.Context) (trace.SpanExporter, error) { return traceExporter, nil } -func newTraceProvider(ctx context.Context, res *resource.Resource) (*trace.TracerProvider, error) { - traceExporter, err := newExporter(ctx) +func initConn() (*grpc.ClientConn, error) { + // It connects the OpenTelemetry Collector through local gRPC connection. + // You may replace `localhost:4317` with your endpoint. + conn, err := grpc.NewClient(otelEndpoint, + // Note the use of insecure transport here. TLS is recommended in production. + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create gRPC connection to collector: %w", err) } - traceProvider := trace.NewTracerProvider( - trace.WithBatcher(traceExporter, - trace.WithBatchTimeout(time.Second), - ), - trace.WithResource(res), - ) - return traceProvider, nil + return conn, nil } func newMeterProvider(res *resource.Resource) (*metric.MeterProvider, error) { diff --git a/pkg/kine/logstructured/sqllog/sql.go b/pkg/kine/logstructured/sqllog/sql.go index cb560cb9..28d5f426 100644 --- a/pkg/kine/logstructured/sqllog/sql.go +++ b/pkg/kine/logstructured/sqllog/sql.go @@ -11,6 +11,7 @@ import ( "github.com/canonical/k8s-dqlite/pkg/kine/broadcaster" "github.com/canonical/k8s-dqlite/pkg/kine/server" "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/attribute" ) const SupersededCount = 100 @@ -124,12 +125,12 @@ func (s *SQLLog) DoCompact(ctx context.Context) error { } func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { - _, span := tracer.Start(ctx, "sqllog.compactor") + ctx, span := tracer.Start(ctx, "sqllog.compactor") defer span.End() - span.AddEvent(fmt.Sprintf("nextEnd: %d", nextEnd)) + span.SetAttributes(attribute.Int64("nextEnd", nextEnd)) currentRev, err := s.d.CurrentRevision(ctx) - span.AddEvent(fmt.Sprintf("get current revision: %d", currentRev)) + span.SetAttributes(attribute.Int64("currentRev", currentRev)) if err != nil { span.RecordError(err) logrus.Errorf("failed to get current revision: %v", err) @@ -142,7 +143,7 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { logrus.Errorf("failed to get compact revision: %v", err) return nextEnd, fmt.Errorf("failed to get compact revision: %v", err) } - span.AddEvent(fmt.Sprintf("get compact revision: %d", cursor)) + span.SetAttributes(attribute.Int64("cursor", cursor)) end := nextEnd nextEnd = currentRev diff --git a/pkg/kine/server/create.go b/pkg/kine/server/create.go index b5070382..ecb28c64 100644 --- a/pkg/kine/server/create.go +++ b/pkg/kine/server/create.go @@ -21,7 +21,8 @@ func isCreate(txn *etcdserverpb.TxnRequest) *etcdserverpb.PutRequest { } func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest, txn *etcdserverpb.TxnRequest) (*etcdserverpb.TxnResponse, error) { - ctx, span := tracer.Start(ctx, "backend.create") + createCnt.Add(ctx, 1) + ctx, span := tracer.Start(ctx, "limited.create") defer span.End() span.SetAttributes( attribute.String("key", string(put.Key)), @@ -36,18 +37,16 @@ func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest return nil, unsupported("prevKv") } - createCnt.Add(ctx, 1) - rev, err := l.backend.Create(ctx, string(put.Key), put.Value, put.Lease) span.SetAttributes(attribute.Int64("revision", rev)) if err == ErrKeyExists { - span.RecordError(err) + span.AddEvent("key exists") return &etcdserverpb.TxnResponse{ Header: txnHeader(rev), Succeeded: false, }, nil } else if err != nil { - span.RecordError(err) + defer span.RecordError(err) return nil, err } diff --git a/pkg/kine/server/delete.go b/pkg/kine/server/delete.go index e1caa96f..edd8cf6a 100644 --- a/pkg/kine/server/delete.go +++ b/pkg/kine/server/delete.go @@ -29,13 +29,14 @@ func isDelete(txn *etcdserverpb.TxnRequest) (int64, string, bool) { } func (l *LimitedServer) delete(ctx context.Context, key string, revision int64) (*etcdserverpb.TxnResponse, error) { - ctx, span := tracer.Start(ctx, "backend.delete") + deleteCnt.Add(ctx, 1) + ctx, span := tracer.Start(ctx, "limited.delete") defer span.End() span.SetAttributes( attribute.String("key", key), attribute.Int64("revision", revision), ) - deleteCnt.Add(ctx, 1) + rev, kv, ok, err := l.backend.Delete(ctx, key, revision) if err != nil { span.RecordError(err) diff --git a/pkg/kine/server/get.go b/pkg/kine/server/get.go index cde9e509..66502cfe 100644 --- a/pkg/kine/server/get.go +++ b/pkg/kine/server/get.go @@ -9,7 +9,8 @@ import ( ) func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { - ctx, span := tracer.Start(ctx, "backend.get") + getCnt.Add(ctx, 1) + ctx, span := tracer.Start(ctx, "limited.get") defer span.End() span.SetAttributes( @@ -19,10 +20,10 @@ func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) ( attribute.Int64("revision", r.Revision), ) if r.Limit != 0 && len(r.RangeEnd) != 0 { - span.RecordError(fmt.Errorf("invalid combination of rangeEnd and limit, limit should be 0 got %d", r.Limit)) - return nil, fmt.Errorf("invalid combination of rangeEnd and limit, limit should be 0 got %d", r.Limit) + err := fmt.Errorf("invalid combination of rangeEnd and limit, limit should be 0 got %d", r.Limit) + span.RecordError(err) + return nil, err } - getCnt.Add(ctx, 1) rev, kv, err := l.backend.Get(ctx, string(r.Key), string(r.RangeEnd), r.Limit, r.Revision) if err != nil { diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index 965019a9..f4016a91 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -12,6 +12,7 @@ import ( ) func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { + listCnt.Add(ctx, 1) ctx, span := tracer.Start(ctx, "list") defer span.End() @@ -29,18 +30,13 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) } start := string(bytes.TrimRight(r.Key, "\x00")) revision := r.Revision + span.SetAttributes( + attribute.String("prefix", prefix), + attribute.String("start", start), + attribute.Int64("revision", revision), + ) if r.CountOnly { - ctx, span := tracer.Start(ctx, "backend.count") - defer span.End() - span.SetAttributes( - attribute.String("key", string(r.Key)), - attribute.String("rangeEnd", string(r.RangeEnd)), - attribute.String("prefix", prefix), - attribute.String("start", start), - attribute.Int64("revision", revision), - ) - countCnt.Add(ctx, 1) rev, count, err := l.backend.Count(ctx, prefix, start, revision) if err != nil { span.RecordError(err) @@ -59,27 +55,20 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) if limit > 0 { limit++ } - - span.SetAttributes( - attribute.String("start", start), - attribute.Int64("limit", limit), - attribute.Int64("revision", revision), - ) - - listCnt.Add(ctx, 1) + span.SetAttributes(attribute.Int64("limit", limit)) rev, kvs, err := l.backend.List(ctx, prefix, start, limit, revision) if err != nil { span.RecordError(err) return nil, err } - span.End() resp := &RangeResponse{ Header: txnHeader(rev), Count: int64(len(kvs)), Kvs: kvs, } + span.SetAttributes(attribute.Int64("list-count", resp.Count)) // count the actual number of results if there are more items in the db. if limit > 0 && resp.Count > r.Limit { @@ -90,16 +79,6 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) revision = rev } // count the actual number of results if there are more items in the db. - ctx, span := tracer.Start(ctx, "backend.count") - defer span.End() - span.SetAttributes( - attribute.String("key", string(r.Key)), - attribute.String("rangeEnd", string(r.RangeEnd)), - attribute.String("prefix", prefix), - attribute.String("start", start), - attribute.Int64("revision", revision), - ) - countCnt.Add(ctx, 1) rev, resp.Count, err = l.backend.Count(ctx, prefix, start, revision) if err != nil { span.RecordError(err) diff --git a/pkg/kine/server/maintenance.go b/pkg/kine/server/maintenance.go index b789b8bc..bc05aa83 100644 --- a/pkg/kine/server/maintenance.go +++ b/pkg/kine/server/maintenance.go @@ -5,7 +5,6 @@ import ( "fmt" "go.etcd.io/etcd/api/v3/etcdserverpb" - "go.opentelemetry.io/otel/attribute" ) var _ etcdserverpb.MaintenanceServer = (*KVServerBridge)(nil) @@ -15,14 +14,10 @@ func (s *KVServerBridge) Alarm(context.Context, *etcdserverpb.AlarmRequest) (*et } func (s *KVServerBridge) Status(ctx context.Context, r *etcdserverpb.StatusRequest) (*etcdserverpb.StatusResponse, error) { - ctx, span := tracer.Start(ctx, "backend.dbSize") - defer span.End() size, err := s.limited.dbSize(ctx) if err != nil { - span.RecordError(err) return nil, err } - span.SetAttributes(attribute.Int64("size", size)) return &etcdserverpb.StatusResponse{ Header: &etcdserverpb.ResponseHeader{}, DbSize: size, diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go index faed621c..cc3e22b9 100644 --- a/pkg/kine/server/otel.go +++ b/pkg/kine/server/otel.go @@ -3,11 +3,12 @@ package server import ( "fmt" + "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" ) -const name = "k8s-dqlite" +const name = "limited-server" var ( tracer = otel.Tracer(name) @@ -21,10 +22,36 @@ var ( ) func init() { - listCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) - getCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) - countCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.count", name), metric.WithDescription("Number of count requests")) - createCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.create", name), metric.WithDescription("Number of create requests")) - deleteCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.delete", name), metric.WithDescription("Number of delete requests")) - updateCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.update", name), metric.WithDescription("Number of update requests")) + var err error + + listCnt, err = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create list counter") + } + + getCnt, err = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create get counter") + } + + countCnt, err = meter.Int64Counter(fmt.Sprintf("%s.count", name), metric.WithDescription("Number of count requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create count counter") + } + + createCnt, err = meter.Int64Counter(fmt.Sprintf("%s.create", name), metric.WithDescription("Number of create requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + + deleteCnt, err = meter.Int64Counter(fmt.Sprintf("%s.delete", name), metric.WithDescription("Number of delete requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create delete counter") + } + + updateCnt, err = meter.Int64Counter(fmt.Sprintf("%s.update", name), metric.WithDescription("Number of update requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create update counter") + } + } diff --git a/pkg/kine/server/update.go b/pkg/kine/server/update.go index 72312532..085c7c2a 100644 --- a/pkg/kine/server/update.go +++ b/pkg/kine/server/update.go @@ -31,34 +31,23 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value ok bool err error ) - ctx, updateSpan := tracer.Start(ctx, "update") - defer updateSpan.End() - updateSpan.SetAttributes( + updateCnt.Add(ctx, 1) + + ctx, span := tracer.Start(ctx, "limited.update") + defer span.End() + span.SetAttributes( attribute.String("key", key), - attribute.Int64("revision", rev), attribute.Int64("lease", lease), + attribute.Int64("revision", rev), ) if rev == 0 { - ctx, span := tracer.Start(ctx, "backend.create") - defer span.End() - span.SetAttributes( - attribute.String("key", key), - attribute.Int64("lease", lease), - ) - createCnt.Add(ctx, 1) - rev, err = l.backend.Create(ctx, key, value, lease) ok = true - span.SetAttributes(attribute.Bool("ok", ok)) - span.RecordError(err) - span.End() - ctx, span = tracer.Start(ctx, "backend.get") - defer span.End() span.SetAttributes( - attribute.String("key", key), attribute.Int64("revision", rev), + attribute.Bool("ok", ok), ) if err == ErrKeyExists { span.AddEvent("key exists") @@ -72,15 +61,6 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value } span.End() } else { - ctx, span := tracer.Start(ctx, "backend.update") - defer span.End() - span.SetAttributes( - attribute.String("key", key), - attribute.Int64("revision", rev), - attribute.Int64("lease", lease), - ) - updateCnt.Add(ctx, 1) - rev, kv, ok, err = l.backend.Update(ctx, key, value, rev, lease) span.RecordError(err) span.SetAttributes(attribute.Bool("ok", ok)) From f56eb46daf1b37ae69e2c617379e01b9408b9e47 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 17 Jul 2024 09:34:20 +0200 Subject: [PATCH 26/43] move counters closer to where they are used --- pkg/kine/server/create.go | 16 +++++++++++++ pkg/kine/server/delete.go | 16 +++++++++++++ pkg/kine/server/get.go | 14 +++++++++++ pkg/kine/server/list.go | 14 +++++++++++ pkg/kine/server/otel.go | 49 ++------------------------------------- pkg/kine/server/update.go | 15 ++++++++++++ 6 files changed, 77 insertions(+), 47 deletions(-) diff --git a/pkg/kine/server/create.go b/pkg/kine/server/create.go index ecb28c64..8c811f3c 100644 --- a/pkg/kine/server/create.go +++ b/pkg/kine/server/create.go @@ -2,11 +2,27 @@ package server import ( "context" + "fmt" + "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" ) +var ( + createCnt metric.Int64Counter +) + +func init() { + var err error + + createCnt, err = meter.Int64Counter(fmt.Sprintf("%s.create", name), metric.WithDescription("Number of create requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } +} + func isCreate(txn *etcdserverpb.TxnRequest) *etcdserverpb.PutRequest { if len(txn.Compare) == 1 && txn.Compare[0].Target == etcdserverpb.Compare_MOD && diff --git a/pkg/kine/server/delete.go b/pkg/kine/server/delete.go index edd8cf6a..98b8537e 100644 --- a/pkg/kine/server/delete.go +++ b/pkg/kine/server/delete.go @@ -2,11 +2,27 @@ package server import ( "context" + "fmt" + "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" ) +var ( + deleteCnt metric.Int64Counter +) + +func init() { + var err error + + deleteCnt, err = meter.Int64Counter(fmt.Sprintf("%s.delete", name), metric.WithDescription("Number of delete requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create delete counter") + } +} + func isDelete(txn *etcdserverpb.TxnRequest) (int64, string, bool) { if len(txn.Compare) == 0 && len(txn.Failure) == 0 && diff --git a/pkg/kine/server/get.go b/pkg/kine/server/get.go index 66502cfe..74edc6e1 100644 --- a/pkg/kine/server/get.go +++ b/pkg/kine/server/get.go @@ -4,10 +4,24 @@ import ( "context" "fmt" + "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" ) +var ( + getCnt metric.Int64Counter +) + +func init() { + var err error + getCnt, err = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create get counter") + } +} + func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { getCnt.Add(ctx, 1) ctx, span := tracer.Start(ctx, "limited.get") diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index f4016a91..c01e85e1 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -9,8 +9,22 @@ import ( "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" ) +var ( + listCnt metric.Int64Counter +) + +func init() { + var err error + + listCnt, err = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create list counter") + } +} + func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { listCnt.Add(ctx, 1) ctx, span := tracer.Start(ctx, "list") diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go index cc3e22b9..44b29f99 100644 --- a/pkg/kine/server/otel.go +++ b/pkg/kine/server/otel.go @@ -1,57 +1,12 @@ package server import ( - "fmt" - - "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/metric" ) const name = "limited-server" var ( - tracer = otel.Tracer(name) - meter = otel.Meter(name) - getCnt metric.Int64Counter - listCnt metric.Int64Counter - countCnt metric.Int64Counter - createCnt metric.Int64Counter - deleteCnt metric.Int64Counter - updateCnt metric.Int64Counter + tracer = otel.Tracer(name) + meter = otel.Meter(name) ) - -func init() { - var err error - - listCnt, err = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create list counter") - } - - getCnt, err = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create get counter") - } - - countCnt, err = meter.Int64Counter(fmt.Sprintf("%s.count", name), metric.WithDescription("Number of count requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create count counter") - } - - createCnt, err = meter.Int64Counter(fmt.Sprintf("%s.create", name), metric.WithDescription("Number of create requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create create counter") - } - - deleteCnt, err = meter.Int64Counter(fmt.Sprintf("%s.delete", name), metric.WithDescription("Number of delete requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create delete counter") - } - - updateCnt, err = meter.Int64Counter(fmt.Sprintf("%s.update", name), metric.WithDescription("Number of update requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create update counter") - } - -} diff --git a/pkg/kine/server/update.go b/pkg/kine/server/update.go index 085c7c2a..48ddc2b7 100644 --- a/pkg/kine/server/update.go +++ b/pkg/kine/server/update.go @@ -4,10 +4,25 @@ import ( "context" "fmt" + "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" ) +var ( + updateCnt metric.Int64Counter +) + +func init() { + var err error + + updateCnt, err = meter.Int64Counter(fmt.Sprintf("%s.update", name), metric.WithDescription("Number of update requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create update counter") + } +} + func isUpdate(txn *etcdserverpb.TxnRequest) (int64, string, []byte, int64, bool) { if len(txn.Compare) == 1 && txn.Compare[0].Target == etcdserverpb.Compare_MOD && From 2733d4ed70d689f8958fe918f025871f639210a6 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 17 Jul 2024 15:04:44 +0200 Subject: [PATCH 27/43] traces in logstructured --- pkg/kine/logstructured/logstructured.go | 100 ++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/pkg/kine/logstructured/logstructured.go b/pkg/kine/logstructured/logstructured.go index c3f59a92..4c2a1cb0 100644 --- a/pkg/kine/logstructured/logstructured.go +++ b/pkg/kine/logstructured/logstructured.go @@ -2,12 +2,21 @@ package logstructured import ( "context" + "fmt" "sync" "sync/atomic" "time" "github.com/canonical/k8s-dqlite/pkg/kine/server" "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" +) + +const name = "logstructured" + +var ( + tracer = otel.Tracer(name) ) type Log interface { @@ -57,9 +66,19 @@ func (l *LogStructured) Wait() { } func (l *LogStructured) Get(ctx context.Context, key, rangeEnd string, limit, revision int64) (revRet int64, kvRet *server.KeyValue, errRet error) { + ctx, span := tracer.Start(ctx, "logstructured.Get") + span.SetAttributes( + attribute.String("key", key), + attribute.String("rangeEnd", rangeEnd), + attribute.Int64("limit", limit), + attribute.Int64("revision", revision), + ) defer func() { l.adjustRevision(ctx, &revRet) logrus.Debugf("GET %s, rev=%d => rev=%d, kv=%v, err=%v", key, revision, revRet, kvRet != nil, errRet) + span.SetAttributes(attribute.Int64("adjusted-revision", revRet)) + span.RecordError(errRet) + span.End() }() rev, event, err := l.get(ctx, key, rangeEnd, limit, revision, false) @@ -70,8 +89,19 @@ func (l *LogStructured) Get(ctx context.Context, key, rangeEnd string, limit, re } func (l *LogStructured) get(ctx context.Context, key, rangeEnd string, limit, revision int64, includeDeletes bool) (int64, *server.Event, error) { + ctx, span := tracer.Start(ctx, "logstructured.get") + defer span.End() + span.SetAttributes( + attribute.String("key", key), + attribute.String("rangeEnd", rangeEnd), + attribute.Int64("limit", limit), + attribute.Int64("revision", revision), + attribute.Bool("includeDeletes", includeDeletes), + ) rev, events, err := l.log.List(ctx, key, rangeEnd, limit, revision, includeDeletes) + span.RecordError(err) if err == server.ErrCompacted { + span.AddEvent("ignoring compacted error") // ignore compacted when getting by revision err = nil } @@ -98,9 +128,19 @@ func (l *LogStructured) adjustRevision(ctx context.Context, rev *int64) { } func (l *LogStructured) Create(ctx context.Context, key string, value []byte, lease int64) (revRet int64, errRet error) { + ctx, span := tracer.Start(ctx, "logstructured.Create") + span.SetAttributes( + attribute.String("key", key), + attribute.Int64("lease", lease), + attribute.Int64("value-size", int64(len(value))), + ) defer func() { l.adjustRevision(ctx, &revRet) logrus.Debugf("CREATE %s, size=%d, lease=%d => rev=%d, err=%v", key, len(value), lease, revRet, errRet) + span.SetAttributes(attribute.Int64("returned-revision", revRet)) + span.RecordError(errRet) + span.End() + }() rev, prevEvent, err := l.get(ctx, key, "", 1, 0, true) @@ -130,13 +170,25 @@ func (l *LogStructured) Create(ctx context.Context, key string, value []byte, le } func (l *LogStructured) Delete(ctx context.Context, key string, revision int64) (revRet int64, kvRet *server.KeyValue, deletedRet bool, errRet error) { + ctx, span := tracer.Start(ctx, "logstructured.Delete") + defer func() { l.adjustRevision(ctx, &revRet) logrus.Debugf("DELETE %s, rev=%d => rev=%d, kv=%v, deleted=%v, err=%v", key, revision, revRet, kvRet != nil, deletedRet, errRet) + span.SetAttributes( + attribute.String("key", key), + attribute.Int64("revision", revision), + attribute.Int64("adjusted-revision", revRet), + attribute.Bool("deleted", deletedRet), + attribute.Bool("kv-found", kvRet != nil), + ) + span.RecordError(errRet) + span.End() }() rev, event, err := l.get(ctx, key, "", 1, 0, true) if err != nil { + span.RecordError(err) return 0, nil, false, err } @@ -162,8 +214,12 @@ func (l *LogStructured) Delete(ctx context.Context, key string, revision int64) if err != nil { // If error on Append we assume it's a UNIQUE constraint error, so we fetch the latest (if we can) // and return that the delete failed + span.AddEvent("Failed to append delete event") + span.RecordError(err) latestRev, latestEvent, latestErr := l.get(ctx, key, "", 1, 0, true) if latestErr != nil || latestEvent == nil { + span.RecordError(latestErr) + span.SetAttributes(attribute.Bool("latest-event-found", latestEvent != nil)) return rev, event.KV, false, nil } return latestRev, latestEvent.KV, false, nil @@ -172,8 +228,20 @@ func (l *LogStructured) Delete(ctx context.Context, key string, revision int64) } func (l *LogStructured) List(ctx context.Context, prefix, startKey string, limit, revision int64) (revRet int64, kvRet []*server.KeyValue, errRet error) { + ctx, span := tracer.Start(ctx, "logstructured.List") + defer func() { logrus.Debugf("LIST %s, start=%s, limit=%d, rev=%d => rev=%d, kvs=%d, err=%v", prefix, startKey, limit, revision, revRet, len(kvRet), errRet) + span.SetAttributes( + attribute.String("prefix", prefix), + attribute.String("startKey", startKey), + attribute.Int64("limit", limit), + attribute.Int64("revision", revision), + attribute.Int64("adjusted-revision", revRet), + attribute.Int64("kv-count", int64(len(kvRet))), + ) + span.RecordError(errRet) + span.End() }() rev, events, err := l.log.List(ctx, prefix, startKey, limit, revision, false) @@ -202,8 +270,18 @@ func (l *LogStructured) List(ctx context.Context, prefix, startKey string, limit } func (l *LogStructured) Count(ctx context.Context, prefix, startKey string, revision int64) (revRet int64, count int64, err error) { + ctx, span := tracer.Start(ctx, "logstructured.Count") defer func() { logrus.Debugf("COUNT prefix=%s startKey=%s => rev=%d, count=%d, err=%v", prefix, startKey, revRet, count, err) + span.SetAttributes( + attribute.String("prefix", prefix), + attribute.String("startKey", startKey), + attribute.Int64("revision", revision), + attribute.Int64("adjusted-revision", revRet), + attribute.Int64("count", count), + ) + span.RecordError(err) + span.End() }() rev, count, err := l.log.Count(ctx, prefix, startKey, revision) if err != nil { @@ -223,6 +301,8 @@ func (l *LogStructured) Count(ctx context.Context, prefix, startKey string, revi } func (l *LogStructured) Update(ctx context.Context, key string, value []byte, revision, lease int64) (revRet int64, kvRet *server.KeyValue, updateRet bool, errRet error) { + ctx, span := tracer.Start(ctx, "logstructured.Update") + defer func() { l.adjustRevision(ctx, &revRet) kvRev := int64(0) @@ -230,6 +310,16 @@ func (l *LogStructured) Update(ctx context.Context, key string, value []byte, re kvRev = kvRet.ModRevision } logrus.Debugf("UPDATE %s, value=%d, rev=%d, lease=%v => rev=%d, kvrev=%d, updated=%v, err=%v", key, len(value), revision, lease, revRet, kvRev, updateRet, errRet) + span.SetAttributes( + attribute.String("key", key), + attribute.Int64("revision", revision), + attribute.Int64("lease", lease), + attribute.Int64("value-size", int64(len(value))), + attribute.Int64("adjusted-revision", revRet), + attribute.Int64("kv-mod-revision", kvRev), + attribute.Bool("updated", updateRet), + ) + span.End() }() rev, event, err := l.get(ctx, key, "", 1, 0, false) @@ -335,6 +425,12 @@ func (l *LogStructured) ttl(ctx context.Context) { func (l *LogStructured) Watch(ctx context.Context, prefix string, revision int64) <-chan []*server.Event { logrus.Debugf("WATCH %s, revision=%d", prefix, revision) + ctx, span := tracer.Start(ctx, "logstructured.Watch") + defer span.End() + span.SetAttributes( + attribute.String("prefix", prefix), + attribute.Int64("revision", revision), + ) // starting watching right away so we don't miss anything ctx, cancel := context.WithCancel(ctx) @@ -350,10 +446,14 @@ func (l *LogStructured) Watch(ctx context.Context, prefix string, revision int64 rev, kvs, err := l.log.After(ctx, prefix, revision, 0) if err != nil { logrus.Errorf("failed to list %s for revision %d", prefix, revision) + msg := fmt.Sprintf("failed to list %s for revision %d", prefix, revision) + span.AddEvent(msg) + logrus.Errorf(msg) cancel() } logrus.Debugf("WATCH LIST key=%s rev=%d => rev=%d kvs=%d", prefix, revision, rev, len(kvs)) + span.SetAttributes(attribute.Int64("current-revision", rev), attribute.Int64("kvs-count", int64(len(kvs)))) l.wg.Add(1) go func() { From ceb9c8f717f60432d4c920b1a124c42d791b6a1f Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Thu, 18 Jul 2024 13:13:00 +0200 Subject: [PATCH 28/43] capture more logs on compaction --- pkg/kine/drivers/generic/generic.go | 35 ++++++++++++++++++-- pkg/kine/drivers/generic/otel.go | 46 +++++++++++++++++++++++++++ pkg/kine/logstructured/sqllog/otel.go | 8 ++++- pkg/kine/logstructured/sqllog/sql.go | 2 +- pkg/kine/server/update.go | 13 +------- 5 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 pkg/kine/drivers/generic/otel.go diff --git a/pkg/kine/drivers/generic/generic.go b/pkg/kine/drivers/generic/generic.go index 309b33bb..de1daa03 100644 --- a/pkg/kine/drivers/generic/generic.go +++ b/pkg/kine/drivers/generic/generic.go @@ -13,6 +13,7 @@ import ( "github.com/Rican7/retry/jitter" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/attribute" ) var ( @@ -477,15 +478,19 @@ func (d *Generic) executePrepared(ctx context.Context, txName, sql string, prepa } func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) { + getCompactRevCnt.Add(ctx, 1) + ctx, span := tracer.Start(ctx, "get_compact_revision") var compact, target sql.NullInt64 start := time.Now() var err error defer func() { + span.RecordError(err) if err == sql.ErrNoRows { err = nil } recordOpResult("revision_interval_sql", err, start) recordTxResult("revision_interval_sql", err) + span.End() }() done, err := d.AdmissionControlPolicy.Admit(ctx, "revision_interval_sql") @@ -511,21 +516,40 @@ func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) if err == sql.ErrNoRows { return 0, 0, nil } - + span.SetAttributes(attribute.Int64("compact", compact.Int64), attribute.Int64("target", target.Int64)) return compact.Int64, target.Int64, err } func (d *Generic) SetCompactRevision(ctx context.Context, revision int64) error { + setCompactRevCnt.Add(ctx, 1) + ctx, span := tracer.Start(ctx, "set_compact_revision") + defer span.End() + span.SetAttributes(attribute.Int64("revision", revision)) + _, err := d.executePrepared(ctx, "update_compact_sql", d.UpdateCompactSQL, d.updateCompactSQLPrepared, revision) + span.RecordError(err) return err } func (d *Generic) GetRevision(ctx context.Context, revision int64) (*sql.Rows, error) { - return d.queryPrepared(ctx, "get_revision_sql", d.GetRevisionSQL, d.getRevisionSQLPrepared, revision) + getRevisionCnt.Add(ctx, 1) + ctx, span := tracer.Start(ctx, "get_revision") + defer span.End() + span.SetAttributes(attribute.Int64("revision", revision)) + + result, err := d.queryPrepared(ctx, "get_revision_sql", d.GetRevisionSQL, d.getRevisionSQLPrepared, revision) + span.RecordError(err) + return result, err } func (d *Generic) DeleteRevision(ctx context.Context, revision int64) error { + deleteRevCnt.Add(ctx, 1) + ctx, span := tracer.Start(ctx, "delete_revision") + defer span.End() + span.SetAttributes(attribute.Int64("revision", revision)) + _, err := d.executePrepared(ctx, "delete_sql", d.DeleteSQL, d.deleteSQLPrepared, revision) + span.RecordError(err) return err } @@ -562,20 +586,27 @@ func (d *Generic) List(ctx context.Context, prefix, startKey string, limit, revi } func (d *Generic) CurrentRevision(ctx context.Context) (int64, error) { + currentRevCnt.Add(ctx, 1) + ctx, span := tracer.Start(ctx, "current_revision") + defer span.End() var id int64 var err error done, err := d.AdmissionControlPolicy.Admit(ctx, "rev_sql") if err != nil { + span.RecordError(err) return 0, fmt.Errorf("denied: %w", err) } row := d.queryRow(ctx, "rev_sql", revSQL) done() err = row.Scan(&id) + span.RecordError(err) if err == sql.ErrNoRows { + span.AddEvent("no rows") return 0, nil } + span.SetAttributes(attribute.Int64("id", id)) return id, err } diff --git a/pkg/kine/drivers/generic/otel.go b/pkg/kine/drivers/generic/otel.go new file mode 100644 index 00000000..5e18755b --- /dev/null +++ b/pkg/kine/drivers/generic/otel.go @@ -0,0 +1,46 @@ +package generic + +import ( + "fmt" + + "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +const name = "generic" + +var ( + tracer = otel.Tracer(name) + meter = otel.Meter(name) + setCompactRevCnt metric.Int64Counter + getRevisionCnt metric.Int64Counter + deleteRevCnt metric.Int64Counter + currentRevCnt metric.Int64Counter + getCompactRevCnt metric.Int64Counter +) + +func init() { + var err error + setCompactRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + getRevisionCnt, err = meter.Int64Counter(fmt.Sprintf("%s.get_revision", name), metric.WithDescription("Number of get revision requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + deleteRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.delete_revision", name), metric.WithDescription("Number of delete revision requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + currentRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.current_revision", name), metric.WithDescription("Current revision")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + getCompactRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.get_compact_revision", name), metric.WithDescription("Get compact revision")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + +} diff --git a/pkg/kine/logstructured/sqllog/otel.go b/pkg/kine/logstructured/sqllog/otel.go index e7c87999..2330f4fc 100644 --- a/pkg/kine/logstructured/sqllog/otel.go +++ b/pkg/kine/logstructured/sqllog/otel.go @@ -3,6 +3,7 @@ package sqllog import ( "fmt" + "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" ) @@ -16,5 +17,10 @@ var ( ) func init() { - compactCnt, _ = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) + var err error + compactCnt, err = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + } diff --git a/pkg/kine/logstructured/sqllog/sql.go b/pkg/kine/logstructured/sqllog/sql.go index 28d5f426..6c0af6e3 100644 --- a/pkg/kine/logstructured/sqllog/sql.go +++ b/pkg/kine/logstructured/sqllog/sql.go @@ -232,7 +232,7 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { return nextEnd, fmt.Errorf("failed to record compact revision: %v", err) } } - + span.SetAttributes(attribute.Int64("new-nextEnd", nextEnd), attribute.Int64("cursor-ended:", cursor)) return nextEnd, nil } diff --git a/pkg/kine/server/update.go b/pkg/kine/server/update.go index 48ddc2b7..f0979dbe 100644 --- a/pkg/kine/server/update.go +++ b/pkg/kine/server/update.go @@ -64,23 +64,12 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value attribute.Int64("revision", rev), attribute.Bool("ok", ok), ) - if err == ErrKeyExists { - span.AddEvent("key exists") - rev, kv, err = l.backend.Get(ctx, key, "", 1, rev) - if err != nil { - span.RecordError(err) - } - span.AddEvent(fmt.Sprintf("got revision %d", rev)) - } else { - ok = true - } - span.End() } else { rev, kv, ok, err = l.backend.Update(ctx, key, value, rev, lease) - span.RecordError(err) span.SetAttributes(attribute.Bool("ok", ok)) } if err != nil { + span.RecordError(err) return nil, err } From ada277b9456df8ae153e9dd4fccd9e9b9d620cce Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Thu, 18 Jul 2024 13:17:58 +0200 Subject: [PATCH 29/43] cleanup --- pkg/kine/drivers/generic/generic.go | 39 +++++++++++++++++++++++ pkg/kine/drivers/generic/otel.go | 46 --------------------------- pkg/kine/logstructured/sqllog/otel.go | 26 --------------- pkg/kine/logstructured/sqllog/sql.go | 19 +++++++++++ 4 files changed, 58 insertions(+), 72 deletions(-) delete mode 100644 pkg/kine/drivers/generic/otel.go delete mode 100644 pkg/kine/logstructured/sqllog/otel.go diff --git a/pkg/kine/drivers/generic/generic.go b/pkg/kine/drivers/generic/generic.go index de1daa03..0184395c 100644 --- a/pkg/kine/drivers/generic/generic.go +++ b/pkg/kine/drivers/generic/generic.go @@ -13,9 +13,48 @@ import ( "github.com/Rican7/retry/jitter" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" ) +const name = "generic" + +var ( + tracer = otel.Tracer(name) + meter = otel.Meter(name) + setCompactRevCnt metric.Int64Counter + getRevisionCnt metric.Int64Counter + deleteRevCnt metric.Int64Counter + currentRevCnt metric.Int64Counter + getCompactRevCnt metric.Int64Counter +) + +func init() { + var err error + setCompactRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + getRevisionCnt, err = meter.Int64Counter(fmt.Sprintf("%s.get_revision", name), metric.WithDescription("Number of get revision requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + deleteRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.delete_revision", name), metric.WithDescription("Number of delete revision requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + currentRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.current_revision", name), metric.WithDescription("Current revision")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + getCompactRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.get_compact_revision", name), metric.WithDescription("Get compact revision")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + +} + var ( columns = "kv.id as theid, kv.name, kv.created, kv.deleted, kv.create_revision, kv.prev_revision, kv.lease, kv.value, kv.old_value" diff --git a/pkg/kine/drivers/generic/otel.go b/pkg/kine/drivers/generic/otel.go deleted file mode 100644 index 5e18755b..00000000 --- a/pkg/kine/drivers/generic/otel.go +++ /dev/null @@ -1,46 +0,0 @@ -package generic - -import ( - "fmt" - - "github.com/sirupsen/logrus" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/metric" -) - -const name = "generic" - -var ( - tracer = otel.Tracer(name) - meter = otel.Meter(name) - setCompactRevCnt metric.Int64Counter - getRevisionCnt metric.Int64Counter - deleteRevCnt metric.Int64Counter - currentRevCnt metric.Int64Counter - getCompactRevCnt metric.Int64Counter -) - -func init() { - var err error - setCompactRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create create counter") - } - getRevisionCnt, err = meter.Int64Counter(fmt.Sprintf("%s.get_revision", name), metric.WithDescription("Number of get revision requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create create counter") - } - deleteRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.delete_revision", name), metric.WithDescription("Number of delete revision requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create create counter") - } - currentRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.current_revision", name), metric.WithDescription("Current revision")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create create counter") - } - getCompactRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.get_compact_revision", name), metric.WithDescription("Get compact revision")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create create counter") - } - -} diff --git a/pkg/kine/logstructured/sqllog/otel.go b/pkg/kine/logstructured/sqllog/otel.go deleted file mode 100644 index 2330f4fc..00000000 --- a/pkg/kine/logstructured/sqllog/otel.go +++ /dev/null @@ -1,26 +0,0 @@ -package sqllog - -import ( - "fmt" - - "github.com/sirupsen/logrus" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/metric" -) - -const name = "sqllog" - -var ( - tracer = otel.Tracer(name) - meter = otel.Meter(name) - compactCnt metric.Int64Counter -) - -func init() { - var err error - compactCnt, err = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create create counter") - } - -} diff --git a/pkg/kine/logstructured/sqllog/sql.go b/pkg/kine/logstructured/sqllog/sql.go index 6c0af6e3..125fb7e5 100644 --- a/pkg/kine/logstructured/sqllog/sql.go +++ b/pkg/kine/logstructured/sqllog/sql.go @@ -11,11 +11,30 @@ import ( "github.com/canonical/k8s-dqlite/pkg/kine/broadcaster" "github.com/canonical/k8s-dqlite/pkg/kine/server" "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" ) const SupersededCount = 100 +const name = "sqllog" + +var ( + tracer = otel.Tracer(name) + meter = otel.Meter(name) + compactCnt metric.Int64Counter +) + +func init() { + var err error + compactCnt, err = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + +} + type SQLLog struct { d Dialect broadcaster broadcaster.Broadcaster From cda87ce7d63df92d7ec17c6cd1d7b9ea11987364 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Thu, 18 Jul 2024 13:35:26 +0200 Subject: [PATCH 30/43] naming --- cmd/tracing.go | 4 ++-- pkg/kine/drivers/generic/generic.go | 10 +++++----- pkg/kine/server/list.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/tracing.go b/cmd/tracing.go index 955ca1ae..6a4cec96 100644 --- a/cmd/tracing.go +++ b/cmd/tracing.go @@ -20,6 +20,7 @@ import ( var ( otelEndpoint = "localhost:4317" + resourceName = "k8s-dqlite" ) // setupOTelSDK bootstraps the OpenTelemetry pipeline. @@ -43,7 +44,7 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er res, err := resource.New(ctx, resource.WithAttributes( - semconv.ServiceNameKey.String("k8s-dqlite"), + semconv.ServiceNameKey.String(resourceName), ), ) if err != nil { @@ -96,7 +97,6 @@ func newExporter(ctx context.Context) (trace.SpanExporter, error) { func initConn() (*grpc.ClientConn, error) { // It connects the OpenTelemetry Collector through local gRPC connection. - // You may replace `localhost:4317` with your endpoint. conn, err := grpc.NewClient(otelEndpoint, // Note the use of insecure transport here. TLS is recommended in production. grpc.WithTransportCredentials(insecure.NewCredentials()), diff --git a/pkg/kine/drivers/generic/generic.go b/pkg/kine/drivers/generic/generic.go index 0184395c..5583c0dd 100644 --- a/pkg/kine/drivers/generic/generic.go +++ b/pkg/kine/drivers/generic/generic.go @@ -518,7 +518,7 @@ func (d *Generic) executePrepared(ctx context.Context, txName, sql string, prepa func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) { getCompactRevCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "get_compact_revision") + ctx, span := tracer.Start(ctx, "generic.get_compact_revision") var compact, target sql.NullInt64 start := time.Now() var err error @@ -561,7 +561,7 @@ func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) func (d *Generic) SetCompactRevision(ctx context.Context, revision int64) error { setCompactRevCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "set_compact_revision") + ctx, span := tracer.Start(ctx, "generic.set_compact_revision") defer span.End() span.SetAttributes(attribute.Int64("revision", revision)) @@ -572,7 +572,7 @@ func (d *Generic) SetCompactRevision(ctx context.Context, revision int64) error func (d *Generic) GetRevision(ctx context.Context, revision int64) (*sql.Rows, error) { getRevisionCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "get_revision") + ctx, span := tracer.Start(ctx, "generic.get_revision") defer span.End() span.SetAttributes(attribute.Int64("revision", revision)) @@ -583,7 +583,7 @@ func (d *Generic) GetRevision(ctx context.Context, revision int64) (*sql.Rows, e func (d *Generic) DeleteRevision(ctx context.Context, revision int64) error { deleteRevCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "delete_revision") + ctx, span := tracer.Start(ctx, "generic.delete_revision") defer span.End() span.SetAttributes(attribute.Int64("revision", revision)) @@ -626,7 +626,7 @@ func (d *Generic) List(ctx context.Context, prefix, startKey string, limit, revi func (d *Generic) CurrentRevision(ctx context.Context) (int64, error) { currentRevCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "current_revision") + ctx, span := tracer.Start(ctx, "generic.current_revision") defer span.End() var id int64 var err error diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index c01e85e1..032b5e7e 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -27,7 +27,7 @@ func init() { func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { listCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "list") + ctx, span := tracer.Start(ctx, "limited.list") defer span.End() span.SetAttributes( From 8577851001bbd6e7b0792298d68ab09d509b83a1 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Thu, 18 Jul 2024 13:49:27 +0200 Subject: [PATCH 31/43] pass conn address --- cmd/root.go | 4 +++- cmd/tracing.go | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 6e50c2f4..08449857 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -29,6 +29,7 @@ var ( minTLSVersion string metrics bool metricsAddress string + otelAddress string watchAvailableStorageInterval time.Duration watchAvailableStorageMinBytes uint64 @@ -61,7 +62,7 @@ var ( if rootCmdOpts.metrics { go func() { - otelShutdown, err := setupOTelSDK(cmd.Context()) + otelShutdown, err := setupOTelSDK(cmd.Context(), rootCmdOpts.otelAddress) if err != nil { logrus.WithError(err).Warning("Failed to setup OpenTelemetry SDK") } @@ -144,6 +145,7 @@ func init() { rootCmd.Flags().UintVar(&rootCmdOpts.clientSessionCacheSize, "tls-client-session-cache-size", 0, "ClientCacheSession size for dial TLS config") rootCmd.Flags().StringVar(&rootCmdOpts.minTLSVersion, "min-tls-version", "tls12", "Minimum TLS version for dqlite endpoint (tls10|tls11|tls12|tls13). Default is tls12") rootCmd.Flags().BoolVar(&rootCmdOpts.metrics, "metrics", true, "enable metrics endpoint") + rootCmd.Flags().StringVar(&rootCmdOpts.otelAddress, "otel-listen", "127.0.0.1:4317", "listen address for OpenTelemetry endpoint") rootCmd.Flags().StringVar(&rootCmdOpts.metricsAddress, "metrics-listen", "127.0.0.1:9042", "listen address for metrics endpoint") rootCmd.Flags().DurationVar(&rootCmdOpts.watchAvailableStorageInterval, "watch-storage-available-size-interval", 5*time.Second, "Interval to check if the disk is running low on space. Set to 0 to disable the periodic disk size check") rootCmd.Flags().Uint64Var(&rootCmdOpts.watchAvailableStorageMinBytes, "watch-storage-available-size-min-bytes", 10*1024*1024, "Minimum required available disk size (in bytes) to continue operation. If available disk space gets below this threshold, then the --low-available-storage-action is performed") diff --git a/cmd/tracing.go b/cmd/tracing.go index 6a4cec96..3f41aa57 100644 --- a/cmd/tracing.go +++ b/cmd/tracing.go @@ -25,7 +25,7 @@ var ( // setupOTelSDK bootstraps the OpenTelemetry pipeline. // If it does not return an error, make sure to call shutdown for proper cleanup. -func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, err error) { +func setupOTelSDK(ctx context.Context, otelEndpoint string) (shutdown func(context.Context) error, err error) { logrus.SetLevel(logrus.TraceLevel) var shutdownFuncs []func(context.Context) error @@ -51,7 +51,7 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er logrus.WithError(err).Warning("Otel failed to create resource") } - tracerProvider, err := newTraceProvider(ctx, res) + tracerProvider, err := newTraceProvider(ctx, otelEndpoint, res) if err != nil { handleErr(err) return @@ -70,8 +70,8 @@ func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, er return } -func newTraceProvider(ctx context.Context, res *resource.Resource) (*trace.TracerProvider, error) { - traceExporter, err := newExporter(ctx) +func newTraceProvider(ctx context.Context, otelEndpoint string, res *resource.Resource) (*trace.TracerProvider, error) { + traceExporter, err := newExporter(ctx, otelEndpoint) if err != nil { return nil, err } @@ -85,8 +85,8 @@ func newTraceProvider(ctx context.Context, res *resource.Resource) (*trace.Trace return traceProvider, nil } -func newExporter(ctx context.Context) (trace.SpanExporter, error) { - conn, _ := initConn() +func newExporter(ctx context.Context, otelEndpoint string) (trace.SpanExporter, error) { + conn, _ := initConn(otelEndpoint) traceExporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn)) if err != nil { @@ -95,7 +95,7 @@ func newExporter(ctx context.Context) (trace.SpanExporter, error) { return traceExporter, nil } -func initConn() (*grpc.ClientConn, error) { +func initConn(otelEndpoint string) (*grpc.ClientConn, error) { // It connects the OpenTelemetry Collector through local gRPC connection. conn, err := grpc.NewClient(otelEndpoint, // Note the use of insecure transport here. TLS is recommended in production. From 0bb3126b24fb47d765e25fb9de0c4086d69f1833 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Thu, 18 Jul 2024 13:58:12 +0200 Subject: [PATCH 32/43] cleanup --- cmd/tracing.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/tracing.go b/cmd/tracing.go index 3f41aa57..3e73ebf2 100644 --- a/cmd/tracing.go +++ b/cmd/tracing.go @@ -19,7 +19,6 @@ import ( ) var ( - otelEndpoint = "localhost:4317" resourceName = "k8s-dqlite" ) From 04ab82112c51698d02266c6d63f8e1e56684a7ab Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Thu, 18 Jul 2024 18:51:26 +0200 Subject: [PATCH 33/43] rewrite otel setup, disable otel by default --- cmd/root.go | 10 +++++- cmd/tracing.go | 93 ++++++++++++++++++++++++++------------------------ 2 files changed, 57 insertions(+), 46 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 08449857..0582e0de 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -27,6 +27,7 @@ var ( diskMode bool clientSessionCacheSize uint minTLSVersion string + otel bool metrics bool metricsAddress string otelAddress string @@ -60,8 +61,9 @@ var ( }() } - if rootCmdOpts.metrics { + if rootCmdOpts.otel { go func() { + logrus.WithField("address", rootCmdOpts.otelAddress).Print("Enable otel endpoint") otelShutdown, err := setupOTelSDK(cmd.Context(), rootCmdOpts.otelAddress) if err != nil { logrus.WithError(err).Warning("Failed to setup OpenTelemetry SDK") @@ -72,6 +74,11 @@ var ( logrus.WithError(err).Warning("Failed to shutdown OpenTelemetry SDK") } }() + }() + } + + if rootCmdOpts.metrics { + go func() { logrus.WithField("address", rootCmdOpts.metricsAddress).Print("Enable metrics endpoint") mux := http.NewServeMux() mux.Handle("/metrics", promhttp.Handler()) @@ -145,6 +152,7 @@ func init() { rootCmd.Flags().UintVar(&rootCmdOpts.clientSessionCacheSize, "tls-client-session-cache-size", 0, "ClientCacheSession size for dial TLS config") rootCmd.Flags().StringVar(&rootCmdOpts.minTLSVersion, "min-tls-version", "tls12", "Minimum TLS version for dqlite endpoint (tls10|tls11|tls12|tls13). Default is tls12") rootCmd.Flags().BoolVar(&rootCmdOpts.metrics, "metrics", true, "enable metrics endpoint") + rootCmd.Flags().BoolVar(&rootCmdOpts.otel, "otel", false, "enable traces endpoint") rootCmd.Flags().StringVar(&rootCmdOpts.otelAddress, "otel-listen", "127.0.0.1:4317", "listen address for OpenTelemetry endpoint") rootCmd.Flags().StringVar(&rootCmdOpts.metricsAddress, "metrics-listen", "127.0.0.1:9042", "listen address for metrics endpoint") rootCmd.Flags().DurationVar(&rootCmdOpts.watchAvailableStorageInterval, "watch-storage-available-size-interval", 5*time.Second, "Interval to check if the disk is running low on space. Set to 0 to disable the periodic disk size check") diff --git a/cmd/tracing.go b/cmd/tracing.go index 3e73ebf2..4e8fb3d1 100644 --- a/cmd/tracing.go +++ b/cmd/tracing.go @@ -25,20 +25,10 @@ var ( // setupOTelSDK bootstraps the OpenTelemetry pipeline. // If it does not return an error, make sure to call shutdown for proper cleanup. func setupOTelSDK(ctx context.Context, otelEndpoint string) (shutdown func(context.Context) error, err error) { - logrus.SetLevel(logrus.TraceLevel) - var shutdownFuncs []func(context.Context) error - - shutdown = func(ctx context.Context) error { - var err error - for _, fn := range shutdownFuncs { - err = errors.Join(err, fn(ctx)) - } - shutdownFuncs = nil - return err - } - - handleErr := func(inErr error) { - err = errors.Join(inErr, shutdown(ctx)) + conn, err := initConn(otelEndpoint) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create gRPC connection") + return nil, fmt.Errorf("failed to create gRPC connection to collector: %w", err) } res, err := resource.New(ctx, @@ -50,48 +40,43 @@ func setupOTelSDK(ctx context.Context, otelEndpoint string) (shutdown func(conte logrus.WithError(err).Warning("Otel failed to create resource") } - tracerProvider, err := newTraceProvider(ctx, otelEndpoint, res) + traceExporter, err := newTraceExporter(ctx, conn) if err != nil { - handleErr(err) - return + logrus.WithError(err).Warning("Otel failed to create trace exporter") + return nil, err } - shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown) - otel.SetTracerProvider(tracerProvider) - meterProvider, err := newMeterProvider(res) + tracerProvider, err := newTraceProvider(traceExporter, res) if err != nil { - handleErr(err) - return + logrus.WithError(err).Warning("Otel failed to create trace provider") + return nil, err } - shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown) - otel.SetMeterProvider(meterProvider) - - return -} + otel.SetTracerProvider(tracerProvider) -func newTraceProvider(ctx context.Context, otelEndpoint string, res *resource.Resource) (*trace.TracerProvider, error) { - traceExporter, err := newExporter(ctx, otelEndpoint) + meterProvider, err := newMeterProvider(res) if err != nil { + logrus.WithError(err).Warning("Otel failed to create meter provider") return nil, err } + otel.SetMeterProvider(meterProvider) - traceProvider := trace.NewTracerProvider( - trace.WithBatcher(traceExporter, - trace.WithBatchTimeout(time.Second), - ), - trace.WithResource(res), - ) - return traceProvider, nil -} - -func newExporter(ctx context.Context, otelEndpoint string) (trace.SpanExporter, error) { - conn, _ := initConn(otelEndpoint) - - traceExporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn)) - if err != nil { - return nil, fmt.Errorf("failed to create trace exporter: %w", err) + shutdown = func(ctx context.Context) error { + var shutdownErrs error + err = meterProvider.Shutdown(ctx) + if err != nil { + shutdownErrs = errors.Join(shutdownErrs, err) + } + err = tracerProvider.Shutdown(ctx) + if err != nil { + shutdownErrs = errors.Join(shutdownErrs, err) + } + err = conn.Close() + if err != nil { + shutdownErrs = errors.Join(shutdownErrs, err) + } + return shutdownErrs } - return traceExporter, nil + return } func initConn(otelEndpoint string) (*grpc.ClientConn, error) { @@ -107,6 +92,24 @@ func initConn(otelEndpoint string) (*grpc.ClientConn, error) { return conn, nil } +func newTraceExporter(ctx context.Context, conn *grpc.ClientConn) (trace.SpanExporter, error) { + exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn)) + if err != nil { + return nil, fmt.Errorf("failed to create trace exporter: %w", err) + } + return exporter, nil +} + +func newTraceProvider(traceExporter trace.SpanExporter, res *resource.Resource) (*trace.TracerProvider, error) { + traceProvider := trace.NewTracerProvider( + trace.WithBatcher(traceExporter, + trace.WithBatchTimeout(time.Second), + ), + trace.WithResource(res), + ) + return traceProvider, nil +} + func newMeterProvider(res *resource.Resource) (*metric.MeterProvider, error) { metricExporter, err := stdoutmetric.New() if err != nil { From f081376e52333bf61b0286ca40c5353c1f9c3bc3 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Fri, 19 Jul 2024 09:12:21 +0200 Subject: [PATCH 34/43] move otel shutdown --- cmd/root.go | 19 +++++++++++-------- cmd/tracing.go | 1 + 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 0582e0de..a029c4c3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -61,19 +61,16 @@ var ( }() } + var otelShutdown func(context.Context) error + if rootCmdOpts.otel { go func() { - logrus.WithField("address", rootCmdOpts.otelAddress).Print("Enable otel endpoint") - otelShutdown, err := setupOTelSDK(cmd.Context(), rootCmdOpts.otelAddress) + var err error + logrus.WithField("address", rootCmdOpts.metricsAddress).Print("Enable otel endpoint") + otelShutdown, err = setupOTelSDK(cmd.Context(), rootCmdOpts.otelAddress) if err != nil { logrus.WithError(err).Warning("Failed to setup OpenTelemetry SDK") } - defer func() { - err = errors.Join(err, otelShutdown(cmd.Context())) - if err != nil { - logrus.WithError(err).Warning("Failed to shutdown OpenTelemetry SDK") - } - }() }() } @@ -129,6 +126,12 @@ var ( if err := instance.Shutdown(stopCtx); err != nil { logrus.WithError(err).Fatal("Failed to shutdown server") } + if rootCmdOpts.metrics && otelShutdown != nil { + err = errors.Join(err, otelShutdown(stopCtx)) + if err != nil { + logrus.WithError(err).Warning("Failed to shutdown OpenTelemetry SDK") + } + } }, } ) diff --git a/cmd/tracing.go b/cmd/tracing.go index 4e8fb3d1..90207365 100644 --- a/cmd/tracing.go +++ b/cmd/tracing.go @@ -55,6 +55,7 @@ func setupOTelSDK(ctx context.Context, otelEndpoint string) (shutdown func(conte meterProvider, err := newMeterProvider(res) if err != nil { + tracerProvider.Shutdown(ctx) logrus.WithError(err).Warning("Otel failed to create meter provider") return nil, err } From b42ceeae5070d934c9bdd08dfc3eb56e4433c8ed Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Fri, 19 Jul 2024 12:37:42 +0200 Subject: [PATCH 35/43] typo --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index a029c4c3..f76b88fc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -66,7 +66,7 @@ var ( if rootCmdOpts.otel { go func() { var err error - logrus.WithField("address", rootCmdOpts.metricsAddress).Print("Enable otel endpoint") + logrus.WithField("address", rootCmdOpts.otelAddress).Print("Enable otel endpoint") otelShutdown, err = setupOTelSDK(cmd.Context(), rootCmdOpts.otelAddress) if err != nil { logrus.WithError(err).Warning("Failed to setup OpenTelemetry SDK") From bd4d75f797083a672e5c7f1ad0525a060be6b345 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 22 Jul 2024 10:50:02 +0200 Subject: [PATCH 36/43] counter init and naming --- pkg/kine/drivers/generic/generic.go | 29 +++++++++-------- pkg/kine/logstructured/logstructured.go | 25 ++++++++------ pkg/kine/logstructured/sqllog/sql.go | 20 +++++++----- pkg/kine/server/create.go | 17 +--------- pkg/kine/server/delete.go | 17 +--------- pkg/kine/server/get.go | 18 +---------- pkg/kine/server/limited.go | 6 ++-- pkg/kine/server/list.go | 16 +-------- pkg/kine/server/otel.go | 43 +++++++++++++++++++++++-- pkg/kine/server/update.go | 17 +--------- 10 files changed, 90 insertions(+), 118 deletions(-) diff --git a/pkg/kine/drivers/generic/generic.go b/pkg/kine/drivers/generic/generic.go index 5583c0dd..d997f61d 100644 --- a/pkg/kine/drivers/generic/generic.go +++ b/pkg/kine/drivers/generic/generic.go @@ -16,13 +16,14 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" ) -const name = "generic" +const otelName = "generic" var ( - tracer = otel.Tracer(name) - meter = otel.Meter(name) + otelTracer trace.Tracer + otelMeter metric.Meter setCompactRevCnt metric.Int64Counter getRevisionCnt metric.Int64Counter deleteRevCnt metric.Int64Counter @@ -32,23 +33,25 @@ var ( func init() { var err error - setCompactRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) + otelTracer = otel.Tracer(otelName) + otelMeter = otel.Meter(otelName) + setCompactRevCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.compact", otelName), metric.WithDescription("Number of compact requests")) if err != nil { logrus.WithError(err).Warning("Otel failed to create create counter") } - getRevisionCnt, err = meter.Int64Counter(fmt.Sprintf("%s.get_revision", name), metric.WithDescription("Number of get revision requests")) + getRevisionCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.get_revision", otelName), metric.WithDescription("Number of get revision requests")) if err != nil { logrus.WithError(err).Warning("Otel failed to create create counter") } - deleteRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.delete_revision", name), metric.WithDescription("Number of delete revision requests")) + deleteRevCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.delete_revision", otelName), metric.WithDescription("Number of delete revision requests")) if err != nil { logrus.WithError(err).Warning("Otel failed to create create counter") } - currentRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.current_revision", name), metric.WithDescription("Current revision")) + currentRevCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.current_revision", otelName), metric.WithDescription("Current revision")) if err != nil { logrus.WithError(err).Warning("Otel failed to create create counter") } - getCompactRevCnt, err = meter.Int64Counter(fmt.Sprintf("%s.get_compact_revision", name), metric.WithDescription("Get compact revision")) + getCompactRevCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.get_compact_revision", otelName), metric.WithDescription("Get compact revision")) if err != nil { logrus.WithError(err).Warning("Otel failed to create create counter") } @@ -518,7 +521,7 @@ func (d *Generic) executePrepared(ctx context.Context, txName, sql string, prepa func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) { getCompactRevCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "generic.get_compact_revision") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.get_compact_revision", otelName)) var compact, target sql.NullInt64 start := time.Now() var err error @@ -561,7 +564,7 @@ func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) func (d *Generic) SetCompactRevision(ctx context.Context, revision int64) error { setCompactRevCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "generic.set_compact_revision") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.set_compact_revision", otelName)) defer span.End() span.SetAttributes(attribute.Int64("revision", revision)) @@ -572,7 +575,7 @@ func (d *Generic) SetCompactRevision(ctx context.Context, revision int64) error func (d *Generic) GetRevision(ctx context.Context, revision int64) (*sql.Rows, error) { getRevisionCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "generic.get_revision") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.get_revision", otelName)) defer span.End() span.SetAttributes(attribute.Int64("revision", revision)) @@ -583,7 +586,7 @@ func (d *Generic) GetRevision(ctx context.Context, revision int64) (*sql.Rows, e func (d *Generic) DeleteRevision(ctx context.Context, revision int64) error { deleteRevCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "generic.delete_revision") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.delete_revision", otelName)) defer span.End() span.SetAttributes(attribute.Int64("revision", revision)) @@ -626,7 +629,7 @@ func (d *Generic) List(ctx context.Context, prefix, startKey string, limit, revi func (d *Generic) CurrentRevision(ctx context.Context) (int64, error) { currentRevCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "generic.current_revision") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.current_revision", otelName)) defer span.End() var id int64 var err error diff --git a/pkg/kine/logstructured/logstructured.go b/pkg/kine/logstructured/logstructured.go index 4c2a1cb0..fc65ac37 100644 --- a/pkg/kine/logstructured/logstructured.go +++ b/pkg/kine/logstructured/logstructured.go @@ -11,14 +11,19 @@ import ( "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) -const name = "logstructured" +const otelName = "logstructured" var ( - tracer = otel.Tracer(name) + otelTracer trace.Tracer ) +func init() { + otelTracer = otel.Tracer(otelName) +} + type Log interface { Start(ctx context.Context) error Wait() @@ -66,7 +71,7 @@ func (l *LogStructured) Wait() { } func (l *LogStructured) Get(ctx context.Context, key, rangeEnd string, limit, revision int64) (revRet int64, kvRet *server.KeyValue, errRet error) { - ctx, span := tracer.Start(ctx, "logstructured.Get") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Get", otelName)) span.SetAttributes( attribute.String("key", key), attribute.String("rangeEnd", rangeEnd), @@ -89,7 +94,7 @@ func (l *LogStructured) Get(ctx context.Context, key, rangeEnd string, limit, re } func (l *LogStructured) get(ctx context.Context, key, rangeEnd string, limit, revision int64, includeDeletes bool) (int64, *server.Event, error) { - ctx, span := tracer.Start(ctx, "logstructured.get") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.get", otelName)) defer span.End() span.SetAttributes( attribute.String("key", key), @@ -128,7 +133,7 @@ func (l *LogStructured) adjustRevision(ctx context.Context, rev *int64) { } func (l *LogStructured) Create(ctx context.Context, key string, value []byte, lease int64) (revRet int64, errRet error) { - ctx, span := tracer.Start(ctx, "logstructured.Create") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Create", otelName)) span.SetAttributes( attribute.String("key", key), attribute.Int64("lease", lease), @@ -170,7 +175,7 @@ func (l *LogStructured) Create(ctx context.Context, key string, value []byte, le } func (l *LogStructured) Delete(ctx context.Context, key string, revision int64) (revRet int64, kvRet *server.KeyValue, deletedRet bool, errRet error) { - ctx, span := tracer.Start(ctx, "logstructured.Delete") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Delete", otelName)) defer func() { l.adjustRevision(ctx, &revRet) @@ -228,7 +233,7 @@ func (l *LogStructured) Delete(ctx context.Context, key string, revision int64) } func (l *LogStructured) List(ctx context.Context, prefix, startKey string, limit, revision int64) (revRet int64, kvRet []*server.KeyValue, errRet error) { - ctx, span := tracer.Start(ctx, "logstructured.List") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.List", otelName)) defer func() { logrus.Debugf("LIST %s, start=%s, limit=%d, rev=%d => rev=%d, kvs=%d, err=%v", prefix, startKey, limit, revision, revRet, len(kvRet), errRet) @@ -270,7 +275,7 @@ func (l *LogStructured) List(ctx context.Context, prefix, startKey string, limit } func (l *LogStructured) Count(ctx context.Context, prefix, startKey string, revision int64) (revRet int64, count int64, err error) { - ctx, span := tracer.Start(ctx, "logstructured.Count") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Count", otelName)) defer func() { logrus.Debugf("COUNT prefix=%s startKey=%s => rev=%d, count=%d, err=%v", prefix, startKey, revRet, count, err) span.SetAttributes( @@ -301,7 +306,7 @@ func (l *LogStructured) Count(ctx context.Context, prefix, startKey string, revi } func (l *LogStructured) Update(ctx context.Context, key string, value []byte, revision, lease int64) (revRet int64, kvRet *server.KeyValue, updateRet bool, errRet error) { - ctx, span := tracer.Start(ctx, "logstructured.Update") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Update", otelName)) defer func() { l.adjustRevision(ctx, &revRet) @@ -425,7 +430,7 @@ func (l *LogStructured) ttl(ctx context.Context) { func (l *LogStructured) Watch(ctx context.Context, prefix string, revision int64) <-chan []*server.Event { logrus.Debugf("WATCH %s, revision=%d", prefix, revision) - ctx, span := tracer.Start(ctx, "logstructured.Watch") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Watch", otelName)) defer span.End() span.SetAttributes( attribute.String("prefix", prefix), diff --git a/pkg/kine/logstructured/sqllog/sql.go b/pkg/kine/logstructured/sqllog/sql.go index 125fb7e5..5d8bce09 100644 --- a/pkg/kine/logstructured/sqllog/sql.go +++ b/pkg/kine/logstructured/sqllog/sql.go @@ -14,25 +14,29 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" ) -const SupersededCount = 100 - -const name = "sqllog" +const ( + SupersededCount = 100 + otelName = "sqllog" +) var ( - tracer = otel.Tracer(name) - meter = otel.Meter(name) + otelTracer trace.Tracer + otelMeter metric.Meter compactCnt metric.Int64Counter ) func init() { var err error - compactCnt, err = meter.Int64Counter(fmt.Sprintf("%s.compact", name), metric.WithDescription("Number of compact requests")) + otelTracer = otel.Tracer(otelName) + otelMeter = otel.Meter(otelName) + + compactCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.compact", otelName), metric.WithDescription("Number of compact requests")) if err != nil { logrus.WithError(err).Warning("Otel failed to create create counter") } - } type SQLLog struct { @@ -144,7 +148,7 @@ func (s *SQLLog) DoCompact(ctx context.Context) error { } func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { - ctx, span := tracer.Start(ctx, "sqllog.compactor") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.compactor", otelName)) defer span.End() span.SetAttributes(attribute.Int64("nextEnd", nextEnd)) diff --git a/pkg/kine/server/create.go b/pkg/kine/server/create.go index 8c811f3c..cc3bf31b 100644 --- a/pkg/kine/server/create.go +++ b/pkg/kine/server/create.go @@ -4,25 +4,10 @@ import ( "context" "fmt" - "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric" ) -var ( - createCnt metric.Int64Counter -) - -func init() { - var err error - - createCnt, err = meter.Int64Counter(fmt.Sprintf("%s.create", name), metric.WithDescription("Number of create requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create create counter") - } -} - func isCreate(txn *etcdserverpb.TxnRequest) *etcdserverpb.PutRequest { if len(txn.Compare) == 1 && txn.Compare[0].Target == etcdserverpb.Compare_MOD && @@ -38,7 +23,7 @@ func isCreate(txn *etcdserverpb.TxnRequest) *etcdserverpb.PutRequest { func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest, txn *etcdserverpb.TxnRequest) (*etcdserverpb.TxnResponse, error) { createCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "limited.create") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.create", otelName)) defer span.End() span.SetAttributes( attribute.String("key", string(put.Key)), diff --git a/pkg/kine/server/delete.go b/pkg/kine/server/delete.go index 98b8537e..8da9e515 100644 --- a/pkg/kine/server/delete.go +++ b/pkg/kine/server/delete.go @@ -4,25 +4,10 @@ import ( "context" "fmt" - "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric" ) -var ( - deleteCnt metric.Int64Counter -) - -func init() { - var err error - - deleteCnt, err = meter.Int64Counter(fmt.Sprintf("%s.delete", name), metric.WithDescription("Number of delete requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create delete counter") - } -} - func isDelete(txn *etcdserverpb.TxnRequest) (int64, string, bool) { if len(txn.Compare) == 0 && len(txn.Failure) == 0 && @@ -46,7 +31,7 @@ func isDelete(txn *etcdserverpb.TxnRequest) (int64, string, bool) { func (l *LimitedServer) delete(ctx context.Context, key string, revision int64) (*etcdserverpb.TxnResponse, error) { deleteCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "limited.delete") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.delete", otelName)) defer span.End() span.SetAttributes( attribute.String("key", key), diff --git a/pkg/kine/server/get.go b/pkg/kine/server/get.go index 74edc6e1..9089eddb 100644 --- a/pkg/kine/server/get.go +++ b/pkg/kine/server/get.go @@ -4,27 +4,13 @@ import ( "context" "fmt" - "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric" ) -var ( - getCnt metric.Int64Counter -) - -func init() { - var err error - getCnt, err = meter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create get counter") - } -} - func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { getCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "limited.get") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.get", otelName)) defer span.End() span.SetAttributes( @@ -45,8 +31,6 @@ func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) ( return nil, err } - span.End() - resp := &RangeResponse{ Header: txnHeader(rev), } diff --git a/pkg/kine/server/limited.go b/pkg/kine/server/limited.go index 61e0f298..476f28fa 100644 --- a/pkg/kine/server/limited.go +++ b/pkg/kine/server/limited.go @@ -12,13 +12,11 @@ type LimitedServer struct { } func (l *LimitedServer) Range(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Range", otelName)) + defer span.End() if len(r.RangeEnd) == 0 { - ctx, span := tracer.Start(ctx, "range.get") - defer span.End() return l.get(ctx, r) } - ctx, span := tracer.Start(ctx, "range.list") - defer span.End() return l.list(ctx, r) } diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index 032b5e7e..c3b4dc50 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -9,25 +9,11 @@ import ( "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric" ) -var ( - listCnt metric.Int64Counter -) - -func init() { - var err error - - listCnt, err = meter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create list counter") - } -} - func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { listCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "limited.list") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.list", otelName)) defer span.End() span.SetAttributes( diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go index 44b29f99..3b1becef 100644 --- a/pkg/kine/server/otel.go +++ b/pkg/kine/server/otel.go @@ -1,12 +1,49 @@ package server import ( + "fmt" + + "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" ) -const name = "limited-server" +const otelName = "limited-server" var ( - tracer = otel.Tracer(name) - meter = otel.Meter(name) + otelTracer trace.Tracer + otelMeter metric.Meter + createCnt metric.Int64Counter + deleteCnt metric.Int64Counter + getCnt metric.Int64Counter + listCnt metric.Int64Counter + updateCnt metric.Int64Counter ) + +func init() { + var err error + otelTracer = otel.Tracer(otelName) + otelMeter = otel.Meter(otelName) + + createCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.create", name), metric.WithDescription("Number of create requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create create counter") + } + deleteCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.delete", name), metric.WithDescription("Number of delete requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create delete counter") + } + getCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create get counter") + } + listCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create list counter") + } + updateCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.update", name), metric.WithDescription("Number of update requests")) + if err != nil { + logrus.WithError(err).Warning("Otel failed to create update counter") + } +} diff --git a/pkg/kine/server/update.go b/pkg/kine/server/update.go index f0979dbe..87cffc8b 100644 --- a/pkg/kine/server/update.go +++ b/pkg/kine/server/update.go @@ -4,25 +4,10 @@ import ( "context" "fmt" - "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric" ) -var ( - updateCnt metric.Int64Counter -) - -func init() { - var err error - - updateCnt, err = meter.Int64Counter(fmt.Sprintf("%s.update", name), metric.WithDescription("Number of update requests")) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create update counter") - } -} - func isUpdate(txn *etcdserverpb.TxnRequest) (int64, string, []byte, int64, bool) { if len(txn.Compare) == 1 && txn.Compare[0].Target == etcdserverpb.Compare_MOD && @@ -48,7 +33,7 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value ) updateCnt.Add(ctx, 1) - ctx, span := tracer.Start(ctx, "limited.update") + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.update", otelName)) defer span.End() span.SetAttributes( attribute.String("key", key), From fcdae605b3013b779b9758b40a6b57892cce05f3 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 22 Jul 2024 10:50:56 +0200 Subject: [PATCH 37/43] more renaming --- pkg/kine/server/otel.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/kine/server/otel.go b/pkg/kine/server/otel.go index 3b1becef..9737a3fb 100644 --- a/pkg/kine/server/otel.go +++ b/pkg/kine/server/otel.go @@ -26,23 +26,23 @@ func init() { otelTracer = otel.Tracer(otelName) otelMeter = otel.Meter(otelName) - createCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.create", name), metric.WithDescription("Number of create requests")) + createCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.create", otelName), metric.WithDescription("Number of create requests")) if err != nil { logrus.WithError(err).Warning("Otel failed to create create counter") } - deleteCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.delete", name), metric.WithDescription("Number of delete requests")) + deleteCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.delete", otelName), metric.WithDescription("Number of delete requests")) if err != nil { logrus.WithError(err).Warning("Otel failed to create delete counter") } - getCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.get", name), metric.WithDescription("Number of get requests")) + getCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.get", otelName), metric.WithDescription("Number of get requests")) if err != nil { logrus.WithError(err).Warning("Otel failed to create get counter") } - listCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.list", name), metric.WithDescription("Number of list requests")) + listCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.list", otelName), metric.WithDescription("Number of list requests")) if err != nil { logrus.WithError(err).Warning("Otel failed to create list counter") } - updateCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.update", name), metric.WithDescription("Number of update requests")) + updateCnt, err = otelMeter.Int64Counter(fmt.Sprintf("%s.update", otelName), metric.WithDescription("Number of update requests")) if err != nil { logrus.WithError(err).Warning("Otel failed to create update counter") } From da1622a9b859a638234fc098c40e249f958b719f Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 22 Jul 2024 11:27:02 +0200 Subject: [PATCH 38/43] comments --- cmd/root.go | 18 ++++++++---------- cmd/tracing.go | 32 ++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index f76b88fc..152a1cce 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -27,9 +27,9 @@ var ( diskMode bool clientSessionCacheSize uint minTLSVersion string - otel bool metrics bool metricsAddress string + otel bool otelAddress string watchAvailableStorageInterval time.Duration @@ -64,14 +64,12 @@ var ( var otelShutdown func(context.Context) error if rootCmdOpts.otel { - go func() { - var err error - logrus.WithField("address", rootCmdOpts.otelAddress).Print("Enable otel endpoint") - otelShutdown, err = setupOTelSDK(cmd.Context(), rootCmdOpts.otelAddress) - if err != nil { - logrus.WithError(err).Warning("Failed to setup OpenTelemetry SDK") - } - }() + var err error + logrus.WithField("address", rootCmdOpts.otelAddress).Print("Enable otel endpoint") + otelShutdown, err = setupOTelSDK(cmd.Context(), rootCmdOpts.otelAddress) + if err != nil { + logrus.WithError(err).Warning("Failed to setup OpenTelemetry SDK") + } } if rootCmdOpts.metrics { @@ -126,7 +124,7 @@ var ( if err := instance.Shutdown(stopCtx); err != nil { logrus.WithError(err).Fatal("Failed to shutdown server") } - if rootCmdOpts.metrics && otelShutdown != nil { + if rootCmdOpts.otel && otelShutdown != nil { err = errors.Join(err, otelShutdown(stopCtx)) if err != nil { logrus.WithError(err).Warning("Failed to shutdown OpenTelemetry SDK") diff --git a/cmd/tracing.go b/cmd/tracing.go index 90207365..0578ce79 100644 --- a/cmd/tracing.go +++ b/cmd/tracing.go @@ -27,7 +27,6 @@ var ( func setupOTelSDK(ctx context.Context, otelEndpoint string) (shutdown func(context.Context) error, err error) { conn, err := initConn(otelEndpoint) if err != nil { - logrus.WithError(err).Warning("Otel failed to create gRPC connection") return nil, fmt.Errorf("failed to create gRPC connection to collector: %w", err) } @@ -42,22 +41,27 @@ func setupOTelSDK(ctx context.Context, otelEndpoint string) (shutdown func(conte traceExporter, err := newTraceExporter(ctx, conn) if err != nil { - logrus.WithError(err).Warning("Otel failed to create trace exporter") - return nil, err + return nil, fmt.Errorf("failed to create trace exporter: %w", err) } - tracerProvider, err := newTraceProvider(traceExporter, res) - if err != nil { - logrus.WithError(err).Warning("Otel failed to create trace provider") - return nil, err - } + tracerProvider := newTraceProvider(traceExporter, res) otel.SetTracerProvider(tracerProvider) meterProvider, err := newMeterProvider(res) if err != nil { - tracerProvider.Shutdown(ctx) - logrus.WithError(err).Warning("Otel failed to create meter provider") - return nil, err + var shutdownErrs error + shutdownErr := tracerProvider.Shutdown(ctx) + if shutdownErr != nil { + shutdownErrs = errors.Join(shutdownErrs, shutdownErr) + } + shutdownErr = conn.Close() + if shutdownErr != nil { + shutdownErrs = errors.Join(shutdownErrs, shutdownErr) + } + if shutdownErrs != nil { + logrus.WithError(shutdownErrs).Warning("Failed to shutdown OpenTelemetry SDK") + return nil, fmt.Errorf("failed to create meter provider: %w", err) + } } otel.SetMeterProvider(meterProvider) @@ -77,7 +81,7 @@ func setupOTelSDK(ctx context.Context, otelEndpoint string) (shutdown func(conte } return shutdownErrs } - return + return shutdown, nil } func initConn(otelEndpoint string) (*grpc.ClientConn, error) { @@ -101,14 +105,14 @@ func newTraceExporter(ctx context.Context, conn *grpc.ClientConn) (trace.SpanExp return exporter, nil } -func newTraceProvider(traceExporter trace.SpanExporter, res *resource.Resource) (*trace.TracerProvider, error) { +func newTraceProvider(traceExporter trace.SpanExporter, res *resource.Resource) *trace.TracerProvider { traceProvider := trace.NewTracerProvider( trace.WithBatcher(traceExporter, trace.WithBatchTimeout(time.Second), ), trace.WithResource(res), ) - return traceProvider, nil + return traceProvider } func newMeterProvider(res *resource.Resource) (*metric.MeterProvider, error) { From 90a04ecfe4c1e9be46a35faa9a1f423bde17b12d Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 22 Jul 2024 11:45:09 +0200 Subject: [PATCH 39/43] shutdown conn --- cmd/tracing.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/tracing.go b/cmd/tracing.go index 0578ce79..b6eabb82 100644 --- a/cmd/tracing.go +++ b/cmd/tracing.go @@ -41,6 +41,10 @@ func setupOTelSDK(ctx context.Context, otelEndpoint string) (shutdown func(conte traceExporter, err := newTraceExporter(ctx, conn) if err != nil { + connErr := conn.Close() + if connErr != nil { + logrus.WithError(connErr).Warning("Failed to shut down otel gRPC connection") + } return nil, fmt.Errorf("failed to create trace exporter: %w", err) } From 2b1887c47a7b641091d1c8a87e27b168e4caba6b Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 22 Jul 2024 11:48:16 +0200 Subject: [PATCH 40/43] handling errs --- pkg/kine/drivers/generic/generic.go | 39 ++++++++++++++++--------- pkg/kine/logstructured/logstructured.go | 7 +++-- pkg/kine/logstructured/sqllog/sql.go | 15 ++++------ pkg/kine/server/create.go | 7 +++-- pkg/kine/server/delete.go | 7 +++-- pkg/kine/server/get.go | 8 +++-- pkg/kine/server/list.go | 9 +++--- pkg/kine/server/update.go | 6 ++-- 8 files changed, 59 insertions(+), 39 deletions(-) diff --git a/pkg/kine/drivers/generic/generic.go b/pkg/kine/drivers/generic/generic.go index d997f61d..f43850fc 100644 --- a/pkg/kine/drivers/generic/generic.go +++ b/pkg/kine/drivers/generic/generic.go @@ -526,10 +526,10 @@ func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) start := time.Now() var err error defer func() { - span.RecordError(err) if err == sql.ErrNoRows { err = nil } + span.RecordError(err) recordOpResult("revision_interval_sql", err, start) recordTxResult("revision_interval_sql", err) span.End() @@ -563,35 +563,44 @@ func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) } func (d *Generic) SetCompactRevision(ctx context.Context, revision int64) error { + var err error setCompactRevCnt.Add(ctx, 1) ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.set_compact_revision", otelName)) - defer span.End() + defer func() { + span.RecordError(err) + span.End() + }() span.SetAttributes(attribute.Int64("revision", revision)) - _, err := d.executePrepared(ctx, "update_compact_sql", d.UpdateCompactSQL, d.updateCompactSQLPrepared, revision) - span.RecordError(err) + _, err = d.executePrepared(ctx, "update_compact_sql", d.UpdateCompactSQL, d.updateCompactSQLPrepared, revision) return err } func (d *Generic) GetRevision(ctx context.Context, revision int64) (*sql.Rows, error) { + var err error getRevisionCnt.Add(ctx, 1) ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.get_revision", otelName)) - defer span.End() + defer func() { + span.RecordError(err) + span.End() + }() span.SetAttributes(attribute.Int64("revision", revision)) result, err := d.queryPrepared(ctx, "get_revision_sql", d.GetRevisionSQL, d.getRevisionSQLPrepared, revision) - span.RecordError(err) return result, err } func (d *Generic) DeleteRevision(ctx context.Context, revision int64) error { + var err error deleteRevCnt.Add(ctx, 1) ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.delete_revision", otelName)) - defer span.End() + defer func() { + span.RecordError(err) + span.End() + }() span.SetAttributes(attribute.Int64("revision", revision)) - _, err := d.executePrepared(ctx, "delete_sql", d.DeleteSQL, d.deleteSQLPrepared, revision) - span.RecordError(err) + _, err = d.executePrepared(ctx, "delete_sql", d.DeleteSQL, d.deleteSQLPrepared, revision) return err } @@ -628,22 +637,24 @@ func (d *Generic) List(ctx context.Context, prefix, startKey string, limit, revi } func (d *Generic) CurrentRevision(ctx context.Context) (int64, error) { - currentRevCnt.Add(ctx, 1) - ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.current_revision", otelName)) - defer span.End() var id int64 var err error + currentRevCnt.Add(ctx, 1) + ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.current_revision", otelName)) + defer func() { + span.RecordError(err) + span.End() + }() + done, err := d.AdmissionControlPolicy.Admit(ctx, "rev_sql") if err != nil { - span.RecordError(err) return 0, fmt.Errorf("denied: %w", err) } row := d.queryRow(ctx, "rev_sql", revSQL) done() err = row.Scan(&id) - span.RecordError(err) if err == sql.ErrNoRows { span.AddEvent("no rows") return 0, nil diff --git a/pkg/kine/logstructured/logstructured.go b/pkg/kine/logstructured/logstructured.go index fc65ac37..11dc4c8b 100644 --- a/pkg/kine/logstructured/logstructured.go +++ b/pkg/kine/logstructured/logstructured.go @@ -94,8 +94,12 @@ func (l *LogStructured) Get(ctx context.Context, key, rangeEnd string, limit, re } func (l *LogStructured) get(ctx context.Context, key, rangeEnd string, limit, revision int64, includeDeletes bool) (int64, *server.Event, error) { + var err error ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.get", otelName)) - defer span.End() + defer func() { + span.RecordError(err) + span.End() + }() span.SetAttributes( attribute.String("key", key), attribute.String("rangeEnd", rangeEnd), @@ -104,7 +108,6 @@ func (l *LogStructured) get(ctx context.Context, key, rangeEnd string, limit, re attribute.Bool("includeDeletes", includeDeletes), ) rev, events, err := l.log.List(ctx, key, rangeEnd, limit, revision, includeDeletes) - span.RecordError(err) if err == server.ErrCompacted { span.AddEvent("ignoring compacted error") // ignore compacted when getting by revision diff --git a/pkg/kine/logstructured/sqllog/sql.go b/pkg/kine/logstructured/sqllog/sql.go index 5d8bce09..678fc0ca 100644 --- a/pkg/kine/logstructured/sqllog/sql.go +++ b/pkg/kine/logstructured/sqllog/sql.go @@ -148,21 +148,23 @@ func (s *SQLLog) DoCompact(ctx context.Context) error { } func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { + var err error ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.compactor", otelName)) - defer span.End() + defer func() { + span.RecordError(err) + span.End() + }() span.SetAttributes(attribute.Int64("nextEnd", nextEnd)) currentRev, err := s.d.CurrentRevision(ctx) span.SetAttributes(attribute.Int64("currentRev", currentRev)) if err != nil { - span.RecordError(err) logrus.Errorf("failed to get current revision: %v", err) return nextEnd, fmt.Errorf("failed to get current revision: %v", err) } cursor, _, err := s.d.GetCompactRevision(ctx) if err != nil { - span.RecordError(err) logrus.Errorf("failed to get compact revision: %v", err) return nextEnd, fmt.Errorf("failed to get compact revision: %v", err) } @@ -185,14 +187,12 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { for ; cursor <= end; cursor++ { rows, err := s.d.GetRevision(ctx, cursor) if err != nil { - span.RecordError(err) logrus.Errorf("failed to get revision %d: %v", cursor, err) return nextEnd, fmt.Errorf("failed to get revision %d: %v", cursor, err) } events, err := RowsToEvents(rows) if err != nil { - span.RecordError(err) logrus.Errorf("failed to convert to events: %v", err) return nextEnd, fmt.Errorf("failed to convert to events: %v", err) } @@ -213,7 +213,6 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { if event.PrevKV != nil && event.PrevKV.ModRevision != 0 { if savedCursor != cursor { if err := s.d.SetCompactRevision(ctx, cursor); err != nil { - span.RecordError(err) span.AddEvent(fmt.Sprintf("failed to record compact revision: %v", err)) logrus.Errorf("failed to record compact revision: %v", err) return nextEnd, fmt.Errorf("failed to record compact revision: %v", err) @@ -223,7 +222,6 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { } if err := s.d.DeleteRevision(ctx, event.PrevKV.ModRevision); err != nil { - span.RecordError(err) span.AddEvent(fmt.Sprintf("failed to delete revision %d", event.PrevKV.ModRevision)) logrus.Errorf("failed to delete revision %d: %v", event.PrevKV.ModRevision, err) return nextEnd, fmt.Errorf("failed to delete revision %d: %v", event.PrevKV.ModRevision, err) @@ -233,7 +231,6 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { if event.Delete { if !setRev && savedCursor != cursor { if err := s.d.SetCompactRevision(ctx, cursor); err != nil { - span.RecordError(err) logrus.Errorf("failed to record compact revision: %v", err) return nextEnd, fmt.Errorf("failed to record compact revision: %v", err) } @@ -241,7 +238,6 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { } if err := s.d.DeleteRevision(ctx, cursor); err != nil { - span.RecordError(err) logrus.Errorf("failed to delete current revision %d: %v", cursor, err) return nextEnd, fmt.Errorf("failed to delete current revision %d: %v", cursor, err) } @@ -250,7 +246,6 @@ func (s *SQLLog) compactor(ctx context.Context, nextEnd int64) (int64, error) { if savedCursor != cursor { if err := s.d.SetCompactRevision(ctx, cursor); err != nil { - span.RecordError(err) logrus.Errorf("failed to record compact revision: %v", err) return nextEnd, fmt.Errorf("failed to record compact revision: %v", err) } diff --git a/pkg/kine/server/create.go b/pkg/kine/server/create.go index cc3bf31b..2779a0d3 100644 --- a/pkg/kine/server/create.go +++ b/pkg/kine/server/create.go @@ -22,9 +22,13 @@ func isCreate(txn *etcdserverpb.TxnRequest) *etcdserverpb.PutRequest { } func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest, txn *etcdserverpb.TxnRequest) (*etcdserverpb.TxnResponse, error) { + var err error createCnt.Add(ctx, 1) ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.create", otelName)) - defer span.End() + defer func() { + span.RecordError(err) + span.End() + }() span.SetAttributes( attribute.String("key", string(put.Key)), attribute.Int64("lease", put.Lease), @@ -47,7 +51,6 @@ func (l *LimitedServer) create(ctx context.Context, put *etcdserverpb.PutRequest Succeeded: false, }, nil } else if err != nil { - defer span.RecordError(err) return nil, err } diff --git a/pkg/kine/server/delete.go b/pkg/kine/server/delete.go index 8da9e515..0a5e63b3 100644 --- a/pkg/kine/server/delete.go +++ b/pkg/kine/server/delete.go @@ -30,9 +30,13 @@ func isDelete(txn *etcdserverpb.TxnRequest) (int64, string, bool) { } func (l *LimitedServer) delete(ctx context.Context, key string, revision int64) (*etcdserverpb.TxnResponse, error) { + var err error deleteCnt.Add(ctx, 1) ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.delete", otelName)) - defer span.End() + defer func() { + span.RecordError(err) + span.End() + }() span.SetAttributes( attribute.String("key", key), attribute.Int64("revision", revision), @@ -40,7 +44,6 @@ func (l *LimitedServer) delete(ctx context.Context, key string, revision int64) rev, kv, ok, err := l.backend.Delete(ctx, key, revision) if err != nil { - span.RecordError(err) return nil, err } span.SetAttributes(attribute.Bool("ok", ok)) diff --git a/pkg/kine/server/get.go b/pkg/kine/server/get.go index 9089eddb..2f1ce6d5 100644 --- a/pkg/kine/server/get.go +++ b/pkg/kine/server/get.go @@ -9,9 +9,13 @@ import ( ) func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { + var err error getCnt.Add(ctx, 1) ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.get", otelName)) - defer span.End() + defer func() { + span.RecordError(err) + span.End() + }() span.SetAttributes( attribute.String("key", string(r.Key)), @@ -21,13 +25,11 @@ func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) ( ) if r.Limit != 0 && len(r.RangeEnd) != 0 { err := fmt.Errorf("invalid combination of rangeEnd and limit, limit should be 0 got %d", r.Limit) - span.RecordError(err) return nil, err } rev, kv, err := l.backend.Get(ctx, string(r.Key), string(r.RangeEnd), r.Limit, r.Revision) if err != nil { - span.RecordError(err) return nil, err } diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index c3b4dc50..8557cbcb 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -12,9 +12,13 @@ import ( ) func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) (*RangeResponse, error) { + var err error listCnt.Add(ctx, 1) ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.list", otelName)) - defer span.End() + defer func() { + span.RecordError(err) + span.End() + }() span.SetAttributes( attribute.String("key", string(r.Key)), @@ -39,7 +43,6 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) if r.CountOnly { rev, count, err := l.backend.Count(ctx, prefix, start, revision) if err != nil { - span.RecordError(err) return nil, err } span.SetAttributes(attribute.Int64("count", count)) @@ -59,7 +62,6 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) rev, kvs, err := l.backend.List(ctx, prefix, start, limit, revision) if err != nil { - span.RecordError(err) return nil, err } @@ -81,7 +83,6 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) // count the actual number of results if there are more items in the db. rev, resp.Count, err = l.backend.Count(ctx, prefix, start, revision) if err != nil { - span.RecordError(err) return nil, err } span.SetAttributes(attribute.Int64("count", resp.Count)) diff --git a/pkg/kine/server/update.go b/pkg/kine/server/update.go index 87cffc8b..94acfd49 100644 --- a/pkg/kine/server/update.go +++ b/pkg/kine/server/update.go @@ -34,7 +34,10 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value updateCnt.Add(ctx, 1) ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.update", otelName)) - defer span.End() + defer func() { + span.RecordError(err) + span.End() + }() span.SetAttributes( attribute.String("key", key), attribute.Int64("lease", lease), @@ -54,7 +57,6 @@ func (l *LimitedServer) update(ctx context.Context, rev int64, key string, value span.SetAttributes(attribute.Bool("ok", ok)) } if err != nil { - span.RecordError(err) return nil, err } From 7dbfb7418d75ca42b9690cb8a044fd1040c76db7 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 22 Jul 2024 12:05:01 +0200 Subject: [PATCH 41/43] adjust ignore compacted --- pkg/kine/logstructured/logstructured.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kine/logstructured/logstructured.go b/pkg/kine/logstructured/logstructured.go index 11dc4c8b..6dd53754 100644 --- a/pkg/kine/logstructured/logstructured.go +++ b/pkg/kine/logstructured/logstructured.go @@ -109,7 +109,7 @@ func (l *LogStructured) get(ctx context.Context, key, rangeEnd string, limit, re ) rev, events, err := l.log.List(ctx, key, rangeEnd, limit, revision, includeDeletes) if err == server.ErrCompacted { - span.AddEvent("ignoring compacted error") + span.AddEvent("key already compacted") // ignore compacted when getting by revision err = nil } From b9c235eda8910b6a469d1fde5a9b608640b3b62b Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 22 Jul 2024 12:27:00 +0200 Subject: [PATCH 42/43] spaces --- pkg/kine/logstructured/logstructured.go | 1 - pkg/kine/server/list.go | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kine/logstructured/logstructured.go b/pkg/kine/logstructured/logstructured.go index 6dd53754..0f6ff105 100644 --- a/pkg/kine/logstructured/logstructured.go +++ b/pkg/kine/logstructured/logstructured.go @@ -310,7 +310,6 @@ func (l *LogStructured) Count(ctx context.Context, prefix, startKey string, revi func (l *LogStructured) Update(ctx context.Context, key string, value []byte, revision, lease int64) (revRet int64, kvRet *server.KeyValue, updateRet bool, errRet error) { ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.Update", otelName)) - defer func() { l.adjustRevision(ctx, &revRet) kvRev := int64(0) diff --git a/pkg/kine/server/list.go b/pkg/kine/server/list.go index 8557cbcb..2d7f770d 100644 --- a/pkg/kine/server/list.go +++ b/pkg/kine/server/list.go @@ -80,13 +80,14 @@ func (l *LimitedServer) list(ctx context.Context, r *etcdserverpb.RangeRequest) if revision == 0 { revision = rev } + // count the actual number of results if there are more items in the db. rev, resp.Count, err = l.backend.Count(ctx, prefix, start, revision) if err != nil { return nil, err } - span.SetAttributes(attribute.Int64("count", resp.Count)) + span.SetAttributes(attribute.Int64("count", resp.Count)) logrus.Tracef("LIST COUNT key=%s, end=%s, revision=%d, currentRev=%d count=%d", r.Key, r.RangeEnd, revision, rev, resp.Count) resp.Header = txnHeader(rev) } From 9023e1ee739a197bc8fdbb4733ec3f75545da4ab Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 22 Jul 2024 13:50:41 +0200 Subject: [PATCH 43/43] tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 0c166c9a..37acc898 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( go.opentelemetry.io/otel/metric v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 + go.opentelemetry.io/otel/trace v1.28.0 golang.org/x/sync v0.7.0 golang.org/x/sys v0.21.0 google.golang.org/grpc v1.65.0 @@ -66,7 +67,6 @@ require ( go.etcd.io/etcd/raft/v3 v3.5.12 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect