reduce writes to the operations collection

rockset · Mar 2, 2024 · 1f2fe24 · 1f2fe24
1 parent 6c52b5d
commit 1f2fe24
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 6 deletions.
diff --git a/go.mod b/go.mod
@@ -4,6 +4,7 @@ go 1.21
 
 require (
 	github.com/hashicorp/go-hclog v1.6.1
+	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/jaegertracing/jaeger v1.53.0
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/rockset/rockset-go-client v0.23.0

diff --git a/go.sum b/go.sum
@@ -138,6 +138,8 @@ github.com/hashicorp/go-plugin v1.6.0 h1:wgd4KxHJTVGGqWBq4QPB1i5BZNEx9BR8+OFmHDm
 github.com/hashicorp/go-plugin v1.6.0/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE=

diff --git a/storage/spanstore/store.go b/storage/spanstore/store.go
@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/golang-lru/v2/expirable"
 	"github.com/jaegertracing/jaeger/model"
 	"github.com/jaegertracing/jaeger/storage/spanstore"
 	"github.com/opentracing/opentracing-go"
@@ -61,6 +62,7 @@ type Store struct {
 	writer  *writer.Writer
 	config  Config
 	counter int
+	cache   *expirable.LRU[string, Operation]
 }
 
 func New(logger hclog.Logger, rc *rockset.RockClient, config Config) (*Store, error) {
@@ -81,6 +83,7 @@ func New(logger hclog.Logger, rc *rockset.RockClient, config Config) (*Store, er
 		rc:     rc,
 		writer: w,
 		config: config,
+		cache:  expirable.NewLRU[string, Operation](100, nil, 5*time.Minute),
 	}, nil
 }
 

diff --git a/storage/spanstore/writer.go b/storage/spanstore/writer.go
@@ -59,22 +59,31 @@ func (s Store) WriteSpan(_ context.Context, span *model.Span) error {
 	}
 
 	// TODO we should batch these updates, to reduce the number of writes
+	// cache the id to avoid repeatedly updating the operations collection with the same information
+	id := span.Process.ServiceName + ":" + span.OperationName
+	if s.cache.Contains(id) {
+		return nil
+	}
+
 	kind := unspecified
 	for _, tag := range span.Tags {
 		if tag.Key == "span.kind" {
 			kind = tag.VStr
 		}
 	}
 
+	op := Operation{
+		ID:        id,
+		Service:   span.Process.ServiceName,
+		Operation: span.OperationName,
+		Kind:      kind,
+	}
+	s.cache.Add(id, op)
+
 	s.writer.C() <- writer.Request{
 		Workspace:  s.config.Workspace,
 		Collection: s.config.Operations,
-		Data: Operation{
-			ID:        span.Process.ServiceName + ":" + span.OperationName,
-			Service:   span.Process.ServiceName,
-			Operation: span.OperationName,
-			Kind:      kind,
-		},
+		Data:       op,
 	}
 
 	return nil