Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cache parsed file descriptors #1128

Merged
merged 18 commits into from
Jan 23, 2024
Merged

Cache parsed file descriptors #1128

Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
14f6056
chore: update repo semaphore config
ConfluentJenkins Dec 16, 2022
472de03
chore: update repo semaphore config
ConfluentJenkins Dec 16, 2022
497d60a
chore: update repo semaphore config
ConfluentJenkins Dec 16, 2022
0efc0ea
chore: update repo semaphore config
ConfluentJenkins Dec 16, 2022
7f39e3c
chore: update repo semaphore config
ConfluentJenkins Dec 17, 2022
98c126a
chore: update repo semaphore config
ConfluentJenkins Dec 17, 2022
d72cb17
chore: update repo semaphore config
ConfluentJenkins Dec 17, 2022
2598709
chore: update repo semaphore config
ConfluentJenkins Dec 17, 2022
049146d
chore: update repo semaphore config
ConfluentJenkins Dec 18, 2022
cad18fa
chore: update repo semaphore config
ConfluentJenkins Dec 18, 2022
3e3238e
chore: update repo semaphore config
ConfluentJenkins Dec 18, 2022
c30b15a
chore: update repo semaphore config
ConfluentJenkins Dec 18, 2022
bb7ce71
chore: update repo semaphore config
ConfluentJenkins Dec 19, 2022
51b08dd
chore: update repo semaphore config
ConfluentJenkins Dec 19, 2022
c1cd76a
chore: update repo semaphore config
ConfluentJenkins Dec 19, 2022
361d40e
chore: update repo semaphore config
ConfluentJenkins Dec 19, 2022
566c808
Cache parsed file descriptors
rayokota Jan 18, 2024
092ba5b
Clean up import
rayokota Jan 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 24 additions & 4 deletions schemaregistry/serde/protobuf/protobuf.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@ import (
"io"
"log"
"strings"
"sync"

"github.com/confluentinc/confluent-kafka-go/v2/schemaregistry"
"github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/cache"
"github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/confluent"
"github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/confluent/types"
"github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde"
Expand Down Expand Up @@ -69,7 +71,9 @@ type Serializer struct {
// Deserializer represents a Protobuf deserializer
type Deserializer struct {
serde.BaseDeserializer
ProtoRegistry *protoregistry.Types
ProtoRegistry *protoregistry.Types
schemaToDescCache cache.Cache
schemaToDescCacheLock sync.RWMutex
}

var _ serde.Serializer = new(Serializer)
Expand Down Expand Up @@ -337,8 +341,14 @@ func ignoreFile(name string) bool {

// NewDeserializer creates a Protobuf deserializer for Protobuf-generated objects
func NewDeserializer(client schemaregistry.Client, serdeType serde.Type, conf *DeserializerConfig) (*Deserializer, error) {
s := &Deserializer{}
err := s.ConfigureDeserializer(client, serdeType, &conf.DeserializerConfig)
cache, err := cache.NewLRUCache(1000)
if err != nil {
return nil, err
}
s := &Deserializer{
schemaToDescCache: cache,
}
err = s.ConfigureDeserializer(client, serdeType, &conf.DeserializerConfig)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -405,6 +415,12 @@ func (s *Deserializer) DeserializeInto(topic string, payload []byte, msg interfa
}

func (s *Deserializer) toFileDesc(info schemaregistry.SchemaInfo) (*desc.FileDescriptor, error) {
s.schemaToDescCacheLock.RLock()
value, ok := s.schemaToDescCache.Get(info.Schema)
s.schemaToDescCacheLock.RUnlock()
if ok {
return value.(*desc.FileDescriptor), nil
}
deps := make(map[string]string)
err := serde.ResolveReferences(s.Client, info, deps)
if err != nil {
Expand Down Expand Up @@ -433,7 +449,11 @@ func (s *Deserializer) toFileDesc(info schemaregistry.SchemaInfo) (*desc.FileDes
if len(fileDescriptors) != 1 {
return nil, fmt.Errorf("could not resolve schema")
}
return fileDescriptors[0], nil
fd := fileDescriptors[0]
s.schemaToDescCacheLock.Lock()
s.schemaToDescCache.Put(info.Schema, fd)
s.schemaToDescCacheLock.Unlock()
return fd, nil
}

func readMessageIndexes(payload []byte) (int, []int, error) {
Expand Down