-
Notifications
You must be signed in to change notification settings - Fork 290
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
use consistent-hash load balancer with kubernetes endpoint resolver for
dispatch
- Loading branch information
Showing
6 changed files
with
168 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| package balancer | ||
|
|
||
| import ( | ||
| "math/rand" | ||
| "time" | ||
|
|
||
| "google.golang.org/grpc/balancer" | ||
| "google.golang.org/grpc/balancer/base" | ||
| "google.golang.org/grpc/grpclog" | ||
|
|
||
| "github.com/authzed/spicedb/pkg/consistent" | ||
| ) | ||
|
|
||
| type ctxKey string | ||
|
|
||
| const ( | ||
| // BalancerName is the name of consistent-hashring balancer. | ||
| BalancerName = "consistent-hashring" | ||
|
|
||
| // CtxKey is the key for the grpc request's context.Context which points to | ||
| // the key to hash for the request. The value it points to must be []byte | ||
| CtxKey ctxKey = "requestKey" | ||
| ) | ||
|
|
||
| var ( | ||
| logger = grpclog.Component("consistenthashring") | ||
| r = rand.New(rand.NewSource(time.Now().UnixNano())) | ||
| ) | ||
|
|
||
| // NewConsistentHashringBuilder creates a new ConsistentBalancerBuilder that | ||
| // will create a balancer with the given config. | ||
| // Before making a connection, register it with grpc with: | ||
| // `balancer.Register(consistent.NewConsistentHashringBuilder(hasher, factor, spread))` | ||
| func NewConsistentHashringBuilder(hasher consistent.HasherFunc, replicationFactor, spread uint8) balancer.Builder { | ||
| return base.NewBalancerBuilder( | ||
| BalancerName, | ||
| &consistentHashringPickerBuilder{hasher: hasher, replicationFactor: replicationFactor, spread: spread}, | ||
| base.Config{HealthCheck: true}, | ||
| ) | ||
| } | ||
|
|
||
| type subConnMember struct { | ||
| balancer.SubConn | ||
| key string | ||
| } | ||
|
|
||
| // Key implements consistent.Member | ||
| // This value is what will be hashed for placement on the consistent hash ring. | ||
| func (s subConnMember) Key() string { | ||
| return s.key | ||
| } | ||
|
|
||
| var _ consistent.Member = &subConnMember{} | ||
|
|
||
| type consistentHashringPickerBuilder struct { | ||
| hasher consistent.HasherFunc | ||
| replicationFactor, spread uint8 | ||
| } | ||
|
|
||
| func (b *consistentHashringPickerBuilder) Build(info base.PickerBuildInfo) balancer.Picker { | ||
| logger.Infof("consistentHashringPicker: Build called with info: %v", info) | ||
| if len(info.ReadySCs) == 0 { | ||
| return base.NewErrPicker(balancer.ErrNoSubConnAvailable) | ||
| } | ||
| hashring := consistent.NewHashring(b.hasher, b.replicationFactor) | ||
| for sc, scInfo := range info.ReadySCs { | ||
| if err := hashring.Add(subConnMember{ | ||
| SubConn: sc, | ||
| key: scInfo.Address.Addr + scInfo.Address.ServerName, | ||
| }); err != nil { | ||
| return base.NewErrPicker(err) | ||
| } | ||
| } | ||
| return &consistentHashringPicker{ | ||
| hashring: hashring, | ||
| spread: b.spread, | ||
| } | ||
| } | ||
|
|
||
| type consistentHashringPicker struct { | ||
| hashring *consistent.Hashring | ||
| spread uint8 | ||
| } | ||
|
|
||
| func (p *consistentHashringPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { | ||
| key := info.Ctx.Value(CtxKey).([]byte) | ||
| members, err := p.hashring.FindN(key, p.spread) | ||
| if err != nil { | ||
| return balancer.PickResult{}, err | ||
| } | ||
| chosen := members[r.Intn(int(p.spread))].(subConnMember) | ||
| return balancer.PickResult{ | ||
| SubConn: chosen.SubConn, | ||
| }, nil | ||
| } |