From 9bf71d3d728db5f8cd095c179714a2f40723ca2e Mon Sep 17 00:00:00 2001 From: Manish R Jain Date: Fri, 13 Jul 2018 19:28:39 -0700 Subject: [PATCH] Fix up predicate snapshot (#2487) Fix up both the way we create a snapshot and the way we stream it. Make the logic robust and add tests to ensure that the calculation of snapshot is right. Streaming no longer needs to worry about `min_ts`. We no longer send a snapshot upfront. Raft would request it as it desires, and the follower sends the snapshot details it needs from the leader. The leader in turn checks if its snapshot matches with what the follower wants. This is a better logic, but not entirely foolproof (comment in predicate.go explains). Bring in Badger changes, which add a `DropAll` method, which is called before applying the snapshot. --- conn/node.go | 3 +- conn/raft_server.go | 13 +- posting/lists.go | 3 +- posting/mvcc.go | 4 + posting/oracle.go | 6 +- protos/intern/internal.pb.go | 443 +++++++++--------- protos/internal.proto | 18 +- vendor/github.com/dgraph-io/badger/db.go | 13 +- vendor/github.com/dgraph-io/badger/errors.go | 4 + vendor/github.com/dgraph-io/badger/levels.go | 56 +++ .../github.com/dgraph-io/badger/managed_db.go | 107 +++++ vendor/vendor.json | 44 +- worker/draft.go | 205 +++++--- worker/draft_test.go | 106 +++++ worker/predicate.go | 139 ++---- worker/stream_lists_test.go | 4 +- 16 files changed, 731 insertions(+), 437 deletions(-) create mode 100644 worker/draft_test.go diff --git a/conn/node.go b/conn/node.go index adb51f485af..8f9e3639613 100644 --- a/conn/node.go +++ b/conn/node.go @@ -28,6 +28,7 @@ import ( var ( ErrDuplicateRaftId = x.Errorf("Node is already part of group") + ErrNoNode = fmt.Errorf("No node has been set up yet") ) type Node struct { @@ -418,7 +419,7 @@ func (n *Node) AddToCluster(ctx context.Context, pid uint64) error { func (n *Node) ProposePeerRemoval(ctx context.Context, id uint64) error { if n.Raft() == nil { - return errNoNode + return ErrNoNode } if _, ok := n.Peer(id); !ok && id != n.RaftContext.Id { return x.Errorf("Node %d not part of group", id) diff --git a/conn/raft_server.go b/conn/raft_server.go index bcbea3af6e6..fdea1ce5115 100644 --- a/conn/raft_server.go +++ b/conn/raft_server.go @@ -10,7 +10,6 @@ package conn import ( "context" "encoding/binary" - "fmt" "math/rand" "sync" @@ -115,7 +114,7 @@ func (w *RaftServer) IsPeer(ctx context.Context, rc *intern.RaftContext) (*inter error) { node := w.GetNode() if node == nil || node.Raft() == nil { - return &intern.PeerResponse{}, errNoNode + return &intern.PeerResponse{}, ErrNoNode } if node._confState == nil { @@ -139,7 +138,7 @@ func (w *RaftServer) JoinCluster(ctx context.Context, // TODO: Uncomment this after groups is removed. node := w.GetNode() if node == nil || node.Raft() == nil { - return nil, errNoNode + return nil, ErrNoNode } // Only process one JoinCluster request at a time. node.joinLock.Lock() @@ -168,14 +167,12 @@ func (w *RaftServer) JoinCluster(ctx context.Context, return &api.Payload{}, err } -var ( - errNoNode = fmt.Errorf("No node has been set up yet") -) +var () func (w *RaftServer) applyMessage(ctx context.Context, msg raftpb.Message) error { node := w.GetNode() if node == nil || node.Raft() == nil { - return errNoNode + return ErrNoNode } c := make(chan error, 1) @@ -198,7 +195,7 @@ func (w *RaftServer) RaftMessage(ctx context.Context, if rc != nil { n := w.GetNode() if n == nil { - return &api.Payload{}, errNoNode + return &api.Payload{}, ErrNoNode } n.Connect(rc.Id, rc.Addr) } diff --git a/posting/lists.go b/posting/lists.go index d8ea71001ed..37d2096a534 100644 --- a/posting/lists.go +++ b/posting/lists.go @@ -269,7 +269,8 @@ func GetNoStore(key []byte) (*List, error) { return getNew(key, pstore) // This retrieves a new *List and sets refcount to 1. } -// This doesn't sync, so call this only when you don't care about dirty posting lists in // memory(for example before populating snapshot) or after calling syncAllMarks +// This doesn't sync, so call this only when you don't care about dirty posting lists in +// memory(for example before populating snapshot) or after calling syncAllMarks func EvictLRU() { lcache.Reset() } diff --git a/posting/mvcc.go b/posting/mvcc.go index 326f1669908..906a2b71c25 100644 --- a/posting/mvcc.go +++ b/posting/mvcc.go @@ -74,6 +74,10 @@ func (tx *Txn) CommitMutations(ctx context.Context, commitTs uint64) error { sort.SliceStable(tx.deltas, func(i, j int) bool { return bytes.Compare(tx.deltas[i].key, tx.deltas[j].key) < 0 }) + // TODO: Simplify this. All we need to do is to get the PL for the key, and if it has the + // postings for the startTs, we commit them. Otherwise, we skip. + // Also, if the snapshot read ts is above the commit ts, then we just delete the postings from + // memory, instead of writing them back again. var prevKey []byte var pl *intern.PostingList var plist *List diff --git a/posting/oracle.go b/posting/oracle.go index c1b46ed8592..ded97bdc94a 100644 --- a/posting/oracle.go +++ b/posting/oracle.go @@ -88,8 +88,8 @@ func (o *oracle) RegisterStartTs(ts uint64) *Txn { return txn } -// MinPendingStartTs returns the min start ts which is currently pending a commit or abort decision. -func (o *oracle) MinPendingStartTs() uint64 { +// minPendingStartTs returns the min start ts which is currently pending a commit or abort decision. +func (o *oracle) minPendingStartTs() uint64 { o.RLock() defer o.RUnlock() min := uint64(math.MaxUint64) @@ -107,7 +107,7 @@ func (o *oracle) PurgeTs() uint64 { // o.MinPendingStartTs can be inf, but we don't want Zero to delete new // records that haven't yet reached us. So, we also consider MaxAssigned // that we have received so far, so only records below MaxAssigned are purged. - return x.Min(o.MinPendingStartTs()-1, o.MaxAssigned()) + return x.Min(o.minPendingStartTs()-1, o.MaxAssigned()) } func (o *oracle) TxnOlderThan(dur time.Duration) (res []uint64) { diff --git a/protos/intern/internal.pb.go b/protos/intern/internal.pb.go index 9090f6d4a1f..2b896b298a2 100644 --- a/protos/intern/internal.pb.go +++ b/protos/intern/internal.pb.go @@ -1126,9 +1126,9 @@ func (m *KeyValues) GetKv() []*KV { } type Snapshot struct { - Context *RaftContext `protobuf:"bytes,1,opt,name=context" json:"context,omitempty"` - Index uint64 `protobuf:"varint,2,opt,name=index,proto3" json:"index,omitempty"` - MinPendingStartTs uint64 `protobuf:"varint,3,opt,name=min_pending_start_ts,json=minPendingStartTs,proto3" json:"min_pending_start_ts,omitempty"` + Context *RaftContext `protobuf:"bytes,1,opt,name=context" json:"context,omitempty"` + Index uint64 `protobuf:"varint,2,opt,name=index,proto3" json:"index,omitempty"` + ReadTs uint64 `protobuf:"varint,3,opt,name=read_ts,json=readTs,proto3" json:"read_ts,omitempty"` } func (m *Snapshot) Reset() { *m = Snapshot{} } @@ -1150,9 +1150,9 @@ func (m *Snapshot) GetIndex() uint64 { return 0 } -func (m *Snapshot) GetMinPendingStartTs() uint64 { +func (m *Snapshot) GetReadTs() uint64 { if m != nil { - return m.MinPendingStartTs + return m.ReadTs } return 0 } @@ -2520,7 +2520,7 @@ type WorkerClient interface { // Data serving RPCs. Mutate(ctx context.Context, in *Mutations, opts ...grpc.CallOption) (*api.TxnContext, error) ServeTask(ctx context.Context, in *Query, opts ...grpc.CallOption) (*Result, error) - PredicateAndSchemaData(ctx context.Context, in *SnapshotMeta, opts ...grpc.CallOption) (Worker_PredicateAndSchemaDataClient, error) + StreamSnapshot(ctx context.Context, in *Snapshot, opts ...grpc.CallOption) (Worker_StreamSnapshotClient, error) Sort(ctx context.Context, in *SortMessage, opts ...grpc.CallOption) (*SortResult, error) Schema(ctx context.Context, in *SchemaRequest, opts ...grpc.CallOption) (*SchemaResult, error) PurgeTs(ctx context.Context, in *api.Payload, opts ...grpc.CallOption) (*Num, error) @@ -2555,12 +2555,12 @@ func (c *workerClient) ServeTask(ctx context.Context, in *Query, opts ...grpc.Ca return out, nil } -func (c *workerClient) PredicateAndSchemaData(ctx context.Context, in *SnapshotMeta, opts ...grpc.CallOption) (Worker_PredicateAndSchemaDataClient, error) { - stream, err := grpc.NewClientStream(ctx, &_Worker_serviceDesc.Streams[0], c.cc, "/intern.Worker/PredicateAndSchemaData", opts...) +func (c *workerClient) StreamSnapshot(ctx context.Context, in *Snapshot, opts ...grpc.CallOption) (Worker_StreamSnapshotClient, error) { + stream, err := grpc.NewClientStream(ctx, &_Worker_serviceDesc.Streams[0], c.cc, "/intern.Worker/StreamSnapshot", opts...) if err != nil { return nil, err } - x := &workerPredicateAndSchemaDataClient{stream} + x := &workerStreamSnapshotClient{stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -2570,16 +2570,16 @@ func (c *workerClient) PredicateAndSchemaData(ctx context.Context, in *SnapshotM return x, nil } -type Worker_PredicateAndSchemaDataClient interface { +type Worker_StreamSnapshotClient interface { Recv() (*KVS, error) grpc.ClientStream } -type workerPredicateAndSchemaDataClient struct { +type workerStreamSnapshotClient struct { grpc.ClientStream } -func (x *workerPredicateAndSchemaDataClient) Recv() (*KVS, error) { +func (x *workerStreamSnapshotClient) Recv() (*KVS, error) { m := new(KVS) if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err @@ -2672,7 +2672,7 @@ type WorkerServer interface { // Data serving RPCs. Mutate(context.Context, *Mutations) (*api.TxnContext, error) ServeTask(context.Context, *Query) (*Result, error) - PredicateAndSchemaData(*SnapshotMeta, Worker_PredicateAndSchemaDataServer) error + StreamSnapshot(*Snapshot, Worker_StreamSnapshotServer) error Sort(context.Context, *SortMessage) (*SortResult, error) Schema(context.Context, *SchemaRequest) (*SchemaResult, error) PurgeTs(context.Context, *api.Payload) (*Num, error) @@ -2721,24 +2721,24 @@ func _Worker_ServeTask_Handler(srv interface{}, ctx context.Context, dec func(in return interceptor(ctx, in, info, handler) } -func _Worker_PredicateAndSchemaData_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(SnapshotMeta) +func _Worker_StreamSnapshot_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(Snapshot) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(WorkerServer).PredicateAndSchemaData(m, &workerPredicateAndSchemaDataServer{stream}) + return srv.(WorkerServer).StreamSnapshot(m, &workerStreamSnapshotServer{stream}) } -type Worker_PredicateAndSchemaDataServer interface { +type Worker_StreamSnapshotServer interface { Send(*KVS) error grpc.ServerStream } -type workerPredicateAndSchemaDataServer struct { +type workerStreamSnapshotServer struct { grpc.ServerStream } -func (x *workerPredicateAndSchemaDataServer) Send(m *KVS) error { +func (x *workerStreamSnapshotServer) Send(m *KVS) error { return x.ServerStream.SendMsg(m) } @@ -2893,8 +2893,8 @@ var _Worker_serviceDesc = grpc.ServiceDesc{ }, Streams: []grpc.StreamDesc{ { - StreamName: "PredicateAndSchemaData", - Handler: _Worker_PredicateAndSchemaData_Handler, + StreamName: "StreamSnapshot", + Handler: _Worker_StreamSnapshot_Handler, ServerStreams: true, }, { @@ -4144,10 +4144,10 @@ func (m *Snapshot) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintInternal(dAtA, i, uint64(m.Index)) } - if m.MinPendingStartTs != 0 { + if m.ReadTs != 0 { dAtA[i] = 0x18 i++ - i = encodeVarintInternal(dAtA, i, uint64(m.MinPendingStartTs)) + i = encodeVarintInternal(dAtA, i, uint64(m.ReadTs)) } return i, nil } @@ -5693,8 +5693,8 @@ func (m *Snapshot) Size() (n int) { if m.Index != 0 { n += 1 + sovInternal(uint64(m.Index)) } - if m.MinPendingStartTs != 0 { - n += 1 + sovInternal(uint64(m.MinPendingStartTs)) + if m.ReadTs != 0 { + n += 1 + sovInternal(uint64(m.ReadTs)) } return n } @@ -9880,9 +9880,9 @@ func (m *Snapshot) Unmarshal(dAtA []byte) error { } case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field MinPendingStartTs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ReadTs", wireType) } - m.MinPendingStartTs = 0 + m.ReadTs = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowInternal @@ -9892,7 +9892,7 @@ func (m *Snapshot) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.MinPendingStartTs |= (uint64(b) & 0x7F) << shift + m.ReadTs |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -13107,198 +13107,197 @@ var ( func init() { proto.RegisterFile("internal.proto", fileDescriptorInternal) } var fileDescriptorInternal = []byte{ - // 3085 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x19, 0xcb, 0x6e, 0x23, 0xc7, - 0x51, 0x33, 0x24, 0x87, 0xc3, 0x22, 0xa9, 0xe5, 0xb6, 0xe5, 0x5d, 0x9a, 0x76, 0x64, 0xb9, 0x6d, - 0xef, 0xca, 0xf6, 0x5a, 0xeb, 0x95, 0xd7, 0x8f, 0x38, 0x71, 0x02, 0xad, 0xc8, 0x5d, 0xd3, 0xab, - 0x97, 0x9b, 0xd4, 0x3a, 0xf6, 0x21, 0x44, 0x8b, 0xd3, 0xa2, 0x06, 0x1a, 0xce, 0x8c, 0xa7, 0x87, - 0x02, 0xe5, 0x53, 0x90, 0x5b, 0x90, 0x1f, 0x08, 0x72, 0x0c, 0x90, 0x5b, 0x80, 0x00, 0x39, 0xe6, - 0x1e, 0x20, 0x01, 0x02, 0x24, 0xf9, 0x83, 0xc4, 0x39, 0x04, 0x08, 0x90, 0x43, 0xfe, 0x20, 0xe8, - 0xc7, 0x3c, 0x48, 0x71, 0x65, 0xe5, 0x75, 0xe2, 0x54, 0x75, 0x55, 0x57, 0xd7, 0xa3, 0xab, 0xaa, - 0x8b, 0xb0, 0xec, 0xfa, 0x31, 0x8b, 0x7c, 0xea, 0x6d, 0x84, 0x51, 0x10, 0x07, 0xc8, 0x52, 0x70, - 0xab, 0x42, 0x43, 0x57, 0xa1, 0x70, 0x0b, 0x8a, 0x3b, 0x2e, 0x8f, 0x11, 0x82, 0xe2, 0xc4, 0x75, - 0x78, 0xd3, 0x58, 0x2b, 0xac, 0x5b, 0x44, 0x7e, 0xe3, 0x4f, 0xa0, 0xd2, 0xa7, 0xfc, 0xf4, 0x09, - 0xf5, 0x26, 0x0c, 0x35, 0xa0, 0x70, 0x46, 0xbd, 0xa6, 0xb1, 0x66, 0xac, 0xd7, 0x88, 0xf8, 0x44, - 0x9b, 0x60, 0x9f, 0x51, 0x6f, 0x10, 0x9f, 0x87, 0xac, 0x69, 0xae, 0x19, 0xeb, 0xcb, 0x9b, 0x37, - 0x37, 0x94, 0x80, 0x8d, 0x83, 0x80, 0xc7, 0xae, 0x3f, 0xda, 0x78, 0x42, 0xbd, 0xfe, 0x79, 0xc8, - 0x48, 0xf9, 0x4c, 0x7d, 0xe0, 0x7d, 0xa8, 0xf6, 0xa2, 0xe1, 0xc3, 0x89, 0x3f, 0x8c, 0xdd, 0xc0, - 0x17, 0x52, 0x7d, 0x3a, 0x66, 0x72, 0xd7, 0x0a, 0x91, 0xdf, 0x02, 0x47, 0xa3, 0x11, 0x6f, 0x16, - 0xd6, 0x0a, 0x02, 0x27, 0xbe, 0x51, 0x13, 0xca, 0x2e, 0xdf, 0x0e, 0x26, 0x7e, 0xdc, 0x2c, 0xae, - 0x19, 0xeb, 0x36, 0x49, 0x40, 0xfc, 0x8b, 0x02, 0x94, 0x3e, 0x99, 0xb0, 0xe8, 0x5c, 0xf2, 0xc5, - 0x71, 0x94, 0xec, 0x25, 0xbe, 0xd1, 0x0a, 0x94, 0x3c, 0xea, 0x8f, 0x78, 0xd3, 0x94, 0x9b, 0x29, - 0x00, 0x3d, 0x0f, 0x15, 0x7a, 0x1c, 0xb3, 0x68, 0x30, 0x71, 0x9d, 0x66, 0x61, 0xcd, 0x58, 0xb7, - 0x88, 0x2d, 0x11, 0x87, 0xae, 0x83, 0x9e, 0x03, 0xdb, 0x09, 0x06, 0xc3, 0xbc, 0x2c, 0x27, 0x90, - 0xb2, 0xd0, 0x6d, 0xb0, 0x27, 0xae, 0x33, 0xf0, 0x5c, 0x1e, 0x37, 0x4b, 0x6b, 0xc6, 0x7a, 0x75, - 0xb3, 0x96, 0x28, 0x2c, 0x6c, 0x48, 0xca, 0x13, 0xd7, 0x91, 0xc6, 0xdc, 0x00, 0x9b, 0x47, 0xc3, - 0xc1, 0xf1, 0xc4, 0x1f, 0x36, 0x2d, 0x49, 0xf8, 0x4c, 0x42, 0x98, 0xd3, 0x9e, 0x94, 0xb9, 0x02, - 0x84, 0x7a, 0x11, 0x3b, 0x63, 0x11, 0x67, 0xcd, 0xb2, 0x12, 0xa9, 0x41, 0x74, 0x1f, 0xaa, 0xc7, - 0x74, 0xc8, 0xe2, 0x41, 0x48, 0x23, 0x3a, 0x6e, 0xda, 0xb3, 0x9b, 0x3d, 0x14, 0x4b, 0x07, 0x62, - 0x85, 0x13, 0x38, 0x4e, 0x01, 0xf4, 0x1e, 0xd4, 0x25, 0xc4, 0x07, 0xc7, 0xae, 0x17, 0xb3, 0xa8, - 0x59, 0x91, 0x7c, 0x28, 0xe5, 0x93, 0xd8, 0x7e, 0xc4, 0x18, 0xa9, 0x29, 0x42, 0x85, 0x41, 0xdf, - 0x00, 0x60, 0xd3, 0x90, 0xfa, 0xce, 0x80, 0x7a, 0x5e, 0x13, 0xe4, 0x59, 0x2a, 0x0a, 0xb3, 0xe5, - 0x79, 0xe8, 0xa6, 0x38, 0x27, 0x75, 0x06, 0x31, 0x6f, 0xd6, 0xd7, 0x8c, 0xf5, 0x22, 0xb1, 0x04, - 0xd8, 0xe7, 0xc2, 0x32, 0x9e, 0xeb, 0x0f, 0x04, 0xd4, 0x5c, 0xd6, 0x96, 0x11, 0x31, 0xb6, 0xe3, - 0xfa, 0x84, 0x51, 0x87, 0x94, 0x3d, 0xf5, 0x81, 0xdf, 0x85, 0x8a, 0x0c, 0x27, 0x69, 0xa6, 0xd7, - 0xc0, 0x3a, 0x13, 0x80, 0x8a, 0xba, 0xea, 0xe6, 0xf5, 0xe4, 0x7c, 0x69, 0xd4, 0x11, 0x4d, 0x80, - 0x57, 0xc1, 0xde, 0xa1, 0xfe, 0x28, 0x09, 0x55, 0xe1, 0x47, 0xc9, 0x54, 0x21, 0xf2, 0x1b, 0xff, - 0xc9, 0x04, 0x8b, 0x30, 0x3e, 0xf1, 0x62, 0xf4, 0x06, 0x80, 0xf0, 0xd2, 0x98, 0xc6, 0x91, 0x3b, - 0xd5, 0x3b, 0xcf, 0xfa, 0xa9, 0x32, 0x71, 0x9d, 0x5d, 0xb9, 0x8c, 0xee, 0x43, 0x4d, 0x4a, 0x48, - 0xc8, 0xcd, 0xd9, 0x83, 0xa4, 0x67, 0x25, 0x55, 0x49, 0xa6, 0xb9, 0x6e, 0x80, 0x25, 0x03, 0x44, - 0x05, 0x69, 0x9d, 0x68, 0x08, 0xbd, 0xaa, 0x6f, 0x1c, 0x67, 0xc3, 0x78, 0xe0, 0x30, 0x9e, 0x44, - 0x50, 0x3d, 0xc5, 0xb6, 0x19, 0x8f, 0xd1, 0x3b, 0xa0, 0xac, 0x9e, 0x08, 0x2d, 0x49, 0xa1, 0x68, - 0xc6, 0xab, 0x5c, 0x49, 0x95, 0x74, 0x5a, 0xea, 0x3d, 0xa8, 0x0a, 0x5d, 0x13, 0x2e, 0x4b, 0x72, - 0x35, 0x52, 0xcd, 0xb4, 0x79, 0x08, 0x08, 0x22, 0xcd, 0x22, 0x4c, 0x25, 0xa2, 0x55, 0x45, 0x95, - 0xfc, 0xbe, 0xba, 0xaf, 0x3a, 0x50, 0xda, 0x8f, 0x1c, 0x16, 0x2d, 0xbc, 0x59, 0x08, 0x8a, 0x0e, - 0xe3, 0x43, 0x79, 0xf1, 0x6d, 0x22, 0xbf, 0xb3, 0xdb, 0x56, 0xc8, 0xdd, 0x36, 0xfc, 0x07, 0x03, - 0xaa, 0xbd, 0x20, 0x8a, 0x77, 0x19, 0xe7, 0x74, 0xc4, 0xd0, 0xcb, 0x50, 0x0a, 0xc4, 0xb6, 0xda, - 0x35, 0xf5, 0x44, 0x01, 0x29, 0x8b, 0xa8, 0xb5, 0x39, 0x27, 0x9a, 0x97, 0x3b, 0x71, 0x05, 0x4a, - 0xea, 0xbe, 0x8a, 0xbb, 0x5c, 0x22, 0x0a, 0x10, 0x4e, 0x0a, 0x8e, 0x8f, 0x39, 0x53, 0x4e, 0x28, - 0x11, 0x0d, 0xfd, 0x0f, 0x82, 0xf8, 0x08, 0x40, 0x28, 0xf4, 0x9f, 0xc4, 0xdb, 0x95, 0x65, 0x9c, - 0x40, 0x95, 0xd0, 0xe3, 0x78, 0x3b, 0xf0, 0x63, 0x36, 0x8d, 0xd1, 0x32, 0x98, 0xae, 0x23, 0x1d, - 0x60, 0x11, 0xd3, 0x75, 0x84, 0xca, 0xa3, 0x28, 0x98, 0x84, 0xd2, 0xfe, 0x75, 0xa2, 0x00, 0xe9, - 0x28, 0xc7, 0x89, 0xa4, 0x1d, 0x84, 0xa3, 0x1c, 0x27, 0x42, 0x2f, 0x42, 0x95, 0xfb, 0x34, 0xe4, - 0x27, 0x41, 0x2c, 0x54, 0x2e, 0x4a, 0x95, 0x21, 0x41, 0xf5, 0x39, 0xfe, 0x8d, 0x01, 0xd6, 0x2e, - 0x1b, 0x1f, 0xb1, 0xe8, 0x82, 0x94, 0xe7, 0xc0, 0x96, 0x1b, 0x0f, 0x5c, 0x47, 0x0b, 0x2a, 0x4b, - 0xb8, 0xeb, 0x2c, 0x14, 0x75, 0x03, 0x2c, 0x8f, 0x51, 0xe1, 0x5a, 0x15, 0xf6, 0x1a, 0x12, 0x16, - 0xa7, 0xe3, 0x81, 0x23, 0x74, 0x2e, 0xa9, 0x05, 0x3a, 0x6e, 0x33, 0xea, 0x88, 0xb3, 0x79, 0x94, - 0xc7, 0x83, 0x49, 0xe8, 0xd0, 0x98, 0xc9, 0x54, 0x59, 0x14, 0xf1, 0xcb, 0xe3, 0x43, 0x89, 0x41, - 0xaf, 0xc3, 0xf5, 0xa1, 0x37, 0xe1, 0x22, 0x57, 0xbb, 0xfe, 0x71, 0x30, 0x08, 0x7c, 0xef, 0x5c, - 0x7a, 0xcd, 0x26, 0xd7, 0xf4, 0x42, 0xd7, 0x3f, 0x0e, 0xf6, 0x7d, 0xef, 0x1c, 0xff, 0xd8, 0x84, - 0xd2, 0x23, 0x69, 0x86, 0xfb, 0x50, 0x1e, 0x4b, 0x85, 0x92, 0xc4, 0xd2, 0x4a, 0xdc, 0x21, 0xd7, - 0x37, 0x94, 0xb6, 0xbc, 0xe3, 0xc7, 0xd1, 0x39, 0x49, 0x48, 0x05, 0x57, 0x4c, 0x8f, 0x3c, 0x16, - 0x73, 0x1d, 0x6f, 0x73, 0x5c, 0x7d, 0xb5, 0xa8, 0xb9, 0x34, 0x69, 0xeb, 0x63, 0xa8, 0xe5, 0xb7, - 0x13, 0x65, 0xf2, 0x94, 0x9d, 0x4b, 0x1b, 0x16, 0x89, 0xf8, 0x44, 0xaf, 0x40, 0x49, 0xe6, 0x0e, - 0x69, 0xc1, 0xea, 0xe6, 0x72, 0xb2, 0xab, 0x62, 0x23, 0x6a, 0xf1, 0x03, 0xf3, 0x7d, 0x43, 0xec, - 0x95, 0x17, 0x92, 0xdf, 0xab, 0x72, 0xf9, 0x5e, 0x8a, 0x2d, 0xb7, 0x17, 0xfe, 0x87, 0x01, 0xb5, - 0xcf, 0x59, 0x14, 0x1c, 0x44, 0x41, 0x18, 0x70, 0xea, 0xa1, 0x5b, 0x60, 0x29, 0x4d, 0x9f, 0x72, - 0x0e, 0xbd, 0x2a, 0xe8, 0x94, 0x6e, 0xd2, 0xb5, 0x17, 0x65, 0xe8, 0x55, 0xb4, 0x0a, 0x30, 0xa6, - 0xd3, 0x1d, 0x46, 0x39, 0xeb, 0x3a, 0x49, 0x58, 0x65, 0x18, 0xd4, 0x02, 0x7b, 0x4c, 0xa7, 0xfd, - 0xa9, 0xdf, 0xe7, 0xd2, 0xeb, 0x45, 0x92, 0xc2, 0xe8, 0x05, 0xa8, 0x8c, 0xe9, 0x54, 0xc4, 0x77, - 0xd7, 0xd1, 0x5e, 0xcf, 0x10, 0xe8, 0x25, 0x28, 0xc4, 0x53, 0x5f, 0xe6, 0xac, 0xea, 0xe6, 0x35, - 0x79, 0x3d, 0xfa, 0x53, 0x5f, 0xdf, 0x04, 0x22, 0xd6, 0x12, 0xcb, 0xd8, 0xa9, 0x65, 0xf0, 0xaf, - 0x0b, 0x70, 0x4d, 0x3b, 0xe2, 0xc4, 0x0d, 0x7b, 0xb1, 0x88, 0x9e, 0x26, 0x94, 0x65, 0x2a, 0x60, - 0x91, 0xf6, 0x47, 0x02, 0xa2, 0x6f, 0x81, 0x25, 0x03, 0x39, 0x71, 0xf5, 0xcb, 0xb3, 0xc6, 0x48, - 0xb7, 0x50, 0xae, 0xd7, 0x3e, 0xd7, 0x2c, 0xe8, 0x7d, 0x28, 0x7d, 0xc9, 0xa2, 0x40, 0xa5, 0xb9, - 0xea, 0x26, 0x7e, 0x1a, 0xaf, 0x30, 0xbf, 0x66, 0x55, 0x0c, 0xff, 0x47, 0x9b, 0xad, 0x8b, 0xa4, - 0x36, 0x0e, 0xce, 0x98, 0xd3, 0x2c, 0xcb, 0x53, 0xcd, 0xbb, 0x37, 0x59, 0x6e, 0x7d, 0x04, 0xd5, - 0x9c, 0x52, 0xf9, 0x18, 0xab, 0xab, 0x18, 0x7b, 0x79, 0x36, 0xc6, 0xea, 0x33, 0xb7, 0x20, 0x1f, - 0xae, 0x1f, 0x01, 0x64, 0x2a, 0xfe, 0x37, 0x81, 0x8f, 0x7f, 0x64, 0xc0, 0xb5, 0xed, 0xc0, 0xf7, - 0x99, 0xec, 0x8b, 0x94, 0xf3, 0xb2, 0x78, 0x35, 0x2e, 0x8d, 0xd7, 0x37, 0xa1, 0xc4, 0x05, 0x83, - 0x96, 0x72, 0xf3, 0x29, 0xde, 0x20, 0x8a, 0x4a, 0xa4, 0x9c, 0x31, 0x9d, 0x0e, 0x42, 0xe6, 0x3b, - 0xae, 0x3f, 0x92, 0x31, 0xae, 0x7c, 0x70, 0xa0, 0x30, 0xf8, 0x67, 0x06, 0x58, 0x2a, 0xd4, 0x67, - 0xd2, 0x9f, 0x31, 0x9b, 0xfe, 0x5e, 0x80, 0x4a, 0x18, 0x31, 0xc7, 0x1d, 0x26, 0x92, 0x2b, 0x24, - 0x43, 0x88, 0xec, 0x7c, 0x1c, 0x44, 0x43, 0x26, 0xb7, 0xb7, 0x89, 0x02, 0x44, 0xdb, 0x29, 0x0b, - 0x8f, 0x4c, 0x62, 0x2a, 0x43, 0xda, 0x02, 0x21, 0xb2, 0x97, 0x60, 0xe1, 0x21, 0x1d, 0xaa, 0x06, - 0xb0, 0x40, 0x14, 0x20, 0x32, 0xaa, 0xf2, 0x9b, 0x0c, 0x75, 0x9b, 0x68, 0x08, 0xff, 0xca, 0x84, - 0x5a, 0xdb, 0x8d, 0xd8, 0x30, 0x66, 0x4e, 0xc7, 0x19, 0x49, 0x42, 0xe6, 0xc7, 0x6e, 0x7c, 0xae, - 0xb3, 0xb7, 0x86, 0xd2, 0xd2, 0x6d, 0xce, 0x36, 0xc5, 0xca, 0x2f, 0x05, 0xd9, 0xcb, 0x2b, 0x00, - 0xbd, 0x0b, 0xa0, 0x3a, 0x21, 0xd9, 0xcf, 0x17, 0x2f, 0xef, 0xe7, 0x2b, 0x92, 0x54, 0x7c, 0x0a, - 0x23, 0x29, 0x3e, 0x57, 0x65, 0x77, 0x4b, 0x36, 0xfb, 0x13, 0x11, 0xce, 0xb2, 0x1f, 0x38, 0x62, - 0x9e, 0x0c, 0x57, 0xd9, 0x0f, 0x1c, 0x31, 0x2f, 0x6d, 0xdf, 0xca, 0xea, 0x48, 0xe2, 0x1b, 0xdd, - 0x06, 0x33, 0x08, 0xa5, 0x8e, 0x39, 0xa1, 0x79, 0x05, 0x37, 0xf6, 0x43, 0x62, 0x06, 0x21, 0xc2, - 0x60, 0xa9, 0x86, 0xb5, 0x59, 0x91, 0x61, 0x0e, 0x32, 0x3d, 0xc8, 0x8e, 0x89, 0xe8, 0x15, 0x7c, - 0x03, 0xcc, 0xfd, 0x10, 0x95, 0xa1, 0xd0, 0xeb, 0xf4, 0x1b, 0x4b, 0xe2, 0xa3, 0xdd, 0xd9, 0x69, - 0x18, 0xf8, 0xef, 0x06, 0x54, 0x76, 0x27, 0x31, 0x15, 0x31, 0xc6, 0x2f, 0x73, 0xee, 0x73, 0x60, - 0xf3, 0x98, 0x46, 0xb2, 0x5e, 0x9a, 0x2a, 0x71, 0x48, 0xb8, 0xcf, 0xd1, 0xeb, 0x50, 0x62, 0xce, - 0x88, 0x25, 0x77, 0x7f, 0x65, 0xd1, 0x59, 0x89, 0x22, 0x41, 0x77, 0xc0, 0xe2, 0xc3, 0x13, 0x36, - 0xa6, 0xcd, 0xe2, 0x2c, 0x71, 0x4f, 0x62, 0x55, 0x89, 0x23, 0x9a, 0x46, 0xbe, 0x3b, 0xa2, 0x20, - 0x94, 0x8d, 0x77, 0x49, 0xbf, 0x3b, 0xa2, 0x20, 0x14, 0x6d, 0xf7, 0x26, 0x3c, 0xeb, 0x8e, 0xfc, - 0x20, 0x62, 0x03, 0xd7, 0x77, 0xd8, 0x74, 0x30, 0x0c, 0xfc, 0x63, 0xcf, 0x1d, 0xc6, 0xd2, 0xae, - 0x36, 0x79, 0x46, 0x2d, 0x76, 0xc5, 0xda, 0xb6, 0x5e, 0xc2, 0xb7, 0xa1, 0xf2, 0x98, 0x9d, 0xcb, - 0xfe, 0x95, 0xa3, 0x16, 0x98, 0xa7, 0x67, 0xba, 0x16, 0x42, 0x72, 0x8a, 0xc7, 0x4f, 0x88, 0x79, - 0x7a, 0x86, 0x7f, 0x60, 0x80, 0xdd, 0xd3, 0xdd, 0x00, 0x7a, 0x53, 0x64, 0x4c, 0x99, 0x67, 0xf5, - 0xad, 0x4b, 0x9f, 0x1a, 0xb9, 0x66, 0x84, 0x24, 0x34, 0xc2, 0xc1, 0xf2, 0x44, 0xda, 0x4a, 0x0a, - 0x40, 0x77, 0x61, 0x65, 0xec, 0xfa, 0xc9, 0x15, 0x1b, 0xa4, 0xa6, 0x54, 0x77, 0xed, 0xfa, 0xd8, - 0xf5, 0xf5, 0x5d, 0xeb, 0x29, 0xa3, 0xe2, 0x9f, 0x9a, 0x60, 0xa7, 0x75, 0xea, 0x2e, 0x54, 0xc6, - 0x89, 0x93, 0xf4, 0x9d, 0x4e, 0xdb, 0xf1, 0xd4, 0x7b, 0x24, 0xa3, 0xd1, 0xca, 0x15, 0x17, 0x29, - 0x97, 0x25, 0x87, 0xd2, 0x95, 0x92, 0xc3, 0x6d, 0xb8, 0x36, 0xf4, 0x18, 0xf5, 0x07, 0xd9, 0xdd, - 0x56, 0xa1, 0xbb, 0x2c, 0xd1, 0x07, 0xe9, 0x05, 0xd7, 0xc9, 0xae, 0x9c, 0x55, 0xe6, 0xd7, 0xa0, - 0xe4, 0x30, 0x2f, 0xa6, 0xf3, 0x4f, 0xb4, 0xfd, 0x88, 0x0e, 0x3d, 0xd6, 0x16, 0x4b, 0x44, 0x51, - 0xa0, 0x3b, 0x60, 0x27, 0xed, 0x97, 0x7e, 0x98, 0xa5, 0x4d, 0x7c, 0xe2, 0x08, 0x92, 0x52, 0xe0, - 0x97, 0xa0, 0xf0, 0xf8, 0x49, 0xef, 0x52, 0x17, 0x7e, 0x1f, 0xcc, 0xc7, 0x4f, 0xf2, 0x09, 0xb8, - 0xa6, 0xce, 0xa4, 0x9f, 0xec, 0x66, 0xf6, 0x64, 0x6f, 0x81, 0x3d, 0xe1, 0x2c, 0xda, 0x65, 0x31, - 0xd5, 0xb7, 0x3f, 0x85, 0x45, 0xb5, 0x14, 0x6f, 0x4e, 0x37, 0xf0, 0x75, 0x65, 0x4a, 0x40, 0xfc, - 0xcf, 0x02, 0x94, 0x75, 0x06, 0x10, 0x7b, 0x4e, 0xd2, 0x1e, 0x51, 0x7c, 0x66, 0xe9, 0xc4, 0xcc, - 0xa7, 0x93, 0xfc, 0x70, 0xa0, 0x70, 0xb5, 0xe1, 0x00, 0xfa, 0x0e, 0xd4, 0x42, 0xb5, 0x96, 0x4f, - 0x42, 0xcf, 0xcf, 0xf3, 0xe9, 0x5f, 0xc9, 0x5b, 0x0d, 0x33, 0x40, 0x5c, 0x21, 0xf9, 0x40, 0x8a, - 0xe9, 0x48, 0x3a, 0xbc, 0x46, 0xca, 0x02, 0xee, 0xd3, 0xd1, 0x53, 0x52, 0xd1, 0x15, 0xb2, 0x89, - 0xe8, 0x89, 0x83, 0xb0, 0x59, 0x93, 0x19, 0x42, 0x64, 0xa0, 0x7c, 0x72, 0xa8, 0xcf, 0x26, 0x87, - 0xe7, 0xa1, 0x32, 0x0c, 0xc6, 0x63, 0x57, 0xae, 0x2d, 0xab, 0xfa, 0xad, 0x10, 0x7d, 0x8e, 0xbf, - 0x84, 0xb2, 0x56, 0x18, 0x55, 0xa1, 0xdc, 0xee, 0x3c, 0xdc, 0x3a, 0xdc, 0x11, 0xe9, 0x09, 0xc0, - 0x7a, 0xd0, 0xdd, 0xdb, 0x22, 0x9f, 0x35, 0x0c, 0x91, 0xaa, 0xba, 0x7b, 0xfd, 0x86, 0x89, 0x2a, - 0x50, 0x7a, 0xb8, 0xb3, 0xbf, 0xd5, 0x6f, 0x14, 0x90, 0x0d, 0xc5, 0x07, 0xfb, 0xfb, 0x3b, 0x8d, - 0x22, 0xaa, 0x81, 0xdd, 0xde, 0xea, 0x77, 0xfa, 0xdd, 0xdd, 0x4e, 0xa3, 0x24, 0x68, 0x1f, 0x75, - 0xf6, 0x1b, 0x96, 0xf8, 0x38, 0xec, 0xb6, 0x1b, 0x65, 0xb1, 0x7e, 0xb0, 0xd5, 0xeb, 0x7d, 0xba, - 0x4f, 0xda, 0x0d, 0x5b, 0xec, 0xdb, 0xeb, 0x93, 0xee, 0xde, 0xa3, 0x46, 0x05, 0xdf, 0x83, 0x6a, - 0xce, 0x68, 0x82, 0x83, 0x74, 0x1e, 0x36, 0x96, 0x84, 0x98, 0x27, 0x5b, 0x3b, 0x87, 0x9d, 0x86, - 0x81, 0x96, 0x01, 0xe4, 0xe7, 0x60, 0x67, 0x6b, 0xef, 0x51, 0xc3, 0xc4, 0x3f, 0x34, 0x52, 0x1e, - 0xf9, 0xe8, 0x7e, 0x03, 0x6c, 0x6d, 0xea, 0xa4, 0xa9, 0xbe, 0x36, 0xe7, 0x17, 0x92, 0x12, 0x88, - 0x30, 0x1b, 0x9e, 0xb0, 0xe1, 0x29, 0x9f, 0x8c, 0x75, 0x54, 0xa4, 0xb0, 0x7a, 0x3b, 0x0b, 0x9b, - 0xe8, 0x7c, 0xa0, 0xa1, 0x74, 0x00, 0x55, 0x94, 0xf4, 0x6a, 0x00, 0x75, 0x1f, 0x20, 0x1b, 0x71, - 0x2c, 0x68, 0x87, 0x57, 0xa0, 0x44, 0x3d, 0x97, 0x72, 0x5d, 0xde, 0x14, 0x80, 0x09, 0x54, 0x73, - 0x83, 0x11, 0xe1, 0x30, 0xea, 0x79, 0x83, 0x53, 0x76, 0xce, 0x25, 0xaf, 0x4d, 0xca, 0xd4, 0xf3, - 0x1e, 0xb3, 0x73, 0x8e, 0xd6, 0xa1, 0xa4, 0xe6, 0x2a, 0xe6, 0x82, 0x17, 0xb8, 0x64, 0x27, 0x8a, - 0x00, 0xdf, 0x01, 0x4b, 0x3d, 0xcb, 0x73, 0x31, 0x63, 0x3c, 0xb5, 0x02, 0x7d, 0xa8, 0xcf, 0x2d, - 0x1f, 0xf1, 0xe8, 0xae, 0x9e, 0xe1, 0x70, 0x35, 0x39, 0x32, 0x66, 0xfb, 0x33, 0x45, 0xa8, 0xc7, - 0x37, 0x92, 0x01, 0xb7, 0xc1, 0xbe, 0x74, 0x42, 0xa6, 0x0d, 0x61, 0x66, 0x86, 0x58, 0x30, 0x33, - 0xc3, 0x11, 0x40, 0x36, 0xe7, 0xd1, 0x61, 0xac, 0x76, 0x11, 0x61, 0xbc, 0x21, 0x5c, 0xe4, 0x7a, - 0x4e, 0xc4, 0xfc, 0x0b, 0xda, 0x67, 0xd3, 0xa1, 0x94, 0x06, 0xbd, 0x02, 0x45, 0x39, 0xce, 0x2a, - 0xcc, 0x26, 0xac, 0x74, 0x96, 0x25, 0x57, 0xf1, 0x11, 0xd4, 0x55, 0x71, 0x23, 0xec, 0x8b, 0x09, - 0xe3, 0x97, 0xb6, 0x50, 0xab, 0x00, 0x69, 0x9a, 0x4d, 0x06, 0x74, 0x39, 0x8c, 0x08, 0x94, 0x63, - 0x97, 0x79, 0x4e, 0xa2, 0x95, 0x86, 0xf0, 0x7b, 0x50, 0x4b, 0x64, 0xc8, 0xf7, 0xf7, 0xed, 0xb4, - 0xcc, 0x26, 0x71, 0x29, 0x1c, 0xa2, 0x48, 0xf6, 0x02, 0x27, 0xad, 0xb0, 0xf8, 0x2f, 0x66, 0xc2, - 0xa9, 0x5f, 0x97, 0x33, 0x4d, 0x9c, 0x31, 0xdf, 0xc4, 0xcd, 0x36, 0x44, 0xe6, 0x95, 0x1b, 0xa2, - 0x6f, 0x43, 0xc5, 0x91, 0xdd, 0x80, 0x7b, 0x96, 0xa4, 0xbe, 0xd5, 0x45, 0x95, 0x5f, 0xf7, 0x0c, - 0xee, 0x19, 0x23, 0x19, 0x83, 0x38, 0x53, 0x1c, 0x9c, 0x32, 0xdf, 0xfd, 0x52, 0x3e, 0xa3, 0x85, - 0xe2, 0x19, 0x22, 0x9b, 0x74, 0xa8, 0x0e, 0x41, 0x4f, 0x3a, 0x92, 0x29, 0x8f, 0x95, 0x9b, 0xf2, - 0xdc, 0x00, 0x6b, 0x12, 0x72, 0x16, 0xc5, 0x49, 0xe7, 0xa8, 0xa0, 0xb4, 0xfb, 0xaa, 0x68, 0x5a, - 0xea, 0x8f, 0xf0, 0x37, 0xa1, 0x92, 0x9e, 0x45, 0xe4, 0x9b, 0xbd, 0xfd, 0xbd, 0x8e, 0xca, 0x0e, - 0xdd, 0xbd, 0x76, 0xe7, 0x7b, 0x0d, 0x43, 0x64, 0x2c, 0xd2, 0x79, 0xd2, 0x21, 0xbd, 0x4e, 0xc3, - 0x14, 0x99, 0xa5, 0xdd, 0xd9, 0xe9, 0xf4, 0x3b, 0x8d, 0xc2, 0xc7, 0x45, 0xbb, 0xdc, 0xb0, 0x89, - 0xcd, 0xa6, 0xa1, 0xe7, 0x0e, 0xdd, 0x18, 0x7f, 0x06, 0xf6, 0x2e, 0x0d, 0x2f, 0xbc, 0x08, 0xb2, - 0x82, 0x34, 0xd1, 0xa3, 0x04, 0x5d, 0x3c, 0x5e, 0x83, 0xb2, 0xce, 0x1a, 0x3a, 0xb2, 0x2e, 0x64, - 0x95, 0x64, 0x1d, 0xff, 0xd2, 0x80, 0x95, 0xdd, 0xe0, 0x8c, 0xa5, 0x55, 0xf8, 0x80, 0x9e, 0x7b, - 0x01, 0x75, 0xbe, 0xc6, 0x8d, 0xb7, 0xe0, 0x1a, 0x0f, 0x26, 0xd1, 0x90, 0x0d, 0xe6, 0x46, 0x19, - 0x75, 0x85, 0x7e, 0xa4, 0xc3, 0x11, 0x43, 0xdd, 0x61, 0x3c, 0xce, 0xa8, 0x0a, 0x92, 0xaa, 0x2a, - 0x90, 0x09, 0x4d, 0xda, 0x4e, 0x14, 0xaf, 0xd2, 0x4e, 0xe0, 0xdf, 0x1b, 0x50, 0xef, 0x4c, 0xc3, - 0x20, 0x8a, 0x93, 0xa3, 0x3e, 0x2b, 0xfa, 0xf9, 0x2f, 0x92, 0xcb, 0x50, 0x24, 0xa5, 0x88, 0x7d, - 0xd1, 0xbd, 0x74, 0xce, 0x72, 0x1f, 0x2c, 0xb1, 0xd9, 0x84, 0xeb, 0x50, 0x7a, 0x21, 0x91, 0x39, - 0xb3, 0xf1, 0x46, 0x4f, 0xd2, 0x10, 0x4d, 0x9b, 0x9f, 0x71, 0x15, 0xf3, 0x33, 0x2e, 0xfc, 0x01, - 0x58, 0x8a, 0x34, 0xe7, 0xe7, 0x2a, 0x94, 0x7b, 0x87, 0xdb, 0xdb, 0x9d, 0x5e, 0xaf, 0x61, 0xa0, - 0x3a, 0x54, 0xda, 0x87, 0x07, 0x3b, 0xdd, 0xed, 0xad, 0xbe, 0xf6, 0xf5, 0xc3, 0xad, 0xee, 0x4e, - 0xa7, 0xdd, 0x28, 0xe0, 0x6d, 0xa8, 0xf4, 0xa7, 0xbe, 0x66, 0xcf, 0x97, 0x41, 0xe3, 0x92, 0x32, - 0x68, 0xce, 0x95, 0xc1, 0x4f, 0xa1, 0x9a, 0x6b, 0x89, 0xd0, 0xab, 0x50, 0x8c, 0xa7, 0xfe, 0xc5, - 0x01, 0x70, 0x22, 0x87, 0xc8, 0x65, 0xf4, 0x12, 0xd4, 0xc4, 0xab, 0x8d, 0x72, 0xee, 0x8e, 0x7c, - 0xe6, 0xe8, 0x5d, 0xc5, 0x4b, 0x6e, 0x4b, 0xa3, 0xf0, 0x8b, 0x50, 0x17, 0x0f, 0x65, 0x77, 0xcc, - 0x78, 0x4c, 0xc7, 0xa1, 0x2c, 0xdc, 0x3a, 0x49, 0x17, 0x89, 0x19, 0x73, 0x7c, 0x0b, 0x6a, 0x07, - 0x8c, 0x45, 0x84, 0xf1, 0x30, 0xf0, 0xb9, 0x7c, 0x32, 0x69, 0xcb, 0xaa, 0xaa, 0xa0, 0x21, 0x7c, - 0x04, 0x15, 0xd1, 0xec, 0x3e, 0xa0, 0xf1, 0xf0, 0xe4, 0xdf, 0x6d, 0x88, 0x6f, 0x41, 0x39, 0x54, - 0x1e, 0xd1, 0xad, 0xab, 0x9a, 0xee, 0x69, 0x2f, 0x91, 0x64, 0x11, 0xdf, 0x84, 0xc2, 0xde, 0x64, - 0x9c, 0xff, 0x4f, 0xa5, 0x28, 0x1b, 0x34, 0xfc, 0x10, 0x6a, 0x49, 0x0f, 0x28, 0x9b, 0x32, 0x61, - 0x4b, 0xcf, 0x65, 0x7e, 0xce, 0xce, 0xb6, 0x42, 0xf4, 0xf9, 0x25, 0x61, 0xb3, 0xf9, 0x3b, 0x03, - 0x8a, 0xe2, 0x84, 0x22, 0x6f, 0x77, 0x86, 0x27, 0x01, 0x9a, 0x39, 0x48, 0x6b, 0x06, 0xc2, 0x4b, - 0xe8, 0x2d, 0x35, 0x6d, 0x4c, 0x46, 0xb4, 0xd7, 0xf3, 0x4a, 0x4a, 0x43, 0x5c, 0xe0, 0xd8, 0x84, - 0xea, 0xc7, 0x81, 0xeb, 0x6f, 0xab, 0x21, 0x1c, 0x5a, 0x64, 0x96, 0x0b, 0x3c, 0xef, 0x80, 0xd5, - 0xe5, 0xc2, 0x07, 0x8b, 0xc9, 0xd3, 0xf7, 0x51, 0xde, 0x4d, 0x78, 0x69, 0xf3, 0xe7, 0x05, 0x28, - 0x7e, 0xce, 0xa2, 0x00, 0xdd, 0x87, 0xb2, 0x9e, 0x12, 0xa0, 0xb9, 0x69, 0x40, 0x2b, 0xbd, 0x8a, - 0x73, 0x63, 0x04, 0xbc, 0x84, 0xde, 0x05, 0x4b, 0xe7, 0xfb, 0xd9, 0x51, 0x46, 0xeb, 0x69, 0xd7, - 0x17, 0x2f, 0xad, 0x1b, 0x6f, 0x19, 0xe8, 0x2e, 0x58, 0x2a, 0x52, 0xe7, 0x6c, 0xb7, 0xa8, 0xb5, - 0xc7, 0x4b, 0x92, 0xa1, 0xda, 0x3b, 0x09, 0x26, 0x9e, 0xd3, 0x63, 0xd1, 0x19, 0x43, 0x73, 0x83, - 0xb3, 0xd6, 0x1c, 0x8c, 0x97, 0xd0, 0x9b, 0x00, 0x2a, 0x7c, 0x0f, 0x5d, 0x87, 0xa3, 0x6a, 0xb2, - 0xbe, 0x37, 0x19, 0xb7, 0x1a, 0x52, 0x64, 0x12, 0xdc, 0x5d, 0x87, 0x2b, 0xf2, 0x5c, 0x78, 0x7f, - 0x2d, 0xf9, 0xdb, 0x50, 0xdf, 0x96, 0xb7, 0x6e, 0x3f, 0xda, 0x3a, 0x0a, 0xa2, 0x18, 0xcd, 0x8f, - 0xd2, 0x5a, 0xf3, 0x08, 0xbc, 0x84, 0x3e, 0x00, 0xbb, 0x1f, 0x9d, 0x2b, 0xfa, 0x67, 0x73, 0xb7, - 0x31, 0x13, 0xdc, 0x5a, 0x8c, 0xc6, 0x4b, 0x9b, 0x7f, 0x2b, 0x80, 0xf5, 0x69, 0x10, 0x9d, 0xb2, - 0x08, 0x6d, 0x80, 0x25, 0xdf, 0x6a, 0xb9, 0x50, 0x4a, 0xdf, 0x6e, 0x8b, 0xc4, 0xde, 0x81, 0x8a, - 0x34, 0x5a, 0x9f, 0xf2, 0xd3, 0xcc, 0x4d, 0xf2, 0x7f, 0xbd, 0xcc, 0x6e, 0xaa, 0xde, 0xe3, 0x25, - 0xf4, 0x5d, 0xb8, 0x91, 0x16, 0x81, 0x2d, 0xdf, 0x51, 0x45, 0xb5, 0x4d, 0x63, 0x8a, 0x56, 0xe6, - 0x1f, 0x52, 0xe2, 0x12, 0xb5, 0xaa, 0xd9, 0x7b, 0xa9, 0x27, 0x3d, 0x75, 0x0f, 0x8a, 0x3d, 0xa1, - 0x61, 0xf6, 0xaf, 0x5c, 0xf6, 0xff, 0x44, 0x0b, 0xe5, 0x91, 0xa9, 0xcc, 0xf7, 0xc0, 0x52, 0x72, - 0x32, 0xb3, 0xcc, 0x74, 0x3a, 0xad, 0x95, 0x79, 0xb4, 0x66, 0xbc, 0x05, 0xe5, 0x83, 0x49, 0x34, - 0x62, 0x7d, 0x3e, 0x17, 0x47, 0x79, 0x07, 0xe2, 0x25, 0xf4, 0x3e, 0x58, 0x2a, 0xa5, 0x67, 0x02, - 0x66, 0x52, 0x7c, 0x6b, 0x31, 0x1a, 0x2f, 0xa1, 0x7b, 0xd0, 0x20, 0x6c, 0xc8, 0xdc, 0x5c, 0x69, - 0x44, 0x79, 0x95, 0xe7, 0xef, 0xe1, 0xba, 0x81, 0x3e, 0x84, 0xfa, 0x4c, 0x29, 0x45, 0x69, 0x59, - 0x59, 0x54, 0x61, 0xe7, 0x37, 0x78, 0xd0, 0xf8, 0xed, 0x57, 0xab, 0xc6, 0x1f, 0xbf, 0x5a, 0x35, - 0xfe, 0xfc, 0xd5, 0xaa, 0xf1, 0x93, 0xbf, 0xae, 0x2e, 0x1d, 0x59, 0xf2, 0xdf, 0xe4, 0xb7, 0xff, - 0x15, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x5c, 0x91, 0x4e, 0x72, 0x1e, 0x00, 0x00, + // 3060 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x39, 0x4b, 0x6f, 0x23, 0xc7, + 0xd1, 0x9a, 0x21, 0x39, 0x1c, 0x16, 0x49, 0x2d, 0xdd, 0x5e, 0xef, 0xd2, 0xb4, 0x3f, 0x59, 0x1e, + 0xdb, 0xbb, 0xb2, 0xbd, 0x96, 0xbd, 0xf2, 0xfa, 0xf1, 0xf9, 0xfb, 0x1c, 0x40, 0x2b, 0x71, 0xd7, + 0xf4, 0xea, 0xe5, 0x26, 0xb5, 0x8e, 0x7d, 0x08, 0xd1, 0xe2, 0xb4, 0xa4, 0x81, 0x86, 0x33, 0xe3, + 0xe9, 0xa1, 0x40, 0xf9, 0x98, 0x5b, 0x90, 0x3f, 0x10, 0xe4, 0x18, 0x20, 0xb7, 0x00, 0x01, 0x72, + 0xcc, 0x3d, 0x40, 0x02, 0x04, 0x48, 0x72, 0xce, 0x25, 0x71, 0x6e, 0x01, 0x72, 0xc8, 0x3f, 0x08, + 0xba, 0xba, 0xe7, 0x41, 0x8a, 0x2b, 0x6f, 0x5e, 0x27, 0x4e, 0x55, 0x57, 0x75, 0x75, 0x3d, 0xba, + 0xaa, 0xba, 0x08, 0xcb, 0x5e, 0x90, 0xf0, 0x38, 0x60, 0xfe, 0x7a, 0x14, 0x87, 0x49, 0x48, 0x2c, + 0x05, 0x77, 0x6a, 0x2c, 0xf2, 0x14, 0xca, 0xe9, 0x40, 0x79, 0xc7, 0x13, 0x09, 0x21, 0x50, 0x9e, + 0x78, 0xae, 0x68, 0x1b, 0xab, 0xa5, 0x35, 0x8b, 0xe2, 0xb7, 0xf3, 0x19, 0xd4, 0x06, 0x4c, 0x9c, + 0x3d, 0x66, 0xfe, 0x84, 0x93, 0x16, 0x94, 0xce, 0x99, 0xdf, 0x36, 0x56, 0x8d, 0xb5, 0x06, 0x95, + 0x9f, 0x64, 0x03, 0xec, 0x73, 0xe6, 0x0f, 0x93, 0x8b, 0x88, 0xb7, 0xcd, 0x55, 0x63, 0x6d, 0x79, + 0xe3, 0xe6, 0xba, 0x12, 0xb0, 0x7e, 0x10, 0x8a, 0xc4, 0x0b, 0x4e, 0xd6, 0x1f, 0x33, 0x7f, 0x70, + 0x11, 0x71, 0x5a, 0x3d, 0x57, 0x1f, 0xce, 0x3e, 0xd4, 0xfb, 0xf1, 0xe8, 0xc1, 0x24, 0x18, 0x25, + 0x5e, 0x18, 0x48, 0xa9, 0x01, 0x1b, 0x73, 0xdc, 0xb5, 0x46, 0xf1, 0x5b, 0xe2, 0x58, 0x7c, 0x22, + 0xda, 0xa5, 0xd5, 0x92, 0xc4, 0xc9, 0x6f, 0xd2, 0x86, 0xaa, 0x27, 0xb6, 0xc2, 0x49, 0x90, 0xb4, + 0xcb, 0xab, 0xc6, 0x9a, 0x4d, 0x53, 0xd0, 0xf9, 0x59, 0x09, 0x2a, 0x9f, 0x4d, 0x78, 0x7c, 0x81, + 0x7c, 0x49, 0x12, 0xa7, 0x7b, 0xc9, 0x6f, 0x72, 0x1d, 0x2a, 0x3e, 0x0b, 0x4e, 0x44, 0xdb, 0xc4, + 0xcd, 0x14, 0x40, 0x5e, 0x80, 0x1a, 0x3b, 0x4e, 0x78, 0x3c, 0x9c, 0x78, 0x6e, 0xbb, 0xb4, 0x6a, + 0xac, 0x59, 0xd4, 0x46, 0xc4, 0xa1, 0xe7, 0x92, 0xe7, 0xc1, 0x76, 0xc3, 0xe1, 0xa8, 0x28, 0xcb, + 0x0d, 0x51, 0x16, 0xb9, 0x0d, 0xf6, 0xc4, 0x73, 0x87, 0xbe, 0x27, 0x92, 0x76, 0x65, 0xd5, 0x58, + 0xab, 0x6f, 0x34, 0x52, 0x85, 0xa5, 0x0d, 0x69, 0x75, 0xe2, 0xb9, 0x68, 0xcc, 0x75, 0xb0, 0x45, + 0x3c, 0x1a, 0x1e, 0x4f, 0x82, 0x51, 0xdb, 0x42, 0xc2, 0x67, 0x53, 0xc2, 0x82, 0xf6, 0xb4, 0x2a, + 0x14, 0x20, 0xd5, 0x8b, 0xf9, 0x39, 0x8f, 0x05, 0x6f, 0x57, 0x95, 0x48, 0x0d, 0x92, 0x7b, 0x50, + 0x3f, 0x66, 0x23, 0x9e, 0x0c, 0x23, 0x16, 0xb3, 0x71, 0xdb, 0x9e, 0xdd, 0xec, 0x81, 0x5c, 0x3a, + 0x90, 0x2b, 0x82, 0xc2, 0x71, 0x06, 0x90, 0x0f, 0xa0, 0x89, 0x90, 0x18, 0x1e, 0x7b, 0x7e, 0xc2, + 0xe3, 0x76, 0x0d, 0xf9, 0x48, 0xc6, 0x87, 0xd8, 0x41, 0xcc, 0x39, 0x6d, 0x28, 0x42, 0x85, 0x21, + 0xff, 0x03, 0xc0, 0xa7, 0x11, 0x0b, 0xdc, 0x21, 0xf3, 0xfd, 0x36, 0xe0, 0x59, 0x6a, 0x0a, 0xb3, + 0xe9, 0xfb, 0xe4, 0xa6, 0x3c, 0x27, 0x73, 0x87, 0x89, 0x68, 0x37, 0x57, 0x8d, 0xb5, 0x32, 0xb5, + 0x24, 0x38, 0x10, 0xd2, 0x32, 0xbe, 0x17, 0x0c, 0x25, 0xd4, 0x5e, 0xd6, 0x96, 0x91, 0x31, 0xb6, + 0xe3, 0x05, 0x94, 0x33, 0x97, 0x56, 0x7d, 0xf5, 0xe1, 0xbc, 0x0f, 0x35, 0x0c, 0x27, 0x34, 0xd3, + 0xeb, 0x60, 0x9d, 0x4b, 0x40, 0x45, 0x5d, 0x7d, 0xe3, 0x99, 0xf4, 0x7c, 0x59, 0xd4, 0x51, 0x4d, + 0xe0, 0xac, 0x80, 0xbd, 0xc3, 0x82, 0x93, 0x34, 0x54, 0xa5, 0x1f, 0x91, 0xa9, 0x46, 0xf1, 0xdb, + 0xf9, 0x83, 0x09, 0x16, 0xe5, 0x62, 0xe2, 0x27, 0xe4, 0x4d, 0x00, 0xe9, 0xa5, 0x31, 0x4b, 0x62, + 0x6f, 0xaa, 0x77, 0x9e, 0xf5, 0x53, 0x6d, 0xe2, 0xb9, 0xbb, 0xb8, 0x4c, 0xee, 0x41, 0x03, 0x25, + 0xa4, 0xe4, 0xe6, 0xec, 0x41, 0xb2, 0xb3, 0xd2, 0x3a, 0x92, 0x69, 0xae, 0x1b, 0x60, 0x61, 0x80, + 0xa8, 0x20, 0x6d, 0x52, 0x0d, 0x91, 0xd7, 0xf4, 0x8d, 0x13, 0x7c, 0x94, 0x0c, 0x5d, 0x2e, 0xd2, + 0x08, 0x6a, 0x66, 0xd8, 0x6d, 0x2e, 0x12, 0xf2, 0x1e, 0x28, 0xab, 0xa7, 0x42, 0x2b, 0x28, 0x94, + 0xcc, 0x78, 0x55, 0x28, 0xa9, 0x48, 0xa7, 0xa5, 0xde, 0x85, 0xba, 0xd4, 0x35, 0xe5, 0xb2, 0x90, + 0xab, 0x95, 0x69, 0xa6, 0xcd, 0x43, 0x41, 0x12, 0x69, 0x16, 0x69, 0x2a, 0x19, 0xad, 0x2a, 0xaa, + 0xf0, 0xfb, 0xe9, 0x7d, 0xd5, 0x85, 0xca, 0x7e, 0xec, 0xf2, 0x78, 0xe1, 0xcd, 0x22, 0x50, 0x76, + 0xb9, 0x18, 0xe1, 0xc5, 0xb7, 0x29, 0x7e, 0xe7, 0xb7, 0xad, 0x54, 0xb8, 0x6d, 0xce, 0xef, 0x0c, + 0xa8, 0xf7, 0xc3, 0x38, 0xd9, 0xe5, 0x42, 0xb0, 0x13, 0x4e, 0x5e, 0x81, 0x4a, 0x28, 0xb7, 0xd5, + 0xae, 0x69, 0xa6, 0x0a, 0xa0, 0x2c, 0xaa, 0xd6, 0xe6, 0x9c, 0x68, 0x5e, 0xed, 0xc4, 0xeb, 0x50, + 0x51, 0xf7, 0x55, 0xde, 0xe5, 0x0a, 0x55, 0x80, 0x74, 0x52, 0x78, 0x7c, 0x2c, 0xb8, 0x72, 0x42, + 0x85, 0x6a, 0xe8, 0x3f, 0x10, 0xc4, 0x47, 0x00, 0x52, 0xa1, 0x7f, 0x25, 0xde, 0x9e, 0x5a, 0xc6, + 0x29, 0xd4, 0x29, 0x3b, 0x4e, 0xb6, 0xc2, 0x20, 0xe1, 0xd3, 0x84, 0x2c, 0x83, 0xe9, 0xb9, 0xe8, + 0x00, 0x8b, 0x9a, 0x9e, 0x2b, 0x55, 0x3e, 0x89, 0xc3, 0x49, 0x84, 0xf6, 0x6f, 0x52, 0x05, 0xa0, + 0xa3, 0x5c, 0x37, 0x46, 0x3b, 0x48, 0x47, 0xb9, 0x6e, 0x4c, 0x5e, 0x82, 0xba, 0x08, 0x58, 0x24, + 0x4e, 0xc3, 0x44, 0xaa, 0x5c, 0x46, 0x95, 0x21, 0x45, 0x0d, 0x84, 0xf3, 0x2b, 0x03, 0xac, 0x5d, + 0x3e, 0x3e, 0xe2, 0xf1, 0x25, 0x29, 0xcf, 0x83, 0x8d, 0x1b, 0x0f, 0x3d, 0x57, 0x0b, 0xaa, 0x22, + 0xdc, 0x73, 0x17, 0x8a, 0xba, 0x01, 0x96, 0xcf, 0x99, 0x74, 0xad, 0x0a, 0x7b, 0x0d, 0x49, 0x8b, + 0xb3, 0xf1, 0xd0, 0x95, 0x3a, 0x57, 0xd4, 0x02, 0x1b, 0x6f, 0x73, 0xe6, 0xca, 0xb3, 0xf9, 0x4c, + 0x24, 0xc3, 0x49, 0xe4, 0xb2, 0x84, 0x63, 0xaa, 0x2c, 0xcb, 0xf8, 0x15, 0xc9, 0x21, 0x62, 0xc8, + 0x1b, 0xf0, 0xcc, 0xc8, 0x9f, 0x08, 0x99, 0xab, 0xbd, 0xe0, 0x38, 0x1c, 0x86, 0x81, 0x7f, 0x81, + 0x5e, 0xb3, 0xe9, 0x35, 0xbd, 0xd0, 0x0b, 0x8e, 0xc3, 0xfd, 0xc0, 0xbf, 0x70, 0x7e, 0x68, 0x42, + 0xe5, 0x21, 0x9a, 0xe1, 0x1e, 0x54, 0xc7, 0xa8, 0x50, 0x9a, 0x58, 0x3a, 0xa9, 0x3b, 0x70, 0x7d, + 0x5d, 0x69, 0x2b, 0xba, 0x41, 0x12, 0x5f, 0xd0, 0x94, 0x54, 0x72, 0x25, 0xec, 0xc8, 0xe7, 0x89, + 0xd0, 0xf1, 0x36, 0xc7, 0x35, 0x50, 0x8b, 0x9a, 0x4b, 0x93, 0x76, 0x3e, 0x85, 0x46, 0x71, 0x3b, + 0x59, 0x26, 0xcf, 0xf8, 0x05, 0xda, 0xb0, 0x4c, 0xe5, 0x27, 0x79, 0x15, 0x2a, 0x98, 0x3b, 0xd0, + 0x82, 0xf5, 0x8d, 0xe5, 0x74, 0x57, 0xc5, 0x46, 0xd5, 0xe2, 0x47, 0xe6, 0x87, 0x86, 0xdc, 0xab, + 0x28, 0xa4, 0xb8, 0x57, 0xed, 0xea, 0xbd, 0x14, 0x5b, 0x61, 0x2f, 0xe7, 0x6f, 0x06, 0x34, 0xbe, + 0xe4, 0x71, 0x78, 0x10, 0x87, 0x51, 0x28, 0x98, 0x4f, 0x6e, 0x81, 0xa5, 0x34, 0x7d, 0xc2, 0x39, + 0xf4, 0xaa, 0xa4, 0x53, 0xba, 0xa1, 0x6b, 0x2f, 0xcb, 0xd0, 0xab, 0x64, 0x05, 0x60, 0xcc, 0xa6, + 0x3b, 0x9c, 0x09, 0xde, 0x73, 0xd3, 0xb0, 0xca, 0x31, 0xa4, 0x03, 0xf6, 0x98, 0x4d, 0x07, 0xd3, + 0x60, 0x20, 0xd0, 0xeb, 0x65, 0x9a, 0xc1, 0xe4, 0x45, 0xa8, 0x8d, 0xd9, 0x54, 0xc6, 0x77, 0xcf, + 0xd5, 0x5e, 0xcf, 0x11, 0xe4, 0x65, 0x28, 0x25, 0xd3, 0x00, 0x73, 0x56, 0x7d, 0xe3, 0x1a, 0x5e, + 0x8f, 0xc1, 0x34, 0xd0, 0x37, 0x81, 0xca, 0xb5, 0xd4, 0x32, 0x76, 0x66, 0x19, 0xe7, 0x97, 0x25, + 0xb8, 0xa6, 0x1d, 0x71, 0xea, 0x45, 0xfd, 0x44, 0x46, 0x4f, 0x1b, 0xaa, 0x98, 0x0a, 0x78, 0xac, + 0xfd, 0x91, 0x82, 0xe4, 0xff, 0xc0, 0xc2, 0x40, 0x4e, 0x5d, 0xfd, 0xca, 0xac, 0x31, 0xb2, 0x2d, + 0x94, 0xeb, 0xb5, 0xcf, 0x35, 0x0b, 0xf9, 0x10, 0x2a, 0x5f, 0xf3, 0x38, 0x54, 0x69, 0xae, 0xbe, + 0xe1, 0x3c, 0x89, 0x57, 0x9a, 0x5f, 0xb3, 0x2a, 0x86, 0xff, 0xa2, 0xcd, 0xd6, 0x64, 0x52, 0x1b, + 0x87, 0xe7, 0xdc, 0x6d, 0x57, 0xf1, 0x54, 0xf3, 0xee, 0x4d, 0x97, 0x3b, 0x9f, 0x40, 0xbd, 0xa0, + 0x54, 0x31, 0xc6, 0x9a, 0x2a, 0xc6, 0x5e, 0x99, 0x8d, 0xb1, 0xe6, 0xcc, 0x2d, 0x28, 0x86, 0xeb, + 0x27, 0x00, 0xb9, 0x8a, 0xff, 0x4e, 0xe0, 0x3b, 0x3f, 0x30, 0xe0, 0xda, 0x56, 0x18, 0x04, 0x1c, + 0xfb, 0x22, 0xe5, 0xbc, 0x3c, 0x5e, 0x8d, 0x2b, 0xe3, 0xf5, 0x2d, 0xa8, 0x08, 0xc9, 0xa0, 0xa5, + 0xdc, 0x7c, 0x82, 0x37, 0xa8, 0xa2, 0x92, 0x29, 0x67, 0xcc, 0xa6, 0xc3, 0x88, 0x07, 0xae, 0x17, + 0x9c, 0x60, 0x8c, 0x2b, 0x1f, 0x1c, 0x28, 0x8c, 0xf3, 0x13, 0x03, 0x2c, 0x15, 0xea, 0x33, 0xe9, + 0xcf, 0x98, 0x4d, 0x7f, 0x2f, 0x42, 0x2d, 0x8a, 0xb9, 0xeb, 0x8d, 0x52, 0xc9, 0x35, 0x9a, 0x23, + 0x64, 0x76, 0x3e, 0x0e, 0xe3, 0x11, 0xc7, 0xed, 0x6d, 0xaa, 0x00, 0xd9, 0x76, 0x62, 0xe1, 0xc1, + 0x24, 0xa6, 0x32, 0xa4, 0x2d, 0x11, 0x32, 0x7b, 0x49, 0x16, 0x11, 0xb1, 0x91, 0x6a, 0x00, 0x4b, + 0x54, 0x01, 0x32, 0xa3, 0x2a, 0xbf, 0x61, 0xa8, 0xdb, 0x54, 0x43, 0xce, 0x2f, 0x4c, 0x68, 0x6c, + 0x7b, 0x31, 0x1f, 0x25, 0xdc, 0xed, 0xba, 0x27, 0x48, 0xc8, 0x83, 0xc4, 0x4b, 0x2e, 0x74, 0xf6, + 0xd6, 0x50, 0x56, 0xba, 0xcd, 0xd9, 0xa6, 0x58, 0xf9, 0xa5, 0x84, 0xbd, 0xbc, 0x02, 0xc8, 0xfb, + 0x00, 0xaa, 0x13, 0xc2, 0x7e, 0xbe, 0x7c, 0x75, 0x3f, 0x5f, 0x43, 0x52, 0xf9, 0x29, 0x8d, 0xa4, + 0xf8, 0x3c, 0x95, 0xdd, 0x2d, 0x6c, 0xf6, 0x27, 0x32, 0x9c, 0xb1, 0x1f, 0x38, 0xe2, 0x3e, 0x86, + 0x2b, 0xf6, 0x03, 0x47, 0xdc, 0xcf, 0xda, 0xb7, 0xaa, 0x3a, 0x92, 0xfc, 0x26, 0xb7, 0xc1, 0x0c, + 0x23, 0xd4, 0xb1, 0x20, 0xb4, 0xa8, 0xe0, 0xfa, 0x7e, 0x44, 0xcd, 0x30, 0x22, 0x0e, 0x58, 0xaa, + 0x61, 0x6d, 0xd7, 0x30, 0xcc, 0x01, 0xd3, 0x03, 0x76, 0x4c, 0x54, 0xaf, 0x38, 0x37, 0xc0, 0xdc, + 0x8f, 0x48, 0x15, 0x4a, 0xfd, 0xee, 0xa0, 0xb5, 0x24, 0x3f, 0xb6, 0xbb, 0x3b, 0x2d, 0xc3, 0xf9, + 0xab, 0x01, 0xb5, 0xdd, 0x49, 0xc2, 0x64, 0x8c, 0x89, 0xab, 0x9c, 0xfb, 0x3c, 0xd8, 0x22, 0x61, + 0x31, 0xd6, 0x4b, 0x53, 0x25, 0x0e, 0x84, 0x07, 0x82, 0xbc, 0x01, 0x15, 0xee, 0x9e, 0xf0, 0xf4, + 0xee, 0x5f, 0x5f, 0x74, 0x56, 0xaa, 0x48, 0xc8, 0x1d, 0xb0, 0xc4, 0xe8, 0x94, 0x8f, 0x59, 0xbb, + 0x3c, 0x4b, 0xdc, 0x47, 0xac, 0x2a, 0x71, 0x54, 0xd3, 0xe0, 0xbb, 0x23, 0x0e, 0x23, 0x6c, 0xbc, + 0x2b, 0xfa, 0xdd, 0x11, 0x87, 0x91, 0x6c, 0xbb, 0x37, 0xe0, 0x39, 0xef, 0x24, 0x08, 0x63, 0x3e, + 0xf4, 0x02, 0x97, 0x4f, 0x87, 0xa3, 0x30, 0x38, 0xf6, 0xbd, 0x51, 0x82, 0x76, 0xb5, 0xe9, 0xb3, + 0x6a, 0xb1, 0x27, 0xd7, 0xb6, 0xf4, 0x92, 0x73, 0x1b, 0x6a, 0x8f, 0xf8, 0x05, 0xf6, 0xaf, 0x82, + 0x74, 0xc0, 0x3c, 0x3b, 0xd7, 0xb5, 0x10, 0xd2, 0x53, 0x3c, 0x7a, 0x4c, 0xcd, 0xb3, 0x73, 0xe7, + 0x14, 0xec, 0xbe, 0x6e, 0x06, 0xc8, 0x5b, 0x32, 0x61, 0x62, 0x9a, 0xd5, 0x97, 0x2e, 0x7b, 0x69, + 0x14, 0x7a, 0x11, 0x9a, 0xd2, 0x48, 0xff, 0xe2, 0x81, 0xb4, 0x91, 0x14, 0x50, 0xec, 0xaf, 0x4a, + 0xc5, 0xfe, 0xca, 0xf9, 0xb1, 0x09, 0x76, 0x56, 0x8e, 0xde, 0x86, 0xda, 0x38, 0xf5, 0x85, 0xbe, + 0xba, 0x59, 0xd7, 0x9d, 0x39, 0x89, 0xe6, 0x34, 0x5a, 0x87, 0xf2, 0x22, 0x1d, 0xf2, 0x1c, 0x50, + 0x79, 0xaa, 0x1c, 0x70, 0x1b, 0xae, 0x8d, 0x7c, 0xce, 0x82, 0x61, 0x7e, 0x85, 0x55, 0x84, 0x2e, + 0x23, 0xfa, 0x20, 0xbb, 0xc7, 0x3a, 0xa7, 0x55, 0xf3, 0x02, 0xfc, 0x3a, 0x54, 0x5c, 0xee, 0x27, + 0x6c, 0xfe, 0x25, 0xb6, 0x1f, 0xb3, 0x91, 0xcf, 0xb7, 0xe5, 0x12, 0x55, 0x14, 0xe4, 0x0e, 0xd8, + 0x69, 0x97, 0xa5, 0xdf, 0x5f, 0x59, 0xaf, 0x9e, 0x1a, 0x9c, 0x66, 0x14, 0xce, 0xcb, 0x50, 0x7a, + 0xf4, 0xb8, 0x7f, 0xa5, 0xa7, 0xbe, 0x07, 0xe6, 0xa3, 0xc7, 0xc5, 0x3c, 0xdb, 0x50, 0x67, 0xd2, + 0x2f, 0x73, 0x33, 0x7f, 0x99, 0x77, 0xc0, 0x9e, 0x08, 0x1e, 0xef, 0xf2, 0x84, 0xe9, 0x4b, 0x9e, + 0xc1, 0xb2, 0x28, 0xca, 0xa7, 0xa5, 0x17, 0x06, 0xba, 0x00, 0xa5, 0xa0, 0xf3, 0xf7, 0x12, 0x54, + 0xf5, 0x45, 0x97, 0x7b, 0x4e, 0xb2, 0x56, 0x50, 0x7e, 0xe6, 0x59, 0xc3, 0x2c, 0x66, 0x8d, 0xe2, + 0x0c, 0xa0, 0xf4, 0x74, 0x33, 0x00, 0xf2, 0x1d, 0x68, 0x44, 0x6a, 0xad, 0x98, 0x6b, 0x5e, 0x98, + 0xe7, 0xd3, 0xbf, 0xc8, 0x5b, 0x8f, 0x72, 0x40, 0xde, 0x14, 0x7c, 0x07, 0x25, 0xec, 0x04, 0x1d, + 0xde, 0xa0, 0x55, 0x09, 0x0f, 0xd8, 0xc9, 0x13, 0x32, 0xce, 0x53, 0x24, 0x0d, 0xd9, 0xfa, 0x86, + 0x51, 0xbb, 0x81, 0x89, 0x40, 0x26, 0x9a, 0x62, 0x0e, 0x68, 0xce, 0xe6, 0x80, 0x17, 0xa0, 0x36, + 0x0a, 0xc7, 0x63, 0x0f, 0xd7, 0x96, 0x55, 0x99, 0x56, 0x88, 0x81, 0x70, 0xbe, 0x86, 0xaa, 0x56, + 0x98, 0xd4, 0xa1, 0xba, 0xdd, 0x7d, 0xb0, 0x79, 0xb8, 0x23, 0xb3, 0x10, 0x80, 0x75, 0xbf, 0xb7, + 0xb7, 0x49, 0xbf, 0x68, 0x19, 0x32, 0x23, 0xf5, 0xf6, 0x06, 0x2d, 0x93, 0xd4, 0xa0, 0xf2, 0x60, + 0x67, 0x7f, 0x73, 0xd0, 0x2a, 0x11, 0x1b, 0xca, 0xf7, 0xf7, 0xf7, 0x77, 0x5a, 0x65, 0xd2, 0x00, + 0x7b, 0x7b, 0x73, 0xd0, 0x1d, 0xf4, 0x76, 0xbb, 0xad, 0x8a, 0xa4, 0x7d, 0xd8, 0xdd, 0x6f, 0x59, + 0xf2, 0xe3, 0xb0, 0xb7, 0xdd, 0xaa, 0xca, 0xf5, 0x83, 0xcd, 0x7e, 0xff, 0xf3, 0x7d, 0xba, 0xdd, + 0xb2, 0xe5, 0xbe, 0xfd, 0x01, 0xed, 0xed, 0x3d, 0x6c, 0xd5, 0x9c, 0xbb, 0x50, 0x2f, 0x18, 0x4d, + 0x72, 0xd0, 0xee, 0x83, 0xd6, 0x92, 0x14, 0xf3, 0x78, 0x73, 0xe7, 0xb0, 0xdb, 0x32, 0xc8, 0x32, + 0x00, 0x7e, 0x0e, 0x77, 0x36, 0xf7, 0x1e, 0xb6, 0x4c, 0xe7, 0xfb, 0x46, 0xc6, 0x83, 0x6f, 0xeb, + 0x37, 0xc1, 0xd6, 0xa6, 0x4e, 0x7b, 0xe7, 0x6b, 0x73, 0x7e, 0xa1, 0x19, 0x81, 0x0c, 0xb3, 0xd1, + 0x29, 0x1f, 0x9d, 0x89, 0xc9, 0x58, 0x47, 0x45, 0x06, 0xab, 0x27, 0xb2, 0xb4, 0x49, 0x9a, 0x04, + 0x14, 0x94, 0xcd, 0x99, 0xca, 0x48, 0xaf, 0xe6, 0x4c, 0xf7, 0x00, 0xf2, 0x49, 0xc6, 0x82, 0xae, + 0xf7, 0x3a, 0x54, 0x98, 0xef, 0x31, 0xa1, 0xab, 0x98, 0x02, 0x1c, 0x0a, 0xf5, 0xc2, 0xfc, 0x43, + 0x3a, 0x8c, 0xf9, 0xfe, 0xf0, 0x8c, 0x5f, 0x08, 0xe4, 0xb5, 0x69, 0x95, 0xf9, 0xfe, 0x23, 0x7e, + 0x21, 0xc8, 0x1a, 0x54, 0xd4, 0xf8, 0xc4, 0x5c, 0xf0, 0xd0, 0x46, 0x76, 0xaa, 0x08, 0x9c, 0x3b, + 0x60, 0xa9, 0xd7, 0x77, 0x21, 0x66, 0x8c, 0x27, 0x16, 0x9a, 0x8f, 0xf5, 0xb9, 0xf1, 0xad, 0x4e, + 0xde, 0xd6, 0xa3, 0x1a, 0xa1, 0x06, 0x44, 0xc6, 0x6c, 0x1b, 0xa6, 0x08, 0xf5, 0x94, 0x06, 0x19, + 0x9c, 0x6d, 0xb0, 0xaf, 0x1c, 0x84, 0x69, 0x43, 0x98, 0xb9, 0x21, 0x16, 0x8c, 0xc6, 0x9c, 0x18, + 0x20, 0x1f, 0xe7, 0xe8, 0x30, 0x56, 0xbb, 0xc8, 0x30, 0x5e, 0x97, 0x2e, 0xf2, 0x7c, 0x37, 0xe6, + 0xc1, 0x25, 0xed, 0xf3, 0x21, 0x50, 0x46, 0x43, 0x5e, 0x85, 0x32, 0x4e, 0xad, 0x4a, 0xb3, 0x09, + 0x2b, 0x1b, 0x59, 0xe1, 0xaa, 0x73, 0x04, 0x4d, 0x55, 0xc3, 0x28, 0xff, 0x6a, 0xc2, 0xc5, 0x95, + 0x9d, 0xd2, 0x0a, 0x40, 0x96, 0x66, 0xd3, 0x39, 0x5c, 0x01, 0x23, 0x03, 0xe5, 0xd8, 0xe3, 0xbe, + 0x9b, 0x6a, 0xa5, 0x21, 0xe7, 0x03, 0x68, 0xa4, 0x32, 0xf0, 0x99, 0x7d, 0x3b, 0xab, 0xa6, 0x69, + 0x5c, 0x4a, 0x87, 0x28, 0x92, 0xbd, 0xd0, 0xcd, 0x0a, 0xa9, 0xf3, 0x67, 0x33, 0xe5, 0xd4, 0x8f, + 0xc8, 0x99, 0x5e, 0xcd, 0x98, 0xef, 0xd5, 0x66, 0xfb, 0x1e, 0xf3, 0xa9, 0xfb, 0x9e, 0xff, 0x87, + 0x9a, 0x8b, 0x45, 0xdf, 0x3b, 0x4f, 0x53, 0xdf, 0xca, 0xa2, 0x02, 0xaf, 0x5b, 0x03, 0xef, 0x9c, + 0xd3, 0x9c, 0x41, 0x9e, 0x29, 0x09, 0xcf, 0x78, 0xe0, 0x7d, 0x8d, 0xaf, 0x65, 0xa9, 0x78, 0x8e, + 0xc8, 0x07, 0x1a, 0xaa, 0x11, 0xd0, 0x03, 0x8d, 0x74, 0x98, 0x63, 0x15, 0x86, 0x39, 0x37, 0xc0, + 0x9a, 0x44, 0x82, 0xc7, 0x49, 0xda, 0x20, 0x2a, 0x28, 0x6b, 0xb2, 0x6a, 0x9a, 0x96, 0x05, 0x27, + 0xce, 0xff, 0x42, 0x2d, 0x3b, 0x8b, 0xcc, 0x37, 0x7b, 0xfb, 0x7b, 0x5d, 0x95, 0x1d, 0x7a, 0x7b, + 0xdb, 0xdd, 0xef, 0xb6, 0x0c, 0x99, 0xb1, 0x68, 0xf7, 0x71, 0x97, 0xf6, 0xbb, 0x2d, 0x53, 0x66, + 0x96, 0xed, 0xee, 0x4e, 0x77, 0xd0, 0x6d, 0x95, 0x3e, 0x2d, 0xdb, 0xd5, 0x96, 0x4d, 0x6d, 0x3e, + 0x8d, 0x7c, 0x6f, 0xe4, 0x25, 0xce, 0x17, 0x60, 0xef, 0xb2, 0xe8, 0x52, 0xe3, 0x9f, 0x17, 0xa4, + 0x89, 0x9e, 0x18, 0xe8, 0xe2, 0xf1, 0x3a, 0x54, 0x75, 0xd6, 0xd0, 0x91, 0x75, 0x29, 0xab, 0xa4, + 0xeb, 0xce, 0xcf, 0x0d, 0xb8, 0xbe, 0x1b, 0x9e, 0xf3, 0xac, 0x0a, 0x1f, 0xb0, 0x0b, 0x3f, 0x64, + 0xee, 0xb7, 0xb8, 0xf1, 0x16, 0x5c, 0x13, 0xe1, 0x24, 0x1e, 0xf1, 0xe1, 0xdc, 0xc4, 0xa2, 0xa9, + 0xd0, 0x0f, 0x75, 0x38, 0x3a, 0xd0, 0x74, 0xb9, 0x48, 0x72, 0xaa, 0x12, 0x52, 0xd5, 0x25, 0x32, + 0xa5, 0xc9, 0xda, 0x89, 0xf2, 0xd3, 0xb4, 0x13, 0xce, 0x6f, 0x0d, 0x68, 0x76, 0xa7, 0x51, 0x18, + 0x27, 0xe9, 0x51, 0x9f, 0x93, 0x6d, 0xfb, 0x57, 0xe9, 0x65, 0x28, 0xd3, 0x4a, 0xcc, 0xbf, 0xea, + 0x5d, 0x39, 0x4e, 0xb9, 0x07, 0x96, 0xdc, 0x6c, 0x22, 0x74, 0x28, 0xbd, 0x98, 0xca, 0x9c, 0xd9, + 0x78, 0xbd, 0x8f, 0x34, 0x54, 0xd3, 0x16, 0x5b, 0xad, 0xf2, 0x4c, 0xab, 0xf5, 0x11, 0x58, 0x8a, + 0xb4, 0xe0, 0xe7, 0x3a, 0x54, 0xfb, 0x87, 0x5b, 0x5b, 0xdd, 0x7e, 0xbf, 0x65, 0x90, 0x26, 0xd4, + 0xb6, 0x0f, 0x0f, 0x76, 0x7a, 0x5b, 0x9b, 0x03, 0xed, 0xeb, 0x07, 0x9b, 0xbd, 0x9d, 0xee, 0x76, + 0xab, 0xe4, 0x6c, 0x41, 0x6d, 0x30, 0x0d, 0x34, 0x7b, 0xb1, 0x0c, 0x1a, 0x57, 0x94, 0x41, 0x73, + 0xae, 0x0c, 0x7e, 0x0e, 0xf5, 0x42, 0x4b, 0x44, 0x5e, 0x83, 0x72, 0x32, 0x0d, 0x2e, 0xcf, 0x79, + 0x53, 0x39, 0x14, 0x97, 0xc9, 0xcb, 0xd0, 0x90, 0x8f, 0x33, 0x26, 0x84, 0x77, 0x12, 0x70, 0x57, + 0xef, 0x2a, 0x1f, 0x6c, 0x9b, 0x1a, 0xe5, 0xbc, 0x04, 0x4d, 0xf9, 0x1e, 0xf6, 0xc6, 0x5c, 0x24, + 0x6c, 0x1c, 0x61, 0xe1, 0xd6, 0x49, 0xba, 0x4c, 0xcd, 0x44, 0x38, 0xb7, 0xa0, 0x71, 0xc0, 0x79, + 0x4c, 0xb9, 0x88, 0xc2, 0x40, 0xe0, 0xcb, 0x48, 0x5b, 0x56, 0x55, 0x05, 0x0d, 0x39, 0x47, 0x50, + 0x93, 0x4d, 0xed, 0x7d, 0x96, 0x8c, 0x4e, 0xff, 0xd9, 0xc6, 0xf7, 0x16, 0x54, 0x23, 0xe5, 0x11, + 0xdd, 0xba, 0xaa, 0x21, 0x9e, 0xf6, 0x12, 0x4d, 0x17, 0x9d, 0x9b, 0x50, 0xda, 0x9b, 0x8c, 0x8b, + 0x7f, 0x9d, 0x94, 0xb1, 0x41, 0x73, 0x1e, 0x40, 0x23, 0xed, 0x01, 0xb1, 0x29, 0x93, 0xb6, 0xf4, + 0x3d, 0x1e, 0x14, 0xec, 0x6c, 0x2b, 0xc4, 0x40, 0x5c, 0x11, 0x36, 0x1b, 0xbf, 0x31, 0xa0, 0x2c, + 0x4f, 0x28, 0xf3, 0x76, 0x77, 0x74, 0x1a, 0x92, 0x99, 0x83, 0x74, 0x66, 0x20, 0x67, 0x89, 0xbc, + 0xa3, 0x86, 0x8a, 0xe9, 0x24, 0xf6, 0x99, 0xa2, 0x92, 0x68, 0x88, 0x4b, 0x1c, 0x1b, 0x50, 0xff, + 0x34, 0xf4, 0x82, 0x2d, 0x35, 0x6b, 0x23, 0x8b, 0xcc, 0x72, 0x89, 0xe7, 0x3d, 0xb0, 0x7a, 0x42, + 0xfa, 0x60, 0x31, 0x79, 0xf6, 0x0c, 0x2a, 0xba, 0xc9, 0x59, 0xda, 0xf8, 0x69, 0x09, 0xca, 0x5f, + 0xf2, 0x38, 0x24, 0xf7, 0xa0, 0xaa, 0x87, 0x01, 0x64, 0xee, 0xd1, 0xdf, 0xc9, 0xae, 0xe2, 0xdc, + 0xb4, 0xc0, 0x59, 0x22, 0xef, 0x83, 0xa5, 0xf3, 0xfd, 0xec, 0xc4, 0xa2, 0xf3, 0xa4, 0xeb, 0xeb, + 0x2c, 0xad, 0x19, 0xef, 0x18, 0xe4, 0x6d, 0xb0, 0x54, 0xa4, 0xce, 0xd9, 0x6e, 0x51, 0x6b, 0xef, + 0x2c, 0x21, 0x43, 0xbd, 0x7f, 0x1a, 0x4e, 0x7c, 0xb7, 0xcf, 0xe3, 0x73, 0x4e, 0xe6, 0xe6, 0x63, + 0x9d, 0x39, 0xd8, 0x59, 0x22, 0x6f, 0x01, 0xa8, 0xf0, 0x3d, 0xf4, 0x5c, 0x41, 0xea, 0xe9, 0xfa, + 0xde, 0x64, 0xdc, 0x69, 0xa1, 0xc8, 0x34, 0xb8, 0x7b, 0xae, 0x50, 0xe4, 0x85, 0xf0, 0xfe, 0x56, + 0xf2, 0x77, 0xa1, 0xb9, 0x85, 0xb7, 0x6e, 0x3f, 0xde, 0x3c, 0x0a, 0xe3, 0x84, 0xcc, 0x4f, 0xcc, + 0x3a, 0xf3, 0x08, 0x67, 0x89, 0x7c, 0x04, 0xf6, 0x20, 0xbe, 0x50, 0xf4, 0xcf, 0x15, 0x6e, 0x63, + 0x2e, 0xb8, 0xb3, 0x18, 0xed, 0x2c, 0x6d, 0xfc, 0xb1, 0x04, 0xd6, 0xe7, 0x61, 0x7c, 0xc6, 0x63, + 0xb2, 0x0e, 0x16, 0xbe, 0xd5, 0x0a, 0xa1, 0x94, 0xbd, 0xdd, 0x16, 0x89, 0xbd, 0x03, 0x35, 0x34, + 0xda, 0x80, 0x89, 0xb3, 0xdc, 0x4d, 0xf8, 0xf7, 0x5d, 0x6e, 0x37, 0x55, 0xef, 0x51, 0xb3, 0xe5, + 0x7e, 0x12, 0x73, 0x36, 0xce, 0xde, 0xa7, 0x97, 0x1e, 0x50, 0x9d, 0x7a, 0xfe, 0x46, 0xea, 0xa3, + 0x77, 0xee, 0x42, 0xb9, 0x2f, 0xb5, 0xca, 0xff, 0x70, 0xcb, 0xff, 0x7a, 0xe8, 0x90, 0x22, 0x32, + 0x93, 0xf3, 0x01, 0x58, 0xaa, 0x60, 0xe7, 0xa6, 0x98, 0xe9, 0x6e, 0x3a, 0xd7, 0xe7, 0xd1, 0x9a, + 0xf1, 0x16, 0x54, 0x0f, 0x26, 0xf1, 0x09, 0x1f, 0x88, 0xb9, 0xd8, 0x29, 0x3a, 0xcd, 0x59, 0x22, + 0x1f, 0x82, 0xa5, 0xd2, 0x78, 0x2e, 0x60, 0x26, 0xad, 0x77, 0x16, 0xa3, 0x9d, 0x25, 0x72, 0x17, + 0x5a, 0x94, 0x8f, 0xb8, 0x57, 0x28, 0x87, 0xa4, 0xa8, 0xf2, 0xfc, 0xdd, 0x5b, 0x33, 0xc8, 0xc7, + 0xd0, 0x9c, 0x29, 0x9f, 0x24, 0x2b, 0x25, 0x8b, 0xaa, 0xea, 0xfc, 0x06, 0xf7, 0x5b, 0xbf, 0xfe, + 0x66, 0xc5, 0xf8, 0xfd, 0x37, 0x2b, 0xc6, 0x9f, 0xbe, 0x59, 0x31, 0x7e, 0xf4, 0x97, 0x95, 0xa5, + 0x23, 0x0b, 0xff, 0x28, 0x7e, 0xf7, 0x1f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xc7, 0xde, 0xd3, 0xd9, + 0x4d, 0x1e, 0x00, 0x00, } diff --git a/protos/internal.proto b/protos/internal.proto index 1f199510ebb..36097188712 100644 --- a/protos/internal.proto +++ b/protos/internal.proto @@ -194,9 +194,9 @@ message KeyValues { } message Snapshot { - RaftContext context = 1; - uint64 index = 2; - uint64 min_pending_start_ts = 3; + RaftContext context = 1; + uint64 index = 2; + uint64 read_ts = 3; } message Proposal { @@ -401,12 +401,12 @@ service Zero { service Worker { // Data serving RPCs. - rpc Mutate (Mutations) returns (api.TxnContext) {} - rpc ServeTask (Query) returns (Result) {} - rpc PredicateAndSchemaData (SnapshotMeta) returns (stream KVS) {} - rpc Sort (SortMessage) returns (SortResult) {} - rpc Schema (SchemaRequest) returns (SchemaResult) {} - rpc PurgeTs (api.Payload) returns (Num) {} + rpc Mutate (Mutations) returns (api.TxnContext) {} + rpc ServeTask (Query) returns (Result) {} + rpc StreamSnapshot (Snapshot) returns (stream KVS) {} + rpc Sort (SortMessage) returns (SortResult) {} + rpc Schema (SchemaRequest) returns (SchemaResult) {} + rpc PurgeTs (api.Payload) returns (Num) {} rpc Export (ExportPayload) returns (ExportPayload) {} rpc ReceivePredicate(stream KVS) returns (api.Payload) {} diff --git a/vendor/github.com/dgraph-io/badger/db.go b/vendor/github.com/dgraph-io/badger/db.go index dc18576f3dc..a8660a5fc10 100644 --- a/vendor/github.com/dgraph-io/badger/db.go +++ b/vendor/github.com/dgraph-io/badger/db.go @@ -25,6 +25,7 @@ import ( "path/filepath" "strconv" "sync" + "sync/atomic" "time" "github.com/dgraph-io/badger/options" @@ -73,6 +74,8 @@ type DB struct { writeCh chan *request flushChan chan flushTask // For flushing memtables. + blockWrites int32 + orc *oracle } @@ -622,6 +625,9 @@ func (db *DB) writeRequests(reqs []*request) error { } func (db *DB) sendToWriteCh(entries []*Entry) (*request, error) { + if atomic.LoadInt32(&db.blockWrites) == 1 { + return nil, ErrBlockedWrites + } var count, size int64 for _, e := range entries { size += int64(e.estimateSize(db.opt.ValueThreshold)) @@ -850,7 +856,12 @@ func (db *DB) flushMemtable(lc *y.Closer) error { // Update s.imm. Need a lock. db.Lock() - y.AssertTrue(ft.mt == db.imm[0]) //For now, single threaded. + // This is a single-threaded operation. ft.mt corresponds to the head of + // db.imm list. Once we flush it, we advance db.imm. The next ft.mt + // which would arrive here would match db.imm[0], because we acquire a + // lock over DB when pushing to flushChan. + // TODO: This logic is dirty AF. Any change and this could easily break. + y.AssertTrue(ft.mt == db.imm[0]) db.imm = db.imm[1:] ft.mt.DecrRef() // Return memory. db.Unlock() diff --git a/vendor/github.com/dgraph-io/badger/errors.go b/vendor/github.com/dgraph-io/badger/errors.go index c567d3c5e05..1de35826e96 100644 --- a/vendor/github.com/dgraph-io/badger/errors.go +++ b/vendor/github.com/dgraph-io/badger/errors.go @@ -96,6 +96,10 @@ var ( // ErrTruncateNeeded is returned when the value log gets corrupt, and requires truncation of // corrupt data to allow Badger to run properly. ErrTruncateNeeded = errors.New("Value log truncate required to run DB. This might result in data loss.") + + // ErrBlockedWrites is returned if the user called DropAll. During the process of dropping all + // data from Badger, we stop accepting new writes, by returning this error. + ErrBlockedWrites = errors.New("Writes are blocked possibly due to DropAll") ) // Key length can't be more than uint16, as determined by table::header. diff --git a/vendor/github.com/dgraph-io/badger/levels.go b/vendor/github.com/dgraph-io/badger/levels.go index d666c563542..31b7fe6bfd4 100644 --- a/vendor/github.com/dgraph-io/badger/levels.go +++ b/vendor/github.com/dgraph-io/badger/levels.go @@ -170,6 +170,62 @@ func (s *levelsController) cleanupLevels() error { return firstErr } +// This function picks all tables from all levels, creates a manifest changeset, +// applies it, and then decrements the refs of these tables, which would result +// in their deletion. It spares one table from L0, to keep the badgerHead key +// persisted, so we don't lose where we are w.r.t. value log. +// NOTE: This function in itself isn't sufficient to completely delete all the +// data. After this, one would still need to iterate over the KV pairs and mark +// them as deleted. +func (s *levelsController) deleteLSMTree() (int, error) { + var all []*table.Table + var keepOne *table.Table + for _, l := range s.levels { + l.RLock() + if l.level == 0 && len(l.tables) > 1 { + // Skip the last table. We do this to keep the badgerMove key persisted. + lastIdx := len(l.tables) - 1 + keepOne = l.tables[lastIdx] + all = append(all, l.tables[:lastIdx]...) + } else { + all = append(all, l.tables...) + } + l.RUnlock() + } + if len(all) == 0 { + return 0, nil + } + + // Generate the manifest changes. + changes := []*protos.ManifestChange{} + for _, table := range all { + changes = append(changes, makeTableDeleteChange(table.ID())) + } + changeSet := protos.ManifestChangeSet{Changes: changes} + if err := s.kv.manifest.addChanges(changeSet.Changes); err != nil { + return 0, err + } + + for _, l := range s.levels { + l.Lock() + l.totalSize = 0 + if l.level == 0 && len(l.tables) > 1 { + l.tables = []*table.Table{keepOne} + l.totalSize += keepOne.Size() + } else { + l.tables = l.tables[:0] + } + l.Unlock() + } + // Now allow deletion of tables. + for _, table := range all { + if err := table.DecrRef(); err != nil { + return 0, err + } + } + return len(all), nil +} + func (s *levelsController) startCompact(lc *y.Closer) { n := s.kv.opt.NumCompactors lc.AddRunning(n - 1) diff --git a/vendor/github.com/dgraph-io/badger/managed_db.go b/vendor/github.com/dgraph-io/badger/managed_db.go index c33cb343489..c3898130e90 100644 --- a/vendor/github.com/dgraph-io/badger/managed_db.go +++ b/vendor/github.com/dgraph-io/badger/managed_db.go @@ -16,6 +16,16 @@ package badger +import ( + "math" + "sync" + "sync/atomic" + "time" + + "github.com/dgraph-io/badger/y" + "github.com/pkg/errors" +) + // ManagedDB allows end users to manage the transactions themselves. Transaction // start and commit timestamps are set by end-user. // @@ -84,3 +94,100 @@ func (db *ManagedDB) GetSequence(_ []byte, _ uint64) (*Sequence, error) { func (db *ManagedDB) SetDiscardTs(ts uint64) { db.orc.setDiscardTs(ts) } + +var errDone = errors.New("Done deleting keys") + +// DropAll would drop all the data stored in Badger. It does this in the following way. +// - Stop accepting new writes. +// - Pause the compactions. +// - Pick all tables from all levels, create a changeset to delete all these +// tables and apply it to manifest. DO not pick up the latest table from level +// 0, to preserve the (persistent) badgerHead key. +// - Iterate over the KVs in Level 0, and run deletes on them via transactions. +// - The deletions are done at the same timestamp as the latest version of the +// key. Thus, we could write the keys back at the same timestamp as before. +func (db *ManagedDB) DropAll() error { + // Stop accepting new writes. + atomic.StoreInt32(&db.blockWrites, 1) + + // Wait for writeCh to reach size of zero. This is not ideal, but a very + // simple way to allow writeCh to flush out, before we proceed. + tick := time.NewTicker(100 * time.Millisecond) + for range tick.C { + if len(db.writeCh) == 0 { + break + } + } + tick.Stop() + + // Stop the compactions. + if db.closers.compactors != nil { + db.closers.compactors.SignalAndWait() + } + + _, err := db.lc.deleteLSMTree() + // Allow writes so that we can run transactions. Ideally, the user must ensure that they're not + // doing more writes concurrently while this operation is happening. + atomic.StoreInt32(&db.blockWrites, 0) + // Need compactions to happen so deletes below can be flushed out. + if db.closers.compactors != nil { + db.closers.compactors = y.NewCloser(1) + db.lc.startCompact(db.closers.compactors) + } + if err != nil { + return err + } + + type KV struct { + key []byte + version uint64 + } + + var kvs []KV + getKeys := func() error { + txn := db.NewTransactionAt(math.MaxUint64, false) + defer txn.Discard() + + opts := DefaultIteratorOptions + opts.PrefetchValues = false + itr := txn.NewIterator(opts) + defer itr.Close() + + for itr.Rewind(); itr.Valid(); itr.Next() { + item := itr.Item() + kvs = append(kvs, KV{item.KeyCopy(nil), item.Version()}) + } + return nil + } + if err := getKeys(); err != nil { + return err + } + + var wg sync.WaitGroup + errCh := make(chan error, 1) + for _, kv := range kvs { + wg.Add(1) + txn := db.NewTransactionAt(math.MaxUint64, true) + if err := txn.Delete(kv.key); err != nil { + return err + } + if err := txn.CommitAt(kv.version, func(rerr error) { + if rerr != nil { + select { + case errCh <- rerr: + default: + } + } + wg.Done() + }); err != nil { + return err + } + } + wg.Wait() + select { + case err := <-errCh: + return err + default: + return nil + } +} diff --git a/vendor/vendor.json b/vendor/vendor.json index c5037e14f12..9707445d604 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -163,50 +163,50 @@ "revisionTime": "2016-09-07T16:21:46Z" }, { - "checksumSHA1": "1qvmTlilsJCsxarfPoawlGF7jQw=", + "checksumSHA1": "sfSALvncyx7ObOeivKM/+V7IMKM=", "path": "github.com/dgraph-io/badger", - "revision": "937af5f844102c2849b070cdb0ec33d82aa97cc3", - "revisionTime": "2018-07-11T22:35:56Z", + "revision": "dade338bf301bc3d6e63167c2ac40395e7c8d4cc", + "revisionTime": "2018-07-12T22:02:33Z", "version": "HEAD", "versionExact": "HEAD" }, { "checksumSHA1": "oOuT7ebEiZ1ViHLKdFxKFOvobAQ=", "path": "github.com/dgraph-io/badger/options", - "revision": "937af5f844102c2849b070cdb0ec33d82aa97cc3", - "revisionTime": "2018-07-11T22:35:56Z", + "revision": "dade338bf301bc3d6e63167c2ac40395e7c8d4cc", + "revisionTime": "2018-07-12T22:02:33Z", "version": "HEAD", "versionExact": "HEAD" }, { "checksumSHA1": "gGTDnTVVw5kcT2P5NXZV1YSckOU=", "path": "github.com/dgraph-io/badger/protos", - "revision": "937af5f844102c2849b070cdb0ec33d82aa97cc3", - "revisionTime": "2018-07-11T22:35:56Z", + "revision": "dade338bf301bc3d6e63167c2ac40395e7c8d4cc", + "revisionTime": "2018-07-12T22:02:33Z", "version": "HEAD", "versionExact": "HEAD" }, { "checksumSHA1": "00T6XbLV4d95J7hm6kTXDReaQHM=", "path": "github.com/dgraph-io/badger/skl", - "revision": "937af5f844102c2849b070cdb0ec33d82aa97cc3", - "revisionTime": "2018-07-11T22:35:56Z", + "revision": "dade338bf301bc3d6e63167c2ac40395e7c8d4cc", + "revisionTime": "2018-07-12T22:02:33Z", "version": "HEAD", "versionExact": "HEAD" }, { "checksumSHA1": "I33KkP2lnYqJDasvvsAlebzkeko=", "path": "github.com/dgraph-io/badger/table", - "revision": "937af5f844102c2849b070cdb0ec33d82aa97cc3", - "revisionTime": "2018-07-11T22:35:56Z", + "revision": "dade338bf301bc3d6e63167c2ac40395e7c8d4cc", + "revisionTime": "2018-07-12T22:02:33Z", "version": "HEAD", "versionExact": "HEAD" }, { "checksumSHA1": "v2pJQ5NbS034cLP+GM1WLlGnByY=", "path": "github.com/dgraph-io/badger/y", - "revision": "937af5f844102c2849b070cdb0ec33d82aa97cc3", - "revisionTime": "2018-07-11T22:35:56Z", + "revision": "dade338bf301bc3d6e63167c2ac40395e7c8d4cc", + "revisionTime": "2018-07-12T22:02:33Z", "version": "HEAD", "versionExact": "HEAD" }, @@ -271,10 +271,10 @@ "revisionTime": "2017-08-10T00:29:00Z" }, { - "checksumSHA1": "OaECjBPfByj/uEEGFqFhqK0Kgzw=", + "checksumSHA1": "ag4CscDKi8ZBExU8jJf03rxonF0=", "path": "github.com/golang/protobuf/proto", - "revision": "0cb4f732569330e7b40d8fcee91c41aafaaddc4f", - "revisionTime": "2018-07-10T19:34:13Z" + "revision": "70b3af33377e7aa25ae42977bed93cc6b90f0373", + "revisionTime": "2018-07-12T04:22:25Z" }, { "checksumSHA1": "z4copNgeTN77OymdDKqLaIK/vSI=", @@ -495,20 +495,20 @@ { "checksumSHA1": "GtamqiJoL7PGHsN454AoffBFMa8=", "path": "golang.org/x/net/context", - "revision": "039a4258aec0ad3c79b905677cceeab13b296a77", - "revisionTime": "2018-05-05T01:56:23Z" + "revision": "d0887baf81f4598189d4e12a37c6da86f0bba4d0", + "revisionTime": "2018-07-12T19:54:13Z" }, { "checksumSHA1": "UxahDzW2v4mf/+aFxruuupaoIwo=", "path": "golang.org/x/net/internal/timeseries", - "revision": "039a4258aec0ad3c79b905677cceeab13b296a77", - "revisionTime": "2018-05-05T01:56:23Z" + "revision": "d0887baf81f4598189d4e12a37c6da86f0bba4d0", + "revisionTime": "2018-07-12T19:54:13Z" }, { "checksumSHA1": "rJn3m/27kO+2IU6KCCZ74Miby+8=", "path": "golang.org/x/net/trace", - "revision": "039a4258aec0ad3c79b905677cceeab13b296a77", - "revisionTime": "2018-05-05T01:56:23Z" + "revision": "d0887baf81f4598189d4e12a37c6da86f0bba4d0", + "revisionTime": "2018-07-12T19:54:13Z" }, { "checksumSHA1": "nf+CWRQfmxmZkdYRpxEuA8u+qwI=", diff --git a/worker/draft.go b/worker/draft.go index bd9589362b5..0a3cf0b924f 100644 --- a/worker/draft.go +++ b/worker/draft.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "math" + "sync/atomic" "time" "github.com/coreos/etcd/raft" @@ -46,6 +47,8 @@ type node struct { done chan struct{} // to check whether node is running or not gid uint32 + streaming int32 // Used to avoid calculating snapshot + canCampaign bool elog trace.EventLog } @@ -385,12 +388,11 @@ func (n *node) applyCommitted(proposal *intern.Proposal, index uint64) error { } else if proposal.Snapshot != nil { snap := proposal.Snapshot n.elog.Printf("Creating snapshot: %+v", snap) - x.Printf("Creating snapshot at index: %d. MinPendingStartTs: %d.\n", - snap.Index, snap.MinPendingStartTs) + x.Printf("Creating snapshot at index: %d. ReadTs: %d.\n", snap.Index, snap.ReadTs) data, err := snap.Marshal() x.Check(err) // We can now discard all invalid versions of keys below this ts. - pstore.SetDiscardTs(snap.MinPendingStartTs - 1) + pstore.SetDiscardTs(snap.ReadTs) return n.Store.CreateSnapshot(snap.Index, n.ConfState(), data) } else { @@ -462,17 +464,27 @@ func (n *node) leaderBlocking() (*conn.Pool, error) { return pool, nil } +func (n *node) Snapshot() (*intern.Snapshot, error) { + if n == nil || n.Store == nil { + return nil, conn.ErrNoNode + } + snap, err := n.Store.Snapshot() + if err != nil { + return nil, err + } + res := &intern.Snapshot{} + if err := res.Unmarshal(snap.Data); err != nil { + return nil, err + } + return res, nil +} + func (n *node) retrieveSnapshot() error { pool, err := n.leaderBlocking() if err != nil { return err } - // Wait for watermarks to sync since populateShard writes directly to db, otherwise - // the values might get overwritten - // Safe to keep this line - n.applyAllMarks(n.ctx) - // Need to clear pl's stored in memory for the case when retrieving snapshot with // index greater than this node's last index // Should invalidate/remove pl's to this group only ideally @@ -494,6 +506,20 @@ func (n *node) retrieveSnapshot() error { return nil } +func (n *node) proposeSnapshot() error { + snap, err := n.calculateSnapshot(1000) + if err != nil || snap == nil { + return err + } + proposal := &intern.Proposal{ + Snapshot: snap, + } + n.elog.Printf("Proposing snapshot: %+v\n", snap) + data, err := proposal.Marshal() + x.Check(err) + return n.Raft().Propose(n.ctx, data) +} + func (n *node) Run() { firstRun := true var leader bool @@ -512,8 +538,8 @@ func (n *node) Run() { // We use disk based storage for Raft. So, we're not too concerned about // snapshotting. We just need to do enough, so that we don't have a huge backlog of // entries to process on a restart. - if err := n.calculateSnapshot(1000); err != nil { - x.Errorf("While taking snapshot: %v\n", err) + if err := n.proposeSnapshot(); err != nil { + x.Errorf("While calculating and proposing snapshot: %v", err) } go n.abortOldTransactions() } @@ -697,66 +723,74 @@ func (n *node) abortOldTransactions() { } // calculateSnapshot would calculate a snapshot index, considering these factors: -// - We still keep at least keepN number of Raft entries. If we cut too short, -// then the chances that a crashed follower needs to retrieve the entire state -// from leader increases. So, we keep a buffer to allow a follower to catch up. -// - We can discard at least half of keepN number of entries. +// - We only start discarding once we have at least discardN entries. // - We are not overshooting the max applied entry. That is, we're not removing // Raft entries before they get applied. // - We are considering the minimum start ts that has yet to be committed or // aborted. This way, we still keep all the mutations corresponding to this // start ts in the Raft logs. This is important, because we don't persist // pre-writes to disk in pstore. +// - Considering the above, find the maximum commit timestamp that we have seen. +// That would tell us about the maximum timestamp used to do any commits. This +// ts is what we can use for future reads of this snapshot. // - Finally, this function would propose this snapshot index, so the entire // group can apply it to their Raft stores. -func (n *node) calculateSnapshot(keepN int) error { +// +// Txn0 | S0 | | | C0 | | | +// Txn1 | | S1 | | | | C1 | +// Txn2 | | | S2 | C2 | | | +// Txn3 | | | | | S3 | | +// Txn4 | | | | | | | S4 +// Index | i1 | i2 | i3 | i4 | i5 | i6 | i7 +// +// At i7, min pending start ts = S3, therefore snapshotIdx = i5 - 1 = i4. +// At i7, max commit ts = C1, therefore readTs = C1. +func (n *node) calculateSnapshot(discardN int) (*intern.Snapshot, error) { tr := trace.New("Dgraph.Internal", "Propose.Snapshot") defer tr.Finish() - // Each member of the Raft group is taking snapshots independently, as mentioned in Section 7 of - // Raft paper. We want to avoid taking snapshots too close to the LastIndex, so that in case the - // leader changes, and the followers haven't yet caught up to this index, they would need to - // retrieve the entire state (snapshot) of the DB. So, we should always wait to accumulate skip - // entries before we start taking a snapshot. - count, err := n.Store.NumEntries() - if err != nil { - tr.LazyPrintf("Error: %v", err) - tr.SetError() - return err + if atomic.LoadInt32(&n.streaming) > 0 { + tr.LazyPrintf("Skipping calculateSnapshot due to streaming") + return nil, nil } - tr.LazyPrintf("Found Raft entries: %d", count) - if count < 2*keepN { - // We wait to build up at least 2*keepN entries, and then discard keepN entries. - tr.LazyPrintf("Skipping due to insufficient entries") - return nil - } - discard := count - keepN first, err := n.Store.FirstIndex() if err != nil { tr.LazyPrintf("Error: %v", err) tr.SetError() - return err + return nil, err } tr.LazyPrintf("First index: %d", first) - last := first + uint64(discard) - if last > n.Applied.DoneUntil() { - tr.LazyPrintf("Skipping because last index: %d > applied", last) - return nil + + last := n.Applied.DoneUntil() + if int(last-first) < discardN { + tr.LazyPrintf("Skipping due to insufficient entries") + return nil, nil } - entries, err := n.Store.Entries(first, last, math.MaxUint64) + tr.LazyPrintf("Found Raft entries: %d", last-first) + + entries, err := n.Store.Entries(first, last+1, math.MaxUint64) if err != nil { tr.LazyPrintf("Error: %v", err) tr.SetError() - return err - } - - // We find the minimum start ts for which a decision to commit or abort is still pending. We - // should not discard mutations corresponding to this start ts, because we don't persist - // mutations until they are committed. - minPending := posting.Oracle().MinPendingStartTs() - tr.LazyPrintf("Found min pending start ts: %d", minPending) - var snapshotIdx uint64 + return nil, err + } + + // We iterate over Raft entries, parsing them to Proposals. We do two + // things: + // 1. Create a start timestamp -> first raft index map. + // 2. We find the max commit timestamp present, as maxCommitTs (/ ReadTs). + // As we see transaction commits, we remove the entries from the startToIdx + // map, so that by the end of iteration, we only keep the entries in map, + // which correspond to mutations which haven't yet been committed, aka + // pending mutations. We pick the lowest start ts, and the index + // corresponding to that becomes our snapshotIdx. We keep all the Raft + // entries including this one, so that on a replay, we can pick all the + // mutations correctly. + + // This map holds a start ts as key, and Raft index as value. + startToIdx := make(map[uint64]uint64) + var maxCommitTs, snapshotIdx uint64 for _, entry := range entries { if entry.Type != raftpb.EntryNormal { continue @@ -765,42 +799,55 @@ func (n *node) calculateSnapshot(keepN int) error { if err := proposal.Unmarshal(entry.Data); err != nil { tr.LazyPrintf("Error: %v", err) tr.SetError() - return err + return nil, err } - mu := proposal.Mutations - if mu != nil && mu.StartTs >= minPending { - break + if proposal.Mutations != nil { + ts := proposal.Mutations.StartTs + if _, has := startToIdx[ts]; !has { + startToIdx[ts] = entry.Index + } + } else if proposal.Delta != nil { + for _, txn := range proposal.Delta.GetTxns() { + delete(startToIdx, txn.StartTs) + maxCommitTs = x.Max(maxCommitTs, txn.CommitTs) + } + snapshotIdx = entry.Index + } + } + if maxCommitTs == 0 { + tr.LazyPrintf("maxCommitTs is zero") + return nil, nil + } + var minPendingTs uint64 = math.MaxUint64 + // It is possible that this loop doesn't execute, because all transactions have been committed. + // In that case, we'll default to snapshotIdx value from above. + for startTs, index := range startToIdx { + if minPendingTs < startTs { + continue } - snapshotIdx = entry.Index + // Pick the lowest startTs, and the corresponding Raft index - 1. We + // deduct one so that the Raft entry can be picked fully during replay. + // This becomes our snapshotIdx. + minPendingTs, snapshotIdx = startTs, index-1 } - tr.LazyPrintf("Got snapshotIdx: %d. Discarding: %d", snapshotIdx, snapshotIdx-first) - // We should discard at least half of keepN. Otherwise, why bother. - if snapshotIdx == 0 || int(snapshotIdx-first) < int(float64(keepN)*0.5) { + + numDiscarding := snapshotIdx - first + 1 + tr.LazyPrintf("Got snapshotIdx: %d. MaxCommitTs: %d. Discarding: %d", + snapshotIdx, maxCommitTs, numDiscarding) + if int(numDiscarding) < discardN { tr.LazyPrintf("Skipping snapshot because insufficient discard entries") x.Printf("Skipping snapshot at index: %d. Insufficient discard entries: %d."+ - " MinPendingStartTs: %d\n", snapshotIdx, snapshotIdx-first, minPending) - return nil + " MinPendingStartTs: %d\n", snapshotIdx, numDiscarding, minPendingTs) + return nil, nil } snap := &intern.Snapshot{ - Context: n.RaftContext, - Index: snapshotIdx, - MinPendingStartTs: minPending, + Context: n.RaftContext, + Index: snapshotIdx, + ReadTs: maxCommitTs, } - proposal := &intern.Proposal{ - Snapshot: snap, - } - tr.LazyPrintf("Proposing snapshot: %+v\n", snap) - - data, err := proposal.Marshal() - x.Check(err) - if err := n.Raft().Propose(context.Background(), data); err != nil { - tr.LazyPrintf("Error: %v", err) - tr.SetError() - return err - } - tr.LazyPrintf("Done best-effort proposing.") - return nil + tr.LazyPrintf("Got snapshot: %+v", snap) + return snap, nil } func (n *node) joinPeers() error { @@ -883,12 +930,16 @@ func (n *node) InitAndStartNode() { n.SetRaft(raft.RestartNode(n.Cfg)) } else { x.Printf("New Node for group: %d\n", n.gid) - if peerId, hasPeer := groups().MyPeer(); hasPeer { + if _, hasPeer := groups().MyPeer(); hasPeer { // Get snapshot before joining peers as it can take time to retrieve it and we dont // want the quorum to be inactive when it happens. - x.Printf("Retrieving snapshot from peer: %d", peerId) - n.retryUntilSuccess(n.retrieveSnapshot, time.Second) + // TODO: This is an optimization, which adds complexity because it requires us to + // understand the Raft state of the node. Let's instead have the node retrieve the + // snapshot as needed after joining the group, instead of us forcing one upfront. + // + // x.Printf("Retrieving snapshot from peer: %d", peerId) + // n.retryUntilSuccess(n.retrieveSnapshot, time.Second) x.Println("Trying to join peers.") n.retryUntilSuccess(n.joinPeers, time.Second) diff --git a/worker/draft_test.go b/worker/draft_test.go new file mode 100644 index 00000000000..656b63fa708 --- /dev/null +++ b/worker/draft_test.go @@ -0,0 +1,106 @@ +package worker + +import ( + "io/ioutil" + "os" + "testing" + + pb "github.com/coreos/etcd/raft/raftpb" + "github.com/dgraph-io/badger" + "github.com/dgraph-io/dgraph/protos/intern" + "github.com/dgraph-io/dgraph/raftwal" + "github.com/dgraph-io/dgraph/x" + "github.com/stretchr/testify/require" +) + +func openBadger(dir string) (*badger.DB, error) { + opt := badger.DefaultOptions + opt.Dir = dir + opt.ValueDir = dir + + return badger.Open(opt) +} + +func getEntryForMutation(index, startTs uint64) pb.Entry { + proposal := intern.Proposal{Mutations: &intern.Mutations{StartTs: startTs}} + data, err := proposal.Marshal() + x.Check(err) + return pb.Entry{Index: index, Term: 1, Type: pb.EntryNormal, Data: data} +} + +func getEntryForCommit(index, startTs, commitTs uint64) pb.Entry { + delta := &intern.OracleDelta{} + delta.Txns = append(delta.Txns, &intern.TxnStatus{StartTs: startTs, CommitTs: commitTs}) + proposal := intern.Proposal{Delta: delta} + data, err := proposal.Marshal() + x.Check(err) + return pb.Entry{Index: index, Term: 1, Type: pb.EntryNormal, Data: data} +} + +func TestCalculateSnapshot(t *testing.T) { + dir, err := ioutil.TempDir("", "badger") + require.NoError(t, err) + defer os.RemoveAll(dir) + + db, err := openBadger(dir) + require.NoError(t, err) + ds := raftwal.Init(db, 0, 0) + + n := newNode(ds, 1, 1, "") + var entries []pb.Entry + // Txn: 1 -> 5 // 5 should be the ReadTs. + // Txn: 2 // Should correspond to the index. Subtract 1 from the index. + // Txn: 3 -> 4 + entries = append(entries, getEntryForMutation(1, 1)) + entries = append(entries, getEntryForMutation(2, 3)) + entries = append(entries, getEntryForMutation(3, 2)) // Start ts can be jumbled. + entries = append(entries, getEntryForCommit(4, 3, 4)) // But commit ts would be serial. + entries = append(entries, getEntryForCommit(5, 1, 5)) + require.NoError(t, n.Store.Save(pb.HardState{}, entries, pb.Snapshot{})) + n.Applied.SetDoneUntil(5) + snap, err := n.calculateSnapshot(1) + require.NoError(t, err) + require.Equal(t, uint64(5), snap.ReadTs) + require.Equal(t, uint64(2), snap.Index) + + // Check state of Raft store. + err = n.Store.CreateSnapshot(snap.Index, nil, nil) + require.NoError(t, err) + + first, err := n.Store.FirstIndex() + require.NoError(t, err) + require.Equal(t, uint64(3), first) + + last, err := n.Store.LastIndex() + require.NoError(t, err) + require.Equal(t, uint64(5), last) + + // This time commit all txns. + // Txn: 7 -> 8 + // Txn: 2 -> 9 + entries = entries[:0] + entries = append(entries, getEntryForMutation(6, 7)) + entries = append(entries, getEntryForCommit(7, 7, 8)) + entries = append(entries, getEntryForCommit(8, 2, 9)) + require.NoError(t, n.Store.Save(pb.HardState{}, entries, pb.Snapshot{})) + n.Applied.SetDoneUntil(8) + snap, err = n.calculateSnapshot(1) + require.NoError(t, err) + require.Equal(t, uint64(9), snap.ReadTs) + require.Equal(t, uint64(8), snap.Index) + + // Check state of Raft store. + err = n.Store.CreateSnapshot(snap.Index, nil, nil) + require.NoError(t, err) + first, err = n.Store.FirstIndex() + require.NoError(t, err) + require.Equal(t, uint64(9), first) + + entries = entries[:0] + entries = append(entries, getEntryForMutation(9, 11)) + require.NoError(t, n.Store.Save(pb.HardState{}, entries, pb.Snapshot{})) + n.Applied.SetDoneUntil(9) + snap, err = n.calculateSnapshot(0) + require.NoError(t, err) + require.Nil(t, snap) +} diff --git a/worker/predicate.go b/worker/predicate.go index e67541f8d30..41ae49e0b8e 100644 --- a/worker/predicate.go +++ b/worker/predicate.go @@ -8,8 +8,8 @@ package worker import ( - "bytes" "context" + "errors" "io" "math" "sync" @@ -88,55 +88,33 @@ func (n *node) populateShard(ps *badger.ManagedDB, pl *conn.Pool) (int, error) { conn := pl.Get() c := intern.NewWorkerClient(conn) - n.RLock() ctx := n.ctx - group := n.gid - stream, err := c.PredicateAndSchemaData(ctx, - &intern.SnapshotMeta{ - ClientTs: n.RaftContext.SnapshotTs, - GroupId: group, - }) - n.RUnlock() + snap, err := n.Snapshot() if err != nil { - if tr, ok := trace.FromContext(ctx); ok { - tr.LazyPrintf(err.Error()) - } return 0, err } - if tr, ok := trace.FromContext(ctx); ok { - tr.LazyPrintf("Streaming data for group: %v", group) - } - - kvs, err := stream.Recv() + stream, err := c.StreamSnapshot(ctx, snap) if err != nil { return 0, err } + // Before we write anything, we should drop all the data stored in ps. + if err := ps.DropAll(); err != nil { + return 0, err + } - x.AssertTrue(len(kvs.Kv) == 1) - ikv := kvs.Kv[0] - // First key has the snapshot ts from the leader. - x.AssertTrue(bytes.Equal(ikv.Key, []byte("min_ts"))) - n.Lock() - // TODO: Fix up all this min ts logic. It's wrong. - n.RaftContext.SnapshotTs = ikv.Version - n.Unlock() - - kvChan := make(chan *intern.KVS, 1000) + kvChan := make(chan *intern.KVS, 100) che := make(chan error, 1) go writeBatch(ctx, ps, kvChan, che) // We can use count to check the number of posting lists returned in tests. count := 0 for { - kvs, err = stream.Recv() + kvs, err := stream.Recv() if err == io.EOF { x.Printf("EOF has been reached\n") break } if err != nil { - if tr, ok := trace.FromContext(ctx); ok { - tr.LazyPrintf(err.Error()) - } close(kvChan) return count, err } @@ -146,15 +124,9 @@ func (n *node) populateShard(ps *badger.ManagedDB, pl *conn.Pool) (int, error) { count += len(kvs.Kv) // OK case <-ctx.Done(): - if tr, ok := trace.FromContext(ctx); ok { - tr.LazyPrintf("Context timed out while streaming group: %v", group) - } close(kvChan) return 0, ctx.Err() case err := <-che: - if tr, ok := trace.FromContext(ctx); ok { - tr.LazyPrintf("Error while doing a batch write for group: %v", group) - } close(kvChan) // Important: Don't put return count, err // There was a compiler bug which was fixed in 1.8.1 @@ -166,76 +138,57 @@ func (n *node) populateShard(ps *badger.ManagedDB, pl *conn.Pool) (int, error) { close(kvChan) if err := <-che; err != nil { - if tr, ok := trace.FromContext(ctx); ok { - tr.LazyPrintf("Error while doing a batch write for group: %v", group) - } return count, err } - if tr, ok := trace.FromContext(ctx); ok { - tr.LazyPrintf("Streaming complete for group: %v", group) - } x.Printf("Got %d keys. DONE.\n", count) return count, nil } -func (w *grpcWorker) PredicateAndSchemaData(m *intern.SnapshotMeta, stream intern.Worker_PredicateAndSchemaDataServer) error { - clientTs := m.ClientTs +func (w *grpcWorker) StreamSnapshot(reqSnap *intern.Snapshot, + stream intern.Worker_StreamSnapshotServer) error { + n := groups().Node + if n == nil { + return conn.ErrNoNode + } + // Indicate that we're streaming right now. Used to cancel + // calculateSnapshot. However, this logic isn't foolproof. A leader might + // have already proposed a snapshot, which it can apply while this streaming + // is going on. That can happen after the reqSnap check we're doing below. + // However, I don't think we need to tackle this edge case for now. + atomic.AddInt32(&n.streaming, 1) + defer atomic.AddInt32(&n.streaming, -1) if !x.IsTestRun() { - if !groups().ServesGroup(m.GroupId) { - return x.Errorf("Group %d not served.", m.GroupId) - } - n := groups().Node if !n.AmLeader() { - return x.Errorf("Not leader of group: %d", m.GroupId) + return errNotLeader } } + snap, err := n.Snapshot() + if err != nil { + return err + } + x.Printf("Got StreamSnapshot request. Mine: %+v. Requested: %+v\n", snap, reqSnap) + if snap.Index != reqSnap.Index || snap.ReadTs != reqSnap.ReadTs { + return errors.New("Mismatching snapshot request") + } - // Any commit which happens in the future will have commitTs greater than - // this. - // TODO: Ensure all deltas have made to disk and read in memory before checking disk. + // We have matched the requested snapshot with what this node, the leader of the group, has. + // Now, we read all the posting lists stored below MinPendingStartTs, and stream them over the + // wire. The MinPendingStartTs stored as part of creation of Snapshot, tells us the readTs to + // use to read from the store. Anything below this timestamp, we should pick up. + // + // TODO: This would also pick up schema updates done "after" the snapshot index. Guess that + // might be OK. Otherwise, we'd want to version the schemas as well. Currently, they're stored + // at timestamp=1. + + // TODO: Confirm that the bug is now over. // BUG: There's a bug here due to which a node which doesn't see any transactions, but has real // data fails to send that over, because of min_ts. - // TODO: This min_ts makes no sense. Let's fix this. - min_ts := posting.Oracle().PurgeTs() - x.Printf("Got min_ts: %d\n", min_ts) - // snap, err := groups().Node.Snapshot() - // if err != nil { - // return err - // } - // index := snap.Metadata.Index - - // TODO: Why are we using MinTs() in the place when we should be using - // snapshot index? This is wrong. - - // TODO: We are not using the snapshot time, because we don't store the - // transaction timestamp in the snapshot. If we did, we'd just use that - // instead of this. This causes issues if the server had received a snapshot - // to begin with, but had no active transactions. Then mints is always zero, - // hence nothing is read or sent in the stream. - - // UPDATE: This doesn't look too bad. So, we're keeping track of the - // transaction timestamps on the side. And we're using those to figure out - // what to stream here. The snapshot index is not really being used for - // anything here. - // This whole transaction tracking business is complex and must be - // simplified to its essence. - - // Send ts as first KV. - if err := stream.Send(&intern.KVS{ - Kv: []*intern.KV{&intern.KV{ - Key: []byte("min_ts"), - Version: min_ts, - }}, - }); err != nil { - return err - } - sl := streamLists{stream: stream, db: pstore} sl.chooseKey = func(key []byte, version uint64) bool { - pk := x.Parse(key) - return version > clientTs || pk.IsSchema() + // Pick all keys. + return true } sl.itemToKv = func(key []byte, itr *badger.Iterator) (*intern.KV, error) { item := itr.Item() @@ -253,6 +206,10 @@ func (w *grpcWorker) PredicateAndSchemaData(m *intern.SnapshotMeta, stream inter } return kv, nil } + // We should keep reading the posting list instead of copying the key-value pairs directly, + // to consolidate the read logic in one place. This is more robust than trying to replicate + // a simplified key-value copy logic here, which still understands the BitCompletePosting + // and other bits about the way we store posting lists. l, err := posting.ReadPostingList(key, itr) if err != nil { return nil, err @@ -260,7 +217,7 @@ func (w *grpcWorker) PredicateAndSchemaData(m *intern.SnapshotMeta, stream inter return l.MarshalToKv() } - if err := sl.orchestrate(stream.Context(), "Sending SNAPSHOT", min_ts); err != nil { + if err := sl.orchestrate(stream.Context(), "Sending SNAPSHOT", snap.ReadTs); err != nil { return err } diff --git a/worker/stream_lists_test.go b/worker/stream_lists_test.go index 920cde681c1..db17e321cb0 100644 --- a/worker/stream_lists_test.go +++ b/worker/stream_lists_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/require" ) -func openBadger(dir string) (*badger.ManagedDB, error) { +func openManaged(dir string) (*badger.ManagedDB, error) { opt := badger.DefaultOptions opt.Dir = dir opt.ValueDir = dir @@ -47,7 +47,7 @@ func TestOrchestrate(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - db, err := openBadger(dir) + db, err := openManaged(dir) require.NoError(t, err) var count int