From 59472eaad64113d8f9767c6b4f74ef753fe9e3f6 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 14 Jul 2021 16:24:58 +0800 Subject: [PATCH 01/19] save work --- dm/ctl/master/import_export_config.go | 115 +++ dm/master/scheduler/scheduler.go | 12 + dm/master/server.go | 50 ++ dm/pb/dmmaster.pb.go | 1189 +++++++++++++++++++++---- dm/pbmock/dmmaster.go | 35 + dm/proto/dmmaster.proto | 24 + 6 files changed, 1250 insertions(+), 175 deletions(-) create mode 100644 dm/ctl/master/import_export_config.go diff --git a/dm/ctl/master/import_export_config.go b/dm/ctl/master/import_export_config.go new file mode 100644 index 0000000000..695cb81f69 --- /dev/null +++ b/dm/ctl/master/import_export_config.go @@ -0,0 +1,115 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package master + +import ( + "context" + "io/ioutil" + "os" + "path" + + "github.com/spf13/cobra" + + "github.com/pingcap/dm/dm/ctl/common" + "github.com/pingcap/dm/dm/pb" +) + +var ( + taskDirname = "tasks" + sourceDirname = "sources" + yamlSuffix = ".yaml" +) + +// NewExportCfgCmd creates a exportCfg command. +func NewExportCfgCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "export-config [--dir directory]", + Short: "Export the configurations of sources and tasks.", + RunE: exportCfgFunc, + } + cmd.Flags().StringP("dir", "d", "configs", "specify the destinary directory, default is `./configs`") + return cmd +} + +// exportCfgFunc gets config. +func exportCfgFunc(cmd *cobra.Command, args []string) error { + dir, err := cmd.Flags().GetString("dir") + if err != nil { + common.PrintLinesf("can not get directory") + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), common.GlobalConfig().RPCTimeout) + defer cancel() + + resp := &pb.ImportExportCfgsResponse{} + err = common.SendRequest( + ctx, + "ImportExportCfgs", + &pb.ImportExportCfgsRequest{ + Op: pb.CfgsOp_Export, + Dir: dir, + }, + &resp, + ) + + if err != nil { + common.PrintLinesf("can not export configs to %s", dir) + return err + } + + if !resp.Result { + common.PrettyPrintResponse(resp) + return nil + } + + if err = os.MkdirAll(dir, 0o755); err != nil { + common.PrintLinesf("can not create directory %s", dir) + return err + } + + taskDir := path.Join(dir, taskDirname) + if err = os.MkdirAll(taskDir, 0o755); err != nil { + common.PrintLinesf("can not create directory of task configs %s", taskDir) + return err + } + + sourceDir := path.Join(dir, sourceDirname) + if err = os.MkdirAll(sourceDirname, 0o755); err != nil { + common.PrintLinesf("can not create directory of source configs %s", path.Join(dir+sourceDir)) + return err + } + + for _, taskCfg := range resp.Tasks { + taskFile := path.Join(taskDir, taskCfg.Name) + taskFile += yamlSuffix + err = ioutil.WriteFile(taskFile, []byte(taskCfg.Content), 0o644) + if err != nil { + common.PrintLinesf("can not write task config to file %s", taskFile) + return err + } + } + + for _, sourceCfg := range resp.Sources { + sourceFile := path.Join(sourceDir, sourceCfg.Name, ".yaml") + sourceFile += yamlSuffix + err = ioutil.WriteFile(sourceFile, []byte(sourceCfg.Content), 0o644) + if err != nil { + common.PrintLinesf("can not write source config to file %s", sourceFile) + return err + } + } + + return nil +} diff --git a/dm/master/scheduler/scheduler.go b/dm/master/scheduler/scheduler.go index 41bc926684..698427eda4 100644 --- a/dm/master/scheduler/scheduler.go +++ b/dm/master/scheduler/scheduler.go @@ -381,6 +381,18 @@ func (s *Scheduler) RemoveSourceCfg(source string) error { return nil } +// GetSourceCfgs gets all source cfgs, return nil when error happens. +func (s *Scheduler) GetSourceCfgs() map[string]*config.SourceConfig { + s.mu.RLock() + defer s.mu.RUnlock() + clone := make(map[string]*config.SourceConfig, len(s.sourceCfgs)) + for sourceID, sourceCfg := range s.sourceCfgs { + cloneCfg := sourceCfg.Clone() + clone[sourceID] = cloneCfg + } + return clone +} + // GetSourceCfgIDs gets all added source ID. func (s *Scheduler) GetSourceCfgIDs() []string { s.mu.RLock() diff --git a/dm/master/server.go b/dm/master/server.go index 61a9d0ca94..9a78cad564 100644 --- a/dm/master/server.go +++ b/dm/master/server.go @@ -2071,6 +2071,56 @@ func (s *Server) GetCfg(ctx context.Context, req *pb.GetCfgRequest) (*pb.GetCfgR }, nil } +func (s *Server) exportCfgs() ([]*pb.Config, []*pb.Config) { + subTaskCfgsMap := s.scheduler.GetSubTaskCfgs() + sourceCfgsMap := s.scheduler.GetSourceCfgs() + taskCfgs := make([]*pb.Config, 0, len(subTaskCfgsMap)) + sourceCfgs := make([]*pb.Config, 0, len(sourceCfgsMap)) + + for task, subTaskCfgs := range subTaskCfgsMap { + subTaskCfgList := make([]*config.SubTaskConfig, 0, len(subTaskCfgs)) + for _, subTaskCfg := range subTaskCfgs { + clone := subTaskCfg + subTaskCfgList = append(subTaskCfgList, &clone) + } + sort.Slice(subTaskCfgList, func(i, j int) bool { + return subTaskCfgList[i].SourceID < subTaskCfgList[j].SourceID + }) + taskCfg := config.FromSubTaskConfigs(subTaskCfgList...) + taskCfgs = append(taskCfgs, &pb.Config{Name: task, Content: taskCfg.String()}) + } + + for sourceID, sourceCfg := range sourceCfgsMap { + sourceCfgs = append(sourceCfgs, &pb.Config{Name: sourceID, Content: sourceCfg.String()}) + } + + return taskCfgs, sourceCfgs +} + +// ImportExportCfgs implements MasterServer.ImportExportCfgs. +func (s *Server) ImportExportCfgs(ctx context.Context, req *pb.ImportExportCfgsRequest) (*pb.ImportExportCfgsResponse, error) { + var ( + resp = &pb.ImportExportCfgsResponse{} + err2 error + ) + shouldRet := s.sharedLogic(ctx, req, &resp, &err2) + if shouldRet { + return resp, err2 + } + + switch req.Op { + case pb.CfgsOp_Import: + resp.Tasks, resp.Sources = s.exportCfgs() + case pb.CfgsOp_Export: + default: + resp.Msg = fmt.Sprintf("invalid configs op '%s'", req.Op) + return resp, nil + } + + resp.Result = true + return resp, nil +} + // HandleError implements MasterServer.HandleError. func (s *Server) HandleError(ctx context.Context, req *pb.HandleErrorRequest) (*pb.HandleErrorResponse, error) { var ( diff --git a/dm/pb/dmmaster.pb.go b/dm/pb/dmmaster.pb.go index 216fb036ea..f9458b79d0 100644 --- a/dm/pb/dmmaster.pb.go +++ b/dm/pb/dmmaster.pb.go @@ -152,6 +152,31 @@ func (RelayOpV2) EnumDescriptor() ([]byte, []int) { return fileDescriptor_f9bef11f2a341f03, []int{3} } +type CfgsOp int32 + +const ( + CfgsOp_Export CfgsOp = 0 + CfgsOp_Import CfgsOp = 1 +) + +var CfgsOp_name = map[int32]string{ + 0: "Export", + 1: "Import", +} + +var CfgsOp_value = map[string]int32{ + "Export": 0, + "Import": 1, +} + +func (x CfgsOp) String() string { + return proto.EnumName(CfgsOp_name, int32(x)) +} + +func (CfgsOp) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_f9bef11f2a341f03, []int{4} +} + type StartTaskRequest struct { Task string `protobuf:"bytes,1,opt,name=task,proto3" json:"task,omitempty"` Sources []string `protobuf:"bytes,2,rep,name=sources,proto3" json:"sources,omitempty"` @@ -3105,11 +3130,184 @@ func (m *OperateRelayResponse) GetMsg() string { return "" } +type ImportExportCfgsRequest struct { + Op CfgsOp `protobuf:"varint,1,opt,name=op,proto3,enum=pb.CfgsOp" json:"op,omitempty"` + Dir string `protobuf:"bytes,2,opt,name=dir,proto3" json:"dir,omitempty"` +} + +func (m *ImportExportCfgsRequest) Reset() { *m = ImportExportCfgsRequest{} } +func (m *ImportExportCfgsRequest) String() string { return proto.CompactTextString(m) } +func (*ImportExportCfgsRequest) ProtoMessage() {} +func (*ImportExportCfgsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_f9bef11f2a341f03, []int{49} +} +func (m *ImportExportCfgsRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ImportExportCfgsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ImportExportCfgsRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ImportExportCfgsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportExportCfgsRequest.Merge(m, src) +} +func (m *ImportExportCfgsRequest) XXX_Size() int { + return m.Size() +} +func (m *ImportExportCfgsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ImportExportCfgsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ImportExportCfgsRequest proto.InternalMessageInfo + +func (m *ImportExportCfgsRequest) GetOp() CfgsOp { + if m != nil { + return m.Op + } + return CfgsOp_Export +} + +func (m *ImportExportCfgsRequest) GetDir() string { + if m != nil { + return m.Dir + } + return "" +} + +type Config struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` +} + +func (m *Config) Reset() { *m = Config{} } +func (m *Config) String() string { return proto.CompactTextString(m) } +func (*Config) ProtoMessage() {} +func (*Config) Descriptor() ([]byte, []int) { + return fileDescriptor_f9bef11f2a341f03, []int{50} +} +func (m *Config) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Config) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Config.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Config) XXX_Merge(src proto.Message) { + xxx_messageInfo_Config.Merge(m, src) +} +func (m *Config) XXX_Size() int { + return m.Size() +} +func (m *Config) XXX_DiscardUnknown() { + xxx_messageInfo_Config.DiscardUnknown(m) +} + +var xxx_messageInfo_Config proto.InternalMessageInfo + +func (m *Config) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Config) GetContent() string { + if m != nil { + return m.Content + } + return "" +} + +type ImportExportCfgsResponse struct { + Result bool `protobuf:"varint,1,opt,name=result,proto3" json:"result,omitempty"` + Msg string `protobuf:"bytes,2,opt,name=msg,proto3" json:"msg,omitempty"` + Sources []*Config `protobuf:"bytes,3,rep,name=sources,proto3" json:"sources,omitempty"` + Tasks []*Config `protobuf:"bytes,4,rep,name=tasks,proto3" json:"tasks,omitempty"` +} + +func (m *ImportExportCfgsResponse) Reset() { *m = ImportExportCfgsResponse{} } +func (m *ImportExportCfgsResponse) String() string { return proto.CompactTextString(m) } +func (*ImportExportCfgsResponse) ProtoMessage() {} +func (*ImportExportCfgsResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_f9bef11f2a341f03, []int{51} +} +func (m *ImportExportCfgsResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ImportExportCfgsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ImportExportCfgsResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ImportExportCfgsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportExportCfgsResponse.Merge(m, src) +} +func (m *ImportExportCfgsResponse) XXX_Size() int { + return m.Size() +} +func (m *ImportExportCfgsResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ImportExportCfgsResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ImportExportCfgsResponse proto.InternalMessageInfo + +func (m *ImportExportCfgsResponse) GetResult() bool { + if m != nil { + return m.Result + } + return false +} + +func (m *ImportExportCfgsResponse) GetMsg() string { + if m != nil { + return m.Msg + } + return "" +} + +func (m *ImportExportCfgsResponse) GetSources() []*Config { + if m != nil { + return m.Sources + } + return nil +} + +func (m *ImportExportCfgsResponse) GetTasks() []*Config { + if m != nil { + return m.Tasks + } + return nil +} + func init() { proto.RegisterEnum("pb.SourceOp", SourceOp_name, SourceOp_value) proto.RegisterEnum("pb.LeaderOp", LeaderOp_name, LeaderOp_value) proto.RegisterEnum("pb.CfgType", CfgType_name, CfgType_value) proto.RegisterEnum("pb.RelayOpV2", RelayOpV2_name, RelayOpV2_value) + proto.RegisterEnum("pb.CfgsOp", CfgsOp_name, CfgsOp_value) proto.RegisterType((*StartTaskRequest)(nil), "pb.StartTaskRequest") proto.RegisterType((*StartTaskResponse)(nil), "pb.StartTaskResponse") proto.RegisterType((*OperateTaskRequest)(nil), "pb.OperateTaskRequest") @@ -3159,140 +3357,150 @@ func init() { proto.RegisterType((*TransferSourceResponse)(nil), "pb.TransferSourceResponse") proto.RegisterType((*OperateRelayRequest)(nil), "pb.OperateRelayRequest") proto.RegisterType((*OperateRelayResponse)(nil), "pb.OperateRelayResponse") + proto.RegisterType((*ImportExportCfgsRequest)(nil), "pb.ImportExportCfgsRequest") + proto.RegisterType((*Config)(nil), "pb.Config") + proto.RegisterType((*ImportExportCfgsResponse)(nil), "pb.ImportExportCfgsResponse") } func init() { proto.RegisterFile("dmmaster.proto", fileDescriptor_f9bef11f2a341f03) } var fileDescriptor_f9bef11f2a341f03 = []byte{ - // 2040 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x59, 0x5f, 0x6f, 0x1b, 0xc7, - 0x11, 0xe7, 0x91, 0x34, 0x45, 0x0d, 0x25, 0x86, 0x5e, 0x51, 0xd4, 0xf9, 0x2c, 0xd3, 0xca, 0x36, - 0x09, 0x04, 0xa1, 0xb0, 0x60, 0xb5, 0x4f, 0x01, 0x52, 0x34, 0x26, 0x1d, 0x47, 0xa8, 0x5c, 0xa5, - 0x27, 0x3b, 0x4d, 0x50, 0xa0, 0xc8, 0x91, 0x5c, 0x52, 0x07, 0x1d, 0xef, 0xce, 0x77, 0x47, 0xa9, - 0x82, 0x91, 0x97, 0x7e, 0x80, 0xfe, 0x41, 0x1f, 0xf2, 0xd8, 0x87, 0x7e, 0x93, 0x3e, 0xf5, 0x31, - 0x40, 0x81, 0xa2, 0x8f, 0x85, 0xdd, 0x0f, 0x52, 0xec, 0xec, 0xde, 0xdd, 0xde, 0x1f, 0xaa, 0xa5, - 0x81, 0xea, 0xed, 0x66, 0x66, 0x39, 0xf3, 0x9b, 0x3f, 0x3b, 0x3b, 0xbb, 0x84, 0xf6, 0x64, 0x3e, - 0xb7, 0xc2, 0x88, 0x05, 0x8f, 0xfc, 0xc0, 0x8b, 0x3c, 0x52, 0xf5, 0x47, 0x46, 0x7b, 0x32, 0xbf, - 0xf2, 0x82, 0x8b, 0x98, 0x67, 0xec, 0xce, 0x3c, 0x6f, 0xe6, 0xb0, 0x43, 0xcb, 0xb7, 0x0f, 0x2d, - 0xd7, 0xf5, 0x22, 0x2b, 0xb2, 0x3d, 0x37, 0x14, 0x52, 0xfa, 0x0d, 0x74, 0xce, 0x22, 0x2b, 0x88, - 0x5e, 0x58, 0xe1, 0x85, 0xc9, 0x5e, 0x2d, 0x58, 0x18, 0x11, 0x02, 0xf5, 0xc8, 0x0a, 0x2f, 0x74, - 0x6d, 0x4f, 0xdb, 0x5f, 0x37, 0xf1, 0x9b, 0xe8, 0xb0, 0x16, 0x7a, 0x8b, 0x60, 0xcc, 0x42, 0xbd, - 0xba, 0x57, 0xdb, 0x5f, 0x37, 0x63, 0x92, 0xf4, 0x01, 0x02, 0x36, 0xf7, 0x2e, 0xd9, 0x73, 0x16, - 0x59, 0x7a, 0x6d, 0x4f, 0xdb, 0x6f, 0x9a, 0x0a, 0x87, 0xbe, 0x82, 0xbb, 0x8a, 0x85, 0xd0, 0xf7, - 0xdc, 0x90, 0x91, 0x1e, 0x34, 0x02, 0x16, 0x2e, 0x9c, 0x08, 0x8d, 0x34, 0x4d, 0x49, 0x91, 0x0e, - 0xd4, 0xe6, 0xe1, 0x4c, 0xaf, 0xa2, 0x65, 0xfe, 0x49, 0x8e, 0x52, 0xc3, 0xb5, 0xbd, 0xda, 0x7e, - 0xeb, 0x48, 0x7f, 0xe4, 0x8f, 0x1e, 0x0d, 0xbc, 0xf9, 0xdc, 0x73, 0x7f, 0x89, 0x7e, 0xc6, 0x4a, - 0x13, 0x48, 0xf4, 0xd7, 0x40, 0x4e, 0x7d, 0x16, 0x58, 0x11, 0x53, 0xdd, 0x32, 0xa0, 0xea, 0xf9, - 0x68, 0xaf, 0x7d, 0x04, 0x5c, 0x09, 0x17, 0x9e, 0xfa, 0x66, 0xd5, 0xf3, 0xb9, 0xcb, 0xae, 0x35, - 0x67, 0xd2, 0x30, 0x7e, 0xab, 0x2e, 0xd7, 0x32, 0x2e, 0xd3, 0xdf, 0x6b, 0xb0, 0x95, 0x31, 0x20, - 0xbd, 0xba, 0xc9, 0x42, 0xea, 0x71, 0xb5, 0xcc, 0xe3, 0x5a, 0xa9, 0xc7, 0xf5, 0xff, 0xd5, 0xe3, - 0x4f, 0xe1, 0xee, 0x4b, 0x7f, 0x92, 0x73, 0x78, 0xa5, 0x3c, 0xd2, 0x00, 0x88, 0xaa, 0xe2, 0x56, - 0x12, 0xf5, 0x19, 0xf4, 0x7e, 0xb1, 0x60, 0xc1, 0xf5, 0x59, 0x64, 0x45, 0x8b, 0xf0, 0xc4, 0x0e, - 0x23, 0x05, 0x3b, 0x26, 0x44, 0x2b, 0x4f, 0x48, 0x0e, 0xfb, 0x25, 0xec, 0x14, 0xf4, 0xac, 0xec, - 0xc0, 0xe3, 0xbc, 0x03, 0x3b, 0xdc, 0x01, 0x45, 0x6f, 0x11, 0xff, 0x00, 0xb6, 0xce, 0xce, 0xbd, - 0xab, 0xe1, 0xf0, 0xe4, 0xc4, 0x1b, 0x5f, 0x84, 0xef, 0x16, 0xf8, 0x3f, 0x6b, 0xb0, 0x26, 0x35, - 0x90, 0x36, 0x54, 0x8f, 0x87, 0xf2, 0x77, 0xd5, 0xe3, 0x61, 0xa2, 0xa9, 0xaa, 0x68, 0x22, 0x50, - 0x9f, 0x7b, 0x13, 0x26, 0x4b, 0x06, 0xbf, 0x49, 0x17, 0xee, 0x78, 0x57, 0x2e, 0x0b, 0xf4, 0x3a, - 0x32, 0x05, 0xc1, 0x57, 0x0e, 0x87, 0x27, 0xa1, 0x7e, 0x07, 0x0d, 0xe2, 0x37, 0x8f, 0x47, 0x78, - 0xed, 0x8e, 0xd9, 0x44, 0x6f, 0x20, 0x57, 0x52, 0xc4, 0x80, 0xe6, 0xc2, 0x95, 0x92, 0x35, 0x94, - 0x24, 0x34, 0x1d, 0x43, 0x37, 0xeb, 0xe6, 0xca, 0xb1, 0x7d, 0x1f, 0xee, 0x38, 0xfc, 0xa7, 0x32, - 0xb2, 0x2d, 0x1e, 0x59, 0xa9, 0xce, 0x14, 0x12, 0xea, 0x40, 0xf7, 0xa5, 0xcb, 0x3f, 0x63, 0xbe, - 0x0c, 0x66, 0x3e, 0x24, 0x14, 0x36, 0x02, 0xe6, 0x3b, 0xd6, 0x98, 0x9d, 0xa2, 0xc7, 0xc2, 0x4a, - 0x86, 0x47, 0xf6, 0xa0, 0x35, 0xf5, 0x82, 0x31, 0x33, 0xb1, 0x0d, 0xc9, 0xa6, 0xa4, 0xb2, 0xe8, - 0xa7, 0xb0, 0x9d, 0xb3, 0xb6, 0xaa, 0x4f, 0xd4, 0x84, 0x7b, 0xb2, 0x09, 0xc4, 0xe5, 0xed, 0x58, - 0xd7, 0x31, 0xea, 0xfb, 0x4a, 0x2b, 0x40, 0x6f, 0x51, 0x2a, 0x7b, 0xc1, 0xf2, 0x5a, 0xf8, 0x4e, - 0x03, 0xa3, 0x4c, 0xa9, 0x04, 0x77, 0xa3, 0xd6, 0xff, 0x6f, 0x87, 0xf9, 0x4e, 0x83, 0x9d, 0x2f, - 0x16, 0xc1, 0xac, 0xcc, 0x59, 0xc5, 0x1f, 0x2d, 0x7b, 0x38, 0x18, 0xd0, 0xb4, 0x5d, 0x6b, 0x1c, - 0xd9, 0x97, 0x4c, 0xa2, 0x4a, 0x68, 0xac, 0x6d, 0x7b, 0x2e, 0xb2, 0x53, 0x33, 0xf1, 0x9b, 0xaf, - 0x9f, 0xda, 0x0e, 0xc3, 0xad, 0x2f, 0x4a, 0x39, 0xa1, 0xb1, 0x72, 0x17, 0xa3, 0xa1, 0x1d, 0xe8, - 0x77, 0x50, 0x22, 0x29, 0xfa, 0x1b, 0xd0, 0x8b, 0xc0, 0x6e, 0xa5, 0x7d, 0x7d, 0x05, 0x9d, 0xc1, - 0x39, 0x1b, 0x5f, 0xfc, 0xb7, 0xa6, 0xdb, 0x83, 0x06, 0x0b, 0x82, 0x81, 0x2b, 0x32, 0x53, 0x33, - 0x25, 0xc5, 0xe3, 0x76, 0x65, 0x05, 0x2e, 0x17, 0x88, 0x20, 0xc4, 0x24, 0xfd, 0x04, 0xee, 0x2a, - 0x9a, 0x57, 0x2e, 0xcd, 0x73, 0xe8, 0xca, 0x2a, 0x3a, 0x43, 0xa8, 0x31, 0xb8, 0x5d, 0xa5, 0x7e, - 0x36, 0xb8, 0x7f, 0x42, 0x9c, 0x16, 0xd0, 0xd8, 0x73, 0xa7, 0xf6, 0x4c, 0x56, 0xa5, 0xa4, 0x78, - 0x52, 0x84, 0xc7, 0xc7, 0x43, 0x79, 0x12, 0x26, 0x34, 0x5d, 0xc0, 0x76, 0xce, 0xd2, 0xad, 0x44, - 0xfe, 0x29, 0x6c, 0x9b, 0x6c, 0x66, 0xf3, 0xd1, 0x27, 0x5e, 0x72, 0xe3, 0xb9, 0x61, 0x4d, 0x26, - 0x01, 0x0b, 0x43, 0x69, 0x36, 0x26, 0xe9, 0x13, 0xe8, 0xe5, 0xd5, 0xac, 0x1c, 0xeb, 0x9f, 0x40, - 0xf7, 0x74, 0x3a, 0x75, 0x6c, 0x97, 0x3d, 0x67, 0xf3, 0x51, 0x06, 0x49, 0x74, 0xed, 0x27, 0x48, - 0xf8, 0x77, 0xd9, 0x98, 0xc1, 0x3b, 0x51, 0xee, 0xf7, 0x2b, 0x43, 0xf8, 0x71, 0x92, 0xee, 0x13, - 0x66, 0x4d, 0x52, 0x08, 0x85, 0x74, 0x0b, 0xb1, 0x48, 0x37, 0x1a, 0xce, 0xfe, 0x6a, 0x65, 0xc3, - 0xbf, 0xd3, 0x00, 0x9e, 0xe3, 0x00, 0x7a, 0xec, 0x4e, 0xbd, 0xd2, 0xe0, 0x1b, 0xd0, 0x9c, 0xa3, - 0x5f, 0xc7, 0x43, 0xfc, 0x65, 0xdd, 0x4c, 0x68, 0x7e, 0x6a, 0x59, 0x8e, 0x9d, 0x34, 0x68, 0x41, - 0xf0, 0x5f, 0xf8, 0x8c, 0x05, 0x2f, 0xcd, 0x13, 0xd1, 0x9e, 0xd6, 0xcd, 0x84, 0xe6, 0xc3, 0xe6, - 0xd8, 0xb1, 0x99, 0x1b, 0xa1, 0x54, 0x9c, 0x6b, 0x0a, 0x87, 0x8e, 0x00, 0x44, 0x22, 0x97, 0xe2, - 0x21, 0x50, 0xe7, 0xd9, 0x8f, 0x53, 0xc0, 0xbf, 0x39, 0x8e, 0x30, 0xb2, 0x66, 0xf1, 0x91, 0x2a, - 0x08, 0xec, 0x37, 0x58, 0x6e, 0xb2, 0x13, 0x49, 0x8a, 0x9e, 0x40, 0x87, 0x4f, 0x18, 0x22, 0x68, - 0x22, 0x67, 0x71, 0x68, 0xb4, 0xb4, 0xaa, 0xcb, 0x26, 0xca, 0xd8, 0x76, 0x2d, 0xb5, 0x4d, 0x7f, - 0x2e, 0xb4, 0x89, 0x28, 0x2e, 0xd5, 0xb6, 0x0f, 0x6b, 0x62, 0xd0, 0x17, 0x27, 0x46, 0xeb, 0xa8, - 0xcd, 0xd3, 0x99, 0x86, 0xde, 0x8c, 0xc5, 0xb1, 0x3e, 0x11, 0x85, 0x9b, 0xf4, 0x89, 0x4b, 0x42, - 0x46, 0x5f, 0x1a, 0x3a, 0x33, 0x16, 0xd3, 0xbf, 0x68, 0xb0, 0x26, 0xd4, 0x84, 0xe4, 0x11, 0x34, - 0x1c, 0xf4, 0x1a, 0x55, 0xb5, 0x8e, 0xba, 0x58, 0x53, 0xb9, 0x58, 0x7c, 0x5e, 0x31, 0xe5, 0x2a, - 0xbe, 0x5e, 0xc0, 0xc2, 0x28, 0x28, 0xeb, 0x55, 0x6f, 0xf9, 0x7a, 0xb1, 0x8a, 0xaf, 0x17, 0x66, - 0x31, 0x42, 0xca, 0x7a, 0xd5, 0x1b, 0xbe, 0x5e, 0xac, 0x7a, 0xd2, 0x84, 0x86, 0xa8, 0x25, 0x7e, - 0xc9, 0x40, 0xbd, 0x99, 0x1d, 0xd8, 0xcb, 0xc0, 0x6d, 0x26, 0xb0, 0x7a, 0x19, 0x58, 0xcd, 0xc4, - 0x7c, 0x2f, 0x63, 0xbe, 0x19, 0x9b, 0xe1, 0xe5, 0xc1, 0xd3, 0x17, 0x57, 0xa3, 0x20, 0x28, 0x03, - 0xa2, 0x9a, 0x5c, 0xb9, 0xed, 0x7d, 0x08, 0x6b, 0x02, 0x7c, 0x66, 0x28, 0x92, 0xa1, 0x36, 0x63, - 0x19, 0xfd, 0x87, 0x96, 0xf6, 0xf2, 0xf1, 0x39, 0x9b, 0x5b, 0xcb, 0x7b, 0x39, 0x8a, 0xd3, 0x0b, - 0x4d, 0x61, 0x70, 0x5c, 0x7a, 0xa1, 0xe1, 0x5b, 0x6e, 0x62, 0x45, 0xd6, 0xc8, 0x0a, 0x93, 0x63, - 0x37, 0xa6, 0xb9, 0xf7, 0x91, 0x35, 0x72, 0x98, 0x3c, 0x75, 0x05, 0x81, 0x9b, 0x03, 0xed, 0xe9, - 0x0d, 0xb9, 0x39, 0x90, 0xe2, 0xab, 0xa7, 0xce, 0x22, 0x3c, 0xd7, 0xd7, 0xc4, 0x96, 0x46, 0x82, - 0xa3, 0xe1, 0xa3, 0xa4, 0xde, 0x44, 0x26, 0x7e, 0xab, 0x27, 0x87, 0xf4, 0xeb, 0x56, 0x4e, 0x8e, - 0x03, 0xe8, 0x3e, 0x63, 0xd1, 0xd9, 0x62, 0xc4, 0x8f, 0xd6, 0xc1, 0x74, 0x76, 0xc3, 0xc1, 0x41, - 0x5f, 0xc2, 0x76, 0x6e, 0xed, 0xca, 0x10, 0x09, 0xd4, 0xc7, 0xd3, 0x59, 0x1c, 0x70, 0xfc, 0xa6, - 0x43, 0xd8, 0x7c, 0xc6, 0x22, 0xc5, 0xf6, 0x43, 0xe5, 0xa8, 0x90, 0x83, 0xdd, 0x60, 0x3a, 0x7b, - 0x71, 0xed, 0xb3, 0x1b, 0xce, 0x8d, 0x13, 0x68, 0xc7, 0x5a, 0x56, 0x46, 0xd5, 0x81, 0xda, 0x78, - 0x9a, 0x8c, 0x84, 0xe3, 0xe9, 0x8c, 0x6e, 0xc3, 0xd6, 0x33, 0x26, 0xf7, 0x65, 0x8a, 0x8c, 0xee, - 0x63, 0xb4, 0x14, 0xb6, 0x34, 0x25, 0x15, 0x68, 0xa9, 0x82, 0x3f, 0x6a, 0x40, 0x3e, 0xb7, 0xdc, - 0x89, 0xc3, 0x9e, 0x06, 0x81, 0x17, 0x2c, 0x9d, 0x83, 0x51, 0xfa, 0x4e, 0x45, 0xba, 0x0b, 0xeb, - 0x23, 0xdb, 0x75, 0xbc, 0xd9, 0x17, 0x5e, 0x28, 0xab, 0x34, 0x65, 0x60, 0x89, 0xbd, 0x72, 0x92, - 0xbb, 0x0e, 0xff, 0xa6, 0x21, 0x6c, 0x65, 0x20, 0xdd, 0x4a, 0x81, 0x3d, 0x83, 0xed, 0x17, 0x81, - 0xe5, 0x86, 0x53, 0x16, 0x64, 0x87, 0xaf, 0xf4, 0x3c, 0xd1, 0xd4, 0xf3, 0x44, 0x69, 0x3b, 0xc2, - 0xb2, 0xa4, 0xf8, 0x70, 0x92, 0x57, 0xb4, 0xf2, 0x01, 0x3d, 0x49, 0x1e, 0x2a, 0x32, 0x03, 0xfb, - 0x03, 0x25, 0x2b, 0x9b, 0xca, 0x3d, 0xe2, 0xcb, 0xa3, 0x78, 0x10, 0x94, 0x48, 0xab, 0x4b, 0x90, - 0x8a, 0xd4, 0xc4, 0x48, 0x7f, 0x9a, 0xb4, 0xa8, 0x77, 0x9c, 0xbe, 0x0f, 0x46, 0xd0, 0x8c, 0x47, - 0x51, 0xb2, 0x05, 0xef, 0x1d, 0xbb, 0x97, 0x96, 0x63, 0x4f, 0x62, 0x56, 0xa7, 0x42, 0xde, 0x83, - 0x16, 0xbe, 0x22, 0x09, 0x56, 0x47, 0x23, 0x1d, 0xd8, 0x10, 0xcf, 0x15, 0x92, 0x53, 0x25, 0x6d, - 0x80, 0xb3, 0xc8, 0xf3, 0x25, 0x5d, 0x43, 0xfa, 0xdc, 0xbb, 0x92, 0x74, 0xfd, 0xe0, 0x67, 0xd0, - 0x8c, 0xe7, 0x1f, 0xc5, 0x46, 0xcc, 0xea, 0x54, 0xc8, 0x5d, 0xd8, 0x7c, 0x7a, 0x69, 0x8f, 0xa3, - 0x84, 0xa5, 0x91, 0x1d, 0xd8, 0x1a, 0x58, 0xee, 0x98, 0x39, 0x59, 0x41, 0xf5, 0xe0, 0x2b, 0x58, - 0x93, 0x5b, 0x94, 0x43, 0x93, 0xba, 0x38, 0xd9, 0xa9, 0x90, 0x0d, 0x68, 0xf2, 0x86, 0x81, 0x94, - 0xc6, 0x61, 0x88, 0xfd, 0x83, 0x34, 0xc2, 0x14, 0xa5, 0x83, 0xb4, 0x80, 0x89, 0x10, 0x91, 0xae, - 0x1f, 0x0c, 0x61, 0x3d, 0xc9, 0x06, 0xe9, 0x42, 0x47, 0xea, 0x4e, 0x78, 0x9d, 0x0a, 0xf7, 0x1d, - 0x83, 0x81, 0xbc, 0x2f, 0x8f, 0x3a, 0x9a, 0x08, 0x8f, 0xe7, 0xc7, 0x8c, 0xea, 0xd1, 0x5f, 0xdb, - 0xd0, 0x10, 0x66, 0xc9, 0xd7, 0xb0, 0x9e, 0x3c, 0xc0, 0x11, 0x3c, 0x52, 0xf3, 0x2f, 0x7e, 0xc6, - 0x76, 0x8e, 0x2b, 0xf2, 0x47, 0x1f, 0xfe, 0xf6, 0xef, 0xff, 0xfe, 0x53, 0xf5, 0x1e, 0xed, 0x1e, - 0x5a, 0xbe, 0x1d, 0x1e, 0x5e, 0x3e, 0xb6, 0x1c, 0xff, 0xdc, 0x7a, 0x7c, 0xc8, 0x37, 0x6a, 0xf8, - 0xb1, 0x76, 0x40, 0xa6, 0xd0, 0x52, 0xde, 0xc1, 0x48, 0x8f, 0xab, 0x29, 0xbe, 0xbc, 0x19, 0x3b, - 0x05, 0xbe, 0x34, 0xf0, 0x11, 0x1a, 0xd8, 0x33, 0xee, 0x97, 0x19, 0x38, 0x7c, 0xcd, 0xfb, 0xdc, - 0xb7, 0xdc, 0xce, 0x27, 0x00, 0xe9, 0xdb, 0x14, 0x41, 0xb4, 0x85, 0xe7, 0x2e, 0xa3, 0x97, 0x67, - 0x4b, 0x23, 0x15, 0xe2, 0x40, 0x4b, 0x79, 0xc6, 0x21, 0x46, 0xee, 0x5d, 0x47, 0x79, 0x77, 0x32, - 0xee, 0x97, 0xca, 0xa4, 0xa6, 0x0f, 0x10, 0x6e, 0x9f, 0xec, 0xe6, 0xe0, 0x86, 0xb8, 0x54, 0xe2, - 0x25, 0x03, 0xd8, 0x50, 0x5f, 0x4b, 0x08, 0x7a, 0x5f, 0xf2, 0x4c, 0x64, 0xe8, 0x45, 0x41, 0x02, - 0xf9, 0x33, 0xd8, 0xcc, 0xbc, 0x4f, 0x10, 0x5c, 0x5c, 0xf6, 0x40, 0x62, 0xdc, 0x2b, 0x91, 0x24, - 0x7a, 0xbe, 0x86, 0x5e, 0xf1, 0x3d, 0x01, 0xa3, 0xf8, 0x40, 0x49, 0x4a, 0xf1, 0x4e, 0x6f, 0xf4, - 0x97, 0x89, 0x13, 0xd5, 0xa7, 0xd0, 0xc9, 0xdf, 0xbb, 0x09, 0x86, 0x6f, 0xc9, 0x33, 0x81, 0xb1, - 0x5b, 0x2e, 0x4c, 0x14, 0x7e, 0x0c, 0xeb, 0xc9, 0xa5, 0x57, 0x14, 0x6a, 0xfe, 0x76, 0x2d, 0x0a, - 0xb5, 0x70, 0x33, 0xa6, 0x15, 0x32, 0x83, 0xcd, 0xcc, 0x3d, 0x54, 0xc4, 0xab, 0xec, 0x12, 0x2c, - 0xe2, 0x55, 0x7a, 0x69, 0xa5, 0xef, 0x63, 0x82, 0xef, 0x1b, 0xbd, 0x7c, 0x82, 0x45, 0x6f, 0xe7, - 0xa5, 0x78, 0x0c, 0xed, 0xec, 0x95, 0x91, 0xdc, 0x13, 0x0d, 0xb4, 0xe4, 0x36, 0x6a, 0x18, 0x65, - 0xa2, 0x04, 0x73, 0x00, 0x9b, 0x99, 0x9b, 0x9f, 0xc4, 0x5c, 0x72, 0x99, 0x94, 0x98, 0xcb, 0xae, - 0x89, 0xf4, 0x87, 0x88, 0xf9, 0xa3, 0x83, 0x0f, 0x72, 0x98, 0xe5, 0x00, 0x79, 0xf8, 0x9a, 0x4f, - 0x10, 0xdf, 0xc6, 0xc5, 0x79, 0x91, 0xc4, 0x49, 0x34, 0xb3, 0x4c, 0x9c, 0x32, 0xb7, 0xc7, 0x4c, - 0x9c, 0xb2, 0x37, 0x44, 0xfa, 0x21, 0xda, 0x7c, 0x68, 0x18, 0x39, 0x9b, 0x62, 0xc0, 0x3e, 0x7c, - 0xed, 0xf9, 0xb8, 0x6d, 0x7f, 0x05, 0x90, 0x8e, 0xc8, 0x62, 0xdb, 0x16, 0xa6, 0x74, 0xb1, 0x6d, - 0x8b, 0x93, 0x34, 0xed, 0xa3, 0x0d, 0x9d, 0xf4, 0xca, 0xfd, 0x22, 0xd3, 0x34, 0xe3, 0x62, 0xf4, - 0xcc, 0x64, 0x5c, 0x1d, 0x95, 0xb3, 0x19, 0xcf, 0x0c, 0x9b, 0x74, 0x0f, 0xad, 0x18, 0xc6, 0x76, - 0x3e, 0xe3, 0xb8, 0x8c, 0x3b, 0xe1, 0xe0, 0xb4, 0x96, 0x0e, 0x81, 0xc2, 0x4e, 0xd9, 0x0c, 0x29, - 0xec, 0x94, 0x4e, 0x8c, 0x71, 0xa7, 0x23, 0xfd, 0xbc, 0x9d, 0xc5, 0x48, 0x6d, 0x76, 0xe4, 0x05, - 0x34, 0xc4, 0x54, 0x47, 0xee, 0x4a, 0x65, 0x8a, 0x7e, 0xa2, 0xb2, 0xa4, 0xe2, 0x1f, 0xa0, 0xe2, - 0x07, 0xe4, 0xa6, 0x16, 0x4a, 0xbe, 0x81, 0x96, 0x32, 0x08, 0x89, 0x3e, 0x5d, 0x1c, 0xd6, 0x44, - 0x9f, 0x2e, 0x99, 0x98, 0x96, 0x46, 0x89, 0xf1, 0x55, 0xb8, 0x2d, 0x06, 0xb0, 0xa1, 0x0e, 0x8a, - 0xa2, 0xe9, 0x95, 0x4c, 0x94, 0x86, 0x5e, 0x14, 0x24, 0x1b, 0xe2, 0x18, 0xda, 0xd9, 0x89, 0x47, - 0xec, 0xad, 0xd2, 0x71, 0x4a, 0xec, 0xad, 0xf2, 0x01, 0x89, 0x56, 0x38, 0x1e, 0x75, 0x24, 0x21, - 0xea, 0x11, 0x94, 0x69, 0x4a, 0x7a, 0x51, 0x10, 0x2b, 0x79, 0xa2, 0xff, 0xed, 0x4d, 0x5f, 0xfb, - 0xfe, 0x4d, 0x5f, 0xfb, 0xd7, 0x9b, 0xbe, 0xf6, 0x87, 0xb7, 0xfd, 0xca, 0xf7, 0x6f, 0xfb, 0x95, - 0x7f, 0xbe, 0xed, 0x57, 0x46, 0x0d, 0xfc, 0xf7, 0xec, 0x47, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, - 0x8d, 0x0a, 0x25, 0x8c, 0x81, 0x1b, 0x00, 0x00, + // 2151 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x59, 0xdd, 0x6f, 0x1b, 0xc7, + 0x11, 0xe7, 0x91, 0x32, 0x45, 0x8d, 0x3e, 0x4c, 0xad, 0x24, 0xea, 0x7c, 0x96, 0x69, 0x66, 0xeb, + 0x04, 0x82, 0x50, 0x58, 0xb0, 0x5a, 0xf4, 0x21, 0x40, 0x8a, 0xc6, 0xa4, 0xa3, 0x10, 0x95, 0xab, + 0xf4, 0x64, 0xa7, 0x09, 0x0a, 0x14, 0x39, 0x92, 0x4b, 0xea, 0xa0, 0xe3, 0xdd, 0xf9, 0xee, 0x28, + 0x45, 0x30, 0xf2, 0xd2, 0xe7, 0xa2, 0x1f, 0xc8, 0x43, 0x1e, 0xfb, 0xd0, 0x7f, 0xa6, 0x8f, 0x01, + 0x0a, 0x14, 0x7d, 0x2c, 0xec, 0xfe, 0x21, 0xc5, 0xce, 0xee, 0xdd, 0xed, 0x7d, 0x50, 0x2d, 0x0d, + 0x44, 0x2f, 0xc2, 0xcd, 0xce, 0x72, 0xe6, 0x37, 0x1f, 0x3b, 0x3b, 0x3b, 0x82, 0x8d, 0xd1, 0x74, + 0x6a, 0x85, 0x11, 0x0b, 0x1e, 0xfb, 0x81, 0x17, 0x79, 0xa4, 0xea, 0x0f, 0x8c, 0x8d, 0xd1, 0xf4, + 0xca, 0x0b, 0x2e, 0xe2, 0x35, 0x63, 0x6f, 0xe2, 0x79, 0x13, 0x87, 0x1d, 0x5a, 0xbe, 0x7d, 0x68, + 0xb9, 0xae, 0x17, 0x59, 0x91, 0xed, 0xb9, 0xa1, 0xe0, 0xd2, 0xaf, 0xa0, 0x79, 0x16, 0x59, 0x41, + 0xf4, 0xc2, 0x0a, 0x2f, 0x4c, 0xf6, 0x6a, 0xc6, 0xc2, 0x88, 0x10, 0x58, 0x8a, 0xac, 0xf0, 0x42, + 0xd7, 0x3a, 0xda, 0xfe, 0x8a, 0x89, 0xdf, 0x44, 0x87, 0xe5, 0xd0, 0x9b, 0x05, 0x43, 0x16, 0xea, + 0xd5, 0x4e, 0x6d, 0x7f, 0xc5, 0x8c, 0x49, 0xd2, 0x06, 0x08, 0xd8, 0xd4, 0xbb, 0x64, 0xcf, 0x59, + 0x64, 0xe9, 0xb5, 0x8e, 0xb6, 0xdf, 0x30, 0x95, 0x15, 0xfa, 0x0a, 0x36, 0x15, 0x0d, 0xa1, 0xef, + 0xb9, 0x21, 0x23, 0x2d, 0xa8, 0x07, 0x2c, 0x9c, 0x39, 0x11, 0x2a, 0x69, 0x98, 0x92, 0x22, 0x4d, + 0xa8, 0x4d, 0xc3, 0x89, 0x5e, 0x45, 0xcd, 0xfc, 0x93, 0x1c, 0xa5, 0x8a, 0x6b, 0x9d, 0xda, 0xfe, + 0xea, 0x91, 0xfe, 0xd8, 0x1f, 0x3c, 0xee, 0x7a, 0xd3, 0xa9, 0xe7, 0xfe, 0x06, 0xed, 0x8c, 0x85, + 0x26, 0x90, 0xe8, 0xef, 0x80, 0x9c, 0xfa, 0x2c, 0xb0, 0x22, 0xa6, 0x9a, 0x65, 0x40, 0xd5, 0xf3, + 0x51, 0xdf, 0xc6, 0x11, 0x70, 0x21, 0x9c, 0x79, 0xea, 0x9b, 0x55, 0xcf, 0xe7, 0x26, 0xbb, 0xd6, + 0x94, 0x49, 0xc5, 0xf8, 0xad, 0x9a, 0x5c, 0xcb, 0x98, 0x4c, 0xff, 0xa4, 0xc1, 0x56, 0x46, 0x81, + 0xb4, 0xea, 0x26, 0x0d, 0xa9, 0xc5, 0xd5, 0x32, 0x8b, 0x6b, 0xa5, 0x16, 0x2f, 0xfd, 0xbf, 0x16, + 0x7f, 0x0c, 0x9b, 0x2f, 0xfd, 0x51, 0xce, 0xe0, 0x85, 0xe2, 0x48, 0x03, 0x20, 0xaa, 0x88, 0x5b, + 0x09, 0xd4, 0x27, 0xd0, 0xfa, 0xf5, 0x8c, 0x05, 0xd7, 0x67, 0x91, 0x15, 0xcd, 0xc2, 0x13, 0x3b, + 0x8c, 0x14, 0xec, 0x18, 0x10, 0xad, 0x3c, 0x20, 0x39, 0xec, 0x97, 0xb0, 0x5b, 0x90, 0xb3, 0xb0, + 0x01, 0x4f, 0xf2, 0x06, 0xec, 0x72, 0x03, 0x14, 0xb9, 0x45, 0xfc, 0x5d, 0xd8, 0x3a, 0x3b, 0xf7, + 0xae, 0x7a, 0xbd, 0x93, 0x13, 0x6f, 0x78, 0x11, 0xbe, 0x9b, 0xe3, 0xff, 0xaa, 0xc1, 0xb2, 0x94, + 0x40, 0x36, 0xa0, 0xda, 0xef, 0xc9, 0xdf, 0x55, 0xfb, 0xbd, 0x44, 0x52, 0x55, 0x91, 0x44, 0x60, + 0x69, 0xea, 0x8d, 0x98, 0x4c, 0x19, 0xfc, 0x26, 0xdb, 0x70, 0xc7, 0xbb, 0x72, 0x59, 0xa0, 0x2f, + 0xe1, 0xa2, 0x20, 0xf8, 0xce, 0x5e, 0xef, 0x24, 0xd4, 0xef, 0xa0, 0x42, 0xfc, 0xe6, 0xfe, 0x08, + 0xaf, 0xdd, 0x21, 0x1b, 0xe9, 0x75, 0x5c, 0x95, 0x14, 0x31, 0xa0, 0x31, 0x73, 0x25, 0x67, 0x19, + 0x39, 0x09, 0x4d, 0x87, 0xb0, 0x9d, 0x35, 0x73, 0x61, 0xdf, 0xbe, 0x07, 0x77, 0x1c, 0xfe, 0x53, + 0xe9, 0xd9, 0x55, 0xee, 0x59, 0x29, 0xce, 0x14, 0x1c, 0xea, 0xc0, 0xf6, 0x4b, 0x97, 0x7f, 0xc6, + 0xeb, 0xd2, 0x99, 0x79, 0x97, 0x50, 0x58, 0x0b, 0x98, 0xef, 0x58, 0x43, 0x76, 0x8a, 0x16, 0x0b, + 0x2d, 0x99, 0x35, 0xd2, 0x81, 0xd5, 0xb1, 0x17, 0x0c, 0x99, 0x89, 0x65, 0x48, 0x16, 0x25, 0x75, + 0x89, 0x7e, 0x0c, 0x3b, 0x39, 0x6d, 0x8b, 0xda, 0x44, 0x4d, 0xb8, 0x27, 0x8b, 0x40, 0x9c, 0xde, + 0x8e, 0x75, 0x1d, 0xa3, 0xbe, 0xaf, 0x94, 0x02, 0xb4, 0x16, 0xb9, 0xb2, 0x16, 0xcc, 0xcf, 0x85, + 0xef, 0x34, 0x30, 0xca, 0x84, 0x4a, 0x70, 0x37, 0x4a, 0xfd, 0x61, 0x2b, 0xcc, 0x77, 0x1a, 0xec, + 0x7e, 0x36, 0x0b, 0x26, 0x65, 0xc6, 0x2a, 0xf6, 0x68, 0xd9, 0xcb, 0xc1, 0x80, 0x86, 0xed, 0x5a, + 0xc3, 0xc8, 0xbe, 0x64, 0x12, 0x55, 0x42, 0x63, 0x6e, 0xdb, 0x53, 0x11, 0x9d, 0x9a, 0x89, 0xdf, + 0x7c, 0xff, 0xd8, 0x76, 0x18, 0x1e, 0x7d, 0x91, 0xca, 0x09, 0x8d, 0x99, 0x3b, 0x1b, 0xf4, 0xec, + 0x40, 0xbf, 0x83, 0x1c, 0x49, 0xd1, 0xaf, 0x41, 0x2f, 0x02, 0xbb, 0x95, 0xf2, 0xf5, 0x05, 0x34, + 0xbb, 0xe7, 0x6c, 0x78, 0xf1, 0xbf, 0x8a, 0x6e, 0x0b, 0xea, 0x2c, 0x08, 0xba, 0xae, 0x88, 0x4c, + 0xcd, 0x94, 0x14, 0xf7, 0xdb, 0x95, 0x15, 0xb8, 0x9c, 0x21, 0x9c, 0x10, 0x93, 0xf4, 0x23, 0xd8, + 0x54, 0x24, 0x2f, 0x9c, 0x9a, 0xe7, 0xb0, 0x2d, 0xb3, 0xe8, 0x0c, 0xa1, 0xc6, 0xe0, 0xf6, 0x94, + 0xfc, 0x59, 0xe3, 0xf6, 0x09, 0x76, 0x9a, 0x40, 0x43, 0xcf, 0x1d, 0xdb, 0x13, 0x99, 0x95, 0x92, + 0xe2, 0x41, 0x11, 0x16, 0xf7, 0x7b, 0xf2, 0x26, 0x4c, 0x68, 0x3a, 0x83, 0x9d, 0x9c, 0xa6, 0x5b, + 0xf1, 0xfc, 0x33, 0xd8, 0x31, 0xd9, 0xc4, 0xe6, 0xad, 0x4f, 0xbc, 0xe5, 0xc6, 0x7b, 0xc3, 0x1a, + 0x8d, 0x02, 0x16, 0x86, 0x52, 0x6d, 0x4c, 0xd2, 0xa7, 0xd0, 0xca, 0x8b, 0x59, 0xd8, 0xd7, 0x3f, + 0x87, 0xed, 0xd3, 0xf1, 0xd8, 0xb1, 0x5d, 0xf6, 0x9c, 0x4d, 0x07, 0x19, 0x24, 0xd1, 0xb5, 0x9f, + 0x20, 0xe1, 0xdf, 0x65, 0x6d, 0x06, 0xaf, 0x44, 0xb9, 0xdf, 0x2f, 0x0c, 0xe1, 0xa7, 0x49, 0xb8, + 0x4f, 0x98, 0x35, 0x4a, 0x21, 0x14, 0xc2, 0x2d, 0xd8, 0x22, 0xdc, 0xa8, 0x38, 0xfb, 0xab, 0x85, + 0x15, 0xff, 0x51, 0x03, 0x78, 0x8e, 0x0d, 0x68, 0xdf, 0x1d, 0x7b, 0xa5, 0xce, 0x37, 0xa0, 0x31, + 0x45, 0xbb, 0xfa, 0x3d, 0xfc, 0xe5, 0x92, 0x99, 0xd0, 0xfc, 0xd6, 0xb2, 0x1c, 0x3b, 0x29, 0xd0, + 0x82, 0xe0, 0xbf, 0xf0, 0x19, 0x0b, 0x5e, 0x9a, 0x27, 0xa2, 0x3c, 0xad, 0x98, 0x09, 0xcd, 0x9b, + 0xcd, 0xa1, 0x63, 0x33, 0x37, 0x42, 0xae, 0xb8, 0xd7, 0x94, 0x15, 0x3a, 0x00, 0x10, 0x81, 0x9c, + 0x8b, 0x87, 0xc0, 0x12, 0x8f, 0x7e, 0x1c, 0x02, 0xfe, 0xcd, 0x71, 0x84, 0x91, 0x35, 0x89, 0xaf, + 0x54, 0x41, 0x60, 0xbd, 0xc1, 0x74, 0x93, 0x95, 0x48, 0x52, 0xf4, 0x04, 0x9a, 0xbc, 0xc3, 0x10, + 0x4e, 0x13, 0x31, 0x8b, 0x5d, 0xa3, 0xa5, 0x59, 0x5d, 0xd6, 0x51, 0xc6, 0xba, 0x6b, 0xa9, 0x6e, + 0xfa, 0x2b, 0x21, 0x4d, 0x78, 0x71, 0xae, 0xb4, 0x7d, 0x58, 0x16, 0x8d, 0xbe, 0xb8, 0x31, 0x56, + 0x8f, 0x36, 0x78, 0x38, 0x53, 0xd7, 0x9b, 0x31, 0x3b, 0x96, 0x27, 0xbc, 0x70, 0x93, 0x3c, 0xf1, + 0x48, 0xc8, 0xc8, 0x4b, 0x5d, 0x67, 0xc6, 0x6c, 0xfa, 0x37, 0x0d, 0x96, 0x85, 0x98, 0x90, 0x3c, + 0x86, 0xba, 0x83, 0x56, 0xa3, 0xa8, 0xd5, 0xa3, 0x6d, 0xcc, 0xa9, 0x9c, 0x2f, 0x3e, 0xad, 0x98, + 0x72, 0x17, 0xdf, 0x2f, 0x60, 0xa1, 0x17, 0x94, 0xfd, 0xaa, 0xb5, 0x7c, 0xbf, 0xd8, 0xc5, 0xf7, + 0x0b, 0xb5, 0xe8, 0x21, 0x65, 0xbf, 0x6a, 0x0d, 0xdf, 0x2f, 0x76, 0x3d, 0x6d, 0x40, 0x5d, 0xe4, + 0x12, 0x7f, 0x64, 0xa0, 0xdc, 0xcc, 0x09, 0x6c, 0x65, 0xe0, 0x36, 0x12, 0x58, 0xad, 0x0c, 0xac, + 0x46, 0xa2, 0xbe, 0x95, 0x51, 0xdf, 0x88, 0xd5, 0xf0, 0xf4, 0xe0, 0xe1, 0x8b, 0xb3, 0x51, 0x10, + 0x94, 0x01, 0x51, 0x55, 0x2e, 0x5c, 0xf6, 0xde, 0x87, 0x65, 0x01, 0x3e, 0xd3, 0x14, 0x49, 0x57, + 0x9b, 0x31, 0x8f, 0xfe, 0x53, 0x4b, 0x6b, 0xf9, 0xf0, 0x9c, 0x4d, 0xad, 0xf9, 0xb5, 0x1c, 0xd9, + 0xe9, 0x83, 0xa6, 0xd0, 0x38, 0xce, 0x7d, 0xd0, 0xf0, 0x23, 0x37, 0xb2, 0x22, 0x6b, 0x60, 0x85, + 0xc9, 0xb5, 0x1b, 0xd3, 0xdc, 0xfa, 0xc8, 0x1a, 0x38, 0x4c, 0xde, 0xba, 0x82, 0xc0, 0xc3, 0x81, + 0xfa, 0xf4, 0xba, 0x3c, 0x1c, 0x48, 0xf1, 0xdd, 0x63, 0x67, 0x16, 0x9e, 0xeb, 0xcb, 0xe2, 0x48, + 0x23, 0xc1, 0xd1, 0xf0, 0x56, 0x52, 0x6f, 0xe0, 0x22, 0x7e, 0xab, 0x37, 0x87, 0xb4, 0xeb, 0x56, + 0x6e, 0x8e, 0x03, 0xd8, 0x3e, 0x66, 0xd1, 0xd9, 0x6c, 0xc0, 0xaf, 0xd6, 0xee, 0x78, 0x72, 0xc3, + 0xc5, 0x41, 0x5f, 0xc2, 0x4e, 0x6e, 0xef, 0xc2, 0x10, 0x09, 0x2c, 0x0d, 0xc7, 0x93, 0xd8, 0xe1, + 0xf8, 0x4d, 0x7b, 0xb0, 0x7e, 0xcc, 0x22, 0x45, 0xf7, 0x43, 0xe5, 0xaa, 0x90, 0x8d, 0x5d, 0x77, + 0x3c, 0x79, 0x71, 0xed, 0xb3, 0x1b, 0xee, 0x8d, 0x13, 0xd8, 0x88, 0xa5, 0x2c, 0x8c, 0xaa, 0x09, + 0xb5, 0xe1, 0x38, 0x69, 0x09, 0x87, 0xe3, 0x09, 0xdd, 0x81, 0xad, 0x63, 0x26, 0xcf, 0x65, 0x8a, + 0x8c, 0xee, 0xa3, 0xb7, 0x94, 0x65, 0xa9, 0x4a, 0x0a, 0xd0, 0x52, 0x01, 0x7f, 0xd1, 0x80, 0x7c, + 0x6a, 0xb9, 0x23, 0x87, 0x3d, 0x0b, 0x02, 0x2f, 0x98, 0xdb, 0x07, 0x23, 0xf7, 0x9d, 0x92, 0x74, + 0x0f, 0x56, 0x06, 0xb6, 0xeb, 0x78, 0x93, 0xcf, 0xbc, 0x50, 0x66, 0x69, 0xba, 0x80, 0x29, 0xf6, + 0xca, 0x49, 0xde, 0x3a, 0xfc, 0x9b, 0x86, 0xb0, 0x95, 0x81, 0x74, 0x2b, 0x09, 0x76, 0x0c, 0x3b, + 0x2f, 0x02, 0xcb, 0x0d, 0xc7, 0x2c, 0xc8, 0x36, 0x5f, 0xe9, 0x7d, 0xa2, 0xa9, 0xf7, 0x89, 0x52, + 0x76, 0x84, 0x66, 0x49, 0xf1, 0xe6, 0x24, 0x2f, 0x68, 0xe1, 0x0b, 0x7a, 0x94, 0x0c, 0x2a, 0x32, + 0x0d, 0xfb, 0x03, 0x25, 0x2a, 0xeb, 0xca, 0x3b, 0xe2, 0xf3, 0xa3, 0xb8, 0x11, 0x94, 0x48, 0xab, + 0x73, 0x90, 0x8a, 0xd0, 0xc4, 0x48, 0x7f, 0x91, 0x94, 0xa8, 0x77, 0xec, 0xbe, 0xe9, 0x31, 0xec, + 0xf6, 0xa7, 0xbe, 0x17, 0x44, 0xcf, 0xbe, 0xe6, 0x7f, 0xbb, 0xe3, 0x49, 0x38, 0x77, 0x6c, 0xc3, + 0x99, 0x32, 0x81, 0x9a, 0x50, 0x1b, 0xd9, 0xb1, 0xdf, 0xf8, 0x27, 0xfd, 0x19, 0xd4, 0xbb, 0xa2, + 0x6b, 0x9d, 0xd3, 0x09, 0x0e, 0x3d, 0x37, 0x62, 0xb2, 0x13, 0x5f, 0x31, 0x63, 0x92, 0xfe, 0x41, + 0x03, 0xbd, 0x88, 0x60, 0xe1, 0x84, 0x79, 0x94, 0x4f, 0x18, 0x81, 0x18, 0x11, 0xa5, 0x99, 0xdc, + 0xe1, 0x25, 0x35, 0xbc, 0x88, 0x5f, 0x5f, 0xea, 0x1e, 0xc1, 0x38, 0x18, 0x40, 0x23, 0x6e, 0xcd, + 0xc9, 0x16, 0xdc, 0xed, 0xbb, 0x97, 0x96, 0x63, 0x8f, 0xe2, 0xa5, 0x66, 0x85, 0xdc, 0x85, 0x55, + 0x9c, 0xaa, 0x89, 0xa5, 0xa6, 0x46, 0x9a, 0xb0, 0x26, 0xc6, 0x37, 0x72, 0xa5, 0x4a, 0x36, 0x00, + 0xce, 0x22, 0xcf, 0x97, 0x74, 0x0d, 0xe9, 0x73, 0xef, 0x4a, 0xd2, 0x4b, 0x07, 0xbf, 0x84, 0x46, + 0xdc, 0x0f, 0x2a, 0x3a, 0xe2, 0xa5, 0x66, 0x85, 0x6c, 0xc2, 0xfa, 0xb3, 0x4b, 0x7b, 0x18, 0x25, + 0x4b, 0x1a, 0xd9, 0x85, 0xad, 0xae, 0xe5, 0x0e, 0x99, 0x93, 0x65, 0x54, 0x0f, 0xbe, 0x80, 0x65, + 0x59, 0xb2, 0x38, 0x34, 0x29, 0x8b, 0x93, 0xcd, 0x0a, 0x59, 0x83, 0x06, 0x2f, 0xa0, 0x48, 0x69, + 0x1c, 0x86, 0xa8, 0x27, 0x48, 0x23, 0x4c, 0x71, 0x94, 0x90, 0x16, 0x30, 0x11, 0x22, 0xd2, 0x4b, + 0x07, 0x3d, 0x58, 0x49, 0xb2, 0x93, 0x6c, 0x43, 0x53, 0xca, 0x4e, 0xd6, 0x9a, 0x15, 0x6e, 0x3b, + 0x3a, 0x03, 0xd7, 0x3e, 0x3f, 0x6a, 0x6a, 0xc2, 0x3d, 0x9e, 0x1f, 0x2f, 0x54, 0x0f, 0x3a, 0x50, + 0x17, 0x79, 0x43, 0x00, 0xea, 0x22, 0xc4, 0xcd, 0x0a, 0xff, 0x16, 0x41, 0x6f, 0x6a, 0x47, 0xdf, + 0xde, 0x85, 0xba, 0x00, 0x46, 0xbe, 0x84, 0x95, 0x64, 0x64, 0x49, 0xb0, 0x09, 0xc9, 0xcf, 0x48, + 0x8d, 0x9d, 0xdc, 0xaa, 0xc8, 0x14, 0xfa, 0xf0, 0xf7, 0xff, 0xf8, 0xcf, 0xb7, 0xd5, 0x7b, 0x74, + 0xfb, 0xd0, 0xf2, 0xed, 0xf0, 0xf0, 0xf2, 0x89, 0xe5, 0xf8, 0xe7, 0xd6, 0x93, 0x43, 0x8c, 0xea, + 0x87, 0xda, 0x01, 0x19, 0xc3, 0xaa, 0x32, 0x39, 0x24, 0x2d, 0x2e, 0xa6, 0x38, 0xab, 0x34, 0x76, + 0x0b, 0xeb, 0x52, 0xc1, 0x07, 0xa8, 0xa0, 0x63, 0xdc, 0x2f, 0x53, 0x70, 0xf8, 0x9a, 0x67, 0xf9, + 0x37, 0x5c, 0xcf, 0x47, 0x00, 0xe9, 0x34, 0x8f, 0x20, 0xda, 0xc2, 0x80, 0xd0, 0x68, 0xe5, 0x97, + 0xa5, 0x92, 0x0a, 0x71, 0x60, 0x55, 0x19, 0x7c, 0x11, 0x23, 0x37, 0x09, 0x53, 0x26, 0x75, 0xc6, + 0xfd, 0x52, 0x9e, 0x94, 0xf4, 0x08, 0xe1, 0xb6, 0xc9, 0x5e, 0x0e, 0x6e, 0x88, 0x5b, 0x25, 0x5e, + 0xd2, 0x85, 0x35, 0x75, 0xbe, 0x44, 0xd0, 0xfa, 0x92, 0xc1, 0x9a, 0xa1, 0x17, 0x19, 0x09, 0xe4, + 0x4f, 0x60, 0x3d, 0x33, 0xd1, 0x21, 0xb8, 0xb9, 0x6c, 0xa4, 0x64, 0xdc, 0x2b, 0xe1, 0x24, 0x72, + 0xbe, 0x84, 0x56, 0x71, 0x02, 0x83, 0x5e, 0x7c, 0xa0, 0x04, 0xa5, 0x38, 0x05, 0x31, 0xda, 0xf3, + 0xd8, 0x89, 0xe8, 0x53, 0x68, 0xe6, 0x27, 0x15, 0x04, 0xdd, 0x37, 0x67, 0xb0, 0x62, 0xec, 0x95, + 0x33, 0x13, 0x81, 0x1f, 0xc2, 0x4a, 0x32, 0x26, 0x10, 0x89, 0x9a, 0x9f, 0x47, 0x88, 0x44, 0x2d, + 0xcc, 0x12, 0x68, 0x85, 0x4c, 0x60, 0x3d, 0xf3, 0x72, 0x17, 0xfe, 0x2a, 0x1b, 0x1b, 0x08, 0x7f, + 0x95, 0x3e, 0xf3, 0xe9, 0x7b, 0x18, 0xe0, 0xfb, 0x46, 0x2b, 0x1f, 0x60, 0x51, 0xea, 0x78, 0x2a, + 0xf6, 0x61, 0x23, 0xfb, 0xc8, 0x26, 0xf7, 0xc4, 0x95, 0x53, 0xf2, 0x7e, 0x37, 0x8c, 0x32, 0x56, + 0x82, 0x39, 0x80, 0xf5, 0xcc, 0x5b, 0x59, 0x62, 0x2e, 0x79, 0x7e, 0x4b, 0xcc, 0x65, 0x0f, 0x6b, + 0xfa, 0x63, 0xc4, 0xfc, 0xc1, 0xc1, 0xa3, 0x1c, 0x66, 0xd9, 0x72, 0x1f, 0xbe, 0xe6, 0x3d, 0xd7, + 0x37, 0x71, 0x72, 0x5e, 0x24, 0x7e, 0x12, 0xe5, 0x2e, 0xe3, 0xa7, 0xcc, 0x7b, 0x3b, 0xe3, 0xa7, + 0xec, 0x9b, 0x9a, 0xbe, 0x8f, 0x3a, 0x1f, 0x1a, 0x46, 0x4e, 0xa7, 0x78, 0x92, 0x1c, 0xbe, 0xf6, + 0x7c, 0x3c, 0xb6, 0xbf, 0x05, 0x48, 0x1f, 0x15, 0xe2, 0xd8, 0x16, 0xde, 0x35, 0xe2, 0xd8, 0x16, + 0xdf, 0x1e, 0xb4, 0x8d, 0x3a, 0x74, 0xd2, 0x2a, 0xb7, 0x8b, 0x8c, 0xd3, 0x88, 0x8b, 0x66, 0x3d, + 0x13, 0x71, 0xf5, 0x71, 0x91, 0x8d, 0x78, 0xa6, 0x3d, 0xa7, 0x1d, 0xd4, 0x62, 0x18, 0x3b, 0xf9, + 0x88, 0xe3, 0x36, 0x6e, 0x84, 0x83, 0xfd, 0x6d, 0xda, 0x36, 0x0b, 0x3d, 0x65, 0x5d, 0xb7, 0xd0, + 0x53, 0xda, 0x63, 0xc7, 0x95, 0x8e, 0xb4, 0xf3, 0x7a, 0x66, 0x03, 0xb5, 0xd8, 0x91, 0x17, 0x50, + 0x17, 0x7d, 0x30, 0xd9, 0x94, 0xc2, 0x14, 0xf9, 0x44, 0x5d, 0x92, 0x82, 0x7f, 0x84, 0x82, 0x1f, + 0x90, 0x9b, 0x4a, 0x28, 0xf9, 0x0a, 0x56, 0x95, 0xd6, 0x51, 0xd4, 0xe9, 0x62, 0x7b, 0x2b, 0xea, + 0x74, 0x49, 0x8f, 0x39, 0xd7, 0x4b, 0x8c, 0xef, 0xc2, 0x63, 0xd1, 0x85, 0x35, 0xb5, 0xb5, 0x16, + 0x45, 0xaf, 0xa4, 0x07, 0x37, 0xf4, 0x22, 0x23, 0x39, 0x10, 0x7d, 0xd8, 0xc8, 0xf6, 0x88, 0xe2, + 0x6c, 0x95, 0x36, 0xa0, 0xe2, 0x6c, 0x95, 0xb7, 0x94, 0xb4, 0xc2, 0xf1, 0xa8, 0x4d, 0x1c, 0x51, + 0xaf, 0xa0, 0x4c, 0x51, 0xd2, 0x8b, 0x0c, 0xb5, 0xc2, 0xe5, 0xbb, 0x28, 0x51, 0xe1, 0xe6, 0x74, + 0x77, 0xa2, 0xc2, 0xcd, 0x6b, 0xbc, 0x68, 0xe5, 0xa9, 0xfe, 0xf7, 0x37, 0x6d, 0xed, 0xfb, 0x37, + 0x6d, 0xed, 0xdf, 0x6f, 0xda, 0xda, 0x9f, 0xdf, 0xb6, 0x2b, 0xdf, 0xbf, 0x6d, 0x57, 0xfe, 0xf5, + 0xb6, 0x5d, 0x19, 0xd4, 0xf1, 0x1f, 0x98, 0x3f, 0xf9, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x08, + 0x8c, 0xd5, 0xa0, 0x04, 0x1d, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -3341,6 +3549,7 @@ type MasterClient interface { GetMasterCfg(ctx context.Context, in *GetMasterCfgRequest, opts ...grpc.CallOption) (*GetMasterCfgResponse, error) TransferSource(ctx context.Context, in *TransferSourceRequest, opts ...grpc.CallOption) (*TransferSourceResponse, error) OperateRelay(ctx context.Context, in *OperateRelayRequest, opts ...grpc.CallOption) (*OperateRelayResponse, error) + ImportExportCfgs(ctx context.Context, in *ImportExportCfgsRequest, opts ...grpc.CallOption) (*ImportExportCfgsResponse, error) } type masterClient struct { @@ -3540,6 +3749,15 @@ func (c *masterClient) OperateRelay(ctx context.Context, in *OperateRelayRequest return out, nil } +func (c *masterClient) ImportExportCfgs(ctx context.Context, in *ImportExportCfgsRequest, opts ...grpc.CallOption) (*ImportExportCfgsResponse, error) { + out := new(ImportExportCfgsResponse) + err := c.cc.Invoke(ctx, "/pb.Master/ImportExportCfgs", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // MasterServer is the server API for Master service. type MasterServer interface { StartTask(context.Context, *StartTaskRequest) (*StartTaskResponse, error) @@ -3576,6 +3794,7 @@ type MasterServer interface { GetMasterCfg(context.Context, *GetMasterCfgRequest) (*GetMasterCfgResponse, error) TransferSource(context.Context, *TransferSourceRequest) (*TransferSourceResponse, error) OperateRelay(context.Context, *OperateRelayRequest) (*OperateRelayResponse, error) + ImportExportCfgs(context.Context, *ImportExportCfgsRequest) (*ImportExportCfgsResponse, error) } // UnimplementedMasterServer can be embedded to have forward compatible implementations. @@ -3645,6 +3864,9 @@ func (*UnimplementedMasterServer) TransferSource(ctx context.Context, req *Trans func (*UnimplementedMasterServer) OperateRelay(ctx context.Context, req *OperateRelayRequest) (*OperateRelayResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method OperateRelay not implemented") } +func (*UnimplementedMasterServer) ImportExportCfgs(ctx context.Context, req *ImportExportCfgsRequest) (*ImportExportCfgsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ImportExportCfgs not implemented") +} func RegisterMasterServer(s *grpc.Server, srv MasterServer) { s.RegisterService(&_Master_serviceDesc, srv) @@ -4028,6 +4250,24 @@ func _Master_OperateRelay_Handler(srv interface{}, ctx context.Context, dec func return interceptor(ctx, in, info, handler) } +func _Master_ImportExportCfgs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ImportExportCfgsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MasterServer).ImportExportCfgs(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/pb.Master/ImportExportCfgs", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MasterServer).ImportExportCfgs(ctx, req.(*ImportExportCfgsRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _Master_serviceDesc = grpc.ServiceDesc{ ServiceName: "pb.Master", HandlerType: (*MasterServer)(nil), @@ -4116,6 +4356,10 @@ var _Master_serviceDesc = grpc.ServiceDesc{ MethodName: "OperateRelay", Handler: _Master_OperateRelay_Handler, }, + { + MethodName: "ImportExportCfgs", + Handler: _Master_ImportExportCfgs_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "dmmaster.proto", @@ -6469,71 +6713,211 @@ func (m *OperateRelayResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func encodeVarintDmmaster(dAtA []byte, offset int, v uint64) int { - offset -= sovDmmaster(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ +func (m *ImportExportCfgsRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err } - dAtA[offset] = uint8(v) - return base + return dAtA[:n], nil } -func (m *StartTaskRequest) Size() (n int) { - if m == nil { - return 0 - } + +func (m *ImportExportCfgsRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ImportExportCfgsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i var l int _ = l - l = len(m.Task) - if l > 0 { - n += 1 + l + sovDmmaster(uint64(l)) - } - if len(m.Sources) > 0 { - for _, s := range m.Sources { - l = len(s) - n += 1 + l + sovDmmaster(uint64(l)) - } + if len(m.Dir) > 0 { + i -= len(m.Dir) + copy(dAtA[i:], m.Dir) + i = encodeVarintDmmaster(dAtA, i, uint64(len(m.Dir))) + i-- + dAtA[i] = 0x12 } - if m.RemoveMeta { - n += 2 + if m.Op != 0 { + i = encodeVarintDmmaster(dAtA, i, uint64(m.Op)) + i-- + dAtA[i] = 0x8 } - return n + return len(dAtA) - i, nil } -func (m *StartTaskResponse) Size() (n int) { - if m == nil { - return 0 +func (m *Config) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err } + return dAtA[:n], nil +} + +func (m *Config) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Config) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i var l int _ = l - if m.Result { - n += 2 - } - l = len(m.Msg) - if l > 0 { - n += 1 + l + sovDmmaster(uint64(l)) + if len(m.Content) > 0 { + i -= len(m.Content) + copy(dAtA[i:], m.Content) + i = encodeVarintDmmaster(dAtA, i, uint64(len(m.Content))) + i-- + dAtA[i] = 0x12 } - if len(m.Sources) > 0 { - for _, e := range m.Sources { - l = e.Size() - n += 1 + l + sovDmmaster(uint64(l)) - } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintDmmaster(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa } - return n + return len(dAtA) - i, nil } -func (m *OperateTaskRequest) Size() (n int) { - if m == nil { - return 0 +func (m *ImportExportCfgsResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err } + return dAtA[:n], nil +} + +func (m *ImportExportCfgsResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ImportExportCfgsResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i var l int _ = l - if m.Op != 0 { - n += 1 + sovDmmaster(uint64(m.Op)) + if len(m.Tasks) > 0 { + for iNdEx := len(m.Tasks) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Tasks[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintDmmaster(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } } - l = len(m.Name) + if len(m.Sources) > 0 { + for iNdEx := len(m.Sources) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Sources[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintDmmaster(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + } + if len(m.Msg) > 0 { + i -= len(m.Msg) + copy(dAtA[i:], m.Msg) + i = encodeVarintDmmaster(dAtA, i, uint64(len(m.Msg))) + i-- + dAtA[i] = 0x12 + } + if m.Result { + i-- + if m.Result { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func encodeVarintDmmaster(dAtA []byte, offset int, v uint64) int { + offset -= sovDmmaster(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *StartTaskRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Task) + if l > 0 { + n += 1 + l + sovDmmaster(uint64(l)) + } + if len(m.Sources) > 0 { + for _, s := range m.Sources { + l = len(s) + n += 1 + l + sovDmmaster(uint64(l)) + } + } + if m.RemoveMeta { + n += 2 + } + return n +} + +func (m *StartTaskResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Result { + n += 2 + } + l = len(m.Msg) + if l > 0 { + n += 1 + l + sovDmmaster(uint64(l)) + } + if len(m.Sources) > 0 { + for _, e := range m.Sources { + l = e.Size() + n += 1 + l + sovDmmaster(uint64(l)) + } + } + return n +} + +func (m *OperateTaskRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Op != 0 { + n += 1 + sovDmmaster(uint64(m.Op)) + } + l = len(m.Name) if l > 0 { n += 1 + l + sovDmmaster(uint64(l)) } @@ -7531,6 +7915,67 @@ func (m *OperateRelayResponse) Size() (n int) { return n } +func (m *ImportExportCfgsRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Op != 0 { + n += 1 + sovDmmaster(uint64(m.Op)) + } + l = len(m.Dir) + if l > 0 { + n += 1 + l + sovDmmaster(uint64(l)) + } + return n +} + +func (m *Config) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovDmmaster(uint64(l)) + } + l = len(m.Content) + if l > 0 { + n += 1 + l + sovDmmaster(uint64(l)) + } + return n +} + +func (m *ImportExportCfgsResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Result { + n += 2 + } + l = len(m.Msg) + if l > 0 { + n += 1 + l + sovDmmaster(uint64(l)) + } + if len(m.Sources) > 0 { + for _, e := range m.Sources { + l = e.Size() + n += 1 + l + sovDmmaster(uint64(l)) + } + } + if len(m.Tasks) > 0 { + for _, e := range m.Tasks { + l = e.Size() + n += 1 + l + sovDmmaster(uint64(l)) + } + } + return n +} + func sovDmmaster(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -14119,6 +14564,400 @@ func (m *OperateRelayResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *ImportExportCfgsRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDmmaster + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ImportExportCfgsRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ImportExportCfgsRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Op", wireType) + } + m.Op = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDmmaster + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Op |= CfgsOp(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Dir", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDmmaster + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDmmaster + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthDmmaster + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Dir = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipDmmaster(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthDmmaster + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthDmmaster + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Config) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDmmaster + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Config: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Config: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDmmaster + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDmmaster + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthDmmaster + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Content", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDmmaster + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDmmaster + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthDmmaster + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Content = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipDmmaster(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthDmmaster + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthDmmaster + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ImportExportCfgsResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDmmaster + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ImportExportCfgsResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ImportExportCfgsResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Result", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDmmaster + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Result = bool(v != 0) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Msg", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDmmaster + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDmmaster + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthDmmaster + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Msg = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Sources", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDmmaster + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthDmmaster + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthDmmaster + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Sources = append(m.Sources, &Config{}) + if err := m.Sources[len(m.Sources)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Tasks", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDmmaster + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthDmmaster + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthDmmaster + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Tasks = append(m.Tasks, &Config{}) + if err := m.Tasks[len(m.Tasks)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipDmmaster(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthDmmaster + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthDmmaster + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipDmmaster(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/dm/pbmock/dmmaster.go b/dm/pbmock/dmmaster.go index 39f9123c3f..f7436976ae 100644 --- a/dm/pbmock/dmmaster.go +++ b/dm/pbmock/dmmaster.go @@ -136,6 +136,26 @@ func (mr *MockMasterClientMockRecorder) HandleError(arg0, arg1 interface{}, arg2 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandleError", reflect.TypeOf((*MockMasterClient)(nil).HandleError), varargs...) } +// ImportExportCfgs mocks base method. +func (m *MockMasterClient) ImportExportCfgs(arg0 context.Context, arg1 *pb.ImportExportCfgsRequest, arg2 ...grpc.CallOption) (*pb.ImportExportCfgsResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ImportExportCfgs", varargs...) + ret0, _ := ret[0].(*pb.ImportExportCfgsResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ImportExportCfgs indicates an expected call of ImportExportCfgs. +func (mr *MockMasterClientMockRecorder) ImportExportCfgs(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImportExportCfgs", reflect.TypeOf((*MockMasterClient)(nil).ImportExportCfgs), varargs...) +} + // ListMember mocks base method. func (m *MockMasterClient) ListMember(arg0 context.Context, arg1 *pb.ListMemberRequest, arg2 ...grpc.CallOption) (*pb.ListMemberResponse, error) { m.ctrl.T.Helper() @@ -554,6 +574,21 @@ func (mr *MockMasterServerMockRecorder) HandleError(arg0, arg1 interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandleError", reflect.TypeOf((*MockMasterServer)(nil).HandleError), arg0, arg1) } +// ImportExportCfgs mocks base method. +func (m *MockMasterServer) ImportExportCfgs(arg0 context.Context, arg1 *pb.ImportExportCfgsRequest) (*pb.ImportExportCfgsResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ImportExportCfgs", arg0, arg1) + ret0, _ := ret[0].(*pb.ImportExportCfgsResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ImportExportCfgs indicates an expected call of ImportExportCfgs. +func (mr *MockMasterServerMockRecorder) ImportExportCfgs(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImportExportCfgs", reflect.TypeOf((*MockMasterServer)(nil).ImportExportCfgs), arg0, arg1) +} + // ListMember mocks base method. func (m *MockMasterServer) ListMember(arg0 context.Context, arg1 *pb.ListMemberRequest) (*pb.ListMemberResponse, error) { m.ctrl.T.Helper() diff --git a/dm/proto/dmmaster.proto b/dm/proto/dmmaster.proto index 9f4a7e2b02..9cd0d6a7a4 100644 --- a/dm/proto/dmmaster.proto +++ b/dm/proto/dmmaster.proto @@ -108,6 +108,8 @@ service Master { rpc TransferSource(TransferSourceRequest) returns(TransferSourceResponse) {} rpc OperateRelay(OperateRelayRequest) returns(OperateRelayResponse) {} + + rpc ImportExportCfgs(ImportExportCfgsRequest) returns(ImportExportCfgsResponse) {} } message StartTaskRequest { @@ -458,4 +460,26 @@ enum RelayOpV2 { InvalidRelayOpV2 = 0; StartRelayV2 = 1; StopRelayV2 = 2; +} + +message ImportExportCfgsRequest { + CfgsOp op = 1; + string dir = 2; +} + +message Config { + string name = 1; + string content = 2; +} + +message ImportExportCfgsResponse { + bool result = 1; + string msg = 2; + repeated Config sources = 3; + repeated Config tasks = 4; +} + +enum CfgsOp { + Export = 0; + Import = 1; } \ No newline at end of file From 98055b4d2fbec935f492267cdb44926577a0fa1e Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 15 Jul 2021 10:25:55 +0800 Subject: [PATCH 02/19] add export-config cmd --- dm/ctl/common/util.go | 12 +- dm/ctl/ctl.go | 1 + dm/ctl/master/import_export_config.go | 99 ++- dm/master/server.go | 50 -- dm/pb/dmmaster.pb.go | 1187 ++++--------------------- dm/pbmock/dmmaster.go | 35 - dm/proto/dmmaster.proto | 24 - pkg/ha/source.go | 25 + 8 files changed, 270 insertions(+), 1163 deletions(-) diff --git a/dm/ctl/common/util.go b/dm/ctl/common/util.go index 25106a34ea..0fa4522018 100644 --- a/dm/ctl/common/util.go +++ b/dm/ctl/common/util.go @@ -58,9 +58,9 @@ var ( type CtlClient struct { mu sync.RWMutex tls *toolutils.TLS - etcdClient *clientv3.Client conn *grpc.ClientConn - MasterClient pb.MasterClient // exposed to be used in test + MasterClient pb.MasterClient // exposed to be used in test + EtcdClient *clientv3.Client // exposed to be used in export config } func (c *CtlClient) updateMasterClient() error { @@ -76,7 +76,7 @@ func (c *CtlClient) updateMasterClient() error { c.conn.Close() } - endpoints := c.etcdClient.Endpoints() + endpoints := c.EtcdClient.Endpoints() for _, endpoint := range endpoints { //nolint:staticcheck conn, err = grpc.Dial(utils.UnwrapScheme(endpoint), c.tls.ToGRPCDialOption(), grpc.WithBackoffMaxDelay(3*time.Second), grpc.WithBlock(), grpc.WithTimeout(3*time.Second)) @@ -176,7 +176,7 @@ func InitClient(addr string, securityCfg config.Security) error { GlobalCtlClient = &CtlClient{ tls: tls, - etcdClient: etcdClient, + EtcdClient: etcdClient, } return GlobalCtlClient.updateMasterClient() @@ -355,7 +355,7 @@ func SyncMasterEndpoints(ctx context.Context) { clientURLs := []string{} updateF := func() { clientURLs = clientURLs[:0] - resp, err := GlobalCtlClient.etcdClient.MemberList(ctx) + resp, err := GlobalCtlClient.EtcdClient.MemberList(ctx) if err != nil { return } @@ -366,7 +366,7 @@ func SyncMasterEndpoints(ctx context.Context) { if utils.NonRepeatStringsEqual(clientURLs, lastClientUrls) { return } - GlobalCtlClient.etcdClient.SetEndpoints(clientURLs...) + GlobalCtlClient.EtcdClient.SetEndpoints(clientURLs...) lastClientUrls = make([]string, len(clientURLs)) copy(lastClientUrls, clientURLs) } diff --git a/dm/ctl/ctl.go b/dm/ctl/ctl.go index 70d1474af9..d2eb23b7dc 100644 --- a/dm/ctl/ctl.go +++ b/dm/ctl/ctl.go @@ -75,6 +75,7 @@ func NewRootCmd() *cobra.Command { master.NewTransferSourceCmd(), master.NewStartRelayCmd(), master.NewStopRelayCmd(), + master.NewExportCfgCmd(), newDecryptCmd(), newEncryptCmd(), ) diff --git a/dm/ctl/master/import_export_config.go b/dm/ctl/master/import_export_config.go index 695cb81f69..53f7e60f4e 100644 --- a/dm/ctl/master/import_export_config.go +++ b/dm/ctl/master/import_export_config.go @@ -14,15 +14,16 @@ package master import ( - "context" "io/ioutil" "os" "path" + "sort" "github.com/spf13/cobra" + "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/dm/ctl/common" - "github.com/pingcap/dm/dm/pb" + "github.com/pingcap/dm/pkg/ha" ) var ( @@ -50,66 +51,94 @@ func exportCfgFunc(cmd *cobra.Command, args []string) error { return err } - ctx, cancel := context.WithTimeout(context.Background(), common.GlobalConfig().RPCTimeout) - defer cancel() - - resp := &pb.ImportExportCfgsResponse{} - err = common.SendRequest( - ctx, - "ImportExportCfgs", - &pb.ImportExportCfgsRequest{ - Op: pb.CfgsOp_Export, - Dir: dir, - }, - &resp, - ) + // get all source cfgs + sourceCfgsMap, _, err := ha.GetSourceCfg(common.GlobalCtlClient.EtcdClient, "", 0) + if err != nil { + common.PrintLinesf("can not get source configs from etcd") + return err + } + // try to get all source cfgs before v2.0.2 + if len(sourceCfgsMap) == 0 { + sourceCfgsMap, _, err = ha.GetAllSourceCfgBeforeV202(common.GlobalCtlClient.EtcdClient) + if err != nil { + common.PrintLinesf("can not get source configs from etcd") + return err + } + } + // get all task configs. + subTaskCfgsMap, _, err := ha.GetAllSubTaskCfg(common.GlobalCtlClient.EtcdClient) if err != nil { - common.PrintLinesf("can not export configs to %s", dir) + common.PrintLinesf("can not get subtask configs from etcd") return err } - if !resp.Result { - common.PrettyPrintResponse(resp) - return nil + taskCfgsMap := make(map[string]string, len(subTaskCfgsMap)) + subTaskCfgsListMap := make(map[string][]*config.SubTaskConfig, len(subTaskCfgsMap)) + + // from source => task => subtask to task => source => subtask + for _, subTaskCfgs := range subTaskCfgsMap { + for task, subTaskCfg := range subTaskCfgs { + clone := subTaskCfg + if subTaskCfgList, ok := subTaskCfgsListMap[task]; ok { + subTaskCfgsListMap[task] = append(subTaskCfgList, &clone) + } else { + subTaskCfgsListMap[task] = []*config.SubTaskConfig{&clone} + } + } + } + // from task => source => subtask to task => taskCfg + for task, subTaskCfgs := range subTaskCfgsListMap { + sort.Slice(subTaskCfgs, func(i, j int) bool { + return subTaskCfgs[i].SourceID < subTaskCfgs[j].SourceID + }) + taskCfg := config.FromSubTaskConfigs(subTaskCfgs...) + taskCfgsMap[task] = taskCfg.String() } + // create directory if err = os.MkdirAll(dir, 0o755); err != nil { - common.PrintLinesf("can not create directory %s", dir) + common.PrintLinesf("can not create directory `%s`", dir) return err } - taskDir := path.Join(dir, taskDirname) if err = os.MkdirAll(taskDir, 0o755); err != nil { - common.PrintLinesf("can not create directory of task configs %s", taskDir) + common.PrintLinesf("can not create directory of task configs `%s`", taskDir) return err } - sourceDir := path.Join(dir, sourceDirname) - if err = os.MkdirAll(sourceDirname, 0o755); err != nil { - common.PrintLinesf("can not create directory of source configs %s", path.Join(dir+sourceDir)) + if err = os.MkdirAll(sourceDir, 0o755); err != nil { + common.PrintLinesf("can not create directory of source configs `%s`", sourceDir) return err } - for _, taskCfg := range resp.Tasks { - taskFile := path.Join(taskDir, taskCfg.Name) - taskFile += yamlSuffix - err = ioutil.WriteFile(taskFile, []byte(taskCfg.Content), 0o644) + // write sourceCfg files + for source, sourceCfg := range sourceCfgsMap { + sourceFile := path.Join(sourceDir, source) + sourceFile += yamlSuffix + fileContent, err2 := sourceCfg.Yaml() + if err2 != nil { + common.PrintLinesf("fail to marshal source config of `%s`", source) + return err2 + } + err = ioutil.WriteFile(sourceFile, []byte(fileContent), 0o644) if err != nil { - common.PrintLinesf("can not write task config to file %s", taskFile) + common.PrintLinesf("fail to write source config to file `%s`", sourceFile) return err } } - for _, sourceCfg := range resp.Sources { - sourceFile := path.Join(sourceDir, sourceCfg.Name, ".yaml") - sourceFile += yamlSuffix - err = ioutil.WriteFile(sourceFile, []byte(sourceCfg.Content), 0o644) + // write taskCfg files + for task, taskCfg := range taskCfgsMap { + taskFile := path.Join(taskDir, task) + taskFile += yamlSuffix + err = ioutil.WriteFile(taskFile, []byte(taskCfg), 0o644) if err != nil { - common.PrintLinesf("can not write source config to file %s", sourceFile) + common.PrintLinesf("can not write task config to file `%s`", taskFile) return err } } + common.PrintLinesf("export configs to directory `%s` succeed", dir) return nil } diff --git a/dm/master/server.go b/dm/master/server.go index 9a78cad564..61a9d0ca94 100644 --- a/dm/master/server.go +++ b/dm/master/server.go @@ -2071,56 +2071,6 @@ func (s *Server) GetCfg(ctx context.Context, req *pb.GetCfgRequest) (*pb.GetCfgR }, nil } -func (s *Server) exportCfgs() ([]*pb.Config, []*pb.Config) { - subTaskCfgsMap := s.scheduler.GetSubTaskCfgs() - sourceCfgsMap := s.scheduler.GetSourceCfgs() - taskCfgs := make([]*pb.Config, 0, len(subTaskCfgsMap)) - sourceCfgs := make([]*pb.Config, 0, len(sourceCfgsMap)) - - for task, subTaskCfgs := range subTaskCfgsMap { - subTaskCfgList := make([]*config.SubTaskConfig, 0, len(subTaskCfgs)) - for _, subTaskCfg := range subTaskCfgs { - clone := subTaskCfg - subTaskCfgList = append(subTaskCfgList, &clone) - } - sort.Slice(subTaskCfgList, func(i, j int) bool { - return subTaskCfgList[i].SourceID < subTaskCfgList[j].SourceID - }) - taskCfg := config.FromSubTaskConfigs(subTaskCfgList...) - taskCfgs = append(taskCfgs, &pb.Config{Name: task, Content: taskCfg.String()}) - } - - for sourceID, sourceCfg := range sourceCfgsMap { - sourceCfgs = append(sourceCfgs, &pb.Config{Name: sourceID, Content: sourceCfg.String()}) - } - - return taskCfgs, sourceCfgs -} - -// ImportExportCfgs implements MasterServer.ImportExportCfgs. -func (s *Server) ImportExportCfgs(ctx context.Context, req *pb.ImportExportCfgsRequest) (*pb.ImportExportCfgsResponse, error) { - var ( - resp = &pb.ImportExportCfgsResponse{} - err2 error - ) - shouldRet := s.sharedLogic(ctx, req, &resp, &err2) - if shouldRet { - return resp, err2 - } - - switch req.Op { - case pb.CfgsOp_Import: - resp.Tasks, resp.Sources = s.exportCfgs() - case pb.CfgsOp_Export: - default: - resp.Msg = fmt.Sprintf("invalid configs op '%s'", req.Op) - return resp, nil - } - - resp.Result = true - return resp, nil -} - // HandleError implements MasterServer.HandleError. func (s *Server) HandleError(ctx context.Context, req *pb.HandleErrorRequest) (*pb.HandleErrorResponse, error) { var ( diff --git a/dm/pb/dmmaster.pb.go b/dm/pb/dmmaster.pb.go index f9458b79d0..216fb036ea 100644 --- a/dm/pb/dmmaster.pb.go +++ b/dm/pb/dmmaster.pb.go @@ -152,31 +152,6 @@ func (RelayOpV2) EnumDescriptor() ([]byte, []int) { return fileDescriptor_f9bef11f2a341f03, []int{3} } -type CfgsOp int32 - -const ( - CfgsOp_Export CfgsOp = 0 - CfgsOp_Import CfgsOp = 1 -) - -var CfgsOp_name = map[int32]string{ - 0: "Export", - 1: "Import", -} - -var CfgsOp_value = map[string]int32{ - "Export": 0, - "Import": 1, -} - -func (x CfgsOp) String() string { - return proto.EnumName(CfgsOp_name, int32(x)) -} - -func (CfgsOp) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_f9bef11f2a341f03, []int{4} -} - type StartTaskRequest struct { Task string `protobuf:"bytes,1,opt,name=task,proto3" json:"task,omitempty"` Sources []string `protobuf:"bytes,2,rep,name=sources,proto3" json:"sources,omitempty"` @@ -3130,184 +3105,11 @@ func (m *OperateRelayResponse) GetMsg() string { return "" } -type ImportExportCfgsRequest struct { - Op CfgsOp `protobuf:"varint,1,opt,name=op,proto3,enum=pb.CfgsOp" json:"op,omitempty"` - Dir string `protobuf:"bytes,2,opt,name=dir,proto3" json:"dir,omitempty"` -} - -func (m *ImportExportCfgsRequest) Reset() { *m = ImportExportCfgsRequest{} } -func (m *ImportExportCfgsRequest) String() string { return proto.CompactTextString(m) } -func (*ImportExportCfgsRequest) ProtoMessage() {} -func (*ImportExportCfgsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f9bef11f2a341f03, []int{49} -} -func (m *ImportExportCfgsRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ImportExportCfgsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ImportExportCfgsRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *ImportExportCfgsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportExportCfgsRequest.Merge(m, src) -} -func (m *ImportExportCfgsRequest) XXX_Size() int { - return m.Size() -} -func (m *ImportExportCfgsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ImportExportCfgsRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ImportExportCfgsRequest proto.InternalMessageInfo - -func (m *ImportExportCfgsRequest) GetOp() CfgsOp { - if m != nil { - return m.Op - } - return CfgsOp_Export -} - -func (m *ImportExportCfgsRequest) GetDir() string { - if m != nil { - return m.Dir - } - return "" -} - -type Config struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` -} - -func (m *Config) Reset() { *m = Config{} } -func (m *Config) String() string { return proto.CompactTextString(m) } -func (*Config) ProtoMessage() {} -func (*Config) Descriptor() ([]byte, []int) { - return fileDescriptor_f9bef11f2a341f03, []int{50} -} -func (m *Config) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Config) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Config.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Config) XXX_Merge(src proto.Message) { - xxx_messageInfo_Config.Merge(m, src) -} -func (m *Config) XXX_Size() int { - return m.Size() -} -func (m *Config) XXX_DiscardUnknown() { - xxx_messageInfo_Config.DiscardUnknown(m) -} - -var xxx_messageInfo_Config proto.InternalMessageInfo - -func (m *Config) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -func (m *Config) GetContent() string { - if m != nil { - return m.Content - } - return "" -} - -type ImportExportCfgsResponse struct { - Result bool `protobuf:"varint,1,opt,name=result,proto3" json:"result,omitempty"` - Msg string `protobuf:"bytes,2,opt,name=msg,proto3" json:"msg,omitempty"` - Sources []*Config `protobuf:"bytes,3,rep,name=sources,proto3" json:"sources,omitempty"` - Tasks []*Config `protobuf:"bytes,4,rep,name=tasks,proto3" json:"tasks,omitempty"` -} - -func (m *ImportExportCfgsResponse) Reset() { *m = ImportExportCfgsResponse{} } -func (m *ImportExportCfgsResponse) String() string { return proto.CompactTextString(m) } -func (*ImportExportCfgsResponse) ProtoMessage() {} -func (*ImportExportCfgsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_f9bef11f2a341f03, []int{51} -} -func (m *ImportExportCfgsResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ImportExportCfgsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ImportExportCfgsResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *ImportExportCfgsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportExportCfgsResponse.Merge(m, src) -} -func (m *ImportExportCfgsResponse) XXX_Size() int { - return m.Size() -} -func (m *ImportExportCfgsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ImportExportCfgsResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_ImportExportCfgsResponse proto.InternalMessageInfo - -func (m *ImportExportCfgsResponse) GetResult() bool { - if m != nil { - return m.Result - } - return false -} - -func (m *ImportExportCfgsResponse) GetMsg() string { - if m != nil { - return m.Msg - } - return "" -} - -func (m *ImportExportCfgsResponse) GetSources() []*Config { - if m != nil { - return m.Sources - } - return nil -} - -func (m *ImportExportCfgsResponse) GetTasks() []*Config { - if m != nil { - return m.Tasks - } - return nil -} - func init() { proto.RegisterEnum("pb.SourceOp", SourceOp_name, SourceOp_value) proto.RegisterEnum("pb.LeaderOp", LeaderOp_name, LeaderOp_value) proto.RegisterEnum("pb.CfgType", CfgType_name, CfgType_value) proto.RegisterEnum("pb.RelayOpV2", RelayOpV2_name, RelayOpV2_value) - proto.RegisterEnum("pb.CfgsOp", CfgsOp_name, CfgsOp_value) proto.RegisterType((*StartTaskRequest)(nil), "pb.StartTaskRequest") proto.RegisterType((*StartTaskResponse)(nil), "pb.StartTaskResponse") proto.RegisterType((*OperateTaskRequest)(nil), "pb.OperateTaskRequest") @@ -3357,150 +3159,140 @@ func init() { proto.RegisterType((*TransferSourceResponse)(nil), "pb.TransferSourceResponse") proto.RegisterType((*OperateRelayRequest)(nil), "pb.OperateRelayRequest") proto.RegisterType((*OperateRelayResponse)(nil), "pb.OperateRelayResponse") - proto.RegisterType((*ImportExportCfgsRequest)(nil), "pb.ImportExportCfgsRequest") - proto.RegisterType((*Config)(nil), "pb.Config") - proto.RegisterType((*ImportExportCfgsResponse)(nil), "pb.ImportExportCfgsResponse") } func init() { proto.RegisterFile("dmmaster.proto", fileDescriptor_f9bef11f2a341f03) } var fileDescriptor_f9bef11f2a341f03 = []byte{ - // 2151 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x59, 0xdd, 0x6f, 0x1b, 0xc7, - 0x11, 0xe7, 0x91, 0x32, 0x45, 0x8d, 0x3e, 0x4c, 0xad, 0x24, 0xea, 0x7c, 0x96, 0x69, 0x66, 0xeb, - 0x04, 0x82, 0x50, 0x58, 0xb0, 0x5a, 0xf4, 0x21, 0x40, 0x8a, 0xc6, 0xa4, 0xa3, 0x10, 0x95, 0xab, - 0xf4, 0x64, 0xa7, 0x09, 0x0a, 0x14, 0x39, 0x92, 0x4b, 0xea, 0xa0, 0xe3, 0xdd, 0xf9, 0xee, 0x28, - 0x45, 0x30, 0xf2, 0xd2, 0xe7, 0xa2, 0x1f, 0xc8, 0x43, 0x1e, 0xfb, 0xd0, 0x7f, 0xa6, 0x8f, 0x01, - 0x0a, 0x14, 0x7d, 0x2c, 0xec, 0xfe, 0x21, 0xc5, 0xce, 0xee, 0xdd, 0xed, 0x7d, 0x50, 0x2d, 0x0d, - 0x44, 0x2f, 0xc2, 0xcd, 0xce, 0x72, 0xe6, 0x37, 0x1f, 0x3b, 0x3b, 0x3b, 0x82, 0x8d, 0xd1, 0x74, - 0x6a, 0x85, 0x11, 0x0b, 0x1e, 0xfb, 0x81, 0x17, 0x79, 0xa4, 0xea, 0x0f, 0x8c, 0x8d, 0xd1, 0xf4, - 0xca, 0x0b, 0x2e, 0xe2, 0x35, 0x63, 0x6f, 0xe2, 0x79, 0x13, 0x87, 0x1d, 0x5a, 0xbe, 0x7d, 0x68, - 0xb9, 0xae, 0x17, 0x59, 0x91, 0xed, 0xb9, 0xa1, 0xe0, 0xd2, 0xaf, 0xa0, 0x79, 0x16, 0x59, 0x41, - 0xf4, 0xc2, 0x0a, 0x2f, 0x4c, 0xf6, 0x6a, 0xc6, 0xc2, 0x88, 0x10, 0x58, 0x8a, 0xac, 0xf0, 0x42, - 0xd7, 0x3a, 0xda, 0xfe, 0x8a, 0x89, 0xdf, 0x44, 0x87, 0xe5, 0xd0, 0x9b, 0x05, 0x43, 0x16, 0xea, - 0xd5, 0x4e, 0x6d, 0x7f, 0xc5, 0x8c, 0x49, 0xd2, 0x06, 0x08, 0xd8, 0xd4, 0xbb, 0x64, 0xcf, 0x59, - 0x64, 0xe9, 0xb5, 0x8e, 0xb6, 0xdf, 0x30, 0x95, 0x15, 0xfa, 0x0a, 0x36, 0x15, 0x0d, 0xa1, 0xef, - 0xb9, 0x21, 0x23, 0x2d, 0xa8, 0x07, 0x2c, 0x9c, 0x39, 0x11, 0x2a, 0x69, 0x98, 0x92, 0x22, 0x4d, - 0xa8, 0x4d, 0xc3, 0x89, 0x5e, 0x45, 0xcd, 0xfc, 0x93, 0x1c, 0xa5, 0x8a, 0x6b, 0x9d, 0xda, 0xfe, - 0xea, 0x91, 0xfe, 0xd8, 0x1f, 0x3c, 0xee, 0x7a, 0xd3, 0xa9, 0xe7, 0xfe, 0x06, 0xed, 0x8c, 0x85, - 0x26, 0x90, 0xe8, 0xef, 0x80, 0x9c, 0xfa, 0x2c, 0xb0, 0x22, 0xa6, 0x9a, 0x65, 0x40, 0xd5, 0xf3, - 0x51, 0xdf, 0xc6, 0x11, 0x70, 0x21, 0x9c, 0x79, 0xea, 0x9b, 0x55, 0xcf, 0xe7, 0x26, 0xbb, 0xd6, - 0x94, 0x49, 0xc5, 0xf8, 0xad, 0x9a, 0x5c, 0xcb, 0x98, 0x4c, 0xff, 0xa4, 0xc1, 0x56, 0x46, 0x81, - 0xb4, 0xea, 0x26, 0x0d, 0xa9, 0xc5, 0xd5, 0x32, 0x8b, 0x6b, 0xa5, 0x16, 0x2f, 0xfd, 0xbf, 0x16, - 0x7f, 0x0c, 0x9b, 0x2f, 0xfd, 0x51, 0xce, 0xe0, 0x85, 0xe2, 0x48, 0x03, 0x20, 0xaa, 0x88, 0x5b, - 0x09, 0xd4, 0x27, 0xd0, 0xfa, 0xf5, 0x8c, 0x05, 0xd7, 0x67, 0x91, 0x15, 0xcd, 0xc2, 0x13, 0x3b, - 0x8c, 0x14, 0xec, 0x18, 0x10, 0xad, 0x3c, 0x20, 0x39, 0xec, 0x97, 0xb0, 0x5b, 0x90, 0xb3, 0xb0, - 0x01, 0x4f, 0xf2, 0x06, 0xec, 0x72, 0x03, 0x14, 0xb9, 0x45, 0xfc, 0x5d, 0xd8, 0x3a, 0x3b, 0xf7, - 0xae, 0x7a, 0xbd, 0x93, 0x13, 0x6f, 0x78, 0x11, 0xbe, 0x9b, 0xe3, 0xff, 0xaa, 0xc1, 0xb2, 0x94, - 0x40, 0x36, 0xa0, 0xda, 0xef, 0xc9, 0xdf, 0x55, 0xfb, 0xbd, 0x44, 0x52, 0x55, 0x91, 0x44, 0x60, - 0x69, 0xea, 0x8d, 0x98, 0x4c, 0x19, 0xfc, 0x26, 0xdb, 0x70, 0xc7, 0xbb, 0x72, 0x59, 0xa0, 0x2f, - 0xe1, 0xa2, 0x20, 0xf8, 0xce, 0x5e, 0xef, 0x24, 0xd4, 0xef, 0xa0, 0x42, 0xfc, 0xe6, 0xfe, 0x08, - 0xaf, 0xdd, 0x21, 0x1b, 0xe9, 0x75, 0x5c, 0x95, 0x14, 0x31, 0xa0, 0x31, 0x73, 0x25, 0x67, 0x19, - 0x39, 0x09, 0x4d, 0x87, 0xb0, 0x9d, 0x35, 0x73, 0x61, 0xdf, 0xbe, 0x07, 0x77, 0x1c, 0xfe, 0x53, - 0xe9, 0xd9, 0x55, 0xee, 0x59, 0x29, 0xce, 0x14, 0x1c, 0xea, 0xc0, 0xf6, 0x4b, 0x97, 0x7f, 0xc6, - 0xeb, 0xd2, 0x99, 0x79, 0x97, 0x50, 0x58, 0x0b, 0x98, 0xef, 0x58, 0x43, 0x76, 0x8a, 0x16, 0x0b, - 0x2d, 0x99, 0x35, 0xd2, 0x81, 0xd5, 0xb1, 0x17, 0x0c, 0x99, 0x89, 0x65, 0x48, 0x16, 0x25, 0x75, - 0x89, 0x7e, 0x0c, 0x3b, 0x39, 0x6d, 0x8b, 0xda, 0x44, 0x4d, 0xb8, 0x27, 0x8b, 0x40, 0x9c, 0xde, - 0x8e, 0x75, 0x1d, 0xa3, 0xbe, 0xaf, 0x94, 0x02, 0xb4, 0x16, 0xb9, 0xb2, 0x16, 0xcc, 0xcf, 0x85, - 0xef, 0x34, 0x30, 0xca, 0x84, 0x4a, 0x70, 0x37, 0x4a, 0xfd, 0x61, 0x2b, 0xcc, 0x77, 0x1a, 0xec, - 0x7e, 0x36, 0x0b, 0x26, 0x65, 0xc6, 0x2a, 0xf6, 0x68, 0xd9, 0xcb, 0xc1, 0x80, 0x86, 0xed, 0x5a, - 0xc3, 0xc8, 0xbe, 0x64, 0x12, 0x55, 0x42, 0x63, 0x6e, 0xdb, 0x53, 0x11, 0x9d, 0x9a, 0x89, 0xdf, - 0x7c, 0xff, 0xd8, 0x76, 0x18, 0x1e, 0x7d, 0x91, 0xca, 0x09, 0x8d, 0x99, 0x3b, 0x1b, 0xf4, 0xec, - 0x40, 0xbf, 0x83, 0x1c, 0x49, 0xd1, 0xaf, 0x41, 0x2f, 0x02, 0xbb, 0x95, 0xf2, 0xf5, 0x05, 0x34, - 0xbb, 0xe7, 0x6c, 0x78, 0xf1, 0xbf, 0x8a, 0x6e, 0x0b, 0xea, 0x2c, 0x08, 0xba, 0xae, 0x88, 0x4c, - 0xcd, 0x94, 0x14, 0xf7, 0xdb, 0x95, 0x15, 0xb8, 0x9c, 0x21, 0x9c, 0x10, 0x93, 0xf4, 0x23, 0xd8, - 0x54, 0x24, 0x2f, 0x9c, 0x9a, 0xe7, 0xb0, 0x2d, 0xb3, 0xe8, 0x0c, 0xa1, 0xc6, 0xe0, 0xf6, 0x94, - 0xfc, 0x59, 0xe3, 0xf6, 0x09, 0x76, 0x9a, 0x40, 0x43, 0xcf, 0x1d, 0xdb, 0x13, 0x99, 0x95, 0x92, - 0xe2, 0x41, 0x11, 0x16, 0xf7, 0x7b, 0xf2, 0x26, 0x4c, 0x68, 0x3a, 0x83, 0x9d, 0x9c, 0xa6, 0x5b, - 0xf1, 0xfc, 0x33, 0xd8, 0x31, 0xd9, 0xc4, 0xe6, 0xad, 0x4f, 0xbc, 0xe5, 0xc6, 0x7b, 0xc3, 0x1a, - 0x8d, 0x02, 0x16, 0x86, 0x52, 0x6d, 0x4c, 0xd2, 0xa7, 0xd0, 0xca, 0x8b, 0x59, 0xd8, 0xd7, 0x3f, - 0x87, 0xed, 0xd3, 0xf1, 0xd8, 0xb1, 0x5d, 0xf6, 0x9c, 0x4d, 0x07, 0x19, 0x24, 0xd1, 0xb5, 0x9f, - 0x20, 0xe1, 0xdf, 0x65, 0x6d, 0x06, 0xaf, 0x44, 0xb9, 0xdf, 0x2f, 0x0c, 0xe1, 0xa7, 0x49, 0xb8, - 0x4f, 0x98, 0x35, 0x4a, 0x21, 0x14, 0xc2, 0x2d, 0xd8, 0x22, 0xdc, 0xa8, 0x38, 0xfb, 0xab, 0x85, - 0x15, 0xff, 0x51, 0x03, 0x78, 0x8e, 0x0d, 0x68, 0xdf, 0x1d, 0x7b, 0xa5, 0xce, 0x37, 0xa0, 0x31, - 0x45, 0xbb, 0xfa, 0x3d, 0xfc, 0xe5, 0x92, 0x99, 0xd0, 0xfc, 0xd6, 0xb2, 0x1c, 0x3b, 0x29, 0xd0, - 0x82, 0xe0, 0xbf, 0xf0, 0x19, 0x0b, 0x5e, 0x9a, 0x27, 0xa2, 0x3c, 0xad, 0x98, 0x09, 0xcd, 0x9b, - 0xcd, 0xa1, 0x63, 0x33, 0x37, 0x42, 0xae, 0xb8, 0xd7, 0x94, 0x15, 0x3a, 0x00, 0x10, 0x81, 0x9c, - 0x8b, 0x87, 0xc0, 0x12, 0x8f, 0x7e, 0x1c, 0x02, 0xfe, 0xcd, 0x71, 0x84, 0x91, 0x35, 0x89, 0xaf, - 0x54, 0x41, 0x60, 0xbd, 0xc1, 0x74, 0x93, 0x95, 0x48, 0x52, 0xf4, 0x04, 0x9a, 0xbc, 0xc3, 0x10, - 0x4e, 0x13, 0x31, 0x8b, 0x5d, 0xa3, 0xa5, 0x59, 0x5d, 0xd6, 0x51, 0xc6, 0xba, 0x6b, 0xa9, 0x6e, - 0xfa, 0x2b, 0x21, 0x4d, 0x78, 0x71, 0xae, 0xb4, 0x7d, 0x58, 0x16, 0x8d, 0xbe, 0xb8, 0x31, 0x56, - 0x8f, 0x36, 0x78, 0x38, 0x53, 0xd7, 0x9b, 0x31, 0x3b, 0x96, 0x27, 0xbc, 0x70, 0x93, 0x3c, 0xf1, - 0x48, 0xc8, 0xc8, 0x4b, 0x5d, 0x67, 0xc6, 0x6c, 0xfa, 0x37, 0x0d, 0x96, 0x85, 0x98, 0x90, 0x3c, - 0x86, 0xba, 0x83, 0x56, 0xa3, 0xa8, 0xd5, 0xa3, 0x6d, 0xcc, 0xa9, 0x9c, 0x2f, 0x3e, 0xad, 0x98, - 0x72, 0x17, 0xdf, 0x2f, 0x60, 0xa1, 0x17, 0x94, 0xfd, 0xaa, 0xb5, 0x7c, 0xbf, 0xd8, 0xc5, 0xf7, - 0x0b, 0xb5, 0xe8, 0x21, 0x65, 0xbf, 0x6a, 0x0d, 0xdf, 0x2f, 0x76, 0x3d, 0x6d, 0x40, 0x5d, 0xe4, - 0x12, 0x7f, 0x64, 0xa0, 0xdc, 0xcc, 0x09, 0x6c, 0x65, 0xe0, 0x36, 0x12, 0x58, 0xad, 0x0c, 0xac, - 0x46, 0xa2, 0xbe, 0x95, 0x51, 0xdf, 0x88, 0xd5, 0xf0, 0xf4, 0xe0, 0xe1, 0x8b, 0xb3, 0x51, 0x10, - 0x94, 0x01, 0x51, 0x55, 0x2e, 0x5c, 0xf6, 0xde, 0x87, 0x65, 0x01, 0x3e, 0xd3, 0x14, 0x49, 0x57, - 0x9b, 0x31, 0x8f, 0xfe, 0x53, 0x4b, 0x6b, 0xf9, 0xf0, 0x9c, 0x4d, 0xad, 0xf9, 0xb5, 0x1c, 0xd9, - 0xe9, 0x83, 0xa6, 0xd0, 0x38, 0xce, 0x7d, 0xd0, 0xf0, 0x23, 0x37, 0xb2, 0x22, 0x6b, 0x60, 0x85, - 0xc9, 0xb5, 0x1b, 0xd3, 0xdc, 0xfa, 0xc8, 0x1a, 0x38, 0x4c, 0xde, 0xba, 0x82, 0xc0, 0xc3, 0x81, - 0xfa, 0xf4, 0xba, 0x3c, 0x1c, 0x48, 0xf1, 0xdd, 0x63, 0x67, 0x16, 0x9e, 0xeb, 0xcb, 0xe2, 0x48, - 0x23, 0xc1, 0xd1, 0xf0, 0x56, 0x52, 0x6f, 0xe0, 0x22, 0x7e, 0xab, 0x37, 0x87, 0xb4, 0xeb, 0x56, - 0x6e, 0x8e, 0x03, 0xd8, 0x3e, 0x66, 0xd1, 0xd9, 0x6c, 0xc0, 0xaf, 0xd6, 0xee, 0x78, 0x72, 0xc3, - 0xc5, 0x41, 0x5f, 0xc2, 0x4e, 0x6e, 0xef, 0xc2, 0x10, 0x09, 0x2c, 0x0d, 0xc7, 0x93, 0xd8, 0xe1, - 0xf8, 0x4d, 0x7b, 0xb0, 0x7e, 0xcc, 0x22, 0x45, 0xf7, 0x43, 0xe5, 0xaa, 0x90, 0x8d, 0x5d, 0x77, - 0x3c, 0x79, 0x71, 0xed, 0xb3, 0x1b, 0xee, 0x8d, 0x13, 0xd8, 0x88, 0xa5, 0x2c, 0x8c, 0xaa, 0x09, - 0xb5, 0xe1, 0x38, 0x69, 0x09, 0x87, 0xe3, 0x09, 0xdd, 0x81, 0xad, 0x63, 0x26, 0xcf, 0x65, 0x8a, - 0x8c, 0xee, 0xa3, 0xb7, 0x94, 0x65, 0xa9, 0x4a, 0x0a, 0xd0, 0x52, 0x01, 0x7f, 0xd1, 0x80, 0x7c, - 0x6a, 0xb9, 0x23, 0x87, 0x3d, 0x0b, 0x02, 0x2f, 0x98, 0xdb, 0x07, 0x23, 0xf7, 0x9d, 0x92, 0x74, - 0x0f, 0x56, 0x06, 0xb6, 0xeb, 0x78, 0x93, 0xcf, 0xbc, 0x50, 0x66, 0x69, 0xba, 0x80, 0x29, 0xf6, - 0xca, 0x49, 0xde, 0x3a, 0xfc, 0x9b, 0x86, 0xb0, 0x95, 0x81, 0x74, 0x2b, 0x09, 0x76, 0x0c, 0x3b, - 0x2f, 0x02, 0xcb, 0x0d, 0xc7, 0x2c, 0xc8, 0x36, 0x5f, 0xe9, 0x7d, 0xa2, 0xa9, 0xf7, 0x89, 0x52, - 0x76, 0x84, 0x66, 0x49, 0xf1, 0xe6, 0x24, 0x2f, 0x68, 0xe1, 0x0b, 0x7a, 0x94, 0x0c, 0x2a, 0x32, - 0x0d, 0xfb, 0x03, 0x25, 0x2a, 0xeb, 0xca, 0x3b, 0xe2, 0xf3, 0xa3, 0xb8, 0x11, 0x94, 0x48, 0xab, - 0x73, 0x90, 0x8a, 0xd0, 0xc4, 0x48, 0x7f, 0x91, 0x94, 0xa8, 0x77, 0xec, 0xbe, 0xe9, 0x31, 0xec, - 0xf6, 0xa7, 0xbe, 0x17, 0x44, 0xcf, 0xbe, 0xe6, 0x7f, 0xbb, 0xe3, 0x49, 0x38, 0x77, 0x6c, 0xc3, - 0x99, 0x32, 0x81, 0x9a, 0x50, 0x1b, 0xd9, 0xb1, 0xdf, 0xf8, 0x27, 0xfd, 0x19, 0xd4, 0xbb, 0xa2, - 0x6b, 0x9d, 0xd3, 0x09, 0x0e, 0x3d, 0x37, 0x62, 0xb2, 0x13, 0x5f, 0x31, 0x63, 0x92, 0xfe, 0x41, - 0x03, 0xbd, 0x88, 0x60, 0xe1, 0x84, 0x79, 0x94, 0x4f, 0x18, 0x81, 0x18, 0x11, 0xa5, 0x99, 0xdc, - 0xe1, 0x25, 0x35, 0xbc, 0x88, 0x5f, 0x5f, 0xea, 0x1e, 0xc1, 0x38, 0x18, 0x40, 0x23, 0x6e, 0xcd, - 0xc9, 0x16, 0xdc, 0xed, 0xbb, 0x97, 0x96, 0x63, 0x8f, 0xe2, 0xa5, 0x66, 0x85, 0xdc, 0x85, 0x55, - 0x9c, 0xaa, 0x89, 0xa5, 0xa6, 0x46, 0x9a, 0xb0, 0x26, 0xc6, 0x37, 0x72, 0xa5, 0x4a, 0x36, 0x00, - 0xce, 0x22, 0xcf, 0x97, 0x74, 0x0d, 0xe9, 0x73, 0xef, 0x4a, 0xd2, 0x4b, 0x07, 0xbf, 0x84, 0x46, - 0xdc, 0x0f, 0x2a, 0x3a, 0xe2, 0xa5, 0x66, 0x85, 0x6c, 0xc2, 0xfa, 0xb3, 0x4b, 0x7b, 0x18, 0x25, - 0x4b, 0x1a, 0xd9, 0x85, 0xad, 0xae, 0xe5, 0x0e, 0x99, 0x93, 0x65, 0x54, 0x0f, 0xbe, 0x80, 0x65, - 0x59, 0xb2, 0x38, 0x34, 0x29, 0x8b, 0x93, 0xcd, 0x0a, 0x59, 0x83, 0x06, 0x2f, 0xa0, 0x48, 0x69, - 0x1c, 0x86, 0xa8, 0x27, 0x48, 0x23, 0x4c, 0x71, 0x94, 0x90, 0x16, 0x30, 0x11, 0x22, 0xd2, 0x4b, - 0x07, 0x3d, 0x58, 0x49, 0xb2, 0x93, 0x6c, 0x43, 0x53, 0xca, 0x4e, 0xd6, 0x9a, 0x15, 0x6e, 0x3b, - 0x3a, 0x03, 0xd7, 0x3e, 0x3f, 0x6a, 0x6a, 0xc2, 0x3d, 0x9e, 0x1f, 0x2f, 0x54, 0x0f, 0x3a, 0x50, - 0x17, 0x79, 0x43, 0x00, 0xea, 0x22, 0xc4, 0xcd, 0x0a, 0xff, 0x16, 0x41, 0x6f, 0x6a, 0x47, 0xdf, - 0xde, 0x85, 0xba, 0x00, 0x46, 0xbe, 0x84, 0x95, 0x64, 0x64, 0x49, 0xb0, 0x09, 0xc9, 0xcf, 0x48, - 0x8d, 0x9d, 0xdc, 0xaa, 0xc8, 0x14, 0xfa, 0xf0, 0xf7, 0xff, 0xf8, 0xcf, 0xb7, 0xd5, 0x7b, 0x74, - 0xfb, 0xd0, 0xf2, 0xed, 0xf0, 0xf0, 0xf2, 0x89, 0xe5, 0xf8, 0xe7, 0xd6, 0x93, 0x43, 0x8c, 0xea, - 0x87, 0xda, 0x01, 0x19, 0xc3, 0xaa, 0x32, 0x39, 0x24, 0x2d, 0x2e, 0xa6, 0x38, 0xab, 0x34, 0x76, - 0x0b, 0xeb, 0x52, 0xc1, 0x07, 0xa8, 0xa0, 0x63, 0xdc, 0x2f, 0x53, 0x70, 0xf8, 0x9a, 0x67, 0xf9, - 0x37, 0x5c, 0xcf, 0x47, 0x00, 0xe9, 0x34, 0x8f, 0x20, 0xda, 0xc2, 0x80, 0xd0, 0x68, 0xe5, 0x97, - 0xa5, 0x92, 0x0a, 0x71, 0x60, 0x55, 0x19, 0x7c, 0x11, 0x23, 0x37, 0x09, 0x53, 0x26, 0x75, 0xc6, - 0xfd, 0x52, 0x9e, 0x94, 0xf4, 0x08, 0xe1, 0xb6, 0xc9, 0x5e, 0x0e, 0x6e, 0x88, 0x5b, 0x25, 0x5e, - 0xd2, 0x85, 0x35, 0x75, 0xbe, 0x44, 0xd0, 0xfa, 0x92, 0xc1, 0x9a, 0xa1, 0x17, 0x19, 0x09, 0xe4, - 0x4f, 0x60, 0x3d, 0x33, 0xd1, 0x21, 0xb8, 0xb9, 0x6c, 0xa4, 0x64, 0xdc, 0x2b, 0xe1, 0x24, 0x72, - 0xbe, 0x84, 0x56, 0x71, 0x02, 0x83, 0x5e, 0x7c, 0xa0, 0x04, 0xa5, 0x38, 0x05, 0x31, 0xda, 0xf3, - 0xd8, 0x89, 0xe8, 0x53, 0x68, 0xe6, 0x27, 0x15, 0x04, 0xdd, 0x37, 0x67, 0xb0, 0x62, 0xec, 0x95, - 0x33, 0x13, 0x81, 0x1f, 0xc2, 0x4a, 0x32, 0x26, 0x10, 0x89, 0x9a, 0x9f, 0x47, 0x88, 0x44, 0x2d, - 0xcc, 0x12, 0x68, 0x85, 0x4c, 0x60, 0x3d, 0xf3, 0x72, 0x17, 0xfe, 0x2a, 0x1b, 0x1b, 0x08, 0x7f, - 0x95, 0x3e, 0xf3, 0xe9, 0x7b, 0x18, 0xe0, 0xfb, 0x46, 0x2b, 0x1f, 0x60, 0x51, 0xea, 0x78, 0x2a, - 0xf6, 0x61, 0x23, 0xfb, 0xc8, 0x26, 0xf7, 0xc4, 0x95, 0x53, 0xf2, 0x7e, 0x37, 0x8c, 0x32, 0x56, - 0x82, 0x39, 0x80, 0xf5, 0xcc, 0x5b, 0x59, 0x62, 0x2e, 0x79, 0x7e, 0x4b, 0xcc, 0x65, 0x0f, 0x6b, - 0xfa, 0x63, 0xc4, 0xfc, 0xc1, 0xc1, 0xa3, 0x1c, 0x66, 0xd9, 0x72, 0x1f, 0xbe, 0xe6, 0x3d, 0xd7, - 0x37, 0x71, 0x72, 0x5e, 0x24, 0x7e, 0x12, 0xe5, 0x2e, 0xe3, 0xa7, 0xcc, 0x7b, 0x3b, 0xe3, 0xa7, - 0xec, 0x9b, 0x9a, 0xbe, 0x8f, 0x3a, 0x1f, 0x1a, 0x46, 0x4e, 0xa7, 0x78, 0x92, 0x1c, 0xbe, 0xf6, - 0x7c, 0x3c, 0xb6, 0xbf, 0x05, 0x48, 0x1f, 0x15, 0xe2, 0xd8, 0x16, 0xde, 0x35, 0xe2, 0xd8, 0x16, - 0xdf, 0x1e, 0xb4, 0x8d, 0x3a, 0x74, 0xd2, 0x2a, 0xb7, 0x8b, 0x8c, 0xd3, 0x88, 0x8b, 0x66, 0x3d, - 0x13, 0x71, 0xf5, 0x71, 0x91, 0x8d, 0x78, 0xa6, 0x3d, 0xa7, 0x1d, 0xd4, 0x62, 0x18, 0x3b, 0xf9, - 0x88, 0xe3, 0x36, 0x6e, 0x84, 0x83, 0xfd, 0x6d, 0xda, 0x36, 0x0b, 0x3d, 0x65, 0x5d, 0xb7, 0xd0, - 0x53, 0xda, 0x63, 0xc7, 0x95, 0x8e, 0xb4, 0xf3, 0x7a, 0x66, 0x03, 0xb5, 0xd8, 0x91, 0x17, 0x50, - 0x17, 0x7d, 0x30, 0xd9, 0x94, 0xc2, 0x14, 0xf9, 0x44, 0x5d, 0x92, 0x82, 0x7f, 0x84, 0x82, 0x1f, - 0x90, 0x9b, 0x4a, 0x28, 0xf9, 0x0a, 0x56, 0x95, 0xd6, 0x51, 0xd4, 0xe9, 0x62, 0x7b, 0x2b, 0xea, - 0x74, 0x49, 0x8f, 0x39, 0xd7, 0x4b, 0x8c, 0xef, 0xc2, 0x63, 0xd1, 0x85, 0x35, 0xb5, 0xb5, 0x16, - 0x45, 0xaf, 0xa4, 0x07, 0x37, 0xf4, 0x22, 0x23, 0x39, 0x10, 0x7d, 0xd8, 0xc8, 0xf6, 0x88, 0xe2, - 0x6c, 0x95, 0x36, 0xa0, 0xe2, 0x6c, 0x95, 0xb7, 0x94, 0xb4, 0xc2, 0xf1, 0xa8, 0x4d, 0x1c, 0x51, - 0xaf, 0xa0, 0x4c, 0x51, 0xd2, 0x8b, 0x0c, 0xb5, 0xc2, 0xe5, 0xbb, 0x28, 0x51, 0xe1, 0xe6, 0x74, - 0x77, 0xa2, 0xc2, 0xcd, 0x6b, 0xbc, 0x68, 0xe5, 0xa9, 0xfe, 0xf7, 0x37, 0x6d, 0xed, 0xfb, 0x37, - 0x6d, 0xed, 0xdf, 0x6f, 0xda, 0xda, 0x9f, 0xdf, 0xb6, 0x2b, 0xdf, 0xbf, 0x6d, 0x57, 0xfe, 0xf5, - 0xb6, 0x5d, 0x19, 0xd4, 0xf1, 0x1f, 0x98, 0x3f, 0xf9, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x08, - 0x8c, 0xd5, 0xa0, 0x04, 0x1d, 0x00, 0x00, + // 2040 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x59, 0x5f, 0x6f, 0x1b, 0xc7, + 0x11, 0xe7, 0x91, 0x34, 0x45, 0x0d, 0x25, 0x86, 0x5e, 0x51, 0xd4, 0xf9, 0x2c, 0xd3, 0xca, 0x36, + 0x09, 0x04, 0xa1, 0xb0, 0x60, 0xb5, 0x4f, 0x01, 0x52, 0x34, 0x26, 0x1d, 0x47, 0xa8, 0x5c, 0xa5, + 0x27, 0x3b, 0x4d, 0x50, 0xa0, 0xc8, 0x91, 0x5c, 0x52, 0x07, 0x1d, 0xef, 0xce, 0x77, 0x47, 0xa9, + 0x82, 0x91, 0x97, 0x7e, 0x80, 0xfe, 0x41, 0x1f, 0xf2, 0xd8, 0x87, 0x7e, 0x93, 0x3e, 0xf5, 0x31, + 0x40, 0x81, 0xa2, 0x8f, 0x85, 0xdd, 0x0f, 0x52, 0xec, 0xec, 0xde, 0xdd, 0xde, 0x1f, 0xaa, 0xa5, + 0x81, 0xea, 0xed, 0x66, 0x66, 0x39, 0xf3, 0x9b, 0x3f, 0x3b, 0x3b, 0xbb, 0x84, 0xf6, 0x64, 0x3e, + 0xb7, 0xc2, 0x88, 0x05, 0x8f, 0xfc, 0xc0, 0x8b, 0x3c, 0x52, 0xf5, 0x47, 0x46, 0x7b, 0x32, 0xbf, + 0xf2, 0x82, 0x8b, 0x98, 0x67, 0xec, 0xce, 0x3c, 0x6f, 0xe6, 0xb0, 0x43, 0xcb, 0xb7, 0x0f, 0x2d, + 0xd7, 0xf5, 0x22, 0x2b, 0xb2, 0x3d, 0x37, 0x14, 0x52, 0xfa, 0x0d, 0x74, 0xce, 0x22, 0x2b, 0x88, + 0x5e, 0x58, 0xe1, 0x85, 0xc9, 0x5e, 0x2d, 0x58, 0x18, 0x11, 0x02, 0xf5, 0xc8, 0x0a, 0x2f, 0x74, + 0x6d, 0x4f, 0xdb, 0x5f, 0x37, 0xf1, 0x9b, 0xe8, 0xb0, 0x16, 0x7a, 0x8b, 0x60, 0xcc, 0x42, 0xbd, + 0xba, 0x57, 0xdb, 0x5f, 0x37, 0x63, 0x92, 0xf4, 0x01, 0x02, 0x36, 0xf7, 0x2e, 0xd9, 0x73, 0x16, + 0x59, 0x7a, 0x6d, 0x4f, 0xdb, 0x6f, 0x9a, 0x0a, 0x87, 0xbe, 0x82, 0xbb, 0x8a, 0x85, 0xd0, 0xf7, + 0xdc, 0x90, 0x91, 0x1e, 0x34, 0x02, 0x16, 0x2e, 0x9c, 0x08, 0x8d, 0x34, 0x4d, 0x49, 0x91, 0x0e, + 0xd4, 0xe6, 0xe1, 0x4c, 0xaf, 0xa2, 0x65, 0xfe, 0x49, 0x8e, 0x52, 0xc3, 0xb5, 0xbd, 0xda, 0x7e, + 0xeb, 0x48, 0x7f, 0xe4, 0x8f, 0x1e, 0x0d, 0xbc, 0xf9, 0xdc, 0x73, 0x7f, 0x89, 0x7e, 0xc6, 0x4a, + 0x13, 0x48, 0xf4, 0xd7, 0x40, 0x4e, 0x7d, 0x16, 0x58, 0x11, 0x53, 0xdd, 0x32, 0xa0, 0xea, 0xf9, + 0x68, 0xaf, 0x7d, 0x04, 0x5c, 0x09, 0x17, 0x9e, 0xfa, 0x66, 0xd5, 0xf3, 0xb9, 0xcb, 0xae, 0x35, + 0x67, 0xd2, 0x30, 0x7e, 0xab, 0x2e, 0xd7, 0x32, 0x2e, 0xd3, 0xdf, 0x6b, 0xb0, 0x95, 0x31, 0x20, + 0xbd, 0xba, 0xc9, 0x42, 0xea, 0x71, 0xb5, 0xcc, 0xe3, 0x5a, 0xa9, 0xc7, 0xf5, 0xff, 0xd5, 0xe3, + 0x4f, 0xe1, 0xee, 0x4b, 0x7f, 0x92, 0x73, 0x78, 0xa5, 0x3c, 0xd2, 0x00, 0x88, 0xaa, 0xe2, 0x56, + 0x12, 0xf5, 0x19, 0xf4, 0x7e, 0xb1, 0x60, 0xc1, 0xf5, 0x59, 0x64, 0x45, 0x8b, 0xf0, 0xc4, 0x0e, + 0x23, 0x05, 0x3b, 0x26, 0x44, 0x2b, 0x4f, 0x48, 0x0e, 0xfb, 0x25, 0xec, 0x14, 0xf4, 0xac, 0xec, + 0xc0, 0xe3, 0xbc, 0x03, 0x3b, 0xdc, 0x01, 0x45, 0x6f, 0x11, 0xff, 0x00, 0xb6, 0xce, 0xce, 0xbd, + 0xab, 0xe1, 0xf0, 0xe4, 0xc4, 0x1b, 0x5f, 0x84, 0xef, 0x16, 0xf8, 0x3f, 0x6b, 0xb0, 0x26, 0x35, + 0x90, 0x36, 0x54, 0x8f, 0x87, 0xf2, 0x77, 0xd5, 0xe3, 0x61, 0xa2, 0xa9, 0xaa, 0x68, 0x22, 0x50, + 0x9f, 0x7b, 0x13, 0x26, 0x4b, 0x06, 0xbf, 0x49, 0x17, 0xee, 0x78, 0x57, 0x2e, 0x0b, 0xf4, 0x3a, + 0x32, 0x05, 0xc1, 0x57, 0x0e, 0x87, 0x27, 0xa1, 0x7e, 0x07, 0x0d, 0xe2, 0x37, 0x8f, 0x47, 0x78, + 0xed, 0x8e, 0xd9, 0x44, 0x6f, 0x20, 0x57, 0x52, 0xc4, 0x80, 0xe6, 0xc2, 0x95, 0x92, 0x35, 0x94, + 0x24, 0x34, 0x1d, 0x43, 0x37, 0xeb, 0xe6, 0xca, 0xb1, 0x7d, 0x1f, 0xee, 0x38, 0xfc, 0xa7, 0x32, + 0xb2, 0x2d, 0x1e, 0x59, 0xa9, 0xce, 0x14, 0x12, 0xea, 0x40, 0xf7, 0xa5, 0xcb, 0x3f, 0x63, 0xbe, + 0x0c, 0x66, 0x3e, 0x24, 0x14, 0x36, 0x02, 0xe6, 0x3b, 0xd6, 0x98, 0x9d, 0xa2, 0xc7, 0xc2, 0x4a, + 0x86, 0x47, 0xf6, 0xa0, 0x35, 0xf5, 0x82, 0x31, 0x33, 0xb1, 0x0d, 0xc9, 0xa6, 0xa4, 0xb2, 0xe8, + 0xa7, 0xb0, 0x9d, 0xb3, 0xb6, 0xaa, 0x4f, 0xd4, 0x84, 0x7b, 0xb2, 0x09, 0xc4, 0xe5, 0xed, 0x58, + 0xd7, 0x31, 0xea, 0xfb, 0x4a, 0x2b, 0x40, 0x6f, 0x51, 0x2a, 0x7b, 0xc1, 0xf2, 0x5a, 0xf8, 0x4e, + 0x03, 0xa3, 0x4c, 0xa9, 0x04, 0x77, 0xa3, 0xd6, 0xff, 0x6f, 0x87, 0xf9, 0x4e, 0x83, 0x9d, 0x2f, + 0x16, 0xc1, 0xac, 0xcc, 0x59, 0xc5, 0x1f, 0x2d, 0x7b, 0x38, 0x18, 0xd0, 0xb4, 0x5d, 0x6b, 0x1c, + 0xd9, 0x97, 0x4c, 0xa2, 0x4a, 0x68, 0xac, 0x6d, 0x7b, 0x2e, 0xb2, 0x53, 0x33, 0xf1, 0x9b, 0xaf, + 0x9f, 0xda, 0x0e, 0xc3, 0xad, 0x2f, 0x4a, 0x39, 0xa1, 0xb1, 0x72, 0x17, 0xa3, 0xa1, 0x1d, 0xe8, + 0x77, 0x50, 0x22, 0x29, 0xfa, 0x1b, 0xd0, 0x8b, 0xc0, 0x6e, 0xa5, 0x7d, 0x7d, 0x05, 0x9d, 0xc1, + 0x39, 0x1b, 0x5f, 0xfc, 0xb7, 0xa6, 0xdb, 0x83, 0x06, 0x0b, 0x82, 0x81, 0x2b, 0x32, 0x53, 0x33, + 0x25, 0xc5, 0xe3, 0x76, 0x65, 0x05, 0x2e, 0x17, 0x88, 0x20, 0xc4, 0x24, 0xfd, 0x04, 0xee, 0x2a, + 0x9a, 0x57, 0x2e, 0xcd, 0x73, 0xe8, 0xca, 0x2a, 0x3a, 0x43, 0xa8, 0x31, 0xb8, 0x5d, 0xa5, 0x7e, + 0x36, 0xb8, 0x7f, 0x42, 0x9c, 0x16, 0xd0, 0xd8, 0x73, 0xa7, 0xf6, 0x4c, 0x56, 0xa5, 0xa4, 0x78, + 0x52, 0x84, 0xc7, 0xc7, 0x43, 0x79, 0x12, 0x26, 0x34, 0x5d, 0xc0, 0x76, 0xce, 0xd2, 0xad, 0x44, + 0xfe, 0x29, 0x6c, 0x9b, 0x6c, 0x66, 0xf3, 0xd1, 0x27, 0x5e, 0x72, 0xe3, 0xb9, 0x61, 0x4d, 0x26, + 0x01, 0x0b, 0x43, 0x69, 0x36, 0x26, 0xe9, 0x13, 0xe8, 0xe5, 0xd5, 0xac, 0x1c, 0xeb, 0x9f, 0x40, + 0xf7, 0x74, 0x3a, 0x75, 0x6c, 0x97, 0x3d, 0x67, 0xf3, 0x51, 0x06, 0x49, 0x74, 0xed, 0x27, 0x48, + 0xf8, 0x77, 0xd9, 0x98, 0xc1, 0x3b, 0x51, 0xee, 0xf7, 0x2b, 0x43, 0xf8, 0x71, 0x92, 0xee, 0x13, + 0x66, 0x4d, 0x52, 0x08, 0x85, 0x74, 0x0b, 0xb1, 0x48, 0x37, 0x1a, 0xce, 0xfe, 0x6a, 0x65, 0xc3, + 0xbf, 0xd3, 0x00, 0x9e, 0xe3, 0x00, 0x7a, 0xec, 0x4e, 0xbd, 0xd2, 0xe0, 0x1b, 0xd0, 0x9c, 0xa3, + 0x5f, 0xc7, 0x43, 0xfc, 0x65, 0xdd, 0x4c, 0x68, 0x7e, 0x6a, 0x59, 0x8e, 0x9d, 0x34, 0x68, 0x41, + 0xf0, 0x5f, 0xf8, 0x8c, 0x05, 0x2f, 0xcd, 0x13, 0xd1, 0x9e, 0xd6, 0xcd, 0x84, 0xe6, 0xc3, 0xe6, + 0xd8, 0xb1, 0x99, 0x1b, 0xa1, 0x54, 0x9c, 0x6b, 0x0a, 0x87, 0x8e, 0x00, 0x44, 0x22, 0x97, 0xe2, + 0x21, 0x50, 0xe7, 0xd9, 0x8f, 0x53, 0xc0, 0xbf, 0x39, 0x8e, 0x30, 0xb2, 0x66, 0xf1, 0x91, 0x2a, + 0x08, 0xec, 0x37, 0x58, 0x6e, 0xb2, 0x13, 0x49, 0x8a, 0x9e, 0x40, 0x87, 0x4f, 0x18, 0x22, 0x68, + 0x22, 0x67, 0x71, 0x68, 0xb4, 0xb4, 0xaa, 0xcb, 0x26, 0xca, 0xd8, 0x76, 0x2d, 0xb5, 0x4d, 0x7f, + 0x2e, 0xb4, 0x89, 0x28, 0x2e, 0xd5, 0xb6, 0x0f, 0x6b, 0x62, 0xd0, 0x17, 0x27, 0x46, 0xeb, 0xa8, + 0xcd, 0xd3, 0x99, 0x86, 0xde, 0x8c, 0xc5, 0xb1, 0x3e, 0x11, 0x85, 0x9b, 0xf4, 0x89, 0x4b, 0x42, + 0x46, 0x5f, 0x1a, 0x3a, 0x33, 0x16, 0xd3, 0xbf, 0x68, 0xb0, 0x26, 0xd4, 0x84, 0xe4, 0x11, 0x34, + 0x1c, 0xf4, 0x1a, 0x55, 0xb5, 0x8e, 0xba, 0x58, 0x53, 0xb9, 0x58, 0x7c, 0x5e, 0x31, 0xe5, 0x2a, + 0xbe, 0x5e, 0xc0, 0xc2, 0x28, 0x28, 0xeb, 0x55, 0x6f, 0xf9, 0x7a, 0xb1, 0x8a, 0xaf, 0x17, 0x66, + 0x31, 0x42, 0xca, 0x7a, 0xd5, 0x1b, 0xbe, 0x5e, 0xac, 0x7a, 0xd2, 0x84, 0x86, 0xa8, 0x25, 0x7e, + 0xc9, 0x40, 0xbd, 0x99, 0x1d, 0xd8, 0xcb, 0xc0, 0x6d, 0x26, 0xb0, 0x7a, 0x19, 0x58, 0xcd, 0xc4, + 0x7c, 0x2f, 0x63, 0xbe, 0x19, 0x9b, 0xe1, 0xe5, 0xc1, 0xd3, 0x17, 0x57, 0xa3, 0x20, 0x28, 0x03, + 0xa2, 0x9a, 0x5c, 0xb9, 0xed, 0x7d, 0x08, 0x6b, 0x02, 0x7c, 0x66, 0x28, 0x92, 0xa1, 0x36, 0x63, + 0x19, 0xfd, 0x87, 0x96, 0xf6, 0xf2, 0xf1, 0x39, 0x9b, 0x5b, 0xcb, 0x7b, 0x39, 0x8a, 0xd3, 0x0b, + 0x4d, 0x61, 0x70, 0x5c, 0x7a, 0xa1, 0xe1, 0x5b, 0x6e, 0x62, 0x45, 0xd6, 0xc8, 0x0a, 0x93, 0x63, + 0x37, 0xa6, 0xb9, 0xf7, 0x91, 0x35, 0x72, 0x98, 0x3c, 0x75, 0x05, 0x81, 0x9b, 0x03, 0xed, 0xe9, + 0x0d, 0xb9, 0x39, 0x90, 0xe2, 0xab, 0xa7, 0xce, 0x22, 0x3c, 0xd7, 0xd7, 0xc4, 0x96, 0x46, 0x82, + 0xa3, 0xe1, 0xa3, 0xa4, 0xde, 0x44, 0x26, 0x7e, 0xab, 0x27, 0x87, 0xf4, 0xeb, 0x56, 0x4e, 0x8e, + 0x03, 0xe8, 0x3e, 0x63, 0xd1, 0xd9, 0x62, 0xc4, 0x8f, 0xd6, 0xc1, 0x74, 0x76, 0xc3, 0xc1, 0x41, + 0x5f, 0xc2, 0x76, 0x6e, 0xed, 0xca, 0x10, 0x09, 0xd4, 0xc7, 0xd3, 0x59, 0x1c, 0x70, 0xfc, 0xa6, + 0x43, 0xd8, 0x7c, 0xc6, 0x22, 0xc5, 0xf6, 0x43, 0xe5, 0xa8, 0x90, 0x83, 0xdd, 0x60, 0x3a, 0x7b, + 0x71, 0xed, 0xb3, 0x1b, 0xce, 0x8d, 0x13, 0x68, 0xc7, 0x5a, 0x56, 0x46, 0xd5, 0x81, 0xda, 0x78, + 0x9a, 0x8c, 0x84, 0xe3, 0xe9, 0x8c, 0x6e, 0xc3, 0xd6, 0x33, 0x26, 0xf7, 0x65, 0x8a, 0x8c, 0xee, + 0x63, 0xb4, 0x14, 0xb6, 0x34, 0x25, 0x15, 0x68, 0xa9, 0x82, 0x3f, 0x6a, 0x40, 0x3e, 0xb7, 0xdc, + 0x89, 0xc3, 0x9e, 0x06, 0x81, 0x17, 0x2c, 0x9d, 0x83, 0x51, 0xfa, 0x4e, 0x45, 0xba, 0x0b, 0xeb, + 0x23, 0xdb, 0x75, 0xbc, 0xd9, 0x17, 0x5e, 0x28, 0xab, 0x34, 0x65, 0x60, 0x89, 0xbd, 0x72, 0x92, + 0xbb, 0x0e, 0xff, 0xa6, 0x21, 0x6c, 0x65, 0x20, 0xdd, 0x4a, 0x81, 0x3d, 0x83, 0xed, 0x17, 0x81, + 0xe5, 0x86, 0x53, 0x16, 0x64, 0x87, 0xaf, 0xf4, 0x3c, 0xd1, 0xd4, 0xf3, 0x44, 0x69, 0x3b, 0xc2, + 0xb2, 0xa4, 0xf8, 0x70, 0x92, 0x57, 0xb4, 0xf2, 0x01, 0x3d, 0x49, 0x1e, 0x2a, 0x32, 0x03, 0xfb, + 0x03, 0x25, 0x2b, 0x9b, 0xca, 0x3d, 0xe2, 0xcb, 0xa3, 0x78, 0x10, 0x94, 0x48, 0xab, 0x4b, 0x90, + 0x8a, 0xd4, 0xc4, 0x48, 0x7f, 0x9a, 0xb4, 0xa8, 0x77, 0x9c, 0xbe, 0x0f, 0x46, 0xd0, 0x8c, 0x47, + 0x51, 0xb2, 0x05, 0xef, 0x1d, 0xbb, 0x97, 0x96, 0x63, 0x4f, 0x62, 0x56, 0xa7, 0x42, 0xde, 0x83, + 0x16, 0xbe, 0x22, 0x09, 0x56, 0x47, 0x23, 0x1d, 0xd8, 0x10, 0xcf, 0x15, 0x92, 0x53, 0x25, 0x6d, + 0x80, 0xb3, 0xc8, 0xf3, 0x25, 0x5d, 0x43, 0xfa, 0xdc, 0xbb, 0x92, 0x74, 0xfd, 0xe0, 0x67, 0xd0, + 0x8c, 0xe7, 0x1f, 0xc5, 0x46, 0xcc, 0xea, 0x54, 0xc8, 0x5d, 0xd8, 0x7c, 0x7a, 0x69, 0x8f, 0xa3, + 0x84, 0xa5, 0x91, 0x1d, 0xd8, 0x1a, 0x58, 0xee, 0x98, 0x39, 0x59, 0x41, 0xf5, 0xe0, 0x2b, 0x58, + 0x93, 0x5b, 0x94, 0x43, 0x93, 0xba, 0x38, 0xd9, 0xa9, 0x90, 0x0d, 0x68, 0xf2, 0x86, 0x81, 0x94, + 0xc6, 0x61, 0x88, 0xfd, 0x83, 0x34, 0xc2, 0x14, 0xa5, 0x83, 0xb4, 0x80, 0x89, 0x10, 0x91, 0xae, + 0x1f, 0x0c, 0x61, 0x3d, 0xc9, 0x06, 0xe9, 0x42, 0x47, 0xea, 0x4e, 0x78, 0x9d, 0x0a, 0xf7, 0x1d, + 0x83, 0x81, 0xbc, 0x2f, 0x8f, 0x3a, 0x9a, 0x08, 0x8f, 0xe7, 0xc7, 0x8c, 0xea, 0xd1, 0x5f, 0xdb, + 0xd0, 0x10, 0x66, 0xc9, 0xd7, 0xb0, 0x9e, 0x3c, 0xc0, 0x11, 0x3c, 0x52, 0xf3, 0x2f, 0x7e, 0xc6, + 0x76, 0x8e, 0x2b, 0xf2, 0x47, 0x1f, 0xfe, 0xf6, 0xef, 0xff, 0xfe, 0x53, 0xf5, 0x1e, 0xed, 0x1e, + 0x5a, 0xbe, 0x1d, 0x1e, 0x5e, 0x3e, 0xb6, 0x1c, 0xff, 0xdc, 0x7a, 0x7c, 0xc8, 0x37, 0x6a, 0xf8, + 0xb1, 0x76, 0x40, 0xa6, 0xd0, 0x52, 0xde, 0xc1, 0x48, 0x8f, 0xab, 0x29, 0xbe, 0xbc, 0x19, 0x3b, + 0x05, 0xbe, 0x34, 0xf0, 0x11, 0x1a, 0xd8, 0x33, 0xee, 0x97, 0x19, 0x38, 0x7c, 0xcd, 0xfb, 0xdc, + 0xb7, 0xdc, 0xce, 0x27, 0x00, 0xe9, 0xdb, 0x14, 0x41, 0xb4, 0x85, 0xe7, 0x2e, 0xa3, 0x97, 0x67, + 0x4b, 0x23, 0x15, 0xe2, 0x40, 0x4b, 0x79, 0xc6, 0x21, 0x46, 0xee, 0x5d, 0x47, 0x79, 0x77, 0x32, + 0xee, 0x97, 0xca, 0xa4, 0xa6, 0x0f, 0x10, 0x6e, 0x9f, 0xec, 0xe6, 0xe0, 0x86, 0xb8, 0x54, 0xe2, + 0x25, 0x03, 0xd8, 0x50, 0x5f, 0x4b, 0x08, 0x7a, 0x5f, 0xf2, 0x4c, 0x64, 0xe8, 0x45, 0x41, 0x02, + 0xf9, 0x33, 0xd8, 0xcc, 0xbc, 0x4f, 0x10, 0x5c, 0x5c, 0xf6, 0x40, 0x62, 0xdc, 0x2b, 0x91, 0x24, + 0x7a, 0xbe, 0x86, 0x5e, 0xf1, 0x3d, 0x01, 0xa3, 0xf8, 0x40, 0x49, 0x4a, 0xf1, 0x4e, 0x6f, 0xf4, + 0x97, 0x89, 0x13, 0xd5, 0xa7, 0xd0, 0xc9, 0xdf, 0xbb, 0x09, 0x86, 0x6f, 0xc9, 0x33, 0x81, 0xb1, + 0x5b, 0x2e, 0x4c, 0x14, 0x7e, 0x0c, 0xeb, 0xc9, 0xa5, 0x57, 0x14, 0x6a, 0xfe, 0x76, 0x2d, 0x0a, + 0xb5, 0x70, 0x33, 0xa6, 0x15, 0x32, 0x83, 0xcd, 0xcc, 0x3d, 0x54, 0xc4, 0xab, 0xec, 0x12, 0x2c, + 0xe2, 0x55, 0x7a, 0x69, 0xa5, 0xef, 0x63, 0x82, 0xef, 0x1b, 0xbd, 0x7c, 0x82, 0x45, 0x6f, 0xe7, + 0xa5, 0x78, 0x0c, 0xed, 0xec, 0x95, 0x91, 0xdc, 0x13, 0x0d, 0xb4, 0xe4, 0x36, 0x6a, 0x18, 0x65, + 0xa2, 0x04, 0x73, 0x00, 0x9b, 0x99, 0x9b, 0x9f, 0xc4, 0x5c, 0x72, 0x99, 0x94, 0x98, 0xcb, 0xae, + 0x89, 0xf4, 0x87, 0x88, 0xf9, 0xa3, 0x83, 0x0f, 0x72, 0x98, 0xe5, 0x00, 0x79, 0xf8, 0x9a, 0x4f, + 0x10, 0xdf, 0xc6, 0xc5, 0x79, 0x91, 0xc4, 0x49, 0x34, 0xb3, 0x4c, 0x9c, 0x32, 0xb7, 0xc7, 0x4c, + 0x9c, 0xb2, 0x37, 0x44, 0xfa, 0x21, 0xda, 0x7c, 0x68, 0x18, 0x39, 0x9b, 0x62, 0xc0, 0x3e, 0x7c, + 0xed, 0xf9, 0xb8, 0x6d, 0x7f, 0x05, 0x90, 0x8e, 0xc8, 0x62, 0xdb, 0x16, 0xa6, 0x74, 0xb1, 0x6d, + 0x8b, 0x93, 0x34, 0xed, 0xa3, 0x0d, 0x9d, 0xf4, 0xca, 0xfd, 0x22, 0xd3, 0x34, 0xe3, 0x62, 0xf4, + 0xcc, 0x64, 0x5c, 0x1d, 0x95, 0xb3, 0x19, 0xcf, 0x0c, 0x9b, 0x74, 0x0f, 0xad, 0x18, 0xc6, 0x76, + 0x3e, 0xe3, 0xb8, 0x8c, 0x3b, 0xe1, 0xe0, 0xb4, 0x96, 0x0e, 0x81, 0xc2, 0x4e, 0xd9, 0x0c, 0x29, + 0xec, 0x94, 0x4e, 0x8c, 0x71, 0xa7, 0x23, 0xfd, 0xbc, 0x9d, 0xc5, 0x48, 0x6d, 0x76, 0xe4, 0x05, + 0x34, 0xc4, 0x54, 0x47, 0xee, 0x4a, 0x65, 0x8a, 0x7e, 0xa2, 0xb2, 0xa4, 0xe2, 0x1f, 0xa0, 0xe2, + 0x07, 0xe4, 0xa6, 0x16, 0x4a, 0xbe, 0x81, 0x96, 0x32, 0x08, 0x89, 0x3e, 0x5d, 0x1c, 0xd6, 0x44, + 0x9f, 0x2e, 0x99, 0x98, 0x96, 0x46, 0x89, 0xf1, 0x55, 0xb8, 0x2d, 0x06, 0xb0, 0xa1, 0x0e, 0x8a, + 0xa2, 0xe9, 0x95, 0x4c, 0x94, 0x86, 0x5e, 0x14, 0x24, 0x1b, 0xe2, 0x18, 0xda, 0xd9, 0x89, 0x47, + 0xec, 0xad, 0xd2, 0x71, 0x4a, 0xec, 0xad, 0xf2, 0x01, 0x89, 0x56, 0x38, 0x1e, 0x75, 0x24, 0x21, + 0xea, 0x11, 0x94, 0x69, 0x4a, 0x7a, 0x51, 0x10, 0x2b, 0x79, 0xa2, 0xff, 0xed, 0x4d, 0x5f, 0xfb, + 0xfe, 0x4d, 0x5f, 0xfb, 0xd7, 0x9b, 0xbe, 0xf6, 0x87, 0xb7, 0xfd, 0xca, 0xf7, 0x6f, 0xfb, 0x95, + 0x7f, 0xbe, 0xed, 0x57, 0x46, 0x0d, 0xfc, 0xf7, 0xec, 0x47, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, + 0x8d, 0x0a, 0x25, 0x8c, 0x81, 0x1b, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -3549,7 +3341,6 @@ type MasterClient interface { GetMasterCfg(ctx context.Context, in *GetMasterCfgRequest, opts ...grpc.CallOption) (*GetMasterCfgResponse, error) TransferSource(ctx context.Context, in *TransferSourceRequest, opts ...grpc.CallOption) (*TransferSourceResponse, error) OperateRelay(ctx context.Context, in *OperateRelayRequest, opts ...grpc.CallOption) (*OperateRelayResponse, error) - ImportExportCfgs(ctx context.Context, in *ImportExportCfgsRequest, opts ...grpc.CallOption) (*ImportExportCfgsResponse, error) } type masterClient struct { @@ -3749,15 +3540,6 @@ func (c *masterClient) OperateRelay(ctx context.Context, in *OperateRelayRequest return out, nil } -func (c *masterClient) ImportExportCfgs(ctx context.Context, in *ImportExportCfgsRequest, opts ...grpc.CallOption) (*ImportExportCfgsResponse, error) { - out := new(ImportExportCfgsResponse) - err := c.cc.Invoke(ctx, "/pb.Master/ImportExportCfgs", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - // MasterServer is the server API for Master service. type MasterServer interface { StartTask(context.Context, *StartTaskRequest) (*StartTaskResponse, error) @@ -3794,7 +3576,6 @@ type MasterServer interface { GetMasterCfg(context.Context, *GetMasterCfgRequest) (*GetMasterCfgResponse, error) TransferSource(context.Context, *TransferSourceRequest) (*TransferSourceResponse, error) OperateRelay(context.Context, *OperateRelayRequest) (*OperateRelayResponse, error) - ImportExportCfgs(context.Context, *ImportExportCfgsRequest) (*ImportExportCfgsResponse, error) } // UnimplementedMasterServer can be embedded to have forward compatible implementations. @@ -3864,9 +3645,6 @@ func (*UnimplementedMasterServer) TransferSource(ctx context.Context, req *Trans func (*UnimplementedMasterServer) OperateRelay(ctx context.Context, req *OperateRelayRequest) (*OperateRelayResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method OperateRelay not implemented") } -func (*UnimplementedMasterServer) ImportExportCfgs(ctx context.Context, req *ImportExportCfgsRequest) (*ImportExportCfgsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ImportExportCfgs not implemented") -} func RegisterMasterServer(s *grpc.Server, srv MasterServer) { s.RegisterService(&_Master_serviceDesc, srv) @@ -4250,24 +4028,6 @@ func _Master_OperateRelay_Handler(srv interface{}, ctx context.Context, dec func return interceptor(ctx, in, info, handler) } -func _Master_ImportExportCfgs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ImportExportCfgsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(MasterServer).ImportExportCfgs(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/pb.Master/ImportExportCfgs", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MasterServer).ImportExportCfgs(ctx, req.(*ImportExportCfgsRequest)) - } - return interceptor(ctx, in, info, handler) -} - var _Master_serviceDesc = grpc.ServiceDesc{ ServiceName: "pb.Master", HandlerType: (*MasterServer)(nil), @@ -4356,10 +4116,6 @@ var _Master_serviceDesc = grpc.ServiceDesc{ MethodName: "OperateRelay", Handler: _Master_OperateRelay_Handler, }, - { - MethodName: "ImportExportCfgs", - Handler: _Master_ImportExportCfgs_Handler, - }, }, Streams: []grpc.StreamDesc{}, Metadata: "dmmaster.proto", @@ -6713,209 +6469,69 @@ func (m *OperateRelayResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *ImportExportCfgsRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err +func encodeVarintDmmaster(dAtA []byte, offset int, v uint64) int { + offset -= sovDmmaster(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ } - return dAtA[:n], nil -} - -func (m *ImportExportCfgsRequest) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) + dAtA[offset] = uint8(v) + return base } - -func (m *ImportExportCfgsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i +func (m *StartTaskRequest) Size() (n int) { + if m == nil { + return 0 + } var l int _ = l - if len(m.Dir) > 0 { - i -= len(m.Dir) - copy(dAtA[i:], m.Dir) - i = encodeVarintDmmaster(dAtA, i, uint64(len(m.Dir))) - i-- - dAtA[i] = 0x12 + l = len(m.Task) + if l > 0 { + n += 1 + l + sovDmmaster(uint64(l)) } - if m.Op != 0 { - i = encodeVarintDmmaster(dAtA, i, uint64(m.Op)) - i-- - dAtA[i] = 0x8 + if len(m.Sources) > 0 { + for _, s := range m.Sources { + l = len(s) + n += 1 + l + sovDmmaster(uint64(l)) + } } - return len(dAtA) - i, nil -} - -func (m *Config) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err + if m.RemoveMeta { + n += 2 } - return dAtA[:n], nil -} - -func (m *Config) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) + return n } -func (m *Config) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i +func (m *StartTaskResponse) Size() (n int) { + if m == nil { + return 0 + } var l int _ = l - if len(m.Content) > 0 { - i -= len(m.Content) - copy(dAtA[i:], m.Content) - i = encodeVarintDmmaster(dAtA, i, uint64(len(m.Content))) - i-- - dAtA[i] = 0x12 + if m.Result { + n += 2 } - if len(m.Name) > 0 { - i -= len(m.Name) - copy(dAtA[i:], m.Name) - i = encodeVarintDmmaster(dAtA, i, uint64(len(m.Name))) - i-- - dAtA[i] = 0xa + l = len(m.Msg) + if l > 0 { + n += 1 + l + sovDmmaster(uint64(l)) } - return len(dAtA) - i, nil -} - -func (m *ImportExportCfgsResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err + if len(m.Sources) > 0 { + for _, e := range m.Sources { + l = e.Size() + n += 1 + l + sovDmmaster(uint64(l)) + } } - return dAtA[:n], nil -} - -func (m *ImportExportCfgsResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) + return n } -func (m *ImportExportCfgsResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i +func (m *OperateTaskRequest) Size() (n int) { + if m == nil { + return 0 + } var l int _ = l - if len(m.Tasks) > 0 { - for iNdEx := len(m.Tasks) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Tasks[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintDmmaster(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x22 - } - } - if len(m.Sources) > 0 { - for iNdEx := len(m.Sources) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Sources[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintDmmaster(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a - } - } - if len(m.Msg) > 0 { - i -= len(m.Msg) - copy(dAtA[i:], m.Msg) - i = encodeVarintDmmaster(dAtA, i, uint64(len(m.Msg))) - i-- - dAtA[i] = 0x12 - } - if m.Result { - i-- - if m.Result { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func encodeVarintDmmaster(dAtA []byte, offset int, v uint64) int { - offset -= sovDmmaster(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *StartTaskRequest) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.Task) - if l > 0 { - n += 1 + l + sovDmmaster(uint64(l)) - } - if len(m.Sources) > 0 { - for _, s := range m.Sources { - l = len(s) - n += 1 + l + sovDmmaster(uint64(l)) - } - } - if m.RemoveMeta { - n += 2 - } - return n -} - -func (m *StartTaskResponse) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Result { - n += 2 - } - l = len(m.Msg) - if l > 0 { - n += 1 + l + sovDmmaster(uint64(l)) - } - if len(m.Sources) > 0 { - for _, e := range m.Sources { - l = e.Size() - n += 1 + l + sovDmmaster(uint64(l)) - } - } - return n -} - -func (m *OperateTaskRequest) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Op != 0 { - n += 1 + sovDmmaster(uint64(m.Op)) + if m.Op != 0 { + n += 1 + sovDmmaster(uint64(m.Op)) } l = len(m.Name) if l > 0 { @@ -7915,67 +7531,6 @@ func (m *OperateRelayResponse) Size() (n int) { return n } -func (m *ImportExportCfgsRequest) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Op != 0 { - n += 1 + sovDmmaster(uint64(m.Op)) - } - l = len(m.Dir) - if l > 0 { - n += 1 + l + sovDmmaster(uint64(l)) - } - return n -} - -func (m *Config) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.Name) - if l > 0 { - n += 1 + l + sovDmmaster(uint64(l)) - } - l = len(m.Content) - if l > 0 { - n += 1 + l + sovDmmaster(uint64(l)) - } - return n -} - -func (m *ImportExportCfgsResponse) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Result { - n += 2 - } - l = len(m.Msg) - if l > 0 { - n += 1 + l + sovDmmaster(uint64(l)) - } - if len(m.Sources) > 0 { - for _, e := range m.Sources { - l = e.Size() - n += 1 + l + sovDmmaster(uint64(l)) - } - } - if len(m.Tasks) > 0 { - for _, e := range m.Tasks { - l = e.Size() - n += 1 + l + sovDmmaster(uint64(l)) - } - } - return n -} - func sovDmmaster(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -14564,400 +14119,6 @@ func (m *OperateRelayResponse) Unmarshal(dAtA []byte) error { } return nil } -func (m *ImportExportCfgsRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDmmaster - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ImportExportCfgsRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ImportExportCfgsRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Op", wireType) - } - m.Op = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDmmaster - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Op |= CfgsOp(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Dir", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDmmaster - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthDmmaster - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthDmmaster - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Dir = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipDmmaster(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthDmmaster - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthDmmaster - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *Config) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDmmaster - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Config: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Config: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDmmaster - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthDmmaster - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthDmmaster - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Name = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Content", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDmmaster - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthDmmaster - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthDmmaster - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Content = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipDmmaster(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthDmmaster - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthDmmaster - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *ImportExportCfgsResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDmmaster - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ImportExportCfgsResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ImportExportCfgsResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Result", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDmmaster - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.Result = bool(v != 0) - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Msg", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDmmaster - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthDmmaster - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthDmmaster - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Msg = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Sources", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDmmaster - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthDmmaster - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthDmmaster - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Sources = append(m.Sources, &Config{}) - if err := m.Sources[len(m.Sources)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Tasks", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDmmaster - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthDmmaster - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthDmmaster - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Tasks = append(m.Tasks, &Config{}) - if err := m.Tasks[len(m.Tasks)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipDmmaster(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthDmmaster - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthDmmaster - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func skipDmmaster(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/dm/pbmock/dmmaster.go b/dm/pbmock/dmmaster.go index f7436976ae..39f9123c3f 100644 --- a/dm/pbmock/dmmaster.go +++ b/dm/pbmock/dmmaster.go @@ -136,26 +136,6 @@ func (mr *MockMasterClientMockRecorder) HandleError(arg0, arg1 interface{}, arg2 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandleError", reflect.TypeOf((*MockMasterClient)(nil).HandleError), varargs...) } -// ImportExportCfgs mocks base method. -func (m *MockMasterClient) ImportExportCfgs(arg0 context.Context, arg1 *pb.ImportExportCfgsRequest, arg2 ...grpc.CallOption) (*pb.ImportExportCfgsResponse, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "ImportExportCfgs", varargs...) - ret0, _ := ret[0].(*pb.ImportExportCfgsResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ImportExportCfgs indicates an expected call of ImportExportCfgs. -func (mr *MockMasterClientMockRecorder) ImportExportCfgs(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImportExportCfgs", reflect.TypeOf((*MockMasterClient)(nil).ImportExportCfgs), varargs...) -} - // ListMember mocks base method. func (m *MockMasterClient) ListMember(arg0 context.Context, arg1 *pb.ListMemberRequest, arg2 ...grpc.CallOption) (*pb.ListMemberResponse, error) { m.ctrl.T.Helper() @@ -574,21 +554,6 @@ func (mr *MockMasterServerMockRecorder) HandleError(arg0, arg1 interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandleError", reflect.TypeOf((*MockMasterServer)(nil).HandleError), arg0, arg1) } -// ImportExportCfgs mocks base method. -func (m *MockMasterServer) ImportExportCfgs(arg0 context.Context, arg1 *pb.ImportExportCfgsRequest) (*pb.ImportExportCfgsResponse, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ImportExportCfgs", arg0, arg1) - ret0, _ := ret[0].(*pb.ImportExportCfgsResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ImportExportCfgs indicates an expected call of ImportExportCfgs. -func (mr *MockMasterServerMockRecorder) ImportExportCfgs(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImportExportCfgs", reflect.TypeOf((*MockMasterServer)(nil).ImportExportCfgs), arg0, arg1) -} - // ListMember mocks base method. func (m *MockMasterServer) ListMember(arg0 context.Context, arg1 *pb.ListMemberRequest) (*pb.ListMemberResponse, error) { m.ctrl.T.Helper() diff --git a/dm/proto/dmmaster.proto b/dm/proto/dmmaster.proto index 9cd0d6a7a4..9f4a7e2b02 100644 --- a/dm/proto/dmmaster.proto +++ b/dm/proto/dmmaster.proto @@ -108,8 +108,6 @@ service Master { rpc TransferSource(TransferSourceRequest) returns(TransferSourceResponse) {} rpc OperateRelay(OperateRelayRequest) returns(OperateRelayResponse) {} - - rpc ImportExportCfgs(ImportExportCfgsRequest) returns(ImportExportCfgsResponse) {} } message StartTaskRequest { @@ -460,26 +458,4 @@ enum RelayOpV2 { InvalidRelayOpV2 = 0; StartRelayV2 = 1; StopRelayV2 = 2; -} - -message ImportExportCfgsRequest { - CfgsOp op = 1; - string dir = 2; -} - -message Config { - string name = 1; - string content = 2; -} - -message ImportExportCfgsResponse { - bool result = 1; - string msg = 2; - repeated Config sources = 3; - repeated Config tasks = 4; -} - -enum CfgsOp { - Export = 0; - Import = 1; } \ No newline at end of file diff --git a/pkg/ha/source.go b/pkg/ha/source.go index 66f6159c1c..db0eeb1768 100644 --- a/pkg/ha/source.go +++ b/pkg/ha/source.go @@ -37,6 +37,31 @@ func PutSourceCfg(cli *clientv3.Client, cfg *config.SourceConfig) (int64, error) return rev, err } +// GetAllSourceCfgBeforeV202 gets all upstream source configs before v2.0.2. +// This func only use for export-config command. +func GetAllSourceCfgBeforeV202(cli *clientv3.Client) (map[string]*config.SourceConfig, int64, error) { + ctx, cancel := context.WithTimeout(cli.Ctx(), etcdutil.DefaultRequestTimeout) + defer cancel() + + var ( + scm = make(map[string]*config.SourceConfig) + resp *clientv3.GetResponse + err error + ) + resp, err = cli.Get(ctx, common.UpstreamConfigKeyAdapterV1.Path(), clientv3.WithPrefix()) + + if err != nil { + return scm, 0, err + } + + scm, err = sourceCfgFromResp("", resp) + if err != nil { + return scm, 0, err + } + + return scm, resp.Header.Revision, nil +} + // GetSourceCfg gets upstream source configs. // k/v: source ID -> source config. // if the source config for the sourceID not exist, return with `err == nil`. From 03a54450a74153ccfbfb4e607ca6354db463c60e Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 15 Jul 2021 19:29:45 +0800 Subject: [PATCH 03/19] add import config --- dm/ctl/ctl.go | 1 + dm/ctl/master/import_export_config.go | 211 +++++++++++++++++++++++++- 2 files changed, 205 insertions(+), 7 deletions(-) diff --git a/dm/ctl/ctl.go b/dm/ctl/ctl.go index d2eb23b7dc..e7bc7f7eb1 100644 --- a/dm/ctl/ctl.go +++ b/dm/ctl/ctl.go @@ -76,6 +76,7 @@ func NewRootCmd() *cobra.Command { master.NewStartRelayCmd(), master.NewStopRelayCmd(), master.NewExportCfgCmd(), + master.NewImportCfgCmd(), newDecryptCmd(), newEncryptCmd(), ) diff --git a/dm/ctl/master/import_export_config.go b/dm/ctl/master/import_export_config.go index 53f7e60f4e..87e3bab94c 100644 --- a/dm/ctl/master/import_export_config.go +++ b/dm/ctl/master/import_export_config.go @@ -14,22 +14,30 @@ package master import ( + "context" + "encoding/json" "io/ioutil" "os" "path" "sort" + "strings" "github.com/spf13/cobra" + "github.com/pingcap/errors" + "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/dm/ctl/common" + "github.com/pingcap/dm/dm/pb" "github.com/pingcap/dm/pkg/ha" + "github.com/pingcap/dm/pkg/utils" ) var ( - taskDirname = "tasks" - sourceDirname = "sources" - yamlSuffix = ".yaml" + taskDirname = "tasks" + sourceDirname = "sources" + relayWorkersFilename = "relay_workers.json" + yamlSuffix = ".yaml" ) // NewExportCfgCmd creates a exportCfg command. @@ -43,7 +51,18 @@ func NewExportCfgCmd() *cobra.Command { return cmd } -// exportCfgFunc gets config. +// NewImportCfgCmd creates a importCfg command. +func NewImportCfgCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "import-config [--dir directory]", + Short: "Import the configurations of sources and tasks.", + RunE: importCfgFunc, + } + cmd.Flags().StringP("dir", "d", "configs", "specify the configs directory, default is `./configs`") + return cmd +} + +// exportCfgFunc exports configs. func exportCfgFunc(cmd *cobra.Command, args []string) error { dir, err := cmd.Flags().GetString("dir") if err != nil { @@ -51,15 +70,16 @@ func exportCfgFunc(cmd *cobra.Command, args []string) error { return err } + cli := common.GlobalCtlClient.EtcdClient // get all source cfgs - sourceCfgsMap, _, err := ha.GetSourceCfg(common.GlobalCtlClient.EtcdClient, "", 0) + sourceCfgsMap, _, err := ha.GetSourceCfg(cli, "", 0) if err != nil { common.PrintLinesf("can not get source configs from etcd") return err } // try to get all source cfgs before v2.0.2 if len(sourceCfgsMap) == 0 { - sourceCfgsMap, _, err = ha.GetAllSourceCfgBeforeV202(common.GlobalCtlClient.EtcdClient) + sourceCfgsMap, _, err = ha.GetAllSourceCfgBeforeV202(cli) if err != nil { common.PrintLinesf("can not get source configs from etcd") return err @@ -67,12 +87,19 @@ func exportCfgFunc(cmd *cobra.Command, args []string) error { } // get all task configs. - subTaskCfgsMap, _, err := ha.GetAllSubTaskCfg(common.GlobalCtlClient.EtcdClient) + subTaskCfgsMap, _, err := ha.GetAllSubTaskCfg(cli) if err != nil { common.PrintLinesf("can not get subtask configs from etcd") return err } + // get all relay configs. + relayWorkers, _, err := ha.GetAllRelayConfig(cli) + if err != nil { + common.PrintLinesf("can not get relay workers from etcd") + return err + } + taskCfgsMap := make(map[string]string, len(subTaskCfgsMap)) subTaskCfgsListMap := make(map[string][]*config.SubTaskConfig, len(subTaskCfgsMap)) @@ -139,6 +166,176 @@ func exportCfgFunc(cmd *cobra.Command, args []string) error { } } + if len(relayWorkers) > 0 { + relayWorkers, err := json.Marshal(relayWorkers) + if err != nil { + common.PrintLinesf("fail to marshal relay workers") + return err + } + + relayWorkersFile := path.Join(dir, relayWorkersFilename) + err = ioutil.WriteFile(relayWorkersFile, relayWorkers, 0o644) + if err != nil { + common.PrintLinesf("can not write relay workers to file `%s`", relayWorkersFile) + return err + } + } + common.PrintLinesf("export configs to directory `%s` succeed", dir) return nil } + +// importCfgFunc imports configs. +func importCfgFunc(cmd *cobra.Command, args []string) error { + dir, err := cmd.Flags().GetString("dir") + if err != nil { + common.PrintLinesf("can not get directory") + return err + } + + // get all source cfgs + if !utils.IsDirExists(dir) { + return errors.Errorf("config directory `%s` not exists", dir) + } + + var ( + sourceCfgs []string + taskCfgs []string + relayWorkers map[string]map[string]struct{} + taskDir = path.Join(dir, taskDirname) + taskDirExist = utils.IsDirExists(taskDir) + sourceDir = path.Join(dir, sourceDirname) + sourceDirExist = utils.IsDirExists(sourceDir) + relayWorkersFile = path.Join(dir, relayWorkersFilename) + relayWorkersExist = utils.IsFileExists(relayWorkersFile) + sourceResp = &pb.OperateSourceResponse{} + taskResp = &pb.StartTaskResponse{} + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if sourceDirExist { + if sourceCfgs, err = collectDirCfgs(sourceDir); err != nil { + common.PrintLinesf("fail to collect source config files from source configs directory `%s`", sourceDir) + return err + } + } + if taskDirExist { + if taskCfgs, err = collectDirCfgs(taskDir); err != nil { + common.PrintLinesf("fail to collect task config files from task configs directory `%s`", taskDir) + return err + } + } + if relayWorkersExist { + content, err2 := common.GetFileContent(relayWorkersFile) + if err2 != nil { + common.PrintLinesf("fail to read relay workers config `%s`", relayWorkersFile) + return err2 + } + err = json.Unmarshal(content, &relayWorkers) + if err != nil { + common.PrintLinesf("fail to unmarshal relay workers config `%s`", relayWorkersFile) + return err + } + } + + if len(sourceCfgs) > 0 { + common.PrintLinesf("start creating sources") + } + + // Do not use batch for `operate-source start source1, source2` if we want to support idemponent import-config. + // Because `operate-source start` will revert all batch sources if any source error. + // e.g. ErrSchedulerSourceCfgExist + for _, sourceCfg := range sourceCfgs { + err = common.SendRequest( + ctx, + "OperateSource", + &pb.OperateSourceRequest{ + Config: []string{sourceCfg}, + Op: pb.SourceOp_StartSource, + }, + &sourceResp, + ) + + if err != nil { + common.PrintLinesf("fail to create sources") + return err + } + + if !sourceResp.Result && !strings.Contains(sourceResp.Msg, "already exist") { + common.PrettyPrintResponse(sourceResp) + return errors.Errorf("fail to create sources") + } + } + + if len(taskCfgs) > 0 { + common.PrintLinesf("start creating tasks") + } + + for _, taskCfg := range taskCfgs { + err = common.SendRequest( + ctx, + "StartTask", + &pb.StartTaskRequest{ + Task: taskCfg, + }, + &taskResp, + ) + + if err != nil { + common.PrintLinesf("fail to create tasks") + return err + } + + if !taskResp.Result && !strings.Contains(taskResp.Msg, "already exist") { + common.PrettyPrintResponse(taskResp) + return errors.Errorf("fail to create tasks") + } + } + + if len(relayWorkers) > 0 { + common.PrintLinesf("start creating relay workers") + } + + for source, workerSet := range relayWorkers { + workers := make([]string, 0, len(workerSet)) + for worker := range workerSet { + workers = append(workers, worker) + } + resp := &pb.OperateRelayResponse{} + err = common.SendRequest( + ctx, + "OperateRelay", + &pb.OperateRelayRequest{ + Op: pb.RelayOpV2_StartRelayV2, + Source: source, + Worker: workers, + }, + &resp, + ) + + if err != nil { + return err + } + } + + common.PrintLinesf("import configs from directory `%s` succeed", dir) + return nil +} + +func collectDirCfgs(dir string) ([]string, error) { + files, err := ioutil.ReadDir(dir) + if err != nil { + return nil, err + } + + cfgs := make([]string, 0, len(files)) + for _, f := range files { + cfg, err2 := common.GetFileContent(path.Join(dir, f.Name())) + if err2 != nil { + return nil, err2 + } + cfgs = append(cfgs, string(cfg)) + } + return cfgs, nil +} From 4c68c048c63b904c89c6a36d4684071be4492ba2 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Fri, 16 Jul 2021 11:25:16 +0800 Subject: [PATCH 04/19] refactor --- dm/ctl/master/import_export_config.go | 330 ++++++++++++++------------ tests/_utils/run_sql | 1 - 2 files changed, 184 insertions(+), 147 deletions(-) diff --git a/dm/ctl/master/import_export_config.go b/dm/ctl/master/import_export_config.go index 87e3bab94c..8dc562eae5 100644 --- a/dm/ctl/master/import_export_config.go +++ b/dm/ctl/master/import_export_config.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/spf13/cobra" + "go.etcd.io/etcd/clientv3" "github.com/pingcap/errors" @@ -70,83 +71,144 @@ func exportCfgFunc(cmd *cobra.Command, args []string) error { return err } - cli := common.GlobalCtlClient.EtcdClient - // get all source cfgs - sourceCfgsMap, _, err := ha.GetSourceCfg(cli, "", 0) + // get all configs + sourceCfgsMap, subTaskCfgsMap, relayWorkersSet, err := getAllCfgs(common.GlobalCtlClient.EtcdClient) if err != nil { - common.PrintLinesf("can not get source configs from etcd") return err } + // create directory + taskDir, sourceDir, err := createDirectory(dir) + if err != nil { + return err + } + // write sourceCfg files + if err = writeSourceCfgs(sourceDir, sourceCfgsMap); err != nil { + return err + } + // write taskCfg files + if err = writeTaskCfgs(taskDir, subTaskCfgsMap); err != nil { + return err + } + // write relayWorkers + if err = writeRelayWorkers(path.Join(dir, relayWorkersFilename), relayWorkersSet); err != nil { + return err + } + + common.PrintLinesf("export configs to directory `%s` succeed", dir) + return nil +} + +// importCfgFunc imports configs. +func importCfgFunc(cmd *cobra.Command, args []string) error { + dir, err := cmd.Flags().GetString("dir") + if err != nil { + common.PrintLinesf("can not get directory") + return err + } + + sourceCfgs, taskCfgs, relayWorkers, err := collectCfgs(dir) + if err != nil { + return err + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := createSources(ctx, sourceCfgs); err != nil { + return err + } + if err := createTasks(ctx, taskCfgs); err != nil { + return err + } + if len(relayWorkers) > 0 { + common.PrintLinesf("The original relay workers have been exported to `%s`.", path.Join(dir, relayWorkersFilename)) + common.PrintLinesf("Currently unsupport recover relay workers. You may need to execute `transfer-source` and `start-relay` command manually.") + } + + common.PrintLinesf("import configs from directory `%s` succeed", dir) + return nil +} + +func collectDirCfgs(dir string) ([]string, error) { + files, err := ioutil.ReadDir(dir) + if err != nil { + return nil, err + } + + cfgs := make([]string, 0, len(files)) + for _, f := range files { + cfg, err2 := common.GetFileContent(path.Join(dir, f.Name())) + if err2 != nil { + return nil, err2 + } + cfgs = append(cfgs, string(cfg)) + } + return cfgs, nil +} + +// getSourceCfgs gets all source cfgs. +func getSourceCfgs(cli *clientv3.Client) (map[string]*config.SourceConfig, error) { + sourceCfgsMap, _, err := ha.GetSourceCfg(cli, "", 0) + if err != nil { + return nil, err + } // try to get all source cfgs before v2.0.2 if len(sourceCfgsMap) == 0 { sourceCfgsMap, _, err = ha.GetAllSourceCfgBeforeV202(cli) if err != nil { - common.PrintLinesf("can not get source configs from etcd") - return err + return nil, err } } + return sourceCfgsMap, nil +} - // get all task configs. +func getAllCfgs(cli *clientv3.Client) (map[string]*config.SourceConfig, map[string]map[string]config.SubTaskConfig, map[string]map[string]struct{}, error) { + // get all source cfgs + sourceCfgsMap, err := getSourceCfgs(cli) + if err != nil { + common.PrintLinesf("can not get source configs from etcd") + return nil, nil, nil, err + } + // get all task cfgs subTaskCfgsMap, _, err := ha.GetAllSubTaskCfg(cli) if err != nil { common.PrintLinesf("can not get subtask configs from etcd") - return err + return nil, nil, nil, err } - // get all relay configs. relayWorkers, _, err := ha.GetAllRelayConfig(cli) if err != nil { common.PrintLinesf("can not get relay workers from etcd") - return err - } - - taskCfgsMap := make(map[string]string, len(subTaskCfgsMap)) - subTaskCfgsListMap := make(map[string][]*config.SubTaskConfig, len(subTaskCfgsMap)) - - // from source => task => subtask to task => source => subtask - for _, subTaskCfgs := range subTaskCfgsMap { - for task, subTaskCfg := range subTaskCfgs { - clone := subTaskCfg - if subTaskCfgList, ok := subTaskCfgsListMap[task]; ok { - subTaskCfgsListMap[task] = append(subTaskCfgList, &clone) - } else { - subTaskCfgsListMap[task] = []*config.SubTaskConfig{&clone} - } - } - } - // from task => source => subtask to task => taskCfg - for task, subTaskCfgs := range subTaskCfgsListMap { - sort.Slice(subTaskCfgs, func(i, j int) bool { - return subTaskCfgs[i].SourceID < subTaskCfgs[j].SourceID - }) - taskCfg := config.FromSubTaskConfigs(subTaskCfgs...) - taskCfgsMap[task] = taskCfg.String() + return nil, nil, nil, err } + return sourceCfgsMap, subTaskCfgsMap, relayWorkers, nil +} - // create directory - if err = os.MkdirAll(dir, 0o755); err != nil { +func createDirectory(dir string) (string, string, error) { + if err := os.MkdirAll(dir, 0o755); err != nil { common.PrintLinesf("can not create directory `%s`", dir) - return err + return "", "", err } taskDir := path.Join(dir, taskDirname) - if err = os.MkdirAll(taskDir, 0o755); err != nil { + if err := os.MkdirAll(taskDir, 0o755); err != nil { common.PrintLinesf("can not create directory of task configs `%s`", taskDir) - return err + return "", "", err } sourceDir := path.Join(dir, sourceDirname) - if err = os.MkdirAll(sourceDir, 0o755); err != nil { + if err := os.MkdirAll(sourceDir, 0o755); err != nil { common.PrintLinesf("can not create directory of source configs `%s`", sourceDir) - return err + return "", "", err } + return taskDir, sourceDir, nil +} - // write sourceCfg files +func writeSourceCfgs(sourceDir string, sourceCfgsMap map[string]*config.SourceConfig) error { for source, sourceCfg := range sourceCfgsMap { sourceFile := path.Join(sourceDir, source) sourceFile += yamlSuffix - fileContent, err2 := sourceCfg.Yaml() - if err2 != nil { + fileContent, err := sourceCfg.Yaml() + if err != nil { common.PrintLinesf("fail to marshal source config of `%s`", source) - return err2 + return err } err = ioutil.WriteFile(sourceFile, []byte(fileContent), 0o644) if err != nil { @@ -154,100 +216,119 @@ func exportCfgFunc(cmd *cobra.Command, args []string) error { return err } } + return nil +} + +func writeTaskCfgs(taskDir string, subTaskCfgsMap map[string]map[string]config.SubTaskConfig) error { + subTaskCfgsListMap := make(map[string][]*config.SubTaskConfig, len(subTaskCfgsMap)) + // from source => task => subtask to task => source => subtask + for _, subTaskCfgs := range subTaskCfgsMap { + for task, subTaskCfg := range subTaskCfgs { + clone := subTaskCfg + if subTaskCfgList, ok := subTaskCfgsListMap[task]; ok { + subTaskCfgsListMap[task] = append(subTaskCfgList, &clone) + } else { + subTaskCfgsListMap[task] = []*config.SubTaskConfig{&clone} + } + } + } + // from task => source => subtask to task => taskCfg + for task, subTaskCfgs := range subTaskCfgsListMap { + sort.Slice(subTaskCfgs, func(i, j int) bool { + return subTaskCfgs[i].SourceID < subTaskCfgs[j].SourceID + }) + taskCfg := config.FromSubTaskConfigs(subTaskCfgs...) - // write taskCfg files - for task, taskCfg := range taskCfgsMap { taskFile := path.Join(taskDir, task) taskFile += yamlSuffix - err = ioutil.WriteFile(taskFile, []byte(taskCfg), 0o644) - if err != nil { + if err := ioutil.WriteFile(taskFile, []byte(taskCfg.String()), 0o644); err != nil { common.PrintLinesf("can not write task config to file `%s`", taskFile) return err } } + return nil +} - if len(relayWorkers) > 0 { - relayWorkers, err := json.Marshal(relayWorkers) - if err != nil { - common.PrintLinesf("fail to marshal relay workers") - return err - } +func writeRelayWorkers(relayWorkersFile string, relayWorkersSet map[string]map[string]struct{}) error { + if len(relayWorkersSet) == 0 { + return nil + } - relayWorkersFile := path.Join(dir, relayWorkersFilename) - err = ioutil.WriteFile(relayWorkersFile, relayWorkers, 0o644) - if err != nil { - common.PrintLinesf("can not write relay workers to file `%s`", relayWorkersFile) - return err + // from source => workerSet to source => workerList + relayWorkers := make(map[string][]string, len(relayWorkersSet)) + for source, workerSet := range relayWorkersSet { + workers := make([]string, 0, len(workerSet)) + for worker := range workerSet { + workers = append(workers, worker) } + relayWorkers[source] = workers } - common.PrintLinesf("export configs to directory `%s` succeed", dir) - return nil -} - -// importCfgFunc imports configs. -func importCfgFunc(cmd *cobra.Command, args []string) error { - dir, err := cmd.Flags().GetString("dir") + content, err := json.Marshal(relayWorkers) if err != nil { - common.PrintLinesf("can not get directory") + common.PrintLinesf("fail to marshal relay workers") return err } - // get all source cfgs - if !utils.IsDirExists(dir) { - return errors.Errorf("config directory `%s` not exists", dir) + err = ioutil.WriteFile(relayWorkersFile, content, 0o644) + if err != nil { + common.PrintLinesf("can not write relay workers to file `%s`", relayWorkersFile) + return err } + return nil +} +func collectCfgs(dir string) (sourceCfgs []string, taskCfgs []string, relayWorkers map[string][]string, err error) { var ( - sourceCfgs []string - taskCfgs []string - relayWorkers map[string]map[string]struct{} - taskDir = path.Join(dir, taskDirname) - taskDirExist = utils.IsDirExists(taskDir) - sourceDir = path.Join(dir, sourceDirname) - sourceDirExist = utils.IsDirExists(sourceDir) - relayWorkersFile = path.Join(dir, relayWorkersFilename) - relayWorkersExist = utils.IsFileExists(relayWorkersFile) - sourceResp = &pb.OperateSourceResponse{} - taskResp = &pb.StartTaskResponse{} + sourceDir = path.Join(dir, sourceDirname) + taskDir = path.Join(dir, taskDirname) + relayWorkersFile = path.Join(dir, relayWorkersFilename) + content []byte ) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + if !utils.IsDirExists(dir) { + return nil, nil, nil, errors.Errorf("config directory `%s` not exists", dir) + } - if sourceDirExist { + if utils.IsDirExists(sourceDir) { if sourceCfgs, err = collectDirCfgs(sourceDir); err != nil { common.PrintLinesf("fail to collect source config files from source configs directory `%s`", sourceDir) - return err + return } } - if taskDirExist { + if utils.IsDirExists(taskDir) { if taskCfgs, err = collectDirCfgs(taskDir); err != nil { common.PrintLinesf("fail to collect task config files from task configs directory `%s`", taskDir) - return err + return } } - if relayWorkersExist { - content, err2 := common.GetFileContent(relayWorkersFile) - if err2 != nil { + if utils.IsFileExists(relayWorkersFile) { + content, err = common.GetFileContent(relayWorkersFile) + if err != nil { common.PrintLinesf("fail to read relay workers config `%s`", relayWorkersFile) - return err2 + return } err = json.Unmarshal(content, &relayWorkers) if err != nil { common.PrintLinesf("fail to unmarshal relay workers config `%s`", relayWorkersFile) - return err + return } } + // nolint:nakedret + return +} - if len(sourceCfgs) > 0 { - common.PrintLinesf("start creating sources") +func createSources(ctx context.Context, sourceCfgs []string) error { + if len(sourceCfgs) == 0 { + return nil } + common.PrintLinesf("start creating sources") + sourceResp := &pb.OperateSourceResponse{} // Do not use batch for `operate-source start source1, source2` if we want to support idemponent import-config. // Because `operate-source start` will revert all batch sources if any source error. // e.g. ErrSchedulerSourceCfgExist for _, sourceCfg := range sourceCfgs { - err = common.SendRequest( + err := common.SendRequest( ctx, "OperateSource", &pb.OperateSourceRequest{ @@ -256,7 +337,6 @@ func importCfgFunc(cmd *cobra.Command, args []string) error { }, &sourceResp, ) - if err != nil { common.PrintLinesf("fail to create sources") return err @@ -267,13 +347,18 @@ func importCfgFunc(cmd *cobra.Command, args []string) error { return errors.Errorf("fail to create sources") } } + return nil +} - if len(taskCfgs) > 0 { - common.PrintLinesf("start creating tasks") +func createTasks(ctx context.Context, taskCfgs []string) error { + if len(taskCfgs) == 0 { + return nil } + common.PrintLinesf("start creating tasks") + taskResp := &pb.StartTaskResponse{} for _, taskCfg := range taskCfgs { - err = common.SendRequest( + err := common.SendRequest( ctx, "StartTask", &pb.StartTaskRequest{ @@ -281,61 +366,14 @@ func importCfgFunc(cmd *cobra.Command, args []string) error { }, &taskResp, ) - if err != nil { common.PrintLinesf("fail to create tasks") return err } - if !taskResp.Result && !strings.Contains(taskResp.Msg, "already exist") { common.PrettyPrintResponse(taskResp) return errors.Errorf("fail to create tasks") } } - - if len(relayWorkers) > 0 { - common.PrintLinesf("start creating relay workers") - } - - for source, workerSet := range relayWorkers { - workers := make([]string, 0, len(workerSet)) - for worker := range workerSet { - workers = append(workers, worker) - } - resp := &pb.OperateRelayResponse{} - err = common.SendRequest( - ctx, - "OperateRelay", - &pb.OperateRelayRequest{ - Op: pb.RelayOpV2_StartRelayV2, - Source: source, - Worker: workers, - }, - &resp, - ) - - if err != nil { - return err - } - } - - common.PrintLinesf("import configs from directory `%s` succeed", dir) return nil } - -func collectDirCfgs(dir string) ([]string, error) { - files, err := ioutil.ReadDir(dir) - if err != nil { - return nil, err - } - - cfgs := make([]string, 0, len(files)) - for _, f := range files { - cfg, err2 := common.GetFileContent(path.Join(dir, f.Name())) - if err2 != nil { - return nil, err2 - } - cfgs = append(cfgs, string(cfg)) - } - return cfgs, nil -} diff --git a/tests/_utils/run_sql b/tests/_utils/run_sql index ed5609dc41..4674e4b945 100755 --- a/tests/_utils/run_sql +++ b/tests/_utils/run_sql @@ -5,7 +5,6 @@ set -eu -TIDB_PORT=4000 user="root" if [[ "$2" = $TIDB_PORT ]]; then user="test" From 734045e096b97c788051c1fa872e441df241c809 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Fri, 16 Jul 2021 15:56:22 +0800 Subject: [PATCH 05/19] omitempty --- dm/config/source_config.go | 75 +++++++++++++- dm/config/task.go | 139 ++++++++++++++++++++++++++ dm/ctl/master/import_export_config.go | 8 +- 3 files changed, 219 insertions(+), 3 deletions(-) diff --git a/dm/config/source_config.go b/dm/config/source_config.go index db70e5ee83..d6c7e89398 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -45,7 +45,7 @@ type PurgeConfig struct { RemainSpace int64 `yaml:"remain-space" toml:"remain-space" json:"remain-space"` // if remain space in @RelayBaseDir less than @RemainSpace (GB), then it can be purged } -// SourceConfig is the configuration for Worker. +// SourceConfig is the configuration for source. type SourceConfig struct { EnableGTID bool `yaml:"enable-gtid" toml:"enable-gtid" json:"enable-gtid"` AutoFixGTID bool `yaml:"auto-fix-gtid" toml:"auto-fix-gtid" json:"auto-fix-gtid"` @@ -353,3 +353,76 @@ func (c *SourceConfig) check(metaData *toml.MetaData, err error) error { c.adjust() return nil } + +// YamlForDowngrade returns YAML format represents of config for downgrade. +func (c *SourceConfig) YamlForDowngrade() (string, error) { + s := NewSourceConfigForDowngrade(c) + return s.Yaml() +} + +// SourceConfigForDowngrade is the base configuration for source in v2.0. +// This config is used for downgrade(export-config) from a higher dmctl version. +// When we add any new config item into SourceConfig, we should update it also. +type SourceConfigForDowngrade struct { + EnableGTID bool `yaml:"enable-gtid"` + AutoFixGTID bool `yaml:"auto-fix-gtid"` + RelayDir string `yaml:"relay-dir"` + MetaDir string `yaml:"meta-dir"` + Flavor string `yaml:"flavor"` + Charset string `yaml:"charset"` + EnableRelay bool `yaml:"enable-relay"` + RelayBinLogName string `yaml:"relay-binlog-name"` + RelayBinlogGTID string `yaml:"relay-binlog-gtid"` + UUIDSuffix int `yaml:"-"` + SourceID string `yaml:"source-id"` + From DBConfig `yaml:"from"` + Purge PurgeConfig `yaml:"purge"` + Checker CheckerConfig `yaml:"checker"` + ServerID uint32 `yaml:"server-id"` + Tracer map[string]interface{} `yaml:"tracer"` + // any new config item, we mark it omitempty + CaseSensitive bool `yaml:"case-sensitive,omitempty"` + Filters []*bf.BinlogEventRule `yaml:"filters,omitempty"` +} + +// NewSourceConfigForDowngrade creates a new base config for downgrade. +func NewSourceConfigForDowngrade(sourceCfg *SourceConfig) *SourceConfigForDowngrade { + return &SourceConfigForDowngrade{ + EnableGTID: sourceCfg.EnableGTID, + AutoFixGTID: sourceCfg.AutoFixGTID, + RelayDir: sourceCfg.RelayDir, + MetaDir: sourceCfg.MetaDir, + Flavor: sourceCfg.Flavor, + Charset: sourceCfg.Charset, + EnableRelay: sourceCfg.EnableRelay, + RelayBinLogName: sourceCfg.RelayBinLogName, + RelayBinlogGTID: sourceCfg.RelayBinlogGTID, + UUIDSuffix: sourceCfg.UUIDSuffix, + SourceID: sourceCfg.SourceID, + From: sourceCfg.From, + Purge: sourceCfg.Purge, + Checker: sourceCfg.Checker, + ServerID: sourceCfg.ServerID, + Tracer: sourceCfg.Tracer, + CaseSensitive: sourceCfg.CaseSensitive, + Filters: sourceCfg.Filters, + } +} + +// omitDefaultVals change default value to empty value for new config item. +// If any default value for new config item is not empty(0 or false or nil), +// we should change it to empty. +func (c *SourceConfigForDowngrade) omitDefaultVals() { + // e.g. + // if s.newConfig == defaultVal { + // s.newConfig == 0 + // } +} + +// Yaml omitDefaultVals and returns YAML format representation of the config. +func (c *SourceConfigForDowngrade) Yaml() (string, error) { + // omit default values, so we can ignore them for later marshal + c.omitDefaultVals() + b, err := yaml.Marshal(c) + return string(b), err +} diff --git a/dm/config/task.go b/dm/config/task.go index 170d789723..06cb06cfcc 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -935,3 +935,142 @@ func checkValidExpr(expr string) error { _, _, err := defaultParser.Parse(expr, "", "") return err } + +// YamlForDowngrade returns YAML format represents of config for downgrade. +func (c *TaskConfig) YamlForDowngrade() (string, error) { + t := NewTaskConfigForDowngrade(c) + return t.Yaml() +} + +// MySQLInstanceForDowngrade represents a sync config of a MySQL instance for downgrade. +type MySQLInstanceForDowngrade struct { + SourceID string `yaml:"source-id"` + Meta *Meta `yaml:"meta"` + FilterRules []string `yaml:"filter-rules"` + ColumnMappingRules []string `yaml:"column-mapping-rules"` + RouteRules []string `yaml:"route-rules"` + BWListName string `yaml:"black-white-list"` + BAListName string `yaml:"block-allow-list"` + MydumperConfigName string `yaml:"mydumper-config-name"` + Mydumper *MydumperConfig `yaml:"mydumper"` + MydumperThread int `yaml:"mydumper-thread"` + LoaderConfigName string `yaml:"loader-config-name"` + Loader *LoaderConfig `yaml:"loader"` + LoaderThread int `yaml:"loader-thread"` + SyncerConfigName string `yaml:"syncer-config-name"` + Syncer *SyncerConfig `yaml:"syncer"` + SyncerThread int `yaml:"syncer-thread"` + // new config item + ExpressionFilters []string `yaml:"expression-filters,omitempty"` +} + +// NewMySQLInstancesForDowngrade creates []* MySQLInstanceForDowngrade. +func NewMySQLInstancesForDowngrade(mysqlInstances []*MySQLInstance) []*MySQLInstanceForDowngrade { + mysqlInstancesForDowngrade := make([]*MySQLInstanceForDowngrade, 0, len(mysqlInstances)) + for _, m := range mysqlInstances { + newMySQLInstance := &MySQLInstanceForDowngrade{ + SourceID: m.SourceID, + Meta: m.Meta, + FilterRules: m.FilterRules, + ColumnMappingRules: m.ColumnMappingRules, + RouteRules: m.RouteRules, + BWListName: m.BWListName, + BAListName: m.BAListName, + MydumperConfigName: m.MydumperConfigName, + Mydumper: m.Mydumper, + MydumperThread: m.MydumperThread, + LoaderConfigName: m.LoaderConfigName, + Loader: m.Loader, + LoaderThread: m.LoaderThread, + SyncerConfigName: m.SyncerConfigName, + Syncer: m.Syncer, + SyncerThread: m.SyncerThread, + ExpressionFilters: m.ExpressionFilters, + } + mysqlInstancesForDowngrade = append(mysqlInstancesForDowngrade, newMySQLInstance) + } + return mysqlInstancesForDowngrade +} + +// TaskConfigForDowngrade is the base configuration for task in v2.0. +// This config is used for downgrade(export-config) from a higher dmctl version. +// When we add any new config item into SourceConfig, we should update it also. +type TaskConfigForDowngrade struct { + Name string `yaml:"name"` + TaskMode string `yaml:"task-mode"` + IsSharding bool `yaml:"is-sharding"` + ShardMode string `yaml:"shard-mode"` + IgnoreCheckingItems []string `yaml:"ignore-checking-items"` + MetaSchema string `yaml:"meta-schema"` + EnableHeartbeat bool `yaml:"enable-heartbeat"` + HeartbeatUpdateInterval int `yaml:"heartbeat-update-interval"` + HeartbeatReportInterval int `yaml:"heartbeat-report-interval"` + Timezone string `yaml:"timezone"` + CaseSensitive bool `yaml:"case-sensitive"` + TargetDB *DBConfig `yaml:"target-database"` + OnlineDDLScheme string `yaml:"online-ddl-scheme"` + Routes map[string]*router.TableRule `yaml:"routes"` + Filters map[string]*bf.BinlogEventRule `yaml:"filters"` + ColumnMappings map[string]*column.Rule `yaml:"column-mappings"` + BWList map[string]*filter.Rules `yaml:"black-white-list"` + BAList map[string]*filter.Rules `yaml:"block-allow-list"` + Mydumpers map[string]*MydumperConfig `yaml:"mydumpers"` + Loaders map[string]*LoaderConfig `yaml:"loaders"` + Syncers map[string]*SyncerConfig `yaml:"syncers"` + CleanDumpFile bool `yaml:"clean-dump-file"` + EnableANSIQuotes bool `yaml:"ansi-quotes"` + RemoveMeta bool `yaml:"remove-meta"` + // new config item + MySQLInstances []*MySQLInstanceForDowngrade `yaml:"mysql-instances"` + ExprFilter map[string]*ExpressionFilter `yaml:"expression-filter,omitempty"` +} + +// NewTaskConfigForDowngrade create new TaskConfigForDowngrade. +func NewTaskConfigForDowngrade(taskConfig *TaskConfig) *TaskConfigForDowngrade { + return &TaskConfigForDowngrade{ + Name: taskConfig.Name, + TaskMode: taskConfig.TaskMode, + IsSharding: taskConfig.IsSharding, + ShardMode: taskConfig.ShardMode, + IgnoreCheckingItems: taskConfig.IgnoreCheckingItems, + MetaSchema: taskConfig.MetaSchema, + EnableHeartbeat: taskConfig.EnableHeartbeat, + HeartbeatUpdateInterval: taskConfig.HeartbeatUpdateInterval, + HeartbeatReportInterval: taskConfig.HeartbeatReportInterval, + Timezone: taskConfig.Timezone, + CaseSensitive: taskConfig.CaseSensitive, + TargetDB: taskConfig.TargetDB, + OnlineDDLScheme: taskConfig.OnlineDDLScheme, + Routes: taskConfig.Routes, + Filters: taskConfig.Filters, + ColumnMappings: taskConfig.ColumnMappings, + BWList: taskConfig.BWList, + BAList: taskConfig.BAList, + Mydumpers: taskConfig.Mydumpers, + Loaders: taskConfig.Loaders, + Syncers: taskConfig.Syncers, + CleanDumpFile: taskConfig.CleanDumpFile, + EnableANSIQuotes: taskConfig.EnableANSIQuotes, + RemoveMeta: taskConfig.RemoveMeta, + MySQLInstances: NewMySQLInstancesForDowngrade(taskConfig.MySQLInstances), + ExprFilter: taskConfig.ExprFilter, + } +} + +// omitDefaultVals change default value to empty value for new config item. +// If any default value for new config item is not empty(0 or false or nil), +// we should change it to empty. +func (c *TaskConfigForDowngrade) omitDefaultVals() { + // e.g. + // if c.newConfig == defaultVal { + // c.newConfig == 0 + // } +} + +// Yaml omitDefaultVals and returns YAML format representation of config. +func (c *TaskConfigForDowngrade) Yaml() (string, error) { + // omit default values, so we can ignore them for later marshal + c.omitDefaultVals() + b, err := yaml.Marshal(c) + return string(b), err +} diff --git a/dm/ctl/master/import_export_config.go b/dm/ctl/master/import_export_config.go index 8dc562eae5..bc045a2faf 100644 --- a/dm/ctl/master/import_export_config.go +++ b/dm/ctl/master/import_export_config.go @@ -205,7 +205,7 @@ func writeSourceCfgs(sourceDir string, sourceCfgsMap map[string]*config.SourceCo for source, sourceCfg := range sourceCfgsMap { sourceFile := path.Join(sourceDir, source) sourceFile += yamlSuffix - fileContent, err := sourceCfg.Yaml() + fileContent, err := sourceCfg.YamlForDowngrade() if err != nil { common.PrintLinesf("fail to marshal source config of `%s`", source) return err @@ -241,7 +241,11 @@ func writeTaskCfgs(taskDir string, subTaskCfgsMap map[string]map[string]config.S taskFile := path.Join(taskDir, task) taskFile += yamlSuffix - if err := ioutil.WriteFile(taskFile, []byte(taskCfg.String()), 0o644); err != nil { + taskContent, err := taskCfg.YamlForDowngrade() + if err != nil { + common.PrintLinesf("fail to marshal source config of `%s`", task) + } + if err := ioutil.WriteFile(taskFile, []byte(taskContent), 0o644); err != nil { common.PrintLinesf("can not write task config to file `%s`", taskFile) return err } From 2cc8fcd682a838a29201e7b74253270d9611d4c9 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Fri, 16 Jul 2021 21:07:02 +0800 Subject: [PATCH 06/19] encrypt passwd --- dm/config/source_config.go | 15 ++++++++++++--- dm/config/task.go | 16 +++++++++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/dm/config/source_config.go b/dm/config/source_config.go index d6c7e89398..b10f9f5c35 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -357,6 +357,17 @@ func (c *SourceConfig) check(metaData *toml.MetaData, err error) error { // YamlForDowngrade returns YAML format represents of config for downgrade. func (c *SourceConfig) YamlForDowngrade() (string, error) { s := NewSourceConfigForDowngrade(c) + + // encrypt password + cipher, err := utils.Encrypt(utils.DecryptOrPlaintext(c.From.Password)) + if err != nil { + return "", err + } + s.From.Password = cipher + + // omit default values, so we can ignore them for later marshal + s.omitDefaultVals() + return s.Yaml() } @@ -419,10 +430,8 @@ func (c *SourceConfigForDowngrade) omitDefaultVals() { // } } -// Yaml omitDefaultVals and returns YAML format representation of the config. +// Yaml returns YAML format representation of the config. func (c *SourceConfigForDowngrade) Yaml() (string, error) { - // omit default values, so we can ignore them for later marshal - c.omitDefaultVals() b, err := yaml.Marshal(c) return string(b), err } diff --git a/dm/config/task.go b/dm/config/task.go index 06cb06cfcc..5e8fb47e1d 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -33,6 +33,7 @@ import ( "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/terror" + "github.com/pingcap/dm/pkg/utils" ) // Online DDL Scheme. @@ -939,6 +940,17 @@ func checkValidExpr(expr string) error { // YamlForDowngrade returns YAML format represents of config for downgrade. func (c *TaskConfig) YamlForDowngrade() (string, error) { t := NewTaskConfigForDowngrade(c) + + // encrypt password + cipher, err := utils.Encrypt(utils.DecryptOrPlaintext(t.TargetDB.Password)) + if err != nil { + return "", err + } + t.TargetDB.Password = cipher + + // omit default values, so we can ignore them for later marshal + t.omitDefaultVals() + return t.Yaml() } @@ -1067,10 +1079,8 @@ func (c *TaskConfigForDowngrade) omitDefaultVals() { // } } -// Yaml omitDefaultVals and returns YAML format representation of config. +// Yaml returns YAML format representation of config. func (c *TaskConfigForDowngrade) Yaml() (string, error) { - // omit default values, so we can ignore them for later marshal - c.omitDefaultVals() b, err := yaml.Marshal(c) return string(b), err } From 2f6297448cc4a210dc26b58bc67ff19f9b89ce25 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Fri, 16 Jul 2021 21:16:37 +0800 Subject: [PATCH 07/19] fix it --- tests/dmctl_command/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dmctl_command/run.sh b/tests/dmctl_command/run.sh index 932fad8d0e..3bf5e863cd 100644 --- a/tests/dmctl_command/run.sh +++ b/tests/dmctl_command/run.sh @@ -6,7 +6,7 @@ cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) source $cur/../_utils/test_prepare WORK_DIR=$TEST_DIR/$TEST_NAME -help_cnt=46 +help_cnt=48 function run() { # check dmctl output with help flag From 83d7967e3cdb8f9f459fdf81f56bacd183a69243 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Sun, 18 Jul 2021 20:35:35 +0800 Subject: [PATCH 08/19] rename --- dm/config/source_config.go | 2 +- dm/config/task.go | 2 +- dm/ctl/ctl.go | 4 ++-- dm/ctl/master/import_export_config.go | 26 +++++++++++++------------- pkg/ha/source.go | 2 +- tests/_utils/run_sql | 1 + 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/dm/config/source_config.go b/dm/config/source_config.go index b10f9f5c35..33dd0b4013 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -372,7 +372,7 @@ func (c *SourceConfig) YamlForDowngrade() (string, error) { } // SourceConfigForDowngrade is the base configuration for source in v2.0. -// This config is used for downgrade(export-config) from a higher dmctl version. +// This config is used for downgrade(export-configs) from a higher dmctl version. // When we add any new config item into SourceConfig, we should update it also. type SourceConfigForDowngrade struct { EnableGTID bool `yaml:"enable-gtid"` diff --git a/dm/config/task.go b/dm/config/task.go index 5e8fb47e1d..9b43cfc087 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -1005,7 +1005,7 @@ func NewMySQLInstancesForDowngrade(mysqlInstances []*MySQLInstance) []*MySQLInst } // TaskConfigForDowngrade is the base configuration for task in v2.0. -// This config is used for downgrade(export-config) from a higher dmctl version. +// This config is used for downgrade(export-configs) from a higher dmctl version. // When we add any new config item into SourceConfig, we should update it also. type TaskConfigForDowngrade struct { Name string `yaml:"name"` diff --git a/dm/ctl/ctl.go b/dm/ctl/ctl.go index e7bc7f7eb1..44a1d3ad01 100644 --- a/dm/ctl/ctl.go +++ b/dm/ctl/ctl.go @@ -75,8 +75,8 @@ func NewRootCmd() *cobra.Command { master.NewTransferSourceCmd(), master.NewStartRelayCmd(), master.NewStopRelayCmd(), - master.NewExportCfgCmd(), - master.NewImportCfgCmd(), + master.NewExportCfgsCmd(), + master.NewImportCfgsCmd(), newDecryptCmd(), newEncryptCmd(), ) diff --git a/dm/ctl/master/import_export_config.go b/dm/ctl/master/import_export_config.go index bc045a2faf..1b78d2c8b8 100644 --- a/dm/ctl/master/import_export_config.go +++ b/dm/ctl/master/import_export_config.go @@ -41,30 +41,30 @@ var ( yamlSuffix = ".yaml" ) -// NewExportCfgCmd creates a exportCfg command. -func NewExportCfgCmd() *cobra.Command { +// NewExportCfgsCmd creates a exportCfg command. +func NewExportCfgsCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "export-config [--dir directory]", + Use: "export-configs [--dir directory]", Short: "Export the configurations of sources and tasks.", - RunE: exportCfgFunc, + RunE: exportCfgsFunc, } cmd.Flags().StringP("dir", "d", "configs", "specify the destinary directory, default is `./configs`") return cmd } -// NewImportCfgCmd creates a importCfg command. -func NewImportCfgCmd() *cobra.Command { +// NewImportCfgsCmd creates a importCfg command. +func NewImportCfgsCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "import-config [--dir directory]", + Use: "import-configs [--dir directory]", Short: "Import the configurations of sources and tasks.", - RunE: importCfgFunc, + RunE: importCfgsFunc, } cmd.Flags().StringP("dir", "d", "configs", "specify the configs directory, default is `./configs`") return cmd } -// exportCfgFunc exports configs. -func exportCfgFunc(cmd *cobra.Command, args []string) error { +// exportCfgsFunc exports configs. +func exportCfgsFunc(cmd *cobra.Command, args []string) error { dir, err := cmd.Flags().GetString("dir") if err != nil { common.PrintLinesf("can not get directory") @@ -98,8 +98,8 @@ func exportCfgFunc(cmd *cobra.Command, args []string) error { return nil } -// importCfgFunc imports configs. -func importCfgFunc(cmd *cobra.Command, args []string) error { +// importCfgsFunc imports configs. +func importCfgsFunc(cmd *cobra.Command, args []string) error { dir, err := cmd.Flags().GetString("dir") if err != nil { common.PrintLinesf("can not get directory") @@ -328,7 +328,7 @@ func createSources(ctx context.Context, sourceCfgs []string) error { common.PrintLinesf("start creating sources") sourceResp := &pb.OperateSourceResponse{} - // Do not use batch for `operate-source start source1, source2` if we want to support idemponent import-config. + // Do not use batch for `operate-source start source1, source2` if we want to support idemponent import-configs. // Because `operate-source start` will revert all batch sources if any source error. // e.g. ErrSchedulerSourceCfgExist for _, sourceCfg := range sourceCfgs { diff --git a/pkg/ha/source.go b/pkg/ha/source.go index db0eeb1768..d83ba0aaa3 100644 --- a/pkg/ha/source.go +++ b/pkg/ha/source.go @@ -38,7 +38,7 @@ func PutSourceCfg(cli *clientv3.Client, cfg *config.SourceConfig) (int64, error) } // GetAllSourceCfgBeforeV202 gets all upstream source configs before v2.0.2. -// This func only use for export-config command. +// This func only use for export-configs command. func GetAllSourceCfgBeforeV202(cli *clientv3.Client) (map[string]*config.SourceConfig, int64, error) { ctx, cancel := context.WithTimeout(cli.Ctx(), etcdutil.DefaultRequestTimeout) defer cancel() diff --git a/tests/_utils/run_sql b/tests/_utils/run_sql index 4674e4b945..ed5609dc41 100755 --- a/tests/_utils/run_sql +++ b/tests/_utils/run_sql @@ -5,6 +5,7 @@ set -eu +TIDB_PORT=4000 user="root" if [[ "$2" = $TIDB_PORT ]]; then user="test" From a6994ccdb18d0da08bfb4d52e465e0745a09661f Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Sun, 18 Jul 2021 21:49:30 +0800 Subject: [PATCH 09/19] add it --- tests/new_relay/conf/source1.yaml | 1 + tests/new_relay/configs/relay_workers.json | 1 + .../configs/sources/mysql-replica-01.yaml | 32 ++++++++ tests/new_relay/configs/tasks/test.yaml | 79 +++++++++++++++++++ tests/new_relay/data/db1.increment5.sql | 2 + tests/new_relay/run.sh | 43 ++++++++++ 6 files changed, 158 insertions(+) create mode 100644 tests/new_relay/configs/relay_workers.json create mode 100644 tests/new_relay/configs/sources/mysql-replica-01.yaml create mode 100644 tests/new_relay/configs/tasks/test.yaml create mode 100644 tests/new_relay/data/db1.increment5.sql diff --git a/tests/new_relay/conf/source1.yaml b/tests/new_relay/conf/source1.yaml index 406e7ae254..96e7c670da 100644 --- a/tests/new_relay/conf/source1.yaml +++ b/tests/new_relay/conf/source1.yaml @@ -1,4 +1,5 @@ source-id: mysql-replica-01 +server-id: 123456 flavor: 'mysql' enable-gtid: true relay-binlog-name: '' diff --git a/tests/new_relay/configs/relay_workers.json b/tests/new_relay/configs/relay_workers.json new file mode 100644 index 0000000000..ae362cafea --- /dev/null +++ b/tests/new_relay/configs/relay_workers.json @@ -0,0 +1 @@ +{"mysql-replica-01":["worker1","worker2"]} \ No newline at end of file diff --git a/tests/new_relay/configs/sources/mysql-replica-01.yaml b/tests/new_relay/configs/sources/mysql-replica-01.yaml new file mode 100644 index 0000000000..1597ba0140 --- /dev/null +++ b/tests/new_relay/configs/sources/mysql-replica-01.yaml @@ -0,0 +1,32 @@ +enable-gtid: true +auto-fix-gtid: false +relay-dir: relay-dir +meta-dir: "" +flavor: mysql +charset: "" +enable-relay: true +relay-binlog-name: "" +relay-binlog-gtid: "" +source-id: mysql-replica-01 +from: + host: 127.0.0.1 + port: 3306 + user: root + max-allowed-packet: null + session: + time_zone: "+00:00" + security: null +purge: + interval: 3600 + expires: 0 + remain-space: 15 +checker: + check-enable: false + backoff-rollback: 5m0s + backoff-max: 5m0s + check-interval: 5s + backoff-min: 1s + backoff-jitter: true + backoff-factor: 2 +server-id: 123456 +tracer: {} diff --git a/tests/new_relay/configs/tasks/test.yaml b/tests/new_relay/configs/tasks/test.yaml new file mode 100644 index 0000000000..5e3589b472 --- /dev/null +++ b/tests/new_relay/configs/tasks/test.yaml @@ -0,0 +1,79 @@ +name: test +task-mode: all +is-sharding: false +shard-mode: "" +ignore-checking-items: [] +meta-schema: dm_meta +enable-heartbeat: false +heartbeat-update-interval: 1 +heartbeat-report-interval: 1 +timezone: "" +case-sensitive: false +target-database: + host: 127.0.0.1 + port: 4111 + user: root + max-allowed-packet: null + session: + tidb_txn_mode: optimistic + time_zone: "+00:00" + security: null +online-ddl-scheme: "" +routes: {} +filters: {} +column-mappings: {} +black-white-list: {} +block-allow-list: + balist-01: + do-tables: [] + do-dbs: + - new_relay + ignore-tables: [] + ignore-dbs: [] +mydumpers: + dump-01: + mydumper-path: ./bin/mydumper + threads: 4 + chunk-filesize: "64" + statement-size: 0 + rows: 0 + where: "" + skip-tz-utc: true + extra-args: "" +loaders: + load-01: + pool-size: 16 + dir: ./dumped_data.test +syncers: + sync-01: + meta-file: "" + worker-count: 16 + batch: 100 + queue-size: 1024 + checkpoint-flush-interval: 30 + max-retry: 0 + auto-fix-gtid: false + enable-gtid: false + disable-detect: false + safe-mode: false + enable-ansi-quotes: false +clean-dump-file: false +ansi-quotes: false +remove-meta: false +mysql-instances: +- source-id: mysql-replica-01 + meta: null + filter-rules: [] + column-mapping-rules: [] + route-rules: [] + black-white-list: "" + block-allow-list: balist-01 + mydumper-config-name: dump-01 + mydumper: null + mydumper-thread: 0 + loader-config-name: load-01 + loader: null + loader-thread: 0 + syncer-config-name: sync-01 + syncer: null + syncer-thread: 0 diff --git a/tests/new_relay/data/db1.increment5.sql b/tests/new_relay/data/db1.increment5.sql new file mode 100644 index 0000000000..6cffd32080 --- /dev/null +++ b/tests/new_relay/data/db1.increment5.sql @@ -0,0 +1,2 @@ +alter table new_relay.t1 add column new_col1 int, add column new_col2 int; +insert into new_relay.t1 (id, name, info, new_col1, new_col2) values (13, 'gentest', '{"id": 133}', 13, 13), (14, 'gentest', '{"id": 134}', 14, 14); diff --git a/tests/new_relay/run.sh b/tests/new_relay/run.sh index 927be18385..8d10522d5d 100755 --- a/tests/new_relay/run.sh +++ b/tests/new_relay/run.sh @@ -116,6 +116,49 @@ function run() { run_sql_file $cur/data/db1.increment4.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml + + # export-configs + run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "export-configs -d /tmp/configs" \ + "export configs to directory .* succeed" 1 + + # check configs + sed '/password/d' /tmp/configs/tasks/test.yaml | diff $cur/configs/tasks/test.yaml - || exit 1 + sed '/password/d' /tmp/configs/sources/mysql-replica-01.yaml | diff $cur/configs/sources/mysql-replica-01.yaml - || exit 1 + diff /tmp/configs/relay_workers.json $cur/configs/relay_workers.json || exit 1 + + # destroy cluster + cleanup_process $* + rm -rf $WORK_DIR + mkdir $WORK_DIR + + # insert new data + run_sql_file $cur/data/db1.increment5.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 + + # deploy new cluster + run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml + check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT + run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT + run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT + + # import configs + run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "import-configs -d /tmp/configs" \ + "creating sources" 1 \ + "creating tasks" 1 \ + "The original relay workers have been exported to" 1 \ + "Currently unsupport recover relay workers.*transfer-source.*start-relay" 1 + + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "operate-source show" \ + "mysql-replica-01" 1 + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status -s $SOURCE_ID1" \ + "\"result\": true" 2 + + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml } cleanup_data $TEST_NAME From 3183886b7d09a4d5336e734576d5dec25869d508 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 19 Jul 2021 15:42:30 +0800 Subject: [PATCH 10/19] add downgrade test --- tests/tiup/conf/source1.yaml | 3 +- tests/tiup/conf/source2.yaml | 3 +- tests/tiup/lib.sh | 46 ++++++++++++++++++++++-- tests/tiup/upgrade-from-v2.sh | 66 ++++++++++++++++++++++++++++++++--- 4 files changed, 110 insertions(+), 8 deletions(-) diff --git a/tests/tiup/conf/source1.yaml b/tests/tiup/conf/source1.yaml index c49cc78ae2..484bbdb662 100644 --- a/tests/tiup/conf/source1.yaml +++ b/tests/tiup/conf/source1.yaml @@ -1,9 +1,10 @@ source-id: mysql-replica-01 enable-gtid: false enable-relay: false +server-id: 123456 from: host: mysql1 user: root port: 3306 checker: - check-enable: false + check-enable: true diff --git a/tests/tiup/conf/source2.yaml b/tests/tiup/conf/source2.yaml index 30903199f5..bcbe1bd306 100644 --- a/tests/tiup/conf/source2.yaml +++ b/tests/tiup/conf/source2.yaml @@ -1,9 +1,10 @@ source-id: mariadb-replica-02 enable-gtid: true enable-relay: true +server-id: 654321 from: host: mariadb2 user: root port: 3306 checker: - check-enable: false + check-enable: true diff --git a/tests/tiup/lib.sh b/tests/tiup/lib.sh index 1f5fca6d17..8dd627e1e4 100755 --- a/tests/tiup/lib.sh +++ b/tests/tiup/lib.sh @@ -172,11 +172,52 @@ function exec_incremental_stage2() { exec_sql mariadb2 3306 "INSERT INTO $DB6.$TBL3 (c1, c2, c3) VALUES (214, '214', 214);" } +function exec_incremental_stage3() { + # prepare incremental data + exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL1 (c1, c2) VALUES (301, '301');" + exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL2 (c1, c2) VALUES (302, '302');" + exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (311, '311');" + exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (312, '312');" + + # prepare optimistic incremental data + exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2, c3, c4) VALUES (301, '301', 301, 301);" + exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c3, c4) VALUES (302, '302', 302, 302);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (311, '311', 311, 311);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c3, c4) VALUES (312, '312', 312, 312);" + + # prepare pessimistic incremental data + exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2, c3) VALUES (303, '303', 303);" + exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL2 (c1, c2, c3) VALUES (304, '304', 304);" + exec_sql mariadb2 3306 "INSERT INTO $DB6.$TBL2 (c1, c2, c3) VALUES (313, '313', 313);" + exec_sql mariadb2 3306 "INSERT INTO $DB6.$TBL3 (c1, c2, c3) VALUES (314, '314', 314);" +} + +function exec_incremental_stage4() { + # prepare incremental data + exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL1 (c1, c2) VALUES (401, '401');" + exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL2 (c1, c2) VALUES (402, '402');" + exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (411, '411');" + exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (412, '412');" + + # prepare optimistic incremental data + exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2, c3, c4) VALUES (401, '401', 401, 401);" + exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c3, c4) VALUES (402, '402', 402, 402);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (411, '411', 411, 411);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c3, c4) VALUES (412, '412', 412, 412);" + + # prepare pessimistic incremental data + exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2, c3) VALUES (403, '403', 403);" + exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL2 (c1, c2, c3) VALUES (404, '404', 404);" + exec_sql mariadb2 3306 "INSERT INTO $DB6.$TBL2 (c1, c2, c3) VALUES (413, '413', 413);" + exec_sql mariadb2 3306 "INSERT INTO $DB6.$TBL3 (c1, c2, c3) VALUES (414, '414', 414);" +} + function patch_nightly_with_tiup_mirror() { # clone packages for upgrade. # FIXME: use nightly version of grafana and prometheus after https://github.com/pingcap/tiup/issues/1334 fixed. tiup mirror clone tidb-dm-nightly-linux-amd64 --os=linux --arch=amd64 \ --alertmanager=v0.17.0 --grafana=v5.0.1 --prometheus=v5.0.1 \ + --dm-master=v2.0.1 --dm-worker=v2.0.1 \ --tiup=v$(tiup --version | grep 'tiup' | awk -F ' ' '{print $1}') --dm=v$(tiup --version | grep 'tiup' | awk -F ' ' '{print $1}') # change tiup mirror @@ -186,8 +227,9 @@ function patch_nightly_with_tiup_mirror() { # binary files have already been built and packaged. tiup mirror genkey tiup mirror grant gmhdbjd --name gmhdbjd - tiup mirror publish dm-master nightly /tmp/dm-master-nightly-linux-amd64.tar.gz dm-master/dm-master --arch amd64 --os linux --desc="dm-master component of Data Migration Platform" - tiup mirror publish dm-worker nightly /tmp/dm-worker-nightly-linux-amd64.tar.gz dm-worker/dm-worker --arch amd64 --os linux --desc="dm-worker component of Data Migration Platform" + mv tidb-dm-nightly-linux-amd64/keys/*-pingcap.json ./pingcap.json + tiup mirror publish dm-master nightly /tmp/dm-master-nightly-linux-amd64.tar.gz dm-master/dm-master --arch amd64 --os linux --desc="dm-master component of Data Migration Platform" -k pingcap.json + tiup mirror publish dm-worker nightly /tmp/dm-worker-nightly-linux-amd64.tar.gz dm-worker/dm-worker --arch amd64 --os linux --desc="dm-worker component of Data Migration Platform" -k pingcap.json tiup mirror publish dmctl nightly /tmp/dmctl-nightly-linux-amd64.tar.gz dmctl/dmctl --arch amd64 --os linux --desc="dmctl component of Data Migration Platform" tiup list diff --git a/tests/tiup/upgrade-from-v2.sh b/tests/tiup/upgrade-from-v2.sh index b0e595d8c8..995ef32b8a 100755 --- a/tests/tiup/upgrade-from-v2.sh +++ b/tests/tiup/upgrade-from-v2.sh @@ -33,6 +33,10 @@ function migrate_in_previous_v2() { sed -i "s/enable-gtid: true/enable-gtid: false/g" $CUR/conf/source2.yaml fi + sed -i "s/enable-heartbeat: true/enable-heartbeat: false/g" $CUR/conf/task.yaml + sed -i "s/enable-heartbeat: true/enable-heartbeat: false/g" $CUR/conf/task_pessimistic.yaml + sed -i "s/enable-heartbeat: true/enable-heartbeat: false/g" $CUR/conf/task_optimistic.yaml + tiup dmctl:$PRE_VER --master-addr=master1:8261 operate-source create $CUR/conf/source1.yaml tiup dmctl:$PRE_VER --master-addr=master1:8261 operate-source create $CUR/conf/source2.yaml @@ -51,12 +55,17 @@ function migrate_in_previous_v2() { function upgrade_to_current_v2() { if [[ "$CUR_VER" == "nightly" && "$ref" == "refs/pull"* ]]; then - patch_nightly_with_tiup_mirror + patch_nightly_with_tiup_mirror $PRE_VER fi - tiup dm upgrade --yes $CLUSTER_NAME $CUR_VER + # uninstall previous dmctl, otherwise dmctl:nightly still use PRE_VER. # FIXME: It may be a bug in tiup mirror. tiup uninstall dmctl --all + + # export-configs in PRE_VER + tiup dmctl:$CUR_VER --master-addr=master1:8261 export-configs -d old_configs + + tiup dm upgrade --yes $CLUSTER_NAME $CUR_VER } function migrate_in_v2() { @@ -81,11 +90,56 @@ function migrate_in_v2() { echo "check locks" run_dmctl_with_retry $CUR_VER "show-ddl-locks" "no DDL lock exists" 1 - export DM_MASTER_ADDR="master1:8261" - tiup dmctl:$CUR_VER stop-task $TASK_NAME + # export-configs in CUR_VER + tiup dmctl:$CUR_VER --master-addr=master1:8261 export-configs -d new_configs +} + +function diff_configs() { + sed '/password/d' old_configs/tasks/upgrade_via_tiup.yaml >/tmp/old_task.yaml + sed '/password/d' old_configs/tasks/upgrade_via_tiup_pessimistic.yaml >/tmp/old_task_pessimistic.yaml + sed '/password/d' old_configs/tasks/upgrade_via_tiup_optimistic.yaml >/tmp/old_task_optimistic.yaml + sed '/password/d' new_configs/tasks/upgrade_via_tiup.yaml >/tmp/new_task.yaml + sed '/password/d' new_configs/tasks/upgrade_via_tiup_pessimistic.yaml >/tmp/new_task_pessimistic.yaml + sed '/password/d' new_configs/tasks/upgrade_via_tiup_optimistic.yaml >/tmp/new_task_optimistic.yaml + + sed '/password/d' old_configs/sources/mysql-replica-01.yaml >/tmp/old_source1.yaml + sed '/password/d' old_configs/sources/mariadb-replica-02.yaml >/tmp/old_source2.yaml + sed '/password/d' new_configs/sources/mysql-replica-01.yaml >/tmp/new_source1.yaml + sed '/password/d' new_configs/sources/mariadb-replica-02.yaml >/tmp/new_source2.yaml + + diff /tmp/old_task.yaml /tmp/new_task.yaml || exit 1 + diff /tmp/old_task_pessimistic.yaml /tmp/new_task_pessimistic.yaml || exit 1 + diff /tmp/old_task_optimistic.yaml /tmp/new_task_optimistic.yaml || exit 1 + diff /tmp/old_source1.yaml /tmp/new_source1.yaml || exit 1 + diff /tmp/old_source2.yaml /tmp/new_source2.yaml || exit 1 +} + +function downgrade_to_previous_v2() { + # destory current cluster + tiup dm destroy --yes $CLUSTER_NAME + + exec_incremental_stage3 + + # deploy previous cluster + tiup dm deploy --yes $CLUSTER_NAME $PRE_VER $CUR/conf/topo.yaml + tiup dm start --yes $CLUSTER_NAME + + # import-configs + tiup dmctl:$CUR_VER --master-addr=master1:8261 import-configs -d new_configs + + exec_incremental_stage4 + + run_dmctl_with_retry $CUR_VER "operate-source show" "mysql-replica-01" 1 "mariadb-replica-02" 1 + run_dmctl_with_retry $CUR_VER "list-member --worker" "\"stage\": \"bound\"" 2 + + check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml + check_sync_diff $WORK_DIR $CUR/conf/diff_config_optimistic.toml + check_sync_diff $WORK_DIR $CUR/conf/diff_config_pessimistic.toml } function destroy_v2_by_tiup() { + export DM_MASTER_ADDR="master1:8261" + tiup dmctl:$CUR_VER stop-task $TASK_NAME tiup dm destroy --yes $CLUSTER_NAME } @@ -100,6 +154,10 @@ function test() { migrate_in_v2 + diff_configs + + downgrade_to_previous_v2 + destroy_v2_by_tiup } From 22c162c98d08b5bd0eab66d69a622fa00933ebae Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 19 Jul 2021 16:01:39 +0800 Subject: [PATCH 11/19] fix test --- tests/new_relay/configs/tasks/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/new_relay/configs/tasks/test.yaml b/tests/new_relay/configs/tasks/test.yaml index 5e3589b472..a1e5183c3d 100644 --- a/tests/new_relay/configs/tasks/test.yaml +++ b/tests/new_relay/configs/tasks/test.yaml @@ -11,7 +11,7 @@ timezone: "" case-sensitive: false target-database: host: 127.0.0.1 - port: 4111 + port: 4000 user: root max-allowed-packet: null session: From 7e224db3d4790adc03e8f2ccf352667a18db458b Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 20 Jul 2021 15:48:31 +0800 Subject: [PATCH 12/19] use sub command --- dm/config/source_config.go | 2 +- dm/config/task.go | 2 +- dm/ctl/ctl.go | 3 +-- .../{import_export_config.go => config.go} | 26 ++++++++++++++----- pkg/ha/source.go | 2 +- tests/new_relay/run.sh | 8 +++--- tests/tiup/upgrade-from-v2.sh | 12 ++++----- 7 files changed, 33 insertions(+), 22 deletions(-) rename dm/ctl/master/{import_export_config.go => config.go} (94%) diff --git a/dm/config/source_config.go b/dm/config/source_config.go index 33dd0b4013..7dfb874267 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -372,7 +372,7 @@ func (c *SourceConfig) YamlForDowngrade() (string, error) { } // SourceConfigForDowngrade is the base configuration for source in v2.0. -// This config is used for downgrade(export-configs) from a higher dmctl version. +// This config is used for downgrade(config export) from a higher dmctl version. // When we add any new config item into SourceConfig, we should update it also. type SourceConfigForDowngrade struct { EnableGTID bool `yaml:"enable-gtid"` diff --git a/dm/config/task.go b/dm/config/task.go index 9b43cfc087..7570d2130f 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -1005,7 +1005,7 @@ func NewMySQLInstancesForDowngrade(mysqlInstances []*MySQLInstance) []*MySQLInst } // TaskConfigForDowngrade is the base configuration for task in v2.0. -// This config is used for downgrade(export-configs) from a higher dmctl version. +// This config is used for downgrade(config export) from a higher dmctl version. // When we add any new config item into SourceConfig, we should update it also. type TaskConfigForDowngrade struct { Name string `yaml:"name"` diff --git a/dm/ctl/ctl.go b/dm/ctl/ctl.go index 44a1d3ad01..0f94229335 100644 --- a/dm/ctl/ctl.go +++ b/dm/ctl/ctl.go @@ -75,8 +75,7 @@ func NewRootCmd() *cobra.Command { master.NewTransferSourceCmd(), master.NewStartRelayCmd(), master.NewStopRelayCmd(), - master.NewExportCfgsCmd(), - master.NewImportCfgsCmd(), + master.NewConfigCmd(), newDecryptCmd(), newEncryptCmd(), ) diff --git a/dm/ctl/master/import_export_config.go b/dm/ctl/master/config.go similarity index 94% rename from dm/ctl/master/import_export_config.go rename to dm/ctl/master/config.go index 1b78d2c8b8..92ad6daa5e 100644 --- a/dm/ctl/master/import_export_config.go +++ b/dm/ctl/master/config.go @@ -41,10 +41,21 @@ var ( yamlSuffix = ".yaml" ) -// NewExportCfgsCmd creates a exportCfg command. -func NewExportCfgsCmd() *cobra.Command { +// NewConfigCmd creates a exportCfg command. +func NewConfigCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "export-configs [--dir directory]", + Use: "config", + Short: "Commands to import/export config", + } + cmd.AddCommand(newExportCfgsCmd()) + cmd.AddCommand(newImportCfgsCmd()) + return cmd +} + +// newExportCfgsCmd creates a exportCfg command. +func newExportCfgsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "export [--dir directory]", Short: "Export the configurations of sources and tasks.", RunE: exportCfgsFunc, } @@ -52,10 +63,10 @@ func NewExportCfgsCmd() *cobra.Command { return cmd } -// NewImportCfgsCmd creates a importCfg command. -func NewImportCfgsCmd() *cobra.Command { +// newImportCfgsCmd creates a importCfg command. +func newImportCfgsCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "import-configs [--dir directory]", + Use: "import [--dir directory]", Short: "Import the configurations of sources and tasks.", RunE: importCfgsFunc, } @@ -265,6 +276,7 @@ func writeRelayWorkers(relayWorkersFile string, relayWorkersSet map[string]map[s for worker := range workerSet { workers = append(workers, worker) } + sort.Strings(workers) relayWorkers[source] = workers } @@ -328,7 +340,7 @@ func createSources(ctx context.Context, sourceCfgs []string) error { common.PrintLinesf("start creating sources") sourceResp := &pb.OperateSourceResponse{} - // Do not use batch for `operate-source start source1, source2` if we want to support idemponent import-configs. + // Do not use batch for `operate-source start source1, source2` if we want to support idemponent config import. // Because `operate-source start` will revert all batch sources if any source error. // e.g. ErrSchedulerSourceCfgExist for _, sourceCfg := range sourceCfgs { diff --git a/pkg/ha/source.go b/pkg/ha/source.go index d83ba0aaa3..89f31c2b5a 100644 --- a/pkg/ha/source.go +++ b/pkg/ha/source.go @@ -38,7 +38,7 @@ func PutSourceCfg(cli *clientv3.Client, cfg *config.SourceConfig) (int64, error) } // GetAllSourceCfgBeforeV202 gets all upstream source configs before v2.0.2. -// This func only use for export-configs command. +// This func only use for config export command. func GetAllSourceCfgBeforeV202(cli *clientv3.Client) (map[string]*config.SourceConfig, int64, error) { ctx, cancel := context.WithTimeout(cli.Ctx(), etcdutil.DefaultRequestTimeout) defer cancel() diff --git a/tests/new_relay/run.sh b/tests/new_relay/run.sh index 8d10522d5d..79270a262e 100755 --- a/tests/new_relay/run.sh +++ b/tests/new_relay/run.sh @@ -117,15 +117,15 @@ function run() { run_sql_file $cur/data/db1.increment4.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml - # export-configs + # config export run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ - "export-configs -d /tmp/configs" \ + "config export -d /tmp/configs" \ "export configs to directory .* succeed" 1 # check configs sed '/password/d' /tmp/configs/tasks/test.yaml | diff $cur/configs/tasks/test.yaml - || exit 1 sed '/password/d' /tmp/configs/sources/mysql-replica-01.yaml | diff $cur/configs/sources/mysql-replica-01.yaml - || exit 1 - diff /tmp/configs/relay_workers.json $cur/configs/relay_workers.json || exit 1 + diff <(jq --sort-keys . /tmp/configs/relay_workers.json) <(jq --sort-keys . $cur/configs/relay_workers.json) || exit 1 # destroy cluster cleanup_process $* @@ -145,7 +145,7 @@ function run() { # import configs run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ - "import-configs -d /tmp/configs" \ + "config import -d /tmp/configs" \ "creating sources" 1 \ "creating tasks" 1 \ "The original relay workers have been exported to" 1 \ diff --git a/tests/tiup/upgrade-from-v2.sh b/tests/tiup/upgrade-from-v2.sh index 995ef32b8a..ffb68d3297 100755 --- a/tests/tiup/upgrade-from-v2.sh +++ b/tests/tiup/upgrade-from-v2.sh @@ -62,8 +62,8 @@ function upgrade_to_current_v2() { # FIXME: It may be a bug in tiup mirror. tiup uninstall dmctl --all - # export-configs in PRE_VER - tiup dmctl:$CUR_VER --master-addr=master1:8261 export-configs -d old_configs + # config export in PRE_VER + tiup dmctl:$CUR_VER --master-addr=master1:8261 config export -d old_configs tiup dm upgrade --yes $CLUSTER_NAME $CUR_VER } @@ -90,8 +90,8 @@ function migrate_in_v2() { echo "check locks" run_dmctl_with_retry $CUR_VER "show-ddl-locks" "no DDL lock exists" 1 - # export-configs in CUR_VER - tiup dmctl:$CUR_VER --master-addr=master1:8261 export-configs -d new_configs + # config export in CUR_VER + tiup dmctl:$CUR_VER --master-addr=master1:8261 config export -d new_configs } function diff_configs() { @@ -124,8 +124,8 @@ function downgrade_to_previous_v2() { tiup dm deploy --yes $CLUSTER_NAME $PRE_VER $CUR/conf/topo.yaml tiup dm start --yes $CLUSTER_NAME - # import-configs - tiup dmctl:$CUR_VER --master-addr=master1:8261 import-configs -d new_configs + # config import + tiup dmctl:$CUR_VER --master-addr=master1:8261 config import -d new_configs exec_incremental_stage4 From edf8937df9e106dfe8173fc0ee8235ab2a44732e Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 20 Jul 2021 15:53:26 +0800 Subject: [PATCH 13/19] address comment --- dm/config/source_config_test.go | 9 +++++++++ dm/config/task_test.go | 9 +++++++++ dm/ctl/master/config.go | 2 +- tests/new_relay/run.sh | 2 +- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/dm/config/source_config_test.go b/dm/config/source_config_test.go index 3aaddb3653..42b95a2545 100644 --- a/dm/config/source_config_test.go +++ b/dm/config/source_config_test.go @@ -19,6 +19,7 @@ import ( "fmt" "io/ioutil" "path" + "reflect" "strings" "time" @@ -235,6 +236,14 @@ func (t *testConfig) TestConfigVerify(c *C) { } } +func (t *testConfig) TestSourceConfigForDowngrade(c *C) { + cfg := newSourceConfig() + cfgForDowngrade := NewSourceConfigForDowngrade(cfg) + cfgReflect := reflect.Indirect(reflect.ValueOf(cfg)) + cfgForDowngradeReflect := reflect.Indirect(reflect.ValueOf(cfgForDowngrade)) + c.Assert(cfgReflect.NumField(), Equals, cfgForDowngradeReflect.NumField()) +} + func subtestFlavor(c *C, cfg *SourceConfig, sqlInfo, expectedFlavor, expectedError string) { cfg.Flavor = "" db, mock, err := sqlmock.New() diff --git a/dm/config/task_test.go b/dm/config/task_test.go index dd817bb3a6..be7cbe1573 100644 --- a/dm/config/task_test.go +++ b/dm/config/task_test.go @@ -16,6 +16,7 @@ package config import ( "io/ioutil" "path" + "reflect" "sort" "strings" @@ -1011,3 +1012,11 @@ func (t *testConfig) TestExclusiveAndWrongExprFilterFields(c *C) { err = cfg.adjust() c.Assert(terror.ErrConfigExprFilterWrongGrammar.Equal(err), IsTrue) } + +func (t *testConfig) TestTaskConfigForDowngrade(c *C) { + cfg := NewTaskConfig() + cfgForDowngrade := NewTaskConfigForDowngrade(cfg) + cfgReflect := reflect.Indirect(reflect.ValueOf(cfg)) + cfgForDowngradeReflect := reflect.Indirect(reflect.ValueOf(cfgForDowngrade)) + c.Assert(cfgReflect.NumField(), Equals, cfgForDowngradeReflect.NumField()+1) // without flag +} diff --git a/dm/ctl/master/config.go b/dm/ctl/master/config.go index 92ad6daa5e..27a1aad52d 100644 --- a/dm/ctl/master/config.go +++ b/dm/ctl/master/config.go @@ -132,7 +132,7 @@ func importCfgsFunc(cmd *cobra.Command, args []string) error { } if len(relayWorkers) > 0 { common.PrintLinesf("The original relay workers have been exported to `%s`.", path.Join(dir, relayWorkersFilename)) - common.PrintLinesf("Currently unsupport recover relay workers. You may need to execute `transfer-source` and `start-relay` command manually.") + common.PrintLinesf("Currently DM doesn't support recover relay workers. You may need to execute `transfer-source` and `start-relay` command manually.") } common.PrintLinesf("import configs from directory `%s` succeed", dir) diff --git a/tests/new_relay/run.sh b/tests/new_relay/run.sh index 79270a262e..16d2e7331b 100755 --- a/tests/new_relay/run.sh +++ b/tests/new_relay/run.sh @@ -149,7 +149,7 @@ function run() { "creating sources" 1 \ "creating tasks" 1 \ "The original relay workers have been exported to" 1 \ - "Currently unsupport recover relay workers.*transfer-source.*start-relay" 1 + "Currently DM doesn't support recover relay workers.*transfer-source.*start-relay" 1 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "operate-source show" \ From d80fdd26352538b37284ecbedfb07a0d937a35ad Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 20 Jul 2021 17:08:59 +0800 Subject: [PATCH 14/19] fix test --- tests/tiup/lib.sh | 8 ++++---- tests/tiup/upgrade-from-v2.sh | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/tiup/lib.sh b/tests/tiup/lib.sh index 8dd627e1e4..d4fe3299da 100755 --- a/tests/tiup/lib.sh +++ b/tests/tiup/lib.sh @@ -180,10 +180,10 @@ function exec_incremental_stage3() { exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (312, '312');" # prepare optimistic incremental data - exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2, c3, c4) VALUES (301, '301', 301, 301);" + exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL_LOWER1 (c1, c2, c3, c4) VALUES (301, '301', 301, 301);" exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c3, c4) VALUES (302, '302', 302, 302);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (311, '311', 311, 311);" - exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c3, c4) VALUES (312, '312', 312, 312);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2, c3, c4) VALUES (312, '312', 312, 312);" # prepare pessimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2, c3) VALUES (303, '303', 303);" @@ -200,10 +200,10 @@ function exec_incremental_stage4() { exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (412, '412');" # prepare optimistic incremental data - exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2, c3, c4) VALUES (401, '401', 401, 401);" + exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL_LOWER1 (c1, c2, c3, c4) VALUES (401, '401', 401, 401);" exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c3, c4) VALUES (402, '402', 402, 402);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (411, '411', 411, 411);" - exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c3, c4) VALUES (412, '412', 412, 412);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2, c3, c4) VALUES (412, '412', 412, 412);" # prepare pessimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2, c3) VALUES (403, '403', 403);" diff --git a/tests/tiup/upgrade-from-v2.sh b/tests/tiup/upgrade-from-v2.sh index ffb68d3297..99b4a426c3 100755 --- a/tests/tiup/upgrade-from-v2.sh +++ b/tests/tiup/upgrade-from-v2.sh @@ -95,6 +95,8 @@ function migrate_in_v2() { } function diff_configs() { + echo "diff configs between different version" + sed '/password/d' old_configs/tasks/upgrade_via_tiup.yaml >/tmp/old_task.yaml sed '/password/d' old_configs/tasks/upgrade_via_tiup_pessimistic.yaml >/tmp/old_task_pessimistic.yaml sed '/password/d' old_configs/tasks/upgrade_via_tiup_optimistic.yaml >/tmp/old_task_optimistic.yaml @@ -115,6 +117,8 @@ function diff_configs() { } function downgrade_to_previous_v2() { + echo "downgrade to previous version $PRE_VER" + # destory current cluster tiup dm destroy --yes $CLUSTER_NAME From c1620bde1c2734bcc5355d7cac5e2f26c30800f0 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 20 Jul 2021 17:41:44 +0800 Subject: [PATCH 15/19] fix test --- tests/dmctl_command/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dmctl_command/run.sh b/tests/dmctl_command/run.sh index 3bf5e863cd..0b08a1bb20 100644 --- a/tests/dmctl_command/run.sh +++ b/tests/dmctl_command/run.sh @@ -6,7 +6,7 @@ cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) source $cur/../_utils/test_prepare WORK_DIR=$TEST_DIR/$TEST_NAME -help_cnt=48 +help_cnt=47 function run() { # check dmctl output with help flag From 5d86646cc2f10e8a270b63597880aef266d57501 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 22 Jul 2021 21:16:53 +0800 Subject: [PATCH 16/19] address comment --- dm/config/source_config.go | 9 +++++---- dm/config/task.go | 9 +++++---- dm/ctl/master/config.go | 16 ++++------------ .../configs/sources/mysql-replica-01.yaml | 3 +-- tests/new_relay/configs/tasks/test.yaml | 1 - tests/tiup/lib.sh | 2 +- 6 files changed, 16 insertions(+), 24 deletions(-) diff --git a/dm/config/source_config.go b/dm/config/source_config.go index 7dfb874267..e9063c8bed 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -424,10 +424,11 @@ func NewSourceConfigForDowngrade(sourceCfg *SourceConfig) *SourceConfigForDowngr // If any default value for new config item is not empty(0 or false or nil), // we should change it to empty. func (c *SourceConfigForDowngrade) omitDefaultVals() { - // e.g. - // if s.newConfig == defaultVal { - // s.newConfig == 0 - // } + if len(c.From.Session) > 0 { + if timeZone, ok := c.From.Session["time_zone"]; ok && timeZone == defaultTimeZone { + delete(c.From.Session, "time_zone") + } + } } // Yaml returns YAML format representation of the config. diff --git a/dm/config/task.go b/dm/config/task.go index 7570d2130f..bd8c85f6d6 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -1073,10 +1073,11 @@ func NewTaskConfigForDowngrade(taskConfig *TaskConfig) *TaskConfigForDowngrade { // If any default value for new config item is not empty(0 or false or nil), // we should change it to empty. func (c *TaskConfigForDowngrade) omitDefaultVals() { - // e.g. - // if c.newConfig == defaultVal { - // c.newConfig == 0 - // } + if len(c.TargetDB.Session) > 0 { + if timeZone, ok := c.TargetDB.Session["time_zone"]; ok && timeZone == defaultTimeZone { + delete(c.TargetDB.Session, "time_zone") + } + } } // Yaml returns YAML format representation of config. diff --git a/dm/ctl/master/config.go b/dm/ctl/master/config.go index 27a1aad52d..5a9931567c 100644 --- a/dm/ctl/master/config.go +++ b/dm/ctl/master/config.go @@ -59,7 +59,7 @@ func newExportCfgsCmd() *cobra.Command { Short: "Export the configurations of sources and tasks.", RunE: exportCfgsFunc, } - cmd.Flags().StringP("dir", "d", "configs", "specify the destinary directory, default is `./configs`") + cmd.Flags().StringP("dir", "d", "configs", "specify the output directory, default is `./configs`") return cmd } @@ -195,10 +195,6 @@ func getAllCfgs(cli *clientv3.Client) (map[string]*config.SourceConfig, map[stri } func createDirectory(dir string) (string, string, error) { - if err := os.MkdirAll(dir, 0o755); err != nil { - common.PrintLinesf("can not create directory `%s`", dir) - return "", "", err - } taskDir := path.Join(dir, taskDirname) if err := os.MkdirAll(taskDir, 0o755); err != nil { common.PrintLinesf("can not create directory of task configs `%s`", taskDir) @@ -232,18 +228,14 @@ func writeSourceCfgs(sourceDir string, sourceCfgsMap map[string]*config.SourceCo func writeTaskCfgs(taskDir string, subTaskCfgsMap map[string]map[string]config.SubTaskConfig) error { subTaskCfgsListMap := make(map[string][]*config.SubTaskConfig, len(subTaskCfgsMap)) - // from source => task => subtask to task => source => subtask + // from source => task => subtask to task => subtask for _, subTaskCfgs := range subTaskCfgsMap { for task, subTaskCfg := range subTaskCfgs { clone := subTaskCfg - if subTaskCfgList, ok := subTaskCfgsListMap[task]; ok { - subTaskCfgsListMap[task] = append(subTaskCfgList, &clone) - } else { - subTaskCfgsListMap[task] = []*config.SubTaskConfig{&clone} - } + subTaskCfgsListMap[task] = append(subTaskCfgsListMap[task], &clone) } } - // from task => source => subtask to task => taskCfg + // from task => subtask to task => taskCfg for task, subTaskCfgs := range subTaskCfgsListMap { sort.Slice(subTaskCfgs, func(i, j int) bool { return subTaskCfgs[i].SourceID < subTaskCfgs[j].SourceID diff --git a/tests/new_relay/configs/sources/mysql-replica-01.yaml b/tests/new_relay/configs/sources/mysql-replica-01.yaml index 1597ba0140..c33a2116cf 100644 --- a/tests/new_relay/configs/sources/mysql-replica-01.yaml +++ b/tests/new_relay/configs/sources/mysql-replica-01.yaml @@ -13,8 +13,7 @@ from: port: 3306 user: root max-allowed-packet: null - session: - time_zone: "+00:00" + session: {} security: null purge: interval: 3600 diff --git a/tests/new_relay/configs/tasks/test.yaml b/tests/new_relay/configs/tasks/test.yaml index a1e5183c3d..17b0698cfc 100644 --- a/tests/new_relay/configs/tasks/test.yaml +++ b/tests/new_relay/configs/tasks/test.yaml @@ -16,7 +16,6 @@ target-database: max-allowed-packet: null session: tidb_txn_mode: optimistic - time_zone: "+00:00" security: null online-ddl-scheme: "" routes: {} diff --git a/tests/tiup/lib.sh b/tests/tiup/lib.sh index d4fe3299da..85736ac023 100755 --- a/tests/tiup/lib.sh +++ b/tests/tiup/lib.sh @@ -217,7 +217,7 @@ function patch_nightly_with_tiup_mirror() { # FIXME: use nightly version of grafana and prometheus after https://github.com/pingcap/tiup/issues/1334 fixed. tiup mirror clone tidb-dm-nightly-linux-amd64 --os=linux --arch=amd64 \ --alertmanager=v0.17.0 --grafana=v5.0.1 --prometheus=v5.0.1 \ - --dm-master=v2.0.1 --dm-worker=v2.0.1 \ + --dm-master=$1 --dm-worker=$1 \ --tiup=v$(tiup --version | grep 'tiup' | awk -F ' ' '{print $1}') --dm=v$(tiup --version | grep 'tiup' | awk -F ' ' '{print $1}') # change tiup mirror From cad5db127860ae1f659e271b5eb05b5eaa95455b Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Fri, 23 Jul 2021 10:05:48 +0800 Subject: [PATCH 17/19] add clone test --- dm/config/source_config_test.go | 10 ++++- dm/config/task_test.go | 72 ++++++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/dm/config/source_config_test.go b/dm/config/source_config_test.go index 42b95a2545..db9045b493 100644 --- a/dm/config/source_config_test.go +++ b/dm/config/source_config_test.go @@ -237,11 +237,19 @@ func (t *testConfig) TestConfigVerify(c *C) { } func (t *testConfig) TestSourceConfigForDowngrade(c *C) { - cfg := newSourceConfig() + cfg, err := LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) + + // make sure all new field were added cfgForDowngrade := NewSourceConfigForDowngrade(cfg) cfgReflect := reflect.Indirect(reflect.ValueOf(cfg)) cfgForDowngradeReflect := reflect.Indirect(reflect.ValueOf(cfgForDowngrade)) c.Assert(cfgReflect.NumField(), Equals, cfgForDowngradeReflect.NumField()) + + // make sure all field were copied + cfgForClone := &SourceConfigForDowngrade{} + Clone(cfgForClone, cfg) + c.Assert(cfgForDowngrade, DeepEquals, cfgForClone) } func subtestFlavor(c *C, cfg *SourceConfig, sqlInfo, expectedFlavor, expectedError string) { diff --git a/dm/config/task_test.go b/dm/config/task_test.go index be7cbe1573..6e5f536bf2 100644 --- a/dm/config/task_test.go +++ b/dm/config/task_test.go @@ -30,8 +30,7 @@ import ( "github.com/coreos/go-semver/semver" ) -func (t *testConfig) TestUnusedTaskConfig(c *C) { - correctTaskConfig := `--- +var correctTaskConfig = `--- name: test task-mode: all shard-mode: "pessimistic" @@ -136,6 +135,8 @@ mysql-instances: loader-config-name: "global2" syncer-config-name: "global2" ` + +func (t *testConfig) TestUnusedTaskConfig(c *C) { taskConfig := NewTaskConfig() err := taskConfig.Decode(correctTaskConfig) c.Assert(err, IsNil) @@ -1015,8 +1016,75 @@ func (t *testConfig) TestExclusiveAndWrongExprFilterFields(c *C) { func (t *testConfig) TestTaskConfigForDowngrade(c *C) { cfg := NewTaskConfig() + err := cfg.Decode(correctTaskConfig) + c.Assert(err, IsNil) + cfgForDowngrade := NewTaskConfigForDowngrade(cfg) + + // make sure all new field were added cfgReflect := reflect.Indirect(reflect.ValueOf(cfg)) cfgForDowngradeReflect := reflect.Indirect(reflect.ValueOf(cfgForDowngrade)) c.Assert(cfgReflect.NumField(), Equals, cfgForDowngradeReflect.NumField()+1) // without flag + + // make sure all field were copied + cfgForClone := &TaskConfigForDowngrade{} + Clone(cfgForClone, cfg) + c.Assert(cfgForDowngrade, DeepEquals, cfgForClone) +} + +// Clone clones src to dest. +func Clone(dest, src interface{}) { + cloneValues(reflect.ValueOf(dest), reflect.ValueOf(src)) +} + +// cloneValues clone src to dest recursively. +// Note: pointer still use shadow copy. +func cloneValues(dest, src reflect.Value) { + destType := dest.Type() + srcType := src.Type() + if destType.Kind() == reflect.Ptr { + destType = destType.Elem() + } + if srcType.Kind() == reflect.Ptr { + srcType = srcType.Elem() + } + + if destType.Kind() == reflect.Slice { + slice := reflect.MakeSlice(destType, src.Len(), src.Cap()) + for i := 0; i < src.Len(); i++ { + if slice.Index(i).Type().Kind() == reflect.Ptr { + newVal := reflect.New(slice.Index(i).Type().Elem()) + cloneValues(newVal, src.Index(i)) + slice.Index(i).Set(newVal) + } else { + cloneValues(slice.Index(i).Addr(), src.Index(i).Addr()) + } + } + dest.Set(slice) + return + } + + destFieldsMap := map[string]int{} + for i := 0; i < destType.NumField(); i++ { + destFieldsMap[destType.Field(i).Name] = i + } + for i := 0; i < srcType.NumField(); i++ { + if j, ok := destFieldsMap[srcType.Field(i).Name]; ok { + destField := dest.Elem().Field(j) + srcField := src.Elem().Field(i) + destFieldType := destField.Type() + srcFieldType := srcField.Type() + if destFieldType.Kind() == reflect.Ptr { + destFieldType = destFieldType.Elem() + } + if srcFieldType.Kind() == reflect.Ptr { + srcFieldType = srcFieldType.Elem() + } + if destFieldType != srcFieldType { + cloneValues(destField, srcField) + } else { + destField.Set(srcField) + } + } + } } From 5bec0a3f45ac7f7b5a55ea964b9e455d5c815143 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 26 Jul 2021 10:50:07 +0800 Subject: [PATCH 18/19] fix typo --- dm/config/task_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/config/task_test.go b/dm/config/task_test.go index 6e5f536bf2..a1352dc957 100644 --- a/dm/config/task_test.go +++ b/dm/config/task_test.go @@ -1038,7 +1038,7 @@ func Clone(dest, src interface{}) { } // cloneValues clone src to dest recursively. -// Note: pointer still use shadow copy. +// Note: pointer still use shallow copy. func cloneValues(dest, src reflect.Value) { destType := dest.Type() srcType := src.Type() From 9e04ecf001da94a7debe4181103a88fcd91552dd Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 27 Jul 2021 09:29:26 +0800 Subject: [PATCH 19/19] add new config --- dm/config/task.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dm/config/task.go b/dm/config/task.go index bd8c85f6d6..e1246ad500 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -1035,6 +1035,7 @@ type TaskConfigForDowngrade struct { // new config item MySQLInstances []*MySQLInstanceForDowngrade `yaml:"mysql-instances"` ExprFilter map[string]*ExpressionFilter `yaml:"expression-filter,omitempty"` + OnlineDDL bool `yaml:"online-ddl,omitempty"` } // NewTaskConfigForDowngrade create new TaskConfigForDowngrade. @@ -1066,6 +1067,7 @@ func NewTaskConfigForDowngrade(taskConfig *TaskConfig) *TaskConfigForDowngrade { RemoveMeta: taskConfig.RemoveMeta, MySQLInstances: NewMySQLInstancesForDowngrade(taskConfig.MySQLInstances), ExprFilter: taskConfig.ExprFilter, + OnlineDDL: taskConfig.OnlineDDL, } }