diff --git a/bcs-scenarios/bcs-gitops-manager/cmd/gitgenerator-webhook/webhook/webhook.go b/bcs-scenarios/bcs-gitops-manager/cmd/gitgenerator-webhook/webhook/webhook.go index 43ed7f217f..320d178c81 100644 --- a/bcs-scenarios/bcs-gitops-manager/cmd/gitgenerator-webhook/webhook/webhook.go +++ b/bcs-scenarios/bcs-gitops-manager/cmd/gitgenerator-webhook/webhook/webhook.go @@ -364,8 +364,8 @@ func (s *AdmissionWebhookServer) interceptAppSyncWithForbid(ctx context.Context, blog.Errorf("not need intercept with forbidden flag because unmarshal application request error") return nil } - state := reqApp.Status.OperationState - if state == nil || state.Phase != synccommon.OperationRunning { + // 如果没有设置app的operation,就忽略 + if reqApp.Operation == nil { return nil } app, err := s.argoStore.GetApplication(ctx, req.Name) diff --git a/bcs-services/bcs-bscp/cmd/data-service/app/app.go b/bcs-services/bcs-bscp/cmd/data-service/app/app.go index 8c3f746687..510a9f2848 100644 --- a/bcs-services/bcs-bscp/cmd/data-service/app/app.go +++ b/bcs-services/bcs-bscp/cmd/data-service/app/app.go @@ -14,6 +14,7 @@ package app import ( + "context" "fmt" "math" "net" @@ -34,6 +35,7 @@ import ( "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/cc" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/criteria/uuid" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/dal/dao" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/dal/repository" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/dal/vault" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/logs" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/metrics" @@ -42,6 +44,8 @@ import ( "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/runtime/ctl" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/runtime/shutdown" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/serviced" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/space" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/thirdparty/esb/client" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/tools" ) @@ -80,12 +84,15 @@ func Run(opt *options.Option) error { } type dataService struct { - serve *grpc.Server - gwServe *http.Server - service *service.Service - sd serviced.Service - daoSet dao.Set - vault vault.Set + serve *grpc.Server + gwServe *http.Server + service *service.Service + sd serviced.Service + daoSet dao.Set + vault vault.Set + esb client.Client + spaceMgr *space.Manager + repo repository.Provider } // prepare do prepare jobs before run data service. @@ -139,28 +146,61 @@ func (ds *dataService) prepare(opt *options.Option) error { state := crontab.NewSyncClientOnlineState(ds.daoSet, ds.sd) state.Run() - // initial Vault set + // initialize vault + if ds.vault, err = initVault(); err != nil { + return err + } + + // initialize esb client + settings := cc.DataService().Esb + esbCli, err := client.NewClient(&settings, metrics.Register()) + if err != nil { + return fmt.Errorf("new esb client failed, err: %v", err) + } + ds.esb = esbCli + + // initialize space manager + spaceMgr, err := space.NewSpaceMgr(context.Background(), esbCli) + if err != nil { + return fmt.Errorf("init space manager failed, err: %v", err) + } + ds.spaceMgr = spaceMgr + + // initialize repo provider + repo, err := repository.NewProvider(cc.DataService().Repo) + if err != nil { + return fmt.Errorf("new repo provider failed, err: %v", err) + } + ds.repo = repo + + // sync files from master to slave repo + if cc.DataService().Repo.EnableHA { + repoSyncer := service.NewRepoSyncer(ds.daoSet, ds.repo, ds.spaceMgr, ds.sd) + repoSyncer.Run() + } + + return nil +} + +func initVault() (vault.Set, error) { vaultSet, err := vault.NewSet(cc.DataService().Vault) if err != nil { - return fmt.Errorf("initial vault set failed, err: %v", err) + return nil, fmt.Errorf("initial vault set failed, err: %v", err) } // 挂载目录 exists, err := vaultSet.IsMountPathExists(vault.MountPath) if err != nil { - return fmt.Errorf("error checking mount path: %v", err) + return nil, fmt.Errorf("error checking mount path: %v", err) } if !exists { mountConfig := &api.MountInput{ Type: "kv-v2", } if err = vaultSet.CreateMountPath(vault.MountPath, mountConfig); err != nil { - return fmt.Errorf("initial vault mount path failed, err: %v", err) + return nil, fmt.Errorf("initial vault mount path failed, err: %v", err) } } - - ds.vault = vaultSet - - return nil + return vaultSet, nil } // listenAndServe listen the grpc serve and set up the shutdown gracefully job. @@ -197,7 +237,7 @@ func (ds *dataService) listenAndServe() error { } serve := grpc.NewServer(opts...) - svc, err := service.NewService(ds.sd, ds.daoSet, ds.vault) + svc, err := service.NewService(ds.sd, ds.daoSet, ds.vault, ds.esb, ds.repo) if err != nil { return err } diff --git a/bcs-services/bcs-bscp/cmd/data-service/service/metric.go b/bcs-services/bcs-bscp/cmd/data-service/service/metric.go new file mode 100644 index 0000000000..f3de70c874 --- /dev/null +++ b/bcs-services/bcs-bscp/cmd/data-service/service/metric.go @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/metrics" +) + +var ( + metricInstance *metric + once sync.Once +) + +func initMetric() *metric { + once.Do(func() { + m := new(metric) + labels := prometheus.Labels{} + m.syncQueueLen = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metrics.Namespace, + Subsystem: metrics.RepoSyncSubSys, + Name: "sync_queue_len", + Help: "the length of sync queue for repo sync", + ConstLabels: labels, + }) + metrics.Register().MustRegister(m.syncQueueLen) + + m.ackQueueLen = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metrics.Namespace, + Subsystem: metrics.RepoSyncSubSys, + Name: "ack_queue_len", + Help: "the length of ack queue for repo sync", + ConstLabels: labels, + }) + metrics.Register().MustRegister(m.ackQueueLen) + + metricInstance = m + + }) + return metricInstance +} + +type metric struct { + // syncQueueLen records the length of sync queue for repo sync + syncQueueLen prometheus.Gauge + + // ackQueueLen records the length of ack queue for repo sync + ackQueueLen prometheus.Gauge +} diff --git a/bcs-services/bcs-bscp/cmd/data-service/service/repo_syncer.go b/bcs-services/bcs-bscp/cmd/data-service/service/repo_syncer.go new file mode 100644 index 0000000000..331ae65845 --- /dev/null +++ b/bcs-services/bcs-bscp/cmd/data-service/service/repo_syncer.go @@ -0,0 +1,388 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "context" + "sort" + "strconv" + "sync" + "sync/atomic" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/cc" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/dal/dao" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/dal/repository" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/kit" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/logs" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/runtime/shutdown" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/serviced" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/space" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/tools" +) + +// NewRepoSyncer new repo syncer +func NewRepoSyncer(set dao.Set, repo repository.Provider, spaceMgr *space.Manager, sd serviced.Service) *RepoSyncer { + return &RepoSyncer{ + set: set, + repo: repo, + spaceMgr: spaceMgr, + state: sd, + metric: initMetric(), + } +} + +// RepoSyncer is repo syncer which sync file from master to slave repo +type RepoSyncer struct { + set dao.Set + repo repository.Provider + spaceMgr *space.Manager + state serviced.Service + metric *metric +} + +// Run runs the repo syncer +func (s *RepoSyncer) Run() { + logs.Infof("begin run repo syncer, sync period is %d seconds", cc.DataService().Repo.SyncPeriodSeconds) + kt := kit.New() + ctx, cancel := context.WithCancel(kt.Ctx) + kt.Ctx = ctx + + go func() { + notifier := shutdown.AddNotifier() + <-notifier.Signal + cancel() + notifier.Done() + }() + + go s.collectMetrics() + + // sync incremental files + go s.syncIncremental(kt) + + go func() { + // sync all files at once after service starts a while + time.Sleep(time.Second * 10) + if s.state.IsMaster() { + s.syncAll(kt) + } else { + logs.Infof("current service instance is slave, skip the task of syncing all repo files") + } + + // sync all files periodically + ticker := time.NewTicker(time.Duration(cc.DataService().Repo.SyncPeriodSeconds) * time.Second) + defer ticker.Stop() + for { + select { + case <-kt.Ctx.Done(): + logs.Infof("stop repo syncer success") + return + case <-ticker.C: + if !s.state.IsMaster() { + logs.Infof("current service instance is slave, skip the task of syncing all repo files") + continue + } + s.syncAll(kt) + } + } + }() +} + +// collectMetrics collects metrics for repo syncer periodically +func (s *RepoSyncer) collectMetrics() { + syncMgr := s.repo.SyncManager() + client := syncMgr.QueueClient() + syncQueue := syncMgr.QueueName() + ackQueue := syncMgr.AckQueueName() + ctx := context.Background() + for { + time.Sleep(time.Second * 5) + if syncQueueLen, err := client.LLen(ctx, syncQueue); err != nil { + logs.Errorf("get sync queue length failed, err: %v", err) + } else { + s.metric.syncQueueLen.Set(float64(syncQueueLen)) + } + + if syncQueueLen, err := client.LLen(ctx, ackQueue); err != nil { + logs.Errorf("get sync queue length failed, err: %v", err) + } else { + s.metric.syncQueueLen.Set(float64(syncQueueLen)) + } + } +} + +type syncStat struct { + bizID int32 + total int32 + success int32 + failed int32 + skip int32 + costSeconds float64 +} + +type noFiles struct { + bizID int32 + fileSigns []string +} + +var ( + stats []syncStat + noFileInMaster []noFiles + syncFailedCnt int32 +) + +const failedLimit = 100 + +// syncAll syncs all files from master to slave repo +func (s *RepoSyncer) syncAll(kt *kit.Kit) { + logs.Infof("start to sync all repo files") + start := time.Now() + + // clear related data for every sync cycle + stats = make([]syncStat, 0) + noFileInMaster = make([]noFiles, 0) + syncFailedCnt = 0 + + // get all sorted bizs + allBizs := s.spaceMgr.AllCMDBSpaces() + bizs := make([]int, 0, len(allBizs)) + for biz := range allBizs { + bizID, _ := strconv.Atoi(biz) + bizs = append(bizs, bizID) + } + sort.Ints(bizs) + + // sync files for all bizs + // we think the file count would not be too large for every biz, eg:<100000 + // so, we directly retrieve all file signatures under one biz from the db + // this syncs biz serially (one by one) , and sync files under every biz concurrently + for _, biz := range bizs { + bizID := uint32(biz) + var allSigns []string + var normalSigns, releasedNormalSigns, tmplSigns, releasedTmplSigns []string + var err error + // 未发版的普通配置项 + if normalSigns, err = s.set.Content().ListAllCISigns(kt, bizID); err != nil { + logs.Errorf("list normal ci signs failed, err: %v, rid: %s", err, kt.Rid) + } else { + allSigns = append(allSigns, normalSigns...) + } + + // 已发版的普通配置项 + if releasedNormalSigns, err = s.set.ReleasedCI().ListAllCISigns(kt, bizID); err != nil { + logs.Errorf("list released normal ci signs failed, err: %v, rid: %s", err, kt.Rid) + } else { + allSigns = append(allSigns, releasedNormalSigns...) + } + + // 未发版的模版配置项 + if tmplSigns, err = s.set.TemplateRevision().ListAllCISigns(kt, bizID); err != nil { + logs.Errorf("list template ci signs failed, err: %v, rid: %s", err, kt.Rid) + } else { + allSigns = append(allSigns, tmplSigns...) + } + + // 已发版的模版配置项 + if releasedTmplSigns, err = s.set.ReleasedAppTemplate().ListAllCISigns(kt, bizID); err != nil { + logs.Errorf("list released template ci signs failed, err: %v, rid: %s", err, kt.Rid) + } else { + allSigns = append(allSigns, releasedTmplSigns...) + } + + allSigns = tools.RemoveDuplicateStrings(allSigns) + s.syncOneBiz(kt, bizID, allSigns) + if atomic.LoadInt32(&syncFailedCnt) > failedLimit { + logs.Infof("sync all repo files failed too many times(> %d), stop the sync task, "+ + "you should check the health of repo service, cost time: %s, rid: %s", failedLimit, + time.Since(start), kt.Rid) + return + } + } + + logs.Infof("sync all repo files finished, cost time: %s, rid: %s, stats: %#v", time.Since(start), kt.Rid, stats) + if len(noFileInMaster) > 0 { + logs.Warnf("sync all repo files found some files not in master, please check the master repo, rid: %s, "+ + "info: %#v", kt.Rid, noFileInMaster) + } +} + +// syncOneBiz syncs all files under one biz concurrently +func (s *RepoSyncer) syncOneBiz(kt *kit.Kit, bizID uint32, signs []string) { + start := time.Now() + syncMgr := s.repo.SyncManager() + var success, failed, skip int32 + var nofiles []string + var mu sync.Mutex + + // sync files concurrently + g, _ := errgroup.WithContext(context.Background()) + g.SetLimit(10) + for _, si := range signs { + sign := si + g.Go(func() error { + if atomic.LoadInt32(&syncFailedCnt) > failedLimit { + return nil + } + kt2 := kt.Clone() + kt2.BizID = bizID + isSkip, err := syncMgr.Sync(kt2, sign) + if err != nil { + logs.Errorf("sync file sign %s for biz %d failed, err: %v, rid: %s", sign, bizID, err, kt2.Rid) + atomic.AddInt32(&failed, 1) + atomic.AddInt32(&syncFailedCnt, 1) + if err == repository.ErrNoFileInMaster { + mu.Lock() + nofiles = append(nofiles, sign) + mu.Unlock() + } + return err + } + if isSkip { + atomic.AddInt32(&skip, 1) + } else { + atomic.AddInt32(&success, 1) + } + return nil + }) + } + _ = g.Wait() + + if len(nofiles) > 0 { + noFileInMaster = append(noFileInMaster, noFiles{ + bizID: int32(bizID), + fileSigns: nofiles, + }) + } + + cost := time.Since(start) + stat := syncStat{ + bizID: int32(bizID), + total: int32(len(signs)), + success: success, + failed: failed, + skip: skip, + costSeconds: cost.Seconds(), + } + stats = append(stats, stat) + logs.Infof("sync biz [%d] repo files finished, cost time: %s, rid: %s, stat: %#v", bizID, cost, kt.Rid, stat) +} + +// syncIncremental syncs incremental files +func (s *RepoSyncer) syncIncremental(kt *kit.Kit) { + syncMgr := s.repo.SyncManager() + client := syncMgr.QueueClient() + syncQueue := syncMgr.QueueName() + ackQueue := syncMgr.AckQueueName() + + // consider the service crash, the ackQueue may have some messages, we move them to syncQueue and handle them again + mvCnt := 0 + for { + kt2 := kt.Clone() + msg, err := client.RPopLPush(kt2.Ctx, ackQueue, syncQueue) + if err != nil { + logs.Errorf("move msg from ackQueue to syncQueue failed, err: %v, rid: %s", err, kt2.Rid) + } + // msg is empty which means occurring redis.Nil and ackQueue is empty + if msg == "" { + break + } + mvCnt++ + } + if mvCnt > 0 { + logs.Errorf("have moved %d msg from ackQueue to syncQueue", mvCnt) + } + + // sync files concurrently + workerCount := 10 + var wg sync.WaitGroup + for i := 0; i < workerCount; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + kt2 := kt.Clone() + // block pop msg from syncQueue and push one to ackQueue + msg, err := client.BRPopLPush(kt2.Ctx, syncQueue, ackQueue, 0) + if err != nil { + logs.Errorf("pop msg from syncQueue failed, err: %v, rid: %s", err, kt2.Rid) + time.Sleep(time.Second) + continue + } + + var bizID uint32 + var sign string + bizID, sign, err = syncMgr.ParseQueueMsg(msg) + if err != nil { + logs.Errorf("parse queue msg failed, err: %v, rid: %s", err, kt2.Rid) + // remove the invalid msg from ackQueue + if err = client.LRem(kt2.Ctx, ackQueue, 0, msg); err != nil { + logs.Errorf("remove msg from ackQueue failed, err: %v, rid: %s", err, kt2.Rid) + } + continue + } + + // sync the file with the specific bizID and signature + kt2.BizID = bizID + if _, err = syncMgr.Sync(kt2, sign); err != nil { + logs.Errorf("sync file failed, err: %v, rid: %s", err, kt2.Rid) + // if failed for no file in master, continue and not to retry it + if err == repository.ErrNoFileInMaster { + continue + } + // if failed for other reasons, retry after a delay + go retryMessage(kt2, syncMgr, ackQueue, msg, sign) + continue + } else { + // if success, remove the msg from ackQueue + if err = client.LRem(kt2.Ctx, ackQueue, 0, msg); err != nil { + logs.Errorf("remove msg from ackQueue failed, err: %v, rid: %s", err, kt2.Rid) + } + } + } + }() + } + + // wait for all workers to finish (they won't, as this is an infinite loop) + wg.Wait() +} + +// retryMessage retry handle the failed msg after a delay +// if retry failed beyond the limit, finish the retry and all-sync mechanism will handle it +func retryMessage(kt *kit.Kit, syncMgr *repository.SyncManager, ackQueue, msg, sign string) { + limit := 3 + count := 0 + for { + // wait for 1 minute before retrying + time.Sleep(1 * time.Minute) + + if count == 0 { + // remove the msg from ackQueue, + if err := syncMgr.QueueClient().LRem(kt.Ctx, ackQueue, 0, msg); err != nil { + logs.Errorf("remove msg from ackQueue during retry failed, err: %v, rid: %s", err, kt.Rid) + continue + } + } + count++ + + if _, err := syncMgr.Sync(kt, sign); err != nil { + logs.Errorf("sync retry count %d failed, err: %v, rid: %s", count, err, kt.Rid) + // retry failed beyond the limit, return and all-sync mechanism will handle it + if count >= limit { + return + } + } + return + } +} diff --git a/bcs-services/bcs-bscp/cmd/data-service/service/service.go b/bcs-services/bcs-bscp/cmd/data-service/service/service.go index 258955ab4b..7009e29da7 100644 --- a/bcs-services/bcs-bscp/cmd/data-service/service/service.go +++ b/bcs-services/bcs-bscp/cmd/data-service/service/service.go @@ -19,11 +19,9 @@ import ( "fmt" "net/http" - "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/cc" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/dal/dao" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/dal/repository" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/dal/vault" - "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/metrics" pbds "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/protocol/data-service" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/serviced" "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/thirdparty/esb/client" @@ -42,7 +40,8 @@ type Service struct { } // NewService create a service instance. -func NewService(sd serviced.Service, daoSet dao.Set, vaultSet vault.Set) (*Service, error) { +func NewService(sd serviced.Service, daoSet dao.Set, vaultSet vault.Set, esb client.Client, repo repository.Provider) ( + *Service, error) { state, ok := sd.(serviced.State) if !ok { return nil, errors.New("discover convert state failed") @@ -52,24 +51,11 @@ func NewService(sd serviced.Service, daoSet dao.Set, vaultSet vault.Set) (*Servi return nil, fmt.Errorf("new gateway failed, err: %v", err) } - // initialize esb client - settings := cc.DataService().Esb - esbCli, err := client.NewClient(&settings, metrics.Register()) - if err != nil { - return nil, err - } - - // initialize repo provider - repo, err := repository.NewProvider(cc.DataService().Repo) - if err != nil { - return nil, err - } - svc := &Service{ dao: daoSet, vault: vaultSet, gateway: gateway, - esb: esbCli, + esb: esb, repo: repo, tmplProc: tmplprocess.NewTmplProcessor(), } diff --git a/bcs-services/bcs-bscp/cmd/feed-server/service/rpc_sidecar.go b/bcs-services/bcs-bscp/cmd/feed-server/service/rpc_sidecar.go index 6713acf075..8a707fe411 100644 --- a/bcs-services/bcs-bscp/cmd/feed-server/service/rpc_sidecar.go +++ b/bcs-services/bcs-bscp/cmd/feed-server/service/rpc_sidecar.go @@ -459,7 +459,10 @@ func (s *Service) GetDownloadURL(ctx context.Context, req *pbfs.GetDownloadURLRe } if !s.rl.Enable() { - return &pbfs.GetDownloadURLResp{Url: downloadLink, WaitTimeMil: 0}, nil + return &pbfs.GetDownloadURLResp{ + Url: downloadLink[0], // 保留Url兼容老版客户端,DownloadLink方法返回无错误则downloadLink长度必大于0,无需判断 + Urls: downloadLink, + WaitTimeMil: 0}, nil } // 对于单个大文件下载,受限于单个客户端和服务端之间的带宽(比如为10MB/s=80Mb/s),而在存储服务端支持更高带宽的情况下(比如100MB/s), // 哪怕单个文件2GB,在限流器阈值比单个客户端下载带宽高的情况下,还是应该允许其他客户端去存储服务端下载, @@ -488,7 +491,10 @@ func (s *Service) GetDownloadURL(ctx context.Context, req *pbfs.GetDownloadURLRe if bWaitTimeMil < gWaitTimeMil { wt = gWaitTimeMil } - return &pbfs.GetDownloadURLResp{Url: downloadLink, WaitTimeMil: wt}, nil + return &pbfs.GetDownloadURLResp{ + Url: downloadLink[0], // 保留Url兼容老版客户端,DownloadLink方法返回无错误则downloadLink长度必大于0,无需判断 + Urls: downloadLink, + WaitTimeMil: wt}, nil } // PullKvMeta pull an app's latest release metadata only when the app's configures is kv type. diff --git a/bcs-services/bcs-bscp/etc/bcs-bscp.yml b/bcs-services/bcs-bscp/etc/bcs-bscp.yml index 80e76a0cca..5d9483bc5b 100644 --- a/bcs-services/bcs-bscp/etc/bcs-bscp.yml +++ b/bcs-services/bcs-bscp/etc/bcs-bscp.yml @@ -56,13 +56,33 @@ redisCluster: password: db: 1 -# feed server -repository: - endpoints: - - xxx - token: xx - project: bscp - user: xx downstream: bounceIntervalHour: 48 +# 文件存储配置,兼容原有配置,原有配置为主存储master配置 +repository: + # 文件存储类型,当前支持bkrepo、s3类型,s3实现为cos存储,默认为bkrepo + storageType: bkrepo + #storageType: s3 + bkRepo: + endpoints: + project: + username: + password: + redisCluster: + db: + endpoints: + password: + # 是否开启高可用,使用主从模式,支持读高可用(写主并同步给从,主从均可读),默认为false(即只使用主存储master配置),为true时需有slave配置 + enableHA: + # 高可用下的主从同步周期,用于全量同步,单位为秒,默认是1天,最小值为1小时,低于最小值会重置为默认值 + syncPeriodSeconds: + slave: + #storageType: bkrepo + storageType: s3 + s3: + endpoint: + accessKeyID: + secretAccessKey: + useSSL: + bucketName: diff --git a/bcs-services/bcs-bscp/pkg/cc/types.go b/bcs-services/bcs-bscp/pkg/cc/types.go index 3ed488b994..1e63181305 100644 --- a/bcs-services/bcs-bscp/pkg/cc/types.go +++ b/bcs-services/bcs-bscp/pkg/cc/types.go @@ -344,10 +344,18 @@ const ( // Repository defines all the repo related runtime. type Repository struct { - StorageType StorageMode `yaml:"storageType"` - S3 S3Storage `yaml:"s3"` - BkRepo BkRepoStorage `yaml:"bkRepo"` - RedisCluster RedisCluster `yaml:"redisCluster"` + BaseRepo `yaml:",inline"` + RedisCluster RedisCluster `yaml:"redisCluster"` + EnableHA bool `yaml:"enableHA"` + SyncPeriodSeconds uint `yaml:"syncPeriodSeconds"` + Slave BaseRepo `yaml:"slave"` +} + +// BaseRepo 文件存储的基础部分 +type BaseRepo struct { + StorageType StorageMode `yaml:"storageType"` + S3 S3Storage `yaml:"s3"` + BkRepo BkRepoStorage `yaml:"bkRepo"` } // BkRepoStorage BKRepo 存储类型 @@ -395,52 +403,83 @@ func (s Repository) OneEndpoint() (string, error) { return addr, nil } +const defaultSyncPeriodSeconds = 24 * 3600 // default is 1 day +const minSyncPeriodSeconds = 3600 // min is 1 hour + func (s *Repository) trySetDefault() { if len(s.StorageType) == 0 { s.StorageType = BkRepo } s.RedisCluster.trySetDefault() + if s.EnableHA { + if len(s.Slave.StorageType) == 0 { + s.Slave.StorageType = S3 + } + } + if s.SyncPeriodSeconds == 0 || s.SyncPeriodSeconds < minSyncPeriodSeconds { + s.SyncPeriodSeconds = defaultSyncPeriodSeconds + } } // validate repo runtime. func (s Repository) validate() error { - switch strings.ToUpper(string(s.StorageType)) { + if err := s.BaseRepo.validate(); err != nil { + return fmt.Errorf("repository master config err: %v", err) + } + + if err := s.RedisCluster.validate(); err != nil { + return fmt.Errorf("repository redis cluster config err: %v", err) + } + + if s.EnableHA { + if err := s.Slave.validate(); err != nil { + return fmt.Errorf("repository slave config err: %v", err) + } + } + + return nil +} + +// validate repo base part. +func (b BaseRepo) validate() error { + switch strings.ToUpper(string(b.StorageType)) { case string(S3): - if len(s.S3.Endpoint) == 0 { + if len(b.S3.Endpoint) == 0 { return errors.New("s3 endpoint is not set") } - if len(s.S3.AccessKeyID) == 0 { + if len(b.S3.AccessKeyID) == 0 { return errors.New("s3 accessKeyID is not set") } - if len(s.S3.SecretAccessKey) == 0 { + if len(b.S3.SecretAccessKey) == 0 { return errors.New("s3 secretAccessKey is not set") } - if len(s.S3.BucketName) == 0 { + if len(b.S3.BucketName) == 0 { return errors.New("s3 bucketName is not set") } case string(BkRepo): - if len(s.BkRepo.Endpoints) == 0 { + if len(b.BkRepo.Endpoints) == 0 { return errors.New("bk_repo endpoints is not set") } - if len(s.BkRepo.Username) == 0 { + if len(b.BkRepo.Username) == 0 { return errors.New("repo basic auth username is not set") } - if len(s.BkRepo.Password) == 0 { + if len(b.BkRepo.Password) == 0 { return errors.New("repo basic auth password is not set") } - if len(s.BkRepo.Project) == 0 { + if len(b.BkRepo.Project) == 0 { return errors.New("repo project is not set") } - if err := s.BkRepo.TLS.validate(); err != nil { + if err := b.BkRepo.TLS.validate(); err != nil { return fmt.Errorf("repo tls, %v", err) } - + default: + return fmt.Errorf("unsupported storage type: %s", string(b.StorageType)) } return nil diff --git a/bcs-services/bcs-bscp/pkg/dal/bedis/base.go b/bcs-services/bcs-bscp/pkg/dal/bedis/base.go index 8d9fe1be19..1035bbf0d9 100644 --- a/bcs-services/bcs-bscp/pkg/dal/bedis/base.go +++ b/bcs-services/bcs-bscp/pkg/dal/bedis/base.go @@ -562,3 +562,58 @@ func (bs *bedis) LTrim(ctx context.Context, key string, start, stop int64) (stri return value, nil } + +// LRem removes the first count occurrences of elements equal to element from the list stored at key. +// count argument influences the operation in the following ways: +// count > 0: Remove elements equal to element moving from head to tail. +// count < 0: Remove elements equal to element moving from tail to head. +// count = 0: Remove all elements equal to element. +func (bs *bedis) LRem(ctx context.Context, key string, count int64, value interface{}) error { + start := time.Now() + _, err := bs.client.LRem(ctx, key, count, value).Result() + if err != nil { + bs.mc.errCounter.With(prm.Labels{"cmd": "lrem"}).Inc() + return err + } + bs.logSlowCmd(ctx, key, time.Since(start)) + bs.mc.cmdLagMS.With(prm.Labels{"cmd": "lrem"}).Observe(float64(time.Since(start).Milliseconds())) + return nil +} + +// RPopLPush atomically returns and removes the last element (tail) of the list stored at source, +// and pushes the element at the first element (head) of the list stored at destination. +func (bs *bedis) RPopLPush(ctx context.Context, source, destination string) (string, error) { + start := time.Now() + value, err := bs.client.RPopLPush(ctx, source, destination).Result() + if err != nil { + if IsNilError(err) { + return "", nil + } + bs.mc.errCounter.With(prm.Labels{"cmd": "rpoplpush"}).Inc() + return "", err + } + bs.logSlowCmd(ctx, "", time.Since(start)) + bs.mc.cmdLagMS.With(prm.Labels{"cmd": "rpoplpush"}).Observe(float64(time.Since(start).Milliseconds())) + + return value, nil +} + +// BRPopLPush is the blocking variant of RPOPLPUSH. +// When source contains elements, this command behaves exactly like RPOPLPUSH. +// When source is empty, Redis will block the connection until another client pushes to it or until timeout is reached. +// A timeout of zero can be used to block indefinitely. +func (bs *bedis) BRPopLPush(ctx context.Context, source, destination string, ttlSeconds int) (string, error) { + start := time.Now() + value, err := bs.client.BRPopLPush(ctx, source, destination, time.Duration(ttlSeconds)*time.Second).Result() + if err != nil { + if IsNilError(err) { + return "", nil + } + bs.mc.errCounter.With(prm.Labels{"cmd": "brpoplpush"}).Inc() + return "", err + } + bs.logSlowCmd(ctx, "", time.Since(start)) + bs.mc.cmdLagMS.With(prm.Labels{"cmd": "brpoplpush"}).Observe(float64(time.Since(start).Milliseconds())) + + return value, nil +} diff --git a/bcs-services/bcs-bscp/pkg/dal/bedis/bedis.go b/bcs-services/bcs-bscp/pkg/dal/bedis/bedis.go index 4aaf700637..7af206ee49 100644 --- a/bcs-services/bcs-bscp/pkg/dal/bedis/bedis.go +++ b/bcs-services/bcs-bscp/pkg/dal/bedis/bedis.go @@ -66,6 +66,9 @@ type RedisClient interface { Keys(ctx context.Context, pattern string) *redis.StringSliceCmd LLen(ctx context.Context, key string) *redis.IntCmd LTrim(ctx context.Context, key string, start, stop int64) *redis.StatusCmd + LRem(ctx context.Context, key string, count int64, value interface{}) *redis.IntCmd + RPopLPush(ctx context.Context, source, destination string) *redis.StringCmd + BRPopLPush(ctx context.Context, source, destination string, timeout time.Duration) *redis.StringCmd } // Client defines all the bscp used redis command @@ -94,6 +97,9 @@ type Client interface { Keys(ctx context.Context, pattern string) ([]string, error) LLen(ctx context.Context, key string) (int64, error) LTrim(ctx context.Context, key string, start, stop int64) (string, error) + LRem(ctx context.Context, key string, count int64, value interface{}) error + RPopLPush(ctx context.Context, source, destination string) (string, error) + BRPopLPush(ctx context.Context, source, destination string, ttlSeconds int) (string, error) } // NewRedisCache create a redis cluster client. diff --git a/bcs-services/bcs-bscp/pkg/dal/dao/content.go b/bcs-services/bcs-bscp/pkg/dal/dao/content.go index 19ca11fc29..4b823195ec 100644 --- a/bcs-services/bcs-bscp/pkg/dal/dao/content.go +++ b/bcs-services/bcs-bscp/pkg/dal/dao/content.go @@ -36,6 +36,8 @@ type Content interface { Get(kit *kit.Kit, id, bizID uint32) (*table.Content, error) // BatchDeleteWithTx batch delete content data instance with transaction. BatchDeleteWithTx(kit *kit.Kit, tx *gen.QueryTx, contentIDs []uint32) error + // ListAllCISigns lists all non-template ci signatures of one biz, and only belongs to existing apps + ListAllCISigns(kit *kit.Kit, bizID uint32) ([]string, error) } var _ Content = new(contentDao) @@ -192,3 +194,26 @@ func (dao *contentDao) validateAttachmentResExist(kit *kit.Kit, am *table.Conten return nil } + +// ListAllCISigns lists all non-template ci signatures of one biz, and only belongs to existing apps +func (dao *contentDao) ListAllCISigns(kit *kit.Kit, bizID uint32) ([]string, error) { + am := dao.genQ.App + aq := dao.genQ.App.WithContext(kit.Ctx) + var appIDs []uint32 + if err := aq.Select(am.ID.Distinct()). + Where(am.BizID.Eq(bizID)). + Pluck(am.ID, &appIDs); err != nil { + return nil, err + } + + m := dao.genQ.Content + q := dao.genQ.Content.WithContext(kit.Ctx) + var signs []string + if err := q.Select(m.Signature.Distinct()). + Where(m.BizID.Eq(bizID), m.AppID.In(appIDs...)). + Pluck(m.Signature, &signs); err != nil { + return nil, err + } + + return signs, nil +} diff --git a/bcs-services/bcs-bscp/pkg/dal/dao/release_ci.go b/bcs-services/bcs-bscp/pkg/dal/dao/release_ci.go index fee92d09e7..4f02c33911 100644 --- a/bcs-services/bcs-bscp/pkg/dal/dao/release_ci.go +++ b/bcs-services/bcs-bscp/pkg/dal/dao/release_ci.go @@ -47,6 +47,8 @@ type ReleasedCI interface { ListAllByReleaseIDs(kit *kit.Kit, releasedIDs []uint32, bizID uint32) ([]*table.ReleasedConfigItem, error) // BatchDeleteByReleaseIDWithTx batch delete by release id with transaction. BatchDeleteByReleaseIDWithTx(kit *kit.Kit, tx *gen.QueryTx, bizID, appID, releaseID uint32) error + // ListAllCISigns lists all released non-template ci signatures of one biz, and only belongs to existing apps + ListAllCISigns(kit *kit.Kit, bizID uint32) ([]string, error) } var _ ReleasedCI = new(releasedCIDao) @@ -214,3 +216,26 @@ func (dao *releasedCIDao) BatchDeleteByReleaseIDWithTx(kit *kit.Kit, tx *gen.Que _, err := m.WithContext(kit.Ctx).Where(m.BizID.Eq(bizID), m.AppID.Eq(appID), m.ReleaseID.Eq(releaseID)).Delete() return err } + +// ListAllCISigns lists all released non-template ci signatures of one biz, and only belongs to existing apps +func (dao *releasedCIDao) ListAllCISigns(kit *kit.Kit, bizID uint32) ([]string, error) { + am := dao.genQ.App + aq := dao.genQ.App.WithContext(kit.Ctx) + var appIDs []uint32 + if err := aq.Select(am.ID.Distinct()). + Where(am.BizID.Eq(bizID)). + Pluck(am.ID, &appIDs); err != nil { + return nil, err + } + + m := dao.genQ.ReleasedConfigItem + q := dao.genQ.ReleasedConfigItem.WithContext(kit.Ctx) + var signs []string + if err := q.Select(m.Signature.Distinct()). + Where(m.BizID.Eq(bizID), m.AppID.In(appIDs...)). + Pluck(m.Signature, &signs); err != nil { + return nil, err + } + + return signs, nil +} diff --git a/bcs-services/bcs-bscp/pkg/dal/dao/released_app_template.go b/bcs-services/bcs-bscp/pkg/dal/dao/released_app_template.go index 91fb5461ad..0d033ca91c 100644 --- a/bcs-services/bcs-bscp/pkg/dal/dao/released_app_template.go +++ b/bcs-services/bcs-bscp/pkg/dal/dao/released_app_template.go @@ -41,6 +41,8 @@ type ReleasedAppTemplate interface { BatchDeleteByAppIDWithTx(kit *kit.Kit, tx *gen.QueryTx, appID, bizID uint32) error // BatchDeleteByReleaseIDWithTx batch delete by release id with transaction. BatchDeleteByReleaseIDWithTx(kit *kit.Kit, tx *gen.QueryTx, bizID, appID, releaseID uint32) error + // ListAllCISigns lists all released template ci signatures of one biz, and only belongs to existing apps + ListAllCISigns(kit *kit.Kit, bizID uint32) ([]string, error) } var _ ReleasedAppTemplate = new(releasedAppTemplateDao) @@ -187,3 +189,26 @@ func (dao *releasedAppTemplateDao) BatchDeleteByReleaseIDWithTx(kit *kit.Kit, tx _, err := m.WithContext(kit.Ctx).Where(m.BizID.Eq(bizID), m.AppID.Eq(appID), m.ReleaseID.Eq(releaseID)).Delete() return err } + +// ListAllCISigns lists all released template ci signatures of one biz, and only belongs to existing apps +func (dao *releasedAppTemplateDao) ListAllCISigns(kit *kit.Kit, bizID uint32) ([]string, error) { + am := dao.genQ.App + aq := dao.genQ.App.WithContext(kit.Ctx) + var appIDs []uint32 + if err := aq.Select(am.ID.Distinct()). + Where(am.BizID.Eq(bizID)). + Pluck(am.ID, &appIDs); err != nil { + return nil, err + } + + m := dao.genQ.ReleasedAppTemplate + q := dao.genQ.ReleasedAppTemplate.WithContext(kit.Ctx) + var signs []string + if err := q.Select(m.Signature.Distinct()). + Where(m.BizID.Eq(bizID), m.AppID.In(appIDs...)). + Pluck(m.Signature, &signs); err != nil { + return nil, err + } + + return signs, nil +} diff --git a/bcs-services/bcs-bscp/pkg/dal/dao/template_revision.go b/bcs-services/bcs-bscp/pkg/dal/dao/template_revision.go index 68051efdc8..7e9192845a 100644 --- a/bcs-services/bcs-bscp/pkg/dal/dao/template_revision.go +++ b/bcs-services/bcs-bscp/pkg/dal/dao/template_revision.go @@ -61,6 +61,8 @@ type TemplateRevision interface { // ListLatestGroupByTemplateIdsWithTx Lists the latest version groups by template ids with transaction. ListLatestGroupByTemplateIdsWithTx(kit *kit.Kit, tx *gen.QueryTx, bizID uint32, templateIDs []uint32) ([]*table.TemplateRevision, error) + // ListAllCISigns lists all template ci of one biz, no need care about apps for the unbound template ci + ListAllCISigns(kit *kit.Kit, bizID uint32) ([]string, error) } var _ TemplateRevision = new(templateRevisionDao) @@ -354,3 +356,17 @@ func (dao *templateRevisionDao) ListLatestRevisionsGroupByTemplateIds(kit *kit.K } return find, nil } + +// ListAllCISigns lists all template ci of one biz, no need care about apps for the unbound template ci +func (dao *templateRevisionDao) ListAllCISigns(kit *kit.Kit, bizID uint32) ([]string, error) { + m := dao.genQ.TemplateRevision + q := dao.genQ.TemplateRevision.WithContext(kit.Ctx) + var signs []string + if err := q.Select(m.Signature.Distinct()). + Where(m.BizID.Eq(bizID)). + Pluck(m.Signature, &signs); err != nil { + return nil, err + } + + return signs, nil +} diff --git a/bcs-services/bcs-bscp/pkg/dal/repository/bkrepo.go b/bcs-services/bcs-bscp/pkg/dal/repository/bkrepo.go index 10631ebe21..ef5e51c824 100644 --- a/bcs-services/bcs-bscp/pkg/dal/repository/bkrepo.go +++ b/bcs-services/bcs-bscp/pkg/dal/repository/bkrepo.go @@ -87,6 +87,11 @@ func (r *RepoCreated) Exist(name string) bool { return ok } +// SyncManager implements HAEnhancer interface +func (c *bkrepoClient) SyncManager() *SyncManager { + return nil +} + func (c *bkrepoClient) ensureRepo(kt *kit.Kit) error { repoName, err := repo.GenRepoName(kt.BizID) if err != nil { @@ -113,7 +118,7 @@ func (c *bkrepoClient) ensureRepo(kt *kit.Kit) error { return nil } -// Upload file to bkrepo +// Upload uploads file to bkrepo func (c *bkrepoClient) Upload(kt *kit.Kit, sign string, body io.Reader) (*ObjectMetadata, error) { if err := c.ensureRepo(kt); err != nil { return nil, errors.Wrap(err, "ensure repo failed") @@ -164,7 +169,7 @@ func (c *bkrepoClient) Upload(kt *kit.Kit, sign string, body io.Reader) (*Object return metadata, nil } -// Download download file from bkrepo +// Download downloads file from bkrepo func (c *bkrepoClient) Download(kt *kit.Kit, sign string) (io.ReadCloser, int64, error) { node, err := repo.GenNodePath(&repo.NodeOption{Project: c.project, BizID: kt.BizID, Sign: sign}) if err != nil { @@ -377,15 +382,15 @@ func (c *bkrepoClient) URIDecorator(bizID uint32) DecoratorInter { } // DownloadLink bkrepo file download link -func (c *bkrepoClient) DownloadLink(kt *kit.Kit, sign string, fetchLimit uint32) (string, error) { +func (c *bkrepoClient) DownloadLink(kt *kit.Kit, sign string, fetchLimit uint32) ([]string, error) { repoName, err := repo.GenRepoName(kt.BizID) if err != nil { - return "", err + return nil, err } objPath, err := repo.GenNodeFullPath(sign) if err != nil { - return "", err + return nil, err } // get file download url. @@ -399,10 +404,10 @@ func (c *bkrepoClient) DownloadLink(kt *kit.Kit, sign string, fetchLimit uint32) }) if err != nil { - return "", errors.Wrap(err, "generate temp download url failed") + return nil, errors.Wrap(err, "generate temp download url failed") } - return url, nil + return []string{url}, nil } // AsyncDownload bkrepo @@ -416,7 +421,7 @@ func (c *bkrepoClient) AsyncDownloadStatus(kt *kit.Kit, sign string, taskID stri } // newBKRepoClient new bkrepo client -func newBKRepoClient(settings cc.Repository) (BaseProvider, error) { +func newBKRepoClient(settings cc.BaseRepo) (BaseProvider, error) { cli, err := repo.NewClient(settings, metrics.Register()) if err != nil { return nil, err @@ -445,20 +450,21 @@ func newBKRepoClient(settings cc.Repository) (BaseProvider, error) { } // newBKRepoProvider new bkrepo provider -func newBKRepoProvider(settings cc.Repository) (Provider, error) { - p, err := newBKRepoClient(settings) +func newBKRepoProvider(repo cc.BaseRepo, redis cc.RedisCluster) (Provider, error) { + p, err := newBKRepoClient(repo) if err != nil { return nil, err } var c VariableCacher - c, err = newVariableCacher(settings.RedisCluster, p) + c, err = newVariableCacher(redis, p) if err != nil { return nil, err } return &repoProvider{ BaseProvider: p, + HAEnhancer: p.(*bkrepoClient), VariableCacher: c, }, nil } diff --git a/bcs-services/bcs-bscp/pkg/dal/repository/cos.go b/bcs-services/bcs-bscp/pkg/dal/repository/cos.go index 60f0e0dfe7..c4ca1fb180 100644 --- a/bcs-services/bcs-bscp/pkg/dal/repository/cos.go +++ b/bcs-services/bcs-bscp/pkg/dal/repository/cos.go @@ -44,7 +44,12 @@ type cosClient struct { innerClient *cos.Client } -// Upload upload file to cos +// SyncManager implements HAEnhancer interface +func (c *cosClient) SyncManager() *SyncManager { + return nil +} + +// Upload uploads file to cos func (c *cosClient) Upload(kt *kit.Kit, sign string, body io.Reader) (*ObjectMetadata, error) { node, err := repo.GenS3NodeFullPath(kt.BizID, sign) if err != nil { @@ -77,7 +82,7 @@ func (c *cosClient) Upload(kt *kit.Kit, sign string, body io.Reader) (*ObjectMet return metadata, nil } -// Download download file from cos +// Download downloads file from cos func (c *cosClient) Download(kt *kit.Kit, sign string) (io.ReadCloser, int64, error) { node, err := repo.GenS3NodeFullPath(kt.BizID, sign) if err != nil { @@ -170,10 +175,10 @@ func (c *cosClient) URIDecorator(bizID uint32) DecoratorInter { } // DownloadLink cos file download link -func (c *cosClient) DownloadLink(kt *kit.Kit, sign string, fetchLimit uint32) (string, error) { +func (c *cosClient) DownloadLink(kt *kit.Kit, sign string, fetchLimit uint32) ([]string, error) { node, err := repo.GenS3NodeFullPath(kt.BizID, sign) if err != nil { - return "", err + return nil, err } opt := &cos.PresignedURLOptions{ @@ -186,10 +191,10 @@ func (c *cosClient) DownloadLink(kt *kit.Kit, sign string, fetchLimit uint32) (s u, err := c.innerClient.Object.GetPresignedURL(kt.Ctx, http.MethodGet, node, c.conf.AccessKeyID, c.conf.SecretAccessKey, time.Hour, opt) if err != nil { - return "", err + return nil, err } - return u.String(), nil + return []string{u.String()}, nil } // AsyncDownload cos @@ -229,20 +234,21 @@ func newCosClient(conf cc.S3Storage) (BaseProvider, error) { } // newCosProvider new cos provider -func newCosProvider(settings cc.Repository) (Provider, error) { - p, err := newCosClient(settings.S3) +func newCosProvider(repo cc.BaseRepo, redis cc.RedisCluster) (Provider, error) { + p, err := newCosClient(repo.S3) if err != nil { return nil, err } var c VariableCacher - c, err = newVariableCacher(settings.RedisCluster, p) + c, err = newVariableCacher(redis, p) if err != nil { return nil, err } return &repoProvider{ BaseProvider: p, + HAEnhancer: p.(*cosClient), VariableCacher: c, }, nil } diff --git a/bcs-services/bcs-bscp/pkg/dal/repository/ha.go b/bcs-services/bcs-bscp/pkg/dal/repository/ha.go new file mode 100644 index 0000000000..e0b6512cec --- /dev/null +++ b/bcs-services/bcs-bscp/pkg/dal/repository/ha.go @@ -0,0 +1,310 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package repository + +import ( + "errors" + "fmt" + "io" + "strconv" + "strings" + + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/cc" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/criteria/errf" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/dal/bedis" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/kit" + "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/logs" +) + +// haClient is client for high availability +// write to master repo only, read from master or slave repo +// there is synchronization mechanism which sync data from master to slave +type haClient struct { + master Provider + slave Provider + syncMgr *SyncManager +} + +// SyncManager is sync manager +type SyncManager struct { + queue *syncQueue + master Provider + slave Provider +} + +type syncQueue struct { + client bedis.Client +} + +// SyncManager implements HAEnhancer interface +func (c *haClient) SyncManager() *SyncManager { + return c.syncMgr +} + +// Upload Uploads file to ha repo master +// push file metadata to queue for sync +func (c *haClient) Upload(kt *kit.Kit, sign string, body io.Reader) (*ObjectMetadata, error) { + md, err := c.master.Upload(kt, sign, body) + if err != nil { + return nil, err + } + _ = c.syncMgr.PushToQueue(kt, sign) + return md, err +} + +func haErr(masterErr, slaveErr error) error { + return fmt.Errorf("master error: %v, slave error: %v", masterErr, slaveErr) +} + +// Download downloads file from ha repo, read priority: master > slave +func (c *haClient) Download(kt *kit.Kit, sign string) (io.ReadCloser, int64, error) { + masterBody, masterSize, masterErr := c.master.Download(kt, sign) + if masterErr == nil { + return masterBody, masterSize, nil + } + + slaveBody, slaveSize, slaveErr := c.slave.Download(kt, sign) + if slaveErr == nil { + return slaveBody, slaveSize, nil + } + + return nil, 0, haErr(masterErr, slaveErr) +} + +// Metadata ha repo file metadata +func (c *haClient) Metadata(kt *kit.Kit, sign string) (*ObjectMetadata, error) { + masterMD, masterErr := c.master.Metadata(kt, sign) + if masterErr == nil { + return masterMD, nil + } + + slaveMD, slaveErr := c.slave.Metadata(kt, sign) + if slaveErr == nil { + return slaveMD, nil + } + + return nil, haErr(masterErr, slaveErr) +} + +// InitMultipartUpload init multipart upload file for ha repo master +// push only after the completion of multipart upload +func (c *haClient) InitMultipartUpload(kt *kit.Kit, sign string) (string, error) { + return c.master.InitMultipartUpload(kt, sign) +} + +// MultipartUpload upload one part of the file to ha repo master +// push only after the completion of multipart upload +func (c *haClient) MultipartUpload(kt *kit.Kit, sign string, uploadID string, partNum uint32, + body io.Reader) error { + return c.master.MultipartUpload(kt, sign, uploadID, partNum, body) +} + +// CompleteMultipartUpload complete multipart upload and return metadata for ha repo master +// push only after the completion of multipart upload +func (c *haClient) CompleteMultipartUpload(kt *kit.Kit, sign string, uploadID string) (*ObjectMetadata, error) { + md, err := c.master.CompleteMultipartUpload(kt, sign, uploadID) + if err != nil { + return nil, err + } + _ = c.syncMgr.PushToQueue(kt, sign) + return md, err +} + +// URIDecorator .. +func (c *haClient) URIDecorator(bizID uint32) DecoratorInter { + return newUriDecoratorInter(bizID) +} + +// DownloadLink ha repo file download link, get download url from master and slave +func (c *haClient) DownloadLink(kt *kit.Kit, sign string, fetchLimit uint32) ([]string, error) { + var urls []string + masterUrl, masterErr := c.master.DownloadLink(kt, sign, fetchLimit) + if masterErr == nil { + urls = append(urls, masterUrl...) + } + + slaveUrl, slaveErr := c.slave.DownloadLink(kt, sign, fetchLimit) + if slaveErr == nil { + urls = append(urls, slaveUrl...) + } + + if masterErr != nil && slaveErr != nil { + return nil, haErr(masterErr, slaveErr) + } + + return urls, nil +} + +// AsyncDownload ha repo +func (c *haClient) AsyncDownload(kt *kit.Kit, sign string) (string, error) { + return "", nil +} + +// AsyncDownloadStatus ha repo +func (c *haClient) AsyncDownloadStatus(kt *kit.Kit, sign string, taskID string) (bool, error) { + return false, nil +} + +// newHAClient new ha client +func newHAClient(settings cc.Repository) (BaseProvider, error) { + var master, slave Provider + var syncMgr *SyncManager + var err error + if master, err = newMasterProvider(settings); err != nil { + return nil, err + } + if slave, err = newSlaveProvider(settings); err != nil { + return nil, err + } + if syncMgr, err = newSyncManager(settings.RedisCluster, master, slave); err != nil { + return nil, err + } + + return &haClient{ + master: master, + slave: slave, + syncMgr: syncMgr, + }, nil +} + +// newMasterProvider new master provider +func newMasterProvider(settings cc.Repository) (Provider, error) { + switch strings.ToUpper(string(settings.StorageType)) { + case string(cc.S3): + return newCosProvider(settings.BaseRepo, settings.RedisCluster) + case string(cc.BkRepo): + return newBKRepoProvider(settings.BaseRepo, settings.RedisCluster) + } + return nil, fmt.Errorf("unsupported storage type: %s", settings.StorageType) +} + +// newSlaveProvider new slave provider +func newSlaveProvider(settings cc.Repository) (Provider, error) { + switch strings.ToUpper(string(settings.Slave.StorageType)) { + case string(cc.S3): + return newCosProvider(settings.Slave, settings.RedisCluster) + case string(cc.BkRepo): + return newBKRepoProvider(settings.Slave, settings.RedisCluster) + } + return nil, fmt.Errorf("unsupported storage type: %s", settings.StorageType) +} + +// newHAProvider new ha provider +func newHAProvider(settings cc.Repository) (Provider, error) { + p, err := newHAClient(settings) + if err != nil { + return nil, err + } + + var c VariableCacher + c, err = newVariableCacher(settings.RedisCluster, p) + if err != nil { + return nil, err + } + + return &repoProvider{ + BaseProvider: p, + HAEnhancer: p.(*haClient), + VariableCacher: c, + }, nil +} + +// newSyncManager new a sync manager +func newSyncManager(redisConf cc.RedisCluster, master, slave Provider) (*SyncManager, error) { + // init redis client + client, err := bedis.NewRedisCache(redisConf) + if err != nil { + return nil, fmt.Errorf("new redis cluster failed, err: %v", err) + } + + return &SyncManager{ + queue: &syncQueue{client: client}, + master: master, + slave: slave, + }, nil +} + +// PushToQueue pushes the file metadata msg to queue so that the master's write operations can be received by slave +func (s *SyncManager) PushToQueue(kt *kit.Kit, sign string) error { + if err := s.QueueClient().LPush(kt.Ctx, s.QueueName(), s.QueueMsg(kt.BizID, sign)); err != nil { + logs.Errorf("push file metadata to queue failed, redis lpush err: %v, rid: %s", err, kt.Rid) + } + return nil +} + +// QueueClient returns client for sync queue +func (s *SyncManager) QueueClient() bedis.Client { + return s.queue.client +} + +// QueueName returns name of sync queue +func (s *SyncManager) QueueName() string { + return "sync_repo_queue" +} + +// AckQueueName returns name of ack queue +func (s *SyncManager) AckQueueName() string { + return "sync_repo_ack_queue" +} + +// QueueMsg returns msg for sync queue +func (s *SyncManager) QueueMsg(bizID uint32, sign string) string { + return fmt.Sprintf("%d_%s", bizID, sign) +} + +// ParseQueueMsg parses msg of sync queue +func (s *SyncManager) ParseQueueMsg(msg string) (uint32, string, error) { + elements := strings.Split(msg, "_") + if len(elements) != 2 { + return 0, "", fmt.Errorf("parse queue msg into two elements by '_' failed, msg: %s", msg) + } + + bizID, err := strconv.ParseInt(elements[0], 10, 64) + if err != nil { + return 0, "", fmt.Errorf("parse queue msg's bizID failed, msg: %s, bizID: %s", msg, elements[0]) + } + return uint32(bizID), elements[1], nil +} + +// ErrNoFileInMaster is error of no file in master +var ErrNoFileInMaster = errors.New("file not found in master") + +// Sync syncs file from master to slave +func (s *SyncManager) Sync(kt *kit.Kit, sign string) (skip bool, err error) { + _, err = s.slave.Metadata(kt, sign) + // the file already exists in slave repo, return directly + if err == nil { + return true, nil + } + // if the error is not 404 which means the slave repo service has some trouble, return directly + if err != errf.ErrFileContentNotFound { + return false, fmt.Errorf("sync file from master to slave failed, slave metadata err: %v", err) + } + + reader, _, err := s.master.Download(kt, sign) + if err != nil { + if err == errf.ErrFileContentNotFound { + return false, ErrNoFileInMaster + } + return false, fmt.Errorf("sync file from master to slave failed, master download err: %v", err) + } + defer reader.Close() + + // stream the downloaded content to slave repo for uploading + _, err = s.slave.Upload(kt, sign, reader) + if err != nil { + return false, fmt.Errorf("sync file from master to slave failed, slave upload err: %v", err) + } + + return false, nil +} diff --git a/bcs-services/bcs-bscp/pkg/dal/repository/repository.go b/bcs-services/bcs-bscp/pkg/dal/repository/repository.go index c6e83477d2..f3a34b784b 100644 --- a/bcs-services/bcs-bscp/pkg/dal/repository/repository.go +++ b/bcs-services/bcs-bscp/pkg/dal/repository/repository.go @@ -14,7 +14,6 @@ package repository import ( - "fmt" "io" "net" "net/http" @@ -79,7 +78,7 @@ type DecoratorInter interface { // ObjectDownloader 文件下载 type ObjectDownloader interface { - DownloadLink(kt *kit.Kit, sign string, fetchLimit uint32) (string, error) + DownloadLink(kt *kit.Kit, sign string, fetchLimit uint32) ([]string, error) AsyncDownload(kt *kit.Kit, sign string) (string, error) AsyncDownloadStatus(kt *kit.Kit, sign string, taskID string) (bool, error) URIDecorator(bizID uint32) DecoratorInter @@ -96,9 +95,15 @@ type BaseProvider interface { Metadata(kt *kit.Kit, sign string) (*ObjectMetadata, error) } +// HAEnhancer high availability enhancer interface +type HAEnhancer interface { + SyncManager() *SyncManager +} + // Provider repo provider interface type Provider interface { BaseProvider + HAEnhancer VariableCacher } @@ -208,16 +213,15 @@ func newUriDecoratorInter(bizID uint32) DecoratorInter { // repoProvider implements interface Provider type repoProvider struct { BaseProvider + HAEnhancer VariableCacher } // NewProvider init provider factory by storage type func NewProvider(conf cc.Repository) (Provider, error) { - switch strings.ToUpper(string(conf.StorageType)) { - case string(cc.S3): - return newCosProvider(conf) - case string(cc.BkRepo): - return newBKRepoProvider(conf) + if conf.EnableHA { + return newHAProvider(conf) } - return nil, fmt.Errorf("store with type %s is not supported", conf.StorageType) + + return newMasterProvider(conf) } diff --git a/bcs-services/bcs-bscp/pkg/metrics/metric.go b/bcs-services/bcs-bscp/pkg/metrics/metric.go index 421a6ed643..0b5cb823d0 100644 --- a/bcs-services/bcs-bscp/pkg/metrics/metric.go +++ b/bcs-services/bcs-bscp/pkg/metrics/metric.go @@ -71,6 +71,9 @@ const ( // RestfulSubSys defines rest server's sub system RestfulSubSys = "restful" + + // RepoSyncSubSys defines repo syncer sub system + RepoSyncSubSys = "repo_syncer" ) // labels diff --git a/bcs-services/bcs-bscp/pkg/protocol/feed-server/feed_server.pb.go b/bcs-services/bcs-bscp/pkg/protocol/feed-server/feed_server.pb.go index 4e7b518d3a..0197b6cdbb 100644 --- a/bcs-services/bcs-bscp/pkg/protocol/feed-server/feed_server.pb.go +++ b/bcs-services/bcs-bscp/pkg/protocol/feed-server/feed_server.pb.go @@ -1003,8 +1003,9 @@ type GetDownloadURLResp struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` - WaitTimeMil int64 `protobuf:"varint,2,opt,name=wait_time_mil,json=waitTimeMil,proto3" json:"wait_time_mil,omitempty"` // the time(milliseconds) that the client should wait before downloading file + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` // compatible with old versions + WaitTimeMil int64 `protobuf:"varint,2,opt,name=wait_time_mil,json=waitTimeMil,proto3" json:"wait_time_mil,omitempty"` // the time(milliseconds) that the client should wait before downloading file + Urls []string `protobuf:"bytes,3,rep,name=urls,proto3" json:"urls,omitempty"` // used for high availability } func (x *GetDownloadURLResp) Reset() { @@ -1053,6 +1054,13 @@ func (x *GetDownloadURLResp) GetWaitTimeMil() int64 { return 0 } +func (x *GetDownloadURLResp) GetUrls() []string { + if x != nil { + return x.Urls + } + return nil +} + // App 对外简单版本 type App struct { state protoimpl.MessageState @@ -2086,176 +2094,177 @@ var file_feed_server_proto_rawDesc = []byte{ 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x4a, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x44, 0x6f, + 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x5e, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x22, 0x0a, 0x0d, 0x77, 0x61, 0x69, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6d, 0x69, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x77, 0x61, 0x69, 0x74, 0x54, 0x69, 0x6d, 0x65, - 0x4d, 0x69, 0x6c, 0x22, 0x78, 0x0a, 0x03, 0x41, 0x70, 0x70, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, - 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x2c, 0x0a, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x62, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x73, - 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x3a, 0x0a, - 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x70, 0x70, 0x73, 0x52, 0x65, 0x71, 0x12, 0x15, 0x0a, 0x06, - 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x62, 0x69, - 0x7a, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x05, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x22, 0x2d, 0x0a, 0x0c, 0x4c, 0x69, 0x73, - 0x74, 0x41, 0x70, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x12, 0x1d, 0x0a, 0x04, 0x61, 0x70, 0x70, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x41, - 0x70, 0x70, 0x52, 0x04, 0x61, 0x70, 0x70, 0x73, 0x22, 0x66, 0x0a, 0x0d, 0x50, 0x75, 0x6c, 0x6c, - 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, 0x71, 0x12, 0x15, 0x0a, 0x06, 0x62, 0x69, 0x7a, + 0x4d, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x22, 0x78, 0x0a, 0x03, 0x41, 0x70, 0x70, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x2c, 0x0a, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x62, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x52, + 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x22, 0x3a, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x70, 0x70, 0x73, 0x52, 0x65, 0x71, + 0x12, 0x15, 0x0a, 0x06, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x05, 0x62, 0x69, 0x7a, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x61, 0x74, 0x63, 0x68, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x22, 0x2d, 0x0a, + 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x70, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x12, 0x1d, 0x0a, + 0x04, 0x61, 0x70, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, + 0x66, 0x73, 0x2e, 0x41, 0x70, 0x70, 0x52, 0x04, 0x61, 0x70, 0x70, 0x73, 0x22, 0x66, 0x0a, 0x0d, + 0x50, 0x75, 0x6c, 0x6c, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, 0x71, 0x12, 0x15, 0x0a, + 0x06, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x62, + 0x69, 0x7a, 0x49, 0x64, 0x12, 0x28, 0x0a, 0x08, 0x61, 0x70, 0x70, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x41, 0x70, + 0x70, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x07, 0x61, 0x70, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x14, + 0x0a, 0x05, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x6d, + 0x61, 0x74, 0x63, 0x68, 0x22, 0x58, 0x0a, 0x0e, 0x50, 0x75, 0x6c, 0x6c, 0x4b, 0x76, 0x4d, 0x65, + 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x6c, 0x65, + 0x61, 0x73, 0x65, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x08, 0x6b, 0x76, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x4b, + 0x76, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x07, 0x6b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x22, 0xd5, + 0x01, 0x0a, 0x06, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x17, 0x0a, 0x07, 0x6b, + 0x76, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, 0x76, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x2c, 0x0a, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x62, 0x62, 0x61, 0x73, 0x65, 0x2e, + 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x0d, 0x6b, 0x76, 0x5f, 0x61, 0x74, 0x74, 0x61, 0x63, 0x68, 0x6d, + 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x62, 0x6b, 0x76, + 0x2e, 0x4b, 0x76, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0c, 0x6b, + 0x76, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x39, 0x0a, 0x0c, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x16, 0x2e, 0x70, 0x62, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x53, 0x70, 0x65, 0x63, 0x22, 0x62, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4b, 0x76, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x65, 0x71, 0x12, 0x15, 0x0a, 0x06, 0x62, 0x69, 0x7a, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x62, 0x69, 0x7a, 0x49, 0x64, 0x12, 0x28, + 0x0a, 0x08, 0x61, 0x70, 0x70, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x0d, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x41, 0x70, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x52, + 0x07, 0x61, 0x70, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x3f, 0x0a, 0x0e, 0x47, 0x65, + 0x74, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x65, 0x73, 0x70, 0x12, 0x17, 0x0a, 0x07, + 0x6b, 0x76, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, + 0x76, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xee, 0x01, 0x0a, 0x10, + 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, + 0x12, 0x15, 0x0a, 0x06, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x05, 0x62, 0x69, 0x7a, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0b, 0x62, 0x6b, 0x5f, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x6b, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x64, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x64, 0x12, 0x25, 0x0a, + 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x46, + 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, + 0x61, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x44, 0x69, 0x72, 0x22, 0x2c, 0x0a, 0x11, + 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, + 0x70, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x22, 0x48, 0x0a, 0x16, 0x41, 0x73, + 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x71, 0x12, 0x15, 0x0a, 0x06, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x62, 0x69, 0x7a, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, + 0x61, 0x73, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, + 0x73, 0x6b, 0x49, 0x64, 0x22, 0x4c, 0x0a, 0x17, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, + 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x12, + 0x31, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x19, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, + 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x22, 0x68, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, + 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x65, 0x71, 0x12, 0x15, 0x0a, 0x06, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x62, 0x69, 0x7a, 0x49, 0x64, 0x12, 0x28, 0x0a, 0x08, 0x61, 0x70, 0x70, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x41, 0x70, 0x70, 0x4d, 0x65, 0x74, - 0x61, 0x52, 0x07, 0x61, 0x70, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x61, - 0x74, 0x63, 0x68, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x61, 0x74, 0x63, 0x68, - 0x22, 0x58, 0x0a, 0x0e, 0x50, 0x75, 0x6c, 0x6c, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, - 0x73, 0x70, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, - 0x64, 0x12, 0x27, 0x0a, 0x08, 0x6b, 0x76, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x4b, 0x76, 0x4d, 0x65, 0x74, - 0x61, 0x52, 0x07, 0x6b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x22, 0xd5, 0x01, 0x0a, 0x06, 0x4b, - 0x76, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x76, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, 0x76, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x2c, 0x0a, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x62, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x76, 0x69, - 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x37, - 0x0a, 0x0d, 0x6b, 0x76, 0x5f, 0x61, 0x74, 0x74, 0x61, 0x63, 0x68, 0x6d, 0x65, 0x6e, 0x74, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x62, 0x6b, 0x76, 0x2e, 0x4b, 0x76, 0x41, - 0x74, 0x74, 0x61, 0x63, 0x68, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0c, 0x6b, 0x76, 0x41, 0x74, 0x74, - 0x61, 0x63, 0x68, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x39, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x74, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, - 0x70, 0x62, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x53, 0x70, - 0x65, 0x63, 0x22, 0x62, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x52, 0x65, 0x71, 0x12, 0x15, 0x0a, 0x06, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x05, 0x62, 0x69, 0x7a, 0x49, 0x64, 0x12, 0x28, 0x0a, 0x08, 0x61, 0x70, - 0x70, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, - 0x62, 0x66, 0x73, 0x2e, 0x41, 0x70, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x07, 0x61, 0x70, 0x70, - 0x4d, 0x65, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x3f, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x4b, 0x76, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x65, 0x73, 0x70, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x76, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, 0x76, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xee, 0x01, 0x0a, 0x10, 0x41, 0x73, 0x79, 0x6e, - 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x12, 0x15, 0x0a, 0x06, - 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x62, 0x69, - 0x7a, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0b, 0x62, 0x6b, 0x5f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x6b, 0x41, 0x67, 0x65, 0x6e, - 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, - 0x12, 0x2b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4d, - 0x65, 0x74, 0x61, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x19, 0x0a, - 0x08, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x66, 0x69, 0x6c, 0x65, 0x44, 0x69, 0x72, 0x22, 0x2c, 0x0a, 0x11, 0x41, 0x73, 0x79, 0x6e, - 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x12, 0x17, 0x0a, - 0x07, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x22, 0x48, 0x0a, 0x16, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, - 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, - 0x12, 0x15, 0x0a, 0x06, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x05, 0x62, 0x69, 0x7a, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x61, 0x73, 0x6b, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, - 0x22, 0x4c, 0x0a, 0x17, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, - 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x12, 0x31, 0x0a, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x70, 0x62, - 0x66, 0x73, 0x2e, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x68, - 0x0a, 0x13, 0x47, 0x65, 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x65, 0x71, 0x12, 0x15, 0x0a, 0x06, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x62, 0x69, 0x7a, 0x49, 0x64, 0x12, 0x28, 0x0a, 0x08, - 0x61, 0x70, 0x70, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, - 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x41, 0x70, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x07, 0x61, - 0x70, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x2a, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x53, - 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x22, 0x37, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, - 0x65, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x12, 0x20, 0x0a, 0x04, 0x64, - 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x70, 0x62, 0x66, 0x73, - 0x2e, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3f, 0x0a, - 0x13, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, - 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x49, 0x4e, 0x47, - 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x02, 0x32, 0xf7, - 0x07, 0x0a, 0x08, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x3a, 0x0a, 0x09, 0x48, - 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x16, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, - 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x1a, 0x13, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x09, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x69, 0x6e, 0x67, 0x12, 0x13, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x13, 0x2e, 0x70, 0x62, 0x66, 0x73, - 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, - 0x12, 0x38, 0x0a, 0x05, 0x57, 0x61, 0x74, 0x63, 0x68, 0x12, 0x13, 0x2e, 0x70, 0x62, 0x66, 0x73, - 0x2e, 0x53, 0x69, 0x64, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x16, - 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x46, 0x65, 0x65, 0x64, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x48, 0x0a, 0x0f, 0x50, 0x75, - 0x6c, 0x6c, 0x41, 0x70, 0x70, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x18, 0x2e, - 0x70, 0x62, 0x66, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x41, 0x70, 0x70, 0x46, 0x69, 0x6c, 0x65, - 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, 0x71, 0x1a, 0x19, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x50, - 0x75, 0x6c, 0x6c, 0x41, 0x70, 0x70, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, - 0x73, 0x70, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, - 0x6f, 0x61, 0x64, 0x55, 0x52, 0x4c, 0x12, 0x17, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x47, 0x65, - 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x1a, - 0x18, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, - 0x61, 0x64, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x6a, 0x0a, 0x0a, 0x50, - 0x75, 0x6c, 0x6c, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x13, 0x2e, 0x70, 0x62, 0x66, 0x73, - 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, 0x71, 0x1a, 0x14, - 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, - 0x52, 0x65, 0x73, 0x70, 0x22, 0x31, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2b, 0x3a, 0x01, 0x2a, 0x22, - 0x26, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x66, 0x65, 0x65, 0x64, 0x2f, 0x62, 0x69, - 0x7a, 0x2f, 0x7b, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x7d, 0x2f, 0x6b, 0x76, 0x73, 0x2f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x6c, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4b, 0x76, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x13, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x47, 0x65, 0x74, - 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x70, 0x62, 0x66, - 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x22, 0x33, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2d, 0x3a, 0x01, 0x2a, 0x22, 0x28, 0x2f, 0x61, 0x70, - 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x66, 0x65, 0x65, 0x64, 0x2f, 0x62, 0x69, 0x7a, 0x2f, 0x7b, 0x62, - 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x7d, 0x2f, 0x6b, 0x76, 0x73, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x2f, - 0x7b, 0x6b, 0x65, 0x79, 0x7d, 0x12, 0x33, 0x0a, 0x08, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x70, 0x70, - 0x73, 0x12, 0x11, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x70, 0x70, - 0x73, 0x52, 0x65, 0x71, 0x1a, 0x12, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x41, 0x70, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x41, 0x73, - 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x16, 0x2e, 0x70, 0x62, - 0x66, 0x73, 0x2e, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, - 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x41, 0x73, 0x79, 0x6e, 0x63, - 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x54, - 0x0a, 0x13, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1c, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x41, 0x73, 0x79, - 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x71, 0x1a, 0x1d, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x41, 0x73, 0x79, 0x6e, 0x63, - 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x22, 0x00, 0x12, 0x7f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, - 0x65, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x19, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, + 0x61, 0x52, 0x07, 0x61, 0x70, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x2a, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x52, 0x65, 0x71, 0x1a, 0x1a, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x69, - 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, - 0x34, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2e, 0x3a, 0x01, 0x2a, 0x22, 0x29, 0x2f, 0x61, 0x70, 0x69, - 0x2f, 0x76, 0x31, 0x2f, 0x66, 0x65, 0x65, 0x64, 0x2f, 0x62, 0x69, 0x7a, 0x2f, 0x7b, 0x62, 0x69, - 0x7a, 0x5f, 0x69, 0x64, 0x7d, 0x2f, 0x6b, 0x76, 0x73, 0x2f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2f, - 0x7b, 0x6b, 0x65, 0x79, 0x7d, 0x12, 0x80, 0x01, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x53, 0x69, 0x6e, - 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x19, 0x2e, 0x70, 0x62, 0x66, 0x73, - 0x2e, 0x47, 0x65, 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x52, 0x65, 0x71, 0x1a, 0x19, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x53, - 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x22, - 0x37, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x31, 0x3a, 0x01, 0x2a, 0x22, 0x2c, 0x2f, 0x61, 0x70, 0x69, - 0x2f, 0x76, 0x31, 0x2f, 0x66, 0x65, 0x65, 0x64, 0x2f, 0x62, 0x69, 0x7a, 0x2f, 0x7b, 0x62, 0x69, - 0x7a, 0x5f, 0x69, 0x64, 0x7d, 0x2f, 0x6b, 0x76, 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2f, 0x7b, 0x6b, 0x65, 0x79, 0x7d, 0x42, 0x57, 0x5a, 0x55, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x54, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x42, 0x6c, - 0x75, 0x65, 0x4b, 0x69, 0x6e, 0x67, 0x2f, 0x62, 0x6b, 0x2d, 0x62, 0x63, 0x73, 0x2f, 0x62, 0x63, - 0x73, 0x2d, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x62, 0x63, 0x73, 0x2d, 0x62, - 0x73, 0x63, 0x70, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, - 0x2f, 0x66, 0x65, 0x65, 0x64, 0x2d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3b, 0x70, 0x62, 0x66, - 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x65, 0x73, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x37, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x53, + 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x12, + 0x20, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, + 0x70, 0x62, 0x66, 0x73, 0x2e, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x04, 0x64, 0x61, 0x74, + 0x61, 0x2a, 0x3f, 0x0a, 0x13, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, + 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, + 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, + 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, + 0x10, 0x02, 0x32, 0xf7, 0x07, 0x0a, 0x08, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, + 0x3a, 0x0a, 0x09, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x16, 0x2e, 0x70, + 0x62, 0x66, 0x73, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x1a, 0x13, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x48, 0x61, 0x6e, 0x64, + 0x73, 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x09, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x12, 0x13, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x1a, 0x13, 0x2e, + 0x70, 0x62, 0x66, 0x73, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x52, 0x65, + 0x73, 0x70, 0x22, 0x00, 0x12, 0x38, 0x0a, 0x05, 0x57, 0x61, 0x74, 0x63, 0x68, 0x12, 0x13, 0x2e, + 0x70, 0x62, 0x66, 0x73, 0x2e, 0x53, 0x69, 0x64, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x65, + 0x74, 0x61, 0x1a, 0x16, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x46, 0x65, 0x65, 0x64, 0x57, 0x61, + 0x74, 0x63, 0x68, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x48, + 0x0a, 0x0f, 0x50, 0x75, 0x6c, 0x6c, 0x41, 0x70, 0x70, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, + 0x61, 0x12, 0x18, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x41, 0x70, 0x70, + 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, 0x71, 0x1a, 0x19, 0x2e, 0x70, 0x62, + 0x66, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x41, 0x70, 0x70, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, + 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x44, + 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x55, 0x52, 0x4c, 0x12, 0x17, 0x2e, 0x70, 0x62, 0x66, + 0x73, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x55, 0x52, 0x4c, + 0x52, 0x65, 0x71, 0x1a, 0x18, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, + 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, + 0x6a, 0x0a, 0x0a, 0x50, 0x75, 0x6c, 0x6c, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x13, 0x2e, + 0x70, 0x62, 0x66, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x52, + 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x4b, 0x76, + 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x22, 0x31, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2b, + 0x3a, 0x01, 0x2a, 0x22, 0x26, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x66, 0x65, 0x65, + 0x64, 0x2f, 0x62, 0x69, 0x7a, 0x2f, 0x7b, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x7d, 0x2f, 0x6b, + 0x76, 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x6c, 0x0a, 0x0a, 0x47, + 0x65, 0x74, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x13, 0x2e, 0x70, 0x62, 0x66, 0x73, + 0x2e, 0x47, 0x65, 0x74, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x14, + 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x22, 0x33, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2d, 0x3a, 0x01, 0x2a, 0x22, + 0x28, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x66, 0x65, 0x65, 0x64, 0x2f, 0x62, 0x69, + 0x7a, 0x2f, 0x7b, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x7d, 0x2f, 0x6b, 0x76, 0x73, 0x2f, 0x64, + 0x61, 0x74, 0x61, 0x2f, 0x7b, 0x6b, 0x65, 0x79, 0x7d, 0x12, 0x33, 0x0a, 0x08, 0x4c, 0x69, 0x73, + 0x74, 0x41, 0x70, 0x70, 0x73, 0x12, 0x11, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x41, 0x70, 0x70, 0x73, 0x52, 0x65, 0x71, 0x1a, 0x12, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x41, 0x70, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x42, + 0x0a, 0x0d, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x12, + 0x16, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, + 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x41, + 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, + 0x22, 0x00, 0x12, 0x54, 0x0a, 0x13, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, + 0x6f, 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1c, 0x2e, 0x70, 0x62, 0x66, 0x73, + 0x2e, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x1a, 0x1d, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x41, + 0x73, 0x79, 0x6e, 0x63, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x7f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, + 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x19, 0x2e, 0x70, + 0x62, 0x66, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x1a, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, 0x47, + 0x65, 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x22, 0x34, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2e, 0x3a, 0x01, 0x2a, 0x22, 0x29, + 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x66, 0x65, 0x65, 0x64, 0x2f, 0x62, 0x69, 0x7a, + 0x2f, 0x7b, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x7d, 0x2f, 0x6b, 0x76, 0x73, 0x2f, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2f, 0x7b, 0x6b, 0x65, 0x79, 0x7d, 0x12, 0x80, 0x01, 0x0a, 0x0f, 0x47, 0x65, + 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x19, 0x2e, + 0x70, 0x62, 0x66, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x19, 0x2e, 0x70, 0x62, 0x66, 0x73, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4b, 0x76, 0x4d, 0x65, 0x74, 0x61, 0x52, + 0x65, 0x73, 0x70, 0x22, 0x37, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x31, 0x3a, 0x01, 0x2a, 0x22, 0x2c, + 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x66, 0x65, 0x65, 0x64, 0x2f, 0x62, 0x69, 0x7a, + 0x2f, 0x7b, 0x62, 0x69, 0x7a, 0x5f, 0x69, 0x64, 0x7d, 0x2f, 0x6b, 0x76, 0x73, 0x2f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x7b, 0x6b, 0x65, 0x79, 0x7d, 0x42, 0x57, 0x5a, 0x55, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x54, 0x65, 0x6e, 0x63, 0x65, + 0x6e, 0x74, 0x42, 0x6c, 0x75, 0x65, 0x4b, 0x69, 0x6e, 0x67, 0x2f, 0x62, 0x6b, 0x2d, 0x62, 0x63, + 0x73, 0x2f, 0x62, 0x63, 0x73, 0x2d, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x62, + 0x63, 0x73, 0x2d, 0x62, 0x73, 0x63, 0x70, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x2f, 0x66, 0x65, 0x65, 0x64, 0x2d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x3b, 0x70, 0x62, 0x66, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/bcs-services/bcs-bscp/pkg/protocol/feed-server/feed_server.proto b/bcs-services/bcs-bscp/pkg/protocol/feed-server/feed_server.proto index 06874846d7..2df7aed515 100644 --- a/bcs-services/bcs-bscp/pkg/protocol/feed-server/feed_server.proto +++ b/bcs-services/bcs-bscp/pkg/protocol/feed-server/feed_server.proto @@ -161,8 +161,9 @@ message GetDownloadURLReq { } message GetDownloadURLResp { - string url = 1; - int64 wait_time_mil = 2; // the time(milliseconds) that the client should wait before downloading file + string url = 1; // compatible with old versions + int64 wait_time_mil = 2; // the time(milliseconds) that the client should wait before downloading file + repeated string urls = 3; // used for high availability } // App 对外简单版本 diff --git a/bcs-services/bcs-bscp/pkg/space/space.go b/bcs-services/bcs-bscp/pkg/space/space.go index 190048efb8..cd9990ffef 100644 --- a/bcs-services/bcs-bscp/pkg/space/space.go +++ b/bcs-services/bcs-bscp/pkg/space/space.go @@ -143,8 +143,8 @@ func (s *Manager) AllSpaces() []*Space { return s.cachedSpace } -// allCMDBSpaces 返回全量CMDB空间 -func (s *Manager) allCMDBSpaces() map[string]struct{} { +// AllCMDBSpaces 返回全量CMDB空间 +func (s *Manager) AllCMDBSpaces() map[string]struct{} { s.mtx.Lock() defer s.mtx.Unlock() @@ -242,7 +242,7 @@ func BuildSpaceUid(t Type, id string) string { // HasCMDBSpace checks if cmdb space exists func (s *Manager) HasCMDBSpace(spaceId string) bool { - if _, ok := s.allCMDBSpaces()[spaceId]; ok { + if _, ok := s.AllCMDBSpaces()[spaceId]; ok { return true } @@ -259,7 +259,7 @@ func (s *Manager) HasCMDBSpace(spaceId string) bool { s.requestedCmdbSpaces[spaceId] = struct{}{} s.mtx.Unlock() - if _, ok := s.allCMDBSpaces()[spaceId]; ok { + if _, ok := s.AllCMDBSpaces()[spaceId]; ok { return true } return false diff --git a/bcs-services/bcs-bscp/pkg/thirdparty/repo/client.go b/bcs-services/bcs-bscp/pkg/thirdparty/repo/client.go index 8c3e76fada..d354e08d5b 100644 --- a/bcs-services/bcs-bscp/pkg/thirdparty/repo/client.go +++ b/bcs-services/bcs-bscp/pkg/thirdparty/repo/client.go @@ -30,7 +30,7 @@ import ( // Client is repo client. type Client struct { - config cc.Repository + config cc.BaseRepo // http client instance client rest.ClientInterface // http header info @@ -38,7 +38,7 @@ type Client struct { } // NewClient new repo client. -func NewClient(repoSetting cc.Repository, reg prometheus.Registerer) (*Client, error) { +func NewClient(repoSetting cc.BaseRepo, reg prometheus.Registerer) (*Client, error) { tls := &tools.TLSConfig{ InsecureSkipVerify: repoSetting.BkRepo.TLS.InsecureSkipVerify, CertFile: repoSetting.BkRepo.TLS.CertFile, diff --git a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/api/utils.go b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/api/utils.go index b53198f73a..94686c8bb6 100644 --- a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/api/utils.go +++ b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/api/utils.go @@ -241,8 +241,8 @@ func MapToAwsTaints(taints []*proto.Taint) []*eks.Taint { for _, v := range taints { key := v.Key value := v.Value - effect := v.Effect - result = append(result, &eks.Taint{Key: &key, Value: &value, Effect: &effect}) + result = append(result, &eks.Taint{Key: &key, Value: &value, Effect: aws.String(taintTransEffect(v.Effect))}) + } return result } @@ -308,8 +308,8 @@ func CreateTagSpecs(instanceTags map[string]*string) []*ec2.LaunchTemplateTagSpe } } -// generateAwsCreateLaunchTemplateInput generate Aws CreateLaunchTemplateInput -func generateAwsCreateLaunchTemplateInput(input *CreateLaunchTemplateInput) *ec2.CreateLaunchTemplateInput { +// generateAwsLaunchTemplateInput generate Aws CreateLaunchTemplateInput +func generateAwsLaunchTemplateInput(input *CreateLaunchTemplateInput) *ec2.CreateLaunchTemplateInput { awsInput := &ec2.CreateLaunchTemplateInput{ LaunchTemplateName: input.LaunchTemplateName, TagSpecifications: generateAwsTagSpecs(input.TagSpecifications), diff --git a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/cloud.go b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/cloud.go index 8ce7f04f29..b348cebe9c 100644 --- a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/cloud.go +++ b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/cloud.go @@ -14,6 +14,7 @@ package aws import ( "fmt" + "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/common" "sync" "github.com/aws/aws-sdk-go/service/ec2" @@ -41,6 +42,22 @@ type CloudInfoManager struct { // InitCloudClusterDefaultInfo init cluster defaultConfig func (c *CloudInfoManager) InitCloudClusterDefaultInfo(cls *cmproto.Cluster, opt *cloudprovider.InitClusterConfigOption) error { + // call aws interface to init cluster defaultConfig + if c == nil || cls == nil { + return fmt.Errorf("%s InitCloudClusterDefaultInfo request is empty", cloudName) + } + + if opt == nil || opt.Cloud == nil { + return fmt.Errorf("%s InitCloudClusterDefaultInfo option is empty", cloudName) + } + + if len(cls.ManageType) == 0 { + cls.ManageType = common.ClusterManageTypeIndependent + } + if len(cls.ClusterCategory) == 0 { + cls.ClusterCategory = common.Builder + } + return nil } diff --git a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/nodegroup.go b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/nodegroup.go index d88237b84f..c5e6803c0e 100644 --- a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/nodegroup.go +++ b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/nodegroup.go @@ -107,10 +107,14 @@ func (ng *NodeGroup) generateUpdateNodegroupConfigInput(group *proto.NodeGroup, input := &eks.UpdateNodegroupConfigInput{ ClusterName: &cluster, NodegroupName: &group.CloudNodeGroupID, - Labels: &eks.UpdateLabelsPayload{ - AddOrUpdateLabels: aws.StringMap(group.Labels), - }, } + + if len(group.GetNodeTemplate().GetLabels()) > 0 { + input.Labels = &eks.UpdateLabelsPayload{ + AddOrUpdateLabels: aws.StringMap(group.GetNodeTemplate().GetLabels()), + } + } + if group.AutoScaling != nil { input.ScalingConfig = &eks.NodegroupScalingConfig{ MaxSize: aws.Int64(int64(group.AutoScaling.MaxSize)), diff --git a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/taskmgr.go b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/taskmgr.go index 976b835bfb..d43d30dce3 100644 --- a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/taskmgr.go +++ b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/taskmgr.go @@ -14,6 +14,7 @@ package aws import ( "fmt" + icommon "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/common" "strconv" "strings" "sync" @@ -27,7 +28,6 @@ import ( "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks" "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/cloudprovider/common" "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/cloudprovider/template" - icommon "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/common" ) var taskMgr sync.Once @@ -160,69 +160,76 @@ func (t *Task) BuildCreateClusterTask(cls *proto.Cluster, opt *cloudprovider.Cre createClusterTask.BuildCreateCloudNodeGroupStep(task) // step4: check cluster nodegroups status createClusterTask.BuildCheckCloudNodeGroupStatusStep(task) - if desireSize != 0 { - // step5: check cluster nodegroups status - createClusterTask.BuildCheckClusterNodesStatusStep(task) - // step6: update nodes to DB - createClusterTask.BuildUpdateNodesToDBStep(task) - // step7: install cluster watch component - common.BuildWatchComponentTaskStep(task, cls, "") - // step8: 若需要则设置节点注解 - common.BuildNodeAnnotationsTaskStep(task, cls.ClusterID, nil, func() map[string]string { - if opt.NodeTemplate != nil && len(opt.NodeTemplate.GetAnnotations()) > 0 { - return opt.NodeTemplate.GetAnnotations() - } - return nil - }()) - // step9 install gse agent - common.BuildInstallGseAgentTaskStep(task, &common.GseInstallInfo{ - ClusterId: cls.ClusterID, - BusinessId: cls.BusinessID, - CloudArea: cls.GetClusterBasicSettings().GetArea(), - User: nodeGroups[0].GetLaunchTemplate().GetInitLoginUsername(), - Passwd: nodeGroups[0].GetLaunchTemplate().GetInitLoginPassword(), - KeyInfo: nodeGroups[0].GetLaunchTemplate().GetKeyPair(), - AllowReviseCloudId: icommon.True, - }, cloudprovider.WithStepAllowSkip(true)) - } } - // step10: transfer host module - moduleID := cls.GetClusterBasicSettings().GetModule().GetWorkerModuleID() - if moduleID != "" { - common.BuildTransferHostModuleStep(task, cls.BusinessID, cls.GetClusterBasicSettings().GetModule(). - GetWorkerModuleID(), cls.GetClusterBasicSettings().GetModule().GetMasterModuleID()) - } + if desireSize != 0 { + // step5: check cluster nodegroups status + createClusterTask.BuildCheckClusterNodesStatusStep(task) + // step6: update nodes to DB + createClusterTask.BuildUpdateNodesToDBStep(task) + // step7: install gse agent + common.BuildInstallGseAgentTaskStep(task, &common.GseInstallInfo{ + ClusterId: cls.ClusterID, + BusinessId: cls.BusinessID, + CloudArea: cls.GetClusterBasicSettings().GetArea(), + User: nodeGroups[0].GetLaunchTemplate().GetInitLoginUsername(), + Passwd: nodeGroups[0].GetLaunchTemplate().GetInitLoginPassword(), + KeyInfo: nodeGroups[0].GetLaunchTemplate().GetKeyPair(), + AllowReviseCloudId: icommon.True, + }, cloudprovider.WithStepAllowSkip(true)) + + // step8: transfer host module + moduleID := cls.GetClusterBasicSettings().GetModule().GetWorkerModuleID() + if moduleID != "" { + common.BuildTransferHostModuleStep(task, cls.BusinessID, cls.GetClusterBasicSettings().GetModule(). + GetWorkerModuleID(), cls.GetClusterBasicSettings().GetModule().GetMasterModuleID()) + } - // step11: 业务后置自定义流程: 支持标准运维任务 或者 后置脚本 - if opt.NodeTemplate != nil && len(opt.NodeTemplate.UserScript) > 0 { - common.BuildJobExecuteScriptStep(task, common.JobExecParas{ - ClusterID: cls.ClusterID, - Content: opt.NodeTemplate.UserScript, - // dynamic node ips - NodeIps: "", - Operator: opt.Operator, - StepName: common.PostInitStepJob, - Translate: common.PostInitJob, - }) - } - // business post define sops task or script - if opt.NodeTemplate != nil && opt.NodeTemplate.ScaleOutExtraAddons != nil { - err := template.BuildSopsFactory{ - StepName: template.UserAfterInit, - Cluster: cls, - Extra: template.ExtraInfo{ + // step9: 业务后置自定义流程: 支持标准运维任务 或者 后置脚本 + if len(nodeGroups) > 0 && nodeGroups[0].GetNodeTemplate() != nil && + len(nodeGroups[0].GetNodeTemplate().UserScript) > 0 { + common.BuildJobExecuteScriptStep(task, common.JobExecParas{ + ClusterID: cls.ClusterID, + Content: nodeGroups[0].GetNodeTemplate().UserScript, // dynamic node ips - NodeIPList: "", - NodeOperator: opt.Operator, - ShowSopsUrl: true, - TranslateMethod: template.UserPostInit, - }}.BuildSopsStep(task, opt.NodeTemplate.ScaleOutExtraAddons, false) - if err != nil { - return nil, fmt.Errorf("BuildCreateClusterTask business BuildBkSopsStepAction failed: %v", err) + NodeIps: "", + Operator: opt.Operator, + StepName: common.PostInitStepJob, + Translate: common.PostInitJob, + }) + } + // business post define sops task or script + if len(nodeGroups) > 0 && nodeGroups[0].GetNodeTemplate() != nil && nodeGroups[0].GetNodeTemplate().ScaleOutExtraAddons != nil { + err := template.BuildSopsFactory{ + StepName: template.UserAfterInit, + Cluster: cls, + Extra: template.ExtraInfo{ + // dynamic node ips + NodeIPList: "", + NodeOperator: opt.Operator, + ShowSopsUrl: true, + TranslateMethod: template.UserPostInit, + }}.BuildSopsStep(task, nodeGroups[0].GetNodeTemplate().ScaleOutExtraAddons, false) + if err != nil { + return nil, fmt.Errorf("BuildCreateClusterTask business BuildBkSopsStepAction failed: %v", err) + } } + + // step10: 若需要则设置节点注解 + common.BuildNodeAnnotationsTaskStep(task, cls.ClusterID, nil, func() map[string]string { + if len(nodeGroups) > 0 && len(nodeGroups[0].GetNodeTemplate().GetAnnotations()) > 0 { + return nodeGroups[0].GetNodeTemplate().GetAnnotations() + } + return nil + }()) + + // step11: remove inner nodes taints + common.BuildRemoveInnerTaintTaskStep(task, cls.ClusterID, cls.Provider) } + // step12: install cluster watch component + common.BuildWatchComponentTaskStep(task, cls, "") + // set current step if len(task.StepSequence) == 0 { return nil, fmt.Errorf("BuildCreateClusterTask task StepSequence empty") @@ -698,7 +705,7 @@ func (t *Task) BuildUpdateDesiredNodesTask(desired uint32, group *proto.NodeGrou common.BuildNodeAnnotationsTaskStep(task, opt.Cluster.ClusterID, nil, cloudprovider.GetAnnotationsByNg(opt.NodeGroup)) // step8: remove inner nodes taints - common.BuildRemoveInnerTaintTaskStep(task, group) + common.BuildRemoveInnerTaintTaskStep(task, group.ClusterID, group.Provider) // set current step if len(task.StepSequence) == 0 { diff --git a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks/createCluster.go b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks/createCluster.go index b19b2d5f8f..4f2172bcfc 100644 --- a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks/createCluster.go +++ b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks/createCluster.go @@ -104,35 +104,45 @@ func CreateEKSClusterTask(taskID string, stepName string) error { return nil } +// createEKSCluster 创建一个Amazon EKS集群,并更新集群信息。 func createEKSCluster(ctx context.Context, info *cloudprovider.CloudDependBasicInfo) ( string, error) { + // 从上下文中获取任务ID taskID := cloudprovider.GetTaskIDFromContext(ctx) cluster := info.Cluster + // 创建一个新的AWS客户端集 client, err := api.NewAWSClientSet(info.CmOption) if err != nil { return "", fmt.Errorf("create eksService failed") } + // 获取IAM角色信息 role, err := client.GetRole(&iam.GetRoleInput{RoleName: aws.String(cluster.GetClusterIamRole())}) if err != nil { + // 如果获取角色失败,记录错误日志并返回错误 blog.Errorf("GetRole[%s] failed, %v", taskID, err) return "", err } + // generateCreateClusterInput 生成创建集群的输入参数 input, err := generateCreateClusterInput(info, role.Arn) if err != nil { + // 如果生成输入参数失败,返回错误 return "", fmt.Errorf("generateCreateClusterInput failed, %v", err) } + // CreateEksCluster 调用API创建EKS集群 eksCluster, err := client.CreateEksCluster(input) if err != nil { + // 如果创建集群失败,返回错误 return "", fmt.Errorf("call CreateEksCluster failed, %v", err) } info.Cluster.SystemID = *eksCluster.Name info.Cluster.VpcID = *eksCluster.ResourcesVpcConfig.VpcId + // 更新集群信息 err = cloudprovider.UpdateCluster(info.Cluster) if err != nil { blog.Errorf("createEKSCluster[%s] UpdateCluster[%s] failed %s", @@ -146,12 +156,13 @@ func createEKSCluster(ctx context.Context, info *cloudprovider.CloudDependBasicI return *eksCluster.Name, nil } +// generateCreateClusterInput 根据提供的云提供商基本信息和角色ARN生成创建EKS集群的输入参数 func generateCreateClusterInput(info *cloudprovider.CloudDependBasicInfo, roleArn *string) ( *eks.CreateClusterInput, error) { var ( - cluster = info.Cluster - subnetIds = make([]string, 0) + cluster = info.Cluster // 获取集群信息 + subnetIds = make([]string, 0) // 初始化子网ID切片 err error ) @@ -164,25 +175,28 @@ func generateCreateClusterInput(info *cloudprovider.CloudDependBasicInfo, roleAr if len(info.Cluster.GetNetworkSettings().GetSubnetSource().GetExisted().GetIds()) > 0 { subnetIds = append(subnetIds, info.Cluster.GetNetworkSettings().GetSubnetSource().GetExisted().GetIds()...) } - } else { - subnetIds = strings.Split(cluster.ClusterBasicSettings.SubnetID, ",") } + // 如果subnetIds为空,则返回错误 + if len(subnetIds) == 0 { + return nil, errors.New("generateCreateClusterInput subnetIds is empty") + } + info.Cluster.NetworkSettings.EniSubnetIDs = subnetIds sgs := strings.Split(cluster.ClusterAdvanceSettings.ClusterConnectSetting.SecurityGroup, ",") input := &eks.CreateClusterInput{ - // 默认启用集群访问权限,认证模式为 AccessConfig: &eks.CreateAccessConfigRequest{ - AuthenticationMode: aws.String(api.ClusterAuthenticationModeAM), - BootstrapClusterCreatorAdminPermissions: aws.Bool(true), + AuthenticationMode: aws.String(api.ClusterAuthenticationModeAM), // 设置认证模式 + BootstrapClusterCreatorAdminPermissions: aws.Bool(true), // 默认启用集群创建者管理员权限 }, - Name: aws.String(cluster.ClusterName), - RoleArn: roleArn, + Name: aws.String(cluster.ClusterName), // 设置集群名称 + RoleArn: roleArn, // 设置角色ARN ResourcesVpcConfig: &eks.VpcConfigRequest{ - SubnetIds: aws.StringSlice(subnetIds), - SecurityGroupIds: aws.StringSlice(sgs), - EndpointPrivateAccess: aws.Bool(!cluster.ClusterAdvanceSettings.ClusterConnectSetting.IsExtranet), - EndpointPublicAccess: aws.Bool(cluster.ClusterAdvanceSettings.ClusterConnectSetting.IsExtranet), + SubnetIds: aws.StringSlice(subnetIds), // 设置子网ID + SecurityGroupIds: aws.StringSlice(sgs), // 设置安全组ID + EndpointPrivateAccess: aws.Bool(!cluster.ClusterAdvanceSettings.ClusterConnectSetting.IsExtranet), // 设置私有访问端点 + EndpointPublicAccess: aws.Bool(cluster.ClusterAdvanceSettings.ClusterConnectSetting.IsExtranet), // 设置公共访问端点 PublicAccessCidrs: func() []*string { + // 如果集群的网络设置中的公共访问CIDR存在,则返回对应的切片,否则返回nil if cluster.ClusterAdvanceSettings.ClusterConnectSetting.Internet == nil || cluster.ClusterAdvanceSettings.ClusterConnectSetting.Internet.PublicAccessCidrs == nil { return nil @@ -191,17 +205,19 @@ func generateCreateClusterInput(info *cloudprovider.CloudDependBasicInfo, roleAr }(), }, UpgradePolicy: func(setting *proto.ClusterBasicSetting) *eks.UpgradePolicyRequest { + // 如果升级策略未设置,则默认使用EXTENDED策略 if setting.UpgradePolicy == nil { - // 默认使用EXTENDED升级策略, 与aws保持一致 return &eks.UpgradePolicyRequest{SupportType: aws.String(api.ClusterUpdatePolicyExtended)} } + // 如果升级策略不是EXTENDED或STANDARD,则使用EXTENDED策略 if setting.UpgradePolicy.SupportType != api.ClusterUpdatePolicyExtended && setting.UpgradePolicy.SupportType != api.ClusterUpdatePolicyStandard { return &eks.UpgradePolicyRequest{SupportType: aws.String(api.ClusterUpdatePolicyExtended)} } + // 否则使用设置的升级策略 return &eks.UpgradePolicyRequest{SupportType: aws.String(setting.UpgradePolicy.SupportType)} }(cluster.ClusterBasicSettings), - Version: aws.String(cluster.ClusterBasicSettings.Version), + Version: aws.String(cluster.ClusterBasicSettings.Version), // 设置Kubernetes版本 } input.KubernetesNetworkConfig = generateKubernetesNetworkConfig(cluster) if len(cluster.ClusterBasicSettings.ClusterTags) > 0 { @@ -211,6 +227,7 @@ func generateCreateClusterInput(info *cloudprovider.CloudDependBasicInfo, roleAr return input, nil } +// generateKubernetesNetworkConfig network config func generateKubernetesNetworkConfig(cluster *proto.Cluster) *eks.KubernetesNetworkConfigRequest { req := &eks.KubernetesNetworkConfigRequest{} if cluster != nil && cluster.NetworkSettings != nil { @@ -345,10 +362,12 @@ func checkClusterStatus(ctx context.Context, info *cloudprovider.CloudDependBasi return nil } +// createAddon 函数用于在AWS EKS集群上创建默认的addons。 func createAddon(ctx context.Context, info *cloudprovider.CloudDependBasicInfo) error { + // 从上下文中获取任务ID,用于日志记录 taskID := cloudprovider.GetTaskIDFromContext(ctx) - // get awsCloud client + // 初始化AWS EKS客户端 cli, err := api.NewEksClient(info.CmOption) if err != nil { blog.Errorf("checkClusterStatus[%s] get aws client failed: %s", taskID, err.Error()) @@ -356,11 +375,14 @@ func createAddon(ctx context.Context, info *cloudprovider.CloudDependBasicInfo) return retErr } + // 遍历默认的addons列表 for _, addon := range defaultAddons { + // 调用CreateAddon方法创建addon _, err = cli.CreateAddon(&eks.CreateAddonInput{ - ClusterName: aws.String(info.Cluster.ClusterName), - AddonName: aws.String(addon), + ClusterName: aws.String(info.Cluster.ClusterName), // 设置集群名称 + AddonName: aws.String(addon), // 设置要创建的addon名称 }) + // 如果创建addon失败,则直接返回错误 if err != nil { return err } @@ -551,10 +573,10 @@ func checkClusterNodesStatus(ctx context.Context, info *cloudprovider.CloudDepen if errors.Is(err, context.DeadlineExceeded) { running, failure := make([]string, 0), make([]string, 0) - nodes, err := k8sOperator.ListClusterNodes(context.Background(), info.Cluster.ClusterID) // nolint - if err != nil { + nodes, errLocal := k8sOperator.ListClusterNodes(context.Background(), info.Cluster.ClusterID) // nolint + if errLocal != nil { blog.Errorf("checkClusterNodesStatus[%s] cluster[%s] failed: %v", taskID, info.Cluster.ClusterID, err) - return nil, nil, err + return nil, nil, errLocal } for _, ins := range nodes { @@ -620,9 +642,6 @@ func UpdateEKSNodesToDBTask(taskID string, stepName string) error { return retErr } - // sync clusterData to pass-cc - providerutils.SyncClusterInfoToPassCC(taskID, dependInfo.Cluster) - // sync cluster perms providerutils.AuthClusterResourceCreatorPerm(ctx, dependInfo.Cluster.ClusterID, dependInfo.Cluster.ClusterName, dependInfo.Cluster.Creator) @@ -643,7 +662,7 @@ func updateNodeToDB(ctx context.Context, state *cloudprovider.TaskState, info *c addSuccessNodes := state.Task.CommonParams[cloudprovider.SuccessClusterNodeIDsKey.String()] addFailureNodes := state.Task.CommonParams[cloudprovider.FailedClusterNodeIDsKey.String()] - nodeIPs, instanceIDs := make([]string, 0), make([]string, 0) + nodeIPs, instanceIDs, nodeNames := make([]string, 0), make([]string, 0), make([]string, 0) nmClient := api.NodeManager{} nodes := make([]*proto.Node, 0) @@ -685,6 +704,7 @@ func updateNodeToDB(ctx context.Context, state *cloudprovider.TaskState, info *c if utils.StringInSlice(n.NodeID, successInstanceID) { n.Status = common.StatusRunning nodeIPs = append(nodeIPs, n.InnerIP) + nodeNames = append(nodeNames, n.GetNodeName()) } else { n.Status = common.StatusAddNodesFailed } @@ -696,6 +716,9 @@ func updateNodeToDB(ctx context.Context, state *cloudprovider.TaskState, info *c } } state.Task.CommonParams[cloudprovider.NodeIPsKey.String()] = strings.Join(nodeIPs, ",") + state.Task.CommonParams[cloudprovider.NodeNamesKey.String()] = strings.Join(nodeNames, ",") + // dynamic inject paras + state.Task.CommonParams[cloudprovider.DynamicNodeIPListKey.String()] = strings.Join(nodeIPs, ",") return nil } diff --git a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks/createNodeGroup.go b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks/createNodeGroup.go index c3a06ab668..1793830407 100644 --- a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks/createNodeGroup.go +++ b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks/createNodeGroup.go @@ -191,13 +191,13 @@ func generateCreateNodegroupInput(group *proto.NodeGroup, cluster *proto.Cluster nodeGroup.CapacityType != aws.String(eks.CapacityTypesSpot) { nodeGroup.CapacityType = aws.String(eks.CapacityTypesOnDemand) } - if len(group.Labels) != 0 { - nodeGroup.Labels = aws.StringMap(group.Labels) + if group.NodeTemplate != nil && len(group.NodeTemplate.Labels) > 0 { + nodeGroup.Labels = aws.StringMap(group.NodeTemplate.Labels) } if len(group.Tags) != 0 { nodeGroup.Tags = aws.StringMap(group.Tags) } - if group.NodeTemplate != nil { + if group.NodeTemplate != nil && len(group.NodeTemplate.Taints) > 0 { nodeGroup.Taints = api.MapToTaints(group.NodeTemplate.Taints) } diff --git a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks/deleteCluster.go b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks/deleteCluster.go index c7d638cedf..6d4055cce9 100644 --- a/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks/deleteCluster.go +++ b/bcs-services/bcs-cluster-manager/internal/cloudprovider/aws/tasks/deleteCluster.go @@ -15,6 +15,7 @@ package tasks import ( "context" "fmt" + "strings" "time" "github.com/Tencent/bk-bcs/bcs-common/common/blog" @@ -73,6 +74,11 @@ func DeleteEKSClusterTask(taskID string, stepName string) error { func deleteEKSCluster(ctx context.Context, info *cloudprovider.CloudDependBasicInfo) error { taskID := cloudprovider.GetTaskIDFromContext(ctx) cluster := info.Cluster + + if cluster.GetSystemID() == "" { + return nil + } + // get aws client cli, err := api.NewEksClient(info.CmOption) if err != nil { @@ -81,13 +87,21 @@ func deleteEKSCluster(ctx context.Context, info *cloudprovider.CloudDependBasicI return fmt.Errorf("get aws client failed, %s", err.Error()) } + // check cluster if exist + _, err = cli.GetEksCluster(cluster.SystemID) + if err != nil { + if strings.Contains(err.Error(), eks.ErrCodeResourceNotFoundException){ + return nil + } + return err + } + + // check cluster if node group exist, and batch delete nodegroups first, or the cluster can't be deleted ngList, err := cli.ListNodegroups(cluster.SystemID) if err != nil { blog.Errorf("deleteEKSCluster[%s]: call aws ListNodegroups failed: %v", taskID, err) return fmt.Errorf("call aws ListNodegroups failed: %s", err.Error()) } - - // delete nodegroups first, or the cluster can't be deleted for _, ng := range ngList { err = retry.Do(func() error { _, err = cli.DeleteNodegroup(&eks.DeleteNodegroupInput{ @@ -126,6 +140,7 @@ func deleteEKSCluster(ctx context.Context, info *cloudprovider.CloudDependBasicI return err } + // delete cluster _, err = cli.DeleteEksCluster(cluster.SystemID) if err != nil { blog.Errorf("deleteEKSCluster[%s]: call aws DeleteEKSCluster failed: %v", taskID, err) diff --git a/bcs-services/bcs-cluster-manager/internal/cloudprovider/common/taint.go b/bcs-services/bcs-cluster-manager/internal/cloudprovider/common/taint.go index 77728bc19a..4fe2a0aaa6 100644 --- a/bcs-services/bcs-cluster-manager/internal/cloudprovider/common/taint.go +++ b/bcs-services/bcs-cluster-manager/internal/cloudprovider/common/taint.go @@ -48,12 +48,11 @@ var ( // BuildRemoveInnerTaintTaskStep 删除预置的污点 // NOCC:tosa/fn_length(忽略) // nolint function name should not exceed 35 characters -func BuildRemoveInnerTaintTaskStep(task *proto.Task, group *proto.NodeGroup) { +func BuildRemoveInnerTaintTaskStep(task *proto.Task, clusterId, provider string) { removeTaintStep := cloudprovider.InitTaskStep(RemoveClusterNodesInnerTaintStep) - removeTaintStep.Params[cloudprovider.ClusterIDKey.String()] = group.ClusterID - removeTaintStep.Params[cloudprovider.NodeGroupIDKey.String()] = group.NodeGroupID - removeTaintStep.Params[cloudprovider.CloudIDKey.String()] = group.Provider + removeTaintStep.Params[cloudprovider.ClusterIDKey.String()] = clusterId + removeTaintStep.Params[cloudprovider.CloudIDKey.String()] = provider task.Steps[RemoveClusterNodesInnerTaintStep.StepMethod] = removeTaintStep task.StepSequence = append(task.StepSequence, RemoveClusterNodesInnerTaintStep.StepMethod) @@ -80,22 +79,21 @@ func RemoveClusterNodesInnerTaintTask(taskID string, stepName string) error { // step login started here // extract parameter && check validate clusterID := step.Params[cloudprovider.ClusterIDKey.String()] - nodeGroupID := step.Params[cloudprovider.NodeGroupIDKey.String()] cloudID := step.Params[cloudprovider.CloudIDKey.String()] // inject success nodesNames nodeNames := strings.Split(state.Task.CommonParams[cloudprovider.NodeNamesKey.String()], ",") - removeTaints := strings.Split(state.Task.CommonParams[cloudprovider.RemoveTaintsKey.String()], ",") + removeTaints := cloudprovider.ParseNodeIpOrIdFromCommonMap(state.Task.CommonParams, + cloudprovider.RemoveTaintsKey.String(), ",") - if len(clusterID) == 0 || len(nodeGroupID) == 0 || len(cloudID) == 0 || len(nodeNames) == 0 { + if len(clusterID) == 0 || len(cloudID) == 0 || len(nodeNames) == 0 { blog.Errorf("RemoveClusterNodesTaintTask[%s]: check parameter validate failed", taskID) retErr := fmt.Errorf("RemoveClusterNodesTaintTask check parameters failed") _ = state.UpdateStepFailure(start, stepName, retErr) return retErr } dependInfo, err := cloudprovider.GetClusterDependBasicInfo(cloudprovider.GetBasicInfoReq{ - ClusterID: clusterID, - CloudID: cloudID, - NodeGroupID: nodeGroupID, + ClusterID: clusterID, + CloudID: cloudID, }) if err != nil { blog.Errorf("RemoveClusterNodesTaintTask[%s]: GetClusterDependBasicInfo failed: %s", taskID, err.Error()) diff --git a/bcs-services/bcs-cluster-manager/internal/cloudprovider/google/taskmgr.go b/bcs-services/bcs-cluster-manager/internal/cloudprovider/google/taskmgr.go index 96039c0f7c..80fa27d47d 100644 --- a/bcs-services/bcs-cluster-manager/internal/cloudprovider/google/taskmgr.go +++ b/bcs-services/bcs-cluster-manager/internal/cloudprovider/google/taskmgr.go @@ -596,7 +596,7 @@ func (t *Task) BuildUpdateDesiredNodesTask(desired uint32, group *proto.NodeGrou cloudprovider.GetAnnotationsByNg(opt.NodeGroup)) // step5: remove inner nodes taints - common.BuildRemoveInnerTaintTaskStep(task, group) + common.BuildRemoveInnerTaintTaskStep(task, group.ClusterID, group.Provider) // set current step if len(task.StepSequence) == 0 { diff --git a/bcs-services/bcs-cluster-manager/internal/cloudprovider/huawei/taskmgr.go b/bcs-services/bcs-cluster-manager/internal/cloudprovider/huawei/taskmgr.go index 628f3394ff..17257207fb 100644 --- a/bcs-services/bcs-cluster-manager/internal/cloudprovider/huawei/taskmgr.go +++ b/bcs-services/bcs-cluster-manager/internal/cloudprovider/huawei/taskmgr.go @@ -780,7 +780,7 @@ func (t *Task) BuildUpdateDesiredNodesTask(desired uint32, group *proto.NodeGrou cloudprovider.GetAnnotationsByNg(opt.NodeGroup)) // step6: remove inner nodes taints - common.BuildRemoveInnerTaintTaskStep(task, group) + common.BuildRemoveInnerTaintTaskStep(task, group.ClusterID, group.Provider) // set current step if len(task.StepSequence) == 0 { diff --git a/bcs-ui/frontend/src/api/modules/cluster-manager.ts b/bcs-ui/frontend/src/api/modules/cluster-manager.ts index d60f6eb935..ffa3cba68c 100644 --- a/bcs-ui/frontend/src/api/modules/cluster-manager.ts +++ b/bcs-ui/frontend/src/api/modules/cluster-manager.ts @@ -66,6 +66,8 @@ export const cloudInstanceTypes = request('get', '/clouds/$cloudId/instancetypes export const cloudInstanceTypesByLevel = request('get', '/clouds/$cloudId/regions/$region/clusterlevels/$level/instancetypes'); export const cloudCidrconflict = request('get', '/clouds/$cloudId/vpcs/$vpc/cidrconflict'); export const addSubnets = request('post', '/clusters/$clusterId/subnets'); +export const cloudRoles = request('get', '/clouds/$cloudId/serviceroles'); +export const recommendNodeGroupConf = request('get', '/cloud/$cloudId/recommendNodeGroupConf'); // node 操作 export const getK8sNodes = request('get', '/cluster/$clusterId/node'); diff --git a/bcs-ui/frontend/src/common/constant.ts b/bcs-ui/frontend/src/common/constant.ts index 3e068ca00d..1d9dd92cb0 100644 --- a/bcs-ui/frontend/src/common/constant.ts +++ b/bcs-ui/frontend/src/common/constant.ts @@ -24,6 +24,7 @@ export const TAINT_VALUE = '^([A-Za-z0-9]([-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?)?$'; export const NAME_REGEX = '^[0-9a-zA-Z-]+$'; export const SECRET_REGEX = '^[0-9a-zA-Z-~]+$'; export const SECRETKEY_REGEX = '^[0-9a-zA-Z/-]+$'; +export const CLUSTER_NAME_REGEX = '^[0-9A-Za-z][A-Za-z0-9-_]*'; // 集群环境 export const CLUSTER_ENV = { diff --git a/bcs-ui/frontend/src/components/cluster-selector/cluster-select-popover.vue b/bcs-ui/frontend/src/components/cluster-selector/cluster-select-popover.vue index 4192c7779f..fbd11cce79 100644 --- a/bcs-ui/frontend/src/components/cluster-selector/cluster-select-popover.vue +++ b/bcs-ui/frontend/src/components/cluster-selector/cluster-select-popover.vue @@ -31,6 +31,8 @@
  • // popover场景的集群选择器 -import { PropType, ref } from 'vue'; +import { PropType, ref, watch } from 'vue'; import CollapseTitle from './collapse-title.vue'; import useClusterSelector, { ClusterType } from './use-cluster-selector'; @@ -100,6 +102,11 @@ const props = defineProps({ type: Boolean, default: true, }, + // 是否展示选中的样式 + isShow: { + type: Boolean, + default: false, + }, }); const emits = defineEmits(['change', 'click']); @@ -107,6 +114,7 @@ const emits = defineEmits(['change', 'click']); const normalStatusList = ['RUNNING']; const hoverClusterID = ref(); +const clusterIdRef = ref([]); const { keyword, localValue, @@ -124,4 +132,14 @@ const handleClick = (cluster) => { emits('click', cluster.clusterID); }; +watch(() => props.isShow, (val) => { + if(val) { + // 如果已选择集群,滚动到活动项目 + const selectedClusterId = localValue.value; + if (selectedClusterId) { + clusterIdRef.value?.find(item => item.id === selectedClusterId).scrollIntoView({ behavior: 'smooth', block: 'center' }); + } + } + +}); diff --git a/bcs-ui/frontend/src/components/combination-input.vue b/bcs-ui/frontend/src/components/combination-input.vue new file mode 100644 index 0000000000..ba7256e5a1 --- /dev/null +++ b/bcs-ui/frontend/src/components/combination-input.vue @@ -0,0 +1,251 @@ + + + diff --git a/bcs-ui/frontend/src/components/ip-selector/ip-selector.vue b/bcs-ui/frontend/src/components/ip-selector/ip-selector.vue index 092723d8e9..0f6225f45a 100644 --- a/bcs-ui/frontend/src/components/ip-selector/ip-selector.vue +++ b/bcs-ui/frontend/src/components/ip-selector/ip-selector.vue @@ -130,6 +130,9 @@ const getRegionName = (region) => { const $biz = computed(() => $store.state.curProject.businessID); const $scope = 'biz'; +// 是否显示不可用主机 +const isHostOnlyValid = ref(false); + // 获取主机云和占有信息 const handleGetHostAvailableAndCloudInfo = async (hostData: IHostData[]) => { const ipList = hostData.filter(item => !!item.ip).map(item => item.ip); @@ -159,11 +162,13 @@ const handleGetHostAvailableAndCloudInfo = async (hostData: IHostData[]) => { } }; // 获取topo树当前页的主机列表 -const fetchTopologyHostsNodes = async (params) => { +const fetchTopologyHostsNodes = async (params, hostOnlyValid) => { + isHostOnlyValid.value = hostOnlyValid; const data: {data: IHostData[]} = await topologyHostsNodes({ ...params, $biz: $biz.value, $scope, + showAvailableNode: hostOnlyValid, }).catch(() => []); await handleGetHostAvailableAndCloudInfo(data.data); return topologyHostsNodesAdapter(data); @@ -174,6 +179,7 @@ const fetchHostCheck = async (params) => { ...params, $biz: $biz.value, $scope, + showAvailableNode: isHostOnlyValid.value, // todo 手动输入IP场景切换 "仅显示可用" 不会触发fetchHostCheck函数 }).catch(() => []); await handleGetHostAvailableAndCloudInfo(data); return hostCheckAdapter(data); diff --git a/bcs-ui/frontend/src/components/ip-selector/ipv6-selector.ts b/bcs-ui/frontend/src/components/ip-selector/ipv6-selector.ts index 6d485aa194..6b783efb16 100644 --- a/bcs-ui/frontend/src/components/ip-selector/ipv6-selector.ts +++ b/bcs-ui/frontend/src/components/ip-selector/ipv6-selector.ts @@ -89,6 +89,7 @@ const IpSelector = createFactory({ nameStyle: 'camelCase', // 主机列表全选模式,false: 本页全选;true: 跨页全选 hostTableDefaultSelectAllMode: false, + hostOnlyValid: true, // 自定义主机列表列 hostTableCustomColumnList: [ // { diff --git a/bcs-ui/frontend/src/components/monaco-editor/ai-editor.vue b/bcs-ui/frontend/src/components/monaco-editor/ai-editor.vue new file mode 100644 index 0000000000..b092eef87f --- /dev/null +++ b/bcs-ui/frontend/src/components/monaco-editor/ai-editor.vue @@ -0,0 +1,152 @@ + + + diff --git a/bcs-ui/frontend/src/composables/use-focus-on-error-field.ts b/bcs-ui/frontend/src/composables/use-focus-on-error-field.ts new file mode 100644 index 0000000000..1958fcbb2f --- /dev/null +++ b/bcs-ui/frontend/src/composables/use-focus-on-error-field.ts @@ -0,0 +1,28 @@ +import { ref } from 'vue'; + +export function useFocusOnErrorField() { + const errorClasses = ref(['form-error-tip', 'error-tips', 'is-error']); + + const focusOnErrorField = async () => { + // 自动滚动到第一个错误的位置 + const firstErrDom = findFirstErrorElement(); + firstErrDom?.scrollIntoView({ + block: 'center', + behavior: 'smooth', + }); + }; + + const findFirstErrorElement = () => { + for (const className of errorClasses.value) { + const elements = document.getElementsByClassName(className); + if (elements.length > 0) { + return elements[0]; + } + } + return null; + }; + + return { + focusOnErrorField, + }; +} \ No newline at end of file diff --git a/bcs-ui/frontend/src/i18n/en-US.yaml b/bcs-ui/frontend/src/i18n/en-US.yaml index b5937d58d7..68f2de96d6 100644 --- a/bcs-ui/frontend/src/i18n/en-US.yaml +++ b/bcs-ui/frontend/src/i18n/en-US.yaml @@ -496,6 +496,9 @@ generic: cidrConflict: 'CIDR conflicts with {cidr}, please re-enter.' fieldRequired: '{0} required' fieldRepeat: '{0} repeat' + clusterName: >- + The cluster name can only contain numbers, letters, mid-line lines (-), down line + (_), The beginning must be numbers or letters title: curVersion: Current diffVersion: Comparison version @@ -818,6 +821,7 @@ cluster: importType: Import Type cloudToken: Cloud Token kubeConfig: Cluster KubeConfig + title: Import a public cloud cluster - Cluster KubeConfig network: text: Network intranet: Intranet @@ -851,6 +855,7 @@ cluster: notSupport: >- If the number of controller plane nodes is less than three, high availability cannot be supported + role: Service role msg: managedClusterInfo: >- Create a custody cluster must add at least one node to run the necessary @@ -899,6 +904,49 @@ cluster: createCloudToken: New voucher fileImport: File Import textKubeConfig: Usability testing + aws: + securityGroup: + text: Safety group + desc: >- + This selection will apply to the Safety group of the EKS-managed elastic + network interfaces created in your control plane subnet. To create a new + Safety group, please go to the appropriate page in the VPC console. + vpcLink: >- + 'This selection will apply to the Safety group of the EKS-managed elastic + network interfaces created in your control plane subnet. To create a new + Safety group, please go to the appropriate page in the {0}.' + vpcMaster: VPC console + cidrTips: + tips1: 0.0.0.0/0 is not allowed + tips2: Invalid + nodePool: Add Node pools + editNodePool: Edit Node pools + cidrWhitelist: CIDR Whitelist + label: + defaultNodePool: Default Node Pool + skipOrNot: Skip or not + validate: + subnet: The node pool subnet cannot be empty + runtime: + extended: + label: EXTENDED + text: >- + This option supports Kubernetes versions for 26 months from the date of release. + Extended support requires an additional hourly fee, which begins after the + standard support period ends. After the extended support period ends, + your cluster will be automatically upgraded to the next version. + standard: + label: STANDARD + text: >- + This option supports Kubernetes versions for 14 months from the date of release, + at no additional cost. After the standard support period ends, your cluster will + be automatically upgraded to the next version. + tips: + subnetEmpty: The subnet is empty, please select a subnet first + awsDesc: Amazon VPC CNI :AWS network plugin that enables Pod networking within the cluster. + awsLink: Overview of Amazon VPC CNI functionality. + azure: + chargingModel: Charging model detail: title: overview: Overview @@ -1090,6 +1138,8 @@ cluster: The name does not exceed 255 characters, only supports Chinese, English, numbers, and scratch lines, separators (-) and decimal points nodeQuotaMaxSize: 'Maximum node quota is {maximum}' + nodeCountsMaxSize: The number of Node count cannot be greater than the Node quota + minNodes: The number of Node count must be greater than 0 label: nameAndID: Node pool name (ID) nodeQuota: Node quota @@ -3349,6 +3399,7 @@ tke: text: CLB internet: Internet intranet: Intranet + internetAndIntranet: Internet and Intranet desc: >- Used to connect the cluster APIServer via the BlueKing container management platform. Depending on the actual network situation, choose @@ -3496,6 +3547,7 @@ tke: re-enter. supportCidrList: 'Supports only network segments between {0}.' formErr: The form was filled in incorrectly. Please modify it + leastTwoDifferentAZs: Subnets specified must be in at least two different AZs configRec: l100: text: Node Count 1~100 diff --git a/bcs-ui/frontend/src/i18n/zh-CN.yaml b/bcs-ui/frontend/src/i18n/zh-CN.yaml index ae2e7b56ac..295995b94d 100644 --- a/bcs-ui/frontend/src/i18n/zh-CN.yaml +++ b/bcs-ui/frontend/src/i18n/zh-CN.yaml @@ -460,6 +460,7 @@ generic: cidrConflict: 'CIDR与 {cidr} 冲突,请重新输入' fieldRequired: '{0} 必填' fieldRepeat: '{0} 重复' + clusterName: 集群名称只能包含数字、字母、中划线(-)、下划线(_),且开头必须是数字或字母 title: createPublicCluster: 新建公有云集群 diffVersion: 线上版本 @@ -726,6 +727,7 @@ cluster: cpuQuota: CPU配额(核) memQuota: 内存配额(GiB) importType: 导入方式 + title: 导入集群 - kubeconfig 集群 cloudToken: 云凭证 kubeConfig: 集群kubeconfig network: @@ -739,6 +741,7 @@ cluster: bcs-apiserver-proxy 以 daemonset 形式部署在 Kubernetes 集群中,利用 IPVS 通过虚拟 IP(VIP)实现集群内组件到 APIserver 的反向代理,并自动发现控制面节点 IP 以维护 IPVS 规则,确保部分控制面节点故障时集群正常运行。 + role: 集群服务角色 msg: managedClusterInfo: 创建托管集群必须添加至少一个节点,以运行必要的服务 independentClusterInfo: @@ -769,6 +772,39 @@ cluster: createCloudToken: 新增凭证 fileImport: 文件导入 textKubeConfig: 可用性测试 + aws: + securityGroup: + text: 安全组 + desc: 选择将应用到在您的控制平面子网中创建的 EKS 托管弹性网络接口的安全组。 要创建新的安全组,请转到 VPC 控制台 中的相应页面。 + vpcLink: '选择将应用到在您的控制平面子网中创建的 EKS 托管弹性网络接口的安全组。 要创建新的安全组,请转到 {0} 中的相应页面。' + vpcMaster: VPC 控制台 + cidrTips: + tips1: 不允许0.0.0.0/0 + tips2: cidr不合法 + nodePool: 添加节点池 + editNodePool: 编辑节点池 + cidrWhitelist: CIDR白名单 + label: + defaultNodePool: 默认节点池 + skipOrNot: 是否跳过 + validate: + subnet: 节点池子网不能为空 + runtime: + extended: + label: 已扩展 + text: >- + 此选项在发布之日起 26 个月内支持 Kubernetes 版本。扩展支持期需要额外支付每小时的费用,该费用在标准支持期结束后开始。 + 扩展支持结束后,您的集群将自动升级到下一个版本。 + standard: + label: 标准 + text: >- + 此选项在发布之日起 14 个月内支持 Kubernetes 版本。没有额外费用。标准支持结束后,您的集群将自动升级到下一个版本。 + tips: + subnetEmpty: 子网为空,请先选择子网 + awsDesc: Amazon VPC CNI :AWS 的网络插件,可以在集群内启用 Pod 联网。 + awsLink: Amazon VPC CNI功能概述 + azure: + chargingModel: 收费模式 detail: title: overview: 集群总览 @@ -893,6 +929,8 @@ cluster: validate: name: 名称不超过255个字符,仅支持中文、英文、数字、下划线,分隔符(-)及小数点 nodeQuotaMaxSize: '节点配额上限为{maximum}' + nodeCountsMaxSize: '节点数量不能大于节点配额' + minNodes: 节点数要大于0 label: nameAndID: 节点池名称 (ID) nodeQuota: 节点配额 @@ -2750,6 +2788,7 @@ tke: 用于蓝鲸容器管理平台连接集群APIServer,根据实际网络情况选择公网访问与内网访问,蓝鲸平台所在网络与被添加集群所在网络内网可连通优先选择内网访问,否则需要选择外网访问,选择不正确会导致添加集群失败 internet: 公网访问 intranet: 内网访问 + internetAndIntranet: 公网和内网访问 securityGroup: text: 安全组 defaultSecurityGroupTips: 云梯默认安全组已对自研云网段配置好安全规则,无需用户再额外配置;业务如有特殊安全规则配置可创建自定义安全组,结合云梯默认安全组使用 @@ -2846,6 +2885,7 @@ tke: supportCidrList: '仅支持 {0} 之间网段' minIpNum2: '容器网段不能少于 {0} 个 IP, 请重新输入' formErr: 表单填写有误,请修改 + leastTwoDifferentAZs: 指定的子网必须至少位于两个不同的可用区中 configRec: l100: text: 节点数1~100 diff --git a/bcs-ui/frontend/src/router/cluster-manage.ts b/bcs-ui/frontend/src/router/cluster-manage.ts index b7930dfc34..2c8781014d 100644 --- a/bcs-ui/frontend/src/router/cluster-manage.ts +++ b/bcs-ui/frontend/src/router/cluster-manage.ts @@ -12,6 +12,10 @@ const CreateTencentCloudCluster = () => import(/* webpackChunkName: 'cluster' */ const CreateVCluster = () => import(/* webpackChunkName: 'cluster' */'@/views/cluster-manage/add/create/vcluster/vcluster.vue'); // ee版本创建集群流程 const CreateK8SCluster = () => import(/* webpackChunkName: 'cluster' */'@/views/cluster-manage/add/create/k8s.vue'); +// 创建aws云集群 +const CreateAWSCloudCluster = () => import(/* webpackChunkName: 'cluster' */'@/views/cluster-manage/add/create/aws-cloud/index.vue'); +// 创建aws云集群 +const CreateAzureCloudCluster = () => import(/* webpackChunkName: 'cluster' */'@/views/cluster-manage/add/create/azure-cloud/index.vue'); // import模式 const ImportCluster = () => import(/* webpackChunkName: 'cluster' */'@/views/cluster-manage/add/import/import-cluster.vue'); @@ -118,6 +122,26 @@ export default [ hideMenu: true, }, }, + // 创建亚马逊云集群 + { + path: 'clusters/cloud/aws', + name: 'CreateAWSCloudCluster', + component: CreateAWSCloudCluster, + meta: { + menuId: 'CLUSTER', + hideMenu: true, + }, + }, + // 创建微软云集群 + { + path: 'clusters/cloud/azure', + name: 'CreateAzureCloudCluster', + component: CreateAzureCloudCluster, + meta: { + menuId: 'CLUSTER', + hideMenu: true, + }, + }, // 创建k8s原生集群 { path: 'clusters/k8s', diff --git a/bcs-ui/frontend/src/views/cluster-manage/add/components/login-type.vue b/bcs-ui/frontend/src/views/cluster-manage/add/components/login-type.vue index 4f5234fa76..51f01f57d0 100644 --- a/bcs-ui/frontend/src/views/cluster-manage/add/components/login-type.vue +++ b/bcs-ui/frontend/src/views/cluster-manage/add/components/login-type.vue @@ -1,6 +1,6 @@ + + diff --git a/bcs-ui/frontend/src/views/cluster-manage/detail/network/index.vue b/bcs-ui/frontend/src/views/cluster-manage/detail/network/index.vue index dcf17253b9..97cf246be0 100644 --- a/bcs-ui/frontend/src/views/cluster-manage/detail/network/index.vue +++ b/bcs-ui/frontend/src/views/cluster-manage/detail/network/index.vue @@ -4,6 +4,7 @@ diff --git a/bcs-ui/frontend/src/views/project-manage/project/project.vue b/bcs-ui/frontend/src/views/project-manage/project/project.vue index ffe7e245f0..4bd04d4f21 100644 --- a/bcs-ui/frontend/src/views/project-manage/project/project.vue +++ b/bcs-ui/frontend/src/views/project-manage/project/project.vue @@ -92,6 +92,7 @@ import ProjectCreate from './project-create.vue'; import useProjects from './use-project'; import { bus } from '@/common/bus'; +import { IProject } from '@/composables/use-app'; import useDebouncedRef from '@/composables/use-debounce'; import $router from '@/router'; @@ -101,16 +102,16 @@ export default defineComponent({ ProjectCreate, }, setup: () => { - const { getAllProjectList } = useProjects(); + const { getProjectList } = useProjects(); const pagination = ref({ current: 1, count: 0, limit: 20, }); - const projectList = ref([]); + const projectList = ref([]); const keyword = useDebouncedRef('', 300); const showCreateDialog = ref(false); - const curProjectData = ref(null); + const curProjectData = ref(); const handleGotoProject = (row) => { $router.push({ name: 'clusterMain', @@ -130,7 +131,7 @@ export default defineComponent({ showCreateDialog.value = true; }; const handleCreateProject = () => { - curProjectData.value = null; + curProjectData.value = undefined; showCreateDialog.value = true; }; const handlePageChange = (page) => { @@ -146,14 +147,14 @@ export default defineComponent({ const webAnnotations = ref({ perms: {} }); const handleGetProjectList = async () => { isLoading.value = true; - const { data = [], web_annotations: webPerms, total } = await getAllProjectList({ - searchName: keyword.value, + const { data, web_annotations: webPerms } = await getProjectList({ + searchKey: keyword.value, offset: pagination.value.current - 1, limit: pagination.value.limit, }); - projectList.value = data; + projectList.value = data.results; webAnnotations.value = webPerms; - pagination.value.count = total; + pagination.value.count = data.total; isLoading.value = false; }; diff --git a/bcs-ui/frontend/src/views/project-manage/project/use-project.ts b/bcs-ui/frontend/src/views/project-manage/project/use-project.ts index 07dbe97765..a89e0238e4 100644 --- a/bcs-ui/frontend/src/views/project-manage/project/use-project.ts +++ b/bcs-ui/frontend/src/views/project-manage/project/use-project.ts @@ -2,7 +2,6 @@ import { businessList, createProject as handleCreateProject, editProject, - fetchAllProjectList, fetchProjectList, getProject, } from '@/api/modules/project'; @@ -44,29 +43,6 @@ export default function useProjects() { return data as IProjectListData; }; - // 获取所有项目列表(项目列表页面) - async function getAllProjectList(params: any = {}, config = {}) { - const result = await fetchAllProjectList(params, { needRes: true, ...config }) - .catch(() => ({ - data: {}, - webAnnotations: { - perms: {}, - }, - })); - return { - data: result.data.results.map(project => ({ - ...project, - cc_app_id: project.businessID, - cc_app_name: project.businessName, - project_id: project.projectID, - project_name: project.name, - project_code: project.projectCode, - })), - total: result.data.total, - web_annotations: result.web_annotations, - }; - }; - async function updateProject(params: any) { const result = await editProject(params).then(() => true) .catch(() => false); @@ -115,7 +91,6 @@ export default function useProjects() { return { fetchProjectInfo, getProjectList, - getAllProjectList, updateProject, getBusinessList, createProject, diff --git a/bcs-ui/frontend/src/views/resource-view/view-manage/view-selector.vue b/bcs-ui/frontend/src/views/resource-view/view-manage/view-selector.vue index 6734d742f0..9e7348487b 100644 --- a/bcs-ui/frontend/src/views/resource-view/view-manage/view-selector.vue +++ b/bcs-ui/frontend/src/views/resource-view/view-manage/view-selector.vue @@ -83,6 +83,7 @@ cluster-type="all" :value="clusterID" :selectable="isDefaultView" + :isShow="!isHide" v-show="viewMode === 'cluster'" @click="changeClusterView" />