From 48c2c31772f7dfc031f2272199ab4e4406428d8c Mon Sep 17 00:00:00 2001 From: johnypeng <2651903873@qq.com> Date: Tue, 6 Feb 2024 20:15:18 +0800 Subject: [PATCH 1/3] feat: cmdline supports to get file info --- client/types.go | 3 + cmd/bscp/get.go | 199 ++++++++++++++++++++++++++++++++++++++++------- cmd/bscp/root.go | 1 + 3 files changed, 176 insertions(+), 27 deletions(-) diff --git a/client/types.go b/client/types.go index ad85df9bb..716de66f3 100644 --- a/client/types.go +++ b/client/types.go @@ -14,6 +14,7 @@ package client import ( "fmt" + "path" pbci "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/protocol/core/config-item" pbhook "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/protocol/core/hook" @@ -50,6 +51,7 @@ type ConfigItemFile struct { func (c *ConfigItemFile) GetContent() ([]byte, error) { if cache.Enable { if hit, bytes := cache.GetCache().GetFileContent(c.FileMeta); hit { + logger.Info("get file content from cache success", slog.String("file", path.Join(c.Path, c.Name))) return bytes, nil } } @@ -59,6 +61,7 @@ func (c *ConfigItemFile) GetContent() ([]byte, error) { c.FileMeta.ContentSpec.ByteSize, downloader.DownloadToBytes, bytes, ""); err != nil { return nil, fmt.Errorf("download file failed, err: %s", err.Error()) } + logger.Info("get file content by downloading from repo success", slog.String("file", path.Join(c.Path, c.Name))) return bytes, nil } diff --git a/cmd/bscp/get.go b/cmd/bscp/get.go index 6ca170c24..76250f267 100644 --- a/cmd/bscp/get.go +++ b/cmd/bscp/get.go @@ -13,12 +13,15 @@ package main import ( + "context" "encoding/json" "fmt" "os" - "sync" + "path" + "github.com/dustin/go-humanize" "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" "github.com/TencentBlueKing/bscp-go/client" "github.com/TencentBlueKing/bscp-go/pkg/logger" @@ -33,13 +36,14 @@ const ( outputFormatJson = "json" outputFormatValue = "value" outputFormatValueJson = "value_json" + outputFormatContent = "content" ) var ( getCmd = &cobra.Command{ Use: "get", - Short: "Display app or kv resources", - Long: `Display app or kv resources`, + Short: "Display app, file or kv resources", + Long: `Display app, file or kv resources`, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { // 设置日志等级, get 命令默认是 error if logLevel == "" { @@ -69,6 +73,15 @@ var ( }, } + getFileCmd = &cobra.Command{ + Use: "file [res...]", + Short: "Display file resources", + Long: `Display file resources`, + RunE: func(cmd *cobra.Command, args []string) error { + return runGetFile(args) + }, + } + getKvCmd = &cobra.Command{ Use: "kv [res...]", Short: "Display kv resources", @@ -89,6 +102,11 @@ func init() { // app 参数 getAppCmd.PersistentFlags().StringVarP(&outputFormat, "output", "o", "", "output format, One of: json") + // file 参数 + getFileCmd.Flags().StringVarP(&appName, "app", "a", "", "app name") + getFileCmd.Flags().StringVarP(&labelsStr, "labels", "l", "", "labels") + getFileCmd.PersistentFlags().StringVarP(&outputFormat, "output", "o", "", "output format, One of: json|content") + // kv 参数 getKvCmd.Flags().StringVarP(&appName, "app", "a", "", "app name") getKvCmd.Flags().StringVarP(&labelsStr, "labels", "l", "", "labels") @@ -145,7 +163,141 @@ func runGetApp(args []string) error { } } -func runGetListKv(bscp client.Client, app string, match []string) error { +func runGetFileList(bscp client.Client, app string, match []string) error { + var opts []client.AppOption + if len(match) > 0 { + opts = append(opts, client.WithAppKey(match[0])) + } + opts = append(opts, client.WithAppLabels(labels)) + + release, err := bscp.PullFiles(app, opts...) + if err != nil { + return err + } + + tableOutput := func() error { + table := newTable() + table.SetHeader([]string{"File", "ContentID", "Size"}) + for _, v := range release.FileItems { + table.Append([]string{ + path.Join(v.Path, v.Name), + v.FileMeta.ContentSpec.Signature, + humanize.IBytes(v.FileMeta.ContentSpec.ByteSize), + }) + } + + table.Render() + return nil + } + + switch outputFormat { + case outputFormatJson: + return jsonOutput(release.FileItems) + case outputFormatTable: + return tableOutput() + default: + return fmt.Errorf( + `unable to match a printer suitable for the output format "%s", allowed formats are: json,content`, + outputFormat) + } +} + +func runGetFileContents(bscp client.Client, app string, contentIDs []string) error { + release, err := bscp.PullFiles(app, client.WithAppLabels(labels)) + if err != nil { + return err + } + + fileMap := make(map[string]*client.ConfigItemFile) + allFiles := make([]*client.ConfigItemFile, len(release.FileItems)) + for idx, f := range release.FileItems { + fileMap[f.FileMeta.ContentSpec.Signature] = f + allFiles[idx] = f + } + + files := allFiles + if len(contentIDs) > 0 { + files = []*client.ConfigItemFile{} + for _, id := range contentIDs { + if _, ok := fileMap[id]; !ok { + return fmt.Errorf("the file content id %s is not exist for the latest release of app %s", id, app) + } + files = append(files, fileMap[id]) + } + } + + var contents [][]byte + contents, err = getfileContents(files) + if err != nil { + return err + } + + output := "" + for idx, file := range files { + output += fmt.Sprintf("***start No.%d***\nfile: %s\ncontentID: %s\nconent: \n%s\n***end No.%d***\n\n", + idx+1, path.Join(file.Path, file.Name), file.FileMeta.ContentSpec.Signature, contents[idx], idx+1) + } + + _, err = fmt.Fprint(os.Stdout, output) + return err +} + +// getfileContents get file contents concurrently +func getfileContents(files []*client.ConfigItemFile) ([][]byte, error) { + contents := make([][]byte, len(files)) + g, _ := errgroup.WithContext(context.Background()) + g.SetLimit(10) + + for i, f := range files { + idx, file := i, f + g.Go(func() error { + content, err := file.GetContent() + if err != nil { + return err + } + contents[idx] = content + return nil + }) + } + + return contents, g.Wait() +} + +// runGetFile executes the get file command. +func runGetFile(args []string) error { + baseConf, err := initBaseConf() + if err != nil { + return err + } + + if appName == "" { + return fmt.Errorf("app must not be empty") + } + + bscp, err := client.New( + client.WithFeedAddrs(baseConf.GetFeedAddrs()), + client.WithBizID(baseConf.Biz), + client.WithToken(baseConf.Token), + client.WithFileCache(client.FileCache{ + Enabled: *baseConf.FileCache.Enabled, + CacheDir: baseConf.FileCache.CacheDir, + ThresholdGB: baseConf.FileCache.ThresholdGB, + }), + ) + + if err != nil { + return err + } + + if outputFormat == outputFormatContent { + return runGetFileContents(bscp, appName, args) + } + + return runGetFileList(bscp, appName, args) + +} + +func runGetKvList(bscp client.Client, app string, match []string) error { release, err := bscp.PullKvs(app, match, client.WithAppLabels(labels)) if err != nil { return err @@ -175,7 +327,8 @@ func runGetListKv(bscp client.Client, app string, match []string) error { return tableOutput() default: return fmt.Errorf( - `unable to match a printer suitable for the output format "%s", allowed formats are: json,value`, outputFormat) + `unable to match a printer suitable for the output format "%s", allowed formats are: json,value,value_json`, + outputFormat) } } @@ -206,9 +359,10 @@ func runGetKvValues(bscp client.Client, app string, keys []string) error { } } - values, hitError := getKvValues(bscp, app, keys) - if hitError != nil { - return hitError + var values []string + values, err = getKvValues(bscp, app, keys) + if err != nil { + return err } output := make(map[string]any, len(keys)) @@ -224,32 +378,23 @@ func runGetKvValues(bscp client.Client, app string, keys []string) error { // getKvValues get kv values concurrently func getKvValues(bscp client.Client, app string, keys []string) ([]string, error) { - var hitError error values := make([]string, len(keys)) - pipe := make(chan struct{}, 10) - wg := sync.WaitGroup{} - - for idx, key := range keys { - wg.Add(1) - - pipe <- struct{}{} - go func(idx int, key string) { - defer func() { - wg.Done() - <-pipe - }() + g, _ := errgroup.WithContext(context.Background()) + g.SetLimit(10) + for i, k := range keys { + idx, key := i, k + g.Go(func() error { value, err := bscp.Get(app, key, client.WithAppLabels(labels)) if err != nil { - hitError = fmt.Errorf("get kv value failed for key: %s, err:%v", key, err) - return + return err } values[idx] = value - }(idx, key) + return nil + }) } - wg.Wait() - return values, hitError + return values, g.Wait() } // runGetKv executes the get kv command. @@ -285,6 +430,6 @@ func runGetKv(args []string) error { case outputFormatValueJson: return runGetKvValues(bscp, appName, args) default: - return runGetListKv(bscp, appName, args) + return runGetKvList(bscp, appName, args) } } diff --git a/cmd/bscp/root.go b/cmd/bscp/root.go index d436fc9dc..340d11ff5 100644 --- a/cmd/bscp/root.go +++ b/cmd/bscp/root.go @@ -51,6 +51,7 @@ func init() { rootCmd.CompletionOptions.DisableDefaultCmd = true getCmd.AddCommand(getAppCmd) + getCmd.AddCommand(getFileCmd) getCmd.AddCommand(getKvCmd) rootCmd.AddCommand(getCmd) From 3e7fd34f9dadc352fc7f32451bc347e28a4aa83e Mon Sep 17 00:00:00 2001 From: johnypeng <2651903873@qq.com> Date: Tue, 20 Feb 2024 19:55:23 +0800 Subject: [PATCH 2/3] feat: add revision info for file detail --- client/client.go | 1 + cmd/bscp/get.go | 11 +++++++++-- go.mod | 2 +- go.sum | 2 ++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/client/client.go b/client/client.go index 7b1a743f2..f4ebd5285 100644 --- a/client/client.go +++ b/client/client.go @@ -206,6 +206,7 @@ func (c *client) PullFiles(app string, opts ...AppOption) (*Release, error) { ContentSpec: meta.CommitSpec.Content, ConfigItemSpec: meta.ConfigItemSpec, ConfigItemAttachment: meta.ConfigItemAttachment, + ConfigItemRevision: meta.ConfigItemRevision, RepositoryPath: meta.RepositorySpec.Path, }, } diff --git a/cmd/bscp/get.go b/cmd/bscp/get.go index 76250f267..0f06d01e6 100644 --- a/cmd/bscp/get.go +++ b/cmd/bscp/get.go @@ -177,12 +177,14 @@ func runGetFileList(bscp client.Client, app string, match []string) error { tableOutput := func() error { table := newTable() - table.SetHeader([]string{"File", "ContentID", "Size"}) + table.SetHeader([]string{"File", "ContentID", "Size", "Reviser", "UpdateAt"}) for _, v := range release.FileItems { table.Append([]string{ path.Join(v.Path, v.Name), v.FileMeta.ContentSpec.Signature, humanize.IBytes(v.FileMeta.ContentSpec.ByteSize), + v.FileMeta.ConfigItemRevision.Reviser, + refineOutputTime(v.FileMeta.ConfigItemRevision.UpdateAt), }) } @@ -232,6 +234,12 @@ func runGetFileContents(bscp client.Client, app string, contentIDs []string) err return err } + // output only content when getting for just one file which is convenient to save it directly in a file + if len(contentIDs) == 1 { + _, err = fmt.Fprint(os.Stdout, string(contents[0])) + return err + } + output := "" for idx, file := range files { output += fmt.Sprintf("***start No.%d***\nfile: %s\ncontentID: %s\nconent: \n%s\n***end No.%d***\n\n", @@ -294,7 +302,6 @@ func runGetFile(args []string) error { } return runGetFileList(bscp, appName, args) - } func runGetKvList(bscp client.Client, app string, match []string) error { diff --git a/go.mod b/go.mod index 7289617e6..48d6595ba 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/Tencent/bk-bcs/bcs-common v0.0.0-20230912015319-acb7495967f5 - github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240111082738-ca8d43177bfd + github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240220073632-303c7a871f3b github.com/denisbrodbeck/machineid v1.0.1 github.com/dustin/go-humanize v1.0.1 github.com/fsnotify/fsnotify v1.7.0 diff --git a/go.sum b/go.sum index 1dd5c8b0e..7abf03ca0 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,8 @@ github.com/Tencent/bk-bcs/bcs-common v0.0.0-20230912015319-acb7495967f5 h1:8B8di github.com/Tencent/bk-bcs/bcs-common v0.0.0-20230912015319-acb7495967f5/go.mod h1:/9h4LrWU1cVp46TF0DW5sUrqLtUyyMW0A9mnqhCTavA= github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240111082738-ca8d43177bfd h1:tsuqMvHPukaQF2cjBfsa94IEwgmvJu4iMhg4vwARLkE= github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240111082738-ca8d43177bfd/go.mod h1:PewYJmxWgfuSI3Y8GKjkg6Gr2vtVdomh++hUPEJ+zOk= +github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240220073632-303c7a871f3b h1:N6CELtoDBk205kJv0NuLzrAJGNcoYp1mleKVh8V18s0= +github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240220073632-303c7a871f3b/go.mod h1:N+2MnKgdnz9f7kmR+M/T9fF0FBKeGJVc0vWu1NR93Yw= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= From e8cebb13f67c61a3535f0ce0faa921faf23dbbc7 Mon Sep 17 00:00:00 2001 From: johnypeng <2651903873@qq.com> Date: Tue, 20 Feb 2024 20:03:03 +0800 Subject: [PATCH 3/3] chore: update go.sum --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 7abf03ca0..7a1ca453b 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Tencent/bk-bcs/bcs-common v0.0.0-20230912015319-acb7495967f5 h1:8B8di9Hxgh9aIIMLVkIhQnsJmAwKd1VxgjfHlvJ+864= github.com/Tencent/bk-bcs/bcs-common v0.0.0-20230912015319-acb7495967f5/go.mod h1:/9h4LrWU1cVp46TF0DW5sUrqLtUyyMW0A9mnqhCTavA= -github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240111082738-ca8d43177bfd h1:tsuqMvHPukaQF2cjBfsa94IEwgmvJu4iMhg4vwARLkE= -github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240111082738-ca8d43177bfd/go.mod h1:PewYJmxWgfuSI3Y8GKjkg6Gr2vtVdomh++hUPEJ+zOk= github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240220073632-303c7a871f3b h1:N6CELtoDBk205kJv0NuLzrAJGNcoYp1mleKVh8V18s0= github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240220073632-303c7a871f3b/go.mod h1:N+2MnKgdnz9f7kmR+M/T9fF0FBKeGJVc0vWu1NR93Yw= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=