feat: cmdline supports to get file info

client/client.go

@@ -206,6 +206,7 @@ func (c *client) PullFiles(app string, opts ...AppOption) (*Release, error) {
 				ContentSpec:          meta.CommitSpec.Content,
 				ConfigItemSpec:       meta.ConfigItemSpec,
 				ConfigItemAttachment: meta.ConfigItemAttachment,
+				ConfigItemRevision:   meta.ConfigItemRevision,
 				RepositoryPath:       meta.RepositorySpec.Path,
 			},
 		}

client/types.go

@@ -14,6 +14,7 @@ package client
 
 import (
 	"fmt"
+	"path"
 
 	pbci "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/protocol/core/config-item"
 	pbhook "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/protocol/core/hook"
@@ -50,6 +51,7 @@ type ConfigItemFile struct {
 func (c *ConfigItemFile) GetContent() ([]byte, error) {
 	if cache.Enable {
 		if hit, bytes := cache.GetCache().GetFileContent(c.FileMeta); hit {
+			logger.Info("get file content from cache success", slog.String("file", path.Join(c.Path, c.Name)))
 			return bytes, nil
 		}
 	}
@@ -59,6 +61,7 @@ func (c *ConfigItemFile) GetContent() ([]byte, error) {
 		c.FileMeta.ContentSpec.ByteSize, downloader.DownloadToBytes, bytes, ""); err != nil {
 		return nil, fmt.Errorf("download file failed, err: %s", err.Error())
 	}
+	logger.Info("get file content by downloading from repo success", slog.String("file", path.Join(c.Path, c.Name)))
 	return bytes, nil
 }
 

cmd/bscp/get.go

@@ -13,12 +13,15 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
-	"sync"
+	"path"
 
+	"github.com/dustin/go-humanize"
 	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/TencentBlueKing/bscp-go/client"
 	"github.com/TencentBlueKing/bscp-go/pkg/logger"
@@ -33,13 +36,14 @@ const (
 	outputFormatJson      = "json"
 	outputFormatValue     = "value"
 	outputFormatValueJson = "value_json"
+	outputFormatContent   = "content"
 )
 
 var (
 	getCmd = &cobra.Command{
 		Use:   "get",
-		Short: "Display app or kv resources",
-		Long:  `Display app or kv resources`,
+		Short: "Display app, file or kv resources",
+		Long:  `Display app, file or kv resources`,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			// 设置日志等级, get 命令默认是 error
 			if logLevel == "" {
@@ -69,6 +73,15 @@ var (
 		},
 	}
 
+	getFileCmd = &cobra.Command{
+		Use:   "file [res...]",
+		Short: "Display file resources",
+		Long:  `Display file resources`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runGetFile(args)
+		},
+	}
+
 	getKvCmd = &cobra.Command{
 		Use:   "kv [res...]",
 		Short: "Display kv resources",
@@ -89,6 +102,11 @@ func init() {
 	// app 参数
 	getAppCmd.PersistentFlags().StringVarP(&outputFormat, "output", "o", "", "output format, One of: json")
 
+	// file 参数
+	getFileCmd.Flags().StringVarP(&appName, "app", "a", "", "app name")
+	getFileCmd.Flags().StringVarP(&labelsStr, "labels", "l", "", "labels")
+	getFileCmd.PersistentFlags().StringVarP(&outputFormat, "output", "o", "", "output format, One of: json|content")
+
 	// kv 参数
 	getKvCmd.Flags().StringVarP(&appName, "app", "a", "", "app name")
 	getKvCmd.Flags().StringVarP(&labelsStr, "labels", "l", "", "labels")
@@ -145,7 +163,148 @@ func runGetApp(args []string) error {
 	}
 }
 
-func runGetListKv(bscp client.Client, app string, match []string) error {
+func runGetFileList(bscp client.Client, app string, match []string) error {
+	var opts []client.AppOption
+	if len(match) > 0 {
+		opts = append(opts, client.WithAppKey(match[0]))
+	}
+	opts = append(opts, client.WithAppLabels(labels))
+
+	release, err := bscp.PullFiles(app, opts...)
+	if err != nil {
+		return err
+	}
+
+	tableOutput := func() error {
+		table := newTable()
+		table.SetHeader([]string{"File", "ContentID", "Size", "Reviser", "UpdateAt"})
+		for _, v := range release.FileItems {
+			table.Append([]string{
+				path.Join(v.Path, v.Name),
+				v.FileMeta.ContentSpec.Signature,
+				humanize.IBytes(v.FileMeta.ContentSpec.ByteSize),
+				v.FileMeta.ConfigItemRevision.Reviser,
+				refineOutputTime(v.FileMeta.ConfigItemRevision.UpdateAt),
+			})
+		}
+
+		table.Render()
+		return nil
+	}
+
+	switch outputFormat {
+	case outputFormatJson:
+		return jsonOutput(release.FileItems)
+	case outputFormatTable:
+		return tableOutput()
+	default:
+		return fmt.Errorf(
+			`unable to match a printer suitable for the output format "%s", allowed formats are: json,content`,
+			outputFormat)
+	}
+}
+
+func runGetFileContents(bscp client.Client, app string, contentIDs []string) error {
+	release, err := bscp.PullFiles(app, client.WithAppLabels(labels))
+	if err != nil {
+		return err
+	}
+
+	fileMap := make(map[string]*client.ConfigItemFile)
+	allFiles := make([]*client.ConfigItemFile, len(release.FileItems))
+	for idx, f := range release.FileItems {
+		fileMap[f.FileMeta.ContentSpec.Signature] = f
+		allFiles[idx] = f
+	}
+
+	files := allFiles
+	if len(contentIDs) > 0 {
+		files = []*client.ConfigItemFile{}
+		for _, id := range contentIDs {
+			if _, ok := fileMap[id]; !ok {
+				return fmt.Errorf("the file content id %s is not exist for the latest release of app %s", id, app)
+			}
+			files = append(files, fileMap[id])
+		}
+	}
+
+	var contents [][]byte
+	contents, err = getfileContents(files)
+	if err != nil {
+		return err
+	}
+
+	// output only content when getting for just one file which is convenient to save it directly in a file
+	if len(contentIDs) == 1 {
+		_, err = fmt.Fprint(os.Stdout, string(contents[0]))
+		return err
+	}
+
+	output := ""
+	for idx, file := range files {
+		output += fmt.Sprintf("***start No.%d***\nfile: %s\ncontentID: %s\nconent: \n%s\n***end No.%d***\n\n",
+			idx+1, path.Join(file.Path, file.Name), file.FileMeta.ContentSpec.Signature, contents[idx], idx+1)
+	}
+
+	_, err = fmt.Fprint(os.Stdout, output)
+	return err
+}
+
+// getfileContents get file contents concurrently
+func getfileContents(files []*client.ConfigItemFile) ([][]byte, error) {
+	contents := make([][]byte, len(files))
+	g, _ := errgroup.WithContext(context.Background())
+	g.SetLimit(10)
+
+	for i, f := range files {
+		idx, file := i, f
+		g.Go(func() error {
+			content, err := file.GetContent()
+			if err != nil {
+				return err
+			}
+			contents[idx] = content
+			return nil
+		})
+	}
+
+	return contents, g.Wait()
+}
+
+// runGetFile executes the get file command.
+func runGetFile(args []string) error {
+	baseConf, err := initBaseConf()
+	if err != nil {
+		return err
+	}
+
+	if appName == "" {
+		return fmt.Errorf("app must not be empty")
+	}
+
+	bscp, err := client.New(
+		client.WithFeedAddrs(baseConf.GetFeedAddrs()),
+		client.WithBizID(baseConf.Biz),
+		client.WithToken(baseConf.Token),
+		client.WithFileCache(client.FileCache{
+			Enabled:     *baseConf.FileCache.Enabled,
+			CacheDir:    baseConf.FileCache.CacheDir,
+			ThresholdGB: baseConf.FileCache.ThresholdGB,
+		}),
+	)
+
+	if err != nil {
+		return err
+	}
+
+	if outputFormat == outputFormatContent {
+		return runGetFileContents(bscp, appName, args)
+	}
+
+	return runGetFileList(bscp, appName, args)
+}
+
+func runGetKvList(bscp client.Client, app string, match []string) error {
 	release, err := bscp.PullKvs(app, match, client.WithAppLabels(labels))
 	if err != nil {
 		return err
@@ -175,7 +334,8 @@ func runGetListKv(bscp client.Client, app string, match []string) error {
 		return tableOutput()
 	default:
 		return fmt.Errorf(
-			`unable to match a printer suitable for the output format "%s", allowed formats are: json,value`, outputFormat)
+			`unable to match a printer suitable for the output format "%s", allowed formats are: json,value,value_json`,
+			outputFormat)
 	}
 }
 
@@ -206,9 +366,10 @@ func runGetKvValues(bscp client.Client, app string, keys []string) error {
 		}
 	}
 
-	values, hitError := getKvValues(bscp, app, keys)
-	if hitError != nil {
-		return hitError
+	var values []string
+	values, err = getKvValues(bscp, app, keys)
+	if err != nil {
+		return err
 	}
 
 	output := make(map[string]any, len(keys))
@@ -224,32 +385,23 @@ func runGetKvValues(bscp client.Client, app string, keys []string) error {
 
 // getKvValues get kv values concurrently
 func getKvValues(bscp client.Client, app string, keys []string) ([]string, error) {
-	var hitError error
 	values := make([]string, len(keys))
-	pipe := make(chan struct{}, 10)
-	wg := sync.WaitGroup{}
-
-	for idx, key := range keys {
-		wg.Add(1)
-
-		pipe <- struct{}{}
-		go func(idx int, key string) {
-			defer func() {
-				wg.Done()
-				<-pipe
-			}()
+	g, _ := errgroup.WithContext(context.Background())
+	g.SetLimit(10)
 
+	for i, k := range keys {
+		idx, key := i, k
+		g.Go(func() error {
 			value, err := bscp.Get(app, key, client.WithAppLabels(labels))
 			if err != nil {
-				hitError = fmt.Errorf("get kv value failed for key: %s, err:%v", key, err)
-				return
+				return err
 			}
 			values[idx] = value
-		}(idx, key)
+			return nil
+		})
 	}
-	wg.Wait()
 
-	return values, hitError
+	return values, g.Wait()
 }
 
 // runGetKv executes the get kv command.
@@ -285,6 +437,6 @@ func runGetKv(args []string) error {
 	case outputFormatValueJson:
 		return runGetKvValues(bscp, appName, args)
 	default:
-		return runGetListKv(bscp, appName, args)
+		return runGetKvList(bscp, appName, args)
 	}
 }

cmd/bscp/root.go

@@ -51,6 +51,7 @@ func init() {
 	rootCmd.CompletionOptions.DisableDefaultCmd = true
 
 	getCmd.AddCommand(getAppCmd)
+	getCmd.AddCommand(getFileCmd)
 	getCmd.AddCommand(getKvCmd)
 	rootCmd.AddCommand(getCmd)
 

go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/Tencent/bk-bcs/bcs-common v0.0.0-20230912015319-acb7495967f5
-	github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240111082738-ca8d43177bfd
+	github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240220073632-303c7a871f3b
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dustin/go-humanize v1.0.1
 	github.com/fsnotify/fsnotify v1.7.0

go.sum

@@ -40,8 +40,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Tencent/bk-bcs/bcs-common v0.0.0-20230912015319-acb7495967f5 h1:8B8di9Hxgh9aIIMLVkIhQnsJmAwKd1VxgjfHlvJ+864=
 github.com/Tencent/bk-bcs/bcs-common v0.0.0-20230912015319-acb7495967f5/go.mod h1:/9h4LrWU1cVp46TF0DW5sUrqLtUyyMW0A9mnqhCTavA=
-github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240111082738-ca8d43177bfd h1:tsuqMvHPukaQF2cjBfsa94IEwgmvJu4iMhg4vwARLkE=
-github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240111082738-ca8d43177bfd/go.mod h1:PewYJmxWgfuSI3Y8GKjkg6Gr2vtVdomh++hUPEJ+zOk=
+github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240220073632-303c7a871f3b h1:N6CELtoDBk205kJv0NuLzrAJGNcoYp1mleKVh8V18s0=
+github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240220073632-303c7a871f3b/go.mod h1:N+2MnKgdnz9f7kmR+M/T9fF0FBKeGJVc0vWu1NR93Yw=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=