cliccl/load: update load show with summary subcommand to display backup meta information

pkg/blobs/client.go

@@ -128,8 +128,8 @@ type localClient struct {
 	localStorage *LocalStorage
 }
 
-// newLocalClient instantiates a local blob service client.
-func newLocalClient(externalIODir string) (BlobClient, error) {
+// NewLocalClient instantiates a local blob service client.
+func NewLocalClient(externalIODir string) (BlobClient, error) {
 	storage, err := NewLocalStorage(externalIODir)
 	if err != nil {
 		return nil, errors.Wrap(err, "creating local client")
@@ -168,7 +168,7 @@ func NewBlobClientFactory(
 ) BlobClientFactory {
 	return func(ctx context.Context, dialing roachpb.NodeID) (BlobClient, error) {
 		if dialing == 0 || localNodeID == dialing {
-			return newLocalClient(externalIODir)
+			return NewLocalClient(externalIODir)
 		}
 		conn, err := dialer.Dial(ctx, dialing, rpc.DefaultClass)
 		if err != nil {

pkg/blobs/testutils.go

@@ -20,7 +20,7 @@ import (
 // in tests that use nodelocal storage.
 func TestBlobServiceClient(externalIODir string) BlobClientFactory {
 	return func(ctx context.Context, dialing roachpb.NodeID) (BlobClient, error) {
-		return newLocalClient(externalIODir)
+		return NewLocalClient(externalIODir)
 	}
 }
 

pkg/ccl/cliccl/BUILD.bazel

@@ -23,9 +23,15 @@ go_library(
         "//pkg/ccl/storageccl/engineccl/enginepbccl:enginepbccl_go_proto",
         "//pkg/ccl/workloadccl/cliccl",
         "//pkg/cli",
+        "//pkg/cli/cliflags",
+        "//pkg/keys",
+        "//pkg/roachpb",
         "//pkg/security",
+        "//pkg/server",
         "//pkg/settings/cluster",
         "//pkg/sql/catalog/descpb",
+        "//pkg/sql/catalog/tabledesc",
+        "//pkg/sql/sessiondata",
         "//pkg/storage/cloud",
         "//pkg/storage/cloudimpl",
         "//pkg/storage/enginepb",
@@ -48,11 +54,22 @@ go_library(
 go_test(
     name = "cliccl_test",
     size = "small",
-    srcs = ["main_test.go"],
+    srcs = [
+        "load_test.go",
+        "main_test.go",
+    ],
+    embed = [":cliccl"],
     deps = [
+        "//pkg/base",
         "//pkg/build",
         "//pkg/ccl/utilccl",
+        "//pkg/cli",
         "//pkg/server",
+        "//pkg/testutils",
         "//pkg/testutils/serverutils",
+        "//pkg/testutils/sqlutils",
+        "//pkg/util/leaktest",
+        "//pkg/util/log",
+        "@com_github_stretchr_testify//require",
     ],
 )

pkg/ccl/cliccl/load.go

@@ -11,16 +11,23 @@ package cliccl
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/blobs"
 	"github.com/cockroachdb/cockroach/pkg/ccl/backupccl"
 	"github.com/cockroachdb/cockroach/pkg/cli"
+	"github.com/cockroachdb/cockroach/pkg/cli/cliflags"
+	"github.com/cockroachdb/cockroach/pkg/keys"
+	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/security"
+	"github.com/cockroachdb/cockroach/pkg/server"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
+	"github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc"
+	"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
 	"github.com/cockroachdb/cockroach/pkg/storage/cloud"
 	"github.com/cockroachdb/cockroach/pkg/storage/cloudimpl"
 	"github.com/cockroachdb/cockroach/pkg/util/humanizeutil"
@@ -29,51 +36,92 @@ import (
 	"github.com/spf13/cobra"
 )
 
+var externalIODir string
+
 func init() {
-	loadShowCmd := &cobra.Command{
-		Use:   "show <basepath>",
+
+	loadShowSummaryCmd := &cobra.Command{
+		Use:   "summary <backup_path>",
+		Short: "show backups summary",
+		Long:  "Shows summary of meta information about a SQL backup.",
+		Args:  cobra.ExactArgs(1),
+		RunE:  cli.MaybeDecorateGRPCError(runLoadShowSummary),
+	}
+
+	loadShowCmds := &cobra.Command{
+		Use:   "show [command]",
 		Short: "show backups",
 		Long:  "Shows information about a SQL backup.",
-		RunE:  cli.MaybeDecorateGRPCError(runLoadShow),
+		Args:  cobra.MinimumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return cmd.Usage()
+		},
 	}
 
 	loadCmds := &cobra.Command{
 		Use:   "load [command]",
-		Short: "loading commands",
+		Short: "load backup commands",
 		Long:  `Commands for bulk loading external files.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return cmd.Usage()
 		},
 	}
+
+	loadFlags := loadCmds.Flags()
+	loadFlags.StringVarP(
+		&externalIODir,
+		cliflags.ExternalIODir.Name,
+		cliflags.ExternalIODir.Shorthand,
+		"", /*value*/
+		cliflags.ExternalIODir.Usage())
+
 	cli.AddCmd(loadCmds)
-	loadCmds.AddCommand(loadShowCmd)
+	loadCmds.AddCommand(loadShowCmds)
+	loadShowCmds.AddCommand(loadShowSummaryCmd)
+	loadShowSummaryCmd.Flags().AddFlagSet(loadFlags)
 }
 
-func runLoadShow(cmd *cobra.Command, args []string) error {
-	if len(args) != 1 {
-		return errors.New("basepath argument is required")
+func newBlobFactory(ctx context.Context, dialing roachpb.NodeID) (blobs.BlobClient, error) {
+	if dialing != 0 {
+		return nil, errors.Errorf("accessing node %d during nodelocal access is unsupported for CLI inspection; only local access is supported with nodelocal://self", dialing)
+	}
+	if externalIODir == "" {
+		externalIODir = filepath.Join(server.DefaultStorePath, "extern")
 	}
+	return blobs.NewLocalClient(externalIODir)
+}
 
-	ctx := context.Background()
-	basepath := args[0]
-	if !strings.Contains(basepath, "://") {
-		basepath = cloudimpl.MakeLocalStorageURI(basepath)
+func runLoadShowSummary(cmd *cobra.Command, args []string) error {
+
+	path := args[0]
+	if !strings.Contains(path, "://") {
+		path = cloudimpl.MakeLocalStorageURI(path)
 	}
 
+	ctx := context.Background()
 	externalStorageFromURI := func(ctx context.Context, uri string,
 		user security.SQLUsername) (cloud.ExternalStorage, error) {
 		return cloudimpl.ExternalStorageFromURI(ctx, uri, base.ExternalIODirConfig{},
-			cluster.NoSettings, blobs.TestEmptyBlobClientFactory, user, nil, nil)
+			cluster.NoSettings, newBlobFactory, user, nil /*Internal Executor*/, nil /*kvDB*/)
 	}
+
 	// This reads the raw backup descriptor (with table descriptors possibly not
 	// upgraded from the old FK representation, or even older formats). If more
 	// fields are added to the output, the table descriptors may need to be
 	// upgraded.
-	desc, err := backupccl.ReadBackupManifestFromURI(ctx, basepath, security.RootUserName(),
+	desc, err := backupccl.ReadBackupManifestFromURI(ctx, path, security.RootUserName(),
 		externalStorageFromURI, nil)
 	if err != nil {
 		return err
 	}
+	showMeta(desc)
+	showSpans(desc)
+	showFiles(desc)
+	showDescriptors(desc)
+	return nil
+}
+
+func showMeta(desc backupccl.BackupManifest) {
 	start := timeutil.Unix(0, desc.StartTime.WallTime).Format(time.RFC3339Nano)
 	end := timeutil.Unix(0, desc.EndTime.WallTime).Format(time.RFC3339Nano)
 	fmt.Printf("StartTime: %s (%s)\n", start, desc.StartTime)
@@ -85,35 +133,104 @@ func runLoadShow(cmd *cobra.Command, args []string) error {
 	fmt.Printf("ClusterID: %s\n", desc.ClusterID)
 	fmt.Printf("NodeID: %s\n", desc.NodeID)
 	fmt.Printf("BuildInfo: %s\n", desc.BuildInfo.Short())
+}
+
+func showSpans(desc backupccl.BackupManifest) {
 	fmt.Printf("Spans:\n")
+	if len(desc.Spans) == 0 {
+		fmt.Printf("	(No spans included in the specified backup path.)\n")
+	}
 	for _, s := range desc.Spans {
 		fmt.Printf("	%s\n", s)
 	}
+}
+
+func showFiles(desc backupccl.BackupManifest) {
 	fmt.Printf("Files:\n")
+	if len(desc.Files) == 0 {
+		fmt.Printf("	(No sst files included in the specified backup path.)\n")
+	}
 	for _, f := range desc.Files {
 		fmt.Printf("	%s:\n", f.Path)
 		fmt.Printf("		Span: %s\n", f.Span)
-		fmt.Printf("		Sha512: %0128x\n", f.Sha512)
 		fmt.Printf("		DataSize: %d (%s)\n", f.EntryCounts.DataSize, humanizeutil.IBytes(f.EntryCounts.DataSize))
 		fmt.Printf("		Rows: %d\n", f.EntryCounts.Rows)
 		fmt.Printf("		IndexEntries: %d\n", f.EntryCounts.IndexEntries)
 	}
+}
+
+func showDescriptors(desc backupccl.BackupManifest) {
 	// Note that these descriptors could be from any past version of the cluster,
 	// in case more fields need to be added to the output.
-	fmt.Printf("Descriptors:\n")
+	dbIDs := make([]descpb.ID, 0, len(desc.Descriptors))
+	dbIDToName := make(map[descpb.ID]string)
+	schemaIDs := make([]descpb.ID, 0, len(desc.Descriptors))
+	schemaIDs = append(schemaIDs, keys.PublicSchemaID)
+	schemaIDToFullyQualifiedName := make(map[descpb.ID]string)
+	schemaIDToFullyQualifiedName[keys.PublicSchemaID] = sessiondata.PublicSchemaName
+	typeIDs := make([]descpb.ID, 0, len(desc.Descriptors))
+	typeIDToFullyQualifiedName := make(map[descpb.ID]string)
+	tableIDs := make([]descpb.ID, 0, len(desc.Descriptors))
+	tableIDToFullyQualifiedName := make(map[descpb.ID]string)
 	for i := range desc.Descriptors {
 		d := &desc.Descriptors[i]
-		table, database, _, _ := descpb.FromDescriptor(d)
-		var typeName string
-		if table != nil {
-			typeName = "table"
-		} else if database != nil {
-			typeName = "database"
-		} else {
-			continue
+		id := descpb.GetDescriptorID(d)
+		tableDesc, databaseDesc, typeDesc, schemaDesc := descpb.FromDescriptor(d)
+		if databaseDesc != nil {
+			dbIDToName[id] = descpb.GetDescriptorName(d)
+			dbIDs = append(dbIDs, id)
+		} else if schemaDesc != nil {
+			dbName := dbIDToName[schemaDesc.GetParentID()]
+			schemaName := descpb.GetDescriptorName(d)
+			schemaIDToFullyQualifiedName[id] = dbName + "." + schemaName
+			schemaIDs = append(schemaIDs, id)
+		} else if typeDesc != nil {
+			parentSchema := schemaIDToFullyQualifiedName[typeDesc.GetParentSchemaID()]
+			if parentSchema == sessiondata.PublicSchemaName {
+				parentSchema = dbIDToName[typeDesc.GetParentID()] + "." + parentSchema
+			}
+			typeName := descpb.GetDescriptorName(d)
+			typeIDToFullyQualifiedName[id] = parentSchema + "." + typeName
+			typeIDs = append(typeIDs, id)
+		} else if tableDesc != nil {
+			tbDesc := tabledesc.NewBuilder(tableDesc).BuildImmutable()
+			parentSchema := schemaIDToFullyQualifiedName[tbDesc.GetParentSchemaID()]
+			if parentSchema == sessiondata.PublicSchemaName {
+				parentSchema = dbIDToName[tableDesc.GetParentID()] + "." + parentSchema
+			}
+			tableName := descpb.GetDescriptorName(d)
+			tableIDToFullyQualifiedName[id] = parentSchema + "." + tableName
+			tableIDs = append(tableIDs, id)
 		}
-		fmt.Printf("	%d: %s (%s)\n",
-			descpb.GetDescriptorID(d), descpb.GetDescriptorName(d), typeName)
 	}
-	return nil
+
+	fmt.Printf("Databases:\n")
+	for _, id := range dbIDs {
+		fmt.Printf("	%d: %s\n",
+			id, dbIDToName[id])
+	}
+
+	fmt.Printf("Schemas:\n")
+	for _, id := range schemaIDs {
+		fmt.Printf("	%d: %s\n",
+			id, schemaIDToFullyQualifiedName[id])
+	}
+
+	fmt.Printf("Types:\n")
+	if len(typeIDs) == 0 {
+		fmt.Printf("	(No user-defined types included in the specified backup path.)\n")
+	}
+	for _, id := range typeIDs {
+		fmt.Printf("	%d: %s\n",
+			id, typeIDToFullyQualifiedName[id])
+	}
+
+	fmt.Printf("Tables:\n")
+	if len(tableIDs) == 0 {
+		fmt.Printf("	(No tables included in the specified backup path.)\n")
+	}
+	for _, id := range tableIDs {
+		fmt.Printf("	%d: %s\n",
+			id, tableIDToFullyQualifiedName[id])
+	}
 }