cliccl: add load show backups to display backup collection

pkg/ccl/backupccl/backup_destination.go

@@ -27,7 +27,7 @@ import (
 	"github.com/cockroachdb/errors"
 )
 
-// fetchPreviousBackup takes a list of URIs of previous backups and returns
+// fetchPreviousBackups takes a list of URIs of previous backups and returns
 // their manifest as well as the encryption options of the first backup in the
 // chain.
 func fetchPreviousBackups(
@@ -123,7 +123,7 @@ func resolveDest(
 		if exists {
 			// The backup in the auto-append directory is the full backup.
 			prevBackupURIs = append(prevBackupURIs, defaultURI)
-			priors, err := findPriorBackupLocations(ctx, defaultStore)
+			priors, err := FindPriorBackupLocations(ctx, defaultStore)
 			for _, prior := range priors {
 				priorURI, err := url.Parse(defaultURI)
 				if err != nil {
@@ -137,7 +137,7 @@ func resolveDest(
 			}
 
 			// Pick a piece-specific suffix and update the destination path(s).
-			partName := endTime.GoTime().Format(dateBasedIncFolderName)
+			partName := endTime.GoTime().Format(DateBasedIncFolderName)
 			partName = path.Join(chosenSuffix, partName)
 			defaultURI, urisByLocalityKV, err = getURIsByLocalityKV(to, partName)
 			if err != nil {
@@ -272,7 +272,7 @@ func resolveBackupCollection(
 		chosenSuffix = strings.TrimPrefix(subdir, "/")
 		chosenSuffix = "/" + chosenSuffix
 	} else {
-		chosenSuffix = endTime.GoTime().Format(dateBasedIntoFolderName)
+		chosenSuffix = endTime.GoTime().Format(DateBasedIntoFolderName)
 	}
 	return collectionURI, chosenSuffix, nil
 }

pkg/ccl/backupccl/manifest_handling.go

@@ -72,8 +72,14 @@ const (
 	// BackupFormatDescriptorTrackingVersion added tracking of complete DBs.
 	BackupFormatDescriptorTrackingVersion uint32 = 1
 
-	dateBasedIncFolderName  = "/20060102/150405.00"
-	dateBasedIntoFolderName = "/2006/01/02-150405.00"
+	// DateBasedIncFolderName is the date format used when creating sub-directories
+	// storing incremental backups for auto-appendable backups.
+	// It is exported for testing backup inspection tooling.
+	DateBasedIncFolderName = "/20060102/150405.00"
+	// DateBasedIntoFolderName is the date format used when creating sub-directories
+	// for storing backups in a collection.
+	// Also exported for testing backup inspection tooling.
+	DateBasedIntoFolderName = "/2006/01/02-150405.00"
 	latestFileName          = "LATEST"
 )
 
@@ -104,7 +110,7 @@ func (r BackupFileDescriptors) Less(i, j int) bool {
 }
 
 // ReadBackupManifestFromURI creates an export store from the given URI, then
-// reads and unmarshals a BackupManifest at the standard location in the
+// reads and unmarshalls a BackupManifest at the standard location in the
 // export storage.
 func ReadBackupManifestFromURI(
 	ctx context.Context,
@@ -119,10 +125,12 @@ func ReadBackupManifestFromURI(
 		return BackupManifest{}, err
 	}
 	defer exportStore.Close()
-	return readBackupManifestFromStore(ctx, exportStore, encryption)
+	return ReadBackupManifestFromStore(ctx, exportStore, encryption)
 }
 
-func readBackupManifestFromStore(
+// ReadBackupManifestFromStore reads and unmarshalls a BackupManifest
+// from an export store.
+func ReadBackupManifestFromStore(
 	ctx context.Context,
 	exportStore cloud.ExternalStorage,
 	encryption *jobspb.BackupEncryptionOptions,
@@ -592,12 +600,12 @@ func findPriorBackupNames(ctx context.Context, store cloud.ExternalStorage) ([]s
 	return prev, nil
 }
 
-// findPriorBackupLocations finds "appended" incremental backups by searching
+// FindPriorBackupLocations finds "appended" incremental backups by searching
 // for the subdirectories matching the naming pattern (e.g. YYMMDD/HHmmss.ss).
 // Using file-system searching rather than keeping an explicit list allows
 // layers to be manually moved/removed/etc without needing to update/maintain
 // said list.
-func findPriorBackupLocations(ctx context.Context, store cloud.ExternalStorage) ([]string, error) {
+func FindPriorBackupLocations(ctx context.Context, store cloud.ExternalStorage) ([]string, error) {
 	backupManifestSuffix := backupManifestName
 	prev, err := store.ListFiles(ctx, incBackupSubdirGlob+backupManifestSuffix)
 	if err != nil {
@@ -641,7 +649,7 @@ func resolveBackupManifests(
 	localityInfo []jobspb.RestoreDetails_BackupLocalityInfo,
 	_ error,
 ) {
-	baseManifest, err := readBackupManifestFromStore(ctx, baseStores[0], encryption)
+	baseManifest, err := ReadBackupManifestFromStore(ctx, baseStores[0], encryption)
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -666,7 +674,7 @@ func resolveBackupManifests(
 				defer stores[j].Close()
 			}
 
-			mainBackupManifests[i], err = readBackupManifestFromStore(ctx, stores[0], encryption)
+			mainBackupManifests[i], err = ReadBackupManifestFromStore(ctx, stores[0], encryption)
 			if err != nil {
 				return nil, nil, nil, err
 			}
@@ -997,3 +1005,18 @@ func checkForPreviousBackup(
 func tempCheckpointFileNameForJob(jobID jobspb.JobID) string {
 	return fmt.Sprintf("%s-%d", backupManifestCheckpointName, jobID)
 }
+
+// ListFullBackupsInCollection lists full backup paths in the collection
+// of an export store
+func ListFullBackupsInCollection(
+	ctx context.Context, store cloud.ExternalStorage,
+) ([]string, error) {
+	backupPaths, err := store.ListFiles(ctx, "/*/*/*/"+backupManifestName)
+	if err != nil {
+		return nil, err
+	}
+	for i, backupPath := range backupPaths {
+		backupPaths[i] = strings.TrimSuffix(backupPath, "/"+backupManifestName)
+	}
+	return backupPaths, nil
+}

pkg/ccl/backupccl/show.go

@@ -185,7 +185,7 @@ func showBackupPlanHook(
 		}
 
 		manifests := make([]BackupManifest, len(incPaths)+1)
-		manifests[0], err = readBackupManifestFromStore(ctx, store, encryption)
+		manifests[0], err = ReadBackupManifestFromStore(ctx, store, encryption)
 		if err != nil {
 			return err
 		}
@@ -532,12 +532,12 @@ func showBackupsInCollectionPlanHook(
 			return errors.Wrapf(err, "connect to external storage")
 		}
 		defer store.Close()
-		res, err := store.ListFiles(ctx, "/*/*/*/"+backupManifestName)
+		res, err := ListFullBackupsInCollection(ctx, store)
 		if err != nil {
 			return err
 		}
 		for _, i := range res {
-			resultsCh <- tree.Datums{tree.NewDString(strings.TrimSuffix(i, "/"+backupManifestName))}
+			resultsCh <- tree.Datums{tree.NewDString(i)}
 		}
 		return nil
 	}

pkg/ccl/cliccl/BUILD.bazel

@@ -62,14 +62,17 @@ go_test(
     deps = [
         "//pkg/base",
         "//pkg/build",
+        "//pkg/ccl/backupccl",
         "//pkg/ccl/utilccl",
         "//pkg/cli",
         "//pkg/server",
         "//pkg/testutils",
         "//pkg/testutils/serverutils",
         "//pkg/testutils/sqlutils",
+        "//pkg/util/hlc",
         "//pkg/util/leaktest",
         "//pkg/util/log",
+        "//pkg/util/timeutil",
         "@com_github_stretchr_testify//require",
     ],
 )

pkg/ccl/cliccl/load.go

@@ -11,8 +11,11 @@ package cliccl
 import (
 	"context"
 	"fmt"
+	"net/url"
+	"os"
 	"path/filepath"
 	"strings"
+	"text/tabwriter"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
@@ -48,6 +51,22 @@ func init() {
 		RunE:  cli.MaybeDecorateGRPCError(runLoadShowSummary),
 	}
 
+	loadShowBackupsCmd := &cobra.Command{
+		Use:   "backups <backup_path>",
+		Short: "show backups in collections",
+		Long:  "Shows full backups in a backup collections.",
+		Args:  cobra.ExactArgs(1),
+		RunE:  cli.MaybeDecorateGRPCError(runLoadShowBackups),
+	}
+
+	loadShowIncrementalCmd := &cobra.Command{
+		Use:   "incremental <backup_path>",
+		Short: "show incremental backups",
+		Long:  "Shows incremental chain of a SQL backup.",
+		Args:  cobra.ExactArgs(1),
+		RunE:  cli.MaybeDecorateGRPCError(runLoadShowIncremental),
+	}
+
 	loadShowCmds := &cobra.Command{
 		Use:   "show [command]",
 		Short: "show backups",
@@ -77,8 +96,14 @@ func init() {
 
 	cli.AddCmd(loadCmds)
 	loadCmds.AddCommand(loadShowCmds)
-	loadShowCmds.AddCommand(loadShowSummaryCmd)
+	loadShowCmds.AddCommand([]*cobra.Command{
+		loadShowSummaryCmd,
+		loadShowBackupsCmd,
+		loadShowIncrementalCmd,
+	}...)
 	loadShowSummaryCmd.Flags().AddFlagSet(loadFlags)
+	loadShowBackupsCmd.Flags().AddFlagSet(loadFlags)
+	loadShowIncrementalCmd.Flags().AddFlagSet(loadFlags)
 }
 
 func newBlobFactory(ctx context.Context, dialing roachpb.NodeID) (blobs.BlobClient, error) {
@@ -121,6 +146,97 @@ func runLoadShowSummary(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func runLoadShowBackups(cmd *cobra.Command, args []string) error {
+
+	path := args[0]
+	if !strings.Contains(path, "://") {
+		path = cloudimpl.MakeLocalStorageURI(path)
+	}
+	ctx := context.Background()
+	store, err := cloudimpl.ExternalStorageFromURI(ctx, path, base.ExternalIODirConfig{},
+		cluster.NoSettings, newBlobFactory, security.RootUserName(), nil /*Internal Executor*/, nil /*kvDB*/)
+	if err != nil {
+		return errors.Wrapf(err, "connect to external storage")
+	}
+	defer store.Close()
+
+	backupPaths, err := backupccl.ListFullBackupsInCollection(ctx, store)
+	if err != nil {
+		return errors.Wrapf(err, "list full backups in collection")
+	}
+
+	if len(backupPaths) == 0 {
+		fmt.Println("no backups found.")
+	}
+
+	for _, backupPath := range backupPaths {
+		fmt.Println("./" + backupPath)
+	}
+
+	return nil
+}
+
+func runLoadShowIncremental(cmd *cobra.Command, args []string) error {
+
+	path := args[0]
+	if !strings.Contains(path, "://") {
+		path = cloudimpl.MakeLocalStorageURI(path)
+	}
+
+	uri, err := url.Parse(path)
+	if err != nil {
+		return err
+	}
+
+	ctx := context.Background()
+	store, err := cloudimpl.ExternalStorageFromURI(ctx, uri.String(), base.ExternalIODirConfig{},
+		cluster.NoSettings, newBlobFactory, security.RootUserName(), nil /*Internal Executor*/, nil /*kvDB*/)
+	if err != nil {
+		return errors.Wrapf(err, "connect to external storage")
+	}
+	defer store.Close()
+
+	incPaths, err := backupccl.FindPriorBackupLocations(ctx, store)
+	if err != nil {
+		return err
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 28, 1, 2, ' ', 0)
+	basepath := uri.Path
+	manifestPaths := append([]string{""}, incPaths...)
+	stores := make([]cloud.ExternalStorage, len(manifestPaths))
+	stores[0] = store
+
+	for i := range manifestPaths {
+
+		if i > 0 {
+			uri.Path = filepath.Join(basepath, manifestPaths[i])
+			stores[i], err = cloudimpl.ExternalStorageFromURI(ctx, uri.String(), base.ExternalIODirConfig{},
+				cluster.NoSettings, newBlobFactory, security.RootUserName(), nil /*Internal Executor*/, nil /*kvDB*/)
+			if err != nil {
+				return errors.Wrapf(err, "connect to external storage")
+			}
+			defer stores[i].Close()
+		}
+
+		manifest, err := backupccl.ReadBackupManifestFromStore(ctx, stores[i], nil)
+		if err != nil {
+			return err
+		}
+		startTime := manifest.StartTime.GoTime().Format(time.RFC3339)
+		endTime := manifest.EndTime.GoTime().Format(time.RFC3339)
+		if i == 0 {
+			startTime = "-"
+		}
+		fmt.Fprintf(w, "%s	%s	%s\n", uri.Path, startTime, endTime)
+	}
+
+	if err := w.Flush(); err != nil {
+		return err
+	}
+	return nil
+}
+
 func showMeta(desc backupccl.BackupManifest) {
 	start := timeutil.Unix(0, desc.StartTime.WallTime).Format(time.RFC3339Nano)
 	end := timeutil.Unix(0, desc.EndTime.WallTime).Format(time.RFC3339Nano)