cliccl: add load show incremental to display incremental backups

Previously, `load show` only supports showing metadata of a single backup
manifest. Users are ignorant of the incremental backups and have no ways to
inspect the incremental backup manifests unless they know the incremental
paths beforehand.

This PR updates `load show` with `incremental` subcommand to display
incremental backups, i.e. displaying the auto appended incremental backup
paths of a full backup. After listing the incremental paths, users are able to
inspect an incremental backup manifest with `load show summary`.

see cockroachdb#61131

Release note (cli change): Update `load show` with `incremental`
subcommand to display incremental backups. User can list
incremental backup paths of a full backup by running
`cockroach load show incremental <backup_url>.`

pkg/ccl/backupccl/backup_destination.go

@@ -27,7 +27,7 @@ import (
 	"github.com/cockroachdb/errors"
 )
 
-// fetchPreviousBackup takes a list of URIs of previous backups and returns
+// fetchPreviousBackups takes a list of URIs of previous backups and returns
 // their manifest as well as the encryption options of the first backup in the
 // chain.
 func fetchPreviousBackups(
@@ -123,7 +123,7 @@ func resolveDest(
 		if exists {
 			// The backup in the auto-append directory is the full backup.
 			prevBackupURIs = append(prevBackupURIs, defaultURI)
-			priors, err := findPriorBackupLocations(ctx, defaultStore)
+			priors, err := FindPriorBackupLocations(ctx, defaultStore)
 			for _, prior := range priors {
 				priorURI, err := url.Parse(defaultURI)
 				if err != nil {
@@ -137,7 +137,7 @@ func resolveDest(
 			}
 
 			// Pick a piece-specific suffix and update the destination path(s).
-			partName := endTime.GoTime().Format(dateBasedIncFolderName)
+			partName := endTime.GoTime().Format(DateBasedIncFolderName)
 			partName = path.Join(chosenSuffix, partName)
 			defaultURI, urisByLocalityKV, err = getURIsByLocalityKV(to, partName)
 			if err != nil {

pkg/ccl/backupccl/manifest_handling.go

@@ -71,8 +71,10 @@ const (
 const (
 	// BackupFormatDescriptorTrackingVersion added tracking of complete DBs.
 	BackupFormatDescriptorTrackingVersion uint32 = 1
-
-	dateBasedIncFolderName  = "/20060102/150405.00"
+	// DateBasedIncFolderName is the date format used when creating sub-directories
+	// storing incremental backups for auto-appendable backups.
+	// It is exported for testing backup inspection tooling.
+	DateBasedIncFolderName  = "/20060102/150405.00"
 	dateBasedIntoFolderName = "/2006/01/02-150405.00"
 	latestFileName          = "LATEST"
 )
@@ -104,7 +106,7 @@ func (r BackupFileDescriptors) Less(i, j int) bool {
 }
 
 // ReadBackupManifestFromURI creates an export store from the given URI, then
-// reads and unmarshals a BackupManifest at the standard location in the
+// reads and unmarshalls a BackupManifest at the standard location in the
 // export storage.
 func ReadBackupManifestFromURI(
 	ctx context.Context,
@@ -119,10 +121,12 @@ func ReadBackupManifestFromURI(
 		return BackupManifest{}, err
 	}
 	defer exportStore.Close()
-	return readBackupManifestFromStore(ctx, exportStore, encryption)
+	return ReadBackupManifestFromStore(ctx, exportStore, encryption)
 }
 
-func readBackupManifestFromStore(
+// ReadBackupManifestFromStore reads and unmarshalls a BackupManifest
+// from an export store.
+func ReadBackupManifestFromStore(
 	ctx context.Context,
 	exportStore cloud.ExternalStorage,
 	encryption *jobspb.BackupEncryptionOptions,
@@ -592,12 +596,12 @@ func findPriorBackupNames(ctx context.Context, store cloud.ExternalStorage) ([]s
 	return prev, nil
 }
 
-// findPriorBackupLocations finds "appended" incremental backups by searching
+// FindPriorBackupLocations finds "appended" incremental backups by searching
 // for the subdirectories matching the naming pattern (e.g. YYMMDD/HHmmss.ss).
 // Using file-system searching rather than keeping an explicit list allows
 // layers to be manually moved/removed/etc without needing to update/maintain
 // said list.
-func findPriorBackupLocations(ctx context.Context, store cloud.ExternalStorage) ([]string, error) {
+func FindPriorBackupLocations(ctx context.Context, store cloud.ExternalStorage) ([]string, error) {
 	backupManifestSuffix := backupManifestName
 	prev, err := store.ListFiles(ctx, incBackupSubdirGlob+backupManifestSuffix)
 	if err != nil {
@@ -641,7 +645,7 @@ func resolveBackupManifests(
 	localityInfo []jobspb.RestoreDetails_BackupLocalityInfo,
 	_ error,
 ) {
-	baseManifest, err := readBackupManifestFromStore(ctx, baseStores[0], encryption)
+	baseManifest, err := ReadBackupManifestFromStore(ctx, baseStores[0], encryption)
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -666,7 +670,7 @@ func resolveBackupManifests(
 				defer stores[j].Close()
 			}
 
-			mainBackupManifests[i], err = readBackupManifestFromStore(ctx, stores[0], encryption)
+			mainBackupManifests[i], err = ReadBackupManifestFromStore(ctx, stores[0], encryption)
 			if err != nil {
 				return nil, nil, nil, err
 			}

pkg/ccl/backupccl/show.go

@@ -185,7 +185,7 @@ func showBackupPlanHook(
 		}
 
 		manifests := make([]BackupManifest, len(incPaths)+1)
-		manifests[0], err = readBackupManifestFromStore(ctx, store, encryption)
+		manifests[0], err = ReadBackupManifestFromStore(ctx, store, encryption)
 		if err != nil {
 			return err
 		}

pkg/ccl/cliccl/BUILD.bazel

@@ -62,14 +62,17 @@ go_test(
     deps = [
         "//pkg/base",
         "//pkg/build",
+        "//pkg/ccl/backupccl",
         "//pkg/ccl/utilccl",
         "//pkg/cli",
         "//pkg/server",
         "//pkg/testutils",
         "//pkg/testutils/serverutils",
         "//pkg/testutils/sqlutils",
+        "//pkg/util/hlc",
         "//pkg/util/leaktest",
         "//pkg/util/log",
+        "//pkg/util/timeutil",
         "@com_github_stretchr_testify//require",
     ],
 )

pkg/ccl/cliccl/load.go

@@ -11,8 +11,11 @@ package cliccl
 import (
 	"context"
 	"fmt"
+	"net/url"
+	"os"
 	"path/filepath"
 	"strings"
+	"text/tabwriter"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
@@ -48,6 +51,14 @@ func init() {
 		RunE:  cli.MaybeDecorateGRPCError(runLoadShowSummary),
 	}
 
+	loadShowIncrementalCmd := &cobra.Command{
+		Use:   "incremental <backup_path>",
+		Short: "show incremental backups",
+		Long:  "Shows incremental chain of a SQL backup.",
+		Args:  cobra.ExactArgs(1),
+		RunE:  cli.MaybeDecorateGRPCError(runLoadShowIncremental),
+	}
+
 	loadShowCmds := &cobra.Command{
 		Use:   "show [command]",
 		Short: "show backups",
@@ -78,7 +89,9 @@ func init() {
 	cli.AddCmd(loadCmds)
 	loadCmds.AddCommand(loadShowCmds)
 	loadShowCmds.AddCommand(loadShowSummaryCmd)
+	loadShowCmds.AddCommand(loadShowIncrementalCmd)
 	loadShowSummaryCmd.Flags().AddFlagSet(loadFlags)
+	loadShowIncrementalCmd.Flags().AddFlagSet(loadFlags)
 }
 
 func newBlobFactory(ctx context.Context, dialing roachpb.NodeID) (blobs.BlobClient, error) {
@@ -121,6 +134,67 @@ func runLoadShowSummary(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func runLoadShowIncremental(cmd *cobra.Command, args []string) error {
+
+	path := args[0]
+	if !strings.Contains(path, "://") {
+		path = cloudimpl.MakeLocalStorageURI(path)
+	}
+
+	uri, err := url.Parse(path)
+	if err != nil {
+		return err
+	}
+
+	ctx := context.Background()
+	store, err := cloudimpl.ExternalStorageFromURI(ctx, uri.String(), base.ExternalIODirConfig{},
+		cluster.NoSettings, newBlobFactory, security.RootUserName(), nil /*Internal Executor*/, nil /*kvDB*/)
+	if err != nil {
+		return errors.Wrapf(err, "connect to external storage")
+	}
+	defer store.Close()
+
+	incPaths, err := backupccl.FindPriorBackupLocations(ctx, store)
+	if err != nil {
+		return err
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 28, 1, 2, ' ', 0)
+	basepath := uri.Path
+	manifestPaths := append([]string{""}, incPaths...)
+	stores := make([]cloud.ExternalStorage, len(manifestPaths))
+	stores[0] = store
+
+	for i := range manifestPaths {
+
+		if i > 0 {
+			uri.Path = filepath.Join(basepath, manifestPaths[i])
+			stores[i], err = cloudimpl.ExternalStorageFromURI(ctx, uri.String(), base.ExternalIODirConfig{},
+				cluster.NoSettings, newBlobFactory, security.RootUserName(), nil /*Internal Executor*/, nil /*kvDB*/)
+			if err != nil {
+				return errors.Wrapf(err, "connect to external storage")
+			}
+			defer stores[i].Close()
+		}
+
+		manifest, err := backupccl.ReadBackupManifestFromStore(ctx, stores[i], nil)
+		if err != nil {
+			return err
+		}
+		startTime := manifest.StartTime.GoTime().Format(time.RFC3339)
+		endTime := manifest.EndTime.GoTime().Format(time.RFC3339)
+		if i == 0 {
+			startTime = "-"
+		}
+		fmt.Fprintf(w, "%s	%s	%s\n", uri.Path, startTime, endTime)
+	}
+
+	if err := w.Flush(); err != nil {
+		return err
+	}
+	return nil
+}
+
 func showMeta(desc backupccl.BackupManifest) {
 	start := timeutil.Unix(0, desc.StartTime.WallTime).Format(time.RFC3339Nano)
 	end := timeutil.Unix(0, desc.EndTime.WallTime).Format(time.RFC3339Nano)

pkg/ccl/cliccl/load_test.go

@@ -9,21 +9,28 @@
 package cliccl
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"strings"
 	"testing"
+	"text/tabwriter"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
+	"github.com/cockroachdb/cockroach/pkg/ccl/backupccl"
 	"github.com/cockroachdb/cockroach/pkg/cli"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
+	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/stretchr/testify/require"
 )
 
-func TestLoadShow(t *testing.T) {
+func TestLoadShowSummary(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
@@ -102,3 +109,54 @@ Tables:
 		}
 	})
 }
+
+func TestLoadShowIncremental(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	c := cli.NewCLITest(cli.TestCLIParams{T: t, NoServer: true})
+	defer c.Cleanup()
+
+	ctx := context.Background()
+	dir, cleanFn := testutils.TempDir(t)
+	defer cleanFn()
+	srv, db, _ := serverutils.StartServer(t, base.TestServerArgs{ExternalIODir: dir, Insecure: true})
+	defer srv.Stopper().Stop(ctx)
+
+	sqlDB := sqlutils.MakeSQLRunner(db)
+	sqlDB.Exec(t, `CREATE DATABASE testDB`)
+	sqlDB.Exec(t, `USE testDB`)
+	const backupPath = "nodelocal://0/fooFolder"
+	initTS := hlc.Timestamp{WallTime: timeutil.Now().UnixNano()}
+	initAOST := initTS.AsOfSystemTime()
+	sqlDB.Exec(t, fmt.Sprintf(`BACKUP DATABASE testDB TO $1 AS OF SYSTEM TIME '%s'`, initAOST), backupPath)
+
+	// We do two more incremental backups on the full backups.
+	incTS := hlc.Timestamp{WallTime: timeutil.Now().UnixNano()}
+	incAOST := incTS.AsOfSystemTime()
+	sqlDB.Exec(t, fmt.Sprintf(`BACKUP DATABASE testDB TO $1 AS OF SYSTEM TIME '%s'`, incAOST), backupPath)
+	incTS2 := hlc.Timestamp{WallTime: timeutil.Now().UnixNano()}
+	incAOST2 := incTS2.AsOfSystemTime()
+	sqlDB.Exec(t, fmt.Sprintf(`BACKUP DATABASE testDB TO $1 AS OF SYSTEM TIME '%s'`, incAOST2), backupPath)
+
+	out, err := c.RunWithCapture(fmt.Sprintf("load show incremental %s --external-io-dir=%s", backupPath, dir))
+	require.NoError(t, err)
+	expectedIncFolder := incTS.GoTime().Format(backupccl.DateBasedIncFolderName)
+	expectedIncFolder2 := incTS2.GoTime().Format(backupccl.DateBasedIncFolderName)
+
+	var buf bytes.Buffer
+	w := tabwriter.NewWriter(&buf, 28, 1, 2, ' ', 0)
+	fmt.Fprintf(w, "/fooFolder	-	%s\n", initTS.GoTime().Format(time.RFC3339))
+	fmt.Fprintf(w, "/fooFolder%s	%s	%s\n", expectedIncFolder, initTS.GoTime().Format(time.RFC3339), incTS.GoTime().Format(time.RFC3339))
+	fmt.Fprintf(w, "/fooFolder%s	%s	%s\n", expectedIncFolder2, incTS.GoTime().Format(time.RFC3339), incTS2.GoTime().Format(time.RFC3339))
+	if err := w.Flush(); err != nil {
+		t.Fatalf("TestLoadShowIncremental: flush: %v", err)
+	}
+	checkExpectedOutput(t, buf.String(), out)
+}
+
+func checkExpectedOutput(t *testing.T, expected string, out string) {
+	endOfCmd := strings.Index(out, "\n")
+	output := out[endOfCmd+1:]
+	require.Equal(t, expected, output)
+}