Skip to content

Commit

Permalink
cli: export TestCLI for cliccl
Browse files Browse the repository at this point in the history
Previously, clitest only has private access within cli package,
while cliccl lack sufficient tests for ccl cli tools.

With this commit, clitest utils are exported for cliccl and cliccl
can reuse them for unit tests. The `cliTest` struct and
the `cliTestParams` struct are renamed as `TestCLI` and `TestCLIParams`.

Release justification: Low risk, high benefit changes to existing functionality.
Release note: None
  • Loading branch information
Elliebababa committed Mar 10, 2021
1 parent 0f49072 commit 5c95fcf
Show file tree
Hide file tree
Showing 16 changed files with 635 additions and 468 deletions.
19 changes: 15 additions & 4 deletions pkg/ccl/cliccl/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ go_library(
"cliccl.go",
"debug.go",
"demo.go",
"inspect.go",
"load.go",
"mtproxy.go",
"start.go",
Expand All @@ -25,13 +24,14 @@ go_library(
"//pkg/ccl/workloadccl/cliccl",
"//pkg/cli",
"//pkg/cli/cliflags",
"//pkg/keys",
"//pkg/roachpb",
"//pkg/rpc/nodedialer",
"//pkg/security",
"//pkg/server",
"//pkg/settings/cluster",
"//pkg/sql/catalog/catalogkv",
"//pkg/sql/catalog/descpb",
"//pkg/sql/catalog/tabledesc",
"//pkg/sql/sessiondata",
"//pkg/storage/cloud",
"//pkg/storage/cloudimpl",
"//pkg/storage/enginepb",
Expand All @@ -55,11 +55,22 @@ go_library(
go_test(
name = "cliccl_test",
size = "small",
srcs = ["main_test.go"],
srcs = [
"load_test.go",
"main_test.go",
],
embed = [":cliccl"],
deps = [
"//pkg/base",
"//pkg/build",
"//pkg/ccl/utilccl",
"//pkg/cli",
"//pkg/server",
"//pkg/testutils",
"//pkg/testutils/serverutils",
"//pkg/testutils/sqlutils",
"//pkg/util/leaktest",
"//pkg/util/log",
"@com_github_stretchr_testify//require",
],
)
85 changes: 54 additions & 31 deletions pkg/ccl/cliccl/load.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,34 +11,37 @@ package cliccl
import (
"context"
"fmt"
"path/filepath"
"strings"
"time"

"github.com/cockroachdb/cockroach/pkg/base"
"github.com/cockroachdb/cockroach/pkg/blobs"
"github.com/cockroachdb/cockroach/pkg/ccl/backupccl"
"github.com/cockroachdb/cockroach/pkg/cli"
"github.com/cockroachdb/cockroach/pkg/cli/cliflags"
"github.com/cockroachdb/cockroach/pkg/keys"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/security"
"github.com/cockroachdb/cockroach/pkg/server"
"github.com/cockroachdb/cockroach/pkg/settings/cluster"
"github.com/cockroachdb/cockroach/pkg/sql/catalog/catalogkv"
"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
"github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc"
"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
"github.com/cockroachdb/cockroach/pkg/storage/cloud"
"github.com/cockroachdb/cockroach/pkg/storage/cloudimpl"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
"github.com/cockroachdb/cockroach/pkg/util/humanizeutil"
"github.com/cockroachdb/cockroach/pkg/util/timeutil"
"github.com/cockroachdb/errors"
"github.com/spf13/cobra"
"path/filepath"
"strings"
"time"
)

const (
descriptors = "descriptors"
files = "files"
spans = "spans"
metadata ="metadata"
files = "files"
spans = "spans"
metadata = "metadata"
)

var externalIODir string
Expand All @@ -48,7 +51,7 @@ func init() {
Use: "show [descriptors|files|spans|metadata] <backup_path>",
Short: "show backups",
Long: "Shows subset(s) of meta information about a SQL backup.",
Args: cobra.MinimumNArgs(1),
Args: cobra.MinimumNArgs(1),
RunE: cli.MaybeDecorateGRPCError(runLoadShow),
}

Expand All @@ -61,29 +64,30 @@ func init() {
},
}

f := loadCmds.Flags()
f.StringVarP(
loadFlags := loadCmds.Flags()
loadFlags.StringVarP(
&externalIODir,
cliflags.ExternalIODir.Name,
cliflags.ExternalIODir.Shorthand,
""/*value*/,
"", /*value*/
cliflags.ExternalIODir.Usage())

cli.AddCmd(loadCmds)
loadCmds.AddCommand(loadShowCmd)
loadShowCmd.Flags().AddFlagSet(loadFlags)
}

func newBlobFactory(ctx context.Context, dialing roachpb.NodeID) (blobs.BlobClient, error) {
if dialing != 0 {
return nil, errors.Errorf(`only support nodelocal (0/self) under offline inspection`)
}
return nil, errors.Errorf(`only support nodelocal (0/self) under offline inspection`)
}
if externalIODir == "" {
externalIODir = filepath.Join(server.DefaultStorePath, "extern")
}
return blobs.NewLocalClient(externalIODir)
}

func parseShowArgs(args [] string) (options map[string]bool, path string, err error) {
func parseShowArgs(args []string) (options map[string]bool, path string, err error) {
options = make(map[string]bool)
for _, arg := range args {
switch strings.ToLower(arg) {
Expand Down Expand Up @@ -111,7 +115,7 @@ func parseShowArgs(args [] string) (options map[string]bool, path string, err er
}

if len(args) == len(options) {
return nil, "", errors.New("backup_path argument is required")
return nil, "", errors.New("backup_path argument is required")
}
return options, path, nil
}
Expand All @@ -121,7 +125,7 @@ func runLoadShow(cmd *cobra.Command, args []string) error {
var options map[string]bool
var path string
var err error
if options, path, err = parseShowArgs(args); err!=nil {
if options, path, err = parseShowArgs(args); err != nil {
return err
}

Expand All @@ -138,7 +142,7 @@ func runLoadShow(cmd *cobra.Command, args []string) error {
externalStorageFromURI := func(ctx context.Context, uri string,
user security.SQLUsername) (cloud.ExternalStorage, error) {
return cloudimpl.ExternalStorageFromURI(ctx, uri, base.ExternalIODirConfig{},
cluster.NoSettings, newBlobFactory, user, nil/*Internal Executor*/, nil/*kvDB*/)
cluster.NoSettings, newBlobFactory, user, nil /*Internal Executor*/, nil /*kvDB*/)
}

// This reads the raw backup descriptor (with table descriptors possibly not
Expand All @@ -164,7 +168,9 @@ func runLoadShow(cmd *cobra.Command, args []string) error {
}

if _, ok := options[descriptors]; ok {
showDescriptor(desc)
if err := showDescriptor(desc); err != nil {
return err
}
}

return nil
Expand All @@ -185,7 +191,7 @@ func showMeta(desc backupccl.BackupManifest) {
}

func showSpans(desc backupccl.BackupManifest, showHeaders bool) {
tabfmt:= ""
tabfmt := ""
if showHeaders {
fmt.Printf("Spans:\n")
tabfmt = "\t"
Expand All @@ -196,46 +202,63 @@ func showSpans(desc backupccl.BackupManifest, showHeaders bool) {
}

func showFiles(desc backupccl.BackupManifest, showHeaders bool) {
tabfmt:= ""
tabfmt := ""
if showHeaders {
fmt.Printf("Files:\n")
tabfmt = "\t"
}
for _, f := range desc.Files {
fmt.Printf("%s%s:\n", tabfmt, f.Path)
fmt.Printf(" Span: %s\n", f.Span)
fmt.Printf(" Sha512: %0128x\n", f.Sha512)
fmt.Printf(" DataSize: %d (%s)\n", f.EntryCounts.DataSize, humanizeutil.IBytes(f.EntryCounts.DataSize))
fmt.Printf(" Rows: %d\n", f.EntryCounts.Rows)
fmt.Printf(" IndexEntries: %d\n", f.EntryCounts.IndexEntries)
fmt.Printf("%s Span: %s\n", tabfmt, f.Span)
fmt.Printf("%s Sha512: %0128x\n", tabfmt, f.Sha512)
fmt.Printf("%s DataSize: %d (%s)\n", tabfmt, f.EntryCounts.DataSize, humanizeutil.IBytes(f.EntryCounts.DataSize))
fmt.Printf("%s Rows: %d\n", tabfmt, f.EntryCounts.Rows)
fmt.Printf("%s IndexEntries: %d\n", tabfmt, f.EntryCounts.IndexEntries)
}
}

func showDescriptor(desc backupccl.BackupManifest) {
func showDescriptor(desc backupccl.BackupManifest) error {
// Note that these descriptors could be from any past version of the cluster,
// in case more fields need to be added to the output.
dbIDs := make([]descpb.ID, 0)
dbIDToName := make(map[descpb.ID]string)
schemaIDs := make([]descpb.ID, 0)
schemaIDs = append(schemaIDs, keys.PublicSchemaID)
schemaIDToName := make(map[descpb.ID]string)
schemaIDToName[keys.PublicSchemaID] = sessiondata.PublicSchemaName
for i := range desc.Descriptors {
d := &desc.Descriptors[i]
id := descpb.GetDescriptorID(d)
if d.GetDatabase() != nil {
dbIDToName[id] = descpb.GetDescriptorName(d)
dbIDs = append(dbIDs, id)
} else if d.GetSchema() != nil {
schemaIDToName[id] = descpb.GetDescriptorName(d)
schemaIDs = append(schemaIDs, id)
}
}

fmt.Printf("Databases:\n")
for i := range dbIDToName {
for _, id := range dbIDs {
fmt.Printf(" %s\n",
dbIDToName[i])
dbIDToName[id])
}

fmt.Printf("Schemas:\n")
for _, id := range schemaIDs {
fmt.Printf(" %s\n",
schemaIDToName[id])
}

fmt.Printf("Tables:\n")
for i := range desc.Descriptors {
d := &desc.Descriptors[i]
if descpb.TableFromDescriptor(d, hlc.Timestamp{}) != nil {
desc := catalogkv.UnwrapDescriptorRaw(nil, d)
fmt.Printf(" %s (%s) \n",
descpb.GetDescriptorName(d), dbIDToName[desc.GetParentID()])
tbDesc := tabledesc.NewImmutable(*descpb.TableFromDescriptor(d, hlc.Timestamp{}))
dbName := dbIDToName[tbDesc.GetParentID()]
schemaName := schemaIDToName[tbDesc.GetParentSchemaID()]
fmt.Printf(" %s.%s.%s\n", dbName, schemaName, descpb.GetDescriptorName(d))
}
}
return nil
}
109 changes: 109 additions & 0 deletions pkg/ccl/cliccl/load_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
// Copyright 2021 The Cockroach Authors.
//
// Licensed as a CockroachDB Enterprise file under the Cockroach Community
// License (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt

package cliccl

import (
"context"
"fmt"
"strings"
"testing"

"github.com/cockroachdb/cockroach/pkg/base"
"github.com/cockroachdb/cockroach/pkg/cli"
"github.com/cockroachdb/cockroach/pkg/testutils"
"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
"github.com/cockroachdb/cockroach/pkg/util/leaktest"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/stretchr/testify/require"
)

func TestLoadShow(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)

c := cli.NewCLITest(cli.TestCLIParams{T: t, NoServer: true})
defer c.Cleanup()

ctx := context.Background()
dir, cleanFn := testutils.TempDir(t)
defer cleanFn()
srv, db, _ := serverutils.StartServer(t, base.TestServerArgs{ExternalIODir: dir, Insecure: true})
defer srv.Stopper().Stop(ctx)

sqlDB := sqlutils.MakeSQLRunner(db)
sqlDB.Exec(t, `CREATE DATABASE testDB`)
sqlDB.Exec(t, `CREATE SCHEMA testDB.testschema`)
sqlDB.Exec(t, `CREATE TABLE testDB.testschema.fooTable (a INT)`)
sqlDB.Exec(t, `INSERT INTO testDB.testschema.fooTable VALUES (123)`)
const backupPath = "nodelocal://0/fooFolder"
sqlDB.Exec(t, `BACKUP testDB.testSchema.fooTable TO $1`, backupPath)

// Test load show with metadata option
expectedMetadataOutputSubstr := []string{"StartTime:", "EndTime:", "DataSize: 20 (20 B)", "Rows: 1", "IndexEntries: 0", "FormatVersion: 1", "ClusterID:", "NodeID: 0", "BuildInfo:"}
t.Run("show-metadata", func(t *testing.T) {
out, err := c.RunWithCapture(fmt.Sprintf("load show %s metadata --external-io-dir=%s", backupPath, dir))
require.NoError(t, err)
for _, substr := range expectedMetadataOutputSubstr {
require.True(t, strings.Contains(out, substr))
}
})

// Test load show with spans option
expectedSpansOutput := "/Table/54/{1-2}\n"
t.Run("show-spans", func(t *testing.T) {
out, err := c.RunWithCapture(fmt.Sprintf("load show %s spans --external-io-dir=%s", backupPath, dir))
require.NoError(t, err)
checkExpectedOutput(t, expectedSpansOutput, out)
})

// Test load show with files option
expectedFilesOutputSubstr := []string{".sst", "Span: /Table/54/{1-2}", "Sha512:", "DataSize: 20 (20 B)", "Rows: 1", "IndexEntries: 0"}
t.Run("show-files", func(t *testing.T) {
out, err := c.RunWithCapture(fmt.Sprintf("load show %s files --external-io-dir=%s", backupPath, dir))
require.NoError(t, err)
for _, substr := range expectedFilesOutputSubstr {
require.Contains(t, out, substr)
}
})

// Test load show with descriptors option
expectedDescOutput :=
`Databases:
testdb
Schemas:
public
testschema
Tables:
testdb.testschema.footable
`
t.Run("show-descriptors", func(t *testing.T) {
out, err := c.RunWithCapture(fmt.Sprintf("load show %s descriptors --external-io-dir=%s", backupPath, dir))
require.NoError(t, err)
checkExpectedOutput(t, expectedDescOutput, out)
})

// Test load show without options should output all information
t.Run("show-without-options", func(t *testing.T) {
out, err := c.RunWithCapture(fmt.Sprintf("load show %s --external-io-dir=%s", backupPath, dir))
require.NoError(t, err)
expectedOutputSubstr := append(expectedMetadataOutputSubstr, "Spans:\n\t"+expectedSpansOutput)
expectedOutputSubstr = append(expectedOutputSubstr, "Files:\n\t")
expectedOutputSubstr = append(expectedOutputSubstr, expectedFilesOutputSubstr...)
for _, substr := range expectedOutputSubstr {
require.Contains(t, out, substr)
}
})
}

func checkExpectedOutput(t *testing.T, expected string, out string) {
endOfCmd := strings.Index(out, "\n")
out = out[endOfCmd+1:]
require.Equal(t, expected, out)
}
2 changes: 2 additions & 0 deletions pkg/cli/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ go_library(
"//pkg/roachpb",
"//pkg/rpc",
"//pkg/security",
"//pkg/security/securitytest",
"//pkg/server",
"//pkg/server/debug",
"//pkg/server/dumpstore",
Expand Down Expand Up @@ -140,6 +141,7 @@ go_library(
"//pkg/storage/cloud",
"//pkg/storage/cloudimpl",
"//pkg/storage/enginepb",
"//pkg/testutils/serverutils",
"//pkg/ts/tspb",
"//pkg/util",
"//pkg/util/cgroups",
Expand Down
Loading

0 comments on commit 5c95fcf

Please sign in to comment.