Skip to content

Commit

Permalink
cliccl: update load show descriptors to display user defined schemas
Browse files Browse the repository at this point in the history
Previously, `load show descriptors` display tables and databases of a backup.
And output the create statement of each tables which may not be useful.

This commit updates `load show descriptors` to display user defined schemas
and removes the output of create statement after tables to keep output conceise.

Release justification: non-production code changes.
Release note: none
  • Loading branch information
Elliebababa committed Mar 8, 2021
1 parent c9ea249 commit b6f00f3
Show file tree
Hide file tree
Showing 4 changed files with 153 additions and 68 deletions.
18 changes: 14 additions & 4 deletions pkg/ccl/cliccl/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,14 @@ go_library(
"//pkg/ccl/workloadccl/cliccl",
"//pkg/cli",
"//pkg/cli/cliflags",
"//pkg/keys",
"//pkg/roachpb",
"//pkg/security",
"//pkg/server",
"//pkg/settings/cluster",
"//pkg/sql",
"//pkg/sql/catalog",
"//pkg/sql/catalog/descpb",
"//pkg/sql/catalog/tabledesc",
"//pkg/sql/sessiondatapb",
"//pkg/sql/sessiondata",
"//pkg/storage/cloud",
"//pkg/storage/cloudimpl",
"//pkg/storage/enginepb",
Expand All @@ -56,11 +55,22 @@ go_library(
go_test(
name = "cliccl_test",
size = "small",
srcs = ["main_test.go"],
srcs = [
"load_test.go",
"main_test.go",
],
embed = [":cliccl"],
deps = [
"//pkg/base",
"//pkg/build",
"//pkg/ccl/utilccl",
"//pkg/cli",
"//pkg/server",
"//pkg/testutils",
"//pkg/testutils/serverutils",
"//pkg/testutils/sqlutils",
"//pkg/util/leaktest",
"//pkg/util/log",
"@com_github_stretchr_testify//require",
],
)
93 changes: 30 additions & 63 deletions pkg/ccl/cliccl/load.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ package cliccl
import (
"context"
"fmt"
"net/url"
"path/filepath"
"strings"
"time"
Expand All @@ -21,21 +20,19 @@ import (
"github.com/cockroachdb/cockroach/pkg/ccl/backupccl"
"github.com/cockroachdb/cockroach/pkg/cli"
"github.com/cockroachdb/cockroach/pkg/cli/cliflags"
"github.com/cockroachdb/cockroach/pkg/keys"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/security"
"github.com/cockroachdb/cockroach/pkg/server"
"github.com/cockroachdb/cockroach/pkg/settings/cluster"
"github.com/cockroachdb/cockroach/pkg/sql"
"github.com/cockroachdb/cockroach/pkg/sql/catalog"
"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
"github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc"
"github.com/cockroachdb/cockroach/pkg/sql/sessiondatapb"
"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
"github.com/cockroachdb/cockroach/pkg/storage/cloud"
"github.com/cockroachdb/cockroach/pkg/storage/cloudimpl"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
"github.com/cockroachdb/cockroach/pkg/util/humanizeutil"
"github.com/cockroachdb/cockroach/pkg/util/timeutil"
"github.com/cockroachdb/cockroach/pkg/util/uuid"
"github.com/cockroachdb/errors"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -67,8 +64,8 @@ func init() {
},
}

f := loadCmds.Flags()
f.StringVarP(
loadFlags := loadCmds.Flags()
loadFlags.StringVarP(
&externalIODir,
cliflags.ExternalIODir.Name,
cliflags.ExternalIODir.Shorthand,
Expand All @@ -77,6 +74,7 @@ func init() {

cli.AddCmd(loadCmds)
loadCmds.AddCommand(loadShowCmd)
loadShowCmd.Flags().AddFlagSet(loadFlags)
}

func newBlobFactory(ctx context.Context, dialing roachpb.NodeID) (blobs.BlobClient, error) {
Expand Down Expand Up @@ -170,8 +168,7 @@ func runLoadShow(cmd *cobra.Command, args []string) error {
}

if _, ok := options[descriptors]; ok {
err = showDescriptor(desc)
if err != nil {
if err := showDescriptor(desc); err != nil {
return err
}
}
Expand Down Expand Up @@ -212,30 +209,45 @@ func showFiles(desc backupccl.BackupManifest, showHeaders bool) {
}
for _, f := range desc.Files {
fmt.Printf("%s%s:\n", tabfmt, f.Path)
fmt.Printf(" Span: %s\n", f.Span)
fmt.Printf(" Sha512: %0128x\n", f.Sha512)
fmt.Printf(" DataSize: %d (%s)\n", f.EntryCounts.DataSize, humanizeutil.IBytes(f.EntryCounts.DataSize))
fmt.Printf(" Rows: %d\n", f.EntryCounts.Rows)
fmt.Printf(" IndexEntries: %d\n", f.EntryCounts.IndexEntries)
fmt.Printf("%s Span: %s\n", tabfmt, f.Span)
fmt.Printf("%s Sha512: %0128x\n", tabfmt, f.Sha512)
fmt.Printf("%s DataSize: %d (%s)\n", tabfmt, f.EntryCounts.DataSize, humanizeutil.IBytes(f.EntryCounts.DataSize))
fmt.Printf("%s Rows: %d\n", tabfmt, f.EntryCounts.Rows)
fmt.Printf("%s IndexEntries: %d\n", tabfmt, f.EntryCounts.IndexEntries)
}
}

func showDescriptor(desc backupccl.BackupManifest) error {
// Note that these descriptors could be from any past version of the cluster,
// in case more fields need to be added to the output.
dbIDs := make([]descpb.ID, 0)
dbIDToName := make(map[descpb.ID]string)
schemaIDs := make([]descpb.ID, 0)
schemaIDs = append(schemaIDs, keys.PublicSchemaID)
schemaIDToName := make(map[descpb.ID]string)
schemaIDToName[keys.PublicSchemaID] = sessiondata.PublicSchemaName
for i := range desc.Descriptors {
d := &desc.Descriptors[i]
id := descpb.GetDescriptorID(d)
if d.GetDatabase() != nil {
dbIDToName[id] = descpb.GetDescriptorName(d)
dbIDs = append(dbIDs, id)
} else if d.GetSchema() != nil {
schemaIDToName[id] = descpb.GetDescriptorName(d)
schemaIDs = append(schemaIDs, id)
}
}

fmt.Printf("Databases:\n")
for i := range dbIDToName {
for _, id := range dbIDs {
fmt.Printf(" %s\n",
dbIDToName[i])
dbIDToName[id])
}

fmt.Printf("Schemas:\n")
for _, id := range schemaIDs {
fmt.Printf(" %s\n",
schemaIDToName[id])
}

fmt.Printf("Tables:\n")
Expand All @@ -244,54 +256,9 @@ func showDescriptor(desc backupccl.BackupManifest) error {
if descpb.TableFromDescriptor(d, hlc.Timestamp{}) != nil {
tbDesc := tabledesc.NewImmutable(*descpb.TableFromDescriptor(d, hlc.Timestamp{}))
dbName := dbIDToName[tbDesc.GetParentID()]
fmt.Printf(" %s (%s) \n", descpb.GetDescriptorName(d), dbName)
err := showCreate(tbDesc, desc.Descriptors, dbName)
if err != nil {
return err
}
schemaName := schemaIDToName[tbDesc.GetParentSchemaID()]
fmt.Printf(" %s.%s.%s\n", dbName, schemaName, descpb.GetDescriptorName(d))
}
}
return nil
}

func showCreate(desc catalog.TableDescriptor, allDescs []descpb.Descriptor, dbName string) error {

fakeNodeInfo := sql.NodeInfo{
AdminURL: func() *url.URL { return nil },
PGURL: func(*url.Userinfo) (*url.URL, error) { return nil, nil },
ClusterID: func() uuid.UUID { return uuid.UUID{} },
NodeID: nil,
}

p, _ := sql.NewInternalPlanner(
"showCreateInCli",
nil, /*kv.Txn*/
security.RootUserName(),
&sql.MemoryMetrics{},
&sql.ExecutorConfig{
RPCContext: nil,
NodeInfo: fakeNodeInfo,
},
sessiondatapb.SessionData{},
)

displayOptions := sql.ShowCreateDisplayOptions{
FKDisplayMode: sql.OmitMissingFKClausesFromCreate,
IgnoreComments: true,
}

createStmt, err := p.(sql.PlanHookState).ShowCreate(
context.Background(),
dbName,
allDescs,
desc,
displayOptions)
if err != nil {
return err
}

createStmt = "\t\t" + createStmt
createStmt = strings.ReplaceAll(createStmt, "\n", "\n\t\t")
fmt.Println(createStmt)
return nil
}
109 changes: 109 additions & 0 deletions pkg/ccl/cliccl/load_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
// Copyright 2021 The Cockroach Authors.
//
// Licensed as a CockroachDB Enterprise file under the Cockroach Community
// License (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt

package cliccl

import (
"context"
"fmt"
"strings"
"testing"

"github.com/cockroachdb/cockroach/pkg/base"
"github.com/cockroachdb/cockroach/pkg/cli"
"github.com/cockroachdb/cockroach/pkg/testutils"
"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
"github.com/cockroachdb/cockroach/pkg/util/leaktest"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/stretchr/testify/require"
)

func TestLoadShow(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)

c := cli.NewCliTest(cli.TestCliParams{T: t, NoServer: true})
defer c.Cleanup()

ctx := context.Background()
dir, cleanFn := testutils.TempDir(t)
defer cleanFn()
srv, db, _ := serverutils.StartServer(t, base.TestServerArgs{ExternalIODir: dir, Insecure: true})
defer srv.Stopper().Stop(ctx)

sqlDB := sqlutils.MakeSQLRunner(db)
sqlDB.Exec(t, `CREATE DATABASE testDB`)
sqlDB.Exec(t, `CREATE SCHEMA testDB.testschema`)
sqlDB.Exec(t, `CREATE TABLE testDB.testschema.fooTable (a INT)`)
sqlDB.Exec(t, `INSERT INTO testDB.testschema.fooTable VALUES (123)`)
const backupPath = "nodelocal://0/fooFolder"
sqlDB.Exec(t, `BACKUP testDB.testSchema.fooTable TO $1`, backupPath)

// Test load show with metadata option
expectedMetadataOutputSubstr := []string{"StartTime:", "EndTime:", "DataSize: 20 (20 B)", "Rows: 1", "IndexEntries: 0", "FormatVersion: 1", "ClusterID:", "NodeID: 0", "BuildInfo:"}
t.Run("show-metadata", func(t *testing.T) {
out, err := c.RunWithCapture(fmt.Sprintf("load show %s metadata --external-io-dir=%s", backupPath, dir))
require.NoError(t, err)
for _, substr := range expectedMetadataOutputSubstr {
require.True(t, strings.Contains(out, substr))
}
})

// Test load show with spans option
expectedSpansOutput := "/Table/54/{1-2}\n"
t.Run("show-spans", func(t *testing.T) {
out, err := c.RunWithCapture(fmt.Sprintf("load show %s spans --external-io-dir=%s", backupPath, dir))
require.NoError(t, err)
checkExpectedOutput(t, expectedSpansOutput, out)
})

// Test load show with files option
expectedFilesOutputSubstr := []string{".sst", "Span: /Table/54/{1-2}", "Sha512:", "DataSize: 20 (20 B)", "Rows: 1", "IndexEntries: 0"}
t.Run("show-files", func(t *testing.T) {
out, err := c.RunWithCapture(fmt.Sprintf("load show %s files --external-io-dir=%s", backupPath, dir))
require.NoError(t, err)
for _, substr := range expectedFilesOutputSubstr {
require.Contains(t, out, substr)
}
})

// Test load show with descriptors option
expectedDescOutput :=
`Databases:
testdb
Schemas:
public
testschema
Tables:
testdb.testschema.footable
`
t.Run("show-descriptors", func(t *testing.T) {
out, err := c.RunWithCapture(fmt.Sprintf("load show %s descriptors --external-io-dir=%s", backupPath, dir))
require.NoError(t, err)
checkExpectedOutput(t, expectedDescOutput, out)
})

// Test load show without options should output all information
t.Run("show-without-options", func(t *testing.T) {
out, err := c.RunWithCapture(fmt.Sprintf("load show %s --external-io-dir=%s", backupPath, dir))
require.NoError(t, err)
expectedOutputSubstr := append(expectedMetadataOutputSubstr, "Spans:\n\t"+expectedSpansOutput)
expectedOutputSubstr = append(expectedOutputSubstr, "Files:\n\t")
expectedOutputSubstr = append(expectedOutputSubstr, expectedFilesOutputSubstr...)
for _, substr := range expectedOutputSubstr {
require.Contains(t, out, substr)
}
})
}

func checkExpectedOutput(t *testing.T, expected string, out string) {
endOfCmd := strings.Index(out, "\n")
out = out[endOfCmd+1:]
require.Equal(t, expected, out)
}
1 change: 0 additions & 1 deletion pkg/cli/testutils.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,6 @@ func createTestCerts(certsDir string) (cleanup func() error) {
filepath.Join(security.EmbeddedCertsDir, security.EmbeddedNodeKey),
filepath.Join(security.EmbeddedCertsDir, security.EmbeddedRootCert),
filepath.Join(security.EmbeddedCertsDir, security.EmbeddedRootKey),

filepath.Join(security.EmbeddedCertsDir, security.EmbeddedTenantClientCACert),
}

Expand Down

0 comments on commit b6f00f3

Please sign in to comment.