cloudimpl: add FileTableStorage as an ExternalStorage option

This change introduces a new FileTableStorage flavor of ExternalStorage.
FileTableStorage is backed by the UserFileTableSystem and allows a user
to write and read file blobs to/from user scoped SQL tables.

Unit tests were also added.

This is commit #2 of 2 to intgrate the UserFileTableStorage.

Release note: None

pkg/roachpb/api.proto

@@ -1263,6 +1263,7 @@ enum ExternalStorageProvider {
  GoogleCloud = 4;
  Azure = 5;
  Workload = 6;
+ FileTable = 7;
 }
 
 message ExternalStorage {
@@ -1327,12 +1328,27 @@ message ExternalStorage {
  int64 batch_begin = 6;
  int64 batch_end = 7;
  }
+ message FileTable {
+ option (gogoproto.equal) = true;
+
+ // User interacting with the external storage. This is used to check access
+ // privileges of the requested user scoped tables.
+ string user = 1;
+
+ // QualifiedTableName specifies the database.schema.tablename which the
+ // FileTableSystem should interact with when servicing reads/writes.
+ string qualified_table_name = 2;
+
+ // Path is the filename being read/written to via the FileTableSystem.
+ string path = 3;
+ }
  LocalFilePath LocalFile = 2 [(gogoproto.nullable) = false];
  Http HttpPath = 3 [(gogoproto.nullable) = false];
  GCS GoogleCloudConfig = 4;
  S3 S3Config = 5;
  Azure AzureConfig = 6;
  Workload WorkloadConfig = 7;
+ FileTable FileTableConfig = 8 [(gogoproto.nullable) = false];
 }
 
 // WriteBatchRequest is arguments to the WriteBatch() method, to apply the

pkg/storage/cloudimpl/tests/azure_storage_test.go → pkg/storage/cloudimpl/cloudimpltests/azure_storage_test.go

@@ -8,7 +8,7 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-package tests
+package cloudimpltests
 
 import (
  "fmt"

pkg/storage/cloudimpl/tests/external_storage_test.go → pkg/storage/cloudimpl/cloudimpltests/external_storage_test.go

@@ -8,7 +8,7 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-package tests
+package cloudimpltests
 
 import (
  "bytes"

pkg/storage/cloudimpl/cloudimpltests/file_table_storage_test.go

@@ -0,0 +1,104 @@
+// Copyright 2020 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+package cloudimpltests
+
+import (
+ "bytes"
+ "context"
+ gosql "database/sql"
+ "fmt"
+ "testing"
+
+ "github.com/cockroachdb/cockroach/pkg/base"
+ "github.com/cockroachdb/cockroach/pkg/blobs"
+ "github.com/cockroachdb/cockroach/pkg/security"
+ "github.com/cockroachdb/cockroach/pkg/settings/cluster"
+ "github.com/cockroachdb/cockroach/pkg/sql"
+ "github.com/cockroachdb/cockroach/pkg/sql/tests"
+ "github.com/cockroachdb/cockroach/pkg/storage/cloudimpl"
+ "github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+ "github.com/cockroachdb/cockroach/pkg/util/leaktest"
+ "github.com/stretchr/testify/require"
+)
+
+func TestPutUserFileTable(t *testing.T) {
+ defer leaktest.AfterTest(t)()
+
+ qualifiedTableName := "defaultdb.public.user_file_table_test"
+ filename := "path/to/file"
+
+ ctx := context.Background()
+ params, _ := tests.CreateTestServerParams()
+ s, _, kvDB := serverutils.StartServer(t, params)
+ defer s.Stopper().Stop(ctx)
+
+ dest := cloudimpl.MakeUserFileStorageURI(qualifiedTableName, filename)
+
+ ie := s.InternalExecutor().(*sql.InternalExecutor)
+ testExportStore(t, dest, false, security.RootUser, ie, kvDB)
+
+ testListFiles(t, "userfile://defaultdb.public.file_list_table/listing-test/basepath",
+ security.RootUser, ie, kvDB)
+}
+
+func createUserGrantAllPrivieleges(username, database string, sqlDB *gosql.DB) error {
+ _, err := sqlDB.Exec(fmt.Sprintf("CREATE USER %s", username))
+ if err != nil {
+ return err
+ }
+ _, err = sqlDB.Exec(fmt.Sprintf("GRANT ALL ON DATABASE %s TO %s", database, username))
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func TestUserScoping(t *testing.T) {
+ defer leaktest.AfterTest(t)()
+
+ qualifiedTableName := "defaultdb.public.user_file_table_test"
+ filename := "path/to/file"
+
+ ctx := context.Background()
+ params, _ := tests.CreateTestServerParams()
+ s, sqlDB, kvDB := serverutils.StartServer(t, params)
+ defer s.Stopper().Stop(ctx)
+
+ dest := cloudimpl.MakeUserFileStorageURI(qualifiedTableName, "")
+ ie := s.InternalExecutor().(*sql.InternalExecutor)
+
+ // Create two users and grant them all privileges on defaultdb.
+ user1 := "foo"
+ require.NoError(t, createUserGrantAllPrivieleges(user1, "defaultdb", sqlDB))
+ user2 := "bar"
+ require.NoError(t, createUserGrantAllPrivieleges(user2, "defaultdb", sqlDB))
+
+ // Write file as user1.
+ fileTableSystem1, err := cloudimpl.ExternalStorageFromURI(ctx, dest, base.ExternalIODirConfig{},
+ cluster.NoSettings, blobs.TestEmptyBlobClientFactory, user1, ie, kvDB)
+ require.NoError(t, err)
+ require.NoError(t, fileTableSystem1.WriteFile(ctx, filename, bytes.NewReader([]byte("aaa"))))
+
+ // Attempt to read/write file as user2 and expect to fail.
+ fileTableSystem2, err := cloudimpl.ExternalStorageFromURI(ctx, dest, base.ExternalIODirConfig{},
+ cluster.NoSettings, blobs.TestEmptyBlobClientFactory, user2, ie, kvDB)
+ require.NoError(t, err)
+ _, err = fileTableSystem2.ReadFile(ctx, filename)
+ require.Error(t, err)
+ require.Error(t, fileTableSystem2.WriteFile(ctx, filename, bytes.NewReader([]byte("aaa"))))
+
+ // Read file as root and expect to succeed.
+ fileTableSystem3, err := cloudimpl.ExternalStorageFromURI(ctx, dest, base.ExternalIODirConfig{},
+ cluster.NoSettings, blobs.TestEmptyBlobClientFactory, security.RootUser, ie, kvDB)
+ require.NoError(t, err)
+ _, err = fileTableSystem3.ReadFile(ctx, filename)
+ require.NoError(t, err)
+}

pkg/storage/cloudimpl/tests/gcs_storage_test.go → pkg/storage/cloudimpl/cloudimpltests/gcs_storage_test.go

@@ -8,7 +8,7 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-package tests
+package cloudimpltests
 
 import (
  "context"

pkg/storage/cloudimpl/tests/http_storage_test.go → pkg/storage/cloudimpl/cloudimpltests/http_storage_test.go

@@ -8,7 +8,7 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-package tests
+package cloudimpltests
 
 import (
  "bytes"

pkg/storage/cloudimpl/tests/main_test.go → pkg/storage/cloudimpl/cloudimpltests/main_test.go

@@ -8,7 +8,7 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-package tests
+package cloudimpltests
 
 import (
  "os"

pkg/storage/cloudimpl/tests/nodelocal_storage_test.go → pkg/storage/cloudimpl/cloudimpltests/nodelocal_storage_test.go

@@ -8,7 +8,7 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-package tests
+package cloudimpltests
 
 import (
  "context"

pkg/storage/cloudimpl/tests/s3_storage_test.go → pkg/storage/cloudimpl/cloudimpltests/s3_storage_test.go

@@ -8,7 +8,7 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-package tests
+package cloudimpltests
 
 import (
  "context"

pkg/storage/cloudimpl/external_storage.go

@@ -14,6 +14,7 @@ import (
  "context"
  "io"
  "net/url"
+ "path"
  "strconv"
  "strings"
  "time"
@@ -28,7 +29,6 @@ import (
  "github.com/cockroachdb/cockroach/pkg/settings/cluster"
  "github.com/cockroachdb/cockroach/pkg/sql"
  "github.com/cockroachdb/cockroach/pkg/storage/cloud"
- "github.com/cockroachdb/cockroach/pkg/util/log"
  "github.com/cockroachdb/cockroach/pkg/util/retry"
  "github.com/cockroachdb/cockroach/pkg/util/sysutil"
  "github.com/cockroachdb/errors"
@@ -184,13 +184,26 @@ func ExternalStorageConfFromURI(path, user string) (roachpb.ExternalStorage, err
  }
  conf.Provider = roachpb.ExternalStorageProvider_LocalFile
  conf.LocalFile.Path = uri.Path
- log.Infof(context.Background(), "THIS IS THE PATH %s", conf.LocalFile.Path)
  conf.LocalFile.NodeID = roachpb.NodeID(nodeID)
  case "experimental-workload", "workload":
  conf.Provider = roachpb.ExternalStorageProvider_Workload
  if conf.WorkloadConfig, err = ParseWorkloadConfig(uri); err != nil {
  return conf, err
  }
+ case "userfile":
+ qualifiedTableName := uri.Host
+ if qualifiedTableName == "" {
+ return conf, errors.Errorf("host component of userfile URI must be a qualified table name")
+ }
+
+ if user == "" {
+ return conf, errors.Errorf("user creating the FileTable ExternalStorage must be specified")
+ }
+
+ conf.Provider = roachpb.ExternalStorageProvider_FileTable
+ conf.FileTableConfig.User = user
+ conf.FileTableConfig.QualifiedTableName = qualifiedTableName
+ conf.FileTableConfig.Path = uri.Path
  default:
  return conf, errors.Errorf("unsupported storage scheme: %q", uri.Scheme)
  }
@@ -281,6 +294,9 @@ func MakeExternalStorage(
  case roachpb.ExternalStorageProvider_Workload:
  telemetry.Count("external-io.workload")
  return makeWorkloadStorage(dest.WorkloadConfig)
+ case roachpb.ExternalStorageProvider_FileTable:
+ telemetry.Count("external-io.filetable")
+ return makeFileTableStorage(ctx, dest.FileTableConfig, ie, kvDB)
  }
  return nil, errors.Errorf("unsupported external destination type: %s", dest.Provider.String())
 }
@@ -381,6 +397,14 @@ func isResumableHTTPError(err error) bool {
  sysutil.IsErrConnectionRefused(err)
 }
 
+func getPrefixBeforeWildcard(p string) string {
+ globIndex := strings.IndexAny(p, "*?[")
+ if globIndex < 0 {
+ return p
+ }
+ return path.Dir(p[:globIndex])
+}
+
 // MaxDelayedRetryAttempts is the number of times the delayedRetry method will
 // re-run the provided function.
 const MaxDelayedRetryAttempts = 3