Merge branch 'master' into make-explicit-implicit-dir-test-independent

internal/auth/auth.go

@@ -24,6 +24,24 @@ import (
 	storagev1 "google.golang.org/api/storage/v1"
 )
 
+const universeDomainDefault = "googleapis.com"
+
+func getUniverseDomain(ctx context.Context, contents []byte, scope string) (string, error) {
+	creds, err := google.CredentialsFromJSON(ctx, contents, scope)
+	if err != nil {
+		err = fmt.Errorf("CredentialsFromJSON(): %w", err)
+		return "", err
+	}
+
+	domain, err := creds.GetUniverseDomain()
+	if err != nil {
+		err = fmt.Errorf("GetUniverseDomain(): %w", err)
+		return "", err
+	}
+
+	return domain, nil
+}
+
 // Create token source from the JSON file at the supplide path.
 func newTokenSourceFromPath(
 	ctx context.Context,
@@ -37,21 +55,38 @@ func newTokenSourceFromPath(
 		return
 	}
 
+	// By default, a standard OAuth 2.0 token source is created
 	// Create a config struct based on its contents.
 	jwtConfig, err := google.JWTConfigFromJSON(contents, scope)
 	if err != nil {
 		err = fmt.Errorf("JWTConfigFromJSON: %w", err)
-		return
 	}
-
 	// Create the token source.
 	ts = jwtConfig.TokenSource(ctx)
 
+	domain, err := getUniverseDomain(ctx, contents, scope)
+	if err != nil {
+		return
+	}
+
+	// For non-GDU universe domains, token exchange is impossible and services
+	// must support self-signed JWTs with scopes.
+	// Override the token source to use self-signed JWT.
+	if domain != universeDomainDefault {
+		// Create self signed JWT access token.
+		ts, err = google.JWTAccessTokenSourceWithScope(contents, scope)
+		if err != nil {
+			err = fmt.Errorf("JWTAccessTokenSourceWithScope: %w", err)
+			return
+		}
+	}
 	return
 }
 
-// GetTokenSource returns a TokenSource for GCS API given a key file, or
-// with the default credentials.
+// GetTokenSource generates the token-source for GCS endpoint by following oauth2.0 authentication
+// for key-file and default-credential flow.
+// It also supports generating the self-signed JWT tokenSource for key-file authentication which can be
+// used by custom-endpoint(e.g. TPC).
 func GetTokenSource(
 	ctx context.Context,
 	keyFile string,

internal/auth/auth_test.go

@@ -0,0 +1,76 @@
+// Copyright 2024 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	. "github.com/jacobsa/ogletest"
+	storagev1 "google.golang.org/api/storage/v1"
+)
+
+const tpcUniverseDomain = "apis-tpclp.goog"
+
+func TestAuth(t *testing.T) { RunTests(t) }
+
+////////////////////////////////////////////////////////////////////////
+// Boilerplate
+////////////////////////////////////////////////////////////////////////
+
+type AuthTest struct {
+}
+
+func init() {
+	RegisterTestSuite(&AuthTest{})
+}
+
+func (t *AuthTest) SetUp(ti *TestInfo) {
+}
+
+////////////////////////////////////////////////////////////////////////
+// Tests for AuthTest
+////////////////////////////////////////////////////////////////////////
+
+func (t *AuthTest) TestGetUniverseDomainForGoogle() {
+	contents, err := os.ReadFile("testdata/google_creds.json")
+	AssertEq(nil, err)
+
+	domain, err := getUniverseDomain(context.Background(), contents, storagev1.DevstorageFullControlScope)
+
+	ExpectEq(nil, err)
+	ExpectEq(universeDomainDefault, domain)
+}
+
+func (t *AuthTest) TestGetUniverseDomainForTPC() {
+	contents, err := os.ReadFile("testdata/tpc_creds.json")
+	AssertEq(nil, err)
+
+	domain, err := getUniverseDomain(context.Background(), contents, storagev1.DevstorageFullControlScope)
+
+	ExpectEq(nil, err)
+	ExpectEq(tpcUniverseDomain, domain)
+}
+
+func (t *AuthTest) TestGetUniverseDomainForEmptyCreds() {
+	contents, err := os.ReadFile("testdata/empty_creds.json")
+	AssertEq(nil, err)
+
+	_, err = getUniverseDomain(context.Background(), contents, storagev1.DevstorageFullControlScope)
+
+	ExpectNe(nil, err)
+	ExpectEq("CredentialsFromJSON(): unexpected end of JSON input", err.Error())
+}

internal/auth/testdata/google_creds.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "project_id",
+  "private_key_id": "private_key_id",
+  "private_key": "private_key",
+  "client_email": "client_email",
+  "client_id": "client_id",
+  "auth_uri": "auth_uri",
+  "token_uri": "token_uri",
+  "auth_provider_x509_cert_url": "auth_provider_x509_cert_url",
+  "client_x509_cert_url": "client_x509_cert_url",
+  "universe_domain": "googleapis.com"
+}

internal/auth/testdata/tpc_creds.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "project_id",
+  "private_key_id": "private_key_id",
+  "private_key": "private_key",
+  "client_email": "client_email",
+  "client_id": "client_id",
+  "auth_uri": "auth_uri",
+  "token_uri": "token_uri",
+  "auth_provider_x509_cert_url": "auth_provider_x509_cert_url",
+  "client_x509_cert_url": "client_x509_cert_url",
+  "universe_domain": "apis-tpclp.goog"
+}

perfmetrics/scripts/run_e2e_tests.sh

@@ -31,6 +31,8 @@ TEST_DIR_PARALLEL=(
   "list_large_dir"
   "explicit_dir"
   "implicit_dir"
+  "rename_dir_limit"
+  "read_large_files"
 )
 # These tests never become parallel as it is changing bucket permissions.
 TEST_DIR_NON_PARALLEL_GROUP_1=(
@@ -43,8 +45,6 @@ TEST_DIR_NON_PARALLEL_GROUP_1=(
 # directory independence.
 TEST_DIR_NON_PARALLEL_GROUP_2=(
   "operations"
-  "read_large_files"
-  "rename_dir_limit"
 )
 
 TEST_DIR_HNS_GROUP=(

tools/integration_tests/managed_folders/admin_permissions_test.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"path"
 	"testing"
+	"time"
 
 	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/creds_tests"
 	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations"
@@ -50,23 +51,17 @@ var (
 // levels apply additively (union) throughout the resource hierarchy.
 // Hence here managed folder will have admin permission throughout all the tests.
 type managedFoldersAdminPermission struct {
-	managedFolderPermission string
 	bucketPermission        string
 }
 
 func (s *managedFoldersAdminPermission) Setup(t *testing.T) {
-	bucket, testDir = setup.GetBucketAndObjectBasedOnTypeOfMount(TestDirForManagedFolderTest)
 	createDirectoryStructureForNonEmptyManagedFolders(t)
-	if s.managedFolderPermission != "nil" {
-		providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, s.managedFolderPermission, t)
-		providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, s.managedFolderPermission, t)
-	}
 }
 
 func (s *managedFoldersAdminPermission) Teardown(t *testing.T) {
-	revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, s.managedFolderPermission, t)
-	revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, s.managedFolderPermission, t)
-	cleanup(bucket, testDir, serviceAccount, s.managedFolderPermission, t)
+	// The 'gsutil rm -rf' command doesn't work on managed folders.
+	// We'll clean up the test directory but leave managed folders.
+	setup.CleanupDirectoryOnGCS(path.Join(bucket, testDir))
 }
 
 func (s *managedFoldersAdminPermission) TestCreateObjectInManagedFolder(t *testing.T) {
@@ -207,13 +202,27 @@ func TestManagedFolders_FolderAdminPermission(t *testing.T) {
 
 	for i := 0; i < len(permissions); i++ {
 		log.Printf("Running tests with flags, bucket have %s permission and managed folder have %s permissions: %s", permissions[i][0], permissions[i][1], flags)
+		bucket, testDir = setup.GetBucketAndObjectBasedOnTypeOfMount(TestDirForManagedFolderTest)
 		ts.bucketPermission = permissions[i][0]
 		if ts.bucketPermission == ViewPermission {
 			creds_tests.RevokePermission(serviceAccount, AdminPermission, setup.TestBucket())
 			creds_tests.ApplyPermissionToServiceAccount(serviceAccount, ViewPermission)
 			defer creds_tests.RevokePermission(serviceAccount, ViewPermission, setup.TestBucket())
 		}
-		ts.managedFolderPermission = permissions[i][1]
+		managedFolderPermission := permissions[i][1]
+		if managedFolderPermission != "nil" {
+			providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, managedFolderPermission, t)
+			providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, managedFolderPermission, t)
+			// Waiting for 10 seconds as it usually takes 10 seconds for policy changes to propagate.
+			time.Sleep(10 * time.Second)
+		}
+
 		test_setup.RunTests(t, ts)
+		revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, managedFolderPermission, t)
+		revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, managedFolderPermission, t)
 	}
+	t.Cleanup(func() {
+		operations.DeleteManagedFoldersInBucket(path.Join(testDir, ManagedFolder1), setup.TestBucket())
+		operations.DeleteManagedFoldersInBucket(path.Join(testDir, ManagedFolder2), setup.TestBucket())
+	})
 }

tools/integration_tests/read_large_files/concurrent_read_files_test.go

@@ -51,8 +51,7 @@ func readFile(fileInLocalDisk string, fileInMntDir string) error {
 }
 
 func TestReadFilesConcurrently(t *testing.T) {
-	// Clean the mountedDirectory before running test.
-	setup.CleanMntDir()
+	testDir := setup.SetupTestDirectory(DirForReadLargeFilesTests)
 
 	filesInLocalDisk := [NumberOfFilesInLocalDiskForConcurrentRead]string{FileOne, FileTwo, FileThree}
 	var filesPathInLocalDisk []string
@@ -62,7 +61,7 @@ func TestReadFilesConcurrently(t *testing.T) {
 		fileInLocalDisk := path.Join(os.Getenv("HOME"), filesInLocalDisk[i])
 		filesPathInLocalDisk = append(filesPathInLocalDisk, fileInLocalDisk)
 
-		file := path.Join(setup.MntDir(), filesInLocalDisk[i])
+		file := path.Join(testDir, filesInLocalDisk[i])
 		filesPathInMntDir = append(filesPathInMntDir, file)
 
 		createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t)

tools/integration_tests/read_large_files/random_read_large_file_test.go

@@ -26,11 +26,9 @@ import (
 )
 
 func TestReadLargeFileRandomly(t *testing.T) {
-	// Clean the mountedDirectory before running test.
-	setup.CleanMntDir()
-
+	testDir := setup.SetupTestDirectory(DirForReadLargeFilesTests)
 	fileInLocalDisk := path.Join(os.Getenv("HOME"), FiveHundredMBFile)
-	file := path.Join(setup.MntDir(), FiveHundredMBFile)
+	file := path.Join(testDir, FiveHundredMBFile)
 	// Create and copy the local file in mountedDirectory.
 	createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t)
 

tools/integration_tests/read_large_files/read_large_files_test.go

@@ -35,6 +35,7 @@ const ChunkSize = 200 * OneMB
 const NumberOfRandomReadCalls = 200
 const MinReadableByteFromFile = 0
 const MaxReadableByteFromFile = 500 * OneMB
+const DirForReadLargeFilesTests = "dirForReadLargeFilesTests"
 
 func createMountConfigsAndEquivalentFlags() (flags [][]string) {
 	cacheDirPath := path.Join(os.Getenv("HOME"), "cache-dri")

tools/integration_tests/read_large_files/seq_read_large_file_test.go

@@ -25,12 +25,10 @@ import (
 )
 
 func TestReadLargeFileSequentially(t *testing.T) {
-	// Clean the mountedDirectory before running test.
-	setup.CleanMntDir()
-
+	testDir := setup.SetupTestDirectory(DirForReadLargeFilesTests)
 	// Create file of 500 MB with random data in local disk and copy it in mntDir.
 	fileInLocalDisk := path.Join(os.Getenv("HOME"), FiveHundredMBFile)
-	file := path.Join(setup.MntDir(), FiveHundredMBFile)
+	file := path.Join(testDir, FiveHundredMBFile)
 	createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t)
 
 	// Sequentially read the data from file.