From 879b823fbfe95206b34d4f6502bb5853bff1c95f Mon Sep 17 00:00:00 2001 From: Tulsi Shah <46474643+Tulsishah@users.noreply.github.com> Date: Thu, 18 Apr 2024 12:12:33 +0530 Subject: [PATCH 1/4] Fix flaky e2e tests for managed folder (#1848) * adding check for permisisons * updating file path and for loop condition * adding log statement to check * testing with sleep * applying permisison out of setup * deletion after creation * lint fix * lint fix * lint fix * review comment * lint fix * adding comment * adding t.Cleanup * small fix * removing managedFolderPermission from struct * review comments --- .../managed_folders/admin_permissions_test.go | 29 ++++++++++++------- .../util/operations/dir_operations.go | 4 +-- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tools/integration_tests/managed_folders/admin_permissions_test.go b/tools/integration_tests/managed_folders/admin_permissions_test.go index ebd2e19fbb..8dce6c0e71 100644 --- a/tools/integration_tests/managed_folders/admin_permissions_test.go +++ b/tools/integration_tests/managed_folders/admin_permissions_test.go @@ -24,6 +24,7 @@ import ( "os" "path" "testing" + "time" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/creds_tests" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" @@ -50,23 +51,17 @@ var ( // levels apply additively (union) throughout the resource hierarchy. // Hence here managed folder will have admin permission throughout all the tests. type managedFoldersAdminPermission struct { - managedFolderPermission string bucketPermission string } func (s *managedFoldersAdminPermission) Setup(t *testing.T) { - bucket, testDir = setup.GetBucketAndObjectBasedOnTypeOfMount(TestDirForManagedFolderTest) createDirectoryStructureForNonEmptyManagedFolders(t) - if s.managedFolderPermission != "nil" { - providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, s.managedFolderPermission, t) - providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, s.managedFolderPermission, t) - } } func (s *managedFoldersAdminPermission) Teardown(t *testing.T) { - revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, s.managedFolderPermission, t) - revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, s.managedFolderPermission, t) - cleanup(bucket, testDir, serviceAccount, s.managedFolderPermission, t) + // The 'gsutil rm -rf' command doesn't work on managed folders. + // We'll clean up the test directory but leave managed folders. + setup.CleanupDirectoryOnGCS(path.Join(bucket, testDir)) } func (s *managedFoldersAdminPermission) TestCreateObjectInManagedFolder(t *testing.T) { @@ -207,13 +202,27 @@ func TestManagedFolders_FolderAdminPermission(t *testing.T) { for i := 0; i < len(permissions); i++ { log.Printf("Running tests with flags, bucket have %s permission and managed folder have %s permissions: %s", permissions[i][0], permissions[i][1], flags) + bucket, testDir = setup.GetBucketAndObjectBasedOnTypeOfMount(TestDirForManagedFolderTest) ts.bucketPermission = permissions[i][0] if ts.bucketPermission == ViewPermission { creds_tests.RevokePermission(serviceAccount, AdminPermission, setup.TestBucket()) creds_tests.ApplyPermissionToServiceAccount(serviceAccount, ViewPermission) defer creds_tests.RevokePermission(serviceAccount, ViewPermission, setup.TestBucket()) } - ts.managedFolderPermission = permissions[i][1] + managedFolderPermission := permissions[i][1] + if managedFolderPermission != "nil" { + providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, managedFolderPermission, t) + providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, managedFolderPermission, t) + // Waiting for 10 seconds as it usually takes 10 seconds for policy changes to propagate. + time.Sleep(10 * time.Second) + } + test_setup.RunTests(t, ts) + revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, managedFolderPermission, t) + revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, managedFolderPermission, t) } + t.Cleanup(func() { + operations.DeleteManagedFoldersInBucket(path.Join(testDir, ManagedFolder1), setup.TestBucket()) + operations.DeleteManagedFoldersInBucket(path.Join(testDir, ManagedFolder2), setup.TestBucket()) + }) } diff --git a/tools/integration_tests/util/operations/dir_operations.go b/tools/integration_tests/util/operations/dir_operations.go index 38fa6f92a7..ba2d26f02e 100644 --- a/tools/integration_tests/util/operations/dir_operations.go +++ b/tools/integration_tests/util/operations/dir_operations.go @@ -179,12 +179,10 @@ func DeleteManagedFoldersInBucket(managedFolderPath, bucket string) { } func CreateManagedFoldersInBucket(managedFolderPath, bucket string) { - // Delete if already exist. - DeleteManagedFoldersInBucket(managedFolderPath, bucket) gcloudCreateManagedFolderCmd := fmt.Sprintf("alpha storage managed-folders create gs://%s/%s", bucket, managedFolderPath) _, err := ExecuteGcloudCommandf(gcloudCreateManagedFolderCmd) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "The specified managed folder already exists") { log.Fatalf(fmt.Sprintf("Error while creating managed folder: %v", err)) } } From 7e33d03cb8c171069db738d3531842eccbf28b00 Mon Sep 17 00:00:00 2001 From: Tulsi Shah <46474643+Tulsishah@users.noreply.github.com> Date: Thu, 18 Apr 2024 14:07:09 +0530 Subject: [PATCH 2/4] Change authentication flow for TPC (#1840) * create token on bases of universe domain * adding unit tests * small fix in comment * lint fix * lint fix * lint fix * review comments * small fix * review comment * updating comment * updating comment --- internal/auth/auth.go | 43 ++++++++++++-- internal/auth/auth_test.go | 76 ++++++++++++++++++++++++ internal/auth/testdata/empty_creds.json | 0 internal/auth/testdata/google_creds.json | 13 ++++ internal/auth/testdata/tpc_creds.json | 13 ++++ 5 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 internal/auth/auth_test.go create mode 100644 internal/auth/testdata/empty_creds.json create mode 100644 internal/auth/testdata/google_creds.json create mode 100644 internal/auth/testdata/tpc_creds.json diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 18505b0a51..5a5179c1fb 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -24,6 +24,24 @@ import ( storagev1 "google.golang.org/api/storage/v1" ) +const universeDomainDefault = "googleapis.com" + +func getUniverseDomain(ctx context.Context, contents []byte, scope string) (string, error) { + creds, err := google.CredentialsFromJSON(ctx, contents, scope) + if err != nil { + err = fmt.Errorf("CredentialsFromJSON(): %w", err) + return "", err + } + + domain, err := creds.GetUniverseDomain() + if err != nil { + err = fmt.Errorf("GetUniverseDomain(): %w", err) + return "", err + } + + return domain, nil +} + // Create token source from the JSON file at the supplide path. func newTokenSourceFromPath( ctx context.Context, @@ -37,21 +55,38 @@ func newTokenSourceFromPath( return } + // By default, a standard OAuth 2.0 token source is created // Create a config struct based on its contents. jwtConfig, err := google.JWTConfigFromJSON(contents, scope) if err != nil { err = fmt.Errorf("JWTConfigFromJSON: %w", err) - return } - // Create the token source. ts = jwtConfig.TokenSource(ctx) + domain, err := getUniverseDomain(ctx, contents, scope) + if err != nil { + return + } + + // For non-GDU universe domains, token exchange is impossible and services + // must support self-signed JWTs with scopes. + // Override the token source to use self-signed JWT. + if domain != universeDomainDefault { + // Create self signed JWT access token. + ts, err = google.JWTAccessTokenSourceWithScope(contents, scope) + if err != nil { + err = fmt.Errorf("JWTAccessTokenSourceWithScope: %w", err) + return + } + } return } -// GetTokenSource returns a TokenSource for GCS API given a key file, or -// with the default credentials. +// GetTokenSource generates the token-source for GCS endpoint by following oauth2.0 authentication +// for key-file and default-credential flow. +// It also supports generating the self-signed JWT tokenSource for key-file authentication which can be +// used by custom-endpoint(e.g. TPC). func GetTokenSource( ctx context.Context, keyFile string, diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go new file mode 100644 index 0000000000..681577c537 --- /dev/null +++ b/internal/auth/auth_test.go @@ -0,0 +1,76 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "context" + "os" + "testing" + + . "github.com/jacobsa/ogletest" + storagev1 "google.golang.org/api/storage/v1" +) + +const tpcUniverseDomain = "apis-tpclp.goog" + +func TestAuth(t *testing.T) { RunTests(t) } + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type AuthTest struct { +} + +func init() { + RegisterTestSuite(&AuthTest{}) +} + +func (t *AuthTest) SetUp(ti *TestInfo) { +} + +//////////////////////////////////////////////////////////////////////// +// Tests for AuthTest +//////////////////////////////////////////////////////////////////////// + +func (t *AuthTest) TestGetUniverseDomainForGoogle() { + contents, err := os.ReadFile("testdata/google_creds.json") + AssertEq(nil, err) + + domain, err := getUniverseDomain(context.Background(), contents, storagev1.DevstorageFullControlScope) + + ExpectEq(nil, err) + ExpectEq(universeDomainDefault, domain) +} + +func (t *AuthTest) TestGetUniverseDomainForTPC() { + contents, err := os.ReadFile("testdata/tpc_creds.json") + AssertEq(nil, err) + + domain, err := getUniverseDomain(context.Background(), contents, storagev1.DevstorageFullControlScope) + + ExpectEq(nil, err) + ExpectEq(tpcUniverseDomain, domain) +} + +func (t *AuthTest) TestGetUniverseDomainForEmptyCreds() { + contents, err := os.ReadFile("testdata/empty_creds.json") + AssertEq(nil, err) + + _, err = getUniverseDomain(context.Background(), contents, storagev1.DevstorageFullControlScope) + + ExpectNe(nil, err) + ExpectEq("CredentialsFromJSON(): unexpected end of JSON input", err.Error()) +} diff --git a/internal/auth/testdata/empty_creds.json b/internal/auth/testdata/empty_creds.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/auth/testdata/google_creds.json b/internal/auth/testdata/google_creds.json new file mode 100644 index 0000000000..59d8020acb --- /dev/null +++ b/internal/auth/testdata/google_creds.json @@ -0,0 +1,13 @@ +{ + "type": "service_account", + "project_id": "project_id", + "private_key_id": "private_key_id", + "private_key": "private_key", + "client_email": "client_email", + "client_id": "client_id", + "auth_uri": "auth_uri", + "token_uri": "token_uri", + "auth_provider_x509_cert_url": "auth_provider_x509_cert_url", + "client_x509_cert_url": "client_x509_cert_url", + "universe_domain": "googleapis.com" +} diff --git a/internal/auth/testdata/tpc_creds.json b/internal/auth/testdata/tpc_creds.json new file mode 100644 index 0000000000..c78e7e72ce --- /dev/null +++ b/internal/auth/testdata/tpc_creds.json @@ -0,0 +1,13 @@ +{ + "type": "service_account", + "project_id": "project_id", + "private_key_id": "private_key_id", + "private_key": "private_key", + "client_email": "client_email", + "client_id": "client_id", + "auth_uri": "auth_uri", + "token_uri": "token_uri", + "auth_provider_x509_cert_url": "auth_provider_x509_cert_url", + "client_x509_cert_url": "client_x509_cert_url", + "universe_domain": "apis-tpclp.goog" +} From d5eedac3bd348907c9b9953cd7db836d1069b231 Mon Sep 17 00:00:00 2001 From: Tulsi Shah <46474643+Tulsishah@users.noreply.github.com> Date: Fri, 19 Apr 2024 14:08:40 +0530 Subject: [PATCH 3/4] Making read large files test package parallell (#1849) * making read large files parallell * removing cleanup * creating test dir * creating global testDir variable * rebasing and cleanup at the end * rebasing and cleanup at the end * removing cleanup * removing from global --- perfmetrics/scripts/run_e2e_tests.sh | 2 +- .../read_large_files/concurrent_read_files_test.go | 5 ++--- .../read_large_files/random_read_large_file_test.go | 6 ++---- .../read_large_files/read_large_files_test.go | 1 + .../read_large_files/seq_read_large_file_test.go | 6 ++---- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/perfmetrics/scripts/run_e2e_tests.sh b/perfmetrics/scripts/run_e2e_tests.sh index d8d4898350..eb80861076 100755 --- a/perfmetrics/scripts/run_e2e_tests.sh +++ b/perfmetrics/scripts/run_e2e_tests.sh @@ -29,6 +29,7 @@ TEST_DIR_PARALLEL=( "gzip" "write_large_files" "list_large_dir" + "read_large_files" ) # These tests never become parallel as it is changing bucket permissions. TEST_DIR_NON_PARALLEL_GROUP_1=( @@ -43,7 +44,6 @@ TEST_DIR_NON_PARALLEL_GROUP_2=( "explicit_dir" "implicit_dir" "operations" - "read_large_files" "rename_dir_limit" ) diff --git a/tools/integration_tests/read_large_files/concurrent_read_files_test.go b/tools/integration_tests/read_large_files/concurrent_read_files_test.go index e4f9e1a4c0..a16e7c3405 100644 --- a/tools/integration_tests/read_large_files/concurrent_read_files_test.go +++ b/tools/integration_tests/read_large_files/concurrent_read_files_test.go @@ -51,8 +51,7 @@ func readFile(fileInLocalDisk string, fileInMntDir string) error { } func TestReadFilesConcurrently(t *testing.T) { - // Clean the mountedDirectory before running test. - setup.CleanMntDir() + testDir := setup.SetupTestDirectory(DirForReadLargeFilesTests) filesInLocalDisk := [NumberOfFilesInLocalDiskForConcurrentRead]string{FileOne, FileTwo, FileThree} var filesPathInLocalDisk []string @@ -62,7 +61,7 @@ func TestReadFilesConcurrently(t *testing.T) { fileInLocalDisk := path.Join(os.Getenv("HOME"), filesInLocalDisk[i]) filesPathInLocalDisk = append(filesPathInLocalDisk, fileInLocalDisk) - file := path.Join(setup.MntDir(), filesInLocalDisk[i]) + file := path.Join(testDir, filesInLocalDisk[i]) filesPathInMntDir = append(filesPathInMntDir, file) createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t) diff --git a/tools/integration_tests/read_large_files/random_read_large_file_test.go b/tools/integration_tests/read_large_files/random_read_large_file_test.go index d607dc5f1f..c68164c48a 100644 --- a/tools/integration_tests/read_large_files/random_read_large_file_test.go +++ b/tools/integration_tests/read_large_files/random_read_large_file_test.go @@ -26,11 +26,9 @@ import ( ) func TestReadLargeFileRandomly(t *testing.T) { - // Clean the mountedDirectory before running test. - setup.CleanMntDir() - + testDir := setup.SetupTestDirectory(DirForReadLargeFilesTests) fileInLocalDisk := path.Join(os.Getenv("HOME"), FiveHundredMBFile) - file := path.Join(setup.MntDir(), FiveHundredMBFile) + file := path.Join(testDir, FiveHundredMBFile) // Create and copy the local file in mountedDirectory. createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t) diff --git a/tools/integration_tests/read_large_files/read_large_files_test.go b/tools/integration_tests/read_large_files/read_large_files_test.go index 41e5fc87a2..f0dc398ad9 100644 --- a/tools/integration_tests/read_large_files/read_large_files_test.go +++ b/tools/integration_tests/read_large_files/read_large_files_test.go @@ -35,6 +35,7 @@ const ChunkSize = 200 * OneMB const NumberOfRandomReadCalls = 200 const MinReadableByteFromFile = 0 const MaxReadableByteFromFile = 500 * OneMB +const DirForReadLargeFilesTests = "dirForReadLargeFilesTests" func createMountConfigsAndEquivalentFlags() (flags [][]string) { cacheDirPath := path.Join(os.Getenv("HOME"), "cache-dri") diff --git a/tools/integration_tests/read_large_files/seq_read_large_file_test.go b/tools/integration_tests/read_large_files/seq_read_large_file_test.go index 56ff0281d1..91e71131ff 100644 --- a/tools/integration_tests/read_large_files/seq_read_large_file_test.go +++ b/tools/integration_tests/read_large_files/seq_read_large_file_test.go @@ -25,12 +25,10 @@ import ( ) func TestReadLargeFileSequentially(t *testing.T) { - // Clean the mountedDirectory before running test. - setup.CleanMntDir() - + testDir := setup.SetupTestDirectory(DirForReadLargeFilesTests) // Create file of 500 MB with random data in local disk and copy it in mntDir. fileInLocalDisk := path.Join(os.Getenv("HOME"), FiveHundredMBFile) - file := path.Join(setup.MntDir(), FiveHundredMBFile) + file := path.Join(testDir, FiveHundredMBFile) createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t) // Sequentially read the data from file. From ede0d0f61fa90107fd6a894c5ae96fd2ee72f12a Mon Sep 17 00:00:00 2001 From: Tulsi Shah <46474643+Tulsishah@users.noreply.github.com> Date: Fri, 19 Apr 2024 15:00:11 +0530 Subject: [PATCH 4/4] Making rename dir limit test package parallell (#1850) * making rename dir limit test parallell * setup test dir * lint tests fix * small fix * formating * creating global testDir variable * rebasing and cleanup at the end * removing cleanup * review comment * passing destDir directly to the method --- perfmetrics/scripts/run_e2e_tests.sh | 2 +- .../rename_dir_limit/move_dir_test.go | 53 +++++----- .../rename_dir_limit/rename_dir_limit_test.go | 1 + .../rename_dir_limit/rename_dir_test.go | 98 ++++++++----------- 4 files changed, 67 insertions(+), 87 deletions(-) diff --git a/perfmetrics/scripts/run_e2e_tests.sh b/perfmetrics/scripts/run_e2e_tests.sh index eb80861076..d55cc0a577 100755 --- a/perfmetrics/scripts/run_e2e_tests.sh +++ b/perfmetrics/scripts/run_e2e_tests.sh @@ -29,6 +29,7 @@ TEST_DIR_PARALLEL=( "gzip" "write_large_files" "list_large_dir" + "rename_dir_limit" "read_large_files" ) # These tests never become parallel as it is changing bucket permissions. @@ -44,7 +45,6 @@ TEST_DIR_NON_PARALLEL_GROUP_2=( "explicit_dir" "implicit_dir" "operations" - "rename_dir_limit" ) TEST_DIR_HNS_GROUP=( diff --git a/tools/integration_tests/rename_dir_limit/move_dir_test.go b/tools/integration_tests/rename_dir_limit/move_dir_test.go index 9db57d6843..5681053f88 100644 --- a/tools/integration_tests/rename_dir_limit/move_dir_test.go +++ b/tools/integration_tests/rename_dir_limit/move_dir_test.go @@ -54,9 +54,6 @@ func checkIfSrcDirectoryGetsRemovedAfterMoveOperation(srcDirPath string, t *test // srcMoveDir/srcMoveFile -- File // srcMoveDir/subSrcMoveDir -- Dir func createSrcDirectoryWithObjectsForMoveDirTest(dirPath string, t *testing.T) { - // Clean the mountedDirectory before running test. - setup.CleanMntDir() - // testBucket/srcMoveDir err := os.Mkdir(dirPath, setup.FilePermission_0600) if err != nil { @@ -133,11 +130,12 @@ func checkIfMovedDirectoryHasCorrectData(destDir string, t *testing.T) { // destMoveDir/srcMoveFile -- File // destMoveDir/subSrcMoveDir -- Dir func TestMoveDirectoryInNonExistingDirectory(t *testing.T) { - srcDir := path.Join(setup.MntDir(), SrcMoveDirectory) + testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) + srcDir := path.Join(testDir, SrcMoveDirectory) createSrcDirectoryWithObjectsForMoveDirTest(srcDir, t) - destDir := path.Join(setup.MntDir(), DestMoveDirectoryNotExist) + destDir := path.Join(testDir, DestMoveDirectoryNotExist) err := operations.Move(srcDir, destDir) if err != nil { @@ -158,13 +156,14 @@ func TestMoveDirectoryInNonExistingDirectory(t *testing.T) { // destMoveDir/srcMoveDir/srcMoveFile -- File // destMoveDir/srcMoveDir/subSrcMoveDir -- Dir func TestMoveDirectoryInEmptyDirectory(t *testing.T) { - srcDir := path.Join(setup.MntDir(), SrcMoveDirectory) + testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) + srcDir := path.Join(testDir, SrcMoveDirectory) createSrcDirectoryWithObjectsForMoveDirTest(srcDir, t) // Create below directory // destMoveDir -- Dir - destDir := path.Join(setup.MntDir(), DestMoveDirectory) + destDir := path.Join(testDir, DestMoveDirectory) err := os.Mkdir(destDir, setup.FilePermission_0600) if err != nil { t.Errorf("Error in creating directory: %v", err) @@ -193,23 +192,23 @@ func TestMoveDirectoryInEmptyDirectory(t *testing.T) { checkIfSrcDirectoryGetsRemovedAfterMoveOperation(srcDir, t) } -func createDestNonEmptyDirectoryForMoveTest(t *testing.T) { - destDir := path.Join(setup.MntDir(), DestNonEmptyMoveDirectory) - operations.CreateDirectoryWithNFiles(0, destDir, "", t) +func createDestNonEmptyDirectoryForMoveTest(dirPath string, t *testing.T) { + operations.CreateDirectoryWithNFiles(0, dirPath, "", t) - destSubDir := path.Join(destDir, SubDirInNonEmptyDestMoveDirectory) + destSubDir := path.Join(dirPath, SubDirInNonEmptyDestMoveDirectory) operations.CreateDirectoryWithNFiles(0, destSubDir, "", t) } func TestMoveDirectoryInNonEmptyDirectory(t *testing.T) { - srcDir := path.Join(setup.MntDir(), SrcMoveDirectory) + testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) + srcDir := path.Join(testDir, SrcMoveDirectory) createSrcDirectoryWithObjectsForMoveDirTest(srcDir, t) // Create below directory // destMoveDir -- Dir - destDir := path.Join(setup.MntDir(), DestNonEmptyMoveDirectory) - createDestNonEmptyDirectoryForMoveTest(t) + destDir := path.Join(testDir, DestNonEmptyMoveDirectory) + createDestNonEmptyDirectoryForMoveTest(destDir, t) err := operations.Move(srcDir, destDir) if err != nil { @@ -269,17 +268,15 @@ func checkIfMovedEmptyDirectoryHasNoData(destSrc string, t *testing.T) { // destNonEmptyMoveDirectory/subDirInNonEmptyDestMoveDirectory // destNonEmptyMoveDirectory/emptySrcDirectoryMoveTest func TestMoveEmptyDirectoryInNonEmptyDirectory(t *testing.T) { - // Clean the mountedDirectory before running test. - setup.CleanMntDir() - - srcDir := path.Join(setup.MntDir(), EmptySrcDirectoryMoveTest) + testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) + srcDir := path.Join(testDir, EmptySrcDirectoryMoveTest) operations.CreateDirectoryWithNFiles(0, srcDir, "", t) // Create below directory // destNonEmptyMoveDirectory -- Dir // destNonEmptyMoveDirectory/subDirInNonEmptyDestMoveDirectory -- Dir - destDir := path.Join(setup.MntDir(), DestNonEmptyMoveDirectory) - createDestNonEmptyDirectoryForMoveTest(t) + destDir := path.Join(testDir, DestNonEmptyMoveDirectory) + createDestNonEmptyDirectoryForMoveTest(destDir, t) err := operations.Move(srcDir, destDir) if err != nil { @@ -325,15 +322,13 @@ func TestMoveEmptyDirectoryInNonEmptyDirectory(t *testing.T) { // Output // destEmptyMoveDirectory/emptySrcDirectoryMoveTest func TestMoveEmptyDirectoryInEmptyDirectory(t *testing.T) { - // Clean the mountedDirectory before running test. - setup.CleanMntDir() - - srcDir := path.Join(setup.MntDir(), EmptySrcDirectoryMoveTest) + testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) + srcDir := path.Join(testDir, EmptySrcDirectoryMoveTest) operations.CreateDirectoryWithNFiles(0, srcDir, "", t) // Create below directory // destMoveDir -- Dir - destDir := path.Join(setup.MntDir(), DestEmptyMoveDirectory) + destDir := path.Join(testDir, DestEmptyMoveDirectory) operations.CreateDirectoryWithNFiles(0, destDir, "", t) err := operations.Move(srcDir, destDir) @@ -371,14 +366,12 @@ func TestMoveEmptyDirectoryInEmptyDirectory(t *testing.T) { // Output // destMoveDirectoryNotExist func TestMoveEmptyDirectoryInNonExistingDirectory(t *testing.T) { - // Clean the mountedDirectory before running test. - setup.CleanMntDir() - - srcDir := path.Join(setup.MntDir(), EmptySrcDirectoryMoveTest) + testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) + srcDir := path.Join(testDir, EmptySrcDirectoryMoveTest) operations.CreateDirectoryWithNFiles(0, srcDir, "", t) // destMoveDirectoryNotExist -- Dir - destDir := path.Join(setup.MntDir(), DestMoveDirectoryNotExist) + destDir := path.Join(testDir, DestMoveDirectoryNotExist) _, err := os.Stat(destDir) if err == nil { diff --git a/tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go b/tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go index e612c83243..0f37279483 100644 --- a/tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go +++ b/tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go @@ -26,6 +26,7 @@ import ( "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" ) +const DirForRenameDirLimitTests = "dirForRenameDirLimitTests" const DirectoryWithThreeFiles = "directoryWithThreeFiles" const DirectoryWithTwoFiles = "directoryWithTwoFiles" const DirectoryWithFourFiles = "directoryWithFourFiles" diff --git a/tools/integration_tests/rename_dir_limit/rename_dir_test.go b/tools/integration_tests/rename_dir_limit/rename_dir_test.go index 73b00841f5..0dee0c6a08 100644 --- a/tools/integration_tests/rename_dir_limit/rename_dir_test.go +++ b/tools/integration_tests/rename_dir_limit/rename_dir_test.go @@ -27,19 +27,17 @@ import ( // As --rename-directory-limit = 3, and the number of objects in the directory is three, // which is equal to the limit, the operation should get successful. func TestRenameDirectoryWithThreeFiles(t *testing.T) { - // Clean the mountedDirectory before running test. - setup.CleanMntDir() - + testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) // Create directory structure - // testBucket/directoryWithThreeFiles -- Dir - // testBucket/directoryWithThreeFiles/temp1.txt -- File - // testBucket/directoryWithThreeFiles/temp2.txt -- File - // testBucket/directoryWithThreeFiles/temp3.txt -- File - dirPath := path.Join(setup.MntDir(), DirectoryWithThreeFiles) + // testBucket/dirForRenameDirLimitTests/directoryWithThreeFiles -- Dir + // testBucket/dirForRenameDirLimitTests/directoryWithThreeFiles/temp1.txt -- File + // testBucket/dirForRenameDirLimitTests/directoryWithThreeFiles/temp2.txt -- File + // testBucket/dirForRenameDirLimitTests/directoryWithThreeFiles/temp3.txt -- File + dirPath := path.Join(testDir, DirectoryWithThreeFiles) operations.CreateDirectoryWithNFiles(3, dirPath, PrefixTempFile, t) - oldDirPath := path.Join(setup.MntDir(), DirectoryWithThreeFiles) - newDirPath := path.Join(setup.MntDir(), RenamedDirectory) + oldDirPath := path.Join(testDir, DirectoryWithThreeFiles) + newDirPath := path.Join(testDir, RenamedDirectory) // Cleaning the directory before renaming. operations.RemoveDir(newDirPath) @@ -54,19 +52,17 @@ func TestRenameDirectoryWithThreeFiles(t *testing.T) { // As --rename-directory-limit = 3, and the number of objects in the directory is two, // which is less than the limit, the operation should get successful. func TestRenameDirectoryWithTwoFiles(t *testing.T) { - // Clean the mountedDirectory before running test. - setup.CleanMntDir() - + testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) // Create directory structure - // testBucket/directoryWithTwoFiles -- Dir - // testBucket/directoryWithTwoFiles/temp1.txt -- File - // testBucket/directoryWithTwoFiles/temp2.txt -- File - dirPath := path.Join(setup.MntDir(), DirectoryWithTwoFiles) + // testBucket/dirForRenameDirLimitTests/directoryWithTwoFiles -- Dir + // testBucket/dirForRenameDirLimitTests/directoryWithTwoFiles/temp1.txt -- File + // testBucket/dirForRenameDirLimitTests/directoryWithTwoFiles/temp2.txt -- File + dirPath := path.Join(testDir, DirectoryWithTwoFiles) operations.CreateDirectoryWithNFiles(2, dirPath, PrefixTempFile, t) - oldDirPath := path.Join(setup.MntDir(), DirectoryWithTwoFiles) - newDirPath := path.Join(setup.MntDir(), RenamedDirectory) + oldDirPath := path.Join(testDir, DirectoryWithTwoFiles) + newDirPath := path.Join(testDir, RenamedDirectory) // Cleaning the directory before renaming. operations.RemoveDir(newDirPath) @@ -81,25 +77,19 @@ func TestRenameDirectoryWithTwoFiles(t *testing.T) { // As --rename-directory-limit = 3, and the number of objects in the directory is two, // which is greater than the limit, the operation should get fail. func TestRenameDirectoryWithFourFiles(t *testing.T) { - // Clean the mountedDirectory before running test. - setup.CleanMntDir() - + testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) // Creating directory structure - // testBucket/directoryWithFourFiles -- Dir - // testBucket/directoryWithFourFiles/temp1.txt -- File - // testBucket/directoryWithFourFiles/temp2.txt -- File - // testBucket/directoryWithFourFiles/temp3.txt -- File - // testBucket/directoryWithFourFiles/temp4.txt -- File - dirPath := path.Join(setup.MntDir(), DirectoryWithFourFiles) - - operations.CreateDirectoryWithNFiles(4, dirPath, PrefixTempFile, t) - - oldDirPath := path.Join(setup.MntDir(), DirectoryWithFourFiles) - newDirPath := path.Join(setup.MntDir(), RenamedDirectory) + // testBucket/dirForRenameDirLimitTests/directoryWithFourFiles -- Dir + // testBucket/dirForRenameDirLimitTests/directoryWithFourFiles/temp1.txt -- File + // testBucket/dirForRenameDirLimitTests/directoryWithFourFiles/temp2.txt -- File + // testBucket/dirForRenameDirLimitTests/directoryWithFourFiles/temp3.txt -- File + // testBucket/dirForRenameDirLimitTests/directoryWithFourFiles/temp4.txt -- File + oldDirPath := path.Join(testDir, DirectoryWithFourFiles) + operations.CreateDirectoryWithNFiles(4, oldDirPath, PrefixTempFile, t) + newDirPath := path.Join(testDir, RenamedDirectory) // Cleaning the directory before renaming. operations.RemoveDir(newDirPath) - err := os.Rename(oldDirPath, newDirPath) if err == nil { @@ -110,22 +100,20 @@ func TestRenameDirectoryWithFourFiles(t *testing.T) { // As --rename-directory-limit = 3, and the number of objects in the directory is three, // which is equal to limit, the operation should get successful. func TestRenameDirectoryWithTwoFilesAndOneEmptyDirectory(t *testing.T) { - // Clean the mountedDirectory before running test. - setup.CleanMntDir() - + testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) // Creating directory structure - // testBucket/directoryWithTwoFilesOneEmptyDirectory -- Dir - // testBucket/directoryWithTwoFilesOneEmptyDirectory/a.txt -- File - // testBucket/directoryWithTwoFilesOneEmptyDirectory/b.txt -- File - // testBucket/directoryWithTwoFilesOneEmptyDirectory/emptySubDirectory -- Dir - dirPath := path.Join(setup.MntDir(), DirectoryWithTwoFilesOneEmptyDirectory) - subDirPath := path.Join(setup.MntDir(), DirectoryWithTwoFilesOneEmptyDirectory, EmptySubDirectory) + // testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneEmptyDirectory -- Dir + // testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneEmptyDirectory/a.txt -- File + // testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneEmptyDirectory/b.txt -- File + // testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneEmptyDirectory/emptySubDirectory -- Dir + dirPath := path.Join(testDir, DirectoryWithTwoFilesOneEmptyDirectory) + subDirPath := path.Join(testDir, DirectoryWithTwoFilesOneEmptyDirectory, EmptySubDirectory) operations.CreateDirectoryWithNFiles(2, dirPath, PrefixTempFile, t) operations.CreateDirectoryWithNFiles(0, subDirPath, PrefixTempFile, t) - oldDirPath := path.Join(setup.MntDir(), DirectoryWithTwoFilesOneEmptyDirectory) - newDirPath := path.Join(setup.MntDir(), RenamedDirectory) + oldDirPath := path.Join(testDir, DirectoryWithTwoFilesOneEmptyDirectory) + newDirPath := path.Join(testDir, RenamedDirectory) // Cleaning the directory before renaming. operations.RemoveDir(newDirPath) @@ -140,24 +128,22 @@ func TestRenameDirectoryWithTwoFilesAndOneEmptyDirectory(t *testing.T) { // As --rename-directory-limit = 3, and the number of objects in the directory is Four, // which is greater than the limit, the operation should get fail. func TestRenameDirectoryWithTwoFilesAndOneNonEmptyDirectory(t *testing.T) { - // Clean the mountedDirectory before running test. - setup.CleanMntDir() - + testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) // Creating directory structure - // testBucket/directoryWithTwoFilesOneNonEmptyDirectory -- Dir - // testBucket/directoryWithTwoFilesOneNonEmptyDirectory/temp1.txt -- File - // testBucket/directoryWithTwoFilesOneNonEmptyDirectory/temp2.txt -- File - // testBucket/directoryWithTwoFilesOneNonEmptyDirectory/NonEmptySubDirectory -- Dir - // testBucket/directoryWithTwoFilesOneNonEmptyDirectory/NonEmptySubDirectory/temp3.txt -- File + // testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneNonEmptyDirectory -- Dir + // testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneNonEmptyDirectory/temp1.txt -- File + // testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneNonEmptyDirectory/temp2.txt -- File + // testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneNonEmptyDirectory/NonEmptySubDirectory -- Dir + // testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneNonEmptyDirectory/NonEmptySubDirectory/temp3.txt -- File - dirPath := path.Join(setup.MntDir(), DirectoryWithTwoFilesOneNonEmptyDirectory) + dirPath := path.Join(testDir, DirectoryWithTwoFilesOneNonEmptyDirectory) subDirPath := path.Join(dirPath, NonEmptySubDirectory) operations.CreateDirectoryWithNFiles(2, dirPath, PrefixTempFile, t) operations.CreateDirectoryWithNFiles(1, subDirPath, PrefixTempFile, t) - oldDirPath := path.Join(setup.MntDir(), DirectoryWithTwoFilesOneNonEmptyDirectory) - newDirPath := path.Join(setup.MntDir(), RenamedDirectory) + oldDirPath := path.Join(testDir, DirectoryWithTwoFilesOneNonEmptyDirectory) + newDirPath := path.Join(testDir, RenamedDirectory) // Cleaning the directory before renaming. operations.RemoveDir(newDirPath)