From b5aea8a274eca6c272bf593d3daaa2f88f3c08ba Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Tue, 28 May 2024 13:49:57 +0000 Subject: [PATCH 01/33] convert cleanup gcs dir function in storage client --- .../implicit_dir/implicit_dir_test.go | 15 ++++++++--- .../implicit_dir/local_file_test.go | 4 --- .../interrupt/interrupt_test.go | 25 +++++++++++++++++- .../local_file/local_file_test.go | 2 +- .../log_rotation/log_rotation_test.go | 23 +++++++++++++++- .../list_empty_managed_folders_test.go | 2 +- .../managed_folders/managed_folders_test.go | 22 +++++++++++++++- .../managed_folders/test_helper.go | 6 +++-- .../managed_folders/view_permissions_test.go | 2 +- .../read_cache/setup_test.go | 22 +++++++++++++--- .../readonly_creds/readonly_creds_test.go | 2 +- tools/integration_tests/util/setup/setup.go | 26 +++++++++++++++---- 12 files changed, 126 insertions(+), 25 deletions(-) diff --git a/tools/integration_tests/implicit_dir/implicit_dir_test.go b/tools/integration_tests/implicit_dir/implicit_dir_test.go index 06819f2595..15a0939cb6 100644 --- a/tools/integration_tests/implicit_dir/implicit_dir_test.go +++ b/tools/integration_tests/implicit_dir/implicit_dir_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "cloud.google.com/go/storage" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup" @@ -36,6 +37,11 @@ const NumberOfFilesInExplicitDirInImplicitSubDir = 1 const NumberOfFilesInExplicitDirInImplicitDir = 1 const DirForImplicitDirTests = "dirForImplicitDirTests" +var ( + storageClient *storage.Client + ctx context.Context +) + func TestMain(m *testing.M) { setup.ParseSetUpFlags() @@ -50,6 +56,10 @@ func TestMain(m *testing.M) { log.Fatalf("client.CreateStorageClient: %v", err) } + // Close storage client and release resources. + defer cancel() + defer storageClient.Close() + flagsSet := [][]string{{"--implicit-dirs"}} if !testing.Short() { @@ -58,10 +68,7 @@ func TestMain(m *testing.M) { successCode := implicit_and_explicit_dir_setup.RunTestsForImplicitDirAndExplicitDir(flagsSet, m) - // Close storage client and release resources. - storageClient.Close() - cancel() // Clean up test directory created. - setup.CleanupDirectoryOnGCS(path.Join(setup.TestBucket(), testDirName)) + setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) os.Exit(successCode) } diff --git a/tools/integration_tests/implicit_dir/local_file_test.go b/tools/integration_tests/implicit_dir/local_file_test.go index aad952eb8e..46d4690d45 100644 --- a/tools/integration_tests/implicit_dir/local_file_test.go +++ b/tools/integration_tests/implicit_dir/local_file_test.go @@ -19,11 +19,9 @@ import ( "path/filepath" "testing" - "cloud.google.com/go/storage" . "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "golang.org/x/net/context" ) const ( @@ -32,8 +30,6 @@ const ( var ( testDirPath string - storageClient *storage.Client - ctx context.Context ) // ////////////////////////////////////////////////////////////////////// diff --git a/tools/integration_tests/interrupt/interrupt_test.go b/tools/integration_tests/interrupt/interrupt_test.go index 5700f06c2b..b20270cdb7 100644 --- a/tools/integration_tests/interrupt/interrupt_test.go +++ b/tools/integration_tests/interrupt/interrupt_test.go @@ -15,11 +15,16 @@ package interrupt import ( + "context" + "log" "os" "path" "testing" + "time" + "cloud.google.com/go/storage" "github.com/googlecloudplatform/gcsfuse/v2/internal/config" + "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" ) @@ -28,6 +33,11 @@ const ( testDirName = "InterruptTest" ) +var ( + storageClient *storage.Client + ctx context.Context +) + //////////////////////////////////////////////////////////////////////// // TestMain //////////////////////////////////////////////////////////////////////// @@ -36,6 +46,19 @@ func TestMain(m *testing.M) { setup.ParseSetUpFlags() setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() + ctx := context.Background() + var cancel context.CancelFunc + + ctx, cancel = context.WithTimeout(ctx, time.Minute*15) + storageClient, err := client.CreateStorageClient(ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + + defer cancel() + defer storageClient.Close() + setup.RunTestsForMountedDirectoryFlag(m) // Else run tests for testBucket. @@ -56,6 +79,6 @@ func TestMain(m *testing.M) { successCode := static_mounting.RunTests(flags, m) // Clean up test directory created. - setup.CleanupDirectoryOnGCS(path.Join(setup.TestBucket(), testDirName)) + setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) os.Exit(successCode) } diff --git a/tools/integration_tests/local_file/local_file_test.go b/tools/integration_tests/local_file/local_file_test.go index 0f51343515..40f8668c45 100644 --- a/tools/integration_tests/local_file/local_file_test.go +++ b/tools/integration_tests/local_file/local_file_test.go @@ -121,6 +121,6 @@ func TestMain(m *testing.M) { storageClient.Close() cancel() // Clean up test directory created. - setup.CleanupDirectoryOnGCS(path.Join(setup.TestBucket(), testDirName)) + setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) os.Exit(successCode) } diff --git a/tools/integration_tests/log_rotation/log_rotation_test.go b/tools/integration_tests/log_rotation/log_rotation_test.go index 255fbb8ef7..93255c07f8 100644 --- a/tools/integration_tests/log_rotation/log_rotation_test.go +++ b/tools/integration_tests/log_rotation/log_rotation_test.go @@ -17,11 +17,16 @@ package log_rotation import ( + "context" + "log" "os" "path" "testing" + "time" + "cloud.google.com/go/storage" "github.com/googlecloudplatform/gcsfuse/v2/internal/config" + "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" ) @@ -39,6 +44,9 @@ const ( var logDirPath string var logFilePath string +var storageClient *storage.Client +var ctx context.Context + func getMountConfigForLogRotation(maxFileSizeMB, backupFileCount int, compress bool, logFilePath string) config.MountConfig { mountConfig := config.MountConfig{ @@ -60,6 +68,19 @@ func getMountConfigForLogRotation(maxFileSizeMB, backupFileCount int, compress b //////////////////////////////////////////////////////////////////////// func TestMain(m *testing.M) { + ctx := context.Background() + var cancel context.CancelFunc + + ctx, cancel = context.WithTimeout(ctx, time.Minute*15) + storageClient, err := client.CreateStorageClient(ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + + defer cancel() + defer storageClient.Close() + setup.ParseSetUpFlags() setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() @@ -97,6 +118,6 @@ func TestMain(m *testing.M) { successCode := static_mounting.RunTests(flags, m) // Clean up test directory created. - setup.CleanupDirectoryOnGCS(path.Join(setup.TestBucket(), testDirName)) + setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) os.Exit(successCode) } diff --git a/tools/integration_tests/managed_folders/list_empty_managed_folders_test.go b/tools/integration_tests/managed_folders/list_empty_managed_folders_test.go index 9f6c9a59fb..7e92e83c2e 100644 --- a/tools/integration_tests/managed_folders/list_empty_managed_folders_test.go +++ b/tools/integration_tests/managed_folders/list_empty_managed_folders_test.go @@ -57,7 +57,7 @@ func (s *enableEmptyManagedFoldersTrue) Teardown(t *testing.T) { bucket, testDir := setup.GetBucketAndObjectBasedOnTypeOfMount(TestDirForEmptyManagedFoldersTest) operations.DeleteManagedFoldersInBucket(path.Join(testDir, EmptyManagedFolder1), setup.TestBucket()) operations.DeleteManagedFoldersInBucket(path.Join(testDir, EmptyManagedFolder2), setup.TestBucket()) - setup.CleanupDirectoryOnGCS(path.Join(bucket, testDir)) + setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(bucket, testDir)) } //////////////////////////////////////////////////////////////////////// diff --git a/tools/integration_tests/managed_folders/managed_folders_test.go b/tools/integration_tests/managed_folders/managed_folders_test.go index 999e3b7cd6..8ff3bad490 100644 --- a/tools/integration_tests/managed_folders/managed_folders_test.go +++ b/tools/integration_tests/managed_folders/managed_folders_test.go @@ -16,11 +16,15 @@ package managed_folders import ( + "context" "log" "os" "path" "testing" + "time" + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/only_dir_mounting" @@ -39,7 +43,9 @@ var ( // Mount directory is where our tests run. mountDir string // Root directory is the directory to be unmounted. - rootDir string + rootDir string + storageClient *storage.Client + ctx context.Context ) //////////////////////////////////////////////////////////////////////// @@ -49,6 +55,20 @@ var ( func TestMain(m *testing.M) { setup.ParseSetUpFlags() + ctx := context.Background() + var cancel context.CancelFunc + + ctx, cancel = context.WithTimeout(ctx, time.Minute*15) + storageClient, err := client.CreateStorageClient(ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + + defer cancel() + defer storageClient.Close() + + setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() if setup.MountedDirectory() != "" { diff --git a/tools/integration_tests/managed_folders/test_helper.go b/tools/integration_tests/managed_folders/test_helper.go index 4bd1ad5a68..8ef8dcc5a3 100644 --- a/tools/integration_tests/managed_folders/test_helper.go +++ b/tools/integration_tests/managed_folders/test_helper.go @@ -15,6 +15,7 @@ package managed_folders import ( + "context" "encoding/json" "fmt" "io/fs" @@ -25,6 +26,7 @@ import ( "strings" "testing" + "cloud.google.com/go/storage" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" ) @@ -113,12 +115,12 @@ func createDirectoryStructureForNonEmptyManagedFolders(t *testing.T) { operations.CopyFileInBucket(path.Join("/tmp", FileInNonEmptyManagedFoldersTest), testDir, bucket, t) } -func cleanup(bucket, testDir, serviceAccount, iam_role string, t *testing.T) { +func cleanup(ctx context.Context, client *storage.Client, bucket, testDir, serviceAccount, iam_role string, t *testing.T) { revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, iam_role, t) revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, iam_role, t) operations.DeleteManagedFoldersInBucket(path.Join(testDir, ManagedFolder1), setup.TestBucket()) operations.DeleteManagedFoldersInBucket(path.Join(testDir, ManagedFolder2), setup.TestBucket()) - setup.CleanupDirectoryOnGCS(path.Join(bucket, testDir)) + setup.CleanupDirectoryOnGCS(ctx, client, path.Join(bucket, testDir)) } func listNonEmptyManagedFolders(t *testing.T) { diff --git a/tools/integration_tests/managed_folders/view_permissions_test.go b/tools/integration_tests/managed_folders/view_permissions_test.go index 9611942c72..8db57ad11b 100644 --- a/tools/integration_tests/managed_folders/view_permissions_test.go +++ b/tools/integration_tests/managed_folders/view_permissions_test.go @@ -148,7 +148,7 @@ func TestManagedFolders_FolderViewPermission(t *testing.T) { bucket, testDir = setup.GetBucketAndObjectBasedOnTypeOfMount(TestDirForManagedFolderTest) // Create directory structure for testing. createDirectoryStructureForNonEmptyManagedFolders(t) - defer cleanup(bucket, testDir, serviceAccount, IAMRoleForViewPermission, t) + defer cleanup(ctx, storageClient, bucket, testDir, serviceAccount, IAMRoleForViewPermission, t) // Run tests. log.Printf("Running tests with flags and managed folder have nil permissions: %s", flags) diff --git a/tools/integration_tests/read_cache/setup_test.go b/tools/integration_tests/read_cache/setup_test.go index 409e512a4d..509f67f5e9 100644 --- a/tools/integration_tests/read_cache/setup_test.go +++ b/tools/integration_tests/read_cache/setup_test.go @@ -20,6 +20,7 @@ import ( "os" "path" "testing" + "time" "cloud.google.com/go/storage" "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" @@ -73,7 +74,9 @@ var ( // mount directory is where our tests run. mountDir string // root directory is the directory to be unmounted. - rootDir string + rootDir string + storageClient *storage.Client + ctx context.Context ) //////////////////////////////////////////////////////////////////////// @@ -122,6 +125,19 @@ func createConfigFile(cacheSize int64, cacheFileForRangeRead bool, fileName stri //////////////////////////////////////////////////////////////////////// func TestMain(m *testing.M) { + ctx := context.Background() + var cancel context.CancelFunc + + ctx, cancel = context.WithTimeout(ctx, time.Minute*15) + storageClient, err := client.CreateStorageClient(ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + + defer cancel() + defer storageClient.Close() + setup.ParseSetUpFlags() setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() @@ -153,10 +169,10 @@ func TestMain(m *testing.M) { mountDir = rootDir mountFunc = only_dir_mounting.MountGcsfuseWithOnlyDir successCode = m.Run() - setup.CleanupDirectoryOnGCS(path.Join(setup.TestBucket(), setup.OnlyDirMounted(), testDirName)) + setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), setup.OnlyDirMounted(), testDirName)) } // Clean up test directory created. - setup.CleanupDirectoryOnGCS(path.Join(setup.TestBucket(), testDirName)) + setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) os.Exit(successCode) } diff --git a/tools/integration_tests/readonly_creds/readonly_creds_test.go b/tools/integration_tests/readonly_creds/readonly_creds_test.go index 3cc9cd19de..602b9683d2 100644 --- a/tools/integration_tests/readonly_creds/readonly_creds_test.go +++ b/tools/integration_tests/readonly_creds/readonly_creds_test.go @@ -67,6 +67,6 @@ func TestMain(m *testing.M) { flags := [][]string{{"--implicit-dirs=true"}, {"--implicit-dirs=false"}} successCode := creds_tests.RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(flags, "objectViewer", m) - setup.CleanupDirectoryOnGCS(path.Join(setup.TestBucket(), testDirName)) + setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) os.Exit(successCode) } diff --git a/tools/integration_tests/util/setup/setup.go b/tools/integration_tests/util/setup/setup.go index 21f9f8ec18..d400f4a622 100644 --- a/tools/integration_tests/util/setup/setup.go +++ b/tools/integration_tests/util/setup/setup.go @@ -15,6 +15,7 @@ package setup import ( + "context" "flag" "fmt" "log" @@ -28,8 +29,10 @@ import ( "testing" "time" + "cloud.google.com/go/storage" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" "github.com/googlecloudplatform/gcsfuse/v2/tools/util" + "google.golang.org/api/iterator" ) var testBucket = flag.String("testbucket", "", "The GCS bucket used for the test.") @@ -370,11 +373,24 @@ func SetupTestDirectory(testDirName string) string { } // CleanupDirectoryOnGCS cleans up the object/directory path passed in parameter. -func CleanupDirectoryOnGCS(directoryPathOnGCS string) { - _, err := operations.ExecuteGcloudCommandf("storage rm -r gs://%s", directoryPathOnGCS) - if err != nil { - log.Printf("Error while cleaning up directory %s from GCS: %v", - directoryPathOnGCS, err) +func CleanupDirectoryOnGCS(ctx context.Context, client *storage.Client, directoryPathOnGCS string) { + if directoryPathOnGCS[len(directoryPathOnGCS)-1] == '/' { + directoryPathOnGCS = directoryPathOnGCS[:len(directoryPathOnGCS)-1] + } + + bucket := client.Bucket(TestBucket()) + it := bucket.Objects(ctx, &storage.Query{Prefix: directoryPathOnGCS + "/"}) + for { + attrs, err := it.Next() + if err == iterator.Done { + break // No more objects found + } + if err != nil { + log.Printf("error iterating objects: %v", err) + } + if err := bucket.Object(attrs.Name).Delete(ctx); err != nil { + log.Printf("error deleting object %s: %v", attrs.Name, err) + } } } From e42eee91e83906d11d2eea9e55c63c988398d7f7 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Tue, 28 May 2024 13:58:01 +0000 Subject: [PATCH 02/33] lint fix --- .../log_rotation/log_rotation_test.go | 15 ++++++++------- .../managed_folders/managed_folders_test.go | 1 - tools/integration_tests/read_cache/setup_test.go | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/integration_tests/log_rotation/log_rotation_test.go b/tools/integration_tests/log_rotation/log_rotation_test.go index 93255c07f8..2c12f8cacb 100644 --- a/tools/integration_tests/log_rotation/log_rotation_test.go +++ b/tools/integration_tests/log_rotation/log_rotation_test.go @@ -41,11 +41,12 @@ const ( logFileCount = activeLogFileCount + backupLogFileCount ) -var logDirPath string -var logFilePath string - -var storageClient *storage.Client -var ctx context.Context +var ( + logDirPath string + logFilePath string + storageClient *storage.Client + ctx context.Context +) func getMountConfigForLogRotation(maxFileSizeMB, backupFileCount int, compress bool, logFilePath string) config.MountConfig { @@ -68,6 +69,8 @@ func getMountConfigForLogRotation(maxFileSizeMB, backupFileCount int, compress b //////////////////////////////////////////////////////////////////////// func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + ctx := context.Background() var cancel context.CancelFunc @@ -81,8 +84,6 @@ func TestMain(m *testing.M) { defer cancel() defer storageClient.Close() - setup.ParseSetUpFlags() - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() // Run tests for mountedDirectory only if --mountedDirectory flag is set. diff --git a/tools/integration_tests/managed_folders/managed_folders_test.go b/tools/integration_tests/managed_folders/managed_folders_test.go index 8ff3bad490..dddfb8d4a5 100644 --- a/tools/integration_tests/managed_folders/managed_folders_test.go +++ b/tools/integration_tests/managed_folders/managed_folders_test.go @@ -68,7 +68,6 @@ func TestMain(m *testing.M) { defer cancel() defer storageClient.Close() - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() if setup.MountedDirectory() != "" { diff --git a/tools/integration_tests/read_cache/setup_test.go b/tools/integration_tests/read_cache/setup_test.go index 509f67f5e9..fad8bccc2d 100644 --- a/tools/integration_tests/read_cache/setup_test.go +++ b/tools/integration_tests/read_cache/setup_test.go @@ -134,7 +134,7 @@ func TestMain(m *testing.M) { log.Printf("Error creating storage client: %v\n", err) os.Exit(1) } - + defer cancel() defer storageClient.Close() From 8c4dc5c1b37cc5457f5e412edea894357c342d09 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Tue, 28 May 2024 13:59:59 +0000 Subject: [PATCH 03/33] lint fix --- .../integration_tests/log_rotation/log_rotation_test.go | 5 +++-- .../managed_folders/managed_folders_test.go | 5 +++-- tools/integration_tests/read_cache/setup_test.go | 9 +++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tools/integration_tests/log_rotation/log_rotation_test.go b/tools/integration_tests/log_rotation/log_rotation_test.go index 2c12f8cacb..70c4973d90 100644 --- a/tools/integration_tests/log_rotation/log_rotation_test.go +++ b/tools/integration_tests/log_rotation/log_rotation_test.go @@ -71,11 +71,12 @@ func getMountConfigForLogRotation(maxFileSizeMB, backupFileCount int, compress b func TestMain(m *testing.M) { setup.ParseSetUpFlags() - ctx := context.Background() + var err error + ctx = context.Background() var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Minute*15) - storageClient, err := client.CreateStorageClient(ctx) + storageClient, err = client.CreateStorageClient(ctx) if err != nil { log.Printf("Error creating storage client: %v\n", err) os.Exit(1) diff --git a/tools/integration_tests/managed_folders/managed_folders_test.go b/tools/integration_tests/managed_folders/managed_folders_test.go index dddfb8d4a5..bef806c033 100644 --- a/tools/integration_tests/managed_folders/managed_folders_test.go +++ b/tools/integration_tests/managed_folders/managed_folders_test.go @@ -55,11 +55,12 @@ var ( func TestMain(m *testing.M) { setup.ParseSetUpFlags() - ctx := context.Background() + var err error + ctx = context.Background() var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Minute*15) - storageClient, err := client.CreateStorageClient(ctx) + storageClient, err = client.CreateStorageClient(ctx) if err != nil { log.Printf("Error creating storage client: %v\n", err) os.Exit(1) diff --git a/tools/integration_tests/read_cache/setup_test.go b/tools/integration_tests/read_cache/setup_test.go index fad8bccc2d..bb81e430bf 100644 --- a/tools/integration_tests/read_cache/setup_test.go +++ b/tools/integration_tests/read_cache/setup_test.go @@ -125,11 +125,14 @@ func createConfigFile(cacheSize int64, cacheFileForRangeRead bool, fileName stri //////////////////////////////////////////////////////////////////////// func TestMain(m *testing.M) { - ctx := context.Background() + setup.ParseSetUpFlags() + + var err error + ctx = context.Background() var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Minute*15) - storageClient, err := client.CreateStorageClient(ctx) + storageClient, err = client.CreateStorageClient(ctx) if err != nil { log.Printf("Error creating storage client: %v\n", err) os.Exit(1) @@ -138,8 +141,6 @@ func TestMain(m *testing.M) { defer cancel() defer storageClient.Close() - setup.ParseSetUpFlags() - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() setup.RunTestsForMountedDirectoryFlag(m) From 5fe00f40dc0d98697758dc2903f9eafafc549e34 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Tue, 28 May 2024 14:01:45 +0000 Subject: [PATCH 04/33] lint fix --- tools/integration_tests/interrupt/interrupt_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/integration_tests/interrupt/interrupt_test.go b/tools/integration_tests/interrupt/interrupt_test.go index b20270cdb7..67dc33d069 100644 --- a/tools/integration_tests/interrupt/interrupt_test.go +++ b/tools/integration_tests/interrupt/interrupt_test.go @@ -44,13 +44,13 @@ var ( func TestMain(m *testing.M) { setup.ParseSetUpFlags() - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() - ctx := context.Background() + var err error + ctx = context.Background() var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Minute*15) - storageClient, err := client.CreateStorageClient(ctx) + storageClient, err = client.CreateStorageClient(ctx) if err != nil { log.Printf("Error creating storage client: %v\n", err) os.Exit(1) @@ -58,7 +58,6 @@ func TestMain(m *testing.M) { defer cancel() defer storageClient.Close() - setup.RunTestsForMountedDirectoryFlag(m) // Else run tests for testBucket. From 278ae38bd887725e112e2268057af48b386d5522 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Thu, 30 May 2024 04:34:26 +0000 Subject: [PATCH 05/33] rebasing and trigger integration tests From e6c6465ff514c6b8a9e95339e9c6b1b9cff10ca1 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Thu, 30 May 2024 11:04:53 +0000 Subject: [PATCH 06/33] removing gcloud command in dynamic mounting --- .../managed_folders/managed_folders_test.go | 2 +- .../dynamic_mounting/dynamic_mounting.go | 42 ++++++++++++++----- .../testdata/create_bucket.sh | 29 ------------- .../testdata/delete_bucket.sh | 18 -------- 4 files changed, 32 insertions(+), 59 deletions(-) delete mode 100755 tools/integration_tests/util/mounting/dynamic_mounting/testdata/create_bucket.sh delete mode 100755 tools/integration_tests/util/mounting/dynamic_mounting/testdata/delete_bucket.sh diff --git a/tools/integration_tests/managed_folders/managed_folders_test.go b/tools/integration_tests/managed_folders/managed_folders_test.go index bef806c033..1384808351 100644 --- a/tools/integration_tests/managed_folders/managed_folders_test.go +++ b/tools/integration_tests/managed_folders/managed_folders_test.go @@ -59,7 +59,7 @@ func TestMain(m *testing.M) { ctx = context.Background() var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, time.Minute*15) + ctx, cancel = context.WithTimeout(ctx, time.Minute*20) storageClient, err = client.CreateStorageClient(ctx) if err != nil { log.Printf("Error creating storage client: %v\n", err) diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index e6c7c01d4f..38bbf6fb9c 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -15,12 +15,15 @@ package dynamic_mounting import ( + "context" "fmt" "log" "path" "testing" + "time" "cloud.google.com/go/compute/metadata" + "cloud.google.com/go/storage" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" @@ -98,33 +101,50 @@ func executeTestsForDynamicMounting(flags [][]string, m *testing.M) (successCode return } -func CreateTestBucketForDynamicMounting() (bucketName string) { - project_id, err := metadata.ProjectID() +func CreateTestBucketForDynamicMounting(ctx context.Context, client *storage.Client) (bucketName string) { + projectID, err := metadata.ProjectID() if err != nil { log.Printf("Error in fetching project id: %v", err) } - // Create bucket with name gcsfuse-dynamic-mounting-test-xxxxx - setup.RunScriptForTestData("../util/mounting/dynamic_mounting/testdata/create_bucket.sh", testBucketForDynamicMounting, project_id) - - return testBucketForDynamicMounting + // Create bucket handle and attributes + storageClassAndLocation := &storage.BucketAttrs{ + Location: "west", + } + bucket := client.Bucket(bucketName) + if err := bucket.Create(ctx, projectID, storageClassAndLocation); err != nil { + log.Fatalf("DynamicBucket(%q).Create: %w", bucketName, err) + } + return bucketName } -func DeleteTestBucketForDynamicMounting(bucketName string) { - // Deleting bucket after testing. - setup.RunScriptForTestData("../util/mounting/dynamic_mounting/testdata/delete_bucket.sh", bucketName) +func DeleteTestBucketForDynamicMounting(ctx context.Context, client *storage.Client, bucketName string) { + bucket := client.Bucket(bucketName) + if err := bucket.Delete(ctx); err != nil { + log.Printf("Bucket(%q).Delete: %w", bucketName, err) + } } func RunTests(flags [][]string, m *testing.M) (successCode int) { log.Println("Running dynamic mounting tests...") - CreateTestBucketForDynamicMounting() + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + log.Fatalf("storage.NewClient: %w", err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(ctx, time.Minute*15) + defer cancel() + + testBucketForDynamicMounting = CreateTestBucketForDynamicMounting(ctx, client) successCode = executeTestsForDynamicMounting(flags, m) log.Printf("Test log: %s\n", setup.LogFile()) - DeleteTestBucketForDynamicMounting(testBucketForDynamicMounting) + DeleteTestBucketForDynamicMounting(ctx, client, testBucketForDynamicMounting) return successCode } diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/testdata/create_bucket.sh b/tools/integration_tests/util/mounting/dynamic_mounting/testdata/create_bucket.sh deleted file mode 100755 index 7da59f770a..0000000000 --- a/tools/integration_tests/util/mounting/dynamic_mounting/testdata/create_bucket.sh +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2023 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Create bucket for testing. - -BUCKET_NAME=$1 -PROJECT_ID=$2 -gcloud alpha storage buckets create gs://$BUCKET_NAME --project=$PROJECT_ID --location=us-west1 --uniform-bucket-level-access 2> ~/output.txt -if [ $? -eq 1 ]; then - if grep "HTTPError 409" ~/output.txt; then - echo "Bucket already exist." - rm ~/output.txt - else - echo $(cat output.txt) - rm ~/output.txt - exit 1 - fi -fi -rm ~/output.txt diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/testdata/delete_bucket.sh b/tools/integration_tests/util/mounting/dynamic_mounting/testdata/delete_bucket.sh deleted file mode 100755 index 5aa0463383..0000000000 --- a/tools/integration_tests/util/mounting/dynamic_mounting/testdata/delete_bucket.sh +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Delete bucket after testing. - -BUCKET_NAME=$1 - -echo $(gcloud alpha storage rm --recursive gs://$BUCKET_NAME/) From 482376b5439bb1710643da34f2b69d5fe0ea753e Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Thu, 30 May 2024 11:07:56 +0000 Subject: [PATCH 07/33] small fix --- tools/integration_tests/read_cache/remount_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/integration_tests/read_cache/remount_test.go b/tools/integration_tests/read_cache/remount_test.go index 36016af8ac..676c6fe7d6 100644 --- a/tools/integration_tests/read_cache/remount_test.go +++ b/tools/integration_tests/read_cache/remount_test.go @@ -91,8 +91,8 @@ func (s *remountTest) TestCacheIsNotReusedOnDynamicRemount(t *testing.T) { runTestsOnlyForDynamicMount(t) testBucket1 := setup.TestBucket() testFileName1 := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) - testBucket2 := dynamic_mounting.CreateTestBucketForDynamicMounting() - defer dynamic_mounting.DeleteTestBucketForDynamicMounting(testBucket2) + testBucket2 := dynamic_mounting.CreateTestBucketForDynamicMounting(ctx, storageClient) + defer dynamic_mounting.DeleteTestBucketForDynamicMounting(ctx, storageClient, testBucket2) setup.SetDynamicBucketMounted(testBucket2) defer setup.SetDynamicBucketMounted("") // Introducing a sleep of 7 seconds after bucket creation to address propagation delays. From 644e56f964f13c644659444a05e0e119e358c96e Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Thu, 30 May 2024 11:09:46 +0000 Subject: [PATCH 08/33] lint fix --- .../util/mounting/dynamic_mounting/dynamic_mounting.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index 38bbf6fb9c..5392b17522 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -113,7 +113,7 @@ func CreateTestBucketForDynamicMounting(ctx context.Context, client *storage.Cli } bucket := client.Bucket(bucketName) if err := bucket.Create(ctx, projectID, storageClassAndLocation); err != nil { - log.Fatalf("DynamicBucket(%q).Create: %w", bucketName, err) + log.Fatalf("DynamicBucket(%q).Create: %v", bucketName, err) } return bucketName } @@ -121,7 +121,7 @@ func CreateTestBucketForDynamicMounting(ctx context.Context, client *storage.Cli func DeleteTestBucketForDynamicMounting(ctx context.Context, client *storage.Client, bucketName string) { bucket := client.Bucket(bucketName) if err := bucket.Delete(ctx); err != nil { - log.Printf("Bucket(%q).Delete: %w", bucketName, err) + log.Printf("Bucket(%q).Delete: %v", bucketName, err) } } @@ -131,7 +131,7 @@ func RunTests(flags [][]string, m *testing.M) (successCode int) { ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { - log.Fatalf("storage.NewClient: %w", err) + log.Fatalf("storage.NewClient: %v", err) } defer client.Close() From 5d2102e2371ea16b1697a45b1557205be0f0e421 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Thu, 30 May 2024 12:03:13 +0000 Subject: [PATCH 09/33] small fix --- perfmetrics/scripts/run_test_n2_standard_96_machine.sh | 0 .../util/mounting/dynamic_mounting/dynamic_mounting.go | 10 +++++----- 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 perfmetrics/scripts/run_test_n2_standard_96_machine.sh diff --git a/perfmetrics/scripts/run_test_n2_standard_96_machine.sh b/perfmetrics/scripts/run_test_n2_standard_96_machine.sh new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index 5392b17522..e7fd18afe8 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -101,7 +101,7 @@ func executeTestsForDynamicMounting(flags [][]string, m *testing.M) (successCode return } -func CreateTestBucketForDynamicMounting(ctx context.Context, client *storage.Client) (bucketName string) { +func CreateTestBucketForDynamicMounting(ctx context.Context, client *storage.Client) (testBucketForDynamicMounting string) { projectID, err := metadata.ProjectID() if err != nil { log.Printf("Error in fetching project id: %v", err) @@ -111,11 +111,11 @@ func CreateTestBucketForDynamicMounting(ctx context.Context, client *storage.Cli storageClassAndLocation := &storage.BucketAttrs{ Location: "west", } - bucket := client.Bucket(bucketName) + bucket := client.Bucket(testBucketForDynamicMounting) if err := bucket.Create(ctx, projectID, storageClassAndLocation); err != nil { - log.Fatalf("DynamicBucket(%q).Create: %v", bucketName, err) + log.Fatalf("DynamicBucket(%q).Create: %v", testBucketForDynamicMounting, err) } - return bucketName + return testBucketForDynamicMounting } func DeleteTestBucketForDynamicMounting(ctx context.Context, client *storage.Client, bucketName string) { @@ -138,7 +138,7 @@ func RunTests(flags [][]string, m *testing.M) (successCode int) { ctx, cancel := context.WithTimeout(ctx, time.Minute*15) defer cancel() - testBucketForDynamicMounting = CreateTestBucketForDynamicMounting(ctx, client) + CreateTestBucketForDynamicMounting(ctx, client) successCode = executeTestsForDynamicMounting(flags, m) From e5e27ae68a58f76f411fec0d8503568e5977a136 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Thu, 30 May 2024 12:03:34 +0000 Subject: [PATCH 10/33] small fix --- perfmetrics/scripts/run_test_n2_standard_96_machine.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 perfmetrics/scripts/run_test_n2_standard_96_machine.sh diff --git a/perfmetrics/scripts/run_test_n2_standard_96_machine.sh b/perfmetrics/scripts/run_test_n2_standard_96_machine.sh deleted file mode 100644 index e69de29bb2..0000000000 From 02fc741e303878f41e1e24e40762ede82141b8ce Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Fri, 31 May 2024 03:58:21 +0000 Subject: [PATCH 11/33] small changes --- .../operations/operations_test.go | 29 +++++++++---------- .../dynamic_mounting/dynamic_mounting.go | 5 ++-- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/tools/integration_tests/operations/operations_test.go b/tools/integration_tests/operations/operations_test.go index e9f70de2b8..cd8e152d1a 100644 --- a/tools/integration_tests/operations/operations_test.go +++ b/tools/integration_tests/operations/operations_test.go @@ -23,9 +23,6 @@ import ( "github.com/googlecloudplatform/gcsfuse/v2/internal/config" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/creds_tests" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/dynamic_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/only_dir_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/persistent_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" ) @@ -163,19 +160,19 @@ func TestMain(m *testing.M) { mountConfigFlags := createMountConfigsAndEquivalentFlags() flagsSet = append(flagsSet, mountConfigFlags...) - successCode := static_mounting.RunTests(flagsSet, m) - - if successCode == 0 { - successCode = only_dir_mounting.RunTests(flagsSet, onlyDirMounted, m) - } - - if successCode == 0 { - successCode = persistent_mounting.RunTests(flagsSet, m) - } - - if successCode == 0 { - successCode = dynamic_mounting.RunTests(flagsSet, m) - } + //successCode := static_mounting.RunTests(flagsSet, m) + // + //if successCode == 0 { + // successCode = only_dir_mounting.RunTests(flagsSet, onlyDirMounted, m) + //} + // + //if successCode == 0 { + // successCode = persistent_mounting.RunTests(flagsSet, m) + //} + + // if successCode == 0 { + successCode := dynamic_mounting.RunTests(flagsSet, m) + //} if successCode == 0 { // Test for admin permission on test bucket. diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index e7fd18afe8..b579438676 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -101,7 +101,7 @@ func executeTestsForDynamicMounting(flags [][]string, m *testing.M) (successCode return } -func CreateTestBucketForDynamicMounting(ctx context.Context, client *storage.Client) (testBucketForDynamicMounting string) { +func CreateTestBucketForDynamicMounting(ctx context.Context, client *storage.Client) (bucketName string) { projectID, err := metadata.ProjectID() if err != nil { log.Printf("Error in fetching project id: %v", err) @@ -109,8 +109,9 @@ func CreateTestBucketForDynamicMounting(ctx context.Context, client *storage.Cli // Create bucket handle and attributes storageClassAndLocation := &storage.BucketAttrs{ - Location: "west", + Location: "us-west1", } + bucket := client.Bucket(testBucketForDynamicMounting) if err := bucket.Create(ctx, projectID, storageClassAndLocation); err != nil { log.Fatalf("DynamicBucket(%q).Create: %v", testBucketForDynamicMounting, err) From 45194ad73be2fc5362b5e4c589d012fb4d93e639 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Fri, 31 May 2024 04:01:53 +0000 Subject: [PATCH 12/33] removing testing changes --- .../operations/operations_test.go | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tools/integration_tests/operations/operations_test.go b/tools/integration_tests/operations/operations_test.go index cd8e152d1a..e9f70de2b8 100644 --- a/tools/integration_tests/operations/operations_test.go +++ b/tools/integration_tests/operations/operations_test.go @@ -23,6 +23,9 @@ import ( "github.com/googlecloudplatform/gcsfuse/v2/internal/config" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/creds_tests" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/dynamic_mounting" + "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/persistent_mounting" + "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" ) @@ -160,19 +163,19 @@ func TestMain(m *testing.M) { mountConfigFlags := createMountConfigsAndEquivalentFlags() flagsSet = append(flagsSet, mountConfigFlags...) - //successCode := static_mounting.RunTests(flagsSet, m) - // - //if successCode == 0 { - // successCode = only_dir_mounting.RunTests(flagsSet, onlyDirMounted, m) - //} - // - //if successCode == 0 { - // successCode = persistent_mounting.RunTests(flagsSet, m) - //} - - // if successCode == 0 { - successCode := dynamic_mounting.RunTests(flagsSet, m) - //} + successCode := static_mounting.RunTests(flagsSet, m) + + if successCode == 0 { + successCode = only_dir_mounting.RunTests(flagsSet, onlyDirMounted, m) + } + + if successCode == 0 { + successCode = persistent_mounting.RunTests(flagsSet, m) + } + + if successCode == 0 { + successCode = dynamic_mounting.RunTests(flagsSet, m) + } if successCode == 0 { // Test for admin permission on test bucket. From 54af9e7ccdfbcb339b5a040d7aa14912c3065ed6 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Fri, 31 May 2024 04:42:36 +0000 Subject: [PATCH 13/33] increaing context timeout --- .../util/mounting/dynamic_mounting/dynamic_mounting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index b579438676..1717cad047 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -136,7 +136,7 @@ func RunTests(flags [][]string, m *testing.M) (successCode int) { } defer client.Close() - ctx, cancel := context.WithTimeout(ctx, time.Minute*15) + ctx, cancel := context.WithTimeout(ctx, time.Minute*25) defer cancel() CreateTestBucketForDynamicMounting(ctx, client) From 0abe9727e4ea316507cd9b3ee60e86cb238ce409 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Fri, 31 May 2024 09:19:23 +0000 Subject: [PATCH 14/33] deleting bucket data and then bucket --- .../dynamic_mounting/dynamic_mounting.go | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index 1717cad047..97c679e27a 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -121,6 +121,27 @@ func CreateTestBucketForDynamicMounting(ctx context.Context, client *storage.Cli func DeleteTestBucketForDynamicMounting(ctx context.Context, client *storage.Client, bucketName string) { bucket := client.Bucket(bucketName) + + // Iterate through objects and delete them + query := &storage.Query{} + it := bucket.Objects(ctx, query) + for { + objAttrs, err := it.Next() + if err == storage.ErrObjectNotExist { + break // No more objects + } + if err != nil { + log.Fatalf("Error iterating through objects: %v", err) + } + + obj := bucket.Object(objAttrs.Name) + err = obj.Delete(ctx) + if err != nil { + log.Fatalf("Failed to delete object %s: %v", objAttrs.Name, err) + } + fmt.Printf("Deleted object: %s\n", objAttrs.Name) + } + if err := bucket.Delete(ctx); err != nil { log.Printf("Bucket(%q).Delete: %v", bucketName, err) } From 423b12885d17959e54304943b81f280a77e20328 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Fri, 31 May 2024 12:08:45 +0000 Subject: [PATCH 15/33] small fix --- .../util/mounting/dynamic_mounting/dynamic_mounting.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index 97c679e27a..214bbb05fc 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -19,6 +19,7 @@ import ( "fmt" "log" "path" + "strings" "testing" "time" @@ -127,7 +128,7 @@ func DeleteTestBucketForDynamicMounting(ctx context.Context, client *storage.Cli it := bucket.Objects(ctx, query) for { objAttrs, err := it.Next() - if err == storage.ErrObjectNotExist { + if strings.Contains(err.Error(), "no more items in iterator") { break // No more objects } if err != nil { From 65ec503538d86200f02801440220d1794ee4f238 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 10 Jun 2024 08:18:48 +0000 Subject: [PATCH 16/33] rebasing and small fix --- .../util/mounting/dynamic_mounting/dynamic_mounting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index 214bbb05fc..43c7745970 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -128,7 +128,7 @@ func DeleteTestBucketForDynamicMounting(ctx context.Context, client *storage.Cli it := bucket.Objects(ctx, query) for { objAttrs, err := it.Next() - if strings.Contains(err.Error(), "no more items in iterator") { + if err != nil && strings.Contains(err.Error(), "no more items in iterator") { break // No more objects } if err != nil { From 15988c9042f826318be9460615ed958b0cad95b2 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 10 Jun 2024 08:22:38 +0000 Subject: [PATCH 17/33] rebasing and small fix --- .../util/mounting/dynamic_mounting/dynamic_mounting.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index 43c7745970..491ec7d75a 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -140,7 +140,6 @@ func DeleteTestBucketForDynamicMounting(ctx context.Context, client *storage.Cli if err != nil { log.Fatalf("Failed to delete object %s: %v", objAttrs.Name, err) } - fmt.Printf("Deleted object: %s\n", objAttrs.Name) } if err := bucket.Delete(ctx); err != nil { From 67dc72023a6e47d9391d06c2db28a4346dc01cdf Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 10 Jun 2024 08:32:44 +0000 Subject: [PATCH 18/33] testing crash --- tools/integration_tests/util/client/gcs_helper.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/integration_tests/util/client/gcs_helper.go b/tools/integration_tests/util/client/gcs_helper.go index c3825a8b5b..fbe911345e 100644 --- a/tools/integration_tests/util/client/gcs_helper.go +++ b/tools/integration_tests/util/client/gcs_helper.go @@ -157,6 +157,7 @@ func CreateNFilesInDir(ctx context.Context, storageClient *storage.Client, numFi func ValidateCRCWithGCS(gotCRC32Value uint32, objectPath string, ctx context.Context, storageClient *storage.Client, t *testing.T) { attr, err := StatObject(ctx, storageClient, objectPath) + log.Println("Object Path: ", objectPath) if err != nil { t.Errorf("Failed to fetch object attributes: %v", err) } From 6792f2b6d423d7517460f85dbc071ae8ea3ab243 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 10 Jun 2024 08:41:34 +0000 Subject: [PATCH 19/33] adding stat cache ttl 0 --- tools/integration_tests/read_cache/remount_test.go | 8 ++++---- tools/integration_tests/util/client/gcs_helper.go | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/integration_tests/read_cache/remount_test.go b/tools/integration_tests/read_cache/remount_test.go index 676c6fe7d6..231b867b04 100644 --- a/tools/integration_tests/read_cache/remount_test.go +++ b/tools/integration_tests/read_cache/remount_test.go @@ -132,11 +132,11 @@ func TestRemountTest(t *testing.T) { } // Define flag set to run the tests. flagsSet := [][]string{ - {"--implicit-dirs=true"}, - {"--implicit-dirs=false"}, + {"--implicit-dirs=true", "--stat-cache-ttl=0"}, + {"--implicit-dirs=false", "--stat-cache-ttl=0"}, } - setup.AppendFlagsToAllFlagsInTheFlagsSet(&flagsSet, "--config-file="+createConfigFile(cacheCapacityInMB, false, configFileName)) - setup.AppendFlagsToAllFlagsInTheFlagsSet(&flagsSet, "--o=ro", "") + setup.AppendFlagsToAllFlagsInTheFlagsSet(&flagsSet, "--config-file="+createConfigFile(cacheCapacityInMB, false, configFileName), "--stat-cache-ttl=0") + setup.AppendFlagsToAllFlagsInTheFlagsSet(&flagsSet, "--o=ro", "", "--stat-cache-ttl=0") // Create storage client before running tests. ts := &remountTest{ctx: context.Background()} diff --git a/tools/integration_tests/util/client/gcs_helper.go b/tools/integration_tests/util/client/gcs_helper.go index fbe911345e..c3825a8b5b 100644 --- a/tools/integration_tests/util/client/gcs_helper.go +++ b/tools/integration_tests/util/client/gcs_helper.go @@ -157,7 +157,6 @@ func CreateNFilesInDir(ctx context.Context, storageClient *storage.Client, numFi func ValidateCRCWithGCS(gotCRC32Value uint32, objectPath string, ctx context.Context, storageClient *storage.Client, t *testing.T) { attr, err := StatObject(ctx, storageClient, objectPath) - log.Println("Object Path: ", objectPath) if err != nil { t.Errorf("Failed to fetch object attributes: %v", err) } From b520d6c58e485d7140081bef174fb5ae6617c4f3 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 10 Jun 2024 08:46:03 +0000 Subject: [PATCH 20/33] increasing timeout --- tools/integration_tests/read_cache/remount_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/integration_tests/read_cache/remount_test.go b/tools/integration_tests/read_cache/remount_test.go index 231b867b04..8b69817be8 100644 --- a/tools/integration_tests/read_cache/remount_test.go +++ b/tools/integration_tests/read_cache/remount_test.go @@ -96,7 +96,7 @@ func (s *remountTest) TestCacheIsNotReusedOnDynamicRemount(t *testing.T) { setup.SetDynamicBucketMounted(testBucket2) defer setup.SetDynamicBucketMounted("") // Introducing a sleep of 7 seconds after bucket creation to address propagation delays. - time.Sleep(7 * time.Second) + time.Sleep(10 * time.Second) client.SetupTestDirectory(s.ctx, s.storageClient, testDirName) testFileName2 := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) @@ -132,11 +132,11 @@ func TestRemountTest(t *testing.T) { } // Define flag set to run the tests. flagsSet := [][]string{ - {"--implicit-dirs=true", "--stat-cache-ttl=0"}, - {"--implicit-dirs=false", "--stat-cache-ttl=0"}, + {"--implicit-dirs=true"}, + {"--implicit-dirs=false"}, } - setup.AppendFlagsToAllFlagsInTheFlagsSet(&flagsSet, "--config-file="+createConfigFile(cacheCapacityInMB, false, configFileName), "--stat-cache-ttl=0") - setup.AppendFlagsToAllFlagsInTheFlagsSet(&flagsSet, "--o=ro", "", "--stat-cache-ttl=0") + setup.AppendFlagsToAllFlagsInTheFlagsSet(&flagsSet, "--config-file="+createConfigFile(cacheCapacityInMB, false, configFileName)) + setup.AppendFlagsToAllFlagsInTheFlagsSet(&flagsSet, "--o=ro", "") // Create storage client before running tests. ts := &remountTest{ctx: context.Background()} From cfec0be1a85fe12ab7224a9c3ad1f864342efcab Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 10 Jun 2024 08:48:10 +0000 Subject: [PATCH 21/33] increasing timeout in comment --- tools/integration_tests/read_cache/remount_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/integration_tests/read_cache/remount_test.go b/tools/integration_tests/read_cache/remount_test.go index 8b69817be8..e65e746a6d 100644 --- a/tools/integration_tests/read_cache/remount_test.go +++ b/tools/integration_tests/read_cache/remount_test.go @@ -95,7 +95,7 @@ func (s *remountTest) TestCacheIsNotReusedOnDynamicRemount(t *testing.T) { defer dynamic_mounting.DeleteTestBucketForDynamicMounting(ctx, storageClient, testBucket2) setup.SetDynamicBucketMounted(testBucket2) defer setup.SetDynamicBucketMounted("") - // Introducing a sleep of 7 seconds after bucket creation to address propagation delays. + // Introducing a sleep of 10 seconds after bucket creation to address propagation delays. time.Sleep(10 * time.Second) client.SetupTestDirectory(s.ctx, s.storageClient, testDirName) testFileName2 := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) From 1f6b66a4c824471717f17070bb879fec52f3c0d6 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 10 Jun 2024 09:28:35 +0000 Subject: [PATCH 22/33] setting bucket and dir path using GetBucketAndObjectBasedOnTypeOfMount --- .../util/mounting/dynamic_mounting/dynamic_mounting.go | 4 ++-- tools/integration_tests/util/setup/setup.go | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index 491ec7d75a..b74546eb87 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -19,12 +19,12 @@ import ( "fmt" "log" "path" - "strings" "testing" "time" "cloud.google.com/go/compute/metadata" "cloud.google.com/go/storage" + "google.golang.org/api/iterator" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" @@ -128,7 +128,7 @@ func DeleteTestBucketForDynamicMounting(ctx context.Context, client *storage.Cli it := bucket.Objects(ctx, query) for { objAttrs, err := it.Next() - if err != nil && strings.Contains(err.Error(), "no more items in iterator") { + if err == iterator.Done { break // No more objects } if err != nil { diff --git a/tools/integration_tests/util/setup/setup.go b/tools/integration_tests/util/setup/setup.go index d400f4a622..97905ad09d 100644 --- a/tools/integration_tests/util/setup/setup.go +++ b/tools/integration_tests/util/setup/setup.go @@ -378,8 +378,10 @@ func CleanupDirectoryOnGCS(ctx context.Context, client *storage.Client, director directoryPathOnGCS = directoryPathOnGCS[:len(directoryPathOnGCS)-1] } - bucket := client.Bucket(TestBucket()) - it := bucket.Objects(ctx, &storage.Query{Prefix: directoryPathOnGCS + "/"}) + bucket, dirPath := GetBucketAndObjectBasedOnTypeOfMount("") + bucketHandle := client.Bucket(bucket) + + it := bucketHandle.Objects(ctx, &storage.Query{Prefix: dirPath + "/"}) for { attrs, err := it.Next() if err == iterator.Done { @@ -388,7 +390,7 @@ func CleanupDirectoryOnGCS(ctx context.Context, client *storage.Client, director if err != nil { log.Printf("error iterating objects: %v", err) } - if err := bucket.Object(attrs.Name).Delete(ctx); err != nil { + if err := bucketHandle.Object(attrs.Name).Delete(ctx); err != nil { log.Printf("error deleting object %s: %v", attrs.Name, err) } } From d60e53fa66d5d633fdf45c0c011a15c0018defb0 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 10 Jun 2024 10:03:34 +0000 Subject: [PATCH 23/33] increase timeout of client --- tools/integration_tests/local_file/local_file_test.go | 2 +- tools/integration_tests/readonly_creds/readonly_creds_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/integration_tests/local_file/local_file_test.go b/tools/integration_tests/local_file/local_file_test.go index 40f8668c45..57136c44e4 100644 --- a/tools/integration_tests/local_file/local_file_test.go +++ b/tools/integration_tests/local_file/local_file_test.go @@ -81,7 +81,7 @@ func TestMain(m *testing.M) { setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() // Create storage client before running tests. - ctx, cancel = context.WithTimeout(ctx, time.Minute*15) + ctx, cancel = context.WithTimeout(ctx, time.Minute*20) storageClient, err = client.CreateStorageClient(ctx) if err != nil { log.Fatalf("client.CreateStorageClient: %v", err) diff --git a/tools/integration_tests/readonly_creds/readonly_creds_test.go b/tools/integration_tests/readonly_creds/readonly_creds_test.go index 602b9683d2..706ebeca41 100644 --- a/tools/integration_tests/readonly_creds/readonly_creds_test.go +++ b/tools/integration_tests/readonly_creds/readonly_creds_test.go @@ -51,7 +51,7 @@ func TestMain(m *testing.M) { // Create test directory. ctx := context.Background() var storageClient *storage.Client - closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*2) + closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15) defer func() { err := closeStorageClient() if err != nil { From 226a51e46c4e182419a6db309293f96419e0946b Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 10 Jun 2024 10:12:21 +0000 Subject: [PATCH 24/33] lint fix --- tools/integration_tests/util/setup/setup.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/integration_tests/util/setup/setup.go b/tools/integration_tests/util/setup/setup.go index 97905ad09d..0a08472835 100644 --- a/tools/integration_tests/util/setup/setup.go +++ b/tools/integration_tests/util/setup/setup.go @@ -374,10 +374,6 @@ func SetupTestDirectory(testDirName string) string { // CleanupDirectoryOnGCS cleans up the object/directory path passed in parameter. func CleanupDirectoryOnGCS(ctx context.Context, client *storage.Client, directoryPathOnGCS string) { - if directoryPathOnGCS[len(directoryPathOnGCS)-1] == '/' { - directoryPathOnGCS = directoryPathOnGCS[:len(directoryPathOnGCS)-1] - } - bucket, dirPath := GetBucketAndObjectBasedOnTypeOfMount("") bucketHandle := client.Bucket(bucket) From 474830e90e1fc603acda5f5f161efdd63537ac9e Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 10 Jun 2024 10:14:18 +0000 Subject: [PATCH 25/33] lint fix --- tools/integration_tests/util/setup/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/integration_tests/util/setup/setup.go b/tools/integration_tests/util/setup/setup.go index 0a08472835..c9d8b78b1c 100644 --- a/tools/integration_tests/util/setup/setup.go +++ b/tools/integration_tests/util/setup/setup.go @@ -374,7 +374,7 @@ func SetupTestDirectory(testDirName string) string { // CleanupDirectoryOnGCS cleans up the object/directory path passed in parameter. func CleanupDirectoryOnGCS(ctx context.Context, client *storage.Client, directoryPathOnGCS string) { - bucket, dirPath := GetBucketAndObjectBasedOnTypeOfMount("") + bucket, dirPath := GetBucketAndObjectBasedOnTypeOfMount(directoryPathOnGCS) bucketHandle := client.Bucket(bucket) it := bucketHandle.Objects(ctx, &storage.Query{Prefix: dirPath + "/"}) From 5f6c0c4d62f63f83ee88bcc46b58924352a15088 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 10 Jun 2024 10:49:12 +0000 Subject: [PATCH 26/33] fix cancel client --- tools/integration_tests/local_file/local_file_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/integration_tests/local_file/local_file_test.go b/tools/integration_tests/local_file/local_file_test.go index 57136c44e4..7b959e4aea 100644 --- a/tools/integration_tests/local_file/local_file_test.go +++ b/tools/integration_tests/local_file/local_file_test.go @@ -81,12 +81,15 @@ func TestMain(m *testing.M) { setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() // Create storage client before running tests. - ctx, cancel = context.WithTimeout(ctx, time.Minute*20) + ctx, cancel = context.WithTimeout(ctx, time.Minute*15) storageClient, err = client.CreateStorageClient(ctx) if err != nil { log.Fatalf("client.CreateStorageClient: %v", err) } + // Close storage client and release resources. + defer cancel() + defer storageClient.Close() // To run mountedDirectory tests, we need both testBucket and mountedDirectory // flags to be set, as local_file tests validates content from the bucket. if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { @@ -117,9 +120,6 @@ func TestMain(m *testing.M) { successCode = dynamic_mounting.RunTests(flagsSet, m) } - // Close storage client and release resources. - storageClient.Close() - cancel() // Clean up test directory created. setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) os.Exit(successCode) From 59e16e267d44217bff08347848bc41b589907f1b Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 10 Jun 2024 11:32:31 +0000 Subject: [PATCH 27/33] small fix --- tools/integration_tests/util/setup/setup.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/integration_tests/util/setup/setup.go b/tools/integration_tests/util/setup/setup.go index c9d8b78b1c..d285053a18 100644 --- a/tools/integration_tests/util/setup/setup.go +++ b/tools/integration_tests/util/setup/setup.go @@ -384,10 +384,10 @@ func CleanupDirectoryOnGCS(ctx context.Context, client *storage.Client, director break // No more objects found } if err != nil { - log.Printf("error iterating objects: %v", err) + log.Fatalf("Error iterating objects: %v", err) } if err := bucketHandle.Object(attrs.Name).Delete(ctx); err != nil { - log.Printf("error deleting object %s: %v", attrs.Name, err) + log.Printf("Error deleting object %s: %v", attrs.Name, err) } } } From b4f3c953e86298d9245428db335f90ecb5bc9e19 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Tue, 11 Jun 2024 11:28:46 +0000 Subject: [PATCH 28/33] review comment --- .../local_file/local_file_test.go | 2 +- .../operations/operations_test.go | 26 ++++++++++++++++++- .../dynamic_mounting/dynamic_mounting.go | 13 +--------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/tools/integration_tests/local_file/local_file_test.go b/tools/integration_tests/local_file/local_file_test.go index 7b959e4aea..cc2aaa6e6e 100644 --- a/tools/integration_tests/local_file/local_file_test.go +++ b/tools/integration_tests/local_file/local_file_test.go @@ -117,7 +117,7 @@ func TestMain(m *testing.M) { } if successCode == 0 { - successCode = dynamic_mounting.RunTests(flagsSet, m) + successCode = dynamic_mounting.RunTests(flagsSet, ctx, storageClient, m) } // Clean up test directory created. diff --git a/tools/integration_tests/operations/operations_test.go b/tools/integration_tests/operations/operations_test.go index e9f70de2b8..018a31a30c 100644 --- a/tools/integration_tests/operations/operations_test.go +++ b/tools/integration_tests/operations/operations_test.go @@ -16,11 +16,16 @@ package operations_test import ( + "context" + "log" "os" "path" "testing" + "time" + "cloud.google.com/go/storage" "github.com/googlecloudplatform/gcsfuse/v2/internal/config" + "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/creds_tests" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/dynamic_mounting" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/only_dir_mounting" @@ -87,6 +92,11 @@ const ContentInFileInDirThreeInCreateThreeLevelDirTest = "Hello world!!" const Content = "line 1\nline 2\n" const onlyDirMounted = "OnlyDirMountOperations" +var ( + storageClient *storage.Client + ctx context.Context +) + func createMountConfigsAndEquivalentFlags() (flags [][]string) { cacheDirPath := path.Join(os.Getenv("HOME"), "operations-cache-dir") @@ -137,9 +147,23 @@ func createMountConfigsAndEquivalentFlags() (flags [][]string) { func TestMain(m *testing.M) { setup.ParseSetUpFlags() + ctx = context.Background() + var cancel context.CancelFunc + var err error setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() + // Create storage client before running tests. + ctx, cancel = context.WithTimeout(ctx, time.Minute*15) + storageClient, err = client.CreateStorageClient(ctx) + if err != nil { + log.Fatalf("client.CreateStorageClient: %v", err) + } + + // Close storage client and release resources. + defer cancel() + defer storageClient.Close() + // To run mountedDirectory tests, we need both testBucket and mountedDirectory // flags to be set, as operations tests validates content from the bucket. if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { @@ -174,7 +198,7 @@ func TestMain(m *testing.M) { } if successCode == 0 { - successCode = dynamic_mounting.RunTests(flagsSet, m) + successCode = dynamic_mounting.RunTests(flagsSet, ctx, storageClient, m) } if successCode == 0 { diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index b74546eb87..d042a17326 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -20,7 +20,6 @@ import ( "log" "path" "testing" - "time" "cloud.google.com/go/compute/metadata" "cloud.google.com/go/storage" @@ -147,19 +146,9 @@ func DeleteTestBucketForDynamicMounting(ctx context.Context, client *storage.Cli } } -func RunTests(flags [][]string, m *testing.M) (successCode int) { +func RunTests(flags [][]string, ctx context.Context, client *storage.Client, m *testing.M) (successCode int) { log.Println("Running dynamic mounting tests...") - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - log.Fatalf("storage.NewClient: %v", err) - } - defer client.Close() - - ctx, cancel := context.WithTimeout(ctx, time.Minute*25) - defer cancel() - CreateTestBucketForDynamicMounting(ctx, client) successCode = executeTestsForDynamicMounting(flags, m) From 4a9297093cd76ef06faea943eabe7583b1813d0d Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Wed, 12 Jun 2024 06:50:36 +0000 Subject: [PATCH 29/33] review comments --- .../implicit_dir/implicit_dir_test.go | 21 +++++--------- .../interrupt/interrupt_test.go | 20 +++++-------- .../local_file/local_file_test.go | 22 ++++++-------- .../log_rotation/log_rotation_test.go | 21 +++++--------- .../managed_folders/managed_folders_test.go | 19 +++++------- .../operations/operations_test.go | 29 +++++++------------ .../read_cache/setup_test.go | 19 +++++------- .../dynamic_mounting/dynamic_mounting.go | 2 +- 8 files changed, 58 insertions(+), 95 deletions(-) diff --git a/tools/integration_tests/implicit_dir/implicit_dir_test.go b/tools/integration_tests/implicit_dir/implicit_dir_test.go index 15a0939cb6..b0624e30c9 100644 --- a/tools/integration_tests/implicit_dir/implicit_dir_test.go +++ b/tools/integration_tests/implicit_dir/implicit_dir_test.go @@ -45,20 +45,15 @@ var ( func TestMain(m *testing.M) { setup.ParseSetUpFlags() - ctx = context.Background() - var cancel context.CancelFunc - var err error - // Create storage client before running tests. - ctx, cancel = context.WithTimeout(ctx, time.Minute*15) - storageClient, err = client.CreateStorageClient(ctx) - if err != nil { - log.Fatalf("client.CreateStorageClient: %v", err) - } - - // Close storage client and release resources. - defer cancel() - defer storageClient.Close() + ctx = context.Background() + closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() flagsSet := [][]string{{"--implicit-dirs"}} diff --git a/tools/integration_tests/interrupt/interrupt_test.go b/tools/integration_tests/interrupt/interrupt_test.go index 67dc33d069..5ec6bef4ed 100644 --- a/tools/integration_tests/interrupt/interrupt_test.go +++ b/tools/integration_tests/interrupt/interrupt_test.go @@ -45,19 +45,15 @@ var ( func TestMain(m *testing.M) { setup.ParseSetUpFlags() - var err error ctx = context.Background() - var cancel context.CancelFunc - - ctx, cancel = context.WithTimeout(ctx, time.Minute*15) - storageClient, err = client.CreateStorageClient(ctx) - if err != nil { - log.Printf("Error creating storage client: %v\n", err) - os.Exit(1) - } - - defer cancel() - defer storageClient.Close() + closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() + setup.RunTestsForMountedDirectoryFlag(m) // Else run tests for testBucket. diff --git a/tools/integration_tests/local_file/local_file_test.go b/tools/integration_tests/local_file/local_file_test.go index cc2aaa6e6e..d0499c6e7d 100644 --- a/tools/integration_tests/local_file/local_file_test.go +++ b/tools/integration_tests/local_file/local_file_test.go @@ -75,21 +75,17 @@ func TestMain(m *testing.M) { setup.ParseSetUpFlags() ctx = context.Background() - var cancel context.CancelFunc - var err error - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() // Create storage client before running tests. - ctx, cancel = context.WithTimeout(ctx, time.Minute*15) - storageClient, err = client.CreateStorageClient(ctx) - if err != nil { - log.Fatalf("client.CreateStorageClient: %v", err) - } - - // Close storage client and release resources. - defer cancel() - defer storageClient.Close() + ctx = context.Background() + closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() // To run mountedDirectory tests, we need both testBucket and mountedDirectory // flags to be set, as local_file tests validates content from the bucket. if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { @@ -117,7 +113,7 @@ func TestMain(m *testing.M) { } if successCode == 0 { - successCode = dynamic_mounting.RunTests(flagsSet, ctx, storageClient, m) + successCode = dynamic_mounting.RunTests(ctx, storageClient, flagsSet, m) } // Clean up test directory created. diff --git a/tools/integration_tests/log_rotation/log_rotation_test.go b/tools/integration_tests/log_rotation/log_rotation_test.go index 70c4973d90..95d6b9e5c3 100644 --- a/tools/integration_tests/log_rotation/log_rotation_test.go +++ b/tools/integration_tests/log_rotation/log_rotation_test.go @@ -71,20 +71,15 @@ func getMountConfigForLogRotation(maxFileSizeMB, backupFileCount int, compress b func TestMain(m *testing.M) { setup.ParseSetUpFlags() - var err error ctx = context.Background() - var cancel context.CancelFunc - - ctx, cancel = context.WithTimeout(ctx, time.Minute*15) - storageClient, err = client.CreateStorageClient(ctx) - if err != nil { - log.Printf("Error creating storage client: %v\n", err) - os.Exit(1) - } - - defer cancel() - defer storageClient.Close() - + closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() + setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() // Run tests for mountedDirectory only if --mountedDirectory flag is set. diff --git a/tools/integration_tests/managed_folders/managed_folders_test.go b/tools/integration_tests/managed_folders/managed_folders_test.go index 1384808351..0f84f632a6 100644 --- a/tools/integration_tests/managed_folders/managed_folders_test.go +++ b/tools/integration_tests/managed_folders/managed_folders_test.go @@ -55,19 +55,14 @@ var ( func TestMain(m *testing.M) { setup.ParseSetUpFlags() - var err error ctx = context.Background() - var cancel context.CancelFunc - - ctx, cancel = context.WithTimeout(ctx, time.Minute*20) - storageClient, err = client.CreateStorageClient(ctx) - if err != nil { - log.Printf("Error creating storage client: %v\n", err) - os.Exit(1) - } - - defer cancel() - defer storageClient.Close() + closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() diff --git a/tools/integration_tests/operations/operations_test.go b/tools/integration_tests/operations/operations_test.go index 018a31a30c..ba91d3ec42 100644 --- a/tools/integration_tests/operations/operations_test.go +++ b/tools/integration_tests/operations/operations_test.go @@ -92,11 +92,6 @@ const ContentInFileInDirThreeInCreateThreeLevelDirTest = "Hello world!!" const Content = "line 1\nline 2\n" const onlyDirMounted = "OnlyDirMountOperations" -var ( - storageClient *storage.Client - ctx context.Context -) - func createMountConfigsAndEquivalentFlags() (flags [][]string) { cacheDirPath := path.Join(os.Getenv("HOME"), "operations-cache-dir") @@ -147,23 +142,19 @@ func createMountConfigsAndEquivalentFlags() (flags [][]string) { func TestMain(m *testing.M) { setup.ParseSetUpFlags() - ctx = context.Background() - var cancel context.CancelFunc - var err error setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() // Create storage client before running tests. - ctx, cancel = context.WithTimeout(ctx, time.Minute*15) - storageClient, err = client.CreateStorageClient(ctx) - if err != nil { - log.Fatalf("client.CreateStorageClient: %v", err) - } - - // Close storage client and release resources. - defer cancel() - defer storageClient.Close() - + ctx := context.Background() + var storageClient *storage.Client + closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() // To run mountedDirectory tests, we need both testBucket and mountedDirectory // flags to be set, as operations tests validates content from the bucket. if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { @@ -198,7 +189,7 @@ func TestMain(m *testing.M) { } if successCode == 0 { - successCode = dynamic_mounting.RunTests(flagsSet, ctx, storageClient, m) + successCode = dynamic_mounting.RunTests(ctx, storageClient, flagsSet, m) } if successCode == 0 { diff --git a/tools/integration_tests/read_cache/setup_test.go b/tools/integration_tests/read_cache/setup_test.go index bb81e430bf..a4328f6a1b 100644 --- a/tools/integration_tests/read_cache/setup_test.go +++ b/tools/integration_tests/read_cache/setup_test.go @@ -127,19 +127,14 @@ func createConfigFile(cacheSize int64, cacheFileForRangeRead bool, fileName stri func TestMain(m *testing.M) { setup.ParseSetUpFlags() - var err error ctx = context.Background() - var cancel context.CancelFunc - - ctx, cancel = context.WithTimeout(ctx, time.Minute*15) - storageClient, err = client.CreateStorageClient(ctx) - if err != nil { - log.Printf("Error creating storage client: %v\n", err) - os.Exit(1) - } - - defer cancel() - defer storageClient.Close() + closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index d042a17326..897bfe1f60 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -146,7 +146,7 @@ func DeleteTestBucketForDynamicMounting(ctx context.Context, client *storage.Cli } } -func RunTests(flags [][]string, ctx context.Context, client *storage.Client, m *testing.M) (successCode int) { +func RunTests(ctx context.Context, client *storage.Client, flags [][]string, m *testing.M) (successCode int) { log.Println("Running dynamic mounting tests...") CreateTestBucketForDynamicMounting(ctx, client) From 8499e04ca22adb095bd65460f4640f81d8dfe123 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Wed, 12 Jun 2024 06:54:06 +0000 Subject: [PATCH 30/33] lint tests --- tools/integration_tests/interrupt/interrupt_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/integration_tests/interrupt/interrupt_test.go b/tools/integration_tests/interrupt/interrupt_test.go index 5ec6bef4ed..e1936ee441 100644 --- a/tools/integration_tests/interrupt/interrupt_test.go +++ b/tools/integration_tests/interrupt/interrupt_test.go @@ -53,7 +53,7 @@ func TestMain(m *testing.M) { log.Fatalf("closeStorageClient failed: %v", err) } }() - + setup.RunTestsForMountedDirectoryFlag(m) // Else run tests for testBucket. From 9acc4eff4ce0e7ea0b0665989e7bfb89624b1702 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Wed, 12 Jun 2024 06:54:22 +0000 Subject: [PATCH 31/33] lint tests --- tools/integration_tests/log_rotation/log_rotation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/integration_tests/log_rotation/log_rotation_test.go b/tools/integration_tests/log_rotation/log_rotation_test.go index 95d6b9e5c3..7487c3db10 100644 --- a/tools/integration_tests/log_rotation/log_rotation_test.go +++ b/tools/integration_tests/log_rotation/log_rotation_test.go @@ -79,7 +79,7 @@ func TestMain(m *testing.M) { log.Fatalf("closeStorageClient failed: %v", err) } }() - + setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() // Run tests for mountedDirectory only if --mountedDirectory flag is set. From 04671bcea0863fc95aaf437e931e07623fd64c71 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Wed, 12 Jun 2024 08:08:14 +0000 Subject: [PATCH 32/33] increasing timeout --- tools/integration_tests/managed_folders/managed_folders_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/integration_tests/managed_folders/managed_folders_test.go b/tools/integration_tests/managed_folders/managed_folders_test.go index 0f84f632a6..7e29edebcf 100644 --- a/tools/integration_tests/managed_folders/managed_folders_test.go +++ b/tools/integration_tests/managed_folders/managed_folders_test.go @@ -56,7 +56,7 @@ func TestMain(m *testing.M) { setup.ParseSetUpFlags() ctx = context.Background() - closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15) + closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*25) defer func() { err := closeStorageClient() if err != nil { From 4e2601b604269c6d0f2ef225a9b1ad1c27597775 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Thu, 13 Jun 2024 05:31:06 +0000 Subject: [PATCH 33/33] review comment --- tools/integration_tests/interrupt/interrupt_test.go | 8 ++------ tools/integration_tests/local_file/local_file_test.go | 1 - .../integration_tests/log_rotation/log_rotation_test.go | 9 ++++----- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/tools/integration_tests/interrupt/interrupt_test.go b/tools/integration_tests/interrupt/interrupt_test.go index e1936ee441..16fd831997 100644 --- a/tools/integration_tests/interrupt/interrupt_test.go +++ b/tools/integration_tests/interrupt/interrupt_test.go @@ -33,11 +33,6 @@ const ( testDirName = "InterruptTest" ) -var ( - storageClient *storage.Client - ctx context.Context -) - //////////////////////////////////////////////////////////////////////// // TestMain //////////////////////////////////////////////////////////////////////// @@ -45,7 +40,8 @@ var ( func TestMain(m *testing.M) { setup.ParseSetUpFlags() - ctx = context.Background() + var storageClient *storage.Client + ctx := context.Background() closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15) defer func() { err := closeStorageClient() diff --git a/tools/integration_tests/local_file/local_file_test.go b/tools/integration_tests/local_file/local_file_test.go index d0499c6e7d..946eb8314d 100644 --- a/tools/integration_tests/local_file/local_file_test.go +++ b/tools/integration_tests/local_file/local_file_test.go @@ -74,7 +74,6 @@ func NewFileShouldGetSyncedToGCSAtClose(ctx context.Context, storageClient *stor func TestMain(m *testing.M) { setup.ParseSetUpFlags() - ctx = context.Background() setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() // Create storage client before running tests. diff --git a/tools/integration_tests/log_rotation/log_rotation_test.go b/tools/integration_tests/log_rotation/log_rotation_test.go index 7487c3db10..9bd70eed6e 100644 --- a/tools/integration_tests/log_rotation/log_rotation_test.go +++ b/tools/integration_tests/log_rotation/log_rotation_test.go @@ -42,10 +42,8 @@ const ( ) var ( - logDirPath string - logFilePath string - storageClient *storage.Client - ctx context.Context + logDirPath string + logFilePath string ) func getMountConfigForLogRotation(maxFileSizeMB, backupFileCount int, compress bool, @@ -71,7 +69,8 @@ func getMountConfigForLogRotation(maxFileSizeMB, backupFileCount int, compress b func TestMain(m *testing.M) { setup.ParseSetUpFlags() - ctx = context.Background() + var storageClient *storage.Client + ctx := context.Background() closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15) defer func() { err := closeStorageClient()