moving from gsutil to gcloud in e2e tests

tools/integration_tests/gzip/gzip_test.go

@@ -61,7 +61,7 @@ func setup_testdata(m *testing.M) error {
 		filesize                    int
 		keepCacheControlNoTransform bool // if true, no-transform is reset as ''
 		enableGzipEncodedContent    bool // if true, original file content is gzip-encoded
-		enableGzipContentEncoding   bool // if true, the content is uploaded as gsutil cp -Z i.e. with content-encoding: gzip header in GCS
+		enableGzipContentEncoding   bool // if true, the content is uploaded as gcloud storage cp -Z i.e. with content-encoding: gzip header in GCS
 	}{
 		{
 			filename:                    TextContentWithContentEncodingWithNoTransformFilename,

tools/integration_tests/list_large_dir/testdata/create_implicit_dir.sh

@@ -27,5 +27,5 @@ while [ $a -le $NUMBER_OF_FILES ]
 do
    dir=$IMPLICIT_DIR$a
    a=`expr $a + 1`
-   gsutil cp testFile.txt gs://$TEST_BUCKET/$dir/
+   gcloud storage cp testFile.txt gs://$TEST_BUCKET/$dir/
 done

tools/integration_tests/list_large_dir/testdata/delete_objects.sh

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Here $1 refers to the testBucket argument
-gsutil -m rm -a gs://$1/**
+gcloud storage rm -r gs://$1/**
 
 # If bucket is empty it will throw an CommandException.
 if [ $? -eq 1 ]; then

tools/integration_tests/list_large_dir/testdata/upload_files_to_bucket.sh

@@ -19,6 +19,6 @@ DIR_WITH_TWELVE_THOUSAND_FILES=$2
 FILES=$3
 
 cd ~/$DIR_WITH_TWELVE_THOUSAND_FILES
-gsutil -m mv $FILES* gs://$TEST_BUCKET/
+gcloud storage mv $FILES* gs://$TEST_BUCKET/
 cd ../
 rm -r ~/$DIR_WITH_TWELVE_THOUSAND_FILES

tools/integration_tests/managed_folders/admin_permissions_test.go

@@ -51,17 +51,21 @@ var (
 // levels apply additively (union) throughout the resource hierarchy.
 // Hence here managed folder will have admin permission throughout all the tests.
 type managedFoldersAdminPermission struct {
-	bucketPermission        string
+	bucketPermission string
 }
 
 func (s *managedFoldersAdminPermission) Setup(t *testing.T) {
 	createDirectoryStructureForNonEmptyManagedFolders(t)
 }
 
 func (s *managedFoldersAdminPermission) Teardown(t *testing.T) {
-	// The 'gsutil rm -rf' command doesn't work on managed folders.
-	// We'll clean up the test directory but leave managed folders.
-	setup.CleanupDirectoryOnGCS(path.Join(bucket, testDir))
+	// Due to bucket view permissions, it prevents cleaning resources outside of managed folders. So we are cleaning managed folders resources only.
+	if s.bucketPermission == ViewPermission {
+		setup.CleanUpDir(path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1))
+		setup.CleanUpDir(path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder2))
+		return
+	}
+	setup.CleanUpDir(path.Join(setup.MntDir(), TestDirForManagedFolderTest))
 }
 
 func (s *managedFoldersAdminPermission) TestCreateObjectInManagedFolder(t *testing.T) {
@@ -187,9 +191,7 @@ func TestManagedFolders_FolderAdminPermission(t *testing.T) {
 
 	// Fetch credentials and apply permission on bucket.
 	serviceAccount, localKeyFilePath = creds_tests.CreateCredentials()
-	creds_tests.ApplyPermissionToServiceAccount(serviceAccount, AdminPermission)
-	// Revoke permission on bucket.
-	defer creds_tests.RevokePermission(serviceAccount, AdminPermission, setup.TestBucket())
+	creds_tests.ApplyPermissionToServiceAccount(serviceAccount, AdminPermission, setup.TestBucket())
 
 	flags := []string{"--implicit-dirs", "--key-file=" + localKeyFilePath, "--rename-dir-limit=5", "--stat-cache-ttl=0"}
 
@@ -206,7 +208,7 @@ func TestManagedFolders_FolderAdminPermission(t *testing.T) {
 		ts.bucketPermission = permissions[i][0]
 		if ts.bucketPermission == ViewPermission {
 			creds_tests.RevokePermission(serviceAccount, AdminPermission, setup.TestBucket())
-			creds_tests.ApplyPermissionToServiceAccount(serviceAccount, ViewPermission)
+			creds_tests.ApplyPermissionToServiceAccount(serviceAccount, ViewPermission, setup.TestBucket())
 			defer creds_tests.RevokePermission(serviceAccount, ViewPermission, setup.TestBucket())
 		}
 		managedFolderPermission := permissions[i][1]

tools/integration_tests/managed_folders/view_permissions_test.go

@@ -137,7 +137,7 @@ func TestManagedFolders_FolderViewPermission(t *testing.T) {
 
 	// Fetch credentials and apply permission on bucket.
 	serviceAccount, localKeyFilePath := creds_tests.CreateCredentials()
-	creds_tests.ApplyPermissionToServiceAccount(serviceAccount, ViewPermission)
+	creds_tests.ApplyPermissionToServiceAccount(serviceAccount, ViewPermission, setup.TestBucket())
 	defer creds_tests.RevokePermission(serviceAccount, ViewPermission, setup.TestBucket())
 
 	flags := []string{"--implicit-dirs", "--key-file=" + localKeyFilePath, "--rename-dir-limit=3"}

tools/integration_tests/readonly/testdata/create_objects.sh

@@ -14,8 +14,8 @@
 
 # Here $1 refers to the testBucket argument
 echo "This is from directory Test file a" >> a.txt
-gsutil mv a.txt gs://$1/Test/
+gcloud storage mv a.txt gs://$1/Test/
 echo "This is from file Test1" >> Test1.txt
-gsutil mv Test1.txt gs://$1/
+gcloud storage mv Test1.txt gs://$1/
 echo "This is from directory Test/b file b" >> b.txt
-gsutil mv b.txt gs://$1/Test/b/
+gcloud storage mv b.txt gs://$1/Test/b/

tools/integration_tests/readonly/testdata/delete_objects.sh

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Here $1 refers to the testBucket argument
-gsutil rm -a gs://$1/**
+gcloud storage rm -r gs://$1/**
 
 # If bucket is empty it will throw an CommandException.
 if [ $? -eq 1 ]; then

tools/integration_tests/util/creds_tests/creds.go

@@ -77,9 +77,9 @@ func CreateCredentials() (serviceAccount, localKeyFilePath string) {
 	return
 }
 
-func ApplyPermissionToServiceAccount(serviceAccount, permission string) {
+func ApplyPermissionToServiceAccount(serviceAccount, permission, bucket string) {
 	// Provide permission to service account for testing.
-	_, err := operations.ExecuteGsutilCommandf(fmt.Sprintf("iam ch serviceAccount:%s:%s gs://%s", serviceAccount, permission, setup.TestBucket()))
+	_, err := operations.ExecuteGcloudCommandf(fmt.Sprintf("storage buckets add-iam-policy-binding gs://%s --member=serviceAccount:%s --role=roles/storage.%s", bucket, serviceAccount, permission))
 	if err != nil {
 		setup.LogAndExit(fmt.Sprintf("Error while setting permissions to SA: %v", err))
 	}
@@ -89,17 +89,17 @@ func ApplyPermissionToServiceAccount(serviceAccount, permission string) {
 }
 
 func RevokePermission(serviceAccount, permission, bucket string) {
-	cmd := fmt.Sprintf("iam ch -d serviceAccount:%s:%s gs://%s", serviceAccount, permission, bucket)
-	// Revoke the permission after testing.
-	_, err := operations.ExecuteGsutilCommandf(cmd)
+	// Revoke the permission to service account after testing.
+	cmd := fmt.Sprintf("storage buckets remove-iam-policy-binding gs://%s --member=serviceAccount:%s --role=roles/storage.%s", bucket, serviceAccount, permission)
+	_, err := operations.ExecuteGcloudCommandf(cmd)
 	if err != nil {
 		setup.LogAndExit(fmt.Sprintf("Error in unsetting permissions to SA: %v", err))
 	}
 }
 
 func RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(testFlagSet [][]string, permission string, m *testing.M) (successCode int) {
 	serviceAccount, localKeyFilePath := CreateCredentials()
-	ApplyPermissionToServiceAccount(serviceAccount, permission)
+	ApplyPermissionToServiceAccount(serviceAccount, permission, setup.TestBucket())
 	defer RevokePermission(serviceAccount, permission, setup.TestBucket())
 
 	// Without –key-file flag and GOOGLE_APPLICATION_CREDENTIALS

tools/integration_tests/util/mounting/only_dir_mounting/testdata/create_objects.sh

@@ -13,4 +13,4 @@
 # limitations under the License.
 
 touch a.txt
-gsutil mv a.txt gs://$1/
+gcloud storage mv a.txt gs://$1/

tools/integration_tests/util/mounting/only_dir_mounting/testdata/delete_objects.sh

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Here $1 refers to the testBucket argument
-gsutil rm -a gs://$1/**
+gcloud storage rm -r gs://$1/**
 
 # If bucket is empty it will throw an CommandException.
 if [ $? -eq 1 ]; then

tools/integration_tests/util/operations/file_operations.go

@@ -420,10 +420,8 @@ func AreFilesIdentical(filepath1, filepath2 string) (bool, error) {
 // Returns size of a give GCS object with path (without 'gs://').
 // Fails if the object doesn't exist or permission to read object's metadata is not
 // available.
-// Uses 'gsutil du -s gs://gcsObjPath'.
-// Alternative 'gcloud storage du -s gs://gcsObjPath', but it doesn't work on kokoro VM.
 func GetGcsObjectSize(gcsObjPath string) (int, error) {
-	stdout, err := ExecuteGsutilCommandf("du -s gs://%s", gcsObjPath)
+	stdout, err := ExecuteGcloudCommandf("storage du -s gs://%s", gcsObjPath)
 	if err != nil {
 		return 0, err
 	}
@@ -442,10 +440,8 @@ func GetGcsObjectSize(gcsObjPath string) (int, error) {
 // Downloads given GCS object (with path without 'gs://') to localPath.
 // Fails if the object doesn't exist or permission to read object is not
 // available.
-// Uses 'gsutil cp gs://gcsObjPath localPath'
-// Alternative 'gcloud storage cp gs://gcsObjPath localPath' but it doesn't work on kokoro VM.
 func DownloadGcsObject(gcsObjPath, localPath string) error {
-	_, err := ExecuteGsutilCommandf("cp gs://%s %s", gcsObjPath, localPath)
+	_, err := ExecuteGcloudCommandf("storage cp gs://%s %s", gcsObjPath, localPath)
 	if err != nil {
 		return err
 	}
@@ -456,16 +452,12 @@ func DownloadGcsObject(gcsObjPath, localPath string) error {
 // Uploads given local file to GCS object (with path without 'gs://').
 // Fails if the file doesn't exist or permission to write to object/bucket is not
 // available.
-// Uses 'gsutil cp localPath gs://gcsObjPath'
-// Alternative 'gcloud storage cp localPath gs://gcsObjPath' but it doesn't work on kokoro VM.
 func UploadGcsObject(localPath, gcsObjPath string, uploadGzipEncoded bool) error {
 	var err error
 	if uploadGzipEncoded {
-		// Using gsutil instead of `gcloud alpha` here as `gcloud alpha`
-		// option `-Z` isn't supported on the kokoro VM.
-		_, err = ExecuteGsutilCommandf("cp -Z %s gs://%s", localPath, gcsObjPath)
+		_, err = ExecuteGcloudCommandf("storage cp -Z %s gs://%s", localPath, gcsObjPath)
 	} else {
-		_, err = ExecuteGsutilCommandf("cp %s gs://%s", localPath, gcsObjPath)
+		_, err = ExecuteGcloudCommandf("storage cp %s gs://%s", localPath, gcsObjPath)
 	}
 
 	return err
@@ -474,22 +466,16 @@ func UploadGcsObject(localPath, gcsObjPath string, uploadGzipEncoded bool) error
 // Deletes a given GCS object (with path without 'gs://').
 // Fails if the object doesn't exist or permission to delete object is not
 // available.
-// Uses 'gsutil rm gs://gcsObjPath'
-// Alternative 'gcloud storage rm gs://gcsObjPath' but it doesn't work on kokoro VM.
 func DeleteGcsObject(gcsObjPath string) error {
-	_, err := ExecuteGsutilCommandf("rm gs://%s", gcsObjPath)
+	_, err := ExecuteGcloudCommandf("rm gs://%s", gcsObjPath)
 	return err
 }
 
 // Clears cache-control attributes on given GCS object (with path without 'gs://').
 // Fails if the file doesn't exist or permission to modify object's metadata is not
 // available.
-// Uses 'gsutil setmeta -h "Cache-Control:" gs://<path>'
-// Preferred approach is 'gcloud storage objects update gs://gs://gcsObjPath --cache-control=' ' ' but it doesn't work on kokoro VM.
 func ClearCacheControlOnGcsObject(gcsObjPath string) error {
-	// Using gsutil instead of `gcloud alpha` here as `gcloud alpha`
-	// implementation for updating object metadata is missing on the kokoro VM.
-	_, err := ExecuteGsutilCommandf("setmeta -h \"Cache-Control:\" gs://%s ", gcsObjPath)
+	_, err := ExecuteGcloudCommandf("storage objects update --cache-control='' gs://%s", gcsObjPath)
 	return err
 }
 

tools/integration_tests/util/operations/operations.go

@@ -53,11 +53,6 @@ func executeToolCommandf(tool string, format string, args ...any) ([]byte, error
 	return stdout.Bytes(), nil
 }
 
-// Executes any given gsutil command with given args.
-func ExecuteGsutilCommandf(format string, args ...any) ([]byte, error) {
-	return executeToolCommandf("gsutil", format, args...)
-}
-
 // Executes any given gcloud command with given args.
 func ExecuteGcloudCommandf(format string, args ...any) ([]byte, error) {
 	return executeToolCommandf("gcloud", format, args...)

tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/testdata/create_objects.sh

@@ -15,7 +15,7 @@
 # Here $1 refers to the testBucket argument
 echo "This is from directory fileInImplicitDir1 file implicitDirectory" > fileInImplicitDir1
 # bucket/implicitDirectory/fileInImplicitDir1
-gsutil cp fileInImplicitDir1 gs://$1/implicitDirectory/
+gcloud storage cp fileInImplicitDir1 gs://$1/implicitDirectory/
 echo "This is from directory implicitDirectory/implicitSubDirectory file fileInImplicitDir2" > fileInImplicitDir2
 # bucket/implicitDirectory/implicitSubDirectory/fileInImplicitDir2
-gsutil cp fileInImplicitDir2 gs://$1/implicitDirectory/implicitSubDirectory/
+gcloud storage cp fileInImplicitDir2 gs://$1/implicitDirectory/implicitSubDirectory/

tools/integration_tests/util/setup/setup.go

@@ -362,7 +362,7 @@ func CleanUpDir(directoryPath string) {
 
 	for _, d := range dir {
 		err := os.RemoveAll(path.Join([]string{directoryPath, d.Name()}...))
-		if err != nil {
+		if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
 			log.Printf("Error in removing directory: %v", err)
 		}
 	}
@@ -387,7 +387,7 @@ func SetupTestDirectory(testDirName string) string {
 
 // CleanupDirectoryOnGCS cleans up the object/directory path passed in parameter.
 func CleanupDirectoryOnGCS(directoryPathOnGCS string) {
-	_, err := operations.ExecuteGsutilCommandf("rm -rf gs://%s", directoryPathOnGCS)
+	_, err := operations.ExecuteGcloudCommandf("storage rm -r gs://%s", directoryPathOnGCS)
 	if err != nil {
 		log.Printf("Error while cleaning up directory %s from GCS: %v",
 			directoryPathOnGCS, err)