Sequentially read large file

tools/integration_tests/read_large_files/read_large_files_test.go

@@ -0,0 +1,50 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Provides integration tests for read large files sequentially and randomly.
+package read_large_files
+
+import (
+	"log"
+	"os"
+	"testing"
+
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+func TestMain(m *testing.M) {
+	setup.ParseSetUpFlags()
+
+	flags := [][]string{{"--implicit-dirs"}, {"--enable-storage-client-library=false", "--implicit-dirs"}}
+
+	setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet()
+
+	if setup.TestBucket() != "" && setup.MountedDirectory() != "" {
+		log.Print("Both --testbucket and --mountedDirectory can't be specified at the same time.")
+		os.Exit(1)
+	}
+
+	// Run tests for mountedDirectory only if --mountedDirectory flag is set.
+	setup.RunTestsForMountedDirectoryFlag(m)
+
+	// Run tests for testBucket
+	setup.SetUpTestDirForTestBucketFlag()
+
+	successCode := static_mounting.RunTests(flags, m)
+
+	setup.RemoveBinFileCopiedForTesting()
+
+	os.Exit(successCode)
+}

tools/integration_tests/read_large_files/read_one_large_file_sequentially_test.go

@@ -0,0 +1,66 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package read_large_files
+
+import (
+	"bytes"
+	"os"
+	"path"
+	"strconv"
+	"testing"
+
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+const FiveHundredMB = 500 * 1024 * 1024
+const FiveHundredMBFile = "fiveHundredMBFile.txt"
+const chunkSize = 200 * 1024 * 1024
+
+func TestReadLargeFileSequentially(t *testing.T) {
+	// Clean the mountedDirectory before running test.
+	setup.CleanMntDir()
+
+	// Create file of 500 MB with random data in local disk.
+	fileInLocalDisk := path.Join(os.Getenv("HOME"), FiveHundredMBFile)
+	setup.RunScriptForTestData("testdata/write_content_of_fix_size_in_file.sh", fileInLocalDisk, strconv.Itoa(FiveHundredMB))
+
+	// Copy the file in mounted directory.
+	file := path.Join(setup.MntDir(), FiveHundredMBFile)
+	err := operations.CopyFile(fileInLocalDisk, file)
+	if err != nil {
+		t.Errorf("Error in copying file:%v", err)
+	}
+
+	// Sequentially read the data from file.
+	content, err := operations.ReadFileSequentially(file, chunkSize)
+	if err != nil {
+		t.Errorf("Error in reading file: %v", err)
+	}
+
+	// Read actual content from file located in local disk.
+	actualContent, err := operations.ReadFile(fileInLocalDisk)
+	if err != nil {
+		t.Errorf("Error in reading file: %v", err)
+	}
+
+	// Compare actual content and expect content.
+	if bytes.Equal(actualContent, content) == false {
+		t.Errorf("Error in reading file sequentially.")
+	}
+
+	// Removing file after testing.
+	operations.RemoveFile(fileInLocalDisk)
+}

tools/integration_tests/read_large_files/testdata/write_content_of_fix_size_in_file.sh

@@ -0,0 +1,20 @@
+# Copyright 2023 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FILE_PATH=$1
+FILE_SIZE=$2
+TEST_BUCKET=$3
+
+# It will write filesize random data in a file.
+head -c $FILE_SIZE </dev/urandom > $FILE_PATH

tools/integration_tests/run_tests_mounted_directory.sh

@@ -254,7 +254,11 @@ gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR
 GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/list_large_dir/...  -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME
 sudo umount $MOUNT_DIR
 
-# Run integration tests for list_large_dir with --only-dir mounting.
-gcsfuse --only-dir testDir --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR
-GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/list_large_dir/...  -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir
+# Run integration tests for read_large_files directory with static mounting
+gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR
+GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_large_files/...  -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR
+sudo umount $MOUNT_DIR
+
+gcsfuse --enable-storage-client-library=false --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR
+GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_large_files/...  -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR
 sudo umount $MOUNT_DIR

tools/integration_tests/util/operations/file_operations.go

@@ -141,3 +141,54 @@ func CloseFile(file *os.File) {
 		log.Printf("error in closing: %v", err)
 	}
 }
+
+func RemoveFile(filePath string) {
+	err := os.Remove(filePath)
+	if err != nil {
+		log.Printf("Error in removing file:%v", err)
+	}
+}
+
+func ReadFileSequentially(filePath string, chunkSize int64) (content []byte, err error) {
+	chunk := make([]byte, chunkSize)
+	var offset int64 = 0
+
+	file, err := os.OpenFile(filePath, os.O_RDONLY|syscall.O_DIRECT, FilePermission_0600)
+	if err != nil {
+		log.Printf("Error in opening file:%v", err)
+	}
+
+	// Closing the file at the end.
+	defer CloseFile(file)
+
+	for err != io.EOF {
+		var numberOfBytes int
+
+		// Reading 200 MB chunk sequentially from the file.
+		numberOfBytes, err = file.ReadAt(chunk, offset)
+		// If the file reaches the end, write the remaining content in the buffer and return.
+		if err == io.EOF {
+
+			for i := offset; i < offset+int64(numberOfBytes); i++ {
+				// Adding remaining bytes.
+				content = append(content, chunk[i-offset])
+			}
+			err = nil
+			return
+		}
+		if err != nil {
+			return
+		}
+		// Write bytes in the buffer to compare with original content.
+		content = append(content, chunk...)
+
+		// The number of bytes read is not equal to 200MB.
+		if int64(numberOfBytes) != chunkSize {
+			log.Printf("Incorrect number of bytes read from file.")
+		}
+
+		// The offset will shift to read the next chunk.
+		offset = offset + chunkSize
+	}
+	return
+}